diff --git a/clang-tools-extra/clang-tidy/CMakeLists.txt b/clang-tools-extra/clang-tidy/CMakeLists.txt
index 774915f3cc451..6dadb27177119 100644
--- a/clang-tools-extra/clang-tidy/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/CMakeLists.txt
@@ -37,12 +37,15 @@ if(CLANG_ENABLE_STATIC_ANALYZER)
   )
 endif()
 
+# Checks.
+# If you add a check, also add it to ClangTidyForceLinker.h in this directory.
 add_subdirectory(android)
 add_subdirectory(abseil)
 add_subdirectory(boost)
 add_subdirectory(bugprone)
 add_subdirectory(cert)
 add_subdirectory(cppcoreguidelines)
+add_subdirectory(darwin)
 add_subdirectory(fuchsia)
 add_subdirectory(google)
 add_subdirectory(hicpp)
@@ -56,9 +59,38 @@ endif()
 add_subdirectory(objc)
 add_subdirectory(openmp)
 add_subdirectory(performance)
-add_subdirectory(plugin)
 add_subdirectory(portability)
 add_subdirectory(readability)
+add_subdirectory(zircon)
+set(ALL_CLANG_TIDY_CHECKS
+  clangTidyAndroidModule
+  clangTidyAbseilModule
+  clangTidyBoostModule
+  clangTidyBugproneModule
+  clangTidyCERTModule
+  clangTidyCppCoreGuidelinesModule
+  clangTidyDarwinModule
+  clangTidyFuchsiaModule
+  clangTidyGoogleModule
+  clangTidyHICPPModule
+  clangTidyLinuxKernelModule
+  clangTidyLLVMModule
+  clangTidyMiscModule
+  clangTidyModernizeModule
+  clangTidyObjCModule
+  clangTidyOpenMPModule
+  clangTidyPerformanceModule
+  clangTidyPortabilityModule
+  clangTidyReadabilityModule
+  clangTidyZirconModule
+  )
+if(CLANG_ENABLE_STATIC_ANALYZER)
+  list(APPEND ALL_CLANG_TIDY_CHECKS clangTidyMPIModule)
+endif()
+set(ALL_CLANG_TIDY_CHECKS ${ALL_CLANG_TIDY_CHECKS} PARENT_SCOPE)
+
+# Other subtargets. These may reference ALL_CLANG_TIDY_CHECKS
+# and must be below its definition.
+add_subdirectory(plugin)
 add_subdirectory(tool)
 add_subdirectory(utils)
-add_subdirectory(zircon)
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 4c4745f810247..21984aea0e856 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -384,8 +384,8 @@ ClangTidyASTConsumerFactory::CreateASTConsumer(
   if (WorkingDir)
     Context.setCurrentBuildDirectory(WorkingDir.get());
 
-  std::vector<std::unique_ptr<ClangTidyCheck>> Checks;
-  CheckFactories->createChecks(&Context, Checks);
+  std::vector<std::unique_ptr<ClangTidyCheck>> Checks =
+      CheckFactories->createChecks(&Context);
 
   ast_matchers::MatchFinder::MatchFinderOptions FinderOptions;
 
@@ -459,8 +459,8 @@ std::vector<std::string> ClangTidyASTConsumerFactory::getCheckNames() {
 
 ClangTidyOptions::OptionMap ClangTidyASTConsumerFactory::getCheckOptions() {
   ClangTidyOptions::OptionMap Options;
-  std::vector<std::unique_ptr<ClangTidyCheck>> Checks;
-  CheckFactories->createChecks(&Context, Checks);
+  std::vector<std::unique_ptr<ClangTidyCheck>> Checks =
+      CheckFactories->createChecks(&Context);
   for (const auto &Check : Checks)
     Check->storeOptions(Options);
   return Options;
diff --git a/clang-tools-extra/clang-tidy/ClangTidyForceLinker.h b/clang-tools-extra/clang-tidy/ClangTidyForceLinker.h
index ec3b5278f74e4..3ab700118587b 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyForceLinker.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyForceLinker.h
@@ -50,6 +50,11 @@ extern volatile int CppCoreGuidelinesModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED CppCoreGuidelinesModuleAnchorDestination =
     CppCoreGuidelinesModuleAnchorSource;
 
+// This anchor is used to force the linker to link the DarwinModule.
+extern volatile int DarwinModuleAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED DarwinModuleAnchorDestination =
+    DarwinModuleAnchorSource;
+
 // This anchor is used to force the linker to link the FuchsiaModule.
 extern volatile int FuchsiaModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED FuchsiaModuleAnchorDestination =
@@ -75,7 +80,8 @@ extern volatile int ModernizeModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED ModernizeModuleAnchorDestination =
     ModernizeModuleAnchorSource;
 
-#if CLANG_ENABLE_STATIC_ANALYZER
+#if CLANG_ENABLE_STATIC_ANALYZER &&                                            \
+    !defined(CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS)
 // This anchor is used to force the linker to link the MPIModule.
 extern volatile int MPIModuleAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED MPIModuleAnchorDestination =
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
index 7d6de87a6ae88..ff83e7e940541 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.cpp
@@ -20,13 +20,14 @@ void ClangTidyCheckFactories::registerCheckFactory(StringRef Name,
   Factories[Name] = std::move(Factory);
 }
 
-void ClangTidyCheckFactories::createChecks(
-    ClangTidyContext *Context,
-    std::vector<std::unique_ptr<ClangTidyCheck>> &Checks) {
+std::vector<std::unique_ptr<ClangTidyCheck>>
+ClangTidyCheckFactories::createChecks(ClangTidyContext *Context) {
+  std::vector<std::unique_ptr<ClangTidyCheck>> Checks;
   for (const auto &Factory : Factories) {
     if (Context->isCheckEnabled(Factory.first))
       Checks.emplace_back(Factory.second(Factory.first, Context));
   }
+  return Checks;
 }
 
 ClangTidyOptions ClangTidyModule::getModuleOptions() {
diff --git a/clang-tools-extra/clang-tidy/ClangTidyModule.h b/clang-tools-extra/clang-tidy/ClangTidyModule.h
index 894093d95b909..5c484f1a68ddb 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyModule.h
+++ b/clang-tools-extra/clang-tidy/ClangTidyModule.h
@@ -13,6 +13,7 @@
 #include "llvm/ADT/StringRef.h"
 #include <functional>
 #include <map>
+#include <memory>
 #include <string>
 #include <utility>
 
@@ -25,9 +26,8 @@ namespace tidy {
 /// this object.
 class ClangTidyCheckFactories {
 public:
-  typedef std::function<ClangTidyCheck *(StringRef Name,
-                                         ClangTidyContext *Context)>
-      CheckFactory;
+  using CheckFactory = std::function<std::unique_ptr<ClangTidyCheck>(
+      StringRef Name, ClangTidyContext *Context)>;
 
   /// Registers check \p Factory with name \p Name.
   ///
@@ -58,16 +58,13 @@ class ClangTidyCheckFactories {
   template <typename CheckType> void registerCheck(StringRef CheckName) {
     registerCheckFactory(CheckName,
                          [](StringRef Name, ClangTidyContext *Context) {
-                           return new CheckType(Name, Context);
+                           return std::make_unique<CheckType>(Name, Context);
                          });
   }
 
-  /// Create instances of all checks matching \p CheckRegexString and
-  /// store them in \p Checks.
-  ///
-  /// The caller takes ownership of the return \c ClangTidyChecks.
-  void createChecks(ClangTidyContext *Context,
-                    std::vector<std::unique_ptr<ClangTidyCheck>> &Checks);
+  /// Create instances of checks that are enabled.
+  std::vector<std::unique_ptr<ClangTidyCheck>>
+  createChecks(ClangTidyContext *Context);
 
   typedef std::map<std::string, CheckFactory> FactoryMap;
   FactoryMap::const_iterator begin() const { return Factories.begin(); }
diff --git a/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
index c690e797e0e9e..4da91d1162ace 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
@@ -324,7 +324,7 @@ void ArgumentCommentCheck::checkCallArgs(ASTContext *Ctx,
           << FixItHint::CreateInsertion(Args[I]->getBeginLoc(), ArgComment);
     }
   }
-} // namespace bugprone
+}
 
 void ArgumentCommentCheck::check(const MatchFinder::MatchResult &Result) {
   const auto *E = Result.Nodes.getNodeAs<Expr>("expr");
@@ -337,7 +337,7 @@ void ArgumentCommentCheck::check(const MatchFinder::MatchResult &Result) {
                   llvm::makeArrayRef(Call->getArgs(), Call->getNumArgs()));
   } else {
     const auto *Construct = cast<CXXConstructExpr>(E);
-    if (Construct->getNumArgs() == 1 &&
+    if (Construct->getNumArgs() > 0 &&
         Construct->getArg(0)->getSourceRange() == Construct->getSourceRange()) {
       // Ignore implicit construction.
       return;
diff --git a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
index 895bd03f32b48..14058a152c1b9 100644
--- a/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/BugproneTidyModule.cpp
@@ -23,6 +23,7 @@
 #include "ForwardingReferenceOverloadCheck.h"
 #include "InaccurateEraseCheck.h"
 #include "IncorrectRoundingsCheck.h"
+#include "InfiniteLoopCheck.h"
 #include "IntegerDivisionCheck.h"
 #include "LambdaFunctionNameCheck.h"
 #include "MacroParenthesesCheck.h"
@@ -88,6 +89,8 @@ class BugproneModule : public ClangTidyModule {
         "bugprone-inaccurate-erase");
     CheckFactories.registerCheck<IncorrectRoundingsCheck>(
         "bugprone-incorrect-roundings");
+    CheckFactories.registerCheck<InfiniteLoopCheck>(
+        "bugprone-infinite-loop");
     CheckFactories.registerCheck<IntegerDivisionCheck>(
         "bugprone-integer-division");
     CheckFactories.registerCheck<LambdaFunctionNameCheck>(
diff --git a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
index 705ee97514e7e..b1d8bbe5c82dd 100644
--- a/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/bugprone/CMakeLists.txt
@@ -15,6 +15,7 @@ add_clang_library(clangTidyBugproneModule
   ForwardingReferenceOverloadCheck.cpp
   InaccurateEraseCheck.cpp
   IncorrectRoundingsCheck.cpp
+  InfiniteLoopCheck.cpp
   IntegerDivisionCheck.cpp
   LambdaFunctionNameCheck.cpp
   MacroParenthesesCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
new file mode 100644
index 0000000000000..81ae45a008961
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.cpp
@@ -0,0 +1,188 @@
+//===--- InfiniteLoopCheck.cpp - clang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "InfiniteLoopCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Analysis/Analyses/ExprMutationAnalyzer.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+static internal::Matcher<Stmt>
+loopEndingStmt(internal::Matcher<Stmt> Internal) {
+  return stmt(anyOf(breakStmt(Internal), returnStmt(Internal),
+                    gotoStmt(Internal), cxxThrowExpr(Internal),
+                    callExpr(Internal, callee(functionDecl(isNoReturn())))));
+}
+
+/// Return whether `S` is a reference to the declaration of `Var`.
+static bool isAccessForVar(const Stmt *S, const VarDecl *Var) {
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(S))
+    return DRE->getDecl() == Var;
+
+  return false;
+}
+
+/// Return whether `Var` has a pointer or reference in `S`.
+static bool isPtrOrReferenceForVar(const Stmt *S, const VarDecl *Var) {
+  if (const auto *DS = dyn_cast<DeclStmt>(S)) {
+    for (const Decl *D : DS->getDeclGroup()) {
+      if (const auto *LeftVar = dyn_cast<VarDecl>(D)) {
+        if (LeftVar->hasInit() && LeftVar->getType()->isReferenceType()) {
+          return isAccessForVar(LeftVar->getInit(), Var);
+        }
+      }
+    }
+  } else if (const auto *UnOp = dyn_cast<UnaryOperator>(S)) {
+    if (UnOp->getOpcode() == UO_AddrOf)
+      return isAccessForVar(UnOp->getSubExpr(), Var);
+  }
+
+  return false;
+}
+
+/// Return whether `Var` has a pointer or reference in `S`.
+static bool hasPtrOrReferenceInStmt(const Stmt *S, const VarDecl *Var) {
+  if (isPtrOrReferenceForVar(S, Var))
+    return true;
+
+  for (const Stmt *Child : S->children()) {
+    if (!Child)
+      continue;
+
+    if (hasPtrOrReferenceInStmt(Child, Var))
+      return true;
+  }
+
+  return false;
+}
+
+/// Return whether `Var` has a pointer or reference in `Func`.
+static bool hasPtrOrReferenceInFunc(const FunctionDecl *Func,
+                                    const VarDecl *Var) {
+  return hasPtrOrReferenceInStmt(Func->getBody(), Var);
+}
+
+/// Return whether `Var` was changed in `LoopStmt`.
+static bool isChanged(const Stmt *LoopStmt, const VarDecl *Var,
+                      ASTContext *Context) {
+  if (const auto *ForLoop = dyn_cast<ForStmt>(LoopStmt))
+    return (ForLoop->getInc() &&
+            ExprMutationAnalyzer(*ForLoop->getInc(), *Context)
+                .isMutated(Var)) ||
+           (ForLoop->getBody() &&
+            ExprMutationAnalyzer(*ForLoop->getBody(), *Context)
+                .isMutated(Var)) ||
+           (ForLoop->getCond() &&
+            ExprMutationAnalyzer(*ForLoop->getCond(), *Context).isMutated(Var));
+
+  return ExprMutationAnalyzer(*LoopStmt, *Context).isMutated(Var);
+}
+
+/// Return whether `Cond` is a variable that is possibly changed in `LoopStmt`.
+static bool isVarThatIsPossiblyChanged(const FunctionDecl *Func,
+                                       const Stmt *LoopStmt, const Stmt *Cond,
+                                       ASTContext *Context) {
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(Cond)) {
+    if (const auto *Var = dyn_cast<VarDecl>(DRE->getDecl())) {
+      if (!Var->isLocalVarDeclOrParm())
+        return true;
+
+      if (Var->getType().isVolatileQualified())
+        return true;
+
+      if (!Var->getType().getTypePtr()->isIntegerType())
+        return true;
+
+      return hasPtrOrReferenceInFunc(Func, Var) ||
+             isChanged(LoopStmt, Var, Context);
+      // FIXME: Track references.
+    }
+  } else if (isa<MemberExpr>(Cond) || isa<CallExpr>(Cond)) {
+    // FIXME: Handle MemberExpr.
+    return true;
+  }
+
+  return false;
+}
+
+/// Return whether at least one variable of `Cond` changed in `LoopStmt`.
+static bool isAtLeastOneCondVarChanged(const FunctionDecl *Func,
+                                       const Stmt *LoopStmt, const Stmt *Cond,
+                                       ASTContext *Context) {
+  if (isVarThatIsPossiblyChanged(Func, LoopStmt, Cond, Context))
+    return true;
+
+  for (const Stmt *Child : Cond->children()) {
+    if (!Child)
+      continue;
+
+    if (isAtLeastOneCondVarChanged(Func, LoopStmt, Child, Context))
+      return true;
+  }
+  return false;
+}
+
+/// Return the variable names in `Cond`.
+static std::string getCondVarNames(const Stmt *Cond) {
+  if (const auto *DRE = dyn_cast<DeclRefExpr>(Cond)) {
+    if (const auto *Var = dyn_cast<VarDecl>(DRE->getDecl()))
+      return Var->getName();
+  }
+
+  std::string Result;
+  for (const Stmt *Child : Cond->children()) {
+    if (!Child)
+      continue;
+
+    std::string NewNames = getCondVarNames(Child);
+    if (!Result.empty() && !NewNames.empty())
+      Result += ", ";
+    Result += NewNames;
+  }
+  return Result;
+}
+
+void InfiniteLoopCheck::registerMatchers(MatchFinder *Finder) {
+  const auto LoopCondition = allOf(
+      hasCondition(
+          expr(forFunction(functionDecl().bind("func"))).bind("condition")),
+      unless(hasBody(hasDescendant(
+          loopEndingStmt(forFunction(equalsBoundNode("func")))))));
+
+  Finder->addMatcher(stmt(anyOf(whileStmt(LoopCondition), doStmt(LoopCondition),
+                                forStmt(LoopCondition)))
+                         .bind("loop-stmt"),
+                     this);
+}
+
+void InfiniteLoopCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *Cond = Result.Nodes.getNodeAs<Expr>("condition");
+  const auto *LoopStmt = Result.Nodes.getNodeAs<Stmt>("loop-stmt");
+  const auto *Func = Result.Nodes.getNodeAs<FunctionDecl>("func");
+
+  if (isAtLeastOneCondVarChanged(Func, LoopStmt, Cond, Result.Context))
+    return;
+
+  std::string CondVarNames = getCondVarNames(Cond);
+  if (CondVarNames.empty())
+    return;
+
+  diag(LoopStmt->getBeginLoc(),
+       "this loop is infinite; none of its condition variables (%0)"
+       " are updated in the loop body")
+      << CondVarNames;
+}
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.h b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.h
new file mode 100644
index 0000000000000..45620bf3286a7
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/bugprone/InfiniteLoopCheck.h
@@ -0,0 +1,35 @@
+//===--- InfiniteLoopCheck.h - clang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_INFINITELOOPCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_INFINITELOOPCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace bugprone {
+
+/// Finds obvious infinite loops (loops where the condition variable is
+/// not changed at all).
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/bugprone-infinite-loop.html
+class InfiniteLoopCheck : public ClangTidyCheck {
+public:
+  InfiniteLoopCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace bugprone
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_BUGPRONE_INFINITELOOPCHECK_H
diff --git a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
index 4e984dede0e23..4a470e5c9def6 100644
--- a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.cpp
@@ -28,26 +28,31 @@ static StringRef getFunctionSpelling(const MatchFinder::MatchResult &Result,
 }
 
 void PosixReturnCheck::registerMatchers(MatchFinder *Finder) {
-  Finder->addMatcher(binaryOperator(hasOperatorName("<"),
-                                    hasLHS(callExpr(callee(functionDecl(
-                                        matchesName("^::posix_"),
-                                        unless(hasName("::posix_openpt")))))),
-                                    hasRHS(integerLiteral(equals(0))))
-                         .bind("ltzop"),
-                     this);
-  Finder->addMatcher(binaryOperator(hasOperatorName(">="),
-                                    hasLHS(callExpr(callee(functionDecl(
-                                        matchesName("^::posix_"),
-                                        unless(hasName("::posix_openpt")))))),
-                                    hasRHS(integerLiteral(equals(0))))
-                         .bind("atop"),
-                     this);
+  Finder->addMatcher(
+      binaryOperator(
+          hasOperatorName("<"),
+          hasLHS(callExpr(callee(functionDecl(
+              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
+              unless(hasName("::posix_openpt")))))),
+          hasRHS(integerLiteral(equals(0))))
+          .bind("ltzop"),
+      this);
+  Finder->addMatcher(
+      binaryOperator(
+          hasOperatorName(">="),
+          hasLHS(callExpr(callee(functionDecl(
+              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
+              unless(hasName("::posix_openpt")))))),
+          hasRHS(integerLiteral(equals(0))))
+          .bind("atop"),
+      this);
   Finder->addMatcher(
       binaryOperator(
           anyOf(hasOperatorName("=="), hasOperatorName("!="),
                 hasOperatorName("<="), hasOperatorName("<")),
           hasLHS(callExpr(callee(functionDecl(
-              matchesName("^::posix_"), unless(hasName("::posix_openpt")))))),
+              anyOf(matchesName("^::posix_"), matchesName("^::pthread_")),
+              unless(hasName("::posix_openpt")))))),
           hasRHS(unaryOperator(hasOperatorName("-"),
                                hasUnaryOperand(integerLiteral()))))
           .bind("binop"),
diff --git a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
index 979a17fed3506..fe5c94c6ac7ca 100644
--- a/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
+++ b/clang-tools-extra/clang-tidy/bugprone/PosixReturnCheck.h
@@ -1,4 +1,4 @@
-//===--- PosixReturnCheck.h - clang-tidy-------------------------*- C++ -*-===//
+//===--- PosixReturnCheck.h - clang-tidy ------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
index be85285932509..13c15bc9d227b 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
@@ -3,6 +3,7 @@ set(LLVM_LINK_COMPONENTS support)
 add_clang_library(clangTidyCppCoreGuidelinesModule
   AvoidGotoCheck.cpp
   CppCoreGuidelinesTidyModule.cpp
+  InitVariablesCheck.cpp
   InterfacesGlobalInitCheck.cpp
   MacroUsageCheck.cpp
   NarrowingConversionsCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
index 7c7fd1b72cb78..8886eb8337958 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
@@ -15,6 +15,7 @@
 #include "../modernize/UseOverrideCheck.h"
 #include "../readability/MagicNumbersCheck.h"
 #include "AvoidGotoCheck.h"
+#include "InitVariablesCheck.h"
 #include "InterfacesGlobalInitCheck.h"
 #include "MacroUsageCheck.h"
 #include "NarrowingConversionsCheck.h"
@@ -49,6 +50,8 @@ class CppCoreGuidelinesModule : public ClangTidyModule {
         "cppcoreguidelines-avoid-magic-numbers");
     CheckFactories.registerCheck<modernize::UseOverrideCheck>(
         "cppcoreguidelines-explicit-virtual-functions");
+    CheckFactories.registerCheck<InitVariablesCheck>(
+        "cppcoreguidelines-init-variables");
     CheckFactories.registerCheck<InterfacesGlobalInitCheck>(
         "cppcoreguidelines-interfaces-global-init");
     CheckFactories.registerCheck<MacroUsageCheck>(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
new file mode 100644
index 0000000000000..8a628317a302f
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.cpp
@@ -0,0 +1,105 @@
+//===--- InitVariablesCheck.cpp - clang-tidy ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "InitVariablesCheck.h"
+
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace cppcoreguidelines {
+
+namespace {
+AST_MATCHER(VarDecl, isLocalVarDecl) { return Node.isLocalVarDecl(); }
+} // namespace
+
+InitVariablesCheck::InitVariablesCheck(StringRef Name,
+                                       ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      IncludeStyle(utils::IncludeSorter::parseIncludeStyle(
+          Options.getLocalOrGlobal("IncludeStyle", "llvm"))),
+      MathHeader(Options.get("MathHeader", "math.h")) {}
+
+void InitVariablesCheck::registerMatchers(MatchFinder *Finder) {
+  Finder->addMatcher(varDecl(unless(hasInitializer(anything())),
+                             unless(isInstantiated()), isLocalVarDecl(),
+                             unless(isStaticLocal()), isDefinition())
+                         .bind("vardecl"),
+                     this);
+}
+
+void InitVariablesCheck::registerPPCallbacks(const SourceManager &SM,
+                                             Preprocessor *PP,
+                                             Preprocessor *ModuleExpanderPP) {
+  IncludeInserter =
+      std::make_unique<utils::IncludeInserter>(SM, getLangOpts(), IncludeStyle);
+  PP->addPPCallbacks(IncludeInserter->CreatePPCallbacks());
+}
+
+void InitVariablesCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *MatchedDecl = Result.Nodes.getNodeAs<VarDecl>("vardecl");
+  const ASTContext &Context = *Result.Context;
+  const SourceManager &Source = Context.getSourceManager();
+
+  // We want to warn about cases where the type name
+  // comes from a macro like this:
+  //
+  // TYPENAME_FROM_MACRO var;
+  //
+  // but not if the entire declaration comes from
+  // one:
+  //
+  // DEFINE_SOME_VARIABLE();
+  //
+  // or if the definition comes from a macro like SWAP
+  // that uses an internal temporary variable.
+  //
+  // Thus check that the variable name does
+  // not come from a macro expansion.
+  if (MatchedDecl->getEndLoc().isMacroID())
+    return;
+
+  QualType TypePtr = MatchedDecl->getType();
+  const char *InitializationString = nullptr;
+  bool AddMathInclude = false;
+
+  if (TypePtr->isIntegerType())
+    InitializationString = " = 0";
+  else if (TypePtr->isFloatingType()) {
+    InitializationString = " = NAN";
+    AddMathInclude = true;
+  } else if (TypePtr->isAnyPointerType()) {
+    if (getLangOpts().CPlusPlus11)
+      InitializationString = " = nullptr";
+    else
+      InitializationString = " = NULL";
+  }
+
+  if (InitializationString) {
+    auto Diagnostic =
+        diag(MatchedDecl->getLocation(), "variable %0 is not initialized")
+        << MatchedDecl
+        << FixItHint::CreateInsertion(
+               MatchedDecl->getLocation().getLocWithOffset(
+                   MatchedDecl->getName().size()),
+               InitializationString);
+    if (AddMathInclude) {
+      auto IncludeHint = IncludeInserter->CreateIncludeInsertion(
+          Source.getFileID(MatchedDecl->getBeginLoc()), MathHeader, false);
+      if (IncludeHint)
+        Diagnostic << *IncludeHint;
+    }
+  }
+}
+
+} // namespace cppcoreguidelines
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
new file mode 100644
index 0000000000000..0cacf9e533cf8
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/InitVariablesCheck.h
@@ -0,0 +1,42 @@
+//===--- InitVariablesCheck.h - clang-tidy ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_INITVARIABLESCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_INITVARIABLESCHECK_H
+
+#include "../ClangTidyCheck.h"
+#include "../utils/IncludeInserter.h"
+#include "../utils/OptionsUtils.h"
+
+namespace clang {
+namespace tidy {
+namespace cppcoreguidelines {
+
+/// Find uninitialized local variables.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines-init-variables.html
+class InitVariablesCheck : public ClangTidyCheck {
+public:
+  InitVariablesCheck(StringRef Name, ClangTidyContext *Context);
+  void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
+                           Preprocessor *ModuleExpanderPP) override;
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+
+private:
+  std::unique_ptr<clang::tidy::utils::IncludeInserter> IncludeInserter;
+  const utils::IncludeSorter::IncludeStyle IncludeStyle;
+  const std::string MathHeader;
+};
+
+} // namespace cppcoreguidelines
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_INITVARIABLESCHECK_H
diff --git a/clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.cpp b/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.cpp
similarity index 96%
rename from clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.cpp
rename to clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.cpp
index ac3d2b288efaa..2a52725c19c8d 100644
--- a/clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.cpp
+++ b/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.cpp
@@ -14,7 +14,7 @@ using namespace clang::ast_matchers;
 
 namespace clang {
 namespace tidy {
-namespace objc {
+namespace darwin {
 
 void AvoidSpinlockCheck::registerMatchers(MatchFinder *Finder) {
   Finder->addMatcher(
@@ -31,6 +31,6 @@ void AvoidSpinlockCheck::check(const MatchFinder::MatchResult &Result) {
        "deprecated OSSpinLock");
 }
 
-}  // namespace objc
+}  // namespace darwin
 }  // namespace tidy
 }  // namespace clang
diff --git a/clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.h b/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
similarity index 73%
rename from clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.h
rename to clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
index dd409625f3a13..6ea10c550d901 100644
--- a/clang-tools-extra/clang-tidy/objc/AvoidSpinlockCheck.h
+++ b/clang-tools-extra/clang-tidy/darwin/AvoidSpinlockCheck.h
@@ -6,20 +6,20 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_AVOID_SPINLOCK_H
-#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_AVOID_SPINLOCK_H
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_AVOIDSPINLOCKCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_AVOIDSPINLOCKCHECK_H
 
 #include "../ClangTidyCheck.h"
 
 namespace clang {
 namespace tidy {
-namespace objc {
+namespace darwin {
 
 /// Finds usages of OSSpinlock, which is deprecated due to potential livelock
 /// problems.
 ///
 /// For the user-facing documentation see:
-/// http://clang.llvm.org/extra/clang-tidy/checks/objc-avoid-spinlock.html
+/// http://clang.llvm.org/extra/clang-tidy/checks/darwin-avoid-spinlock.html
 class AvoidSpinlockCheck : public ClangTidyCheck {
  public:
   AvoidSpinlockCheck(StringRef Name, ClangTidyContext *Context)
@@ -28,8 +28,8 @@ class AvoidSpinlockCheck : public ClangTidyCheck {
   void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
 };
 
-}  // namespace objc
+}  // namespace darwin
 }  // namespace tidy
 }  // namespace clang
 
-#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_AVOID_SPINLOCK_H
+#endif  // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_AVOIDSPINLOCKCHECK_H
diff --git a/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt b/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt
new file mode 100644
index 0000000000000..c650efb2a5f27
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/darwin/CMakeLists.txt
@@ -0,0 +1,16 @@
+set(LLVM_LINK_COMPONENTS support)
+
+add_clang_library(clangTidyDarwinModule
+  AvoidSpinlockCheck.cpp
+  DarwinTidyModule.cpp
+  DispatchOnceNonstaticCheck.cpp
+
+  LINK_LIBS
+  clangAnalysis
+  clangAST
+  clangASTMatchers
+  clangBasic
+  clangLex
+  clangTidy
+  clangTidyUtils
+  )
diff --git a/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp b/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp
new file mode 100644
index 0000000000000..d11ef1fa01c98
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/darwin/DarwinTidyModule.cpp
@@ -0,0 +1,40 @@
+//===--- MiscTidyModule.cpp - clang-tidy ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../ClangTidy.h"
+#include "../ClangTidyModule.h"
+#include "../ClangTidyModuleRegistry.h"
+#include "AvoidSpinlockCheck.h"
+#include "DispatchOnceNonstaticCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace darwin {
+
+class DarwinModule : public ClangTidyModule {
+public:
+  void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
+    CheckFactories.registerCheck<AvoidSpinlockCheck>(
+        "darwin-avoid-spinlock");
+    CheckFactories.registerCheck<DispatchOnceNonstaticCheck>(
+        "darwin-dispatch-once-nonstatic");
+  }
+};
+
+} // namespace darwin
+
+// Register the DarwinTidyModule using this statically initialized variable.
+static ClangTidyModuleRegistry::Add<darwin::DarwinModule>
+    X("darwin-module", "Adds Darwin-specific lint checks.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the DarwinModule.
+volatile int DarwinModuleAnchorSource = 0;
+
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.cpp b/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.cpp
new file mode 100644
index 0000000000000..dac7f83309714
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.cpp
@@ -0,0 +1,62 @@
+//===--- DispatchOnceNonstaticCheck.cpp - clang-tidy ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "DispatchOnceNonstaticCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Basic/Diagnostic.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace darwin {
+
+void DispatchOnceNonstaticCheck::registerMatchers(MatchFinder *Finder) {
+  // Find variables without static or global storage. VarDecls do not include
+  // struct/class members, which are FieldDecls.
+  Finder->addMatcher(
+      varDecl(hasLocalStorage(), hasType(asString("dispatch_once_t")))
+          .bind("non-static-var"),
+      this);
+
+  // Members of structs or classes might be okay, if the use is at static or
+  // global scope. These will be ignored for now. But ObjC ivars can be
+  // flagged immediately, since they cannot be static.
+  Finder->addMatcher(
+      objcIvarDecl(hasType(asString("dispatch_once_t"))).bind("ivar"), this);
+}
+
+void DispatchOnceNonstaticCheck::check(const MatchFinder::MatchResult &Result) {
+  if (const auto *VD = Result.Nodes.getNodeAs<VarDecl>("non-static-var")) {
+    if (const auto *PD = dyn_cast<ParmVarDecl>(VD)) {
+      // Catch function/method parameters, as any dispatch_once_t should be
+      // passed by pointer instead.
+      diag(PD->getTypeSpecStartLoc(),
+           "dispatch_once_t variables must have static or global storage "
+           "duration; function parameters should be pointer references");
+    } else {
+      diag(VD->getTypeSpecStartLoc(), "dispatch_once_t variables must have "
+                                      "static or global storage duration")
+          << FixItHint::CreateInsertion(VD->getTypeSpecStartLoc(), "static ");
+    }
+  }
+
+  if (const auto *D = Result.Nodes.getNodeAs<ObjCIvarDecl>("ivar")) {
+    diag(D->getTypeSpecStartLoc(),
+         "dispatch_once_t variables must have static or global storage "
+         "duration and cannot be Objective-C instance variables");
+  }
+}
+
+} // namespace darwin
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.h b/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.h
new file mode 100644
index 0000000000000..be0719a778ea0
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/darwin/DispatchOnceNonstaticCheck.h
@@ -0,0 +1,35 @@
+//===--- DispatchOnceNonstaticCheck.h - clang-tidy --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_DISPATCHONCENONSTATICCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_DISPATCHONCENONSTATICCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace darwin {
+
+/// Finds variables of type dispatch_once_t that do not have static or global
+/// storage duration, as required by the libdispatch documentation.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/darwin-dispatch-once-nonstatic.html
+class DispatchOnceNonstaticCheck : public ClangTidyCheck {
+public:
+  DispatchOnceNonstaticCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace darwin
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_DARWIN_DISPATCHONCENONSTATICCHECK_H
diff --git a/clang-tools-extra/clang-tidy/google/TodoCommentCheck.cpp b/clang-tools-extra/clang-tidy/google/TodoCommentCheck.cpp
index 787c30548d4af..ec099f0e400c9 100644
--- a/clang-tools-extra/clang-tidy/google/TodoCommentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/TodoCommentCheck.cpp
@@ -55,6 +55,8 @@ TodoCommentCheck::TodoCommentCheck(StringRef Name, ClangTidyContext *Context)
       Handler(std::make_unique<TodoCommentHandler>(
           *this, Context->getOptions().User)) {}
 
+TodoCommentCheck::~TodoCommentCheck() = default;
+
 void TodoCommentCheck::registerPPCallbacks(const SourceManager &SM,
                                            Preprocessor *PP,
                                            Preprocessor *ModuleExpanderPP) {
diff --git a/clang-tools-extra/clang-tidy/google/TodoCommentCheck.h b/clang-tools-extra/clang-tidy/google/TodoCommentCheck.h
index d1343b74fdbf1..8c32dddc28373 100644
--- a/clang-tools-extra/clang-tidy/google/TodoCommentCheck.h
+++ b/clang-tools-extra/clang-tidy/google/TodoCommentCheck.h
@@ -22,6 +22,8 @@ namespace readability {
 class TodoCommentCheck : public ClangTidyCheck {
 public:
   TodoCommentCheck(StringRef Name, ClangTidyContext *Context);
+  ~TodoCommentCheck();
+
   void registerPPCallbacks(const SourceManager &SM, Preprocessor *PP,
                            Preprocessor *ModuleExpanderPP) override;
 
diff --git a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
index c098268196a0a..8b32d8ddcc93b 100644
--- a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
@@ -33,6 +33,13 @@ std::string LLVMHeaderGuardCheck::getHeaderGuard(StringRef Filename,
   if (PosToolsClang != StringRef::npos)
     Guard = Guard.substr(PosToolsClang + std::strlen("tools/"));
 
+  // Unlike LLVM svn, LLVM git monorepo is named llvm-project, so we replace
+  // "/llvm-project/" with the cannonical "/llvm/".
+  const static StringRef LLVMProject = "/llvm-project/";
+  size_t PosLLVMProject = Guard.rfind(LLVMProject);
+  if (PosLLVMProject != StringRef::npos)
+    Guard = Guard.replace(PosLLVMProject, LLVMProject.size(), "/llvm/");
+
   // The remainder is LLVM_FULL_PATH_TO_HEADER_H
   size_t PosLLVM = Guard.rfind("llvm/");
   if (PosLLVM != StringRef::npos)
diff --git a/clang-tools-extra/clang-tidy/objc/CMakeLists.txt b/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
index 4eeb14844a58d..68dda6530f7f7 100644
--- a/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/objc/CMakeLists.txt
@@ -2,8 +2,8 @@ set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangTidyObjCModule
   AvoidNSErrorInitCheck.cpp
-  AvoidSpinlockCheck.cpp
   ForbiddenSubclassingCheck.cpp
+  MissingHashCheck.cpp
   ObjCTidyModule.cpp
   PropertyDeclarationCheck.cpp
   SuperSelfCheck.cpp
diff --git a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp
new file mode 100644
index 0000000000000..0da5571b2f26c
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.cpp
@@ -0,0 +1,62 @@
+//===--- MissingHashCheck.cpp - clang-tidy --------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MissingHashCheck.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang {
+namespace tidy {
+namespace objc {
+
+namespace {
+
+AST_MATCHER_P(ObjCImplementationDecl, hasInterface,
+              ast_matchers::internal::Matcher<ObjCInterfaceDecl>, Base) {
+  const ObjCInterfaceDecl *InterfaceDecl = Node.getClassInterface();
+  return Base.matches(*InterfaceDecl, Finder, Builder);
+}
+
+AST_MATCHER_P(ObjCContainerDecl, hasInstanceMethod,
+              ast_matchers::internal::Matcher<ObjCMethodDecl>, Base) {
+  // Check each instance method against the provided matcher.
+  for (const auto *I : Node.instance_methods()) {
+    if (Base.matches(*I, Finder, Builder))
+      return true;
+  }
+  return false;
+}
+
+} // namespace
+
+void MissingHashCheck::registerMatchers(MatchFinder *Finder) {
+  // This check should only be applied to Objective-C sources.
+  if (!getLangOpts().ObjC)
+    return;
+
+  Finder->addMatcher(
+      objcMethodDecl(
+          hasName("isEqual:"), isInstanceMethod(),
+          hasDeclContext(objcImplementationDecl(
+                             hasInterface(isDirectlyDerivedFrom("NSObject")),
+                             unless(hasInstanceMethod(hasName("hash"))))
+                             .bind("impl"))),
+      this);
+}
+
+void MissingHashCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *ID = Result.Nodes.getNodeAs<ObjCImplementationDecl>("impl");
+  diag(ID->getLocation(), "%0 implements -isEqual: without implementing -hash")
+      << ID;
+}
+
+} // namespace objc
+} // namespace tidy
+} // namespace clang
diff --git a/clang-tools-extra/clang-tidy/objc/MissingHashCheck.h b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.h
new file mode 100644
index 0000000000000..4ac74d5865218
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/objc/MissingHashCheck.h
@@ -0,0 +1,35 @@
+//===--- MissingHashCheck.h - clang-tidy ------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_MISSINGHASHCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_MISSINGHASHCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang {
+namespace tidy {
+namespace objc {
+
+/// Finds Objective-C implementations that implement -isEqual: without also
+/// appropriately implementing -hash.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/objc-missing-hash.html
+class MissingHashCheck : public ClangTidyCheck {
+public:
+  MissingHashCheck(StringRef Name, ClangTidyContext *Context)
+      : ClangTidyCheck(Name, Context) {}
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+};
+
+} // namespace objc
+} // namespace tidy
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_OBJC_MISSINGHASHCHECK_H
diff --git a/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp b/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
index 636e2c02af649..69913125451ce 100644
--- a/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/objc/ObjCTidyModule.cpp
@@ -10,8 +10,8 @@
 #include "../ClangTidyModule.h"
 #include "../ClangTidyModuleRegistry.h"
 #include "AvoidNSErrorInitCheck.h"
-#include "AvoidSpinlockCheck.h"
 #include "ForbiddenSubclassingCheck.h"
+#include "MissingHashCheck.h"
 #include "PropertyDeclarationCheck.h"
 #include "SuperSelfCheck.h"
 
@@ -26,10 +26,10 @@ class ObjCModule : public ClangTidyModule {
   void addCheckFactories(ClangTidyCheckFactories &CheckFactories) override {
     CheckFactories.registerCheck<AvoidNSErrorInitCheck>(
         "objc-avoid-nserror-init");
-    CheckFactories.registerCheck<AvoidSpinlockCheck>(
-        "objc-avoid-spinlock");
     CheckFactories.registerCheck<ForbiddenSubclassingCheck>(
         "objc-forbidden-subclassing");
+    CheckFactories.registerCheck<MissingHashCheck>(
+        "objc-missing-hash");
     CheckFactories.registerCheck<PropertyDeclarationCheck>(
         "objc-property-declaration");
     CheckFactories.registerCheck<SuperSelfCheck>(
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
index 622649ac4594c..e93ed7e3a33b7 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.cpp
@@ -29,26 +29,37 @@ namespace {
 //   for (int i = 0; i < 10 + 1; ++i) {
 //     v.push_back(i);
 //   }
+//
+//   SomeProto p;
+//   for (int i = 0; i < 10 + 1; ++i) {
+//     p.add_xxx(i);
+//   }
 // }
 // \endcode
 //
 // The matcher names are bound to following parts of the AST:
 //   - LoopCounterName: The entire for loop (as ForStmt).
 //   - LoopParentName: The body of function f (as CompoundStmt).
-//   - VectorVarDeclName: 'v' in  (as VarDecl).
+//   - VectorVarDeclName: 'v' (as VarDecl).
 //   - VectorVarDeclStmatName: The entire 'std::vector<T> v;' statement (as
 //     DeclStmt).
 //   - PushBackOrEmplaceBackCallName: 'v.push_back(i)' (as cxxMemberCallExpr).
 //   - LoopInitVarName: 'i' (as VarDecl).
 //   - LoopEndExpr: '10+1' (as Expr).
+// If EnableProto, the proto related names are bound to the following parts:
+//   - ProtoVarDeclName: 'p' (as VarDecl).
+//   - ProtoVarDeclStmtName: The entire 'SomeProto p;' statement (as DeclStmt).
+//   - ProtoAddFieldCallName: 'p.add_xxx(i)' (as cxxMemberCallExpr).
 static const char LoopCounterName[] = "for_loop_counter";
 static const char LoopParentName[] = "loop_parent";
 static const char VectorVarDeclName[] = "vector_var_decl";
 static const char VectorVarDeclStmtName[] = "vector_var_decl_stmt";
 static const char PushBackOrEmplaceBackCallName[] = "append_call";
+static const char ProtoVarDeclName[] = "proto_var_decl";
+static const char ProtoVarDeclStmtName[] = "proto_var_decl_stmt";
+static const char ProtoAddFieldCallName[] = "proto_add_field";
 static const char LoopInitVarName[] = "loop_init_var";
 static const char LoopEndExprName[] = "loop_end_expr";
-
 static const char RangeLoopName[] = "for_range_loop";
 
 ast_matchers::internal::Matcher<Expr> supportedContainerTypesMatcher() {
@@ -63,34 +74,35 @@ InefficientVectorOperationCheck::InefficientVectorOperationCheck(
     StringRef Name, ClangTidyContext *Context)
     : ClangTidyCheck(Name, Context),
       VectorLikeClasses(utils::options::parseStringList(
-          Options.get("VectorLikeClasses", "::std::vector"))) {}
+          Options.get("VectorLikeClasses", "::std::vector"))),
+      EnableProto(Options.getLocalOrGlobal("EnableProto", false)) {}
 
 void InefficientVectorOperationCheck::storeOptions(
     ClangTidyOptions::OptionMap &Opts) {
   Options.store(Opts, "VectorLikeClasses",
                 utils::options::serializeStringList(VectorLikeClasses));
+  Options.store(Opts, "EnableProto", EnableProto);
 }
 
-void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
-  const auto VectorDecl = cxxRecordDecl(hasAnyName(SmallVector<StringRef, 5>(
-      VectorLikeClasses.begin(), VectorLikeClasses.end())));
-  const auto VectorDefaultConstructorCall = cxxConstructExpr(
-      hasType(VectorDecl),
+void InefficientVectorOperationCheck::AddMatcher(
+    const DeclarationMatcher &TargetRecordDecl, StringRef VarDeclName,
+    StringRef VarDeclStmtName, const DeclarationMatcher &AppendMethodDecl,
+    StringRef AppendCallName, MatchFinder *Finder) {
+  const auto DefaultConstructorCall = cxxConstructExpr(
+      hasType(TargetRecordDecl),
       hasDeclaration(cxxConstructorDecl(isDefaultConstructor())));
-  const auto VectorVarDecl =
-      varDecl(hasInitializer(VectorDefaultConstructorCall))
-          .bind(VectorVarDeclName);
-  const auto VectorAppendCallExpr =
-      cxxMemberCallExpr(
-          callee(cxxMethodDecl(hasAnyName("push_back", "emplace_back"))),
-          on(hasType(VectorDecl)),
-          onImplicitObjectArgument(declRefExpr(to(VectorVarDecl))))
-          .bind(PushBackOrEmplaceBackCallName);
-  const auto VectorAppendCall = expr(ignoringImplicit(VectorAppendCallExpr));
-  const auto VectorVarDefStmt =
-      declStmt(hasSingleDecl(equalsBoundNode(VectorVarDeclName)))
-          .bind(VectorVarDeclStmtName);
+  const auto TargetVarDecl =
+      varDecl(hasInitializer(DefaultConstructorCall)).bind(VarDeclName);
+  const auto TargetVarDefStmt =
+      declStmt(hasSingleDecl(equalsBoundNode(VarDeclName)))
+          .bind(VarDeclStmtName);
 
+  const auto AppendCallExpr =
+      cxxMemberCallExpr(
+          callee(AppendMethodDecl), on(hasType(TargetRecordDecl)),
+          onImplicitObjectArgument(declRefExpr(to(TargetVarDecl))))
+          .bind(AppendCallName);
+  const auto AppendCall = expr(ignoringImplicit(AppendCallExpr));
   const auto LoopVarInit =
       declStmt(hasSingleDecl(varDecl(hasInitializer(integerLiteral(equals(0))))
                                  .bind(LoopInitVarName)));
@@ -99,14 +111,16 @@ void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
 
   // Matchers for the loop whose body has only 1 push_back/emplace_back calling
   // statement.
-  const auto HasInterestingLoopBody =
-      hasBody(anyOf(compoundStmt(statementCountIs(1), has(VectorAppendCall)),
-                    VectorAppendCall));
+  const auto HasInterestingLoopBody = hasBody(
+      anyOf(compoundStmt(statementCountIs(1), has(AppendCall)), AppendCall));
   const auto InInterestingCompoundStmt =
-      hasParent(compoundStmt(has(VectorVarDefStmt)).bind(LoopParentName));
+      hasParent(compoundStmt(has(TargetVarDefStmt)).bind(LoopParentName));
 
   // Match counter-based for loops:
-  //  for (int i = 0; i < n; ++i) { v.push_back(...); }
+  //  for (int i = 0; i < n; ++i) {
+  //    v.push_back(...);
+  //    // Or: proto.add_xxx(...);
+  //  }
   //
   // FIXME: Support more types of counter-based loops like decrement loops.
   Finder->addMatcher(
@@ -123,7 +137,10 @@ void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
       this);
 
   // Match for-range loops:
-  //   for (const auto& E : data) { v.push_back(...); }
+  //   for (const auto& E : data) {
+  //     v.push_back(...);
+  //     // Or: proto.add_xxx(...);
+  //   }
   //
   // FIXME: Support more complex range-expressions.
   Finder->addMatcher(
@@ -134,6 +151,28 @@ void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
       this);
 }
 
+void InefficientVectorOperationCheck::registerMatchers(MatchFinder *Finder) {
+  const auto VectorDecl = cxxRecordDecl(hasAnyName(SmallVector<StringRef, 5>(
+      VectorLikeClasses.begin(), VectorLikeClasses.end())));
+  const auto AppendMethodDecl =
+      cxxMethodDecl(hasAnyName("push_back", "emplace_back"));
+  AddMatcher(VectorDecl, VectorVarDeclName, VectorVarDeclStmtName,
+             AppendMethodDecl, PushBackOrEmplaceBackCallName, Finder);
+
+  if (EnableProto) {
+    const auto ProtoDecl =
+        cxxRecordDecl(isDerivedFrom("::proto2::MessageLite"));
+
+    // A method's name starts with "add_" might not mean it's an add field
+    // call; it could be the getter for a proto field of which the name starts
+    // with "add_". So we exlude const methods.
+    const auto AddFieldMethodDecl =
+        cxxMethodDecl(matchesName("::add_"), unless(isConst()));
+    AddMatcher(ProtoDecl, ProtoVarDeclName, ProtoVarDeclStmtName,
+               AddFieldMethodDecl, ProtoAddFieldCallName, Finder);
+  }
+}
+
 void InefficientVectorOperationCheck::check(
     const MatchFinder::MatchResult &Result) {
   auto* Context = Result.Context;
@@ -148,64 +187,84 @@ void InefficientVectorOperationCheck::check(
       Result.Nodes.getNodeAs<CXXForRangeStmt>(RangeLoopName);
   const auto *VectorAppendCall =
       Result.Nodes.getNodeAs<CXXMemberCallExpr>(PushBackOrEmplaceBackCallName);
+  const auto *ProtoVarDecl = Result.Nodes.getNodeAs<VarDecl>(ProtoVarDeclName);
+  const auto *ProtoAddFieldCall =
+      Result.Nodes.getNodeAs<CXXMemberCallExpr>(ProtoAddFieldCallName);
   const auto *LoopEndExpr = Result.Nodes.getNodeAs<Expr>(LoopEndExprName);
   const auto *LoopParent = Result.Nodes.getNodeAs<CompoundStmt>(LoopParentName);
 
+  const CXXMemberCallExpr *AppendCall =
+      VectorAppendCall ? VectorAppendCall : ProtoAddFieldCall;
+  assert(AppendCall && "no append call expression");
+
   const Stmt *LoopStmt = ForLoop;
   if (!LoopStmt)
     LoopStmt = RangeLoop;
 
-  llvm::SmallPtrSet<const DeclRefExpr *, 16> AllVectorVarRefs =
-      utils::decl_ref_expr::allDeclRefExprs(*VectorVarDecl, *LoopParent,
+  const auto *TargetVarDecl = VectorVarDecl;
+  if (!TargetVarDecl)
+    TargetVarDecl = ProtoVarDecl;
+
+  llvm::SmallPtrSet<const DeclRefExpr *, 16> AllVarRefs =
+      utils::decl_ref_expr::allDeclRefExprs(*TargetVarDecl, *LoopParent,
                                             *Context);
-  for (const auto *Ref : AllVectorVarRefs) {
-    // Skip cases where there are usages (defined as DeclRefExpr that refers to
-    // "v") of vector variable `v` before the for loop. We consider these usages
-    // are operations causing memory preallocation (e.g. "v.resize(n)",
-    // "v.reserve(n)").
+  for (const auto *Ref : AllVarRefs) {
+    // Skip cases where there are usages (defined as DeclRefExpr that refers
+    // to "v") of vector variable / proto variable `v` before the for loop. We
+    // consider these usages are operations causing memory preallocation (e.g.
+    // "v.resize(n)", "v.reserve(n)").
     //
-    // FIXME: make it more intelligent to identify the pre-allocating operations
-    // before the for loop.
+    // FIXME: make it more intelligent to identify the pre-allocating
+    // operations before the for loop.
     if (SM.isBeforeInTranslationUnit(Ref->getLocation(),
                                      LoopStmt->getBeginLoc())) {
       return;
     }
   }
 
-  llvm::StringRef VectorVarName = Lexer::getSourceText(
+  std::string PartialReserveStmt;
+  if (VectorAppendCall != nullptr) {
+    PartialReserveStmt = ".reserve";
+  } else {
+    llvm::StringRef FieldName = ProtoAddFieldCall->getMethodDecl()->getName();
+    FieldName.consume_front("add_");
+    std::string MutableFieldName = ("mutable_" + FieldName).str();
+    PartialReserveStmt = "." + MutableFieldName +
+                         "()->Reserve"; // e.g., ".mutable_xxx()->Reserve"
+  }
+
+  llvm::StringRef VarName = Lexer::getSourceText(
       CharSourceRange::getTokenRange(
-          VectorAppendCall->getImplicitObjectArgument()->getSourceRange()),
+          AppendCall->getImplicitObjectArgument()->getSourceRange()),
       SM, Context->getLangOpts());
 
-  std::string ReserveStmt;
+  std::string ReserveSize;
   // Handle for-range loop cases.
   if (RangeLoop) {
     // Get the range-expression in a for-range statement represented as
     // `for (range-declarator: range-expression)`.
-    StringRef RangeInitExpName = Lexer::getSourceText(
-        CharSourceRange::getTokenRange(
-            RangeLoop->getRangeInit()->getSourceRange()),
-        SM, Context->getLangOpts());
-
-    ReserveStmt =
-        (VectorVarName + ".reserve(" + RangeInitExpName + ".size()" + ");\n")
-            .str();
+    StringRef RangeInitExpName =
+        Lexer::getSourceText(CharSourceRange::getTokenRange(
+                                 RangeLoop->getRangeInit()->getSourceRange()),
+                             SM, Context->getLangOpts());
+    ReserveSize = (RangeInitExpName + ".size()").str();
   } else if (ForLoop) {
     // Handle counter-based loop cases.
     StringRef LoopEndSource = Lexer::getSourceText(
         CharSourceRange::getTokenRange(LoopEndExpr->getSourceRange()), SM,
         Context->getLangOpts());
-    ReserveStmt = (VectorVarName + ".reserve(" + LoopEndSource + ");\n").str();
+    ReserveSize = LoopEndSource;
   }
 
-  auto Diag =
-      diag(VectorAppendCall->getBeginLoc(),
-           "%0 is called inside a loop; "
-           "consider pre-allocating the vector capacity before the loop")
-      << VectorAppendCall->getMethodDecl()->getDeclName();
-
-  if (!ReserveStmt.empty())
+  auto Diag = diag(AppendCall->getBeginLoc(),
+                   "%0 is called inside a loop; consider pre-allocating the "
+                   "container capacity before the loop")
+              << AppendCall->getMethodDecl()->getDeclName();
+  if (!ReserveSize.empty()) {
+    std::string ReserveStmt =
+        (VarName + PartialReserveStmt + "(" + ReserveSize + ");\n").str();
     Diag << FixItHint::CreateInsertion(LoopStmt->getBeginLoc(), ReserveStmt);
+  }
 }
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
index bfd84d100c279..e224dcc90685f 100644
--- a/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
+++ b/clang-tools-extra/clang-tidy/performance/InefficientVectorOperationCheck.h
@@ -18,6 +18,9 @@ namespace performance {
 /// Finds possible inefficient `std::vector` operations (e.g. `push_back`) in
 /// for loops that may cause unnecessary memory reallocations.
 ///
+/// When EnableProto, also finds calls that add element to protobuf repeated
+/// field without calling Reserve() first.
+///
 /// For the user-facing documentation see:
 /// http://clang.llvm.org/extra/clang-tidy/checks/performance-inefficient-vector-operation.html
 class InefficientVectorOperationCheck : public ClangTidyCheck {
@@ -28,7 +31,14 @@ class InefficientVectorOperationCheck : public ClangTidyCheck {
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
 
 private:
+  void AddMatcher(const ast_matchers::DeclarationMatcher &TargetRecordDecl,
+                  StringRef VarDeclName, StringRef VarDeclStmtName,
+                  const ast_matchers::DeclarationMatcher &AppendMethodDecl,
+                  StringRef AppendCallName, ast_matchers::MatchFinder *Finder);
   const std::vector<std::string> VectorLikeClasses;
+
+  // If true, also check inefficient operations for proto repeated fields.
+  bool EnableProto;
 };
 
 } // namespace performance
diff --git a/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt b/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
index c92f44df097cb..4adc3f269770d 100644
--- a/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/plugin/CMakeLists.txt
@@ -8,30 +8,6 @@ add_clang_library(clangTidyPlugin
   clangFrontend
   clangSema
   clangTidy
-  clangTidyAbseilModule
-  clangTidyAndroidModule
-  clangTidyBoostModule
-  clangTidyBugproneModule
-  clangTidyCERTModule
-  clangTidyCppCoreGuidelinesModule
-  clangTidyFuchsiaModule
-  clangTidyGoogleModule
-  clangTidyHICPPModule
-  clangTidyLinuxKernelModule
-  clangTidyLLVMModule
-  clangTidyMiscModule
-  clangTidyModernizeModule
-  clangTidyObjCModule
-  clangTidyOpenMPModule
-  clangTidyPerformanceModule
-  clangTidyPortabilityModule
-  clangTidyReadabilityModule
-  clangTidyZirconModule
   clangTooling
+  ${ALL_CLANG_TIDY_CHECKS}
   )
-
-if(CLANG_ENABLE_STATIC_ANALYZER)
-  target_link_libraries(clangTidyPlugin PRIVATE
-    clangTidyMPIModule
-  )
-endif()
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index dc24f663ae353..3228940413257 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -193,6 +193,8 @@ IdentifierNamingCheck::IdentifierNamingCheck(StringRef Name,
   IgnoreFailedSplit = Options.get("IgnoreFailedSplit", 0);
 }
 
+IdentifierNamingCheck::~IdentifierNamingCheck() = default;
+
 void IdentifierNamingCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
   auto const toString = [](CaseType Type) {
     switch (Type) {
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
index 5d5f056049484..250dc361ff74a 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.h
@@ -34,6 +34,7 @@ namespace readability {
 class IdentifierNamingCheck : public ClangTidyCheck {
 public:
   IdentifierNamingCheck(StringRef Name, ClangTidyContext *Context);
+  ~IdentifierNamingCheck();
 
   void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
   void registerMatchers(ast_matchers::MatchFinder *Finder) override;
diff --git a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
index cbd87d15eef87..fc2b4ebd3b410 100644
--- a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt
@@ -17,34 +17,11 @@ target_link_libraries(clang-tidy
   clangASTMatchers
   clangBasic
   clangTidy
-  clangTidyAndroidModule
-  clangTidyAbseilModule
-  clangTidyBoostModule
-  clangTidyBugproneModule
-  clangTidyCERTModule
-  clangTidyCppCoreGuidelinesModule
-  clangTidyFuchsiaModule
-  clangTidyGoogleModule
-  clangTidyHICPPModule
-  clangTidyLinuxKernelModule
-  clangTidyLLVMModule
-  clangTidyMiscModule
-  clangTidyModernizeModule
-  clangTidyObjCModule
-  clangTidyOpenMPModule
-  clangTidyPerformanceModule
-  clangTidyPortabilityModule
-  clangTidyReadabilityModule
-  clangTidyZirconModule
   clangTooling
   clangToolingCore
+  ${ALL_CLANG_TIDY_CHECKS}
   )
 
-if(CLANG_ENABLE_STATIC_ANALYZER)
-  target_link_libraries(clang-tidy PRIVATE
-    clangTidyMPIModule
-  )
-endif()
 
 install(PROGRAMS clang-tidy-diff.py
   DESTINATION share/clang
diff --git a/clang-tools-extra/clang-tidy/utils/LexerUtils.h b/clang-tools-extra/clang-tidy/utils/LexerUtils.h
index 06a03b6e52628..8330266e6341b 100644
--- a/clang-tools-extra/clang-tidy/utils/LexerUtils.h
+++ b/clang-tools-extra/clang-tidy/utils/LexerUtils.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_UTILS_LEXER_UTILS_H
 
 #include "clang/AST/ASTContext.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Lex/Lexer.h"
 
 namespace clang {
@@ -70,6 +71,11 @@ SourceLocation findNextAnyTokenKind(SourceLocation Start,
     if (PotentialMatch.isOneOf(TK, TKs...))
       return PotentialMatch.getLocation();
 
+    // If we reach the end of the file, and eof is not the target token, we stop
+    // the loop, otherwise we will get infinite loop (findNextToken will return
+    // eof on eof).
+    if (PotentialMatch.is(tok::eof))
+      return SourceLocation();
     Start = PotentialMatch.getLastLoc();
   }
 }
diff --git a/clang-tools-extra/clangd/CMakeLists.txt b/clang-tools-extra/clangd/CMakeLists.txt
index 8d622c2c572a6..d428c8ad26945 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -56,6 +56,7 @@ add_clang_library(clangDaemon
   FuzzyMatch.cpp
   GlobalCompilationDatabase.cpp
   Headers.cpp
+  HeaderSourceSwitch.cpp
   IncludeFixer.cpp
   JSONTransport.cpp
   Logger.cpp
@@ -66,6 +67,7 @@ add_clang_library(clangDaemon
   RIFF.cpp
   Selection.cpp
   SemanticHighlighting.cpp
+  SemanticSelection.cpp
   SourceCode.cpp
   QueryDriverDatabase.cpp
   Threading.cpp
@@ -115,25 +117,6 @@ add_clang_library(clangDaemon
   clangSema
   clangSerialization
   clangTidy
-  clangTidyAndroidModule
-  clangTidyAbseilModule
-  clangTidyBoostModule
-  clangTidyBugproneModule
-  clangTidyCERTModule
-  clangTidyCppCoreGuidelinesModule
-  clangTidyFuchsiaModule
-  clangTidyGoogleModule
-  clangTidyHICPPModule
-  clangTidyLinuxKernelModule
-  clangTidyLLVMModule
-  clangTidyMiscModule
-  clangTidyModernizeModule
-  clangTidyObjCModule
-  clangTidyOpenMPModule
-  clangTidyPerformanceModule
-  clangTidyPortabilityModule
-  clangTidyReadabilityModule
-  clangTidyZirconModule
   clangTooling
   clangToolingCore
   clangToolingInclusions
@@ -141,6 +124,7 @@ add_clang_library(clangDaemon
   clangToolingSyntax
   ${LLVM_PTHREAD_LIB}
   ${CLANGD_ATOMIC_LIB}
+  ${ALL_CLANG_TIDY_CHECKS}
   )
 
 add_subdirectory(refactor/tweaks)
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index 0930c80da06f6..692bd24072a47 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -22,14 +22,18 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include <cstddef>
+#include <memory>
 #include <string>
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -127,6 +131,21 @@ llvm::Error validateEdits(const DraftStore &DraftMgr, const Tweak::Effect &E) {
           llvm::to_string(InvalidFileCount - 1) + " others)");
 }
 
+// Converts a list of Ranges to a LinkedList of SelectionRange.
+SelectionRange render(const std::vector<Range> &Ranges) {
+  if (Ranges.empty())
+    return {};
+  SelectionRange Result;
+  Result.range = Ranges[0];
+  auto *Next = &Result.parent;
+  for (const auto &R : llvm::make_range(Ranges.begin() + 1, Ranges.end())) {
+    *Next = std::make_unique<SelectionRange>();
+    Next->get()->range = R;
+    Next = &Next->get()->parent;
+  }
+  return Result;
+}
+
 } // namespace
 
 // MessageHandler dispatches incoming LSP messages.
@@ -536,6 +555,7 @@ void ClangdLSPServer::onInitialize(const InitializeParams &Params,
             {"documentHighlightProvider", true},
             {"hoverProvider", true},
             {"renameProvider", std::move(RenameProvider)},
+            {"selectionRangeProvider", true},
             {"documentSymbolProvider", true},
             {"workspaceSymbolProvider", true},
             {"referencesProvider", true},
@@ -680,7 +700,7 @@ void ClangdLSPServer::onCommand(const ExecuteCommandParams &Params,
       WorkspaceEdit WE;
       WE.changes.emplace();
       for (const auto &It : R->ApplyEdits) {
-        (*WE.changes)[URI::create(It.first()).toString()] =
+        (*WE.changes)[URI::createFile(It.first()).toString()] =
             It.second.asTextEdits();
       }
       // ApplyEdit will take care of calling Reply().
@@ -1018,10 +1038,16 @@ void ClangdLSPServer::onGoToDeclaration(
 void ClangdLSPServer::onSwitchSourceHeader(
     const TextDocumentIdentifier &Params,
     Callback<llvm::Optional<URIForFile>> Reply) {
-  if (auto Result = Server->switchSourceHeader(Params.uri.file()))
-    Reply(URIForFile::canonicalize(*Result, Params.uri.file()));
-  else
-    Reply(llvm::None);
+  Server->switchSourceHeader(
+      Params.uri.file(),
+      [Reply = std::move(Reply),
+       Params](llvm::Expected<llvm::Optional<clangd::Path>> Path) mutable {
+        if (!Path)
+          return Reply(Path.takeError());
+        if (*Path)
+          Reply(URIForFile::canonicalize(**Path, Params.uri.file()));
+        return Reply(llvm::None);
+      });
 }
 
 void ClangdLSPServer::onDocumentHighlight(
@@ -1125,6 +1151,30 @@ void ClangdLSPServer::onSymbolInfo(const TextDocumentPositionParams &Params,
                      std::move(Reply));
 }
 
+void ClangdLSPServer::onSelectionRange(
+    const SelectionRangeParams &Params,
+    Callback<std::vector<SelectionRange>> Reply) {
+  if (Params.positions.size() != 1) {
+    elog("{0} positions provided to SelectionRange. Supports exactly one "
+         "position.",
+         Params.positions.size());
+    return Reply(llvm::make_error<LSPError>(
+        "SelectionRange supports exactly one position",
+        ErrorCode::InvalidRequest));
+  }
+  Server->semanticRanges(
+      Params.textDocument.uri.file(), Params.positions[0],
+      [Reply = std::move(Reply)](
+          llvm::Expected<std::vector<Range>> Ranges) mutable {
+        if (!Ranges) {
+          return Reply(Ranges.takeError());
+        }
+        std::vector<SelectionRange> Result;
+        Result.emplace_back(render(std::move(*Ranges)));
+        return Reply(std::move(Result));
+      });
+}
+
 ClangdLSPServer::ClangdLSPServer(
     class Transport &Transp, const FileSystemProvider &FSProvider,
     const clangd::CodeCompleteOptions &CCOpts,
@@ -1167,6 +1217,7 @@ ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("textDocument/symbolInfo", &ClangdLSPServer::onSymbolInfo);
   MsgHandler->bind("textDocument/typeHierarchy", &ClangdLSPServer::onTypeHierarchy);
   MsgHandler->bind("typeHierarchy/resolve", &ClangdLSPServer::onResolveTypeHierarchy);
+  MsgHandler->bind("textDocument/selectionRange", &ClangdLSPServer::onSelectionRange);
   // clang-format on
 }
 
diff --git a/clang-tools-extra/clangd/ClangdLSPServer.h b/clang-tools-extra/clangd/ClangdLSPServer.h
index 4e7e042bb9ada..3f9f1cc9c90dc 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -107,6 +107,8 @@ class ClangdLSPServer : private DiagnosticsConsumer {
   void onChangeConfiguration(const DidChangeConfigurationParams &);
   void onSymbolInfo(const TextDocumentPositionParams &,
                     Callback<std::vector<SymbolDetails>>);
+  void onSelectionRange(const SelectionRangeParams &,
+                        Callback<std::vector<SelectionRange>>);
 
   std::vector<Fix> getFixes(StringRef File, const clangd::Diagnostic &D);
 
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 5b850e6ab4772..b59bb4136259c 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -11,12 +11,14 @@
 #include "FindSymbols.h"
 #include "Format.h"
 #include "FormattedString.h"
+#include "HeaderSourceSwitch.h"
 #include "Headers.h"
 #include "Logger.h"
 #include "ParsedAST.h"
 #include "Preamble.h"
 #include "Protocol.h"
 #include "SemanticHighlighting.h"
+#include "SemanticSelection.h"
 #include "SourceCode.h"
 #include "TUScheduler.h"
 #include "Trace.h"
@@ -125,8 +127,8 @@ ClangdServer::ClangdServer(const GlobalCompilationDatabase &CDB,
       // critical paths.
       WorkScheduler(
           CDB, Opts.AsyncThreadsCount, Opts.StorePreamblesInMemory,
-          std::make_unique<UpdateIndexCallbacks>(
-              DynamicIdx.get(), DiagConsumer, Opts.SemanticHighlighting),
+          std::make_unique<UpdateIndexCallbacks>(DynamicIdx.get(), DiagConsumer,
+                                                 Opts.SemanticHighlighting),
           Opts.UpdateDebounce, Opts.RetentionPolicy) {
   // Adds an index to the stack, at higher priority than existing indexes.
   auto AddIndex = [&](SymbolIndex *Idx) {
@@ -447,61 +449,24 @@ void ClangdServer::locateSymbolAt(PathRef File, Position Pos,
   WorkScheduler.runWithAST("Definitions", File, std::move(Action));
 }
 
-llvm::Optional<Path> ClangdServer::switchSourceHeader(PathRef Path) {
-
-  llvm::StringRef SourceExtensions[] = {".cpp", ".c", ".cc", ".cxx",
-                                        ".c++", ".m", ".mm"};
-  llvm::StringRef HeaderExtensions[] = {".h", ".hh", ".hpp", ".hxx", ".inc"};
-
-  llvm::StringRef PathExt = llvm::sys::path::extension(Path);
-
-  // Lookup in a list of known extensions.
-  auto SourceIter =
-      llvm::find_if(SourceExtensions, [&PathExt](PathRef SourceExt) {
-        return SourceExt.equals_lower(PathExt);
-      });
-  bool IsSource = SourceIter != std::end(SourceExtensions);
-
-  auto HeaderIter =
-      llvm::find_if(HeaderExtensions, [&PathExt](PathRef HeaderExt) {
-        return HeaderExt.equals_lower(PathExt);
-      });
-
-  bool IsHeader = HeaderIter != std::end(HeaderExtensions);
-
-  // We can only switch between the known extensions.
-  if (!IsSource && !IsHeader)
-    return None;
-
-  // Array to lookup extensions for the switch. An opposite of where original
-  // extension was found.
-  llvm::ArrayRef<llvm::StringRef> NewExts;
-  if (IsSource)
-    NewExts = HeaderExtensions;
-  else
-    NewExts = SourceExtensions;
-
-  // Storage for the new path.
-  llvm::SmallString<128> NewPath = llvm::StringRef(Path);
-
-  // Instance of vfs::FileSystem, used for file existence checks.
-  auto FS = FSProvider.getFileSystem();
-
-  // Loop through switched extension candidates.
-  for (llvm::StringRef NewExt : NewExts) {
-    llvm::sys::path::replace_extension(NewPath, NewExt);
-    if (FS->exists(NewPath))
-      return NewPath.str().str(); // First str() to convert from SmallString to
-                                  // StringRef, second to convert from StringRef
-                                  // to std::string
-
-    // Also check NewExt in upper-case, just in case.
-    llvm::sys::path::replace_extension(NewPath, NewExt.upper());
-    if (FS->exists(NewPath))
-      return NewPath.str().str();
-  }
-
-  return None;
+void ClangdServer::switchSourceHeader(
+    PathRef Path, Callback<llvm::Optional<clangd::Path>> CB) {
+  // We want to return the result as fast as possible, stragety is:
+  //  1) use the file-only heuristic, it requires some IO but it is much
+  //     faster than building AST, but it only works when .h/.cc files are in
+  //     the same directory.
+  //  2) if 1) fails, we use the AST&Index approach, it is slower but supports
+  //     different code layout.
+  if (auto CorrespondingFile =
+          getCorrespondingHeaderOrSource(Path, FSProvider.getFileSystem()))
+    return CB(std::move(CorrespondingFile));
+  auto Action = [Path, CB = std::move(CB),
+                 this](llvm::Expected<InputsAndAST> InpAST) mutable {
+    if (!InpAST)
+      return CB(InpAST.takeError());
+    CB(getCorrespondingHeaderOrSource(Path, InpAST->AST, Index));
+  };
+  WorkScheduler.runWithAST("SwitchHeaderSource", Path, std::move(Action));
 }
 
 llvm::Expected<tooling::Replacements>
@@ -620,6 +585,17 @@ void ClangdServer::symbolInfo(PathRef File, Position Pos,
   WorkScheduler.runWithAST("SymbolInfo", File, std::move(Action));
 }
 
+void ClangdServer::semanticRanges(PathRef File, Position Pos,
+                                  Callback<std::vector<Range>> CB) {
+  auto Action =
+      [Pos, CB = std::move(CB)](llvm::Expected<InputsAndAST> InpAST) mutable {
+        if (!InpAST)
+          return CB(InpAST.takeError());
+        CB(clangd::getSemanticRanges(InpAST->AST, Pos));
+      };
+  WorkScheduler.runWithAST("SemanticRanges", File, std::move(Action));
+}
+
 std::vector<std::pair<Path, std::size_t>>
 ClangdServer::getUsedBytesPerFile() const {
   return WorkScheduler.getUsedBytesPerFile();
diff --git a/clang-tools-extra/clangd/ClangdServer.h b/clang-tools-extra/clangd/ClangdServer.h
index 83a74ff54c51b..c04dc50849f8f 100644
--- a/clang-tools-extra/clangd/ClangdServer.h
+++ b/clang-tools-extra/clangd/ClangdServer.h
@@ -51,8 +51,6 @@ class DiagnosticsConsumer {
   virtual void onFileUpdated(PathRef File, const TUStatus &Status){};
 
   /// Called by ClangdServer when some \p Highlightings for \p File are ready.
-  /// \p NumLines are the number of lines in the file where the highlightings
-  /// where generated from.
   virtual void
   onHighlightingsReady(PathRef File,
                        std::vector<HighlightingToken> Highlightings) {}
@@ -194,9 +192,10 @@ class ClangdServer {
   void locateSymbolAt(PathRef File, Position Pos,
                       Callback<std::vector<LocatedSymbol>> CB);
 
-  /// Helper function that returns a path to the corresponding source file when
-  /// given a header file and vice versa.
-  llvm::Optional<Path> switchSourceHeader(PathRef Path);
+  /// Switch to a corresponding source file when given a header file, and vice
+  /// versa.
+  void switchSourceHeader(PathRef Path,
+                          Callback<llvm::Optional<clangd::Path>> CB);
 
   /// Get document highlights for a given position.
   void findDocumentHighlights(PathRef File, Position Pos,
@@ -279,6 +278,10 @@ class ClangdServer {
   void symbolInfo(PathRef File, Position Pos,
                   Callback<std::vector<SymbolDetails>> CB);
 
+  /// Get semantic ranges around a specified position in a file.
+  void semanticRanges(PathRef File, Position Pos,
+                      Callback<std::vector<Range>> CB);
+
   /// Returns estimated memory usage for each of the currently open files.
   /// The order of results is unspecified.
   /// Overall memory usage of clangd may be significantly more than reported
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index b96636aaf63be..bc826ba0adcd6 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -318,11 +318,8 @@ struct CodeCompletionBuilder {
     // Turn absolute path into a literal string that can be #included.
     auto Inserted = [&](llvm::StringRef Header)
         -> llvm::Expected<std::pair<std::string, bool>> {
-      auto DeclaringURI =
-          URI::parse(C.IndexResult->CanonicalDeclaration.FileURI);
-      if (!DeclaringURI)
-        return DeclaringURI.takeError();
-      auto ResolvedDeclaring = URI::resolve(*DeclaringURI, FileName);
+      auto ResolvedDeclaring =
+          URI::resolve(C.IndexResult->CanonicalDeclaration.FileURI, FileName);
       if (!ResolvedDeclaring)
         return ResolvedDeclaring.takeError();
       auto ResolvedInserted = toHeaderFile(Header, FileName);
@@ -1033,8 +1030,8 @@ void loadMainFilePreambleMacros(const Preprocessor &PP,
       PP.getIdentifierTable().getExternalIdentifierLookup();
   if (!PreambleIdentifiers || !PreambleMacros)
     return;
-  for (const auto &MacroName : Preamble.MainFileMacros)
-    if (auto *II = PreambleIdentifiers->get(MacroName))
+  for (const auto &MacroName : Preamble.Macros.Names)
+    if (auto *II = PreambleIdentifiers->get(MacroName.getKey()))
       if (II->isOutOfDate())
         PreambleMacros->updateOutOfDateIdentifier(*II);
 }
diff --git a/clang-tools-extra/clangd/CollectMacros.h b/clang-tools-extra/clangd/CollectMacros.h
new file mode 100644
index 0000000000000..21227e1ed570f
--- /dev/null
+++ b/clang-tools-extra/clangd/CollectMacros.h
@@ -0,0 +1,74 @@
+//===--- CollectMacros.h -----------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_COLLECTEDMACROS_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_COLLECTEDMACROS_H
+
+#include "Protocol.h"
+#include "SourceCode.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Lex/PPCallbacks.h"
+#include <string>
+
+namespace clang {
+namespace clangd {
+
+struct MainFileMacros {
+  llvm::StringSet<> Names;
+  // Instead of storing SourceLocation, we have to store the token range because
+  // SourceManager from preamble is not available when we build the AST.
+  std::vector<Range> Ranges;
+};
+
+/// Collects macro definitions and expansions in the main file. It is used to:
+///  - collect macros in the preamble section of the main file (in Preamble.cpp)
+///  - collect macros after the preamble of the main file (in ParsedAST.cpp)
+class CollectMainFileMacros : public PPCallbacks {
+public:
+  explicit CollectMainFileMacros(const SourceManager &SM,
+                                 const LangOptions &LangOpts,
+                                 MainFileMacros &Out)
+      : SM(SM), LangOpts(LangOpts), Out(Out) {}
+
+  void FileChanged(SourceLocation Loc, FileChangeReason,
+                   SrcMgr::CharacteristicKind, FileID) override {
+    InMainFile = isInsideMainFile(Loc, SM);
+  }
+
+  void MacroDefined(const Token &MacroName, const MacroDirective *MD) override {
+    add(MacroName, MD->getMacroInfo());
+  }
+
+  void MacroExpands(const Token &MacroName, const MacroDefinition &MD,
+                    SourceRange Range, const MacroArgs *Args) override {
+    add(MacroName, MD.getMacroInfo());
+  }
+
+private:
+  void add(const Token &MacroNameTok, const MacroInfo *MI) {
+    if (!InMainFile)
+      return;
+    auto Loc = MacroNameTok.getLocation();
+    if (Loc.isMacroID())
+      return;
+
+    if (auto Range = getTokenRange(SM, LangOpts, MacroNameTok.getLocation())) {
+      Out.Names.insert(MacroNameTok.getIdentifierInfo()->getName());
+      Out.Ranges.push_back(*Range);
+    }
+  }
+  const SourceManager &SM;
+  const LangOptions &LangOpts;
+  bool InMainFile = true;
+  MainFileMacros &Out;
+};
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_COLLECTEDMACROS_H
diff --git a/clang-tools-extra/clangd/FileDistance.cpp b/clang-tools-extra/clangd/FileDistance.cpp
index a6a65ab7ead25..a9ce3e514bcc1 100644
--- a/clang-tools-extra/clangd/FileDistance.cpp
+++ b/clang-tools-extra/clangd/FileDistance.cpp
@@ -208,7 +208,7 @@ createScopeFileDistance(llvm::ArrayRef<std::string> QueryScopes) {
     Param.MaxUpTraversals = std::max(Path.second - 1, 0);
     Sources[Path.first] = std::move(Param);
   }
-  return FileDistance(Sources, Opts);
+  return FileDistance(std::move(Sources), Opts);
 }
 
 ScopeDistance::ScopeDistance(llvm::ArrayRef<std::string> QueryScopes)
diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index 287a8a4df0b43..619f1a5cfdbcd 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -43,18 +43,11 @@ llvm::Expected<Location> symbolToLocation(const Symbol &Sym,
                                           llvm::StringRef HintPath) {
   // Prefer the definition over e.g. a function declaration in a header
   auto &CD = Sym.Definition ? Sym.Definition : Sym.CanonicalDeclaration;
-  auto Uri = URI::parse(CD.FileURI);
-  if (!Uri) {
-    return llvm::make_error<llvm::StringError>(
-        formatv("Could not parse URI '{0}' for symbol '{1}'.", CD.FileURI,
-                Sym.Name),
-        llvm::inconvertibleErrorCode());
-  }
-  auto Path = URI::resolve(*Uri, HintPath);
+  auto Path = URI::resolve(CD.FileURI, HintPath);
   if (!Path) {
     return llvm::make_error<llvm::StringError>(
-        formatv("Could not resolve path for URI '{0}' for symbol '{1}'.",
-                Uri->toString(), Sym.Name),
+        formatv("Could not resolve path for symbol '{0}': {1}",
+                Sym.Name, llvm::toString(Path.takeError())),
         llvm::inconvertibleErrorCode());
   }
   Location L;
diff --git a/clang-tools-extra/clangd/FindTarget.cpp b/clang-tools-extra/clangd/FindTarget.cpp
index 2631185b47c89..ffe808d579637 100644
--- a/clang-tools-extra/clangd/FindTarget.cpp
+++ b/clang-tools-extra/clangd/FindTarget.cpp
@@ -10,22 +10,35 @@
 #include "AST.h"
 #include "Logger.h"
 #include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/DeclVisitor.h"
 #include "clang/AST/DeclarationName.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
 #include "clang/AST/ExprObjC.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/PrettyPrinter.h"
+#include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/StmtVisitor.h"
+#include "clang/AST/TemplateBase.h"
 #include "clang/AST/Type.h"
+#include "clang/AST/TypeLoc.h"
 #include "clang/AST/TypeLocVisitor.h"
+#include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/raw_ostream.h"
+#include <utility>
 
 namespace clang {
 namespace clangd {
 namespace {
+using ast_type_traits::DynTypedNode;
 
 LLVM_ATTRIBUTE_UNUSED std::string
 nodeToString(const ast_type_traits::DynTypedNode &N) {
@@ -177,6 +190,10 @@ struct TargetFinder {
           D = USD;
         Outer.add(D, Flags);
       }
+      void VisitOverloadExpr(const OverloadExpr *OE) {
+        for (auto *D : OE->decls())
+          Outer.add(D, Flags);
+      }
       void VisitCXXConstructExpr(const CXXConstructExpr *CCE) {
         Outer.add(CCE->getConstructor(), Flags);
       }
@@ -348,12 +365,326 @@ allTargetDecls(const ast_type_traits::DynTypedNode &N) {
 llvm::SmallVector<const Decl *, 1>
 targetDecl(const ast_type_traits::DynTypedNode &N, DeclRelationSet Mask) {
   llvm::SmallVector<const Decl *, 1> Result;
-  for (const auto &Entry : allTargetDecls(N))
+  for (const auto &Entry : allTargetDecls(N)) {
     if (!(Entry.second & ~Mask))
       Result.push_back(Entry.first);
+  }
   return Result;
 }
 
+namespace {
+/// Find declarations explicitly referenced in the source code defined by \p N.
+/// For templates, will prefer to return a template instantiation whenever
+/// possible. However, can also return a template pattern if the specialization
+/// cannot be picked, e.g. in dependent code or when there is no corresponding
+/// Decl for a template instantitation, e.g. for templated using decls:
+///    template <class T> using Ptr = T*;
+///    Ptr<int> x;
+///    ^~~ there is no Decl for 'Ptr<int>', so we return the template pattern.
+llvm::SmallVector<const NamedDecl *, 1>
+explicitReferenceTargets(DynTypedNode N, DeclRelationSet Mask = {}) {
+  assert(!(Mask & (DeclRelation::TemplatePattern |
+                   DeclRelation::TemplateInstantiation)) &&
+         "explicitRefenceTargets handles templates on its own");
+  auto Decls = allTargetDecls(N);
+
+  // We prefer to return template instantiation, but fallback to template
+  // pattern if instantiation is not available.
+  Mask |= DeclRelation::TemplatePattern | DeclRelation::TemplateInstantiation;
+
+  llvm::SmallVector<const NamedDecl *, 1> TemplatePatterns;
+  llvm::SmallVector<const NamedDecl *, 1> Targets;
+  bool SeenTemplateInstantiations = false;
+  for (auto &D : Decls) {
+    if (D.second & ~Mask)
+      continue;
+    if (D.second & DeclRelation::TemplatePattern) {
+      TemplatePatterns.push_back(llvm::cast<NamedDecl>(D.first));
+      continue;
+    }
+    if (D.second & DeclRelation::TemplateInstantiation)
+      SeenTemplateInstantiations = true;
+    Targets.push_back(llvm::cast<NamedDecl>(D.first));
+  }
+  if (!SeenTemplateInstantiations)
+    Targets.insert(Targets.end(), TemplatePatterns.begin(),
+                   TemplatePatterns.end());
+  return Targets;
+}
+
+Optional<ReferenceLoc> refInDecl(const Decl *D) {
+  struct Visitor : ConstDeclVisitor<Visitor> {
+    llvm::Optional<ReferenceLoc> Ref;
+
+    void VisitUsingDirectiveDecl(const UsingDirectiveDecl *D) {
+      Ref = ReferenceLoc{D->getQualifierLoc(),
+                         D->getIdentLocation(),
+                         {D->getNominatedNamespaceAsWritten()}};
+    }
+
+    void VisitUsingDecl(const UsingDecl *D) {
+      Ref = ReferenceLoc{D->getQualifierLoc(), D->getLocation(),
+                         explicitReferenceTargets(DynTypedNode::create(*D),
+                                                  DeclRelation::Underlying)};
+    }
+
+    void VisitNamespaceAliasDecl(const NamespaceAliasDecl *D) {
+      Ref = ReferenceLoc{D->getQualifierLoc(),
+                         D->getTargetNameLoc(),
+                         {D->getAliasedNamespace()}};
+    }
+  };
+
+  Visitor V;
+  V.Visit(D);
+  return V.Ref;
+}
+
+Optional<ReferenceLoc> refInExpr(const Expr *E) {
+  struct Visitor : ConstStmtVisitor<Visitor> {
+    // FIXME: handle more complicated cases, e.g. ObjC, designated initializers.
+    llvm::Optional<ReferenceLoc> Ref;
+
+    void VisitDeclRefExpr(const DeclRefExpr *E) {
+      Ref = ReferenceLoc{
+          E->getQualifierLoc(), E->getNameInfo().getLoc(), {E->getFoundDecl()}};
+    }
+
+    void VisitMemberExpr(const MemberExpr *E) {
+      Ref = ReferenceLoc{E->getQualifierLoc(),
+                         E->getMemberNameInfo().getLoc(),
+                         {E->getFoundDecl()}};
+    }
+
+    void VisitOverloadExpr(const OverloadExpr *E) {
+      Ref = ReferenceLoc{E->getQualifierLoc(), E->getNameInfo().getLoc(),
+                         llvm::SmallVector<const NamedDecl *, 1>(
+                             E->decls().begin(), E->decls().end())};
+    }
+  };
+
+  Visitor V;
+  V.Visit(E);
+  return V.Ref;
+}
+
+Optional<ReferenceLoc> refInTypeLoc(TypeLoc L) {
+  struct Visitor : TypeLocVisitor<Visitor> {
+    llvm::Optional<ReferenceLoc> Ref;
+
+    void VisitElaboratedTypeLoc(ElaboratedTypeLoc L) {
+      // We only know about qualifier, rest if filled by inner locations.
+      Visit(L.getNamedTypeLoc().getUnqualifiedLoc());
+      // Fill in the qualifier.
+      if (!Ref)
+        return;
+      assert(!Ref->Qualifier.hasQualifier() && "qualifier already set");
+      Ref->Qualifier = L.getQualifierLoc();
+    }
+
+    void VisitTagTypeLoc(TagTypeLoc L) {
+      Ref =
+          ReferenceLoc{NestedNameSpecifierLoc(), L.getNameLoc(), {L.getDecl()}};
+    }
+
+    void VisitTemplateTypeParmTypeLoc(TemplateTypeParmTypeLoc L) {
+      Ref =
+          ReferenceLoc{NestedNameSpecifierLoc(), L.getNameLoc(), {L.getDecl()}};
+    }
+
+    void VisitTemplateSpecializationTypeLoc(TemplateSpecializationTypeLoc L) {
+      // We must ensure template type aliases are included in results if they
+      // were written in the source code, e.g. in
+      //    template <class T> using valias = vector<T>;
+      //    ^valias<int> x;
+      // 'explicitReferenceTargets' will return:
+      //    1. valias with mask 'Alias'.
+      //    2. 'vector<int>' with mask 'Underlying'.
+      //  we want to return only #1 in this case.
+      Ref = ReferenceLoc{
+          NestedNameSpecifierLoc(), L.getTemplateNameLoc(),
+          explicitReferenceTargets(DynTypedNode::create(L.getType()),
+                                   DeclRelation::Alias)};
+    }
+    void VisitDeducedTemplateSpecializationTypeLoc(
+        DeducedTemplateSpecializationTypeLoc L) {
+      Ref = ReferenceLoc{
+          NestedNameSpecifierLoc(), L.getNameLoc(),
+          explicitReferenceTargets(DynTypedNode::create(L.getType()),
+                                   DeclRelation::Alias)};
+    }
+
+    void VisitDependentTemplateSpecializationTypeLoc(
+        DependentTemplateSpecializationTypeLoc L) {
+      Ref = ReferenceLoc{
+          L.getQualifierLoc(), L.getTemplateNameLoc(),
+          explicitReferenceTargets(DynTypedNode::create(L.getType()))};
+    }
+
+    void VisitDependentNameTypeLoc(DependentNameTypeLoc L) {
+      Ref = ReferenceLoc{
+          L.getQualifierLoc(), L.getNameLoc(),
+          explicitReferenceTargets(DynTypedNode::create(L.getType()))};
+    }
+
+    void VisitTypedefTypeLoc(TypedefTypeLoc L) {
+      Ref = ReferenceLoc{
+          NestedNameSpecifierLoc(), L.getNameLoc(), {L.getTypedefNameDecl()}};
+    }
+  };
+
+  Visitor V;
+  V.Visit(L.getUnqualifiedLoc());
+  return V.Ref;
+}
+
+class ExplicitReferenceColletor
+    : public RecursiveASTVisitor<ExplicitReferenceColletor> {
+public:
+  ExplicitReferenceColletor(llvm::function_ref<void(ReferenceLoc)> Out)
+      : Out(Out) {
+    assert(Out);
+  }
+
+  bool VisitTypeLoc(TypeLoc TTL) {
+    if (TypeLocsToSkip.count(TTL.getBeginLoc().getRawEncoding()))
+      return true;
+    visitNode(DynTypedNode::create(TTL));
+    return true;
+  }
+
+  bool TraverseElaboratedTypeLoc(ElaboratedTypeLoc L) {
+    // ElaboratedTypeLoc will reports information for its inner type loc.
+    // Otherwise we loose information about inner types loc's qualifier.
+    TypeLoc Inner = L.getNamedTypeLoc().getUnqualifiedLoc();
+    TypeLocsToSkip.insert(Inner.getBeginLoc().getRawEncoding());
+    return RecursiveASTVisitor::TraverseElaboratedTypeLoc(L);
+  }
+
+  bool VisitExpr(Expr *E) {
+    visitNode(DynTypedNode::create(*E));
+    return true;
+  }
+
+  // We re-define Traverse*, since there's no corresponding Visit*.
+  // TemplateArgumentLoc is the only way to get locations for references to
+  // template template parameters.
+  bool TraverseTemplateArgumentLoc(TemplateArgumentLoc A) {
+    switch (A.getArgument().getKind()) {
+    case TemplateArgument::Template:
+    case TemplateArgument::TemplateExpansion:
+      reportReference(ReferenceLoc{A.getTemplateQualifierLoc(),
+                                   A.getTemplateNameLoc(),
+                                   {A.getArgument()
+                                        .getAsTemplateOrTemplatePattern()
+                                        .getAsTemplateDecl()}},
+                      DynTypedNode::create(A.getArgument()));
+      break;
+    case TemplateArgument::Declaration:
+      break; // FIXME: can this actually happen in TemplateArgumentLoc?
+    case TemplateArgument::Integral:
+    case TemplateArgument::Null:
+    case TemplateArgument::NullPtr:
+      break; // no references.
+    case TemplateArgument::Pack:
+    case TemplateArgument::Type:
+    case TemplateArgument::Expression:
+      break; // Handled by VisitType and VisitExpression.
+    };
+    return RecursiveASTVisitor::TraverseTemplateArgumentLoc(A);
+  }
+
+  bool VisitDecl(Decl *D) {
+    visitNode(DynTypedNode::create(*D));
+    return true;
+  }
+
+  // We have to use Traverse* because there is no corresponding Visit*.
+  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc L) {
+    if (!L.getNestedNameSpecifier())
+      return true;
+    visitNode(DynTypedNode::create(L));
+    // Inner type is missing information about its qualifier, skip it.
+    if (auto TL = L.getTypeLoc())
+      TypeLocsToSkip.insert(TL.getBeginLoc().getRawEncoding());
+    return RecursiveASTVisitor::TraverseNestedNameSpecifierLoc(L);
+  }
+
+private:
+  /// Obtain information about a reference directly defined in \p N. Does not
+  /// recurse into child nodes, e.g. do not expect references for constructor
+  /// initializers
+  ///
+  /// Any of the fields in the returned structure can be empty, but not all of
+  /// them, e.g.
+  ///   - for implicitly generated nodes (e.g. MemberExpr from range-based-for),
+  ///     source location information may be missing,
+  ///   - for dependent code, targets may be empty.
+  ///
+  /// (!) For the purposes of this function declarations are not considered to
+  ///     be references. However, declarations can have references inside them,
+  ///     e.g. 'namespace foo = std' references namespace 'std' and this
+  ///     function will return the corresponding reference.
+  llvm::Optional<ReferenceLoc> explicitReference(DynTypedNode N) {
+    if (auto *D = N.get<Decl>())
+      return refInDecl(D);
+    if (auto *E = N.get<Expr>())
+      return refInExpr(E);
+    if (auto *NNSL = N.get<NestedNameSpecifierLoc>())
+      return ReferenceLoc{NNSL->getPrefix(), NNSL->getLocalBeginLoc(),
+                          explicitReferenceTargets(DynTypedNode::create(
+                              *NNSL->getNestedNameSpecifier()))};
+    if (const TypeLoc *TL = N.get<TypeLoc>())
+      return refInTypeLoc(*TL);
+    if (const CXXCtorInitializer *CCI = N.get<CXXCtorInitializer>()) {
+      if (CCI->isBaseInitializer())
+        return refInTypeLoc(CCI->getBaseClassLoc());
+      assert(CCI->isAnyMemberInitializer());
+      return ReferenceLoc{NestedNameSpecifierLoc(),
+                          CCI->getMemberLocation(),
+                          {CCI->getAnyMember()}};
+    }
+    // We do not have location information for other nodes (QualType, etc)
+    return llvm::None;
+  }
+
+  void visitNode(DynTypedNode N) {
+    auto Ref = explicitReference(N);
+    if (!Ref)
+      return;
+    reportReference(*Ref, N);
+  }
+
+  void reportReference(const ReferenceLoc &Ref, DynTypedNode N) {
+    // Our promise is to return only references from the source code. If we lack
+    // location information, skip these nodes.
+    // Normally this should not happen in practice, unless there are bugs in the
+    // traversals or users started the traversal at an implicit node.
+    if (Ref.NameLoc.isInvalid()) {
+      dlog("invalid location at node {0}", nodeToString(N));
+      return;
+    }
+    Out(Ref);
+  }
+
+  llvm::function_ref<void(ReferenceLoc)> Out;
+  /// TypeLocs starting at these locations must be skipped, see
+  /// TraverseElaboratedTypeSpecifierLoc for details.
+  llvm::DenseSet</*SourceLocation*/ unsigned> TypeLocsToSkip;
+};
+} // namespace
+
+void findExplicitReferences(const Stmt *S,
+                            llvm::function_ref<void(ReferenceLoc)> Out) {
+  assert(S);
+  ExplicitReferenceColletor(Out).TraverseStmt(const_cast<Stmt *>(S));
+}
+void findExplicitReferences(const Decl *D,
+                            llvm::function_ref<void(ReferenceLoc)> Out) {
+  assert(D);
+  ExplicitReferenceColletor(Out).TraverseDecl(const_cast<Decl *>(D));
+}
+
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, DeclRelation R) {
   switch (R) {
 #define REL_CASE(X)                                                            \
@@ -378,5 +709,26 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, DeclRelationSet RS) {
   return OS;
 }
 
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, ReferenceLoc R) {
+  // note we cannot print R.NameLoc without a source manager.
+  OS << "targets = {";
+  bool First = true;
+  for (const NamedDecl *T : R.Targets) {
+    if (!First)
+      OS << ", ";
+    else
+      First = false;
+    OS << printQualifiedName(*T) << printTemplateSpecializationArgs(*T);
+  }
+  OS << "}";
+  if (R.Qualifier) {
+    OS << ", qualifier = '";
+    R.Qualifier.getNestedNameSpecifier()->print(OS,
+                                                PrintingPolicy(LangOptions()));
+    OS << "'";
+  }
+  return OS;
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/FindTarget.h b/clang-tools-extra/clangd/FindTarget.h
index 1524603fa306f..80a8e787d6250 100644
--- a/clang-tools-extra/clangd/FindTarget.h
+++ b/clang-tools-extra/clangd/FindTarget.h
@@ -19,9 +19,18 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_FINDTARGET_H
+#define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_FINDTARGET_H
+
 #include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/NestedNameSpecifier.h"
+#include "clang/AST/Stmt.h"
 #include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include <bitset>
 
@@ -69,6 +78,34 @@ class DeclRelationSet;
 llvm::SmallVector<const Decl *, 1>
 targetDecl(const ast_type_traits::DynTypedNode &, DeclRelationSet Mask);
 
+/// Information about a reference written in the source code, independent of the
+/// actual AST node that this reference lives in.
+/// Useful for tools that are source-aware, e.g. refactorings.
+struct ReferenceLoc {
+  /// Contains qualifier written in the code, if any, e.g. 'ns::' for 'ns::foo'.
+  NestedNameSpecifierLoc Qualifier;
+  /// Start location of the last name part, i.e. 'foo' in 'ns::foo<int>'.
+  SourceLocation NameLoc;
+  // FIXME: add info about template arguments.
+  /// A list of targets referenced by this name. Normally this has a single
+  /// element, but multiple is also possible, e.g. in case of using declarations
+  /// or unresolved overloaded functions.
+  /// For dependent and unresolved references, Targets can also be empty.
+  llvm::SmallVector<const NamedDecl *, 1> Targets;
+};
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, ReferenceLoc R);
+
+/// Recursively traverse \p S and report all references explicitly written in
+/// the code. The main use-case is refactorings that need to process all
+/// references in some subrange of the file and apply simple edits, e.g. add
+/// qualifiers.
+/// FIXME: currently this does not report references to overloaded operators.
+/// FIXME: extend to report location information about declaration names too.
+void findExplicitReferences(const Stmt *S,
+                            llvm::function_ref<void(ReferenceLoc)> Out);
+void findExplicitReferences(const Decl *D,
+                            llvm::function_ref<void(ReferenceLoc)> Out);
+
 /// Similar to targetDecl(), however instead of applying a filter, all possible
 /// decls are returned along with their DeclRelationSets.
 /// This is suitable for indexing, where everything is recorded and filtering
@@ -142,3 +179,5 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &, DeclRelationSet);
 
 } // namespace clangd
 } // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_FINDTARGET_H
diff --git a/clang-tools-extra/clangd/HeaderSourceSwitch.cpp b/clang-tools-extra/clangd/HeaderSourceSwitch.cpp
new file mode 100644
index 0000000000000..535c8d6d8e1dc
--- /dev/null
+++ b/clang-tools-extra/clangd/HeaderSourceSwitch.cpp
@@ -0,0 +1,153 @@
+//===--- HeaderSourceSwitch.cpp - --------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "HeaderSourceSwitch.h"
+#include "AST.h"
+#include "Logger.h"
+#include "index/SymbolCollector.h"
+#include "clang/AST/Decl.h"
+
+namespace clang {
+namespace clangd {
+
+llvm::Optional<Path> getCorrespondingHeaderOrSource(
+    const Path &OriginalFile,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS) {
+  llvm::StringRef SourceExtensions[] = {".cpp", ".c", ".cc", ".cxx",
+                                        ".c++", ".m", ".mm"};
+  llvm::StringRef HeaderExtensions[] = {".h", ".hh", ".hpp", ".hxx", ".inc"};
+
+  llvm::StringRef PathExt = llvm::sys::path::extension(OriginalFile);
+
+  // Lookup in a list of known extensions.
+  auto SourceIter =
+      llvm::find_if(SourceExtensions, [&PathExt](PathRef SourceExt) {
+        return SourceExt.equals_lower(PathExt);
+      });
+  bool IsSource = SourceIter != std::end(SourceExtensions);
+
+  auto HeaderIter =
+      llvm::find_if(HeaderExtensions, [&PathExt](PathRef HeaderExt) {
+        return HeaderExt.equals_lower(PathExt);
+      });
+  bool IsHeader = HeaderIter != std::end(HeaderExtensions);
+
+  // We can only switch between the known extensions.
+  if (!IsSource && !IsHeader)
+    return None;
+
+  // Array to lookup extensions for the switch. An opposite of where original
+  // extension was found.
+  llvm::ArrayRef<llvm::StringRef> NewExts;
+  if (IsSource)
+    NewExts = HeaderExtensions;
+  else
+    NewExts = SourceExtensions;
+
+  // Storage for the new path.
+  llvm::SmallString<128> NewPath = llvm::StringRef(OriginalFile);
+
+  // Loop through switched extension candidates.
+  for (llvm::StringRef NewExt : NewExts) {
+    llvm::sys::path::replace_extension(NewPath, NewExt);
+    if (VFS->exists(NewPath))
+      return NewPath.str().str(); // First str() to convert from SmallString to
+                                  // StringRef, second to convert from StringRef
+                                  // to std::string
+
+    // Also check NewExt in upper-case, just in case.
+    llvm::sys::path::replace_extension(NewPath, NewExt.upper());
+    if (VFS->exists(NewPath))
+      return NewPath.str().str();
+  }
+  return None;
+}
+
+llvm::Optional<Path> getCorrespondingHeaderOrSource(const Path &OriginalFile,
+                                                    ParsedAST &AST,
+                                                    const SymbolIndex *Index) {
+  if (!Index) {
+    // FIXME: use the AST to do the inference.
+    return None;
+  }
+  LookupRequest Request;
+  // Find all symbols present in the original file.
+  for (const auto *D : getIndexableLocalDecls(AST)) {
+    if (auto ID = getSymbolID(D))
+      Request.IDs.insert(*ID);
+  }
+  llvm::StringMap<int> Candidates; // Target path => score.
+  auto AwardTarget = [&](const char *TargetURI) {
+    if (auto TargetPath = URI::resolve(TargetURI, OriginalFile)) {
+      if (*TargetPath != OriginalFile) // exclude the original file.
+        ++Candidates[*TargetPath];
+    };
+  };
+  // If we switch from a header, we are looking for the implementation
+  // file, so we use the definition loc; otherwise we look for the header file,
+  // we use the decl loc;
+  //
+  // For each symbol in the original file, we get its target location (decl or
+  // def) from the index, then award that target file.
+  bool IsHeader = AST.getASTContext().getLangOpts().IsHeaderFile;
+  Index->lookup(Request, [&](const Symbol &Sym) {
+    if (IsHeader)
+      AwardTarget(Sym.Definition.FileURI);
+    else
+      AwardTarget(Sym.CanonicalDeclaration.FileURI);
+  });
+  // FIXME: our index doesn't have any interesting information (this could be
+  // that the background-index is not finished), we should use the decl/def
+  // locations from the AST to do the inference (from .cc to .h).
+  if (Candidates.empty())
+    return None;
+
+  // Pickup the winner, who contains most of symbols.
+  // FIXME: should we use other signals (file proximity) to help score?
+  auto Best = Candidates.begin();
+  for (auto It = Candidates.begin(); It != Candidates.end(); ++It) {
+    if (It->second > Best->second)
+      Best = It;
+    else if (It->second == Best->second && It->first() < Best->first())
+      // Select the first one in the lexical order if we have multiple
+      // candidates.
+      Best = It;
+  }
+  return Path(Best->first());
+}
+
+std::vector<const Decl *> getIndexableLocalDecls(ParsedAST &AST) {
+  std::vector<const Decl *> Results;
+  std::function<void(Decl *)> TraverseDecl = [&](Decl *D) {
+    auto *ND = llvm::dyn_cast<NamedDecl>(D);
+    if (!ND || ND->isImplicit())
+      return;
+    if (!SymbolCollector::shouldCollectSymbol(*ND, D->getASTContext(), {},
+                                              /*IsMainFileSymbol=*/false))
+      return;
+    if (!llvm::isa<FunctionDecl>(ND)) {
+      // Visit the children, but we skip function decls as we are not interested
+      // in the function body.
+      if (auto *Scope = llvm::dyn_cast<DeclContext>(ND)) {
+        for (auto *D : Scope->decls())
+          TraverseDecl(D);
+      }
+    }
+    if (llvm::isa<NamespaceDecl>(D))
+      return; // namespace is indexable, but we're not interested.
+    Results.push_back(D);
+  };
+  // Traverses the ParsedAST directly to collect all decls present in the main
+  // file.
+  for (auto *TopLevel : AST.getLocalTopLevelDecls())
+    TraverseDecl(TopLevel);
+  return Results;
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/HeaderSourceSwitch.h b/clang-tools-extra/clangd/HeaderSourceSwitch.h
new file mode 100644
index 0000000000000..a971b385d74ce
--- /dev/null
+++ b/clang-tools-extra/clangd/HeaderSourceSwitch.h
@@ -0,0 +1,37 @@
+//===--- HeaderSourceSwitch.h - ----------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_HEADERSOURCESWITCH_H
+#define LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_HEADERSOURCESWITCH_H
+
+#include "ParsedAST.h"
+#include "llvm/ADT/Optional.h"
+
+namespace clang {
+namespace clangd {
+
+/// Given a header file, returns the best matching source file, and vice visa.
+/// It only uses the filename heuristics to do the inference.
+llvm::Optional<Path> getCorrespondingHeaderOrSource(
+    const Path &OriginalFile,
+    llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS);
+
+/// Given a header file, returns the best matching source file, and vice visa.
+/// The heuristics incorporate with the AST and the index (if provided).
+llvm::Optional<Path> getCorrespondingHeaderOrSource(const Path &OriginalFile,
+                                                    ParsedAST &AST,
+                                                    const SymbolIndex *Index);
+
+/// Returns all indexable decls that are present in the main file of the AST.
+/// Exposed for unittests.
+std::vector<const Decl *> getIndexableLocalDecls(ParsedAST &AST);
+
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_UNITTESTS_CLANGD_HEADERSOURCESWITCH_H
diff --git a/clang-tools-extra/clangd/IncludeFixer.cpp b/clang-tools-extra/clangd/IncludeFixer.cpp
index 081dad83583b4..93dee2eabfc6d 100644
--- a/clang-tools-extra/clangd/IncludeFixer.cpp
+++ b/clang-tools-extra/clangd/IncludeFixer.cpp
@@ -144,10 +144,8 @@ std::vector<Fix> IncludeFixer::fixIncompleteType(const Type &T) const {
 std::vector<Fix> IncludeFixer::fixesForSymbols(const SymbolSlab &Syms) const {
   auto Inserted = [&](const Symbol &Sym, llvm::StringRef Header)
       -> llvm::Expected<std::pair<std::string, bool>> {
-    auto DeclaringURI = URI::parse(Sym.CanonicalDeclaration.FileURI);
-    if (!DeclaringURI)
-      return DeclaringURI.takeError();
-    auto ResolvedDeclaring = URI::resolve(*DeclaringURI, File);
+    auto ResolvedDeclaring =
+        URI::resolve(Sym.CanonicalDeclaration.FileURI, File);
     if (!ResolvedDeclaring)
       return ResolvedDeclaring.takeError();
     auto ResolvedInserted = toHeaderFile(Header, File);
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index 3619144eae92d..7c397504a00bc 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -49,6 +49,11 @@
 #include <algorithm>
 #include <memory>
 
+// Force the linker to link in Clang-tidy modules.
+// clangd doesn't support the static analyzer.
+#define CLANG_TIDY_DISABLE_STATIC_ANALYZER_CHECKS
+#include "../clang-tidy/ClangTidyForceLinker.h"
+
 namespace clang {
 namespace clangd {
 namespace {
@@ -98,33 +103,6 @@ class ClangdFrontendAction : public SyntaxOnlyAction {
   std::vector<Decl *> TopLevelDecls;
 };
 
-// This collects macro expansions/definitions in the main file.
-// (Contrast with CollectMainFileMacros in Preamble.cpp, which collects macro
-// *definitions* in the preamble region of the main file).
-class CollectMainFileMacros : public PPCallbacks {
-  const SourceManager &SM;
-  std::vector<SourceLocation> &MainFileMacroLocs;
-
-  void addLoc(SourceLocation Loc) {
-    if (!Loc.isMacroID() && isInsideMainFile(Loc, SM))
-      MainFileMacroLocs.push_back(Loc);
-  }
-
-public:
-  CollectMainFileMacros(const SourceManager &SM,
-                        std::vector<SourceLocation> &MainFileMacroLocs)
-      : SM(SM), MainFileMacroLocs(MainFileMacroLocs) {}
-
-  void MacroDefined(const Token &MacroNameTok,
-                    const MacroDirective *MD) override {
-    addLoc(MacroNameTok.getLocation());
-  }
-  void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
-                    SourceRange Range, const MacroArgs *Args) override {
-    addLoc(MacroNameTok.getLocation());
-  }
-};
-
 // When using a preamble, only preprocessor events outside its bounds are seen.
 // This is almost what we want: replaying transitive preprocessing wastes time.
 // However this confuses clang-tidy checks: they don't see any #includes!
@@ -288,7 +266,7 @@ ParsedAST::build(std::unique_ptr<clang::CompilerInvocation> CI,
     CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
     CTContext->setASTContext(&Clang->getASTContext());
     CTContext->setCurrentFile(Filename);
-    CTFactories.createChecks(CTContext.getPointer(), CTChecks);
+    CTChecks = CTFactories.createChecks(CTContext.getPointer());
     ASTDiags.setLevelAdjuster([&CTContext](DiagnosticsEngine::Level DiagLevel,
                                            const clang::Diagnostic &Info) {
       if (CTContext) {
@@ -362,11 +340,14 @@ ParsedAST::build(std::unique_ptr<clang::CompilerInvocation> CI,
   // (We can't *just* use the replayed includes, they don't have Resolved path).
   Clang->getPreprocessor().addPPCallbacks(
       collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
-  // Collect the macro expansions in the main file.
-  std::vector<SourceLocation> MainFileMacroExpLocs;
+  // Copy over the macros in the preamble region of the main file, and combine
+  // with non-preamble macros below.
+  MainFileMacros Macros;
+  if (Preamble)
+    Macros = Preamble->Macros;
   Clang->getPreprocessor().addPPCallbacks(
       std::make_unique<CollectMainFileMacros>(Clang->getSourceManager(),
-                                              MainFileMacroExpLocs));
+                                              Clang->getLangOpts(), Macros));
 
   // Copy over the includes from the preamble, then combine with the
   // non-preamble includes below.
@@ -420,9 +401,9 @@ ParsedAST::build(std::unique_ptr<clang::CompilerInvocation> CI,
     Diags.insert(Diags.end(), D.begin(), D.end());
   }
   return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
-                   std::move(Tokens), std::move(MainFileMacroExpLocs),
-                   std::move(ParsedDecls), std::move(Diags),
-                   std::move(Includes), std::move(CanonIncludes));
+                   std::move(Tokens), std::move(Macros), std::move(ParsedDecls),
+                   std::move(Diags), std::move(Includes),
+                   std::move(CanonIncludes));
 }
 
 ParsedAST::ParsedAST(ParsedAST &&Other) = default;
@@ -460,9 +441,7 @@ llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
   return LocalTopLevelDecls;
 }
 
-llvm::ArrayRef<SourceLocation> ParsedAST::getMacros() const {
-  return MacroIdentifierLocs;
-}
+const MainFileMacros &ParsedAST::getMacros() const { return Macros; }
 
 const std::vector<Diag> &ParsedAST::getDiagnostics() const { return Diags; }
 
@@ -509,15 +488,13 @@ const CanonicalIncludes &ParsedAST::getCanonicalIncludes() const {
 ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
                      std::unique_ptr<CompilerInstance> Clang,
                      std::unique_ptr<FrontendAction> Action,
-                     syntax::TokenBuffer Tokens,
-                     std::vector<SourceLocation> MacroIdentifierLocs,
+                     syntax::TokenBuffer Tokens, MainFileMacros Macros,
                      std::vector<Decl *> LocalTopLevelDecls,
                      std::vector<Diag> Diags, IncludeStructure Includes,
                      CanonicalIncludes CanonIncludes)
     : Preamble(std::move(Preamble)), Clang(std::move(Clang)),
       Action(std::move(Action)), Tokens(std::move(Tokens)),
-      MacroIdentifierLocs(std::move(MacroIdentifierLocs)),
-      Diags(std::move(Diags)),
+      Macros(std::move(Macros)), Diags(std::move(Diags)),
       LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
       Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
   assert(this->Clang);
@@ -549,32 +526,4 @@ buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,
 }
 
 } // namespace clangd
-namespace tidy {
-// Force the linker to link in Clang-tidy modules.
-#define LINK_TIDY_MODULE(X)                                                    \
-  extern volatile int X##ModuleAnchorSource;                                   \
-  static int LLVM_ATTRIBUTE_UNUSED X##ModuleAnchorDestination =                \
-      X##ModuleAnchorSource
-LINK_TIDY_MODULE(Abseil);
-LINK_TIDY_MODULE(Android);
-LINK_TIDY_MODULE(Boost);
-LINK_TIDY_MODULE(Bugprone);
-LINK_TIDY_MODULE(CERT);
-LINK_TIDY_MODULE(CppCoreGuidelines);
-LINK_TIDY_MODULE(Fuchsia);
-LINK_TIDY_MODULE(Google);
-LINK_TIDY_MODULE(HICPP);
-LINK_TIDY_MODULE(LinuxKernel);
-LINK_TIDY_MODULE(LLVM);
-LINK_TIDY_MODULE(Misc);
-LINK_TIDY_MODULE(Modernize);
-// LINK_TIDY_MODULE(MPI); // clangd doesn't support static analyzer.
-LINK_TIDY_MODULE(ObjC);
-LINK_TIDY_MODULE(OpenMP);
-LINK_TIDY_MODULE(Performance);
-LINK_TIDY_MODULE(Portability);
-LINK_TIDY_MODULE(Readability);
-LINK_TIDY_MODULE(Zircon);
-#undef LINK_TIDY_MODULE
-} // namespace tidy
 } // namespace clang
diff --git a/clang-tools-extra/clangd/ParsedAST.h b/clang-tools-extra/clangd/ParsedAST.h
index f37e14d805168..0b4a6ab73df83 100644
--- a/clang-tools-extra/clangd/ParsedAST.h
+++ b/clang-tools-extra/clangd/ParsedAST.h
@@ -21,6 +21,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PARSEDAST_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PARSEDAST_H
 
+#include "CollectMacros.h"
 #include "Compiler.h"
 #include "Diagnostics.h"
 #include "Headers.h"
@@ -89,10 +90,9 @@ class ParsedAST {
   const IncludeStructure &getIncludeStructure() const;
   const CanonicalIncludes &getCanonicalIncludes() const;
 
-  /// Gets all macro locations (definition, expansions) present in the main
-  /// file.
-  /// NOTE: macros inside the preamble are not included.
-  llvm::ArrayRef<SourceLocation> getMacros() const;
+  /// Gets all macro references (definition, expansions) present in the main
+  /// file, including those in the preamble region.
+  const MainFileMacros &getMacros() const;
   /// Tokens recorded while parsing the main file.
   /// (!) does not have tokens from the preamble.
   const syntax::TokenBuffer &getTokens() const { return Tokens; }
@@ -101,9 +101,9 @@ class ParsedAST {
   ParsedAST(std::shared_ptr<const PreambleData> Preamble,
             std::unique_ptr<CompilerInstance> Clang,
             std::unique_ptr<FrontendAction> Action, syntax::TokenBuffer Tokens,
-            std::vector<SourceLocation> MainFileMacroExpLocs,
-            std::vector<Decl *> LocalTopLevelDecls, std::vector<Diag> Diags,
-            IncludeStructure Includes, CanonicalIncludes CanonIncludes);
+            MainFileMacros Macros, std::vector<Decl *> LocalTopLevelDecls,
+            std::vector<Diag> Diags, IncludeStructure Includes,
+            CanonicalIncludes CanonIncludes);
 
   // In-memory preambles must outlive the AST, it is important that this member
   // goes before Clang and Action.
@@ -121,10 +121,8 @@ class ParsedAST {
   ///   - Does not have spelled or expanded tokens for files from preamble.
   syntax::TokenBuffer Tokens;
 
-  /// The start locations of all macro definitions/expansions spelled **after**
-  /// preamble.
-  /// Does not include locations from inside other macro expansions.
-  std::vector<SourceLocation> MacroIdentifierLocs;
+  /// All macro definitions and expansions in the main file.
+  MainFileMacros Macros;
   // Data, stored after parsing.
   std::vector<Diag> Diags;
   // Top-level decls inside the current file. Not that this does not include
diff --git a/clang-tools-extra/clangd/Preamble.cpp b/clang-tools-extra/clangd/Preamble.cpp
index d68c1ec97fa43..41e3c1368eb72 100644
--- a/clang-tools-extra/clangd/Preamble.cpp
+++ b/clang-tools-extra/clangd/Preamble.cpp
@@ -24,49 +24,14 @@ bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
          llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
 }
 
-// This collects macro definitions in the *preamble region* of the main file.
-// (Contrast with CollectMainFileMacroExpansions in ParsedAST.cpp, which
-// collects macro *expansions* in the rest of the main file.
-class CollectMainFileMacros : public PPCallbacks {
-public:
-  explicit CollectMainFileMacros(const SourceManager &SM,
-                                 std::vector<std::string> *Out)
-      : SM(SM), Out(Out) {}
-
-  void FileChanged(SourceLocation Loc, FileChangeReason,
-                   SrcMgr::CharacteristicKind, FileID Prev) {
-    InMainFile = SM.isWrittenInMainFile(Loc);
-  }
-
-  void MacroDefined(const Token &MacroName, const MacroDirective *MD) {
-    if (InMainFile)
-      MainFileMacros.insert(MacroName.getIdentifierInfo()->getName());
-  }
-
-  void EndOfMainFile() {
-    for (const auto &Entry : MainFileMacros)
-      Out->push_back(Entry.getKey());
-    llvm::sort(*Out);
-  }
-
-private:
-  const SourceManager &SM;
-  bool InMainFile = true;
-  llvm::StringSet<> MainFileMacros;
-  std::vector<std::string> *Out;
-};
-
 class CppFilePreambleCallbacks : public PreambleCallbacks {
 public:
   CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
-      : File(File), ParsedCallback(ParsedCallback) {
-  }
+      : File(File), ParsedCallback(ParsedCallback) {}
 
   IncludeStructure takeIncludes() { return std::move(Includes); }
 
-  std::vector<std::string> takeMainFileMacros() {
-    return std::move(MainFileMacros);
-  }
+  MainFileMacros takeMacros() { return std::move(Macros); }
 
   CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
 
@@ -79,14 +44,17 @@ class CppFilePreambleCallbacks : public PreambleCallbacks {
 
   void BeforeExecute(CompilerInstance &CI) override {
     CanonIncludes.addSystemHeadersMapping(CI.getLangOpts());
+    LangOpts = &CI.getLangOpts();
     SourceMgr = &CI.getSourceManager();
   }
 
   std::unique_ptr<PPCallbacks> createPPCallbacks() override {
-    assert(SourceMgr && "SourceMgr must be set at this point");
+    assert(SourceMgr && LangOpts &&
+           "SourceMgr and LangOpts must be set at this point");
+
     return std::make_unique<PPChainedCallbacks>(
         collectIncludeStructureCallback(*SourceMgr, &Includes),
-        std::make_unique<CollectMainFileMacros>(*SourceMgr, &MainFileMacros));
+        std::make_unique<CollectMainFileMacros>(*SourceMgr, *LangOpts, Macros));
   }
 
   CommentHandler *getCommentHandler() override {
@@ -99,20 +67,21 @@ class CppFilePreambleCallbacks : public PreambleCallbacks {
   PreambleParsedCallback ParsedCallback;
   IncludeStructure Includes;
   CanonicalIncludes CanonIncludes;
-  std::vector<std::string> MainFileMacros;
+  MainFileMacros Macros;
   std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
-  SourceManager *SourceMgr = nullptr;
+  const clang::LangOptions *LangOpts = nullptr;
+  const SourceManager *SourceMgr = nullptr;
 };
 
 } // namespace
 
 PreambleData::PreambleData(PrecompiledPreamble Preamble,
                            std::vector<Diag> Diags, IncludeStructure Includes,
-                           std::vector<std::string> MainFileMacros,
+                           MainFileMacros Macros,
                            std::unique_ptr<PreambleFileStatusCache> StatCache,
                            CanonicalIncludes CanonIncludes)
     : Preamble(std::move(Preamble)), Diags(std::move(Diags)),
-      Includes(std::move(Includes)), MainFileMacros(std::move(MainFileMacros)),
+      Includes(std::move(Includes)), Macros(std::move(Macros)),
       StatCache(std::move(StatCache)), CanonIncludes(std::move(CanonIncludes)) {
 }
 
@@ -181,7 +150,7 @@ buildPreamble(PathRef FileName, CompilerInvocation &CI,
     return std::make_shared<PreambleData>(
         std::move(*BuiltPreamble), std::move(Diags),
         SerializedDeclsCollector.takeIncludes(),
-        SerializedDeclsCollector.takeMainFileMacros(), std::move(StatCache),
+        SerializedDeclsCollector.takeMacros(), std::move(StatCache),
         SerializedDeclsCollector.takeCanonicalIncludes());
   } else {
     elog("Could not build a preamble for file {0}", FileName);
diff --git a/clang-tools-extra/clangd/Preamble.h b/clang-tools-extra/clangd/Preamble.h
index c1632ff02b8c3..55dcce76d2eec 100644
--- a/clang-tools-extra/clangd/Preamble.h
+++ b/clang-tools-extra/clangd/Preamble.h
@@ -22,6 +22,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H
 
+#include "CollectMacros.h"
 #include "Compiler.h"
 #include "Diagnostics.h"
 #include "FS.h"
@@ -43,8 +44,7 @@ namespace clangd {
 /// be obtained during parsing must be eagerly captured and stored here.
 struct PreambleData {
   PreambleData(PrecompiledPreamble Preamble, std::vector<Diag> Diags,
-               IncludeStructure Includes,
-               std::vector<std::string> MainFileMacros,
+               IncludeStructure Includes, MainFileMacros Macros,
                std::unique_ptr<PreambleFileStatusCache> StatCache,
                CanonicalIncludes CanonIncludes);
 
@@ -57,7 +57,7 @@ struct PreambleData {
   // Macros defined in the preamble section of the main file.
   // Users care about headers vs main-file, not preamble vs non-preamble.
   // These should be treated as main-file entities e.g. for code completion.
-  std::vector<std::string> MainFileMacros;
+  MainFileMacros Macros;
   // Cache of FS operations performed when building the preamble.
   // When reusing a preamble, this cache can be consumed to save IO.
   std::unique_ptr<PreambleFileStatusCache> StatCache;
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index fb1481da5b5e9..9b0fbc5b2f3f7 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -1073,5 +1073,18 @@ llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) {
   };
 }
 
+bool fromJSON(const llvm::json::Value &Params, SelectionRangeParams &P) {
+  llvm::json::ObjectMapper O(Params);
+  return O && O.map("textDocument", P.textDocument) &&
+         O.map("positions", P.positions);
+}
+
+llvm::json::Value toJSON(const SelectionRange &Out) {
+  if (Out.parent) {
+    return llvm::json::Object{{"range", Out.range},
+                              {"parent", toJSON(*Out.parent)}};
+  }
+  return llvm::json::Object{{"range", Out.range}};
+}
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/Protocol.h b/clang-tools-extra/clangd/Protocol.h
index d244febfce456..6540365dccd81 100644
--- a/clang-tools-extra/clangd/Protocol.h
+++ b/clang-tools-extra/clangd/Protocol.h
@@ -30,6 +30,7 @@
 #include "llvm/Support/JSON.h"
 #include "llvm/Support/raw_ostream.h"
 #include <bitset>
+#include <memory>
 #include <string>
 #include <vector>
 
@@ -1222,6 +1223,28 @@ struct SemanticHighlightingParams {
 };
 llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting);
 
+struct SelectionRangeParams {
+  /// The text document.
+  TextDocumentIdentifier textDocument;
+
+  /// The positions inside the text document.
+  std::vector<Position> positions;
+};
+bool fromJSON(const llvm::json::Value &, SelectionRangeParams &);
+
+struct SelectionRange {
+  /**
+   * The range of this selection range.
+   */
+  Range range;
+  /**
+   * The parent selection range containing this range. Therefore `parent.range`
+   * must contain `this.range`.
+   */
+  std::unique_ptr<SelectionRange> parent;
+};
+llvm::json::Value toJSON(const SelectionRange &);
+
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/Selection.cpp b/clang-tools-extra/clangd/Selection.cpp
index 1877b1cf7b673..8451c988ab667 100644
--- a/clang-tools-extra/clangd/Selection.cpp
+++ b/clang-tools-extra/clangd/Selection.cpp
@@ -61,13 +61,13 @@ class SelectedTokens {
 
   // Associates any tokens overlapping [Begin, End) with an AST node.
   // Tokens that were already claimed by another AST node are not claimed again.
-  // Returns whether the node is selected in the sense of SelectionTree.
-  SelectionTree::Selection claim(unsigned Begin, unsigned End) {
+  // Updates Result if the node is selected in the sense of SelectionTree.
+  void claim(unsigned Begin, unsigned End, SelectionTree::Selection &Result) {
     assert(Begin <= End);
 
     // Fast-path for missing the selection entirely.
     if (Begin >= SelEnd || End <= SelBegin)
-      return SelectionTree::Unselected;
+      return;
 
     // We will consider the range (at least partially) selected if it hit any
     // selected and previously unclaimed token.
@@ -98,9 +98,13 @@ class SelectedTokens {
       }
     }
 
-    if (!ClaimedAnyToken)
-      return SelectionTree::Unselected;
-    return PartialSelection ? SelectionTree::Partial : SelectionTree::Complete;
+    // If some tokens were previously claimed (Result != Unselected), we may
+    // upgrade from Partial->Complete, even if no new tokens were claimed.
+    // Important for [[int a]].
+    if (ClaimedAnyToken || Result) {
+      Result = std::max(Result, PartialSelection ? SelectionTree::Partial
+                                                 : SelectionTree::Complete);
+    }
   }
 
 private:
@@ -162,7 +166,9 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
     assert(V.Stack.size() == 1 && "Unpaired push/pop?");
     assert(V.Stack.top() == &V.Nodes.front());
     // We selected TUDecl if tokens were unclaimed (or the file is empty).
-    if (V.Nodes.size() == 1 || V.Claimed.claim(Begin, End)) {
+    SelectionTree::Selection UnclaimedTokens = SelectionTree::Unselected;
+    V.Claimed.claim(Begin, End, UnclaimedTokens);
+    if (UnclaimedTokens || V.Nodes.size() == 1) {
       StringRef FileContent = AST.getSourceManager().getBufferData(File);
       // Don't require the trailing newlines to be selected.
       bool SelectedAll = Begin == 0 && End >= FileContent.rtrim().size();
@@ -333,10 +339,11 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
     Nodes.emplace_back();
     Nodes.back().ASTNode = std::move(Node);
     Nodes.back().Parent = Stack.top();
-    // Early hit detection never selects the whole node.
     Stack.push(&Nodes.back());
-    Nodes.back().Selected =
-        claimRange(Early) ? SelectionTree::Partial : SelectionTree::Unselected;
+    claimRange(Early, Nodes.back().Selected);
+    // Early hit detection never selects the whole node.
+    if (Nodes.back().Selected)
+      Nodes.back().Selected = SelectionTree::Partial;
   }
 
   // Pops a node off the ancestor stack, and finalizes it. Pairs with push().
@@ -344,8 +351,7 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
   void pop() {
     Node &N = *Stack.top();
     dlog("{1}pop: {0}", printNodeToString(N.ASTNode, PrintPolicy), indent(-1));
-    if (auto Sel = claimRange(N.ASTNode.getSourceRange()))
-      N.Selected = Sel;
+    claimRange(N.ASTNode.getSourceRange(), N.Selected);
     if (N.Selected || !N.Children.empty()) {
       // Attach to the tree.
       N.Parent->Children.push_back(&N);
@@ -375,9 +381,10 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
   // Perform hit-testing of a complete Node against the selection.
   // This runs for every node in the AST, and must be fast in common cases.
   // This is usually called from pop(), so we can take children into account.
-  SelectionTree::Selection claimRange(SourceRange S) {
+  // The existing state of Result is relevant (early/late claims can interact).
+  void claimRange(SourceRange S, SelectionTree::Selection &Result) {
     if (!S.isValid())
-      return SelectionTree::Unselected;
+      return;
     // toHalfOpenFileRange() allows selection of constructs in macro args. e.g:
     //   #define LOOP_FOREVER(Body) for(;;) { Body }
     //   void IncrementLots(int &x) {
@@ -391,17 +398,16 @@ class SelectionVisitor : public RecursiveASTVisitor<SelectionVisitor> {
     auto E = SM.getDecomposedLoc(Range->getEnd());
     // Otherwise, nodes in macro expansions can't be selected.
     if (B.first != SelFile || E.first != SelFile)
-      return SelectionTree::Unselected;
+      return;
     // Attempt to claim the remaining range. If there's nothing to claim, only
     // children were selected.
-    SelectionTree::Selection Result = Claimed.claim(B.second, E.second);
+    Claimed.claim(B.second, E.second, Result);
     if (Result)
       dlog("{1}hit selection: {0}",
            SourceRange(SM.getComposedLoc(B.first, B.second),
                        SM.getComposedLoc(E.first, E.second))
                .printToString(SM),
            indent());
-    return Result;
   }
 
   std::string indent(int Offset = 0) {
diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp
index c38bf87cb3352..651490132c29f 100644
--- a/clang-tools-extra/clangd/SemanticHighlighting.cpp
+++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/DeclarationName.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
@@ -24,6 +25,77 @@ namespace clang {
 namespace clangd {
 namespace {
 
+/// Some names are not written in the source code and cannot be highlighted,
+/// e.g. anonymous classes. This function detects those cases.
+bool canHighlightName(DeclarationName Name) {
+  if (Name.getNameKind() == DeclarationName::CXXConstructorName ||
+      Name.getNameKind() == DeclarationName::CXXUsingDirective)
+    return true;
+  auto *II = Name.getAsIdentifierInfo();
+  return II && !II->getName().empty();
+}
+
+llvm::Optional<HighlightingKind> kindForType(const Type *TP);
+llvm::Optional<HighlightingKind> kindForDecl(const NamedDecl *D) {
+  if (auto *TD = dyn_cast<TypedefNameDecl>(D)) {
+    // We try to highlight typedefs as their underlying type.
+    if (auto K = kindForType(TD->getUnderlyingType().getTypePtrOrNull()))
+      return K;
+    // And fallback to a generic kind if this fails.
+    return HighlightingKind::Typedef;
+  }
+  // We highlight class decls, constructor decls and destructor decls as
+  // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we
+  // will visit a TypeLoc where the underlying Type is a CXXRecordDecl).
+  if (auto *RD = llvm::dyn_cast<RecordDecl>(D)) {
+    // We don't want to highlight lambdas like classes.
+    if (RD->isLambda())
+      return llvm::None;
+    return HighlightingKind::Class;
+  }
+  if (isa<ClassTemplateDecl>(D) || isa<RecordDecl>(D) ||
+      isa<CXXConstructorDecl>(D))
+    return HighlightingKind::Class;
+  if (auto *MD = dyn_cast<CXXMethodDecl>(D))
+    return MD->isStatic() ? HighlightingKind::StaticMethod
+                          : HighlightingKind::Method;
+  if (isa<FieldDecl>(D))
+    return HighlightingKind::Field;
+  if (isa<EnumDecl>(D))
+    return HighlightingKind::Enum;
+  if (isa<EnumConstantDecl>(D))
+    return HighlightingKind::EnumConstant;
+  if (isa<ParmVarDecl>(D))
+    return HighlightingKind::Parameter;
+  if (auto *VD = dyn_cast<VarDecl>(D))
+    return VD->isStaticDataMember()
+               ? HighlightingKind::StaticField
+               : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
+                                      : HighlightingKind::Variable;
+  if (isa<BindingDecl>(D))
+    return HighlightingKind::Variable;
+  if (isa<FunctionDecl>(D))
+    return HighlightingKind::Function;
+  if (isa<NamespaceDecl>(D) || isa<NamespaceAliasDecl>(D) ||
+      isa<UsingDirectiveDecl>(D))
+    return HighlightingKind::Namespace;
+  if (isa<TemplateTemplateParmDecl>(D) || isa<TemplateTypeParmDecl>(D) ||
+      isa<NonTypeTemplateParmDecl>(D))
+    return HighlightingKind::TemplateParameter;
+  return llvm::None;
+}
+llvm::Optional<HighlightingKind> kindForType(const Type *TP) {
+  if (!TP)
+    return llvm::None;
+  if (TP->isBuiltinType()) // Builtins are special, they do not have decls.
+    return HighlightingKind::Primitive;
+  if (auto *TD = dyn_cast<TemplateTypeParmType>(TP))
+    return kindForDecl(TD->getDecl());
+  if (auto *TD = TP->getAsTagDecl())
+    return kindForDecl(TD);
+  return llvm::None;
+}
+
 // Collects all semantic tokens in an ASTContext.
 class HighlightingTokenCollector
     : public RecursiveASTVisitor<HighlightingTokenCollector> {
@@ -38,8 +110,8 @@ class HighlightingTokenCollector
     TraverseAST(AST.getASTContext());
     // Add highlightings for macro expansions as they are not traversed by the
     // visitor.
-    for (SourceLocation Loc : AST.getMacros())
-      addToken(Loc, HighlightingKind::Macro);
+    for (const auto &M : AST.getMacros().Ranges)
+      Tokens.push_back({HighlightingKind::Macro, M});
     // Initializer lists can give duplicates of tokens, therefore all tokens
     // must be deduplicated.
     llvm::sort(Tokens);
@@ -70,66 +142,30 @@ class HighlightingTokenCollector
 
   bool VisitNamespaceAliasDecl(NamespaceAliasDecl *NAD) {
     // The target namespace of an alias can not be found in any other way.
-    addToken(NAD->getTargetNameLoc(), HighlightingKind::Namespace);
+    addToken(NAD->getTargetNameLoc(), NAD->getAliasedNamespace());
     return true;
   }
 
   bool VisitMemberExpr(MemberExpr *ME) {
-    const auto *MD = ME->getMemberDecl();
-    if (isa<CXXDestructorDecl>(MD))
-      // When calling the destructor manually like: AAA::~A(); The ~ is a
-      // MemberExpr. Other methods should still be highlighted though.
-      return true;
-    if (isa<CXXConversionDecl>(MD))
-      // The MemberLoc is invalid for C++ conversion operators. We do not
-      // attempt to add tokens with invalid locations.
-      return true;
-    addToken(ME->getMemberLoc(), MD);
+    if (canHighlightName(ME->getMemberNameInfo().getName()))
+      addToken(ME->getMemberLoc(), ME->getMemberDecl());
     return true;
   }
 
   bool VisitNamedDecl(NamedDecl *ND) {
-    // UsingDirectiveDecl's namespaces do not show up anywhere else in the
-    // Visit/Traverse mehods. But they should also be highlighted as a
-    // namespace.
-    if (const auto *UD = dyn_cast<UsingDirectiveDecl>(ND)) {
-      addToken(UD->getIdentLocation(), HighlightingKind::Namespace);
-      return true;
-    }
-
-    // Constructors' TypeLoc has a TypePtr that is a FunctionProtoType. It has
-    // no tag decl and therefore constructors must be gotten as NamedDecls
-    // instead.
-    if (ND->getDeclName().getNameKind() ==
-        DeclarationName::CXXConstructorName) {
+    if (canHighlightName(ND->getDeclName()))
       addToken(ND->getLocation(), ND);
-      return true;
-    }
-
-    if (ND->getDeclName().getNameKind() != DeclarationName::Identifier)
-      return true;
-
-    addToken(ND->getLocation(), ND);
     return true;
   }
 
   bool VisitDeclRefExpr(DeclRefExpr *Ref) {
-    if (Ref->getNameInfo().getName().getNameKind() !=
-        DeclarationName::Identifier)
-      // Only want to highlight identifiers.
-      return true;
-
-    addToken(Ref->getLocation(), Ref->getDecl());
-    return true;
-  }
-
-  bool VisitTypedefNameDecl(TypedefNameDecl *TD) {
-    addTokenForTypedef(TD->getLocation(), TD);
+    if (canHighlightName(Ref->getNameInfo().getName()))
+      addToken(Ref->getLocation(), Ref->getDecl());
     return true;
   }
 
   bool VisitTypedefTypeLoc(TypedefTypeLoc TL) {
-    addTokenForTypedef(TL.getBeginLoc(), TL.getTypedefNameDecl());
+    addToken(TL.getBeginLoc(), TL.getTypedefNameDecl());
     return true;
   }
 
@@ -140,22 +176,29 @@ class HighlightingTokenCollector
     return true;
   }
 
-  bool VisitTypeLoc(TypeLoc &TL) {
-    // This check is for not getting two entries when there are anonymous
-    // structs. It also makes us not highlight certain namespace qualifiers
-    // twice. For elaborated types the actual type is highlighted as an inner
-    // TypeLoc.
-    if (TL.getTypeLocClass() != TypeLoc::TypeLocClass::Elaborated)
-      addType(TL.getBeginLoc(), TL.getTypePtr());
+  bool WalkUpFromTagTypeLoc(TagTypeLoc L) {
+    if (L.isDefinition())
+      return true; // Definition will be highligthed by VisitNamedDecl.
+    return RecursiveASTVisitor::WalkUpFromTagTypeLoc(L);
+  }
+
+  bool WalkUpFromElaboratedTypeLoc(ElaboratedTypeLoc L) {
+    // Avoid highlighting 'struct' or 'enum' keywords.
+    return true;
+  }
+
+  bool VisitTypeLoc(TypeLoc TL) {
+    if (auto K = kindForType(TL.getTypePtr()))
+      addToken(TL.getBeginLoc(), *K);
     return true;
   }
 
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNSLoc) {
-    if (NestedNameSpecifier *NNS = NNSLoc.getNestedNameSpecifier())
+    if (auto *NNS = NNSLoc.getNestedNameSpecifier()) {
       if (NNS->getKind() == NestedNameSpecifier::Namespace ||
           NNS->getKind() == NestedNameSpecifier::NamespaceAlias)
         addToken(NNSLoc.getLocalBeginLoc(), HighlightingKind::Namespace);
-
+    }
     return RecursiveASTVisitor<
         HighlightingTokenCollector>::TraverseNestedNameSpecifierLoc(NNSLoc);
   }
@@ -168,127 +211,21 @@ class HighlightingTokenCollector
   }
 
   bool VisitDeclaratorDecl(DeclaratorDecl *D) {
-    if ((!D->getTypeSourceInfo()))
+    // Highlight 'auto' with its underlying type.
+    auto *AT = D->getType()->getContainedAutoType();
+    if (!AT)
       return true;
-
-    if (auto *AT = D->getType()->getContainedAutoType()) {
-      const auto Deduced = AT->getDeducedType();
-      if (!Deduced.isNull())
-        addType(D->getTypeSpecStartLoc(), Deduced.getTypePtr());
-    }
+    auto K = kindForType(AT->getDeducedType().getTypePtrOrNull());
+    if (!K)
+      return true;
+    addToken(D->getTypeSpecStartLoc(), *K);
     return true;
   }
 
 private:
-  void addTokenForTypedef(SourceLocation Loc, const TypedefNameDecl *TD) {
-    auto *TSI = TD->getTypeSourceInfo();
-    if (!TSI)
-      return;
-    // Try to highlight as underlying type.
-    if (addType(Loc, TSI->getType().getTypePtrOrNull()))
-      return;
-    // Fallback to the typedef highlighting kind.
-    addToken(Loc, HighlightingKind::Typedef);
-  }
-
-  bool addType(SourceLocation Loc, const Type *TP) {
-    if (!TP)
-      return false;
-    if (TP->isBuiltinType()) {
-      // Builtins must be special cased as they do not have a TagDecl.
-      addToken(Loc, HighlightingKind::Primitive);
-      return true;
-    }
-    if (auto *TD = dyn_cast<TemplateTypeParmType>(TP)) {
-      // TemplateTypeParmType also do not have a TagDecl.
-      addToken(Loc, TD->getDecl());
-      return true;
-    }
-    if (auto *TD = TP->getAsTagDecl()) {
-      addToken(Loc, TD);
-      return true;
-    }
-    return false;
-  }
-
-  void addToken(SourceLocation Loc, const NamedDecl *D) {
-    if (D->getDeclName().isIdentifier() && D->getName().empty())
-      // Don't add symbols that don't have any length.
-      return;
-    // We highlight class decls, constructor decls and destructor decls as
-    // `Class` type. The destructor decls are handled in `VisitTypeLoc` (we will
-    // visit a TypeLoc where the underlying Type is a CXXRecordDecl).
-    if (isa<ClassTemplateDecl>(D)) {
-      addToken(Loc, HighlightingKind::Class);
-      return;
-    }
-    if (isa<RecordDecl>(D)) {
-      addToken(Loc, HighlightingKind::Class);
-      return;
-    }
-    if (isa<CXXConstructorDecl>(D)) {
-      addToken(Loc, HighlightingKind::Class);
-      return;
-    }
-    if (auto *MD = dyn_cast<CXXMethodDecl>(D)) {
-      addToken(Loc, MD->isStatic() ? HighlightingKind::StaticMethod
-                                   : HighlightingKind::Method);
-      return;
-    }
-    if (isa<FieldDecl>(D)) {
-      addToken(Loc, HighlightingKind::Field);
-      return;
-    }
-    if (isa<EnumDecl>(D)) {
-      addToken(Loc, HighlightingKind::Enum);
-      return;
-    }
-    if (isa<EnumConstantDecl>(D)) {
-      addToken(Loc, HighlightingKind::EnumConstant);
-      return;
-    }
-    if (isa<ParmVarDecl>(D)) {
-      addToken(Loc, HighlightingKind::Parameter);
-      return;
-    }
-    if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
-      addToken(Loc, VD->isStaticDataMember()
-                        ? HighlightingKind::StaticField
-                        : VD->isLocalVarDecl() ? HighlightingKind::LocalVariable
-                                               : HighlightingKind::Variable);
-      return;
-    }
-    if (isa<BindingDecl>(D)) {
-      addToken(Loc, HighlightingKind::Variable);
-      return;
-    }
-    if (isa<FunctionDecl>(D)) {
-      addToken(Loc, HighlightingKind::Function);
-      return;
-    }
-    if (isa<NamespaceDecl>(D)) {
-      addToken(Loc, HighlightingKind::Namespace);
-      return;
-    }
-    if (isa<NamespaceAliasDecl>(D)) {
-      addToken(Loc, HighlightingKind::Namespace);
-      return;
-    }
-    if (isa<TemplateTemplateParmDecl>(D)) {
-      addToken(Loc, HighlightingKind::TemplateParameter);
-      return;
-    }
-    if (isa<TemplateTypeParmDecl>(D)) {
-      addToken(Loc, HighlightingKind::TemplateParameter);
-      return;
-    }
-    if (isa<NonTypeTemplateParmDecl>(D)) {
-      addToken(Loc, HighlightingKind::TemplateParameter);
-      return;
-    }
-  }
-
   void addToken(SourceLocation Loc, HighlightingKind Kind) {
+    if (Loc.isInvalid())
+      return;
     const auto &SM = AST.getSourceManager();
     if (Loc.isMacroID()) {
       // Only intereseted in highlighting arguments in macros (DEF_X(arg)).
@@ -298,8 +235,8 @@ class HighlightingTokenCollector
     }
 
     // Non top level decls that are included from a header are not filtered by
-    // topLevelDecls. (example: method declarations being included from another
-    // file for a class from another file)
+    // topLevelDecls. (example: method declarations being included from
+    // another file for a class from another file).
     // There are also cases with macros where the spelling loc will not be in
     // the main file and the highlighting would be incorrect.
     if (!isInsideMainFile(Loc, SM))
@@ -314,6 +251,11 @@ class HighlightingTokenCollector
 
     Tokens.push_back({Kind, R.getValue()});
   }
+
+  void addToken(SourceLocation Loc, const NamedDecl *D) {
+    if (auto K = kindForDecl(D))
+      addToken(Loc, *K);
+  }
 };
 
 // Encode binary data into base64.
diff --git a/clang-tools-extra/clangd/SemanticSelection.cpp b/clang-tools-extra/clangd/SemanticSelection.cpp
new file mode 100644
index 0000000000000..91a5582ac29a4
--- /dev/null
+++ b/clang-tools-extra/clangd/SemanticSelection.cpp
@@ -0,0 +1,64 @@
+//===--- SemanticSelection.cpp -----------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#include "SemanticSelection.h"
+#include "ParsedAST.h"
+#include "Protocol.h"
+#include "Selection.h"
+#include "SourceCode.h"
+#include "clang/AST/DeclBase.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+// Adds Range \p R to the Result if it is distinct from the last added Range.
+// Assumes that only consecutive ranges can coincide.
+void addIfDistinct(const Range &R, std::vector<Range> &Result) {
+  if (Result.empty() || Result.back() != R) {
+    Result.push_back(R);
+  }
+}
+} // namespace
+
+llvm::Expected<std::vector<Range>> getSemanticRanges(ParsedAST &AST,
+                                                     Position Pos) {
+  std::vector<Range> Result;
+  const auto &SM = AST.getSourceManager();
+  const auto &LangOpts = AST.getASTContext().getLangOpts();
+
+  auto FID = SM.getMainFileID();
+  auto Offset = positionToOffset(SM.getBufferData(FID), Pos);
+  if (!Offset) {
+    return Offset.takeError();
+  }
+
+  // Get node under the cursor.
+  SelectionTree ST(AST.getASTContext(), AST.getTokens(), *Offset);
+  for (const auto *Node = ST.commonAncestor(); Node != nullptr;
+       Node = Node->Parent) {
+    if (const Decl *D = Node->ASTNode.get<Decl>()) {
+      if (llvm::isa<TranslationUnitDecl>(D)) {
+        break;
+      }
+    }
+
+    auto SR = toHalfOpenFileRange(SM, LangOpts, Node->ASTNode.getSourceRange());
+    if (!SR.hasValue() || SM.getFileID(SR->getBegin()) != SM.getMainFileID()) {
+      continue;
+    }
+    Range R;
+    R.start = sourceLocToPosition(SM, SR->getBegin());
+    R.end = sourceLocToPosition(SM, SR->getEnd());
+    addIfDistinct(R, Result);
+  }
+  return Result;
+}
+
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/SemanticSelection.h b/clang-tools-extra/clangd/SemanticSelection.h
new file mode 100644
index 0000000000000..b0d301e2efb89
--- /dev/null
+++ b/clang-tools-extra/clangd/SemanticSelection.h
@@ -0,0 +1,32 @@
+//===--- SemanticSelection.h -------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Features for giving interesting semantic ranges around the cursor.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSELECTION_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSELECTION_H
+#include "ParsedAST.h"
+#include "Protocol.h"
+#include "llvm/Support/Error.h"
+#include <vector>
+namespace clang {
+namespace clangd {
+
+/// Returns the list of all interesting ranges around the Position \p Pos.
+/// The interesting ranges corresponds to the AST nodes in the SelectionTree
+/// containing \p Pos.
+/// Any range in the result strictly contains all the previous ranges in the
+/// result. front() is the inner most range. back() is the outermost range.
+llvm::Expected<std::vector<Range>> getSemanticRanges(ParsedAST &AST,
+                                                     Position Pos);
+} // namespace clangd
+} // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_SEMANTICSELECTION_H
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index 95f250c9d6a23..05ca7aaaa1e86 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -237,6 +237,45 @@ llvm::Optional<Range> getTokenRange(const SourceManager &SM,
   return halfOpenToRange(SM, CharSourceRange::getCharRange(TokLoc, End));
 }
 
+namespace {
+
+enum TokenFlavor { Identifier, Operator, Whitespace, Other };
+
+bool isOverloadedOperator(const Token &Tok) {
+  switch (Tok.getKind()) {
+#define OVERLOADED_OPERATOR(Name, Spelling, Token, Unary, Binary, MemOnly)     \
+  case tok::Token:
+#define OVERLOADED_OPERATOR_MULTI(Name, Spelling, Unary, Binary, MemOnly)
+#include "clang/Basic/OperatorKinds.def"
+    return true;
+
+  default:
+    break;
+  }
+  return false;
+}
+
+TokenFlavor getTokenFlavor(SourceLocation Loc, const SourceManager &SM,
+                           const LangOptions &LangOpts) {
+  Token Tok;
+  Tok.setKind(tok::NUM_TOKENS);
+  if (Lexer::getRawToken(Loc, Tok, SM, LangOpts,
+                         /*IgnoreWhiteSpace*/ false))
+    return Other;
+
+  // getRawToken will return false without setting Tok when the token is
+  // whitespace, so if the flag is not set, we are sure this is a whitespace.
+  if (Tok.is(tok::TokenKind::NUM_TOKENS))
+    return Whitespace;
+  if (Tok.is(tok::TokenKind::raw_identifier))
+    return Identifier;
+  if (isOverloadedOperator(Tok))
+    return Operator;
+  return Other;
+}
+
+} // namespace
+
 SourceLocation getBeginningOfIdentifier(const Position &Pos,
                                         const SourceManager &SM,
                                         const LangOptions &LangOpts) {
@@ -247,27 +286,57 @@ SourceLocation getBeginningOfIdentifier(const Position &Pos,
     return SourceLocation();
   }
 
-  // GetBeginningOfToken(pos) is almost what we want, but does the wrong thing
-  // if the cursor is at the end of the identifier.
-  // Instead, we lex at GetBeginningOfToken(pos - 1). The cases are:
-  //  1) at the beginning of an identifier, we'll be looking at something
-  //  that isn't an identifier.
-  //  2) at the middle or end of an identifier, we get the identifier.
-  //  3) anywhere outside an identifier, we'll get some non-identifier thing.
-  // We can't actually distinguish cases 1 and 3, but returning the original
-  // location is correct for both!
+  // GetBeginningOfToken(InputLoc) is almost what we want, but does the wrong
+  // thing if the cursor is at the end of the token (identifier or operator).
+  // The cases are:
+  //   1) at the beginning of the token
+  //   2) at the middle of the token
+  //   3) at the end of the token
+  //   4) anywhere outside the identifier or operator
+  // To distinguish all cases, we lex both at the
+  // GetBeginningOfToken(InputLoc-1) and GetBeginningOfToken(InputLoc), for
+  // cases 1 and 4, we just return the original location.
   SourceLocation InputLoc = SM.getComposedLoc(FID, *Offset);
-  if (*Offset == 0) // Case 1 or 3.
+  if (*Offset == 0) // Case 1 or 4.
     return InputLoc;
   SourceLocation Before = SM.getComposedLoc(FID, *Offset - 1);
+  SourceLocation BeforeTokBeginning =
+      Lexer::GetBeginningOfToken(Before, SM, LangOpts);
+  TokenFlavor BeforeKind = getTokenFlavor(BeforeTokBeginning, SM, LangOpts);
+
+  SourceLocation CurrentTokBeginning =
+      Lexer::GetBeginningOfToken(InputLoc, SM, LangOpts);
+  TokenFlavor CurrentKind = getTokenFlavor(CurrentTokBeginning, SM, LangOpts);
+
+  // At the middle of the token.
+  if (BeforeTokBeginning == CurrentTokBeginning) {
+    // For interesting token, we return the beginning of the token.
+    if (CurrentKind == Identifier || CurrentKind == Operator)
+      return CurrentTokBeginning;
+    // otherwise, we return the original loc.
+    return InputLoc;
+  }
 
-  Before = Lexer::GetBeginningOfToken(Before, SM, LangOpts);
-  Token Tok;
-  if (Before.isValid() &&
-      !Lexer::getRawToken(Before, Tok, SM, LangOpts, false) &&
-      Tok.is(tok::raw_identifier))
-    return Before; // Case 2.
-  return InputLoc; // Case 1 or 3.
+  // Whitespace is not interesting.
+  if (BeforeKind == Whitespace)
+    return CurrentTokBeginning;
+  if (CurrentKind == Whitespace)
+    return BeforeTokBeginning;
+
+  // The cursor is at the token boundary, e.g. "Before^Current", we prefer
+  // identifiers to other tokens.
+  if (CurrentKind == Identifier)
+    return CurrentTokBeginning;
+  if (BeforeKind == Identifier)
+    return BeforeTokBeginning;
+  // Then prefer overloaded operators to other tokens.
+  if (CurrentKind == Operator)
+    return CurrentTokBeginning;
+  if (BeforeKind == Operator)
+    return BeforeTokBeginning;
+
+  // Non-interesting case, we just return the original location.
+  return InputLoc;
 }
 
 bool isValidFileRange(const SourceManager &Mgr, SourceRange R) {
@@ -659,6 +728,9 @@ static void lex(llvm::StringRef Code, const format::FormatStyle &Style,
 
   while (!Lex.LexFromRawLexer(Tok))
     A(Tok);
+  // LexFromRawLexer returns true after it lexes last token, so we still have
+  // one more token to report.
+  A(Tok);
 }
 
 llvm::StringMap<unsigned> collectIdentifiers(llvm::StringRef Content,
diff --git a/clang-tools-extra/clangd/SourceCode.h b/clang-tools-extra/clangd/SourceCode.h
index 53a140fe4abc7..3017746c17ee4 100644
--- a/clang-tools-extra/clangd/SourceCode.h
+++ b/clang-tools-extra/clangd/SourceCode.h
@@ -79,7 +79,7 @@ llvm::Expected<SourceLocation> sourceLocationInMainFile(const SourceManager &SM,
                                                         Position P);
 
 /// Get the beginning SourceLocation at a specified \p Pos in the main file.
-/// May be invalid if Pos is, or if there's no identifier.
+/// May be invalid if Pos is, or if there's no identifier or operators.
 /// The returned position is in the main file, callers may prefer to
 /// obtain the macro expansion location.
 SourceLocation getBeginningOfIdentifier(const Position &Pos,
diff --git a/clang-tools-extra/clangd/URI.cpp b/clang-tools-extra/clangd/URI.cpp
index 9f5c92b684ff9..f8a9f59ca1378 100644
--- a/clang-tools-extra/clangd/URI.cpp
+++ b/clang-tools-extra/clangd/URI.cpp
@@ -183,6 +183,17 @@ llvm::Expected<URI> URI::parse(llvm::StringRef OrigUri) {
   return U;
 }
 
+llvm::Expected<std::string> URI::resolve(llvm::StringRef FileURI,
+                                         llvm::StringRef HintPath) {
+  auto Uri = URI::parse(FileURI);
+  if (!Uri)
+    return Uri.takeError();
+  auto Path = URI::resolve(*Uri, HintPath);
+  if (!Path)
+    return Path.takeError();
+  return *Path;
+}
+
 llvm::Expected<URI> URI::create(llvm::StringRef AbsolutePath,
                                 llvm::StringRef Scheme) {
   if (!llvm::sys::path::is_absolute(AbsolutePath))
diff --git a/clang-tools-extra/clangd/URI.h b/clang-tools-extra/clangd/URI.h
index 110b39af663cb..5f120ef9e386f 100644
--- a/clang-tools-extra/clangd/URI.h
+++ b/clang-tools-extra/clangd/URI.h
@@ -63,6 +63,10 @@ class URI {
   static llvm::Expected<std::string> resolve(const URI &U,
                                              llvm::StringRef HintPath = "");
 
+  /// Same as above, in addition it parses the \p FileURI using URI::parse.
+  static llvm::Expected<std::string> resolve(llvm::StringRef FileURI,
+                                             llvm::StringRef HintPath = "");
+
   /// Resolves \p AbsPath into a canonical path of its URI, by converting
   /// \p AbsPath to URI and resolving the URI to get th canonical path.
   /// This ensures that paths with the same URI are resolved into consistent
diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
index 2c8d161c19310..f51891b8aec59 100644
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -9,6 +9,7 @@
 #include "AST.h"
 #include "CodeCompletionStrings.h"
 #include "FindSymbols.h"
+#include "FindTarget.h"
 #include "FormattedString.h"
 #include "Logger.h"
 #include "ParsedAST.h"
@@ -1299,5 +1300,18 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
   return OS;
 }
 
+llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
+                                                 const FunctionDecl *FD) {
+  if (!FD->hasBody())
+    return {};
+  llvm::DenseSet<const Decl *> DeclRefs;
+  findExplicitReferences(FD, [&](ReferenceLoc Ref) {
+    for (const Decl *D : Ref.Targets) {
+      if (!index::isFunctionLocalSymbol(D) && !D->isTemplateParameter())
+        DeclRefs.insert(D);
+    }
+  });
+  return DeclRefs;
+}
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/XRefs.h b/clang-tools-extra/clangd/XRefs.h
index d6b3930f513b9..dc2237cb0fd12 100644
--- a/clang-tools-extra/clangd/XRefs.h
+++ b/clang-tools-extra/clangd/XRefs.h
@@ -17,8 +17,8 @@
 #include "Path.h"
 #include "Protocol.h"
 #include "index/Index.h"
-#include "clang/AST/Type.h"
 #include "index/SymbolLocation.h"
+#include "clang/AST/Type.h"
 #include "clang/Format/Format.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/Optional.h"
@@ -158,6 +158,9 @@ llvm::Optional<QualType> getDeducedType(ParsedAST &AST,
 /// SourceLocationBeg must point to the first character of the token
 bool hasDeducedType(ParsedAST &AST, SourceLocation SourceLocationBeg);
 
+/// Returns all decls that are referenced in the \p FD except local symbols.
+llvm::DenseSet<const Decl *> getNonLocalDeclRefs(ParsedAST &AST,
+                                                 const FunctionDecl *FD);
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md b/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
index 92aad528db942..e888aba3ea205 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/DEVELOPING.md
@@ -48,6 +48,6 @@ please contact clangd-dev@lists.llvm.org.
   # For the first time, you need to login in the account. vsce will ask you for
     the Personal Access Token, and remember it for future commands.
   $ vsce login llvm-vs-code-extensions
-  $ vsce publish
+  # Publish the extension to the VSCode marketplace.
+  $ npm run publish
 ```
-
diff --git a/clang-tools-extra/clangd/clients/clangd-vscode/package.json b/clang-tools-extra/clangd/clients/clangd-vscode/package.json
index c2ef3f853e258..03de7b3e58b05 100644
--- a/clang-tools-extra/clangd/clients/clangd-vscode/package.json
+++ b/clang-tools-extra/clangd/clients/clangd-vscode/package.json
@@ -2,7 +2,7 @@
     "name": "vscode-clangd",
     "displayName": "vscode-clangd",
     "description": "Clang Language Server",
-    "version": "0.0.16",
+    "version": "0.0.18",
     "publisher": "llvm-vs-code-extensions",
     "homepage": "https://clang.llvm.org/extra/clangd.html",
     "engines": {
@@ -33,13 +33,15 @@
         "compile": "tsc -watch -p ./",
         "postinstall": "node ./node_modules/vscode/bin/install",
         "format": "clang-format --style=LLVM -i --glob=\"{src,test}/*.ts\"",
-        "test": "node ./node_modules/vscode/bin/test"
+        "test": "node ./node_modules/vscode/bin/test",
+        "package": "vsce package --baseImagesUrl https://raw.githubusercontent.com/llvm/llvm-project/master/clang-tools-extra/clangd/clients/clangd-vscode/",
+        "publish": "vsce publish --baseImagesUrl https://raw.githubusercontent.com/llvm/llvm-project/master/clang-tools-extra/clangd/clients/clangd-vscode/"
     },
     "dependencies": {
         "jsonc-parser": "^2.1.0",
-        "vscode-languageclient": "^5.3.0-next.6",
-        "vscode-languageserver": "^5.3.0-next.6",
-        "vscode-languageserver-types": "^3.14.0"
+        "vscode-languageclient": "^6.0.0-next.1",
+        "vscode-languageserver": "^6.0.0-next.1",
+        "vscode-languageserver-types": "^3.15.0-next.5"
     },
     "devDependencies": {
         "@types/mocha": "^2.2.32",
@@ -92,7 +94,7 @@
                 },
                 "clangd.semanticHighlighting": {
                     "type": "boolean",
-                    "default": "false",
+                    "default": "true",
                     "description": "Enable semantic highlighting in clangd"
                 }
             }
diff --git a/clang-tools-extra/clangd/index/Background.cpp b/clang-tools-extra/clangd/index/Background.cpp
index 9ea21fdf6c590..ff6a9e5226843 100644
--- a/clang-tools-extra/clangd/index/Background.cpp
+++ b/clang-tools-extra/clangd/index/Background.cpp
@@ -69,13 +69,7 @@ class URIToFileCache {
   llvm::StringRef resolve(llvm::StringRef FileURI) {
     auto I = URIToPathCache.try_emplace(FileURI);
     if (I.second) {
-      auto U = URI::parse(FileURI);
-      if (!U) {
-        elog("Failed to parse URI {0}: {1}", FileURI, U.takeError());
-        assert(false && "Failed to parse URI");
-        return "";
-      }
-      auto Path = URI::resolve(*U, HintPath);
+      auto Path = URI::resolve(FileURI, HintPath);
       if (!Path) {
         elog("Failed to resolve URI {0}: {1}", FileURI, Path.takeError());
         assert(false && "Failed to resolve URI");
diff --git a/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp b/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
index 585d36716d296..ff690f9025733 100644
--- a/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
+++ b/clang-tools-extra/clangd/index/BackgroundIndexLoader.cpp
@@ -24,16 +24,6 @@ namespace clang {
 namespace clangd {
 namespace {
 
-llvm::Optional<Path> uriToAbsolutePath(llvm::StringRef URI, PathRef HintPath) {
-  auto U = URI::parse(URI);
-  if (!U)
-    return llvm::None;
-  auto AbsolutePath = URI::resolve(*U, HintPath);
-  if (!AbsolutePath)
-    return llvm::None;
-  return *AbsolutePath;
-}
-
 /// A helper class to cache BackgroundIndexStorage operations and keep the
 /// inverse dependency mapping.
 class BackgroundIndexLoader {
@@ -79,9 +69,11 @@ BackgroundIndexLoader::loadShard(PathRef StartSourceFile, PathRef DependentTU) {
 
   LS.Shard = std::move(Shard);
   for (const auto &It : *LS.Shard->Sources) {
-    auto AbsPath = uriToAbsolutePath(It.getKey(), StartSourceFile);
-    if (!AbsPath)
+    auto AbsPath = URI::resolve(It.getKey(), StartSourceFile);
+    if (!AbsPath) {
+      elog("Failed to resolve URI: {0}", AbsPath.takeError());
       continue;
+    }
     // A shard contains only edges for non main-file sources.
     if (*AbsPath != StartSourceFile) {
       Edges.push_back(*AbsPath);
diff --git a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
index 86c73ff7420a3..48510e11e8455 100644
--- a/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
+++ b/clang-tools-extra/clangd/index/BackgroundIndexStorage.cpp
@@ -18,6 +18,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include <functional>
@@ -35,34 +36,6 @@ std::string getShardPathFromFilePath(llvm::StringRef ShardRoot,
   return ShardRootSS.str();
 }
 
-llvm::Error
-writeAtomically(llvm::StringRef OutPath,
-                llvm::function_ref<void(llvm::raw_ostream &)> Writer) {
-  // Write to a temporary file first.
-  llvm::SmallString<128> TempPath;
-  int FD;
-  auto EC =
-      llvm::sys::fs::createUniqueFile(OutPath + ".tmp.%%%%%%%%", FD, TempPath);
-  if (EC)
-    return llvm::errorCodeToError(EC);
-  // Make sure temp file is destroyed on failure.
-  auto RemoveOnFail =
-      llvm::make_scope_exit([TempPath] { llvm::sys::fs::remove(TempPath); });
-  llvm::raw_fd_ostream OS(FD, /*shouldClose=*/true);
-  Writer(OS);
-  OS.close();
-  if (OS.has_error())
-    return llvm::errorCodeToError(OS.error());
-  // Then move to real location.
-  EC = llvm::sys::fs::rename(TempPath, OutPath);
-  if (EC)
-    return llvm::errorCodeToError(EC);
-  // If everything went well, we already moved the file to another name. So
-  // don't delete the file, as the name might be taken by another file.
-  RemoveOnFail.release();
-  return llvm::ErrorSuccess();
-}
-
 // Uses disk as a storage for index shards. Creates a directory called
 // ".clangd/index/" under the path provided during construction.
 class DiskBackedIndexStorage : public BackgroundIndexStorage {
@@ -100,9 +73,12 @@ class DiskBackedIndexStorage : public BackgroundIndexStorage {
 
   llvm::Error storeShard(llvm::StringRef ShardIdentifier,
                          IndexFileOut Shard) const override {
-    return writeAtomically(
-        getShardPathFromFilePath(DiskShardRoot, ShardIdentifier),
-        [&Shard](llvm::raw_ostream &OS) { OS << Shard; });
+    auto ShardPath = getShardPathFromFilePath(DiskShardRoot, ShardIdentifier);
+    return llvm::writeFileAtomically(ShardPath + ".tmp.%%%%%%%%", ShardPath,
+                                     [&Shard](llvm::raw_ostream &OS) {
+                                       OS << Shard;
+                                       return llvm::Error::success();
+                                     });
   }
 };
 
diff --git a/clang-tools-extra/clangd/refactor/Rename.cpp b/clang-tools-extra/clangd/refactor/Rename.cpp
index 770064fc61321..f38815a1f14ac 100644
--- a/clang-tools-extra/clangd/refactor/Rename.cpp
+++ b/clang-tools-extra/clangd/refactor/Rename.cpp
@@ -74,14 +74,22 @@ llvm::Optional<ReasonToReject> renamableWithinFile(const Decl &RenameDecl,
                                                    const SymbolIndex *Index) {
   if (llvm::isa<NamespaceDecl>(&RenameDecl))
     return ReasonToReject::UnsupportedSymbol;
+  if (const auto *FD = llvm::dyn_cast<FunctionDecl>(&RenameDecl)) {
+    if (FD->isOverloadedOperator())
+      return ReasonToReject::UnsupportedSymbol;
+  }
   auto &ASTCtx = RenameDecl.getASTContext();
   const auto &SM = ASTCtx.getSourceManager();
   bool MainFileIsHeader = ASTCtx.getLangOpts().IsHeaderFile;
   bool DeclaredInMainFile = isInsideMainFile(RenameDecl.getBeginLoc(), SM);
 
+  if (!DeclaredInMainFile)
+    // We are sure the symbol is used externally, bail out early.
+    return UsedOutsideFile;
+
   // If the symbol is declared in the main file (which is not a header), we
   // rename it.
-  if (DeclaredInMainFile && !MainFileIsHeader)
+  if (!MainFileIsHeader)
     return None;
 
   // Below are cases where the symbol is declared in the header.
diff --git a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
index 1532c951a26bb..1551f41a13184 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/ExtractFunction.cpp
@@ -104,6 +104,11 @@ bool isRootStmt(const Node *N) {
 }
 
 // Returns the (unselected) parent of all RootStmts given the commonAncestor.
+// Returns null if:
+// 1. any node is partially selected
+// 2. If all completely selected nodes don't have the same common parent
+// 3. Any child of Parent isn't a RootStmt.
+// Returns null if any child is not a RootStmt.
 // We only support extraction of RootStmts since it allows us to extract without
 // having to change the selection range. Also, this means that any scope that
 // begins in selection range, ends in selection range and any scope that begins
@@ -111,28 +116,30 @@ bool isRootStmt(const Node *N) {
 const Node *getParentOfRootStmts(const Node *CommonAnc) {
   if (!CommonAnc)
     return nullptr;
+  const Node *Parent = nullptr;
   switch (CommonAnc->Selected) {
   case SelectionTree::Selection::Unselected:
+    // Typicaly a block, with the { and } unselected, could also be ForStmt etc
     // Ensure all Children are RootStmts.
-    return llvm::all_of(CommonAnc->Children, isRootStmt) ? CommonAnc : nullptr;
+    Parent = CommonAnc;
+    break;
   case SelectionTree::Selection::Partial:
-    // Treat Partially selected VarDecl as completely selected since
-    // SelectionTree doesn't always select VarDecls correctly.
-    // FIXME: Remove this after D66872 is upstream)
-    if (!CommonAnc->ASTNode.get<VarDecl>())
-      return nullptr;
-    LLVM_FALLTHROUGH;
+    // Only a fully-selected single statement can be selected.
+    return nullptr;
   case SelectionTree::Selection::Complete:
     // If the Common Ancestor is completely selected, then it's a root statement
     // and its parent will be unselected.
-    const Node *Parent = CommonAnc->Parent;
+    Parent = CommonAnc->Parent;
     // If parent is a DeclStmt, even though it's unselected, we consider it a
     // root statement and return its parent. This is done because the VarDecls
     // claim the entire selection range of the Declaration and DeclStmt is
     // always unselected.
-    return Parent->ASTNode.get<DeclStmt>() ? Parent->Parent : Parent;
+    if (Parent->ASTNode.get<DeclStmt>())
+      Parent = Parent->Parent;
+    break;
   }
-  llvm_unreachable("Unhandled SelectionTree::Selection enum");
+  // Ensure all Children are RootStmts.
+  return llvm::all_of(Parent->Children, isRootStmt) ? Parent : nullptr;
 }
 
 // The ExtractionZone class forms a view of the code wrt Zone.
@@ -218,10 +225,24 @@ computeEnclosingFuncRange(const FunctionDecl *EnclosingFunction,
   return toHalfOpenFileRange(SM, LangOpts, EnclosingFunction->getSourceRange());
 }
 
+// returns true if Child can be a single RootStmt being extracted from
+// EnclosingFunc.
+bool validSingleChild(const Node *Child, const FunctionDecl *EnclosingFunc) {
+  // Don't extract expressions.
+  // FIXME: We should extract expressions that are "statements" i.e. not
+  // subexpressions
+  if (Child->ASTNode.get<Expr>())
+    return false;
+  // Extracting the body of EnclosingFunc would remove it's definition.
+  assert(EnclosingFunc->hasBody() &&
+         "We should always be extracting from a function body.");
+  if (Child->ASTNode.get<Stmt>() == EnclosingFunc->getBody())
+    return false;
+  return true;
+}
+
 // FIXME: Check we're not extracting from the initializer/condition of a control
 // flow structure.
-// FIXME: Check that we don't extract the compound statement of the
-// enclosingFunction.
 llvm::Optional<ExtractionZone> findExtractionZone(const Node *CommonAnc,
                                                   const SourceManager &SM,
                                                   const LangOptions &LangOpts) {
@@ -229,15 +250,14 @@ llvm::Optional<ExtractionZone> findExtractionZone(const Node *CommonAnc,
   ExtZone.Parent = getParentOfRootStmts(CommonAnc);
   if (!ExtZone.Parent || ExtZone.Parent->Children.empty())
     return llvm::None;
-  // Don't extract expressions.
-  // FIXME: We should extract expressions that are "statements" i.e. not
-  // subexpressions
-  if (ExtZone.Parent->Children.size() == 1 &&
-      ExtZone.getLastRootStmt()->ASTNode.get<Expr>())
-    return llvm::None;
   ExtZone.EnclosingFunction = findEnclosingFunction(ExtZone.Parent);
   if (!ExtZone.EnclosingFunction)
     return llvm::None;
+  // When there is a single RootStmt, we must check if it's valid for
+  // extraction.
+  if (ExtZone.Parent->Children.size() == 1 &&
+      !validSingleChild(ExtZone.getLastRootStmt(), ExtZone.EnclosingFunction))
+    return llvm::None;
   if (auto FuncRange =
           computeEnclosingFuncRange(ExtZone.EnclosingFunction, SM, LangOpts))
     ExtZone.EnclosingFuncRange = *FuncRange;
diff --git a/clang-tools-extra/clangd/test/initialize-params.test b/clang-tools-extra/clangd/test/initialize-params.test
index 10a354b27ceba..8a853b7c6223c 100644
--- a/clang-tools-extra/clangd/test/initialize-params.test
+++ b/clang-tools-extra/clangd/test/initialize-params.test
@@ -33,6 +33,7 @@
 # CHECK-NEXT:      "hoverProvider": true,
 # CHECK-NEXT:      "referencesProvider": true,
 # CHECK-NEXT:      "renameProvider": true,
+# CHECK-NEXT:      "selectionRangeProvider": true,
 # CHECK-NEXT:      "signatureHelpProvider": {
 # CHECK-NEXT:        "triggerCharacters": [
 # CHECK-NEXT:          "(",
diff --git a/clang-tools-extra/clangd/test/selection-range.test b/clang-tools-extra/clangd/test/selection-range.test
new file mode 100644
index 0000000000000..e7ff14a678f72
--- /dev/null
+++ b/clang-tools-extra/clangd/test/selection-range.test
@@ -0,0 +1,39 @@
+# RUN: clangd -lit-test < %s | FileCheck -strict-whitespace %s
+{"jsonrpc":"2.0","id":0,"method":"initialize","params":{"processId":123,"rootPath":"clangd","capabilities":{},"trace":"off"}}
+---
+{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"test:///main.cpp","languageId":"cpp","version":1,"text":"void func() {\n}"}}}
+---
+{"jsonrpc":"2.0","id":1,"method":"textDocument/selectionRange","params":{"textDocument":{"uri":"test:///main.cpp"},"positions":[{"line":1,"character":0}]}}
+#      CHECK:  "id": 1
+# CHECK-NEXT:  "jsonrpc": "2.0",
+# CHECK-NEXT:  "result": [
+# CHECK-NEXT:    {
+# CHECK-NEXT:      "parent": {
+# CHECK-NEXT:        "range": {
+# CHECK-NEXT:          "end": {
+# CHECK-NEXT:            "character": 1,
+# CHECK-NEXT:            "line": 1
+# CHECK-NEXT:          },
+# CHECK-NEXT:          "start": {
+# CHECK-NEXT:            "character": 0,
+# CHECK-NEXT:            "line": 0
+# CHECK-NEXT:          }
+# CHECK-NEXT:        }
+# CHECK-NEXT:      },
+# CHECK-NEXT:      "range": {
+# CHECK-NEXT:        "end": {
+# CHECK-NEXT:          "character": 1,
+# CHECK-NEXT:          "line": 1
+# CHECK-NEXT:        },
+# CHECK-NEXT:        "start": {
+# CHECK-NEXT:          "character": 12,
+# CHECK-NEXT:          "line": 0
+# CHECK-NEXT:        }
+# CHECK-NEXT:      }
+# CHECK-NEXT:    }
+# CHECK-NEXT:  ]
+# CHECK-NEXT:}
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}
diff --git a/clang-tools-extra/clangd/unittests/CMakeLists.txt b/clang-tools-extra/clangd/unittests/CMakeLists.txt
index 913401c72fcd8..d25745f94c7b9 100644
--- a/clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ b/clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -46,6 +46,7 @@ add_unittest(ClangdUnitTests ClangdTests
   FuzzyMatchTests.cpp
   GlobalCompilationDatabaseTests.cpp
   HeadersTests.cpp
+  HeaderSourceSwitchTests.cpp
   IndexActionTests.cpp
   IndexTests.cpp
   JSONTransportTests.cpp
@@ -56,6 +57,7 @@ add_unittest(ClangdUnitTests ClangdTests
   RIFFTests.cpp
   SelectionTests.cpp
   SemanticHighlightingTests.cpp
+  SemanticSelectionTests.cpp
   SerializationTests.cpp
   SourceCodeTests.cpp
   SymbolCollectorTests.cpp
diff --git a/clang-tools-extra/clangd/unittests/ClangdTests.cpp b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
index 25c82e4895fc6..3cbd33ab57ebf 100644
--- a/clang-tools-extra/clangd/unittests/ClangdTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ClangdTests.cpp
@@ -765,82 +765,6 @@ int d;
   }
 }
 
-TEST_F(ClangdVFSTest, CheckSourceHeaderSwitch) {
-  MockFSProvider FS;
-  ErrorCheckingDiagConsumer DiagConsumer;
-  MockCompilationDatabase CDB;
-  ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest());
-
-  auto SourceContents = R"cpp(
-  #include "foo.h"
-  int b = a;
-  )cpp";
-
-  auto FooCpp = testPath("foo.cpp");
-  auto FooH = testPath("foo.h");
-  auto Invalid = testPath("main.cpp");
-
-  FS.Files[FooCpp] = SourceContents;
-  FS.Files[FooH] = "int a;";
-  FS.Files[Invalid] = "int main() { \n return 0; \n }";
-
-  Optional<Path> PathResult = Server.switchSourceHeader(FooCpp);
-  EXPECT_TRUE(PathResult.hasValue());
-  ASSERT_EQ(PathResult.getValue(), FooH);
-
-  PathResult = Server.switchSourceHeader(FooH);
-  EXPECT_TRUE(PathResult.hasValue());
-  ASSERT_EQ(PathResult.getValue(), FooCpp);
-
-  SourceContents = R"c(
-  #include "foo.HH"
-  int b = a;
-  )c";
-
-  // Test with header file in capital letters and different extension, source
-  // file with different extension
-  auto FooC = testPath("bar.c");
-  auto FooHH = testPath("bar.HH");
-
-  FS.Files[FooC] = SourceContents;
-  FS.Files[FooHH] = "int a;";
-
-  PathResult = Server.switchSourceHeader(FooC);
-  EXPECT_TRUE(PathResult.hasValue());
-  ASSERT_EQ(PathResult.getValue(), FooHH);
-
-  // Test with both capital letters
-  auto Foo2C = testPath("foo2.C");
-  auto Foo2HH = testPath("foo2.HH");
-  FS.Files[Foo2C] = SourceContents;
-  FS.Files[Foo2HH] = "int a;";
-
-  PathResult = Server.switchSourceHeader(Foo2C);
-  EXPECT_TRUE(PathResult.hasValue());
-  ASSERT_EQ(PathResult.getValue(), Foo2HH);
-
-  // Test with source file as capital letter and .hxx header file
-  auto Foo3C = testPath("foo3.C");
-  auto Foo3HXX = testPath("foo3.hxx");
-
-  SourceContents = R"c(
-  #include "foo3.hxx"
-  int b = a;
-  )c";
-
-  FS.Files[Foo3C] = SourceContents;
-  FS.Files[Foo3HXX] = "int a;";
-
-  PathResult = Server.switchSourceHeader(Foo3C);
-  EXPECT_TRUE(PathResult.hasValue());
-  ASSERT_EQ(PathResult.getValue(), Foo3HXX);
-
-  // Test if asking for a corresponding file that doesn't exist returns an empty
-  // string.
-  PathResult = Server.switchSourceHeader(Invalid);
-  EXPECT_FALSE(PathResult.hasValue());
-}
-
 TEST_F(ClangdThreadingTest, NoConcurrentDiagnostics) {
   class NoConcurrentAccessDiagConsumer : public DiagnosticsConsumer {
   public:
diff --git a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
index 88d7daa7e3b43..bf100630b30b5 100644
--- a/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindTargetTests.cpp
@@ -9,6 +9,11 @@
 
 #include "Selection.h"
 #include "TestTU.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclTemplate.h"
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Testing/Support/Annotations.h"
 #include "gmock/gmock.h"
@@ -388,6 +393,32 @@ TEST_F(TargetDeclTest, Lambda) {
   EXPECT_DECLS("DeclRefExpr", "auto int x = 1");
 }
 
+TEST_F(TargetDeclTest, OverloadExpr) {
+  Code = R"cpp(
+    void func(int*);
+    void func(char*);
+
+    template <class T>
+    void foo(T t) {
+      [[func]](t);
+    };
+  )cpp";
+  EXPECT_DECLS("UnresolvedLookupExpr", "void func(int *)", "void func(char *)");
+
+  Code = R"cpp(
+    struct X {
+      void func(int*);
+      void func(char*);
+    };
+
+    template <class T>
+    void foo(X x, T t) {
+      x.[[func]](t);
+    };
+  )cpp";
+  EXPECT_DECLS("UnresolvedMemberExpr", "void func(int *)", "void func(char *)");
+}
+
 TEST_F(TargetDeclTest, ObjC) {
   Flags = {"-xobjective-c"};
   Code = R"cpp(
@@ -464,6 +495,307 @@ TEST_F(TargetDeclTest, ObjC) {
   EXPECT_DECLS("ObjCObjectTypeLoc");
 }
 
+class FindExplicitReferencesTest : public ::testing::Test {
+protected:
+  struct AllRefs {
+    std::string AnnotatedCode;
+    std::string DumpedReferences;
+  };
+
+  /// Parses \p Code, finds function '::foo' and annotates its body with results
+  /// of findExplicitReferecnces.
+  /// See actual tests for examples of annotation format.
+  AllRefs annotateReferencesInFoo(llvm::StringRef Code) {
+    TestTU TU;
+    TU.Code = Code;
+
+    auto AST = TU.build();
+
+    auto *TestDecl = &findDecl(AST, "foo");
+    if (auto *T = llvm::dyn_cast<FunctionTemplateDecl>(TestDecl))
+      TestDecl = T->getTemplatedDecl();
+    auto &Func = llvm::cast<FunctionDecl>(*TestDecl);
+
+    std::vector<ReferenceLoc> Refs;
+    findExplicitReferences(Func.getBody(), [&Refs](ReferenceLoc R) {
+      Refs.push_back(std::move(R));
+    });
+
+    auto &SM = AST.getSourceManager();
+    llvm::sort(Refs, [&](const ReferenceLoc &L, const ReferenceLoc &R) {
+      return SM.isBeforeInTranslationUnit(L.NameLoc, R.NameLoc);
+    });
+
+    std::string AnnotatedCode;
+    unsigned NextCodeChar = 0;
+    for (unsigned I = 0; I < Refs.size(); ++I) {
+      auto &R = Refs[I];
+
+      SourceLocation Pos = R.NameLoc;
+      assert(Pos.isValid());
+      if (Pos.isMacroID()) // FIXME: figure out how to show macro locations.
+        Pos = SM.getExpansionLoc(Pos);
+      assert(Pos.isFileID());
+
+      FileID File;
+      unsigned Offset;
+      std::tie(File, Offset) = SM.getDecomposedLoc(Pos);
+      if (File == SM.getMainFileID()) {
+        // Print the reference in a source code.
+        assert(NextCodeChar <= Offset);
+        AnnotatedCode += Code.substr(NextCodeChar, Offset - NextCodeChar);
+        AnnotatedCode += "$" + std::to_string(I) + "^";
+
+        NextCodeChar = Offset;
+      }
+    }
+    AnnotatedCode += Code.substr(NextCodeChar);
+
+    std::string DumpedReferences;
+    for (unsigned I = 0; I < Refs.size(); ++I)
+      DumpedReferences += llvm::formatv("{0}: {1}\n", I, Refs[I]);
+
+    return AllRefs{std::move(AnnotatedCode), std::move(DumpedReferences)};
+  }
+};
+
+TEST_F(FindExplicitReferencesTest, All) {
+  std::pair</*Code*/ llvm::StringRef, /*References*/ llvm::StringRef> Cases[] =
+      {
+          // Simple expressions.
+          {R"cpp(
+        int global;
+        int func();
+        void foo(int param) {
+          $0^global = $1^param + $2^func();
+        }
+        )cpp",
+           "0: targets = {global}\n"
+           "1: targets = {param}\n"
+           "2: targets = {func}\n"},
+          {R"cpp(
+        struct X { int a; };
+        void foo(X x) {
+          $0^x.$1^a = 10;
+        }
+        )cpp",
+           "0: targets = {x}\n"
+           "1: targets = {X::a}\n"},
+          // Namespaces and aliases.
+          {R"cpp(
+          namespace ns {}
+          namespace alias = ns;
+          void foo() {
+            using namespace $0^ns;
+            using namespace $1^alias;
+          }
+        )cpp",
+           "0: targets = {ns}\n"
+           "1: targets = {alias}\n"},
+          // Using declarations.
+          {R"cpp(
+          namespace ns { int global; }
+          void foo() {
+            using $0^ns::$1^global;
+          }
+        )cpp",
+           "0: targets = {ns}\n"
+           "1: targets = {ns::global}, qualifier = 'ns::'\n"},
+          // Simple types.
+          {R"cpp(
+         struct Struct { int a; };
+         using Typedef = int;
+         void foo() {
+           $0^Struct x;
+           $1^Typedef y;
+           static_cast<$2^Struct*>(0);
+         }
+       )cpp",
+           "0: targets = {Struct}\n"
+           "1: targets = {Typedef}\n"
+           "2: targets = {Struct}\n"},
+          // Name qualifiers.
+          {R"cpp(
+         namespace a { namespace b { struct S { typedef int type; }; } }
+         void foo() {
+           $0^a::$1^b::$2^S x;
+           using namespace $3^a::$4^b;
+           $5^S::$6^type y;
+         }
+        )cpp",
+           "0: targets = {a}\n"
+           "1: targets = {a::b}, qualifier = 'a::'\n"
+           "2: targets = {a::b::S}, qualifier = 'a::b::'\n"
+           "3: targets = {a}\n"
+           "4: targets = {a::b}, qualifier = 'a::'\n"
+           "5: targets = {a::b::S}\n"
+           "6: targets = {a::b::S::type}, qualifier = 'struct S::'\n"},
+          // Simple templates.
+          {R"cpp(
+          template <class T> struct vector { using value_type = T; };
+          template <> struct vector<bool> { using value_type = bool; };
+          void foo() {
+            $0^vector<int> vi;
+            $1^vector<bool> vb;
+          }
+        )cpp",
+           "0: targets = {vector<int>}\n"
+           "1: targets = {vector<bool>}\n"},
+          // Template type aliases.
+          {R"cpp(
+            template <class T> struct vector { using value_type = T; };
+            template <> struct vector<bool> { using value_type = bool; };
+            template <class T> using valias = vector<T>;
+            void foo() {
+              $0^valias<int> vi;
+              $1^valias<bool> vb;
+            }
+          )cpp",
+           "0: targets = {valias}\n"
+           "1: targets = {valias}\n"},
+          // MemberExpr should know their using declaration.
+          {R"cpp(
+            struct X { void func(int); }
+            struct Y : X {
+              using X::func;
+            };
+            void foo(Y y) {
+              $0^y.$1^func(1);
+            }
+        )cpp",
+           "0: targets = {y}\n"
+           "1: targets = {Y::func}\n"},
+          // DeclRefExpr should know their using declaration.
+          {R"cpp(
+            namespace ns { void bar(int); }
+            using ns::bar;
+
+            void foo() {
+              $0^bar(10);
+            }
+        )cpp",
+           "0: targets = {bar}\n"},
+          // References from a macro.
+          {R"cpp(
+            #define FOO a
+            #define BAR b
+
+            void foo(int a, int b) {
+              $0^FOO+$1^BAR;
+            }
+        )cpp",
+           "0: targets = {a}\n"
+           "1: targets = {b}\n"},
+          // No references from implicit nodes.
+          {R"cpp(
+            struct vector {
+              int *begin();
+              int *end();
+            };
+
+            void foo() {
+              for (int x : $0^vector()) {
+                $1^x = 10;
+              }
+            }
+        )cpp",
+           "0: targets = {vector}\n"
+           "1: targets = {x}\n"},
+          // Handle UnresolvedLookupExpr.
+          {R"cpp(
+            namespace ns1 { void func(char*); }
+            namespace ns2 { void func(int*); }
+            using namespace ns1;
+            using namespace ns2;
+
+            template <class T>
+            void foo(T t) {
+              $0^func($1^t);
+            }
+        )cpp",
+           "0: targets = {ns1::func, ns2::func}\n"
+           "1: targets = {t}\n"},
+          // Handle UnresolvedMemberExpr.
+          {R"cpp(
+            struct X {
+              void func(char*);
+              void func(int*);
+            };
+
+            template <class T>
+            void foo(X x, T t) {
+              $0^x.$1^func($2^t);
+            }
+        )cpp",
+           "0: targets = {x}\n"
+           "1: targets = {X::func, X::func}\n"
+           "2: targets = {t}\n"},
+          // Type template parameters.
+          {R"cpp(
+            template <class T>
+            void foo() {
+              static_cast<$0^T>(0);
+              $1^T();
+              $2^T t;
+            }
+        )cpp",
+           "0: targets = {T}\n"
+           "1: targets = {T}\n"
+           "2: targets = {T}\n"},
+          // Non-type template parameters.
+          {R"cpp(
+            template <int I>
+            void foo() {
+              int x = $0^I;
+            }
+        )cpp",
+           "0: targets = {I}\n"},
+          // Template template parameters.
+          {R"cpp(
+            template <class T> struct vector {};
+
+            template <template<class> class TT, template<class> class ...TP>
+            void foo() {
+              $0^TT<int> x;
+              $1^foo<$2^TT>();
+              $3^foo<$4^vector>()
+              $5^foo<$6^TP...>();
+            }
+        )cpp",
+           "0: targets = {TT}\n"
+           "1: targets = {foo}\n"
+           "2: targets = {TT}\n"
+           "3: targets = {foo}\n"
+           "4: targets = {vector}\n"
+           "5: targets = {foo}\n"
+           "6: targets = {TP}\n"},
+          // Non-type template parameters with declarations.
+          {R"cpp(
+            int func();
+            template <int(*)()> struct wrapper {};
+
+            template <int(*FuncParam)()>
+            void foo() {
+              $0^wrapper<$1^func> w;
+              $2^FuncParam();
+            }
+        )cpp",
+           "0: targets = {wrapper<&func>}\n"
+           "1: targets = {func}\n"
+           "2: targets = {FuncParam}\n"},
+      };
+
+  for (const auto &C : Cases) {
+    llvm::StringRef ExpectedCode = C.first;
+    llvm::StringRef ExpectedRefs = C.second;
+
+    auto Actual =
+        annotateReferencesInFoo(llvm::Annotations(ExpectedCode).code());
+    EXPECT_EQ(ExpectedCode, Actual.AnnotatedCode);
+    EXPECT_EQ(ExpectedRefs, Actual.DumpedReferences) << ExpectedCode;
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp b/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp
new file mode 100644
index 0000000000000..3b5fe86b96b6c
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/HeaderSourceSwitchTests.cpp
@@ -0,0 +1,272 @@
+//===--- HeaderSourceSwitchTests.cpp - ---------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "HeaderSourceSwitch.h"
+
+#include "SyncAPI.h"
+#include "TestFS.h"
+#include "TestTU.h"
+#include "index/MemIndex.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace clang {
+namespace clangd {
+namespace {
+
+TEST(HeaderSourceSwitchTest, FileHeuristic) {
+  MockFSProvider FS;
+  auto FooCpp = testPath("foo.cpp");
+  auto FooH = testPath("foo.h");
+  auto Invalid = testPath("main.cpp");
+
+  FS.Files[FooCpp];
+  FS.Files[FooH];
+  FS.Files[Invalid];
+  Optional<Path> PathResult =
+      getCorrespondingHeaderOrSource(FooCpp, FS.getFileSystem());
+  EXPECT_TRUE(PathResult.hasValue());
+  ASSERT_EQ(PathResult.getValue(), FooH);
+
+  PathResult = getCorrespondingHeaderOrSource(FooH, FS.getFileSystem());
+  EXPECT_TRUE(PathResult.hasValue());
+  ASSERT_EQ(PathResult.getValue(), FooCpp);
+
+  // Test with header file in capital letters and different extension, source
+  // file with different extension
+  auto FooC = testPath("bar.c");
+  auto FooHH = testPath("bar.HH");
+
+  FS.Files[FooC];
+  FS.Files[FooHH];
+  PathResult = getCorrespondingHeaderOrSource(FooC, FS.getFileSystem());
+  EXPECT_TRUE(PathResult.hasValue());
+  ASSERT_EQ(PathResult.getValue(), FooHH);
+
+  // Test with both capital letters
+  auto Foo2C = testPath("foo2.C");
+  auto Foo2HH = testPath("foo2.HH");
+  FS.Files[Foo2C];
+  FS.Files[Foo2HH];
+  PathResult = getCorrespondingHeaderOrSource(Foo2C, FS.getFileSystem());
+  EXPECT_TRUE(PathResult.hasValue());
+  ASSERT_EQ(PathResult.getValue(), Foo2HH);
+
+  // Test with source file as capital letter and .hxx header file
+  auto Foo3C = testPath("foo3.C");
+  auto Foo3HXX = testPath("foo3.hxx");
+
+  FS.Files[Foo3C];
+  FS.Files[Foo3HXX];
+  PathResult = getCorrespondingHeaderOrSource(Foo3C, FS.getFileSystem());
+  EXPECT_TRUE(PathResult.hasValue());
+  ASSERT_EQ(PathResult.getValue(), Foo3HXX);
+
+  // Test if asking for a corresponding file that doesn't exist returns an empty
+  // string.
+  PathResult = getCorrespondingHeaderOrSource(Invalid, FS.getFileSystem());
+  EXPECT_FALSE(PathResult.hasValue());
+}
+
+MATCHER_P(DeclNamed, Name, "") {
+  if (const NamedDecl *ND = dyn_cast<NamedDecl>(arg))
+    if (ND->getQualifiedNameAsString() == Name)
+      return true;
+  return false;
+}
+
+TEST(HeaderSourceSwitchTest, GetLocalDecls) {
+  TestTU TU;
+  TU.HeaderCode = R"cpp(
+  void HeaderOnly();
+  )cpp";
+  TU.Code = R"cpp(
+  void MainF1();
+  class Foo {};
+  namespace ns {
+  class Foo {
+    void method();
+    int field;
+  };
+  } // namespace ns
+
+  // Non-indexable symbols
+  namespace {
+  void Ignore1() {}
+  }
+
+  )cpp";
+
+  auto AST = TU.build();
+  EXPECT_THAT(getIndexableLocalDecls(AST),
+              testing::UnorderedElementsAre(
+                  DeclNamed("MainF1"), DeclNamed("Foo"), DeclNamed("ns::Foo"),
+                  DeclNamed("ns::Foo::method"), DeclNamed("ns::Foo::field")));
+}
+
+TEST(HeaderSourceSwitchTest, FromHeaderToSource) {
+  // build a proper index, which contains symbols:
+  //   A_Sym1, declared in TestTU.h, defined in a.cpp
+  //   B_Sym[1-2], declared in TestTU.h, defined in b.cpp
+  SymbolSlab::Builder AllSymbols;
+  TestTU Testing;
+  Testing.HeaderFilename = "TestTU.h";
+  Testing.HeaderCode = "void A_Sym1();";
+  Testing.Filename = "a.cpp";
+  Testing.Code = "void A_Sym1() {};";
+  for (auto &Sym : Testing.headerSymbols())
+    AllSymbols.insert(Sym);
+
+  Testing.HeaderCode = R"cpp(
+  void B_Sym1();
+  void B_Sym2();
+  )cpp";
+  Testing.Filename = "b.cpp";
+  Testing.Code = R"cpp(
+  void B_Sym1() {}
+  void B_Sym2() {}
+  )cpp";
+  for (auto &Sym : Testing.headerSymbols())
+    AllSymbols.insert(Sym);
+  auto Index = MemIndex::build(std::move(AllSymbols).build(), {}, {});
+
+  // Test for swtich from .h header to .cc source
+  struct {
+    llvm::StringRef HeaderCode;
+    llvm::Optional<std::string> ExpectedSource;
+  } TestCases[] = {
+      {"// empty, no header found", llvm::None},
+      {R"cpp(
+         // no definition found in the index.
+         void NonDefinition();
+       )cpp",
+       llvm::None},
+      {R"cpp(
+         void A_Sym1();
+       )cpp",
+       testPath("a.cpp")},
+      {R"cpp(
+         // b.cpp wins.
+         void A_Sym1();
+         void B_Sym1();
+         void B_Sym2();
+       )cpp",
+       testPath("b.cpp")},
+      {R"cpp(
+         // a.cpp and b.cpp have same scope, but a.cpp because "a.cpp" < "b.cpp".
+         void A_Sym1();
+         void B_Sym1();
+       )cpp",
+       testPath("a.cpp")},
+  };
+  for (const auto &Case : TestCases) {
+    TestTU TU = TestTU::withCode(Case.HeaderCode);
+    TU.Filename = "TestTU.h";
+    TU.ExtraArgs.push_back("-xc++-header"); // inform clang this is a header.
+    auto HeaderAST = TU.build();
+    EXPECT_EQ(Case.ExpectedSource,
+              getCorrespondingHeaderOrSource(testPath(TU.Filename), HeaderAST,
+                                             Index.get()));
+  }
+}
+
+TEST(HeaderSourceSwitchTest, FromSourceToHeader) {
+  // build a proper index, which contains symbols:
+  //   A_Sym1, declared in a.h, defined in TestTU.cpp
+  //   B_Sym[1-2], declared in b.h, defined in TestTU.cpp
+  TestTU TUForIndex = TestTU::withCode(R"cpp(
+  #include "a.h"
+  #include "b.h"
+
+  void A_Sym1() {}
+
+  void B_Sym1() {}
+  void B_Sym2() {}
+  )cpp");
+  TUForIndex.AdditionalFiles["a.h"] = R"cpp(
+  void A_Sym1();
+  )cpp";
+  TUForIndex.AdditionalFiles["b.h"] = R"cpp(
+  void B_Sym1();
+  void B_Sym2();
+  )cpp";
+  TUForIndex.Filename = "TestTU.cpp";
+  auto Index = TUForIndex.index();
+
+  // Test for switching from .cc source file to .h header.
+  struct {
+    llvm::StringRef SourceCode;
+    llvm::Optional<std::string> ExpectedResult;
+  } TestCases[] = {
+      {"// empty, no header found", llvm::None},
+      {R"cpp(
+         // symbol not in index, no header found
+         void Local() {}
+       )cpp",
+       llvm::None},
+
+      {R"cpp(
+         // a.h wins.
+         void A_Sym1() {}
+       )cpp",
+       testPath("a.h")},
+
+      {R"cpp(
+         // b.h wins.
+         void A_Sym1() {}
+         void B_Sym1() {}
+         void B_Sym2() {}
+       )cpp",
+       testPath("b.h")},
+
+      {R"cpp(
+         // a.h and b.h have same scope, but a.h wins because "a.h" < "b.h".
+         void A_Sym1() {}
+         void B_Sym1() {}
+       )cpp",
+       testPath("a.h")},
+  };
+  for (const auto &Case : TestCases) {
+    TestTU TU = TestTU::withCode(Case.SourceCode);
+    TU.Filename = "Test.cpp";
+    auto AST = TU.build();
+    EXPECT_EQ(Case.ExpectedResult,
+              getCorrespondingHeaderOrSource(testPath(TU.Filename), AST,
+                                             Index.get()));
+  }
+}
+
+TEST(HeaderSourceSwitchTest, ClangdServerIntegration) {
+  class IgnoreDiagnostics : public DiagnosticsConsumer {
+    void onDiagnosticsReady(PathRef File,
+                            std::vector<Diag> Diagnostics) override {}
+  } DiagConsumer;
+  MockCompilationDatabase CDB;
+  CDB.ExtraClangFlags = {"-I" +
+                         testPath("src/include")}; // add search directory.
+  MockFSProvider FS;
+  // File heuristic fails here, we rely on the index to find the .h file.
+  std::string CppPath = testPath("src/lib/test.cpp");
+  std::string HeaderPath = testPath("src/include/test.h");
+  FS.Files[HeaderPath] = "void foo();";
+  const std::string FileContent = R"cpp(
+    #include "test.h"
+    void foo() {};
+  )cpp";
+  FS.Files[CppPath] = FileContent;
+  auto Options = ClangdServer::optsForTest();
+  Options.BuildDynamicSymbolIndex = true;
+  ClangdServer Server(CDB, FS, DiagConsumer, Options);
+  runAddDocument(Server, CppPath, FileContent);
+  EXPECT_EQ(HeaderPath,
+            *llvm::cantFail(runSwitchHeaderSource(Server, CppPath)));
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
index 65020bf2a95f7..53e2f4b68e158 100644
--- a/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
+++ b/clang-tools-extra/clangd/unittests/ParsedASTTests.cpp
@@ -229,8 +229,8 @@ TEST(ParsedASTTest, CanBuildInvocationWithUnknownArgs) {
 
 TEST(ParsedASTTest, CollectsMainFileMacroExpansions) {
   Annotations TestCase(R"cpp(
-    #define MACRO_ARGS(X, Y) X Y
-    // - premable ends, macros inside preamble are not considered in main file.
+    #define ^MACRO_ARGS(X, Y) X Y
+    // - preamble ends
     ^ID(int A);
     // Macro arguments included.
     ^MACRO_ARGS(^MACRO_ARGS(^MACRO_EXP(int), A), ^ID(= 2));
@@ -270,12 +270,11 @@ TEST(ParsedASTTest, CollectsMainFileMacroExpansions) {
     int D = DEF;
   )cpp";
   ParsedAST AST = TU.build();
-  const std::vector<SourceLocation> &MacroExpansionLocations = AST.getMacros();
   std::vector<Position> MacroExpansionPositions;
-  for (const auto &L : MacroExpansionLocations)
-    MacroExpansionPositions.push_back(
-        sourceLocToPosition(AST.getSourceManager(), L));
-  EXPECT_EQ(MacroExpansionPositions, TestCase.points());
+  for (const auto &R : AST.getMacros().Ranges)
+    MacroExpansionPositions.push_back(R.start);
+  EXPECT_THAT(MacroExpansionPositions,
+              testing::UnorderedElementsAreArray(TestCase.points()));
 }
 
 } // namespace
diff --git a/clang-tools-extra/clangd/unittests/RenameTests.cpp b/clang-tools-extra/clangd/unittests/RenameTests.cpp
index 99e07e240e203..e3af711fbe36c 100644
--- a/clang-tools-extra/clangd/unittests/RenameTests.cpp
+++ b/clang-tools-extra/clangd/unittests/RenameTests.cpp
@@ -95,7 +95,19 @@ TEST(RenameTest, Renameable) {
     const char *Code;
     const char* ErrorMessage; // null if no error
     bool IsHeaderFile;
+    const SymbolIndex *Index;
   };
+  TestTU OtherFile = TestTU::withCode("Outside s; auto ss = &foo;");
+  const char *CommonHeader = R"cpp(
+    class Outside {};
+    void foo();
+  )cpp";
+  OtherFile.HeaderCode = CommonHeader;
+  OtherFile.Filename = "other.cc";
+  // The index has a "Outside" reference and a "foo" reference.
+  auto OtherFileIndex = OtherFile.index();
+  const SymbolIndex *Index = OtherFileIndex.get();
+
   const bool HeaderFile = true;
   Case Cases[] = {
       {R"cpp(// allow -- function-local
@@ -103,52 +115,55 @@ TEST(RenameTest, Renameable) {
           [[Local]] = 2;
         }
       )cpp",
-       nullptr, HeaderFile},
+       nullptr, HeaderFile, Index},
 
       {R"cpp(// allow -- symbol is indexable and has no refs in index.
         void [[On^lyInThisFile]]();
       )cpp",
-       nullptr, HeaderFile},
+       nullptr, HeaderFile, Index},
 
       {R"cpp(// disallow -- symbol is indexable and has other refs in index.
         void f() {
           Out^side s;
         }
       )cpp",
-       "used outside main file", HeaderFile},
+       "used outside main file", HeaderFile, Index},
 
       {R"cpp(// disallow -- symbol is not indexable.
         namespace {
         class Unin^dexable {};
         }
       )cpp",
-       "not eligible for indexing", HeaderFile},
+       "not eligible for indexing", HeaderFile, Index},
 
       {R"cpp(// disallow -- namespace symbol isn't supported
         namespace fo^o {}
       )cpp",
-       "not a supported kind", HeaderFile},
+       "not a supported kind", HeaderFile, Index},
 
       {
           R"cpp(
          #define MACRO 1
          int s = MAC^RO;
        )cpp",
-          "not a supported kind", HeaderFile},
+          "not a supported kind", HeaderFile, Index},
+
+      {
+
+          R"cpp(
+        struct X { X operator++(int) {} };
+        void f(X x) {x+^+;})cpp",
+          "not a supported kind", HeaderFile, Index},
 
       {R"cpp(// foo is declared outside the file.
         void fo^o() {}
-      )cpp", "used outside main file", !HeaderFile/*cc file*/},
+      )cpp", "used outside main file", !HeaderFile /*cc file*/, Index},
+
+      {R"cpp(
+         // We should detect the symbol is used outside the file from the AST.
+         void fo^o() {})cpp",
+       "used outside main file", !HeaderFile, nullptr /*no index*/},
   };
-  const char *CommonHeader = R"cpp(
-    class Outside {};
-    void foo();
-  )cpp";
-  TestTU OtherFile = TestTU::withCode("Outside s; auto ss = &foo;");
-  OtherFile.HeaderCode = CommonHeader;
-  OtherFile.Filename = "other.cc";
-  // The index has a "Outside" reference and a "foo" reference.
-  auto OtherFileIndex = OtherFile.index();
 
   for (const auto& Case : Cases) {
     Annotations T(Case.Code);
@@ -163,7 +178,7 @@ TEST(RenameTest, Renameable) {
     auto AST = TU.build();
 
     auto Results = renameWithinFile(AST, testPath(TU.Filename), T.point(),
-                                    "dummyNewName", OtherFileIndex.get());
+                                    "dummyNewName", Case.Index);
     bool WantRename = true;
     if (T.ranges().empty())
       WantRename = false;
diff --git a/clang-tools-extra/clangd/unittests/SelectionTests.cpp b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
index 964f0e98e2867..e2cbff37b3db0 100644
--- a/clang-tools-extra/clangd/unittests/SelectionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SelectionTests.cpp
@@ -354,6 +354,8 @@ TEST(SelectionTest, Selected) {
         #define ECHO(X) X
         ECHO(EC^HO([[$C[[int]]) EC^HO(a]]));
       ]])cpp",
+      R"cpp( $C[[^$C[[int]] a^]]; )cpp",
+      R"cpp( $C[[^$C[[int]] a = $C[[5]]^]]; )cpp",
   };
   for (const char *C : Cases) {
     Annotations Test(C);
diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index 50955b3e69e56..06c8d3ae1b6ca 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -425,8 +425,8 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
       // Tokens that share a source range but have conflicting Kinds are not
       // highlighted.
       R"cpp(
-      #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
-      #define DEF_CLASS(T) class T {};
+      #define $Macro[[DEF_MULTIPLE]](X) namespace X { class X { int X; }; }
+      #define $Macro[[DEF_CLASS]](T) class T {};
       // Preamble ends.
       $Macro[[DEF_MULTIPLE]](XYZ);
       $Macro[[DEF_MULTIPLE]](XYZW);
@@ -465,8 +465,8 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
       }
     )cpp",
       R"cpp(
-      #define fail(expr) expr
-      #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); }
+      #define $Macro[[fail]](expr) expr
+      #define $Macro[[assert]](COND) if (!(COND)) { fail("assertion failed" #COND); }
       // Preamble ends.
       $Primitive[[int]] $Variable[[x]];
       $Primitive[[int]] $Variable[[y]];
diff --git a/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
new file mode 100644
index 0000000000000..b9ca0273a8233
--- /dev/null
+++ b/clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp
@@ -0,0 +1,181 @@
+//===-- SemanticSelectionTests.cpp  ----------------*- C++ -*--------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Annotations.h"
+#include "ClangdServer.h"
+#include "Matchers.h"
+#include "Protocol.h"
+#include "SemanticSelection.h"
+#include "SourceCode.h"
+#include "SyncAPI.h"
+#include "TestFS.h"
+#include "TestTU.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/Error.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <vector>
+namespace clang {
+namespace clangd {
+namespace {
+using ::testing::ElementsAreArray;
+
+class IgnoreDiagnostics : public DiagnosticsConsumer {
+  void onDiagnosticsReady(PathRef File,
+                          std::vector<Diag> Diagnostics) override {}
+};
+
+TEST(SemanticSelection, All) {
+  const char *Tests[] = {
+      R"cpp( // Single statement in a function body.
+        [[void func() [[{
+          [[[[int v = [[1^00]]]];]]
+        }]]]]
+      )cpp",
+      R"cpp( // Expression
+        [[void func() [[{
+          int a = 1;
+          // int v = (10 + 2) * (a + a);
+          [[[[int v = [[[[([[[[10^]] + 2]])]] * (a + a)]]]];]]
+        }]]]]
+      )cpp",
+      R"cpp( // Function call.
+        int add(int x, int y) { return x + y; }
+        [[void callee() [[{
+          // int res = add(11, 22);
+          [[[[int res = [[add([[1^1]], 22)]]]];]]
+        }]]]]
+      )cpp",
+      R"cpp( // Tricky macros.
+        #define MUL ) * (
+        [[void func() [[{
+          // int var = (4 + 15 MUL 6 + 10);
+          [[[[int var = [[[[([[4 + [[1^5]]]] MUL]] 6 + 10)]]]];]]
+        }]]]]
+       )cpp",
+      R"cpp( // Cursor inside a macro.
+        #define HASH(x) ((x) % 10)
+        [[void func() [[{
+          [[[[int a = [[HASH([[[[2^3]] + 34]])]]]];]]
+        }]]]]
+       )cpp",
+      R"cpp( // Cursor on a macro.
+        #define HASH(x) ((x) % 10)
+        [[void func() [[{
+          [[[[int a = [[HA^SH(23)]]]];]]
+        }]]]]
+       )cpp",
+      R"cpp( // Multiple declaration.
+        [[void func() [[{
+          [[[[int var1, var^2]], var3;]]
+        }]]]]
+       )cpp",
+      R"cpp( // Before comment.
+        [[void func() [[{
+          int var1 = 1;
+          [[[[int var2 = [[[[var1]]^ /*some comment*/ + 41]]]];]]
+        }]]]]
+       )cpp",
+      // Empty file.
+      "^",
+      // FIXME: We should get the whole DeclStmt as a range.
+      R"cpp( // Single statement in TU.
+        [[int v = [[1^00]]]];
+      )cpp",
+      // FIXME: No node found associated to the position.
+      R"cpp( // Cursor at end of VarDecl.
+        void func() {
+          int v = 100 + 100^;
+        }
+      )cpp",
+      // FIXME: No node found associated to the position.
+      R"cpp( // Cursor in between spaces.
+        void func() {
+          int v = 100 + ^  100;
+        }
+      )cpp",
+      // Structs.
+      R"cpp(
+        struct AAA { struct BBB { static int ccc(); };};
+        [[void func() [[{
+          // int x = AAA::BBB::ccc();
+          [[[[int x = [[[[AAA::BBB::c^cc]]()]]]];]]
+        }]]]]
+      )cpp",
+      R"cpp(
+        struct AAA { struct BBB { static int ccc(); };};
+        [[void func() [[{
+          // int x = AAA::BBB::ccc();
+          [[[[int x = [[[[[[[[[[AA^A]]::]]BBB::]]ccc]]()]]]];]]
+        }]]]]
+      )cpp",
+      R"cpp( // Inside struct.
+        struct A { static int a(); };
+        [[struct B { 
+          [[static int b() [[{
+            [[return [[[[1^1]] + 2]]]];
+          }]]]]
+        }]];
+      )cpp",
+      // Namespaces.
+      R"cpp( 
+        [[namespace nsa { 
+          [[namespace nsb { 
+            static int ccc();
+            [[void func() [[{
+              // int x = nsa::nsb::ccc();
+              [[[[int x = [[[[nsa::nsb::cc^c]]()]]]];]]
+            }]]]]
+          }]]
+        }]]
+      )cpp",
+
+  };
+
+  for (const char *Test : Tests) {
+    auto T = Annotations(Test);
+    auto AST = TestTU::withCode(T.code()).build();
+    EXPECT_THAT(llvm::cantFail(getSemanticRanges(AST, T.point())),
+                ElementsAreArray(T.ranges()))
+        << Test;
+  }
+}
+
+TEST(SemanticSelection, RunViaClangDServer) {
+  MockFSProvider FS;
+  IgnoreDiagnostics DiagConsumer;
+  MockCompilationDatabase CDB;
+  ClangdServer Server(CDB, FS, DiagConsumer, ClangdServer::optsForTest());
+
+  auto FooH = testPath("foo.h");
+  FS.Files[FooH] = R"cpp(
+    int foo(int x);
+    #define HASH(x) ((x) % 10)
+  )cpp";
+
+  auto FooCpp = testPath("Foo.cpp");
+  const char *SourceContents = R"cpp(
+  #include "foo.h"
+  [[void bar(int& inp) [[{
+    // inp = HASH(foo(inp));
+    [[inp = [[HASH([[foo([[in^p]])]])]]]];
+  }]]]]
+  )cpp";
+  Annotations SourceAnnotations(SourceContents);
+  FS.Files[FooCpp] = SourceAnnotations.code();
+  Server.addDocument(FooCpp, SourceAnnotations.code());
+
+  auto Ranges = runSemanticRanges(Server, FooCpp, SourceAnnotations.point());
+  ASSERT_TRUE(bool(Ranges))
+      << "getSemanticRange returned an error: " << Ranges.takeError();
+  EXPECT_THAT(*Ranges, ElementsAreArray(SourceAnnotations.ranges()));
+}
+} // namespace
+} // namespace clangd
+} // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp b/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp
index cd9ccf0481a70..e49f64cef139e 100644
--- a/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SourceCodeTests.cpp
@@ -319,14 +319,29 @@ struct Bar { int func(); };
 Bar* bar;
   )cpp";
   // First ^ is the expected beginning, last is the search position.
-  for (std::string Text : std::vector<std::string>{
+  for (const std::string &Text : std::vector<std::string>{
            "int ^f^oo();", // inside identifier
            "int ^foo();",  // beginning of identifier
            "int ^foo^();", // end of identifier
            "int foo(^);",  // non-identifier
            "^int foo();",  // beginning of file (can't back up)
            "int ^f0^0();", // after a digit (lexing at N-1 is wrong)
-           "int ^λλ^λ();", // UTF-8 handled properly when backing up
+           "/^/ comments", // non-interesting token
+           "void f(int abc) { abc ^ ++; }",    // whitespace
+           "void f(int abc) { ^abc^++; }",     // range of identifier
+           "void f(int abc) { ++^abc^; }",     // range of identifier
+           "void f(int abc) { ++^abc; }",      // range of identifier
+           "void f(int abc) { ^+^+abc; }",     // range of operator
+           "void f(int abc) { ^abc^ ++; }",    // range of identifier
+           "void f(int abc) { abc ^++^; }",    // range of operator
+           "void f(int abc) { ^++^ abc; }",    // range of operator
+           "void f(int abc) { ++ ^abc^; }",    // range of identifier
+           "void f(int abc) { ^++^/**/abc; }", // range of operator
+           "void f(int abc) { ++/**/^abc; }",  // range of identifier
+           "void f(int abc) { ^abc^/**/++; }", // range of identifier
+           "void f(int abc) { abc/**/^++; }",  // range of operator
+           "void f() {^ }", // outside of identifier and operator
+           "int ^λλ^λ();",  // UTF-8 handled properly when backing up
 
            // identifier in macro arg
            "MACRO(bar->^func())",  // beginning of identifier
@@ -441,6 +456,16 @@ TEST(SourceCodeTests, VisibleNamespaces) {
               "c::d",
           },
       },
+      {
+          "",
+          {""},
+      },
+      {
+          R"cpp(
+            // Parse until EOF
+            namespace bar{})cpp",
+          {""},
+      },
   };
   for (const auto& Case : Cases) {
     EXPECT_EQ(Case.second,
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.cpp b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
index af9ef9ffa1e85..812fa7a0f2ecb 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.cpp
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.cpp
@@ -145,5 +145,19 @@ RefSlab getRefs(const SymbolIndex &Index, SymbolID ID) {
   return std::move(Slab).build();
 }
 
+llvm::Expected<std::vector<Range>>
+runSemanticRanges(ClangdServer &Server, PathRef File, Position Pos) {
+  llvm::Optional<llvm::Expected<std::vector<Range>>> Result;
+  Server.semanticRanges(File, Pos, capture(Result));
+  return std::move(*Result);
+}
+
+llvm::Expected<llvm::Optional<clangd::Path>>
+runSwitchHeaderSource(ClangdServer &Server, PathRef File) {
+  llvm::Optional<llvm::Expected<llvm::Optional<clangd::Path>>> Result;
+  Server.switchSourceHeader(File, capture(Result));
+  return std::move(*Result);
+}
+
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/clangd/unittests/SyncAPI.h b/clang-tools-extra/clangd/unittests/SyncAPI.h
index c141652415313..5ffed1fbb120c 100644
--- a/clang-tools-extra/clangd/unittests/SyncAPI.h
+++ b/clang-tools-extra/clangd/unittests/SyncAPI.h
@@ -53,6 +53,12 @@ SymbolSlab runFuzzyFind(const SymbolIndex &Index, StringRef Query);
 SymbolSlab runFuzzyFind(const SymbolIndex &Index, const FuzzyFindRequest &Req);
 RefSlab getRefs(const SymbolIndex &Index, SymbolID ID);
 
+llvm::Expected<std::vector<Range>>
+runSemanticRanges(ClangdServer &Server, PathRef File, Position Pos);
+
+llvm::Expected<llvm::Optional<clangd::Path>>
+runSwitchHeaderSource(ClangdServer &Server, PathRef File);
+
 } // namespace clangd
 } // namespace clang
 
diff --git a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
index 5dbfca5f2462c..db05c70aa7332 100644
--- a/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TUSchedulerTests.cpp
@@ -769,12 +769,14 @@ TEST_F(TUSchedulerTests, CommandLineWarnings) {
   // We should not see warnings from command-line parsing.
   CDB.ExtraClangFlags = {"-Wsome-unknown-warning"};
 
+  // (!) 'Ready' must live longer than TUScheduler.
+  Notification Ready;
+
   TUScheduler S(CDB, /*AsyncThreadsCount=*/getDefaultAsyncThreadsCount(),
                 /*StorePreambleInMemory=*/true, /*ASTCallbacks=*/captureDiags(),
                 /*UpdateDebounce=*/std::chrono::steady_clock::duration::zero(),
                 ASTRetentionPolicy());
 
-  Notification Ready;
   std::vector<Diag> Diagnostics;
   updateWithDiags(S, testPath("foo.cpp"), "void test() {}",
                   WantDiagnostics::Yes, [&](std::vector<Diag> D) {
diff --git a/clang-tools-extra/clangd/unittests/TweakTests.cpp b/clang-tools-extra/clangd/unittests/TweakTests.cpp
index c3384af082228..97cd4a2cafa66 100644
--- a/clang-tools-extra/clangd/unittests/TweakTests.cpp
+++ b/clang-tools-extra/clangd/unittests/TweakTests.cpp
@@ -540,13 +540,12 @@ TEST_F(ExtractFunctionTest, FunctionTest) {
   EXPECT_EQ(apply("int x = 0; [[x++;]]"), "unavailable");
   // We don't support extraction from lambdas.
   EXPECT_EQ(apply("auto lam = [](){ [[int x;]] }; "), "unavailable");
+  // Partial statements aren't extracted.
+  EXPECT_THAT(apply("int [[x = 0]];"), "unavailable");
 
   // Ensure that end of Zone and Beginning of PostZone being adjacent doesn't
   // lead to break being included in the extraction zone.
   EXPECT_THAT(apply("for(;;) { [[int x;]]break; }"), HasSubstr("extracted"));
-  // FIXME: This should be unavailable since partially selected but
-  // selectionTree doesn't always work correctly for VarDecls.
-  EXPECT_THAT(apply("int [[x = 0]];"), HasSubstr("extracted"));
   // FIXME: ExtractFunction should be unavailable inside loop construct
   // initalizer/condition.
   EXPECT_THAT(apply(" for([[int i = 0;]];);"), HasSubstr("extracted"));
@@ -554,7 +553,6 @@ TEST_F(ExtractFunctionTest, FunctionTest) {
   EXPECT_THAT(apply(" [[int a = 5;]] a++; "), StartsWith("fail"));
   // Don't extract return
   EXPECT_THAT(apply(" if(true) [[return;]] "), StartsWith("fail"));
-  
 }
 
 TEST_F(ExtractFunctionTest, FileTest) {
@@ -631,6 +629,16 @@ void f(const int c) {
     F ([[int x = 0;]])
   )cpp";
   EXPECT_EQ(apply(MacroFailInput), "unavailable");
+
+  // Shouldn't crash.
+  EXPECT_EQ(apply("void f([[int a]]);"), "unavailable");
+  // Don't extract if we select the entire function body (CompoundStmt).
+  std::string CompoundFailInput = R"cpp(
+    void f() [[{
+      int a;
+    }]]
+  )cpp";
+  EXPECT_EQ(apply(CompoundFailInput), "unavailable");
 }
 
 TEST_F(ExtractFunctionTest, ControlFlow) {
diff --git a/clang-tools-extra/clangd/unittests/XRefsTests.cpp b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
index e4a4a6f82587a..60cba82492d72 100644
--- a/clang-tools-extra/clangd/unittests/XRefsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/XRefsTests.cpp
@@ -10,6 +10,7 @@
 #include "Matchers.h"
 #include "ParsedAST.h"
 #include "Protocol.h"
+#include "SourceCode.h"
 #include "SyncAPI.h"
 #include "TestFS.h"
 #include "TestIndex.h"
@@ -18,13 +19,19 @@
 #include "index/FileIndex.h"
 #include "index/MemIndex.h"
 #include "index/SymbolCollector.h"
+#include "clang/AST/Decl.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Index/IndexingAction.h"
 #include "llvm/ADT/None.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <string>
+#include <vector>
 
 namespace clang {
 namespace clangd {
@@ -434,6 +441,15 @@ TEST(LocateSymbol, All) {
           auto x = m^akeX();
         }
       )cpp",
+
+      R"cpp(
+        struct X {
+          X& [[operator]]++() {}
+        };
+        void foo(X& x) {
+          +^+x;
+        }
+      )cpp",
   };
   for (const char *Test : Tests) {
     Annotations T(Test);
@@ -2187,6 +2203,84 @@ TEST(GetDeducedType, KwAutoExpansion) {
   }
 }
 
+TEST(GetNonLocalDeclRefs, All) {
+  struct Case {
+    llvm::StringRef AnnotatedCode;
+    std::vector<llvm::StringRef> ExpectedDecls;
+  } Cases[] = {
+      {
+          // VarDecl and ParamVarDecl
+          R"cpp(
+            void bar();
+            void ^foo(int baz) {
+              int x = 10;
+              bar();
+            })cpp",
+          {"bar"},
+      },
+      {
+          // Method from class
+          R"cpp(
+            class Foo { public: void foo(); };
+            class Bar {
+              void foo();
+              void bar();
+            };
+            void Bar::^foo() {
+              Foo f;
+              bar();
+              f.foo();
+            })cpp",
+          {"Bar", "Bar::bar", "Foo", "Foo::foo"},
+      },
+      {
+          // Local types
+          R"cpp(
+            void ^foo() {
+              class Foo { public: void foo() {} };
+              class Bar { public: void bar() {} };
+              Foo f;
+              Bar b;
+              b.bar();
+              f.foo();
+            })cpp",
+          {},
+      },
+      {
+          // Template params
+          R"cpp(
+            template <typename T, template<typename> class Q>
+            void ^foo() {
+              T x;
+              Q<T> y;
+            })cpp",
+          {},
+      },
+  };
+  for (const Case &C : Cases) {
+    Annotations File(C.AnnotatedCode);
+    auto AST = TestTU::withCode(File.code()).build();
+    ASSERT_TRUE(AST.getDiagnostics().empty())
+        << AST.getDiagnostics().begin()->Message;
+    SourceLocation SL = llvm::cantFail(
+        sourceLocationInMainFile(AST.getSourceManager(), File.point()));
+
+    const FunctionDecl *FD =
+        llvm::dyn_cast<FunctionDecl>(&findDecl(AST, [SL](const NamedDecl &ND) {
+          return ND.getLocation() == SL && llvm::isa<FunctionDecl>(ND);
+        }));
+    ASSERT_NE(FD, nullptr);
+
+    auto NonLocalDeclRefs = getNonLocalDeclRefs(AST, FD);
+    std::vector<std::string> Names;
+    for (const Decl *D : NonLocalDeclRefs) {
+      if (const auto *ND = llvm::dyn_cast<NamedDecl>(D))
+        Names.push_back(ND->getQualifiedNameAsString());
+    }
+    EXPECT_THAT(Names, UnorderedElementsAreArray(C.ExpectedDecls));
+  }
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 766bbee49717f..e5be18202c9a8 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -73,11 +73,20 @@ Improvements to clang-tidy
   Finds instances where variables with static storage are initialized
   dynamically in header files.
 
-- New :doc:`linuxkernel-must-use-errs
-  <clang-tidy/checks/linuxkernel-must-use-errs>` check.
+- New :doc:`bugprone-infinite-loop
+  <clang-tidy/checks/bugprone-infinite-loop>` check.
 
-  Checks Linux kernel code to see if it uses the results from the functions in
-  ``linux/err.h``.
+  Finds obvious infinite loops (loops where the condition variable is not
+  changed at all).
+
+- New :doc:`cppcoreguidelines-init-variables
+  <clang-tidy/checks/cppcoreguidelines-init-variables>` check.
+
+- New :doc:`darwin-dispatch-once-nonstatic
+  <clang-tidy/checks/darwin-dispatch-once-nonstatic>` check.
+
+  Finds declarations of ``dispatch_once_t`` variables without static or global
+  storage.
 
 - New :doc:`google-upgrade-googletest-case
   <clang-tidy/checks/google-upgrade-googletest-case>` check.
@@ -85,12 +94,32 @@ Improvements to clang-tidy
   Finds uses of deprecated Googletest APIs with names containing ``case`` and
   replaces them with equivalent APIs with ``suite``.
 
+- New :doc:`linuxkernel-must-use-errs
+  <clang-tidy/checks/linuxkernel-must-use-errs>` check.
+
+  Checks Linux kernel code to see if it uses the results from the functions in
+  ``linux/err.h``.
+
 - New :doc:`llvm-prefer-register-over-unsigned
   <clang-tidy/checks/llvm-prefer-register-over-unsigned>` check.
 
   Finds historical use of ``unsigned`` to hold vregs and physregs and rewrites
   them to use ``Register``
 
+- New :doc:`objc-missing-hash
+  <clang-tidy/checks/objc-missing-hash>` check.
+
+  Finds Objective-C implementations that implement ``-isEqual:`` without also
+  appropriately implementing ``-hash``.
+
+- Improved :doc:`bugprone-posix-return
+  <clang-tidy/checks/bugprone-posix-return>` check.
+
+  Now also checks if any calls to ``pthread_*`` functions expect negative return
+  values.
+
+- The 'objc-avoid-spinlock' check was renamed to :doc:`darwin-avoid-spinlock
+  <clang-tidy/checks/darwin-avoid-spinlock>`
 
 Improvements to include-fixer
 -----------------------------
diff --git a/clang-tools-extra/docs/clang-doc.rst b/clang-tools-extra/docs/clang-doc.rst
index 6f61970a58577..0c47405fb5a97 100644
--- a/clang-tools-extra/docs/clang-doc.rst
+++ b/clang-tools-extra/docs/clang-doc.rst
@@ -12,7 +12,7 @@ source code and comments.
 
 The tool is in a very early development stage, so you might encounter bugs and
 crashes. Submitting reports with information about how to reproduce the issue
-to `the LLVM bugtracker <https://llvm.org/bugs>`_ will definitely help the
+to `the LLVM bugtracker <https://bugs.llvm.org/>`_ will definitely help the
 project. If you have any ideas or suggestions, please to put a feature request
 there.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-infinite-loop.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-infinite-loop.rst
new file mode 100644
index 0000000000000..89502c1882a92
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-infinite-loop.rst
@@ -0,0 +1,32 @@
+.. title:: clang-tidy - bugprone-infinite-loop
+
+bugprone-infinite-loop
+======================
+
+Finds obvious infinite loops (loops where the condition variable is not changed
+at all).
+
+Finding infinite loops is well-known to be impossible (halting problem).
+However, it is possible to detect some obvious infinite loops, for example, if
+the loop condition is not changed. This check detects such loops. A loop is
+considered infinite if it does not have any loop exit statement (``break``,
+``continue``, ``goto``, ``return``, ``throw`` or a call to a function called as
+``[[noreturn]]``) and all of the following conditions hold for every variable in
+the condition:
+
+- It is a local variable.
+- It has no reference or pointer aliases.
+- It is not a structure or class member.
+
+Furthermore, the condition must not contain a function call to consider the loop
+infinite since functions may return different values for different calls.
+
+For example, the following loop is considered infinite `i` is not changed in
+the body:
+
+.. code-block:: c++
+
+  int i = 0, j = 0;
+  while (i < 10) {
+    ++j;
+  }
diff --git a/clang-tools-extra/docs/clang-tidy/checks/bugprone-posix-return.rst b/clang-tools-extra/docs/clang-tidy/checks/bugprone-posix-return.rst
index 2248efab551f4..a5c4ccb0c4d7c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/bugprone-posix-return.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/bugprone-posix-return.rst
@@ -3,9 +3,9 @@
 bugprone-posix-return
 =====================
 
-Checks if any calls to POSIX functions (except ``posix_openpt``) expect negative
-return values. These functions return either ``0`` on success or an ``errno`` on failure,
-which is positive only.
+Checks if any calls to ``pthread_*`` or ``posix_*`` functions
+(except ``posix_openpt``) expect negative return values. These functions return
+either ``0`` on success or an ``errno`` on failure, which is positive only.
 
 Example buggy usage looks like:
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-init-variables.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-init-variables.rst
new file mode 100644
index 0000000000000..e1b0fe3230007
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines-init-variables.rst
@@ -0,0 +1,51 @@
+.. title:: clang-tidy - cppcoreguidelines-init-variables
+
+cppcoreguidelines-init-variables
+================================
+
+Checks whether there are local variables that are declared without an
+initial value. These may lead to unexpected behaviour if there is a
+code path that reads the variable before assigning to it.
+
+Only integers, booleans, floats, doubles and pointers are checked. The
+fix option initializes all detected values with the value of zero. An
+exception is float and double types, which are initialized to NaN.
+
+As an example a function that looks like this:
+
+.. code-block:: c++
+
+   void function() {
+     int x;
+     char *txt;
+     double d;
+
+     // Rest of the function.
+   }
+
+Would be rewritten to look like this:
+
+.. code-block:: c++
+
+   #include <math.h>
+
+   void function() {
+     int x = 0;
+     char *txt = nullptr;
+     double d = NAN;
+
+     // Rest of the function.
+   }
+
+Options
+-------
+
+.. option:: IncludeStyle
+
+   A string specifying which include-style is used, `llvm` or
+   `google`. Default is `llvm`.
+
+.. option:: MathHeader
+
+   A string specifying the header to include to get the definition of
+   `NAN`. Default is `math.h`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/objc-avoid-spinlock.rst b/clang-tools-extra/docs/clang-tidy/checks/darwin-avoid-spinlock.rst
similarity index 79%
rename from clang-tools-extra/docs/clang-tidy/checks/objc-avoid-spinlock.rst
rename to clang-tools-extra/docs/clang-tidy/checks/darwin-avoid-spinlock.rst
index d1b11fcbab68e..fd1f5d294a472 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/objc-avoid-spinlock.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/darwin-avoid-spinlock.rst
@@ -1,7 +1,7 @@
-.. title:: clang-tidy - objc-avoid-spinlock
+.. title:: clang-tidy - darwin-avoid-spinlock
 
-objc-avoid-spinlock
-===================
+darwin-avoid-spinlock
+=====================
 
 Finds usages of ``OSSpinlock``, which is deprecated due to potential livelock
 problems. 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/darwin-dispatch-once-nonstatic.rst b/clang-tools-extra/docs/clang-tidy/checks/darwin-dispatch-once-nonstatic.rst
new file mode 100644
index 0000000000000..4506bfff2cf27
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/darwin-dispatch-once-nonstatic.rst
@@ -0,0 +1,22 @@
+.. title:: clang-tidy - darwin-dispatch-once-nonstatic
+
+darwin-dispatch-once-nonstatic
+==============================
+
+Finds declarations of ``dispatch_once_t`` variables without static or global
+storage. The behavior of using ``dispatch_once_t`` predicates with automatic or
+dynamic storage is undefined by libdispatch, and should be avoided.
+
+It is a common pattern to have functions initialize internal static or global
+data once when the function runs, but programmers have been known to miss the
+static on the ``dispatch_once_t`` predicate, leading to an uninitialized flag
+value at the mercy of the stack.
+
+Programmers have also been known to make ``dispatch_once_t`` variables be
+members of structs or classes, with the intent to lazily perform some expensive
+struct or class member initialization only once; however, this violates the
+libdispatch requirements.
+
+See the discussion section of
+`Apple's dispatch_once documentation <https://developer.apple.com/documentation/dispatch/1447169-dispatch_once>`_
+for more information.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index a906a5b291178..f0f35bd9e621d 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -51,6 +51,7 @@ Clang-Tidy Checks
    bugprone-forwarding-reference-overload
    bugprone-inaccurate-erase
    bugprone-incorrect-roundings
+   bugprone-infinite-loop
    bugprone-integer-division
    bugprone-lambda-function-name
    bugprone-macro-parentheses
@@ -193,6 +194,7 @@ Clang-Tidy Checks
    cppcoreguidelines-avoid-magic-numbers (redirects to readability-magic-numbers) <cppcoreguidelines-avoid-magic-numbers>
    cppcoreguidelines-c-copy-assignment-signature (redirects to misc-unconventional-assign-operator) <cppcoreguidelines-c-copy-assignment-signature>
    cppcoreguidelines-explicit-virtual-functions (redirects to modernize-use-override) <cppcoreguidelines-explicit-virtual-functions>
+   cppcoreguidelines-init-variables
    cppcoreguidelines-interfaces-global-init
    cppcoreguidelines-macro-usage
    cppcoreguidelines-narrowing-conversions
@@ -211,6 +213,8 @@ Clang-Tidy Checks
    cppcoreguidelines-pro-type-vararg
    cppcoreguidelines-slicing
    cppcoreguidelines-special-member-functions
+   darwin-avoid-spinlock
+   darwin-dispatch-once-nonstatic
    fuchsia-default-arguments-calls
    fuchsia-default-arguments-declarations
    fuchsia-header-anon-namespaces (redirects to google-build-namespaces) <fuchsia-header-anon-namespaces>
@@ -323,8 +327,8 @@ Clang-Tidy Checks
    mpi-buffer-deref
    mpi-type-mismatch
    objc-avoid-nserror-init
-   objc-avoid-spinlock
    objc-forbidden-subclassing
+   objc-missing-hash
    objc-property-declaration
    objc-super-self
    openmp-exception-escape
diff --git a/clang-tools-extra/docs/clang-tidy/checks/objc-missing-hash.rst b/clang-tools-extra/docs/clang-tidy/checks/objc-missing-hash.rst
new file mode 100644
index 0000000000000..ea8f775897c21
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/objc-missing-hash.rst
@@ -0,0 +1,16 @@
+.. title:: clang-tidy - objc-missing-hash
+
+objc-missing-hash
+=================
+
+Finds Objective-C implementations that implement ``-isEqual:`` without also
+appropriately implementing ``-hash``.
+
+Apple documentation highlights that objects that are equal must have the same
+hash value:
+https://developer.apple.com/documentation/objectivec/1418956-nsobject/1418795-isequal?language=objc
+
+Note that the check only verifies the presence of ``-hash`` in scenarios where
+its omission could result in unexpected behavior. The verification of the
+implementation of ``-hash`` is the responsibility of the developer, e.g.,
+through the addition of unit tests to verify the implementation.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
index 8cf9318fd7bf7..f327ac27e8f0a 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/performance-inefficient-vector-operation.rst
@@ -6,6 +6,10 @@ performance-inefficient-vector-operation
 Finds possible inefficient ``std::vector`` operations (e.g. ``push_back``,
 ``emplace_back``) that may cause unnecessary memory reallocations.
 
+It can also find calls that add element to protobuf repeated field in a loop
+without calling Reserve() before the loop. Calling Reserve() first can avoid
+unnecessary memory reallocations.
+
 Currently, the check only detects following kinds of loops with a single
 statement body:
 
@@ -21,6 +25,13 @@ statement body:
     // statement before the for statement.
   }
 
+  SomeProto p;
+  for (int i = 0; i < n; ++i) {
+    p.add_xxx(n);
+    // This will trigger the warning since the add_xxx may cause multiple memory
+    // relloacations. This can be avoid by inserting a
+    // 'p.mutable_xxx().Reserve(n)' statement before the for statement.
+  }
 
 * For-range loops like ``for (range-declaration : range_expression)``, the type
   of ``range_expression`` can be ``std::vector``, ``std::array``,
@@ -47,3 +58,9 @@ Options
 
    Semicolon-separated list of names of vector-like classes. By default only
    ``::std::vector`` is considered.
+
+.. option:: EnableProto
+
+   When non-zero, the check will also warn on inefficient operations for proto
+   repeated fields. Otherwise, the check only warns on inefficient vector
+   operations. Default is `0`.
diff --git a/clang-tools-extra/docs/clangd/Installation.rst b/clang-tools-extra/docs/clangd/Installation.rst
index caf49a75d1105..18e1ae9817fc2 100644
--- a/clang-tools-extra/docs/clangd/Installation.rst
+++ b/clang-tools-extra/docs/clangd/Installation.rst
@@ -352,20 +352,28 @@ Clangd will assume the compile command is ``clang $FLAGS some_file.cc``.
 Creating this file by hand is a reasonable place to start if your project is
 quite simple.
 
-Project-wide Index
-==================
-
-By default clangd only has a view on symbols coming from files you are
-currently editing. You can extend this view to whole project by providing a
-project-wide index to clangd.  There are two ways to do this.
-
-- Pass an experimental `-background-index` command line argument.  With
-  this feature enabled, clangd incrementally builds an index of projects
-  that you work on and uses the just-built index automatically.
-
-- Generate an index file using `clangd-indexer
-  <https://github.com/llvm/llvm-project/blob/master/clang-tools-extra/clangd/indexer/IndexerMain.cpp>`__
-  Then you can pass generated index file to clangd using
-  `-index-file=/path/to/index_file`.  *Note that clangd-indexer isn't
-  included alongside clangd in the Debian clang-tools package. You will
-  likely have to build it from source to use this option.*
+Background Indexing
+===================
+
+clangd builds an incremental index of your project (all files listed in the
+compilation database). The index improves code navigation features (go-to-definition,
+find-references) and code completion.
+
+- clangd only uses idle cores to build the index, you can limit the total
+  amount of cores by passing the `-j=<number>` flag;
+- the index is saved to the ``.clangd/index`` in the project root; index shards
+  for common headers e.g. STL will be stored in `$HOME/.clangd/index`;
+- background indexing can be disabled by the ``--background-index=false`` flag;
+  Note that, disabling background-index will limit clangd's knowledge about your
+  codebase to files you are currently editing.
+
+Build Index Manually
+====================
+
+**DISCLAIMER: This is mainly for clangd developers.**
+
+There is a `clangd-indexer <https://github.com/llvm/llvm-project/blob/master/clang-tools-extra/clangd/indexer/IndexerMain.cpp>`__
+which generates an index file for your project. To use the index, pass the flag
+`-index=file=/path/to/index_file` to clangd. *Note that clangd-indexer isn't
+included alongside clangd in the Debian clang-tools package. You will likely
+have to build it from source to use this option.*
diff --git a/clang-tools-extra/modularize/Modularize.cpp b/clang-tools-extra/modularize/Modularize.cpp
index 1905fdf4e2aa8..8d48324700e2e 100644
--- a/clang-tools-extra/modularize/Modularize.cpp
+++ b/clang-tools-extra/modularize/Modularize.cpp
@@ -585,6 +585,8 @@ class CollectEntitiesVisitor
       LinkageLabel = "extern \"C\" {}";
       break;
     case LinkageSpecDecl::lang_cxx:
+    case LinkageSpecDecl::lang_cxx_11:
+    case LinkageSpecDecl::lang_cxx_14:
       LinkageLabel = "extern \"C++\" {}";
       break;
     }
diff --git a/clang-tools-extra/test/clang-tidy/bugprone-argument-comment.cpp b/clang-tools-extra/test/clang-tidy/bugprone-argument-comment.cpp
index 08f87170c42a4..8a4d9e5b644ca 100644
--- a/clang-tools-extra/test/clang-tidy/bugprone-argument-comment.cpp
+++ b/clang-tools-extra/test/clang-tidy/bugprone-argument-comment.cpp
@@ -63,6 +63,28 @@ void ignores_underscores() {
   f3(/*With_Underscores=*/false);
 }
 
+namespace IgnoresImplicit {
+struct S {
+  S(int x);
+  int x;
+};
+
+struct T {
+  // Use two arguments (one defaulted) because simplistic check for implicit
+  // constructor looks for only one argument. We need to default the argument so
+  // that it will still be triggered implicitly.  This is not contrived -- it
+  // comes up in real code, for example std::set(std::initializer_list...).
+  T(S s, int y = 0);
+};
+
+void k(T arg1);
+
+void mynewtest() {
+  int foo = 3;
+  k(/*arg1=*/S(foo));
+}
+} // namespace IgnoresImplicit
+
 namespace ThisEditDistanceAboveThreshold {
 void f4(int xxx);
 void g() { f4(/*xyz=*/0); }
diff --git a/clang-tools-extra/test/clang-tidy/bugprone-infinite-loop.cpp b/clang-tools-extra/test/clang-tidy/bugprone-infinite-loop.cpp
new file mode 100644
index 0000000000000..33d94820b0f69
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/bugprone-infinite-loop.cpp
@@ -0,0 +1,320 @@
+// RUN: %check_clang_tidy %s bugprone-infinite-loop %t -- -- -fexceptions
+
+void simple_infinite_loop1() {
+  int i = 0;
+  int j = 0;
+  while (i < 10) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i) are updated in the loop body [bugprone-infinite-loop]
+    j++;
+  }
+
+  do {
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i) are updated in the loop body [bugprone-infinite-loop]
+    j++;
+  } while (i < 10);
+
+  for (i = 0; i < 10; ++j) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i) are updated in the loop body [bugprone-infinite-loop]
+  }
+}
+
+void simple_infinite_loop2() {
+  int i = 0;
+  int j = 0;
+  int Limit = 10;
+  while (i < Limit) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i, Limit) are updated in the loop body [bugprone-infinite-loop]
+    j++;
+  }
+
+  do {
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i, Limit) are updated in the loop body [bugprone-infinite-loop]
+    j++;
+  } while (i < Limit);
+
+  for (i = 0; i < Limit; ++j) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i, Limit) are updated in the loop body [bugprone-infinite-loop]
+  }
+}
+
+void simple_not_infinite1() {
+  int i = 0;
+  int Limit = 100;
+  while (i < Limit) {
+    // Not an error since 'Limit' is updated.
+    Limit--;
+  }
+  do {
+    Limit--;
+  } while (i < Limit);
+
+  for (i = 0; i < Limit; Limit--) {
+  }
+}
+
+void simple_not_infinite2() {
+  for (int i = 10; i-- > 0;) {
+    // Not an error, since loop variable is modified in its condition part.
+  }
+}
+
+int unknown_function();
+
+void function_call() {
+  int i = 0;
+  while (i < unknown_function()) {
+    // Not an error, since the function may return different values.
+  }
+
+  do {
+    // Not an error, since the function may return different values.
+  } while (i < unknown_function());
+
+  for (i = 0; i < unknown_function();) {
+    // Not an error, since the function may return different values.
+  }
+}
+
+void escape_before1() {
+  int i = 0;
+  int Limit = 100;
+  int *p = &i;
+  while (i < Limit) {
+    // Not an error, since *p is alias of i.
+    (*p)++;
+  }
+
+  do {
+    (*p)++;
+  } while (i < Limit);
+
+  for (i = 0; i < Limit; ++(*p)) {
+  }
+}
+
+void escape_before2() {
+  int i = 0;
+  int Limit = 100;
+  int &ii = i;
+  while (i < Limit) {
+    // Not an error, since ii is alias of i.
+    ii++;
+  }
+
+  do {
+    ii++;
+  } while (i < Limit);
+
+  for (i = 0; i < Limit; ++ii) {
+  }
+}
+
+void escape_inside1() {
+  int i = 0;
+  int Limit = 100;
+  int *p = &i;
+  while (i < Limit) {
+    // Not an error, since *p is alias of i.
+    int *p = &i;
+    (*p)++;
+  }
+
+  do {
+    int *p = &i;
+    (*p)++;
+  } while (i < Limit);
+}
+
+void escape_inside2() {
+  int i = 0;
+  int Limit = 100;
+  while (i < Limit) {
+    // Not an error, since ii is alias of i.
+    int &ii = i;
+    ii++;
+  }
+
+  do {
+    int &ii = i;
+    ii++;
+  } while (i < Limit);
+}
+
+void escape_after1() {
+  int i = 0;
+  int j = 0;
+  int Limit = 10;
+
+  while (i < Limit) {
+    // False negative, but difficult to detect without CFG-based analysis
+  }
+  int *p = &i;
+}
+
+void escape_after2() {
+  int i = 0;
+  int j = 0;
+  int Limit = 10;
+
+  while (i < Limit) {
+    // False negative, but difficult to detect without CFG-based analysis
+  }
+  int &ii = i;
+}
+
+int glob;
+
+void global1(int &x) {
+  int i = 0, Limit = 100;
+  while (x < Limit) {
+    // Not an error since 'x' can be an alias of 'glob'.
+    glob++;
+  }
+}
+
+void global2() {
+  int i = 0, Limit = 100;
+  while (glob < Limit) {
+    // Since 'glob' is declared out of the function we do not warn.
+    i++;
+  }
+}
+
+struct X {
+  int m;
+
+  void change_m();
+
+  void member_expr1(int i) {
+    while (i < m) {
+      // False negative: No warning, since skipping the case where a struct or
+      // class can be found in its condition.
+      ;
+    }
+  }
+
+  void member_expr2(int i) {
+    while (i < m) {
+      --m;
+    }
+  }
+
+  void member_expr3(int i) {
+    while (i < m) {
+      change_m();
+    }
+  }
+};
+
+void array_index() {
+  int i = 0;
+  int v[10];
+  while (i < 10) {
+    v[i++] = 0;
+  }
+
+  i = 0;
+  do {
+    v[i++] = 0;
+  } while (i < 9);
+
+  for (i = 0; i < 10;) {
+    v[i++] = 0;
+  }
+
+  for (i = 0; i < 10; v[i++] = 0) {
+  }
+}
+
+void no_loop_variable() {
+  while (0)
+    ;
+}
+
+void volatile_in_condition() {
+  volatile int cond = 0;
+  while (!cond) {
+  }
+}
+
+namespace std {
+template<typename T> class atomic {
+  T val;
+public:
+  atomic(T v): val(v) {};
+  operator T() { return val; };
+};
+}
+
+void atomic_in_condition() {
+  std::atomic<int> cond = 0;
+  while (!cond) {
+  }
+}
+
+void loop_exit1() {
+  int i = 0;
+  while (i) {
+    if (unknown_function())
+      break;
+  }
+}
+
+void loop_exit2() {
+  int i = 0;
+  while (i) {
+    if (unknown_function())
+      return;
+  }
+}
+
+void loop_exit3() {
+  int i = 0;
+  while (i) {
+    if (unknown_function())
+      goto end;
+  }
+ end:
+  ;
+}
+
+void loop_exit4() {
+  int i = 0;
+  while (i) {
+    if (unknown_function())
+      throw 1;
+  }
+}
+
+[[noreturn]] void exit(int);
+
+void loop_exit5() {
+  int i = 0;
+  while (i) {
+    if (unknown_function())
+      exit(1);
+  }
+}
+
+void loop_exit_in_lambda() {
+  int i = 0;
+  while (i) {
+    // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: this loop is infinite; none of its condition variables (i) are updated in the loop body [bugprone-infinite-loop]
+    auto l = []() { return 0; };
+  }
+}
+
+void lambda_capture() {
+  int i = 0;
+  int Limit = 100;
+  int *p = &i;
+  while (i < Limit) {
+    // Not an error, since i is captured by reference in a lambda.
+    auto l = [&i]() { ++i; };
+  }
+
+  do {
+    int *p = &i;
+    (*p)++;
+  } while (i < Limit);
+}
diff --git a/clang-tools-extra/test/clang-tidy/bugprone-posix-return.cpp b/clang-tools-extra/test/clang-tidy/bugprone-posix-return.cpp
index 160f567cffaeb..271893c707069 100644
--- a/clang-tools-extra/test/clang-tidy/bugprone-posix-return.cpp
+++ b/clang-tools-extra/test/clang-tidy/bugprone-posix-return.cpp
@@ -9,6 +9,16 @@ typedef long off_t;
 typedef decltype(sizeof(int)) size_t;
 typedef struct __posix_spawn_file_actions* posix_spawn_file_actions_t;
 typedef struct __posix_spawnattr* posix_spawnattr_t;
+# define __CPU_SETSIZE 1024
+# define __NCPUBITS (8 * sizeof (__cpu_mask))
+typedef unsigned long int __cpu_mask;
+typedef struct
+{
+  __cpu_mask __bits[__CPU_SETSIZE / __NCPUBITS];
+} cpu_set_t;
+typedef struct _opaque_pthread_t *__darwin_pthread_t;
+typedef __darwin_pthread_t pthread_t;
+typedef struct pthread_attr_t_ *pthread_attr_t;
 
 extern "C" int posix_fadvise(int fd, off_t offset, off_t len, int advice);
 extern "C" int posix_fallocate(int fd, off_t offset, off_t len);
@@ -23,6 +33,12 @@ extern "C" int posix_spawnp(pid_t *pid, const char *file,
                  const posix_spawn_file_actions_t *file_actions,
                  const posix_spawnattr_t *attrp,
                  char *const argv[], char *const envp[]);
+extern "C" int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg);
+extern "C" int pthread_attr_setaffinity_np(pthread_attr_t *attr, size_t cpusetsize, const cpu_set_t *cpuset);
+extern "C" int pthread_attr_setschedpolicy(pthread_attr_t *attr, int policy);
+extern "C" int pthread_attr_init(pthread_attr_t *attr);
+extern "C" int pthread_yield(void);
+
 
 void warningLessThanZero() {
   if (posix_fadvise(0, 0, 0, 0) < 0) {}
@@ -43,11 +59,38 @@ void warningLessThanZero() {
   if (posix_spawnp(NULL, NULL, NULL, NULL, {NULL}, {NULL}) < 0) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:60: warning:
   // CHECK-FIXES: posix_spawnp(NULL, NULL, NULL, NULL, {NULL}, {NULL}) > 0
+  if (pthread_create(NULL, NULL, NULL, NULL) < 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning: the comparison always evaluates to false because pthread_create always returns non-negative values
+  // CHECK-FIXES: pthread_create(NULL, NULL, NULL, NULL) > 0
+  if (pthread_attr_setaffinity_np(NULL, 0, NULL) < 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:50: warning:
+  // CHECK-FIXES: pthread_attr_setaffinity_np(NULL, 0, NULL) > 0
+  if (pthread_attr_setschedpolicy(NULL, 0) < 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:44: warning:
+  // CHECK-FIXES: pthread_attr_setschedpolicy(NULL, 0) > 0)
+  if (pthread_attr_init(NULL) < 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:31: warning:
+  // CHECK-FIXES: pthread_attr_init(NULL) > 0
+  if (pthread_yield() < 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:23: warning:
+  // CHECK-FIXES: pthread_yield() > 0
+
 }
 
 void warningAlwaysTrue() {
   if (posix_fadvise(0, 0, 0, 0) >= 0) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:33: warning: the comparison always evaluates to true because posix_fadvise always returns non-negative values
+  if (pthread_create(NULL, NULL, NULL, NULL) >= 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning: the comparison always evaluates to true because pthread_create always returns non-negative values
+  if (pthread_attr_setaffinity_np(NULL, 0, NULL) >= 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:50: warning:
+  if (pthread_attr_setschedpolicy(NULL, 0) >= 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:44: warning:
+  if (pthread_attr_init(NULL) >= 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:31: warning:
+  if (pthread_yield() >= 0) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:23: warning:
+
 }
 
 void warningEqualsNegative() {
@@ -69,6 +112,15 @@ void warningEqualsNegative() {
   // CHECK-MESSAGES: :[[@LINE-1]]:59: warning:
   if (posix_spawnp(NULL, NULL, NULL, NULL, {NULL}, {NULL}) == -1) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:60: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) == -1) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning: pthread_create
+  if (pthread_create(NULL, NULL, NULL, NULL) != -1) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) <= -1) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) < -1) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+
 }
 
 void WarningWithMacro() {
@@ -85,6 +137,20 @@ void WarningWithMacro() {
   // CHECK-MESSAGES: :[[@LINE-1]]:33: warning:
   if (posix_fadvise(0, 0, 0, 0) < NEGATIVE_ONE) {}
   // CHECK-MESSAGES: :[[@LINE-1]]:33: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) < ZERO) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  // CHECK-FIXES: pthread_create(NULL, NULL, NULL, NULL) > ZERO
+  if (pthread_create(NULL, NULL, NULL, NULL) >= ZERO) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) == NEGATIVE_ONE) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) != NEGATIVE_ONE) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) <= NEGATIVE_ONE) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+  if (pthread_create(NULL, NULL, NULL, NULL) < NEGATIVE_ONE) {}
+  // CHECK-MESSAGES: :[[@LINE-1]]:46: warning:
+
 }
 
 void noWarning() {
@@ -100,6 +166,7 @@ void noWarning() {
 
 namespace i {
 int posix_fadvise(int fd, off_t offset, off_t len, int advice);
+int pthread_yield(void);
 
 void noWarning() {
   if (posix_fadvise(0, 0, 0, 0) < 0) {}
@@ -108,6 +175,12 @@ void noWarning() {
   if (posix_fadvise(0, 0, 0, 0) != -1) {}
   if (posix_fadvise(0, 0, 0, 0) <= -1) {}
   if (posix_fadvise(0, 0, 0, 0) < -1) {}
+    if (pthread_yield() < 0) {}
+    if (pthread_yield() >= 0) {}
+    if (pthread_yield() == -1) {}
+    if (pthread_yield() != -1) {}
+    if (pthread_yield() <= -1) {}
+    if (pthread_yield() < -1) {}
 }
 
 } // namespace i
@@ -115,6 +188,7 @@ void noWarning() {
 class G {
  public:
   int posix_fadvise(int fd, off_t offset, off_t len, int advice);
+  int pthread_yield(void);
 
   void noWarning() {
     if (posix_fadvise(0, 0, 0, 0) < 0) {}
@@ -123,5 +197,11 @@ class G {
     if (posix_fadvise(0, 0, 0, 0) != -1) {}
     if (posix_fadvise(0, 0, 0, 0) <= -1) {}
     if (posix_fadvise(0, 0, 0, 0) < -1) {}
+    if (pthread_yield() < 0) {}
+    if (pthread_yield() >= 0) {}
+    if (pthread_yield() == -1) {}
+    if (pthread_yield() != -1) {}
+    if (pthread_yield() <= -1) {}
+    if (pthread_yield() < -1) {}
   }
 };
diff --git a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
index aa6304baef813..368b674d2d327 100755
--- a/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
+++ b/clang-tools-extra/test/clang-tidy/check_clang_tidy.py
@@ -69,7 +69,8 @@ def run_test_once(args, extra_args):
     clang_tidy_extra_args.append('-format-style=none')
 
   if extension in ['.m', '.mm']:
-    clang_extra_args = ['-fobjc-abi-version=2', '-fobjc-arc'] + clang_extra_args
+    clang_extra_args = ['-fobjc-abi-version=2', '-fobjc-arc', '-fblocks'] + \
+        clang_extra_args
 
   if extension in ['.cpp', '.hpp', '.mm']:
     clang_extra_args.append('-std=' + std)
diff --git a/clang-tools-extra/test/clang-tidy/cppcoreguidelines-init-variables.cpp b/clang-tools-extra/test/clang-tidy/cppcoreguidelines-init-variables.cpp
new file mode 100644
index 0000000000000..893c1d2877983
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/cppcoreguidelines-init-variables.cpp
@@ -0,0 +1,80 @@
+// RUN: %check_clang_tidy %s cppcoreguidelines-init-variables %t
+
+// Ensure that function declarations are not changed.
+void some_func(int x, double d, bool b, const char *p);
+
+// Ensure that function arguments are not changed
+int identity_function(int x) {
+  return x;
+}
+
+int do_not_modify_me;
+
+static int should_not_be_initialized;
+extern int should_not_be_initialized2;
+
+typedef struct {
+  int unaltered1;
+  int unaltered2;
+} UnusedStruct;
+
+typedef int my_int_type;
+#define MACRO_INT int
+#define FULL_DECLARATION() int macrodecl;
+
+template <typename T>
+void template_test_function() {
+  T t;
+  int uninitialized;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: variable 'uninitialized' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  int uninitialized = 0;{{$}}
+}
+
+void init_unit_tests() {
+  int x;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: variable 'x' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  int x = 0;{{$}}
+  my_int_type myint;
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: variable 'myint' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  my_int_type myint = 0;{{$}}
+
+  MACRO_INT macroint;
+  // CHECK-MESSAGES: :[[@LINE-1]]:13: warning: variable 'macroint' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  MACRO_INT macroint = 0;{{$}}
+  FULL_DECLARATION();
+
+  int x0 = 1, x1, x2 = 2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: variable 'x1' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  int x0 = 1, x1 = 0, x2 = 2;{{$}}
+  int y0, y1 = 1, y2;
+  // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: variable 'y0' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-MESSAGES: :[[@LINE-2]]:19: warning: variable 'y2' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  int y0 = 0, y1 = 1, y2 = 0;{{$}}
+  int hasval = 42;
+
+  float f;
+  // CHECK-MESSAGES: :[[@LINE-1]]:9: warning: variable 'f' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  float f = NAN;{{$}}
+  float fval = 85.0;
+  double d;
+  // CHECK-MESSAGES: :[[@LINE-1]]:10: warning: variable 'd' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  double d = NAN;{{$}}
+  double dval = 99.0;
+
+  bool b;
+  // CHECK-MESSAGES: :[[@LINE-1]]:8: warning: variable 'b' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  bool b = 0;{{$}}
+  bool bval = true;
+
+  const char *ptr;
+  // CHECK-MESSAGES: :[[@LINE-1]]:15: warning: variable 'ptr' is not initialized [cppcoreguidelines-init-variables]
+  // CHECK-FIXES: {{^}}  const char *ptr = nullptr;{{$}}
+  const char *ptrval = "a string";
+
+  UnusedStruct u;
+
+  static int does_not_need_an_initializer;
+  extern int does_not_need_an_initializer2;
+  int parens(42);
+  int braces{42};
+}
diff --git a/clang-tools-extra/test/clang-tidy/objc-avoid-spinlock.m b/clang-tools-extra/test/clang-tidy/darwin-avoid-spinlock.m
similarity index 78%
rename from clang-tools-extra/test/clang-tidy/objc-avoid-spinlock.m
rename to clang-tools-extra/test/clang-tidy/darwin-avoid-spinlock.m
index f9f05cb05a286..c870e0a0fed06 100644
--- a/clang-tools-extra/test/clang-tidy/objc-avoid-spinlock.m
+++ b/clang-tools-extra/test/clang-tidy/darwin-avoid-spinlock.m
@@ -1,4 +1,4 @@
-// RUN: %check_clang_tidy %s objc-avoid-spinlock %t
+// RUN: %check_clang_tidy %s darwin-avoid-spinlock %t
 
 typedef int OSSpinLock;
 
@@ -6,10 +6,10 @@ @implementation Foo
 - (void)f {
     int i = 1;
     OSSpinlockLock(&i);
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [objc-avoid-spinlock]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [darwin-avoid-spinlock]
     OSSpinlockTry(&i);
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [objc-avoid-spinlock]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [darwin-avoid-spinlock]
     OSSpinlockUnlock(&i);
-    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [objc-avoid-spinlock]
+    // CHECK-MESSAGES: :[[@LINE-1]]:5: warning: use os_unfair_lock_lock() or dispatch queue APIs instead of the deprecated OSSpinLock [darwin-avoid-spinlock]
 }
 @end
diff --git a/clang-tools-extra/test/clang-tidy/darwin-dispatch-once-nonstatic.mm b/clang-tools-extra/test/clang-tidy/darwin-dispatch-once-nonstatic.mm
new file mode 100644
index 0000000000000..92d6f453c66df
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/darwin-dispatch-once-nonstatic.mm
@@ -0,0 +1,48 @@
+// RUN: %check_clang_tidy %s darwin-dispatch-once-nonstatic %t
+
+typedef int dispatch_once_t;
+extern void dispatch_once(dispatch_once_t *pred, void(^block)(void));
+
+
+void bad_dispatch_once(dispatch_once_t once, void(^block)(void)) {}
+// CHECK-MESSAGES: :[[@LINE-1]]:24: warning: dispatch_once_t variables must have static or global storage duration; function parameters should be pointer references [darwin-dispatch-once-nonstatic]
+
+// file-scope dispatch_once_ts have static storage duration.
+dispatch_once_t global_once;
+static dispatch_once_t file_static_once;
+namespace {
+dispatch_once_t anonymous_once;
+} // end anonymous namespace
+
+int Correct(void) {
+  static int value;
+  static dispatch_once_t once;
+  dispatch_once(&once, ^{
+    value = 1;
+  });
+  return value;
+}
+
+int Incorrect(void) {
+  static int value;
+  dispatch_once_t once;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: dispatch_once_t variables must have static or global storage duration [darwin-dispatch-once-nonstatic]
+  // CHECK-FIXES: static dispatch_once_t once;
+  dispatch_once(&once, ^{
+    value = 1;
+  });
+  return value;
+}
+
+struct OnceStruct {
+  static dispatch_once_t staticOnce; // Allowed
+  int value;
+  dispatch_once_t once;  // Allowed (at this time)
+};
+
+@interface MyObject {
+  dispatch_once_t _once;
+  // CHECK-MESSAGES: :[[@LINE-1]]:3: warning: dispatch_once_t variables must have static or global storage duration and cannot be Objective-C instance variables [darwin-dispatch-once-nonstatic]
+  // CHECK-FIXES: dispatch_once_t _once;
+}
+@end
diff --git a/clang-tools-extra/test/clang-tidy/file-filter-symlinks.cpp b/clang-tools-extra/test/clang-tidy/file-filter-symlinks.cpp
new file mode 100644
index 0000000000000..7efa7d070f69f
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/file-filter-symlinks.cpp
@@ -0,0 +1,19 @@
+// REQUIRES: shell
+
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/dir1/dir2
+// RUN: echo 'class A { A(int); };' > %t/dir1/header.h
+// RUN: ln -s %t/dir1/header.h %t/dir1/header_alias.h
+//
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='dir1/dir2/\.\./header_alias\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='dir1/dir2/\.\./header_alias\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header_alias\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header_alias\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER_ALIAS %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
+// RUN: clang-tidy -checks='-*,google-explicit-constructor' -header-filter='header\.h' -quiet %s -- -I %t 2>&1 | FileCheck --check-prefix=CHECK_HEADER %s
+
+// Check that `-header-filter` operates on the same file paths as paths in
+// diagnostics printed by ClangTidy.
+#include "dir1/dir2/../header_alias.h"
+// CHECK_HEADER_ALIAS: dir1/dir2/../header_alias.h:1:11: warning: single-argument constructors
+// CHECK_HEADER-NOT: warning:
diff --git a/clang-tools-extra/test/clang-tidy/google-objc-global-variable-declaration.mm b/clang-tools-extra/test/clang-tidy/google-objc-global-variable-declaration.mm
index a6b0f6ee47688..345edec1f979e 100644
--- a/clang-tools-extra/test/clang-tidy/google-objc-global-variable-declaration.mm
+++ b/clang-tools-extra/test/clang-tidy/google-objc-global-variable-declaration.mm
@@ -6,5 +6,5 @@
 // CHECK-FIXES: static NSString* const kMyConstString = @"hello";
 
 class MyTest {
-    static int not_objc_style;
+  static int not_objc_style;
 };
diff --git a/clang-tools-extra/test/clang-tidy/objc-missing-hash.m b/clang-tools-extra/test/clang-tidy/objc-missing-hash.m
new file mode 100644
index 0000000000000..b9cc9d023ad13
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/objc-missing-hash.m
@@ -0,0 +1,68 @@
+// RUN: %check_clang_tidy %s objc-missing-hash %t
+
+typedef _Bool BOOL;
+#define YES 1
+#define NO 0
+typedef unsigned int NSUInteger;
+typedef void *id;
+
+@interface NSObject
+- (NSUInteger)hash;
+- (BOOL)isEqual:(id)object;
+@end
+
+@interface MissingHash : NSObject
+@end
+
+@implementation MissingHash
+// CHECK-MESSAGES: :[[@LINE-1]]:17: warning: 'MissingHash' implements -isEqual: without implementing -hash [objc-missing-hash]
+
+- (BOOL)isEqual:(id)object {
+  return YES;
+}
+
+@end
+
+@interface HasHash : NSObject
+@end
+
+@implementation HasHash
+
+- (NSUInteger)hash {
+  return 0;
+}
+
+- (BOOL)isEqual:(id)object {
+  return YES;
+}
+
+@end
+
+@interface NSArray : NSObject
+@end
+
+@interface MayHaveInheritedHash : NSArray
+@end
+
+@implementation MayHaveInheritedHash
+
+- (BOOL)isEqual:(id)object {
+  return YES;
+}
+
+@end
+
+@interface AnotherRootClass
+@end
+
+@interface NotDerivedFromNSObject : AnotherRootClass
+@end
+
+@implementation NotDerivedFromNSObject
+
+- (BOOL)isEqual:(id)object {
+  return NO;
+}
+
+@end
+
diff --git a/clang-tools-extra/test/clang-tidy/performance-inefficient-vector-operation.cpp b/clang-tools-extra/test/clang-tidy/performance-inefficient-vector-operation.cpp
index d5e38eb71f835..ffac24c2b9a82 100644
--- a/clang-tools-extra/test/clang-tidy/performance-inefficient-vector-operation.cpp
+++ b/clang-tools-extra/test/clang-tidy/performance-inefficient-vector-operation.cpp
@@ -1,4 +1,7 @@
-// RUN: %check_clang_tidy %s performance-inefficient-vector-operation %t -- -format-style=llvm
+// RUN: %check_clang_tidy %s performance-inefficient-vector-operation %t -- \
+// RUN: -format-style=llvm \
+// RUN: -config='{CheckOptions: \
+// RUN:  [{key: performance-inefficient-vector-operation.EnableProto, value: 1}]}'
 
 namespace std {
 
@@ -62,13 +65,35 @@ class Bar {
 
 int Op(int);
 
+namespace proto2 {
+class MessageLite {};
+class Message : public MessageLite {};
+} // namespace proto2
+
+class FooProto : public proto2::Message {
+ public:
+  int *add_x();  // repeated int x;
+  void add_x(int x);
+  void mutable_x();
+  void mutable_y();
+  int add_z() const; // optional int add_z;
+};
+
+class BarProto : public proto2::Message {
+ public:
+  int *add_x();
+  void add_x(int x);
+  void mutable_x();
+  void mutable_y();
+};
+
 void f(std::vector<int>& t) {
   {
     std::vector<int> v0;
     // CHECK-FIXES: v0.reserve(10);
     for (int i = 0; i < 10; ++i)
       v0.push_back(i);
-      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'push_back' is called inside a loop; consider pre-allocating the vector capacity before the loop
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'push_back' is called inside a loop; consider pre-allocating the container capacity before the loop
   }
   {
     std::vector<int> v1;
@@ -162,6 +187,15 @@ void f(std::vector<int>& t) {
     }
   }
 
+  {
+    FooProto foo;
+    // CHECK-FIXES: foo.mutable_x()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x(i);
+      // CHECK-MESSAGES: :[[@LINE-1]]:7: warning: 'add_x' is called inside a loop; consider pre-allocating the container capacity before the loop
+    }
+  }
+
   // ---- Non-fixed Cases ----
   {
     std::vector<int> z0;
@@ -274,4 +308,54 @@ void f(std::vector<int>& t) {
       z12.push_back(e);
     }
   }
+
+  {
+    FooProto foo;
+    foo.mutable_x();
+    // CHECK-FIXES-NOT: foo.mutable_x()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x(i);
+    }
+  }
+  {
+    FooProto foo;
+    // CHECK-FIXES-NOT: foo.mutable_x()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x(i);
+      foo.add_x(i);
+    }
+  }
+  {
+    FooProto foo;
+    // CHECK-FIXES-NOT: foo.mutable_x()->Reserve(5);
+    foo.add_x(-1);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x(i);
+    }
+  }
+  {
+    FooProto foo;
+    BarProto bar;
+    bar.mutable_x();
+    // CHECK-FIXES-NOT: foo.mutable_x()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x();
+      bar.add_x();
+    }
+  }
+  {
+    FooProto foo;
+    foo.mutable_y();
+    // CHECK-FIXES-NOT: foo.mutable_x()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_x(i);
+    }
+  }
+  {
+    FooProto foo;
+    // CHECK-FIXES-NOT: foo.mutable_z()->Reserve(5);
+    for (int i = 0; i < 5; i++) {
+      foo.add_z();
+    }
+  }
 }
diff --git a/clang-tools-extra/test/clang-tidy/readability-isolate-declaration-no-infinite-loop.cpp b/clang-tools-extra/test/clang-tidy/readability-isolate-declaration-no-infinite-loop.cpp
new file mode 100644
index 0000000000000..78a8ec738800e
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/readability-isolate-declaration-no-infinite-loop.cpp
@@ -0,0 +1,7 @@
+// RUN: %check_clang_tidy -expect-clang-tidy-error %s readability-isolate-declaration %t
+
+int main(){
+  int a, b
+  // CHECK-MESSAGES: [[@LINE-1]]:3: warning: multiple declarations in a single statement reduces readability
+  // CHECK-MESSAGES: [[@LINE-2]]:11: error: expected ';' at end of declaration [clang-diagnostic-error]
+}
diff --git a/clang-tools-extra/unittests/clang-tidy/LLVMModuleTest.cpp b/clang-tools-extra/unittests/clang-tidy/LLVMModuleTest.cpp
index aed9f6d831ea3..b86f8e63fb899 100644
--- a/clang-tools-extra/unittests/clang-tidy/LLVMModuleTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/LLVMModuleTest.cpp
@@ -211,6 +211,15 @@ TEST(LLVMHeaderGuardCheckTest, FixHeaderGuards) {
                                          "#endif  /* LLVM_ADT_FOO_H\\ \n"
                                          " FOO */",
                                          "include/llvm/ADT/foo.h", None));
+
+  EXPECT_EQ("#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_FOO_H\n"
+            "#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_FOO_H\n"
+            "\n"
+            "\n"
+            "#endif\n",
+            runHeaderGuardCheck(
+                "", "/llvm-project/clang-tools-extra/clangd/foo.h",
+                StringRef("header is missing header guard")));
 }
 #endif
 
diff --git a/clang/docs/ClangCommandLineReference.rst b/clang/docs/ClangCommandLineReference.rst
index c26a6e5ecfc8a..5f6bb9829f5af 100644
--- a/clang/docs/ClangCommandLineReference.rst
+++ b/clang/docs/ClangCommandLineReference.rst
@@ -86,8 +86,6 @@ Pass <arg> to the target offloading toolchain identified by <triple>.
 
 Run the static analyzer
 
-.. option:: --analyze-auto
-
 .. option:: --analyzer-no-default-checks
 
 .. option:: --analyzer-output<arg>
@@ -1944,7 +1942,8 @@ Perform ThinLTO importing using provided function summary index
 
 .. option:: -ftime-trace
 
-Turn on time profiler
+Turn on time profiler. Results can be analyzed with chrome://tracing or
+`Speedscope App <https://www.speedscope.app>`_ for flamegraph visualization
 
 .. option:: -ftime-trace-granularity=<arg>
 
@@ -2194,7 +2193,7 @@ Set EABI type, e.g. 4, 5 or gnu (default depends on triple)
 
 .. option:: -mfentry
 
-Insert calls to fentry at function entry (x86 only)
+Insert calls to fentry at function entry (x86/SystemZ only)
 
 .. option:: -mfloat-abi=<arg>
 
diff --git a/clang/docs/ClangFormatStyleOptions.rst b/clang/docs/ClangFormatStyleOptions.rst
index 7193e3ede290d..0803e65df2e22 100644
--- a/clang/docs/ClangFormatStyleOptions.rst
+++ b/clang/docs/ClangFormatStyleOptions.rst
@@ -1128,6 +1128,34 @@ the configuration (without a prefix: ``Auto``).
         B
       };
 
+  * ``BS_Whitesmiths`` (in configuration: ``Whitesmiths``)
+    Like ``Allman`` but always indent braces and line up code with braces.
+
+    .. code-block:: c++
+
+      try
+        {
+        foo();
+        }
+      catch ()
+        {
+        }
+      void foo() { bar(); }
+      class foo
+        {
+        };
+      if (foo())
+        {
+        }
+      else
+        {
+        }
+      enum X : int
+        {
+        A,
+        B
+        };
+
   * ``BS_GNU`` (in configuration: ``GNU``)
     Always break before braces and add an extra level of indentation to
     braces of control statements, not to those of class, function
@@ -1483,6 +1511,13 @@ the configuration (without a prefix: ``Auto``).
   can also assign negative priorities if you have certain headers that
   always need to be first.
 
+  There is a third and optional field ``SortPriority`` which can used while
+  ``IncludeBloks = IBS_Regroup`` to define the priority in which ``#includes``
+  should be ordered, and value of ``Priority`` defines the order of
+  ``#include blocks`` and also enables to group ``#includes`` of different
+  priority for order.``SortPriority`` is set to the value of ``Priority``
+  as default if it is not assigned.
+
   To configure this in the .clang-format file, use:
 
   .. code-block:: yaml
@@ -1490,12 +1525,14 @@ the configuration (without a prefix: ``Auto``).
     IncludeCategories:
       - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
         Priority:        2
+        SortPriority:    2
       - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
         Priority:        3
       - Regex:           '<[[:alnum:].]+>'
         Priority:        4
       - Regex:           '.*'
         Priority:        1
+        SortPriority:    0
 
 **IncludeIsMainRegex** (``std::string``)
   Specify a regular expression of suffixes that are allowed in the
@@ -2251,22 +2288,38 @@ the configuration (without a prefix: ``Auto``).
      std::unique_ptr<int[]> foo() {} // Won't be affected
 
 **Standard** (``LanguageStandard``)
-  Format compatible with this standard, e.g. use ``A<A<int> >``
-  instead of ``A<A<int>>`` for ``LS_Cpp03``.
+  .. code-block:: c++
+
+     c++03:                                 latest:
+     vector<set<int> > x;           vs.     vector<set<int>> x;
+  Parse and format C++ constructs compatible with this standard.
 
   Possible values:
 
-  * ``LS_Cpp03`` (in configuration: ``Cpp03``)
+  * ``LS_Cpp03`` (in configuration: ``c++03``)
     Use C++03-compatible syntax.
 
-  * ``LS_Cpp11`` (in configuration: ``Cpp11``)
-    Use features of C++11, C++14 and C++1z (e.g. ``A<A<int>>`` instead of
-    ``A<A<int> >``).
+  * ``LS_Cpp11`` (in configuration: ``c++11``)
+    Use C++11-compatible syntax.
+
+  * ``LS_Cpp14`` (in configuration: ``c++14``)
+    Use C++14-compatible syntax.
+
+  * ``LS_Cpp17`` (in configuration: ``c++17``)
+    Use C++17-compatible syntax.
+
+  * ``LS_Cpp20`` (in configuration: ``c++20``)
+    Use C++20-compatible syntax.
+
+  * ``LS_Latest`` (in configuration: ``Latest``)
+    Parse and format using the latest supported language version.
 
   * ``LS_Auto`` (in configuration: ``Auto``)
     Automatic detection based on the input.
 
+  * ``Cpp03``: deprecated alias for ``c++03``
 
+  * ``Cpp11``: deprecated alias for ``Latest``
 
 **StatementMacros** (``std::vector<std::string>``)
   A vector of macros that should be interpreted as complete
diff --git a/clang/docs/InternalsManual.rst b/clang/docs/InternalsManual.rst
index b56abc1e1bc5d..59f3cba699dbb 100644
--- a/clang/docs/InternalsManual.rst
+++ b/clang/docs/InternalsManual.rst
@@ -859,7 +859,7 @@ benefits:
 
 There are unfortunately exceptions to this general approach, such as:
 
-  * A the first declaration of a redeclarable entity maintains a pointer to the
+  * The first declaration of a redeclarable entity maintains a pointer to the
     most recent declaration of that entity, which naturally needs to change as
     more declarations are parsed.
   * Name lookup tables in declaration contexts change after the namespace
@@ -1913,7 +1913,7 @@ declarations like enums, classes, etc. if they are in anonymous namespaces.
 Therefore, we filter the lookup results and consider only those which have the
 same visibility as the declaration we currently import.
 
-We consider two declarations in two anonymous namsepaces to have the same
+We consider two declarations in two anonymous namespaces to have the same
 visibility only if they are imported from the same AST context.
 
 Strategies to Handle Conflicting Names
@@ -2134,7 +2134,7 @@ about them.
 
 Finally, this is not just a problem for semantic analysis.  The code generator
 and other clients have to be able to fold constants (e.g., to initialize global
-variables) and has to handle a superset of what C99 allows.  Further, these
+variables) and have to handle a superset of what C99 allows.  Further, these
 clients can benefit from extended information.  For example, we know that
 "``foo() || 1``" always evaluates to ``true``, but we can't replace the
 expression with ``true`` because it has side effects.
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 3080f6dd07d6b..da3c85e78e57c 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -1161,10 +1161,7 @@ The following type trait primitives are supported by Clang. Those traits marked
 * ``__is_sealed`` (Microsoft):
   Synonym for ``__is_final``.
 * ``__is_signed`` (C++, Embarcadero):
-  Note that this currently returns true for enumeration types if the underlying
-  type is signed, and returns false for floating-point types, in violation of
-  the requirements for ``std::is_signed``. This behavior is likely to change in
-  a future version of Clang.
+  Returns false for enumeration types, and returns true for floating-point types. Note, before Clang 10, returned true for enumeration types if the underlying type was signed, and returned false for floating-point types.
 * ``__is_standard_layout`` (C++, GNU, Microsoft, Embarcadero)
 * ``__is_trivial`` (C++, GNU, Microsoft, Embarcadero)
 * ``__is_trivially_assignable`` (C++, GNU, Microsoft)
@@ -2377,12 +2374,14 @@ macro expansions, the entire macro expansion stack is expressed.
 array subscript access and structure/union member access are relocatable
 under bpf compile-once run-everywhere framework. Debuginfo (typically
 with ``-g``) is needed, otherwise, the compiler will exit with an error.
+The return type for the intrinsic is the same as the type of the
+argument.
 
 **Syntax**:
 
 .. code-block:: c
 
-  const void * __builtin_preserve_access_index(const void * ptr)
+  type __builtin_preserve_access_index(type arg)
 
 **Example of Use**:
 
@@ -2397,7 +2396,8 @@ with ``-g``) is needed, otherwise, the compiler will exit with an error.
     } c[4];
   };
   struct t *v = ...;
-  const void *pb =__builtin_preserve_access_index(&v->c[3].b);
+  int *pb =__builtin_preserve_access_index(&v->c[3].b);
+  __builtin_preserve_access_index(v->j);
 
 Multiprecision Arithmetic Builtins
 ----------------------------------
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 9716360a78d13..95dd51e634755 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -51,7 +51,11 @@ Major New Features
 Improvements to Clang's diagnostics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- ...
+- -Wtautological-overlap-compare will warn on negative numbers and non-int
+  types.
+- -Wtautological-compare for self comparisons and
+  -Wtautological-overlap-compare will now look through member and array
+  access to determine if two operand expressions are the same.
 
 Non-comprehensive list of changes in this release
 -------------------------------------------------
@@ -98,7 +102,18 @@ Attribute Changes in Clang
 Windows Support
 ---------------
 
-- ...
+- Previous Clang versions contained a work-around to avoid an issue with the
+  standard library headers in Visual Studio 2019 versions prior to 16.3. This
+  work-around has now been removed, and users of Visual Studio 2019 are
+  encouraged to upgrade to 16.3 or later, otherwise they may see link errors as
+  below:
+
+  .. code-block:: console
+
+    error LNK2005: "bool const std::_Is_integral<int>" (??$_Is_integral@H@std@@3_NB) already defined
+
+
+
 
 C Language Changes in Clang
 ---------------------------
@@ -113,7 +128,10 @@ C11 Feature Support
 C++ Language Changes in Clang
 -----------------------------
 
-- ...
+- The behaviour of the `gnu_inline` attribute now matches GCC, for cases
+  where used without the `extern` keyword. As this is a change compared to
+  how it behaved in previous Clang versions, a warning is emitted for this
+  combination.
 
 C++1z Feature Support
 ^^^^^^^^^^^^^^^^^^^^^
@@ -192,7 +210,15 @@ AST Matchers
 clang-format
 ------------
 
-- ...
+- The ``Standard`` style option specifies which version of C++ should be used
+  when parsing and formatting C++ code. The set of allowed values has changed:
+  - ``Latest`` will always enable new C++ language features.
+  - ``c++03``, ``c++11``, ``c++14``, ``c++17``, ``c++20`` will pin to exactly
+    that language version.
+  - ``Auto`` is the default and detects style from the code (this is unchanged).
+ The previous values of ``Cpp03`` and ``Cpp11`` are deprecated. Note that
+ ``Cpp11`` is treated as ``Latest``, as this was always clang-format's behavior.
+ (One motivation for this change is the new name describes the behavior better).
 
 libclang
 --------
diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst
index f45d2d5ac0e44..419714d38cdc4 100644
--- a/clang/docs/UsersManual.rst
+++ b/clang/docs/UsersManual.rst
@@ -1128,6 +1128,185 @@ number of cases where the compilation environment is tightly controlled
 and the precompiled header cannot be generated after headers have been
 installed.
 
+.. _controlling-fp-behavior:
+
+Controlling Floating Point Behavior
+-----------------------------------
+
+Clang provides a number of ways to control floating point behavior. The options
+are listed below.
+
+.. option:: -ffast-math
+
+   Enable fast-math mode.  This option lets the
+   compiler make aggressive, potentially-lossy assumptions about
+   floating-point math.  These include:
+
+   * Floating-point math obeys regular algebraic rules for real numbers (e.g.
+     ``+`` and ``*`` are associative, ``x/y == x * (1/y)``, and
+     ``(a + b) * c == a * c + b * c``),
+   * Operands to floating-point operations are not equal to ``NaN`` and
+     ``Inf``, and
+   * ``+0`` and ``-0`` are interchangeable.
+
+   ``-ffast-math`` also defines the ``__FAST_MATH__`` preprocessor
+   macro. Some math libraries recognize this macro and change their behavior.
+   With the exception of ``-ffp-contract=fast``, using any of the options
+   below to disable any of the individual optimizations in ``-ffast-math``
+   will cause ``__FAST_MATH__`` to no longer be set.
+
+  This option implies:
+
+   * ``-fno-honor-infinities``
+
+   * ``-fno-honor-nans``
+
+   * ``-fno-math-errno``
+
+   * ``-ffinite-math``
+
+   * ``-fassociative-math``
+
+   * ``-freciprocal-math``
+
+   * ``-fno-signed-zeros``
+
+   * ``-fno-trapping-math``
+
+   * ``-ffp-contract=fast``
+
+.. option:: -fdenormal-fp-math=<value>
+
+   Select which denormal numbers the code is permitted to require.
+
+   Valid values are: 
+
+   * ``ieee`` - IEEE 754 denormal numbers
+   * ``preserve-sign`` - the sign of a flushed-to-zero number is preserved in the sign of 0
+   * ``positive-zero`` - denormals are flushed to positive zero
+
+   Defaults to ``ieee``.
+
+.. _opt_fstrict-float-cast-overflow:
+
+**-f[no-]strict-float-cast-overflow**
+
+   When a floating-point value is not representable in a destination integer 
+   type, the code has undefined behavior according to the language standard.
+   By default, Clang will not guarantee any particular result in that case.
+   With the 'no-strict' option, Clang attempts to match the overflowing behavior
+   of the target's native float-to-int conversion instructions.
+
+.. _opt_fmath-errno:
+
+**-f[no-]math-errno**
+
+   Require math functions to indicate errors by setting errno.
+   The default varies by ToolChain.  ``-fno-math-errno`` allows optimizations
+   that might cause standard C math functions to not set ``errno``.
+   For example, on some systems, the math function ``sqrt`` is specified
+   as setting ``errno`` to ``EDOM`` when the input is negative. On these
+   systems, the compiler cannot normally optimize a call to ``sqrt`` to use
+   inline code (e.g. the x86 ``sqrtsd`` instruction) without additional
+   checking to ensure that ``errno`` is set appropriately.
+   ``-fno-math-errno`` permits these transformations.
+
+   On some targets, math library functions never set ``errno``, and so
+   ``-fno-math-errno`` is the default. This includes most BSD-derived
+   systems, including Darwin.
+
+.. _opt_ftrapping-math:
+
+**-f[no-]trapping-math**
+
+   ``-fno-trapping-math`` allows optimizations that assume that
+   floating point operations cannot generate traps such as divide-by-zero,
+   overflow and underflow. Defaults to ``-ftrapping-math``.
+   Currently this option has no effect.
+
+.. option:: -ffp-contract=<value>
+
+   Specify when the compiler is permitted to form fused floating-point
+   operations, such as fused multiply-add (FMA). Fused operations are
+   permitted to produce more precise results than performing the same
+   operations separately.
+
+   The C standard permits intermediate floating-point results within an
+   expression to be computed with more precision than their type would
+   normally allow. This permits operation fusing, and Clang takes advantage
+   of this by default. This behavior can be controlled with the
+   ``FP_CONTRACT`` pragma. Please refer to the pragma documentation for a
+   description of how the pragma interacts with this option.
+
+   Valid values are:
+
+   * ``fast`` (everywhere)
+   * ``on`` (according to FP_CONTRACT pragma, default)
+   * ``off`` (never fuse)
+
+.. _opt_fhonor-infinities:
+
+**-f[no-]honor-infinities**
+
+   If both ``-fno-honor-infinities`` and ``-fno-honor-nans`` are used,
+   has the same effect as specifying ``-ffinite-math``.
+
+.. _opt_fhonor-nans:
+
+**-f[no-]honor-nans**
+
+   If both ``-fno-honor-infinities`` and ``-fno-honor-nans`` are used,
+   has the same effect as specifying ``-ffinite-math``.
+
+.. _opt_fsigned-zeros:
+
+**-f[no-]signed-zeros**
+
+   Allow optimizations that ignore the sign of floating point zeros.
+   Defaults to ``-fno-signed-zeros``.
+
+.. _opt_fassociative-math:
+
+**-f[no-]associative-math**
+
+  Allow floating point operations to be reassociated.
+  Defaults to ``-fno-associative-math``.
+
+.. _opt_freciprocal-math:
+
+**-f[no-]reciprocal-math**
+
+  Allow division operations to be transformed into multiplication by a
+  reciprocal. This can be significantly faster than an ordinary division
+  but can also have significantly less precision. Defaults to
+  ``-fno-reciprocal-math``.
+
+.. _opt_funsafe-math-optimizations:
+
+**-f[no-]unsafe-math-optimizations**
+
+   Allow unsafe floating-point optimizations. Also implies:
+
+   * ``-fassociative-math``
+   * ``-freciprocal-math``
+   * ``-fno-signed-zeroes``
+   * ``-fno-trapping-math``.
+
+   Defaults to ``-fno-unsafe-math-optimizations``.
+
+.. _opt_ffinite-math:
+
+**-f[no-]finite-math**
+
+   Allow floating-point optimizations that assume arguments and results are
+   not NaNs or +-Inf.  This defines the ``__FINITE_MATH_ONLY__`` preprocessor macro.
+   Also implies:
+
+   * ``-fno-honor-infinities``
+   * ``-fno-honor-nans``
+
+   Defaults to ``-fno-finite-math``.
+
 .. _controlling-code-generation:
 
 Controlling Code Generation
@@ -1266,36 +1445,6 @@ are listed below.
    This enables better devirtualization. Turned off by default, because it is
    still experimental.
 
-.. option:: -ffast-math
-
-   Enable fast-math mode. This defines the ``__FAST_MATH__`` preprocessor
-   macro, and lets the compiler make aggressive, potentially-lossy assumptions
-   about floating-point math.  These include:
-
-   * Floating-point math obeys regular algebraic rules for real numbers (e.g.
-     ``+`` and ``*`` are associative, ``x/y == x * (1/y)``, and
-     ``(a + b) * c == a * c + b * c``),
-   * operands to floating-point operations are not equal to ``NaN`` and
-     ``Inf``, and
-   * ``+0`` and ``-0`` are interchangeable.
-
-.. option:: -fdenormal-fp-math=[values]
-
-   Select which denormal numbers the code is permitted to require.
-
-   Valid values are: ``ieee``, ``preserve-sign``, and ``positive-zero``,
-   which correspond to IEEE 754 denormal numbers, the sign of a
-   flushed-to-zero number is preserved in the sign of 0, denormals are
-   flushed to positive zero, respectively.
-
-.. option:: -f[no-]strict-float-cast-overflow
-
-   When a floating-point value is not representable in a destination integer 
-   type, the code has undefined behavior according to the language standard.
-   By default, Clang will not guarantee any particular result in that case.
-   With the 'no-strict' option, Clang attempts to match the overflowing behavior
-   of the target's native float-to-int conversion instructions.
-
 .. option:: -fwhole-program-vtables
 
    Enable whole-program vtable optimizations, such as single-implementation
diff --git a/clang/include/clang-c/FatalErrorHandler.h b/clang/include/clang-c/FatalErrorHandler.h
index 74c9a8fe98bbb..ce8ff2cae735f 100644
--- a/clang/include/clang-c/FatalErrorHandler.h
+++ b/clang/include/clang-c/FatalErrorHandler.h
@@ -18,14 +18,14 @@ extern "C" {
  * Installs error handler that prints error message to stderr and calls abort().
  * Replaces currently installed error handler (if any).
  */
-void clang_install_aborting_llvm_fatal_error_handler();
+void clang_install_aborting_llvm_fatal_error_handler(void);
 
 /**
  * Removes currently installed error handler (if any).
  * If no error handler is intalled, the default strategy is to print error
  * message to stderr and call exit(1).
  */
-void clang_uninstall_llvm_fatal_error_handler();
+void clang_uninstall_llvm_fatal_error_handler(void);
 
 #ifdef __cplusplus
 }
diff --git a/clang/include/clang/AST/APValue.h b/clang/include/clang/AST/APValue.h
index 6943479831ecf..63359294ef632 100644
--- a/clang/include/clang/AST/APValue.h
+++ b/clang/include/clang/AST/APValue.h
@@ -53,6 +53,34 @@ class TypeInfoLValue {
 
   void print(llvm::raw_ostream &Out, const PrintingPolicy &Policy) const;
 };
+
+/// Symbolic representation of a dynamic allocation.
+class DynamicAllocLValue {
+  unsigned Index;
+
+public:
+  DynamicAllocLValue() : Index(0) {}
+  explicit DynamicAllocLValue(unsigned Index) : Index(Index + 1) {}
+  unsigned getIndex() { return Index - 1; }
+
+  explicit operator bool() const { return Index != 0; }
+
+  void *getOpaqueValue() {
+    return reinterpret_cast<void *>(static_cast<uintptr_t>(Index)
+                                    << NumLowBitsAvailable);
+  }
+  static DynamicAllocLValue getFromOpaqueValue(void *Value) {
+    DynamicAllocLValue V;
+    V.Index = reinterpret_cast<uintptr_t>(Value) >> NumLowBitsAvailable;
+    return V;
+  }
+
+  static unsigned getMaxIndex() {
+    return (std::numeric_limits<unsigned>::max() >> NumLowBitsAvailable) - 1;
+  }
+
+  static constexpr int NumLowBitsAvailable = 3;
+};
 }
 
 namespace llvm {
@@ -67,6 +95,17 @@ template<> struct PointerLikeTypeTraits<clang::TypeInfoLValue> {
   // to include Type.h.
   static constexpr int NumLowBitsAvailable = 3;
 };
+
+template<> struct PointerLikeTypeTraits<clang::DynamicAllocLValue> {
+  static void *getAsVoidPointer(clang::DynamicAllocLValue V) {
+    return V.getOpaqueValue();
+  }
+  static clang::DynamicAllocLValue getFromVoidPointer(void *P) {
+    return clang::DynamicAllocLValue::getFromOpaqueValue(P);
+  }
+  static constexpr int NumLowBitsAvailable =
+      clang::DynamicAllocLValue::NumLowBitsAvailable;
+};
 }
 
 namespace clang {
@@ -97,13 +136,15 @@ class APValue {
   };
 
   class LValueBase {
-    typedef llvm::PointerUnion<const ValueDecl *, const Expr *, TypeInfoLValue>
+    typedef llvm::PointerUnion<const ValueDecl *, const Expr *, TypeInfoLValue,
+                               DynamicAllocLValue>
         PtrTy;
 
   public:
     LValueBase() : Local{} {}
     LValueBase(const ValueDecl *P, unsigned I = 0, unsigned V = 0);
     LValueBase(const Expr *P, unsigned I = 0, unsigned V = 0);
+    static LValueBase getDynamicAlloc(DynamicAllocLValue LV, QualType Type);
     static LValueBase getTypeInfo(TypeInfoLValue LV, QualType TypeInfo);
 
     template <class T>
@@ -124,6 +165,7 @@ class APValue {
     unsigned getCallIndex() const;
     unsigned getVersion() const;
     QualType getTypeInfoType() const;
+    QualType getDynamicAllocType() const;
 
     friend bool operator==(const LValueBase &LHS, const LValueBase &RHS);
     friend bool operator!=(const LValueBase &LHS, const LValueBase &RHS) {
@@ -140,6 +182,8 @@ class APValue {
       LocalState Local;
       /// The type std::type_info, if this is a TypeInfoLValue.
       void *TypeInfoType;
+      /// The QualType, if this is a DynamicAllocLValue.
+      void *DynamicAllocType;
     };
   };
 
diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h
index 689dfc76efd18..66bd4aa1e3ba0 100644
--- a/clang/include/clang/AST/ASTContext.h
+++ b/clang/include/clang/AST/ASTContext.h
@@ -2583,10 +2583,12 @@ class ASTContext : public RefCountedBase<ASTContext> {
     return T == getObjCSelType();
   }
 
-  bool ObjCQualifiedIdTypesAreCompatible(QualType LHS, QualType RHS,
+  bool ObjCQualifiedIdTypesAreCompatible(const ObjCObjectPointerType *LHS,
+                                         const ObjCObjectPointerType *RHS,
                                          bool ForCompare);
 
-  bool ObjCQualifiedClassTypesAreCompatible(QualType LHS, QualType RHS);
+  bool ObjCQualifiedClassTypesAreCompatible(const ObjCObjectPointerType *LHS,
+                                            const ObjCObjectPointerType *RHS);
 
   // Check the safety of assignment from LHS to RHS
   bool canAssignObjCInterfaces(const ObjCObjectPointerType *LHSOPT,
diff --git a/clang/include/clang/AST/ASTFwd.h b/clang/include/clang/AST/ASTFwd.h
index 93919bbdd52f5..25c3214854437 100644
--- a/clang/include/clang/AST/ASTFwd.h
+++ b/clang/include/clang/AST/ASTFwd.h
@@ -24,7 +24,7 @@ class Stmt;
 #include "clang/AST/StmtNodes.inc"
 class Type;
 #define TYPE(DERIVED, BASE) class DERIVED##Type;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 class CXXCtorInitializer;
 
 } // end namespace clang
diff --git a/clang/include/clang/AST/ASTTypeTraits.h b/clang/include/clang/AST/ASTTypeTraits.h
index a29a04e5d242d..dd4ead2f0c2bd 100644
--- a/clang/include/clang/AST/ASTTypeTraits.h
+++ b/clang/include/clang/AST/ASTTypeTraits.h
@@ -148,7 +148,7 @@ class ASTNodeKind {
 #include "clang/AST/StmtNodes.inc"
     NKI_Type,
 #define TYPE(DERIVED, BASE) NKI_##DERIVED##Type,
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     NKI_OMPClause,
 #define OPENMP_CLAUSE(TextualSpelling, Class) NKI_##Class,
 #include "clang/Basic/OpenMPKinds.def"
@@ -205,7 +205,7 @@ KIND_TO_KIND_ID(OMPClause)
 #define STMT(DERIVED, BASE) KIND_TO_KIND_ID(DERIVED)
 #include "clang/AST/StmtNodes.inc"
 #define TYPE(DERIVED, BASE) KIND_TO_KIND_ID(DERIVED##Type)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 #define OPENMP_CLAUSE(TextualSpelling, Class) KIND_TO_KIND_ID(Class)
 #include "clang/Basic/OpenMPKinds.def"
 #undef KIND_TO_KIND_ID
diff --git a/clang/include/clang/AST/CMakeLists.txt b/clang/include/clang/AST/CMakeLists.txt
index da16987141c20..fc9839878131a 100644
--- a/clang/include/clang/AST/CMakeLists.txt
+++ b/clang/include/clang/AST/CMakeLists.txt
@@ -31,6 +31,10 @@ clang_tablegen(DeclNodes.inc -gen-clang-decl-nodes
   SOURCE ../Basic/DeclNodes.td
   TARGET ClangDeclNodes)
 
+clang_tablegen(TypeNodes.inc -gen-clang-type-nodes
+  SOURCE ../Basic/TypeNodes.td
+  TARGET ClangTypeNodes)
+
 clang_tablegen(CommentNodes.inc -gen-clang-comment-nodes
   SOURCE ../Basic/CommentNodes.td
   TARGET ClangCommentNodes)
diff --git a/clang/include/clang/AST/CommentLexer.h b/clang/include/clang/AST/CommentLexer.h
index 9ddbb7d31d99e..138fdaca0ff61 100644
--- a/clang/include/clang/AST/CommentLexer.h
+++ b/clang/include/clang/AST/CommentLexer.h
@@ -352,8 +352,7 @@ class Lexer {
 
   void lex(Token &T);
 
-  StringRef getSpelling(const Token &Tok, const SourceManager &SourceMgr,
-                        bool *Invalid = nullptr) const;
+  StringRef getSpelling(const Token &Tok, const SourceManager &SourceMgr) const;
 };
 
 } // end namespace comments
diff --git a/clang/include/clang/AST/Decl.h b/clang/include/clang/AST/Decl.h
index 096df484cbe1f..cbac50daf14c7 100644
--- a/clang/include/clang/AST/Decl.h
+++ b/clang/include/clang/AST/Decl.h
@@ -310,6 +310,14 @@ class NamedDecl : public Decl {
   void printQualifiedName(raw_ostream &OS) const;
   void printQualifiedName(raw_ostream &OS, const PrintingPolicy &Policy) const;
 
+  /// Print only the nested name specifier part of a fully-qualified name,
+  /// including the '::' at the end. E.g.
+  ///    when `printQualifiedName(D)` prints "A::B::i",
+  ///    this function prints "A::B::".
+  void printNestedNameSpecifier(raw_ostream &OS) const;
+  void printNestedNameSpecifier(raw_ostream &OS,
+                                const PrintingPolicy &Policy) const;
+
   // FIXME: Remove string version.
   std::string getQualifiedNameAsString() const;
 
@@ -800,12 +808,19 @@ struct EvaluatedStmt {
   /// valid if CheckedICE is true.
   bool IsICE : 1;
 
+  /// Whether this variable is known to have constant destruction. That is,
+  /// whether running the destructor on the initial value is a side-effect
+  /// (and doesn't inspect any state that might have changed during program
+  /// execution). This is currently only computed if the destructor is
+  /// non-trivial.
+  bool HasConstantDestruction : 1;
+
   Stmt *Value;
   APValue Evaluated;
 
-  EvaluatedStmt() : WasEvaluated(false), IsEvaluating(false), CheckedICE(false),
-                    CheckingICE(false), IsICE(false) {}
-
+  EvaluatedStmt()
+      : WasEvaluated(false), IsEvaluating(false), CheckedICE(false),
+        CheckingICE(false), IsICE(false), HasConstantDestruction(false) {}
 };
 
 /// Represents a variable declaration or definition.
@@ -1259,6 +1274,14 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
   /// to untyped APValue if the value could not be evaluated.
   APValue *getEvaluatedValue() const;
 
+  /// Evaluate the destruction of this variable to determine if it constitutes
+  /// constant destruction.
+  ///
+  /// \pre isInitICE()
+  /// \return \c true if this variable has constant destruction, \c false if
+  ///         not.
+  bool evaluateDestruction(SmallVectorImpl<PartialDiagnosticAt> &Notes) const;
+
   /// Determines whether it is already known whether the
   /// initializer is an integral constant expression or not.
   bool isInitKnownICE() const;
@@ -1497,9 +1520,14 @@ class VarDecl : public DeclaratorDecl, public Redeclarable<VarDecl> {
   // has no definition within this source file.
   bool isKnownToBeDefined() const;
 
-  /// Do we need to emit an exit-time destructor for this variable?
+  /// Is destruction of this variable entirely suppressed? If so, the variable
+  /// need not have a usable destructor at all.
   bool isNoDestroy(const ASTContext &) const;
 
+  /// Do we need to emit an exit-time destructor for this variable, and if so,
+  /// what kind?
+  QualType::DestructionKind needsDestruction(const ASTContext &Ctx) const;
+
   // Implement isa/cast/dyncast/etc.
   static bool classof(const Decl *D) { return classofKind(D->getKind()); }
   static bool classofKind(Kind K) { return K >= firstVar && K <= lastVar; }
diff --git a/clang/include/clang/AST/DeclBase.h b/clang/include/clang/AST/DeclBase.h
index d64d0cb425db0..01c2f1809771b 100644
--- a/clang/include/clang/AST/DeclBase.h
+++ b/clang/include/clang/AST/DeclBase.h
@@ -959,7 +959,7 @@ class alignas(8) Decl {
   /// as this declaration, or NULL if there is no previous declaration.
   Decl *getPreviousDecl() { return getPreviousDeclImpl(); }
 
-  /// Retrieve the most recent declaration that declares the same entity
+  /// Retrieve the previous declaration that declares the same entity
   /// as this declaration, or NULL if there is no previous declaration.
   const Decl *getPreviousDecl() const {
     return const_cast<Decl *>(this)->getPreviousDeclImpl();
diff --git a/clang/include/clang/AST/DeclCXX.h b/clang/include/clang/AST/DeclCXX.h
index f9668f1dce1f8..de0aa5f9b900f 100644
--- a/clang/include/clang/AST/DeclCXX.h
+++ b/clang/include/clang/AST/DeclCXX.h
@@ -42,6 +42,7 @@
 #include "llvm/ADT/PointerUnion.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/iterator_range.h"
+#include "llvm/BinaryFormat/Dwarf.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
@@ -445,6 +446,9 @@ class CXXRecordDecl : public RecordDecl {
     /// or an implicitly declared constexpr default constructor.
     unsigned HasConstexprDefaultConstructor : 1;
 
+    /// True if a defaulted destructor for this class would be constexpr.
+    unsigned DefaultedDestructorIsConstexpr : 1;
+
     /// True when this class contains at least one non-static data
     /// member or base class of non-literal or volatile type.
     unsigned HasNonLiteralTypeFieldsOrBases : 1;
@@ -1168,6 +1172,10 @@ class CXXRecordDecl : public RecordDecl {
   /// if this is a closure type.
   CXXMethodDecl *getLambdaCallOperator() const;
 
+  /// Retrieve the dependent lambda call operator of the closure type
+  /// if this is a templated closure type.
+  FunctionTemplateDecl *getDependentLambdaCallOperator() const;
+
   /// Retrieve the lambda static invoker, the address of which
   /// is returned by the conversion operator, and the body of which
   /// is forwarded to the lambda call operator.
@@ -1352,7 +1360,8 @@ class CXXRecordDecl : public RecordDecl {
   /// would be constexpr.
   bool defaultedDefaultConstructorIsConstexpr() const {
     return data().DefaultedDefaultConstructorIsConstexpr &&
-           (!isUnion() || hasInClassInitializer() || !hasVariantMembers());
+           (!isUnion() || hasInClassInitializer() || !hasVariantMembers() ||
+            getASTContext().getLangOpts().CPlusPlus2a);
   }
 
   /// Determine whether this class has a constexpr default constructor.
@@ -1440,6 +1449,16 @@ class CXXRecordDecl : public RecordDecl {
             !(data().HasTrivialSpecialMembers & SMF_MoveAssignment));
   }
 
+  /// Determine whether a defaulted default constructor for this class
+  /// would be constexpr.
+  bool defaultedDestructorIsConstexpr() const {
+    return data().DefaultedDestructorIsConstexpr &&
+           getASTContext().getLangOpts().CPlusPlus2a;
+  }
+
+  /// Determine whether this class has a constexpr destructor.
+  bool hasConstexprDestructor() const;
+
   /// Determine whether this class has a trivial destructor
   /// (C++ [class.dtor]p3)
   bool hasTrivialDestructor() const {
@@ -1531,8 +1550,10 @@ class CXXRecordDecl : public RecordDecl {
   ///
   /// Only in C++17 and beyond, are lambdas literal types.
   bool isLiteral() const {
-    return hasTrivialDestructor() &&
-           (!isLambda() || getASTContext().getLangOpts().CPlusPlus17) &&
+    ASTContext &Ctx = getASTContext();
+    return (Ctx.getLangOpts().CPlusPlus2a ? hasConstexprDestructor()
+                                          : hasTrivialDestructor()) &&
+           (!isLambda() || Ctx.getLangOpts().CPlusPlus17) &&
            !hasNonLiteralTypeFieldsOrBases() &&
            (isAggregate() || isLambda() ||
             hasConstexprNonCopyMoveConstructor() ||
@@ -2219,7 +2240,7 @@ class CXXMethodDecl : public FunctionDecl {
                                     const CXXRecordDecl *Decl);
 
   Qualifiers getMethodQualifiers() const {
-    return getType()->getAs<FunctionProtoType>()->getMethodQuals();
+    return getType()->castAs<FunctionProtoType>()->getMethodQuals();
   }
 
   /// Retrieve the ref-qualifier associated with this method.
@@ -2234,7 +2255,7 @@ class CXXMethodDecl : public FunctionDecl {
   /// };
   /// @endcode
   RefQualifierKind getRefQualifier() const {
-    return getType()->getAs<FunctionProtoType>()->getRefQualifier();
+    return getType()->castAs<FunctionProtoType>()->getRefQualifier();
   }
 
   bool hasInlineBody() const;
@@ -2554,9 +2575,9 @@ class CXXConstructorDecl final
 
   ExplicitSpecifier getExplicitSpecifierInternal() const {
     if (CXXConstructorDeclBits.HasTrailingExplicitSpecifier)
-      return *getCanonicalDecl()->getTrailingObjects<ExplicitSpecifier>();
+      return *getTrailingObjects<ExplicitSpecifier>();
     return ExplicitSpecifier(
-        nullptr, getCanonicalDecl()->CXXConstructorDeclBits.IsSimpleExplicit
+        nullptr, CXXConstructorDeclBits.IsSimpleExplicit
                      ? ExplicitSpecKind::ResolvedTrue
                      : ExplicitSpecKind::ResolvedFalse);
   }
@@ -2597,10 +2618,10 @@ class CXXConstructorDecl final
          InheritedConstructor Inherited = InheritedConstructor());
 
   ExplicitSpecifier getExplicitSpecifier() {
-    return getExplicitSpecifierInternal();
+    return getCanonicalDecl()->getExplicitSpecifierInternal();
   }
   const ExplicitSpecifier getExplicitSpecifier() const {
-    return getExplicitSpecifierInternal();
+    return getCanonicalDecl()->getExplicitSpecifierInternal();
   }
 
   /// Return true if the declartion is already resolved to be explicit.
@@ -2801,9 +2822,9 @@ class CXXDestructorDecl : public CXXMethodDecl {
   CXXDestructorDecl(ASTContext &C, CXXRecordDecl *RD, SourceLocation StartLoc,
                     const DeclarationNameInfo &NameInfo, QualType T,
                     TypeSourceInfo *TInfo, bool isInline,
-                    bool isImplicitlyDeclared)
+                    bool isImplicitlyDeclared, ConstexprSpecKind ConstexprKind)
       : CXXMethodDecl(CXXDestructor, C, RD, StartLoc, NameInfo, T, TInfo,
-                      SC_None, isInline, CSK_unspecified, SourceLocation()) {
+                      SC_None, isInline, ConstexprKind, SourceLocation()) {
     setImplicit(isImplicitlyDeclared);
   }
 
@@ -2813,9 +2834,9 @@ class CXXDestructorDecl : public CXXMethodDecl {
   static CXXDestructorDecl *Create(ASTContext &C, CXXRecordDecl *RD,
                                    SourceLocation StartLoc,
                                    const DeclarationNameInfo &NameInfo,
-                                   QualType T, TypeSourceInfo* TInfo,
-                                   bool isInline,
-                                   bool isImplicitlyDeclared);
+                                   QualType T, TypeSourceInfo *TInfo,
+                                   bool isInline, bool isImplicitlyDeclared,
+                                   ConstexprSpecKind ConstexprKind);
   static CXXDestructorDecl *CreateDeserialized(ASTContext & C, unsigned ID);
 
   void setOperatorDelete(FunctionDecl *OD, Expr *ThisArg);
@@ -2888,7 +2909,7 @@ class CXXConversionDecl : public CXXMethodDecl {
 
   /// Returns the type that this conversion function is converting to.
   QualType getConversionType() const {
-    return getType()->getAs<FunctionType>()->getReturnType();
+    return getType()->castAs<FunctionType>()->getReturnType();
   }
 
   /// Determine whether this conversion function is a conversion from
@@ -2925,8 +2946,10 @@ class LinkageSpecDecl : public Decl, public DeclContext {
   /// ensure a stable ABI for this, we choose the DW_LANG_ encodings
   /// from the dwarf standard.
   enum LanguageIDs {
-    lang_c = /* DW_LANG_C */ 0x0002,
-    lang_cxx = /* DW_LANG_C_plus_plus */ 0x0004
+    lang_c = llvm::dwarf::DW_LANG_C,
+    lang_cxx = llvm::dwarf::DW_LANG_C_plus_plus,
+    lang_cxx_11 = llvm::dwarf::DW_LANG_C_plus_plus_11,
+    lang_cxx_14 = llvm::dwarf::DW_LANG_C_plus_plus_14
   };
 
 private:
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 0b8ea50109061..33845d9f8a92b 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -906,6 +906,11 @@ class Expr : public ValueStmt {
     return skipRValueSubobjectAdjustments(CommaLHSs, Adjustments);
   }
 
+  /// Checks that the two Expr's will refer to the same value as a comparison
+  /// operand.  The caller must ensure that the values referenced by the Expr's
+  /// are not modified between E1 and E2 or the result my be invalid.
+  static bool isSameComparisonOperand(const Expr* E1, const Expr* E2);
+
   static bool classof(const Stmt *T) {
     return T->getStmtClass() >= firstExprConstant &&
            T->getStmtClass() <= lastExprConstant;
diff --git a/clang/include/clang/AST/ExprCXX.h b/clang/include/clang/AST/ExprCXX.h
index e554968caba74..61e7a91d9ff77 100644
--- a/clang/include/clang/AST/ExprCXX.h
+++ b/clang/include/clang/AST/ExprCXX.h
@@ -1907,6 +1907,10 @@ class LambdaExpr final : public Expr,
   /// lambda expression.
   CXXMethodDecl *getCallOperator() const;
 
+  /// Retrieve the function template call operator associated with this
+  /// lambda expression.
+  FunctionTemplateDecl *getDependentCallOperator() const;
+
   /// If this is a generic lambda expression, retrieve the template
   /// parameter list associated with it, or else return null.
   TemplateParameterList *getTemplateParameterList() const;
@@ -2096,8 +2100,7 @@ class CXXNewExpr final
                                  bool IsParenTypeId);
 
   QualType getAllocatedType() const {
-    assert(getType()->isPointerType());
-    return getType()->getAs<PointerType>()->getPointeeType();
+    return getType()->castAs<PointerType>()->getPointeeType();
   }
 
   TypeSourceInfo *getAllocatedTypeSourceInfo() const {
@@ -2275,8 +2278,8 @@ class CXXDeleteExpr : public Expr {
   CXXDeleteExpr(QualType Ty, bool GlobalDelete, bool ArrayForm,
                 bool ArrayFormAsWritten, bool UsualArrayDeleteWantsSize,
                 FunctionDecl *OperatorDelete, Expr *Arg, SourceLocation Loc)
-      : Expr(CXXDeleteExprClass, Ty, VK_RValue, OK_Ordinary, false, false,
-             Arg->isInstantiationDependent(),
+      : Expr(CXXDeleteExprClass, Ty, VK_RValue, OK_Ordinary, false,
+             Arg->isValueDependent(), Arg->isInstantiationDependent(),
              Arg->containsUnexpandedParameterPack()),
         OperatorDelete(OperatorDelete), Argument(Arg) {
     CXXDeleteExprBits.GlobalDelete = GlobalDelete;
diff --git a/clang/include/clang/AST/ExternalASTMerger.h b/clang/include/clang/AST/ExternalASTMerger.h
index d89189da04f02..d3942ee0a4857 100644
--- a/clang/include/clang/AST/ExternalASTMerger.h
+++ b/clang/include/clang/AST/ExternalASTMerger.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_AST_EXTERNALASTMERGER_H
 
 #include "clang/AST/ASTImporter.h"
+#include "clang/AST/ASTImporterSharedState.h"
 #include "clang/AST/ExternalASTSource.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -79,15 +80,27 @@ class ExternalASTMerger : public ExternalASTSource {
   /// import SourceLocations properly.  Additionally, when import occurs for
   /// a DeclContext whose origin has been overridden, then this
   /// ExternalASTMerger must be able to determine that.
-  struct ImporterSource {
+  class ImporterSource {
     ASTContext &AST;
     FileManager &FM;
     const OriginMap &OM;
+
+  public:
+    ImporterSource(ASTContext &_AST, FileManager &_FM, const OriginMap &_OM)
+        : AST(_AST), FM(_FM), OM(_OM) {}
+    ASTContext &getASTContext() const { return AST; }
+    FileManager &getFileManager() const { return FM; }
+    const OriginMap &getOriginMap() const { return OM; }
   };
 
 private:
   /// The target for this ExtenralASTMerger.
   ImporterTarget Target;
+  /// ExternalASTMerger has multiple ASTImporters that import into the same
+  /// TU. This is the shared state for all ASTImporters of this
+  /// ExternalASTMerger.
+  /// See also the CrossTranslationUnitContext that has a similar setup.
+  std::shared_ptr<ASTImporterSharedState> SharedState;
 
 public:
   ExternalASTMerger(const ImporterTarget &Target,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index c0cb6aadc7d65..7d5faf2b8cad8 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -431,7 +431,7 @@ template <typename Derived> class RecursiveASTVisitor {
 // Declare Traverse*() for all concrete Type classes.
 #define ABSTRACT_TYPE(CLASS, BASE)
 #define TYPE(CLASS, BASE) bool Traverse##CLASS##Type(CLASS##Type *T);
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   // The above header #undefs ABSTRACT_TYPE and TYPE upon exit.
 
   // Define WalkUpFrom*() and empty Visit*() for all Type classes.
@@ -444,7 +444,7 @@ template <typename Derived> class RecursiveASTVisitor {
     return true;                                                               \
   }                                                                            \
   bool Visit##CLASS##Type(CLASS##Type *T) { return true; }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 // ---- Methods on TypeLocs ----
 // FIXME: this currently just calls the matching Type methods
@@ -460,7 +460,7 @@ template <typename Derived> class RecursiveASTVisitor {
   bool VisitTypeLoc(TypeLoc TL) { return true; }
 
   // QualifiedTypeLoc and UnqualTypeLoc are not declared in
-  // TypeNodes.def and thus need to be handled specially.
+  // TypeNodes.inc and thus need to be handled specially.
   bool WalkUpFromQualifiedTypeLoc(QualifiedTypeLoc TL) {
     return getDerived().VisitUnqualTypeLoc(TL.getUnqualifiedLoc());
   }
@@ -478,7 +478,7 @@ template <typename Derived> class RecursiveASTVisitor {
     return true;                                                               \
   }                                                                            \
   bool Visit##CLASS##TypeLoc(CLASS##TypeLoc TL) { return true; }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 // ---- Methods on Decls ----
 
@@ -676,7 +676,7 @@ bool RecursiveASTVisitor<Derived>::TraverseType(QualType T) {
 #define TYPE(CLASS, BASE)                                                      \
   case Type::CLASS:                                                            \
     DISPATCH(CLASS##Type, CLASS##Type, const_cast<Type *>(T.getTypePtr()));
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
 
   return true;
@@ -2844,6 +2844,7 @@ bool RecursiveASTVisitor<Derived>::TraverseOMPClause(OMPClause *C) {
   case OMPC_threadprivate:
   case OMPC_uniform:
   case OMPC_device_type:
+  case OMPC_match:
   case OMPC_unknown:
     break;
   }
diff --git a/clang/include/clang/AST/TextNodeDumper.h b/clang/include/clang/AST/TextNodeDumper.h
index 4c2d0710963b2..0ff5a614a864d 100644
--- a/clang/include/clang/AST/TextNodeDumper.h
+++ b/clang/include/clang/AST/TextNodeDumper.h
@@ -146,8 +146,6 @@ class TextNodeDumper
 
   const comments::CommandTraits *Traits;
 
-  const ASTContext *Context;
-
   const char *getCommandName(unsigned CommandID);
 
 public:
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 293f67c7fe9e9..ba835957ab52c 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -126,7 +126,7 @@ using CanQualType = CanQual<Type>;
 
 // Provide forward declarations for all of the *Type classes.
 #define TYPE(Class, Base) class Class##Type;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 /// The collection of all-type qualifiers we support.
 /// Clang supports five independent qualifiers:
@@ -1445,10 +1445,9 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
 public:
   enum TypeClass {
 #define TYPE(Class, Base) Class,
-#define LAST_TYPE(Class) TypeLast = Class,
+#define LAST_TYPE(Class) TypeLast = Class
 #define ABSTRACT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
-    TagFirst = Record, TagLast = Enum
+#include "clang/AST/TypeNodes.inc"
   };
 
 private:
@@ -2064,6 +2063,7 @@ class alignas(8) Type : public ExtQualsTypeCommonBase {
   bool isCARCBridgableType() const;
   bool isTemplateTypeParmType() const;          // C++ template type parameter
   bool isNullPtrType() const;                   // C++11 std::nullptr_t
+  bool isNothrowT() const;                      // C++   std::nothrow_t
   bool isAlignValT() const;                     // C++17 std::align_val_t
   bool isStdByteType() const;                   // C++17 std::byte
   bool isAtomicType() const;                    // C11 _Atomic()
@@ -2427,7 +2427,7 @@ template <> inline const Class##Type *Type::getAs() const { \
 template <> inline const Class##Type *Type::castAs() const { \
   return cast<Class##Type>(CanonicalType); \
 }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 /// This class is used for builtin types like 'int'.  Builtin
 /// types are always canonical and have a literal name field.
@@ -4443,7 +4443,7 @@ class TagType : public Type {
   bool isBeingDefined() const;
 
   static bool classof(const Type *T) {
-    return T->getTypeClass() >= TagFirst && T->getTypeClass() <= TagLast;
+    return T->getTypeClass() == Enum || T->getTypeClass() == Record;
   }
 };
 
@@ -6361,6 +6361,7 @@ inline bool QualType::isCForbiddenLValueType() const {
 /// \returns True for types specified in C++0x [basic.fundamental].
 inline bool Type::isFundamentalType() const {
   return isVoidType() ||
+         isNullPtrType() ||
          // FIXME: It's really annoying that we don't have an
          // 'isArithmeticType()' which agrees with the standard definition.
          (isArithmeticType() && !isEnumeralType());
diff --git a/clang/include/clang/AST/TypeLoc.h b/clang/include/clang/AST/TypeLoc.h
index 40d17f991f1fb..f305680d775cf 100644
--- a/clang/include/clang/AST/TypeLoc.h
+++ b/clang/include/clang/AST/TypeLoc.h
@@ -106,7 +106,7 @@ class TypeLoc {
 #define ABSTRACT_TYPE(Class, Base)
 #define TYPE(Class, Base) \
     Class = Type::Class,
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     Qualified
   };
 
diff --git a/clang/include/clang/AST/TypeLocNodes.def b/clang/include/clang/AST/TypeLocNodes.def
index c0dfe150d6cc0..81448c7e7ce5b 100644
--- a/clang/include/clang/AST/TypeLocNodes.def
+++ b/clang/include/clang/AST/TypeLocNodes.def
@@ -31,7 +31,7 @@
 TYPELOC(Qualified, TypeLoc)
 #define TYPE(Class, Base) UNQUAL_TYPELOC(Class, Base##Loc)
 #define ABSTRACT_TYPE(Class, Base) ABSTRACT_TYPELOC(Class, Base##Loc)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 #undef DECLARATOR_TYPELOC
 #undef TYPESPEC_TYPELOC
diff --git a/clang/include/clang/AST/TypeNodes.def b/clang/include/clang/AST/TypeNodes.def
deleted file mode 100644
index 58a5f880cbe61..0000000000000
--- a/clang/include/clang/AST/TypeNodes.def
+++ /dev/null
@@ -1,135 +0,0 @@
-//===-- TypeNodes.def - Metadata about Type AST nodes -----------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the AST type info database. Each type node is
-//  enumerated by providing its name (e.g., "Builtin" or "Enum") and
-//  base class (e.g., "Type" or "TagType"). Depending on where in the
-//  abstract syntax tree the type will show up, the enumeration uses
-//  one of five different macros:
-//
-//    TYPE(Class, Base) - A type that can show up anywhere in the AST,
-//    and might be dependent, canonical, or non-canonical. All clients
-//    will need to understand these types.
-//
-//    ABSTRACT_TYPE(Class, Base) - An abstract class that shows up in
-//    the type hierarchy but has no concrete instances.
-//
-//    NON_CANONICAL_TYPE(Class, Base) - A type that can show up
-//    anywhere in the AST but will never be a part of a canonical
-//    type. Clients that only need to deal with canonical types
-//    (ignoring, e.g., typedefs and other type aliases used for
-//    pretty-printing) can ignore these types.
-//
-//    DEPENDENT_TYPE(Class, Base) - A type that will only show up
-//    within a C++ template that has not been instantiated, e.g., a
-//    type that is always dependent. Clients that do not need to deal
-//    with uninstantiated C++ templates can ignore these types.
-//
-//    NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) - A type that
-//    is non-canonical unless it is dependent.  Defaults to TYPE because
-//    it is neither reliably dependent nor reliably non-canonical.
-//
-//  There is a sixth macro, independent of the others.  Most clients
-//  will not need to use it.
-//
-//    LEAF_TYPE(Class) - A type that never has inner types.  Clients
-//    which can operate on such types more efficiently may wish to do so.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ABSTRACT_TYPE
-#  define ABSTRACT_TYPE(Class, Base) TYPE(Class, Base)
-#endif
-
-#ifndef NON_CANONICAL_TYPE
-#  define NON_CANONICAL_TYPE(Class, Base) TYPE(Class, Base)
-#endif
-
-#ifndef DEPENDENT_TYPE
-#  define DEPENDENT_TYPE(Class, Base) TYPE(Class, Base)
-#endif
-
-#ifndef NON_CANONICAL_UNLESS_DEPENDENT_TYPE
-#  define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) TYPE(Class, Base)
-#endif
-
-TYPE(Builtin, Type)
-TYPE(Complex, Type)
-TYPE(Pointer, Type)
-TYPE(BlockPointer, Type)
-ABSTRACT_TYPE(Reference, Type)
-TYPE(LValueReference, ReferenceType)
-TYPE(RValueReference, ReferenceType)
-TYPE(MemberPointer, Type)
-ABSTRACT_TYPE(Array, Type)
-TYPE(ConstantArray, ArrayType)
-TYPE(IncompleteArray, ArrayType)
-TYPE(VariableArray, ArrayType)
-DEPENDENT_TYPE(DependentSizedArray, ArrayType)
-DEPENDENT_TYPE(DependentSizedExtVector, Type)
-DEPENDENT_TYPE(DependentAddressSpace, Type)
-TYPE(Vector, Type)
-DEPENDENT_TYPE(DependentVector, Type)
-TYPE(ExtVector, VectorType)
-ABSTRACT_TYPE(Function, Type)
-TYPE(FunctionProto, FunctionType)
-TYPE(FunctionNoProto, FunctionType)
-DEPENDENT_TYPE(UnresolvedUsing, Type)
-NON_CANONICAL_TYPE(Paren, Type)
-NON_CANONICAL_TYPE(Typedef, Type)
-NON_CANONICAL_TYPE(MacroQualified, Type)
-NON_CANONICAL_TYPE(Adjusted, Type)
-NON_CANONICAL_TYPE(Decayed, AdjustedType)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(TypeOfExpr, Type)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(TypeOf, Type)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Decltype, Type)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(UnaryTransform, Type)
-ABSTRACT_TYPE(Tag, Type)
-TYPE(Record, TagType)
-TYPE(Enum, TagType)
-NON_CANONICAL_TYPE(Elaborated, Type)
-NON_CANONICAL_TYPE(Attributed, Type)
-DEPENDENT_TYPE(TemplateTypeParm, Type)
-NON_CANONICAL_TYPE(SubstTemplateTypeParm, Type)
-DEPENDENT_TYPE(SubstTemplateTypeParmPack, Type)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(TemplateSpecialization, Type)
-ABSTRACT_TYPE(Deduced, Type)
-TYPE(Auto, DeducedType)
-TYPE(DeducedTemplateSpecialization, DeducedType)
-DEPENDENT_TYPE(InjectedClassName, Type)
-DEPENDENT_TYPE(DependentName, Type)
-DEPENDENT_TYPE(DependentTemplateSpecialization, Type)
-NON_CANONICAL_UNLESS_DEPENDENT_TYPE(PackExpansion, Type)
-NON_CANONICAL_TYPE(ObjCTypeParam, Type)
-TYPE(ObjCObject, Type)
-TYPE(ObjCInterface, ObjCObjectType)
-TYPE(ObjCObjectPointer, Type)
-TYPE(Pipe, Type)
-TYPE(Atomic, Type)
-
-#ifdef LAST_TYPE
-LAST_TYPE(Atomic)
-#undef LAST_TYPE
-#endif
-
-// These types are always leaves in the type hierarchy.
-#ifdef LEAF_TYPE
-LEAF_TYPE(Enum)
-LEAF_TYPE(Builtin)
-LEAF_TYPE(Record)
-LEAF_TYPE(InjectedClassName)
-LEAF_TYPE(ObjCInterface)
-LEAF_TYPE(TemplateTypeParm)
-#undef LEAF_TYPE
-#endif
-
-#undef NON_CANONICAL_UNLESS_DEPENDENT_TYPE
-#undef DEPENDENT_TYPE
-#undef NON_CANONICAL_TYPE
-#undef ABSTRACT_TYPE
-#undef TYPE
diff --git a/clang/include/clang/AST/TypeVisitor.h b/clang/include/clang/AST/TypeVisitor.h
index 8930ec8539497..17301835fb187 100644
--- a/clang/include/clang/AST/TypeVisitor.h
+++ b/clang/include/clang/AST/TypeVisitor.h
@@ -70,7 +70,7 @@ class TypeVisitor {
     switch (T->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) case Type::CLASS: DISPATCH(CLASS##Type);
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
     llvm_unreachable("Unknown type class!");
   }
@@ -80,7 +80,7 @@ class TypeVisitor {
 #define TYPE(CLASS, PARENT) RetTy Visit##CLASS##Type(const CLASS##Type *T) { \
   DISPATCH(PARENT);                                                          \
 }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
   /// Method called if \c ImpClass doesn't provide specific handler
   /// for some type class.
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index cc7fe78415a59..55932dc4ec8e7 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -724,9 +724,25 @@ def AVRSignal : InheritableAttr, TargetSpecificAttr<TargetAVR> {
 
 def AsmLabel : InheritableAttr {
   let Spellings = [Keyword<"asm">, Keyword<"__asm__">];
-  let Args = [StringArgument<"Label">];
+  let Args = [
+    // Label specifies the mangled name for the decl.
+    StringArgument<"Label">,
+
+    // IsLiteralLabel specifies whether the label is literal (i.e. suppresses
+    // the global C symbol prefix) or not. If not, the mangle-suppression prefix
+    // ('\01') is omitted from the decl name at the LLVM IR level.
+    //
+    // Non-literal labels are used by some external AST sources like LLDB.
+    BoolArgument<"IsLiteralLabel", /*optional=*/0, /*fake=*/1>
+  ];
   let SemaHandler = 0;
-  let Documentation = [Undocumented];
+  let Documentation = [AsmLabelDocs];
+  let AdditionalMembers =
+[{
+bool isEquivalent(AsmLabelAttr *Other) const {
+  return getLabel() == Other->getLabel() && getIsLiteralLabel() == Other->getIsLiteralLabel();
+}
+}];
 }
 
 def Availability : InheritableAttr {
@@ -3526,6 +3542,78 @@ def OMPAllocateDecl : InheritableAttr {
   let Documentation = [Undocumented];
 }
 
+def OMPDeclareVariant : InheritableAttr {
+  let Spellings = [Pragma<"omp", "declare variant">];
+  let Subjects = SubjectList<[Function]>;
+  let SemaHandler = 0;
+  let HasCustomParsing = 1;
+  let InheritEvenIfAlreadyPresent = 1;
+  let Documentation = [OMPDeclareVariantDocs];
+  let Args = [
+    ExprArgument<"VariantFuncRef">,
+    ExprArgument<"Score">,
+    EnumArgument<"CtxSelectorSet", "CtxSelectorSetType",
+                 [ "", "implementation"
+                 ],
+                 [
+                   "CtxSetUnknown", "CtxSetImplementation"
+                 ]>,
+    EnumArgument<"CtxScore", "ScoreType",
+                 [ "", "score"
+                 ],
+                 [
+                   "ScoreUnknown", "ScoreSpecified"
+                 ]>,
+    EnumArgument<"CtxSelector", "CtxSelectorType",
+                 [ "", "vendor"
+                 ],
+                 [
+                   "CtxUnknown", "CtxVendor"
+                 ]>,
+    StringArgument<"ImplVendor", 1>
+  ];
+  let AdditionalMembers = [{
+    void printScore(raw_ostream & OS, const PrintingPolicy &Policy) const {
+      if (const Expr *E = getScore()) {
+        OS << "score(";
+        E->printPretty(OS, nullptr, Policy);
+        OS << "):";
+      }
+    }
+    void printPrettyPragma(raw_ostream & OS, const PrintingPolicy &Policy)
+        const {
+      assert(getCtxSelectorSet() != CtxSetUnknown &&
+             getCtxSelector() != CtxUnknown && "Unknown context selector.");
+      if (const Expr *E = getVariantFuncRef()) {
+        OS << "(";
+        E->printPretty(OS, nullptr, Policy);
+        OS << ")";
+      }
+      // TODO: add printing of real context selectors.
+      OS << " match(";
+      switch (getCtxSelectorSet()) {
+      case CtxSetImplementation:
+        OS << "implementation={";
+        switch (getCtxSelector()) {
+        case CtxVendor:
+          OS << "vendor(";
+          printScore(OS, Policy);
+          OS << getImplVendor();
+          OS << ")";
+          break;
+        case CtxUnknown:
+          llvm_unreachable("Unknown context selector.");
+        }
+        OS << "}";
+        break;
+      case CtxSetUnknown:
+        llvm_unreachable("Unknown context selector set.");
+      }
+      OS << ")";
+    }
+  }];
+}
+
 def InternalLinkage : InheritableAttr {
   let Spellings = [Clang<"internal_linkage">];
   let Subjects = SubjectList<[Var, Function, CXXRecord]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 5dff70e4b8a7f..ce78fcc139ea0 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -2727,6 +2727,30 @@ manipulating bits of the enumerator when issuing warnings.
   }];
 }
 
+def AsmLabelDocs : Documentation {
+  let Category = DocCatDecl;
+  let Content = [{
+This attribute can be used on a function or variable to specify its symbol name.
+
+On some targets, all C symbols are prefixed by default with a single character, typically ``_``.  This was done historically to distinguish them from symbols used by other languages.  (This prefix is also added to the standard Itanium C++ ABI prefix on "mangled" symbol names, so that e.g. on such targets the true symbol name for a C++ variable declared as ``int cppvar;`` would be ``__Z6cppvar``; note the two underscores.)  This prefix is *not* added to the symbol names specified by the ``asm`` attribute; programmers wishing to match a C symbol name must compensate for this.
+
+For example, consider the following C code:
+
+.. code-block:: c
+
+  int var1 asm("altvar") = 1;  // "altvar" in symbol table.
+  int var2 = 1; // "_var2" in symbol table.
+
+  void func1(void) asm("altfunc");
+  void func1(void) {} // "altfunc" in symbol table.
+  void func2(void) {} // "_func2" in symbol table.
+
+Clang's implementation of this attribute is compatible with GCC's, `documented here <https://gcc.gnu.org/onlinedocs/gcc/Asm-Labels.html>`_.
+
+While it is possible to use this attribute to name a special symbol used internally by the compiler, such as an LLVM intrinsic, this is neither recommended nor supported and may cause the compiler to crash or miscompile.  Users who wish to gain access to intrinsic behavior are strongly encouraged to request new builtin functions.
+  }];
+}
+
 def EnumExtensibilityDocs : Documentation {
   let Category = DocCatDecl;
   let Content = [{
@@ -3410,6 +3434,34 @@ where clause is one of the following:
   }];
 }
 
+def OMPDeclareVariantDocs : Documentation {
+  let Category = DocCatFunction;
+  let Heading = "#pragma omp declare variant";
+  let Content = [{
+The `declare variant` directive declares a specialized variant of a base
+ function and specifies the context in which that specialized variant is used.
+ The declare variant directive is a declarative directive.
+The syntax of the `declare variant` construct is as follows:
+
+  .. code-block:: none
+
+    #pragma omp declare variant(variant-func-id) clause new-line
+    [#pragma omp declare variant(variant-func-id) clause new-line]
+    [...]
+    function definition or declaration
+
+where clause is one of the following:
+
+  .. code-block:: none
+
+    match(context-selector-specification)
+
+and where `variant-func-id` is the name of a function variant that is either a
+ base language identifier or, for C++, a template-id.
+
+  }];
+}
+
 def NoStackProtectorDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/AttributeCommonInfo.h b/clang/include/clang/Basic/AttributeCommonInfo.h
index c8fc0a5a044b8..545e7e9a2b47e 100644
--- a/clang/include/clang/Basic/AttributeCommonInfo.h
+++ b/clang/include/clang/Basic/AttributeCommonInfo.h
@@ -74,11 +74,11 @@ class AttributeCommonInfo {
 
 public:
   AttributeCommonInfo(SourceRange AttrRange)
-      : AttrRange(AttrRange), AttrKind(0), SyntaxUsed(0),
+      : AttrRange(AttrRange), ScopeLoc(), AttrKind(0), SyntaxUsed(0),
         SpellingIndex(SpellingNotCalculated) {}
 
   AttributeCommonInfo(SourceLocation AttrLoc)
-      : AttrRange(AttrLoc), AttrKind(0), SyntaxUsed(0),
+      : AttrRange(AttrLoc), ScopeLoc(), AttrKind(0), SyntaxUsed(0),
         SpellingIndex(SpellingNotCalculated) {}
 
   AttributeCommonInfo(const IdentifierInfo *AttrName,
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 808957573b698..b0f38fbccc365 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -461,7 +461,7 @@ BUILTIN(__builtin_rotateleft64, "UWiUWiUWi", "nc")
 BUILTIN(__builtin_rotateright8, "UcUcUc", "nc")
 BUILTIN(__builtin_rotateright16, "UsUsUs", "nc")
 BUILTIN(__builtin_rotateright32, "UZiUZiUZi", "nc")
-BUILTIN(__builtin_rotateright64, "UWiUWiWi", "nc")
+BUILTIN(__builtin_rotateright64, "UWiUWiUWi", "nc")
 
 // Random GCC builtins
 BUILTIN(__builtin_constant_p, "i.", "nctu")
@@ -1005,9 +1005,7 @@ LIBBUILTIN(pthread_create, "",  "fC<2,3>", "pthread.h", ALL_GNU_LANGUAGES)
 LIBBUILTIN(_setjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(__sigsetjmp, "iSJi",   "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(sigsetjmp, "iSJi",     "fj",   "setjmp.h", ALL_LANGUAGES)
-LIBBUILTIN(setjmp_syscall, "iJ",  "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(savectx, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
-LIBBUILTIN(qsetjmp, "iJ",         "fj",   "setjmp.h", ALL_LANGUAGES)
 LIBBUILTIN(getcontext, "iK*",     "fj",   "setjmp.h", ALL_LANGUAGES)
 
 LIBBUILTIN(_longjmp, "vJi",       "fr",    "setjmp.h", ALL_GNU_LANGUAGES)
@@ -1470,7 +1468,7 @@ BUILTIN(__builtin_operator_new, "v*z", "tc")
 BUILTIN(__builtin_operator_delete, "vv*", "tn")
 BUILTIN(__builtin_char_memchr, "c*cC*iz", "n")
 BUILTIN(__builtin_dump_struct, "ivC*v*", "tn")
-BUILTIN(__builtin_preserve_access_index, "vC*vC*", "nU")
+BUILTIN(__builtin_preserve_access_index, "v.", "t")
 
 // Safestack builtins
 BUILTIN(__builtin___get_unsafe_stack_start, "v*", "Fn")
diff --git a/clang/include/clang/Basic/BuiltinsWebAssembly.def b/clang/include/clang/Basic/BuiltinsWebAssembly.def
index 37f945a319532..f825040d61428 100644
--- a/clang/include/clang/Basic/BuiltinsWebAssembly.def
+++ b/clang/include/clang/Basic/BuiltinsWebAssembly.def
@@ -118,5 +118,19 @@ TARGET_BUILTIN(__builtin_wasm_trunc_saturate_u_i32x4_f32x4, "V4iV4f", "nc", "sim
 TARGET_BUILTIN(__builtin_wasm_trunc_saturate_s_i64x2_f64x2, "V2LLiV2d", "nc", "unimplemented-simd128")
 TARGET_BUILTIN(__builtin_wasm_trunc_saturate_u_i64x2_f64x2, "V2LLiV2d", "nc", "unimplemented-simd128")
 
+TARGET_BUILTIN(__builtin_wasm_narrow_s_i8x16_i16x8, "V16cV8sV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_narrow_u_i8x16_i16x8, "V16cV8sV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_narrow_s_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_narrow_u_i16x8_i32x4, "V8sV4iV4i", "nc", "simd128")
+
+TARGET_BUILTIN(__builtin_wasm_widen_low_s_i16x8_i8x16, "V8sV16c", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_high_s_i16x8_i8x16, "V8sV16c", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_low_u_i16x8_i8x16, "V8sV16c", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_high_u_i16x8_i8x16, "V8sV16c", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_low_s_i32x4_i16x8, "V4iV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_high_s_i32x4_i16x8, "V4iV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_low_u_i32x4_i16x8, "V4iV8s", "nc", "simd128")
+TARGET_BUILTIN(__builtin_wasm_widen_high_u_i32x4_i16x8, "V4iV8s", "nc", "simd128")
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/BuiltinsX86.def b/clang/include/clang/Basic/BuiltinsX86.def
index a0ba0ecf36bb8..5ab9dc1c3ac3a 100644
--- a/clang/include/clang/Basic/BuiltinsX86.def
+++ b/clang/include/clang/Basic/BuiltinsX86.def
@@ -751,8 +751,8 @@ TARGET_BUILTIN(__builtin_ia32_bextri_u32, "UiUiIUi", "nc", "tbm")
 // LWP
 TARGET_BUILTIN(__builtin_ia32_llwpcb, "vv*", "n", "lwp")
 TARGET_BUILTIN(__builtin_ia32_slwpcb, "v*", "n", "lwp")
-TARGET_BUILTIN(__builtin_ia32_lwpins32, "UcUiUiUi", "n", "lwp")
-TARGET_BUILTIN(__builtin_ia32_lwpval32, "vUiUiUi", "n", "lwp")
+TARGET_BUILTIN(__builtin_ia32_lwpins32, "UcUiUiIUi", "n", "lwp")
+TARGET_BUILTIN(__builtin_ia32_lwpval32, "vUiUiIUi", "n", "lwp")
 
 // SHA
 TARGET_BUILTIN(__builtin_ia32_sha1rnds4, "V4iV4iV4iIc", "ncV:128:", "sha")
diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def
index 56051af55e7d5..c535f43203e56 100644
--- a/clang/include/clang/Basic/BuiltinsX86_64.def
+++ b/clang/include/clang/Basic/BuiltinsX86_64.def
@@ -86,8 +86,8 @@ TARGET_BUILTIN(__builtin_ia32_bzhi_di, "UOiUOiUOi", "nc", "bmi2")
 TARGET_BUILTIN(__builtin_ia32_pdep_di, "UOiUOiUOi", "nc", "bmi2")
 TARGET_BUILTIN(__builtin_ia32_pext_di, "UOiUOiUOi", "nc", "bmi2")
 TARGET_BUILTIN(__builtin_ia32_bextri_u64, "UOiUOiIUOi", "nc", "tbm")
-TARGET_BUILTIN(__builtin_ia32_lwpins64, "UcUOiUiUi", "n", "lwp")
-TARGET_BUILTIN(__builtin_ia32_lwpval64, "vUOiUiUi", "n", "lwp")
+TARGET_BUILTIN(__builtin_ia32_lwpins64, "UcUOiUiIUi", "n", "lwp")
+TARGET_BUILTIN(__builtin_ia32_lwpval64, "vUOiUiIUi", "n", "lwp")
 TARGET_BUILTIN(__builtin_ia32_vcvtsd2si64, "OiV2dIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_vcvtsd2usi64, "UOiV2dIi", "ncV:128:", "avx512f")
 TARGET_BUILTIN(__builtin_ia32_vcvtss2si64, "OiV4fIi", "ncV:128:", "avx512f")
diff --git a/clang/include/clang/Basic/DiagnosticASTKinds.td b/clang/include/clang/Basic/DiagnosticASTKinds.td
index 7732be56595a4..63207a0e2254a 100644
--- a/clang/include/clang/Basic/DiagnosticASTKinds.td
+++ b/clang/include/clang/Basic/DiagnosticASTKinds.td
@@ -37,8 +37,9 @@ def note_constexpr_virtual_call : Note<
 def note_constexpr_pure_virtual_call : Note<
   "pure virtual function %q0 called">;
 def note_constexpr_polymorphic_unknown_dynamic_type : Note<
-  "%select{||||virtual function called on|dynamic_cast applied to|"
-  "typeid applied to}0 object '%1' whose dynamic type is not constant">;
+  "%select{|||||virtual function called on|dynamic_cast applied to|"
+  "typeid applied to|construction of|destruction of}0 object '%1' "
+  "whose dynamic type is not constant">;
 def note_constexpr_dynamic_cast_to_reference_failed : Note<
   "reference dynamic_cast failed: %select{"
   "static type %1 of operand is a non-public base class of dynamic type %2|"
@@ -53,6 +54,9 @@ def note_constexpr_nonliteral : Note<
 def note_constexpr_non_global : Note<
   "%select{pointer|reference}0 to %select{|subobject of }1"
   "%select{temporary|%3}2 is not a constant expression">;
+def note_constexpr_dynamic_alloc : Note<
+  "%select{pointer|reference}0 to %select{|subobject of }1"
+  "heap-allocated object is not a constant expression">;
 def note_constexpr_uninitialized : Note<
   "%select{|sub}0object of type %1 is not initialized">;
 def note_constexpr_subobject_declared_here : Note<
@@ -100,6 +104,7 @@ def note_constexpr_typeid_polymorphic : Note<
 def note_constexpr_void_comparison : Note<
   "comparison between unequal pointers to void has unspecified result">;
 def note_constexpr_temporary_here : Note<"temporary created here">;
+def note_constexpr_dynamic_alloc_here : Note<"heap allocation performed here">;
 def note_constexpr_conditional_never_const : Note<
   "both arms of conditional operator are unable to produce a "
   "constant expression">;
@@ -109,16 +114,19 @@ def note_constexpr_call_limit_exceeded : Note<
   "constexpr evaluation hit maximum call limit">;
 def note_constexpr_step_limit_exceeded : Note<
   "constexpr evaluation hit maximum step limit; possible infinite loop?">;
+def note_constexpr_heap_alloc_limit_exceeded : Note<
+  "constexpr evaluation hit maximum heap allocation limit">;
 def note_constexpr_this : Note<
   "%select{|implicit }0use of 'this' pointer is only allowed within the "
   "evaluation of a call to a 'constexpr' member function">;
 def note_constexpr_lifetime_ended : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
-  "%select{temporary|variable}1 whose lifetime has ended">;
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 %select{temporary|variable}1 whose "
+  "%plural{8:storage duration|:lifetime}0 has ended">;
 def note_constexpr_access_uninit : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|<ERRPR>|destruction of}0 "
   "%select{object outside its lifetime|uninitialized object}1 "
   "is not allowed in a constant expression">;
 def note_constexpr_use_uninit_reference : Note<
@@ -128,16 +136,21 @@ def note_constexpr_modify_const_type : Note<
   "modification of object of const-qualified type %0 is not allowed "
   "in a constant expression">;
 def note_constexpr_access_volatile_type : Note<
-  "%select{read of|assignment to|increment of|decrement of|<ERROR>|<ERROR>}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "<ERROR>|<ERROR>|<ERROR>|<ERROR>}0 "
   "volatile-qualified type %1 is not allowed in a constant expression">;
 def note_constexpr_access_volatile_obj : Note<
-  "%select{read of|assignment to|increment of|decrement of|<ERROR>|<ERROR>}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "<ERROR>|<ERROR>|<ERROR>|<ERROR>}0 "
   "volatile %select{temporary|object %2|member %2}1 is not allowed in "
   "a constant expression">;
 def note_constexpr_volatile_here : Note<
   "volatile %select{temporary created|object declared|member declared}0 here">;
-def note_constexpr_ltor_mutable : Note<
-  "read of mutable member %0 is not allowed in a constant expression">;
+def note_constexpr_access_mutable : Note<
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
+  "mutable member %1 is not allowed in a constant expression">;
 def note_constexpr_ltor_non_const_int : Note<
   "read of non-const variable %0 is not allowed in a constant expression">;
 def note_constexpr_ltor_non_constexpr : Note<
@@ -145,31 +158,44 @@ def note_constexpr_ltor_non_constexpr : Note<
 def note_constexpr_ltor_incomplete_type : Note<
   "read of incomplete type %0 is not allowed in a constant expression">;
 def note_constexpr_access_null : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
   "dereferenced null pointer is not allowed in a constant expression">;
 def note_constexpr_access_past_end : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
-  "dereferenced one-past-the-end pointer is not allowed in a constant expression">;
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
+  "dereferenced one-past-the-end pointer is not allowed "
+  "in a constant expression">;
 def note_constexpr_access_unsized_array : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
   "element of array without known bound "
   "is not allowed in a constant expression">;
 def note_constexpr_access_inactive_union_member : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|"
+  "construction of subobject of|destruction of}0 "
   "member %1 of union with %select{active member %3|no active member}2 "
   "is not allowed in a constant expression">;
 def note_constexpr_access_static_temporary : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 temporary "
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|reconstruction of|"
+  "destruction of}0 temporary "
   "is not allowed in a constant expression outside the expression that "
   "created the temporary">;
 def note_constexpr_access_unreadable_object : Note<
-  "%select{read of|assignment to|increment of|decrement of|member call on|"
-  "dynamic_cast of|typeid applied to}0 object '%1' whose value is not known">;
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
+  "object '%1' whose value is not known">;
+def note_constexpr_access_deleted_object : Note<
+  "%select{read of|read of|assignment to|increment of|decrement of|"
+  "member call on|dynamic_cast of|typeid applied to|construction of|"
+  "destruction of}0 "
+  "heap allocated object that has been deleted">;
 def note_constexpr_modify_global : Note<
   "a constant expression cannot modify an object that is visible outside "
   "that expression">;
@@ -189,6 +215,13 @@ def note_constexpr_baa_insufficient_alignment : Note<
 def note_constexpr_baa_value_insufficient_alignment : Note<
   "value of the aligned pointer (%0) is not a multiple of the asserted %1 "
   "%plural{1:byte|:bytes}1">;
+def note_constexpr_destroy_out_of_lifetime : Note<
+  "destroying object '%0' whose lifetime has already ended">;
+def note_constexpr_unsupported_destruction : Note<
+  "non-trivial destruction of type %0 in a constant expression is not supported">;
+def note_constexpr_unsupported_tempoarary_nontrivial_dtor : Note<
+  "non-trivial destruction of lifetime-extended temporary with type %0 "
+  "used in the result of a constant expression is not yet supported">;
 def note_constexpr_unsupported_unsized_array : Note<
   "array-to-pointer decay of array member without known bound is not supported">;
 def note_constexpr_unsized_array_indexed : Note<
@@ -228,6 +261,60 @@ def note_constexpr_bit_cast_invalid_subtype : Note<
 def note_constexpr_bit_cast_indet_dest : Note<
   "indeterminate value can only initialize an object of type 'unsigned char'"
   "%select{, 'char',|}1 or 'std::byte'; %0 is invalid">;
+def note_constexpr_pseudo_destructor : Note<
+  "pseudo-destructor call is not permitted in constant expressions "
+  "until C++20">;
+def note_constexpr_construct_complex_elem : Note<
+  "construction of individual component of complex number is not yet supported "
+  "in constant expressions">;
+def note_constexpr_destroy_complex_elem : Note<
+  "destruction of individual component of complex number is not yet supported "
+  "in constant expressions">;
+def note_constexpr_new : Note<
+  "dynamic memory allocation is not permitted in constant expressions "
+  "until C++20">;
+def note_constexpr_new_non_replaceable : Note<
+  "call to %select{placement|class-specific}0 %1">;
+def note_constexpr_new_placement : Note<
+  "this placement new expression is not yet supported in constant expressions">;
+def note_constexpr_placement_new_wrong_type : Note<
+  "placement new would change type of storage from %0 to %1">;
+def note_constexpr_new_negative : Note<
+  "cannot allocate array; evaluated array bound %0 is negative">;
+def note_constexpr_new_too_large : Note<
+  "cannot allocate array; evaluated array bound %0 is too large">;
+def note_constexpr_new_too_small : Note<
+  "cannot allocate array; evaluated array bound %0 is too small to hold "
+  "%1 explicitly initialized elements">;
+def note_constexpr_new_untyped : Note<
+  "cannot allocate untyped memory in a constant expression; "
+  "use 'std::allocator<T>::allocate' to allocate memory of type 'T'">;
+def note_constexpr_new_not_complete_object_type : Note<
+  "cannot allocate memory of %select{incomplete|function}0 type %1">;
+def note_constexpr_operator_new_bad_size : Note<
+  "allocated size %0 is not a multiple of size %1 of element type %2">;
+def note_constexpr_delete_not_heap_alloc : Note<
+  "delete of pointer '%0' that does not point to a heap-allocated object">;
+def note_constexpr_double_delete : Note<
+  "delete of pointer that has already been deleted">;
+def note_constexpr_double_destroy : Note<
+  "destruction of object that is already being destroyed">;
+def note_constexpr_new_delete_mismatch : Note<
+  "%plural{2:'delete' used to delete pointer to object "
+  "allocated with 'std::allocator<...>::allocate'|"
+  ":%select{non-array delete|array delete|'std::allocator<...>::deallocate'}0 "
+  "used to delete pointer to "
+  "%select{array object of type %2|non-array object of type %2|"
+  "object allocated with 'new'}0}1">;
+def note_constexpr_delete_subobject : Note<
+  "delete of pointer%select{ to subobject|}1 '%0' "
+  "%select{|that does not point to complete object}1">;
+def note_constexpr_delete_base_nonvirt_dtor : Note<
+  "delete of object with dynamic type %1 through pointer to "
+  "base class type %0 with non-virtual destructor">;
+def note_constexpr_memory_leak : Note<
+  "allocation performed here was not deallocated"
+  "%plural{0:|: (along with %0 other memory leak%s0)}0">;
 def err_experimental_clang_interp_failed : Error<
   "the experimental clang interpreter failed to evaluate an expression">;
 
@@ -279,7 +366,6 @@ def warn_odr_variable_multiple_def : Warning<
   "external variable %0 defined in multiple translation units">,
   InGroup<ODR>;
 def note_odr_value_here : Note<"declared here with type %0">;
-def note_odr_defined_here : Note<"also defined here">;
 def err_odr_function_type_inconsistent : Error<
   "external function %0 declared with incompatible types in different "
   "translation units (%1 vs. %2)">;
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index 8196a4601c224..484cc317f9658 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -270,8 +270,6 @@ def err_target_unsupported_mcmse : Error<
   "-mcmse is not supported for %0">;
 def err_opt_not_valid_with_opt : Error<
   "option '%0' cannot be specified with '%1'">;
-def err_opt_not_valid_without_opt : Error<
-  "option '%0' cannot be specified without '%1'">;
 def err_opt_not_valid_on_target : Error<
   "option '%0' cannot be specified on this target">;
 
@@ -315,4 +313,9 @@ def err_unknown_analyzer_checker_or_package : Error<
     "no analyzer checkers or packages are associated with '%0'">;
 def note_suggest_disabling_all_checkers : Note<
     "use -analyzer-disable-all-checks to disable all static analyzer checkers">;
+
+// Poison system directories.
+def warn_poison_system_directories : Warning <
+  "include location '%0' is unsafe for cross-compilation">,
+  InGroup<DiagGroup<"poison-system-directories">>, DefaultIgnore;
 }
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 5cc8f70b32f3d..c3f094c0590f7 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -188,9 +188,6 @@ def warn_drv_unknown_argument_clang_cl_with_suggestion : Warning<
 def warn_drv_ycyu_different_arg_clang_cl : Warning<
   "support for '/Yc' and '/Yu' with different filenames not implemented yet; flags ignored">,
   InGroup<ClangClPch>;
-def warn_drv_ycyu_no_fi_arg_clang_cl : Warning<
-  "support for '%0' without a corresponding /FI flag not implemented yet; flag ignored">,
-  InGroup<ClangClPch>;
 def warn_drv_yc_multiple_inputs_clang_cl : Warning<
   "support for '/Yc' with more than one source file not implemented yet; flag ignored">,
   InGroup<ClangClPch>;
diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
index 3ab18913a457c..a798b498d4e9a 100644
--- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td
+++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td
@@ -64,8 +64,6 @@ def err_fe_backend_unsupported : Error<"%0">, BackendInfo;
 
 def err_fe_invalid_code_complete_file : Error<
     "cannot locate code-completion file %0">, DefaultFatal;
-def err_fe_stdout_binary : Error<"unable to change standard output to binary">,
-  DefaultFatal;
 def err_fe_dependency_file_requires_MT : Error<
     "-dependency-file requires at least one -MT or -MQ option">;
 def err_fe_invalid_plugin_name : Error<
@@ -217,10 +215,6 @@ def err_modules_embed_file_not_found :
   DefaultFatal;
 def err_module_header_file_not_found :
   Error<"module header file '%0' not found">, DefaultFatal;
-def err_module_header_file_invalid :
-  Error<"unexpected module header file input '%0'">, DefaultFatal;
-
-def err_interface_stubs : Error<"clang-ifs (-emit-iterface-stubs): %0">;
 
 def err_test_module_file_extension_version : Error<
   "test module file extension '%0' has different version (%1.%2) than expected "
diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td
index 22a42001280d1..0f225bee24c06 100644
--- a/clang/include/clang/Basic/DiagnosticGroups.td
+++ b/clang/include/clang/Basic/DiagnosticGroups.td
@@ -61,9 +61,16 @@ def BoolConversion : DiagGroup<"bool-conversion", [PointerBoolConversion,
                                                    UndefinedBoolConversion]>;
 def IntConversion : DiagGroup<"int-conversion">;
 def EnumConversion : DiagGroup<"enum-conversion">;
-def ImplicitIntConversion : DiagGroup<"implicit-int-conversion">;
+def ObjCSignedCharBoolImplicitIntConversion :
+  DiagGroup<"objc-signed-char-bool-implicit-int-conversion">;
+def ImplicitIntConversion : DiagGroup<"implicit-int-conversion",
+                                     [ObjCSignedCharBoolImplicitIntConversion]>;
 def ImplicitIntFloatConversion : DiagGroup<"implicit-int-float-conversion">;
-def ImplicitFloatConversion : DiagGroup<"implicit-float-conversion", [ImplicitIntFloatConversion]>;
+def ObjCSignedCharBoolImplicitFloatConversion :
+  DiagGroup<"objc-signed-char-bool-implicit-float-conversion">;
+def ImplicitFloatConversion : DiagGroup<"implicit-float-conversion",
+  [ImplicitIntFloatConversion,
+   ObjCSignedCharBoolImplicitFloatConversion]>;
 def ImplicitFixedPointConversion : DiagGroup<"implicit-fixed-point-conversion">;
 
 def FloatOverflowConversion : DiagGroup<"float-overflow-conversion">;
@@ -492,6 +499,7 @@ def StringCompare : DiagGroup<"string-compare">;
 def StringPlusInt : DiagGroup<"string-plus-int">;
 def StringPlusChar : DiagGroup<"string-plus-char">;
 def StrncatSize : DiagGroup<"strncat-size">;
+def IntInBoolContext : DiagGroup<"int-in-bool-context">;
 def TautologicalTypeLimitCompare : DiagGroup<"tautological-type-limit-compare">;
 def TautologicalUnsignedZeroCompare : DiagGroup<"tautological-unsigned-zero-compare">;
 def TautologicalUnsignedEnumZeroCompare : DiagGroup<"tautological-unsigned-enum-zero-compare">;
@@ -555,6 +563,7 @@ def CoveredSwitchDefault : DiagGroup<"covered-switch-default">;
 def SwitchBool     : DiagGroup<"switch-bool">;
 def SwitchEnum     : DiagGroup<"switch-enum">;
 def Switch         : DiagGroup<"switch">;
+def EnumCompareConditional : DiagGroup<"enum-compare-conditional">;
 def EnumCompareSwitch : DiagGroup<"enum-compare-switch">;
 def EnumCompare       : DiagGroup<"enum-compare", [EnumCompareSwitch]>;
 def ImplicitFallthroughPerFunction :
@@ -814,6 +823,7 @@ def Most : DiagGroup<"most", [
     Format,
     Implicit,
     InfiniteRecursion,
+    IntInBoolContext,
     MismatchedTags,
     MissingBraces,
     Move,
@@ -1015,6 +1025,12 @@ def ObjCLiteralComparison : DiagGroup<"objc-literal-compare", [
     ObjCStringComparison
   ]>;
 
+def ObjCSignedCharBool : DiagGroup<"objc-signed-char-bool",
+  [ObjCSignedCharBoolImplicitIntConversion,
+   ObjCSignedCharBoolImplicitFloatConversion,
+   ObjCBoolConstantConversion,
+   TautologicalObjCBoolCompare]>;
+
 // Inline ASM warnings.
 def ASMOperandWidths : DiagGroup<"asm-operand-widths">;
 def ASMIgnoredQualifier : DiagGroup<"asm-ignored-qualifier">;
diff --git a/clang/include/clang/Basic/DiagnosticOptions.def b/clang/include/clang/Basic/DiagnosticOptions.def
index baafd7ac723f6..6d1a1af92821b 100644
--- a/clang/include/clang/Basic/DiagnosticOptions.def
+++ b/clang/include/clang/Basic/DiagnosticOptions.def
@@ -49,6 +49,7 @@ DIAGOPT(Pedantic, 1, 0)         /// -pedantic
 DIAGOPT(PedanticErrors, 1, 0)   /// -pedantic-errors
 DIAGOPT(ShowColumn, 1, 1)       /// Show column number on diagnostics.
 DIAGOPT(ShowLocation, 1, 1)     /// Show source location information.
+DIAGOPT(ShowLevel, 1, 1)        /// Show diagnostic level.
 DIAGOPT(AbsolutePath, 1, 0)     /// Use absolute paths.
 DIAGOPT(ShowCarets, 1, 1)       /// Show carets in diagnostics.
 DIAGOPT(ShowFixits, 1, 1)       /// Show fixit information.
diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td
index 50e7342a5cd60..a7dbcd08b8cd0 100644
--- a/clang/include/clang/Basic/DiagnosticParseKinds.td
+++ b/clang/include/clang/Basic/DiagnosticParseKinds.td
@@ -201,6 +201,7 @@ def err_invalid_token_after_declarator_suggest_equal : Error<
   "invalid %0 at end of declaration; did you mean '='?">;
 def err_expected_statement : Error<"expected statement">;
 def err_expected_lparen_after : Error<"expected '(' after '%0'">;
+def err_expected_lbrace_after : Error<"expected '{' after '%0'">;
 def err_expected_rparen_after : Error<"expected ')' after '%0'">;
 def err_expected_punc : Error<"expected ')' or ',' after '%0'">;
 def err_expected_less_after : Error<"expected '<' after '%0'">;
@@ -1177,8 +1178,8 @@ def err_omp_expected_identifier_for_critical : Error<
   "expected identifier specifying the name of the 'omp critical' directive">;
 def err_omp_expected_reduction_identifier : Error<
   "expected identifier or one of the following operators: '+', '-', '*', '&', '|', '^', '&&', or '||'">;
-def err_omp_decl_in_declare_simd : Error<
-  "function declaration is expected after 'declare simd' directive">;
+def err_omp_decl_in_declare_simd_variant : Error<
+  "function declaration is expected after 'declare %select{simd|variant}0' directive">;
 def err_omp_unknown_map_type : Error<
   "incorrect map type, expected one of 'to', 'from', 'tofrom', 'alloc', 'release', or 'delete'">;
 def err_omp_unknown_map_type_modifier : Error<
@@ -1199,6 +1200,17 @@ def err_omp_mapper_illegal_identifier : Error<
   "illegal OpenMP user-defined mapper identifier">;
 def err_omp_mapper_expected_declarator : Error<
   "expected declarator on 'omp declare mapper' directive">;
+def err_omp_declare_variant_wrong_clause : Error<
+  "expected '%0' clause on 'omp declare variant' directive">;
+def err_omp_declare_variant_no_ctx_selector : Error<
+  "expected context selector in '%0' clause on 'omp declare variant' directive">;
+def err_omp_declare_variant_equal_expected : Error<
+  "expected '=' after '%0' context selector set name on 'omp declare variant' directive">;
+def warn_omp_declare_variant_cs_name_expected : Warning<
+  "unknown context selector in '%0' context selector set of 'omp declare variant' directive, ignored">,
+  InGroup<OpenMPClauses>;
+def err_omp_declare_variant_item_expected : Error<
+  "expected %0 in '%1' context selector of '%2' selector set of 'omp declare variant' directive">;
 def warn_omp_more_one_device_type_clause : Warning<
   "more than one 'device_type' clause is specified">,
   InGroup<OpenMPClauses>;
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 9d36ff6f87d31..ca07d14dd594f 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -2382,7 +2382,13 @@ def err_constexpr_tag : Error<
   "%select{class|struct|interface|union|enum}0 "
   "cannot be marked %sub{select_constexpr_spec_kind}1">;
 def err_constexpr_dtor : Error<
-  "destructor cannot be marked %sub{select_constexpr_spec_kind}0">;
+  "destructor cannot be declared %sub{select_constexpr_spec_kind}0">;
+def err_constexpr_dtor_subobject : Error<
+  "destructor cannot be declared %sub{select_constexpr_spec_kind}0 because "
+  "%select{data member %2|base class %3}1 does not have a "
+  "constexpr destructor">;
+def note_constexpr_dtor_subobject : Note<
+  "%select{data member %1|base class %2}0 declared here">;
 def err_constexpr_wrong_decl_kind : Error<
   "%sub{select_constexpr_spec_kind}0 can only be used "
   "in %select{|variable and function|function|variable}0 declarations">;
@@ -2394,6 +2400,8 @@ def err_constexpr_var_non_literal : Error<
   "constexpr variable cannot have non-literal type %0">;
 def err_constexpr_var_requires_const_init : Error<
   "constexpr variable %0 must be initialized by a constant expression">;
+def err_constexpr_var_requires_const_destruction : Error<
+  "constexpr variable %0 must have constant destruction">;
 def err_constexpr_redecl_mismatch : Error<
   "%select{non-constexpr|constexpr|consteval}1 declaration of %0"
   " follows %select{non-constexpr|constexpr|consteval}2 declaration">;
@@ -2454,9 +2462,13 @@ def err_constexpr_local_var_static : Error<
 def err_constexpr_local_var_non_literal_type : Error<
   "variable of non-literal type %1 cannot be defined in a constexpr "
   "%select{function|constructor}0">;
-def err_constexpr_local_var_no_init : Error<
-  "variables defined in a constexpr %select{function|constructor}0 must be "
-  "initialized">;
+def ext_constexpr_local_var_no_init : ExtWarn<
+  "uninitialized variable in a constexpr %select{function|constructor}0 "
+  "is a C++20 extension">, InGroup<CXX2a>;
+def warn_cxx17_compat_constexpr_local_var_no_init : Warning<
+  "uninitialized variable in a constexpr %select{function|constructor}0 "
+  "is incompatible with C++ standards before C++20">,
+  InGroup<CXXPre2aCompat>, DefaultIgnore;
 def ext_constexpr_function_never_constant_expr : ExtWarn<
   "constexpr %select{function|constructor}0 never produces a "
   "constant expression">, InGroup<DiagGroup<"invalid-constexpr">>, DefaultError;
@@ -2481,10 +2493,8 @@ def warn_cxx11_compat_constexpr_body_multiple_return : Warning<
   InGroup<CXXPre14Compat>, DefaultIgnore;
 def note_constexpr_body_previous_return : Note<
   "previous return statement is here">;
-def err_constexpr_function_try_block : Error<
-  "function try block not allowed in constexpr %select{function|constructor}0">;
 
-// c++2a function try blocks in constexpr
+// C++2a function try blocks in constexpr
 def ext_constexpr_function_try_block_cxx2a : ExtWarn<
   "function try block in constexpr %select{function|constructor}0 is "
   "a C++2a extension">, InGroup<CXX2a>;
@@ -2493,10 +2503,20 @@ def warn_cxx17_compat_constexpr_function_try_block : Warning<
   "incompatible with C++ standards before C++2a">,
   InGroup<CXXPre2aCompat>, DefaultIgnore;
 
-def err_constexpr_union_ctor_no_init : Error<
-  "constexpr union constructor does not initialize any member">;
-def err_constexpr_ctor_missing_init : Error<
-  "constexpr constructor must initialize all members">;
+def ext_constexpr_union_ctor_no_init : ExtWarn<
+  "constexpr union constructor that does not initialize any member "
+  "is a C++20 extension">, InGroup<CXX2a>;
+def warn_cxx17_compat_constexpr_union_ctor_no_init : Warning<
+  "constexpr union constructor that does not initialize any member "
+  "is incompatible with C++ standards before C++20">,
+  InGroup<CXXPre2aCompat>, DefaultIgnore;
+def ext_constexpr_ctor_missing_init : ExtWarn<
+  "constexpr constructor that does not initialize all members "
+  "is a C++20 extension">, InGroup<CXX2a>;
+def warn_cxx17_compat_constexpr_ctor_missing_init : Warning<
+  "constexpr constructor that does not initialize all members "
+  "is incompatible with C++ standards before C++20">,
+  InGroup<CXXPre2aCompat>, DefaultIgnore;
 def note_constexpr_ctor_missing_init : Note<
   "member not initialized by constructor">;
 def note_non_literal_no_constexpr_ctors : Note<
@@ -2511,6 +2531,8 @@ def note_non_literal_user_provided_dtor : Note<
   "%0 is not literal because it has a user-provided destructor">;
 def note_non_literal_nontrivial_dtor : Note<
   "%0 is not literal because it has a non-trivial destructor">;
+def note_non_literal_non_constexpr_dtor : Note<
+  "%0 is not literal because its destructor is not constexpr">;
 def note_non_literal_lambda : Note<
   "lambda closure types are non-literal types before C++17">;
 def warn_private_extern : Warning<
@@ -3014,6 +3036,10 @@ def warn_gnu_inline_attribute_requires_inline : Warning<
   "'gnu_inline' attribute requires function to be marked 'inline',"
   " attribute ignored">,
   InGroup<IgnoredAttributes>;
+def warn_gnu_inline_cplusplus_without_extern : Warning<
+  "'gnu_inline' attribute without 'extern' in C++ treated as externally"
+  " available, this changed in Clang 10">,
+  InGroup<DiagGroup<"gnu-inline-cpp-without-extern">>;
 def err_attribute_vecreturn_only_vector_member : Error<
   "the vecreturn attribute can only be used on a class or structure with one member, which must be a vector">;
 def err_attribute_vecreturn_only_pod_record : Error<
@@ -3316,9 +3342,9 @@ def warn_impcast_integer_precision_constant : Warning<
 def warn_impcast_bitfield_precision_constant : Warning<
   "implicit truncation from %2 to bit-field changes value from %0 to %1">,
   InGroup<BitFieldConstantConversion>;
-def warn_impcast_constant_int_to_objc_bool : Warning<
-  "implicit conversion from constant value %0 to BOOL; "
-  "the only well defined values for BOOL are YES and NO">,
+def warn_impcast_constant_value_to_objc_bool : Warning<
+  "implicit conversion from constant value %0 to 'BOOL'; "
+  "the only well defined values for 'BOOL' are YES and NO">,
   InGroup<ObjCBoolConstantConversion>;
 
 def warn_impcast_fixed_point_range : Warning<
@@ -3334,6 +3360,12 @@ def warn_impcast_literal_float_to_integer_out_of_range : Warning<
 def warn_impcast_float_integer : Warning<
   "implicit conversion turns floating-point number into integer: %0 to %1">,
   InGroup<FloatConversion>, DefaultIgnore;
+def warn_impcast_float_to_objc_signed_char_bool : Warning<
+  "implicit conversion from floating-point type %0 to 'BOOL'">,
+  InGroup<ObjCSignedCharBoolImplicitFloatConversion>;
+def warn_impcast_int_to_objc_signed_char_bool : Warning<
+  "implicit conversion from integral type %0 to 'BOOL'">,
+  InGroup<ObjCSignedCharBoolImplicitIntConversion>, DefaultIgnore;
 
 // Implicit int -> float conversion precision loss warnings.
 def warn_impcast_integer_float_precision : Warning<
@@ -5428,9 +5460,6 @@ def err_arc_mismatched_cast : Error<
   " to %3 is disallowed with ARC">;
 def err_arc_nolifetime_behavior : Error<
   "explicit ownership qualifier on cast result has no effect">;
-def err_arc_objc_object_in_tag : Error<
-  "ARC forbids %select{Objective-C objects|blocks}0 in "
-  "%select{struct|interface|union|<<ERROR>>|enum}1">;
 def err_arc_objc_property_default_assign_on_object : Error<
   "ARC forbids synthesizing a property of an Objective-C object "
   "with unspecified ownership or storage attribute">;
@@ -5723,6 +5752,12 @@ def warn_precedence_conditional : Warning<
 def note_precedence_conditional_first : Note<
   "place parentheses around the '?:' expression to evaluate it first">;
 
+def warn_enum_constant_in_bool_context : Warning<
+  "converting the enum constant to a boolean">,
+  InGroup<IntInBoolContext>, DefaultIgnore;
+def warn_left_shift_in_bool_context : Warning<
+  "converting the result of '<<' to a boolean; did you mean '(%0) != 0'?">,
+  InGroup<IntInBoolContext>, DefaultIgnore;
 def warn_logical_instead_of_bitwise : Warning<
   "use of logical '%0' with constant operand">,
   InGroup<DiagGroup<"constant-logical-operand">>;
@@ -6135,8 +6170,8 @@ def warn_tautological_constant_compare : Warning<
   "%select{%1|%3}0 is always %4">,
   InGroup<TautologicalTypeLimitCompare>, DefaultIgnore;
 def warn_tautological_compare_objc_bool : Warning<
-  "result of comparison of constant %0 with expression of type BOOL"
-  " is always %1, as the only well defined values for BOOL are YES and NO">,
+  "result of comparison of constant %0 with expression of type 'BOOL'"
+  " is always %1, as the only well defined values for 'BOOL' are YES and NO">,
   InGroup<TautologicalObjCBoolCompare>;
 
 def warn_mixed_sign_comparison : Warning<
@@ -6148,10 +6183,22 @@ def warn_out_of_range_compare : Warning<
   InGroup<TautologicalOutOfRangeCompare>;
 def warn_tautological_bool_compare : Warning<warn_out_of_range_compare.Text>,
   InGroup<TautologicalConstantCompare>;
+def warn_integer_constants_in_conditional_always_true : Warning<
+  "converting the result of '?:' with integer constants to a boolean always "
+  "evaluates to 'true'">,
+  InGroup<TautologicalConstantCompare>;
+def warn_left_shift_always : Warning<
+  "converting the result of '<<' to a boolean always evaluates "
+  "to %select{false|true}0">,
+  InGroup<TautologicalConstantCompare>;
 def warn_comparison_of_mixed_enum_types : Warning<
   "comparison of two values with different enumeration types"
   "%diff{ ($ and $)|}0,1">,
   InGroup<EnumCompare>;
+def warn_conditional_mixed_enum_types : Warning<
+  "enumeration type mismatch in conditional expression"
+  "%diff{ ($ and $)|}0,1">,
+  InGroup<EnumCompareConditional>, DefaultIgnore;
 def warn_comparison_of_mixed_enum_types_switch : Warning<
   "comparison of two values with different enumeration types in switch statement"
   "%diff{ ($ and $)|}0,1">,
@@ -7621,8 +7668,6 @@ let CategoryName = "Inline Assembly Issue" in {
     "invalid lvalue in asm input for constraint '%0'">;
   def err_asm_invalid_input_constraint : Error<
     "invalid input constraint '%0' in asm">;
-  def err_asm_immediate_expected : Error<"constraint '%0' expects "
-    "an integer constant expression">;
   def err_asm_tying_incompatible_types : Error<
     "unsupported inline asm: input with type "
     "%diff{$ matching output with type $|}0,1">;
@@ -8151,6 +8196,9 @@ def warn_printf_invalid_objc_flag: Warning<
 def warn_scanf_scanlist_incomplete : Warning<
   "no closing ']' for '%%[' in scanf format string">,
   InGroup<Format>;
+def warn_format_bool_as_character : Warning<
+  "using '%0' format specifier, but argument has boolean value">,
+  InGroup<Format>;
 def note_format_string_defined : Note<"format string is defined here">;
 def note_format_fix_specifier : Note<"did you mean to use '%0'?">;
 def note_printf_c_str: Note<"did you mean to call the %0 method?">;
@@ -9239,10 +9287,10 @@ def err_omp_single_copyprivate_with_nowait : Error<
   "the 'copyprivate' clause must not be used with the 'nowait' clause">;
 def note_omp_nowait_clause_here : Note<
   "'nowait' clause is here">;
-def err_omp_single_decl_in_declare_simd : Error<
-  "single declaration is expected after 'declare simd' directive">;
+def err_omp_single_decl_in_declare_simd_variant : Error<
+  "single declaration is expected after 'declare %select{simd|variant}0' directive">;
 def err_omp_function_expected : Error<
-  "'#pragma omp declare simd' can only be applied to functions">;
+  "'#pragma omp declare %select{simd|variant}0' can only be applied to functions">;
 def err_omp_wrong_cancel_region : Error<
   "one of 'for', 'parallel', 'sections' or 'taskgroup' is expected">;
 def err_omp_parent_cancel_region_nowait : Error<
@@ -9434,6 +9482,34 @@ def note_omp_marked_device_type_here : Note<"marked as 'device_type(%0)' here">;
 def warn_omp_declare_target_after_first_use : Warning<
   "declaration marked as declare target after first use, it may lead to incorrect results">,
   InGroup<OpenMPTarget>;
+def err_omp_declare_variant_incompat_attributes : Error<
+  "'#pragma omp declare variant' is not compatible with any target-specific attributes">;
+def warn_omp_declare_variant_after_used : Warning<
+  "'#pragma omp declare variant' cannot be applied for function after first "
+  "usage; the original function might be used">, InGroup<SourceUsesOpenMP>;
+def warn_omp_declare_variant_after_emitted : Warning<
+  "'#pragma omp declare variant' cannot be applied to the function that was defined already;"
+  " the original function might be used">, InGroup<SourceUsesOpenMP>;
+def err_omp_declare_variant_noproto : Error<
+  "function with '#pragma omp declare variant' must have a prototype">;
+def note_omp_declare_variant_specified_here : Note<
+  "'#pragma omp declare variant' for function specified here">;
+def err_omp_declare_variant_doesnt_support : Error<
+  "'#pragma omp declare variant' does not "
+  "support %select{function templates|virtual functions|"
+  "deduced return types|constructors|destructors|deleted functions|"
+  "defaulted functions|constexpr functions|consteval function}0">;
+def err_omp_declare_variant_diff : Error<
+  "function with '#pragma omp declare variant' has a different %select{calling convention"
+  "|return type|constexpr specification|inline specification|storage class|"
+  "linkage}0">;
+def err_omp_declare_variant_incompat_types : Error<
+  "variant in '#pragma omp declare variant' with type %0 is incompatible with type %1"
+  >;
+def warn_omp_declare_variant_marked_as_declare_variant : Warning<
+  "variant function in '#pragma omp declare variant' is itself marked as '#pragma omp declare variant'"
+  >, InGroup<SourceUsesOpenMP>;
+def note_omp_marked_declare_variant_here : Note<"marked as 'declare variant' here">;
 } // end of OpenMP category
 
 let CategoryName = "Related Result Type Issue" in {
@@ -9883,8 +9959,6 @@ def err_memtag_arg_must_be_pointer : Error<
   "%0 argument of MTE builtin function must be a pointer (%1 invalid)">;
 def err_memtag_arg_must_be_integer : Error<
   "%0 argument of MTE builtin function must be an integer type (%1 invalid)">;
-def err_memtag_arg_must_be_unsigned : Error<
-  "%0 argument  of MTE builtin function must be an unsigned integer type (%1 invalid)">;
 
 def warn_dereference_of_noderef_type : Warning<
   "dereferencing %0; was declared with a 'noderef' type">, InGroup<NoDeref>;
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 57132f5c0ffb9..28eb694ba9a89 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -39,6 +39,8 @@
 FEATURE(address_sanitizer,
         LangOpts.Sanitize.hasOneOf(SanitizerKind::Address |
                                    SanitizerKind::KernelAddress))
+FEATURE(leak_sanitizer,
+        LangOpts.Sanitize.has(SanitizerKind::Leak))
 FEATURE(hwaddress_sanitizer,
         LangOpts.Sanitize.hasOneOf(SanitizerKind::HWAddress |
                                    SanitizerKind::KernelHWAddress))
diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index b39ab50a89d5a..eca8b7d0d822d 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -230,6 +230,8 @@ LANGOPT(SYCLIsHost        , 1, 0, "SYCL host compilation")
 LANGOPT(SYCLAllowFuncPtr  , 1, 0, "Allow function pointers in SYCL device code")
 LANGOPT(SYCLUnnamedLambda , 1, 0, "Allow unnamed lambda SYCL kernels")
 
+LANGOPT(HIPUseNewLaunchAPI, 1, 0, "Use new kernel launching API for HIP")
+
 LANGOPT(SizedDeallocation , 1, 0, "sized deallocation")
 LANGOPT(AlignedAllocation , 1, 0, "aligned allocation")
 LANGOPT(AlignedAllocationUnavailable, 1, 0, "aligned allocation functions are unavailable")
diff --git a/clang/include/clang/Basic/OpenCLOptions.h b/clang/include/clang/Basic/OpenCLOptions.h
index 47310da1d6d95..15661154eab54 100644
--- a/clang/include/clang/Basic/OpenCLOptions.h
+++ b/clang/include/clang/Basic/OpenCLOptions.h
@@ -42,7 +42,7 @@ class OpenCLOptions {
 
   // Is supported as either an extension or an (optional) core feature for
   // OpenCL version \p CLVer.
-  bool isSupported(llvm::StringRef Ext, LangOptions LO) const {
+  bool isSupported(llvm::StringRef Ext, const LangOptions &LO) const {
     // In C++ mode all extensions should work at least as in v2.0.
     auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
     auto I = OptMap.find(Ext)->getValue();
@@ -51,7 +51,7 @@ class OpenCLOptions {
 
   // Is supported (optional) OpenCL core features for OpenCL version \p CLVer.
   // For supported extension, return false.
-  bool isSupportedCore(llvm::StringRef Ext, LangOptions LO) const {
+  bool isSupportedCore(llvm::StringRef Ext, const LangOptions &LO) const {
     // In C++ mode all extensions should work at least as in v2.0.
     auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
     auto I = OptMap.find(Ext)->getValue();
@@ -60,7 +60,7 @@ class OpenCLOptions {
 
   // Is supported OpenCL extension for OpenCL version \p CLVer.
   // For supported (optional) core feature, return false.
-  bool isSupportedExtension(llvm::StringRef Ext, LangOptions LO) const {
+  bool isSupportedExtension(llvm::StringRef Ext, const LangOptions &LO) const {
     // In C++ mode all extensions should work at least as in v2.0.
     auto CLVer = LO.OpenCLCPlusPlus ? 200 : LO.OpenCLVersion;
     auto I = OptMap.find(Ext)->getValue();
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 9a9592a7627b4..e8a40de3e3cdb 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -194,6 +194,12 @@
 #ifndef OPENMP_DEVICE_TYPE_KIND
 #define OPENMP_DEVICE_TYPE_KIND(Name)
 #endif
+#ifndef OPENMP_DECLARE_VARIANT_CLAUSE
+#define OPENMP_DECLARE_VARIANT_CLAUSE(Name)
+#endif
+#ifndef OPENMP_MATCH_KIND
+#define OPENMP_MATCH_KIND(Name)
+#endif
 
 // OpenMP directives.
 OPENMP_DIRECTIVE(threadprivate)
@@ -251,6 +257,7 @@ OPENMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for, "target teams distrib
 OPENMP_DIRECTIVE_EXT(target_teams_distribute_parallel_for_simd, "target teams distribute parallel for simd")
 OPENMP_DIRECTIVE_EXT(target_teams_distribute_simd, "target teams distribute simd")
 OPENMP_DIRECTIVE(allocate)
+OPENMP_DIRECTIVE_EXT(declare_variant, "declare variant")
 
 // OpenMP clauses.
 OPENMP_CLAUSE(allocator, OMPAllocatorClause)
@@ -958,6 +965,15 @@ OPENMP_DEVICE_TYPE_KIND(host)
 OPENMP_DEVICE_TYPE_KIND(nohost)
 OPENMP_DEVICE_TYPE_KIND(any)
 
+// Clauses allowed for OpenMP directive 'declare variant'.
+OPENMP_DECLARE_VARIANT_CLAUSE(match)
+
+// Context selectors for 'match' clause.
+// TODO: add other context selectors.
+OPENMP_MATCH_KIND(implementation)
+
+#undef OPENMP_MATCH_KIND
+#undef OPENMP_DECLARE_VARIANT_CLAUSE
 #undef OPENMP_DEVICE_TYPE_KIND
 #undef OPENMP_ALLOCATE_CLAUSE
 #undef OPENMP_DECLARE_MAPPER_CLAUSE
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 003737b0ab783..fe0ba51becade 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -36,6 +36,7 @@ enum OpenMPClauseKind {
   OMPC_threadprivate,
   OMPC_uniform,
   OMPC_device_type,
+  OMPC_match,
   OMPC_unknown
 };
 
diff --git a/clang/include/clang/Basic/TypeNodes.td b/clang/include/clang/Basic/TypeNodes.td
new file mode 100644
index 0000000000000..b2554de24aaf4
--- /dev/null
+++ b/clang/include/clang/Basic/TypeNodes.td
@@ -0,0 +1,106 @@
+class Type<bit abstract = 0> {
+  bit Abstract = abstract;
+} 
+
+class DerivedType<Type base, bit abstract = 0> : Type<abstract> {
+	Type Base = base;
+}
+
+/// A type node that is only used to represent dependent types in C++.  For
+/// example, DependentTemplateSpecializationType is used to represent types
+/// where the base template-id is dependent (such as `T::foo<U>`).  Code
+/// that only works with non-dependent types can ignore these type nodes.
+class AlwaysDependent {}
+
+/// A type node that is never used to represent a canonical type, which is to
+/// say that it always represents some sort of type "sugar" which can
+/// (supposedly) be erased without affecting the formal behavior of the
+/// language.  For example, in standard C/C++, typedefs do not introduce new
+/// types and do not affect the semantics of the program.  Code that only
+/// works with canonical types can ignore these type nodes.
+///
+/// Note that this simple story about non-canonical types is not the whole
+/// truth.  Languages and extensions often have formation rules which differ
+/// based on how a type is spelled and which therefore are not consistent
+/// with immediately stipping away type sugar.  More critically, attributes on
+/// typedefs can have semantic impacts in ways that are only reflected in our
+/// AST by preserving the typedef sugar; for example, we do not otherwise
+/// represent the alignment attribute on typedefs, and so it is necessary to
+/// preserve typedef structure into most parts of IR generation.
+class NeverCanonical {}
+
+/// A type node that only represents a canonical type in some dependent cases.
+/// For example, `std::vector<int>` (a TemplateSpecializationType) is
+/// considered to be a non-canonical representation for the RecordType
+/// referencing the concrete ClassTemplateSpecializationDecl; but
+/// `std::vector<T>` cannot be resolved to a concrete specialization
+/// and so remains canonical.  Code which only works with non-dependent
+/// canonical types can ignore these nodes.
+class NeverCanonicalUnlessDependent {}
+
+/// A type node which never has component type structure.  Some code may be
+/// able to operate on leaf types faster than they can on non-leaf types.
+///
+/// For example, the function type `void (int)` is not a leaf type because it
+/// is structurally composed of component types (`void` and `int`).
+///
+/// A struct type is a leaf type because its field types are not part of its
+/// type-expression.
+///
+/// Nodes like `TypedefType` which are syntactically leaves but can desugar
+/// to types that may not be leaves should not declare this.
+class LeafType {}
+
+def BuiltinType : Type, LeafType;
+def ComplexType : Type;
+def PointerType : Type;
+def BlockPointerType : Type;
+def ReferenceType : Type<1>;
+def LValueReferenceType : DerivedType<ReferenceType>;
+def RValueReferenceType : DerivedType<ReferenceType>;
+def MemberPointerType : Type;
+def ArrayType : Type<1>;
+def ConstantArrayType : DerivedType<ArrayType>;
+def IncompleteArrayType : DerivedType<ArrayType>;
+def VariableArrayType : DerivedType<ArrayType>;
+def DependentSizedArrayType : DerivedType<ArrayType>, AlwaysDependent;
+def DependentSizedExtVectorType : Type, AlwaysDependent;
+def DependentAddressSpaceType : Type, AlwaysDependent;
+def VectorType : Type;
+def DependentVectorType : Type, AlwaysDependent;
+def ExtVectorType : DerivedType<VectorType>;
+def FunctionType : Type<1>;
+def FunctionProtoType : DerivedType<FunctionType>;
+def FunctionNoProtoType : DerivedType<FunctionType>;
+def UnresolvedUsingType : Type, AlwaysDependent;
+def ParenType : Type, NeverCanonical;
+def TypedefType : Type, NeverCanonical;
+def MacroQualifiedType : Type, NeverCanonical;
+def AdjustedType : Type, NeverCanonical;
+def DecayedType : DerivedType<AdjustedType>, NeverCanonical;
+def TypeOfExprType : Type, NeverCanonicalUnlessDependent;
+def TypeOfType : Type, NeverCanonicalUnlessDependent;
+def DecltypeType : Type, NeverCanonicalUnlessDependent;
+def UnaryTransformType : Type, NeverCanonicalUnlessDependent;
+def TagType : Type<1>;
+def RecordType : DerivedType<TagType>, LeafType;
+def EnumType : DerivedType<TagType>, LeafType;
+def ElaboratedType : Type, NeverCanonical;
+def AttributedType : Type, NeverCanonical;
+def TemplateTypeParmType : Type, AlwaysDependent, LeafType;
+def SubstTemplateTypeParmType : Type, NeverCanonical;
+def SubstTemplateTypeParmPackType : Type, AlwaysDependent;
+def TemplateSpecializationType : Type, NeverCanonicalUnlessDependent;
+def DeducedType : Type<1>;
+def AutoType : DerivedType<DeducedType>;
+def DeducedTemplateSpecializationType : DerivedType<DeducedType>;
+def InjectedClassNameType : Type, AlwaysDependent, LeafType;
+def DependentNameType : Type, AlwaysDependent;
+def DependentTemplateSpecializationType : Type, AlwaysDependent;
+def PackExpansionType : Type, NeverCanonicalUnlessDependent;
+def ObjCTypeParamType : Type, NeverCanonical;
+def ObjCObjectType : Type;
+def ObjCInterfaceType : DerivedType<ObjCObjectType>, LeafType;
+def ObjCObjectPointerType : Type;
+def PipeType : Type;
+def AtomicType : Type;
diff --git a/clang/include/clang/CodeGen/CGFunctionInfo.h b/clang/include/clang/CodeGen/CGFunctionInfo.h
index 1f81072e23d00..5069d9af42a3c 100644
--- a/clang/include/clang/CodeGen/CGFunctionInfo.h
+++ b/clang/include/clang/CodeGen/CGFunctionInfo.h
@@ -109,14 +109,12 @@ class ABIArgInfo {
     UnpaddedCoerceAndExpandType = T;
   }
 
-  ABIArgInfo(Kind K)
-      : TheKind(K), PaddingInReg(false), InReg(false) {
-  }
-
 public:
-  ABIArgInfo()
+  ABIArgInfo(Kind K = Direct)
       : TypeData(nullptr), PaddingType(nullptr), DirectOffset(0),
-        TheKind(Direct), PaddingInReg(false), InReg(false) {}
+        TheKind(K), PaddingInReg(false), InAllocaSRet(false),
+        IndirectByVal(false), IndirectRealign(false), SRetAfterThis(false),
+        InReg(false), CanBeFlattened(false), SignExt(false) {}
 
   static ABIArgInfo getDirect(llvm::Type *T = nullptr, unsigned Offset = 0,
                               llvm::Type *Padding = nullptr,
diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td
index ad6d1f32363dc..35063f8f40b5d 100644
--- a/clang/include/clang/Driver/CC1Options.td
+++ b/clang/include/clang/Driver/CC1Options.td
@@ -691,7 +691,7 @@ let Flags = [CC1Option, CC1AsOption, NoDriverOption] in {
 def version : Flag<["-"], "version">,
   HelpText<"Print the compiler version">;
 def main_file_name : Separate<["-"], "main-file-name">,
-  HelpText<"Main file name to use for debug info">;
+  HelpText<"Main file name to use for debug info and source if missing">;
 def split_dwarf_output : Separate<["-"], "split-dwarf-output">,
   HelpText<"File name to use for split dwarf debug info output">;
 
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 7c3b2de84f601..0d84508947d14 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -621,6 +621,9 @@ def hip_device_lib_EQ : Joined<["--"], "hip-device-lib=">, Group<Link_Group>,
   HelpText<"HIP device library">;
 def fhip_dump_offload_linker_script : Flag<["-"], "fhip-dump-offload-linker-script">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>;
+def fhip_new_launch_api : Flag<["-"], "fhip-new-launch-api">,
+  Flags<[CC1Option]>, HelpText<"Use new kernel launching API for HIP.">;
+def fno_hip_new_launch_api : Flag<["-"], "fno-hip-new-launch-api">;
 def libomptarget_nvptx_path_EQ : Joined<["--"], "libomptarget-nvptx-path=">, Group<i_Group>,
   HelpText<"Path to libomptarget-nvptx libraries">;
 def dD : Flag<["-"], "dD">, Group<d_Group>, Flags<[CC1Option]>,
@@ -2227,6 +2230,12 @@ def msave_restore : Flag<["-"], "msave-restore">, Group<m_riscv_Features_Group>,
   HelpText<"Enable using library calls for save and restore">;
 def mno_save_restore : Flag<["-"], "mno-save-restore">, Group<m_riscv_Features_Group>,
   HelpText<"Disable using library calls for save and restore">;
+def mcmodel_EQ_medlow : Flag<["-"], "mcmodel=medlow">, Group<m_riscv_Features_Group>,
+  Flags<[CC1Option]>, Alias<mcmodel_EQ>, AliasArgs<["small"]>,
+  HelpText<"Equivalent to -mcmodel=small, compatible with RISC-V gcc.">;
+def mcmodel_EQ_medany : Flag<["-"], "mcmodel=medany">, Group<m_riscv_Features_Group>,
+  Flags<[CC1Option]>, Alias<mcmodel_EQ>, AliasArgs<["medium"]>,
+  HelpText<"Equivalent to -mcmodel=medium, compatible with RISC-V gcc.">;
 
 def munaligned_access : Flag<["-"], "munaligned-access">, Group<m_arm_Features_Group>,
   HelpText<"Allow memory accesses to be unaligned (AArch32/AArch64 only)">;
@@ -2431,7 +2440,7 @@ def mpie_copy_relocations : Flag<["-"], "mpie-copy-relocations">, Group<m_Group>
   Flags<[CC1Option]>,
   HelpText<"Use copy relocations support for PIE builds">;
 def mno_pie_copy_relocations : Flag<["-"], "mno-pie-copy-relocations">, Group<m_Group>;
-def mfentry : Flag<["-"], "mfentry">, HelpText<"Insert calls to fentry at function entry (x86 only)">,
+def mfentry : Flag<["-"], "mfentry">, HelpText<"Insert calls to fentry at function entry (x86/SystemZ only)">,
   Flags<[CC1Option]>, Group<m_Group>;
 def mips16 : Flag<["-"], "mips16">, Group<m_mips_Features_Group>;
 def mno_mips16 : Flag<["-"], "mno-mips16">, Group<m_mips_Features_Group>;
@@ -2806,7 +2815,6 @@ def _mhwdiv : Separate<["--"], "mhwdiv">, Alias<mhwdiv_EQ>;
 def _CLASSPATH_EQ : Joined<["--"], "CLASSPATH=">, Alias<fclasspath_EQ>;
 def _CLASSPATH : Separate<["--"], "CLASSPATH">, Alias<fclasspath_EQ>;
 def _all_warnings : Flag<["--"], "all-warnings">, Alias<Wall>;
-def _analyze_auto : Flag<["--"], "analyze-auto">, Flags<[DriverOption]>;
 def _analyzer_no_default_checks : Flag<["--"], "analyzer-no-default-checks">, Flags<[DriverOption]>;
 def _analyzer_output : JoinedOrSeparate<["--"], "analyzer-output">, Flags<[DriverOption]>,
   HelpText<"Static analyzer report output format (html|plist|plist-multi-file|plist-html|text).">;
diff --git a/clang/include/clang/Format/Format.h b/clang/include/clang/Format/Format.h
index 07b2df912da4f..6389e498e483f 100644
--- a/clang/include/clang/Format/Format.h
+++ b/clang/include/clang/Format/Format.h
@@ -733,6 +733,32 @@ struct FormatStyle {
     ///     B
     ///   };
     /// \endcode
+    BS_Whitesmiths,
+    /// Like ``Allman`` but always indent braces and line up code with braces.
+    /// \code
+    ///    try
+    ///     {
+    ///     foo();
+    ///     }
+    ///   catch ()
+    ///     {
+    ///     }
+    ///   void foo() { bar(); }
+    ///   class foo
+    ///     {
+    ///     };
+    ///   if (foo())
+    ///     {
+    ///     }
+    ///   else
+    ///     {
+    ///     }
+    ///   enum X : int
+    ///     {
+    ///     A,
+    ///     B
+    ///     };
+    /// \endcode
     BS_GNU,
     /// Like ``Attach``, but break before functions.
     /// \code
@@ -1919,15 +1945,32 @@ struct FormatStyle {
   /// \endcode
   bool SpacesInSquareBrackets;
 
-  /// Supported language standards.
+  /// Supported language standards for parsing and formatting C++ constructs.
+  /// \code
+  ///    Latest:                                vector<set<int>>
+  ///    c++03                          vs.     vector<set<int> >
+  /// \endcode
+  ///
+  /// The correct way to spell a specific language version is e.g. ``c++11``.
+  /// The historical aliases ``Cpp03`` and ``Cpp11`` are deprecated.
   enum LanguageStandard {
-    /// Use C++03-compatible syntax.
+    /// c++03: Parse and format as C++03.
     LS_Cpp03,
-    /// Use features of C++11, C++14 and C++1z (e.g. ``A<A<int>>`` instead of
-    /// ``A<A<int> >``).
+    /// c++11: Parse and format as C++11.
     LS_Cpp11,
-    /// Automatic detection based on the input.
-    LS_Auto
+    /// c++14: Parse and format as C++14.
+    LS_Cpp14,
+    /// c++17: Parse and format as C++17.
+    LS_Cpp17,
+    /// c++20: Parse and format as C++20.
+    LS_Cpp20,
+    /// Latest: Parse and format using the latest supported language version.
+    /// 'Cpp11' is an alias for LS_Latest for historical reasons.
+    LS_Latest,
+
+    /// Auto: Automatic detection based on the input.
+    /// Parse using the latest language version. Format based on detected input.
+    LS_Auto,
   };
 
   /// Format compatible with this standard, e.g. use ``A<A<int> >``
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index d800ee1289eb2..1bdd2be04c0ef 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -2208,7 +2208,7 @@ class Preprocessor {
       SourceLocation FilenameLoc, CharSourceRange FilenameRange,
       const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
       bool &IsMapped, const DirectoryLookup *LookupFrom,
-      const FileEntry *LookupFromFile, SmallString<128> &NormalizedPath,
+      const FileEntry *LookupFromFile, StringRef LookupFilename,
       SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
       ModuleMap::KnownHeader &SuggestedModule, bool isAngled);
 
diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h
index bc9686ab350e1..b391eb6fd4ff4 100644
--- a/clang/include/clang/Parse/Parser.h
+++ b/clang/include/clang/Parse/Parser.h
@@ -2843,6 +2843,17 @@ class Parser : public CodeCompletionHandler {
   DeclGroupPtrTy ParseOMPDeclareSimdClauses(DeclGroupPtrTy Ptr,
                                             CachedTokens &Toks,
                                             SourceLocation Loc);
+  /// Parses OpenMP context selectors and calls \p Callback for each
+  /// successfully parsed context selector.
+  bool parseOpenMPContextSelectors(
+      SourceLocation Loc,
+      llvm::function_ref<
+          void(SourceRange, const Sema::OpenMPDeclareVariantCtsSelectorData &)>
+          Callback);
+
+  /// Parse clauses for '#pragma omp declare variant'.
+  void ParseOMPDeclareVariantClauses(DeclGroupPtrTy Ptr, CachedTokens &Toks,
+                                     SourceLocation Loc);
   /// Parse clauses for '#pragma omp declare target'.
   DeclGroupPtrTy ParseOMPDeclareTargetClauses();
   /// Parse '#pragma omp end declare target'.
@@ -2936,7 +2947,8 @@ class Parser : public CodeCompletionHandler {
   /// Parses simple expression in parens for single-expression clauses of OpenMP
   /// constructs.
   /// \param RLoc Returned location of right paren.
-  ExprResult ParseOpenMPParensExpr(StringRef ClauseName, SourceLocation &RLoc);
+  ExprResult ParseOpenMPParensExpr(StringRef ClauseName, SourceLocation &RLoc,
+                                   bool IsAddressOfOperand = false);
 
   /// Data used for parsing list of variables in OpenMP clauses.
   struct OpenMPVarListDataTy {
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 435af4e8cb1f4..aa5a3d1289f97 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -4767,6 +4767,12 @@ class Sema {
                            MultiExprArg ArgExprs, SourceLocation RParenLoc,
                            Expr *ExecConfig = nullptr,
                            bool IsExecConfig = false);
+  enum class AtomicArgumentOrder { API, AST };
+  ExprResult
+  BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
+                  SourceLocation RParenLoc, MultiExprArg Args,
+                  AtomicExpr::AtomicOp Op,
+                  AtomicArgumentOrder ArgOrder = AtomicArgumentOrder::API);
   ExprResult
   BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl, SourceLocation LParenLoc,
                         ArrayRef<Expr *> Arg, SourceLocation RParenLoc,
@@ -9228,7 +9234,39 @@ class Sema {
                                      MapT &Map, unsigned Selector = 0,
                                      SourceRange SrcRange = SourceRange());
 
+  /// Marks all the functions that might be required for the currently active
+  /// OpenMP context.
+  void markOpenMPDeclareVariantFuncsReferenced(SourceLocation Loc,
+                                               FunctionDecl *Func,
+                                               bool MightBeOdrUse);
+
 public:
+  /// Struct to store the context selectors info for declare variant directive.
+  struct OpenMPDeclareVariantCtsSelectorData {
+    OMPDeclareVariantAttr::CtxSelectorSetType CtxSet =
+        OMPDeclareVariantAttr::CtxSetUnknown;
+    OMPDeclareVariantAttr::CtxSelectorType Ctx =
+        OMPDeclareVariantAttr::CtxUnknown;
+    StringRef ImplVendor;
+    ExprResult CtxScore;
+    explicit OpenMPDeclareVariantCtsSelectorData() = default;
+    explicit OpenMPDeclareVariantCtsSelectorData(
+        OMPDeclareVariantAttr::CtxSelectorSetType CtxSet,
+        OMPDeclareVariantAttr::CtxSelectorType Ctx, StringRef ImplVendor,
+        ExprResult CtxScore)
+        : CtxSet(CtxSet), Ctx(Ctx), ImplVendor(ImplVendor), CtxScore(CtxScore) {
+    }
+  };
+
+  /// Checks if the variant/multiversion functions are compatible.
+  bool areMultiversionVariantFunctionsCompatible(
+      const FunctionDecl *OldFD, const FunctionDecl *NewFD,
+      const PartialDiagnostic &NoProtoDiagID,
+      const PartialDiagnosticAt &NoteCausedDiagIDAt,
+      const PartialDiagnosticAt &NoSupportDiagIDAt,
+      const PartialDiagnosticAt &DiffDiagIDAt, bool TemplatesSupported,
+      bool ConstexprSupported);
+
   /// Function tries to capture lambda's captured variables in the OpenMP region
   /// before the original lambda is captured.
   void tryCaptureOpenMPLambdas(ValueDecl *V);
@@ -9653,6 +9691,29 @@ class Sema {
       ArrayRef<Expr *> Alignments, ArrayRef<Expr *> Linears,
       ArrayRef<unsigned> LinModifiers, ArrayRef<Expr *> Steps, SourceRange SR);
 
+  /// Checks '\#pragma omp declare variant' variant function and original
+  /// functions after parsing of the associated method/function.
+  /// \param DG Function declaration to which declare variant directive is
+  /// applied to.
+  /// \param VariantRef Expression that references the variant function, which
+  /// must be used instead of the original one, specified in \p DG.
+  /// \returns None, if the function/variant function are not compatible with
+  /// the pragma, pair of original function/variant ref expression otherwise.
+  Optional<std::pair<FunctionDecl *, Expr *>> checkOpenMPDeclareVariantFunction(
+      DeclGroupPtrTy DG, Expr *VariantRef, SourceRange SR);
+
+  /// Called on well-formed '\#pragma omp declare variant' after parsing of
+  /// the associated method/function.
+  /// \param FD Function declaration to which declare variant directive is
+  /// applied to.
+  /// \param VariantRef Expression that references the variant function, which
+  /// must be used instead of the original one, specified in \p DG.
+  /// \param Data Set of context-specific data for the specified context
+  /// selector.
+  void ActOnOpenMPDeclareVariantDirective(
+      FunctionDecl *FD, Expr *VariantRef, SourceRange SR,
+      const Sema::OpenMPDeclareVariantCtsSelectorData &Data);
+
   OMPClause *ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
                                          Expr *Expr,
                                          SourceLocation StartLoc,
diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
index 04d611a94a7f5..ce16095e10c0b 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
@@ -278,13 +278,13 @@ class AnalyzerOptions : public RefCountedBase<AnalyzerOptions> {
 
   // Create an array of all -analyzer-config command line options. Sort it in
   // the constructor.
-  std::vector<StringRef> AnalyzerConfigCmdFlags = {
+  std::vector<llvm::StringLiteral> AnalyzerConfigCmdFlags = {
 #define ANALYZER_OPTION_DEPENDS_ON_USER_MODE(TYPE, NAME, CMDFLAG, DESC,        \
                                              SHALLOW_VAL, DEEP_VAL)            \
   ANALYZER_OPTION(TYPE, NAME, CMDFLAG, DESC, SHALLOW_VAL)
 
 #define ANALYZER_OPTION(TYPE, NAME, CMDFLAG, DESC, DEFAULT_VAL)                \
-    CMDFLAG,
+  llvm::StringLiteral(CMDFLAG),
 
 #include "clang/StaticAnalyzer/Core/AnalyzerOptions.def"
 #undef ANALYZER_OPTION
@@ -415,9 +415,10 @@ inline UserModeKind AnalyzerOptions::getUserMode() const {
 
 inline std::vector<StringRef>
 AnalyzerOptions::getRegisteredCheckers(bool IncludeExperimental) {
-  static const StringRef StaticAnalyzerCheckerNames[] = {
+  static constexpr llvm::StringLiteral StaticAnalyzerCheckerNames[] = {
 #define GET_CHECKERS
-#define CHECKER(FULLNAME, CLASS, HELPTEXT, DOC_URI, IS_HIDDEN) FULLNAME,
+#define CHECKER(FULLNAME, CLASS, HELPTEXT, DOC_URI, IS_HIDDEN)                 \
+  llvm::StringLiteral(FULLNAME),
 #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
 #undef CHECKER
 #undef GET_CHECKERS
@@ -433,9 +434,9 @@ AnalyzerOptions::getRegisteredCheckers(bool IncludeExperimental) {
 
 inline std::vector<StringRef>
 AnalyzerOptions::getRegisteredPackages(bool IncludeExperimental) {
-  static const StringRef StaticAnalyzerPackageNames[] = {
+  static constexpr llvm::StringLiteral StaticAnalyzerPackageNames[] = {
 #define GET_PACKAGES
-#define PACKAGE(FULLNAME) FULLNAME,
+#define PACKAGE(FULLNAME) llvm::StringLiteral(FULLNAME),
 #include "clang/StaticAnalyzer/Checkers/Checkers.inc"
 #undef PACKAGE
 #undef GET_PACKAGES
diff --git a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
index 8d1a672bc6129..fc1cc91388266 100644
--- a/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
+++ b/clang/include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h
@@ -347,7 +347,7 @@ class CallEvent {
   ProgramStateRef invalidateRegions(unsigned BlockCount,
                                     ProgramStateRef Orig = nullptr) const;
 
-  using FrameBindingTy = std::pair<Loc, SVal>;
+  using FrameBindingTy = std::pair<SVal, SVal>;
   using BindingsTy = SmallVectorImpl<FrameBindingTy>;
 
   /// Populates the given SmallVector with the bindings in the callee's stack
diff --git a/clang/include/clang/Tooling/ArgumentsAdjusters.h b/clang/include/clang/Tooling/ArgumentsAdjusters.h
index bf0886034324a..c48a8725aae90 100644
--- a/clang/include/clang/Tooling/ArgumentsAdjusters.h
+++ b/clang/include/clang/Tooling/ArgumentsAdjusters.h
@@ -43,6 +43,10 @@ ArgumentsAdjuster getClangSyntaxOnlyAdjuster();
 /// arguments.
 ArgumentsAdjuster getClangStripOutputAdjuster();
 
+/// Gets an argument adjuster which removes command line arguments related to
+/// diagnostic serialization.
+ArgumentsAdjuster getClangStripSerializeDiagnosticAdjuster();
+
 /// Gets an argument adjuster which removes dependency-file
 /// related command line arguments.
 ArgumentsAdjuster getClangStripDependencyFileAdjuster();
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index 94b370520093b..1d0d26589e04d 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -56,6 +56,9 @@ class CachedFileSystemEntry {
   /// \returns True if the entry is valid.
   bool isValid() const { return !MaybeStat || MaybeStat->isStatusKnown(); }
 
+  /// \returns True if the current entry points to a directory.
+  bool isDirectory() const { return MaybeStat && MaybeStat->isDirectory(); }
+
   /// \returns The error or the file's contents.
   llvm::ErrorOr<StringRef> getContents() const {
     if (!MaybeStat)
diff --git a/clang/include/clang/Tooling/Inclusions/HeaderIncludes.h b/clang/include/clang/Tooling/Inclusions/HeaderIncludes.h
index ec6f0ea45ffed..6e6d6d8fb024b 100644
--- a/clang/include/clang/Tooling/Inclusions/HeaderIncludes.h
+++ b/clang/include/clang/Tooling/Inclusions/HeaderIncludes.h
@@ -32,6 +32,7 @@ class IncludeCategoryManager {
   /// 0. Otherwise, returns the priority of the matching category or INT_MAX.
   /// NOTE: this API is not thread-safe!
   int getIncludePriority(StringRef IncludeName, bool CheckMainHeader) const;
+  int getSortIncludePriority(StringRef IncludeName, bool CheckMainHeader) const;
 
 private:
   bool isMainHeader(StringRef IncludeName) const;
diff --git a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
index a0f236e6fc461..266763a5b1bd0 100644
--- a/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
+++ b/clang/include/clang/Tooling/Inclusions/IncludeStyle.h
@@ -58,6 +58,8 @@ struct IncludeStyle {
     std::string Regex;
     /// The priority to assign to this category.
     int Priority;
+    /// The custom priority to sort before grouping.
+    int SortPriority;
     bool operator==(const IncludeCategory &Other) const {
       return Regex == Other.Regex && Priority == Other.Priority;
     }
diff --git a/clang/include/clang/Tooling/Refactoring/MatchConsumer.h b/clang/include/clang/Tooling/Refactoring/MatchConsumer.h
new file mode 100644
index 0000000000000..d516550d0f02b
--- /dev/null
+++ b/clang/include/clang/Tooling/Refactoring/MatchConsumer.h
@@ -0,0 +1,58 @@
+//===--- MatchConsumer.h - MatchConsumer abstraction ------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file defines the *MatchConsumer* abstraction: a computation over
+/// match results, specifically the `ast_matchers::MatchFinder::MatchResult`
+/// class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_REFACTOR_MATCH_CONSUMER_H_
+#define LLVM_CLANG_TOOLING_REFACTOR_MATCH_CONSUMER_H_
+
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+
+namespace clang {
+namespace tooling {
+
+/// A failable computation over nodes bound by AST matchers.
+///
+/// The computation should report any errors though its return value (rather
+/// than terminating the program) to enable usage in interactive scenarios like
+/// clang-query.
+///
+/// This is a central abstraction of the Transformer framework.
+template <typename T>
+using MatchConsumer =
+    std::function<Expected<T>(const ast_matchers::MatchFinder::MatchResult &)>;
+
+/// Creates an error that signals that a `MatchConsumer` expected a certain node
+/// to be bound by AST matchers, but it was not actually bound.
+inline llvm::Error notBoundError(llvm::StringRef Id) {
+  return llvm::make_error<llvm::StringError>(llvm::errc::invalid_argument,
+                                             "Id not bound: " + Id);
+}
+
+/// Chooses between the two consumers, based on whether \p ID is bound in the
+/// match.
+template <typename T>
+MatchConsumer<T> ifBound(std::string ID, MatchConsumer<T> TrueC,
+                         MatchConsumer<T> FalseC) {
+  return [=](const ast_matchers::MatchFinder::MatchResult &Result) {
+    auto &Map = Result.Nodes.getMap();
+    return (Map.find(ID) != Map.end() ? TrueC : FalseC)(Result);
+  };
+}
+
+} // namespace tooling
+} // namespace clang
+#endif // LLVM_CLANG_TOOLING_REFACTOR_MATCH_CONSUMER_H_
diff --git a/clang/include/clang/Tooling/Refactoring/RangeSelector.h b/clang/include/clang/Tooling/Refactoring/RangeSelector.h
index b117e4d82ad46..d5b5c8fbd8a5b 100644
--- a/clang/include/clang/Tooling/Refactoring/RangeSelector.h
+++ b/clang/include/clang/Tooling/Refactoring/RangeSelector.h
@@ -17,14 +17,14 @@
 
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Tooling/Refactoring/MatchConsumer.h"
 #include "llvm/Support/Error.h"
 #include <functional>
 #include <string>
 
 namespace clang {
 namespace tooling {
-using RangeSelector = std::function<Expected<CharSourceRange>(
-    const ast_matchers::MatchFinder::MatchResult &)>;
+using RangeSelector = MatchConsumer<CharSourceRange>;
 
 inline RangeSelector charRange(CharSourceRange R) {
   return [R](const ast_matchers::MatchFinder::MatchResult &)
@@ -79,6 +79,10 @@ RangeSelector statements(std::string ID);
 // (all source between the braces).
 RangeSelector initListElements(std::string ID);
 
+/// Given an \IfStmt (bound to \p ID), selects the range of the else branch,
+/// starting from the \c else keyword.
+RangeSelector elseBranch(std::string ID);
+
 /// Selects the range from which `S` was expanded (possibly along with other
 /// source), if `S` is an expansion, and `S` itself, otherwise.  Corresponds to
 /// `SourceManager::getExpansionRange`.
diff --git a/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h b/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
index 41a448f035a40..c0f995d85c14c 100644
--- a/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
+++ b/clang/include/clang/Tooling/Refactoring/RecursiveSymbolVisitor.h
@@ -98,7 +98,17 @@ class RecursiveSymbolVisitor
                  TypeBeginLoc, TypeEndLoc))
         return false;
     }
-    return visit(Loc.getType()->getAsCXXRecordDecl(), TypeBeginLoc, TypeEndLoc);
+    if (const Type *TP = Loc.getTypePtr()) {
+      if (TP->getTypeClass() == clang::Type::Record)
+        return visit(TP->getAsCXXRecordDecl(), TypeBeginLoc, TypeEndLoc);
+    }
+    return true;
+  }
+
+  bool VisitTypedefTypeLoc(TypedefTypeLoc TL) {
+    const SourceLocation TypeEndLoc =
+        Lexer::getLocForEndOfToken(TL.getBeginLoc(), 0, SM, LangOpts);
+    return visit(TL.getTypedefNameDecl(), TL.getBeginLoc(), TypeEndLoc);
   }
 
   bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) {
@@ -122,8 +132,7 @@ class RecursiveSymbolVisitor
         ND, SourceRange(BeginLoc, EndLoc));
   }
   bool visit(const NamedDecl *ND, SourceLocation Loc) {
-    return visit(ND, Loc,
-                 Loc.getLocWithOffset(ND->getNameAsString().length() - 1));
+    return visit(ND, Loc, Lexer::getLocForEndOfToken(Loc, 0, SM, LangOpts));
   }
 };
 
diff --git a/clang/include/clang/Tooling/Refactoring/SourceCodeBuilders.h b/clang/include/clang/Tooling/Refactoring/SourceCodeBuilders.h
new file mode 100644
index 0000000000000..797046f3ecffb
--- /dev/null
+++ b/clang/include/clang/Tooling/Refactoring/SourceCodeBuilders.h
@@ -0,0 +1,86 @@
+//===--- SourceCodeBuilders.h - Source-code building facilities -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file collects facilities for generating source code strings.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_BUILDERS_H_
+#define LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_BUILDERS_H_
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include <string>
+
+namespace clang {
+namespace tooling {
+
+/// \name Code analysis utilities.
+/// @{
+/// Ignores implicit object-construction expressions in addition to the normal
+/// implicit expressions that are ignored.
+const Expr *reallyIgnoreImplicit(const Expr &E);
+
+/// Determines whether printing this expression in *any* expression requires
+/// parentheses to preserve its meaning. This analyses is necessarily
+/// conservative because it lacks information about the target context.
+bool mayEverNeedParens(const Expr &E);
+
+/// Determines whether printing this expression to the left of a dot or arrow
+/// operator requires a parentheses to preserve its meaning. Given that
+/// dot/arrow are (effectively) the highest precedence, this is equivalent to
+/// asking whether it ever needs parens.
+inline bool needParensBeforeDotOrArrow(const Expr &E) {
+  return mayEverNeedParens(E);
+}
+
+/// Determines whether printing this expression to the right of a unary operator
+/// requires a parentheses to preserve its meaning.
+bool needParensAfterUnaryOperator(const Expr &E);
+/// @}
+
+/// \name Basic code-string generation utilities.
+/// @{
+
+/// Builds source for an expression, adding parens if needed for unambiguous
+/// parsing.
+llvm::Optional<std::string> buildParens(const Expr &E,
+                                        const ASTContext &Context);
+
+/// Builds idiomatic source for the dereferencing of `E`: prefix with `*` but
+/// simplify when it already begins with `&`.  \returns empty string on failure.
+llvm::Optional<std::string> buildDereference(const Expr &E,
+                                             const ASTContext &Context);
+
+/// Builds idiomatic source for taking the address of `E`: prefix with `&` but
+/// simplify when it already begins with `*`.  \returns empty string on failure.
+llvm::Optional<std::string> buildAddressOf(const Expr &E,
+                                           const ASTContext &Context);
+
+/// Adds a dot to the end of the given expression, but adds parentheses when
+/// needed by the syntax, and simplifies to `->` when possible, e.g.:
+///
+///  `x` becomes `x.`
+///  `*a` becomes `a->`
+///  `a+b` becomes `(a+b).`
+llvm::Optional<std::string> buildDot(const Expr &E, const ASTContext &Context);
+
+/// Adds an arrow to the end of the given expression, but adds parentheses
+/// when needed by the syntax, and simplifies to `.` when possible, e.g.:
+///
+///  `x` becomes `x->`
+///  `&a` becomes `a.`
+///  `a+b` becomes `(a+b)->`
+llvm::Optional<std::string> buildArrow(const Expr &E,
+                                       const ASTContext &Context);
+/// @}
+
+} // namespace tooling
+} // namespace clang
+#endif // LLVM_CLANG_TOOLING_REFACTOR_SOURCE_CODE_BUILDERS_H_
diff --git a/clang/include/clang/Tooling/Refactoring/Stencil.h b/clang/include/clang/Tooling/Refactoring/Stencil.h
index e57a576e55755..3bd66e578b857 100644
--- a/clang/include/clang/Tooling/Refactoring/Stencil.h
+++ b/clang/include/clang/Tooling/Refactoring/Stencil.h
@@ -23,6 +23,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTTypeTraits.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Tooling/Refactoring/MatchConsumer.h"
 #include "clang/Tooling/Refactoring/RangeSelector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
@@ -153,16 +154,32 @@ inline StencilPart node(llvm::StringRef Id) {
   return selection(tooling::node(Id));
 }
 
-/// Variant of \c node() that identifies the node as a statement, for purposes
-/// of deciding whether to include any trailing semicolon.  Only relevant for
-/// Expr nodes, which, by default, are *not* considered as statements.
-/// \returns the source corresponding to the identified node, considered as a
-/// statement.
-/// FIXME: Deprecated. Write `selection(statement(Id))` instead.
-inline StencilPart sNode(llvm::StringRef Id) {
-  return selection(tooling::statement(Id));
+/// Constructs a `MemberExpr` that accesses the named member (\p Member) of the
+/// object bound to \p BaseId. The access is constructed idiomatically: if \p
+/// BaseId is bound to `e` and \p Member identifies member `m`, then returns
+/// `e->m`, when e is a pointer, `e2->m` when e = `*e2` and `e.m` otherwise.
+/// Additionally, `e` is wrapped in parentheses, if needed.
+StencilPart access(llvm::StringRef BaseId, StencilPart Member);
+inline StencilPart access(llvm::StringRef BaseId, llvm::StringRef Member) {
+  return access(BaseId, text(Member));
 }
 
+/// Chooses between the two stencil parts, based on whether \p ID is bound in
+/// the match.
+StencilPart ifBound(llvm::StringRef Id, StencilPart TruePart,
+                    StencilPart FalsePart);
+
+/// Chooses between the two strings, based on whether \p ID is bound in the
+/// match.
+inline StencilPart ifBound(llvm::StringRef Id, llvm::StringRef TrueText,
+                           llvm::StringRef FalseText) {
+  return ifBound(Id, text(TrueText), text(FalseText));
+}
+
+/// Wraps a MatchConsumer in a StencilPart, so that it can be used in a Stencil.
+/// This supports user-defined extensions to the Stencil language.
+StencilPart run(MatchConsumer<std::string> C);
+
 /// For debug use only; semantics are not guaranteed.
 ///
 /// \returns the string resulting from calling the node's print() method.
diff --git a/clang/include/clang/Tooling/Refactoring/Transformer.h b/clang/include/clang/Tooling/Refactoring/Transformer.h
index d5d4adf1f0e89..0971cc3e66793 100644
--- a/clang/include/clang/Tooling/Refactoring/Transformer.h
+++ b/clang/include/clang/Tooling/Refactoring/Transformer.h
@@ -19,6 +19,7 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/ASTMatchers/ASTMatchersInternal.h"
 #include "clang/Tooling/Refactoring/AtomicChange.h"
+#include "clang/Tooling/Refactoring/MatchConsumer.h"
 #include "clang/Tooling/Refactoring/RangeSelector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
@@ -32,11 +33,7 @@
 namespace clang {
 namespace tooling {
 
-// Note that \p TextGenerator is allowed to fail, e.g. when trying to access a
-// matched node that was not bound.  Allowing this to fail simplifies error
-// handling for interactive tools like clang-query.
-using TextGenerator = std::function<Expected<std::string>(
-    const ast_matchers::MatchFinder::MatchResult &)>;
+using TextGenerator = MatchConsumer<std::string>;
 
 /// Wraps a string as a TextGenerator.
 inline TextGenerator text(std::string M) {
@@ -254,6 +251,12 @@ ast_matchers::internal::DynTypedMatcher buildMatcher(const RewriteRule &Rule);
 std::vector<ast_matchers::internal::DynTypedMatcher>
 buildMatchers(const RewriteRule &Rule);
 
+/// Gets the beginning location of the source matched by a rewrite rule. If the
+/// match occurs within a macro expansion, returns the beginning of the
+/// expansion point. `Result` must come from the matching of a rewrite rule.
+SourceLocation
+getRuleMatchLoc(const ast_matchers::MatchFinder::MatchResult &Result);
+
 /// Returns the \c Case of \c Rule that was selected in the match result.
 /// Assumes a matcher built with \c buildMatcher.
 const RewriteRule::Case &
diff --git a/clang/include/clang/module.modulemap b/clang/include/clang/module.modulemap
index fc4a2de2f99e3..f7bf482fbbeaf 100644
--- a/clang/include/clang/module.modulemap
+++ b/clang/include/clang/module.modulemap
@@ -20,7 +20,6 @@ module Clang_AST {
   textual header "AST/BuiltinTypes.def"
   textual header "AST/OperationKinds.def"
   textual header "AST/TypeLocNodes.def"
-  textual header "AST/TypeNodes.def"
 
   module * { export * }
 }
diff --git a/clang/lib/AST/APValue.cpp b/clang/lib/AST/APValue.cpp
index 1993bba9bd1a1..f29d6f5554c54 100644
--- a/clang/lib/AST/APValue.cpp
+++ b/clang/lib/AST/APValue.cpp
@@ -42,6 +42,14 @@ APValue::LValueBase::LValueBase(const ValueDecl *P, unsigned I, unsigned V)
 APValue::LValueBase::LValueBase(const Expr *P, unsigned I, unsigned V)
     : Ptr(P), Local{I, V} {}
 
+APValue::LValueBase APValue::LValueBase::getDynamicAlloc(DynamicAllocLValue LV,
+                                                         QualType Type) {
+  LValueBase Base;
+  Base.Ptr = LV;
+  Base.DynamicAllocType = Type.getAsOpaquePtr();
+  return Base;
+}
+
 APValue::LValueBase APValue::LValueBase::getTypeInfo(TypeInfoLValue LV,
                                                      QualType TypeInfo) {
   LValueBase Base;
@@ -51,11 +59,12 @@ APValue::LValueBase APValue::LValueBase::getTypeInfo(TypeInfoLValue LV,
 }
 
 unsigned APValue::LValueBase::getCallIndex() const {
-  return is<TypeInfoLValue>() ? 0 : Local.CallIndex;
+  return (is<TypeInfoLValue>() || is<DynamicAllocLValue>()) ? 0
+                                                            : Local.CallIndex;
 }
 
 unsigned APValue::LValueBase::getVersion() const {
-  return is<TypeInfoLValue>() ? 0 : Local.Version;
+  return (is<TypeInfoLValue>() || is<DynamicAllocLValue>()) ? 0 : Local.Version;
 }
 
 QualType APValue::LValueBase::getTypeInfoType() const {
@@ -63,6 +72,11 @@ QualType APValue::LValueBase::getTypeInfoType() const {
   return QualType::getFromOpaquePtr(TypeInfoType);
 }
 
+QualType APValue::LValueBase::getDynamicAllocType() const {
+  assert(is<DynamicAllocLValue>() && "not a dynamic allocation lvalue");
+  return QualType::getFromOpaquePtr(DynamicAllocType);
+}
+
 namespace clang {
 bool operator==(const APValue::LValueBase &LHS,
                 const APValue::LValueBase &RHS) {
@@ -111,7 +125,7 @@ llvm::DenseMapInfo<clang::APValue::LValueBase>::getTombstoneKey() {
 
 namespace clang {
 llvm::hash_code hash_value(const APValue::LValueBase &Base) {
-  if (Base.is<TypeInfoLValue>())
+  if (Base.is<TypeInfoLValue>() || Base.is<DynamicAllocLValue>())
     return llvm::hash_value(Base.getOpaqueValue());
   return llvm::hash_combine(Base.getOpaqueValue(), Base.getCallIndex(),
                             Base.getVersion());
@@ -479,7 +493,7 @@ void APValue::printPretty(raw_ostream &Out, const ASTContext &Ctx,
     return;
   case APValue::Vector: {
     Out << '{';
-    QualType ElemTy = Ty->getAs<VectorType>()->getElementType();
+    QualType ElemTy = Ty->castAs<VectorType>()->getElementType();
     getVectorElt(0).printPretty(Out, Ctx, ElemTy);
     for (unsigned i = 1; i != getVectorLength(); ++i) {
       Out << ", ";
@@ -528,13 +542,18 @@ void APValue::printPretty(raw_ostream &Out, const ASTContext &Ctx,
           S = CharUnits::One();
         }
         Out << '&';
-      } else if (!IsReference)
+      } else if (!IsReference) {
         Out << '&';
+      }
 
       if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>())
         Out << *VD;
       else if (TypeInfoLValue TI = Base.dyn_cast<TypeInfoLValue>()) {
         TI.print(Out, Ctx.getPrintingPolicy());
+      } else if (DynamicAllocLValue DA = Base.dyn_cast<DynamicAllocLValue>()) {
+        Out << "{*new "
+            << Base.getDynamicAllocType().stream(Ctx.getPrintingPolicy()) << "#"
+            << DA.getIndex() << "}";
       } else {
         assert(Base.get<const Expr *>() != nullptr &&
                "Expecting non-null Expr");
@@ -563,10 +582,17 @@ void APValue::printPretty(raw_ostream &Out, const ASTContext &Ctx,
     } else if (TypeInfoLValue TI = Base.dyn_cast<TypeInfoLValue>()) {
       TI.print(Out, Ctx.getPrintingPolicy());
       ElemTy = Base.getTypeInfoType();
+    } else if (DynamicAllocLValue DA = Base.dyn_cast<DynamicAllocLValue>()) {
+      Out << "{*new "
+          << Base.getDynamicAllocType().stream(Ctx.getPrintingPolicy()) << "#"
+          << DA.getIndex() << "}";
+      ElemTy = Base.getDynamicAllocType();
     } else {
       const Expr *E = Base.get<const Expr*>();
       assert(E != nullptr && "Expecting non-null Expr");
       E->printPretty(Out, nullptr, Ctx.getPrintingPolicy());
+      // FIXME: This is wrong if E is a MaterializeTemporaryExpr with an lvalue
+      // adjustment.
       ElemTy = E->getType();
     }
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 588f12fd963fb..2443447d3f3ea 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -982,7 +982,7 @@ void ASTContext::PrintStats() const {
   unsigned counts[] = {
 #define TYPE(Name, Parent) 0,
 #define ABSTRACT_TYPE(Name, Parent)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     0 // Extra
   };
 
@@ -1002,7 +1002,7 @@ void ASTContext::PrintStats() const {
   TotalBytes += counts[Idx] * sizeof(Name##Type);                       \
   ++Idx;
 #define ABSTRACT_TYPE(Name, Parent)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
   llvm::errs() << "Total bytes = " << TotalBytes << "\n";
 
@@ -1814,7 +1814,7 @@ TypeInfo ASTContext::getTypeInfoImpl(const Type *T) const {
   case Type::Class:                                                            \
   assert(!T->isDependentType() && "should not see dependent types here");      \
   return getTypeInfo(cast<Class##Type>(T)->desugar().getTypePtr());
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Should not see dependent types");
 
   case Type::FunctionNoProto:
@@ -2448,7 +2448,7 @@ structHasUniqueObjectRepresentations(const ASTContext &Context,
       // have tail padding, so just make sure there isn't an error.
       if (!isStructEmpty(Base.getType())) {
         llvm::Optional<int64_t> Size = structHasUniqueObjectRepresentations(
-            Context, Base.getType()->getAs<RecordType>()->getDecl());
+            Context, Base.getType()->castAs<RecordType>()->getDecl());
         if (!Size)
           return llvm::None;
         Bases.emplace_back(Base.getType(), Size.getValue());
@@ -2539,7 +2539,7 @@ bool ASTContext::hasUniqueObjectRepresentations(QualType Ty) const {
   }
 
   if (Ty->isRecordType()) {
-    const RecordDecl *Record = Ty->getAs<RecordType>()->getDecl();
+    const RecordDecl *Record = Ty->castAs<RecordType>()->getDecl();
 
     if (Record->isInvalidDecl())
       return false;
@@ -3227,7 +3227,7 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const {
 #define TYPE(Class, Base)
 #define ABSTRACT_TYPE(Class, Base)
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("didn't desugar past all non-canonical types?");
 
   // These types should never be variably-modified.
@@ -5684,7 +5684,7 @@ static FloatingRank getFloatingRank(QualType T) {
     return getFloatingRank(CT->getElementType());
 
   assert(T->getAs<BuiltinType>() && "getFloatingRank(): not a floating type");
-  switch (T->getAs<BuiltinType>()->getKind()) {
+  switch (T->castAs<BuiltinType>()->getKind()) {
   default: llvm_unreachable("getFloatingRank(): not a floating type");
   case BuiltinType::Float16:    return Float16Rank;
   case BuiltinType::Half:       return HalfRank;
@@ -6322,14 +6322,14 @@ std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
 
   const BlockDecl *Decl = Expr->getBlockDecl();
   QualType BlockTy =
-      Expr->getType()->getAs<BlockPointerType>()->getPointeeType();
+      Expr->getType()->castAs<BlockPointerType>()->getPointeeType();
+  QualType BlockReturnTy = BlockTy->castAs<FunctionType>()->getReturnType();
   // Encode result type.
   if (getLangOpts().EncodeExtendedBlockSig)
-    getObjCEncodingForMethodParameter(
-        Decl::OBJC_TQ_None, BlockTy->getAs<FunctionType>()->getReturnType(), S,
-        true /*Extended*/);
+    getObjCEncodingForMethodParameter(Decl::OBJC_TQ_None, BlockReturnTy, S,
+                                      true /*Extended*/);
   else
-    getObjCEncodingForType(BlockTy->getAs<FunctionType>()->getReturnType(), S);
+    getObjCEncodingForType(BlockReturnTy, S);
   // Compute size of all parameters.
   // Start with computing size of a pointer in number of bytes.
   // FIXME: There might(should) be a better way of doing this computation!
@@ -7129,7 +7129,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
   case Type::KIND:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(KIND, BASE) \
   case Type::KIND:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("@encode for dependent type!");
   }
   llvm_unreachable("bad type kind!");
@@ -7912,7 +7912,7 @@ Qualifiers::GC ASTContext::getObjCGCAttrKind(QualType Ty) const {
     if (Ty->isObjCObjectPointerType() || Ty->isBlockPointerType())
       return Qualifiers::Strong;
     else if (Ty->isPointerType())
-      return getObjCGCAttrKind(Ty->getAs<PointerType>()->getPointeeType());
+      return getObjCGCAttrKind(Ty->castAs<PointerType>()->getPointeeType());
   } else {
     // It's not valid to set GC attributes on anything that isn't a
     // pointer.
@@ -7949,8 +7949,8 @@ bool ASTContext::areCompatibleVectorTypes(QualType FirstVec,
 
   // Treat Neon vector types and most AltiVec vector types as if they are the
   // equivalent GCC vector types.
-  const auto *First = FirstVec->getAs<VectorType>();
-  const auto *Second = SecondVec->getAs<VectorType>();
+  const auto *First = FirstVec->castAs<VectorType>();
+  const auto *Second = SecondVec->castAs<VectorType>();
   if (First->getNumElements() == Second->getNumElements() &&
       hasSameType(First->getElementType(), Second->getElementType()) &&
       First->getVectorKind() != VectorType::AltiVecPixel &&
@@ -8003,15 +8003,11 @@ ASTContext::ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
 
 /// ObjCQualifiedClassTypesAreCompatible - compare  Class<pr,...> and
 /// Class<pr1, ...>.
-bool ASTContext::ObjCQualifiedClassTypesAreCompatible(QualType lhs,
-                                                      QualType rhs) {
-  const auto *lhsQID = lhs->getAs<ObjCObjectPointerType>();
-  const auto *rhsOPT = rhs->getAs<ObjCObjectPointerType>();
-  assert((lhsQID && rhsOPT) && "ObjCQualifiedClassTypesAreCompatible");
-
-  for (auto *lhsProto : lhsQID->quals()) {
+bool ASTContext::ObjCQualifiedClassTypesAreCompatible(
+    const ObjCObjectPointerType *lhs, const ObjCObjectPointerType *rhs) {
+  for (auto *lhsProto : lhs->quals()) {
     bool match = false;
-    for (auto *rhsProto : rhsOPT->quals()) {
+    for (auto *rhsProto : rhs->quals()) {
       if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto)) {
         match = true;
         break;
@@ -8025,8 +8021,9 @@ bool ASTContext::ObjCQualifiedClassTypesAreCompatible(QualType lhs,
 
 /// ObjCQualifiedIdTypesAreCompatible - We know that one of lhs/rhs is an
 /// ObjCQualifiedIDType.
-bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
-                                                   bool compare) {
+bool ASTContext::ObjCQualifiedIdTypesAreCompatible(
+    const ObjCObjectPointerType *lhs, const ObjCObjectPointerType *rhs,
+    bool compare) {
   // Allow id<P..> and an 'id' or void* type in all cases.
   if (lhs->isVoidPointerType() ||
       lhs->isObjCIdType() || lhs->isObjCClassType())
@@ -8035,16 +8032,12 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
            rhs->isObjCIdType() || rhs->isObjCClassType())
     return true;
 
-  if (const ObjCObjectPointerType *lhsQID = lhs->getAsObjCQualifiedIdType()) {
-    const auto *rhsOPT = rhs->getAs<ObjCObjectPointerType>();
-
-    if (!rhsOPT) return false;
-
-    if (rhsOPT->qual_empty()) {
+  if (lhs->isObjCQualifiedIdType()) {
+    if (rhs->qual_empty()) {
       // If the RHS is a unqualified interface pointer "NSString*",
       // make sure we check the class hierarchy.
-      if (ObjCInterfaceDecl *rhsID = rhsOPT->getInterfaceDecl()) {
-        for (auto *I : lhsQID->quals()) {
+      if (ObjCInterfaceDecl *rhsID = rhs->getInterfaceDecl()) {
+        for (auto *I : lhs->quals()) {
           // when comparing an id<P> on lhs with a static type on rhs,
           // see if static class implements all of id's protocols, directly or
           // through its super class and categories.
@@ -8056,13 +8049,13 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
       return true;
     }
     // Both the right and left sides have qualifiers.
-    for (auto *lhsProto : lhsQID->quals()) {
+    for (auto *lhsProto : lhs->quals()) {
       bool match = false;
 
       // when comparing an id<P> on lhs with a static type on rhs,
       // see if static class implements all of id's protocols, directly or
       // through its super class and categories.
-      for (auto *rhsProto : rhsOPT->quals()) {
+      for (auto *rhsProto : rhs->quals()) {
         if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
             (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
           match = true;
@@ -8071,8 +8064,8 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
       }
       // If the RHS is a qualified interface pointer "NSString<P>*",
       // make sure we check the class hierarchy.
-      if (ObjCInterfaceDecl *rhsID = rhsOPT->getInterfaceDecl()) {
-        for (auto *I : lhsQID->quals()) {
+      if (ObjCInterfaceDecl *rhsID = rhs->getInterfaceDecl()) {
+        for (auto *I : lhs->quals()) {
           // when comparing an id<P> on lhs with a static type on rhs,
           // see if static class implements all of id's protocols, directly or
           // through its super class and categories.
@@ -8089,13 +8082,11 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
     return true;
   }
 
-  const ObjCObjectPointerType *rhsQID = rhs->getAsObjCQualifiedIdType();
-  assert(rhsQID && "One of the LHS/RHS should be id<x>");
+  assert(rhs->isObjCQualifiedIdType() && "One of the LHS/RHS should be id<x>");
 
-  if (const ObjCObjectPointerType *lhsOPT =
-        lhs->getAsObjCInterfacePointerType()) {
+  if (lhs->getInterfaceType()) {
     // If both the right and left sides have qualifiers.
-    for (auto *lhsProto : lhsOPT->quals()) {
+    for (auto *lhsProto : lhs->quals()) {
       bool match = false;
 
       // when comparing an id<P> on rhs with a static type on lhs,
@@ -8103,7 +8094,7 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
       // through its super class and categories.
       // First, lhs protocols in the qualifier list must be found, direct
       // or indirect in rhs's qualifier list or it is a mismatch.
-      for (auto *rhsProto : rhsQID->quals()) {
+      for (auto *rhsProto : rhs->quals()) {
         if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
             (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
           match = true;
@@ -8116,17 +8107,17 @@ bool ASTContext::ObjCQualifiedIdTypesAreCompatible(QualType lhs, QualType rhs,
 
     // Static class's protocols, or its super class or category protocols
     // must be found, direct or indirect in rhs's qualifier list or it is a mismatch.
-    if (ObjCInterfaceDecl *lhsID = lhsOPT->getInterfaceDecl()) {
+    if (ObjCInterfaceDecl *lhsID = lhs->getInterfaceDecl()) {
       llvm::SmallPtrSet<ObjCProtocolDecl *, 8> LHSInheritedProtocols;
       CollectInheritedProtocols(lhsID, LHSInheritedProtocols);
       // This is rather dubious but matches gcc's behavior. If lhs has
       // no type qualifier and its class has no static protocol(s)
       // assume that it is mismatch.
-      if (LHSInheritedProtocols.empty() && lhsOPT->qual_empty())
+      if (LHSInheritedProtocols.empty() && lhs->qual_empty())
         return false;
       for (auto *lhsProto : LHSInheritedProtocols) {
         bool match = false;
-        for (auto *rhsProto : rhsQID->quals()) {
+        for (auto *rhsProto : rhs->quals()) {
           if (ProtocolCompatibleWithProtocol(lhsProto, rhsProto) ||
               (compare && ProtocolCompatibleWithProtocol(rhsProto, lhsProto))) {
             match = true;
@@ -8171,14 +8162,11 @@ bool ASTContext::canAssignObjCInterfaces(const ObjCObjectPointerType *LHSOPT,
   };
 
   if (LHS->isObjCQualifiedId() || RHS->isObjCQualifiedId()) {
-    return finish(ObjCQualifiedIdTypesAreCompatible(QualType(LHSOPT,0),
-                                                    QualType(RHSOPT,0),
-                                                    false));
+    return finish(ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT, false));
   }
 
   if (LHS->isObjCQualifiedClass() && RHS->isObjCQualifiedClass()) {
-    return finish(ObjCQualifiedClassTypesAreCompatible(QualType(LHSOPT,0),
-                                                       QualType(RHSOPT,0)));
+    return finish(ObjCQualifiedClassTypesAreCompatible(LHSOPT, RHSOPT));
   }
 
   // If we have 2 user-defined types, fall into that path.
@@ -8227,8 +8215,8 @@ bool ASTContext::canAssignObjCInterfacesInBlockPointer(
 
   if (LHSOPT->isObjCQualifiedIdType() || RHSOPT->isObjCQualifiedIdType())
     return finish(ObjCQualifiedIdTypesAreCompatible(
-        QualType(BlockReturnType ? LHSOPT : RHSOPT, 0),
-        QualType(BlockReturnType ? RHSOPT : LHSOPT, 0), false));
+        (BlockReturnType ? LHSOPT : RHSOPT),
+        (BlockReturnType ? RHSOPT : LHSOPT), false));
 
   const ObjCInterfaceType* LHS = LHSOPT->getInterfaceType();
   const ObjCInterfaceType* RHS = RHSOPT->getInterfaceType();
@@ -8952,7 +8940,7 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   case Type::Auto:
@@ -8972,8 +8960,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
   case Type::Pointer:
   {
     // Merge two pointer types, while trying to preserve typedef info
-    QualType LHSPointee = LHS->getAs<PointerType>()->getPointeeType();
-    QualType RHSPointee = RHS->getAs<PointerType>()->getPointeeType();
+    QualType LHSPointee = LHS->castAs<PointerType>()->getPointeeType();
+    QualType RHSPointee = RHS->castAs<PointerType>()->getPointeeType();
     if (Unqualified) {
       LHSPointee = LHSPointee.getUnqualifiedType();
       RHSPointee = RHSPointee.getUnqualifiedType();
@@ -8991,8 +8979,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
   case Type::BlockPointer:
   {
     // Merge two block pointer types, while trying to preserve typedef info
-    QualType LHSPointee = LHS->getAs<BlockPointerType>()->getPointeeType();
-    QualType RHSPointee = RHS->getAs<BlockPointerType>()->getPointeeType();
+    QualType LHSPointee = LHS->castAs<BlockPointerType>()->getPointeeType();
+    QualType RHSPointee = RHS->castAs<BlockPointerType>()->getPointeeType();
     if (Unqualified) {
       LHSPointee = LHSPointee.getUnqualifiedType();
       RHSPointee = RHSPointee.getUnqualifiedType();
@@ -9024,8 +9012,8 @@ QualType ASTContext::mergeTypes(QualType LHS, QualType RHS,
   case Type::Atomic:
   {
     // Merge two pointer types, while trying to preserve typedef info
-    QualType LHSValue = LHS->getAs<AtomicType>()->getValueType();
-    QualType RHSValue = RHS->getAs<AtomicType>()->getValueType();
+    QualType LHSValue = LHS->castAs<AtomicType>()->getValueType();
+    QualType RHSValue = RHS->castAs<AtomicType>()->getValueType();
     if (Unqualified) {
       LHSValue = LHSValue.getUnqualifiedType();
       RHSValue = RHSValue.getUnqualifiedType();
@@ -9284,8 +9272,8 @@ QualType ASTContext::mergeObjCGCQualifiers(QualType LHS, QualType RHS) {
   }
 
   if (LHSCan->isObjCObjectPointerType() && RHSCan->isObjCObjectPointerType()) {
-    QualType LHSBaseQT = LHS->getAs<ObjCObjectPointerType>()->getPointeeType();
-    QualType RHSBaseQT = RHS->getAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType LHSBaseQT = LHS->castAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType RHSBaseQT = RHS->castAs<ObjCObjectPointerType>()->getPointeeType();
     QualType ResQT = mergeObjCGCQualifiers(LHSBaseQT, RHSBaseQT);
     if (ResQT == LHSBaseQT)
       return LHS;
@@ -9936,25 +9924,10 @@ static GVALinkage basicGVALinkageForVariable(const ASTContext &Context,
     return StrongLinkage;
 
   case TSK_ExplicitSpecialization:
-    if (Context.getTargetInfo().getCXXABI().isMicrosoft()) {
-      // If this is a fully specialized constexpr variable template, pretend it
-      // was marked inline. MSVC 14.21.27702 headers define _Is_integral in a
-      // header this way, and we don't want to emit non-discardable definitions
-      // of these variables in every TU that includes <type_traits>. This
-      // behavior is non-conforming, since another TU could use an extern
-      // template declaration for this variable, but for constexpr variables,
-      // it's unlikely for a user to want to do that. This behavior can be
-      // removed if the headers change to explicitly mark such variable template
-      // specializations inline.
-      if (isa<VarTemplateSpecializationDecl>(VD) && VD->isConstexpr())
-        return GVA_DiscardableODR;
-
-      // Use ODR linkage for static data members of fully specialized templates
-      // to prevent duplicate definition errors with MSVC.
-      if (VD->isStaticDataMember())
-        return GVA_StrongODR;
-    }
-    return StrongLinkage;
+    return Context.getTargetInfo().getCXXABI().isMicrosoft() &&
+                   VD->isStaticDataMember()
+               ? GVA_StrongODR
+               : StrongLinkage;
 
   case TSK_ExplicitInstantiationDefinition:
     return GVA_StrongODR;
@@ -10100,7 +10073,7 @@ bool ASTContext::DeclMustBeEmitted(const Decl *D) {
     return false;
 
   // Variables that have destruction with side-effects are required.
-  if (VD->getType().isDestructedType())
+  if (VD->needsDestruction(*this))
     return true;
 
   // Variables that have initialization with side-effects are required.
@@ -10583,8 +10556,7 @@ QualType ASTContext::getCorrespondingSaturatedType(QualType Ty) const {
 
   if (Ty->isSaturatedFixedPointType()) return Ty;
 
-  const auto &BT = Ty->getAs<BuiltinType>();
-  switch (BT->getKind()) {
+  switch (Ty->castAs<BuiltinType>()->getKind()) {
     default:
       llvm_unreachable("Not a fixed point type!");
     case BuiltinType::ShortAccum:
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index 15df865852941..30985441031de 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -154,7 +154,7 @@ Underlying = CTy->desugar(); \
 } \
 break; \
 }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
 
     // If it wasn't sugared, we're done.
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 7be3502108341..057444ac99290 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -3244,7 +3244,7 @@ ExpectedDecl ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
     if (GetImportedOrCreateDecl<CXXDestructorDecl>(
         ToFunction, D, Importer.getToContext(), cast<CXXRecordDecl>(DC),
         ToInnerLocStart, NameInfo, T, TInfo, D->isInlineSpecified(),
-        D->isImplicit()))
+        D->isImplicit(), D->getConstexprKind()))
       return ToFunction;
 
     CXXDestructorDecl *ToDtor = cast<CXXDestructorDecl>(ToFunction);
diff --git a/clang/lib/AST/ASTTypeTraits.cpp b/clang/lib/AST/ASTTypeTraits.cpp
index 8615dee4e7958..6b7f6ec51086f 100644
--- a/clang/lib/AST/ASTTypeTraits.cpp
+++ b/clang/lib/AST/ASTTypeTraits.cpp
@@ -37,7 +37,7 @@ const ASTNodeKind::KindInfo ASTNodeKind::AllKindInfo[] = {
 #include "clang/AST/StmtNodes.inc"
   { NKI_None, "Type" },
 #define TYPE(DERIVED, BASE) { NKI_##BASE, #DERIVED "Type" },
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   { NKI_None, "OMPClause" },
 #define OPENMP_CLAUSE(TextualSpelling, Class) {NKI_OMPClause, #Class},
 #include "clang/Basic/OpenMPKinds.def"
@@ -104,7 +104,7 @@ ASTNodeKind ASTNodeKind::getFromNode(const Type &T) {
 #define TYPE(Class, Base)                                                      \
     case Type::Class: return ASTNodeKind(NKI_##Class##Type);
 #define ABSTRACT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
   llvm_unreachable("invalid type kind");
  }
@@ -117,6 +117,7 @@ ASTNodeKind ASTNodeKind::getFromNode(const OMPClause &C) {
   case OMPC_threadprivate:
   case OMPC_uniform:
   case OMPC_device_type:
+  case OMPC_match:
   case OMPC_unknown:
     llvm_unreachable("unexpected OpenMP clause kind");
   }
diff --git a/clang/lib/AST/CommentLexer.cpp b/clang/lib/AST/CommentLexer.cpp
index 19485f6018c02..c1ea3eab075e2 100644
--- a/clang/lib/AST/CommentLexer.cpp
+++ b/clang/lib/AST/CommentLexer.cpp
@@ -850,17 +850,14 @@ void Lexer::lex(Token &T) {
 }
 
 StringRef Lexer::getSpelling(const Token &Tok,
-                             const SourceManager &SourceMgr,
-                             bool *Invalid) const {
+                             const SourceManager &SourceMgr) const {
   SourceLocation Loc = Tok.getLocation();
   std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
 
   bool InvalidTemp = false;
   StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
-  if (InvalidTemp) {
-    *Invalid = true;
+  if (InvalidTemp)
     return StringRef();
-  }
 
   const char *Begin = File.data() + LocInfo.second;
   return StringRef(Begin, Tok.getLength());
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index b27ac907b8bc2..9ebf1c32629fe 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -1558,6 +1558,24 @@ void NamedDecl::printQualifiedName(raw_ostream &OS) const {
 
 void NamedDecl::printQualifiedName(raw_ostream &OS,
                                    const PrintingPolicy &P) const {
+  if (getDeclContext()->isFunctionOrMethod()) {
+    // We do not print '(anonymous)' for function parameters without name.
+    printName(OS);
+    return;
+  }
+  printNestedNameSpecifier(OS, P);
+  if (getDeclName() || isa<DecompositionDecl>(this))
+    OS << *this;
+  else
+    OS << "(anonymous)";
+}
+
+void NamedDecl::printNestedNameSpecifier(raw_ostream &OS) const {
+  printNestedNameSpecifier(OS, getASTContext().getPrintingPolicy());
+}
+
+void NamedDecl::printNestedNameSpecifier(raw_ostream &OS,
+                                         const PrintingPolicy &P) const {
   const DeclContext *Ctx = getDeclContext();
 
   // For ObjC methods and properties, look through categories and use the
@@ -1571,10 +1589,8 @@ void NamedDecl::printQualifiedName(raw_ostream &OS,
         Ctx = ID;
   }
 
-  if (Ctx->isFunctionOrMethod()) {
-    printName(OS);
+  if (Ctx->isFunctionOrMethod())
     return;
-  }
 
   using ContextsTy = SmallVector<const DeclContext *, 8>;
   ContextsTy Contexts;
@@ -1644,11 +1660,6 @@ void NamedDecl::printQualifiedName(raw_ostream &OS,
     }
     OS << "::";
   }
-
-  if (getDeclName() || isa<DecompositionDecl>(this))
-    OS << *this;
-  else
-    OS << "(anonymous)";
 }
 
 void NamedDecl::getNameForDiagnostic(raw_ostream &OS,
@@ -2581,6 +2592,18 @@ bool VarDecl::isNoDestroy(const ASTContext &Ctx) const {
                                  !hasAttr<AlwaysDestroyAttr>()));
 }
 
+QualType::DestructionKind
+VarDecl::needsDestruction(const ASTContext &Ctx) const {
+  if (EvaluatedStmt *Eval = Init.dyn_cast<EvaluatedStmt *>())
+    if (Eval->HasConstantDestruction)
+      return QualType::DK_none;
+
+  if (isNoDestroy(Ctx))
+    return QualType::DK_none;
+
+  return getType().isDestructedType();
+}
+
 MemberSpecializationInfo *VarDecl::getMemberSpecializationInfo() const {
   if (isStaticDataMember())
     // FIXME: Remove ?
@@ -2966,8 +2989,7 @@ bool FunctionDecl::isReplaceableGlobalAllocationFunction(bool *IsAligned) const
     Ty = Ty->getPointeeType();
     if (Ty.getCVRQualifiers() != Qualifiers::Const)
       return false;
-    const CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
-    if (RD && isNamed(RD, "nothrow_t") && RD->isInStdNamespace())
+    if (Ty->isNothrowT())
       Consume();
   }
 
@@ -3251,6 +3273,9 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
       return true;
   }
 
+  if (Context.getLangOpts().CPlusPlus)
+    return false;
+
   if (Context.getLangOpts().GNUInline || hasAttr<GNUInlineAttr>()) {
     // With GNU inlining, a declaration with 'inline' but not 'extern', forces
     // an externally visible definition.
@@ -3279,9 +3304,6 @@ bool FunctionDecl::doesDeclarationForceExternallyVisibleDefinition() const {
     return FoundBody;
   }
 
-  if (Context.getLangOpts().CPlusPlus)
-    return false;
-
   // C99 6.7.4p6:
   //   [...] If all of the file scope declarations for a function in a
   //   translation unit include the inline function specifier without extern,
@@ -3361,6 +3383,8 @@ bool FunctionDecl::isInlineDefinitionExternallyVisible() const {
     // If it's not the case that both 'inline' and 'extern' are
     // specified on the definition, then this inline definition is
     // externally visible.
+    if (Context.getLangOpts().CPlusPlus)
+      return false;
     if (!(isInlineSpecified() && getStorageClass() == SC_Extern))
       return true;
 
diff --git a/clang/lib/AST/DeclCXX.cpp b/clang/lib/AST/DeclCXX.cpp
index 4939c37edc7e8..95b14e494a3b5 100644
--- a/clang/lib/AST/DeclCXX.cpp
+++ b/clang/lib/AST/DeclCXX.cpp
@@ -95,6 +95,7 @@ CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
       HasDefaultedDefaultConstructor(false),
       DefaultedDefaultConstructorIsConstexpr(true),
       HasConstexprDefaultConstructor(false),
+      DefaultedDestructorIsConstexpr(true),
       HasNonLiteralTypeFieldsOrBases(false), ComputedVisibleConversions(false),
       UserProvidedDefaultConstructor(false), DeclaredSpecialMembers(0),
       ImplicitCopyConstructorCanHaveConstParamForVBase(true),
@@ -325,10 +326,12 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
       data().IsStandardLayout = false;
       data().IsCXX11StandardLayout = false;
 
-      // C++11 [dcl.constexpr]p4:
-      //   In the definition of a constexpr constructor [...]
-      //    -- the class shall not have any virtual base classes
+      // C++20 [dcl.constexpr]p3:
+      //   In the definition of a constexpr function [...]
+      //    -- if the function is a constructor or destructor,
+      //       its class shall not have any virtual base classes
       data().DefaultedDefaultConstructorIsConstexpr = false;
+      data().DefaultedDestructorIsConstexpr = false;
 
       // C++1z [class.copy]p8:
       //   The implicitly-declared copy constructor for a class X will have
@@ -520,6 +523,19 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) {
     data().NeedOverloadResolutionForMoveConstructor = true;
     data().NeedOverloadResolutionForDestructor = true;
   }
+
+  // C++2a [dcl.constexpr]p4:
+  //   The definition of a constexpr destructor [shall] satisfy the
+  //   following requirement:
+  //   -- for every subobject of class type or (possibly multi-dimensional)
+  //      array thereof, that class type shall have a constexpr destructor
+  if (!Subobj->hasConstexprDestructor())
+    data().DefaultedDestructorIsConstexpr = false;
+}
+
+bool CXXRecordDecl::hasConstexprDestructor() const {
+  auto *Dtor = getDestructor();
+  return Dtor ? Dtor->isConstexpr() : defaultedDestructorIsConstexpr();
 }
 
 bool CXXRecordDecl::hasAnyDependentBases() const {
@@ -1263,7 +1279,8 @@ void CXXRecordDecl::addedMember(Decl *D) {
     } else {
       // Base element type of field is a non-class type.
       if (!T->isLiteralType(Context) ||
-          (!Field->hasInClassInitializer() && !isUnion()))
+          (!Field->hasInClassInitializer() && !isUnion() &&
+           !Context.getLangOpts().CPlusPlus2a))
         data().DefaultedDefaultConstructorIsConstexpr = false;
 
       // C++11 [class.copy]p23:
@@ -1382,17 +1399,29 @@ static bool allLookupResultsAreTheSame(const DeclContext::lookup_result &R) {
 }
 #endif
 
-CXXMethodDecl* CXXRecordDecl::getLambdaCallOperator() const {
-  if (!isLambda()) return nullptr;
+static NamedDecl* getLambdaCallOperatorHelper(const CXXRecordDecl &RD) {
+  if (!RD.isLambda()) return nullptr;
   DeclarationName Name =
-    getASTContext().DeclarationNames.getCXXOperatorName(OO_Call);
-  DeclContext::lookup_result Calls = lookup(Name);
+    RD.getASTContext().DeclarationNames.getCXXOperatorName(OO_Call);
+  DeclContext::lookup_result Calls = RD.lookup(Name);
 
   assert(!Calls.empty() && "Missing lambda call operator!");
   assert(allLookupResultsAreTheSame(Calls) &&
          "More than one lambda call operator!");
+  return Calls.front();
+}
+
+FunctionTemplateDecl* CXXRecordDecl::getDependentLambdaCallOperator() const {
+  NamedDecl *CallOp = getLambdaCallOperatorHelper(*this);
+  return  dyn_cast_or_null<FunctionTemplateDecl>(CallOp);
+}
+
+CXXMethodDecl *CXXRecordDecl::getLambdaCallOperator() const {
+  NamedDecl *CallOp = getLambdaCallOperatorHelper(*this);
+
+  if (CallOp == nullptr)
+    return nullptr;
 
-  NamedDecl *CallOp = Calls.front();
   if (const auto *CallOpTmpl = dyn_cast<FunctionTemplateDecl>(CallOp))
     return cast<CXXMethodDecl>(CallOpTmpl->getTemplatedDecl());
 
@@ -2570,20 +2599,19 @@ CXXDestructorDecl *
 CXXDestructorDecl::CreateDeserialized(ASTContext &C, unsigned ID) {
   return new (C, ID)
       CXXDestructorDecl(C, nullptr, SourceLocation(), DeclarationNameInfo(),
-                        QualType(), nullptr, false, false);
+                        QualType(), nullptr, false, false, CSK_unspecified);
 }
 
-CXXDestructorDecl *
-CXXDestructorDecl::Create(ASTContext &C, CXXRecordDecl *RD,
-                          SourceLocation StartLoc,
-                          const DeclarationNameInfo &NameInfo,
-                          QualType T, TypeSourceInfo *TInfo,
-                          bool isInline, bool isImplicitlyDeclared) {
+CXXDestructorDecl *CXXDestructorDecl::Create(
+    ASTContext &C, CXXRecordDecl *RD, SourceLocation StartLoc,
+    const DeclarationNameInfo &NameInfo, QualType T, TypeSourceInfo *TInfo,
+    bool isInline, bool isImplicitlyDeclared, ConstexprSpecKind ConstexprKind) {
   assert(NameInfo.getName().getNameKind()
          == DeclarationName::CXXDestructorName &&
          "Name must refer to a destructor");
-  return new (C, RD) CXXDestructorDecl(C, RD, StartLoc, NameInfo, T, TInfo,
-                                       isInline, isImplicitlyDeclared);
+  return new (C, RD)
+      CXXDestructorDecl(C, RD, StartLoc, NameInfo, T, TInfo, isInline,
+                        isImplicitlyDeclared, ConstexprKind);
 }
 
 void CXXDestructorDecl::setOperatorDelete(FunctionDecl *OD, Expr *ThisArg) {
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index d055d9cd31fbb..3d93588a43ba1 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1007,12 +1007,19 @@ void DeclPrinter::VisitCXXRecordDecl(CXXRecordDecl *D) {
 
 void DeclPrinter::VisitLinkageSpecDecl(LinkageSpecDecl *D) {
   const char *l;
-  if (D->getLanguage() == LinkageSpecDecl::lang_c)
+  switch (D->getLanguage()) {
+  case LinkageSpecDecl::lang_c:
     l = "C";
-  else {
-    assert(D->getLanguage() == LinkageSpecDecl::lang_cxx &&
-           "unknown language in linkage specification");
+    break;
+  case LinkageSpecDecl::lang_cxx_14:
+    l = "C++14";
+    break;
+  case LinkageSpecDecl::lang_cxx_11:
+    l = "C++11";
+    break;
+  case LinkageSpecDecl::lang_cxx:
     l = "C++";
+    break;
   }
 
   Out << "extern \"" << l << "\" ";
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index 40c39c845db63..c03ae22fb5d89 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -510,8 +510,12 @@ SourceRange TemplateTypeParmDecl::getSourceRange() const {
   if (hasDefaultArgument() && !defaultArgumentWasInherited())
     return SourceRange(getBeginLoc(),
                        getDefaultArgumentInfo()->getTypeLoc().getEndLoc());
-  else
-    return TypeDecl::getSourceRange();
+  // TypeDecl::getSourceRange returns a range containing name location, which is
+  // wrong for unnamed template parameters. e.g:
+  // it will return <[[typename>]] instead of <[[typename]]>
+  else if (getDeclName().isEmpty())
+    return SourceRange(getBeginLoc());
+  return TypeDecl::getSourceRange();
 }
 
 unsigned TemplateTypeParmDecl::getDepth() const {
diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index e11224a91ae7b..efc050496e23f 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -185,6 +185,12 @@ bool Expr::isKnownToHaveBooleanValue() const {
     return CO->getTrueExpr()->isKnownToHaveBooleanValue() &&
            CO->getFalseExpr()->isKnownToHaveBooleanValue();
 
+  if (isa<ObjCBoolLiteralExpr>(E))
+    return true;
+
+  if (const auto *OVE = dyn_cast<OpaqueValueExpr>(E))
+    return OVE->getSourceExpr()->isKnownToHaveBooleanValue();
+
   return false;
 }
 
@@ -3991,6 +3997,112 @@ bool Expr::refersToGlobalRegisterVar() const {
   return false;
 }
 
+bool Expr::isSameComparisonOperand(const Expr* E1, const Expr* E2) {
+  E1 = E1->IgnoreParens();
+  E2 = E2->IgnoreParens();
+
+  if (E1->getStmtClass() != E2->getStmtClass())
+    return false;
+
+  switch (E1->getStmtClass()) {
+    default:
+      return false;
+    case CXXThisExprClass:
+      return true;
+    case DeclRefExprClass: {
+      // DeclRefExpr without an ImplicitCastExpr can happen for integral
+      // template parameters.
+      const auto *DRE1 = cast<DeclRefExpr>(E1);
+      const auto *DRE2 = cast<DeclRefExpr>(E2);
+      return DRE1->isRValue() && DRE2->isRValue() &&
+             DRE1->getDecl() == DRE2->getDecl();
+    }
+    case ImplicitCastExprClass: {
+      // Peel off implicit casts.
+      while (true) {
+        const auto *ICE1 = dyn_cast<ImplicitCastExpr>(E1);
+        const auto *ICE2 = dyn_cast<ImplicitCastExpr>(E2);
+        if (!ICE1 || !ICE2)
+          return false;
+        if (ICE1->getCastKind() != ICE2->getCastKind())
+          return false;
+        E1 = ICE1->getSubExpr()->IgnoreParens();
+        E2 = ICE2->getSubExpr()->IgnoreParens();
+        // The final cast must be one of these types.
+        if (ICE1->getCastKind() == CK_LValueToRValue ||
+            ICE1->getCastKind() == CK_ArrayToPointerDecay ||
+            ICE1->getCastKind() == CK_FunctionToPointerDecay) {
+          break;
+        }
+      }
+
+      const auto *DRE1 = dyn_cast<DeclRefExpr>(E1);
+      const auto *DRE2 = dyn_cast<DeclRefExpr>(E2);
+      if (DRE1 && DRE2)
+        return declaresSameEntity(DRE1->getDecl(), DRE2->getDecl());
+
+      const auto *Ivar1 = dyn_cast<ObjCIvarRefExpr>(E1);
+      const auto *Ivar2 = dyn_cast<ObjCIvarRefExpr>(E2);
+      if (Ivar1 && Ivar2) {
+        return Ivar1->isFreeIvar() && Ivar2->isFreeIvar() &&
+               declaresSameEntity(Ivar1->getDecl(), Ivar2->getDecl());
+      }
+
+      const auto *Array1 = dyn_cast<ArraySubscriptExpr>(E1);
+      const auto *Array2 = dyn_cast<ArraySubscriptExpr>(E2);
+      if (Array1 && Array2) {
+        if (!isSameComparisonOperand(Array1->getBase(), Array2->getBase()))
+          return false;
+
+        auto Idx1 = Array1->getIdx();
+        auto Idx2 = Array2->getIdx();
+        const auto Integer1 = dyn_cast<IntegerLiteral>(Idx1);
+        const auto Integer2 = dyn_cast<IntegerLiteral>(Idx2);
+        if (Integer1 && Integer2) {
+          if (!llvm::APInt::isSameValue(Integer1->getValue(),
+                                        Integer2->getValue()))
+            return false;
+        } else {
+          if (!isSameComparisonOperand(Idx1, Idx2))
+            return false;
+        }
+
+        return true;
+      }
+
+      // Walk the MemberExpr chain.
+      while (isa<MemberExpr>(E1) && isa<MemberExpr>(E2)) {
+        const auto *ME1 = cast<MemberExpr>(E1);
+        const auto *ME2 = cast<MemberExpr>(E2);
+        if (!declaresSameEntity(ME1->getMemberDecl(), ME2->getMemberDecl()))
+          return false;
+        if (const auto *D = dyn_cast<VarDecl>(ME1->getMemberDecl()))
+          if (D->isStaticDataMember())
+            return true;
+        E1 = ME1->getBase()->IgnoreParenImpCasts();
+        E2 = ME2->getBase()->IgnoreParenImpCasts();
+      }
+
+      if (isa<CXXThisExpr>(E1) && isa<CXXThisExpr>(E2))
+        return true;
+
+      // A static member variable can end the MemberExpr chain with either
+      // a MemberExpr or a DeclRefExpr.
+      auto getAnyDecl = [](const Expr *E) -> const ValueDecl * {
+        if (const auto *DRE = dyn_cast<DeclRefExpr>(E))
+          return DRE->getDecl();
+        if (const auto *ME = dyn_cast<MemberExpr>(E))
+          return ME->getMemberDecl();
+        return nullptr;
+      };
+
+      const ValueDecl *VD1 = getAnyDecl(E1);
+      const ValueDecl *VD2 = getAnyDecl(E2);
+      return declaresSameEntity(VD1, VD2);
+    }
+  }
+}
+
 /// isArrow - Return true if the base expression is a pointer to vector,
 /// return false if the base expression is a vector.
 bool ExtVectorElementExpr::isArrow() const {
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index c5f86a4cc12b4..1d5fd80d0d470 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -124,6 +124,8 @@ CXXNewExpr::CXXNewExpr(bool IsGlobalNew, FunctionDecl *OperatorNew,
 
   if (ArraySize) {
     if (Expr *SizeExpr = *ArraySize) {
+      if (SizeExpr->isValueDependent())
+        ExprBits.ValueDependent = true;
       if (SizeExpr->isInstantiationDependent())
         ExprBits.InstantiationDependent = true;
       if (SizeExpr->containsUnexpandedParameterPack())
@@ -134,6 +136,8 @@ CXXNewExpr::CXXNewExpr(bool IsGlobalNew, FunctionDecl *OperatorNew,
   }
 
   if (Initializer) {
+    if (Initializer->isValueDependent())
+      ExprBits.ValueDependent = true;
     if (Initializer->isInstantiationDependent())
       ExprBits.InstantiationDependent = true;
     if (Initializer->containsUnexpandedParameterPack())
@@ -143,6 +147,8 @@ CXXNewExpr::CXXNewExpr(bool IsGlobalNew, FunctionDecl *OperatorNew,
   }
 
   for (unsigned I = 0; I != PlacementArgs.size(); ++I) {
+    if (PlacementArgs[I]->isValueDependent())
+      ExprBits.ValueDependent = true;
     if (PlacementArgs[I]->isInstantiationDependent())
       ExprBits.InstantiationDependent = true;
     if (PlacementArgs[I]->containsUnexpandedParameterPack())
@@ -1212,6 +1218,11 @@ CXXMethodDecl *LambdaExpr::getCallOperator() const {
   return Record->getLambdaCallOperator();
 }
 
+FunctionTemplateDecl *LambdaExpr::getDependentCallOperator() const {
+  CXXRecordDecl *Record = getLambdaClass();
+  return Record->getDependentLambdaCallOperator();
+}
+
 TemplateParameterList *LambdaExpr::getTemplateParameterList() const {
   CXXRecordDecl *Record = getLambdaClass();
   return Record->getGenericLambdaTemplateParameterList();
diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index a4c247370ee1b..908e801234045 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -66,8 +66,6 @@ using llvm::APSInt;
 using llvm::APFloat;
 using llvm::Optional;
 
-static bool IsGlobalLValue(APValue::LValueBase B);
-
 namespace {
   struct LValue;
   class CallStackFrame;
@@ -98,6 +96,9 @@ namespace {
     if (B.is<TypeInfoLValue>())
       return B.getTypeInfoType();
 
+    if (B.is<DynamicAllocLValue>())
+      return B.getDynamicAllocType();
+
     const Expr *Base = B.get<const Expr*>();
 
     // For a materialized temporary, the type of the temporary we materialized
@@ -134,6 +135,14 @@ namespace {
     return E.getAsBaseOrMember().getInt();
   }
 
+  /// Given an expression, determine the type used to store the result of
+  /// evaluating that expression.
+  static QualType getStorageType(ASTContext &Ctx, Expr *E) {
+    if (E->isRValue())
+      return E->getType();
+    return Ctx.getLValueReferenceType(E->getType());
+  }
+
   /// Given a CallExpr, try to get the alloc_size attribute. May return null.
   static const AllocSizeAttr *getAllocSizeAttr(const CallExpr *CE) {
     const FunctionDecl *Callee = CE->getDirectCallee();
@@ -572,13 +581,26 @@ namespace {
       return 0;
     }
 
-    APValue &createTemporary(const void *Key, bool IsLifetimeExtended);
+    /// Allocate storage for an object of type T in this stack frame.
+    /// Populates LV with a handle to the created object. Key identifies
+    /// the temporary within the stack frame, and must not be reused without
+    /// bumping the temporary version number.
+    template<typename KeyT>
+    APValue &createTemporary(const KeyT *Key, QualType T,
+                             bool IsLifetimeExtended, LValue &LV);
 
     void describe(llvm::raw_ostream &OS) override;
 
     Frame *getCaller() const override { return Caller; }
     SourceLocation getCallLocation() const override { return CallLoc; }
     const FunctionDecl *getCallee() const override { return Callee; }
+
+    bool isStdFunction() const {
+      for (const DeclContext *DC = Callee; DC; DC = DC->getParent())
+        if (DC->isStdNamespace())
+          return true;
+      return false;
+    }
   };
 
   /// Temporarily override 'this'.
@@ -596,18 +618,42 @@ namespace {
     CallStackFrame &Frame;
     const LValue *OldThis;
   };
+}
+
+static bool HandleDestruction(EvalInfo &Info, const Expr *E,
+                              const LValue &This, QualType ThisType);
+static bool HandleDestruction(EvalInfo &Info, SourceLocation Loc,
+                              APValue::LValueBase LVBase, APValue &Value,
+                              QualType T);
 
+namespace {
   /// A cleanup, and a flag indicating whether it is lifetime-extended.
   class Cleanup {
     llvm::PointerIntPair<APValue*, 1, bool> Value;
+    APValue::LValueBase Base;
+    QualType T;
 
   public:
-    Cleanup(APValue *Val, bool IsLifetimeExtended)
-        : Value(Val, IsLifetimeExtended) {}
+    Cleanup(APValue *Val, APValue::LValueBase Base, QualType T,
+            bool IsLifetimeExtended)
+        : Value(Val, IsLifetimeExtended), Base(Base), T(T) {}
 
     bool isLifetimeExtended() const { return Value.getInt(); }
-    void endLifetime() {
+    bool endLifetime(EvalInfo &Info, bool RunDestructors) {
+      if (RunDestructors) {
+        SourceLocation Loc;
+        if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>())
+          Loc = VD->getLocation();
+        else if (const Expr *E = Base.dyn_cast<const Expr*>())
+          Loc = E->getExprLoc();
+        return HandleDestruction(Info, Loc, Base, *Value.getPointer(), T);
+      }
       *Value.getPointer() = APValue();
+      return true;
+    }
+
+    bool hasSideEffect() {
+      return T.isDestructedType();
     }
   };
 
@@ -623,7 +669,13 @@ namespace {
       return llvm::hash_combine(Obj.Base, Obj.Path);
     }
   };
-  enum class ConstructionPhase { None, Bases, AfterBases };
+  enum class ConstructionPhase {
+    None,
+    Bases,
+    AfterBases,
+    Destroying,
+    DestroyingBases
+  };
 }
 
 namespace llvm {
@@ -645,6 +697,37 @@ template<> struct DenseMapInfo<ObjectUnderConstruction> {
 }
 
 namespace {
+  /// A dynamically-allocated heap object.
+  struct DynAlloc {
+    /// The value of this heap-allocated object.
+    APValue Value;
+    /// The allocating expression; used for diagnostics. Either a CXXNewExpr
+    /// or a CallExpr (the latter is for direct calls to operator new inside
+    /// std::allocator<T>::allocate).
+    const Expr *AllocExpr = nullptr;
+
+    enum Kind {
+      New,
+      ArrayNew,
+      StdAllocator
+    };
+
+    /// Get the kind of the allocation. This must match between allocation
+    /// and deallocation.
+    Kind getKind() const {
+      if (auto *NE = dyn_cast<CXXNewExpr>(AllocExpr))
+        return NE->isArray() ? ArrayNew : New;
+      assert(isa<CallExpr>(AllocExpr));
+      return StdAllocator;
+    }
+  };
+
+  struct DynAllocOrder {
+    bool operator()(DynamicAllocLValue L, DynamicAllocLValue R) const {
+      return L.getIndex() < R.getIndex();
+    }
+  };
+
   /// EvalInfo - This is a private struct used by the evaluator to capture
   /// information about a subexpression as it is folded.  It retains information
   /// about the AST context, but also maintains information about the folded
@@ -699,6 +782,15 @@ namespace {
     /// evaluated, if any.
     APValue::LValueBase EvaluatingDecl;
 
+    enum class EvaluatingDeclKind {
+      None,
+      /// We're evaluating the construction of EvaluatingDecl.
+      Ctor,
+      /// We're evaluating the destruction of EvaluatingDecl.
+      Dtor,
+    };
+    EvaluatingDeclKind IsEvaluatingDecl = EvaluatingDeclKind::None;
+
     /// EvaluatingDeclValue - This is the value being constructed for the
     /// declaration whose initializer is being evaluated, if any.
     APValue *EvaluatingDeclValue;
@@ -707,6 +799,14 @@ namespace {
     llvm::DenseMap<ObjectUnderConstruction, ConstructionPhase>
         ObjectsUnderConstruction;
 
+    /// Current heap allocations, along with the location where each was
+    /// allocated. We use std::map here because we need stable addresses
+    /// for the stored APValues.
+    std::map<DynamicAllocLValue, DynAlloc, DynAllocOrder> HeapAllocs;
+
+    /// The number of heap allocations performed so far in this evaluation.
+    unsigned NumHeapAllocs = 0;
+
     struct EvaluatingConstructorRAII {
       EvalInfo &EI;
       ObjectUnderConstruction Object;
@@ -728,9 +828,29 @@ namespace {
       }
     };
 
+    struct EvaluatingDestructorRAII {
+      EvalInfo &EI;
+      ObjectUnderConstruction Object;
+      bool DidInsert;
+      EvaluatingDestructorRAII(EvalInfo &EI, ObjectUnderConstruction Object)
+          : EI(EI), Object(Object) {
+        DidInsert = EI.ObjectsUnderConstruction
+                        .insert({Object, ConstructionPhase::Destroying})
+                        .second;
+      }
+      void startedDestroyingBases() {
+        EI.ObjectsUnderConstruction[Object] =
+            ConstructionPhase::DestroyingBases;
+      }
+      ~EvaluatingDestructorRAII() {
+        if (DidInsert)
+          EI.ObjectsUnderConstruction.erase(Object);
+      }
+    };
+
     ConstructionPhase
-    isEvaluatingConstructor(APValue::LValueBase Base,
-                            ArrayRef<APValue::LValuePathEntry> Path) {
+    isEvaluatingCtorDtor(APValue::LValueBase Base,
+                         ArrayRef<APValue::LValuePathEntry> Path) {
       return ObjectsUnderConstruction.lookup({Base, Path});
     }
 
@@ -811,8 +931,14 @@ namespace {
           HasFoldFailureDiagnostic(false), InConstantContext(false),
           EvalMode(Mode) {}
 
-    void setEvaluatingDecl(APValue::LValueBase Base, APValue &Value) {
+    ~EvalInfo() {
+      discardCleanups();
+    }
+
+    void setEvaluatingDecl(APValue::LValueBase Base, APValue &Value,
+                           EvaluatingDeclKind EDK = EvaluatingDeclKind::Ctor) {
       EvaluatingDecl = Base;
+      IsEvaluatingDecl = EDK;
       EvaluatingDeclValue = &Value;
     }
 
@@ -858,6 +984,68 @@ namespace {
       return true;
     }
 
+    APValue *createHeapAlloc(const Expr *E, QualType T, LValue &LV);
+
+    Optional<DynAlloc*> lookupDynamicAlloc(DynamicAllocLValue DA) {
+      Optional<DynAlloc*> Result;
+      auto It = HeapAllocs.find(DA);
+      if (It != HeapAllocs.end())
+        Result = &It->second;
+      return Result;
+    }
+
+    /// Information about a stack frame for std::allocator<T>::[de]allocate.
+    struct StdAllocatorCaller {
+      unsigned FrameIndex;
+      QualType ElemType;
+      explicit operator bool() const { return FrameIndex != 0; };
+    };
+
+    StdAllocatorCaller getStdAllocatorCaller(StringRef FnName) const {
+      for (const CallStackFrame *Call = CurrentCall; Call != &BottomFrame;
+           Call = Call->Caller) {
+        const auto *MD = dyn_cast_or_null<CXXMethodDecl>(Call->Callee);
+        if (!MD)
+          continue;
+        const IdentifierInfo *FnII = MD->getIdentifier();
+        if (!FnII || !FnII->isStr(FnName))
+          continue;
+
+        const auto *CTSD =
+            dyn_cast<ClassTemplateSpecializationDecl>(MD->getParent());
+        if (!CTSD)
+          continue;
+
+        const IdentifierInfo *ClassII = CTSD->getIdentifier();
+        const TemplateArgumentList &TAL = CTSD->getTemplateArgs();
+        if (CTSD->isInStdNamespace() && ClassII &&
+            ClassII->isStr("allocator") && TAL.size() >= 1 &&
+            TAL[0].getKind() == TemplateArgument::Type)
+          return {Call->Index, TAL[0].getAsType()};
+      }
+
+      return {};
+    }
+
+    void performLifetimeExtension() {
+      // Disable the cleanups for lifetime-extended temporaries.
+      CleanupStack.erase(
+          std::remove_if(CleanupStack.begin(), CleanupStack.end(),
+                         [](Cleanup &C) { return C.isLifetimeExtended(); }),
+          CleanupStack.end());
+     }
+
+    /// Throw away any remaining cleanups at the end of evaluation. If any
+    /// cleanups would have had a side-effect, note that as an unmodeled
+    /// side-effect and return false. Otherwise, return true.
+    bool discardCleanups() {
+      for (Cleanup &C : CleanupStack)
+        if (C.hasSideEffect())
+          if (!noteSideEffect())
+            return false;
+      return true;
+    }
+
   private:
     interp::Frame *getCurrentFrame() override { return CurrentCall; }
     const interp::Frame *getBottomFrame() const override { return &BottomFrame; }
@@ -1101,29 +1289,45 @@ namespace {
       // temporaries created in different iterations of a loop.
       Info.CurrentCall->pushTempVersion();
     }
+    bool destroy(bool RunDestructors = true) {
+      bool OK = cleanup(Info, RunDestructors, OldStackSize);
+      OldStackSize = -1U;
+      return OK;
+    }
     ~ScopeRAII() {
+      if (OldStackSize != -1U)
+        destroy(false);
       // Body moved to a static method to encourage the compiler to inline away
       // instances of this class.
-      cleanup(Info, OldStackSize);
       Info.CurrentCall->popTempVersion();
     }
   private:
-    static void cleanup(EvalInfo &Info, unsigned OldStackSize) {
-      unsigned NewEnd = OldStackSize;
-      for (unsigned I = OldStackSize, N = Info.CleanupStack.size();
-           I != N; ++I) {
-        if (IsFullExpression && Info.CleanupStack[I].isLifetimeExtended()) {
-          // Full-expression cleanup of a lifetime-extended temporary: nothing
-          // to do, just move this cleanup to the right place in the stack.
-          std::swap(Info.CleanupStack[I], Info.CleanupStack[NewEnd]);
-          ++NewEnd;
-        } else {
-          // End the lifetime of the object.
-          Info.CleanupStack[I].endLifetime();
+    static bool cleanup(EvalInfo &Info, bool RunDestructors,
+                        unsigned OldStackSize) {
+      assert(OldStackSize <= Info.CleanupStack.size() &&
+             "running cleanups out of order?");
+
+      // Run all cleanups for a block scope, and non-lifetime-extended cleanups
+      // for a full-expression scope.
+      bool Success = true;
+      for (unsigned I = Info.CleanupStack.size(); I > OldStackSize; --I) {
+        if (!(IsFullExpression &&
+              Info.CleanupStack[I - 1].isLifetimeExtended())) {
+          if (!Info.CleanupStack[I - 1].endLifetime(Info, RunDestructors)) {
+            Success = false;
+            break;
+          }
         }
       }
-      Info.CleanupStack.erase(Info.CleanupStack.begin() + NewEnd,
-                              Info.CleanupStack.end());
+
+      // Compact lifetime-extended cleanups.
+      auto NewEnd = Info.CleanupStack.begin() + OldStackSize;
+      if (IsFullExpression)
+        NewEnd =
+            std::remove_if(NewEnd, Info.CleanupStack.end(),
+                           [](Cleanup &C) { return !C.isLifetimeExtended(); });
+      Info.CleanupStack.erase(NewEnd, Info.CleanupStack.end());
+      return Success;
     }
   };
   typedef ScopeRAII<false> BlockScopeRAII;
@@ -1183,18 +1387,14 @@ CallStackFrame::~CallStackFrame() {
   Info.CurrentCall = Caller;
 }
 
-APValue &CallStackFrame::createTemporary(const void *Key,
-                                         bool IsLifetimeExtended) {
-  unsigned Version = Info.CurrentCall->getTempVersion();
-  APValue &Result = Temporaries[MapKeyTy(Key, Version)];
-  assert(Result.isAbsent() && "temporary created multiple times");
-  Info.CleanupStack.push_back(Cleanup(&Result, IsLifetimeExtended));
-  return Result;
+static bool isRead(AccessKinds AK) {
+  return AK == AK_Read || AK == AK_ReadObjectRepresentation;
 }
 
 static bool isModification(AccessKinds AK) {
   switch (AK) {
   case AK_Read:
+  case AK_ReadObjectRepresentation:
   case AK_MemberCall:
   case AK_DynamicCast:
   case AK_TypeId:
@@ -1202,14 +1402,20 @@ static bool isModification(AccessKinds AK) {
   case AK_Assign:
   case AK_Increment:
   case AK_Decrement:
+  case AK_Construct:
+  case AK_Destroy:
     return true;
   }
   llvm_unreachable("unknown access kind");
 }
 
+static bool isAnyAccess(AccessKinds AK) {
+  return isRead(AK) || isModification(AK);
+}
+
 /// Is this an access per the C++ definition?
 static bool isFormalAccess(AccessKinds AK) {
-  return AK == AK_Read || isModification(AK);
+  return isAnyAccess(AK) && AK != AK_Construct && AK != AK_Destroy;
 }
 
 namespace {
@@ -1305,9 +1511,10 @@ namespace {
       IsNullPtr = false;
     }
 
-    void setNull(QualType PointerTy, uint64_t TargetVal) {
+    void setNull(ASTContext &Ctx, QualType PointerTy) {
       Base = (Expr *)nullptr;
-      Offset = CharUnits::fromQuantity(TargetVal);
+      Offset =
+          CharUnits::fromQuantity(Ctx.getTargetNullPointerValue(PointerTy));
       InvalidBase = false;
       Designator = SubobjectDesignator(PointerTy->getPointeeType());
       IsNullPtr = true;
@@ -1317,6 +1524,12 @@ namespace {
       set(B, true);
     }
 
+    std::string toString(ASTContext &Ctx, QualType T) const {
+      APValue Printable;
+      moveInto(Printable);
+      return Printable.getAsString(Ctx, T);
+    }
+
   private:
     // Check that this LValue is not based on a null pointer. If it is, produce
     // a diagnostic and mark the designator as invalid.
@@ -1539,15 +1752,6 @@ static bool EvaluateFixedPoint(const Expr *E, APFixedPoint &Result,
 // Misc utilities
 //===----------------------------------------------------------------------===//
 
-/// A helper function to create a temporary and set an LValue.
-template <class KeyTy>
-static APValue &createTemporary(const KeyTy *Key, bool IsLifetimeExtended,
-                                LValue &LV, CallStackFrame &Frame) {
-  LV.set({Key, Frame.Info.CurrentCall->Index,
-          Frame.Info.CurrentCall->getTempVersion()});
-  return Frame.createTemporary(Key, IsLifetimeExtended);
-}
-
 /// Negate an APSInt in place, converting it to a signed form if necessary, and
 /// preserving its value (by extending by up to one bit as needed).
 static void negateAsSigned(APSInt &Int) {
@@ -1558,6 +1762,43 @@ static void negateAsSigned(APSInt &Int) {
   Int = -Int;
 }
 
+template<typename KeyT>
+APValue &CallStackFrame::createTemporary(const KeyT *Key, QualType T,
+                                         bool IsLifetimeExtended, LValue &LV) {
+  unsigned Version = getTempVersion();
+  APValue::LValueBase Base(Key, Index, Version);
+  LV.set(Base);
+  APValue &Result = Temporaries[MapKeyTy(Key, Version)];
+  assert(Result.isAbsent() && "temporary created multiple times");
+
+  // If we're creating a temporary immediately in the operand of a speculative
+  // evaluation, don't register a cleanup to be run outside the speculative
+  // evaluation context, since we won't actually be able to initialize this
+  // object.
+  if (Index <= Info.SpeculativeEvaluationDepth) {
+    if (T.isDestructedType())
+      Info.noteSideEffect();
+  } else {
+    Info.CleanupStack.push_back(Cleanup(&Result, Base, T, IsLifetimeExtended));
+  }
+  return Result;
+}
+
+APValue *EvalInfo::createHeapAlloc(const Expr *E, QualType T, LValue &LV) {
+  if (NumHeapAllocs > DynamicAllocLValue::getMaxIndex()) {
+    FFDiag(E, diag::note_constexpr_heap_alloc_limit_exceeded);
+    return nullptr;
+  }
+
+  DynamicAllocLValue DA(NumHeapAllocs++);
+  LV.set(APValue::LValueBase::getDynamicAlloc(DA, T));
+  auto Result = HeapAllocs.emplace(std::piecewise_construct,
+                                   std::forward_as_tuple(DA), std::tuple<>());
+  assert(Result.second && "reused a heap alloc index?");
+  Result.first->second.AllocExpr = E;
+  return &Result.first->second.Value;
+}
+
 /// Produce a string describing the given constexpr call.
 void CallStackFrame::describe(raw_ostream &Out) {
   unsigned ArgIndex = 0;
@@ -1628,7 +1869,7 @@ static bool IsGlobalLValue(APValue::LValueBase B) {
     return isa<FunctionDecl>(D);
   }
 
-  if (B.is<TypeInfoLValue>())
+  if (B.is<TypeInfoLValue>() || B.is<DynamicAllocLValue>())
     return true;
 
   const Expr *E = B.get<const Expr*>();
@@ -1727,15 +1968,39 @@ static void NoteLValueLocation(EvalInfo &Info, APValue::LValueBase Base) {
     Info.Note(VD->getLocation(), diag::note_declared_at);
   else if (const Expr *E = Base.dyn_cast<const Expr*>())
     Info.Note(E->getExprLoc(), diag::note_constexpr_temporary_here);
+  else if (DynamicAllocLValue DA = Base.dyn_cast<DynamicAllocLValue>()) {
+    // FIXME: Produce a note for dangling pointers too.
+    if (Optional<DynAlloc*> Alloc = Info.lookupDynamicAlloc(DA))
+      Info.Note((*Alloc)->AllocExpr->getExprLoc(),
+                diag::note_constexpr_dynamic_alloc_here);
+  }
   // We have no information to show for a typeid(T) object.
 }
 
+enum class CheckEvaluationResultKind {
+  ConstantExpression,
+  FullyInitialized,
+};
+
+/// Materialized temporaries that we've already checked to determine if they're
+/// initializsed by a constant expression.
+using CheckedTemporaries =
+    llvm::SmallPtrSet<const MaterializeTemporaryExpr *, 8>;
+
+static bool CheckEvaluationResult(CheckEvaluationResultKind CERK,
+                                  EvalInfo &Info, SourceLocation DiagLoc,
+                                  QualType Type, const APValue &Value,
+                                  Expr::ConstExprUsage Usage,
+                                  SourceLocation SubobjectLoc,
+                                  CheckedTemporaries &CheckedTemps);
+
 /// Check that this reference or pointer core constant expression is a valid
 /// value for an address or reference constant expression. Return true if we
 /// can fold this expression, whether or not it's a constant expression.
 static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc,
                                           QualType Type, const LValue &LVal,
-                                          Expr::ConstExprUsage Usage) {
+                                          Expr::ConstExprUsage Usage,
+                                          CheckedTemporaries &CheckedTemps) {
   bool IsReferenceType = Type->isReferenceType();
 
   APValue::LValueBase Base = LVal.getLValueBase();
@@ -1761,14 +2026,23 @@ static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc,
           LVal.getLValueCallIndex() == 0) &&
          "have call index for global lvalue");
 
+  if (Base.is<DynamicAllocLValue>()) {
+    Info.FFDiag(Loc, diag::note_constexpr_dynamic_alloc)
+        << IsReferenceType << !Designator.Entries.empty();
+    NoteLValueLocation(Info, Base);
+    return false;
+  }
+
   if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>()) {
     if (const VarDecl *Var = dyn_cast<const VarDecl>(VD)) {
       // Check if this is a thread-local variable.
       if (Var->getTLSKind())
+        // FIXME: Diagnostic!
         return false;
 
       // A dllimport variable never acts like a constant.
       if (Usage == Expr::EvaluateForCodeGen && Var->hasAttr<DLLImportAttr>())
+        // FIXME: Diagnostic!
         return false;
     }
     if (const auto *FD = dyn_cast<const FunctionDecl>(VD)) {
@@ -1784,6 +2058,25 @@ static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc,
       // perform initialization with the address of the thunk.
       if (Info.getLangOpts().CPlusPlus && Usage == Expr::EvaluateForCodeGen &&
           FD->hasAttr<DLLImportAttr>())
+        // FIXME: Diagnostic!
+        return false;
+    }
+  } else if (const auto *MTE = dyn_cast_or_null<MaterializeTemporaryExpr>(
+                 Base.dyn_cast<const Expr *>())) {
+    if (CheckedTemps.insert(MTE).second) {
+      QualType TempType = getType(Base);
+      if (TempType.isDestructedType()) {
+        Info.FFDiag(MTE->getExprLoc(),
+                    diag::note_constexpr_unsupported_tempoarary_nontrivial_dtor)
+            << TempType;
+        return false;
+      }
+
+      APValue *V = Info.Ctx.getMaterializedTemporaryValue(MTE, false);
+      assert(V && "evasluation result refers to uninitialised temporary");
+      if (!CheckEvaluationResult(CheckEvaluationResultKind::ConstantExpression,
+                                 Info, MTE->getExprLoc(), TempType, *V,
+                                 Usage, SourceLocation(), CheckedTemps))
         return false;
     }
   }
@@ -1858,14 +2151,12 @@ static bool CheckLiteralType(EvalInfo &Info, const Expr *E,
   return false;
 }
 
-/// Check that this core constant expression value is a valid value for a
-/// constant expression. If not, report an appropriate diagnostic. Does not
-/// check that the expression is of literal type.
-static bool
-CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, QualType Type,
-                        const APValue &Value,
-                        Expr::ConstExprUsage Usage = Expr::EvaluateForCodeGen,
-                        SourceLocation SubobjectLoc = SourceLocation()) {
+static bool CheckEvaluationResult(CheckEvaluationResultKind CERK,
+                                  EvalInfo &Info, SourceLocation DiagLoc,
+                                  QualType Type, const APValue &Value,
+                                  Expr::ConstExprUsage Usage,
+                                  SourceLocation SubobjectLoc,
+                                  CheckedTemporaries &CheckedTemps) {
   if (!Value.hasValue()) {
     Info.FFDiag(DiagLoc, diag::note_constexpr_uninitialized)
       << true << Type;
@@ -1885,30 +2176,31 @@ CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, QualType Type,
   if (Value.isArray()) {
     QualType EltTy = Type->castAsArrayTypeUnsafe()->getElementType();
     for (unsigned I = 0, N = Value.getArrayInitializedElts(); I != N; ++I) {
-      if (!CheckConstantExpression(Info, DiagLoc, EltTy,
-                                   Value.getArrayInitializedElt(I), Usage,
-                                   SubobjectLoc))
+      if (!CheckEvaluationResult(CERK, Info, DiagLoc, EltTy,
+                                 Value.getArrayInitializedElt(I), Usage,
+                                 SubobjectLoc, CheckedTemps))
         return false;
     }
     if (!Value.hasArrayFiller())
       return true;
-    return CheckConstantExpression(Info, DiagLoc, EltTy, Value.getArrayFiller(),
-                                   Usage, SubobjectLoc);
+    return CheckEvaluationResult(CERK, Info, DiagLoc, EltTy,
+                                 Value.getArrayFiller(), Usage, SubobjectLoc,
+                                 CheckedTemps);
   }
   if (Value.isUnion() && Value.getUnionField()) {
-    return CheckConstantExpression(Info, DiagLoc,
-                                   Value.getUnionField()->getType(),
-                                   Value.getUnionValue(), Usage,
-                                   Value.getUnionField()->getLocation());
+    return CheckEvaluationResult(
+        CERK, Info, DiagLoc, Value.getUnionField()->getType(),
+        Value.getUnionValue(), Usage, Value.getUnionField()->getLocation(),
+        CheckedTemps);
   }
   if (Value.isStruct()) {
     RecordDecl *RD = Type->castAs<RecordType>()->getDecl();
     if (const CXXRecordDecl *CD = dyn_cast<CXXRecordDecl>(RD)) {
       unsigned BaseIndex = 0;
       for (const CXXBaseSpecifier &BS : CD->bases()) {
-        if (!CheckConstantExpression(Info, DiagLoc, BS.getType(),
-                                     Value.getStructBase(BaseIndex), Usage,
-                                     BS.getBeginLoc()))
+        if (!CheckEvaluationResult(CERK, Info, DiagLoc, BS.getType(),
+                                   Value.getStructBase(BaseIndex), Usage,
+                                   BS.getBeginLoc(), CheckedTemps))
           return false;
         ++BaseIndex;
       }
@@ -1917,26 +2209,66 @@ CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, QualType Type,
       if (I->isUnnamedBitfield())
         continue;
 
-      if (!CheckConstantExpression(Info, DiagLoc, I->getType(),
-                                   Value.getStructField(I->getFieldIndex()),
-                                   Usage, I->getLocation()))
+      if (!CheckEvaluationResult(CERK, Info, DiagLoc, I->getType(),
+                                 Value.getStructField(I->getFieldIndex()),
+                                 Usage, I->getLocation(), CheckedTemps))
         return false;
     }
   }
 
-  if (Value.isLValue()) {
+  if (Value.isLValue() &&
+      CERK == CheckEvaluationResultKind::ConstantExpression) {
     LValue LVal;
     LVal.setFrom(Info.Ctx, Value);
-    return CheckLValueConstantExpression(Info, DiagLoc, Type, LVal, Usage);
+    return CheckLValueConstantExpression(Info, DiagLoc, Type, LVal, Usage,
+                                         CheckedTemps);
   }
 
-  if (Value.isMemberPointer())
+  if (Value.isMemberPointer() &&
+      CERK == CheckEvaluationResultKind::ConstantExpression)
     return CheckMemberPointerConstantExpression(Info, DiagLoc, Type, Value, Usage);
 
   // Everything else is fine.
   return true;
 }
 
+/// Check that this core constant expression value is a valid value for a
+/// constant expression. If not, report an appropriate diagnostic. Does not
+/// check that the expression is of literal type.
+static bool
+CheckConstantExpression(EvalInfo &Info, SourceLocation DiagLoc, QualType Type,
+                        const APValue &Value,
+                        Expr::ConstExprUsage Usage = Expr::EvaluateForCodeGen) {
+  CheckedTemporaries CheckedTemps;
+  return CheckEvaluationResult(CheckEvaluationResultKind::ConstantExpression,
+                               Info, DiagLoc, Type, Value, Usage,
+                               SourceLocation(), CheckedTemps);
+}
+
+/// Check that this evaluated value is fully-initialized and can be loaded by
+/// an lvalue-to-rvalue conversion.
+static bool CheckFullyInitialized(EvalInfo &Info, SourceLocation DiagLoc,
+                                  QualType Type, const APValue &Value) {
+  CheckedTemporaries CheckedTemps;
+  return CheckEvaluationResult(
+      CheckEvaluationResultKind::FullyInitialized, Info, DiagLoc, Type, Value,
+      Expr::EvaluateForCodeGen, SourceLocation(), CheckedTemps);
+}
+
+/// Enforce C++2a [expr.const]/4.17, which disallows new-expressions unless
+/// "the allocated storage is deallocated within the evaluation".
+static bool CheckMemoryLeaks(EvalInfo &Info) {
+  if (!Info.HeapAllocs.empty()) {
+    // We can still fold to a constant despite a compile-time memory leak,
+    // so long as the heap allocation isn't referenced in the result (we check
+    // that in CheckConstantExpression).
+    Info.CCEDiag(Info.HeapAllocs.begin()->second.AllocExpr,
+                 diag::note_constexpr_memory_leak)
+        << unsigned(Info.HeapAllocs.size() - 1);
+  }
+  return true;
+}
+
 static bool EvalPointerValueAsBool(const APValue &Value, bool &Result) {
   // A null base expression indicates a null pointer.  These are always
   // evaluatable, and they are false unless the offset is zero.
@@ -2628,9 +2960,10 @@ static APSInt extractStringLiteralCharacter(EvalInfo &Info, const Expr *Lit,
 // FIXME: This is inefficient; we should probably introduce something similar
 // to the LLVM ConstantDataArray to make this cheaper.
 static void expandStringLiteral(EvalInfo &Info, const StringLiteral *S,
-                                APValue &Result) {
-  const ConstantArrayType *CAT =
-      Info.Ctx.getAsConstantArrayType(S->getType());
+                                APValue &Result,
+                                QualType AllocType = QualType()) {
+  const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(
+      AllocType.isNull() ? S->getType() : AllocType);
   assert(CAT && "string literal isn't an array");
   QualType CharType = CAT->getElementType();
   assert(CharType->isIntegerType() && "unexpected character type");
@@ -2694,8 +3027,8 @@ static bool isReadByLvalueToRvalueConversion(QualType T) {
 
 /// Diagnose an attempt to read from any unreadable field within the specified
 /// type, which might be a class type.
-static bool diagnoseUnreadableFields(EvalInfo &Info, const Expr *E,
-                                     QualType T) {
+static bool diagnoseMutableFields(EvalInfo &Info, const Expr *E, AccessKinds AK,
+                                  QualType T) {
   CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
   if (!RD)
     return false;
@@ -2710,17 +3043,17 @@ static bool diagnoseUnreadableFields(EvalInfo &Info, const Expr *E,
     // FIXME: Add core issue number for the union case.
     if (Field->isMutable() &&
         (RD->isUnion() || isReadByLvalueToRvalueConversion(Field->getType()))) {
-      Info.FFDiag(E, diag::note_constexpr_ltor_mutable, 1) << Field;
+      Info.FFDiag(E, diag::note_constexpr_access_mutable, 1) << AK << Field;
       Info.Note(Field->getLocation(), diag::note_declared_at);
       return true;
     }
 
-    if (diagnoseUnreadableFields(Info, E, Field->getType()))
+    if (diagnoseMutableFields(Info, E, AK, Field->getType()))
       return true;
   }
 
   for (auto &BaseSpec : RD->bases())
-    if (diagnoseUnreadableFields(Info, E, BaseSpec.getType()))
+    if (diagnoseMutableFields(Info, E, AK, BaseSpec.getType()))
       return true;
 
   // All mutable fields were empty, and thus not actually read.
@@ -2728,7 +3061,8 @@ static bool diagnoseUnreadableFields(EvalInfo &Info, const Expr *E,
 }
 
 static bool lifetimeStartedInEvaluation(EvalInfo &Info,
-                                        APValue::LValueBase Base) {
+                                        APValue::LValueBase Base,
+                                        bool MutableSubobject = false) {
   // A temporary we created.
   if (Base.getCallIndex())
     return true;
@@ -2737,19 +3071,42 @@ static bool lifetimeStartedInEvaluation(EvalInfo &Info,
   if (!Evaluating)
     return false;
 
-  // The variable whose initializer we're evaluating.
-  if (auto *BaseD = Base.dyn_cast<const ValueDecl*>())
-    if (declaresSameEntity(Evaluating, BaseD))
-      return true;
+  auto *BaseD = Base.dyn_cast<const ValueDecl*>();
 
-  // A temporary lifetime-extended by the variable whose initializer we're
-  // evaluating.
-  if (auto *BaseE = Base.dyn_cast<const Expr *>())
-    if (auto *BaseMTE = dyn_cast<MaterializeTemporaryExpr>(BaseE))
-      if (declaresSameEntity(BaseMTE->getExtendingDecl(), Evaluating))
-        return true;
+  switch (Info.IsEvaluatingDecl) {
+  case EvalInfo::EvaluatingDeclKind::None:
+    return false;
 
-  return false;
+  case EvalInfo::EvaluatingDeclKind::Ctor:
+    // The variable whose initializer we're evaluating.
+    if (BaseD)
+      return declaresSameEntity(Evaluating, BaseD);
+
+    // A temporary lifetime-extended by the variable whose initializer we're
+    // evaluating.
+    if (auto *BaseE = Base.dyn_cast<const Expr *>())
+      if (auto *BaseMTE = dyn_cast<MaterializeTemporaryExpr>(BaseE))
+        return declaresSameEntity(BaseMTE->getExtendingDecl(), Evaluating);
+    return false;
+
+  case EvalInfo::EvaluatingDeclKind::Dtor:
+    // C++2a [expr.const]p6:
+    //   [during constant destruction] the lifetime of a and its non-mutable
+    //   subobjects (but not its mutable subobjects) [are] considered to start
+    //   within e.
+    //
+    // FIXME: We can meaningfully extend this to cover non-const objects, but
+    // we will need special handling: we should be able to access only
+    // subobjects of such objects that are themselves declared const.
+    if (!BaseD ||
+        !(BaseD->getType().isConstQualified() ||
+          BaseD->getType()->isReferenceType()) ||
+        MutableSubobject)
+      return false;
+    return declaresSameEntity(Evaluating, BaseD);
+  }
+
+  llvm_unreachable("unknown evaluating decl kind");
 }
 
 namespace {
@@ -2767,13 +3124,13 @@ struct CompleteObject {
   CompleteObject(APValue::LValueBase Base, APValue *Value, QualType Type)
       : Base(Base), Value(Value), Type(Type) {}
 
-  bool mayReadMutableMembers(EvalInfo &Info) const {
+  bool mayAccessMutableMembers(EvalInfo &Info, AccessKinds AK) const {
     // In C++14 onwards, it is permitted to read a mutable member whose
     // lifetime began within the evaluation.
     // FIXME: Should we also allow this in C++11?
     if (!Info.getLangOpts().CPlusPlus14)
       return false;
-    return lifetimeStartedInEvaluation(Info, Base);
+    return lifetimeStartedInEvaluation(Info, Base, /*MutableSubobject*/true);
   }
 
   explicit operator bool() const { return !Type.isNull(); }
@@ -2821,19 +3178,22 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
   // Walk the designator's path to find the subobject.
   for (unsigned I = 0, N = Sub.Entries.size(); /**/; ++I) {
     // Reading an indeterminate value is undefined, but assigning over one is OK.
-    if (O->isAbsent() || (O->isIndeterminate() && handler.AccessKind != AK_Assign)) {
+    if ((O->isAbsent() && handler.AccessKind != AK_Construct) ||
+        (O->isIndeterminate() && handler.AccessKind != AK_Construct &&
+         handler.AccessKind != AK_Assign &&
+         handler.AccessKind != AK_ReadObjectRepresentation)) {
       if (!Info.checkingPotentialConstantExpression())
         Info.FFDiag(E, diag::note_constexpr_access_uninit)
             << handler.AccessKind << O->isIndeterminate();
       return handler.failed();
     }
 
-    // C++ [class.ctor]p5:
+    // C++ [class.ctor]p5, C++ [class.dtor]p5:
     //    const and volatile semantics are not applied on an object under
-    //    construction.
+    //    {con,de}struction.
     if ((ObjType.isConstQualified() || ObjType.isVolatileQualified()) &&
         ObjType->isRecordType() &&
-        Info.isEvaluatingConstructor(
+        Info.isEvaluatingCtorDtor(
             Obj.Base, llvm::makeArrayRef(Sub.Entries.begin(),
                                          Sub.Entries.begin() + I)) !=
                           ConstructionPhase::None) {
@@ -2876,9 +3236,9 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
       // things we need to check: if there are any mutable subobjects, we
       // cannot perform this read. (This only happens when performing a trivial
       // copy or assignment.)
-      if (ObjType->isRecordType() && handler.AccessKind == AK_Read &&
-          !Obj.mayReadMutableMembers(Info) &&
-          diagnoseUnreadableFields(Info, E, ObjType))
+      if (ObjType->isRecordType() &&
+          !Obj.mayAccessMutableMembers(Info, handler.AccessKind) &&
+          diagnoseMutableFields(Info, E, handler.AccessKind, ObjType))
         return handler.failed();
     }
 
@@ -2916,7 +3276,7 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
 
       if (O->getArrayInitializedElts() > Index)
         O = &O->getArrayInitializedElt(Index);
-      else if (handler.AccessKind != AK_Read) {
+      else if (!isRead(handler.AccessKind)) {
         expandArray(*O, Index);
         O = &O->getArrayInitializedElt(Index);
       } else
@@ -2946,10 +3306,10 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
                                    : O->getComplexFloatReal(), ObjType);
       }
     } else if (const FieldDecl *Field = getAsField(Sub.Entries[I])) {
-      if (Field->isMutable() && handler.AccessKind == AK_Read &&
-          !Obj.mayReadMutableMembers(Info)) {
-        Info.FFDiag(E, diag::note_constexpr_ltor_mutable, 1)
-          << Field;
+      if (Field->isMutable() &&
+          !Obj.mayAccessMutableMembers(Info, handler.AccessKind)) {
+        Info.FFDiag(E, diag::note_constexpr_access_mutable, 1)
+          << handler.AccessKind << Field;
         Info.Note(Field->getLocation(), diag::note_declared_at);
         return handler.failed();
       }
@@ -2960,9 +3320,18 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
         const FieldDecl *UnionField = O->getUnionField();
         if (!UnionField ||
             UnionField->getCanonicalDecl() != Field->getCanonicalDecl()) {
-          Info.FFDiag(E, diag::note_constexpr_access_inactive_union_member)
-            << handler.AccessKind << Field << !UnionField << UnionField;
-          return handler.failed();
+          if (I == N - 1 && handler.AccessKind == AK_Construct) {
+            // Placement new onto an inactive union member makes it active.
+            O->setUnion(Field, APValue());
+          } else {
+            // FIXME: If O->getUnionValue() is absent, report that there's no
+            // active union member rather than reporting the prior active union
+            // member. We'll need to fix nullptr_t to not use APValue() as its
+            // representation first.
+            Info.FFDiag(E, diag::note_constexpr_access_inactive_union_member)
+                << handler.AccessKind << Field << !UnionField << UnionField;
+            return handler.failed();
+          }
         }
         O = &O->getUnionValue();
       } else
@@ -2986,15 +3355,17 @@ findSubobject(EvalInfo &Info, const Expr *E, const CompleteObject &Obj,
 namespace {
 struct ExtractSubobjectHandler {
   EvalInfo &Info;
+  const Expr *E;
   APValue &Result;
-
-  static const AccessKinds AccessKind = AK_Read;
+  const AccessKinds AccessKind;
 
   typedef bool result_type;
   bool failed() { return false; }
   bool found(APValue &Subobj, QualType SubobjType) {
     Result = Subobj;
-    return true;
+    if (AccessKind == AK_ReadObjectRepresentation)
+      return true;
+    return CheckFullyInitialized(Info, E->getExprLoc(), SubobjType, Result);
   }
   bool found(APSInt &Value, QualType SubobjType) {
     Result = APValue(Value);
@@ -3007,14 +3378,13 @@ struct ExtractSubobjectHandler {
 };
 } // end anonymous namespace
 
-const AccessKinds ExtractSubobjectHandler::AccessKind;
-
 /// Extract the designated sub-object of an rvalue.
 static bool extractSubobject(EvalInfo &Info, const Expr *E,
                              const CompleteObject &Obj,
-                             const SubobjectDesignator &Sub,
-                             APValue &Result) {
-  ExtractSubobjectHandler Handler = { Info, Result };
+                             const SubobjectDesignator &Sub, APValue &Result,
+                             AccessKinds AK = AK_Read) {
+  assert(AK == AK_Read || AK == AK_ReadObjectRepresentation);
+  ExtractSubobjectHandler Handler = {Info, E, Result, AK};
   return findSubobject(Info, E, Obj, Sub, Handler);
 }
 
@@ -3160,13 +3530,13 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
     }
   }
 
-  bool IsAccess = isFormalAccess(AK);
+  bool IsAccess = isAnyAccess(AK);
 
   // C++11 DR1311: An lvalue-to-rvalue conversion on a volatile-qualified type
   // is not a constant expression (even if the object is non-volatile). We also
   // apply this rule to C++98, in order to conform to the expected 'volatile'
   // semantics.
-  if (IsAccess && LValType.isVolatileQualified()) {
+  if (isFormalAccess(AK) && LValType.isVolatileQualified()) {
     if (Info.getLangOpts().CPlusPlus)
       Info.FFDiag(E, diag::note_constexpr_access_volatile_type)
         << AK << LValType;
@@ -3201,8 +3571,7 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
     // the variable we're reading must be const.
     if (!Frame) {
       if (Info.getLangOpts().CPlusPlus14 &&
-          declaresSameEntity(
-              VD, Info.EvaluatingDecl.dyn_cast<const ValueDecl *>())) {
+          lifetimeStartedInEvaluation(Info, LVal.Base)) {
         // OK, we can read and modify an object if we're in the process of
         // evaluating its initializer, because its lifetime began in this
         // evaluation.
@@ -3261,6 +3630,14 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
 
     if (!evaluateVarDeclInit(Info, E, VD, Frame, BaseVal, &LVal))
       return CompleteObject();
+  } else if (DynamicAllocLValue DA = LVal.Base.dyn_cast<DynamicAllocLValue>()) {
+    Optional<DynAlloc*> Alloc = Info.lookupDynamicAlloc(DA);
+    if (!Alloc) {
+      Info.FFDiag(E, diag::note_constexpr_access_deleted_object) << AK;
+      return CompleteObject();
+    }
+    return CompleteObject(LVal.Base, &(*Alloc)->Value,
+                          LVal.Base.getDynamicAllocType());
   } else {
     const Expr *Base = LVal.Base.dyn_cast<const Expr*>();
 
@@ -3284,11 +3661,14 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
         //   int x = ++r;
         //   constexpr int k = r;
         // Therefore we use the C++14 rules in C++11 too.
-        const ValueDecl *VD = Info.EvaluatingDecl.dyn_cast<const ValueDecl*>();
-        const ValueDecl *ED = MTE->getExtendingDecl();
+        //
+        // Note that temporaries whose lifetimes began while evaluating a
+        // variable's constructor are not usable while evaluating the
+        // corresponding destructor, not even if they're of const-qualified
+        // types.
         if (!(BaseType.isConstQualified() &&
               BaseType->isIntegralOrEnumerationType()) &&
-            !(VD && VD->getCanonicalDecl() == ED->getCanonicalDecl())) {
+            !lifetimeStartedInEvaluation(Info, LVal.Base)) {
           if (!IsAccess)
             return CompleteObject(LVal.getLValueBase(), nullptr, BaseType);
           Info.FFDiag(E, diag::note_constexpr_access_static_temporary, 1) << AK;
@@ -3340,15 +3720,22 @@ static CompleteObject findCompleteObject(EvalInfo &Info, const Expr *E,
 ///               case of a non-class type).
 /// \param LVal - The glvalue on which we are attempting to perform this action.
 /// \param RVal - The produced value will be placed here.
-static bool handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv,
-                                           QualType Type,
-                                           const LValue &LVal, APValue &RVal) {
+/// \param WantObjectRepresentation - If true, we're looking for the object
+///               representation rather than the value, and in particular,
+///               there is no requirement that the result be fully initialized.
+static bool
+handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv, QualType Type,
+                               const LValue &LVal, APValue &RVal,
+                               bool WantObjectRepresentation = false) {
   if (LVal.Designator.Invalid)
     return false;
 
   // Check for special cases where there is no existing APValue to look at.
   const Expr *Base = LVal.Base.dyn_cast<const Expr*>();
 
+  AccessKinds AK =
+      WantObjectRepresentation ? AK_ReadObjectRepresentation : AK_Read;
+
   if (Base && !LVal.getLValueCallIndex() && !Type.isVolatileQualified()) {
     if (const CompoundLiteralExpr *CLE = dyn_cast<CompoundLiteralExpr>(Base)) {
       // In C99, a CompoundLiteralExpr is an lvalue, and we defer evaluating the
@@ -3362,7 +3749,7 @@ static bool handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv,
       if (!Evaluate(Lit, Info, CLE->getInitializer()))
         return false;
       CompleteObject LitObj(LVal.Base, &Lit, Base->getType());
-      return extractSubobject(Info, Conv, LitObj, LVal.Designator, RVal);
+      return extractSubobject(Info, Conv, LitObj, LVal.Designator, RVal, AK);
     } else if (isa<StringLiteral>(Base) || isa<PredefinedExpr>(Base)) {
       // Special-case character extraction so we don't have to construct an
       // APValue for the whole string.
@@ -3377,7 +3764,7 @@ static bool handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv,
       }
       if (LVal.Designator.isOnePastTheEnd()) {
         if (Info.getLangOpts().CPlusPlus11)
-          Info.FFDiag(Conv, diag::note_constexpr_access_past_end) << AK_Read;
+          Info.FFDiag(Conv, diag::note_constexpr_access_past_end) << AK;
         else
           Info.FFDiag(Conv);
         return false;
@@ -3388,8 +3775,8 @@ static bool handleLValueToRValueConversion(EvalInfo &Info, const Expr *Conv,
     }
   }
 
-  CompleteObject Obj = findCompleteObject(Info, Conv, AK_Read, LVal, Type);
-  return Obj && extractSubobject(Info, Conv, Obj, LVal.Designator, RVal);
+  CompleteObject Obj = findCompleteObject(Info, Conv, AK, LVal, Type);
+  return Obj && extractSubobject(Info, Conv, Obj, LVal.Designator, RVal, AK);
 }
 
 /// Perform an assignment of Val to LVal. Takes ownership of Val.
@@ -3681,7 +4068,7 @@ static bool handleIncDec(EvalInfo &Info, const Expr *E, const LValue &LVal,
 /// Build an lvalue for the object argument of a member function call.
 static bool EvaluateObjectArgument(EvalInfo &Info, const Expr *Object,
                                    LValue &This) {
-  if (Object->getType()->isPointerType())
+  if (Object->getType()->isPointerType() && Object->isRValue())
     return EvaluatePointer(Object, This, Info);
 
   if (Object->isGLValue())
@@ -3843,6 +4230,40 @@ static bool HandleBaseToDerivedCast(EvalInfo &Info, const CastExpr *E,
   return CastToDerivedClass(Info, E, Result, TargetType, NewEntriesSize);
 }
 
+/// Get the value to use for a default-initialized object of type T.
+static APValue getDefaultInitValue(QualType T) {
+  if (auto *RD = T->getAsCXXRecordDecl()) {
+    if (RD->isUnion())
+      return APValue((const FieldDecl*)nullptr);
+
+    APValue Struct(APValue::UninitStruct(), RD->getNumBases(),
+                   std::distance(RD->field_begin(), RD->field_end()));
+
+    unsigned Index = 0;
+    for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
+           End = RD->bases_end(); I != End; ++I, ++Index)
+      Struct.getStructBase(Index) = getDefaultInitValue(I->getType());
+
+    for (const auto *I : RD->fields()) {
+      if (I->isUnnamedBitfield())
+        continue;
+      Struct.getStructField(I->getFieldIndex()) =
+          getDefaultInitValue(I->getType());
+    }
+    return Struct;
+  }
+
+  if (auto *AT =
+          dyn_cast_or_null<ConstantArrayType>(T->getAsArrayTypeUnsafe())) {
+    APValue Array(APValue::UninitArray(), 0, AT->getSize().getZExtValue());
+    if (Array.hasArrayFiller())
+      Array.getArrayFiller() = getDefaultInitValue(AT->getElementType());
+    return Array;
+  }
+
+  return APValue::IndeterminateValue();
+}
+
 namespace {
 enum EvalStmtResult {
   /// Evaluation failed.
@@ -3866,14 +4287,13 @@ static bool EvaluateVarDecl(EvalInfo &Info, const VarDecl *VD) {
     return true;
 
   LValue Result;
-  APValue &Val = createTemporary(VD, true, Result, *Info.CurrentCall);
+  APValue &Val =
+      Info.CurrentCall->createTemporary(VD, VD->getType(), true, Result);
 
   const Expr *InitE = VD->getInit();
   if (!InitE) {
-    Info.FFDiag(VD->getBeginLoc(), diag::note_constexpr_uninitialized)
-        << false << VD->getType();
-    Val = APValue();
-    return false;
+    Val = getDefaultInitValue(VD->getType());
+    return true;
   }
 
   if (InitE->isValueDependent())
@@ -3910,7 +4330,9 @@ static bool EvaluateCond(EvalInfo &Info, const VarDecl *CondDecl,
   FullExpressionRAII Scope(Info);
   if (CondDecl && !EvaluateDecl(Info, CondDecl))
     return false;
-  return EvaluateAsBooleanCondition(Cond, Result, Info);
+  if (!EvaluateAsBooleanCondition(Cond, Result, Info))
+    return false;
+  return Scope.destroy();
 }
 
 namespace {
@@ -3946,7 +4368,12 @@ static EvalStmtResult EvaluateLoopBody(StmtResult &Result, EvalInfo &Info,
                                        const Stmt *Body,
                                        const SwitchCase *Case = nullptr) {
   BlockScopeRAII Scope(Info);
-  switch (EvalStmtResult ESR = EvaluateStmt(Result, Info, Body, Case)) {
+
+  EvalStmtResult ESR = EvaluateStmt(Result, Info, Body, Case);
+  if (ESR != ESR_Failed && ESR != ESR_CaseNotFound && !Scope.destroy())
+    ESR = ESR_Failed;
+
+  switch (ESR) {
   case ESR_Break:
     return ESR_Succeeded;
   case ESR_Succeeded:
@@ -3968,17 +4395,23 @@ static EvalStmtResult EvaluateSwitch(StmtResult &Result, EvalInfo &Info,
   // Evaluate the switch condition.
   APSInt Value;
   {
-    FullExpressionRAII Scope(Info);
     if (const Stmt *Init = SS->getInit()) {
       EvalStmtResult ESR = EvaluateStmt(Result, Info, Init);
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          ESR = ESR_Failed;
         return ESR;
+      }
     }
+
+    FullExpressionRAII CondScope(Info);
     if (SS->getConditionVariable() &&
         !EvaluateDecl(Info, SS->getConditionVariable()))
       return ESR_Failed;
     if (!EvaluateInteger(SS->getCond(), Value, Info))
       return ESR_Failed;
+    if (!CondScope.destroy())
+      return ESR_Failed;
   }
 
   // Find the switch case corresponding to the value of the condition.
@@ -4002,10 +4435,14 @@ static EvalStmtResult EvaluateSwitch(StmtResult &Result, EvalInfo &Info,
   }
 
   if (!Found)
-    return ESR_Succeeded;
+    return Scope.destroy() ? ESR_Failed : ESR_Succeeded;
 
   // Search the switch body for the switch case and evaluate it from there.
-  switch (EvalStmtResult ESR = EvaluateStmt(Result, Info, SS->getBody(), Found)) {
+  EvalStmtResult ESR = EvaluateStmt(Result, Info, SS->getBody(), Found);
+  if (ESR != ESR_Failed && ESR != ESR_CaseNotFound && !Scope.destroy())
+    return ESR_Failed;
+
+  switch (ESR) {
   case ESR_Break:
     return ESR_Succeeded;
   case ESR_Succeeded:
@@ -4032,10 +4469,6 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
   // If we're hunting down a 'case' or 'default' label, recurse through
   // substatements until we hit the label.
   if (Case) {
-    // FIXME: We don't start the lifetime of objects whose initialization we
-    // jump over. However, such objects must be of class type with a trivial
-    // default constructor that initialize all subobjects, so must be empty,
-    // so this almost never matters.
     switch (S->getStmtClass()) {
     case Stmt::CompoundStmtClass:
       // FIXME: Precompute which substatement of a compound statement we
@@ -4061,10 +4494,35 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
       // preceded by our switch label.
       BlockScopeRAII Scope(Info);
 
+      // Step into the init statement in case it brings an (uninitialized)
+      // variable into scope.
+      if (const Stmt *Init = IS->getInit()) {
+        EvalStmtResult ESR = EvaluateStmt(Result, Info, Init, Case);
+        if (ESR != ESR_CaseNotFound) {
+          assert(ESR != ESR_Succeeded);
+          return ESR;
+        }
+      }
+
+      // Condition variable must be initialized if it exists.
+      // FIXME: We can skip evaluating the body if there's a condition
+      // variable, as there can't be any case labels within it.
+      // (The same is true for 'for' statements.)
+
       EvalStmtResult ESR = EvaluateStmt(Result, Info, IS->getThen(), Case);
-      if (ESR != ESR_CaseNotFound || !IS->getElse())
+      if (ESR == ESR_Failed)
+        return ESR;
+      if (ESR != ESR_CaseNotFound)
+        return Scope.destroy() ? ESR : ESR_Failed;
+      if (!IS->getElse())
+        return ESR_CaseNotFound;
+
+      ESR = EvaluateStmt(Result, Info, IS->getElse(), Case);
+      if (ESR == ESR_Failed)
         return ESR;
-      return EvaluateStmt(Result, Info, IS->getElse(), Case);
+      if (ESR != ESR_CaseNotFound)
+        return Scope.destroy() ? ESR : ESR_Failed;
+      return ESR_CaseNotFound;
     }
 
     case Stmt::WhileStmtClass: {
@@ -4077,21 +4535,47 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
 
     case Stmt::ForStmtClass: {
       const ForStmt *FS = cast<ForStmt>(S);
+      BlockScopeRAII Scope(Info);
+
+      // Step into the init statement in case it brings an (uninitialized)
+      // variable into scope.
+      if (const Stmt *Init = FS->getInit()) {
+        EvalStmtResult ESR = EvaluateStmt(Result, Info, Init, Case);
+        if (ESR != ESR_CaseNotFound) {
+          assert(ESR != ESR_Succeeded);
+          return ESR;
+        }
+      }
+
       EvalStmtResult ESR =
           EvaluateLoopBody(Result, Info, FS->getBody(), Case);
       if (ESR != ESR_Continue)
         return ESR;
       if (FS->getInc()) {
         FullExpressionRAII IncScope(Info);
-        if (!EvaluateIgnoredValue(Info, FS->getInc()))
+        if (!EvaluateIgnoredValue(Info, FS->getInc()) || !IncScope.destroy())
           return ESR_Failed;
       }
       break;
     }
 
-    case Stmt::DeclStmtClass:
-      // FIXME: If the variable has initialization that can't be jumped over,
-      // bail out of any immediately-surrounding compound-statement too.
+    case Stmt::DeclStmtClass: {
+      // Start the lifetime of any uninitialized variables we encounter. They
+      // might be used by the selected branch of the switch.
+      const DeclStmt *DS = cast<DeclStmt>(S);
+      for (const auto *D : DS->decls()) {
+        if (const auto *VD = dyn_cast<VarDecl>(D)) {
+          if (VD->hasLocalStorage() && !VD->getInit())
+            if (!EvaluateVarDecl(Info, VD))
+              return ESR_Failed;
+          // FIXME: If the variable has initialization that can't be jumped
+          // over, bail out of any immediately-surrounding compound-statement
+          // too. There can't be any case labels here.
+        }
+      }
+      return ESR_CaseNotFound;
+    }
+
     default:
       return ESR_CaseNotFound;
     }
@@ -4102,8 +4586,10 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
     if (const Expr *E = dyn_cast<Expr>(S)) {
       // Don't bother evaluating beyond an expression-statement which couldn't
       // be evaluated.
+      // FIXME: Do we need the FullExpressionRAII object here?
+      // VisitExprWithCleanups should create one when necessary.
       FullExpressionRAII Scope(Info);
-      if (!EvaluateIgnoredValue(Info, E))
+      if (!EvaluateIgnoredValue(Info, E) || !Scope.destroy())
         return ESR_Failed;
       return ESR_Succeeded;
     }
@@ -4116,12 +4602,12 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
 
   case Stmt::DeclStmtClass: {
     const DeclStmt *DS = cast<DeclStmt>(S);
-    for (const auto *DclIt : DS->decls()) {
+    for (const auto *D : DS->decls()) {
       // Each declaration initialization is its own full-expression.
-      // FIXME: This isn't quite right; if we're performing aggregate
-      // initialization, each braced subexpression is its own full-expression.
       FullExpressionRAII Scope(Info);
-      if (!EvaluateDecl(Info, DclIt) && !Info.noteFailure())
+      if (!EvaluateDecl(Info, D) && !Info.noteFailure())
+        return ESR_Failed;
+      if (!Scope.destroy())
         return ESR_Failed;
     }
     return ESR_Succeeded;
@@ -4135,7 +4621,7 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
               ? EvaluateInPlace(Result.Value, Info, *Result.Slot, RetExpr)
               : Evaluate(Result.Value, Info, RetExpr)))
       return ESR_Failed;
-    return ESR_Returned;
+    return Scope.destroy() ? ESR_Returned : ESR_Failed;
   }
 
   case Stmt::CompoundStmtClass: {
@@ -4146,10 +4632,15 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
       EvalStmtResult ESR = EvaluateStmt(Result, Info, BI, Case);
       if (ESR == ESR_Succeeded)
         Case = nullptr;
-      else if (ESR != ESR_CaseNotFound)
+      else if (ESR != ESR_CaseNotFound) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
     }
-    return Case ? ESR_CaseNotFound : ESR_Succeeded;
+    if (Case)
+      return ESR_CaseNotFound;
+    return Scope.destroy() ? ESR_Succeeded : ESR_Failed;
   }
 
   case Stmt::IfStmtClass: {
@@ -4159,8 +4650,11 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
     BlockScopeRAII Scope(Info);
     if (const Stmt *Init = IS->getInit()) {
       EvalStmtResult ESR = EvaluateStmt(Result, Info, Init);
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
     }
     bool Cond;
     if (!EvaluateCond(Info, IS->getConditionVariable(), IS->getCond(), Cond))
@@ -4168,10 +4662,13 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
 
     if (const Stmt *SubStmt = Cond ? IS->getThen() : IS->getElse()) {
       EvalStmtResult ESR = EvaluateStmt(Result, Info, SubStmt);
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
     }
-    return ESR_Succeeded;
+    return Scope.destroy() ? ESR_Succeeded : ESR_Failed;
   }
 
   case Stmt::WhileStmtClass: {
@@ -4186,8 +4683,13 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
         break;
 
       EvalStmtResult ESR = EvaluateLoopBody(Result, Info, WS->getBody());
-      if (ESR != ESR_Continue)
+      if (ESR != ESR_Continue) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
+      if (!Scope.destroy())
+        return ESR_Failed;
     }
     return ESR_Succeeded;
   }
@@ -4202,7 +4704,8 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
       Case = nullptr;
 
       FullExpressionRAII CondScope(Info);
-      if (!EvaluateAsBooleanCondition(DS->getCond(), Continue, Info))
+      if (!EvaluateAsBooleanCondition(DS->getCond(), Continue, Info) ||
+          !CondScope.destroy())
         return ESR_Failed;
     } while (Continue);
     return ESR_Succeeded;
@@ -4210,14 +4713,17 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
 
   case Stmt::ForStmtClass: {
     const ForStmt *FS = cast<ForStmt>(S);
-    BlockScopeRAII Scope(Info);
+    BlockScopeRAII ForScope(Info);
     if (FS->getInit()) {
       EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getInit());
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && !ForScope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
     }
     while (true) {
-      BlockScopeRAII Scope(Info);
+      BlockScopeRAII IterScope(Info);
       bool Continue = true;
       if (FS->getCond() && !EvaluateCond(Info, FS->getConditionVariable(),
                                          FS->getCond(), Continue))
@@ -4226,16 +4732,22 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
         break;
 
       EvalStmtResult ESR = EvaluateLoopBody(Result, Info, FS->getBody());
-      if (ESR != ESR_Continue)
+      if (ESR != ESR_Continue) {
+        if (ESR != ESR_Failed && (!IterScope.destroy() || !ForScope.destroy()))
+          return ESR_Failed;
         return ESR;
+      }
 
       if (FS->getInc()) {
         FullExpressionRAII IncScope(Info);
-        if (!EvaluateIgnoredValue(Info, FS->getInc()))
+        if (!EvaluateIgnoredValue(Info, FS->getInc()) || !IncScope.destroy())
           return ESR_Failed;
       }
+
+      if (!IterScope.destroy())
+        return ESR_Failed;
     }
-    return ESR_Succeeded;
+    return ForScope.destroy() ? ESR_Succeeded : ESR_Failed;
   }
 
   case Stmt::CXXForRangeStmtClass: {
@@ -4245,22 +4757,34 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
     // Evaluate the init-statement if present.
     if (FS->getInit()) {
       EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getInit());
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && !Scope.destroy())
+          return ESR_Failed;
         return ESR;
+      }
     }
 
     // Initialize the __range variable.
     EvalStmtResult ESR = EvaluateStmt(Result, Info, FS->getRangeStmt());
-    if (ESR != ESR_Succeeded)
+    if (ESR != ESR_Succeeded) {
+      if (ESR != ESR_Failed && !Scope.destroy())
+        return ESR_Failed;
       return ESR;
+    }
 
     // Create the __begin and __end iterators.
     ESR = EvaluateStmt(Result, Info, FS->getBeginStmt());
-    if (ESR != ESR_Succeeded)
+    if (ESR != ESR_Succeeded) {
+      if (ESR != ESR_Failed && !Scope.destroy())
+        return ESR_Failed;
       return ESR;
+    }
     ESR = EvaluateStmt(Result, Info, FS->getEndStmt());
-    if (ESR != ESR_Succeeded)
+    if (ESR != ESR_Succeeded) {
+      if (ESR != ESR_Failed && !Scope.destroy())
+        return ESR_Failed;
       return ESR;
+    }
 
     while (true) {
       // Condition: __begin != __end.
@@ -4276,20 +4800,29 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info,
       // User's variable declaration, initialized by *__begin.
       BlockScopeRAII InnerScope(Info);
       ESR = EvaluateStmt(Result, Info, FS->getLoopVarStmt());
-      if (ESR != ESR_Succeeded)
+      if (ESR != ESR_Succeeded) {
+        if (ESR != ESR_Failed && (!InnerScope.destroy() || !Scope.destroy()))
+          return ESR_Failed;
         return ESR;
+      }
 
       // Loop body.
       ESR = EvaluateLoopBody(Result, Info, FS->getBody());
-      if (ESR != ESR_Continue)
+      if (ESR != ESR_Continue) {
+        if (ESR != ESR_Failed && (!InnerScope.destroy() || !Scope.destroy()))
+          return ESR_Failed;
         return ESR;
+      }
 
       // Increment: ++__begin
       if (!EvaluateIgnoredValue(Info, FS->getInc()))
         return ESR_Failed;
+
+      if (!InnerScope.destroy())
+        return ESR_Failed;
     }
 
-    return ESR_Succeeded;
+    return Scope.destroy() ? ESR_Succeeded : ESR_Failed;
   }
 
   case Stmt::SwitchStmtClass:
@@ -4464,9 +4997,13 @@ static bool checkDynamicType(EvalInfo &Info, const Expr *E, const LValue &This,
 
 /// Check that the pointee of the 'this' pointer in a member function call is
 /// either within its lifetime or in its period of construction or destruction.
-static bool checkNonVirtualMemberCallThisPointer(EvalInfo &Info, const Expr *E,
-                                                 const LValue &This) {
-  return checkDynamicType(Info, E, This, AK_MemberCall, false);
+static bool
+checkNonVirtualMemberCallThisPointer(EvalInfo &Info, const Expr *E,
+                                     const LValue &This,
+                                     const CXXMethodDecl *NamedMember) {
+  return checkDynamicType(
+      Info, E, This,
+      isa<CXXDestructorDecl>(NamedMember) ? AK_Destroy : AK_MemberCall, false);
 }
 
 struct DynamicType {
@@ -4514,16 +5051,19 @@ static Optional<DynamicType> ComputeDynamicType(EvalInfo &Info, const Expr *E,
   ArrayRef<APValue::LValuePathEntry> Path = This.Designator.Entries;
   for (unsigned PathLength = This.Designator.MostDerivedPathLength;
        PathLength <= Path.size(); ++PathLength) {
-    switch (Info.isEvaluatingConstructor(This.getLValueBase(),
-                                         Path.slice(0, PathLength))) {
+    switch (Info.isEvaluatingCtorDtor(This.getLValueBase(),
+                                      Path.slice(0, PathLength))) {
     case ConstructionPhase::Bases:
-      // We're constructing a base class. This is not the dynamic type.
+    case ConstructionPhase::DestroyingBases:
+      // We're constructing or destroying a base class. This is not the dynamic
+      // type.
       break;
 
     case ConstructionPhase::None:
     case ConstructionPhase::AfterBases:
-      // We've finished constructing the base classes, so this is the dynamic
-      // type.
+    case ConstructionPhase::Destroying:
+      // We've finished constructing the base classes and not yet started
+      // destroying them again, so this is the dynamic type.
       return DynamicType{getBaseClassType(This.Designator, PathLength),
                          PathLength};
     }
@@ -4540,8 +5080,9 @@ static Optional<DynamicType> ComputeDynamicType(EvalInfo &Info, const Expr *E,
 static const CXXMethodDecl *HandleVirtualDispatch(
     EvalInfo &Info, const Expr *E, LValue &This, const CXXMethodDecl *Found,
     llvm::SmallVectorImpl<QualType> &CovariantAdjustmentPath) {
-  Optional<DynamicType> DynType =
-      ComputeDynamicType(Info, E, This, AK_MemberCall);
+  Optional<DynamicType> DynType = ComputeDynamicType(
+      Info, E, This,
+      isa<CXXDestructorDecl>(Found) ? AK_Destroy : AK_MemberCall);
   if (!DynType)
     return nullptr;
 
@@ -4677,8 +5218,7 @@ static bool HandleDynamicCast(EvalInfo &Info, const ExplicitCastExpr *E,
     if (!E->isGLValue()) {
       //   The value of a failed cast to pointer type is the null pointer value
       //   of the required result type.
-      auto TargetVal = Info.Ctx.getTargetNullPointerValue(E->getType());
-      Ptr.setNull(E->getType(), TargetVal);
+      Ptr.setNull(Info.Ctx, E->getType());
       return true;
     }
 
@@ -4743,39 +5283,6 @@ struct StartLifetimeOfUnionMemberHandler {
 
   static const AccessKinds AccessKind = AK_Assign;
 
-  APValue getDefaultInitValue(QualType SubobjType) {
-    if (auto *RD = SubobjType->getAsCXXRecordDecl()) {
-      if (RD->isUnion())
-        return APValue((const FieldDecl*)nullptr);
-
-      APValue Struct(APValue::UninitStruct(), RD->getNumBases(),
-                     std::distance(RD->field_begin(), RD->field_end()));
-
-      unsigned Index = 0;
-      for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
-             End = RD->bases_end(); I != End; ++I, ++Index)
-        Struct.getStructBase(Index) = getDefaultInitValue(I->getType());
-
-      for (const auto *I : RD->fields()) {
-        if (I->isUnnamedBitfield())
-          continue;
-        Struct.getStructField(I->getFieldIndex()) =
-            getDefaultInitValue(I->getType());
-      }
-      return Struct;
-    }
-
-    if (auto *AT = dyn_cast_or_null<ConstantArrayType>(
-            SubobjType->getAsArrayTypeUnsafe())) {
-      APValue Array(APValue::UninitArray(), 0, AT->getSize().getZExtValue());
-      if (Array.hasArrayFiller())
-        Array.getArrayFiller() = getDefaultInitValue(AT->getElementType());
-      return Array;
-    }
-
-    return APValue::IndeterminateValue();
-  }
-
   typedef bool result_type;
   bool failed() { return false; }
   bool found(APValue &Subobj, QualType SubobjType) {
@@ -4788,7 +5295,8 @@ struct StartLifetimeOfUnionMemberHandler {
     //  * No variant members' lifetimes begin
     //  * All scalar subobjects whose lifetimes begin have indeterminate values
     assert(SubobjType->isUnionType());
-    if (!declaresSameEntity(Subobj.getUnionField(), Field))
+    if (!declaresSameEntity(Subobj.getUnionField(), Field) ||
+        !Subobj.getUnionValue().hasValue())
       Subobj.setUnion(Field, getDefaultInitValue(Field->getType()));
     return true;
   }
@@ -4820,7 +5328,9 @@ static bool HandleUnionActiveMemberChange(EvalInfo &Info, const Expr *LHSExpr,
     //   -- If E is of the form A.B, S(E) contains the elements of S(A)...
     if (auto *ME = dyn_cast<MemberExpr>(E)) {
       auto *FD = dyn_cast<FieldDecl>(ME->getMemberDecl());
-      if (!FD)
+      // Note that we can't implicitly start the lifetime of a reference,
+      // so we don't need to proceed any further if we reach one.
+      if (!FD || FD->getType()->isReferenceType())
         break;
 
       //    ... and also contains A.B if B names a union member
@@ -4981,8 +5491,8 @@ static bool HandleFunctionCall(SourceLocation CallLoc,
     LValue RHS;
     RHS.setFrom(Info.Ctx, ArgValues[0]);
     APValue RHSValue;
-    if (!handleLValueToRValueConversion(Info, Args[0], Args[0]->getType(),
-                                        RHS, RHSValue))
+    if (!handleLValueToRValueConversion(Info, Args[0], Args[0]->getType(), RHS,
+                                        RHSValue, MD->getParent()->isUnion()))
       return false;
     if (Info.getLangOpts().CPlusPlus2a && MD->isTrivial() &&
         !HandleUnionActiveMemberChange(Info, Args[0], *This))
@@ -5045,7 +5555,8 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
     CXXConstructorDecl::init_const_iterator I = Definition->init_begin();
     {
       FullExpressionRAII InitScope(Info);
-      if (!EvaluateInPlace(Result, Info, This, (*I)->getInit()))
+      if (!EvaluateInPlace(Result, Info, This, (*I)->getInit()) ||
+          !InitScope.destroy())
         return false;
     }
     return EvaluateStmt(Ret, Info, Definition->getBody()) != ESR_Failed;
@@ -5066,7 +5577,7 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
     RHS.setFrom(Info.Ctx, ArgValues[0]);
     return handleLValueToRValueConversion(
         Info, E, Definition->getParamDecl(0)->getType().getNonReferenceType(),
-        RHS, Result);
+        RHS, Result, Definition->getParent()->isUnion());
   }
 
   // Reserve space for the struct members.
@@ -5085,6 +5596,25 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
 #ifndef NDEBUG
   CXXRecordDecl::base_class_const_iterator BaseIt = RD->bases_begin();
 #endif
+  CXXRecordDecl::field_iterator FieldIt = RD->field_begin();
+  auto SkipToField = [&](FieldDecl *FD, bool Indirect) {
+    // We might be initializing the same field again if this is an indirect
+    // field initialization.
+    if (FieldIt == RD->field_end() ||
+        FieldIt->getFieldIndex() > FD->getFieldIndex()) {
+      assert(Indirect && "fields out of order?");
+      return;
+    }
+
+    // Default-initialize any fields with no explicit initializer.
+    for (; !declaresSameEntity(*FieldIt, FD); ++FieldIt) {
+      assert(FieldIt != RD->field_end() && "missing field?");
+      if (!FieldIt->isUnnamedBitfield())
+        Result.getStructField(FieldIt->getFieldIndex()) =
+            getDefaultInitValue(FieldIt->getType());
+    }
+    ++FieldIt;
+  };
   for (const auto *I : Definition->inits()) {
     LValue Subobject = This;
     LValue SubobjectParent = This;
@@ -5113,6 +5643,7 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
         Result = APValue(FD);
         Value = &Result.getUnionValue();
       } else {
+        SkipToField(FD, false);
         Value = &Result.getStructField(FD->getFieldIndex());
       }
     } else if (IndirectFieldDecl *IFD = I->getIndirectMember()) {
@@ -5132,8 +5663,10 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
           if (CD->isUnion())
             *Value = APValue(FD);
           else
-            *Value = APValue(APValue::UninitStruct(), CD->getNumBases(),
-                             std::distance(CD->field_begin(), CD->field_end()));
+            // FIXME: This immediately starts the lifetime of all members of an
+            // anonymous struct. It would be preferable to strictly start member
+            // lifetime in initialization order.
+            *Value = getDefaultInitValue(Info.Ctx.getRecordType(CD));
         }
         // Store Subobject as its parent before updating it for the last element
         // in the chain.
@@ -5143,50 +5676,431 @@ static bool HandleConstructorCall(const Expr *E, const LValue &This,
           return false;
         if (CD->isUnion())
           Value = &Value->getUnionValue();
-        else
+        else {
+          if (C == IndirectFieldChain.front() && !RD->isUnion())
+            SkipToField(FD, true);
           Value = &Value->getStructField(FD->getFieldIndex());
+        }
       }
     } else {
       llvm_unreachable("unknown base initializer kind");
     }
 
-    // Need to override This for implicit field initializers as in this case
-    // This refers to innermost anonymous struct/union containing initializer,
-    // not to currently constructed class.
-    const Expr *Init = I->getInit();
-    ThisOverrideRAII ThisOverride(*Info.CurrentCall, &SubobjectParent,
-                                  isa<CXXDefaultInitExpr>(Init));
-    FullExpressionRAII InitScope(Info);
-    if (!EvaluateInPlace(*Value, Info, Subobject, Init) ||
-        (FD && FD->isBitField() &&
-         !truncateBitfieldValue(Info, Init, *Value, FD))) {
-      // If we're checking for a potential constant expression, evaluate all
-      // initializers even if some of them fail.
-      if (!Info.noteFailure())
-        return false;
-      Success = false;
+    // Need to override This for implicit field initializers as in this case
+    // This refers to innermost anonymous struct/union containing initializer,
+    // not to currently constructed class.
+    const Expr *Init = I->getInit();
+    ThisOverrideRAII ThisOverride(*Info.CurrentCall, &SubobjectParent,
+                                  isa<CXXDefaultInitExpr>(Init));
+    FullExpressionRAII InitScope(Info);
+    if (!EvaluateInPlace(*Value, Info, Subobject, Init) ||
+        (FD && FD->isBitField() &&
+         !truncateBitfieldValue(Info, Init, *Value, FD))) {
+      // If we're checking for a potential constant expression, evaluate all
+      // initializers even if some of them fail.
+      if (!Info.noteFailure())
+        return false;
+      Success = false;
+    }
+
+    // This is the point at which the dynamic type of the object becomes this
+    // class type.
+    if (I->isBaseInitializer() && BasesSeen == RD->getNumBases())
+      EvalObj.finishedConstructingBases();
+  }
+
+  // Default-initialize any remaining fields.
+  if (!RD->isUnion()) {
+    for (; FieldIt != RD->field_end(); ++FieldIt) {
+      if (!FieldIt->isUnnamedBitfield())
+        Result.getStructField(FieldIt->getFieldIndex()) =
+            getDefaultInitValue(FieldIt->getType());
+    }
+  }
+
+  return Success &&
+         EvaluateStmt(Ret, Info, Definition->getBody()) != ESR_Failed &&
+         LifetimeExtendedScope.destroy();
+}
+
+static bool HandleConstructorCall(const Expr *E, const LValue &This,
+                                  ArrayRef<const Expr*> Args,
+                                  const CXXConstructorDecl *Definition,
+                                  EvalInfo &Info, APValue &Result) {
+  ArgVector ArgValues(Args.size());
+  if (!EvaluateArgs(Args, ArgValues, Info, Definition))
+    return false;
+
+  return HandleConstructorCall(E, This, ArgValues.data(), Definition,
+                               Info, Result);
+}
+
+static bool HandleDestructionImpl(EvalInfo &Info, SourceLocation CallLoc,
+                                  const LValue &This, APValue &Value,
+                                  QualType T) {
+  // Objects can only be destroyed while they're within their lifetimes.
+  // FIXME: We have no representation for whether an object of type nullptr_t
+  // is in its lifetime; it usually doesn't matter. Perhaps we should model it
+  // as indeterminate instead?
+  if (Value.isAbsent() && !T->isNullPtrType()) {
+    APValue Printable;
+    This.moveInto(Printable);
+    Info.FFDiag(CallLoc, diag::note_constexpr_destroy_out_of_lifetime)
+      << Printable.getAsString(Info.Ctx, Info.Ctx.getLValueReferenceType(T));
+    return false;
+  }
+
+  // Invent an expression for location purposes.
+  // FIXME: We shouldn't need to do this.
+  OpaqueValueExpr LocE(CallLoc, Info.Ctx.IntTy, VK_RValue);
+
+  // For arrays, destroy elements right-to-left.
+  if (const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(T)) {
+    uint64_t Size = CAT->getSize().getZExtValue();
+    QualType ElemT = CAT->getElementType();
+
+    LValue ElemLV = This;
+    ElemLV.addArray(Info, &LocE, CAT);
+    if (!HandleLValueArrayAdjustment(Info, &LocE, ElemLV, ElemT, Size))
+      return false;
+
+    // Ensure that we have actual array elements available to destroy; the
+    // destructors might mutate the value, so we can't run them on the array
+    // filler.
+    if (Size && Size > Value.getArrayInitializedElts())
+      expandArray(Value, Value.getArraySize() - 1);
+
+    for (; Size != 0; --Size) {
+      APValue &Elem = Value.getArrayInitializedElt(Size - 1);
+      if (!HandleLValueArrayAdjustment(Info, &LocE, ElemLV, ElemT, -1) ||
+          !HandleDestructionImpl(Info, CallLoc, ElemLV, Elem, ElemT))
+        return false;
+    }
+
+    // End the lifetime of this array now.
+    Value = APValue();
+    return true;
+  }
+
+  const CXXRecordDecl *RD = T->getAsCXXRecordDecl();
+  if (!RD) {
+    if (T.isDestructedType()) {
+      Info.FFDiag(CallLoc, diag::note_constexpr_unsupported_destruction) << T;
+      return false;
+    }
+
+    Value = APValue();
+    return true;
+  }
+
+  if (RD->getNumVBases()) {
+    Info.FFDiag(CallLoc, diag::note_constexpr_virtual_base) << RD;
+    return false;
+  }
+
+  const CXXDestructorDecl *DD = RD->getDestructor();
+  if (!DD && !RD->hasTrivialDestructor()) {
+    Info.FFDiag(CallLoc);
+    return false;
+  }
+
+  if (!DD || DD->isTrivial() ||
+      (RD->isAnonymousStructOrUnion() && RD->isUnion())) {
+    // A trivial destructor just ends the lifetime of the object. Check for
+    // this case before checking for a body, because we might not bother
+    // building a body for a trivial destructor. Note that it doesn't matter
+    // whether the destructor is constexpr in this case; all trivial
+    // destructors are constexpr.
+    //
+    // If an anonymous union would be destroyed, some enclosing destructor must
+    // have been explicitly defined, and the anonymous union destruction should
+    // have no effect.
+    Value = APValue();
+    return true;
+  }
+
+  if (!Info.CheckCallLimit(CallLoc))
+    return false;
+
+  const FunctionDecl *Definition = nullptr;
+  const Stmt *Body = DD->getBody(Definition);
+
+  if (!CheckConstexprFunction(Info, CallLoc, DD, Definition, Body))
+    return false;
+
+  CallStackFrame Frame(Info, CallLoc, Definition, &This, nullptr);
+
+  // We're now in the period of destruction of this object.
+  unsigned BasesLeft = RD->getNumBases();
+  EvalInfo::EvaluatingDestructorRAII EvalObj(
+      Info,
+      ObjectUnderConstruction{This.getLValueBase(), This.Designator.Entries});
+  if (!EvalObj.DidInsert) {
+    // C++2a [class.dtor]p19:
+    //   the behavior is undefined if the destructor is invoked for an object
+    //   whose lifetime has ended
+    // (Note that formally the lifetime ends when the period of destruction
+    // begins, even though certain uses of the object remain valid until the
+    // period of destruction ends.)
+    Info.FFDiag(CallLoc, diag::note_constexpr_double_destroy);
+    return false;
+  }
+
+  // FIXME: Creating an APValue just to hold a nonexistent return value is
+  // wasteful.
+  APValue RetVal;
+  StmtResult Ret = {RetVal, nullptr};
+  if (EvaluateStmt(Ret, Info, Definition->getBody()) == ESR_Failed)
+    return false;
+
+  // A union destructor does not implicitly destroy its members.
+  if (RD->isUnion())
+    return true;
+
+  const ASTRecordLayout &Layout = Info.Ctx.getASTRecordLayout(RD);
+
+  // We don't have a good way to iterate fields in reverse, so collect all the
+  // fields first and then walk them backwards.
+  SmallVector<FieldDecl*, 16> Fields(RD->field_begin(), RD->field_end());
+  for (const FieldDecl *FD : llvm::reverse(Fields)) {
+    if (FD->isUnnamedBitfield())
+      continue;
+
+    LValue Subobject = This;
+    if (!HandleLValueMember(Info, &LocE, Subobject, FD, &Layout))
+      return false;
+
+    APValue *SubobjectValue = &Value.getStructField(FD->getFieldIndex());
+    if (!HandleDestructionImpl(Info, CallLoc, Subobject, *SubobjectValue,
+                               FD->getType()))
+      return false;
+  }
+
+  if (BasesLeft != 0)
+    EvalObj.startedDestroyingBases();
+
+  // Destroy base classes in reverse order.
+  for (const CXXBaseSpecifier &Base : llvm::reverse(RD->bases())) {
+    --BasesLeft;
+
+    QualType BaseType = Base.getType();
+    LValue Subobject = This;
+    if (!HandleLValueDirectBase(Info, &LocE, Subobject, RD,
+                                BaseType->getAsCXXRecordDecl(), &Layout))
+      return false;
+
+    APValue *SubobjectValue = &Value.getStructBase(BasesLeft);
+    if (!HandleDestructionImpl(Info, CallLoc, Subobject, *SubobjectValue,
+                               BaseType))
+      return false;
+  }
+  assert(BasesLeft == 0 && "NumBases was wrong?");
+
+  // The period of destruction ends now. The object is gone.
+  Value = APValue();
+  return true;
+}
+
+namespace {
+struct DestroyObjectHandler {
+  EvalInfo &Info;
+  const Expr *E;
+  const LValue &This;
+  const AccessKinds AccessKind;
+
+  typedef bool result_type;
+  bool failed() { return false; }
+  bool found(APValue &Subobj, QualType SubobjType) {
+    return HandleDestructionImpl(Info, E->getExprLoc(), This, Subobj,
+                                 SubobjType);
+  }
+  bool found(APSInt &Value, QualType SubobjType) {
+    Info.FFDiag(E, diag::note_constexpr_destroy_complex_elem);
+    return false;
+  }
+  bool found(APFloat &Value, QualType SubobjType) {
+    Info.FFDiag(E, diag::note_constexpr_destroy_complex_elem);
+    return false;
+  }
+};
+}
+
+/// Perform a destructor or pseudo-destructor call on the given object, which
+/// might in general not be a complete object.
+static bool HandleDestruction(EvalInfo &Info, const Expr *E,
+                              const LValue &This, QualType ThisType) {
+  CompleteObject Obj = findCompleteObject(Info, E, AK_Destroy, This, ThisType);
+  DestroyObjectHandler Handler = {Info, E, This, AK_Destroy};
+  return Obj && findSubobject(Info, E, Obj, This.Designator, Handler);
+}
+
+/// Destroy and end the lifetime of the given complete object.
+static bool HandleDestruction(EvalInfo &Info, SourceLocation Loc,
+                              APValue::LValueBase LVBase, APValue &Value,
+                              QualType T) {
+  // If we've had an unmodeled side-effect, we can't rely on mutable state
+  // (such as the object we're about to destroy) being correct.
+  if (Info.EvalStatus.HasSideEffects)
+    return false;
+
+  LValue LV;
+  LV.set({LVBase});
+  return HandleDestructionImpl(Info, Loc, LV, Value, T);
+}
+
+/// Perform a call to 'perator new' or to `__builtin_operator_new'.
+static bool HandleOperatorNewCall(EvalInfo &Info, const CallExpr *E,
+                                  LValue &Result) {
+  if (Info.checkingPotentialConstantExpression() ||
+      Info.SpeculativeEvaluationDepth)
+    return false;
+
+  // This is permitted only within a call to std::allocator<T>::allocate.
+  auto Caller = Info.getStdAllocatorCaller("allocate");
+  if (!Caller) {
+    Info.FFDiag(E->getExprLoc(), Info.getLangOpts().CPlusPlus2a
+                                     ? diag::note_constexpr_new_untyped
+                                     : diag::note_constexpr_new);
+    return false;
+  }
+
+  QualType ElemType = Caller.ElemType;
+  if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
+    Info.FFDiag(E->getExprLoc(),
+                diag::note_constexpr_new_not_complete_object_type)
+        << (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
+    return false;
+  }
+
+  APSInt ByteSize;
+  if (!EvaluateInteger(E->getArg(0), ByteSize, Info))
+    return false;
+  bool IsNothrow = false;
+  for (unsigned I = 1, N = E->getNumArgs(); I != N; ++I) {
+    EvaluateIgnoredValue(Info, E->getArg(I));
+    IsNothrow |= E->getType()->isNothrowT();
+  }
+
+  CharUnits ElemSize;
+  if (!HandleSizeof(Info, E->getExprLoc(), ElemType, ElemSize))
+    return false;
+  APInt Size, Remainder;
+  APInt ElemSizeAP(ByteSize.getBitWidth(), ElemSize.getQuantity());
+  APInt::udivrem(ByteSize, ElemSizeAP, Size, Remainder);
+  if (Remainder != 0) {
+    // This likely indicates a bug in the implementation of 'std::allocator'.
+    Info.FFDiag(E->getExprLoc(), diag::note_constexpr_operator_new_bad_size)
+        << ByteSize << APSInt(ElemSizeAP, true) << ElemType;
+    return false;
+  }
+
+  if (ByteSize.getActiveBits() > ConstantArrayType::getMaxSizeBits(Info.Ctx)) {
+    if (IsNothrow) {
+      Result.setNull(Info.Ctx, E->getType());
+      return true;
     }
 
-    // This is the point at which the dynamic type of the object becomes this
-    // class type.
-    if (I->isBaseInitializer() && BasesSeen == RD->getNumBases())
-      EvalObj.finishedConstructingBases();
+    Info.FFDiag(E, diag::note_constexpr_new_too_large) << APSInt(Size, true);
+    return false;
   }
 
-  return Success &&
-         EvaluateStmt(Ret, Info, Definition->getBody()) != ESR_Failed;
+  QualType AllocType =
+      Info.Ctx.getConstantArrayType(ElemType, Size, ArrayType::Normal, 0);
+  APValue *Val = Info.createHeapAlloc(E, AllocType, Result);
+  *Val = APValue(APValue::UninitArray(), 0, Size.getZExtValue());
+  Result.addArray(Info, E, cast<ConstantArrayType>(AllocType));
+  return true;
 }
 
-static bool HandleConstructorCall(const Expr *E, const LValue &This,
-                                  ArrayRef<const Expr*> Args,
-                                  const CXXConstructorDecl *Definition,
-                                  EvalInfo &Info, APValue &Result) {
-  ArgVector ArgValues(Args.size());
-  if (!EvaluateArgs(Args, ArgValues, Info, Definition))
+static bool hasVirtualDestructor(QualType T) {
+  if (CXXRecordDecl *RD = T->getAsCXXRecordDecl())
+    if (CXXDestructorDecl *DD = RD->getDestructor())
+      return DD->isVirtual();
+  return false;
+}
+
+/// Check that the given object is a suitable pointer to a heap allocation that
+/// still exists and is of the right kind for the purpose of a deletion.
+///
+/// On success, returns the heap allocation to deallocate. On failure, produces
+/// a diagnostic and returns None.
+static Optional<DynAlloc *> CheckDeleteKind(EvalInfo &Info, const Expr *E,
+                                            const LValue &Pointer,
+                                            DynAlloc::Kind DeallocKind) {
+  auto PointerAsString = [&] {
+    return Pointer.toString(Info.Ctx, Info.Ctx.VoidPtrTy);
+  };
+
+  DynamicAllocLValue DA = Pointer.Base.dyn_cast<DynamicAllocLValue>();
+  if (!DA) {
+    Info.FFDiag(E, diag::note_constexpr_delete_not_heap_alloc)
+        << PointerAsString();
+    if (Pointer.Base)
+      NoteLValueLocation(Info, Pointer.Base);
+    return None;
+  }
+
+  Optional<DynAlloc *> Alloc = Info.lookupDynamicAlloc(DA);
+  if (!Alloc) {
+    Info.FFDiag(E, diag::note_constexpr_double_delete);
+    return None;
+  }
+
+  QualType AllocType = Pointer.Base.getDynamicAllocType();
+  if (DeallocKind != (*Alloc)->getKind()) {
+    Info.FFDiag(E, diag::note_constexpr_new_delete_mismatch)
+        << DeallocKind << (*Alloc)->getKind() << AllocType;
+    NoteLValueLocation(Info, Pointer.Base);
+    return None;
+  }
+
+  bool Subobject = false;
+  if (DeallocKind == DynAlloc::New) {
+    Subobject = Pointer.Designator.MostDerivedPathLength != 0 ||
+                Pointer.Designator.isOnePastTheEnd();
+  } else {
+    Subobject = Pointer.Designator.Entries.size() != 1 ||
+                Pointer.Designator.Entries[0].getAsArrayIndex() != 0;
+  }
+  if (Subobject) {
+    Info.FFDiag(E, diag::note_constexpr_delete_subobject)
+        << PointerAsString() << Pointer.Designator.isOnePastTheEnd();
+    return None;
+  }
+
+  return Alloc;
+}
+
+// Perform a call to 'operator delete' or '__builtin_operator_delete'.
+bool HandleOperatorDeleteCall(EvalInfo &Info, const CallExpr *E) {
+  if (Info.checkingPotentialConstantExpression() ||
+      Info.SpeculativeEvaluationDepth)
     return false;
 
-  return HandleConstructorCall(E, This, ArgValues.data(), Definition,
-                               Info, Result);
+  // This is permitted only within a call to std::allocator<T>::deallocate.
+  if (!Info.getStdAllocatorCaller("deallocate")) {
+    Info.FFDiag(E->getExprLoc());
+    return true;
+  }
+
+  LValue Pointer;
+  if (!EvaluatePointer(E->getArg(0), Pointer, Info))
+    return false;
+  for (unsigned I = 1, N = E->getNumArgs(); I != N; ++I)
+    EvaluateIgnoredValue(Info, E->getArg(I));
+
+  if (Pointer.Designator.Invalid)
+    return false;
+
+  // Deleting a null pointer has no effect.
+  if (Pointer.isNullPointer())
+    return true;
+
+  if (!CheckDeleteKind(Info, E, Pointer, DynAlloc::StdAllocator))
+    return false;
+
+  Info.HeapAllocs.erase(Pointer.Base.get<DynamicAllocLValue>());
+  return true;
 }
 
 //===----------------------------------------------------------------------===//
@@ -5567,7 +6481,7 @@ class BufferToAPValueConverter {
 #define NON_CANONICAL_UNLESS_DEPENDENT(Class, Base)                            \
   case Type::Class:                                                            \
     llvm_unreachable("either dependent or not canonical!");
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
     llvm_unreachable("Unhandled Type::TypeClass");
   }
@@ -5658,9 +6572,9 @@ static bool handleLValueToRValueBitCast(EvalInfo &Info, APValue &DestValue,
   LValue SourceLValue;
   APValue SourceRValue;
   SourceLValue.setFrom(Info.Ctx, SourceValue);
-  if (!handleLValueToRValueConversion(Info, BCE,
-                                      BCE->getSubExpr()->getType().withConst(),
-                                      SourceLValue, SourceRValue))
+  if (!handleLValueToRValueConversion(
+          Info, BCE, BCE->getSubExpr()->getType().withConst(), SourceLValue,
+          SourceRValue, /*WantObjectRepresentation=*/true))
     return false;
 
   // Read out SourceValue into a char buffer.
@@ -5799,10 +6713,16 @@ class ExprEvaluatorBase
     return StmtVisitorTy::Visit(E->getExpr());
   }
 
-  // We cannot create any objects for which cleanups are required, so there is
-  // nothing to do here; all cleanups must come from unevaluated subexpressions.
-  bool VisitExprWithCleanups(const ExprWithCleanups *E)
-    { return StmtVisitorTy::Visit(E->getSubExpr()); }
+  bool VisitExprWithCleanups(const ExprWithCleanups *E) {
+    FullExpressionRAII Scope(Info);
+    return StmtVisitorTy::Visit(E->getSubExpr()) && Scope.destroy();
+  }
+
+  // Temporaries are registered when created, so we don't care about
+  // CXXBindTemporaryExpr.
+  bool VisitCXXBindTemporaryExpr(const CXXBindTemporaryExpr *E) {
+    return StmtVisitorTy::Visit(E->getSubExpr());
+  }
 
   bool VisitCXXReinterpretCastExpr(const CXXReinterpretCastExpr *E) {
     CCEDiag(E, diag::note_constexpr_invalid_cast) << 0;
@@ -5842,7 +6762,11 @@ class ExprEvaluatorBase
   bool VisitBinaryConditionalOperator(const BinaryConditionalOperator *E) {
     // Evaluate and cache the common expression. We treat it as a temporary,
     // even though it's not quite the same thing.
-    if (!Evaluate(Info.CurrentCall->createTemporary(E->getOpaqueValue(), false),
+    LValue CommonLV;
+    if (!Evaluate(Info.CurrentCall->createTemporary(
+                      E->getOpaqueValue(),
+                      getStorageType(Info.Ctx, E->getOpaqueValue()), false,
+                      CommonLV),
                   Info, E->getCommon()))
       return false;
 
@@ -5921,11 +6845,21 @@ class ExprEvaluatorBase
         HasQualifier = ME->hasQualifier();
       } else if (const BinaryOperator *BE = dyn_cast<BinaryOperator>(Callee)) {
         // Indirect bound member calls ('.*' or '->*').
-        Member = dyn_cast_or_null<CXXMethodDecl>(
-            HandleMemberPointerAccess(Info, BE, ThisVal, false));
+        const ValueDecl *D =
+            HandleMemberPointerAccess(Info, BE, ThisVal, false);
+        if (!D)
+          return false;
+        Member = dyn_cast<CXXMethodDecl>(D);
         if (!Member)
           return Error(Callee);
         This = &ThisVal;
+      } else if (const auto *PDE = dyn_cast<CXXPseudoDestructorExpr>(Callee)) {
+        if (!Info.getLangOpts().CPlusPlus2a)
+          Info.CCEDiag(PDE, diag::note_constexpr_pseudo_destructor);
+        // FIXME: If pseudo-destructor calls ever start ending the lifetime of
+        // their callee, we should start calling HandleDestruction here.
+        // For now, we just evaluate the object argument and discard it.
+        return EvaluateObjectArgument(Info, PDE->getBase(), ThisVal);
       } else
         return Error(Callee);
       FD = Member;
@@ -5994,6 +6928,17 @@ class ExprEvaluatorBase
           FD = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization);
         } else
           FD = LambdaCallOp;
+      } else if (FD->isReplaceableGlobalAllocationFunction()) {
+        if (FD->getDeclName().getCXXOverloadedOperator() == OO_New ||
+            FD->getDeclName().getCXXOverloadedOperator() == OO_Array_New) {
+          LValue Ptr;
+          if (!HandleOperatorNewCall(Info, E, Ptr))
+            return false;
+          Ptr.moveInto(Result);
+          return true;
+        } else {
+          return HandleOperatorDeleteCall(Info, E);
+        }
       }
     } else
       return Error(E);
@@ -6009,11 +6954,20 @@ class ExprEvaluatorBase
           return false;
       } else {
         // Check that the 'this' pointer points to an object of the right type.
-        if (!checkNonVirtualMemberCallThisPointer(Info, E, *This))
+        // FIXME: If this is an assignment operator call, we may need to change
+        // the active union member before we check this.
+        if (!checkNonVirtualMemberCallThisPointer(Info, E, *This, NamedMember))
           return false;
       }
     }
 
+    // Destructor calls are different enough that they have their own codepath.
+    if (auto *DD = dyn_cast<CXXDestructorDecl>(FD)) {
+      assert(This && "no 'this' pointer for destructor call");
+      return HandleDestruction(Info, E, *This,
+                               Info.Ctx.getRecordType(DD->getParent()));
+    }
+
     const FunctionDecl *Definition = nullptr;
     Stmt *Body = FD->getBody(Definition);
 
@@ -6149,11 +7103,11 @@ class ExprEvaluatorBase
     if (Info.checkingForUndefinedBehavior())
       return Error(E);
 
-    BlockScopeRAII Scope(Info);
     const CompoundStmt *CS = E->getSubStmt();
     if (CS->body_empty())
       return true;
 
+    BlockScopeRAII Scope(Info);
     for (CompoundStmt::const_body_iterator BI = CS->body_begin(),
                                            BE = CS->body_end();
          /**/; ++BI) {
@@ -6164,7 +7118,7 @@ class ExprEvaluatorBase
                       diag::note_constexpr_stmt_expr_unsupported);
           return false;
         }
-        return this->Visit(FinalExpr);
+        return this->Visit(FinalExpr) && Scope.destroy();
       }
 
       APValue ReturnValue;
@@ -6513,16 +7467,14 @@ bool LValueExprEvaluator::VisitMaterializeTemporaryExpr(
     *Value = APValue();
     Result.set(E);
   } else {
-    Value = &createTemporary(E, E->getStorageDuration() == SD_Automatic, Result,
-                             *Info.CurrentCall);
+    Value = &Info.CurrentCall->createTemporary(
+        E, E->getType(), E->getStorageDuration() == SD_Automatic, Result);
   }
 
   QualType Type = Inner->getType();
 
   // Materialize the temporary itself.
-  if (!EvaluateInPlace(*Value, Info, Result, Inner) ||
-      (E->getStorageDuration() == SD_Static &&
-       !CheckConstantExpression(Info, E->getExprLoc(), Type, *Value))) {
+  if (!EvaluateInPlace(*Value, Info, Result, Inner)) {
     *Value = APValue();
     return false;
   }
@@ -6852,8 +7804,7 @@ class PointerExprEvaluator
     return true;
   }
   bool ZeroInitialization(const Expr *E) {
-    auto TargetVal = Info.Ctx.getTargetNullPointerValue(E->getType());
-    Result.setNull(E->getType(), TargetVal);
+    Result.setNull(Info.Ctx, E->getType());
     return true;
   }
 
@@ -6914,6 +7865,8 @@ class PointerExprEvaluator
     return true;
   }
 
+  bool VisitCXXNewExpr(const CXXNewExpr *E);
+
   bool VisitSourceLocExpr(const SourceLocExpr *E) {
     assert(E->isStringType() && "SourceLocExpr isn't a pointer type?");
     APValue LValResult = E->EvaluateInContext(
@@ -6978,12 +7931,22 @@ bool PointerExprEvaluator::VisitCastExpr(const CastExpr *E) {
     // permitted in constant expressions in C++11. Bitcasts from cv void* are
     // also static_casts, but we disallow them as a resolution to DR1312.
     if (!E->getType()->isVoidPointerType()) {
-      Result.Designator.setInvalid();
-      if (SubExpr->getType()->isVoidPointerType())
-        CCEDiag(E, diag::note_constexpr_invalid_cast)
-          << 3 << SubExpr->getType();
-      else
-        CCEDiag(E, diag::note_constexpr_invalid_cast) << 2;
+      if (!Result.InvalidBase && !Result.Designator.Invalid &&
+          !Result.IsNullPtr &&
+          Info.Ctx.hasSameUnqualifiedType(Result.Designator.getType(Info.Ctx),
+                                          E->getType()->getPointeeType()) &&
+          Info.getStdAllocatorCaller("allocate")) {
+        // Inside a call to std::allocator::allocate and friends, we permit
+        // casting from void* back to cv1 T* for a pointer that points to a
+        // cv2 T.
+      } else {
+        Result.Designator.setInvalid();
+        if (SubExpr->getType()->isVoidPointerType())
+          CCEDiag(E, diag::note_constexpr_invalid_cast)
+            << 3 << SubExpr->getType();
+        else
+          CCEDiag(E, diag::note_constexpr_invalid_cast) << 2;
+      }
     }
     if (E->getCastKind() == CK_AddressSpaceConversion && Result.IsNullPtr)
       ZeroInitialization(E);
@@ -7046,8 +8009,8 @@ bool PointerExprEvaluator::VisitCastExpr(const CastExpr *E) {
       if (!evaluateLValue(SubExpr, Result))
         return false;
     } else {
-      APValue &Value = createTemporary(SubExpr, false, Result,
-                                       *Info.CurrentCall);
+      APValue &Value = Info.CurrentCall->createTemporary(
+          SubExpr, SubExpr->getType(), false, Result);
       if (!EvaluateInPlace(Value, Info, Result, SubExpr))
         return false;
     }
@@ -7220,6 +8183,8 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
 
     return true;
   }
+  case Builtin::BI__builtin_operator_new:
+    return HandleOperatorNewCall(Info, E, Result);
   case Builtin::BI__builtin_launder:
     return evaluatePointer(E->getArg(0), Result);
   case Builtin::BIstrchr:
@@ -7455,6 +8420,8 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
 
     while (true) {
       APValue Val;
+      // FIXME: Set WantObjectRepresentation to true if we're copying a
+      // char-like type?
       if (!handleLValueToRValueConversion(Info, E, T, Src, Val) ||
           !handleAssignment(Info, E, Dest, T, Val))
         return false;
@@ -7469,10 +8436,208 @@ bool PointerExprEvaluator::VisitBuiltinCallExpr(const CallExpr *E,
   }
 
   default:
-    return visitNonBuiltinCallExpr(E);
+    break;
   }
+
+  return visitNonBuiltinCallExpr(E);
 }
 
+static bool EvaluateArrayNewInitList(EvalInfo &Info, LValue &This,
+                                     APValue &Result, const InitListExpr *ILE,
+                                     QualType AllocType);
+
+bool PointerExprEvaluator::VisitCXXNewExpr(const CXXNewExpr *E) {
+  if (!Info.getLangOpts().CPlusPlus2a)
+    Info.CCEDiag(E, diag::note_constexpr_new);
+
+  // We cannot speculatively evaluate a delete expression.
+  if (Info.SpeculativeEvaluationDepth)
+    return false;
+
+  FunctionDecl *OperatorNew = E->getOperatorNew();
+
+  bool IsNothrow = false;
+  bool IsPlacement = false;
+  if (OperatorNew->isReservedGlobalPlacementOperator() &&
+      Info.CurrentCall->isStdFunction() && !E->isArray()) {
+    // FIXME Support array placement new.
+    assert(E->getNumPlacementArgs() == 1);
+    if (!EvaluatePointer(E->getPlacementArg(0), Result, Info))
+      return false;
+    if (Result.Designator.Invalid)
+      return false;
+    IsPlacement = true;
+  } else if (!OperatorNew->isReplaceableGlobalAllocationFunction()) {
+    Info.FFDiag(E, diag::note_constexpr_new_non_replaceable)
+        << isa<CXXMethodDecl>(OperatorNew) << OperatorNew;
+    return false;
+  } else if (E->getNumPlacementArgs()) {
+    // The only new-placement list we support is of the form (std::nothrow).
+    //
+    // FIXME: There is no restriction on this, but it's not clear that any
+    // other form makes any sense. We get here for cases such as:
+    //
+    //   new (std::align_val_t{N}) X(int)
+    //
+    // (which should presumably be valid only if N is a multiple of
+    // alignof(int), and in any case can't be deallocated unless N is
+    // alignof(X) and X has new-extended alignment).
+    if (E->getNumPlacementArgs() != 1 ||
+        !E->getPlacementArg(0)->getType()->isNothrowT())
+      return Error(E, diag::note_constexpr_new_placement);
+
+    LValue Nothrow;
+    if (!EvaluateLValue(E->getPlacementArg(0), Nothrow, Info))
+      return false;
+    IsNothrow = true;
+  }
+
+  const Expr *Init = E->getInitializer();
+  const InitListExpr *ResizedArrayILE = nullptr;
+
+  QualType AllocType = E->getAllocatedType();
+  if (Optional<const Expr*> ArraySize = E->getArraySize()) {
+    const Expr *Stripped = *ArraySize;
+    for (; auto *ICE = dyn_cast<ImplicitCastExpr>(Stripped);
+         Stripped = ICE->getSubExpr())
+      if (ICE->getCastKind() != CK_NoOp &&
+          ICE->getCastKind() != CK_IntegralCast)
+        break;
+
+    llvm::APSInt ArrayBound;
+    if (!EvaluateInteger(Stripped, ArrayBound, Info))
+      return false;
+
+    // C++ [expr.new]p9:
+    //   The expression is erroneous if:
+    //   -- [...] its value before converting to size_t [or] applying the
+    //      second standard conversion sequence is less than zero
+    if (ArrayBound.isSigned() && ArrayBound.isNegative()) {
+      if (IsNothrow)
+        return ZeroInitialization(E);
+
+      Info.FFDiag(*ArraySize, diag::note_constexpr_new_negative)
+          << ArrayBound << (*ArraySize)->getSourceRange();
+      return false;
+    }
+
+    //   -- its value is such that the size of the allocated object would
+    //      exceed the implementation-defined limit
+    if (ConstantArrayType::getNumAddressingBits(Info.Ctx, AllocType,
+                                                ArrayBound) >
+        ConstantArrayType::getMaxSizeBits(Info.Ctx)) {
+      if (IsNothrow)
+        return ZeroInitialization(E);
+
+      Info.FFDiag(*ArraySize, diag::note_constexpr_new_too_large)
+        << ArrayBound << (*ArraySize)->getSourceRange();
+      return false;
+    }
+
+    //   -- the new-initializer is a braced-init-list and the number of
+    //      array elements for which initializers are provided [...]
+    //      exceeds the number of elements to initialize
+    if (Init) {
+      auto *CAT = Info.Ctx.getAsConstantArrayType(Init->getType());
+      assert(CAT && "unexpected type for array initializer");
+
+      unsigned Bits =
+          std::max(CAT->getSize().getBitWidth(), ArrayBound.getBitWidth());
+      llvm::APInt InitBound = CAT->getSize().zextOrSelf(Bits);
+      llvm::APInt AllocBound = ArrayBound.zextOrSelf(Bits);
+      if (InitBound.ugt(AllocBound)) {
+        if (IsNothrow)
+          return ZeroInitialization(E);
+
+        Info.FFDiag(*ArraySize, diag::note_constexpr_new_too_small)
+            << AllocBound.toString(10, /*Signed=*/false)
+            << InitBound.toString(10, /*Signed=*/false)
+            << (*ArraySize)->getSourceRange();
+        return false;
+      }
+
+      // If the sizes differ, we must have an initializer list, and we need
+      // special handling for this case when we initialize.
+      if (InitBound != AllocBound)
+        ResizedArrayILE = cast<InitListExpr>(Init);
+    }
+
+    AllocType = Info.Ctx.getConstantArrayType(AllocType, ArrayBound,
+                                              ArrayType::Normal, 0);
+  } else {
+    assert(!AllocType->isArrayType() &&
+           "array allocation with non-array new");
+  }
+
+  APValue *Val;
+  if (IsPlacement) {
+    AccessKinds AK = AK_Construct;
+    struct FindObjectHandler {
+      EvalInfo &Info;
+      const Expr *E;
+      QualType AllocType;
+      const AccessKinds AccessKind;
+      APValue *Value;
+
+      typedef bool result_type;
+      bool failed() { return false; }
+      bool found(APValue &Subobj, QualType SubobjType) {
+        // FIXME: Reject the cases where [basic.life]p8 would not permit the
+        // old name of the object to be used to name the new object.
+        if (!Info.Ctx.hasSameUnqualifiedType(SubobjType, AllocType)) {
+          Info.FFDiag(E, diag::note_constexpr_placement_new_wrong_type) <<
+            SubobjType << AllocType;
+          return false;
+        }
+        Value = &Subobj;
+        return true;
+      }
+      bool found(APSInt &Value, QualType SubobjType) {
+        Info.FFDiag(E, diag::note_constexpr_construct_complex_elem);
+        return false;
+      }
+      bool found(APFloat &Value, QualType SubobjType) {
+        Info.FFDiag(E, diag::note_constexpr_construct_complex_elem);
+        return false;
+      }
+    } Handler = {Info, E, AllocType, AK, nullptr};
+
+    CompleteObject Obj = findCompleteObject(Info, E, AK, Result, AllocType);
+    if (!Obj || !findSubobject(Info, E, Obj, Result.Designator, Handler))
+      return false;
+
+    Val = Handler.Value;
+
+    // [basic.life]p1:
+    //   The lifetime of an object o of type T ends when [...] the storage
+    //   which the object occupies is [...] reused by an object that is not
+    //   nested within o (6.6.2).
+    *Val = APValue();
+  } else {
+    // Perform the allocation and obtain a pointer to the resulting object.
+    Val = Info.createHeapAlloc(E, AllocType, Result);
+    if (!Val)
+      return false;
+  }
+
+  if (ResizedArrayILE) {
+    if (!EvaluateArrayNewInitList(Info, Result, *Val, ResizedArrayILE,
+                                  AllocType))
+      return false;
+  } else if (Init) {
+    if (!EvaluateInPlace(*Val, Info, Result, Init))
+      return false;
+  } else {
+    *Val = getDefaultInitValue(AllocType);
+  }
+
+  // Array new returns a pointer to the first element, not a pointer to the
+  // array.
+  if (auto *AT = AllocType->getAsArrayTypeUnsafe())
+    Result.addArray(Info, E, cast<ConstantArrayType>(AT));
+
+  return true;
+}
 //===----------------------------------------------------------------------===//
 // Member Pointer Evaluation
 //===----------------------------------------------------------------------===//
@@ -7596,7 +8761,6 @@ namespace {
     bool VisitCXXInheritedCtorInitExpr(const CXXInheritedCtorInitExpr *E);
     bool VisitCXXConstructExpr(const CXXConstructExpr *E, QualType T);
     bool VisitCXXStdInitializerListExpr(const CXXStdInitializerListExpr *E);
-
     bool VisitBinCmp(const BinaryOperator *E);
   };
 }
@@ -7830,15 +8994,11 @@ bool RecordExprEvaluator::VisitCXXConstructExpr(const CXXConstructExpr *E,
     if (Result.hasValue())
       return true;
 
-    // We can get here in two different ways:
-    //  1) We're performing value-initialization, and should zero-initialize
-    //     the object, or
-    //  2) We're performing default-initialization of an object with a trivial
-    //     constexpr default constructor, in which case we should start the
-    //     lifetimes of all the base subobjects (there can be no data member
-    //     subobjects in this case) per [basic.life]p1.
-    // Either way, ZeroInitialization is appropriate.
-    return ZeroInitialization(E, T);
+    if (ZeroInit)
+      return ZeroInitialization(E, T);
+
+    Result = getDefaultInitValue(T);
+    return true;
   }
 
   const FunctionDecl *Definition = nullptr;
@@ -7938,9 +9098,8 @@ bool RecordExprEvaluator::VisitCXXStdInitializerListExpr(
 
 bool RecordExprEvaluator::VisitLambdaExpr(const LambdaExpr *E) {
   const CXXRecordDecl *ClosureClass = E->getLambdaClass();
-  if (ClosureClass->isInvalidDecl()) return false;
-
-  if (Info.checkingPotentialConstantExpression()) return true;
+  if (ClosureClass->isInvalidDecl())
+    return false;
 
   const size_t NumFields =
       std::distance(ClosureClass->field_begin(), ClosureClass->field_end());
@@ -8000,7 +9159,8 @@ class TemporaryExprEvaluator
 
   /// Visit an expression which constructs the value of this temporary.
   bool VisitConstructExpr(const Expr *E) {
-    APValue &Value = createTemporary(E, false, Result, *Info.CurrentCall);
+    APValue &Value =
+        Info.CurrentCall->createTemporary(E, E->getType(), false, Result);
     return EvaluateInPlace(Value, Info, Result, E);
   }
 
@@ -8258,14 +9418,16 @@ namespace {
     bool VisitCallExpr(const CallExpr *E) {
       return handleCallExpr(E, Result, &This);
     }
-    bool VisitInitListExpr(const InitListExpr *E);
+    bool VisitInitListExpr(const InitListExpr *E,
+                           QualType AllocType = QualType());
     bool VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E);
     bool VisitCXXConstructExpr(const CXXConstructExpr *E);
     bool VisitCXXConstructExpr(const CXXConstructExpr *E,
                                const LValue &Subobject,
                                APValue *Value, QualType Type);
-    bool VisitStringLiteral(const StringLiteral *E) {
-      expandStringLiteral(Info, E, Result);
+    bool VisitStringLiteral(const StringLiteral *E,
+                            QualType AllocType = QualType()) {
+      expandStringLiteral(Info, E, Result, AllocType);
       return true;
     }
   };
@@ -8277,6 +9439,15 @@ static bool EvaluateArray(const Expr *E, const LValue &This,
   return ArrayExprEvaluator(Info, This, Result).Visit(E);
 }
 
+static bool EvaluateArrayNewInitList(EvalInfo &Info, LValue &This,
+                                     APValue &Result, const InitListExpr *ILE,
+                                     QualType AllocType) {
+  assert(ILE->isRValue() && ILE->getType()->isArrayType() &&
+         "not an array rvalue");
+  return ArrayExprEvaluator(Info, This, Result)
+      .VisitInitListExpr(ILE, AllocType);
+}
+
 // Return true iff the given array filler may depend on the element index.
 static bool MaybeElementDependentArrayFiller(const Expr *FillerExpr) {
   // For now, just whitelist non-class value-initialization and initialization
@@ -8293,15 +9464,23 @@ static bool MaybeElementDependentArrayFiller(const Expr *FillerExpr) {
   return true;
 }
 
-bool ArrayExprEvaluator::VisitInitListExpr(const InitListExpr *E) {
-  const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(E->getType());
+bool ArrayExprEvaluator::VisitInitListExpr(const InitListExpr *E,
+                                           QualType AllocType) {
+  const ConstantArrayType *CAT = Info.Ctx.getAsConstantArrayType(
+      AllocType.isNull() ? E->getType() : AllocType);
   if (!CAT)
     return Error(E);
 
   // C++11 [dcl.init.string]p1: A char array [...] can be initialized by [...]
   // an appropriately-typed string literal enclosed in braces.
-  if (E->isStringLiteralInit())
-    return Visit(E->getInit(0));
+  if (E->isStringLiteralInit()) {
+    auto *SL = dyn_cast<StringLiteral>(E->getInit(0)->IgnoreParens());
+    // FIXME: Support ObjCEncodeExpr here once we support it in
+    // ArrayExprEvaluator generally.
+    if (!SL)
+      return Error(E);
+    return VisitStringLiteral(SL, AllocType);
+  }
 
   bool Success = true;
 
@@ -8360,8 +9539,12 @@ bool ArrayExprEvaluator::VisitInitListExpr(const InitListExpr *E) {
 }
 
 bool ArrayExprEvaluator::VisitArrayInitLoopExpr(const ArrayInitLoopExpr *E) {
+  LValue CommonLV;
   if (E->getCommonExpr() &&
-      !Evaluate(Info.CurrentCall->createTemporary(E->getCommonExpr(), false),
+      !Evaluate(Info.CurrentCall->createTemporary(
+                    E->getCommonExpr(),
+                    getStorageType(Info.Ctx, E->getCommonExpr()), false,
+                    CommonLV),
                 Info, E->getCommonExpr()->getSourceExpr()))
     return false;
 
@@ -8761,7 +9944,7 @@ EvaluateBuiltinClassifyType(QualType T, const LangOptions &LangOpts) {
 #define DEPENDENT_TYPE(ID, BASE) case Type::ID:
 #define NON_CANONICAL_TYPE(ID, BASE) case Type::ID:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(ID, BASE) case Type::ID:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   case Type::Auto:
   case Type::DeducedTemplateSpecialization:
       llvm_unreachable("unexpected non-canonical or dependent type");
@@ -8981,6 +10164,8 @@ static QualType getObjectType(APValue::LValueBase B) {
       return E->getType();
   } else if (B.is<TypeInfoLValue>()) {
     return B.getTypeInfoType();
+  } else if (B.is<DynamicAllocLValue>()) {
+    return B.getDynamicAllocType();
   }
 
   return QualType();
@@ -11960,17 +13145,86 @@ class VoidExprEvaluator
 
   bool VisitCallExpr(const CallExpr *E) {
     switch (E->getBuiltinCallee()) {
-    default:
-      return ExprEvaluatorBaseTy::VisitCallExpr(E);
     case Builtin::BI__assume:
     case Builtin::BI__builtin_assume:
       // The argument is not evaluated!
       return true;
+
+    case Builtin::BI__builtin_operator_delete:
+      return HandleOperatorDeleteCall(Info, E);
+
+    default:
+      break;
     }
+
+    return ExprEvaluatorBaseTy::VisitCallExpr(E);
   }
+
+  bool VisitCXXDeleteExpr(const CXXDeleteExpr *E);
 };
 } // end anonymous namespace
 
+bool VoidExprEvaluator::VisitCXXDeleteExpr(const CXXDeleteExpr *E) {
+  // We cannot speculatively evaluate a delete expression.
+  if (Info.SpeculativeEvaluationDepth)
+    return false;
+
+  FunctionDecl *OperatorDelete = E->getOperatorDelete();
+  if (!OperatorDelete->isReplaceableGlobalAllocationFunction()) {
+    Info.FFDiag(E, diag::note_constexpr_new_non_replaceable)
+        << isa<CXXMethodDecl>(OperatorDelete) << OperatorDelete;
+    return false;
+  }
+
+  const Expr *Arg = E->getArgument();
+
+  LValue Pointer;
+  if (!EvaluatePointer(Arg, Pointer, Info))
+    return false;
+  if (Pointer.Designator.Invalid)
+    return false;
+
+  // Deleting a null pointer has no effect.
+  if (Pointer.isNullPointer()) {
+    // This is the only case where we need to produce an extension warning:
+    // the only other way we can succeed is if we find a dynamic allocation,
+    // and we will have warned when we allocated it in that case.
+    if (!Info.getLangOpts().CPlusPlus2a)
+      Info.CCEDiag(E, diag::note_constexpr_new);
+    return true;
+  }
+
+  Optional<DynAlloc *> Alloc = CheckDeleteKind(
+      Info, E, Pointer, E->isArrayForm() ? DynAlloc::ArrayNew : DynAlloc::New);
+  if (!Alloc)
+    return false;
+  QualType AllocType = Pointer.Base.getDynamicAllocType();
+
+  // For the non-array case, the designator must be empty if the static type
+  // does not have a virtual destructor.
+  if (!E->isArrayForm() && Pointer.Designator.Entries.size() != 0 &&
+      !hasVirtualDestructor(Arg->getType()->getPointeeType())) {
+    Info.FFDiag(E, diag::note_constexpr_delete_base_nonvirt_dtor)
+        << Arg->getType()->getPointeeType() << AllocType;
+    return false;
+  }
+
+  if (!HandleDestruction(Info, E->getExprLoc(), Pointer.getLValueBase(),
+                         (*Alloc)->Value, AllocType))
+    return false;
+
+  if (!Info.HeapAllocs.erase(Pointer.Base.dyn_cast<DynamicAllocLValue>())) {
+    // The element was already erased. This means the destructor call also
+    // deleted the object.
+    // FIXME: This probably results in undefined behavior before we get this
+    // far, and should be diagnosed elsewhere first.
+    Info.FFDiag(E, diag::note_constexpr_double_delete);
+    return false;
+  }
+
+  return true;
+}
+
 static bool EvaluateVoid(const Expr *E, EvalInfo &Info) {
   assert(E->isRValue() && E->getType()->isVoidType());
   return VoidExprEvaluator(Info).Visit(E);
@@ -12020,13 +13274,14 @@ static bool Evaluate(APValue &Result, EvalInfo &Info, const Expr *E) {
     return true;
   } else if (T->isArrayType()) {
     LValue LV;
-    APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall);
+    APValue &Value =
+        Info.CurrentCall->createTemporary(E, T, false, LV);
     if (!EvaluateArray(E, LV, Value, Info))
       return false;
     Result = Value;
   } else if (T->isRecordType()) {
     LValue LV;
-    APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall);
+    APValue &Value = Info.CurrentCall->createTemporary(E, T, false, LV);
     if (!EvaluateRecord(E, LV, Value, Info))
       return false;
     Result = Value;
@@ -12040,7 +13295,7 @@ static bool Evaluate(APValue &Result, EvalInfo &Info, const Expr *E) {
     QualType Unqual = T.getAtomicUnqualifiedType();
     if (Unqual->isArrayType() || Unqual->isRecordType()) {
       LValue LV;
-      APValue &Value = createTemporary(E, false, LV, *Info.CurrentCall);
+      APValue &Value = Info.CurrentCall->createTemporary(E, Unqual, false, LV);
       if (!EvaluateAtomic(E, &LV, Value, Info))
         return false;
     } else {
@@ -12119,7 +13374,8 @@ static bool EvaluateAsRValue(EvalInfo &Info, const Expr *E, APValue &Result) {
   }
 
   // Check this core constant expression is a constant expression.
-  return CheckConstantExpression(Info, E->getExprLoc(), E->getType(), Result);
+  return CheckConstantExpression(Info, E->getExprLoc(), E->getType(), Result) &&
+         CheckMemoryLeaks(Info);
 }
 
 static bool FastEvaluateAsRValue(const Expr *Exp, Expr::EvalResult &Result,
@@ -12268,10 +13524,12 @@ bool Expr::EvaluateAsLValue(EvalResult &Result, const ASTContext &Ctx,
   EvalInfo Info(Ctx, Result, EvalInfo::EM_ConstantFold);
   Info.InConstantContext = InConstantContext;
   LValue LV;
-  if (!EvaluateLValue(this, LV, Info) || Result.HasSideEffects ||
+  CheckedTemporaries CheckedTemps;
+  if (!EvaluateLValue(this, LV, Info) || !Info.discardCleanups() ||
+      Result.HasSideEffects ||
       !CheckLValueConstantExpression(Info, getExprLoc(),
                                      Ctx.getLValueReferenceType(getType()), LV,
-                                     Expr::EvaluateForCodeGen))
+                                     Expr::EvaluateForCodeGen, CheckedTemps))
     return false;
 
   LV.moveInto(Result.Val);
@@ -12287,11 +13545,15 @@ bool Expr::EvaluateAsConstantExpr(EvalResult &Result, ConstExprUsage Usage,
   EvalInfo Info(Ctx, Result, EM);
   Info.InConstantContext = true;
 
-  if (!::Evaluate(Result.Val, Info, this))
+  if (!::Evaluate(Result.Val, Info, this) || Result.HasSideEffects)
     return false;
 
+  if (!Info.discardCleanups())
+    llvm_unreachable("Unhandled cleanup; missing full expression marker?");
+
   return CheckConstantExpression(Info, getExprLoc(), getType(), Result.Val,
-                                 Usage);
+                                 Usage) &&
+         CheckMemoryLeaks(Info);
 }
 
 bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx,
@@ -12353,7 +13615,50 @@ bool Expr::EvaluateAsInitializer(APValue &Value, const ASTContext &Ctx,
       EStatus.HasSideEffects)
     return false;
 
-  return CheckConstantExpression(Info, DeclLoc, DeclTy, Value);
+  // At this point, any lifetime-extended temporaries are completely
+  // initialized.
+  Info.performLifetimeExtension();
+
+  if (!Info.discardCleanups())
+    llvm_unreachable("Unhandled cleanup; missing full expression marker?");
+
+  return CheckConstantExpression(Info, DeclLoc, DeclTy, Value) &&
+         CheckMemoryLeaks(Info);
+}
+
+bool VarDecl::evaluateDestruction(
+    SmallVectorImpl<PartialDiagnosticAt> &Notes) const {
+  assert(getEvaluatedValue() && !getEvaluatedValue()->isAbsent() &&
+         "cannot evaluate destruction of non-constant-initialized variable");
+
+  Expr::EvalStatus EStatus;
+  EStatus.Diag = &Notes;
+
+  // Make a copy of the value for the destructor to mutate.
+  APValue DestroyedValue = *getEvaluatedValue();
+
+  EvalInfo Info(getASTContext(), EStatus, EvalInfo::EM_ConstantExpression);
+  Info.setEvaluatingDecl(this, DestroyedValue,
+                         EvalInfo::EvaluatingDeclKind::Dtor);
+  Info.InConstantContext = true;
+
+  SourceLocation DeclLoc = getLocation();
+  QualType DeclTy = getType();
+
+  LValue LVal;
+  LVal.set(this);
+
+  // FIXME: Consider storing whether this variable has constant destruction in
+  // the EvaluatedStmt so that CodeGen can query it.
+  if (!HandleDestruction(Info, DeclLoc, LVal.Base, DestroyedValue, DeclTy) ||
+      EStatus.HasSideEffects)
+    return false;
+
+  if (!Info.discardCleanups())
+    llvm_unreachable("Unhandled cleanup; missing full expression marker?");
+
+  ensureEvaluatedStmt()->HasConstantDestruction = true;
+  return true;
 }
 
 /// isEvaluatable - Call EvaluateAsRValue to see if this expression can be
@@ -12959,7 +14264,11 @@ bool Expr::isCXX11ConstantExpr(const ASTContext &Ctx, APValue *Result,
   EvalInfo Info(Ctx, Status, EvalInfo::EM_ConstantExpression);
 
   APValue Scratch;
-  bool IsConstExpr = ::EvaluateAsRValue(Info, this, Result ? *Result : Scratch);
+  bool IsConstExpr =
+      ::EvaluateAsRValue(Info, this, Result ? *Result : Scratch) &&
+      // FIXME: We don't produce a diagnostic for this, but the callers that
+      // call us on arbitrary full-expressions should generally not care.
+      Info.discardCleanups() && !Status.HasSideEffects;
 
   if (!Diags.empty()) {
     IsConstExpr = false;
@@ -13011,7 +14320,8 @@ bool Expr::EvaluateWithSubstitution(APValue &Value, ASTContext &Ctx,
   // Build fake call to Callee.
   CallStackFrame Frame(Info, Callee->getLocation(), Callee, ThisPtr,
                        ArgValues.data());
-  return Evaluate(Value, Info, this) && !Info.EvalStatus.HasSideEffects;
+  return Evaluate(Value, Info, this) && Info.discardCleanups() &&
+         !Info.EvalStatus.HasSideEffects;
 }
 
 bool Expr::isPotentialConstantExpr(const FunctionDecl *FD,
diff --git a/clang/lib/AST/ExternalASTMerger.cpp b/clang/lib/AST/ExternalASTMerger.cpp
index 4dc89f0f31a0a..01161723ac0aa 100644
--- a/clang/lib/AST/ExternalASTMerger.cpp
+++ b/clang/lib/AST/ExternalASTMerger.cpp
@@ -105,13 +105,16 @@ class LazyASTImporter : public ASTImporter {
   llvm::raw_ostream &logs() { return Parent.logs(); }
 public:
   LazyASTImporter(ExternalASTMerger &_Parent, ASTContext &ToContext,
-                  FileManager &ToFileManager, ASTContext &FromContext,
-                  FileManager &FromFileManager,
-                  const ExternalASTMerger::OriginMap &_FromOrigins)
-      : ASTImporter(ToContext, ToFileManager, FromContext, FromFileManager,
-                    /*MinimalImport=*/true),
-        Parent(_Parent), Reverse(FromContext, FromFileManager, ToContext,
-                                 ToFileManager, /*MinimalImport=*/true), FromOrigins(_FromOrigins) {}
+                  FileManager &ToFileManager,
+                  const ExternalASTMerger::ImporterSource &_Source,
+                  std::shared_ptr<ASTImporterSharedState> SharedState)
+      : ASTImporter(ToContext, ToFileManager, _Source.getASTContext(),
+                    _Source.getFileManager(),
+                    /*MinimalImport=*/true, SharedState),
+        Parent(_Parent),
+        Reverse(_Source.getASTContext(), _Source.getFileManager(), ToContext,
+                ToFileManager, /*MinimalImport=*/true),
+        FromOrigins(_Source.getOriginMap()) {}
 
   /// Whenever a DeclContext is imported, ensure that ExternalASTSource's origin
   /// map is kept up to date.  Also set the appropriate flags.
@@ -314,28 +317,30 @@ void ExternalASTMerger::RecordOriginImpl(const DeclContext *ToDC, DCOrigin Origi
 
 ExternalASTMerger::ExternalASTMerger(const ImporterTarget &Target,
                                      llvm::ArrayRef<ImporterSource> Sources) : LogStream(&llvm::nulls()), Target(Target) {
+  SharedState = std::make_shared<ASTImporterSharedState>(
+      *Target.AST.getTranslationUnitDecl());
   AddSources(Sources);
 }
 
 void ExternalASTMerger::AddSources(llvm::ArrayRef<ImporterSource> Sources) {
   for (const ImporterSource &S : Sources) {
-    assert(&S.AST != &Target.AST);
+    assert(&S.getASTContext() != &Target.AST);
     Importers.push_back(std::make_unique<LazyASTImporter>(
-        *this, Target.AST, Target.FM, S.AST, S.FM, S.OM));
+        *this, Target.AST, Target.FM, S, SharedState));
   }
 }
 
 void ExternalASTMerger::RemoveSources(llvm::ArrayRef<ImporterSource> Sources) {
   if (LoggingEnabled())
     for (const ImporterSource &S : Sources)
-      logs() << "(ExternalASTMerger*)" << (void*)this
-             << " removing source (ASTContext*)" << (void*)&S.AST
+      logs() << "(ExternalASTMerger*)" << (void *)this
+             << " removing source (ASTContext*)" << (void *)&S.getASTContext()
              << "\n";
   Importers.erase(
       std::remove_if(Importers.begin(), Importers.end(),
                      [&Sources](std::unique_ptr<ASTImporter> &Importer) -> bool {
                        for (const ImporterSource &S : Sources) {
-                         if (&Importer->getFromContext() == &S.AST)
+                         if (&Importer->getFromContext() == &S.getASTContext())
                            return true;
                        }
                        return false;
@@ -345,7 +350,7 @@ void ExternalASTMerger::RemoveSources(llvm::ArrayRef<ImporterSource> Sources) {
     std::pair<const DeclContext *, DCOrigin> Origin = *OI;
     bool Erase = false;
     for (const ImporterSource &S : Sources) {
-      if (&S.AST == Origin.second.AST) {
+      if (&S.getASTContext() == Origin.second.AST) {
         Erase = true;
         break;
       }
diff --git a/clang/lib/AST/FormatString.cpp b/clang/lib/AST/FormatString.cpp
index ff439f9921c9b..3c96c80e4ccc8 100644
--- a/clang/lib/AST/FormatString.cpp
+++ b/clang/lib/AST/FormatString.cpp
@@ -359,6 +359,7 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
           case BuiltinType::SChar:
           case BuiltinType::UChar:
           case BuiltinType::Char_U:
+          case BuiltinType::Bool:
             return Match;
         }
       return NoMatch;
@@ -386,6 +387,7 @@ ArgType::matchesType(ASTContext &C, QualType argTy) const {
           case BuiltinType::SChar:
           case BuiltinType::Char_U:
           case BuiltinType::UChar:
+          case BuiltinType::Bool:
             if (T == C.UnsignedShortTy || T == C.ShortTy)
               return NoMatchPedantic;
             return T == C.UnsignedCharTy || T == C.SignedCharTy ? Match
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index c9ace131bbd5a..1a8109cedf769 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -275,7 +275,7 @@ bool CheckMutable(InterpState &S, CodePtr OpPC, const Pointer &Ptr) {
 
   const SourceInfo &Loc = S.Current->getSource(OpPC);
   const FieldDecl *Field = Ptr.getField();
-  S.FFDiag(Loc, diag::note_constexpr_ltor_mutable, 1) << Field;
+  S.FFDiag(Loc, diag::note_constexpr_access_mutable, 1) << AK_Read << Field;
   S.Note(Field->getLocation(), diag::note_declared_at);
   return false;
 }
diff --git a/clang/lib/AST/Interp/InterpStack.cpp b/clang/lib/AST/Interp/InterpStack.cpp
index f159fe1955f81..5c803f3d94244 100644
--- a/clang/lib/AST/Interp/InterpStack.cpp
+++ b/clang/lib/AST/Interp/InterpStack.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <cassert>
+#include <cstdlib>
 #include "InterpStack.h"
 
 using namespace clang;
diff --git a/clang/lib/AST/Interp/State.h b/clang/lib/AST/Interp/State.h
index 49af3d18c9bf5..d9a645a3eb3eb 100644
--- a/clang/lib/AST/Interp/State.h
+++ b/clang/lib/AST/Interp/State.h
@@ -25,12 +25,15 @@ namespace clang {
 /// same set of semantic restrictions.
 enum AccessKinds {
   AK_Read,
+  AK_ReadObjectRepresentation,
   AK_Assign,
   AK_Increment,
   AK_Decrement,
   AK_MemberCall,
   AK_DynamicCast,
   AK_TypeId,
+  AK_Construct,
+  AK_Destroy,
 };
 
 // The order of this enum is important for diagnostics.
diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp
index a54a767f45aea..d740b89fb9764 100644
--- a/clang/lib/AST/ItaniumMangle.cpp
+++ b/clang/lib/AST/ItaniumMangle.cpp
@@ -520,7 +520,7 @@ class CXXNameMangler {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define NON_CANONICAL_TYPE(CLASS, PARENT)
 #define TYPE(CLASS, PARENT) void mangleType(const CLASS##Type *T);
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
   void mangleType(const TagType*);
   void mangleType(TemplateName);
@@ -2541,7 +2541,7 @@ void CXXNameMangler::mangleType(QualType T) {
     case Type::CLASS: \
       mangleType(static_cast<const CLASS##Type*>(ty)); \
       break;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
   }
 
diff --git a/clang/lib/AST/JSONNodeDumper.cpp b/clang/lib/AST/JSONNodeDumper.cpp
index b7e8c882244e2..049ce0d6ff24b 100644
--- a/clang/lib/AST/JSONNodeDumper.cpp
+++ b/clang/lib/AST/JSONNodeDumper.cpp
@@ -850,6 +850,12 @@ void JSONNodeDumper::VisitLinkageSpecDecl(const LinkageSpecDecl *LSD) {
   switch (LSD->getLanguage()) {
   case LinkageSpecDecl::lang_c: Lang = "C"; break;
   case LinkageSpecDecl::lang_cxx: Lang = "C++"; break;
+  case LinkageSpecDecl::lang_cxx_11:
+    Lang = "C++11";
+    break;
+  case LinkageSpecDecl::lang_cxx_14:
+    Lang = "C++14";
+    break;
   }
   JOS.attribute("language", Lang);
   attributeOnlyIfTrue("hasBraces", LSD->hasBraces());
diff --git a/clang/lib/AST/Mangle.cpp b/clang/lib/AST/Mangle.cpp
index 28de87fde9e15..b158fe85a475c 100644
--- a/clang/lib/AST/Mangle.cpp
+++ b/clang/lib/AST/Mangle.cpp
@@ -122,15 +122,21 @@ void MangleContext::mangleName(const NamedDecl *D, raw_ostream &Out) {
   if (const AsmLabelAttr *ALA = D->getAttr<AsmLabelAttr>()) {
     // If we have an asm name, then we use it as the mangling.
 
+    // If the label isn't literal, or if this is an alias for an LLVM intrinsic,
+    // do not add a "\01" prefix.
+    if (!ALA->getIsLiteralLabel() || ALA->getLabel().startswith("llvm.")) {
+      Out << ALA->getLabel();
+      return;
+    }
+
     // Adding the prefix can cause problems when one file has a "foo" and
     // another has a "\01foo". That is known to happen on ELF with the
     // tricks normally used for producing aliases (PR9177). Fortunately the
     // llvm mangler on ELF is a nop, so we can just avoid adding the \01
-    // marker.  We also avoid adding the marker if this is an alias for an
-    // LLVM intrinsic.
+    // marker.
     char GlobalPrefix =
         getASTContext().getTargetInfo().getDataLayout().getGlobalPrefix();
-    if (GlobalPrefix && !ALA->getLabel().startswith("llvm."))
+    if (GlobalPrefix)
       Out << '\01'; // LLVM IR Marker for __asm("foo")
 
     Out << ALA->getLabel();
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index 7ad8b52f7fd59..c80388be1eaad 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -364,7 +364,7 @@ class MicrosoftCXXNameMangler {
 #define TYPE(CLASS, PARENT) void mangleType(const CLASS##Type *T, \
                                             Qualifiers Quals, \
                                             SourceRange Range);
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 #undef ABSTRACT_TYPE
 #undef NON_CANONICAL_TYPE
 #undef TYPE
@@ -615,6 +615,8 @@ void MicrosoftCXXNameMangler::mangleMemberDataPointer(const CXXRecordDecl *RD,
   case MSInheritanceAttr::Keyword_multiple_inheritance:    Code = '0'; break;
   case MSInheritanceAttr::Keyword_virtual_inheritance:     Code = 'F'; break;
   case MSInheritanceAttr::Keyword_unspecified_inheritance: Code = 'G'; break;
+  case MSInheritanceAttr::SpellingNotCalculated:
+    llvm_unreachable("not reachable");
   }
 
   Out << '$' << Code;
@@ -646,6 +648,8 @@ MicrosoftCXXNameMangler::mangleMemberFunctionPointer(const CXXRecordDecl *RD,
   case MSInheritanceAttr::Keyword_multiple_inheritance:    Code = 'H'; break;
   case MSInheritanceAttr::Keyword_virtual_inheritance:     Code = 'I'; break;
   case MSInheritanceAttr::Keyword_unspecified_inheritance: Code = 'J'; break;
+  case MSInheritanceAttr::SpellingNotCalculated:
+    llvm_unreachable("not reachable");
   }
 
   // If non-virtual, mangle the name.  If virtual, mangle as a virtual memptr
@@ -1937,7 +1941,7 @@ void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range,
   case Type::CLASS: \
     mangleType(cast<CLASS##Type>(ty), Quals, Range); \
     break;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 #undef ABSTRACT_TYPE
 #undef NON_CANONICAL_TYPE
 #undef TYPE
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0312fe6b76967..bfe272b1d935d 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -44,6 +44,7 @@ OMPClause::child_range OMPClause::used_children() {
   case OMPC_threadprivate:
   case OMPC_uniform:
   case OMPC_device_type:
+  case OMPC_match:
   case OMPC_unknown:
     break;
   }
@@ -129,6 +130,7 @@ const OMPClauseWithPreInit *OMPClauseWithPreInit::get(const OMPClause *C) {
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     break;
   }
 
@@ -206,6 +208,7 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     break;
   }
 
diff --git a/clang/lib/AST/TextNodeDumper.cpp b/clang/lib/AST/TextNodeDumper.cpp
index df3d149f2978e..a540346ad459f 100644
--- a/clang/lib/AST/TextNodeDumper.cpp
+++ b/clang/lib/AST/TextNodeDumper.cpp
@@ -223,7 +223,6 @@ void TextNodeDumper::Visit(const Decl *D) {
     return;
   }
 
-  Context = &D->getASTContext();
   {
     ColorScope Color(OS, ShowColors, DeclKindNameColor);
     OS << D->getDeclKindName() << "Decl";
@@ -688,7 +687,7 @@ void TextNodeDumper::VisitConstantExpr(const ConstantExpr *Node) {
   if (Node->getResultAPValueKind() != APValue::None) {
     ColorScope Color(OS, ShowColors, ValueColor);
     OS << " ";
-    Node->getAPValueResult().printPretty(OS, *Context, Node->getType());
+    Node->getAPValueResult().dump(OS);
   }
 }
 
@@ -1385,6 +1384,8 @@ void TextNodeDumper::VisitVarDecl(const VarDecl *D) {
       break;
     }
   }
+  if (D->needsDestruction(D->getASTContext()))
+    OS << " destroyed";
   if (D->isParameterPack())
     OS << " pack";
 }
@@ -1767,6 +1768,12 @@ void TextNodeDumper::VisitLinkageSpecDecl(const LinkageSpecDecl *D) {
   case LinkageSpecDecl::lang_cxx:
     OS << " C++";
     break;
+  case LinkageSpecDecl::lang_cxx_11:
+    OS << " C++11";
+    break;
+  case LinkageSpecDecl::lang_cxx_14:
+    OS << " C++14";
+    break;
   }
 }
 
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index 411cb749a7b8d..8ab2f1e271efa 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -75,11 +75,11 @@ const IdentifierInfo* QualType::getBaseTypeIdentifier() const {
   if (ty->isPointerType() || ty->isReferenceType())
     return ty->getPointeeType().getBaseTypeIdentifier();
   else if (ty->isRecordType())
-    ND = ty->getAs<RecordType>()->getDecl();
+    ND = ty->castAs<RecordType>()->getDecl();
   else if (ty->isEnumeralType())
-    ND = ty->getAs<EnumType>()->getDecl();
+    ND = ty->castAs<EnumType>()->getDecl();
   else if (ty->getTypeClass() == Type::Typedef)
-    ND = ty->getAs<TypedefType>()->getDecl();
+    ND = ty->castAs<TypedefType>()->getDecl();
   else if (ty->isArrayType())
     return ty->castAsArrayTypeUnsafe()->
         getElementType().getBaseTypeIdentifier();
@@ -298,7 +298,7 @@ QualType QualType::getSingleStepDesugaredTypeImpl(QualType type,
 #define TYPE(CLASS, BASE) \
   static_assert(!std::is_polymorphic<CLASS##Type>::value, \
                 #CLASS "Type should not be polymorphic!");
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 // Check that no type class has a non-trival destructor. Types are
 // allocated with the BumpPtrAllocator from ASTContext and therefore
@@ -310,7 +310,7 @@ QualType QualType::getSingleStepDesugaredTypeImpl(QualType type,
   static_assert(std::is_trivially_destructible<CLASS##Type>::value ||          \
                     std::is_same<CLASS##Type, ConstantArrayType>::value,       \
                 #CLASS "Type should be trivially destructible!");
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 QualType Type::getLocallyUnqualifiedSingleStepDesugaredType() const {
   switch (getTypeClass()) {
@@ -321,7 +321,7 @@ QualType Type::getLocallyUnqualifiedSingleStepDesugaredType() const {
     if (!ty->isSugared()) return QualType(ty, 0); \
     return ty->desugar(); \
   }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
   llvm_unreachable("bad type kind!");
 }
@@ -342,7 +342,7 @@ SplitQualType QualType::getSplitDesugaredType(QualType T) {
       Cur = Ty->desugar(); \
       break; \
     }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
   }
 }
@@ -370,7 +370,7 @@ SplitQualType QualType::getSplitUnqualifiedTypeImpl(QualType type) {
       next = ty->desugar(); \
       break; \
     }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
 
     // Otherwise, split the underlying type.  If that yields qualifiers,
@@ -409,7 +409,7 @@ template<typename T> static const T *getAsSugar(const Type *Cur) {
       Cur = Ty->desugar().getTypePtr(); \
       break; \
     }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
   }
 }
@@ -442,7 +442,7 @@ const Type *Type::getUnqualifiedDesugaredType() const {
       Cur = Ty->desugar().getTypePtr(); \
       break; \
     }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     }
   }
 }
@@ -766,7 +766,7 @@ struct SimpleTransformVisitor : public TypeVisitor<Derived, QualType> {
 #define TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) \
   QualType Visit##Class##Type(const Class##Type *T) { return QualType(T, 0); }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
 #define TRIVIAL_TYPE_CLASS(Class) \
   QualType Visit##Class##Type(const Class##Type *T) { return QualType(T, 0); }
@@ -2490,6 +2490,15 @@ bool QualType::isCXX11PODType(const ASTContext &Context) const {
   return false;
 }
 
+bool Type::isNothrowT() const {
+  if (const auto *RD = getAsCXXRecordDecl()) {
+    IdentifierInfo *II = RD->getIdentifier();
+    if (II && II->isStr("nothrow_t") && RD->isInStdNamespace())
+      return true;
+  }
+  return false;
+}
+
 bool Type::isAlignValT() const {
   if (const auto *ET = getAs<EnumType>()) {
     IdentifierInfo *II = ET->getDecl()->getIdentifier();
@@ -2703,7 +2712,7 @@ const char *Type::getTypeClassName() const {
   switch (TypeBits.TC) {
 #define ABSTRACT_TYPE(Derived, Base)
 #define TYPE(Derived, Base) case Derived: return #Derived;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
 
   llvm_unreachable("Invalid type class.");
@@ -3573,13 +3582,13 @@ static CachedProperties computeCachedProperties(const Type *T) {
   switch (T->getTypeClass()) {
 #define TYPE(Class,Base)
 #define NON_CANONICAL_TYPE(Class,Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("didn't expect a non-canonical type here");
 
 #define TYPE(Class,Base)
 #define DEPENDENT_TYPE(Class,Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class,Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     // Treat instantiation-dependent types as external.
     assert(T->isInstantiationDependentType());
     return CachedProperties(ExternalLinkage, false);
@@ -3676,13 +3685,13 @@ LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) {
   switch (T->getTypeClass()) {
 #define TYPE(Class,Base)
 #define NON_CANONICAL_TYPE(Class,Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("didn't expect a non-canonical type here");
 
 #define TYPE(Class,Base)
 #define DEPENDENT_TYPE(Class,Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class,Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     // Treat instantiation-dependent types as external.
     assert(T->isInstantiationDependentType());
     return LinkageInfo::external();
@@ -3791,7 +3800,7 @@ bool Type::canHaveNullability(bool ResultIfUnknown) const {
   case Type::Class:                             \
     llvm_unreachable("non-canonical type");
 #define TYPE(Class, Parent)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
   // Pointer types.
   case Type::Pointer:
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
index 6e707fb7d52e1..5ae3340eeb5f2 100644
--- a/clang/lib/AST/TypePrinter.cpp
+++ b/clang/lib/AST/TypePrinter.cpp
@@ -125,7 +125,7 @@ namespace {
 #define TYPE(CLASS, PARENT) \
     void print##CLASS##Before(const CLASS##Type *T, raw_ostream &OS); \
     void print##CLASS##After(const CLASS##Type *T, raw_ostream &OS);
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
   private:
     void printBefore(const Type *ty, Qualifiers qs, raw_ostream &OS);
@@ -329,7 +329,7 @@ void TypePrinter::printBefore(const Type *T,Qualifiers Quals, raw_ostream &OS) {
 #define TYPE(CLASS, PARENT) case Type::CLASS: \
     print##CLASS##Before(cast<CLASS##Type>(T), OS); \
     break;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
 
   if (hasAfterQuals) {
@@ -355,7 +355,7 @@ void TypePrinter::printAfter(const Type *T, Qualifiers Quals, raw_ostream &OS) {
 #define TYPE(CLASS, PARENT) case Type::CLASS: \
     print##CLASS##After(cast<CLASS##Type>(T), OS); \
     break;
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   }
 }
 
diff --git a/clang/lib/Analysis/CFG.cpp b/clang/lib/Analysis/CFG.cpp
index 1a45c9f532d92..687cf57059138 100644
--- a/clang/lib/Analysis/CFG.cpp
+++ b/clang/lib/Analysis/CFG.cpp
@@ -70,23 +70,47 @@ static SourceLocation GetEndLoc(Decl *D) {
   return D->getLocation();
 }
 
+/// Returns true on constant values based around a single IntegerLiteral.
+/// Allow for use of parentheses, integer casts, and negative signs.
+static bool IsIntegerLiteralConstantExpr(const Expr *E) {
+  // Allow parentheses
+  E = E->IgnoreParens();
+
+  // Allow conversions to different integer kind.
+  if (const auto *CE = dyn_cast<CastExpr>(E)) {
+    if (CE->getCastKind() != CK_IntegralCast)
+      return false;
+    E = CE->getSubExpr();
+  }
+
+  // Allow negative numbers.
+  if (const auto *UO = dyn_cast<UnaryOperator>(E)) {
+    if (UO->getOpcode() != UO_Minus)
+      return false;
+    E = UO->getSubExpr();
+  }
+
+  return isa<IntegerLiteral>(E);
+}
+
 /// Helper for tryNormalizeBinaryOperator. Attempts to extract an IntegerLiteral
-/// or EnumConstantDecl from the given Expr. If it fails, returns nullptr.
+/// constant expression or EnumConstantDecl from the given Expr. If it fails,
+/// returns nullptr.
 static const Expr *tryTransformToIntOrEnumConstant(const Expr *E) {
   E = E->IgnoreParens();
-  if (isa<IntegerLiteral>(E))
+  if (IsIntegerLiteralConstantExpr(E))
     return E;
   if (auto *DR = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()))
     return isa<EnumConstantDecl>(DR->getDecl()) ? DR : nullptr;
   return nullptr;
 }
 
-/// Tries to interpret a binary operator into `Decl Op Expr` form, if Expr is
-/// an integer literal or an enum constant.
+/// Tries to interpret a binary operator into `Expr Op NumExpr` form, if
+/// NumExpr is an integer literal or an enum constant.
 ///
 /// If this fails, at least one of the returned DeclRefExpr or Expr will be
 /// null.
-static std::tuple<const DeclRefExpr *, BinaryOperatorKind, const Expr *>
+static std::tuple<const Expr *, BinaryOperatorKind, const Expr *>
 tryNormalizeBinaryOperator(const BinaryOperator *B) {
   BinaryOperatorKind Op = B->getOpcode();
 
@@ -108,8 +132,7 @@ tryNormalizeBinaryOperator(const BinaryOperator *B) {
     Constant = tryTransformToIntOrEnumConstant(B->getLHS());
   }
 
-  auto *D = dyn_cast<DeclRefExpr>(MaybeDecl->IgnoreParenImpCasts());
-  return std::make_tuple(D, Op, Constant);
+  return std::make_tuple(MaybeDecl, Op, Constant);
 }
 
 /// For an expression `x == Foo && x == Bar`, this determines whether the
@@ -121,11 +144,11 @@ tryNormalizeBinaryOperator(const BinaryOperator *B) {
 static bool areExprTypesCompatible(const Expr *E1, const Expr *E2) {
   // User intent isn't clear if they're mixing int literals with enum
   // constants.
-  if (isa<IntegerLiteral>(E1) != isa<IntegerLiteral>(E2))
+  if (isa<DeclRefExpr>(E1) != isa<DeclRefExpr>(E2))
     return false;
 
   // Integer literal comparisons, regardless of literal type, are acceptable.
-  if (isa<IntegerLiteral>(E1))
+  if (!isa<DeclRefExpr>(E1))
     return true;
 
   // IntegerLiterals are handled above and only EnumConstantDecls are expected
@@ -1021,34 +1044,34 @@ class CFGBuilder {
     if (!LHS->isComparisonOp() || !RHS->isComparisonOp())
       return {};
 
-    const DeclRefExpr *Decl1;
-    const Expr *Expr1;
+    const Expr *DeclExpr1;
+    const Expr *NumExpr1;
     BinaryOperatorKind BO1;
-    std::tie(Decl1, BO1, Expr1) = tryNormalizeBinaryOperator(LHS);
+    std::tie(DeclExpr1, BO1, NumExpr1) = tryNormalizeBinaryOperator(LHS);
 
-    if (!Decl1 || !Expr1)
+    if (!DeclExpr1 || !NumExpr1)
       return {};
 
-    const DeclRefExpr *Decl2;
-    const Expr *Expr2;
+    const Expr *DeclExpr2;
+    const Expr *NumExpr2;
     BinaryOperatorKind BO2;
-    std::tie(Decl2, BO2, Expr2) = tryNormalizeBinaryOperator(RHS);
+    std::tie(DeclExpr2, BO2, NumExpr2) = tryNormalizeBinaryOperator(RHS);
 
-    if (!Decl2 || !Expr2)
+    if (!DeclExpr2 || !NumExpr2)
       return {};
 
     // Check that it is the same variable on both sides.
-    if (Decl1->getDecl() != Decl2->getDecl())
+    if (!Expr::isSameComparisonOperand(DeclExpr1, DeclExpr2))
       return {};
 
     // Make sure the user's intent is clear (e.g. they're comparing against two
     // int literals, or two things from the same enum)
-    if (!areExprTypesCompatible(Expr1, Expr2))
+    if (!areExprTypesCompatible(NumExpr1, NumExpr2))
       return {};
 
     Expr::EvalResult L1Result, L2Result;
-    if (!Expr1->EvaluateAsInt(L1Result, *Context) ||
-        !Expr2->EvaluateAsInt(L2Result, *Context))
+    if (!NumExpr1->EvaluateAsInt(L1Result, *Context) ||
+        !NumExpr2->EvaluateAsInt(L2Result, *Context))
       return {};
 
     llvm::APSInt L1 = L1Result.Val.getInt();
@@ -1081,6 +1104,10 @@ class CFGBuilder {
     // * Variable x is equal to the largest literal.
     // * Variable x is greater than largest literal.
     bool AlwaysTrue = true, AlwaysFalse = true;
+    // Track value of both subexpressions.  If either side is always
+    // true/false, another warning should have already been emitted.
+    bool LHSAlwaysTrue = true, LHSAlwaysFalse = true;
+    bool RHSAlwaysTrue = true, RHSAlwaysFalse = true;
     for (const llvm::APSInt &Value : Values) {
       TryResult Res1, Res2;
       Res1 = analyzeLogicOperatorCondition(BO1, Value, L1);
@@ -1096,10 +1123,16 @@ class CFGBuilder {
         AlwaysTrue &= (Res1.isTrue() || Res2.isTrue());
         AlwaysFalse &= !(Res1.isTrue() || Res2.isTrue());
       }
+
+      LHSAlwaysTrue &= Res1.isTrue();
+      LHSAlwaysFalse &= Res1.isFalse();
+      RHSAlwaysTrue &= Res2.isTrue();
+      RHSAlwaysFalse &= Res2.isFalse();
     }
 
     if (AlwaysTrue || AlwaysFalse) {
-      if (BuildOpts.Observer)
+      if (!LHSAlwaysTrue && !LHSAlwaysFalse && !RHSAlwaysTrue &&
+          !RHSAlwaysFalse && BuildOpts.Observer)
         BuildOpts.Observer->compareAlwaysTrue(B, AlwaysTrue);
       return TryResult(AlwaysTrue);
     }
diff --git a/clang/lib/Analysis/CallGraph.cpp b/clang/lib/Analysis/CallGraph.cpp
index 6aa8bf395b63f..63f7a8878e02b 100644
--- a/clang/lib/Analysis/CallGraph.cpp
+++ b/clang/lib/Analysis/CallGraph.cpp
@@ -80,7 +80,10 @@ class CGBuilder : public StmtVisitor<CGBuilder> {
   }
 
   void VisitLambdaExpr(LambdaExpr *LE) {
-    if (CXXMethodDecl *MD = LE->getCallOperator())
+    if (FunctionTemplateDecl *FTD = LE->getDependentCallOperator())
+      for (FunctionDecl *FD : FTD->specializations())
+        G->VisitFunctionDecl(FD);
+    else if (CXXMethodDecl *MD = LE->getCallOperator())
       G->VisitFunctionDecl(MD);
   }
 
diff --git a/clang/lib/Analysis/Consumed.cpp b/clang/lib/Analysis/Consumed.cpp
index 17bc29ba88342..cde753e8ec576 100644
--- a/clang/lib/Analysis/Consumed.cpp
+++ b/clang/lib/Analysis/Consumed.cpp
@@ -644,10 +644,10 @@ bool ConsumedStmtVisitor::handleCall(const CallExpr *Call, const Expr *ObjArg,
       continue;
 
     // Adjust state on the caller side.
-    if (isRValueRef(ParamType))
-      setStateForVarOrTmp(StateMap, PInfo, consumed::CS_Consumed);
-    else if (ReturnTypestateAttr *RT = Param->getAttr<ReturnTypestateAttr>())
+    if (ReturnTypestateAttr *RT = Param->getAttr<ReturnTypestateAttr>())
       setStateForVarOrTmp(StateMap, PInfo, mapReturnTypestateAttrState(RT));
+    else if (isRValueRef(ParamType) || isConsumableType(ParamType))
+      setStateForVarOrTmp(StateMap, PInfo, consumed::CS_Consumed);
     else if (isPointerOrRef(ParamType) &&
              (!ParamType->getPointeeType().isConstQualified() ||
               isSetOnReadPtrType(ParamType)))
diff --git a/clang/lib/Analysis/PathDiagnostic.cpp b/clang/lib/Analysis/PathDiagnostic.cpp
index 764cb8ed0e4f8..53235ba076994 100644
--- a/clang/lib/Analysis/PathDiagnostic.cpp
+++ b/clang/lib/Analysis/PathDiagnostic.cpp
@@ -695,14 +695,18 @@ PathDiagnosticLocation::create(const ProgramPoint& P,
     return PathDiagnosticLocation(
         CEB->getLocationContext()->getDecl()->getSourceRange().getEnd(), SMng);
   } else if (Optional<BlockEntrance> BE = P.getAs<BlockEntrance>()) {
-    CFGElement BlockFront = BE->getBlock()->front();
-    if (auto StmtElt = BlockFront.getAs<CFGStmt>()) {
-      return PathDiagnosticLocation(StmtElt->getStmt()->getBeginLoc(), SMng);
-    } else if (auto NewAllocElt = BlockFront.getAs<CFGNewAllocator>()) {
-      return PathDiagnosticLocation(
-          NewAllocElt->getAllocatorExpr()->getBeginLoc(), SMng);
+    if (Optional<CFGElement> BlockFront = BE->getFirstElement()) {
+      if (auto StmtElt = BlockFront->getAs<CFGStmt>()) {
+        return PathDiagnosticLocation(StmtElt->getStmt()->getBeginLoc(), SMng);
+      } else if (auto NewAllocElt = BlockFront->getAs<CFGNewAllocator>()) {
+        return PathDiagnosticLocation(
+            NewAllocElt->getAllocatorExpr()->getBeginLoc(), SMng);
+      }
+      llvm_unreachable("Unexpected CFG element at front of block");
     }
-    llvm_unreachable("Unexpected CFG element at front of block");
+
+    return PathDiagnosticLocation(
+        BE->getBlock()->getTerminatorStmt()->getBeginLoc(), SMng);
   } else if (Optional<FunctionExitPoint> FE = P.getAs<FunctionExitPoint>()) {
     return PathDiagnosticLocation(FE->getStmt(), SMng,
                                   FE->getLocationContext());
diff --git a/clang/lib/Analysis/ReachableCode.cpp b/clang/lib/Analysis/ReachableCode.cpp
index 2fea88ea2eff4..1dab8e309f599 100644
--- a/clang/lib/Analysis/ReachableCode.cpp
+++ b/clang/lib/Analysis/ReachableCode.cpp
@@ -247,7 +247,7 @@ static bool isConfigurationValue(const Stmt *S,
     }
     case Stmt::UnaryOperatorClass: {
       const UnaryOperator *UO = cast<UnaryOperator>(S);
-      if (UO->getOpcode() != UO_LNot)
+      if (UO->getOpcode() != UO_LNot && UO->getOpcode() != UO_Minus)
         return false;
       bool SilenceableCondValNotSet =
           SilenceableCondVal && SilenceableCondVal->getBegin().isInvalid();
diff --git a/clang/lib/Analysis/plugins/CheckerDependencyHandling/CMakeLists.txt b/clang/lib/Analysis/plugins/CheckerDependencyHandling/CMakeLists.txt
index dde2cca133be7..229de54814926 100644
--- a/clang/lib/Analysis/plugins/CheckerDependencyHandling/CMakeLists.txt
+++ b/clang/lib/Analysis/plugins/CheckerDependencyHandling/CMakeLists.txt
@@ -3,7 +3,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/CheckerDependencyHandlingAnalyzerPlugin.exports)
-add_llvm_library(CheckerDependencyHandlingAnalyzerPlugin MODULE CheckerDependencyHandling.cpp PLUGIN_TOOL clang)
+add_llvm_library(CheckerDependencyHandlingAnalyzerPlugin MODULE BUILDTREE_ONLY CheckerDependencyHandling.cpp PLUGIN_TOOL clang)
 
 clang_target_link_libraries(CheckerDependencyHandlingAnalyzerPlugin PRIVATE
   clangAnalysis
diff --git a/clang/lib/Analysis/plugins/CheckerOptionHandling/CMakeLists.txt b/clang/lib/Analysis/plugins/CheckerOptionHandling/CMakeLists.txt
index 744164782d915..432383efba5cc 100644
--- a/clang/lib/Analysis/plugins/CheckerOptionHandling/CMakeLists.txt
+++ b/clang/lib/Analysis/plugins/CheckerOptionHandling/CMakeLists.txt
@@ -3,7 +3,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/CheckerOptionHandlingAnalyzerPlugin.exports)
-add_llvm_library(CheckerOptionHandlingAnalyzerPlugin MODULE CheckerOptionHandling.cpp PLUGIN_TOOL clang)
+add_llvm_library(CheckerOptionHandlingAnalyzerPlugin MODULE BUILDTREE_ONLY CheckerOptionHandling.cpp PLUGIN_TOOL clang)
 
 clang_target_link_libraries(CheckerOptionHandlingAnalyzerPlugin PRIVATE
   clangAnalysis
diff --git a/clang/lib/Analysis/plugins/SampleAnalyzer/CMakeLists.txt b/clang/lib/Analysis/plugins/SampleAnalyzer/CMakeLists.txt
index 0db1780ca7c98..d9b3f05cbd1bb 100644
--- a/clang/lib/Analysis/plugins/SampleAnalyzer/CMakeLists.txt
+++ b/clang/lib/Analysis/plugins/SampleAnalyzer/CMakeLists.txt
@@ -3,7 +3,7 @@ set(LLVM_LINK_COMPONENTS
   )
 
 set(LLVM_EXPORTED_SYMBOL_FILE ${CMAKE_CURRENT_SOURCE_DIR}/SampleAnalyzerPlugin.exports)
-add_llvm_library(SampleAnalyzerPlugin MODULE MainCallChecker.cpp PLUGIN_TOOL clang)
+add_llvm_library(SampleAnalyzerPlugin MODULE BUILDTREE_ONLY MainCallChecker.cpp PLUGIN_TOOL clang)
 
 clang_target_link_libraries(SampleAnalyzerPlugin PRIVATE
   clangAnalysis
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 761df3b55db6b..f69efe1aaaba6 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -56,6 +56,7 @@ OpenMPClauseKind clang::getOpenMPClauseKind(StringRef Str) {
 #include "clang/Basic/OpenMPKinds.def"
       .Case("uniform", OMPC_uniform)
       .Case("device_type", OMPC_device_type)
+      .Case("match", OMPC_match)
       .Default(OMPC_unknown);
 }
 
@@ -74,6 +75,8 @@ const char *clang::getOpenMPClauseName(OpenMPClauseKind Kind) {
     return "threadprivate or thread local";
   case OMPC_device_type:
     return "device_type";
+  case OMPC_match:
+    return "match";
   }
   llvm_unreachable("Invalid OpenMP clause kind");
 }
@@ -200,6 +203,7 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind,
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
+  case OMPC_match:
     break;
   }
   llvm_unreachable("Invalid OpenMP simple clause kind");
@@ -393,6 +397,7 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_unified_shared_memory:
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
+  case OMPC_match:
     break;
   }
   llvm_unreachable("Invalid OpenMP simple clause kind");
@@ -606,8 +611,6 @@ bool clang::isAllowedClauseForDirective(OpenMPDirectiveKind DKind,
       break;
     }
     break;
-  case OMPD_declare_simd:
-    break;
   case OMPD_cancel:
     switch (CKind) {
 #define OPENMP_CANCEL_CLAUSE(Name)                                             \
@@ -833,6 +836,16 @@ bool clang::isAllowedClauseForDirective(OpenMPDirectiveKind DKind,
 #define OPENMP_ALLOCATE_CLAUSE(Name)                                           \
   case OMPC_##Name:                                                            \
     return true;
+#include "clang/Basic/OpenMPKinds.def"
+    default:
+      break;
+    }
+    break;
+  case OMPD_declare_variant:
+    switch (CKind) {
+#define OPENMP_DECLARE_VARIANT_CLAUSE(Name)                                    \
+  case OMPC_##Name:                                                            \
+    return true;
 #include "clang/Basic/OpenMPKinds.def"
     default:
       break;
@@ -849,6 +862,7 @@ bool clang::isAllowedClauseForDirective(OpenMPDirectiveKind DKind,
   case OMPD_taskwait:
   case OMPD_cancellation_point:
   case OMPD_declare_reduction:
+  case OMPD_declare_simd:
     break;
   }
   return false;
@@ -1078,6 +1092,7 @@ void clang::getOpenMPCaptureRegions(
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_requires:
+  case OMPD_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
diff --git a/clang/lib/Basic/Targets/ARM.cpp b/clang/lib/Basic/Targets/ARM.cpp
index 2d23b050bbc9e..437a77afdc998 100644
--- a/clang/lib/Basic/Targets/ARM.cpp
+++ b/clang/lib/Basic/Targets/ARM.cpp
@@ -428,11 +428,10 @@ bool ARMTargetInfo::handleTargetFeatures(std::vector<std::string> &Features,
   for (const auto &Feature : Features) {
     if (Feature == "+soft-float") {
       SoftFloat = true;
-    } else if (Feature == "+vfp2sp" || Feature == "+vfp2d16sp" ||
-               Feature == "+vfp2" || Feature == "+vfp2d16") {
+    } else if (Feature == "+vfp2sp" || Feature == "+vfp2") {
       FPU |= VFP2FPU;
       HW_FP |= HW_FP_SP;
-      if (Feature == "+vfp2" || Feature == "+vfp2d16")
+      if (Feature == "+vfp2")
           HW_FP |= HW_FP_DP;
     } else if (Feature == "+vfp3sp" || Feature == "+vfp3d16sp" ||
                Feature == "+vfp3" || Feature == "+vfp3d16") {
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 0a8df86fc88e3..ab8272c034fd3 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -88,8 +88,14 @@ void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts,
   Builder.defineMacro("__riscv");
   bool Is64Bit = getTriple().getArch() == llvm::Triple::riscv64;
   Builder.defineMacro("__riscv_xlen", Is64Bit ? "64" : "32");
-  // TODO: modify when more code models are supported.
-  Builder.defineMacro("__riscv_cmodel_medlow");
+  StringRef CodeModel = getTargetOpts().CodeModel;
+  if (CodeModel == "default")
+    CodeModel = "small";
+
+  if (CodeModel == "small")
+    Builder.defineMacro("__riscv_cmodel_medlow");
+  else if (CodeModel == "medium")
+    Builder.defineMacro("__riscv_cmodel_medany");
 
   StringRef ABIName = getABI();
   if (ABIName == "ilp32f" || ABIName == "lp64f")
diff --git a/clang/lib/Basic/Targets/SystemZ.cpp b/clang/lib/Basic/Targets/SystemZ.cpp
index d86928a6333b0..ad3915e4d5dd0 100644
--- a/clang/lib/Basic/Targets/SystemZ.cpp
+++ b/clang/lib/Basic/Targets/SystemZ.cpp
@@ -92,7 +92,7 @@ static constexpr ISANameRevision ISARevisions[] = {
   {{"arch10"}, 10}, {{"zEC12"}, 10},
   {{"arch11"}, 11}, {{"z13"}, 11},
   {{"arch12"}, 12}, {{"z14"}, 12},
-  {{"arch13"}, 13},
+  {{"arch13"}, 13}, {{"z15"}, 13}
 };
 
 int SystemZTargetInfo::getISARevision(StringRef Name) const {
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp
index 522bef8ed6f92..150264a40587a 100644
--- a/clang/lib/CodeGen/BackendUtil.cpp
+++ b/clang/lib/CodeGen/BackendUtil.cpp
@@ -911,6 +911,7 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
 
   {
     PrettyStackTraceString CrashInfo("Per-function optimization");
+    llvm::TimeTraceScope TimeScope("PerFunctionPasses", StringRef(""));
 
     PerFunctionPasses.doInitialization();
     for (Function &F : *TheModule)
@@ -921,11 +922,13 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action,
 
   {
     PrettyStackTraceString CrashInfo("Per-module optimization passes");
+    llvm::TimeTraceScope TimeScope("PerModulePasses", StringRef(""));
     PerModulePasses.run(*TheModule);
   }
 
   {
     PrettyStackTraceString CrashInfo("Code generation");
+    llvm::TimeTraceScope TimeScope("CodeGenPasses", StringRef(""));
     CodeGenPasses.run(*TheModule);
   }
 
diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp
index a95cd12c2d648..f9915a1a8f5f0 100644
--- a/clang/lib/CodeGen/CGAtomic.cpp
+++ b/clang/lib/CodeGen/CGAtomic.cpp
@@ -107,7 +107,7 @@ namespace {
         }
         AtomicAlign = ValueAlign = lvalue.getAlignment();
       } else if (lvalue.isVectorElt()) {
-        ValueTy = lvalue.getType()->getAs<VectorType>()->getElementType();
+        ValueTy = lvalue.getType()->castAs<VectorType>()->getElementType();
         ValueSizeInBits = C.getTypeSize(ValueTy);
         AtomicTy = lvalue.getType();
         AtomicSizeInBits = C.getTypeSize(AtomicTy);
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index d3b7033e5671a..37be19275809b 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2366,7 +2366,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
             .toCharUnitsFromBits(TI.getSuitableAlign())
             .getQuantity();
     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
-    AI->setAlignment(SuitableAlignmentInBytes);
+    AI->setAlignment(MaybeAlign(SuitableAlignmentInBytes));
     initializeAlloca(*this, AI, Size, SuitableAlignmentInBytes);
     return RValue::get(AI);
   }
@@ -2379,7 +2379,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
     unsigned AlignmentInBytes =
         CGM.getContext().toCharUnitsFromBits(AlignmentInBits).getQuantity();
     AllocaInst *AI = Builder.CreateAlloca(Builder.getInt8Ty(), Size);
-    AI->setAlignment(AlignmentInBytes);
+    AI->setAlignment(MaybeAlign(AlignmentInBytes));
     initializeAlloca(*this, AI, Size, AlignmentInBytes);
     return RValue::get(AI);
   }
@@ -14200,6 +14200,63 @@ Value *CodeGenFunction::EmitWebAssemblyBuiltinExpr(unsigned BuiltinID,
     Function *Callee = CGM.getIntrinsic(IntNo, A->getType());
     return Builder.CreateCall(Callee, {A, B, C});
   }
+  case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
+  case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
+  case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
+  case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4: {
+    Value *Low = EmitScalarExpr(E->getArg(0));
+    Value *High = EmitScalarExpr(E->getArg(1));
+    unsigned IntNo;
+    switch (BuiltinID) {
+    case WebAssembly::BI__builtin_wasm_narrow_s_i8x16_i16x8:
+    case WebAssembly::BI__builtin_wasm_narrow_s_i16x8_i32x4:
+      IntNo = Intrinsic::wasm_narrow_signed;
+      break;
+    case WebAssembly::BI__builtin_wasm_narrow_u_i8x16_i16x8:
+    case WebAssembly::BI__builtin_wasm_narrow_u_i16x8_i32x4:
+      IntNo = Intrinsic::wasm_narrow_unsigned;
+      break;
+    default:
+      llvm_unreachable("unexpected builtin ID");
+    }
+    Function *Callee =
+        CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Low->getType()});
+    return Builder.CreateCall(Callee, {Low, High});
+  }
+  case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
+  case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
+  case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
+  case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
+  case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
+  case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
+  case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
+  case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8: {
+    Value *Vec = EmitScalarExpr(E->getArg(0));
+    unsigned IntNo;
+    switch (BuiltinID) {
+    case WebAssembly::BI__builtin_wasm_widen_low_s_i16x8_i8x16:
+    case WebAssembly::BI__builtin_wasm_widen_low_s_i32x4_i16x8:
+      IntNo = Intrinsic::wasm_widen_low_signed;
+      break;
+    case WebAssembly::BI__builtin_wasm_widen_high_s_i16x8_i8x16:
+    case WebAssembly::BI__builtin_wasm_widen_high_s_i32x4_i16x8:
+      IntNo = Intrinsic::wasm_widen_high_signed;
+      break;
+    case WebAssembly::BI__builtin_wasm_widen_low_u_i16x8_i8x16:
+    case WebAssembly::BI__builtin_wasm_widen_low_u_i32x4_i16x8:
+      IntNo = Intrinsic::wasm_widen_low_unsigned;
+      break;
+    case WebAssembly::BI__builtin_wasm_widen_high_u_i16x8_i8x16:
+    case WebAssembly::BI__builtin_wasm_widen_high_u_i32x4_i16x8:
+      IntNo = Intrinsic::wasm_widen_high_unsigned;
+      break;
+    default:
+      llvm_unreachable("unexpected builtin ID");
+    }
+    Function *Callee =
+        CGM.getIntrinsic(IntNo, {ConvertType(E->getType()), Vec->getType()});
+    return Builder.CreateCall(Callee, Vec);
+  }
   default:
     return nullptr;
   }
diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp
index 4d4038dae9cfc..05aeef4194d4d 100644
--- a/clang/lib/CodeGen/CGCUDANV.cpp
+++ b/clang/lib/CodeGen/CGCUDANV.cpp
@@ -236,7 +236,8 @@ void CGNVCUDARuntime::emitDeviceStub(CodeGenFunction &CGF,
 
   EmittedKernels.push_back({CGF.CurFn, CGF.CurFuncDecl});
   if (CudaFeatureEnabled(CGM.getTarget().getSDKVersion(),
-                         CudaFeature::CUDA_USES_NEW_LAUNCH))
+                         CudaFeature::CUDA_USES_NEW_LAUNCH) ||
+      CGF.getLangOpts().HIPUseNewLaunchAPI)
     emitDeviceStubBodyNew(CGF, Args);
   else
     emitDeviceStubBodyLegacy(CGF, Args);
@@ -264,14 +265,18 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
 
   llvm::BasicBlock *EndBlock = CGF.createBasicBlock("setup.end");
 
-  // Lookup cudaLaunchKernel function.
+  // Lookup cudaLaunchKernel/hipLaunchKernel function.
   // cudaError_t cudaLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
   //                              void **args, size_t sharedMem,
   //                              cudaStream_t stream);
+  // hipError_t hipLaunchKernel(const void *func, dim3 gridDim, dim3 blockDim,
+  //                            void **args, size_t sharedMem,
+  //                            hipStream_t stream);
   TranslationUnitDecl *TUDecl = CGM.getContext().getTranslationUnitDecl();
   DeclContext *DC = TranslationUnitDecl::castToDeclContext(TUDecl);
+  auto LaunchKernelName = addPrefixToName("LaunchKernel");
   IdentifierInfo &cudaLaunchKernelII =
-      CGM.getContext().Idents.get("cudaLaunchKernel");
+      CGM.getContext().Idents.get(LaunchKernelName);
   FunctionDecl *cudaLaunchKernelFD = nullptr;
   for (const auto &Result : DC->lookup(&cudaLaunchKernelII)) {
     if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Result))
@@ -280,7 +285,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
 
   if (cudaLaunchKernelFD == nullptr) {
     CGM.Error(CGF.CurFuncDecl->getLocation(),
-              "Can't find declaration for cudaLaunchKernel()");
+              "Can't find declaration for " + LaunchKernelName);
     return;
   }
   // Create temporary dim3 grid_dim, block_dim.
@@ -301,7 +306,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
                                /*ShmemSize=*/ShmemSize.getType(),
                                /*Stream=*/Stream.getType()},
                               /*isVarArg=*/false),
-      "__cudaPopCallConfiguration");
+      addUnderscoredPrefixToName("PopCallConfiguration"));
 
   CGF.EmitRuntimeCallOrInvoke(cudaPopConfigFn,
                               {GridDim.getPointer(), BlockDim.getPointer(),
@@ -329,7 +334,7 @@ void CGNVCUDARuntime::emitDeviceStubBodyNew(CodeGenFunction &CGF,
   const CGFunctionInfo &FI =
       CGM.getTypes().arrangeFunctionDeclaration(cudaLaunchKernelFD);
   llvm::FunctionCallee cudaLaunchKernelFn =
-      CGM.CreateRuntimeFunction(FTy, "cudaLaunchKernel");
+      CGM.CreateRuntimeFunction(FTy, LaunchKernelName);
   CGF.EmitCall(FI, CGCallee::forDirect(cudaLaunchKernelFn), ReturnValueSlot(),
                LaunchKernelArgs);
   CGF.EmitBranch(EndBlock);
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 97d50fe709714..43da978a56b10 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -3093,7 +3093,7 @@ void CodeGenFunction::EmitDelegateCallArg(CallArgList &args,
   // Deactivate the cleanup for the callee-destructed param that was pushed.
   if (hasAggregateEvaluationKind(type) && !CurFuncIsThunk &&
       type->getAs<RecordType>()->getDecl()->isParamDestroyedInCallee() &&
-      type.isDestructedType()) {
+      param->needsDestruction(getContext())) {
     EHScopeStack::stable_iterator cleanup =
         CalleeDestructedParamCleanups.lookup(cast<ParmVarDecl>(param));
     assert(cleanup.isValid() &&
@@ -3841,7 +3841,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       AI = CreateTempAlloca(ArgStruct, "argmem");
     }
     auto Align = CallInfo.getArgStructAlignment();
-    AI->setAlignment(Align.getQuantity());
+    AI->setAlignment(llvm::MaybeAlign(Align.getQuantity()));
     AI->setUsedWithInAlloca(true);
     assert(AI->isUsedWithInAlloca() && !AI->isStaticAlloca());
     ArgMemory = Address(AI, Align);
@@ -4143,11 +4143,12 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         auto scalarAlign = CGM.getDataLayout().getPrefTypeAlignment(scalarType);
 
         // Materialize to a temporary.
-        addr = CreateTempAlloca(RV.getScalarVal()->getType(),
-                                CharUnits::fromQuantity(std::max(
-                                    layout->getAlignment(), scalarAlign)),
-                                "tmp",
-                                /*ArraySize=*/nullptr, &AllocaAddr);
+        addr = CreateTempAlloca(
+            RV.getScalarVal()->getType(),
+            CharUnits::fromQuantity(std::max(
+                (unsigned)layout->getAlignment().value(), scalarAlign)),
+            "tmp",
+            /*ArraySize=*/nullptr, &AllocaAddr);
         tempSize = EmitLifetimeStart(scalarSize, AllocaAddr.getPointer());
 
         Builder.CreateStore(RV.getScalarVal(), addr);
diff --git a/clang/lib/CodeGen/CGClass.cpp b/clang/lib/CodeGen/CGClass.cpp
index b4fedb0836ec1..3312a279c1571 100644
--- a/clang/lib/CodeGen/CGClass.cpp
+++ b/clang/lib/CodeGen/CGClass.cpp
@@ -2085,7 +2085,7 @@ static bool canEmitDelegateCallArgs(CodeGenFunction &CGF,
   if (CGF.getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee()) {
     // If the parameters are callee-cleanup, it's not safe to forward.
     for (auto *P : Ctor->parameters())
-      if (P->getType().isDestructedType())
+      if (P->needsDestruction(CGF.getContext()))
         return false;
 
     // Likewise if they're inalloca.
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index 3cf366e4635d5..1280bdb1932e5 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -310,7 +310,7 @@ static void createStoreInstBefore(llvm::Value *value, Address addr,
 static llvm::LoadInst *createLoadInstBefore(Address addr, const Twine &name,
                                             llvm::Instruction *beforeInst) {
   auto load = new llvm::LoadInst(addr.getPointer(), name, beforeInst);
-  load->setAlignment(addr.getAlignment().getQuantity());
+  load->setAlignment(llvm::MaybeAlign(addr.getAlignment().getQuantity()));
   return load;
 }
 
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index e36f6932f86e7..71d99bc551c48 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -561,6 +561,10 @@ void CGDebugInfo::CreateCompileUnit() {
   if (LO.CPlusPlus) {
     if (LO.ObjC)
       LangTag = llvm::dwarf::DW_LANG_ObjC_plus_plus;
+    else if (LO.CPlusPlus14)
+      LangTag = llvm::dwarf::DW_LANG_C_plus_plus_14;
+    else if (LO.CPlusPlus11)
+      LangTag = llvm::dwarf::DW_LANG_C_plus_plus_11;
     else
       LangTag = llvm::dwarf::DW_LANG_C_plus_plus;
   } else if (LO.ObjC) {
@@ -878,6 +882,8 @@ llvm::DIType *CGDebugInfo::CreateType(const PointerType *Ty,
 static bool hasCXXMangling(const TagDecl *TD, llvm::DICompileUnit *TheCU) {
   switch (TheCU->getSourceLanguage()) {
   case llvm::dwarf::DW_LANG_C_plus_plus:
+  case llvm::dwarf::DW_LANG_C_plus_plus_11:
+  case llvm::dwarf::DW_LANG_C_plus_plus_14:
     return true;
   case llvm::dwarf::DW_LANG_ObjC_plus_plus:
     return isa<CXXRecordDecl>(TD) || isa<EnumDecl>(TD);
@@ -2711,6 +2717,8 @@ llvm::DIType *CGDebugInfo::CreateType(const MemberPointerType *Ty,
         break;
       case MSInheritanceAttr::Keyword_unspecified_inheritance:
         break;
+      case MSInheritanceAttr::SpellingNotCalculated:
+        llvm_unreachable("Spelling not yet calculated");
       }
     }
   }
@@ -2994,7 +3002,7 @@ llvm::DIType *CGDebugInfo::CreateTypeNode(QualType Ty, llvm::DIFile *Unit) {
 #define ABSTRACT_TYPE(Class, Base)
 #define NON_CANONICAL_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Dependent types cannot show up in debug information");
 
   case Type::ExtVector:
diff --git a/clang/lib/CodeGen/CGDecl.cpp b/clang/lib/CodeGen/CGDecl.cpp
index 08d71252ef0b5..d0a0f20d2f823 100644
--- a/clang/lib/CodeGen/CGDecl.cpp
+++ b/clang/lib/CodeGen/CGDecl.cpp
@@ -309,14 +309,6 @@ llvm::Constant *CodeGenModule::getOrCreateStaticVarDecl(
   return Addr;
 }
 
-/// hasNontrivialDestruction - Determine whether a type's destruction is
-/// non-trivial. If so, and the variable uses static initialization, we must
-/// register its destructor to run on exit.
-static bool hasNontrivialDestruction(QualType T) {
-  CXXRecordDecl *RD = T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
-  return RD && !RD->hasTrivialDestructor();
-}
-
 /// AddInitializerToStaticVarDecl - Add the initializer for 'D' to the
 /// global variable that has already been created for it.  If the initializer
 /// has a different type than GV does, this may free GV and return a different
@@ -395,7 +387,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
 
   emitter.finalize(GV);
 
-  if (hasNontrivialDestruction(D.getType()) && HaveInsertPoint()) {
+  if (D.needsDestruction(getContext()) && HaveInsertPoint()) {
     // We have a constant initializer, but a nontrivial destructor. We still
     // need to perform a guarded "initialization" in order to register the
     // destructor.
@@ -2037,7 +2029,7 @@ void CodeGenFunction::EmitAutoVarCleanups(const AutoVarEmission &emission) {
   const VarDecl &D = *emission.Variable;
 
   // Check the type for a cleanup.
-  if (QualType::DestructionKind dtorKind = D.getType().isDestructedType())
+  if (QualType::DestructionKind dtorKind = D.needsDestruction(getContext()))
     emitAutoVarTypeCleanup(emission, dtorKind);
 
   // In GC mode, honor objc_precise_lifetime.
@@ -2447,7 +2439,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, ParamValue Arg,
     // cleanup.
     if (hasAggregateEvaluationKind(Ty) && !CurFuncIsThunk &&
         Ty->getAs<RecordType>()->getDecl()->isParamDestroyedInCallee()) {
-      if (QualType::DestructionKind DtorKind = Ty.isDestructedType()) {
+      if (QualType::DestructionKind DtorKind =
+              D.needsDestruction(getContext())) {
         assert((DtorKind == QualType::DK_cxx_destructor ||
                 DtorKind == QualType::DK_nontrivial_c_struct) &&
                "unexpected destructor type");
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 0bc256a78d357..358c856c5fc04 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -75,16 +75,10 @@ static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
   // that isn't balanced out by a destructor call as intended by the
   // attribute. This also checks for -fno-c++-static-destructors and
   // bails even if the attribute is not present.
-  if (D.isNoDestroy(CGF.getContext()))
-    return;
-
-  CodeGenModule &CGM = CGF.CGM;
+  QualType::DestructionKind DtorKind = D.needsDestruction(CGF.getContext());
 
   // FIXME:  __attribute__((cleanup)) ?
 
-  QualType Type = D.getType();
-  QualType::DestructionKind DtorKind = Type.isDestructedType();
-
   switch (DtorKind) {
   case QualType::DK_none:
     return;
@@ -103,6 +97,9 @@ static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
   llvm::FunctionCallee Func;
   llvm::Constant *Argument;
 
+  CodeGenModule &CGM = CGF.CGM;
+  QualType Type = D.getType();
+
   // Special-case non-array C++ destructors, if they have the right signature.
   // Under some ABIs, destructors return this instead of void, and cannot be
   // passed directly to __cxa_atexit if the target does not allow this
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index b442178f03a89..175ec8c5ceb92 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -66,7 +66,7 @@ Address CodeGenFunction::CreateTempAllocaWithoutCast(llvm::Type *Ty,
                                                      const Twine &Name,
                                                      llvm::Value *ArraySize) {
   auto Alloca = CreateTempAlloca(Ty, Name, ArraySize);
-  Alloca->setAlignment(Align.getQuantity());
+  Alloca->setAlignment(llvm::MaybeAlign(Align.getQuantity()));
   return Address(Alloca, Align);
 }
 
diff --git a/clang/lib/CodeGen/CGExprAgg.cpp b/clang/lib/CodeGen/CGExprAgg.cpp
index 0a57870a7c583..9610a04be343a 100644
--- a/clang/lib/CodeGen/CGExprAgg.cpp
+++ b/clang/lib/CodeGen/CGExprAgg.cpp
@@ -1759,7 +1759,7 @@ static CharUnits GetNumNonZeroBytesInInit(const Expr *E, CodeGenFunction &CGF) {
   // referencee.  InitListExprs for unions and arrays can't have references.
   if (const RecordType *RT = E->getType()->getAs<RecordType>()) {
     if (!RT->isUnionType()) {
-      RecordDecl *SD = E->getType()->getAs<RecordType>()->getDecl();
+      RecordDecl *SD = RT->getDecl();
       CharUnits NumNonZeroBytes = CharUnits::Zero();
 
       unsigned ILEElement = 0;
diff --git a/clang/lib/CodeGen/CGExprScalar.cpp b/clang/lib/CodeGen/CGExprScalar.cpp
index e6bf6d7720765..7d01fbf27c4db 100644
--- a/clang/lib/CodeGen/CGExprScalar.cpp
+++ b/clang/lib/CodeGen/CGExprScalar.cpp
@@ -1657,8 +1657,8 @@ Value *ScalarExprEmitter::VisitConvertVectorExpr(ConvertVectorExpr *E) {
   if (SrcTy == DstTy)
     return Src;
 
-  QualType SrcEltType = SrcType->getAs<VectorType>()->getElementType(),
-           DstEltType = DstType->getAs<VectorType>()->getElementType();
+  QualType SrcEltType = SrcType->castAs<VectorType>()->getElementType(),
+           DstEltType = DstType->castAs<VectorType>()->getElementType();
 
   assert(SrcTy->isVectorTy() &&
          "ConvertVector source IR type must be a vector");
@@ -3815,7 +3815,7 @@ Value *ScalarExprEmitter::EmitCompare(const BinaryOperator *E,
       Value *FirstVecArg = LHS,
             *SecondVecArg = RHS;
 
-      QualType ElTy = LHSTy->getAs<VectorType>()->getElementType();
+      QualType ElTy = LHSTy->castAs<VectorType>()->getElementType();
       const BuiltinType *BTy = ElTy->getAs<BuiltinType>();
       BuiltinType::Kind ElementKind = BTy->getKind();
 
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index ee5c12aa35bd7..ee0fd184c64d4 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -1854,7 +1854,8 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
         ivarBuilder.addInt(Int32Ty,
             CGM.getContext().getTypeSizeInChars(ivarTy).getQuantity());
         // Alignment will be stored as a base-2 log of the alignment.
-        int align = llvm::Log2_32(Context.getTypeAlignInChars(ivarTy).getQuantity());
+        unsigned align =
+            llvm::Log2_32(Context.getTypeAlignInChars(ivarTy).getQuantity());
         // Objects that require more than 2^64-byte alignment should be impossible!
         assert(align < 64);
         // uint32_t flags;
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 12880fecbadff..2b92c4b9e6df0 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -2517,14 +2517,12 @@ void CGObjCCommonMac::BuildRCRecordLayout(const llvm::StructLayout *RecLayout,
     }
 
     if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
-      const ConstantArrayType *CArray =
-        dyn_cast_or_null<ConstantArrayType>(Array);
+      auto *CArray = cast<ConstantArrayType>(Array);
       uint64_t ElCount = CArray->getSize().getZExtValue();
       assert(CArray && "only array with known element size is supported");
       FQT = CArray->getElementType();
       while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
-        const ConstantArrayType *CArray =
-          dyn_cast_or_null<ConstantArrayType>(Array);
+        auto *CArray = cast<ConstantArrayType>(Array);
         ElCount *= CArray->getSize().getZExtValue();
         FQT = CArray->getElementType();
       }
@@ -4902,7 +4900,7 @@ LValue CGObjCMac::EmitObjCValueForIvar(CodeGen::CodeGenFunction &CGF,
                                        const ObjCIvarDecl *Ivar,
                                        unsigned CVRQualifiers) {
   const ObjCInterfaceDecl *ID =
-    ObjectTy->getAs<ObjCObjectType>()->getInterface();
+    ObjectTy->castAs<ObjCObjectType>()->getInterface();
   return EmitValueForIvarAtOffset(CGF, ID, BaseValue, Ivar, CVRQualifiers,
                                   EmitIvarOffset(CGF, ID, Ivar));
 }
@@ -7053,7 +7051,7 @@ LValue CGObjCNonFragileABIMac::EmitObjCValueForIvar(
                                                llvm::Value *BaseValue,
                                                const ObjCIvarDecl *Ivar,
                                                unsigned CVRQualifiers) {
-  ObjCInterfaceDecl *ID = ObjectTy->getAs<ObjCObjectType>()->getInterface();
+  ObjCInterfaceDecl *ID = ObjectTy->castAs<ObjCObjectType>()->getInterface();
   llvm::Value *Offset = EmitIvarOffset(CGF, ID, Ivar);
   return EmitValueForIvarAtOffset(CGF, ID, BaseValue, Ivar, CVRQualifiers,
                                   Offset);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 99fa0790e908e..4da84f230ec1e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -1264,6 +1264,51 @@ CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator,
   loadOffloadInfoMetadata();
 }
 
+static bool tryEmitAlias(CodeGenModule &CGM, const GlobalDecl &NewGD,
+                         const GlobalDecl &OldGD, llvm::GlobalValue *OrigAddr,
+                         bool IsForDefinition) {
+  // Emit at least a definition for the aliasee if the the address of the
+  // original function is requested.
+  if (IsForDefinition || OrigAddr)
+    (void)CGM.GetAddrOfGlobal(NewGD);
+  StringRef NewMangledName = CGM.getMangledName(NewGD);
+  llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName);
+  if (Addr && !Addr->isDeclaration()) {
+    const auto *D = cast<FunctionDecl>(OldGD.getDecl());
+    const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD);
+    llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI);
+
+    // Create a reference to the named value.  This ensures that it is emitted
+    // if a deferred decl.
+    llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD);
+
+    // Create the new alias itself, but don't set a name yet.
+    auto *GA =
+        llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule());
+
+    if (OrigAddr) {
+      assert(OrigAddr->isDeclaration() && "Expected declaration");
+
+      GA->takeName(OrigAddr);
+      OrigAddr->replaceAllUsesWith(
+          llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType()));
+      OrigAddr->eraseFromParent();
+    } else {
+      GA->setName(CGM.getMangledName(OldGD));
+    }
+
+    // Set attributes which are particular to an alias; this is a
+    // specialization of the attributes which may be set on a global function.
+    if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() ||
+        D->isWeakImported())
+      GA->setLinkage(llvm::Function::WeakAnyLinkage);
+
+    CGM.SetCommonAttributes(OldGD, GA);
+    return true;
+  }
+  return false;
+}
+
 void CGOpenMPRuntime::clear() {
   InternalVars.clear();
   // Clean non-target variable declarations possibly used only in debug info.
@@ -1277,6 +1322,14 @@ void CGOpenMPRuntime::clear() {
       continue;
     GV->eraseFromParent();
   }
+  // Emit aliases for the deferred aliasees.
+  for (const auto &Pair : DeferredVariantFunction) {
+    StringRef MangledName = CGM.getMangledName(Pair.second.second);
+    llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName);
+    // If not able to emit alias, just emit original declaration.
+    (void)tryEmitAlias(CGM, Pair.second.first, Pair.second.second, Addr,
+                       /*IsForDefinition=*/false);
+  }
 }
 
 std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const {
@@ -2815,6 +2868,8 @@ llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition(
 bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD,
                                                      llvm::GlobalVariable *Addr,
                                                      bool PerformInit) {
+  if (CGM.getLangOpts().OMPTargetTriples.empty())
+    return false;
   Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
   if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||
@@ -3970,16 +4025,16 @@ CGOpenMPRuntime::createOffloadingBinaryDescriptorRegistration() {
   // host entries section. These will be defined by the linker.
   llvm::Type *OffloadEntryTy =
       CGM.getTypes().ConvertTypeForMem(getTgtOffloadEntryQTy());
-  std::string EntriesBeginName = getName({"omp_offloading", "entries_begin"});
   auto *HostEntriesBegin = new llvm::GlobalVariable(
       M, OffloadEntryTy, /*isConstant=*/true,
       llvm::GlobalValue::ExternalLinkage, /*Initializer=*/nullptr,
-      EntriesBeginName);
-  std::string EntriesEndName = getName({"omp_offloading", "entries_end"});
-  auto *HostEntriesEnd =
-      new llvm::GlobalVariable(M, OffloadEntryTy, /*isConstant=*/true,
-                               llvm::GlobalValue::ExternalLinkage,
-                               /*Initializer=*/nullptr, EntriesEndName);
+      "__start_omp_offloading_entries");
+  HostEntriesBegin->setVisibility(llvm::GlobalValue::HiddenVisibility);
+  auto *HostEntriesEnd = new llvm::GlobalVariable(
+      M, OffloadEntryTy, /*isConstant=*/true,
+      llvm::GlobalValue::ExternalLinkage,
+      /*Initializer=*/nullptr, "__stop_omp_offloading_entries");
+  HostEntriesEnd->setVisibility(llvm::GlobalValue::HiddenVisibility);
 
   // Create all device images
   auto *DeviceImageTy = cast<llvm::StructType>(
@@ -4127,8 +4182,7 @@ void CGOpenMPRuntime::createOffloadEntry(
       Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage);
 
   // The entry has to be created in the section the linker expects it to be.
-  std::string Section = getName({"omp_offloading", "entries"});
-  Entry->setSection(Section);
+  Entry->setSection("omp_offloading_entries");
 }
 
 void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() {
@@ -6770,6 +6824,7 @@ emitNumTeamsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_teams_distribute_parallel_for_simd:
   case OMPD_target_update:
   case OMPD_declare_simd:
+  case OMPD_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -7075,6 +7130,7 @@ emitNumThreadsForTargetDirective(CodeGenFunction &CGF,
   case OMPD_teams_distribute_parallel_for_simd:
   case OMPD_target_update:
   case OMPD_declare_simd:
+  case OMPD_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -7246,9 +7302,11 @@ class MappableExprsHandler {
                             OAE->getBase()->IgnoreParenImpCasts())
                             .getCanonicalType();
 
-      // If there is no length associated with the expression, that means we
-      // are using the whole length of the base.
-      if (!OAE->getLength() && OAE->getColonLoc().isValid())
+      // If there is no length associated with the expression and lower bound is
+      // not specified too, that means we are using the whole length of the
+      // base.
+      if (!OAE->getLength() && OAE->getColonLoc().isValid() &&
+          !OAE->getLowerBound())
         return CGF.getTypeSize(BaseTy);
 
       llvm::Value *ElemSize;
@@ -7262,13 +7320,30 @@ class MappableExprsHandler {
 
       // If we don't have a length at this point, that is because we have an
       // array section with a single element.
-      if (!OAE->getLength())
+      if (!OAE->getLength() && OAE->getColonLoc().isInvalid())
         return ElemSize;
 
-      llvm::Value *LengthVal = CGF.EmitScalarExpr(OAE->getLength());
-      LengthVal =
-          CGF.Builder.CreateIntCast(LengthVal, CGF.SizeTy, /*isSigned=*/false);
-      return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
+      if (const Expr *LenExpr = OAE->getLength()) {
+        llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr);
+        LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(),
+                                             CGF.getContext().getSizeType(),
+                                             LenExpr->getExprLoc());
+        return CGF.Builder.CreateNUWMul(LengthVal, ElemSize);
+      }
+      assert(!OAE->getLength() && OAE->getColonLoc().isValid() &&
+             OAE->getLowerBound() && "expected array_section[lb:].");
+      // Size = sizetype - lb * elemtype;
+      llvm::Value *LengthVal = CGF.getTypeSize(BaseTy);
+      llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound());
+      LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(),
+                                       CGF.getContext().getSizeType(),
+                                       OAE->getLowerBound()->getExprLoc());
+      LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize);
+      llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal);
+      llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal);
+      LengthVal = CGF.Builder.CreateSelect(
+          Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0));
+      return LengthVal;
     }
     return CGF.getTypeSize(ExprTy);
   }
@@ -8826,6 +8901,7 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_target_update:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -9173,9 +9249,11 @@ void CGOpenMPRuntime::emitUDMapperArrayInitOrDel(
 }
 
 void CGOpenMPRuntime::emitTargetNumIterationsCall(
-    CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device,
-    const llvm::function_ref<llvm::Value *(
-        CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter) {
+    CodeGenFunction &CGF, const OMPExecutableDirective &D,
+    llvm::Value *DeviceID,
+    llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                     const OMPLoopDirective &D)>
+        SizeEmitter) {
   OpenMPDirectiveKind Kind = D.getDirectiveKind();
   const OMPExecutableDirective *TD = &D;
   // Get nested teams distribute kind directive, if any.
@@ -9184,30 +9262,24 @@ void CGOpenMPRuntime::emitTargetNumIterationsCall(
   if (!TD)
     return;
   const auto *LD = cast<OMPLoopDirective>(TD);
-  auto &&CodeGen = [LD, &Device, &SizeEmitter, this](CodeGenFunction &CGF,
+  auto &&CodeGen = [LD, DeviceID, SizeEmitter, this](CodeGenFunction &CGF,
                                                      PrePostActionTy &) {
-    llvm::Value *NumIterations = SizeEmitter(CGF, *LD);
-
-    // Emit device ID if any.
-    llvm::Value *DeviceID;
-    if (Device)
-      DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
-                                           CGF.Int64Ty, /*isSigned=*/true);
-    else
-      DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF);
-
-    llvm::Value *Args[] = {DeviceID, NumIterations};
-    CGF.EmitRuntimeCall(
-        createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args);
+    if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) {
+      llvm::Value *Args[] = {DeviceID, NumIterations};
+      CGF.EmitRuntimeCall(
+          createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args);
+    }
   };
   emitInlinedDirective(CGF, OMPD_unknown, CodeGen);
 }
 
-void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
-                                     const OMPExecutableDirective &D,
-                                     llvm::Function *OutlinedFn,
-                                     llvm::Value *OutlinedFnID,
-                                     const Expr *IfCond, const Expr *Device) {
+void CGOpenMPRuntime::emitTargetCall(
+    CodeGenFunction &CGF, const OMPExecutableDirective &D,
+    llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
+    const Expr *Device,
+    llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                     const OMPLoopDirective &D)>
+        SizeEmitter) {
   if (!CGF.HaveInsertPoint())
     return;
 
@@ -9226,8 +9298,8 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
   llvm::Value *MapTypesArray = nullptr;
   // Fill up the pointer arrays and transfer execution to the device.
   auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo,
-                    &MapTypesArray, &CS, RequiresOuterTask,
-                    &CapturedVars](CodeGenFunction &CGF, PrePostActionTy &) {
+                    &MapTypesArray, &CS, RequiresOuterTask, &CapturedVars,
+                    SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) {
     // On top of the arrays that were filled up, the target offloading call
     // takes as arguments the device id as well as the host pointer. The host
     // pointer is used by the runtime library to identify the current target
@@ -9259,6 +9331,9 @@ void CGOpenMPRuntime::emitTargetCall(CodeGenFunction &CGF,
     llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D);
     llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D);
 
+    // Emit tripcount for the target loop-based directive.
+    emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter);
+
     bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>();
     // The target region is an outlined function launched by the runtime
     // via calls __tgt_target() or __tgt_target_teams().
@@ -9583,6 +9658,7 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_target_update:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -9715,6 +9791,8 @@ CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF,
 
 void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD,
                                                    llvm::Constant *Addr) {
+  if (CGM.getLangOpts().OMPTargetTriples.empty())
+    return;
   llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =
       OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);
   if (!Res) {
@@ -10205,6 +10283,7 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     case OMPD_teams_distribute_parallel_for:
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -11060,6 +11139,80 @@ Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF,
   return Address(Addr, Align);
 }
 
+/// Checks current context and returns true if it matches the context selector.
+template <OMPDeclareVariantAttr::CtxSelectorSetType CtxSet,
+          OMPDeclareVariantAttr::CtxSelectorType Ctx>
+static bool checkContext(const OMPDeclareVariantAttr *A) {
+  assert(CtxSet != OMPDeclareVariantAttr::CtxSetUnknown &&
+         Ctx != OMPDeclareVariantAttr::CtxUnknown &&
+         "Unknown context selector or context selector set.");
+  return false;
+}
+
+/// Checks for implementation={vendor(<vendor>)} context selector.
+/// \returns true iff <vendor>="llvm", false otherwise.
+template <>
+bool checkContext<OMPDeclareVariantAttr::CtxSetImplementation,
+                  OMPDeclareVariantAttr::CtxVendor>(
+    const OMPDeclareVariantAttr *A) {
+  return !A->getImplVendor().compare("llvm");
+}
+
+/// Finds the variant function that matches current context with its context
+/// selector.
+static const FunctionDecl *getDeclareVariantFunction(const FunctionDecl *FD) {
+  if (!FD->hasAttrs() || !FD->hasAttr<OMPDeclareVariantAttr>())
+    return FD;
+  // Iterate through all DeclareVariant attributes and check context selectors.
+  SmallVector<const OMPDeclareVariantAttr *, 4> MatchingAttributes;
+  for (const auto * A : FD->specific_attrs<OMPDeclareVariantAttr>()) {
+    switch (A->getCtxSelectorSet()) {
+    case OMPDeclareVariantAttr::CtxSetImplementation:
+      switch (A->getCtxSelector()) {
+      case OMPDeclareVariantAttr::CtxVendor:
+        if (checkContext<OMPDeclareVariantAttr::CtxSetImplementation,
+                         OMPDeclareVariantAttr::CtxVendor>(A))
+          MatchingAttributes.push_back(A);
+        break;
+      case OMPDeclareVariantAttr::CtxUnknown:
+        llvm_unreachable(
+            "Unknown context selector in implementation selctor set.");
+      }
+      break;
+    case OMPDeclareVariantAttr::CtxSetUnknown:
+      llvm_unreachable("Unknown context selector set.");
+    }
+  }
+  if (MatchingAttributes.empty())
+    return FD;
+  // TODO: implement score analysis of multiple context selectors.
+  const OMPDeclareVariantAttr *MainAttr = MatchingAttributes.front();
+  return cast<FunctionDecl>(
+      cast<DeclRefExpr>(MainAttr->getVariantFuncRef()->IgnoreParenImpCasts())
+          ->getDecl());
+}
+
+bool CGOpenMPRuntime::emitDeclareVariant(GlobalDecl GD, bool IsForDefinition) {
+  const auto *D = cast<FunctionDecl>(GD.getDecl());
+  // If the original function is defined already, use its definition.
+  StringRef MangledName = CGM.getMangledName(GD);
+  llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName);
+  if (Orig && !Orig->isDeclaration())
+    return false;
+  const FunctionDecl *NewFD = getDeclareVariantFunction(D);
+  // Emit original function if it does not have declare variant attribute or the
+  // context does not match.
+  if (NewFD == D)
+    return false;
+  GlobalDecl NewGD = GD.getWithDecl(NewFD);
+  if (tryEmitAlias(CGM, NewGD, GD, Orig, IsForDefinition)) {
+    DeferredVariantFunction.erase(D);
+    return true;
+  }
+  DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD)));
+  return true;
+}
+
 llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction(
     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
@@ -11280,12 +11433,13 @@ void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction(
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
-void CGOpenMPSIMDRuntime::emitTargetCall(CodeGenFunction &CGF,
-                                         const OMPExecutableDirective &D,
-                                         llvm::Function *OutlinedFn,
-                                         llvm::Value *OutlinedFnID,
-                                         const Expr *IfCond,
-                                         const Expr *Device) {
+void CGOpenMPSIMDRuntime::emitTargetCall(
+    CodeGenFunction &CGF, const OMPExecutableDirective &D,
+    llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond,
+    const Expr *Device,
+    llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                     const OMPLoopDirective &D)>
+        SizeEmitter) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 92e946c670c01..b8137a20d0013 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -15,6 +15,7 @@
 
 #include "CGValue.h"
 #include "clang/AST/DeclOpenMP.h"
+#include "clang/AST/GlobalDecl.h"
 #include "clang/AST/Type.h"
 #include "clang/Basic/OpenMPKinds.h"
 #include "clang/Basic/SourceLocation.h"
@@ -36,7 +37,6 @@ class Value;
 
 namespace clang {
 class Expr;
-class GlobalDecl;
 class OMPDependClause;
 class OMPExecutableDirective;
 class OMPLoopDirective;
@@ -644,6 +644,12 @@ class CGOpenMPRuntime {
   /// must be emitted.
   llvm::SmallDenseSet<const VarDecl *> DeferredGlobalVariables;
 
+  /// Mapping of the original functions to their variants and original global
+  /// decl.
+  llvm::MapVector<CanonicalDeclPtr<const FunctionDecl>,
+                  std::pair<GlobalDecl, GlobalDecl>>
+      DeferredVariantFunction;
+
   /// Flag for keeping track of weather a requires unified_shared_memory
   /// directive is present.
   bool HasRequiresUnifiedSharedMemory = false;
@@ -793,6 +799,17 @@ class CGOpenMPRuntime {
   /// default.
   virtual unsigned getDefaultFirstprivateAddressSpace() const { return 0; }
 
+  /// Emit code that pushes the trip count of loops associated with constructs
+  /// 'target teams distribute' and 'teams distribute parallel for'.
+  /// \param SizeEmitter Emits the int64 value for the number of iterations of
+  /// the associated loop.
+  void emitTargetNumIterationsCall(
+      CodeGenFunction &CGF, const OMPExecutableDirective &D,
+      llvm::Value *DeviceID,
+      llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                       const OMPLoopDirective &D)>
+          SizeEmitter);
+
 public:
   explicit CGOpenMPRuntime(CodeGenModule &CGM)
       : CGOpenMPRuntime(CGM, ".", ".") {}
@@ -1414,15 +1431,6 @@ class CGOpenMPRuntime {
                                           bool IsOffloadEntry,
                                           const RegionCodeGenTy &CodeGen);
 
-  /// Emit code that pushes the trip count of loops associated with constructs
-  /// 'target teams distribute' and 'teams distribute parallel for'.
-  /// \param SizeEmitter Emits the int64 value for the number of iterations of
-  /// the associated loop.
-  virtual void emitTargetNumIterationsCall(
-      CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *Device,
-      const llvm::function_ref<llvm::Value *(
-          CodeGenFunction &CGF, const OMPLoopDirective &D)> &SizeEmitter);
-
   /// Emit the target offloading code associated with \a D. The emitted
   /// code attempts offloading the execution to the device, an the event of
   /// a failure it executes the host version outlined in \a OutlinedFn.
@@ -1433,11 +1441,15 @@ class CGOpenMPRuntime {
   /// directive, or null if no if clause is used.
   /// \param Device Expression evaluated in device clause associated with the
   /// target directive, or null if no device clause is used.
-  virtual void emitTargetCall(CodeGenFunction &CGF,
-                              const OMPExecutableDirective &D,
-                              llvm::Function *OutlinedFn,
-                              llvm::Value *OutlinedFnID, const Expr *IfCond,
-                              const Expr *Device);
+  /// \param SizeEmitter Callback to emit number of iterations for loop-based
+  /// directives.
+  virtual void
+  emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
+                 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
+                 const Expr *IfCond, const Expr *Device,
+                 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                                  const OMPLoopDirective &D)>
+                     SizeEmitter);
 
   /// Emit the target regions enclosed in \a GD function definition or
   /// the function itself in case it is a valid device function. Returns true if
@@ -1646,6 +1658,9 @@ class CGOpenMPRuntime {
 
   /// Return whether the unified_shared_memory has been specified.
   bool hasRequiresUnifiedSharedMemory() const;
+
+  /// Emits the definition of the declare variant function.
+  virtual bool emitDeclareVariant(GlobalDecl GD, bool IsForDefinition);
 };
 
 /// Class supports emissionof SIMD-only code.
@@ -2117,9 +2132,13 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
   /// directive, or null if no if clause is used.
   /// \param Device Expression evaluated in device clause associated with the
   /// target directive, or null if no device clause is used.
-  void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
-                      llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
-                      const Expr *IfCond, const Expr *Device) override;
+  void
+  emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D,
+                 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,
+                 const Expr *IfCond, const Expr *Device,
+                 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF,
+                                                  const OMPLoopDirective &D)>
+                     SizeEmitter) override;
 
   /// Emit the target regions enclosed in \a GD function definition or
   /// the function itself in case it is a valid device function. Returns true if
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
index 09042b92d83b2..c6c595dcc2c6e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeNVPTX.cpp
@@ -795,6 +795,7 @@ static bool hasNestedSPMDDirective(ASTContext &Ctx,
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_target_update:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -865,6 +866,7 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx,
   case OMPD_teams_distribute_parallel_for_simd:
   case OMPD_target_update:
   case OMPD_declare_simd:
+  case OMPD_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
@@ -1028,6 +1030,7 @@ static bool hasNestedLightweightDirective(ASTContext &Ctx,
     case OMPD_teams_distribute_parallel_for_simd:
     case OMPD_target_update:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_declare_reduction:
@@ -1104,6 +1107,7 @@ static bool supportsLightweightRuntime(ASTContext &Ctx,
   case OMPD_teams_distribute_parallel_for_simd:
   case OMPD_target_update:
   case OMPD_declare_simd:
+  case OMPD_declare_variant:
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_declare_reduction:
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 473c6f7950ae7..bb6323e118cc2 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -120,11 +120,27 @@ class OMPTeamsScope final : public OMPLexicalScope {
 class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
   void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
     CodeGenFunction::OMPMapVars PreCondVars;
+    llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
     for (const auto *E : S.counters()) {
       const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+      EmittedAsPrivate.insert(VD->getCanonicalDecl());
       (void)PreCondVars.setVarAddr(
           CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
     }
+    // Mark private vars as undefs.
+    for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
+      for (const Expr *IRef : C->varlists()) {
+        const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
+        if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
+          (void)PreCondVars.setVarAddr(
+              CGF, OrigVD,
+              Address(llvm::UndefValue::get(
+                          CGF.ConvertTypeForMem(CGF.getContext().getPointerType(
+                              OrigVD->getType().getNonReferenceType()))),
+                      CGF.getContext().getDeclAlign(OrigVD)));
+        }
+      }
+    }
     (void)PreCondVars.apply(CGF);
     if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
       for (const auto *I : PreInits->decls())
@@ -4022,6 +4038,7 @@ static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed in 'omp atomic'.");
   }
 }
@@ -4121,18 +4138,21 @@ static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
   CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
                                                     IsOffloadEntry, CodeGen);
   OMPLexicalScope Scope(CGF, S, OMPD_task);
-  auto &&SizeEmitter = [](CodeGenFunction &CGF, const OMPLoopDirective &D) {
-    OMPLoopScope(CGF, D);
-    // Emit calculation of the iterations count.
-    llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
-    NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
-                                              /*isSigned=*/false);
-    return NumIterations;
+  auto &&SizeEmitter =
+      [IsOffloadEntry](CodeGenFunction &CGF,
+                       const OMPLoopDirective &D) -> llvm::Value * {
+    if (IsOffloadEntry) {
+      OMPLoopScope(CGF, D);
+      // Emit calculation of the iterations count.
+      llvm::Value *NumIterations = CGF.EmitScalarExpr(D.getNumIterations());
+      NumIterations = CGF.Builder.CreateIntCast(NumIterations, CGF.Int64Ty,
+                                                /*isSigned=*/false);
+      return NumIterations;
+    }
+    return nullptr;
   };
-  if (IsOffloadEntry)
-    CGM.getOpenMPRuntime().emitTargetNumIterationsCall(CGF, S, Device,
-                                                       SizeEmitter);
-  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
+  CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device,
+                                        SizeEmitter);
 }
 
 static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
diff --git a/clang/lib/CodeGen/CGVTables.cpp b/clang/lib/CodeGen/CGVTables.cpp
index 3055ea3bfd780..a74905fd70fd4 100644
--- a/clang/lib/CodeGen/CGVTables.cpp
+++ b/clang/lib/CodeGen/CGVTables.cpp
@@ -157,7 +157,7 @@ CodeGenFunction::GenerateVarArgsThunk(llvm::Function *Fn,
                                       const CGFunctionInfo &FnInfo,
                                       GlobalDecl GD, const ThunkInfo &Thunk) {
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
-  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
+  const FunctionProtoType *FPT = MD->getType()->castAs<FunctionProtoType>();
   QualType ResultType = FPT->getReturnType();
 
   // Get the original function
@@ -242,7 +242,6 @@ void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD,
   // Build FunctionArgs.
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(GD.getDecl());
   QualType ThisType = MD->getThisType();
-  const FunctionProtoType *FPT = MD->getType()->getAs<FunctionProtoType>();
   QualType ResultType;
   if (IsUnprototyped)
     ResultType = CGM.getContext().VoidTy;
@@ -251,7 +250,7 @@ void CodeGenFunction::StartThunk(llvm::Function *Fn, GlobalDecl GD,
   else if (CGM.getCXXABI().hasMostDerivedReturn(GD))
     ResultType = CGM.getContext().VoidPtrTy;
   else
-    ResultType = FPT->getReturnType();
+    ResultType = MD->getType()->castAs<FunctionProtoType>()->getReturnType();
   FunctionArgList FunctionArgs;
 
   // Create the implicit 'this' parameter declaration.
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 5ae088f6933d9..e2cd56a08d0d6 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -195,7 +195,7 @@ TypeEvaluationKind CodeGenFunction::getEvaluationKind(QualType type) {
 #define NON_CANONICAL_TYPE(name, parent) case Type::name:
 #define DEPENDENT_TYPE(name, parent) case Type::name:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(name, parent) case Type::name:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
       llvm_unreachable("non-canonical or dependent type in IR-generation");
 
     case Type::Auto:
@@ -1881,7 +1881,7 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
 #define NON_CANONICAL_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
       llvm_unreachable("unexpected dependent type!");
 
     // These types are never variably-modified.
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index 29369ae06ea66..8d024b5b836a8 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -2572,6 +2572,11 @@ void CodeGenModule::EmitGlobal(GlobalDecl GD) {
     return;
   }
 
+    // Check if this must be emitted as declare variant.
+  if (LangOpts.OpenMP && isa<FunctionDecl>(Global) && OpenMPRuntime &&
+      OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/false))
+    return;
+
   // If we're deferring emission of a C++ variable with an
   // initializer, remember the order in which it appeared in the file.
   if (getLangOpts().CPlusPlus && isa<VarDecl>(Global) &&
@@ -3091,6 +3096,10 @@ llvm::Constant *CodeGenModule::GetOrCreateLLVMFunction(
         EmitGlobal(GDDef);
       }
     }
+    // Check if this must be emitted as declare variant and emit reference to
+    // the the declare variant function.
+    if (LangOpts.OpenMP && OpenMPRuntime)
+      (void)OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/true);
 
     if (FD->isMultiVersion()) {
       const auto *TA = FD->getAttr<TargetAttr>();
@@ -3957,9 +3966,9 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D,
     return;
 
   llvm::Constant *Init = nullptr;
-  CXXRecordDecl *RD = ASTTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
   bool NeedsGlobalCtor = false;
-  bool NeedsGlobalDtor = RD && !RD->hasTrivialDestructor();
+  bool NeedsGlobalDtor =
+      D->needsDestruction(getContext()) == QualType::DK_cxx_destructor;
 
   const VarDecl *InitDecl;
   const Expr *InitExpr = D->getAnyInitializer(InitDecl);
@@ -4490,6 +4499,11 @@ void CodeGenModule::HandleCXXStaticMemberVarInstantiation(VarDecl *VD) {
 
 void CodeGenModule::EmitGlobalFunctionDefinition(GlobalDecl GD,
                                                  llvm::GlobalValue *GV) {
+  // Check if this must be emitted as declare variant.
+  if (LangOpts.OpenMP && OpenMPRuntime &&
+      OpenMPRuntime->emitDeclareVariant(GD, /*IsForDefinition=*/true))
+    return;
+
   const auto *D = cast<FunctionDecl>(GD.getDecl());
 
   // Compute the function info and LLVM type.
@@ -5295,7 +5309,9 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
 // EmitLinkageSpec - Emit all declarations in a linkage spec.
 void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
   if (LSD->getLanguage() != LinkageSpecDecl::lang_c &&
-      LSD->getLanguage() != LinkageSpecDecl::lang_cxx) {
+      LSD->getLanguage() != LinkageSpecDecl::lang_cxx &&
+      LSD->getLanguage() != LinkageSpecDecl::lang_cxx_11 &&
+      LSD->getLanguage() != LinkageSpecDecl::lang_cxx_14) {
     ErrorUnsupported(LSD, "linkage spec");
     return;
   }
diff --git a/clang/lib/CodeGen/CodeGenPGO.h b/clang/lib/CodeGen/CodeGenPGO.h
index 2e740f7892431..a3778b549910b 100644
--- a/clang/lib/CodeGen/CodeGenPGO.h
+++ b/clang/lib/CodeGen/CodeGenPGO.h
@@ -41,8 +41,8 @@ class CodeGenPGO {
 
 public:
   CodeGenPGO(CodeGenModule &CGM)
-      : CGM(CGM), NumValueSites({{0}}), NumRegionCounters(0), FunctionHash(0),
-        CurrentRegionCount(0) {}
+      : CGM(CGM), FuncNameVar(nullptr), NumValueSites({{0}}),
+        NumRegionCounters(0), FunctionHash(0), CurrentRegionCount(0) {}
 
   /// Whether or not we have PGO region data for the current function. This is
   /// false both when we have no data at all and when our data has been
diff --git a/clang/lib/CodeGen/CodeGenTypes.cpp b/clang/lib/CodeGen/CodeGenTypes.cpp
index dd822d1f3ede3..c87d99e6bdcb0 100644
--- a/clang/lib/CodeGen/CodeGenTypes.cpp
+++ b/clang/lib/CodeGen/CodeGenTypes.cpp
@@ -416,7 +416,7 @@ llvm::Type *CodeGenTypes::ConvertType(QualType T) {
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Non-canonical or dependent types aren't possible.");
 
   case Type::Builtin: {
diff --git a/clang/lib/CodeGen/ItaniumCXXABI.cpp b/clang/lib/CodeGen/ItaniumCXXABI.cpp
index 037185390c983..d6e51c60cc7f8 100644
--- a/clang/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/clang/lib/CodeGen/ItaniumCXXABI.cpp
@@ -350,7 +350,7 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
     // If we have the only definition, we don't need a thread wrapper if we
     // will emit the value as a constant.
     if (isUniqueGVALinkage(getContext().GetGVALinkageForVariable(VD)))
-      return !VD->getType().isDestructedType() && InitDecl->evaluateValue();
+      return !VD->needsDestruction(getContext()) && InitDecl->evaluateValue();
 
     // Otherwise, we need a thread wrapper unless we know that every
     // translation unit will emit the value as a constant. We rely on
@@ -360,7 +360,8 @@ class ItaniumCXXABI : public CodeGen::CGCXXABI {
   }
 
   bool usesThreadWrapperFunction(const VarDecl *VD) const override {
-    return !isEmittedWithConstantInitializer(VD);
+    return !isEmittedWithConstantInitializer(VD) ||
+           VD->needsDestruction(getContext());
   }
   LValue EmitThreadLocalVarDeclLValue(CodeGenFunction &CGF, const VarDecl *VD,
                                       QualType LValType) override;
@@ -579,8 +580,8 @@ CGCallee ItaniumCXXABI::EmitLoadOfMemberFunctionPointer(
 
   const FunctionProtoType *FPT =
     MPT->getPointeeType()->getAs<FunctionProtoType>();
-  const CXXRecordDecl *RD =
-    cast<CXXRecordDecl>(MPT->getClass()->getAs<RecordType>()->getDecl());
+  auto *RD =
+      cast<CXXRecordDecl>(MPT->getClass()->castAs<RecordType>()->getDecl());
 
   llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(
       CGM.getTypes().arrangeCXXMethodType(RD, FPT, /*FD=*/nullptr));
@@ -1142,7 +1143,7 @@ void ItaniumCXXABI::emitVirtualObjectDelete(CodeGenFunction &CGF,
 
     // Grab the vtable pointer as an intptr_t*.
     auto *ClassDecl =
-        cast<CXXRecordDecl>(ElementType->getAs<RecordType>()->getDecl());
+        cast<CXXRecordDecl>(ElementType->castAs<RecordType>()->getDecl());
     llvm::Value *VTable =
         CGF.GetVTablePtr(Ptr, CGF.IntPtrTy->getPointerTo(), ClassDecl);
 
@@ -1345,7 +1346,7 @@ llvm::Value *ItaniumCXXABI::EmitTypeid(CodeGenFunction &CGF,
                                        Address ThisPtr,
                                        llvm::Type *StdTypeInfoPtrTy) {
   auto *ClassDecl =
-      cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
   llvm::Value *Value =
       CGF.GetVTablePtr(ThisPtr, StdTypeInfoPtrTy->getPointerTo(), ClassDecl);
 
@@ -1411,7 +1412,7 @@ llvm::Value *ItaniumCXXABI::EmitDynamicCastToVoid(CodeGenFunction &CGF,
   llvm::Type *DestLTy = CGF.ConvertType(DestTy);
 
   auto *ClassDecl =
-      cast<CXXRecordDecl>(SrcRecordTy->getAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(SrcRecordTy->castAs<RecordType>()->getDecl());
   // Get the vtable pointer.
   llvm::Value *VTable = CGF.GetVTablePtr(ThisAddr, PtrDiffLTy->getPointerTo(),
       ClassDecl);
@@ -2606,7 +2607,7 @@ void ItaniumCXXABI::EmitThreadLocalInitFuncs(
     llvm::GlobalValue *Init = nullptr;
     bool InitIsInitFunc = false;
     bool HasConstantInitialization = false;
-    if (isEmittedWithConstantInitializer(VD)) {
+    if (!usesThreadWrapperFunction(VD)) {
       HasConstantInitialization = true;
     } else if (VD->hasDefinition()) {
       InitIsInitFunc = true;
@@ -3099,8 +3100,8 @@ static bool CanUseSingleInheritance(const CXXRecordDecl *RD) {
     return false;
 
   // Check that the class is dynamic iff the base is.
-  const CXXRecordDecl *BaseDecl =
-    cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+  auto *BaseDecl =
+      cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
   if (!BaseDecl->isEmpty() &&
       BaseDecl->isDynamicClass() != RD->isDynamicClass())
     return false;
@@ -3127,7 +3128,7 @@ void ItaniumRTTIBuilder::BuildVTablePointer(const Type *Ty) {
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   case Type::LValueReference:
@@ -3373,7 +3374,7 @@ llvm::Constant *ItaniumRTTIBuilder::BuildTypeInfo(
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     llvm_unreachable("Non-canonical and dependent types shouldn't get here");
 
   // GCC treats vector types as fundamental types.
@@ -3563,8 +3564,8 @@ static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
 
   unsigned Flags = 0;
 
-  const CXXRecordDecl *BaseDecl =
-    cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
+  auto *BaseDecl =
+      cast<CXXRecordDecl>(Base->getType()->castAs<RecordType>()->getDecl());
 
   if (Base->isVirtual()) {
     // Mark the virtual base as seen.
@@ -3662,8 +3663,8 @@ void ItaniumRTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
     // The __base_type member points to the RTTI for the base type.
     Fields.push_back(ItaniumRTTIBuilder(CXXABI).BuildTypeInfo(Base.getType()));
 
-    const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(Base.getType()->getAs<RecordType>()->getDecl());
+    auto *BaseDecl =
+        cast<CXXRecordDecl>(Base.getType()->castAs<RecordType>()->getDecl());
 
     int64_t OffsetFlags = 0;
 
diff --git a/clang/lib/CodeGen/ModuleBuilder.cpp b/clang/lib/CodeGen/ModuleBuilder.cpp
index 40662b5317632..26a1f41ea7515 100644
--- a/clang/lib/CodeGen/ModuleBuilder.cpp
+++ b/clang/lib/CodeGen/ModuleBuilder.cpp
@@ -65,6 +65,13 @@ namespace {
   private:
     SmallVector<FunctionDecl *, 8> DeferredInlineMemberFuncDefs;
 
+    static llvm::StringRef ExpandModuleName(llvm::StringRef ModuleName,
+                                            const CodeGenOptions &CGO) {
+      if (ModuleName == "-" && !CGO.MainFileName.empty())
+        return CGO.MainFileName;
+      return ModuleName;
+    }
+
   public:
     CodeGeneratorImpl(DiagnosticsEngine &diags, llvm::StringRef ModuleName,
                       const HeaderSearchOptions &HSO,
@@ -73,7 +80,8 @@ namespace {
                       CoverageSourceInfo *CoverageInfo = nullptr)
         : Diags(diags), Ctx(nullptr), HeaderSearchOpts(HSO),
           PreprocessorOpts(PPO), CodeGenOpts(CGO), HandlingTopLevelDecls(0),
-          CoverageInfo(CoverageInfo), M(new llvm::Module(ModuleName, C)) {
+          CoverageInfo(CoverageInfo),
+          M(new llvm::Module(ExpandModuleName(ModuleName, CGO), C)) {
       C.setDiscardValueNames(CGO.DiscardValueNames);
     }
 
@@ -121,7 +129,7 @@ namespace {
     llvm::Module *StartModule(llvm::StringRef ModuleName,
                               llvm::LLVMContext &C) {
       assert(!M && "Replacing existing Module?");
-      M.reset(new llvm::Module(ModuleName, C));
+      M.reset(new llvm::Module(ExpandModuleName(ModuleName, CodeGenOpts), C));
       Initialize(*Ctx);
       return M.get();
     }
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index f2696a33cfbf4..c4d0004f0e288 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -9954,7 +9954,7 @@ llvm::Function *AMDGPUTargetCodeGenInfo::createEnqueuedBlockKernel(
   Builder.SetInsertPoint(BB);
   unsigned BlockAlign = CGF.CGM.getDataLayout().getPrefTypeAlignment(BlockTy);
   auto *BlockPtr = Builder.CreateAlloca(BlockTy, nullptr);
-  BlockPtr->setAlignment(BlockAlign);
+  BlockPtr->setAlignment(llvm::MaybeAlign(BlockAlign));
   Builder.CreateAlignedStore(F->arg_begin(), BlockPtr, BlockAlign);
   auto *Cast = Builder.CreatePointerCast(BlockPtr, InvokeFT->getParamType(0));
   llvm::SmallVector<llvm::Value *, 2> Args;
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index a889b7e018d59..d0f12f97353e5 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -288,8 +288,7 @@ phases::ID Driver::getFinalPhase(const DerivedArgList &DAL,
              (PhaseArg = DAL.getLastArg(options::OPT_rewrite_legacy_objc)) ||
              (PhaseArg = DAL.getLastArg(options::OPT__migrate)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_emit_iterface_stubs)) ||
-             (PhaseArg = DAL.getLastArg(options::OPT__analyze,
-                                        options::OPT__analyze_auto)) ||
+             (PhaseArg = DAL.getLastArg(options::OPT__analyze)) ||
              (PhaseArg = DAL.getLastArg(options::OPT_emit_ast))) {
     FinalPhase = phases::Compile;
 
@@ -2489,15 +2488,13 @@ class OffloadingActionBuilder final {
       return ABRT_Inactive;
     }
 
-    /// Append top level actions generated by the builder. Return true if errors
-    /// were found.
+    /// Append top level actions generated by the builder.
     virtual void appendTopLevelActions(ActionList &AL) {}
 
     /// Append top level actions specific for certain link situations.
     virtual void appendTopLevelLinkAction(ActionList &AL) {}
 
-    /// Append linker actions generated by the builder. Return true if errors
-    /// were found.
+    /// Append linker actions generated by the builder.
     virtual void appendLinkDependences(OffloadAction::DeviceDependences &DA) {}
 
     /// Initialize the builder. Return true if any initialization errors are
@@ -4301,8 +4298,10 @@ Action *Driver::ConstructPhaseAction(
     llvm_unreachable("link action invalid here.");
   case phases::Preprocess: {
     types::ID OutputTy;
-    // -{M, MM} alter the output type.
-    if (Args.hasArg(options::OPT_M, options::OPT_MM)) {
+    // -M and -MM specify the dependency file name by altering the output type,
+    // -if -MD and -MMD are not specified.
+    if (Args.hasArg(options::OPT_M, options::OPT_MM) &&
+        !Args.hasArg(options::OPT_MD, options::OPT_MMD)) {
       OutputTy = types::TY_Dependencies;
     } else {
       OutputTy = Input->getType();
@@ -4350,7 +4349,7 @@ Action *Driver::ConstructPhaseAction(
     if (Args.hasArg(options::OPT_rewrite_legacy_objc))
       return C.MakeAction<CompileJobAction>(Input,
                                             types::TY_RewrittenLegacyObjC);
-    if (Args.hasArg(options::OPT__analyze, options::OPT__analyze_auto))
+    if (Args.hasArg(options::OPT__analyze))
       return C.MakeAction<AnalyzeJobAction>(Input, types::TY_Plist);
     if (Args.hasArg(options::OPT__migrate))
       return C.MakeAction<MigrateJobAction>(Input, types::TY_Remap);
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index 65e1100ee9cd9..947a4c6814eaa 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -460,7 +460,7 @@ void arm::getARMTargetFeatures(const ToolChain &TC,
     //        now just be explicit and disable all known dependent features
     //        as well.
     for (std::string Feature : {
-            "vfp2", "vfp2sp", "vfp2d16", "vfp2d16sp",
+            "vfp2", "vfp2sp",
             "vfp3", "vfp3sp", "vfp3d16", "vfp3d16sp",
             "vfp4", "vfp4sp", "vfp4d16", "vfp4d16sp",
             "fp-armv8", "fp-armv8sp", "fp-armv8d16", "fp-armv8d16sp",
diff --git a/clang/lib/Driver/ToolChains/Arch/Mips.cpp b/clang/lib/Driver/ToolChains/Arch/Mips.cpp
index b512ff64b0c62..384d3acedc2d1 100644
--- a/clang/lib/Driver/ToolChains/Arch/Mips.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/Mips.cpp
@@ -267,6 +267,13 @@ void mips::getMIPSTargetFeatures(const Driver &D, const llvm::Triple &Triple,
       D.Diag(diag::warn_drv_unsupported_longcalls) << (ABICallsArg ? 0 : 1);
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_mxgot, options::OPT_mno_xgot)) {
+    if (A->getOption().matches(options::OPT_mxgot))
+      Features.push_back("+xgot");
+    else
+      Features.push_back("-xgot");
+  }
+
   mips::FloatABI FloatABI = mips::getMipsFloatABI(D, Args);
   if (FloatABI == mips::FloatABI::Soft) {
     // FIXME: Note, this is a hack. We need to pass the selected float
diff --git a/clang/lib/Driver/ToolChains/Arch/PPC.cpp b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
index 30f1a0d9022c2..59ff7cbc787c7 100644
--- a/clang/lib/Driver/ToolChains/Arch/PPC.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/PPC.cpp
@@ -115,7 +115,8 @@ ppc::ReadGOTPtrMode ppc::getPPCReadGOTPtrMode(const Driver &D, const llvm::Tripl
                                               const ArgList &Args) {
   if (Args.getLastArg(options::OPT_msecure_plt))
     return ppc::ReadGOTPtrMode::SecurePlt;
-  if (Triple.isOSNetBSD() || Triple.isOSOpenBSD() || Triple.isMusl())
+  if ((Triple.isOSFreeBSD() && Triple.getOSMajorVersion() >= 13) ||
+      Triple.isOSNetBSD() || Triple.isOSOpenBSD() || Triple.isMusl())
     return ppc::ReadGOTPtrMode::SecurePlt;
   else
     return ppc::ReadGOTPtrMode::Bss;
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 1d40f4e01ba4a..593c32a25f8fa 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -839,12 +839,14 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     }
   }
 
-  if (Args.hasArg(options::OPT_ftest_coverage) ||
-      Args.hasArg(options::OPT_coverage))
+  bool EmitCovNotes = Args.hasArg(options::OPT_ftest_coverage) ||
+                      Args.hasArg(options::OPT_coverage);
+  bool EmitCovData = Args.hasFlag(options::OPT_fprofile_arcs,
+                                  options::OPT_fno_profile_arcs, false) ||
+                     Args.hasArg(options::OPT_coverage);
+  if (EmitCovNotes)
     CmdArgs.push_back("-femit-coverage-notes");
-  if (Args.hasFlag(options::OPT_fprofile_arcs, options::OPT_fno_profile_arcs,
-                   false) ||
-      Args.hasArg(options::OPT_coverage))
+  if (EmitCovData)
     CmdArgs.push_back("-femit-coverage-data");
 
   if (Args.hasFlag(options::OPT_fcoverage_mapping,
@@ -880,40 +882,48 @@ static void addPGOAndCoverageFlags(const ToolChain &TC, Compilation &C,
     CmdArgs.push_back(Args.MakeArgString(Twine("-fprofile-filter-files=" + v)));
   }
 
-  if (C.getArgs().hasArg(options::OPT_c) ||
-      C.getArgs().hasArg(options::OPT_S)) {
-    if (Output.isFilename()) {
-      CmdArgs.push_back("-coverage-notes-file");
-      SmallString<128> OutputFilename;
-      if (Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o))
-        OutputFilename = FinalOutput->getValue();
-      else
-        OutputFilename = llvm::sys::path::filename(Output.getBaseInput());
-      SmallString<128> CoverageFilename = OutputFilename;
-      if (llvm::sys::path::is_relative(CoverageFilename)) {
-        SmallString<128> Pwd;
-        if (!llvm::sys::fs::current_path(Pwd)) {
-          llvm::sys::path::append(Pwd, CoverageFilename);
-          CoverageFilename.swap(Pwd);
-        }
+  // Leave -fprofile-dir= an unused argument unless .gcda emission is
+  // enabled. To be polite, with '-fprofile-arcs -fno-profile-arcs' consider
+  // the flag used. There is no -fno-profile-dir, so the user has no
+  // targeted way to suppress the warning.
+  Arg *FProfileDir = nullptr;
+  if (Args.hasArg(options::OPT_fprofile_arcs) ||
+      Args.hasArg(options::OPT_coverage))
+    FProfileDir = Args.getLastArg(options::OPT_fprofile_dir);
+
+  // Put the .gcno and .gcda files (if needed) next to the object file or
+  // bitcode file in the case of LTO.
+  // FIXME: There should be a simpler way to find the object file for this
+  // input, and this code probably does the wrong thing for commands that
+  // compile and link all at once.
+  if ((Args.hasArg(options::OPT_c) || Args.hasArg(options::OPT_S)) &&
+      (EmitCovNotes || EmitCovData) && Output.isFilename()) {
+    SmallString<128> OutputFilename;
+    if (Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o))
+      OutputFilename = FinalOutput->getValue();
+    else
+      OutputFilename = llvm::sys::path::filename(Output.getBaseInput());
+    SmallString<128> CoverageFilename = OutputFilename;
+    if (llvm::sys::path::is_relative(CoverageFilename)) {
+      SmallString<128> Pwd;
+      if (!llvm::sys::fs::current_path(Pwd)) {
+        llvm::sys::path::append(Pwd, CoverageFilename);
+        CoverageFilename.swap(Pwd);
       }
-      llvm::sys::path::replace_extension(CoverageFilename, "gcno");
-      CmdArgs.push_back(Args.MakeArgString(CoverageFilename));
+    }
+    llvm::sys::path::replace_extension(CoverageFilename, "gcno");
 
-      // Leave -fprofile-dir= an unused argument unless .gcda emission is
-      // enabled. To be polite, with '-fprofile-arcs -fno-profile-arcs' consider
-      // the flag used. There is no -fno-profile-dir, so the user has no
-      // targeted way to suppress the warning.
-      if (Args.hasArg(options::OPT_fprofile_arcs) ||
-          Args.hasArg(options::OPT_coverage)) {
-        CmdArgs.push_back("-coverage-data-file");
-        if (Arg *FProfileDir = Args.getLastArg(options::OPT_fprofile_dir)) {
-          CoverageFilename = FProfileDir->getValue();
-          llvm::sys::path::append(CoverageFilename, OutputFilename);
-        }
-        llvm::sys::path::replace_extension(CoverageFilename, "gcda");
-        CmdArgs.push_back(Args.MakeArgString(CoverageFilename));
+    CmdArgs.push_back("-coverage-notes-file");
+    CmdArgs.push_back(Args.MakeArgString(CoverageFilename));
+
+    if (EmitCovData) {
+      if (FProfileDir) {
+        CoverageFilename = FProfileDir->getValue();
+        llvm::sys::path::append(CoverageFilename, OutputFilename);
       }
+      llvm::sys::path::replace_extension(CoverageFilename, "gcda");
+      CmdArgs.push_back("-coverage-data-file");
+      CmdArgs.push_back(Args.MakeArgString(CoverageFilename));
     }
   }
 }
@@ -1068,7 +1078,6 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
                                     ArgStringList &CmdArgs,
                                     const InputInfo &Output,
                                     const InputInfoList &Inputs) const {
-  Arg *A;
   const bool IsIAMCU = getToolChain().getTriple().isOSIAMCU();
 
   CheckPreprocessingOptions(D, Args);
@@ -1077,9 +1086,20 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_CC);
 
   // Handle dependency file generation.
-  if ((A = Args.getLastArg(options::OPT_M, options::OPT_MM)) ||
-      (A = Args.getLastArg(options::OPT_MD)) ||
-      (A = Args.getLastArg(options::OPT_MMD))) {
+  Arg *ArgM = Args.getLastArg(options::OPT_MM);
+  if (!ArgM)
+    ArgM = Args.getLastArg(options::OPT_M);
+  Arg *ArgMD = Args.getLastArg(options::OPT_MMD);
+  if (!ArgMD)
+    ArgMD = Args.getLastArg(options::OPT_MD);
+
+  // -M and -MM imply -w.
+  if (ArgM)
+    CmdArgs.push_back("-w");
+  else
+    ArgM = ArgMD;
+
+  if (ArgM) {
     // Determine the output location.
     const char *DepFile;
     if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
@@ -1087,11 +1107,10 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
       C.addFailureResultFile(DepFile, &JA);
     } else if (Output.getType() == types::TY_Dependencies) {
       DepFile = Output.getFilename();
-    } else if (A->getOption().matches(options::OPT_M) ||
-               A->getOption().matches(options::OPT_MM)) {
+    } else if (!ArgMD) {
       DepFile = "-";
-    } else if (A->getOption().matches(options::OPT_MMD) &&
-        Args.hasArg(options::OPT_fintelfpga)) {
+    } else if (ArgMD->getOption().matches(options::OPT_MMD) &&
+               Args.hasArg(options::OPT_fintelfpga)) {
       // When generating dependency files for FPGA AOT, the output files will
       // always be named after the source file.
       DepFile = Args.MakeArgString(Twine(getBaseInputStem(Args, Inputs)) + ".d");
@@ -1102,8 +1121,22 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-dependency-file");
     CmdArgs.push_back(DepFile);
 
+    bool HasTarget = false;
+    for (const Arg *A : Args.filtered(options::OPT_MT, options::OPT_MQ)) {
+      HasTarget = true;
+      A->claim();
+      if (A->getOption().matches(options::OPT_MT)) {
+        A->render(Args, CmdArgs);
+      } else {
+        CmdArgs.push_back("-MT");
+        SmallString<128> Quoted;
+        QuoteTarget(A->getValue(), Quoted);
+        CmdArgs.push_back(Args.MakeArgString(Quoted));
+      }
+    }
+
     // Add a default target if one wasn't specified.
-    if (!Args.hasArg(options::OPT_MT) && !Args.hasArg(options::OPT_MQ)) {
+    if (!HasTarget) {
       const char *DepTarget;
 
       // If user provided -o, that is the dependency target, except
@@ -1120,17 +1153,14 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
         DepTarget = Args.MakeArgString(llvm::sys::path::filename(P));
       }
 
-      if (!A->getOption().matches(options::OPT_MD) && !A->getOption().matches(options::OPT_MMD)) {
-        CmdArgs.push_back("-w");
-      }
       CmdArgs.push_back("-MT");
       SmallString<128> Quoted;
       QuoteTarget(DepTarget, Quoted);
       CmdArgs.push_back(Args.MakeArgString(Quoted));
     }
 
-    if (A->getOption().matches(options::OPT_M) ||
-        A->getOption().matches(options::OPT_MD))
+    if (ArgM->getOption().matches(options::OPT_M) ||
+        ArgM->getOption().matches(options::OPT_MD))
       CmdArgs.push_back("-sys-header-deps");
     if ((isa<PrecompileJobAction>(JA) &&
          !Args.hasArg(options::OPT_fno_module_file_deps)) ||
@@ -1139,8 +1169,8 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
   }
 
   if (Args.hasArg(options::OPT_MG)) {
-    if (!A || A->getOption().matches(options::OPT_MD) ||
-        A->getOption().matches(options::OPT_MMD))
+    if (!ArgM || ArgM->getOption().matches(options::OPT_MD) ||
+        ArgM->getOption().matches(options::OPT_MMD))
       D.Diag(diag::err_drv_mg_requires_m_or_mm);
     CmdArgs.push_back("-MG");
   }
@@ -1148,22 +1178,6 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_MP);
   Args.AddLastArg(CmdArgs, options::OPT_MV);
 
-  // Convert all -MQ <target> args to -MT <quoted target>
-  for (const Arg *A : Args.filtered(options::OPT_MT, options::OPT_MQ)) {
-    A->claim();
-
-    if (A->getOption().matches(options::OPT_MQ)) {
-      CmdArgs.push_back("-MT");
-      SmallString<128> Quoted;
-      QuoteTarget(A->getValue(), Quoted);
-      CmdArgs.push_back(Args.MakeArgString(Quoted));
-
-      // -MT flag - no change
-    } else {
-      A->render(Args, CmdArgs);
-    }
-  }
-
   // Add offload include arguments specific for CUDA.  This must happen before
   // we -I or -include anything else, because we must pick up the CUDA headers
   // from the particular CUDA installation, rather than from e.g.
@@ -1683,13 +1697,6 @@ void Clang::AddMIPSTargetArgs(const ArgList &Args,
     CmdArgs.push_back("hard");
   }
 
-  if (Arg *A = Args.getLastArg(options::OPT_mxgot, options::OPT_mno_xgot)) {
-    if (A->getOption().matches(options::OPT_mxgot)) {
-      CmdArgs.push_back("-mllvm");
-      CmdArgs.push_back("-mxgot");
-    }
-  }
-
   if (Arg *A = Args.getLastArg(options::OPT_mldc1_sdc1,
                                options::OPT_mno_ldc1_sdc1)) {
     if (A->getOption().matches(options::OPT_mno_ldc1_sdc1)) {
@@ -2032,7 +2039,7 @@ void Clang::DumpCompilationDatabase(Compilation &C, StringRef Filename,
   }
   auto &CDB = *CompilationDatabase;
   SmallString<128> Buf;
-  if (!llvm::sys::fs::current_path(Buf))
+  if (llvm::sys::fs::current_path(Buf))
     Buf = ".";
   CDB << "{ \"directory\": \"" << escape(Buf) << "\"";
   CDB << ", \"file\": \"" << escape(Input.getFilename()) << "\"";
@@ -4863,6 +4870,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Forward -sycl-std option to -cc1
   Args.AddLastArg(CmdArgs, options::OPT_sycl_std_EQ);
 
+  if (Args.hasFlag(options::OPT_fhip_new_launch_api,
+                   options::OPT_fno_hip_new_launch_api, false))
+    CmdArgs.push_back("-fhip-new-launch-api");
+
   if (Arg *A = Args.getLastArg(options::OPT_fcf_protection_EQ)) {
     CmdArgs.push_back(
         Args.MakeArgString(Twine("-fcf-protection=") + A->getValue()));
@@ -5581,14 +5592,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-fwhole-program-vtables");
   }
 
-  bool RequiresSplitLTOUnit = WholeProgramVTables || Sanitize.needsLTO();
+  bool DefaultsSplitLTOUnit = WholeProgramVTables || Sanitize.needsLTO();
   bool SplitLTOUnit =
       Args.hasFlag(options::OPT_fsplit_lto_unit,
-                   options::OPT_fno_split_lto_unit, RequiresSplitLTOUnit);
-  if (RequiresSplitLTOUnit && !SplitLTOUnit)
-    D.Diag(diag::err_drv_argument_not_allowed_with)
-        << "-fno-split-lto-unit"
-        << (WholeProgramVTables ? "-fwhole-program-vtables" : "-fsanitize=cfi");
+                   options::OPT_fno_split_lto_unit, DefaultsSplitLTOUnit);
+  if (Sanitize.needsLTO() && !SplitLTOUnit)
+    D.Diag(diag::err_drv_argument_not_allowed_with) << "-fno-split-lto-unit"
+                                                    << "-fsanitize=cfi";
   if (SplitLTOUnit)
     CmdArgs.push_back("-fsplit-lto-unit");
 
@@ -6209,7 +6219,7 @@ const char *Clang::getDependencyFileName(const ArgList &Args,
     return Args.MakeArgString(OutputFilename);
   }
 
-  return Args.MakeArgString(std::string(getBaseInputStem(Args, Inputs)) + ".d");
+  return Args.MakeArgString(Twine(getBaseInputStem(Args, Inputs)) + ".d");
 }
 
 // Begin ClangAs
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 30b72391f313e..5dae8effa243d 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1342,17 +1342,6 @@ void tools::AddOpenMPLinkerScript(const ToolChain &TC, Compilation &C,
     LksStream << "  }\n";
   }
 
-  // Add commands to define host entries begin and end. We use 1-byte subalign
-  // so that the linker does not add any padding and the elements in this
-  // section form an array.
-  LksStream << "  .omp_offloading.entries :\n";
-  LksStream << "  ALIGN(0x10)\n";
-  LksStream << "  SUBALIGN(0x01)\n";
-  LksStream << "  {\n";
-  LksStream << "    PROVIDE_HIDDEN(.omp_offloading.entries_begin = .);\n";
-  LksStream << "    *(.omp_offloading.entries)\n";
-  LksStream << "    PROVIDE_HIDDEN(.omp_offloading.entries_end = .);\n";
-  LksStream << "  }\n";
   LksStream << "}\n";
   LksStream << "INSERT BEFORE .data\n";
   LksStream.flush();
diff --git a/clang/lib/Driver/ToolChains/Fuchsia.cpp b/clang/lib/Driver/ToolChains/Fuchsia.cpp
index cb84499320f36..e7d38ff9f227b 100644
--- a/clang/lib/Driver/ToolChains/Fuchsia.cpp
+++ b/clang/lib/Driver/ToolChains/Fuchsia.cpp
@@ -52,7 +52,7 @@ void fuchsia::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-z");
     CmdArgs.push_back("rodynamic");
     CmdArgs.push_back("-z");
-    CmdArgs.push_back("separate-code");
+    CmdArgs.push_back("separate-loadable-segments");
   }
 
   if (!D.SysRoot.empty())
diff --git a/clang/lib/Driver/Types.cpp b/clang/lib/Driver/Types.cpp
index e304fa1846300..7c6f40aef932b 100644
--- a/clang/lib/Driver/Types.cpp
+++ b/clang/lib/Driver/Types.cpp
@@ -329,7 +329,7 @@ void types::getCompilationPhases(const clang::driver::Driver &Driver,
            DAL.getLastArg(options::OPT_rewrite_legacy_objc) ||
            DAL.getLastArg(options::OPT__migrate) ||
            DAL.getLastArg(options::OPT_emit_iterface_stubs) ||
-           DAL.getLastArg(options::OPT__analyze, options::OPT__analyze_auto) ||
+           DAL.getLastArg(options::OPT__analyze) ||
            DAL.getLastArg(options::OPT_emit_ast))
     llvm::copy_if(PhaseList, std::back_inserter(P),
                   [](phases::ID Phase) { return Phase <= phases::Compile; });
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index ede6a797122b6..52afb6744ad3b 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -931,6 +931,11 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) {
     return std::max(State.Stack.back().LastSpace,
                     State.Stack.back().Indent + Style.ContinuationIndentWidth);
 
+  if (Style.BreakBeforeBraces == FormatStyle::BS_Whitesmiths &&
+      State.Line->First->is(tok::kw_enum))
+    return (Style.IndentWidth * State.Line->First->IndentLevel) +
+           Style.IndentWidth;
+
   if (NextNonComment->is(tok::l_brace) && NextNonComment->BlockKind == BK_Block)
     return Current.NestingLevel == 0 ? State.FirstIndent
                                      : State.Stack.back().Indent;
diff --git a/clang/lib/Format/Encoding.h b/clang/lib/Format/Encoding.h
index fe3d5f019858b..a0d664121b2bb 100644
--- a/clang/lib/Format/Encoding.h
+++ b/clang/lib/Format/Encoding.h
@@ -67,7 +67,8 @@ inline unsigned columnWidthWithTabs(StringRef Text, unsigned StartColumn,
     if (TabPos == StringRef::npos)
       return TotalWidth + columnWidth(Tail, Encoding);
     TotalWidth += columnWidth(Tail.substr(0, TabPos), Encoding);
-    TotalWidth += TabWidth - (TotalWidth + StartColumn) % TabWidth;
+    if (TabWidth)
+      TotalWidth += TabWidth - (TotalWidth + StartColumn) % TabWidth;
     Tail = Tail.substr(TabPos + 1);
   }
 }
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 81fdab4ed70aa..dd561cc0d8925 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -67,10 +67,19 @@ template <> struct ScalarEnumerationTraits<FormatStyle::LanguageKind> {
 
 template <> struct ScalarEnumerationTraits<FormatStyle::LanguageStandard> {
   static void enumeration(IO &IO, FormatStyle::LanguageStandard &Value) {
-    IO.enumCase(Value, "Cpp03", FormatStyle::LS_Cpp03);
-    IO.enumCase(Value, "C++03", FormatStyle::LS_Cpp03);
-    IO.enumCase(Value, "Cpp11", FormatStyle::LS_Cpp11);
-    IO.enumCase(Value, "C++11", FormatStyle::LS_Cpp11);
+    IO.enumCase(Value, "c++03", FormatStyle::LS_Cpp03);
+    IO.enumCase(Value, "C++03", FormatStyle::LS_Cpp03); // Legacy alias
+    IO.enumCase(Value, "Cpp03", FormatStyle::LS_Cpp03); // Legacy alias
+
+    IO.enumCase(Value, "c++11", FormatStyle::LS_Cpp11);
+    IO.enumCase(Value, "C++11", FormatStyle::LS_Cpp11); // Legacy alias
+
+    IO.enumCase(Value, "c++14", FormatStyle::LS_Cpp14);
+    IO.enumCase(Value, "c++17", FormatStyle::LS_Cpp17);
+    IO.enumCase(Value, "c++20", FormatStyle::LS_Cpp20);
+
+    IO.enumCase(Value, "Latest", FormatStyle::LS_Latest);
+    IO.enumCase(Value, "Cpp11", FormatStyle::LS_Latest); // Legacy alias
     IO.enumCase(Value, "Auto", FormatStyle::LS_Auto);
   }
 };
@@ -165,6 +174,7 @@ template <> struct ScalarEnumerationTraits<FormatStyle::BraceBreakingStyle> {
     IO.enumCase(Value, "Mozilla", FormatStyle::BS_Mozilla);
     IO.enumCase(Value, "Stroustrup", FormatStyle::BS_Stroustrup);
     IO.enumCase(Value, "Allman", FormatStyle::BS_Allman);
+    IO.enumCase(Value, "Whitesmiths", FormatStyle::BS_Whitesmiths);
     IO.enumCase(Value, "GNU", FormatStyle::BS_GNU);
     IO.enumCase(Value, "WebKit", FormatStyle::BS_WebKit);
     IO.enumCase(Value, "Custom", FormatStyle::BS_Custom);
@@ -657,6 +667,19 @@ static FormatStyle expandPresets(const FormatStyle &Style) {
     Expanded.BraceWrapping.BeforeCatch = true;
     Expanded.BraceWrapping.BeforeElse = true;
     break;
+  case FormatStyle::BS_Whitesmiths:
+    Expanded.BraceWrapping.AfterCaseLabel = true;
+    Expanded.BraceWrapping.AfterClass = true;
+    Expanded.BraceWrapping.AfterControlStatement = true;
+    Expanded.BraceWrapping.AfterEnum = true;
+    Expanded.BraceWrapping.AfterFunction = true;
+    Expanded.BraceWrapping.AfterNamespace = true;
+    Expanded.BraceWrapping.AfterObjCDeclaration = true;
+    Expanded.BraceWrapping.AfterStruct = true;
+    Expanded.BraceWrapping.AfterExternBlock = true;
+    Expanded.BraceWrapping.BeforeCatch = true;
+    Expanded.BraceWrapping.BeforeElse = true;
+    break;
   case FormatStyle::BS_GNU:
     Expanded.BraceWrapping = {true, true, true, true, true, true, true, true,
                               true, true, true, true, true, true, true, true};
@@ -720,9 +743,9 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.ForEachMacros.push_back("Q_FOREACH");
   LLVMStyle.ForEachMacros.push_back("BOOST_FOREACH");
   LLVMStyle.IncludeStyle.IncludeCategories = {
-      {"^\"(llvm|llvm-c|clang|clang-c)/", 2},
-      {"^(<|\"(gtest|gmock|isl|json)/)", 3},
-      {".*", 1}};
+      {"^\"(llvm|llvm-c|clang|clang-c)/", 2, 0},
+      {"^(<|\"(gtest|gmock|isl|json)/)", 3, 0},
+      {".*", 1, 0}};
   LLVMStyle.IncludeStyle.IncludeIsMainRegex = "(Test)?$";
   LLVMStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Preserve;
   LLVMStyle.IndentCaseLabels = false;
@@ -742,7 +765,7 @@ FormatStyle getLLVMStyle(FormatStyle::LanguageKind Language) {
   LLVMStyle.ObjCSpaceBeforeProtocolList = true;
   LLVMStyle.PointerAlignment = FormatStyle::PAS_Right;
   LLVMStyle.SpacesBeforeTrailingComments = 1;
-  LLVMStyle.Standard = FormatStyle::LS_Cpp11;
+  LLVMStyle.Standard = FormatStyle::LS_Latest;
   LLVMStyle.UseTab = FormatStyle::UT_Never;
   LLVMStyle.ReflowComments = true;
   LLVMStyle.SpacesInParentheses = false;
@@ -804,8 +827,10 @@ FormatStyle getGoogleStyle(FormatStyle::LanguageKind Language) {
   GoogleStyle.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
   GoogleStyle.ConstructorInitializerAllOnOneLineOrOnePerLine = true;
   GoogleStyle.DerivePointerAlignment = true;
-  GoogleStyle.IncludeStyle.IncludeCategories = {
-      {"^<ext/.*\\.h>", 2}, {"^<.*\\.h>", 1}, {"^<.*", 2}, {".*", 3}};
+  GoogleStyle.IncludeStyle.IncludeCategories = {{"^<ext/.*\\.h>", 2, 0},
+                                                {"^<.*\\.h>", 1, 0},
+                                                {"^<.*", 2, 0},
+                                                {".*", 3, 0}};
   GoogleStyle.IncludeStyle.IncludeIsMainRegex = "([-_](test|unittest))?$";
   GoogleStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
   GoogleStyle.IndentCaseLabels = true;
@@ -1383,7 +1408,7 @@ class Formatter : public TokenAnalyzer {
                                    : FormatStyle::PAS_Right;
     if (Style.Standard == FormatStyle::LS_Auto)
       Style.Standard = hasCpp03IncompatibleFormat(AnnotatedLines)
-                           ? FormatStyle::LS_Cpp11
+                           ? FormatStyle::LS_Latest
                            : FormatStyle::LS_Cpp03;
     BinPackInconclusiveFunctions =
         HasBinPackedFunction || !HasOnePerLineFunction;
@@ -1417,22 +1442,29 @@ class Cleaner : public TokenAnalyzer {
 
     checkEmptyNamespace(AnnotatedLines);
 
-    for (auto &Line : AnnotatedLines) {
-      if (Line->Affected) {
-        cleanupRight(Line->First, tok::comma, tok::comma);
-        cleanupRight(Line->First, TT_CtorInitializerColon, tok::comma);
-        cleanupRight(Line->First, tok::l_paren, tok::comma);
-        cleanupLeft(Line->First, tok::comma, tok::r_paren);
-        cleanupLeft(Line->First, TT_CtorInitializerComma, tok::l_brace);
-        cleanupLeft(Line->First, TT_CtorInitializerColon, tok::l_brace);
-        cleanupLeft(Line->First, TT_CtorInitializerColon, tok::equal);
-      }
-    }
+    for (auto *Line : AnnotatedLines)
+      cleanupLine(Line);
 
     return {generateFixes(), 0};
   }
 
 private:
+  void cleanupLine(AnnotatedLine *Line) {
+    for (auto *Child : Line->Children) {
+      cleanupLine(Child);
+    }
+
+    if (Line->Affected) {
+      cleanupRight(Line->First, tok::comma, tok::comma);
+      cleanupRight(Line->First, TT_CtorInitializerColon, tok::comma);
+      cleanupRight(Line->First, tok::l_paren, tok::comma);
+      cleanupLeft(Line->First, tok::comma, tok::r_paren);
+      cleanupLeft(Line->First, TT_CtorInitializerComma, tok::l_brace);
+      cleanupLeft(Line->First, TT_CtorInitializerColon, tok::l_brace);
+      cleanupLeft(Line->First, TT_CtorInitializerColon, tok::equal);
+    }
+  }
+
   bool containsOnlyComments(const AnnotatedLine &Line) {
     for (FormatToken *Tok = Line.First; Tok != nullptr; Tok = Tok->Next) {
       if (Tok->isNot(tok::comment))
@@ -1750,6 +1782,7 @@ struct IncludeDirective {
   StringRef Text;
   unsigned Offset;
   int Category;
+  int Priority;
 };
 
 struct JavaImportDirective {
@@ -1801,6 +1834,28 @@ FindCursorIndex(const SmallVectorImpl<IncludeDirective> &Includes,
   return std::make_pair(CursorIndex, OffsetToEOL);
 }
 
+// Replace all "\r\n" with "\n".
+std::string replaceCRLF(const std::string &Code) {
+  std::string NewCode;
+  size_t Pos = 0, LastPos = 0;
+
+  do {
+    Pos = Code.find("\r\n", LastPos);
+    if (Pos == LastPos) {
+      LastPos++;
+      continue;
+    }
+    if (Pos == std::string::npos) {
+      NewCode += Code.substr(LastPos);
+      break;
+    }
+    NewCode += Code.substr(LastPos, Pos - LastPos) + "\n";
+    LastPos = Pos + 2;
+  } while (Pos != std::string::npos);
+
+  return NewCode;
+}
+
 // Sorts and deduplicate a block of includes given by 'Includes' alphabetically
 // adding the necessary replacement to 'Replaces'. 'Includes' must be in strict
 // source order.
@@ -1813,6 +1868,7 @@ static void sortCppIncludes(const FormatStyle &Style,
                             ArrayRef<tooling::Range> Ranges, StringRef FileName,
                             StringRef Code, tooling::Replacements &Replaces,
                             unsigned *Cursor) {
+  tooling::IncludeCategoryManager Categories(Style.IncludeStyle, FileName);
   unsigned IncludesBeginOffset = Includes.front().Offset;
   unsigned IncludesEndOffset =
       Includes.back().Offset + Includes.back().Text.size();
@@ -1820,11 +1876,12 @@ static void sortCppIncludes(const FormatStyle &Style,
   if (!affectsRange(Ranges, IncludesBeginOffset, IncludesEndOffset))
     return;
   SmallVector<unsigned, 16> Indices;
-  for (unsigned i = 0, e = Includes.size(); i != e; ++i)
+  for (unsigned i = 0, e = Includes.size(); i != e; ++i) {
     Indices.push_back(i);
+  }
   llvm::stable_sort(Indices, [&](unsigned LHSI, unsigned RHSI) {
-    return std::tie(Includes[LHSI].Category, Includes[LHSI].Filename) <
-           std::tie(Includes[RHSI].Category, Includes[RHSI].Filename);
+    return std::tie(Includes[LHSI].Priority, Includes[LHSI].Filename) <
+           std::tie(Includes[RHSI].Priority, Includes[RHSI].Filename);
   });
   // The index of the include on which the cursor will be put after
   // sorting/deduplicating.
@@ -1872,7 +1929,8 @@ static void sortCppIncludes(const FormatStyle &Style,
 
   // If the #includes are out of order, we generate a single replacement fixing
   // the entire range of blocks. Otherwise, no replacement is generated.
-  if (result == Code.substr(IncludesBeginOffset, IncludesBlockSize))
+  if (replaceCRLF(result) ==
+      replaceCRLF(Code.substr(IncludesBeginOffset, IncludesBlockSize)))
     return;
 
   auto Err = Replaces.add(tooling::Replacement(
@@ -1939,9 +1997,12 @@ tooling::Replacements sortCppIncludes(const FormatStyle &Style, StringRef Code,
         int Category = Categories.getIncludePriority(
             IncludeName,
             /*CheckMainHeader=*/!MainIncludeFound && FirstIncludeBlock);
+        int Priority = Categories.getSortIncludePriority(
+            IncludeName, !MainIncludeFound && FirstIncludeBlock);
         if (Category == 0)
           MainIncludeFound = true;
-        IncludesInBlock.push_back({IncludeName, Line, Prev, Category});
+        IncludesInBlock.push_back(
+            {IncludeName, Line, Prev, Category, Priority});
       } else if (!IncludesInBlock.empty() && !EmptyLineSkipped) {
         sortCppIncludes(Style, IncludesInBlock, Ranges, FileName, Code,
                         Replaces, Cursor);
@@ -2037,7 +2098,8 @@ static void sortJavaImports(const FormatStyle &Style,
 
   // If the imports are out of order, we generate a single replacement fixing
   // the entire block. Otherwise, no replacement is generated.
-  if (result == Code.substr(Imports.front().Offset, ImportsBlockSize))
+  if (replaceCRLF(result) ==
+      replaceCRLF(Code.substr(Imports.front().Offset, ImportsBlockSize)))
     return;
 
   auto Err = Replaces.add(tooling::Replacement(FileName, Imports.front().Offset,
@@ -2327,7 +2389,7 @@ reformat(const FormatStyle &Style, StringRef Code,
 
   auto Env =
       std::make_unique<Environment>(Code, FileName, Ranges, FirstStartColumn,
-                                     NextStartColumn, LastStartColumn);
+                                    NextStartColumn, LastStartColumn);
   llvm::Optional<std::string> CurrentCode = None;
   tooling::Replacements Fixes;
   unsigned Penalty = 0;
@@ -2402,14 +2464,18 @@ tooling::Replacements sortUsingDeclarations(const FormatStyle &Style,
 
 LangOptions getFormattingLangOpts(const FormatStyle &Style) {
   LangOptions LangOpts;
-  FormatStyle::LanguageStandard LexingStd =
-      Style.Standard == FormatStyle::LS_Auto ? FormatStyle::LS_Cpp11
-                                             : Style.Standard;
+
+  FormatStyle::LanguageStandard LexingStd = Style.Standard;
+  if (LexingStd == FormatStyle::LS_Auto)
+    LexingStd = FormatStyle::LS_Latest;
+  if (LexingStd == FormatStyle::LS_Latest)
+    LexingStd = FormatStyle::LS_Cpp20;
   LangOpts.CPlusPlus = 1;
   LangOpts.CPlusPlus11 = LexingStd >= FormatStyle::LS_Cpp11;
-  LangOpts.CPlusPlus14 = LexingStd >= FormatStyle::LS_Cpp11;
-  LangOpts.CPlusPlus17 = LexingStd >= FormatStyle::LS_Cpp11;
-  LangOpts.CPlusPlus2a = LexingStd >= FormatStyle::LS_Cpp11;
+  LangOpts.CPlusPlus14 = LexingStd >= FormatStyle::LS_Cpp14;
+  LangOpts.CPlusPlus17 = LexingStd >= FormatStyle::LS_Cpp17;
+  LangOpts.CPlusPlus2a = LexingStd >= FormatStyle::LS_Cpp20;
+
   LangOpts.LineComment = 1;
   bool AlternativeOperators = Style.isCpp();
   LangOpts.CXXOperatorNames = AlternativeOperators ? 1 : 0;
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index 009b8849753cf..e59a059fd6b23 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -657,7 +657,8 @@ FormatToken *FormatTokenLexer::getNextToken() {
         ++Column;
         break;
       case '\t':
-        Column += Style.TabWidth - Column % Style.TabWidth;
+        Column +=
+            Style.TabWidth - (Style.TabWidth ? Column % Style.TabWidth : 0);
         break;
       case '\\':
         if (i + 1 == e || (Text[i + 1] != '\r' && Text[i + 1] != '\n'))
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index 9802711834e69..91ac9822c8ace 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -65,7 +65,7 @@ class AnnotatingParser {
   AnnotatingParser(const FormatStyle &Style, AnnotatedLine &Line,
                    const AdditionalKeywords &Keywords)
       : Style(Style), Line(Line), CurrentToken(Line.First), AutoFound(false),
-        Keywords(Keywords) {
+        TrailingReturnFound(false), Keywords(Keywords) {
     Contexts.push_back(Context(tok::unknown, 1, /*IsExpression=*/false));
     resetTokenMetadata(CurrentToken);
   }
@@ -1389,7 +1389,11 @@ class AnnotatingParser {
     } else if (Current.is(tok::arrow) && AutoFound && Line.MustBeDeclaration &&
                Current.NestingLevel == 0) {
       Current.Type = TT_TrailingReturnArrow;
-    } else if (Current.isOneOf(tok::star, tok::amp, tok::ampamp)) {
+      TrailingReturnFound = true;
+    } else if (Current.is(tok::star) ||
+               (Current.isOneOf(tok::amp, tok::ampamp) &&
+                (Current.NestingLevel != 0 || !Line.MightBeFunctionDecl ||
+                 TrailingReturnFound))) {
       Current.Type = determineStarAmpUsage(Current,
                                            Contexts.back().CanBeExpression &&
                                                Contexts.back().IsExpression,
@@ -1412,6 +1416,8 @@ class AnnotatingParser {
         Current.Type = TT_ConditionalExpr;
       }
     } else if (Current.isBinaryOperator() &&
+               !(Line.MightBeFunctionDecl && Current.NestingLevel == 0 &&
+                 Current.isOneOf(tok::amp, tok::ampamp)) &&
                (!Current.Previous || Current.Previous->isNot(tok::l_square)) &&
                (!Current.is(tok::greater) &&
                 Style.Language != FormatStyle::LK_TextProto)) {
@@ -1486,10 +1492,12 @@ class AnnotatingParser {
       // colon after this, this is the only place which annotates the identifier
       // as a selector.)
       Current.Type = TT_SelectorName;
-    } else if (Current.isOneOf(tok::identifier, tok::kw_const) &&
+    } else if (Current.isOneOf(tok::identifier, tok::kw_const, tok::amp,
+                               tok::ampamp) &&
                Current.Previous &&
                !Current.Previous->isOneOf(tok::equal, tok::at) &&
-               Line.MightBeFunctionDecl && Contexts.size() == 1) {
+               Line.MightBeFunctionDecl && !TrailingReturnFound &&
+               Contexts.size() == 1) {
       // Line.MightBeFunctionDecl can only be true after the parentheses of a
       // function declaration have been found.
       Current.Type = TT_TrailingAnnotation;
@@ -1767,6 +1775,7 @@ class AnnotatingParser {
   AnnotatedLine &Line;
   FormatToken *CurrentToken;
   bool AutoFound;
+  bool TrailingReturnFound;
   const AdditionalKeywords &Keywords;
 
   // Set of "<" tokens that do not open a template parameter list. If parseAngle
@@ -2580,7 +2589,8 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
             (Style.PointerAlignment != FormatStyle::PAS_Right &&
              !Line.IsMultiVariableDeclStmt) &&
             Left.Previous &&
-            !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon));
+            !Left.Previous->isOneOf(tok::l_paren, tok::coloncolon,
+                                    tok::l_square));
   if (Right.is(tok::star) && Left.is(tok::l_paren))
     return false;
   const auto SpaceRequiredForArrayInitializerLSquare =
@@ -2679,6 +2689,14 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
       Right.MatchingParen->endsSequence(TT_DictLiteral, tok::at))
     // Objective-C dictionary literal -> no space before closing brace.
     return false;
+  if (Right.Type == TT_TrailingAnnotation &&
+      Right.isOneOf(tok::amp, tok::ampamp) &&
+      Left.isOneOf(tok::kw_const, tok::kw_volatile) &&
+      (!Right.Next || Right.Next->is(tok::semi)))
+    // Match const and volatile ref-qualifiers without any additional
+    // qualifiers such as
+    // void Fn() const &;
+    return Style.PointerAlignment != FormatStyle::PAS_Left;
   return true;
 }
 
diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp
index 9878d45f58635..df30f7db80e65 100644
--- a/clang/lib/Format/UnwrappedLineFormatter.cpp
+++ b/clang/lib/Format/UnwrappedLineFormatter.cpp
@@ -1193,6 +1193,12 @@ void UnwrappedLineFormatter::formatFirstToken(
   if (Newlines)
     Indent = NewlineIndent;
 
+  // If in Whitemsmiths mode, indent start and end of blocks
+  if (Style.BreakBeforeBraces == FormatStyle::BS_Whitesmiths) {
+    if (RootToken.isOneOf(tok::l_brace, tok::r_brace, tok::kw_case))
+      Indent += Style.IndentWidth;
+  }
+
   // Preprocessor directives get indented before the hash only if specified
   if (Style.IndentPPDirectives != FormatStyle::PPDIS_BeforeHash &&
       (Line.Type == LT_PreprocessorDirective ||
diff --git a/clang/lib/Format/WhitespaceManager.cpp b/clang/lib/Format/WhitespaceManager.cpp
index 33c0d53402e6f..5a44500d355fc 100644
--- a/clang/lib/Format/WhitespaceManager.cpp
+++ b/clang/lib/Format/WhitespaceManager.cpp
@@ -815,19 +815,24 @@ void WhitespaceManager::appendIndentText(std::string &Text,
     Text.append(Spaces, ' ');
     break;
   case FormatStyle::UT_Always: {
-    unsigned FirstTabWidth =
-        Style.TabWidth - WhitespaceStartColumn % Style.TabWidth;
-    // Insert only spaces when we want to end up before the next tab.
-    if (Spaces < FirstTabWidth || Spaces == 1) {
+    if (Style.TabWidth) {
+      unsigned FirstTabWidth =
+          Style.TabWidth - WhitespaceStartColumn % Style.TabWidth;
+
+      // Insert only spaces when we want to end up before the next tab.
+      if (Spaces < FirstTabWidth || Spaces == 1) {
+        Text.append(Spaces, ' ');
+        break;
+      }
+      // Align to the next tab.
+      Spaces -= FirstTabWidth;
+      Text.append("\t");
+
+      Text.append(Spaces / Style.TabWidth, '\t');
+      Text.append(Spaces % Style.TabWidth, ' ');
+    } else if (Spaces == 1) {
       Text.append(Spaces, ' ');
-      break;
     }
-    // Align to the next tab.
-    Spaces -= FirstTabWidth;
-    Text.append("\t");
-
-    Text.append(Spaces / Style.TabWidth, '\t');
-    Text.append(Spaces % Style.TabWidth, ' ');
     break;
   }
   case FormatStyle::UT_ForIndentation:
@@ -837,14 +842,16 @@ void WhitespaceManager::appendIndentText(std::string &Text,
       // the first one.
       if (Indentation > Spaces)
         Indentation = Spaces;
-      unsigned Tabs = Indentation / Style.TabWidth;
-      Text.append(Tabs, '\t');
-      Spaces -= Tabs * Style.TabWidth;
+      if (Style.TabWidth) {
+        unsigned Tabs = Indentation / Style.TabWidth;
+        Text.append(Tabs, '\t');
+        Spaces -= Tabs * Style.TabWidth;
+      }
     }
     Text.append(Spaces, ' ');
     break;
   case FormatStyle::UT_ForContinuationAndIndentation:
-    if (WhitespaceStartColumn == 0) {
+    if (WhitespaceStartColumn == 0 && Style.TabWidth) {
       unsigned Tabs = Spaces / Style.TabWidth;
       Text.append(Tabs, '\t');
       Spaces -= Tabs * Style.TabWidth;
diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index 35348ffe13b19..f5e291b7fe17e 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -84,6 +84,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -2301,26 +2302,19 @@ bool ASTUnit::Save(StringRef File) {
   SmallString<128> TempPath;
   TempPath = File;
   TempPath += "-%%%%%%%%";
-  int fd;
-  if (llvm::sys::fs::createUniqueFile(TempPath, fd, TempPath))
-    return true;
-
   // FIXME: Can we somehow regenerate the stat cache here, or do we need to
   // unconditionally create a stat cache when we parse the file?
-  llvm::raw_fd_ostream Out(fd, /*shouldClose=*/true);
-
-  serialize(Out);
-  Out.close();
-  if (Out.has_error()) {
-    Out.clear_error();
-    return true;
-  }
 
-  if (llvm::sys::fs::rename(TempPath, File)) {
-    llvm::sys::fs::remove(TempPath);
+  if (llvm::Error Err = llvm::writeFileAtomically(
+          TempPath, File, [this](llvm::raw_ostream &Out) {
+            return serialize(Out) ? llvm::make_error<llvm::StringError>(
+                                        "ASTUnit serialization failed",
+                                        llvm::inconvertibleErrorCode())
+                                  : llvm::Error::success();
+          })) {
+    consumeError(std::move(Err));
     return true;
   }
-
   return false;
 }
 
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index fd6038d784e8b..2a263124ac583 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -2537,6 +2537,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     Opts.CUDADeviceApproxTranscendentals = 1;
 
   Opts.GPURelocatableDeviceCode = Args.hasArg(OPT_fgpu_rdc);
+  Opts.HIPUseNewLaunchAPI = Args.hasArg(OPT_fhip_new_launch_api);
 
   Opts.SYCLIntHeader = Args.getLastArgValue(OPT_fsycl_int_header);
 
diff --git a/clang/lib/Frontend/InitHeaderSearch.cpp b/clang/lib/Frontend/InitHeaderSearch.cpp
index ad0d23271487a..5d877ee9c0d73 100644
--- a/clang/lib/Frontend/InitHeaderSearch.cpp
+++ b/clang/lib/Frontend/InitHeaderSearch.cpp
@@ -137,6 +137,13 @@ bool InitHeaderSearch::AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
   SmallString<256> MappedPathStorage;
   StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
 
+  // If use system headers while cross-compiling, emit the warning.
+  if (HasSysroot && (MappedPathStr.startswith("/usr/include") ||
+                     MappedPathStr.startswith("/usr/local/include"))) {
+    Headers.getDiags().Report(diag::warn_poison_system_directories)
+        << MappedPathStr;
+  }
+
   // Compute the DirectoryLookup type.
   SrcMgr::CharacteristicKind Type;
   if (Group == Quoted || Group == Angled || Group == IndexHeaderMap) {
diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp
index f632139efa557..3fa2cd4841421 100644
--- a/clang/lib/Frontend/InitPreprocessor.cpp
+++ b/clang/lib/Frontend/InitPreprocessor.cpp
@@ -555,6 +555,7 @@ static void InitializeCPlusPlusFeatureTestMacros(const LangOptions &LangOpts,
   // C++20 features.
   if (LangOpts.CPlusPlus2a) {
     Builder.defineMacro("__cpp_conditional_explicit", "201806L");
+    Builder.defineMacro("__cpp_constexpr_dynamic_alloc", "201907L");
     Builder.defineMacro("__cpp_constinit", "201907L");
   }
   if (LangOpts.Char8)
diff --git a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
index 984b63866422e..dcf645f67f2f3 100644
--- a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
+++ b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
@@ -49,6 +49,8 @@ class InclusionRewriter : public PPCallbacks {
   std::map<unsigned, const Module *> ModuleIncludes;
   /// Tracks where inclusions that enter modules (in a module build) are found.
   std::map<unsigned, const Module *> ModuleEntryIncludes;
+  /// Tracks where #if and #elif directives get evaluated and whether to true.
+  std::map<unsigned, bool> IfConditions;
   /// Used transitively for building up the FileIncludes mapping over the
   /// various \c PPCallbacks callbacks.
   SourceLocation LastInclusionLocation;
@@ -78,6 +80,10 @@ class InclusionRewriter : public PPCallbacks {
                           StringRef SearchPath, StringRef RelativePath,
                           const Module *Imported,
                           SrcMgr::CharacteristicKind FileType) override;
+  void If(SourceLocation Loc, SourceRange ConditionRange,
+          ConditionValueKind ConditionValue) override;
+  void Elif(SourceLocation Loc, SourceRange ConditionRange,
+            ConditionValueKind ConditionValue, SourceLocation IfLoc) override;
   void WriteLineInfo(StringRef Filename, int Line,
                      SrcMgr::CharacteristicKind FileType,
                      StringRef Extra = StringRef());
@@ -89,12 +95,10 @@ class InclusionRewriter : public PPCallbacks {
   void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
                            const MemoryBuffer &FromFile, StringRef EOL,
                            unsigned &NextToWrite, int &Lines);
-  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
-                        const DirectoryLookup *Lookup, Token &Tok,
-                        bool &FileExists);
   const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
   const Module *FindModuleAtLocation(SourceLocation Loc) const;
   const Module *FindEnteredModule(SourceLocation Loc) const;
+  bool IsIfAtLocationTrue(SourceLocation Loc) const;
   StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
 };
 
@@ -203,6 +207,23 @@ void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
     LastInclusionLocation = HashLoc;
 }
 
+void InclusionRewriter::If(SourceLocation Loc, SourceRange ConditionRange,
+                           ConditionValueKind ConditionValue) {
+  auto P = IfConditions.insert(
+      std::make_pair(Loc.getRawEncoding(), ConditionValue == CVK_True));
+  (void)P;
+  assert(P.second && "Unexpected revisitation of the same if directive");
+}
+
+void InclusionRewriter::Elif(SourceLocation Loc, SourceRange ConditionRange,
+                             ConditionValueKind ConditionValue,
+                             SourceLocation IfLoc) {
+  auto P = IfConditions.insert(
+      std::make_pair(Loc.getRawEncoding(), ConditionValue == CVK_True));
+  (void)P;
+  assert(P.second && "Unexpected revisitation of the same elif directive");
+}
+
 /// Simple lookup for a SourceLocation (specifically one denoting the hash in
 /// an inclusion directive) in the map of inclusion information, FileChanges.
 const InclusionRewriter::IncludedFile *
@@ -233,6 +254,13 @@ InclusionRewriter::FindEnteredModule(SourceLocation Loc) const {
   return nullptr;
 }
 
+bool InclusionRewriter::IsIfAtLocationTrue(SourceLocation Loc) const {
+  const auto I = IfConditions.find(Loc.getRawEncoding());
+  if (I != IfConditions.end())
+    return I->second;
+  return false;
+}
+
 /// Detect the likely line ending style of \p FromFile by examining the first
 /// newline found within it.
 static StringRef DetectEOL(const MemoryBuffer &FromFile) {
@@ -346,80 +374,6 @@ StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
   return StringRef();
 }
 
-// Expand __has_include and __has_include_next if possible. If there's no
-// definitive answer return false.
-bool InclusionRewriter::HandleHasInclude(
-    FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
-    bool &FileExists) {
-  // Lex the opening paren.
-  RawLex.LexFromRawLexer(Tok);
-  if (Tok.isNot(tok::l_paren))
-    return false;
-
-  RawLex.LexFromRawLexer(Tok);
-
-  SmallString<128> FilenameBuffer;
-  StringRef Filename;
-  // Since the raw lexer doesn't give us angle_literals we have to parse them
-  // ourselves.
-  // FIXME: What to do if the file name is a macro?
-  if (Tok.is(tok::less)) {
-    RawLex.LexFromRawLexer(Tok);
-
-    FilenameBuffer += '<';
-    do {
-      if (Tok.is(tok::eod)) // Sanity check.
-        return false;
-
-      if (Tok.is(tok::raw_identifier))
-        PP.LookUpIdentifierInfo(Tok);
-
-      // Get the string piece.
-      SmallVector<char, 128> TmpBuffer;
-      bool Invalid = false;
-      StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
-      if (Invalid)
-        return false;
-
-      FilenameBuffer += TmpName;
-
-      RawLex.LexFromRawLexer(Tok);
-    } while (Tok.isNot(tok::greater));
-
-    FilenameBuffer += '>';
-    Filename = FilenameBuffer;
-  } else {
-    if (Tok.isNot(tok::string_literal))
-      return false;
-
-    bool Invalid = false;
-    Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
-    if (Invalid)
-      return false;
-  }
-
-  // Lex the closing paren.
-  RawLex.LexFromRawLexer(Tok);
-  if (Tok.isNot(tok::r_paren))
-    return false;
-
-  // Now ask HeaderInfo if it knows about the header.
-  // FIXME: Subframeworks aren't handled here. Do we care?
-  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
-  const DirectoryLookup *CurDir;
-  const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
-  SmallVector<std::pair<const FileEntry *, const DirectoryEntry *>, 1>
-      Includers;
-  Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
-  // FIXME: Why don't we call PP.LookupFile here?
-  Optional<FileEntryRef> File = PP.getHeaderSearchInfo().LookupFile(
-      Filename, SourceLocation(), isAngled, Lookup, CurDir, Includers, nullptr,
-      nullptr, nullptr, nullptr, nullptr, nullptr);
-
-  FileExists = File.hasValue();
-  return true;
-}
-
 /// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
 /// and including content of included files recursively.
 void InclusionRewriter::Process(FileID FileId,
@@ -519,53 +473,33 @@ void InclusionRewriter::Process(FileID FileId,
           case tok::pp_elif: {
             bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
                          tok::pp_elif);
-            // Rewrite special builtin macros to avoid pulling in host details.
+            bool isTrue = IsIfAtLocationTrue(RawToken.getLocation());
+            OutputContentUpTo(FromFile, NextToWrite,
+                              SM.getFileOffset(HashToken.getLocation()),
+                              LocalEOL, Line, /*EnsureNewline=*/true);
             do {
-              // Walk over the directive.
               RawLex.LexFromRawLexer(RawToken);
-              if (RawToken.is(tok::raw_identifier))
-                PP.LookUpIdentifierInfo(RawToken);
-
-              if (RawToken.is(tok::identifier)) {
-                bool HasFile;
-                SourceLocation Loc = RawToken.getLocation();
-
-                // Rewrite __has_include(x)
-                if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
-                  if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
-                                        HasFile))
-                    continue;
-                  // Rewrite __has_include_next(x)
-                } else if (RawToken.getIdentifierInfo()->isStr(
-                               "__has_include_next")) {
-                  if (DirLookup)
-                    ++DirLookup;
-
-                  if (!HandleHasInclude(FileId, RawLex, DirLookup, RawToken,
-                                        HasFile))
-                    continue;
-                } else {
-                  continue;
-                }
-                // Replace the macro with (0) or (1), followed by the commented
-                // out macro for reference.
-                OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
-                                  LocalEOL, Line, false);
-                OS << '(' << (int) HasFile << ")/*";
-                OutputContentUpTo(FromFile, NextToWrite,
-                                  SM.getFileOffset(RawToken.getLocation()) +
-                                      RawToken.getLength(),
-                                  LocalEOL, Line, false);
-                OS << "*/";
-              }
-            } while (RawToken.isNot(tok::eod));
+            } while (!RawToken.is(tok::eod) && RawToken.isNot(tok::eof));
+            // We need to disable the old condition, but that is tricky.
+            // Trying to comment it out can easily lead to comment nesting.
+            // So instead make the condition harmless by making it enclose
+            // and empty block. Moreover, put it itself inside an #if 0 block
+            // to disable it from getting evaluated (e.g. __has_include_next
+            // warns if used from the primary source file).
+            OS << "#if 0 /* disabled by -frewrite-includes */" << MainEOL;
             if (elif) {
-              OutputContentUpTo(FromFile, NextToWrite,
-                                SM.getFileOffset(RawToken.getLocation()) +
-                                    RawToken.getLength(),
-                                LocalEOL, Line, /*EnsureNewline=*/ true);
-              WriteLineInfo(FileName, Line, FileType);
+              OS << "#if 0" << MainEOL;
             }
+            OutputContentUpTo(FromFile, NextToWrite,
+                              SM.getFileOffset(RawToken.getLocation()) +
+                                  RawToken.getLength(),
+                              LocalEOL, Line, /*EnsureNewline=*/true);
+            // Close the empty block and the disabling block.
+            OS << "#endif" << MainEOL;
+            OS << "#endif /* disabled by -frewrite-includes */" << MainEOL;
+            OS << (elif ? "#elif " : "#if ") << (isTrue ? "1" : "0")
+               << " /* evaluated by -frewrite-includes */" << MainEOL;
+            WriteLineInfo(FileName, Line, FileType);
             break;
           }
           case tok::pp_endif:
diff --git a/clang/lib/Frontend/TextDiagnostic.cpp b/clang/lib/Frontend/TextDiagnostic.cpp
index d6e75a9180cf5..7bb6c5b74d5f6 100644
--- a/clang/lib/Frontend/TextDiagnostic.cpp
+++ b/clang/lib/Frontend/TextDiagnostic.cpp
@@ -683,8 +683,9 @@ void TextDiagnostic::emitDiagnosticMessage(
   if (DiagOpts->ShowColors)
     OS.resetColor();
 
-  printDiagnosticLevel(OS, Level, DiagOpts->ShowColors,
-                       DiagOpts->CLFallbackMode);
+  if (DiagOpts->ShowLevel)
+    printDiagnosticLevel(OS, Level, DiagOpts->ShowColors,
+                         DiagOpts->CLFallbackMode);
   printDiagnosticMessage(OS,
                          /*IsSupplemental*/ Level == DiagnosticsEngine::Note,
                          Message, OS.tell() - StartOfLocationInfo,
diff --git a/clang/lib/Headers/altivec.h b/clang/lib/Headers/altivec.h
index bc26f1089d04d..8352f8f740c24 100644
--- a/clang/lib/Headers/altivec.h
+++ b/clang/lib/Headers/altivec.h
@@ -2761,8 +2761,8 @@ static __inline__ vector double __ATTRS_o_ai vec_xl_len(double *__a,
   return (vector double)__builtin_vsx_lxvl(__a, (__b << 56));
 }
 
-static __inline__ vector double __ATTRS_o_ai vec_xl_len_r(unsigned char *__a,
-                                                          size_t __b) {
+static __inline__ vector unsigned char __ATTRS_o_ai
+vec_xl_len_r(unsigned char *__a, size_t __b) {
   vector unsigned char __res =
       (vector unsigned char)__builtin_vsx_lxvll(__a, (__b << 56));
 #ifdef __LITTLE_ENDIAN__
@@ -2912,10 +2912,11 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 #ifdef __VSX__
 #define vec_cts(__a, __b)                                                      \
   _Generic((__a), vector float                                                 \
-           : __builtin_altivec_vctsxs((__a), (__b)), vector double             \
+           : __builtin_altivec_vctsxs((vector float)(__a), (__b)),             \
+             vector double                                                     \
            : __extension__({                                                   \
              vector double __ret =                                             \
-                 (__a) *                                                       \
+                 (vector double)(__a) *                                        \
                  (vector double)(vector unsigned long long)((0x3ffULL + (__b)) \
                                                             << 52);            \
              __builtin_convertvector(__ret, vector signed long long);          \
@@ -2933,10 +2934,11 @@ static __inline__ vector double __ATTRS_o_ai vec_cpsgn(vector double __a,
 #ifdef __VSX__
 #define vec_ctu(__a, __b)                                                      \
   _Generic((__a), vector float                                                 \
-           : __builtin_altivec_vctuxs((__a), (__b)), vector double             \
+           : __builtin_altivec_vctuxs((vector float)(__a), (__b)),             \
+             vector double                                                     \
            : __extension__({                                                   \
              vector double __ret =                                             \
-                 (__a) *                                                       \
+                 (vector double)(__a) *                                        \
                  (vector double)(vector unsigned long long)((0x3ffULL + __b)   \
                                                             << 52);            \
              __builtin_convertvector(__ret, vector unsigned long long);        \
@@ -6301,19 +6303,20 @@ static __inline__ vector float __ATTRS_o_ai vec_or(vector float __a,
 #ifdef __VSX__
 static __inline__ vector double __ATTRS_o_ai vec_or(vector bool long long __a,
                                                     vector double __b) {
-  return (vector unsigned long long)__a | (vector unsigned long long)__b;
+  return (vector double)((vector unsigned long long)__a |
+                         (vector unsigned long long)__b);
 }
 
 static __inline__ vector double __ATTRS_o_ai vec_or(vector double __a,
                                                     vector bool long long __b) {
-  return (vector unsigned long long)__a | (vector unsigned long long)__b;
+  return (vector double)((vector unsigned long long)__a |
+                         (vector unsigned long long)__b);
 }
 
 static __inline__ vector double __ATTRS_o_ai vec_or(vector double __a,
                                                     vector double __b) {
-  vector unsigned long long __res =
-      (vector unsigned long long)__a | (vector unsigned long long)__b;
-  return (vector double)__res;
+  return (vector double)((vector unsigned long long)__a |
+                         (vector unsigned long long)__b);
 }
 
 static __inline__ vector signed long long __ATTRS_o_ai
@@ -14781,7 +14784,7 @@ static __inline__ int __ATTRS_o_ai vec_all_ne(vector bool long long __a,
 static __inline__ int __ATTRS_o_ai vec_all_ne(vector float __a,
                                               vector float __b) {
 #ifdef __VSX__
-  return __builtin_vsx_xvcmpeqdp_p(__CR6_EQ, __a, __b);
+  return __builtin_vsx_xvcmpeqdp_p(__CR6_EQ, (vector double)__a, (vector double)__b);
 #else
   return __builtin_altivec_vcmpeqfp_p(__CR6_EQ, __a, __b);
 #endif
@@ -16671,13 +16674,13 @@ static __inline__ void __ATTRS_o_ai vec_xst_be(vector unsigned __int128 __vec,
 #endif
 
 #ifdef __POWER9_VECTOR__
-#define vec_test_data_class(__a, __b)                                      \
-        _Generic((__a),                                                    \
-           vector float:                                                   \
-             (vector bool int)__builtin_vsx_xvtstdcsp((__a), (__b)),       \
-           vector double:                                                  \
-             (vector bool long long)__builtin_vsx_xvtstdcdp((__a), (__b))  \
-        )
+#define vec_test_data_class(__a, __b)                                          \
+  _Generic(                                                                    \
+      (__a), vector float                                                      \
+      : (vector bool int)__builtin_vsx_xvtstdcsp((vector float)(__a), (__b)),  \
+        vector double                                                          \
+      : (vector bool long long)__builtin_vsx_xvtstdcdp((vector double)(__a),   \
+                                                       (__b)))
 
 #endif /* #ifdef __POWER9_VECTOR__ */
 
diff --git a/clang/lib/Headers/avx512fintrin.h b/clang/lib/Headers/avx512fintrin.h
index 4e341a10fa948..698e477fe5f36 100644
--- a/clang/lib/Headers/avx512fintrin.h
+++ b/clang/lib/Headers/avx512fintrin.h
@@ -7658,13 +7658,13 @@ _mm512_maskz_getexp_ps (__mmask16 __U, __m512 __A)
 #define _mm512_i32gather_ps(index, addr, scale) \
   (__m512)__builtin_ia32_gathersiv16sf((__v16sf)_mm512_undefined_ps(), \
                                        (void const *)(addr), \
-                                       (__v16sf)(__m512)(index), \
+                                       (__v16si)(__m512)(index), \
                                        (__mmask16)-1, (int)(scale))
 
 #define _mm512_mask_i32gather_ps(v1_old, mask, index, addr, scale) \
   (__m512)__builtin_ia32_gathersiv16sf((__v16sf)(__m512)(v1_old), \
                                        (void const *)(addr), \
-                                       (__v16sf)(__m512)(index), \
+                                       (__v16si)(__m512)(index), \
                                        (__mmask16)(mask), (int)(scale))
 
 #define _mm512_i32gather_epi32(index, addr, scale) \
diff --git a/clang/lib/Headers/ia32intrin.h b/clang/lib/Headers/ia32intrin.h
index 8e38df73187d1..79b7f0655cf01 100644
--- a/clang/lib/Headers/ia32intrin.h
+++ b/clang/lib/Headers/ia32intrin.h
@@ -195,6 +195,74 @@ __writeeflags(unsigned int __f)
 }
 #endif /* !__x86_64__ */
 
+/** Cast a 32-bit float value to a 32-bit unsigned integer value
+ *
+ *  \headerfile <x86intrin.h>
+ *  This intrinsic corresponds to the <c> VMOVD / MOVD </c> instruction in x86_64,
+ *  and corresponds to the <c> VMOVL / MOVL </c> instruction in ia32.
+ *
+ *  \param __A
+ *     A 32-bit float value.
+ *  \returns a 32-bit unsigned integer containing the converted value.
+ */
+static __inline__ unsigned int __attribute__((__always_inline__))
+_castf32_u32(float __A) {
+  unsigned int D;
+  __builtin_memcpy(&D, &__A, sizeof(__A));
+  return D;
+}
+
+/** Cast a 64-bit float value to a 64-bit unsigned integer value
+ *
+ *  \headerfile <x86intrin.h>
+ *  This intrinsic corresponds to the <c> VMOVQ / MOVQ </c> instruction in x86_64,
+ *  and corresponds to the <c> VMOVL / MOVL </c> instruction in ia32.
+ *
+ *  \param __A
+ *     A 64-bit float value.
+ *  \returns a 64-bit unsigned integer containing the converted value.
+ */
+static __inline__ unsigned long long __attribute__((__always_inline__))
+_castf64_u64(double __A) {
+  unsigned long long D;
+  __builtin_memcpy(&D, &__A, sizeof(__A));
+  return D;
+}
+
+/** Cast a 32-bit unsigned integer value to a 32-bit float value
+ *
+ *  \headerfile <x86intrin.h>
+ *  This intrinsic corresponds to the <c> VMOVQ / MOVQ </c> instruction in x86_64,
+ *  and corresponds to the <c> FLDS </c> instruction in ia32.
+ *
+ *  \param __A
+ *     A 32-bit unsigned integer value.
+ *  \returns a 32-bit float value containing the converted value.
+ */
+static __inline__ float __attribute__((__always_inline__))
+_castu32_f32(unsigned int __A) {
+  float D;
+  __builtin_memcpy(&D, &__A, sizeof(__A));
+  return D;
+}
+
+/** Cast a 64-bit unsigned integer value to a 64-bit float value
+ *
+ *  \headerfile <x86intrin.h>
+ *  This intrinsic corresponds to the <c> VMOVQ / MOVQ </c> instruction in x86_64,
+ *  and corresponds to the <c> FLDL </c> instruction in ia32.
+ *
+ *  \param __A
+ *     A 64-bit unsigned integer value.
+ *  \returns a 64-bit float value containing the converted value.
+ */
+static __inline__ double __attribute__((__always_inline__))
+_castu64_f64(unsigned long long __A) {
+  double D;
+  __builtin_memcpy(&D, &__A, sizeof(__A));
+  return D;
+}
+
 /** Adds the unsigned integer operand to the CRC-32C checksum of the
  *     unsigned char operand.
  *
diff --git a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp b/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
index 0a0656699ab54..2ffbcfb7e6768 100644
--- a/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
+++ b/clang/lib/Lex/DependencyDirectivesSourceMinimizer.cpp
@@ -244,13 +244,22 @@ static void skipToNewlineRaw(const char *&First, const char *const End) {
   }
 }
 
-static const char *reverseOverSpaces(const char *First, const char *Last) {
+static const char *findLastNonSpace(const char *First, const char *Last) {
   assert(First <= Last);
   while (First != Last && isHorizontalWhitespace(Last[-1]))
     --Last;
   return Last;
 }
 
+static const char *findFirstTrailingSpace(const char *First,
+                                          const char *Last) {
+  const char *LastNonSpace = findLastNonSpace(First, Last);
+  if (Last == LastNonSpace)
+    return Last;
+  assert(isHorizontalWhitespace(LastNonSpace[0]));
+  return LastNonSpace + 1;
+}
+
 static void skipLineComment(const char *&First, const char *const End) {
   assert(First[0] == '/' && First[1] == '/');
   First += 2;
@@ -382,7 +391,7 @@ void Minimizer::printToNewline(const char *&First, const char *const End) {
       }
 
       // Deal with "//..." and "/*...*/".
-      append(First, reverseOverSpaces(First, Last));
+      append(First, findFirstTrailingSpace(First, Last));
       First = Last;
 
       if (Last[1] == '/') {
@@ -397,15 +406,20 @@ void Minimizer::printToNewline(const char *&First, const char *const End) {
     } while (Last != End && !isVerticalWhitespace(*Last));
 
     // Print out the string.
-    if (Last == End || Last == First || Last[-1] != '\\') {
-      append(First, reverseOverSpaces(First, Last));
+    const char *LastBeforeTrailingSpace = findLastNonSpace(First, Last);
+    if (Last == End || LastBeforeTrailingSpace == First ||
+        LastBeforeTrailingSpace[-1] != '\\') {
+      append(First, LastBeforeTrailingSpace);
       First = Last;
       skipNewline(First, End);
       return;
     }
 
-    // Print up to the backslash, backing up over spaces.
-    append(First, reverseOverSpaces(First, Last - 1));
+    // Print up to the backslash, backing up over spaces. Preserve at least one
+    // space, as the space matters when tokens are separated by a line
+    // continuation.
+    append(First, findFirstTrailingSpace(
+                      First, LastBeforeTrailingSpace - 1));
 
     First = Last;
     skipNewline(First, End);
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index c4f660f88c73f..3b7eaee3c914f 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1716,11 +1716,11 @@ Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
     SourceLocation FilenameLoc, CharSourceRange FilenameRange,
     const Token &FilenameTok, bool &IsFrameworkFound, bool IsImportDecl,
     bool &IsMapped, const DirectoryLookup *LookupFrom,
-    const FileEntry *LookupFromFile, SmallString<128> &NormalizedPath,
+    const FileEntry *LookupFromFile, StringRef LookupFilename,
     SmallVectorImpl<char> &RelativePath, SmallVectorImpl<char> &SearchPath,
     ModuleMap::KnownHeader &SuggestedModule, bool isAngled) {
   Optional<FileEntryRef> File = LookupFile(
-      FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename,
+      FilenameLoc, LookupFilename,
       isAngled, LookupFrom, LookupFromFile, CurDir,
       Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
       &SuggestedModule, &IsMapped, &IsFrameworkFound);
@@ -1739,7 +1739,7 @@ Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
         // Try the lookup again, skipping the cache.
         Optional<FileEntryRef> File = LookupFile(
             FilenameLoc,
-            LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled,
+            LookupFilename, isAngled,
             LookupFrom, LookupFromFile, CurDir, nullptr, nullptr,
             &SuggestedModule, &IsMapped, /*IsFrameworkFound=*/nullptr,
             /*SkipCache*/ true);
@@ -1757,7 +1757,7 @@ Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
   // provide the user with a possible fixit.
   if (isAngled) {
     Optional<FileEntryRef> File = LookupFile(
-        FilenameLoc, LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename,
+        FilenameLoc, LookupFilename,
         false, LookupFrom, LookupFromFile, CurDir,
         Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
         &SuggestedModule, &IsMapped,
@@ -1785,20 +1785,23 @@ Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
       return Filename;
     };
     StringRef TypoCorrectionName = CorrectTypoFilename(Filename);
-    SmallString<128> NormalizedTypoCorrectionPath;
-    if (LangOpts.MSVCCompat) {
-      NormalizedTypoCorrectionPath = TypoCorrectionName.str();
+
 #ifndef _WIN32
+    // Normalize slashes when compiling with -fms-extensions on non-Windows.
+    // This is unnecessary on Windows since the filesystem there handles
+    // backslashes.
+    SmallString<128> NormalizedTypoCorrectionPath;
+    if (LangOpts.MicrosoftExt) {
+      NormalizedTypoCorrectionPath = TypoCorrectionName;
       llvm::sys::path::native(NormalizedTypoCorrectionPath);
-#endif
+      TypoCorrectionName = NormalizedTypoCorrectionPath;
     }
+#endif
+
     Optional<FileEntryRef> File = LookupFile(
-        FilenameLoc,
-        LangOpts.MSVCCompat ? NormalizedTypoCorrectionPath.c_str()
-                            : TypoCorrectionName,
-        isAngled, LookupFrom, LookupFromFile, CurDir,
-        Callbacks ? &SearchPath : nullptr, Callbacks ? &RelativePath : nullptr,
-        &SuggestedModule, &IsMapped,
+        FilenameLoc, TypoCorrectionName, isAngled, LookupFrom, LookupFromFile,
+        CurDir, Callbacks ? &SearchPath : nullptr,
+        Callbacks ? &RelativePath : nullptr, &SuggestedModule, &IsMapped,
         /*IsFrameworkFound=*/nullptr);
     if (File) {
       auto Hint =
@@ -1906,18 +1909,23 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
   // the path.
   ModuleMap::KnownHeader SuggestedModule;
   SourceLocation FilenameLoc = FilenameTok.getLocation();
+  StringRef LookupFilename = Filename;
+
+#ifndef _WIN32
+  // Normalize slashes when compiling with -fms-extensions on non-Windows. This
+  // is unnecessary on Windows since the filesystem there handles backslashes.
   SmallString<128> NormalizedPath;
-  if (LangOpts.MSVCCompat) {
+  if (LangOpts.MicrosoftExt) {
     NormalizedPath = Filename.str();
-#ifndef _WIN32
     llvm::sys::path::native(NormalizedPath);
-#endif
+    LookupFilename = NormalizedPath;
   }
+#endif
 
   Optional<FileEntryRef> File = LookupHeaderIncludeOrImport(
       CurDir, Filename, FilenameLoc, FilenameRange, FilenameTok,
       IsFrameworkFound, IsImportDecl, IsMapped, LookupFrom, LookupFromFile,
-      NormalizedPath, RelativePath, SearchPath, SuggestedModule, isAngled);
+      LookupFilename, RelativePath, SearchPath, SuggestedModule, isAngled);
 
   if (usingPCHWithThroughHeader() && SkippingUntilPCHThroughHeader) {
     if (File && isPCHThroughHeader(&File->getFileEntry()))
@@ -2059,10 +2067,9 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
     // Notify the callback object that we've seen an inclusion directive.
     // FIXME: Use a different callback for a pp-import?
     Callbacks->InclusionDirective(
-        HashLoc, IncludeTok,
-        LangOpts.MSVCCompat ? NormalizedPath.c_str() : Filename, isAngled,
-        FilenameRange, File ? &File->getFileEntry() : nullptr, SearchPath,
-        RelativePath, Action == Import ? SuggestedModule.getModule() : nullptr,
+        HashLoc, IncludeTok, LookupFilename, isAngled, FilenameRange,
+        File ? &File->getFileEntry() : nullptr, SearchPath, RelativePath,
+        Action == Import ? SuggestedModule.getModule() : nullptr,
         FileCharacter);
     if (Action == Skip && File)
       Callbacks->FileSkipped(*File, FilenameTok, FileCharacter);
@@ -2085,7 +2092,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
       !IsMapped && !File->getFileEntry().tryGetRealPathName().empty();
 
   if (CheckIncludePathPortability) {
-    StringRef Name = LangOpts.MSVCCompat ? NormalizedPath.str() : Filename;
+    StringRef Name = LookupFilename;
     StringRef RealPathName = File->getFileEntry().tryGetRealPathName();
     SmallVector<StringRef, 16> Components(llvm::sys::path::begin(Name),
                                           llvm::sys::path::end(Name));
@@ -2782,7 +2789,8 @@ void Preprocessor::HandleDefineDirective(
   // If we need warning for not using the macro, add its location in the
   // warn-because-unused-macro set. If it gets used it will be removed from set.
   if (getSourceManager().isInMainFile(MI->getDefinitionLoc()) &&
-      !Diags->isIgnored(diag::pp_macro_not_used, MI->getDefinitionLoc())) {
+      !Diags->isIgnored(diag::pp_macro_not_used, MI->getDefinitionLoc()) &&
+      !MacroExpansionInDirectivesOverride) {
     MI->setIsWarnIfUnused(true);
     WarnUnusedMacroLocs.insert(MI->getDefinitionLoc());
   }
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index 35a05b258bc35..2cbe0788070d1 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -4684,8 +4684,10 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
     ExprResult AssignedVal;
     EnumAvailabilityDiags.emplace_back(*this);
 
+    EnterExpressionEvaluationContext ConstantEvaluated(
+        Actions, Sema::ExpressionEvaluationContext::ConstantEvaluated);
     if (TryConsumeToken(tok::equal, EqualLoc)) {
-      AssignedVal = ParseConstantExpression();
+      AssignedVal = ParseConstantExpressionInExprEvalContext();
       if (AssignedVal.isInvalid())
         SkipUntil(tok::comma, tok::r_brace, StopBeforeMatch);
     }
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 6bd96051f9e17..c7143fd410410 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -42,6 +42,7 @@ enum OpenMPDirectiveKindEx {
   OMPD_teams_distribute_parallel,
   OMPD_target_teams_distribute_parallel,
   OMPD_mapper,
+  OMPD_variant,
 };
 
 class DeclDirectiveListParserHelper final {
@@ -80,6 +81,7 @@ static unsigned getOpenMPDirectiveKindEx(StringRef S) {
       .Case("reduction", OMPD_reduction)
       .Case("update", OMPD_update)
       .Case("mapper", OMPD_mapper)
+      .Case("variant", OMPD_variant)
       .Default(OMPD_unknown);
 }
 
@@ -93,6 +95,7 @@ static OpenMPDirectiveKind parseOpenMPDirectiveKind(Parser &P) {
       {OMPD_declare, OMPD_mapper, OMPD_declare_mapper},
       {OMPD_declare, OMPD_simd, OMPD_declare_simd},
       {OMPD_declare, OMPD_target, OMPD_declare_target},
+      {OMPD_declare, OMPD_variant, OMPD_declare_variant},
       {OMPD_distribute, OMPD_parallel, OMPD_distribute_parallel},
       {OMPD_distribute_parallel, OMPD_for, OMPD_distribute_parallel_for},
       {OMPD_distribute_parallel_for, OMPD_simd,
@@ -752,6 +755,7 @@ Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
                       /*IsReinject*/ true);
   // Consume the previously pushed token.
   ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
+  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
 
   FNContextRAII FnContext(*this, Ptr);
   OMPDeclareSimdDeclAttr::BranchStateTy BS =
@@ -782,6 +786,232 @@ Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
       LinModifiers, Steps, SourceRange(Loc, EndLoc));
 }
 
+/// Parse optional 'score' '(' <expr> ')' ':'.
+static ExprResult parseContextScore(Parser &P) {
+  ExprResult ScoreExpr;
+  SmallString<16> Buffer;
+  StringRef SelectorName =
+      P.getPreprocessor().getSpelling(P.getCurToken(), Buffer);
+  OMPDeclareVariantAttr::ScoreType ScoreKind =
+      OMPDeclareVariantAttr::ScoreUnknown;
+  (void)OMPDeclareVariantAttr::ConvertStrToScoreType(SelectorName, ScoreKind);
+  if (ScoreKind == OMPDeclareVariantAttr::ScoreUnknown)
+    return ScoreExpr;
+  assert(ScoreKind == OMPDeclareVariantAttr::ScoreSpecified &&
+         "Expected \"score\" clause.");
+  (void)P.ConsumeToken();
+  SourceLocation RLoc;
+  ScoreExpr = P.ParseOpenMPParensExpr(SelectorName, RLoc);
+  // Parse ':'
+  if (P.getCurToken().is(tok::colon))
+    (void)P.ConsumeAnyToken();
+  else
+    P.Diag(P.getCurToken(), diag::warn_pragma_expected_colon)
+        << "context selector score clause";
+  return ScoreExpr;
+}
+
+/// Parse context selector for 'implementation' selector set:
+/// 'vendor' '(' <vendor> ')'
+static void
+parseImplementationSelector(Parser &P,
+                            Sema::OpenMPDeclareVariantCtsSelectorData &Data) {
+  const Token &Tok = P.getCurToken();
+  // Parse inner context selector set name, if any.
+  if (!Tok.is(tok::identifier)) {
+    P.Diag(Tok.getLocation(), diag::warn_omp_declare_variant_cs_name_expected)
+        << "implementation";
+    // Skip until either '}', ')', or end of directive.
+    while (!P.SkipUntil(tok::r_brace, tok::r_paren,
+                        tok::annot_pragma_openmp_end, Parser::StopBeforeMatch))
+      ;
+    return;
+  }
+  SmallString<16> Buffer;
+  StringRef CtxSelectorName = P.getPreprocessor().getSpelling(Tok, Buffer);
+  OMPDeclareVariantAttr::CtxSelectorType CSKind =
+      OMPDeclareVariantAttr::CtxUnknown;
+  (void)OMPDeclareVariantAttr::ConvertStrToCtxSelectorType(CtxSelectorName,
+                                                           CSKind);
+  (void)P.ConsumeToken();
+  switch (CSKind) {
+  case OMPDeclareVariantAttr::CtxVendor: {
+    // Parse '('.
+    BalancedDelimiterTracker T(P, tok::l_paren, tok::annot_pragma_openmp_end);
+    (void)T.expectAndConsume(diag::err_expected_lparen_after,
+                             CtxSelectorName.data());
+    Data.CtxScore = parseContextScore(P);
+    // Parse <vendor>.
+    StringRef VendorName;
+    if (Tok.is(tok::identifier)) {
+      VendorName = P.getPreprocessor().getSpelling(P.getCurToken(), Buffer);
+      (void)P.ConsumeToken();
+    } else {
+      P.Diag(Tok.getLocation(), diag::err_omp_declare_variant_item_expected)
+          << "vendor identifier" << "vendor" << "implementation";
+    }
+    // Parse ')'.
+    (void)T.consumeClose();
+    if (!VendorName.empty())
+      Data.ImplVendor = VendorName;
+    break;
+  }
+  case OMPDeclareVariantAttr::CtxUnknown:
+    P.Diag(Tok.getLocation(), diag::warn_omp_declare_variant_cs_name_expected)
+        << "implementation";
+    // Skip until either '}', ')', or end of directive.
+    while (!P.SkipUntil(tok::r_brace, tok::r_paren,
+                        tok::annot_pragma_openmp_end, Parser::StopBeforeMatch))
+      ;
+    return;
+  }
+  Data.CtxSet = OMPDeclareVariantAttr::CtxSetImplementation;
+  Data.Ctx = CSKind;
+}
+
+/// Parses clauses for 'declare variant' directive.
+/// clause:
+/// <selector_set_name> '=' '{' <context_selectors> '}'
+/// [ ',' <selector_set_name> '=' '{' <context_selectors> '}' ]
+bool Parser::parseOpenMPContextSelectors(
+    SourceLocation Loc,
+    llvm::function_ref<void(SourceRange,
+                            const Sema::OpenMPDeclareVariantCtsSelectorData &)>
+        Callback) {
+  do {
+    // Parse inner context selector set name.
+    if (!Tok.is(tok::identifier)) {
+      Diag(Tok.getLocation(), diag::err_omp_declare_variant_no_ctx_selector)
+          << getOpenMPClauseName(OMPC_match);
+      return true;
+    }
+    SmallString<16> Buffer;
+    StringRef CtxSelectorSetName = PP.getSpelling(Tok, Buffer);
+    // Parse '='.
+    (void)ConsumeToken();
+    if (Tok.isNot(tok::equal)) {
+      Diag(Tok.getLocation(), diag::err_omp_declare_variant_equal_expected)
+          << CtxSelectorSetName;
+      return true;
+    }
+    (void)ConsumeToken();
+    // TBD: add parsing of known context selectors.
+    // Unknown selector - just ignore it completely.
+    Sema::OpenMPDeclareVariantCtsSelectorData Data;
+    {
+      // Parse '{'.
+      BalancedDelimiterTracker TBr(*this, tok::l_brace,
+                                   tok::annot_pragma_openmp_end);
+      if (TBr.expectAndConsume(diag::err_expected_lbrace_after, "="))
+        return true;
+      OMPDeclareVariantAttr::CtxSelectorSetType CSSKind =
+          OMPDeclareVariantAttr::CtxSetUnknown;
+      (void)OMPDeclareVariantAttr::ConvertStrToCtxSelectorSetType(
+          CtxSelectorSetName, CSSKind);
+      switch (CSSKind) {
+      case OMPDeclareVariantAttr::CtxSetImplementation:
+        parseImplementationSelector(*this, Data);
+        break;
+      case OMPDeclareVariantAttr::CtxSetUnknown:
+        // Skip until either '}', ')', or end of directive.
+        while (!SkipUntil(tok::r_brace, tok::r_paren,
+                          tok::annot_pragma_openmp_end, StopBeforeMatch))
+          ;
+        break;
+      }
+      // Parse '}'.
+      (void)TBr.consumeClose();
+    }
+    Callback(SourceRange(Loc, Tok.getLocation()), Data);
+    // Consume ','
+    if (Tok.isNot(tok::r_paren) && Tok.isNot(tok::annot_pragma_openmp_end))
+      (void)ExpectAndConsume(tok::comma);
+  } while (Tok.isAnyIdentifier());
+  return false;
+}
+
+/// Parse clauses for '#pragma omp declare variant ( variant-func-id ) clause'.
+void Parser::ParseOMPDeclareVariantClauses(Parser::DeclGroupPtrTy Ptr,
+                                           CachedTokens &Toks,
+                                           SourceLocation Loc) {
+  PP.EnterToken(Tok, /*IsReinject*/ true);
+  PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+                      /*IsReinject*/ true);
+  // Consume the previously pushed token.
+  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
+  ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
+
+  FNContextRAII FnContext(*this, Ptr);
+  // Parse function declaration id.
+  SourceLocation RLoc;
+  // Parse with IsAddressOfOperand set to true to parse methods as DeclRefExprs
+  // instead of MemberExprs.
+  ExprResult AssociatedFunction =
+      ParseOpenMPParensExpr(getOpenMPDirectiveName(OMPD_declare_variant), RLoc,
+                            /*IsAddressOfOperand=*/true);
+  if (!AssociatedFunction.isUsable()) {
+    if (!Tok.is(tok::annot_pragma_openmp_end))
+      while (!SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch))
+        ;
+    // Skip the last annot_pragma_openmp_end.
+    (void)ConsumeAnnotationToken();
+    return;
+  }
+  Optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
+      Actions.checkOpenMPDeclareVariantFunction(
+          Ptr, AssociatedFunction.get(), SourceRange(Loc, Tok.getLocation()));
+
+  // Parse 'match'.
+  OpenMPClauseKind CKind = Tok.isAnnotation()
+                               ? OMPC_unknown
+                               : getOpenMPClauseKind(PP.getSpelling(Tok));
+  if (CKind != OMPC_match) {
+    Diag(Tok.getLocation(), diag::err_omp_declare_variant_wrong_clause)
+        << getOpenMPClauseName(OMPC_match);
+    while (!SkipUntil(tok::annot_pragma_openmp_end, Parser::StopBeforeMatch))
+      ;
+    // Skip the last annot_pragma_openmp_end.
+    (void)ConsumeAnnotationToken();
+    return;
+  }
+  (void)ConsumeToken();
+  // Parse '('.
+  BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
+  if (T.expectAndConsume(diag::err_expected_lparen_after,
+                         getOpenMPClauseName(OMPC_match))) {
+    while (!SkipUntil(tok::annot_pragma_openmp_end, StopBeforeMatch))
+      ;
+    // Skip the last annot_pragma_openmp_end.
+    (void)ConsumeAnnotationToken();
+    return;
+  }
+
+  // Parse inner context selectors.
+  if (!parseOpenMPContextSelectors(
+          Loc, [this, &DeclVarData](
+                   SourceRange SR,
+                   const Sema::OpenMPDeclareVariantCtsSelectorData &Data) {
+            if (DeclVarData.hasValue())
+              Actions.ActOnOpenMPDeclareVariantDirective(
+                  DeclVarData.getValue().first, DeclVarData.getValue().second,
+                  SR, Data);
+          })) {
+    // Parse ')'.
+    (void)T.consumeClose();
+    // Need to check for extra tokens.
+    if (Tok.isNot(tok::annot_pragma_openmp_end)) {
+      Diag(Tok, diag::warn_omp_extra_tokens_at_eol)
+          << getOpenMPDirectiveName(OMPD_declare_variant);
+    }
+  }
+
+  // Skip last tokens.
+  while (Tok.isNot(tok::annot_pragma_openmp_end))
+    ConsumeAnyToken();
+  // Skip the last annot_pragma_openmp_end.
+  (void)ConsumeAnnotationToken();
+}
+
 /// Parsing of simple OpenMP clauses like 'default' or 'proc_bind'.
 ///
 ///    default-clause:
@@ -1103,13 +1333,15 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
     }
     break;
   }
+  case OMPD_declare_variant:
   case OMPD_declare_simd: {
     // The syntax is:
-    // { #pragma omp declare simd }
+    // { #pragma omp declare {simd|variant} }
     // <function-declaration-or-definition>
     //
-    ConsumeToken();
     CachedTokens Toks;
+    Toks.push_back(Tok);
+    ConsumeToken();
     while(Tok.isNot(tok::annot_pragma_openmp_end)) {
       Toks.push_back(Tok);
       ConsumeAnyToken();
@@ -1133,10 +1365,16 @@ Parser::DeclGroupPtrTy Parser::ParseOpenMPDeclarativeDirectiveWithExtDecl(
       }
     }
     if (!Ptr) {
-      Diag(Loc, diag::err_omp_decl_in_declare_simd);
+      Diag(Loc, diag::err_omp_decl_in_declare_simd_variant)
+          << (DKind == OMPD_declare_simd ? 0 : 1);
       return DeclGroupPtrTy();
     }
-    return ParseOMPDeclareSimdClauses(Ptr, Toks, Loc);
+    if (DKind == OMPD_declare_simd)
+      return ParseOMPDeclareSimdClauses(Ptr, Toks, Loc);
+    assert(DKind == OMPD_declare_variant &&
+           "Expected declare variant directive only");
+    ParseOMPDeclareVariantClauses(Ptr, Toks, Loc);
+    return Ptr;
   }
   case OMPD_declare_target: {
     SourceLocation DTLoc = ConsumeAnyToken();
@@ -1572,6 +1810,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_requires:
+  case OMPD_declare_variant:
     Diag(Tok, diag::err_omp_unexpected_directive)
         << 1 << getOpenMPDirectiveName(DKind);
     SkipUntil(tok::annot_pragma_openmp_end);
@@ -1818,6 +2057,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     break;
   case OMPC_threadprivate:
   case OMPC_uniform:
+  case OMPC_match:
     if (!WrongDirective)
       Diag(Tok, diag::err_omp_unexpected_clause)
           << getOpenMPClauseName(CKind) << getOpenMPDirectiveName(DKind);
@@ -1831,14 +2071,15 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
 /// constructs.
 /// \param RLoc Returned location of right paren.
 ExprResult Parser::ParseOpenMPParensExpr(StringRef ClauseName,
-                                         SourceLocation &RLoc) {
+                                         SourceLocation &RLoc,
+                                         bool IsAddressOfOperand) {
   BalancedDelimiterTracker T(*this, tok::l_paren, tok::annot_pragma_openmp_end);
   if (T.expectAndConsume(diag::err_expected_lparen_after, ClauseName.data()))
     return ExprError();
 
   SourceLocation ELoc = Tok.getLocation();
   ExprResult LHS(ParseCastExpression(
-      /*isUnaryExpression=*/false, /*isAddressOfOperand=*/false, NotTypeCast));
+      /*isUnaryExpression=*/false, IsAddressOfOperand, NotTypeCast));
   ExprResult Val(ParseRHSOfBinaryExpression(LHS, prec::Conditional));
   Val = Actions.ActOnFinishFullExpr(Val.get(), ELoc, /*DiscardedValue*/ false);
 
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index 9bb5b6eac37e2..928bc5aa25b35 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -630,11 +630,11 @@ NamedDecl *Parser::ParseTypeParameter(unsigned Depth, unsigned Position) {
   }
 
   // Grab the template parameter name (if given)
-  SourceLocation NameLoc;
+  SourceLocation NameLoc = Tok.getLocation();
   IdentifierInfo *ParamName = nullptr;
   if (Tok.is(tok::identifier)) {
     ParamName = Tok.getIdentifierInfo();
-    NameLoc = ConsumeToken();
+    ConsumeToken();
   } else if (Tok.isOneOf(tok::equal, tok::comma, tok::greater,
                          tok::greatergreater)) {
     // Unnamed template parameter. Don't have to do anything here, just
@@ -727,11 +727,11 @@ Parser::ParseTemplateTemplateParameter(unsigned Depth, unsigned Position) {
            : diag::ext_variadic_templates);
 
   // Get the identifier, if given.
-  SourceLocation NameLoc;
+  SourceLocation NameLoc = Tok.getLocation();
   IdentifierInfo *ParamName = nullptr;
   if (Tok.is(tok::identifier)) {
     ParamName = Tok.getIdentifierInfo();
-    NameLoc = ConsumeToken();
+    ConsumeToken();
   } else if (Tok.isOneOf(tok::equal, tok::comma, tok::greater,
                          tok::greatergreater)) {
     // Unnamed template parameter. Don't have to do anything here, just
diff --git a/clang/lib/Sema/OpenCLBuiltins.td b/clang/lib/Sema/OpenCLBuiltins.td
index 5da67e339da46..2986140594675 100644
--- a/clang/lib/Sema/OpenCLBuiltins.td
+++ b/clang/lib/Sema/OpenCLBuiltins.td
@@ -20,12 +20,13 @@
 //===----------------------------------------------------------------------===//
 // Versions of OpenCL
 class Version<int _Version> {
-  int Version = _Version;
+  int ID = _Version;
 }
-def CL10: Version<100>;
-def CL11: Version<110>;
-def CL12: Version<120>;
-def CL20: Version<200>;
+def CLAll : Version<  0>;
+def CL10  : Version<100>;
+def CL11  : Version<110>;
+def CL12  : Version<120>;
+def CL20  : Version<200>;
 
 // Address spaces
 // Pointer types need to be assigned an address space.
@@ -61,10 +62,8 @@ class IntList<string _Name, list<int> _List> {
 //                      OpenCL C classes for types
 //===----------------------------------------------------------------------===//
 // OpenCL C basic data types (int, float, image2d_t, ...).
-// Its Child classes can represent concrete types (e.g.: VectorType) or
-// custom types (e.g.: GenType).
-// Instances of these child classes should be used in Builtin function
-// arguments.  See the definition of the "read_imagef" function as example.
+// Its child classes can represent concrete types (e.g. VectorType) or
+// abstract types (e.g. GenType).
 class Type<string _Name, QualType _QTName> {
   // Name of the Type.
   string Name = _Name;
@@ -150,12 +149,8 @@ class TypeList<string _Name, list<Type> _Type> {
 // A GenericType is an abstract type that defines a set of types as a
 // combination of Types and vector sizes.
 //
-// E.g.: If TypeList = <int, float> and VectorList = <1, 2, 4>, then it
-//       represents <int, int2, int4, float, float2, float4>.
-// _Ty          : Name of the GenType.
-// _TypeList    : List of basic data Types.
-// _VectorList  : Sizes of the vector for each type of the _TypeList, 1 being a
-//                scalar.
+// For example, if TypeList = <int, float> and VectorList = <1, 2, 4>, then it
+// represents <int, int2, int4, float, float2, float4>.
 //
 // Some rules apply when using multiple GenericType arguments in a declaration:
 //   1. The number of vector sizes must be equal or 1 for all gentypes in a
@@ -197,8 +192,13 @@ class Builtin<string _Name, list<Type> _Signature> {
   list<Type> Signature = _Signature;
   // OpenCL Extension to which the function belongs (cl_khr_subgroups, ...)
   string Extension = "";
-  // OpenCL Version to which the function belongs (CL10, ...)
-  Version Version = CL10;
+  // Version of OpenCL from which the function is available (e.g.: CL10).
+  // MinVersion is inclusive.
+  Version MinVersion = CL10;
+  // Version of OpenCL from which the function is not supported anymore.
+  // MaxVersion is exclusive.
+  // CLAll makes the function available for all versions.
+  Version MaxVersion = CLAll;
 }
 
 //===----------------------------------------------------------------------===//
@@ -318,6 +318,137 @@ foreach RType = [Float, Double, Half, Char, UChar, Short,
   }
 }
 
+//--------------------------------------------------------------------
+// OpenCL v1.1 s6.11.1, v1.2 s6.12.1, v2.0 s6.13.1 - Work-item Functions
+// --- Table 7 ---
+def : Builtin<"get_work_dim", [UInt]>;
+foreach name = ["get_global_size", "get_global_id", "get_local_size",
+                "get_local_id", "get_num_groups", "get_group_id",
+                "get_global_offset"] in {
+  def : Builtin<name, [Size, UInt]>;
+}
+
+let MinVersion = CL20 in {
+  def : Builtin<"get_enqueued_local_size", [Size, UInt]>;
+  foreach name = ["get_global_linear_id", "get_local_linear_id"] in {
+    def : Builtin<name, [Size]>;
+  }
+}
+
+//--------------------------------------------------------------------
+// OpenCL v1.1 s6.11.7, v1.2 s6.12.7, v2.0 s6.13.7 - Vector Data Load and Store Functions
+// OpenCL Extension v1.1 s9.3.6 and s9.6.6, v1.2 s9.5.6, v2.0 s9.4.6, v2.0 s5.1.6 and 6.1.6 - Vector Data Load and Store Functions
+// --- Table 15 ---
+// Variants for OpenCL versions below 2.0, using pointers to the global, local
+// and private address spaces.
+let MaxVersion = CL20 in {
+  foreach AS = [GlobalAS, LocalAS, PrivateAS] in {
+    foreach VSize = [2, 3, 4, 8, 16] in {
+      foreach name = ["vload" # VSize] in {
+        def : Builtin<name, [VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, AS>]>;
+        def : Builtin<name, [VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, AS>]>;
+        def : Builtin<name, [VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, AS>]>;
+        def : Builtin<name, [VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, AS>]>;
+        def : Builtin<name, [VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, AS>]>;
+        def : Builtin<name, [VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, AS>]>;
+        def : Builtin<name, [VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, AS>]>;
+        def : Builtin<name, [VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, AS>]>;
+        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, AS>]>;
+        def : Builtin<name, [VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, AS>]>;
+        def : Builtin<name, [VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, AS>]>;
+      }
+      foreach name = ["vstore" # VSize] in {
+        def : Builtin<name, [Void, VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, AS>]>;
+        def : Builtin<name, [Void, VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, AS>]>;
+        def : Builtin<name, [Void, VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, AS>]>;
+        def : Builtin<name, [Void, VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, AS>]>;
+        def : Builtin<name, [Void, VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, AS>]>;
+        def : Builtin<name, [Void, VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, AS>]>;
+      }
+      foreach name = ["vloada_half" # VSize] in {
+        def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Half>, AS>]>;
+      }
+      foreach rnd = ["", "_rte", "_rtz", "_rtp", "_rtn"] in {
+        foreach name = ["vstorea_half" # VSize # rnd] in {
+          def : Builtin<name, [Void, VectorType<Float, VSize>, Size, PointerType<Half, AS>]>;
+          def : Builtin<name, [Void, VectorType<Double, VSize>, Size, PointerType<Half, AS>]>;
+        }
+      }
+    }
+  }
+}
+// Variants for OpenCL versions above 2.0, using pointers to the generic
+// address space.
+let MinVersion = CL20 in {
+  foreach VSize = [2, 3, 4, 8, 16] in {
+    foreach name = ["vload" # VSize] in {
+      def : Builtin<name, [VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, GenericAS>]>;
+      def : Builtin<name, [VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, GenericAS>]>;
+      def : Builtin<name, [VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, GenericAS>]>;
+      def : Builtin<name, [VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, GenericAS>]>;
+      def : Builtin<name, [VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, GenericAS>]>;
+      def : Builtin<name, [VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, GenericAS>]>;
+    }
+    foreach name = ["vstore" # VSize] in {
+      def : Builtin<name, [Void, VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, GenericAS>]>;
+      def : Builtin<name, [Void, VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, GenericAS>]>;
+    }
+    foreach name = ["vloada_half" # VSize] in {
+      def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Half>, GenericAS>]>;
+    }
+    foreach rnd = ["", "_rte", "_rtz", "_rtp", "_rtn"] in {
+      foreach name = ["vstorea_half" # VSize # rnd] in {
+        def : Builtin<name, [Void, VectorType<Float, VSize>, Size, PointerType<Half, GenericAS>]>;
+        def : Builtin<name, [Void, VectorType<Double, VSize>, Size, PointerType<Half, GenericAS>]>;
+      }
+    }
+  }
+}
+// Variants using pointers to the constant address space.
+foreach VSize = [2, 3, 4, 8, 16] in {
+  foreach name = ["vload" # VSize] in {
+    def : Builtin<name, [VectorType<Char, VSize>, Size, PointerType<ConstType<Char>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<UChar, VSize>, Size, PointerType<ConstType<UChar>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Short, VSize>, Size, PointerType<ConstType<Short>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<UShort, VSize>, Size, PointerType<ConstType<UShort>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Int, VSize>, Size, PointerType<ConstType<Int>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<UInt, VSize>, Size, PointerType<ConstType<UInt>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Long, VSize>, Size, PointerType<ConstType<Long>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<ULong, VSize>, Size, PointerType<ConstType<ULong>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Float>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Double, VSize>, Size, PointerType<ConstType<Double>, ConstantAS>]>;
+    def : Builtin<name, [VectorType<Half, VSize>, Size, PointerType<ConstType<Half>, ConstantAS>]>;
+  }
+  foreach name = ["vloada_half" # VSize] in {
+    def : Builtin<name, [VectorType<Float, VSize>, Size, PointerType<ConstType<Half>, ConstantAS>]>;
+  }
+  foreach rnd = ["", "_rte", "_rtz", "_rtp", "_rtn"] in {
+    foreach name = ["vstorea_half" # VSize # rnd] in {
+      def : Builtin<name, [Void, VectorType<Float, VSize>, Size, PointerType<Half, ConstantAS>]>;
+      def : Builtin<name, [Void, VectorType<Double, VSize>, Size, PointerType<Half, ConstantAS>]>;
+    }
+  }
+}
+
 //--------------------------------------------------------------------
 // OpenCL v1.1 s6.11.10, v1.2 s6.12.10, v2.0 s6.13.10: Async Copies from Global to Local Memory, Local to Global Memory, and Prefetch
 // OpenCL Extension v2.0 s5.1.7 and s6.1.7: Async Copies from Global to Local Memory, Local to Global Memory, and Prefetch
@@ -356,14 +487,6 @@ foreach Type = [Int, UInt] in {
   }
 }
 
-// OpenCL v1.2 s6.12.1: Work-Item Functions
-def get_work_dim : Builtin<"get_work_dim", [UInt]>;
-foreach name = ["get_global_size", "get_global_id", "get_local_size",
-                "get_local_id", "get_num_groups", "get_group_id",
-                "get_global_offset"] in {
-  def : Builtin<name, [Size, UInt]>;
-}
-
 // OpenCL v1.2 s6.12.2: Math Functions
 foreach name = ["acos", "acosh", "acospi",
                 "asin", "asinh", "asinpi",
@@ -470,6 +593,31 @@ foreach aQual = ["WO", "RW"] in {
   def : Builtin<"write_imagef", [Void, ImageType<Image2dArrayDepth, aQual>, VectorType<Int, 4>, Float]>;
 }
 
+// --- Table 25: Image Query Functions ---
+foreach aQual = ["RO", "WO", "RW"] in {
+  foreach imgTy = [Image1d, Image1dBuffer, Image2d, Image3d,
+                   Image1dArray, Image2dArray, Image2dDepth,
+                   Image2dArrayDepth] in {
+    foreach name = ["get_image_width", "get_image_channel_data_type",
+                    "get_image_channel_order"] in {
+      def : Builtin<name, [Int, ImageType<imgTy, aQual>]>;
+    }
+  }
+  foreach imgTy = [Image2d, Image3d, Image2dArray, Image2dDepth,
+                   Image2dArrayDepth] in {
+    def : Builtin<"get_image_height", [Int, ImageType<imgTy, aQual>]>;
+  }
+  def : Builtin<"get_image_depth", [Int, ImageType<Image3d, aQual>]>;
+  foreach imgTy = [Image2d, Image2dArray, Image2dDepth,
+                   Image2dArrayDepth] in {
+    def : Builtin<"get_image_dim", [VectorType<Int, 2>, ImageType<imgTy, aQual>]>;
+  }
+  def : Builtin<"get_image_dim", [VectorType<Int, 4>, ImageType<Image3d, aQual>]>;
+  foreach imgTy = [Image1dArray, Image2dArray, Image2dArrayDepth] in {
+    def : Builtin<"get_image_array_size", [Size, ImageType<imgTy, aQual>]>;
+  }
+}
+
 // OpenCL extension v2.0 s5.1.9: Built-in Image Read Functions
 // --- Table 8 ---
 foreach aQual = ["RO"] in {
@@ -517,7 +665,7 @@ foreach aQual = ["WO", "RW"] in {
 
 
 // OpenCL v2.0 s9.17.3: Additions to section 6.13.1: Work-Item Functions
-let Version = CL20 in {
+let MinVersion = CL20 in {
   let Extension = "cl_khr_subgroups" in {
     def get_sub_group_size : Builtin<"get_sub_group_size", [UInt]>;
     def get_max_sub_group_size : Builtin<"get_max_sub_group_size", [UInt]>;
diff --git a/clang/lib/Sema/SemaCUDA.cpp b/clang/lib/Sema/SemaCUDA.cpp
index 203c09c57112b..3c9c991c77feb 100644
--- a/clang/lib/Sema/SemaCUDA.cpp
+++ b/clang/lib/Sema/SemaCUDA.cpp
@@ -267,6 +267,18 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
                                                    CXXMethodDecl *MemberDecl,
                                                    bool ConstRHS,
                                                    bool Diagnose) {
+  // If the defaulted special member is defined lexically outside of its
+  // owning class, or the special member already has explicit device or host
+  // attributes, do not infer.
+  bool InClass = MemberDecl->getLexicalParent() == MemberDecl->getParent();
+  bool HasH = MemberDecl->hasAttr<CUDAHostAttr>();
+  bool HasD = MemberDecl->hasAttr<CUDADeviceAttr>();
+  bool HasExplicitAttr =
+      (HasD && !MemberDecl->getAttr<CUDADeviceAttr>()->isImplicit()) ||
+      (HasH && !MemberDecl->getAttr<CUDAHostAttr>()->isImplicit());
+  if (!InClass || HasExplicitAttr)
+    return false;
+
   llvm::Optional<CUDAFunctionTarget> InferredTarget;
 
   // We're going to invoke special member lookup; mark that these special
@@ -371,21 +383,23 @@ bool Sema::inferCUDATargetForImplicitSpecialMember(CXXRecordDecl *ClassDecl,
     }
   }
 
+
+  // If no target was inferred, mark this member as __host__ __device__;
+  // it's the least restrictive option that can be invoked from any target.
+  bool NeedsH = true, NeedsD = true;
   if (InferredTarget.hasValue()) {
-    if (InferredTarget.getValue() == CFT_Device) {
-      MemberDecl->addAttr(CUDADeviceAttr::CreateImplicit(Context));
-    } else if (InferredTarget.getValue() == CFT_Host) {
-      MemberDecl->addAttr(CUDAHostAttr::CreateImplicit(Context));
-    } else {
-      MemberDecl->addAttr(CUDADeviceAttr::CreateImplicit(Context));
-      MemberDecl->addAttr(CUDAHostAttr::CreateImplicit(Context));
-    }
-  } else {
-    // If no target was inferred, mark this member as __host__ __device__;
-    // it's the least restrictive option that can be invoked from any target.
+    if (InferredTarget.getValue() == CFT_Device)
+      NeedsH = false;
+    else if (InferredTarget.getValue() == CFT_Host)
+      NeedsD = false;
+  }
+
+  // We either setting attributes first time, or the inferred ones must match
+  // previously set ones.
+  if (NeedsD && !HasD)
     MemberDecl->addAttr(CUDADeviceAttr::CreateImplicit(Context));
+  if (NeedsH && !HasH)
     MemberDecl->addAttr(CUDAHostAttr::CreateImplicit(Context));
-  }
 
   return false;
 }
@@ -806,7 +820,8 @@ void Sema::inheritCUDATargetAttrs(FunctionDecl *FD,
 
 std::string Sema::getCudaConfigureFuncName() const {
   if (getLangOpts().HIP)
-    return "hipConfigureCall";
+    return getLangOpts().HIPUseNewLaunchAPI ? "__hipPushCallConfiguration"
+                                            : "hipConfigureCall";
 
   // New CUDA kernel launch sequence.
   if (CudaFeatureEnabled(Context.getTargetInfo().getSDKVersion(),
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index f3b617d3520ba..cbf04f7313c27 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -191,7 +191,7 @@ static bool SemaBuiltinAddressof(Sema &S, CallExpr *TheCall) {
   return false;
 }
 
-/// Check the number of arguments, and set the result type to
+/// Check the number of arguments and set the result type to
 /// the argument type.
 static bool SemaBuiltinPreserveAI(Sema &S, CallExpr *TheCall) {
   if (checkArgCount(S, TheCall, 1))
@@ -4535,7 +4535,16 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
                                          AtomicExpr::AtomicOp Op) {
   CallExpr *TheCall = cast<CallExpr>(TheCallResult.get());
   DeclRefExpr *DRE =cast<DeclRefExpr>(TheCall->getCallee()->IgnoreParenCasts());
+  MultiExprArg Args{TheCall->getArgs(), TheCall->getNumArgs()};
+  return BuildAtomicExpr({TheCall->getBeginLoc(), TheCall->getEndLoc()},
+                         DRE->getSourceRange(), TheCall->getRParenLoc(), Args,
+                         Op);
+}
 
+ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange,
+                                 SourceLocation RParenLoc, MultiExprArg Args,
+                                 AtomicExpr::AtomicOp Op,
+                                 AtomicArgumentOrder ArgOrder) {
   // All the non-OpenCL operations take one of the following forms.
   // The OpenCL operations take the __c11 forms with one extra argument for
   // synchronization scope.
@@ -4682,21 +4691,21 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   if (IsOpenCL && Op != AtomicExpr::AO__opencl_atomic_init)
     ++AdjustedNumArgs;
   // Check we have the right number of arguments.
-  if (TheCall->getNumArgs() < AdjustedNumArgs) {
-    Diag(TheCall->getEndLoc(), diag::err_typecheck_call_too_few_args)
-        << 0 << AdjustedNumArgs << TheCall->getNumArgs()
-        << TheCall->getCallee()->getSourceRange();
+  if (Args.size() < AdjustedNumArgs) {
+    Diag(CallRange.getEnd(), diag::err_typecheck_call_too_few_args)
+        << 0 << AdjustedNumArgs << static_cast<unsigned>(Args.size())
+        << ExprRange;
     return ExprError();
-  } else if (TheCall->getNumArgs() > AdjustedNumArgs) {
-    Diag(TheCall->getArg(AdjustedNumArgs)->getBeginLoc(),
+  } else if (Args.size() > AdjustedNumArgs) {
+    Diag(Args[AdjustedNumArgs]->getBeginLoc(),
          diag::err_typecheck_call_too_many_args)
-        << 0 << AdjustedNumArgs << TheCall->getNumArgs()
-        << TheCall->getCallee()->getSourceRange();
+        << 0 << AdjustedNumArgs << static_cast<unsigned>(Args.size())
+        << ExprRange;
     return ExprError();
   }
 
   // Inspect the first argument of the atomic operation.
-  Expr *Ptr = TheCall->getArg(0);
+  Expr *Ptr = Args[0];
   ExprResult ConvertedPtr = DefaultFunctionArrayLvalueConversion(Ptr);
   if (ConvertedPtr.isInvalid())
     return ExprError();
@@ -4704,7 +4713,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   Ptr = ConvertedPtr.get();
   const PointerType *pointerType = Ptr->getType()->getAs<PointerType>();
   if (!pointerType) {
-    Diag(DRE->getBeginLoc(), diag::err_atomic_builtin_must_be_pointer)
+    Diag(ExprRange.getBegin(), diag::err_atomic_builtin_must_be_pointer)
         << Ptr->getType() << Ptr->getSourceRange();
     return ExprError();
   }
@@ -4714,13 +4723,13 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   QualType ValType = AtomTy; // 'C'
   if (IsC11) {
     if (!AtomTy->isAtomicType()) {
-      Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_atomic)
+      Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_atomic)
           << Ptr->getType() << Ptr->getSourceRange();
       return ExprError();
     }
     if ((Form != Load && Form != LoadCopy && AtomTy.isConstQualified()) ||
         AtomTy.getAddressSpace() == LangAS::opencl_constant) {
-      Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_non_const_atomic)
+      Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_non_const_atomic)
           << (AtomTy.isConstQualified() ? 0 : 1) << Ptr->getType()
           << Ptr->getSourceRange();
       return ExprError();
@@ -4728,7 +4737,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
     ValType = AtomTy->getAs<AtomicType>()->getValueType();
   } else if (Form != Load && Form != LoadCopy) {
     if (ValType.isConstQualified()) {
-      Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_non_const_pointer)
+      Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_non_const_pointer)
           << Ptr->getType() << Ptr->getSourceRange();
       return ExprError();
     }
@@ -4739,7 +4748,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
     // gcc does not enforce these rules for GNU atomics, but we do so for sanity.
     if (IsAddSub && !ValType->isIntegerType()
         && !ValType->isPointerType()) {
-      Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_atomic_int_or_ptr)
+      Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_atomic_int_or_ptr)
           << IsC11 << Ptr->getType() << Ptr->getSourceRange();
       return ExprError();
     }
@@ -4747,12 +4756,12 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
       const BuiltinType *BT = ValType->getAs<BuiltinType>();
       if (!BT || (BT->getKind() != BuiltinType::Int &&
                   BT->getKind() != BuiltinType::UInt)) {
-        Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_int32_or_ptr);
+        Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_int32_or_ptr);
         return ExprError();
       }
     }
     if (!IsAddSub && !IsMinMax && !ValType->isIntegerType()) {
-      Diag(DRE->getBeginLoc(), diag::err_atomic_op_bitwise_needs_atomic_int)
+      Diag(ExprRange.getBegin(), diag::err_atomic_op_bitwise_needs_atomic_int)
           << IsC11 << Ptr->getType() << Ptr->getSourceRange();
       return ExprError();
     }
@@ -4764,7 +4773,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   } else if (IsN && !ValType->isIntegerType() && !ValType->isPointerType()) {
     // For __atomic_*_n operations, the value type must be a scalar integral or
     // pointer type which is 1, 2, 4, 8 or 16 bytes in length.
-    Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_atomic_int_or_ptr)
+    Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_atomic_int_or_ptr)
         << IsC11 << Ptr->getType() << Ptr->getSourceRange();
     return ExprError();
   }
@@ -4773,7 +4782,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
       !AtomTy->isScalarType()) {
     // For GNU atomics, require a trivially-copyable type. This is not part of
     // the GNU atomics specification, but we enforce it for sanity.
-    Diag(DRE->getBeginLoc(), diag::err_atomic_op_needs_trivial_copy)
+    Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_trivial_copy)
         << Ptr->getType() << Ptr->getSourceRange();
     return ExprError();
   }
@@ -4789,7 +4798,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   case Qualifiers::OCL_Autoreleasing:
     // FIXME: Can this happen? By this point, ValType should be known
     // to be trivially copyable.
-    Diag(DRE->getBeginLoc(), diag::err_arc_atomic_ownership)
+    Diag(ExprRange.getBegin(), diag::err_arc_atomic_ownership)
         << ValType << Ptr->getSourceRange();
     return ExprError();
   }
@@ -4816,19 +4825,56 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
     IsPassedByAddress = true;
   }
 
+  SmallVector<Expr *, 5> APIOrderedArgs;
+  if (ArgOrder == Sema::AtomicArgumentOrder::AST) {
+    APIOrderedArgs.push_back(Args[0]);
+    switch (Form) {
+    case Init:
+    case Load:
+      APIOrderedArgs.push_back(Args[1]); // Val1/Order
+      break;
+    case LoadCopy:
+    case Copy:
+    case Arithmetic:
+    case Xchg:
+      APIOrderedArgs.push_back(Args[2]); // Val1
+      APIOrderedArgs.push_back(Args[1]); // Order
+      break;
+    case GNUXchg:
+      APIOrderedArgs.push_back(Args[2]); // Val1
+      APIOrderedArgs.push_back(Args[3]); // Val2
+      APIOrderedArgs.push_back(Args[1]); // Order
+      break;
+    case C11CmpXchg:
+      APIOrderedArgs.push_back(Args[2]); // Val1
+      APIOrderedArgs.push_back(Args[4]); // Val2
+      APIOrderedArgs.push_back(Args[1]); // Order
+      APIOrderedArgs.push_back(Args[3]); // OrderFail
+      break;
+    case GNUCmpXchg:
+      APIOrderedArgs.push_back(Args[2]); // Val1
+      APIOrderedArgs.push_back(Args[4]); // Val2
+      APIOrderedArgs.push_back(Args[5]); // Weak
+      APIOrderedArgs.push_back(Args[1]); // Order
+      APIOrderedArgs.push_back(Args[3]); // OrderFail
+      break;
+    }
+  } else
+    APIOrderedArgs.append(Args.begin(), Args.end());
+
   // The first argument's non-CV pointer type is used to deduce the type of
   // subsequent arguments, except for:
   //  - weak flag (always converted to bool)
   //  - memory order (always converted to int)
   //  - scope  (always converted to int)
-  for (unsigned i = 0; i != TheCall->getNumArgs(); ++i) {
+  for (unsigned i = 0; i != APIOrderedArgs.size(); ++i) {
     QualType Ty;
     if (i < NumVals[Form] + 1) {
       switch (i) {
       case 0:
         // The first argument is always a pointer. It has a fixed type.
         // It is always dereferenced, a nullptr is undefined.
-        CheckNonNullArgument(*this, TheCall->getArg(i), DRE->getBeginLoc());
+        CheckNonNullArgument(*this, APIOrderedArgs[i], ExprRange.getBegin());
         // Nothing else to do: we already know all we want about this pointer.
         continue;
       case 1:
@@ -4840,16 +4886,18 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
         if (Form == Init || (Form == Arithmetic && ValType->isIntegerType()))
           Ty = ValType;
         else if (Form == Copy || Form == Xchg) {
-          if (IsPassedByAddress)
+          if (IsPassedByAddress) {
             // The value pointer is always dereferenced, a nullptr is undefined.
-            CheckNonNullArgument(*this, TheCall->getArg(i), DRE->getBeginLoc());
+            CheckNonNullArgument(*this, APIOrderedArgs[i],
+                                 ExprRange.getBegin());
+          }
           Ty = ByValType;
         } else if (Form == Arithmetic)
           Ty = Context.getPointerDiffType();
         else {
-          Expr *ValArg = TheCall->getArg(i);
+          Expr *ValArg = APIOrderedArgs[i];
           // The value pointer is always dereferenced, a nullptr is undefined.
-          CheckNonNullArgument(*this, ValArg, DRE->getBeginLoc());
+          CheckNonNullArgument(*this, ValArg, ExprRange.getBegin());
           LangAS AS = LangAS::Default;
           // Keep address space of non-atomic pointer type.
           if (const PointerType *PtrTy =
@@ -4864,7 +4912,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
         // The third argument to compare_exchange / GNU exchange is the desired
         // value, either by-value (for the C11 and *_n variant) or as a pointer.
         if (IsPassedByAddress)
-          CheckNonNullArgument(*this, TheCall->getArg(i), DRE->getBeginLoc());
+          CheckNonNullArgument(*this, APIOrderedArgs[i], ExprRange.getBegin());
         Ty = ByValType;
         break;
       case 3:
@@ -4879,11 +4927,11 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
 
     InitializedEntity Entity =
         InitializedEntity::InitializeParameter(Context, Ty, false);
-    ExprResult Arg = TheCall->getArg(i);
+    ExprResult Arg = APIOrderedArgs[i];
     Arg = PerformCopyInitialization(Entity, SourceLocation(), Arg);
     if (Arg.isInvalid())
       return true;
-    TheCall->setArg(i, Arg.get());
+    APIOrderedArgs[i] = Arg.get();
   }
 
   // Permute the arguments into a 'consistent' order.
@@ -4892,36 +4940,36 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   switch (Form) {
   case Init:
     // Note, AtomicExpr::getVal1() has a special case for this atomic.
-    SubExprs.push_back(TheCall->getArg(1)); // Val1
+    SubExprs.push_back(APIOrderedArgs[1]); // Val1
     break;
   case Load:
-    SubExprs.push_back(TheCall->getArg(1)); // Order
+    SubExprs.push_back(APIOrderedArgs[1]); // Order
     break;
   case LoadCopy:
   case Copy:
   case Arithmetic:
   case Xchg:
-    SubExprs.push_back(TheCall->getArg(2)); // Order
-    SubExprs.push_back(TheCall->getArg(1)); // Val1
+    SubExprs.push_back(APIOrderedArgs[2]); // Order
+    SubExprs.push_back(APIOrderedArgs[1]); // Val1
     break;
   case GNUXchg:
     // Note, AtomicExpr::getVal2() has a special case for this atomic.
-    SubExprs.push_back(TheCall->getArg(3)); // Order
-    SubExprs.push_back(TheCall->getArg(1)); // Val1
-    SubExprs.push_back(TheCall->getArg(2)); // Val2
+    SubExprs.push_back(APIOrderedArgs[3]); // Order
+    SubExprs.push_back(APIOrderedArgs[1]); // Val1
+    SubExprs.push_back(APIOrderedArgs[2]); // Val2
     break;
   case C11CmpXchg:
-    SubExprs.push_back(TheCall->getArg(3)); // Order
-    SubExprs.push_back(TheCall->getArg(1)); // Val1
-    SubExprs.push_back(TheCall->getArg(4)); // OrderFail
-    SubExprs.push_back(TheCall->getArg(2)); // Val2
+    SubExprs.push_back(APIOrderedArgs[3]); // Order
+    SubExprs.push_back(APIOrderedArgs[1]); // Val1
+    SubExprs.push_back(APIOrderedArgs[4]); // OrderFail
+    SubExprs.push_back(APIOrderedArgs[2]); // Val2
     break;
   case GNUCmpXchg:
-    SubExprs.push_back(TheCall->getArg(4)); // Order
-    SubExprs.push_back(TheCall->getArg(1)); // Val1
-    SubExprs.push_back(TheCall->getArg(5)); // OrderFail
-    SubExprs.push_back(TheCall->getArg(2)); // Val2
-    SubExprs.push_back(TheCall->getArg(3)); // Weak
+    SubExprs.push_back(APIOrderedArgs[4]); // Order
+    SubExprs.push_back(APIOrderedArgs[1]); // Val1
+    SubExprs.push_back(APIOrderedArgs[5]); // OrderFail
+    SubExprs.push_back(APIOrderedArgs[2]); // Val2
+    SubExprs.push_back(APIOrderedArgs[3]); // Weak
     break;
   }
 
@@ -4935,7 +4983,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   }
 
   if (auto ScopeModel = AtomicExpr::getScopeModel(Op)) {
-    auto *Scope = TheCall->getArg(TheCall->getNumArgs() - 1);
+    auto *Scope = Args[Args.size() - 1];
     llvm::APSInt Result(32);
     if (Scope->isIntegerConstantExpr(Result, Context) &&
         !ScopeModel->isValid(Result.getZExtValue())) {
@@ -4945,9 +4993,8 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
     SubExprs.push_back(Scope);
   }
 
-  AtomicExpr *AE =
-      new (Context) AtomicExpr(TheCall->getCallee()->getBeginLoc(), SubExprs,
-                               ResultType, Op, TheCall->getRParenLoc());
+  AtomicExpr *AE = new (Context)
+      AtomicExpr(ExprRange.getBegin(), SubExprs, ResultType, Op, RParenLoc);
 
   if ((Op == AtomicExpr::AO__c11_atomic_load ||
        Op == AtomicExpr::AO__c11_atomic_store ||
@@ -5866,7 +5913,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
           << SourceRange(TheCall->getArg(0)->getBeginLoc(),
                          TheCall->getArg(1)->getEndLoc()));
 
-    numElements = LHSType->getAs<VectorType>()->getNumElements();
+    numElements = LHSType->castAs<VectorType>()->getNumElements();
     unsigned numResElements = TheCall->getNumArgs() - 2;
 
     // Check to see if we have a call with 2 vector arguments, the unary shuffle
@@ -5874,7 +5921,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
     // same number of elts as lhs.
     if (TheCall->getNumArgs() == 2) {
       if (!RHSType->hasIntegerRepresentation() ||
-          RHSType->getAs<VectorType>()->getNumElements() != numElements)
+          RHSType->castAs<VectorType>()->getNumElements() != numElements)
         return ExprError(Diag(TheCall->getBeginLoc(),
                               diag::err_vec_builtin_incompatible_vector)
                          << TheCall->getDirectCallee()
@@ -5887,7 +5934,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
                        << SourceRange(TheCall->getArg(0)->getBeginLoc(),
                                       TheCall->getArg(1)->getEndLoc()));
     } else if (numElements != numResElements) {
-      QualType eltType = LHSType->getAs<VectorType>()->getElementType();
+      QualType eltType = LHSType->castAs<VectorType>()->getElementType();
       resType = Context.getVectorType(eltType, numResElements,
                                       VectorType::GenericVector);
     }
@@ -5944,8 +5991,8 @@ ExprResult Sema::SemaConvertVectorExpr(Expr *E, TypeSourceInfo *TInfo,
                           diag::err_convertvector_non_vector_type));
 
   if (!SrcTy->isDependentType() && !DstTy->isDependentType()) {
-    unsigned SrcElts = SrcTy->getAs<VectorType>()->getNumElements();
-    unsigned DstElts = DstTy->getAs<VectorType>()->getNumElements();
+    unsigned SrcElts = SrcTy->castAs<VectorType>()->getNumElements();
+    unsigned DstElts = DstTy->castAs<VectorType>()->getNumElements();
     if (SrcElts != DstElts)
       return ExprError(Diag(BuiltinLoc,
                             diag::err_convertvector_incompatible_vector)
@@ -8179,6 +8226,22 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
     ExprTy = TET->getUnderlyingExpr()->getType();
   }
 
+  // Diagnose attempts to print a boolean value as a character. Unlike other
+  // -Wformat diagnostics, this is fine from a type perspective, but it still
+  // doesn't make sense.
+  if (FS.getConversionSpecifier().getKind() == ConversionSpecifier::cArg &&
+      E->isKnownToHaveBooleanValue()) {
+    const CharSourceRange &CSR =
+        getSpecifierRange(StartSpecifier, SpecifierLen);
+    SmallString<4> FSString;
+    llvm::raw_svector_ostream os(FSString);
+    FS.toString(os);
+    EmitFormatDiagnostic(S.PDiag(diag::warn_format_bool_as_character)
+                             << FSString,
+                         E->getExprLoc(), false, CSR);
+    return true;
+  }
+
   const analyze_printf::ArgType::MatchKind Match =
       AT.matchesType(S.Context, ExprTy);
   bool Pedantic = Match == analyze_printf::ArgType::NoMatchPedantic;
@@ -10910,6 +10973,26 @@ static void DiagnoseImpCast(Sema &S, Expr *E, QualType T,
   DiagnoseImpCast(S, E, E->getType(), T, CContext, diag, pruneControlFlow);
 }
 
+static bool isObjCSignedCharBool(Sema &S, QualType Ty) {
+  return Ty->isSpecificBuiltinType(BuiltinType::SChar) &&
+      S.getLangOpts().ObjC && S.NSAPIObj->isObjCBOOLType(Ty);
+}
+
+static void adornObjCBoolConversionDiagWithTernaryFixit(
+    Sema &S, Expr *SourceExpr, const Sema::SemaDiagnosticBuilder &Builder) {
+  Expr *Ignored = SourceExpr->IgnoreImplicit();
+  if (const auto *OVE = dyn_cast<OpaqueValueExpr>(Ignored))
+    Ignored = OVE->getSourceExpr();
+  bool NeedsParens = isa<AbstractConditionalOperator>(Ignored) ||
+                     isa<BinaryOperator>(Ignored) ||
+                     isa<CXXOperatorCallExpr>(Ignored);
+  SourceLocation EndLoc = S.getLocForEndOfToken(SourceExpr->getEndLoc());
+  if (NeedsParens)
+    Builder << FixItHint::CreateInsertion(SourceExpr->getBeginLoc(), "(")
+            << FixItHint::CreateInsertion(EndLoc, ")");
+  Builder << FixItHint::CreateInsertion(EndLoc, " ? YES : NO");
+}
+
 /// Diagnose an implicit cast from a floating point value to an integer value.
 static void DiagnoseFloatingImpCast(Sema &S, Expr *E, QualType T,
                                     SourceLocation CContext) {
@@ -10929,6 +11012,13 @@ static void DiagnoseFloatingImpCast(Sema &S, Expr *E, QualType T,
   bool IsConstant =
     E->EvaluateAsFloat(Value, S.Context, Expr::SE_AllowSideEffects);
   if (!IsConstant) {
+    if (isObjCSignedCharBool(S, T)) {
+      return adornObjCBoolConversionDiagWithTernaryFixit(
+          S, E,
+          S.Diag(CContext, diag::warn_impcast_float_to_objc_signed_char_bool)
+              << E->getType());
+    }
+
     return DiagnoseImpCast(S, E, T, CContext,
                            diag::warn_impcast_float_integer, PruneWarnings);
   }
@@ -10940,6 +11030,23 @@ static void DiagnoseFloatingImpCast(Sema &S, Expr *E, QualType T,
   llvm::APFloat::opStatus Result = Value.convertToInteger(
       IntegerValue, llvm::APFloat::rmTowardZero, &isExact);
 
+  // FIXME: Force the precision of the source value down so we don't print
+  // digits which are usually useless (we don't really care here if we
+  // truncate a digit by accident in edge cases).  Ideally, APFloat::toString
+  // would automatically print the shortest representation, but it's a bit
+  // tricky to implement.
+  SmallString<16> PrettySourceValue;
+  unsigned precision = llvm::APFloat::semanticsPrecision(Value.getSemantics());
+  precision = (precision * 59 + 195) / 196;
+  Value.toString(PrettySourceValue, precision);
+
+  if (isObjCSignedCharBool(S, T) && IntegerValue != 0 && IntegerValue != 1) {
+    return adornObjCBoolConversionDiagWithTernaryFixit(
+        S, E,
+        S.Diag(CContext, diag::warn_impcast_constant_value_to_objc_bool)
+            << PrettySourceValue);
+  }
+
   if (Result == llvm::APFloat::opOK && isExact) {
     if (IsLiteral) return;
     return DiagnoseImpCast(S, E, T, CContext, diag::warn_impcast_float_integer,
@@ -10983,16 +11090,6 @@ static void DiagnoseFloatingImpCast(Sema &S, Expr *E, QualType T,
     DiagID = diag::warn_impcast_float_to_integer;
   }
 
-  // FIXME: Force the precision of the source value down so we don't print
-  // digits which are usually useless (we don't really care here if we
-  // truncate a digit by accident in edge cases).  Ideally, APFloat::toString
-  // would automatically print the shortest representation, but it's a bit
-  // tricky to implement.
-  SmallString<16> PrettySourceValue;
-  unsigned precision = llvm::APFloat::semanticsPrecision(Value.getSemantics());
-  precision = (precision * 59 + 195) / 196;
-  Value.toString(PrettySourceValue, precision);
-
   SmallString<16> PrettyTargetValue;
   if (IsBool)
     PrettyTargetValue = Value.isZero() ? "false" : "true";
@@ -11269,9 +11366,79 @@ static bool isSameWidthConstantConversion(Sema &S, Expr *E, QualType T,
   return true;
 }
 
-static bool isObjCSignedCharBool(Sema &S, QualType Ty) {
-  return Ty->isSpecificBuiltinType(BuiltinType::SChar) &&
-         S.getLangOpts().ObjC && S.NSAPIObj->isObjCBOOLType(Ty);
+static const IntegerLiteral *getIntegerLiteral(Expr *E) {
+  const auto *IL = dyn_cast<IntegerLiteral>(E);
+  if (!IL) {
+    if (auto *UO = dyn_cast<UnaryOperator>(E)) {
+      if (UO->getOpcode() == UO_Minus)
+        return dyn_cast<IntegerLiteral>(UO->getSubExpr());
+    }
+  }
+
+  return IL;
+}
+
+static void CheckConditionalWithEnumTypes(Sema &S, SourceLocation Loc,
+                                          Expr *LHS, Expr *RHS) {
+  QualType LHSStrippedType = LHS->IgnoreParenImpCasts()->getType();
+  QualType RHSStrippedType = RHS->IgnoreParenImpCasts()->getType();
+
+  const auto *LHSEnumType = LHSStrippedType->getAs<EnumType>();
+  if (!LHSEnumType)
+    return;
+  const auto *RHSEnumType = RHSStrippedType->getAs<EnumType>();
+  if (!RHSEnumType)
+    return;
+
+  // Ignore anonymous enums.
+  if (!LHSEnumType->getDecl()->hasNameForLinkage())
+    return;
+  if (!RHSEnumType->getDecl()->hasNameForLinkage())
+    return;
+
+  if (S.Context.hasSameUnqualifiedType(LHSStrippedType, RHSStrippedType))
+    return;
+
+  S.Diag(Loc, diag::warn_conditional_mixed_enum_types)
+      << LHSStrippedType << RHSStrippedType << LHS->getSourceRange()
+      << RHS->getSourceRange();
+}
+
+static void DiagnoseIntInBoolContext(Sema &S, Expr *E) {
+  E = E->IgnoreParenImpCasts();
+  SourceLocation ExprLoc = E->getExprLoc();
+
+  if (const auto *BO = dyn_cast<BinaryOperator>(E)) {
+    BinaryOperator::Opcode Opc = BO->getOpcode();
+    Expr::EvalResult Result;
+    // Do not diagnose unsigned shifts.
+    if (Opc == BO_Shl) {
+      const auto *LHS = getIntegerLiteral(BO->getLHS());
+      const auto *RHS = getIntegerLiteral(BO->getRHS());
+      if (LHS && LHS->getValue() == 0)
+        S.Diag(ExprLoc, diag::warn_left_shift_always) << 0;
+      else if (!E->isValueDependent() && LHS && RHS &&
+               RHS->getValue().isNonNegative() &&
+               E->EvaluateAsInt(Result, S.Context, Expr::SE_AllowSideEffects))
+        S.Diag(ExprLoc, diag::warn_left_shift_always)
+            << (Result.Val.getInt() != 0);
+      else if (E->getType()->isSignedIntegerType())
+        S.Diag(ExprLoc, diag::warn_left_shift_in_bool_context) << E;
+    }
+  }
+
+  if (const auto *CO = dyn_cast<ConditionalOperator>(E)) {
+    const auto *LHS = getIntegerLiteral(CO->getTrueExpr());
+    const auto *RHS = getIntegerLiteral(CO->getFalseExpr());
+    if (!LHS || !RHS)
+      return;
+    if ((LHS->getValue() == 0 || LHS->getValue() == 1) &&
+        (RHS->getValue() == 0 || RHS->getValue() == 1))
+      // Do not diagnose common idioms.
+      return;
+    if (LHS->getValue() != 0 && LHS->getValue() != 0)
+      S.Diag(ExprLoc, diag::warn_integer_constants_in_conditional_always_true);
+  }
 }
 
 static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
@@ -11324,19 +11491,13 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   if (isObjCSignedCharBool(S, T) && Source->isIntegralType(S.Context)) {
     Expr::EvalResult Result;
     if (E->EvaluateAsInt(Result, S.getASTContext(),
-                         Expr::SE_AllowSideEffects) &&
-        Result.Val.getInt() != 1 && Result.Val.getInt() != 0) {
-      auto Builder = S.Diag(CC, diag::warn_impcast_constant_int_to_objc_bool)
-                     << Result.Val.getInt().toString(10);
-      Expr *Ignored = E->IgnoreImplicit();
-      bool NeedsParens = isa<AbstractConditionalOperator>(Ignored) ||
-                         isa<BinaryOperator>(Ignored) ||
-                         isa<CXXOperatorCallExpr>(Ignored);
-      SourceLocation EndLoc = S.getLocForEndOfToken(E->getEndLoc());
-      if (NeedsParens)
-        Builder << FixItHint::CreateInsertion(E->getBeginLoc(), "(")
-                << FixItHint::CreateInsertion(EndLoc, ")");
-      Builder << FixItHint::CreateInsertion(EndLoc, " ? YES : NO");
+                         Expr::SE_AllowSideEffects)) {
+      if (Result.Val.getInt() != 1 && Result.Val.getInt() != 0) {
+        adornObjCBoolConversionDiagWithTernaryFixit(
+            S, E,
+            S.Diag(CC, diag::warn_impcast_constant_value_to_objc_bool)
+                << Result.Val.getInt().toString(10));
+      }
       return;
     }
   }
@@ -11571,6 +11732,9 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
 
   S.DiscardMisalignedMemberAddress(Target, E);
 
+  if (Target->isBooleanType())
+    DiagnoseIntInBoolContext(S, E);
+
   if (!Source->isIntegerType() || !Target->isIntegerType())
     return;
 
@@ -11579,6 +11743,14 @@ static void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
   if (Target->isSpecificBuiltinType(BuiltinType::Bool))
     return;
 
+  if (isObjCSignedCharBool(S, T) && !Source->isCharType() &&
+      !E->isKnownToHaveBooleanValue()) {
+    return adornObjCBoolConversionDiagWithTernaryFixit(
+        S, E,
+        S.Diag(CC, diag::warn_impcast_int_to_objc_signed_char_bool)
+            << E->getType());
+  }
+
   IntRange SourceRange = GetExprRange(S.Context, E, S.isConstantEvaluated());
   IntRange TargetRange = IntRange::forTargetOfCanonicalType(S.Context, Target);
 
@@ -11723,6 +11895,11 @@ static void CheckConditionalOperator(Sema &S, ConditionalOperator *E,
   bool Suspicious = false;
   CheckConditionalOperand(S, E->getTrueExpr(), T, CC, Suspicious);
   CheckConditionalOperand(S, E->getFalseExpr(), T, CC, Suspicious);
+  CheckConditionalWithEnumTypes(S, E->getBeginLoc(), E->getTrueExpr(),
+                                E->getFalseExpr());
+
+  if (T->isBooleanType())
+    DiagnoseIntInBoolContext(S, E);
 
   // If -Wconversion would have warned about either of the candidates
   // for a signedness conversion to the context type...
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index b868fb3645f87..4d4966afb0166 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2766,7 +2766,7 @@ void Sema::mergeDeclAttributes(NamedDecl *New, Decl *Old,
 
   if (AsmLabelAttr *NewA = New->getAttr<AsmLabelAttr>()) {
     if (AsmLabelAttr *OldA = Old->getAttr<AsmLabelAttr>()) {
-      if (OldA->getLabel() != NewA->getLabel()) {
+      if (!OldA->isEquivalent(NewA)) {
         // This redeclaration changes __asm__ label.
         Diag(New->getLocation(), diag::err_different_asm_label);
         Diag(OldA->getLocation(), diag::note_previous_declaration);
@@ -3562,7 +3562,12 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, NamedDecl *&OldD,
       }
     }
 
-    if (OldQTypeForComparison == NewQType)
+    // If the function types are compatible, merge the declarations. Ignore the
+    // exception specifier because it was already checked above in
+    // CheckEquivalentExceptionSpec, and we don't want follow-on diagnostics
+    // about incompatible types under -fms-compatibility.
+    if (Context.hasSameFunctionTypeIgnoringExceptionSpec(OldQTypeForComparison,
+                                                         NewQType))
       return MergeCompatibleFunctionDecls(New, Old, S, MergeTypeWithOld);
 
     // If the types are imprecise (due to dependent constructs in friends or
@@ -6986,8 +6991,8 @@ NamedDecl *Sema::ActOnVariableDeclarator(
       }
     }
 
-    NewVD->addAttr(::new (Context)
-                       AsmLabelAttr(Context, SE->getStrTokenLoc(0), Label));
+    NewVD->addAttr(::new (Context) AsmLabelAttr(
+        Context, SE->getStrTokenLoc(0), Label, /*IsLiteralLabel=*/true));
   } else if (!ExtnameUndeclaredIdentifiers.empty()) {
     llvm::DenseMap<IdentifierInfo*,AsmLabelAttr*>::iterator I =
       ExtnameUndeclaredIdentifiers.find(NewVD->getIdentifier());
@@ -8175,10 +8180,10 @@ static FunctionDecl *CreateNewFunctionDecl(Sema &SemaRef, Declarator &D,
     if (DC->isRecord()) {
       R = SemaRef.CheckDestructorDeclarator(D, R, SC);
       CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
-      CXXDestructorDecl *NewDD =
-          CXXDestructorDecl::Create(SemaRef.Context, Record, D.getBeginLoc(),
-                                    NameInfo, R, TInfo, isInline,
-                                    /*isImplicitlyDeclared=*/false);
+      CXXDestructorDecl *NewDD = CXXDestructorDecl::Create(
+          SemaRef.Context, Record, D.getBeginLoc(), NameInfo, R, TInfo,
+          isInline,
+          /*isImplicitlyDeclared=*/false, ConstexprKind);
 
       // If the destructor needs an implicit exception specification, set it
       // now. FIXME: It'd be nice to be able to create the right type to start
@@ -8787,7 +8792,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       // C++11 [dcl.constexpr]p3: functions declared constexpr are required to
       // be either constructors or to return a literal type. Therefore,
       // destructors cannot be declared constexpr.
-      if (isa<CXXDestructorDecl>(NewFD)) {
+      if (isa<CXXDestructorDecl>(NewFD) && !getLangOpts().CPlusPlus2a) {
         Diag(D.getDeclSpec().getConstexprSpecLoc(), diag::err_constexpr_dtor)
             << ConstexprKind;
       }
@@ -8885,8 +8890,9 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   if (Expr *E = (Expr*) D.getAsmLabel()) {
     // The parser guarantees this is a string.
     StringLiteral *SE = cast<StringLiteral>(E);
-    NewFD->addAttr(::new (Context) AsmLabelAttr(Context, SE->getStrTokenLoc(0),
-                                                SE->getString()));
+    NewFD->addAttr(::new (Context)
+                       AsmLabelAttr(Context, SE->getStrTokenLoc(0),
+                                    SE->getString(), /*IsLiteralLabel=*/true));
   } else if (!ExtnameUndeclaredIdentifiers.empty()) {
     llvm::DenseMap<IdentifierInfo*,AsmLabelAttr*>::iterator I =
       ExtnameUndeclaredIdentifiers.find(NewFD->getIdentifier());
@@ -9674,10 +9680,13 @@ static bool HasNonMultiVersionAttributes(const FunctionDecl *FD,
   return false;
 }
 
-static bool CheckMultiVersionAdditionalRules(Sema &S, const FunctionDecl *OldFD,
-                                             const FunctionDecl *NewFD,
-                                             bool CausesMV,
-                                             MultiVersionKind MVType) {
+bool Sema::areMultiversionVariantFunctionsCompatible(
+    const FunctionDecl *OldFD, const FunctionDecl *NewFD,
+    const PartialDiagnostic &NoProtoDiagID,
+    const PartialDiagnosticAt &NoteCausedDiagIDAt,
+    const PartialDiagnosticAt &NoSupportDiagIDAt,
+    const PartialDiagnosticAt &DiffDiagIDAt, bool TemplatesSupported,
+    bool ConstexprSupported) {
   enum DoesntSupport {
     FuncTemplates = 0,
     VirtFuncs = 1,
@@ -9695,123 +9704,85 @@ static bool CheckMultiVersionAdditionalRules(Sema &S, const FunctionDecl *OldFD,
     ConstexprSpec = 2,
     InlineSpec = 3,
     StorageClass = 4,
-    Linkage = 5
+    Linkage = 5,
   };
 
-  bool IsCPUSpecificCPUDispatchMVType =
-      MVType == MultiVersionKind::CPUDispatch ||
-      MVType == MultiVersionKind::CPUSpecific;
-
   if (OldFD && !OldFD->getType()->getAs<FunctionProtoType>()) {
-    S.Diag(OldFD->getLocation(), diag::err_multiversion_noproto);
-    S.Diag(NewFD->getLocation(), diag::note_multiversioning_caused_here);
+    Diag(OldFD->getLocation(), NoProtoDiagID);
+    Diag(NoteCausedDiagIDAt.first, NoteCausedDiagIDAt.second);
     return true;
   }
 
   if (!NewFD->getType()->getAs<FunctionProtoType>())
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_noproto);
-
-  if (!S.getASTContext().getTargetInfo().supportsMultiVersioning()) {
-    S.Diag(NewFD->getLocation(), diag::err_multiversion_not_supported);
-    if (OldFD)
-      S.Diag(OldFD->getLocation(), diag::note_previous_declaration);
-    return true;
-  }
-
-  // For now, disallow all other attributes.  These should be opt-in, but
-  // an analysis of all of them is a future FIXME.
-  if (CausesMV && OldFD && HasNonMultiVersionAttributes(OldFD, MVType)) {
-    S.Diag(OldFD->getLocation(), diag::err_multiversion_no_other_attrs)
-        << IsCPUSpecificCPUDispatchMVType;
-    S.Diag(NewFD->getLocation(), diag::note_multiversioning_caused_here);
-    return true;
-  }
+    return Diag(NewFD->getLocation(), NoProtoDiagID);
 
-  if (HasNonMultiVersionAttributes(NewFD, MVType))
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_no_other_attrs)
-           << IsCPUSpecificCPUDispatchMVType;
-
-  if (NewFD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate)
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_doesnt_support)
-           << IsCPUSpecificCPUDispatchMVType << FuncTemplates;
+  if (!TemplatesSupported &&
+      NewFD->getTemplatedKind() == FunctionDecl::TK_FunctionTemplate)
+    return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+           << FuncTemplates;
 
   if (const auto *NewCXXFD = dyn_cast<CXXMethodDecl>(NewFD)) {
     if (NewCXXFD->isVirtual())
-      return S.Diag(NewCXXFD->getLocation(),
-                    diag::err_multiversion_doesnt_support)
-             << IsCPUSpecificCPUDispatchMVType << VirtFuncs;
+      return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+             << VirtFuncs;
 
-    if (const auto *NewCXXCtor = dyn_cast<CXXConstructorDecl>(NewFD))
-      return S.Diag(NewCXXCtor->getLocation(),
-                    diag::err_multiversion_doesnt_support)
-             << IsCPUSpecificCPUDispatchMVType << Constructors;
+    if (isa<CXXConstructorDecl>(NewCXXFD))
+      return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+             << Constructors;
 
-    if (const auto *NewCXXDtor = dyn_cast<CXXDestructorDecl>(NewFD))
-      return S.Diag(NewCXXDtor->getLocation(),
-                    diag::err_multiversion_doesnt_support)
-             << IsCPUSpecificCPUDispatchMVType << Destructors;
+    if (isa<CXXDestructorDecl>(NewCXXFD))
+      return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+             << Destructors;
   }
 
   if (NewFD->isDeleted())
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_doesnt_support)
-           << IsCPUSpecificCPUDispatchMVType << DeletedFuncs;
+    return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+           << DeletedFuncs;
 
   if (NewFD->isDefaulted())
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_doesnt_support)
-           << IsCPUSpecificCPUDispatchMVType << DefaultedFuncs;
+    return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+           << DefaultedFuncs;
 
-  if (NewFD->isConstexpr() && (MVType == MultiVersionKind::CPUDispatch ||
-                               MVType == MultiVersionKind::CPUSpecific))
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_doesnt_support)
-           << IsCPUSpecificCPUDispatchMVType
+  if (!ConstexprSupported && NewFD->isConstexpr())
+    return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
            << (NewFD->isConsteval() ? ConstevalFuncs : ConstexprFuncs);
 
-  QualType NewQType = S.getASTContext().getCanonicalType(NewFD->getType());
+  QualType NewQType = Context.getCanonicalType(NewFD->getType());
   const auto *NewType = cast<FunctionType>(NewQType);
   QualType NewReturnType = NewType->getReturnType();
 
   if (NewReturnType->isUndeducedType())
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_doesnt_support)
-           << IsCPUSpecificCPUDispatchMVType << DeducedReturn;
-
-  // Only allow transition to MultiVersion if it hasn't been used.
-  if (OldFD && CausesMV && OldFD->isUsed(false))
-    return S.Diag(NewFD->getLocation(), diag::err_multiversion_after_used);
+    return Diag(NoSupportDiagIDAt.first, NoSupportDiagIDAt.second)
+           << DeducedReturn;
 
   // Ensure the return type is identical.
   if (OldFD) {
-    QualType OldQType = S.getASTContext().getCanonicalType(OldFD->getType());
+    QualType OldQType = Context.getCanonicalType(OldFD->getType());
     const auto *OldType = cast<FunctionType>(OldQType);
     FunctionType::ExtInfo OldTypeInfo = OldType->getExtInfo();
     FunctionType::ExtInfo NewTypeInfo = NewType->getExtInfo();
 
     if (OldTypeInfo.getCC() != NewTypeInfo.getCC())
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << CallingConv;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << CallingConv;
 
     QualType OldReturnType = OldType->getReturnType();
 
     if (OldReturnType != NewReturnType)
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << ReturnType;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << ReturnType;
 
     if (OldFD->getConstexprKind() != NewFD->getConstexprKind())
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << ConstexprSpec;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << ConstexprSpec;
 
     if (OldFD->isInlineSpecified() != NewFD->isInlineSpecified())
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << InlineSpec;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << InlineSpec;
 
     if (OldFD->getStorageClass() != NewFD->getStorageClass())
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << StorageClass;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << StorageClass;
 
     if (OldFD->isExternC() != NewFD->isExternC())
-      return S.Diag(NewFD->getLocation(), diag::err_multiversion_diff)
-             << Linkage;
+      return Diag(DiffDiagIDAt.first, DiffDiagIDAt.second) << Linkage;
 
-    if (S.CheckEquivalentExceptionSpec(
+    if (CheckEquivalentExceptionSpec(
             OldFD->getType()->getAs<FunctionProtoType>(), OldFD->getLocation(),
             NewFD->getType()->getAs<FunctionProtoType>(), NewFD->getLocation()))
       return true;
@@ -9819,6 +9790,51 @@ static bool CheckMultiVersionAdditionalRules(Sema &S, const FunctionDecl *OldFD,
   return false;
 }
 
+static bool CheckMultiVersionAdditionalRules(Sema &S, const FunctionDecl *OldFD,
+                                             const FunctionDecl *NewFD,
+                                             bool CausesMV,
+                                             MultiVersionKind MVType) {
+  if (!S.getASTContext().getTargetInfo().supportsMultiVersioning()) {
+    S.Diag(NewFD->getLocation(), diag::err_multiversion_not_supported);
+    if (OldFD)
+      S.Diag(OldFD->getLocation(), diag::note_previous_declaration);
+    return true;
+  }
+
+  bool IsCPUSpecificCPUDispatchMVType =
+      MVType == MultiVersionKind::CPUDispatch ||
+      MVType == MultiVersionKind::CPUSpecific;
+
+  // For now, disallow all other attributes.  These should be opt-in, but
+  // an analysis of all of them is a future FIXME.
+  if (CausesMV && OldFD && HasNonMultiVersionAttributes(OldFD, MVType)) {
+    S.Diag(OldFD->getLocation(), diag::err_multiversion_no_other_attrs)
+        << IsCPUSpecificCPUDispatchMVType;
+    S.Diag(NewFD->getLocation(), diag::note_multiversioning_caused_here);
+    return true;
+  }
+
+  if (HasNonMultiVersionAttributes(NewFD, MVType))
+    return S.Diag(NewFD->getLocation(), diag::err_multiversion_no_other_attrs)
+           << IsCPUSpecificCPUDispatchMVType;
+
+  // Only allow transition to MultiVersion if it hasn't been used.
+  if (OldFD && CausesMV && OldFD->isUsed(false))
+    return S.Diag(NewFD->getLocation(), diag::err_multiversion_after_used);
+
+  return S.areMultiversionVariantFunctionsCompatible(
+      OldFD, NewFD, S.PDiag(diag::err_multiversion_noproto),
+      PartialDiagnosticAt(NewFD->getLocation(),
+                          S.PDiag(diag::note_multiversioning_caused_here)),
+      PartialDiagnosticAt(NewFD->getLocation(),
+                          S.PDiag(diag::err_multiversion_doesnt_support)
+                              << IsCPUSpecificCPUDispatchMVType),
+      PartialDiagnosticAt(NewFD->getLocation(),
+                          S.PDiag(diag::err_multiversion_diff)),
+      /*TemplatesSupported=*/false,
+      /*ConstexprSupported=*/!IsCPUSpecificCPUDispatchMVType);
+}
+
 /// Check the validity of a multiversion function declaration that is the
 /// first of its kind. Also sets the multiversion'ness' of the function itself.
 ///
@@ -10266,7 +10282,7 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
   CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(NewFD);
   if (!getLangOpts().CPlusPlus14 && MD && MD->isConstexpr() &&
       !MD->isStatic() && !isa<CXXConstructorDecl>(MD) &&
-      !MD->getMethodQualifiers().hasConst()) {
+      !isa<CXXDestructorDecl>(MD) && !MD->getMethodQualifiers().hasConst()) {
     CXXMethodDecl *OldMD = nullptr;
     if (OldDecl)
       OldMD = dyn_cast_or_null<CXXMethodDecl>(OldDecl->getAsFunction());
@@ -17544,8 +17560,8 @@ void Sema::ActOnPragmaRedefineExtname(IdentifierInfo* Name,
                                          LookupOrdinaryName);
   AttributeCommonInfo Info(AliasName, SourceRange(AliasNameLoc),
                            AttributeCommonInfo::AS_Pragma);
-  AsmLabelAttr *Attr =
-      AsmLabelAttr::CreateImplicit(Context, AliasName->getName(), Info);
+  AsmLabelAttr *Attr = AsmLabelAttr::CreateImplicit(
+      Context, AliasName->getName(), /*LiteralLabel=*/true, Info);
 
   // If a declaration that:
   // 1) declares a function or a variable
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 2d2ce148b3c7a..5c9a618d87d5c 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -4347,7 +4347,9 @@ static void handleGlobalAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
   const auto *FD = cast<FunctionDecl>(D);
-  if (!FD->getReturnType()->isVoidType()) {
+  if (!FD->getReturnType()->isVoidType() &&
+      !FD->getReturnType()->getAs<AutoType>() &&
+      !FD->getReturnType()->isInstantiationDependentType()) {
     SourceRange RTRange = FD->getReturnTypeSourceRange();
     S.Diag(FD->getTypeSpecStartLoc(), diag::err_kern_type_not_void_return)
         << FD->getType()
@@ -4377,6 +4379,9 @@ static void handleGNUInlineAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     return;
   }
 
+  if (S.LangOpts.CPlusPlus && Fn->getStorageClass() != SC_Extern)
+    S.Diag(AL.getLoc(), diag::warn_gnu_inline_cplusplus_without_extern);
+
   D->addAttr(::new (S.Context) GNUInlineAttr(S.Context, AL));
 }
 
diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp
index a221ab0239253..cbc46a7492264 100644
--- a/clang/lib/Sema/SemaDeclCXX.cpp
+++ b/clang/lib/Sema/SemaDeclCXX.cpp
@@ -1590,6 +1590,36 @@ static bool CheckLiteralType(Sema &SemaRef, Sema::CheckConstexprKind Kind,
   llvm_unreachable("unknown CheckConstexprKind");
 }
 
+/// Determine whether a destructor cannot be constexpr due to
+static bool CheckConstexprDestructorSubobjects(Sema &SemaRef,
+                                               const CXXDestructorDecl *DD,
+                                               Sema::CheckConstexprKind Kind) {
+  auto Check = [&](SourceLocation Loc, QualType T, const FieldDecl *FD) {
+    const CXXRecordDecl *RD =
+        T->getBaseElementTypeUnsafe()->getAsCXXRecordDecl();
+    if (!RD || RD->hasConstexprDestructor())
+      return true;
+
+    if (Kind == Sema::CheckConstexprKind::Diagnose) {
+      SemaRef.Diag(DD->getLocation(), diag::err_constexpr_dtor_subobject)
+          << DD->getConstexprKind() << !FD
+          << (FD ? FD->getDeclName() : DeclarationName()) << T;
+      SemaRef.Diag(Loc, diag::note_constexpr_dtor_subobject)
+          << !FD << (FD ? FD->getDeclName() : DeclarationName()) << T;
+    }
+    return false;
+  };
+
+  const CXXRecordDecl *RD = DD->getParent();
+  for (const CXXBaseSpecifier &B : RD->bases())
+    if (!Check(B.getBaseTypeLoc(), B.getType(), nullptr))
+      return false;
+  for (const FieldDecl *FD : RD->fields())
+    if (!Check(FD->getLocation(), FD->getType(), FD))
+      return false;
+  return true;
+}
+
 // CheckConstexprParameterTypes - Check whether a function's parameter types
 // are all literal types. If so, return true. If not, produce a suitable
 // diagnostic and return false.
@@ -1645,8 +1675,8 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD,
     //  constraints:
     //  - the class shall not have any virtual base classes;
     //
-    // FIXME: This only applies to constructors, not arbitrary member
-    // functions.
+    // FIXME: This only applies to constructors and destructors, not arbitrary
+    // member functions.
     const CXXRecordDecl *RD = MD->getParent();
     if (RD->getNumVBases()) {
       if (Kind == CheckConstexprKind::CheckValid)
@@ -1699,6 +1729,18 @@ bool Sema::CheckConstexprFunctionDefinition(const FunctionDecl *NewFD,
       return false;
   }
 
+  if (auto *Dtor = dyn_cast<CXXDestructorDecl>(NewFD)) {
+    // A destructor can be constexpr only if the defaulted destructor could be;
+    // we don't need to check the members and bases if we already know they all
+    // have constexpr destructors.
+    if (!Dtor->getParent()->defaultedDestructorIsConstexpr()) {
+      if (Kind == CheckConstexprKind::CheckValid)
+        return false;
+      if (!CheckConstexprDestructorSubobjects(*this, Dtor, Kind))
+        return false;
+    }
+  }
+
   // - each of its parameter types shall be a literal type;
   if (!CheckConstexprParameterTypes(*this, NewFD, Kind))
     return false;
@@ -1778,7 +1820,8 @@ static bool CheckConstexprDeclStmt(Sema &SemaRef, const FunctionDecl *Dcl,
     case Decl::Decomposition: {
       // C++1y [dcl.constexpr]p3 allows anything except:
       //   a definition of a variable of non-literal type or of static or
-      //   thread storage duration or for which no initialization is performed.
+      //   thread storage duration or [before C++2a] for which no
+      //   initialization is performed.
       const auto *VD = cast<VarDecl>(DclIt);
       if (VD->isThisDeclarationADefinition()) {
         if (VD->isStaticLocal()) {
@@ -1797,11 +1840,16 @@ static bool CheckConstexprDeclStmt(Sema &SemaRef, const FunctionDecl *Dcl,
         if (!VD->getType()->isDependentType() &&
             !VD->hasInit() && !VD->isCXXForRangeDecl()) {
           if (Kind == Sema::CheckConstexprKind::Diagnose) {
-            SemaRef.Diag(VD->getLocation(),
-                         diag::err_constexpr_local_var_no_init)
-              << isa<CXXConstructorDecl>(Dcl);
+            SemaRef.Diag(
+                VD->getLocation(),
+                SemaRef.getLangOpts().CPlusPlus2a
+                    ? diag::warn_cxx17_compat_constexpr_local_var_no_init
+                    : diag::ext_constexpr_local_var_no_init)
+                << isa<CXXConstructorDecl>(Dcl);
+          } else if (!SemaRef.getLangOpts().CPlusPlus2a) {
+            return false;
           }
-          return false;
+          continue;
         }
       }
       if (Kind == Sema::CheckConstexprKind::Diagnose) {
@@ -1855,6 +1903,11 @@ static bool CheckConstexprCtorInitializer(Sema &SemaRef,
                                           llvm::SmallSet<Decl*, 16> &Inits,
                                           bool &Diagnosed,
                                           Sema::CheckConstexprKind Kind) {
+  // In C++20 onwards, there's nothing to check for validity.
+  if (Kind == Sema::CheckConstexprKind::CheckValid &&
+      SemaRef.getLangOpts().CPlusPlus2a)
+    return true;
+
   if (Field->isInvalidDecl())
     return true;
 
@@ -1873,12 +1926,15 @@ static bool CheckConstexprCtorInitializer(Sema &SemaRef,
   if (!Inits.count(Field)) {
     if (Kind == Sema::CheckConstexprKind::Diagnose) {
       if (!Diagnosed) {
-        SemaRef.Diag(Dcl->getLocation(), diag::err_constexpr_ctor_missing_init);
+        SemaRef.Diag(Dcl->getLocation(),
+                     SemaRef.getLangOpts().CPlusPlus2a
+                         ? diag::warn_cxx17_compat_constexpr_ctor_missing_init
+                         : diag::ext_constexpr_ctor_missing_init);
         Diagnosed = true;
       }
       SemaRef.Diag(Field->getLocation(),
                    diag::note_constexpr_ctor_missing_init);
-    } else {
+    } else if (!SemaRef.getLangOpts().CPlusPlus2a) {
       return false;
     }
   } else if (Field->isAnonymousStructOrUnion()) {
@@ -2121,10 +2177,15 @@ static bool CheckConstexprFunctionBody(Sema &SemaRef, const FunctionDecl *Dcl,
     if (RD->isUnion()) {
       if (Constructor->getNumCtorInitializers() == 0 &&
           RD->hasVariantMembers()) {
-        if (Kind == Sema::CheckConstexprKind::Diagnose)
-          SemaRef.Diag(Dcl->getLocation(),
-                       diag::err_constexpr_union_ctor_no_init);
-        return false;
+        if (Kind == Sema::CheckConstexprKind::Diagnose) {
+          SemaRef.Diag(
+              Dcl->getLocation(),
+              SemaRef.getLangOpts().CPlusPlus2a
+                  ? diag::warn_cxx17_compat_constexpr_union_ctor_no_init
+                  : diag::ext_constexpr_union_ctor_no_init);
+        } else if (!SemaRef.getLangOpts().CPlusPlus2a) {
+          return false;
+        }
       }
     } else if (!Constructor->isDependentContext() &&
                !Constructor->isDelegatingConstructor()) {
@@ -6303,6 +6364,12 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
         DelayedDllExportMemberFunctions.push_back(M);
       }
     }
+
+    // Define defaulted constexpr virtual functions that override a base class
+    // function right away.
+    // FIXME: We can defer doing this until the vtable is marked as used.
+    if (M->isDefaulted() && M->isConstexpr() && M->size_overridden_methods())
+      DefineImplicitSpecialMember(*this, M, M->getLocation());
   };
 
   bool HasMethodWithOverrideControl = false,
@@ -6516,6 +6583,8 @@ specialMemberIsConstexpr(Sema &S, CXXRecordDecl *ClassDecl,
 
   if (CSM == Sema::CXXDefaultConstructor)
     return ClassDecl->hasConstexprDefaultConstructor();
+  if (CSM == Sema::CXXDestructor)
+    return ClassDecl->hasConstexprDestructor();
 
   Sema::SpecialMemberOverloadResult SMOR =
       lookupCallFromSpecialMember(S, ClassDecl, CSM, Quals, ConstRHS);
@@ -6564,6 +6633,8 @@ static bool defaultedSpecialMemberIsConstexpr(
     break;
 
   case Sema::CXXDestructor:
+    return ClassDecl->defaultedDestructorIsConstexpr();
+
   case Sema::CXXInvalid:
     return false;
   }
@@ -6816,13 +6887,14 @@ void Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD) {
   // Do not apply this rule to members of class templates, since core issue 1358
   // makes such functions always instantiate to constexpr functions. For
   // functions which cannot be constexpr (for non-constructors in C++11 and for
-  // destructors in C++1y), this is checked elsewhere.
+  // destructors in C++14 and C++17), this is checked elsewhere.
   //
   // FIXME: This should not apply if the member is deleted.
   bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, RD, CSM,
                                                      HasConstParam);
-  if ((getLangOpts().CPlusPlus14 ? !isa<CXXDestructorDecl>(MD)
-                                 : isa<CXXConstructorDecl>(MD)) &&
+  if ((getLangOpts().CPlusPlus2a ||
+       (getLangOpts().CPlusPlus14 ? !isa<CXXDestructorDecl>(MD)
+                                  : isa<CXXConstructorDecl>(MD))) &&
       MD->isConstexpr() && !Constexpr &&
       MD->getTemplatedKind() == FunctionDecl::TK_NonTemplate) {
     Diag(MD->getBeginLoc(), MD->isConsteval()
@@ -11445,6 +11517,10 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   if (DSM.isAlreadyBeingDeclared())
     return nullptr;
 
+  bool Constexpr = defaultedSpecialMemberIsConstexpr(*this, ClassDecl,
+                                                     CXXDestructor,
+                                                     false);
+
   // Create the actual destructor declaration.
   CanQualType ClassType
     = Context.getCanonicalType(Context.getTypeDeclType(ClassDecl));
@@ -11452,10 +11528,11 @@ CXXDestructorDecl *Sema::DeclareImplicitDestructor(CXXRecordDecl *ClassDecl) {
   DeclarationName Name
     = Context.DeclarationNames.getCXXDestructorName(ClassType);
   DeclarationNameInfo NameInfo(Name, ClassLoc);
-  CXXDestructorDecl *Destructor
-      = CXXDestructorDecl::Create(Context, ClassDecl, ClassLoc, NameInfo,
-                                  QualType(), nullptr, /*isInline=*/true,
-                                  /*isImplicitlyDeclared=*/true);
+  CXXDestructorDecl *Destructor =
+      CXXDestructorDecl::Create(Context, ClassDecl, ClassLoc, NameInfo,
+                                QualType(), nullptr, /*isInline=*/true,
+                                /*isImplicitlyDeclared=*/true,
+                                Constexpr ? CSK_constexpr : CSK_unspecified);
   Destructor->setAccess(AS_public);
   Destructor->setDefaulted();
 
@@ -13323,6 +13400,20 @@ void Sema::FinalizeVarWithDestructor(VarDecl *VD, const RecordType *Record) {
   }
 
   if (Destructor->isTrivial()) return;
+
+  // If the destructor is constexpr, check whether the variable has constant
+  // destruction now.
+  if (Destructor->isConstexpr() && VD->getInit() &&
+      !VD->getInit()->isValueDependent() && VD->evaluateValue()) {
+    SmallVector<PartialDiagnosticAt, 8> Notes;
+    if (!VD->evaluateDestruction(Notes) && VD->isConstexpr()) {
+      Diag(VD->getLocation(),
+           diag::err_constexpr_var_requires_const_destruction) << VD;
+      for (unsigned I = 0, N = Notes.size(); I != N; ++I)
+        Diag(Notes[I].first, Notes[I].second);
+    }
+  }
+
   if (!VD->hasGlobalStorage()) return;
 
   // Emit warning for non-trivial dtor in global scope (a real global,
@@ -13913,6 +14004,10 @@ Decl *Sema::ActOnStartLinkageSpecification(Scope *S, SourceLocation ExternLoc,
     Language = LinkageSpecDecl::lang_c;
   else if (Lang == "C++")
     Language = LinkageSpecDecl::lang_cxx;
+  else if (Lang == "C++11")
+    Language = LinkageSpecDecl::lang_cxx_11;
+  else if (Lang == "C++14")
+    Language = LinkageSpecDecl::lang_cxx_14;
   else {
     Diag(LangStr->getExprLoc(), diag::err_language_linkage_spec_unknown)
       << LangStr->getSourceRange();
@@ -14165,8 +14260,17 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
     if (Converted.isInvalid())
       Failed = true;
 
+    ExprResult FullAssertExpr =
+        ActOnFinishFullExpr(Converted.get(), StaticAssertLoc,
+                            /*DiscardedValue*/ false,
+                            /*IsConstexpr*/ true);
+    if (FullAssertExpr.isInvalid())
+      Failed = true;
+    else
+      AssertExpr = FullAssertExpr.get();
+
     llvm::APSInt Cond;
-    if (!Failed && VerifyIntegerConstantExpression(Converted.get(), &Cond,
+    if (!Failed && VerifyIntegerConstantExpression(AssertExpr, &Cond,
           diag::err_static_assert_expression_is_not_constant,
           /*AllowFold=*/false).isInvalid())
       Failed = true;
@@ -14192,16 +14296,16 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
       }
       Failed = true;
     }
+  } else {
+    ExprResult FullAssertExpr = ActOnFinishFullExpr(AssertExpr, StaticAssertLoc,
+                                                    /*DiscardedValue*/false,
+                                                    /*IsConstexpr*/true);
+    if (FullAssertExpr.isInvalid())
+      Failed = true;
+    else
+      AssertExpr = FullAssertExpr.get();
   }
 
-  ExprResult FullAssertExpr = ActOnFinishFullExpr(AssertExpr, StaticAssertLoc,
-                                                  /*DiscardedValue*/false,
-                                                  /*IsConstexpr*/true);
-  if (FullAssertExpr.isInvalid())
-    Failed = true;
-  else
-    AssertExpr = FullAssertExpr.get();
-
   Decl *Decl = StaticAssertDecl::Create(Context, CurContext, StaticAssertLoc,
                                         AssertExpr, AssertMessage, RParenLoc,
                                         Failed);
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index 5fab2a147879a..a0fef8346c2a5 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -2275,9 +2275,7 @@ static bool isObjCTypeSubstitutable(ASTContext &Context,
   // stricter definition so it is not substitutable for id<A>.
   if (B->isObjCQualifiedIdType()) {
     return A->isObjCQualifiedIdType() &&
-           Context.ObjCQualifiedIdTypesAreCompatible(QualType(A, 0),
-                                                     QualType(B,0),
-                                                     false);
+           Context.ObjCQualifiedIdTypesAreCompatible(A, B, false);
   }
 
   /*
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index 61fda54b32199..a9f7d98a8027b 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -4081,7 +4081,7 @@ static void captureVariablyModifiedType(ASTContext &Context, QualType T,
 #define NON_CANONICAL_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
       T = QualType();
       break;
     // These types are never variably-modified.
@@ -5955,7 +5955,9 @@ ExprResult Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
             << FDecl << Fn->getSourceRange());
 
       // CUDA: Kernel function must have 'void' return type
-      if (!FuncT->getReturnType()->isVoidType())
+      if (!FuncT->getReturnType()->isVoidType() &&
+          !FuncT->getReturnType()->getAs<AutoType>() &&
+          !FuncT->getReturnType()->isInstantiationDependentType())
         return ExprError(Diag(LParenLoc, diag::err_kern_type_not_void_return)
             << Fn->getType() << Fn->getSourceRange());
     } else {
@@ -6776,8 +6778,8 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
   assert(Ty->isVectorType() && "Expected vector type");
 
   SmallVector<Expr *, 8> initExprs;
-  const VectorType *VTy = Ty->getAs<VectorType>();
-  unsigned numElems = Ty->getAs<VectorType>()->getNumElements();
+  const VectorType *VTy = Ty->castAs<VectorType>();
+  unsigned numElems = VTy->getNumElements();
 
   // '(...)' form of vector initialization in AltiVec: the number of
   // initializers must be one or must match the size of the vector.
@@ -6788,7 +6790,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
     // vector. If a single value is specified in the initializer then it will
     // be replicated to all the components of the vector
     if (numExprs == 1) {
-      QualType ElemTy = Ty->getAs<VectorType>()->getElementType();
+      QualType ElemTy = VTy->getElementType();
       ExprResult Literal = DefaultLvalueConversion(exprs[0]);
       if (Literal.isInvalid())
         return ExprError();
@@ -6810,7 +6812,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
     if (getLangOpts().OpenCL &&
         VTy->getVectorKind() == VectorType::GenericVector &&
         numExprs == 1) {
-        QualType ElemTy = Ty->getAs<VectorType>()->getElementType();
+        QualType ElemTy = VTy->getElementType();
         ExprResult Literal = DefaultLvalueConversion(exprs[0]);
         if (Literal.isInvalid())
           return ExprError();
@@ -7109,8 +7111,8 @@ checkConditionalObjectPointersCompatibility(Sema &S, ExprResult &LHS,
   QualType RHSTy = RHS.get()->getType();
 
   // get the "pointed to" types
-  QualType lhptee = LHSTy->getAs<PointerType>()->getPointeeType();
-  QualType rhptee = RHSTy->getAs<PointerType>()->getPointeeType();
+  QualType lhptee = LHSTy->castAs<PointerType>()->getPointeeType();
+  QualType rhptee = RHSTy->castAs<PointerType>()->getPointeeType();
 
   // ignore qualifiers on void (C99 6.5.15p3, clause 6)
   if (lhptee->isVoidType() && rhptee->isIncompleteOrObjectType()) {
@@ -7560,9 +7562,10 @@ QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
       compositeType = RHSOPT->isObjCBuiltinType() ? RHSTy : LHSTy;
     } else if (Context.canAssignObjCInterfaces(RHSOPT, LHSOPT)) {
       compositeType = LHSOPT->isObjCBuiltinType() ? LHSTy : RHSTy;
-    } else if ((LHSTy->isObjCQualifiedIdType() ||
-                RHSTy->isObjCQualifiedIdType()) &&
-               Context.ObjCQualifiedIdTypesAreCompatible(LHSTy, RHSTy, true)) {
+    } else if ((LHSOPT->isObjCQualifiedIdType() ||
+                RHSOPT->isObjCQualifiedIdType()) &&
+               Context.ObjCQualifiedIdTypesAreCompatible(LHSOPT, RHSOPT,
+                                                         true)) {
       // Need to handle "id<xx>" explicitly.
       // GCC allows qualified id and any Objective-C type to devolve to
       // id. Currently localizing to here until clear this should be
@@ -7594,8 +7597,8 @@ QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
       LHS = RHS = true;
       return QualType();
     }
-    QualType lhptee = LHSTy->getAs<PointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->getAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType lhptee = LHSTy->castAs<PointerType>()->getPointeeType();
+    QualType rhptee = RHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
     QualType destPointee
     = Context.getQualifiedType(lhptee, rhptee.getQualifiers());
     QualType destType = Context.getPointerType(destPointee);
@@ -7614,8 +7617,8 @@ QualType Sema::FindCompositeObjCPointerType(ExprResult &LHS, ExprResult &RHS,
       LHS = RHS = true;
       return QualType();
     }
-    QualType lhptee = LHSTy->getAs<ObjCObjectPointerType>()->getPointeeType();
-    QualType rhptee = RHSTy->getAs<PointerType>()->getPointeeType();
+    QualType lhptee = LHSTy->castAs<ObjCObjectPointerType>()->getPointeeType();
+    QualType rhptee = RHSTy->castAs<PointerType>()->getPointeeType();
     QualType destPointee
     = Context.getQualifiedType(rhptee, lhptee.getQualifiers());
     QualType destType = Context.getPointerType(destPointee);
@@ -8120,8 +8123,8 @@ checkObjCPointerTypesForAssignment(Sema &S, QualType LHSType,
       return Sema::IncompatiblePointer;
     return Sema::Compatible;
   }
-  QualType lhptee = LHSType->getAs<ObjCObjectPointerType>()->getPointeeType();
-  QualType rhptee = RHSType->getAs<ObjCObjectPointerType>()->getPointeeType();
+  QualType lhptee = LHSType->castAs<ObjCObjectPointerType>()->getPointeeType();
+  QualType rhptee = RHSType->castAs<ObjCObjectPointerType>()->getPointeeType();
 
   if (!lhptee.isAtLeastAsQualifiedAs(rhptee) &&
       // make an exception for id<P>
@@ -9254,7 +9257,8 @@ static void DiagnoseDivisionSizeofPointerOrArray(Sema &S, Expr *LHS, Expr *RHS,
     }
   } else if (const auto *ArrayTy = S.Context.getAsArrayType(LHSTy)) {
     QualType ArrayElemTy = ArrayTy->getElementType();
-    if (ArrayElemTy->isDependentType() || RHSTy->isDependentType() ||
+    if (ArrayElemTy != S.Context.getBaseElementType(ArrayTy) ||
+        ArrayElemTy->isDependentType() || RHSTy->isDependentType() ||
         S.Context.getTypeSize(ArrayElemTy) == S.Context.getTypeSize(RHSTy))
       return;
     S.Diag(Loc, diag::warn_division_sizeof_array)
@@ -9264,6 +9268,8 @@ static void DiagnoseDivisionSizeofPointerOrArray(Sema &S, Expr *LHS, Expr *RHS,
         S.Diag(LHSArgDecl->getLocation(), diag::note_array_declared_here)
             << LHSArgDecl;
     }
+
+    S.Diag(Loc, diag::note_precedence_silence) << RHS;
   }
 }
 
@@ -9459,8 +9465,8 @@ static bool checkArithmeticBinOpPointerOperands(Sema &S, SourceLocation Loc,
 
   // if both are pointers check if operation is valid wrt address spaces
   if (S.getLangOpts().OpenCL && isLHSPointer && isRHSPointer) {
-    const PointerType *lhsPtr = LHSExpr->getType()->getAs<PointerType>();
-    const PointerType *rhsPtr = RHSExpr->getType()->getAs<PointerType>();
+    const PointerType *lhsPtr = LHSExpr->getType()->castAs<PointerType>();
+    const PointerType *rhsPtr = RHSExpr->getType()->castAs<PointerType>();
     if (!lhsPtr->isAddressSpaceOverlapping(*rhsPtr)) {
       S.Diag(Loc,
              diag::err_typecheck_op_on_nonoverlapping_address_space_pointers)
@@ -10307,20 +10313,18 @@ static void diagnoseLogicalNotOnLHSofCheck(Sema &S, ExprResult &LHS,
       << FixItHint::CreateInsertion(SecondClose, ")");
 }
 
-// Get the decl for a simple expression: a reference to a variable,
-// an implicit C++ field reference, or an implicit ObjC ivar reference.
-static ValueDecl *getCompareDecl(Expr *E) {
-  if (DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E))
-    return DR->getDecl();
-  if (ObjCIvarRefExpr *Ivar = dyn_cast<ObjCIvarRefExpr>(E)) {
-    if (Ivar->isFreeIvar())
-      return Ivar->getDecl();
-  }
-  if (MemberExpr *Mem = dyn_cast<MemberExpr>(E)) {
+// Returns true if E refers to a non-weak array.
+static bool checkForArray(const Expr *E) {
+  const ValueDecl *D = nullptr;
+  if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(E)) {
+    D = DR->getDecl();
+  } else if (const MemberExpr *Mem = dyn_cast<MemberExpr>(E)) {
     if (Mem->isImplicitAccess())
-      return Mem->getMemberDecl();
+      D = Mem->getMemberDecl();
   }
-  return nullptr;
+  if (!D)
+    return false;
+  return D->getType()->isArrayType() && !D->isWeak();
 }
 
 /// Diagnose some forms of syntactically-obvious tautological comparison.
@@ -10353,8 +10357,6 @@ static void diagnoseTautologicalComparison(Sema &S, SourceLocation Loc,
   // obvious cases in the definition of the template anyways. The idea is to
   // warn when the typed comparison operator will always evaluate to the same
   // result.
-  ValueDecl *DL = getCompareDecl(LHSStripped);
-  ValueDecl *DR = getCompareDecl(RHSStripped);
 
   // Used for indexing into %select in warn_comparison_always
   enum {
@@ -10363,7 +10365,8 @@ static void diagnoseTautologicalComparison(Sema &S, SourceLocation Loc,
     AlwaysFalse,
     AlwaysEqual, // std::strong_ordering::equal from operator<=>
   };
-  if (DL && DR && declaresSameEntity(DL, DR)) {
+
+  if (Expr::isSameComparisonOperand(LHS, RHS)) {
     unsigned Result;
     switch (Opc) {
     case BO_EQ: case BO_LE: case BO_GE:
@@ -10383,9 +10386,7 @@ static void diagnoseTautologicalComparison(Sema &S, SourceLocation Loc,
                           S.PDiag(diag::warn_comparison_always)
                               << 0 /*self-comparison*/
                               << Result);
-  } else if (DL && DR &&
-             DL->getType()->isArrayType() && DR->getType()->isArrayType() &&
-             !DL->isWeak() && !DR->isWeak()) {
+  } else if (checkForArray(LHSStripped) && checkForArray(RHSStripped)) {
     // What is it always going to evaluate to?
     unsigned Result;
     switch(Opc) {
@@ -10558,7 +10559,7 @@ static QualType checkArithmeticOrEnumeralThreeWayCompare(Sema &S,
       return QualType();
     }
     QualType IntType =
-        LHSStrippedType->getAs<EnumType>()->getDecl()->getIntegerType();
+        LHSStrippedType->castAs<EnumType>()->getDecl()->getIntegerType();
     assert(IntType->isArithmeticType());
 
     // We can't use `CK_IntegralCast` when the underlying type is 'bool', so we
@@ -10841,8 +10842,8 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
     if (LCanPointeeTy != RCanPointeeTy) {
       // Treat NULL constant as a special case in OpenCL.
       if (getLangOpts().OpenCL && !LHSIsNull && !RHSIsNull) {
-        const PointerType *LHSPtr = LHSType->getAs<PointerType>();
-        if (!LHSPtr->isAddressSpaceOverlapping(*RHSType->getAs<PointerType>())) {
+        const PointerType *LHSPtr = LHSType->castAs<PointerType>();
+        if (!LHSPtr->isAddressSpaceOverlapping(*RHSType->castAs<PointerType>())) {
           Diag(Loc,
                diag::err_typecheck_op_on_nonoverlapping_address_space_pointers)
               << LHSType << RHSType << 0 /* comparison */
@@ -11107,7 +11108,7 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
 // the largest type to the smallest type to avoid cases where long long == long,
 // where long gets picked over long long.
 QualType Sema::GetSignedVectorType(QualType V) {
-  const VectorType *VTy = V->getAs<VectorType>();
+  const VectorType *VTy = V->castAs<VectorType>();
   unsigned TypeSize = Context.getTypeSize(VTy->getElementType());
 
   if (isa<ExtVectorType>(VTy)) {
@@ -11162,7 +11163,7 @@ QualType Sema::CheckVectorCompareOperands(ExprResult &LHS, ExprResult &RHS,
   // If AltiVec, the comparison results in a numeric type, i.e.
   // bool for C++, int for C
   if (getLangOpts().AltiVec &&
-      vType->getAs<VectorType>()->getVectorKind() == VectorType::AltiVecVector)
+      vType->castAs<VectorType>()->getVectorKind() == VectorType::AltiVecVector)
     return Context.getLogicalOperationType();
 
   // For non-floating point types, check for self-comparisons of the form
@@ -11362,10 +11363,22 @@ inline QualType Sema::CheckLogicalOperands(ExprResult &LHS, ExprResult &RHS,
   if (LHS.get()->getType()->isVectorType() || RHS.get()->getType()->isVectorType())
     return CheckVectorLogicalOperands(LHS, RHS, Loc);
 
+  bool EnumConstantInBoolContext = false;
+  for (const ExprResult &HS : {LHS, RHS}) {
+    if (const auto *DREHS = dyn_cast<DeclRefExpr>(HS.get())) {
+      const auto *ECDHS = dyn_cast<EnumConstantDecl>(DREHS->getDecl());
+      if (ECDHS && ECDHS->getInitVal() != 0 && ECDHS->getInitVal() != 1)
+        EnumConstantInBoolContext = true;
+    }
+  }
+
+  if (EnumConstantInBoolContext)
+    Diag(Loc, diag::warn_enum_constant_in_bool_context);
+
   // Diagnose cases where the user write a logical and/or but probably meant a
   // bitwise one.  We do this when the LHS is a non-bool integer and the RHS
   // is a constant.
-  if (LHS.get()->getType()->isIntegerType() &&
+  if (!EnumConstantInBoolContext && LHS.get()->getType()->isIntegerType() &&
       !LHS.get()->getType()->isBooleanType() &&
       RHS.get()->getType()->isIntegerType() && !RHS.get()->isValueDependent() &&
       // Don't warn in macros or template instantiations.
@@ -12159,11 +12172,11 @@ static QualType CheckIncrementDecrementOperand(Sema &S, Expr *Op,
   } else if (S.getLangOpts().AltiVec && ResType->isVectorType()) {
     // OK! ( C/C++ Language Extensions for CBEA(Version 2.6) 10.3 )
   } else if (S.getLangOpts().ZVector && ResType->isVectorType() &&
-             (ResType->getAs<VectorType>()->getVectorKind() !=
+             (ResType->castAs<VectorType>()->getVectorKind() !=
               VectorType::AltiVecBool)) {
     // The z vector extensions allow ++ and -- for non-bool vectors.
   } else if(S.getLangOpts().OpenCL && ResType->isVectorType() &&
-            ResType->getAs<VectorType>()->getElementType()->isIntegerType()) {
+            ResType->castAs<VectorType>()->getElementType()->isIntegerType()) {
     // OpenCL V1.2 6.3 says dec/inc ops operate on integer vector types.
   } else {
     S.Diag(OpLoc, diag::err_typecheck_illegal_increment_decrement)
@@ -12749,7 +12762,7 @@ static ExprResult convertHalfVecBinOp(Sema &S, ExprResult LHS, ExprResult RHS,
   LHS = convertVector(LHS.get(), Context.FloatTy, S);
   auto *BO = new (Context) BinaryOperator(LHS.get(), RHS.get(), Opc, BinOpResTy,
                                           VK, OK, OpLoc, FPFeatures);
-  return convertVector(BO, ResultTy->getAs<VectorType>()->getElementType(), S);
+  return convertVector(BO, ResultTy->castAs<VectorType>()->getElementType(), S);
 }
 
 static std::pair<ExprResult, ExprResult>
@@ -13505,7 +13518,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     else if (resultType->isVectorType() &&
              // The z vector extensions don't allow + or - with bool vectors.
              (!Context.getLangOpts().ZVector ||
-              resultType->getAs<VectorType>()->getVectorKind() !=
+              resultType->castAs<VectorType>()->getVectorKind() !=
               VectorType::AltiVecBool))
       break;
     else if (getLangOpts().CPlusPlus && // C++ [expr.unary.op]p6
@@ -13534,7 +13547,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     else if (resultType->isExtVectorType() && Context.getLangOpts().OpenCL) {
       // OpenCL v1.1 s6.3.f: The bitwise operator not (~) does not operate
       // on vector float types.
-      QualType T = resultType->getAs<ExtVectorType>()->getElementType();
+      QualType T = resultType->castAs<ExtVectorType>()->getElementType();
       if (!T->isIntegerType())
         return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                           << resultType << Input.get()->getSourceRange());
@@ -13579,7 +13592,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
           !Context.getLangOpts().OpenCLCPlusPlus) {
         // OpenCL v1.1 6.3.h: The logical operator not (!) does not
         // operate on vector float types.
-        QualType T = resultType->getAs<ExtVectorType>()->getElementType();
+        QualType T = resultType->castAs<ExtVectorType>()->getElementType();
         if (!T->isIntegerType())
           return ExprError(Diag(OpLoc, diag::err_typecheck_unary_expr)
                            << resultType << Input.get()->getSourceRange());
@@ -14244,7 +14257,7 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
 
   // If the user wrote a function type in some form, try to use that.
   if (!BSI->FunctionType.isNull()) {
-    const FunctionType *FTy = BSI->FunctionType->getAs<FunctionType>();
+    const FunctionType *FTy = BSI->FunctionType->castAs<FunctionType>();
 
     FunctionType::ExtInfo Ext = FTy->getExtInfo();
     if (NoReturn && !Ext.getNoReturn()) Ext = Ext.withNoReturn(true);
@@ -14716,24 +14729,24 @@ bool Sema::DiagnoseAssignmentResult(AssignConvertType ConvTy,
   case IncompatibleObjCQualifiedId: {
     if (SrcType->isObjCQualifiedIdType()) {
       const ObjCObjectPointerType *srcOPT =
-                SrcType->getAs<ObjCObjectPointerType>();
+                SrcType->castAs<ObjCObjectPointerType>();
       for (auto *srcProto : srcOPT->quals()) {
         PDecl = srcProto;
         break;
       }
       if (const ObjCInterfaceType *IFaceT =
-            DstType->getAs<ObjCObjectPointerType>()->getInterfaceType())
+            DstType->castAs<ObjCObjectPointerType>()->getInterfaceType())
         IFace = IFaceT->getDecl();
     }
     else if (DstType->isObjCQualifiedIdType()) {
       const ObjCObjectPointerType *dstOPT =
-        DstType->getAs<ObjCObjectPointerType>();
+        DstType->castAs<ObjCObjectPointerType>();
       for (auto *dstProto : dstOPT->quals()) {
         PDecl = dstProto;
         break;
       }
       if (const ObjCInterfaceType *IFaceT =
-            SrcType->getAs<ObjCObjectPointerType>()->getInterfaceType())
+            SrcType->castAs<ObjCObjectPointerType>()->getInterfaceType())
         IFace = IFaceT->getDecl();
     }
     DiagKind = diag::warn_incompatible_qualified_id;
@@ -15561,6 +15574,7 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func,
   }
 
   if (LangOpts.OpenMP) {
+    markOpenMPDeclareVariantFuncsReferenced(Loc, Func, MightBeOdrUse);
     if (LangOpts.OpenMPIsDevice)
       checkOpenMPDeviceFunction(Loc, Func);
     else
diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp
index 21b50b7c685ad..a0a32d1dd2c61 100644
--- a/clang/lib/Sema/SemaExprCXX.cpp
+++ b/clang/lib/Sema/SemaExprCXX.cpp
@@ -4615,7 +4615,9 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, TypeTrait UTT,
       return RD->hasAttr<FinalAttr>();
     return false;
   case UTT_IsSigned:
-    return T->isSignedIntegerType();
+    // Enum types should always return false.
+    // Floating points should always return true.
+    return !T->isEnumeralType() && (T->isFloatingType() || T->isSignedIntegerType());
   case UTT_IsUnsigned:
     return T->isUnsignedIntegerType();
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index a114164186d0b..21a32dbeea1c3 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -974,7 +974,7 @@ int InitListChecker::numArrayElements(QualType DeclType) {
 }
 
 int InitListChecker::numStructUnionElements(QualType DeclType) {
-  RecordDecl *structDecl = DeclType->getAs<RecordType>()->getDecl();
+  RecordDecl *structDecl = DeclType->castAs<RecordType>()->getDecl();
   int InitializableMembers = 0;
   if (auto *CXXRD = dyn_cast<CXXRecordDecl>(structDecl))
     InitializableMembers += CXXRD->getNumBases();
@@ -1033,7 +1033,7 @@ void InitListChecker::CheckImplicitInitList(const InitializedEntity &Entity,
   else if (T->isRecordType())
     maxElements = numStructUnionElements(T);
   else if (T->isVectorType())
-    maxElements = T->getAs<VectorType>()->getNumElements();
+    maxElements = T->castAs<VectorType>()->getNumElements();
   else
     llvm_unreachable("CheckImplicitInitList(): Illegal type");
 
@@ -1264,7 +1264,7 @@ void InitListChecker::CheckListElementTypes(const InitializedEntity &Entity,
   } else if (DeclType->isRecordType()) {
     assert(DeclType->isAggregateType() &&
            "non-aggregate records should be handed in CheckSubElementType");
-    RecordDecl *RD = DeclType->getAs<RecordType>()->getDecl();
+    RecordDecl *RD = DeclType->castAs<RecordType>()->getDecl();
     auto Bases =
         CXXRecordDecl::base_class_range(CXXRecordDecl::base_class_iterator(),
                                         CXXRecordDecl::base_class_iterator());
@@ -1490,7 +1490,7 @@ void InitListChecker::CheckComplexType(const InitializedEntity &Entity,
         << IList->getSourceRange();
 
   // Initialize the complex number.
-  QualType elementType = DeclType->getAs<ComplexType>()->getElementType();
+  QualType elementType = DeclType->castAs<ComplexType>()->getElementType();
   InitializedEntity ElementEntity =
     InitializedEntity::InitializeElement(SemaRef.Context, 0, Entity);
 
@@ -1636,7 +1636,7 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
                                       unsigned &Index,
                                       InitListExpr *StructuredList,
                                       unsigned &StructuredIndex) {
-  const VectorType *VT = DeclType->getAs<VectorType>();
+  const VectorType *VT = DeclType->castAs<VectorType>();
   unsigned maxElements = VT->getNumElements();
   unsigned numEltsInit = 0;
   QualType elementType = VT->getElementType();
@@ -1706,7 +1706,7 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
       return;
 
     bool isBigEndian = SemaRef.Context.getTargetInfo().isBigEndian();
-    const VectorType *T = Entity.getType()->getAs<VectorType>();
+    const VectorType *T = Entity.getType()->castAs<VectorType>();
     if (isBigEndian && (T->getVectorKind() == VectorType::NeonVector ||
                         T->getVectorKind() == VectorType::NeonPolyVector)) {
       // The ability to use vector initializer lists is a GNU vector extension
@@ -1762,7 +1762,7 @@ void InitListChecker::CheckVectorType(const InitializedEntity &Entity,
       ++numEltsInit;
     } else {
       QualType VecType;
-      const VectorType *IVT = IType->getAs<VectorType>();
+      const VectorType *IVT = IType->castAs<VectorType>();
       unsigned numIElts = IVT->getNumElements();
 
       if (IType->isExtVectorType())
@@ -1995,7 +1995,7 @@ void InitListChecker::CheckStructUnionTypes(
     bool SubobjectIsDesignatorContext, unsigned &Index,
     InitListExpr *StructuredList, unsigned &StructuredIndex,
     bool TopLevelObject) {
-  RecordDecl *structDecl = DeclType->getAs<RecordType>()->getDecl();
+  RecordDecl *structDecl = DeclType->castAs<RecordType>()->getDecl();
 
   // If the record is invalid, some of it's members are invalid. To avoid
   // confusion, we forgo checking the intializer for the entire record.
@@ -2007,7 +2007,7 @@ void InitListChecker::CheckStructUnionTypes(
   }
 
   if (DeclType->isUnionType() && IList->getNumInits() == 0) {
-    RecordDecl *RD = DeclType->getAs<RecordType>()->getDecl();
+    RecordDecl *RD = DeclType->castAs<RecordType>()->getDecl();
 
     if (!VerifyOnly)
       for (FieldDecl *FD : RD->fields()) {
@@ -2082,7 +2082,7 @@ void InitListChecker::CheckStructUnionTypes(
   // anything except look at designated initializers; That's okay,
   // because an error should get printed out elsewhere. It might be
   // worthwhile to skip over the rest of the initializer, though.
-  RecordDecl *RD = DeclType->getAs<RecordType>()->getDecl();
+  RecordDecl *RD = DeclType->castAs<RecordType>()->getDecl();
   RecordDecl::field_iterator FieldEnd = RD->field_end();
   bool CheckForMissingFields =
     !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts());
@@ -4206,7 +4206,7 @@ static void TryReferenceListInitialization(Sema &S,
   }
 
   QualType DestType = Entity.getType();
-  QualType cv1T1 = DestType->getAs<ReferenceType>()->getPointeeType();
+  QualType cv1T1 = DestType->castAs<ReferenceType>()->getPointeeType();
   Qualifiers T1Quals;
   QualType T1 = S.Context.getUnqualifiedArrayType(cv1T1, T1Quals);
 
@@ -4463,7 +4463,7 @@ static OverloadingResult TryRefInitWithConversionFunction(
     Expr *Initializer, bool AllowRValues, bool IsLValueRef,
     InitializationSequence &Sequence) {
   QualType DestType = Entity.getType();
-  QualType cv1T1 = DestType->getAs<ReferenceType>()->getPointeeType();
+  QualType cv1T1 = DestType->castAs<ReferenceType>()->getPointeeType();
   QualType T1 = cv1T1.getUnqualifiedType();
   QualType cv2T2 = Initializer->getType();
   QualType T2 = cv2T2.getUnqualifiedType();
@@ -4656,7 +4656,7 @@ static void TryReferenceInitialization(Sema &S,
                                        Expr *Initializer,
                                        InitializationSequence &Sequence) {
   QualType DestType = Entity.getType();
-  QualType cv1T1 = DestType->getAs<ReferenceType>()->getPointeeType();
+  QualType cv1T1 = DestType->castAs<ReferenceType>()->getPointeeType();
   Qualifiers T1Quals;
   QualType T1 = S.Context.getUnqualifiedArrayType(cv1T1, T1Quals);
   QualType cv2T2 = Initializer->getType();
@@ -7580,34 +7580,27 @@ static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr,
   if (!DestType->isRecordType())
     return;
 
-  const CXXConstructExpr *CCE =
-      dyn_cast<CXXConstructExpr>(InitExpr->IgnoreParens());
-  if (!CCE || CCE->getNumArgs() != 1)
-    return;
+  unsigned DiagID = 0;
+  if (IsReturnStmt) {
+    const CXXConstructExpr *CCE =
+        dyn_cast<CXXConstructExpr>(InitExpr->IgnoreParens());
+    if (!CCE || CCE->getNumArgs() != 1)
+      return;
 
-  if (!CCE->getConstructor()->isCopyOrMoveConstructor())
-    return;
+    if (!CCE->getConstructor()->isCopyOrMoveConstructor())
+      return;
 
-  InitExpr = CCE->getArg(0)->IgnoreImpCasts();
+    InitExpr = CCE->getArg(0)->IgnoreImpCasts();
+  }
 
   // Find the std::move call and get the argument.
   const CallExpr *CE = dyn_cast<CallExpr>(InitExpr->IgnoreParens());
   if (!CE || !CE->isCallToStdMove())
     return;
 
-  const Expr *Arg = CE->getArg(0);
+  const Expr *Arg = CE->getArg(0)->IgnoreImplicit();
 
-  unsigned DiagID = 0;
-
-  if (!IsReturnStmt && !isa<MaterializeTemporaryExpr>(Arg))
-    return;
-
-  if (isa<MaterializeTemporaryExpr>(Arg)) {
-    DiagID = diag::warn_pessimizing_move_on_initialization;
-    const Expr *ArgStripped = Arg->IgnoreImplicit()->IgnoreParens();
-    if (!ArgStripped->isRValue() || !ArgStripped->getType()->isRecordType())
-      return;
-  } else { // IsReturnStmt
+  if (IsReturnStmt) {
     const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Arg->IgnoreParenImpCasts());
     if (!DRE || DRE->refersToEnclosingVariableOrCapture())
       return;
@@ -7634,18 +7627,24 @@ static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr,
       DiagID = diag::warn_redundant_move_on_return;
     else
       DiagID = diag::warn_pessimizing_move_on_return;
+  } else {
+    DiagID = diag::warn_pessimizing_move_on_initialization;
+    const Expr *ArgStripped = Arg->IgnoreImplicit()->IgnoreParens();
+    if (!ArgStripped->isRValue() || !ArgStripped->getType()->isRecordType())
+      return;
   }
 
   S.Diag(CE->getBeginLoc(), DiagID);
 
   // Get all the locations for a fix-it.  Don't emit the fix-it if any location
   // is within a macro.
-  SourceLocation BeginLoc = CCE->getBeginLoc();
-  if (BeginLoc.isMacroID())
+  SourceLocation CallBegin = CE->getCallee()->getBeginLoc();
+  if (CallBegin.isMacroID())
     return;
   SourceLocation RParen = CE->getRParenLoc();
   if (RParen.isMacroID())
     return;
+  SourceLocation LParen;
   SourceLocation ArgLoc = Arg->getBeginLoc();
 
   // Special testing for the argument location.  Since the fix-it needs the
@@ -7656,16 +7655,14 @@ static void CheckMoveOnConstruction(Sema &S, const Expr *InitExpr,
     ArgLoc = S.getSourceManager().getImmediateExpansionRange(ArgLoc).getBegin();
   }
 
-  SourceLocation LParen = ArgLoc.getLocWithOffset(-1);
   if (LParen.isMacroID())
     return;
-  SourceLocation EndLoc = CCE->getEndLoc();
-  if (EndLoc.isMacroID())
-    return;
+
+  LParen = ArgLoc.getLocWithOffset(-1);
 
   S.Diag(CE->getBeginLoc(), diag::note_remove_move)
-      << FixItHint::CreateRemoval(SourceRange(BeginLoc, LParen))
-      << FixItHint::CreateRemoval(SourceRange(RParen, EndLoc));
+      << FixItHint::CreateRemoval(SourceRange(CallBegin, LParen))
+      << FixItHint::CreateRemoval(SourceRange(RParen, RParen));
 }
 
 static void CheckForNullPointerDereference(Sema &S, const Expr *E) {
@@ -8204,7 +8201,7 @@ ExprResult InitializationSequence::Perform(Sema &S,
           Ty = S.Context.getRValueReferenceType(Ty);
         else if ((*ResultType)->isLValueReferenceType())
           Ty = S.Context.getLValueReferenceType(Ty,
-            (*ResultType)->getAs<LValueReferenceType>()->isSpelledAsLValue());
+            (*ResultType)->castAs<LValueReferenceType>()->isSpelledAsLValue());
         *ResultType = Ty;
       }
 
@@ -8672,7 +8669,7 @@ static void diagnoseListInit(Sema &S, const InitializedEntity &Entity,
     // A list-initialization failure for a reference means that we tried to
     // create a temporary of the inner type (per [dcl.init.list]p3.6) and the
     // inner initialization failed.
-    QualType T = DestType->getAs<ReferenceType>()->getPointeeType();
+    QualType T = DestType->castAs<ReferenceType>()->getPointeeType();
     diagnoseListInit(S, InitializedEntity::InitializeTemporary(T), InitList);
     SourceLocation Loc = InitList->getBeginLoc();
     if (auto *D = Entity.getDecl())
@@ -9029,7 +9026,7 @@ bool InitializationSequence::Diagnose(Sema &S,
               << InheritedFrom;
 
             RecordDecl *BaseDecl
-              = Entity.getBaseSpecifier()->getType()->getAs<RecordType>()
+              = Entity.getBaseSpecifier()->getType()->castAs<RecordType>()
                                                                   ->getDecl();
             S.Diag(BaseDecl->getLocation(), diag::note_previous_decl)
               << S.Context.getTagDeclType(BaseDecl);
diff --git a/clang/lib/Sema/SemaLookup.cpp b/clang/lib/Sema/SemaLookup.cpp
index 83825275782b1..fdb8f488471f6 100644
--- a/clang/lib/Sema/SemaLookup.cpp
+++ b/clang/lib/Sema/SemaLookup.cpp
@@ -764,6 +764,13 @@ static void InsertOCLBuiltinDeclarationsFromTable(Sema &S, LookupResult &LR,
         BuiltinTable[FctIndex + SignatureIndex];
     ASTContext &Context = S.Context;
 
+    // Ignore this BIF if its version does not match the language options.
+    if (Context.getLangOpts().OpenCLVersion < OpenCLBuiltin.MinVersion)
+      continue;
+    if ((OpenCLBuiltin.MaxVersion != 0) &&
+        (Context.getLangOpts().OpenCLVersion >= OpenCLBuiltin.MaxVersion))
+      continue;
+
     SmallVector<QualType, 1> RetTypes;
     SmallVector<SmallVector<QualType, 1>, 5> ArgTypes;
 
@@ -2822,7 +2829,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
 #define NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) case Type::Class:
 #define ABSTRACT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
       // T is canonical.  We can also ignore dependent types because
       // we don't need to do ADL at the definition point, but if we
       // wanted to implement template export (or if we find some other
@@ -3701,6 +3708,7 @@ NamedDecl *VisibleDeclsRecord::checkHidden(NamedDecl *ND) {
   return nullptr;
 }
 
+namespace {
 class LookupVisibleHelper {
 public:
   LookupVisibleHelper(VisibleDeclConsumer &Consumer, bool IncludeDependentBases,
@@ -4025,6 +4033,7 @@ class LookupVisibleHelper {
   bool IncludeDependentBases;
   bool LoadExternal;
 };
+} // namespace
 
 void Sema::LookupVisibleDecls(Scope *S, LookupNameKind Kind,
                               VisibleDeclConsumer &Consumer,
@@ -5273,8 +5282,11 @@ static NamedDecl *getDefinitionToImport(NamedDecl *D) {
     return FD->getDefinition();
   if (TagDecl *TD = dyn_cast<TagDecl>(D))
     return TD->getDefinition();
+  // The first definition for this ObjCInterfaceDecl might be in the TU
+  // and not associated with any module. Use the one we know to be complete
+  // and have just seen in a module.
   if (ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(D))
-    return ID->getDefinition();
+    return ID;
   if (ObjCProtocolDecl *PD = dyn_cast<ObjCProtocolDecl>(D))
     return PD->getDefinition();
   if (TemplateDecl *TD = dyn_cast<TemplateDecl>(D))
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 10de0ca91221c..1fca351bfb096 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -31,6 +31,8 @@ static void checkModuleImportContext(Sema &S, Module *M,
         ExternCLoc = LSD->getBeginLoc();
       break;
     case LinkageSpecDecl::lang_cxx:
+    case LinkageSpecDecl::lang_cxx_11:
+    case LinkageSpecDecl::lang_cxx_14:
       break;
     }
     DC = LSD->getParent();
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index dbf8155d029f8..2b659f7f11f98 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -170,7 +170,7 @@ class DSAStackTy {
   OpenMPClauseKind ClauseKindMode = OMPC_unknown;
   Sema &SemaRef;
   bool ForceCapturing = false;
-  /// true if all the vaiables in the target executable directives must be
+  /// true if all the variables in the target executable directives must be
   /// captured by reference.
   bool ForceCaptureByReferenceInTargetExecutable = false;
   CriticalsWithHintsTy Criticals;
@@ -3447,6 +3447,7 @@ void Sema::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind, Scope *CurScope) {
   case OMPD_declare_target:
   case OMPD_end_declare_target:
   case OMPD_requires:
+  case OMPD_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
@@ -4516,6 +4517,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
   case OMPD_declare_mapper:
   case OMPD_declare_simd:
   case OMPD_requires:
+  case OMPD_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
   case OMPD_unknown:
     llvm_unreachable("Unknown OpenMP directive");
@@ -4597,6 +4599,7 @@ StmtResult Sema::ActOnOpenMPExecutableDirective(
       case OMPC_dynamic_allocators:
       case OMPC_atomic_default_mem_order:
       case OMPC_device_type:
+      case OMPC_match:
         llvm_unreachable("Unexpected clause");
       }
       for (Stmt *CC : C->children()) {
@@ -4653,8 +4656,10 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareSimdDirective(
   if (!DG || DG.get().isNull())
     return DeclGroupPtrTy();
 
+  const int SimdId = 0;
   if (!DG.get().isSingleDecl()) {
-    Diag(SR.getBegin(), diag::err_omp_single_decl_in_declare_simd);
+    Diag(SR.getBegin(), diag::err_omp_single_decl_in_declare_simd_variant)
+        << SimdId;
     return DG;
   }
   Decl *ADecl = DG.get().getSingleDecl();
@@ -4663,7 +4668,7 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareSimdDirective(
 
   auto *FD = dyn_cast<FunctionDecl>(ADecl);
   if (!FD) {
-    Diag(ADecl->getLocation(), diag::err_omp_function_expected);
+    Diag(ADecl->getLocation(), diag::err_omp_function_expected) << SimdId;
     return DeclGroupPtrTy();
   }
 
@@ -4885,7 +4890,265 @@ Sema::DeclGroupPtrTy Sema::ActOnOpenMPDeclareSimdDirective(
       const_cast<unsigned *>(LinModifiers.data()), LinModifiers.size(),
       NewSteps.data(), NewSteps.size(), SR);
   ADecl->addAttr(NewAttr);
-  return ConvertDeclToDeclGroup(ADecl);
+  return DG;
+}
+
+Optional<std::pair<FunctionDecl *, Expr *>>
+Sema::checkOpenMPDeclareVariantFunction(Sema::DeclGroupPtrTy DG,
+                                        Expr *VariantRef, SourceRange SR) {
+  if (!DG || DG.get().isNull())
+    return None;
+
+  const int VariantId = 1;
+  // Must be applied only to single decl.
+  if (!DG.get().isSingleDecl()) {
+    Diag(SR.getBegin(), diag::err_omp_single_decl_in_declare_simd_variant)
+        << VariantId << SR;
+    return None;
+  }
+  Decl *ADecl = DG.get().getSingleDecl();
+  if (auto *FTD = dyn_cast<FunctionTemplateDecl>(ADecl))
+    ADecl = FTD->getTemplatedDecl();
+
+  // Decl must be a function.
+  auto *FD = dyn_cast<FunctionDecl>(ADecl);
+  if (!FD) {
+    Diag(ADecl->getLocation(), diag::err_omp_function_expected)
+        << VariantId << SR;
+    return None;
+  }
+
+  auto &&HasMultiVersionAttributes = [](const FunctionDecl *FD) {
+    return FD->hasAttrs() &&
+           (FD->hasAttr<CPUDispatchAttr>() || FD->hasAttr<CPUSpecificAttr>() ||
+            FD->hasAttr<TargetAttr>());
+  };
+  // OpenMP is not compatible with CPU-specific attributes.
+  if (HasMultiVersionAttributes(FD)) {
+    Diag(FD->getLocation(), diag::err_omp_declare_variant_incompat_attributes)
+        << SR;
+    return None;
+  }
+
+  // Allow #pragma omp declare variant only if the function is not used.
+  if (FD->isUsed(false))
+    Diag(SR.getBegin(), diag::warn_omp_declare_variant_after_used)
+        << FD->getLocation();
+
+  // Check if the function was emitted already.
+  const FunctionDecl *Definition;
+  if (!FD->isThisDeclarationADefinition() && FD->isDefined(Definition) &&
+      (LangOpts.EmitAllDecls || Context.DeclMustBeEmitted(Definition)))
+    Diag(SR.getBegin(), diag::warn_omp_declare_variant_after_emitted)
+        << FD->getLocation();
+
+  // The VariantRef must point to function.
+  if (!VariantRef) {
+    Diag(SR.getBegin(), diag::err_omp_function_expected) << VariantId;
+    return None;
+  }
+
+  // Do not check templates, wait until instantiation.
+  if (VariantRef->isTypeDependent() || VariantRef->isValueDependent() ||
+      VariantRef->containsUnexpandedParameterPack() ||
+      VariantRef->isInstantiationDependent() || FD->isDependentContext())
+    return std::make_pair(FD, VariantRef);
+
+  // Convert VariantRef expression to the type of the original function to
+  // resolve possible conflicts.
+  ExprResult VariantRefCast;
+  if (LangOpts.CPlusPlus) {
+    QualType FnPtrType;
+    auto *Method = dyn_cast<CXXMethodDecl>(FD);
+    if (Method && !Method->isStatic()) {
+      const Type *ClassType =
+          Context.getTypeDeclType(Method->getParent()).getTypePtr();
+      FnPtrType = Context.getMemberPointerType(FD->getType(), ClassType);
+      ExprResult ER;
+      {
+        // Build adrr_of unary op to correctly handle type checks for member
+        // functions.
+        Sema::TentativeAnalysisScope Trap(*this);
+        ER = CreateBuiltinUnaryOp(VariantRef->getBeginLoc(), UO_AddrOf,
+                                  VariantRef);
+      }
+      if (!ER.isUsable()) {
+        Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
+            << VariantId << VariantRef->getSourceRange();
+        return None;
+      }
+      VariantRef = ER.get();
+    } else {
+      FnPtrType = Context.getPointerType(FD->getType());
+    }
+    ImplicitConversionSequence ICS =
+        TryImplicitConversion(VariantRef, FnPtrType.getUnqualifiedType(),
+                              /*SuppressUserConversions=*/false,
+                              /*AllowExplicit=*/false,
+                              /*InOverloadResolution=*/false,
+                              /*CStyle=*/false,
+                              /*AllowObjCWritebackConversion=*/false);
+    if (ICS.isFailure()) {
+      Diag(VariantRef->getExprLoc(),
+           diag::err_omp_declare_variant_incompat_types)
+          << VariantRef->getType() << FnPtrType << VariantRef->getSourceRange();
+      return None;
+    }
+    VariantRefCast = PerformImplicitConversion(
+        VariantRef, FnPtrType.getUnqualifiedType(), AA_Converting);
+    if (!VariantRefCast.isUsable())
+      return None;
+    // Drop previously built artificial addr_of unary op for member functions.
+    if (Method && !Method->isStatic()) {
+      Expr *PossibleAddrOfVariantRef = VariantRefCast.get();
+      if (auto *UO = dyn_cast<UnaryOperator>(
+              PossibleAddrOfVariantRef->IgnoreImplicit()))
+        VariantRefCast = UO->getSubExpr();
+    }
+  } else {
+    VariantRefCast = VariantRef;
+  }
+
+  ExprResult ER = CheckPlaceholderExpr(VariantRefCast.get());
+  if (!ER.isUsable() ||
+      !ER.get()->IgnoreParenImpCasts()->getType()->isFunctionType()) {
+    Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
+        << VariantId << VariantRef->getSourceRange();
+    return None;
+  }
+
+  // The VariantRef must point to function.
+  auto *DRE = dyn_cast<DeclRefExpr>(ER.get()->IgnoreParenImpCasts());
+  if (!DRE) {
+    Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
+        << VariantId << VariantRef->getSourceRange();
+    return None;
+  }
+  auto *NewFD = dyn_cast_or_null<FunctionDecl>(DRE->getDecl());
+  if (!NewFD) {
+    Diag(VariantRef->getExprLoc(), diag::err_omp_function_expected)
+        << VariantId << VariantRef->getSourceRange();
+    return None;
+  }
+
+  // Check if variant function is not marked with declare variant directive.
+  if (NewFD->hasAttrs() && NewFD->hasAttr<OMPDeclareVariantAttr>()) {
+    Diag(VariantRef->getExprLoc(),
+         diag::warn_omp_declare_variant_marked_as_declare_variant)
+        << VariantRef->getSourceRange();
+    SourceRange SR =
+        NewFD->specific_attr_begin<OMPDeclareVariantAttr>()->getRange();
+    Diag(SR.getBegin(), diag::note_omp_marked_declare_variant_here) << SR;
+    return None;
+  }
+
+  enum DoesntSupport {
+    VirtFuncs = 1,
+    Constructors = 3,
+    Destructors = 4,
+    DeletedFuncs = 5,
+    DefaultedFuncs = 6,
+    ConstexprFuncs = 7,
+    ConstevalFuncs = 8,
+  };
+  if (const auto *CXXFD = dyn_cast<CXXMethodDecl>(FD)) {
+    if (CXXFD->isVirtual()) {
+      Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+          << VirtFuncs;
+      return None;
+    }
+
+    if (isa<CXXConstructorDecl>(FD)) {
+      Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+          << Constructors;
+      return None;
+    }
+
+    if (isa<CXXDestructorDecl>(FD)) {
+      Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+          << Destructors;
+      return None;
+    }
+  }
+
+  if (FD->isDeleted()) {
+    Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+        << DeletedFuncs;
+    return None;
+  }
+
+  if (FD->isDefaulted()) {
+    Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+        << DefaultedFuncs;
+    return None;
+  }
+
+  if (FD->isConstexpr()) {
+    Diag(FD->getLocation(), diag::err_omp_declare_variant_doesnt_support)
+        << (NewFD->isConsteval() ? ConstevalFuncs : ConstexprFuncs);
+    return None;
+  }
+
+  // Check general compatibility.
+  if (areMultiversionVariantFunctionsCompatible(
+          FD, NewFD, PDiag(diag::err_omp_declare_variant_noproto),
+          PartialDiagnosticAt(
+              SR.getBegin(),
+              PDiag(diag::note_omp_declare_variant_specified_here) << SR),
+          PartialDiagnosticAt(
+              VariantRef->getExprLoc(),
+              PDiag(diag::err_omp_declare_variant_doesnt_support)),
+          PartialDiagnosticAt(VariantRef->getExprLoc(),
+                              PDiag(diag::err_omp_declare_variant_diff)
+                                  << FD->getLocation()),
+          /*TemplatesSupported=*/true, /*ConstexprSupported=*/false))
+    return None;
+  return std::make_pair(FD, cast<Expr>(DRE));
+}
+
+void Sema::ActOnOpenMPDeclareVariantDirective(
+    FunctionDecl *FD, Expr *VariantRef, SourceRange SR,
+    const Sema::OpenMPDeclareVariantCtsSelectorData &Data) {
+  if (Data.CtxSet == OMPDeclareVariantAttr::CtxSetUnknown ||
+      Data.Ctx == OMPDeclareVariantAttr::CtxUnknown)
+    return;
+  Expr *Score = nullptr;
+  OMPDeclareVariantAttr::ScoreType ST = OMPDeclareVariantAttr::ScoreUnknown;
+  if (Data.CtxScore.isUsable()) {
+    ST = OMPDeclareVariantAttr::ScoreSpecified;
+    Score = Data.CtxScore.get();
+    if (!Score->isTypeDependent() && !Score->isValueDependent() &&
+        !Score->isInstantiationDependent() &&
+        !Score->containsUnexpandedParameterPack()) {
+      llvm::APSInt Result;
+      ExprResult ICE = VerifyIntegerConstantExpression(Score, &Result);
+      if (ICE.isInvalid())
+        return;
+    }
+  }
+  auto *NewAttr = OMPDeclareVariantAttr::CreateImplicit(
+      Context, VariantRef, Score, Data.CtxSet, ST, Data.Ctx, Data.ImplVendor,
+      SR);
+  FD->addAttr(NewAttr);
+}
+
+void Sema::markOpenMPDeclareVariantFuncsReferenced(SourceLocation Loc,
+                                                   FunctionDecl *Func,
+                                                   bool MightBeOdrUse) {
+  assert(LangOpts.OpenMP && "Expected OpenMP mode.");
+
+  if (!Func->isDependentContext() && Func->hasAttrs()) {
+    for (OMPDeclareVariantAttr *A :
+         Func->specific_attrs<OMPDeclareVariantAttr>()) {
+      // TODO: add checks for active OpenMP context where possible.
+      Expr *VariantRef = A->getVariantFuncRef();
+      auto *DRE = dyn_cast<DeclRefExpr>(VariantRef->IgnoreParenImpCasts());
+      auto *F = cast<FunctionDecl>(DRE->getDecl());
+      if (!F->isDefined() && F->isTemplateInstantiation())
+        InstantiateFunctionDefinition(Loc, F->getFirstDecl());
+      MarkFunctionReferenced(Loc, F, MightBeOdrUse);
+    }
+  }
 }
 
 StmtResult Sema::ActOnOpenMPParallelDirective(ArrayRef<OMPClause *> Clauses,
@@ -5999,13 +6262,21 @@ Expr *OpenMPIterationSpaceChecker::buildFinalCondition(Scope *S) const {
 Expr *OpenMPIterationSpaceChecker::buildPreCond(
     Scope *S, Expr *Cond,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures) const {
+  // Do not build a precondition when the condition/initialization is dependent
+  // to prevent pessimistic early loop exit.
+  // TODO: this can be improved by calculating min/max values but not sure that
+  // it will be very effective.
+  if (CondDependOnLC || InitDependOnLC)
+    return SemaRef.PerformImplicitConversion(
+        SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(),
+        SemaRef.Context.BoolTy, /*Action=*/Sema::AA_Casting,
+        /*AllowExplicit=*/true).get();
+
   // Try to build LB <op> UB, where <op> is <, >, <=, or >=.
   Sema::TentativeAnalysisScope Trap(SemaRef);
 
-  ExprResult NewLB =
-      InitDependOnLC ? LB : tryBuildCapture(SemaRef, LB, Captures);
-  ExprResult NewUB =
-      CondDependOnLC ? UB : tryBuildCapture(SemaRef, UB, Captures);
+  ExprResult NewLB = tryBuildCapture(SemaRef, LB, Captures);
+  ExprResult NewUB = tryBuildCapture(SemaRef, UB, Captures);
   if (!NewLB.isUsable() || !NewUB.isUsable())
     return nullptr;
 
@@ -7177,7 +7448,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       Built.DependentCounters[Cnt] = nullptr;
       Built.DependentInits[Cnt] = nullptr;
       Built.FinalsConditions[Cnt] = nullptr;
-      if (IS.IsNonRectangularLB) {
+      if (IS.IsNonRectangularLB || IS.IsNonRectangularUB) {
         Built.DependentCounters[Cnt] =
             Built.Counters[NestedLoopCount - 1 - IS.LoopDependentIdx];
         Built.DependentInits[Cnt] =
@@ -9826,6 +10097,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind, Expr *Expr,
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -9895,6 +10167,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_teams:
@@ -9963,6 +10236,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_teams:
@@ -10032,6 +10306,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -10098,6 +10373,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -10165,6 +10441,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -10231,6 +10508,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -10296,6 +10574,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
     case OMPD_declare_reduction:
     case OMPD_declare_mapper:
     case OMPD_declare_simd:
+    case OMPD_declare_variant:
     case OMPD_declare_target:
     case OMPD_end_declare_target:
     case OMPD_simd:
@@ -10369,6 +10648,7 @@ static OpenMPDirectiveKind getOpenMPCaptureRegionForClause(
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Unexpected OpenMP clause.");
   }
   return CaptureRegion;
@@ -10763,6 +11043,7 @@ OMPClause *Sema::ActOnOpenMPSimpleClause(
   case OMPC_reverse_offload:
   case OMPC_dynamic_allocators:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -10942,6 +11223,7 @@ OMPClause *Sema::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -11152,6 +11434,7 @@ OMPClause *Sema::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_is_device_ptr:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -11359,6 +11642,7 @@ OMPClause *Sema::ActOnOpenMPVarListClause(
   case OMPC_dynamic_allocators:
   case OMPC_atomic_default_mem_order:
   case OMPC_device_type:
+  case OMPC_match:
     llvm_unreachable("Clause is not allowed.");
   }
   return Res;
@@ -14494,6 +14778,11 @@ static ExprResult buildUserDefinedMapperRef(Sema &SemaRef, Scope *S,
                                             Expr *UnresolvedMapper) {
   if (MapperIdScopeSpec.isInvalid())
     return ExprError();
+  // Get the actual type for the array type.
+  if (Type->isArrayType()) {
+    assert(Type->getAsArrayTypeUnsafe() && "Expect to get a valid array type");
+    Type = Type->getAsArrayTypeUnsafe()->getElementType().getCanonicalType();
+  }
   // Find all user-defined mappers with the given MapperId.
   SmallVector<UnresolvedSet<8>, 4> Lookups;
   LookupResult Lookup(SemaRef, MapperId, Sema::LookupOMPMapperName);
@@ -14540,11 +14829,14 @@ static ExprResult buildUserDefinedMapperRef(Sema &SemaRef, Scope *S,
         MapperIdScopeSpec.getWithLocInContext(SemaRef.Context), MapperId,
         /*ADL=*/false, /*Overloaded=*/true, URS.begin(), URS.end());
   }
+  SourceLocation Loc = MapperId.getLoc();
   // [OpenMP 5.0], 2.19.7.3 declare mapper Directive, Restrictions
   //  The type must be of struct, union or class type in C and C++
-  if (!Type->isStructureOrClassType() && !Type->isUnionType())
-    return ExprEmpty();
-  SourceLocation Loc = MapperId.getLoc();
+  if (!Type->isStructureOrClassType() && !Type->isUnionType() &&
+      (MapperIdScopeSpec.isSet() || MapperId.getAsString() != "default")) {
+    SemaRef.Diag(Loc, diag::err_omp_mapper_wrong_type);
+    return ExprError();
+  }
   // Perform argument dependent lookup.
   if (SemaRef.getLangOpts().CPlusPlus && !MapperIdScopeSpec.isSet())
     argumentDependentLookup(SemaRef, MapperId, Loc, Type, Lookups);
diff --git a/clang/lib/Sema/SemaOverload.cpp b/clang/lib/Sema/SemaOverload.cpp
index bd6788c1d0d31..cc93ad9195f1b 100644
--- a/clang/lib/Sema/SemaOverload.cpp
+++ b/clang/lib/Sema/SemaOverload.cpp
@@ -257,8 +257,18 @@ isPointerConversionToVoidPointer(ASTContext& Context) const {
 
 /// Skip any implicit casts which could be either part of a narrowing conversion
 /// or after one in an implicit conversion.
-static const Expr *IgnoreNarrowingConversion(const Expr *Converted) {
-  while (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Converted)) {
+static const Expr *IgnoreNarrowingConversion(ASTContext &Ctx,
+                                             const Expr *Converted) {
+  // We can have cleanups wrapping the converted expression; these need to be
+  // preserved so that destructors run if necessary.
+  if (auto *EWC = dyn_cast<ExprWithCleanups>(Converted)) {
+    Expr *Inner =
+        const_cast<Expr *>(IgnoreNarrowingConversion(Ctx, EWC->getSubExpr()));
+    return ExprWithCleanups::Create(Ctx, Inner, EWC->cleanupsHaveSideEffects(),
+                                    EWC->getObjects());
+  }
+
+  while (auto *ICE = dyn_cast<ImplicitCastExpr>(Converted)) {
     switch (ICE->getCastKind()) {
     case CK_NoOp:
     case CK_IntegralCast:
@@ -332,7 +342,7 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
       if (IgnoreFloatToIntegralConversion)
         return NK_Not_Narrowing;
       llvm::APSInt IntConstantValue;
-      const Expr *Initializer = IgnoreNarrowingConversion(Converted);
+      const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted);
       assert(Initializer && "Unknown conversion expression");
 
       // If it's value-dependent, we can't tell whether it's narrowing.
@@ -370,7 +380,7 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
     if (FromType->isRealFloatingType() && ToType->isRealFloatingType() &&
         Ctx.getFloatingTypeOrder(FromType, ToType) == 1) {
       // FromType is larger than ToType.
-      const Expr *Initializer = IgnoreNarrowingConversion(Converted);
+      const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted);
 
       // If it's value-dependent, we can't tell whether it's narrowing.
       if (Initializer->isValueDependent())
@@ -416,7 +426,7 @@ NarrowingKind StandardConversionSequence::getNarrowingKind(
         (FromSigned && !ToSigned)) {
       // Not all values of FromType can be represented in ToType.
       llvm::APSInt InitializerValue;
-      const Expr *Initializer = IgnoreNarrowingConversion(Converted);
+      const Expr *Initializer = IgnoreNarrowingConversion(Ctx, Converted);
 
       // If it's value-dependent, we can't tell whether it's narrowing.
       if (Initializer->isValueDependent())
@@ -1463,14 +1473,14 @@ bool Sema::IsFunctionConversion(QualType FromType, QualType ToType,
   if (TyClass != CanFrom->getTypeClass()) return false;
   if (TyClass != Type::FunctionProto && TyClass != Type::FunctionNoProto) {
     if (TyClass == Type::Pointer) {
-      CanTo = CanTo.getAs<PointerType>()->getPointeeType();
-      CanFrom = CanFrom.getAs<PointerType>()->getPointeeType();
+      CanTo = CanTo.castAs<PointerType>()->getPointeeType();
+      CanFrom = CanFrom.castAs<PointerType>()->getPointeeType();
     } else if (TyClass == Type::BlockPointer) {
-      CanTo = CanTo.getAs<BlockPointerType>()->getPointeeType();
-      CanFrom = CanFrom.getAs<BlockPointerType>()->getPointeeType();
+      CanTo = CanTo.castAs<BlockPointerType>()->getPointeeType();
+      CanFrom = CanFrom.castAs<BlockPointerType>()->getPointeeType();
     } else if (TyClass == Type::MemberPointer) {
-      auto ToMPT = CanTo.getAs<MemberPointerType>();
-      auto FromMPT = CanFrom.getAs<MemberPointerType>();
+      auto ToMPT = CanTo.castAs<MemberPointerType>();
+      auto FromMPT = CanFrom.castAs<MemberPointerType>();
       // A function pointer conversion cannot change the class of the function.
       if (ToMPT->getClass() != FromMPT->getClass())
         return false;
@@ -2273,7 +2283,7 @@ bool Sema::IsPointerConversion(Expr *From, QualType FromType, QualType ToType,
 
   // Blocks: Block pointers can be converted to void*.
   if (FromType->isBlockPointerType() && ToType->isPointerType() &&
-      ToType->getAs<PointerType>()->getPointeeType()->isVoidType()) {
+      ToType->castAs<PointerType>()->getPointeeType()->isVoidType()) {
     ConvertedType = ToType;
     return true;
   }
@@ -3272,7 +3282,7 @@ IsInitializerListConstructorConversion(Sema &S, Expr *From, QualType ToType,
     User.ConversionFunction = Constructor;
     User.FoundConversionFunction = Best->FoundDecl;
     User.After.setAsIdentityConversion();
-    User.After.setFromType(ThisType->getAs<PointerType>()->getPointeeType());
+    User.After.setFromType(ThisType->castAs<PointerType>()->getPointeeType());
     User.After.setAllToTypes(ToType);
     return Result;
   }
@@ -3463,7 +3473,7 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
       User.ConversionFunction = Constructor;
       User.FoundConversionFunction = Best->FoundDecl;
       User.After.setAsIdentityConversion();
-      User.After.setFromType(ThisType->getAs<PointerType>()->getPointeeType());
+      User.After.setFromType(ThisType->castAs<PointerType>()->getPointeeType());
       User.After.setAllToTypes(ToType);
       return Result;
     }
@@ -4105,14 +4115,14 @@ CompareDerivedToBaseConversions(Sema &S, SourceLocation Loc,
       /*FIXME: Remove if Objective-C id conversions get their own rank*/
       FromType1->isPointerType() && FromType2->isPointerType() &&
       ToType1->isPointerType() && ToType2->isPointerType()) {
-    QualType FromPointee1
-      = FromType1->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
-    QualType ToPointee1
-      = ToType1->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
-    QualType FromPointee2
-      = FromType2->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
-    QualType ToPointee2
-      = ToType2->getAs<PointerType>()->getPointeeType().getUnqualifiedType();
+    QualType FromPointee1 =
+        FromType1->castAs<PointerType>()->getPointeeType().getUnqualifiedType();
+    QualType ToPointee1 =
+        ToType1->castAs<PointerType>()->getPointeeType().getUnqualifiedType();
+    QualType FromPointee2 =
+        FromType2->castAs<PointerType>()->getPointeeType().getUnqualifiedType();
+    QualType ToPointee2 =
+        ToType2->castAs<PointerType>()->getPointeeType().getUnqualifiedType();
 
     //   -- conversion of C* to B* is better than conversion of C* to A*,
     if (FromPointee1 == FromPointee2 && ToPointee1 != ToPointee2) {
@@ -4392,7 +4402,7 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
                          bool AllowExplicit) {
   assert(T2->isRecordType() && "Can only find conversions of record types.");
   CXXRecordDecl *T2RecordDecl
-    = dyn_cast<CXXRecordDecl>(T2->getAs<RecordType>()->getDecl());
+    = dyn_cast<CXXRecordDecl>(T2->castAs<RecordType>()->getDecl());
 
   OverloadCandidateSet CandidateSet(
       DeclLoc, OverloadCandidateSet::CSK_InitByUserDefinedConversion);
@@ -4523,7 +4533,7 @@ TryReferenceInit(Sema &S, Expr *Init, QualType DeclType,
   ImplicitConversionSequence ICS;
   ICS.setBad(BadConversionSequence::no_conversion, Init, DeclType);
 
-  QualType T1 = DeclType->getAs<ReferenceType>()->getPointeeType();
+  QualType T1 = DeclType->castAs<ReferenceType>()->getPointeeType();
   QualType T2 = Init->getType();
 
   // If the initializer is the address of an overloaded function, try
@@ -4947,7 +4957,7 @@ TryListConversion(Sema &S, InitListExpr *From, QualType ToType,
     // mention initializer lists in any way. So we go by what list-
     // initialization would do and try to extrapolate from that.
 
-    QualType T1 = ToType->getAs<ReferenceType>()->getPointeeType();
+    QualType T1 = ToType->castAs<ReferenceType>()->getPointeeType();
 
     // If the initializer list has a single element that is reference-related
     // to the parameter type, we initialize the reference from that.
@@ -5217,7 +5227,7 @@ Sema::PerformObjectArgumentInitialization(Expr *From,
                                           CXXMethodDecl *Method) {
   QualType FromRecordType, DestType;
   QualType ImplicitParamRecordType  =
-    Method->getThisType()->getAs<PointerType>()->getPointeeType();
+    Method->getThisType()->castAs<PointerType>()->getPointeeType();
 
   Expr::Classification FromClassification;
   if (const PointerType *PT = From->getType()->getAs<PointerType>()) {
@@ -5465,6 +5475,14 @@ static ExprResult CheckConvertedConstantExpression(Sema &S, Expr *From,
   if (Result.isInvalid())
     return Result;
 
+  // C++2a [intro.execution]p5:
+  //   A full-expression is [...] a constant-expression [...]
+  Result =
+      S.ActOnFinishFullExpr(Result.get(), From->getExprLoc(),
+                            /*DiscardedValue=*/false, /*IsConstexpr=*/true);
+  if (Result.isInvalid())
+    return Result;
+
   // Check for a narrowing implicit conversion.
   APValue PreNarrowingValue;
   QualType PreNarrowingType;
@@ -6996,7 +7014,7 @@ void Sema::AddConversionCandidate(
   if (const PointerType *FromPtrType = ImplicitParamType->getAs<PointerType>())
     ImplicitParamType = FromPtrType->getPointeeType();
   CXXRecordDecl *ConversionContext
-    = cast<CXXRecordDecl>(ImplicitParamType->getAs<RecordType>()->getDecl());
+    = cast<CXXRecordDecl>(ImplicitParamType->castAs<RecordType>()->getDecl());
 
   Candidate.Conversions[0] = TryObjectArgumentInitialization(
       *this, CandidateSet.getLocation(), From->getType(),
@@ -8756,7 +8774,7 @@ class BuiltinOperatorOverloadBuilder {
                   Enum = CandidateTypes[ArgIdx].enumeration_begin(),
                EnumEnd = CandidateTypes[ArgIdx].enumeration_end();
              Enum != EnumEnd; ++Enum) {
-          if (!(*Enum)->getAs<EnumType>()->getDecl()->isScoped())
+          if (!(*Enum)->castAs<EnumType>()->getDecl()->isScoped())
             continue;
 
           if (!AddedTypes.insert(S.Context.getCanonicalType(*Enum)).second)
@@ -9422,13 +9440,15 @@ OverloadCandidateSet::BestViableFunction(Sema &S, SourceLocation Loc,
     const FunctionDecl *Caller = dyn_cast<FunctionDecl>(S.CurContext);
     bool ContainsSameSideCandidate =
         llvm::any_of(Candidates, [&](OverloadCandidate *Cand) {
-          return Cand->Function &&
+          // Check viable function only.
+          return Cand->Viable && Cand->Function &&
                  S.IdentifyCUDAPreference(Caller, Cand->Function) ==
                      Sema::CFP_SameSide;
         });
     if (ContainsSameSideCandidate) {
       auto IsWrongSideCandidate = [&](OverloadCandidate *Cand) {
-        return Cand->Function &&
+        // Check viable function only to avoid unnecessary data copying/moving.
+        return Cand->Viable && Cand->Function &&
                S.IdentifyCUDAPreference(Caller, Cand->Function) ==
                    Sema::CFP_WrongSide;
       };
@@ -10779,12 +10799,12 @@ static void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
       = Cand->Surrogate->getConversionType().getNonReferenceType();
     if (const PointerType *ConvPtrType = ConvType->getAs<PointerType>())
       ConvType = ConvPtrType->getPointeeType();
-    ParamTypes = ConvType->getAs<FunctionProtoType>()->getParamTypes();
+    ParamTypes = ConvType->castAs<FunctionProtoType>()->getParamTypes();
     // Conversion 0 is 'this', which doesn't have a corresponding argument.
     ConvIdx = 1;
   } else if (Cand->Function) {
     ParamTypes =
-        Cand->Function->getType()->getAs<FunctionProtoType>()->getParamTypes();
+        Cand->Function->getType()->castAs<FunctionProtoType>()->getParamTypes();
     if (isa<CXXMethodDecl>(Cand->Function) &&
         !isa<CXXConstructorDecl>(Cand->Function)) {
       // Conversion 0 is 'this', which doesn't have a corresponding argument.
diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp
index a14580d785c90..34c366cc91759 100644
--- a/clang/lib/Sema/SemaStmt.cpp
+++ b/clang/lib/Sema/SemaStmt.cpp
@@ -430,45 +430,44 @@ Sema::ActOnCaseExpr(SourceLocation CaseLoc, ExprResult Val) {
 
   // If we're not inside a switch, let the 'case' statement handling diagnose
   // this. Just clean up after the expression as best we can.
-  if (!getCurFunction()->SwitchStack.empty()) {
-    Expr *CondExpr =
-        getCurFunction()->SwitchStack.back().getPointer()->getCond();
-    if (!CondExpr)
-      return ExprError();
-    QualType CondType = CondExpr->getType();
-
-    auto CheckAndFinish = [&](Expr *E) {
-      if (CondType->isDependentType() || E->isTypeDependent())
-        return ExprResult(E);
-
-      if (getLangOpts().CPlusPlus11) {
-        // C++11 [stmt.switch]p2: the constant-expression shall be a converted
-        // constant expression of the promoted type of the switch condition.
-        llvm::APSInt TempVal;
-        return CheckConvertedConstantExpression(E, CondType, TempVal,
-                                                CCEK_CaseValue);
-      }
+  if (getCurFunction()->SwitchStack.empty())
+    return ActOnFinishFullExpr(Val.get(), Val.get()->getExprLoc(), false,
+                               getLangOpts().CPlusPlus11);
 
-      ExprResult ER = E;
-      if (!E->isValueDependent())
-        ER = VerifyIntegerConstantExpression(E);
-      if (!ER.isInvalid())
-        ER = DefaultLvalueConversion(ER.get());
-      if (!ER.isInvalid())
-        ER = ImpCastExprToType(ER.get(), CondType, CK_IntegralCast);
-      return ER;
-    };
+  Expr *CondExpr =
+      getCurFunction()->SwitchStack.back().getPointer()->getCond();
+  if (!CondExpr)
+    return ExprError();
+  QualType CondType = CondExpr->getType();
 
-    ExprResult Converted = CorrectDelayedTyposInExpr(Val, CheckAndFinish);
-    if (Converted.get() == Val.get())
-      Converted = CheckAndFinish(Val.get());
-    if (Converted.isInvalid())
-      return ExprError();
-    Val = Converted;
-  }
+  auto CheckAndFinish = [&](Expr *E) {
+    if (CondType->isDependentType() || E->isTypeDependent())
+      return ExprResult(E);
+
+    if (getLangOpts().CPlusPlus11) {
+      // C++11 [stmt.switch]p2: the constant-expression shall be a converted
+      // constant expression of the promoted type of the switch condition.
+      llvm::APSInt TempVal;
+      return CheckConvertedConstantExpression(E, CondType, TempVal,
+                                              CCEK_CaseValue);
+    }
 
-  return ActOnFinishFullExpr(Val.get(), Val.get()->getExprLoc(), false,
-                             getLangOpts().CPlusPlus11);
+    ExprResult ER = E;
+    if (!E->isValueDependent())
+      ER = VerifyIntegerConstantExpression(E);
+    if (!ER.isInvalid())
+      ER = DefaultLvalueConversion(ER.get());
+    if (!ER.isInvalid())
+      ER = ImpCastExprToType(ER.get(), CondType, CK_IntegralCast);
+    if (!ER.isInvalid())
+      ER = ActOnFinishFullExpr(ER.get(), ER.get()->getExprLoc(), false);
+    return ER;
+  };
+
+  ExprResult Converted = CorrectDelayedTyposInExpr(Val, CheckAndFinish);
+  if (Converted.get() == Val.get())
+    Converted = CheckAndFinish(Val.get());
+  return Converted;
 }
 
 StmtResult
@@ -3501,6 +3500,14 @@ bool Sema::DeduceFunctionTypeFromReturnExpr(FunctionDecl *FD,
       return true;
   }
 
+  // CUDA: Kernel function must have 'void' return type.
+  if (getLangOpts().CUDA)
+    if (FD->hasAttr<CUDAGlobalAttr>() && !Deduced->isVoidType()) {
+      Diag(FD->getLocation(), diag::err_kern_type_not_void_return)
+          << FD->getType() << FD->getSourceRange();
+      return true;
+    }
+
   //  If a function with a declared return type that contains a placeholder type
   //  has multiple return statements, the return type is deduced for each return
   //  statement. [...] if the type deduced is not the same in each deduction,
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index 91466fa904be4..e189b5235a970 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -1005,15 +1005,10 @@ NamedDecl *Sema::ActOnTypeParameter(Scope *S, bool Typename,
   assert(S->isTemplateParamScope() &&
          "Template type parameter not in template parameter scope!");
 
-  SourceLocation Loc = ParamNameLoc;
-  if (!ParamName)
-    Loc = KeyLoc;
-
   bool IsParameterPack = EllipsisLoc.isValid();
-  TemplateTypeParmDecl *Param
-    = TemplateTypeParmDecl::Create(Context, Context.getTranslationUnitDecl(),
-                                   KeyLoc, Loc, Depth, Position, ParamName,
-                                   Typename, IsParameterPack);
+  TemplateTypeParmDecl *Param = TemplateTypeParmDecl::Create(
+      Context, Context.getTranslationUnitDecl(), KeyLoc, ParamNameLoc, Depth,
+      Position, ParamName, Typename, IsParameterPack);
   Param->setAccess(AS_public);
 
   if (Param->isParameterPack())
@@ -1044,7 +1039,7 @@ NamedDecl *Sema::ActOnTypeParameter(Scope *S, bool Typename,
     assert(DefaultTInfo && "expected source information for type");
 
     // Check for unexpanded parameter packs.
-    if (DiagnoseUnexpandedParameterPack(Loc, DefaultTInfo,
+    if (DiagnoseUnexpandedParameterPack(ParamNameLoc, DefaultTInfo,
                                         UPPC_DefaultArgument))
       return Param;
 
@@ -5503,7 +5498,7 @@ namespace {
     bool Visit##Class##Type(const Class##Type *) { return false; }
 #define NON_CANONICAL_TYPE(Class, Parent) \
     bool Visit##Class##Type(const Class##Type *) { return false; }
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
     bool VisitTagDecl(const TagDecl *Tag);
     bool VisitNestedNameSpecifier(NestedNameSpecifier *NNS);
@@ -6431,8 +6426,11 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
   }
 
   // If either the parameter has a dependent type or the argument is
-  // type-dependent, there's nothing we can check now.
-  if (ParamType->isDependentType() || Arg->isTypeDependent()) {
+  // type-dependent, there's nothing we can check now. The argument only
+  // contains an unexpanded pack during partial ordering, and there's
+  // nothing more we can check in that case.
+  if (ParamType->isDependentType() || Arg->isTypeDependent() ||
+      Arg->containsUnexpandedParameterPack()) {
     // Force the argument to the type of the parameter to maintain invariants.
     auto *PE = dyn_cast<PackExpansionExpr>(Arg);
     if (PE)
diff --git a/clang/lib/Sema/SemaTemplateDeduction.cpp b/clang/lib/Sema/SemaTemplateDeduction.cpp
index db50830a15da0..16d5afbd87e11 100644
--- a/clang/lib/Sema/SemaTemplateDeduction.cpp
+++ b/clang/lib/Sema/SemaTemplateDeduction.cpp
@@ -1486,7 +1486,7 @@ DeduceTemplateArgumentsByTypeMatch(Sema &S,
 #define NON_CANONICAL_TYPE(Class, Base) \
   case Type::Class: llvm_unreachable("deducing non-canonical type: " #Class);
 #define TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
 
     case Type::TemplateTypeParm:
     case Type::SubstTemplateTypeParmPack:
@@ -3093,6 +3093,13 @@ Sema::SubstituteExplicitTemplateArguments(
                   Function->getTypeSpecStartLoc(), Function->getDeclName());
     if (ResultType.isNull() || Trap.hasErrorOccurred())
       return TDK_SubstitutionFailure;
+    // CUDA: Kernel function must have 'void' return type.
+    if (getLangOpts().CUDA)
+      if (Function->hasAttr<CUDAGlobalAttr>() && !ResultType->isVoidType()) {
+        Diag(Function->getLocation(), diag::err_kern_type_not_void_return)
+            << Function->getType() << Function->getSourceRange();
+        return TDK_SubstitutionFailure;
+      }
   }
 
   // Instantiate the types of each of the function parameters given the
@@ -5608,7 +5615,7 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
 #define ABSTRACT_TYPE(Class, Base)
 #define DEPENDENT_TYPE(Class, Base)
 #define NON_CANONICAL_TYPE(Class, Base) case Type::Class:
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
     break;
   }
 }
diff --git a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
index 632f76dc37cb5..cfce84a9c0333 100644
--- a/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/clang/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -349,6 +349,65 @@ static void instantiateOMPDeclareSimdDeclAttr(
       Attr.getRange());
 }
 
+/// Instantiation of 'declare variant' attribute and its arguments.
+static void instantiateOMPDeclareVariantAttr(
+    Sema &S, const MultiLevelTemplateArgumentList &TemplateArgs,
+    const OMPDeclareVariantAttr &Attr, Decl *New) {
+  // Allow 'this' in clauses with varlists.
+  if (auto *FTD = dyn_cast<FunctionTemplateDecl>(New))
+    New = FTD->getTemplatedDecl();
+  auto *FD = cast<FunctionDecl>(New);
+  auto *ThisContext = dyn_cast_or_null<CXXRecordDecl>(FD->getDeclContext());
+
+  auto &&SubstExpr = [FD, ThisContext, &S, &TemplateArgs](Expr *E) {
+    if (auto *DRE = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()))
+      if (auto *PVD = dyn_cast<ParmVarDecl>(DRE->getDecl())) {
+        Sema::ContextRAII SavedContext(S, FD);
+        LocalInstantiationScope Local(S);
+        if (FD->getNumParams() > PVD->getFunctionScopeIndex())
+          Local.InstantiatedLocal(
+              PVD, FD->getParamDecl(PVD->getFunctionScopeIndex()));
+        return S.SubstExpr(E, TemplateArgs);
+      }
+    Sema::CXXThisScopeRAII ThisScope(S, ThisContext, Qualifiers(),
+                                     FD->isCXXInstanceMember());
+    return S.SubstExpr(E, TemplateArgs);
+  };
+
+  // Substitute a single OpenMP clause, which is a potentially-evaluated
+  // full-expression.
+  auto &&Subst = [&SubstExpr, &S](Expr *E) {
+    EnterExpressionEvaluationContext Evaluated(
+        S, Sema::ExpressionEvaluationContext::PotentiallyEvaluated);
+    ExprResult Res = SubstExpr(E);
+    if (Res.isInvalid())
+      return Res;
+    return S.ActOnFinishFullExpr(Res.get(), false);
+  };
+
+  ExprResult VariantFuncRef;
+  if (Expr *E = Attr.getVariantFuncRef())
+    VariantFuncRef = Subst(E);
+
+  ExprResult Score;
+  if (Expr *E = Attr.getScore())
+    Score = Subst(E);
+
+  // Check function/variant ref.
+  Optional<std::pair<FunctionDecl *, Expr *>> DeclVarData =
+      S.checkOpenMPDeclareVariantFunction(
+          S.ConvertDeclToDeclGroup(New), VariantFuncRef.get(), Attr.getRange());
+  if (!DeclVarData)
+    return;
+  // Instantiate the attribute.
+  Sema::OpenMPDeclareVariantCtsSelectorData Data(Attr.getCtxSelectorSet(),
+                                                 Attr.getCtxSelector(),
+                                                 Attr.getImplVendor(), Score);
+  S.ActOnOpenMPDeclareVariantDirective(DeclVarData.getValue().first,
+                                       DeclVarData.getValue().second,
+                                       Attr.getRange(), Data);
+}
+
 static void instantiateDependentAMDGPUFlatWorkGroupSizeAttr(
     Sema &S, const MultiLevelTemplateArgumentList &TemplateArgs,
     const AMDGPUFlatWorkGroupSizeAttr &Attr, Decl *New) {
@@ -506,6 +565,11 @@ void Sema::InstantiateAttrs(const MultiLevelTemplateArgumentList &TemplateArgs,
       continue;
     }
 
+    if (const auto *OMPAttr = dyn_cast<OMPDeclareVariantAttr>(TmplAttr)) {
+      instantiateOMPDeclareVariantAttr(*this, TemplateArgs, *OMPAttr, New);
+      continue;
+    }
+
     if (const auto *AMDGPUFlatWorkGroupSize =
             dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(TmplAttr)) {
       instantiateDependentAMDGPUFlatWorkGroupSizeAttr(
@@ -2091,10 +2155,9 @@ Decl *TemplateDeclInstantiator::VisitCXXMethodDecl(
         Constructor->getConstexprKind());
     Method->setRangeEnd(Constructor->getEndLoc());
   } else if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(D)) {
-    Method = CXXDestructorDecl::Create(SemaRef.Context, Record,
-                                       StartLoc, NameInfo, T, TInfo,
-                                       Destructor->isInlineSpecified(),
-                                       false);
+    Method = CXXDestructorDecl::Create(
+        SemaRef.Context, Record, StartLoc, NameInfo, T, TInfo,
+        Destructor->isInlineSpecified(), false, Destructor->getConstexprKind());
     Method->setRangeEnd(Destructor->getEndLoc());
   } else if (CXXConversionDecl *Conversion = dyn_cast<CXXConversionDecl>(D)) {
     Method = CXXConversionDecl::Create(
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index f3b90c3567934..c8fe686a84a60 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -1635,6 +1635,8 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     case OpenCLAccessAttr::Keyword_read_only:                                  \
       Result = Context.Id##ROTy;                                               \
       break;                                                                   \
+    case OpenCLAccessAttr::SpellingNotCalculated:                              \
+      llvm_unreachable("Spelling not yet calculated");                         \
     }                                                                          \
     break;
 #include "clang/Basic/OpenCLImageTypes.def"
@@ -8254,20 +8256,28 @@ bool Sema::RequireLiteralType(SourceLocation Loc, QualType T,
         return true;
       }
     }
-  } else if (!RD->hasTrivialDestructor()) {
-    // All fields and bases are of literal types, so have trivial destructors.
-    // If this class's destructor is non-trivial it must be user-declared.
+  } else if (getLangOpts().CPlusPlus2a ? !RD->hasConstexprDestructor()
+                                       : !RD->hasTrivialDestructor()) {
+    // All fields and bases are of literal types, so have trivial or constexpr
+    // destructors. If this class's destructor is non-trivial / non-constexpr,
+    // it must be user-declared.
     CXXDestructorDecl *Dtor = RD->getDestructor();
     assert(Dtor && "class has literal fields and bases but no dtor?");
     if (!Dtor)
       return true;
 
-    Diag(Dtor->getLocation(), Dtor->isUserProvided() ?
-         diag::note_non_literal_user_provided_dtor :
-         diag::note_non_literal_nontrivial_dtor) << RD;
-    if (!Dtor->isUserProvided())
-      SpecialMemberIsTrivial(Dtor, CXXDestructor, TAH_IgnoreTrivialABI,
-                             /*Diagnose*/true);
+    if (getLangOpts().CPlusPlus2a) {
+      Diag(Dtor->getLocation(), diag::note_non_literal_non_constexpr_dtor)
+          << RD;
+    } else {
+      Diag(Dtor->getLocation(), Dtor->isUserProvided()
+                                    ? diag::note_non_literal_user_provided_dtor
+                                    : diag::note_non_literal_nontrivial_dtor)
+          << RD;
+      if (!Dtor->isUserProvided())
+        SpecialMemberIsTrivial(Dtor, CXXDestructor, TAH_IgnoreTrivialABI,
+                               /*Diagnose*/ true);
+    }
   }
 
   return true;
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index c2a144a401f6f..84e38c6d78fbe 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -3308,16 +3308,14 @@ class TreeTransform {
   ///
   /// By default, performs semantic analysis to build the new expression.
   /// Subclasses may override this routine to provide different behavior.
-  ExprResult RebuildAtomicExpr(SourceLocation BuiltinLoc,
-                               MultiExprArg SubExprs,
-                               QualType RetTy,
+  ExprResult RebuildAtomicExpr(SourceLocation BuiltinLoc, MultiExprArg SubExprs,
                                AtomicExpr::AtomicOp Op,
                                SourceLocation RParenLoc) {
-    // Just create the expression; there is not any interesting semantic
-    // analysis here because we can't actually build an AtomicExpr until
-    // we are sure it is semantically sound.
-    return new (SemaRef.Context) AtomicExpr(BuiltinLoc, SubExprs, RetTy, Op,
-                                            RParenLoc);
+    // Use this for all of the locations, since we don't know the difference
+    // between the call and the expr at this point.
+    SourceRange Range{BuiltinLoc, RParenLoc};
+    return getSema().BuildAtomicExpr(Range, Range, RParenLoc, SubExprs, Op,
+                                     Sema::AtomicArgumentOrder::AST);
   }
 
 private:
@@ -9221,6 +9219,7 @@ TreeTransform<Derived>::TransformDeclRefExpr(DeclRefExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       QualifierLoc == E->getQualifierLoc() &&
       ND == E->getDecl() &&
+      Found == E->getFoundDecl() &&
       NameInfo.getName() == E->getDecl()->getDeclName() &&
       !E->hasExplicitTemplateArgs()) {
 
@@ -11325,10 +11324,14 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
     }
   }
 
+  LambdaScopeInfo *LSI = getSema().PushLambdaScope();
+  Sema::FunctionScopeRAII FuncScopeCleanup(getSema());
+
   // Transform the template parameters, and add them to the current
   // instantiation scope. The null case is handled correctly.
   auto TPL = getDerived().TransformTemplateParameterList(
       E->getTemplateParameterList());
+  LSI->GLTemplateParameterList = TPL;
 
   // Transform the type of the original lambda's call operator.
   // The transformation MUST be done in the CurrentInstantiationScope since
@@ -11355,10 +11358,6 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
                                                         NewCallOpType);
   }
 
-  LambdaScopeInfo *LSI = getSema().PushLambdaScope();
-  Sema::FunctionScopeRAII FuncScopeCleanup(getSema());
-  LSI->GLTemplateParameterList = TPL;
-
   // Create the local class that will describe the lambda.
   CXXRecordDecl *OldClass = E->getLambdaClass();
   CXXRecordDecl *Class
@@ -12673,7 +12672,6 @@ TreeTransform<Derived>::TransformAsTypeExpr(AsTypeExpr *E) {
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformAtomicExpr(AtomicExpr *E) {
-  QualType RetTy = getDerived().TransformType(E->getType());
   bool ArgumentChanged = false;
   SmallVector<Expr*, 8> SubExprs;
   SubExprs.reserve(E->getNumSubExprs());
@@ -12686,7 +12684,7 @@ TreeTransform<Derived>::TransformAtomicExpr(AtomicExpr *E) {
     return E;
 
   return getDerived().RebuildAtomicExpr(E->getBuiltinLoc(), SubExprs,
-                                        RetTy, E->getOp(), E->getRParenLoc());
+                                        E->getOp(), E->getRParenLoc());
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 10b2a5c4f9dfb..fcb437e0cb1e7 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12202,7 +12202,7 @@ Expected<unsigned> ASTRecordReader::readRecord(llvm::BitstreamCursor &Cursor,
 ////===----------------------------------------------------------------------===//
 
 OMPClause *OMPClauseReader::readClause() {
-  OMPClause *C;
+  OMPClause *C = nullptr;
   switch (Record.readInt()) {
   case OMPC_if:
     C = new (Context) OMPIfClause();
@@ -12403,6 +12403,8 @@ OMPClause *OMPClauseReader::readClause() {
     C = OMPAllocateClause::CreateEmpty(Context, Record.readInt());
     break;
   }
+  assert(C && "Unknown OMPClause type");
+
   Visit(C);
   C->setLocStart(Record.readSourceLocation());
   C->setLocEnd(Record.readSourceLocation());
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index e9cf366983fb3..76928182a1e8e 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -1390,10 +1390,11 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitVarDeclImpl(VarDecl *VD) {
 
   if (uint64_t Val = Record.readInt()) {
     VD->setInit(Record.readExpr());
-    if (Val > 1) { // IsInitKnownICE = 1, IsInitNotICE = 2, IsInitICE = 3
+    if (Val > 1) {
       EvaluatedStmt *Eval = VD->ensureEvaluatedStmt();
       Eval->CheckedICE = true;
-      Eval->IsICE = Val == 3;
+      Eval->IsICE = (Val & 1) != 0;
+      Eval->HasConstantDestruction = (Val & 4) != 0;
     }
   }
 
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 8018ee8b78e98..e1f3d50456b11 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -166,7 +166,7 @@ namespace clang {
 #define TYPE(Class, Base) \
         case Type::Class: Visit##Class##Type(cast<Class##Type>(T)); break;
 #define ABSTRACT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
         }
       }
     }
@@ -177,7 +177,7 @@ namespace clang {
 
 #define TYPE(Class, Base) void Visit##Class##Type(const Class##Type *T);
 #define ABSTRACT_TYPE(Class, Base)
-#include "clang/AST/TypeNodes.def"
+#include "clang/AST/TypeNodes.inc"
   };
 
 } // namespace clang
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index b71315505de90..2c22587957e89 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -968,7 +968,14 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
   Record.push_back(D->getLinkageInternal());
 
   if (D->getInit()) {
-    Record.push_back(!D->isInitKnownICE() ? 1 : (D->isInitICE() ? 3 : 2));
+    if (!D->isInitKnownICE())
+      Record.push_back(1);
+    else {
+      Record.push_back(
+          2 |
+          (D->isInitICE() ? 1 : 0) |
+          (D->ensureEvaluatedStmt()->HasConstantDestruction ? 4 : 0));
+    }
     Record.AddStmt(D->getInit());
   } else {
     Record.push_back(0);
@@ -2140,7 +2147,7 @@ void ASTWriter::WriteDeclAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                         // ImplicitParamKind
   Abv->Add(BitCodeAbbrevOp(0));                         // EscapingByref
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // Linkage
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // IsInitICE (local)
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); // IsInitICE (local)
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // VarKind (local enum)
   // Type Source Info
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp
index 1833ebbdc84cc..54ab17681ee9e 100644
--- a/clang/lib/Serialization/GlobalModuleIndex.cpp
+++ b/clang/lib/Serialization/GlobalModuleIndex.cpp
@@ -10,7 +10,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include "ASTReaderInternals.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Lex/HeaderSearch.h"
@@ -21,10 +20,12 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
 #include "llvm/Bitstream/BitstreamReader.h"
 #include "llvm/Bitstream/BitstreamWriter.h"
 #include "llvm/Support/DJB.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/LockFileManager.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/OnDiskHashTable.h"
@@ -912,37 +913,9 @@ GlobalModuleIndex::writeIndex(FileManager &FileMgr,
                                      "failed writing index");
   }
 
-  // Write the global index file to a temporary file.
-  llvm::SmallString<128> IndexTmpPath;
-  int TmpFD;
-  if (llvm::sys::fs::createUniqueFile(IndexPath + "-%%%%%%%%", TmpFD,
-                                      IndexTmpPath))
-    return llvm::createStringError(std::errc::io_error,
-                                   "failed creating unique file");
-
-  // Open the temporary global index file for output.
-  llvm::raw_fd_ostream Out(TmpFD, true);
-  if (Out.has_error())
-    return llvm::createStringError(Out.error(), "failed outputting to stream");
-
-  // Write the index.
-  Out.write(OutputBuffer.data(), OutputBuffer.size());
-  Out.close();
-  if (Out.has_error())
-    return llvm::createStringError(Out.error(), "failed writing to stream");
-
-  // Remove the old index file. It isn't relevant any more.
-  llvm::sys::fs::remove(IndexPath);
-
-  // Rename the newly-written index file to the proper name.
-  if (std::error_code Err = llvm::sys::fs::rename(IndexTmpPath, IndexPath)) {
-    // Remove the file on failure, don't check whether removal succeeded.
-    llvm::sys::fs::remove(IndexTmpPath);
-    return llvm::createStringError(Err, "failed renaming file \"%s\" to \"%s\"",
-                                   IndexTmpPath.c_str(), IndexPath.c_str());
-  }
-
-  return llvm::Error::success();
+  return llvm::writeFileAtomically(
+      (IndexPath + "-%%%%%%%%").str(), IndexPath,
+      llvm::StringRef(OutputBuffer.data(), OutputBuffer.size()));
 }
 
 namespace {
diff --git a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
index 6be6762468724..e064ca6bd88f6 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -113,8 +113,10 @@ class MacOSKeychainAPIChecker : public Checker<check::PreStmt<CallExpr>,
   const ExplodedNode *getAllocationNode(const ExplodedNode *N, SymbolRef Sym,
                                         CheckerContext &C) const;
 
-  std::unique_ptr<BugReport> generateAllocatedDataNotReleasedReport(
-      const AllocationPair &AP, ExplodedNode *N, CheckerContext &C) const;
+  std::unique_ptr<PathSensitiveBugReport>
+  generateAllocatedDataNotReleasedReport(const AllocationPair &AP,
+                                         ExplodedNode *N,
+                                         CheckerContext &C) const;
 
   /// Mark an AllocationPair interesting for diagnostic reporting.
   void markInteresting(PathSensitiveBugReport *R,
@@ -467,7 +469,7 @@ MacOSKeychainAPIChecker::getAllocationNode(const ExplodedNode *N,
   return AllocNode;
 }
 
-std::unique_ptr<BugReport>
+std::unique_ptr<PathSensitiveBugReport>
 MacOSKeychainAPIChecker::generateAllocatedDataNotReleasedReport(
     const AllocationPair &AP, ExplodedNode *N, CheckerContext &C) const {
   const ADFunctionInfo &FI = FunctionsToTrack[AP.second->AllocatorIdx];
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index 699b5c6b9fa8e..a824499518730 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -6,8 +6,41 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines malloc/free checker, which checks for potential memory
-// leaks, double free, and use-after-free problems.
+// This file defines a variety of memory management related checkers, such as
+// leak, double free, and use-after-free.
+//
+// The following checkers are defined here:
+//
+//   * MallocChecker
+//       Despite its name, it models all sorts of memory allocations and
+//       de- or reallocation, including but not limited to malloc, free,
+//       relloc, new, delete. It also reports on a variety of memory misuse
+//       errors.
+//       Many other checkers interact very closely with this checker, in fact,
+//       most are merely options to this one. Other checkers may register
+//       MallocChecker, but do not enable MallocChecker's reports (more details
+//       to follow around its field, ChecksEnabled).
+//       It also has a boolean "Optimistic" checker option, which if set to true
+//       will cause the checker to model user defined memory management related
+//       functions annotated via the attribute ownership_takes, ownership_holds
+//       and ownership_returns.
+//
+//   * NewDeleteChecker
+//       Enables the modeling of new, new[], delete, delete[] in MallocChecker,
+//       and checks for related double-free and use-after-free errors.
+//
+//   * NewDeleteLeaksChecker
+//       Checks for leaks related to new, new[], delete, delete[].
+//       Depends on NewDeleteChecker.
+//
+//   * MismatchedDeallocatorChecker
+//       Enables checking whether memory is deallocated with the correspending
+//       allocation function in MallocChecker, such as malloc() allocated
+//       regions are only freed by free(), new by delete, new[] by delete[].
+//
+//  InnerPointerChecker interacts very closely with MallocChecker, but unlike
+//  the above checkers, it has it's own file, hence the many InnerPointerChecker
+//  related headers and non-static functions.
 //
 //===----------------------------------------------------------------------===//
 
@@ -37,6 +70,10 @@
 using namespace clang;
 using namespace ento;
 
+//===----------------------------------------------------------------------===//
+// The types of allocation we're modeling.
+//===----------------------------------------------------------------------===//
+
 namespace {
 
 // Used to check correspondence between allocators and deallocators.
@@ -50,57 +87,88 @@ enum AllocationFamily {
   AF_InnerBuffer
 };
 
+struct MemFunctionInfoTy;
+
+} // end of anonymous namespace
+
+/// Determine family of a deallocation expression.
+static AllocationFamily
+getAllocationFamily(const MemFunctionInfoTy &MemFunctionInfo, CheckerContext &C,
+                    const Stmt *S);
+
+/// Print names of allocators and deallocators.
+///
+/// \returns true on success.
+static bool printAllocDeallocName(raw_ostream &os, CheckerContext &C,
+                                  const Expr *E);
+
+/// Print expected name of an allocator based on the deallocator's
+/// family derived from the DeallocExpr.
+static void printExpectedAllocName(raw_ostream &os,
+                                   const MemFunctionInfoTy &MemFunctionInfo,
+                                   CheckerContext &C, const Expr *E);
+
+/// Print expected name of a deallocator based on the allocator's
+/// family.
+static void printExpectedDeallocName(raw_ostream &os, AllocationFamily Family);
+
+//===----------------------------------------------------------------------===//
+// The state of a symbol, in terms of memory management.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
 class RefState {
-  enum Kind { // Reference to allocated memory.
-              Allocated,
-              // Reference to zero-allocated memory.
-              AllocatedOfSizeZero,
-              // Reference to released/freed memory.
-              Released,
-              // The responsibility for freeing resources has transferred from
-              // this reference. A relinquished symbol should not be freed.
-              Relinquished,
-              // We are no longer guaranteed to have observed all manipulations
-              // of this pointer/memory. For example, it could have been
-              // passed as a parameter to an opaque function.
-              Escaped
+  enum Kind {
+    // Reference to allocated memory.
+    Allocated,
+    // Reference to zero-allocated memory.
+    AllocatedOfSizeZero,
+    // Reference to released/freed memory.
+    Released,
+    // The responsibility for freeing resources has transferred from
+    // this reference. A relinquished symbol should not be freed.
+    Relinquished,
+    // We are no longer guaranteed to have observed all manipulations
+    // of this pointer/memory. For example, it could have been
+    // passed as a parameter to an opaque function.
+    Escaped
   };
 
   const Stmt *S;
-  unsigned K : 3; // Kind enum, but stored as a bitfield.
-  unsigned Family : 29; // Rest of 32-bit word, currently just an allocation
-                        // family.
 
-  RefState(Kind k, const Stmt *s, unsigned family)
-    : S(s), K(k), Family(family) {
+  Kind K;
+  AllocationFamily Family;
+
+  RefState(Kind k, const Stmt *s, AllocationFamily family)
+      : S(s), K(k), Family(family) {
     assert(family != AF_None);
   }
+
 public:
   bool isAllocated() const { return K == Allocated; }
   bool isAllocatedOfSizeZero() const { return K == AllocatedOfSizeZero; }
   bool isReleased() const { return K == Released; }
   bool isRelinquished() const { return K == Relinquished; }
   bool isEscaped() const { return K == Escaped; }
-  AllocationFamily getAllocationFamily() const {
-    return (AllocationFamily)Family;
-  }
+  AllocationFamily getAllocationFamily() const { return Family; }
   const Stmt *getStmt() const { return S; }
 
   bool operator==(const RefState &X) const {
     return K == X.K && S == X.S && Family == X.Family;
   }
 
-  static RefState getAllocated(unsigned family, const Stmt *s) {
+  static RefState getAllocated(AllocationFamily family, const Stmt *s) {
     return RefState(Allocated, s, family);
   }
   static RefState getAllocatedOfSizeZero(const RefState *RS) {
     return RefState(AllocatedOfSizeZero, RS->getStmt(),
                     RS->getAllocationFamily());
   }
-  static RefState getReleased(unsigned family, const Stmt *s) {
+  static RefState getReleased(AllocationFamily family, const Stmt *s) {
     return RefState(Released, s, family);
   }
-  static RefState getRelinquished(unsigned family, const Stmt *s) {
+  static RefState getRelinquished(AllocationFamily family, const Stmt *s) {
     return RefState(Relinquished, s, family);
   }
   static RefState getEscaped(const RefState *RS) {
@@ -113,8 +181,8 @@ class RefState {
     ID.AddInteger(Family);
   }
 
-  void dump(raw_ostream &OS) const {
-    switch (static_cast<Kind>(K)) {
+  LLVM_DUMP_METHOD void dump(raw_ostream &OS) const {
+    switch (K) {
 #define CASE(ID) case ID: OS << #ID; break;
     CASE(Allocated)
     CASE(AllocatedOfSizeZero)
@@ -127,24 +195,62 @@ class RefState {
   LLVM_DUMP_METHOD void dump() const { dump(llvm::errs()); }
 };
 
-enum ReallocPairKind {
-  RPToBeFreedAfterFailure,
-  // The symbol has been freed when reallocation failed.
-  RPIsFreeOnFailure,
-  // The symbol does not need to be freed after reallocation fails.
-  RPDoNotTrackAfterFailure
+} // end of anonymous namespace
+
+REGISTER_MAP_WITH_PROGRAMSTATE(RegionState, SymbolRef, RefState)
+
+/// Check if the memory associated with this symbol was released.
+static bool isReleased(SymbolRef Sym, CheckerContext &C);
+
+/// Update the RefState to reflect the new memory allocation.
+/// The optional \p RetVal parameter specifies the newly allocated pointer
+/// value; if unspecified, the value of expression \p E is used.
+static ProgramStateRef MallocUpdateRefState(CheckerContext &C, const Expr *E,
+                                            ProgramStateRef State,
+                                            AllocationFamily Family = AF_Malloc,
+                                            Optional<SVal> RetVal = None);
+
+//===----------------------------------------------------------------------===//
+// The modeling of memory reallocation.
+//
+// The terminology 'toPtr' and 'fromPtr' will be used:
+//   toPtr = realloc(fromPtr, 20);
+//===----------------------------------------------------------------------===//
+
+REGISTER_SET_WITH_PROGRAMSTATE(ReallocSizeZeroSymbols, SymbolRef)
+
+namespace {
+
+/// The state of 'fromPtr' after reallocation is known to have failed.
+enum OwnershipAfterReallocKind {
+  // The symbol needs to be freed (e.g.: realloc)
+  OAR_ToBeFreedAfterFailure,
+  // The symbol has been freed (e.g.: reallocf)
+  OAR_FreeOnFailure,
+  // The symbol doesn't have to freed (e.g.: we aren't sure if, how and where
+  // 'fromPtr' was allocated:
+  //    void Haha(int *ptr) {
+  //      ptr = realloc(ptr, 67);
+  //      // ...
+  //    }
+  // ).
+  OAR_DoNotTrackAfterFailure
 };
 
-/// \class ReallocPair
-/// Stores information about the symbol being reallocated by a call to
-/// 'realloc' to allow modeling failed reallocation later in the path.
+/// Stores information about the 'fromPtr' symbol after reallocation.
+///
+/// This is important because realloc may fail, and that needs special modeling.
+/// Whether reallocation failed or not will not be known until later, so we'll
+/// store whether upon failure 'fromPtr' will be freed, or needs to be freed
+/// later, etc.
 struct ReallocPair {
-  // The symbol which realloc reallocated.
+
+  // The 'fromPtr'.
   SymbolRef ReallocatedSym;
-  ReallocPairKind Kind;
+  OwnershipAfterReallocKind Kind;
 
-  ReallocPair(SymbolRef S, ReallocPairKind K) :
-    ReallocatedSym(S), Kind(K) {}
+  ReallocPair(SymbolRef S, OwnershipAfterReallocKind K)
+      : ReallocatedSym(S), Kind(K) {}
   void Profile(llvm::FoldingSetNodeID &ID) const {
     ID.AddInteger(Kind);
     ID.AddPointer(ReallocatedSym);
@@ -155,42 +261,88 @@ struct ReallocPair {
   }
 };
 
-typedef std::pair<const ExplodedNode*, const MemRegion*> LeakInfo;
-
-class MallocChecker : public Checker<check::DeadSymbols,
-                                     check::PointerEscape,
-                                     check::ConstPointerEscape,
-                                     check::PreStmt<ReturnStmt>,
-                                     check::EndFunction,
-                                     check::PreCall,
-                                     check::PostStmt<CallExpr>,
-                                     check::PostStmt<CXXNewExpr>,
-                                     check::NewAllocator,
-                                     check::PreStmt<CXXDeleteExpr>,
-                                     check::PostStmt<BlockExpr>,
-                                     check::PostObjCMessage,
-                                     check::Location,
-                                     eval::Assume>
-{
-public:
-  MallocChecker()
-      : II_alloca(nullptr), II_win_alloca(nullptr), II_malloc(nullptr),
-        II_free(nullptr), II_realloc(nullptr), II_calloc(nullptr),
-        II_valloc(nullptr), II_reallocf(nullptr), II_strndup(nullptr),
-        II_strdup(nullptr), II_win_strdup(nullptr), II_kmalloc(nullptr),
-        II_kfree(nullptr), II_if_nameindex(nullptr),
-        II_if_freenameindex(nullptr), II_wcsdup(nullptr),
-        II_win_wcsdup(nullptr), II_g_malloc(nullptr), II_g_malloc0(nullptr),
-        II_g_realloc(nullptr), II_g_try_malloc(nullptr),
-        II_g_try_malloc0(nullptr), II_g_try_realloc(nullptr),
-        II_g_free(nullptr), II_g_memdup(nullptr), II_g_malloc_n(nullptr),
-        II_g_malloc0_n(nullptr), II_g_realloc_n(nullptr),
-        II_g_try_malloc_n(nullptr), II_g_try_malloc0_n(nullptr),
-        II_g_try_realloc_n(nullptr) {}
+} // end of anonymous namespace
+
+REGISTER_MAP_WITH_PROGRAMSTATE(ReallocPairs, SymbolRef, ReallocPair)
+
+//===----------------------------------------------------------------------===//
+// Kinds of memory operations, information about resource managing functions.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+enum class MemoryOperationKind { MOK_Allocate, MOK_Free, MOK_Any };
 
+struct MemFunctionInfoTy {
+  /// The value of the MallocChecker:Optimistic is stored in this variable.
+  ///
   /// In pessimistic mode, the checker assumes that it does not know which
   /// functions might free the memory.
+  /// In optimistic mode, the checker assumes that all user-defined functions
+  /// which might free a pointer are annotated.
+  DefaultBool ShouldIncludeOwnershipAnnotatedFunctions;
+
+  // TODO: Change these to CallDescription, and get rid of lazy initialization.
+  mutable IdentifierInfo *II_alloca = nullptr, *II_win_alloca = nullptr,
+                         *II_malloc = nullptr, *II_free = nullptr,
+                         *II_realloc = nullptr, *II_calloc = nullptr,
+                         *II_valloc = nullptr, *II_reallocf = nullptr,
+                         *II_strndup = nullptr, *II_strdup = nullptr,
+                         *II_win_strdup = nullptr, *II_kmalloc = nullptr,
+                         *II_if_nameindex = nullptr,
+                         *II_if_freenameindex = nullptr, *II_wcsdup = nullptr,
+                         *II_win_wcsdup = nullptr, *II_g_malloc = nullptr,
+                         *II_g_malloc0 = nullptr, *II_g_realloc = nullptr,
+                         *II_g_try_malloc = nullptr,
+                         *II_g_try_malloc0 = nullptr,
+                         *II_g_try_realloc = nullptr, *II_g_free = nullptr,
+                         *II_g_memdup = nullptr, *II_g_malloc_n = nullptr,
+                         *II_g_malloc0_n = nullptr, *II_g_realloc_n = nullptr,
+                         *II_g_try_malloc_n = nullptr,
+                         *II_g_try_malloc0_n = nullptr, *II_kfree = nullptr,
+                         *II_g_try_realloc_n = nullptr;
+
+  void initIdentifierInfo(ASTContext &C) const;
+
+  ///@{
+  /// Check if this is one of the functions which can allocate/reallocate
+  /// memory pointed to by one of its arguments.
+  bool isMemFunction(const FunctionDecl *FD, ASTContext &C) const;
+  bool isCMemFunction(const FunctionDecl *FD, ASTContext &C,
+                      AllocationFamily Family,
+                      MemoryOperationKind MemKind) const;
+
+  /// Tells if the callee is one of the builtin new/delete operators, including
+  /// placement operators and other standard overloads.
+  bool isStandardNewDelete(const FunctionDecl *FD, ASTContext &C) const;
+  ///@}
+};
+
+} // end of anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// Definition of the MallocChecker class.
+//===----------------------------------------------------------------------===//
+
+namespace {
+
+class MallocChecker
+    : public Checker<check::DeadSymbols, check::PointerEscape,
+                     check::ConstPointerEscape, check::PreStmt<ReturnStmt>,
+                     check::EndFunction, check::PreCall,
+                     check::PostStmt<CallExpr>, check::PostStmt<CXXNewExpr>,
+                     check::NewAllocator, check::PreStmt<CXXDeleteExpr>,
+                     check::PostStmt<BlockExpr>, check::PostObjCMessage,
+                     check::Location, eval::Assume> {
+public:
+  MemFunctionInfoTy MemFunctionInfo;
+
+  /// Many checkers are essentially built into this one, so enabling them will
+  /// make MallocChecker perform additional modeling and reporting.
   enum CheckKind {
+    /// When a subchecker is enabled but MallocChecker isn't, model memory
+    /// management but do not emit warnings emitted with MallocChecker only
+    /// enabled.
     CK_MallocChecker,
     CK_NewDeleteChecker,
     CK_NewDeleteLeaksChecker,
@@ -199,13 +351,7 @@ class MallocChecker : public Checker<check::DeadSymbols,
     CK_NumCheckKinds
   };
 
-  enum class MemoryOperationKind {
-    MOK_Allocate,
-    MOK_Free,
-    MOK_Any
-  };
-
-  DefaultBool IsOptimistic;
+  using LeakInfo = std::pair<const ExplodedNode *, const MemRegion *>;
 
   DefaultBool ChecksEnabled[CK_NumCheckKinds];
   CheckerNameRef CheckNames[CK_NumCheckKinds];
@@ -248,47 +394,9 @@ class MallocChecker : public Checker<check::DeadSymbols,
   mutable std::unique_ptr<BugType> BT_MismatchedDealloc;
   mutable std::unique_ptr<BugType> BT_OffsetFree[CK_NumCheckKinds];
   mutable std::unique_ptr<BugType> BT_UseZerroAllocated[CK_NumCheckKinds];
-  mutable IdentifierInfo *II_alloca, *II_win_alloca, *II_malloc, *II_free,
-                         *II_realloc, *II_calloc, *II_valloc, *II_reallocf,
-                         *II_strndup, *II_strdup, *II_win_strdup, *II_kmalloc,
-                         *II_kfree, *II_if_nameindex, *II_if_freenameindex,
-                         *II_wcsdup, *II_win_wcsdup, *II_g_malloc,
-                         *II_g_malloc0, *II_g_realloc, *II_g_try_malloc,
-                         *II_g_try_malloc0, *II_g_try_realloc, *II_g_free,
-                         *II_g_memdup, *II_g_malloc_n, *II_g_malloc0_n,
-                         *II_g_realloc_n, *II_g_try_malloc_n,
-                         *II_g_try_malloc0_n, *II_g_try_realloc_n;
-  mutable Optional<uint64_t> KernelZeroFlagVal;
 
-  void initIdentifierInfo(ASTContext &C) const;
-
-  /// Determine family of a deallocation expression.
-  AllocationFamily getAllocationFamily(CheckerContext &C, const Stmt *S) const;
-
-  /// Print names of allocators and deallocators.
-  ///
-  /// \returns true on success.
-  bool printAllocDeallocName(raw_ostream &os, CheckerContext &C,
-                             const Expr *E) const;
-
-  /// Print expected name of an allocator based on the deallocator's
-  /// family derived from the DeallocExpr.
-  void printExpectedAllocName(raw_ostream &os, CheckerContext &C,
-                              const Expr *DeallocExpr) const;
-  /// Print expected name of a deallocator based on the allocator's
-  /// family.
-  void printExpectedDeallocName(raw_ostream &os, AllocationFamily Family) const;
-
-  ///@{
-  /// Check if this is one of the functions which can allocate/reallocate memory
-  /// pointed to by one of its arguments.
-  bool isMemFunction(const FunctionDecl *FD, ASTContext &C) const;
-  bool isCMemFunction(const FunctionDecl *FD,
-                      ASTContext &C,
-                      AllocationFamily Family,
-                      MemoryOperationKind MemKind) const;
-  bool isStandardNewDelete(const FunctionDecl *FD, ASTContext &C) const;
-  ///@}
+  // TODO: Remove mutable by moving the initializtaion to the registry function.
+  mutable Optional<uint64_t> KernelZeroFlagVal;
 
   /// Process C++ operator new()'s allocation, which is the part of C++
   /// new-expression that goes before the constructor.
@@ -296,23 +404,64 @@ class MallocChecker : public Checker<check::DeadSymbols,
                             SVal Target) const;
 
   /// Perform a zero-allocation check.
-  /// The optional \p RetVal parameter specifies the newly allocated pointer
-  /// value; if unspecified, the value of expression \p E is used.
-  ProgramStateRef ProcessZeroAllocation(CheckerContext &C, const Expr *E,
-                                        const unsigned AllocationSizeArg,
-                                        ProgramStateRef State,
-                                        Optional<SVal> RetVal = None) const;
-
+  ///
+  /// \param [in] E The expression that allocates memory.
+  /// \param [in] IndexOfSizeArg Index of the argument that specifies the size
+  ///   of the memory that needs to be allocated. E.g. for malloc, this would be
+  ///   0.
+  /// \param [in] RetVal Specifies the newly allocated pointer value;
+  ///   if unspecified, the value of expression \p E is used.
+  static ProgramStateRef ProcessZeroAllocCheck(CheckerContext &C, const Expr *E,
+                                               const unsigned IndexOfSizeArg,
+                                               ProgramStateRef State,
+                                               Optional<SVal> RetVal = None);
+
+  /// Model functions with the ownership_returns attribute.
+  ///
+  /// User-defined function may have the ownership_returns attribute, which
+  /// annotates that the function returns with an object that was allocated on
+  /// the heap, and passes the ownertship to the callee.
+  ///
+  ///   void __attribute((ownership_returns(malloc, 1))) *my_malloc(size_t);
+  ///
+  /// It has two parameters:
+  ///   - first: name of the resource (e.g. 'malloc')
+  ///   - (OPTIONAL) second: size of the allocated region
+  ///
+  /// \param [in] CE The expression that allocates memory.
+  /// \param [in] Att The ownership_returns attribute.
+  /// \param [in] State The \c ProgramState right before allocation.
+  /// \returns The ProgramState right after allocation.
   ProgramStateRef MallocMemReturnsAttr(CheckerContext &C,
                                        const CallExpr *CE,
                                        const OwnershipAttr* Att,
                                        ProgramStateRef State) const;
+
+  /// Models memory allocation.
+  ///
+  /// \param [in] CE The expression that allocates memory.
+  /// \param [in] SizeEx Size of the memory that needs to be allocated.
+  /// \param [in] Init The value the allocated memory needs to be initialized.
+  /// with. For example, \c calloc initializes the allocated memory to 0,
+  /// malloc leaves it undefined.
+  /// \param [in] State The \c ProgramState right before allocation.
+  /// \returns The ProgramState right after allocation.
   static ProgramStateRef MallocMemAux(CheckerContext &C, const CallExpr *CE,
                                       const Expr *SizeEx, SVal Init,
                                       ProgramStateRef State,
                                       AllocationFamily Family = AF_Malloc);
+
+  /// Models memory allocation.
+  ///
+  /// \param [in] CE The expression that allocates memory.
+  /// \param [in] Size Size of the memory that needs to be allocated.
+  /// \param [in] Init The value the allocated memory needs to be initialized.
+  /// with. For example, \c calloc initializes the allocated memory to 0,
+  /// malloc leaves it undefined.
+  /// \param [in] State The \c ProgramState right before allocation.
+  /// \returns The ProgramState right after allocation.
   static ProgramStateRef MallocMemAux(CheckerContext &C, const CallExpr *CE,
-                                      SVal SizeEx, SVal Init,
+                                      SVal Size, SVal Init,
                                       ProgramStateRef State,
                                       AllocationFamily Family = AF_Malloc);
 
@@ -325,54 +474,125 @@ class MallocChecker : public Checker<check::DeadSymbols,
   performKernelMalloc(const CallExpr *CE, CheckerContext &C,
                       const ProgramStateRef &State) const;
 
-  /// Update the RefState to reflect the new memory allocation.
-  /// The optional \p RetVal parameter specifies the newly allocated pointer
-  /// value; if unspecified, the value of expression \p E is used.
-  static ProgramStateRef
-  MallocUpdateRefState(CheckerContext &C, const Expr *E, ProgramStateRef State,
-                       AllocationFamily Family = AF_Malloc,
-                       Optional<SVal> RetVal = None);
-
+  /// Model functions with the ownership_takes and ownership_holds attributes.
+  ///
+  /// User-defined function may have the ownership_takes and/or ownership_holds
+  /// attributes, which annotates that the function frees the memory passed as a
+  /// parameter.
+  ///
+  ///   void __attribute((ownership_takes(malloc, 1))) my_free(void *);
+  ///   void __attribute((ownership_holds(malloc, 1))) my_hold(void *);
+  ///
+  /// They have two parameters:
+  ///   - first: name of the resource (e.g. 'malloc')
+  ///   - second: index of the parameter the attribute applies to
+  ///
+  /// \param [in] CE The expression that frees memory.
+  /// \param [in] Att The ownership_takes or ownership_holds attribute.
+  /// \param [in] State The \c ProgramState right before allocation.
+  /// \returns The ProgramState right after deallocation.
   ProgramStateRef FreeMemAttr(CheckerContext &C, const CallExpr *CE,
                               const OwnershipAttr* Att,
                               ProgramStateRef State) const;
+
+  /// Models memory deallocation.
+  ///
+  /// \param [in] CE The expression that frees memory.
+  /// \param [in] State The \c ProgramState right before allocation.
+  /// \param [in] Num Index of the argument that needs to be freed. This is
+  ///   normally 0, but for custom free functions it may be different.
+  /// \param [in] Hold Whether the parameter at \p Index has the ownership_holds
+  ///   attribute.
+  /// \param [out] IsKnownToBeAllocated Whether the memory to be freed is known
+  ///   to have been allocated, or in other words, the symbol to be freed was
+  ///   registered as allocated by this checker. In the following case, \c ptr
+  ///   isn't known to be allocated.
+  ///      void Haha(int *ptr) {
+  ///        ptr = realloc(ptr, 67);
+  ///        // ...
+  ///      }
+  /// \param [in] ReturnsNullOnFailure Whether the memory deallocation function
+  ///   we're modeling returns with Null on failure.
+  /// \returns The ProgramState right after deallocation.
   ProgramStateRef FreeMemAux(CheckerContext &C, const CallExpr *CE,
-                             ProgramStateRef state, unsigned Num,
-                             bool Hold,
-                             bool &ReleasedAllocated,
+                             ProgramStateRef State, unsigned Num, bool Hold,
+                             bool &IsKnownToBeAllocated,
                              bool ReturnsNullOnFailure = false) const;
-  ProgramStateRef FreeMemAux(CheckerContext &C, const Expr *Arg,
-                             const Expr *ParentExpr,
-                             ProgramStateRef State,
-                             bool Hold,
-                             bool &ReleasedAllocated,
+
+  /// Models memory deallocation.
+  ///
+  /// \param [in] ArgExpr The variable who's pointee needs to be freed.
+  /// \param [in] ParentExpr The expression that frees the memory.
+  /// \param [in] State The \c ProgramState right before allocation.
+  ///   normally 0, but for custom free functions it may be different.
+  /// \param [in] Hold Whether the parameter at \p Index has the ownership_holds
+  ///   attribute.
+  /// \param [out] IsKnownToBeAllocated Whether the memory to be freed is known
+  ///   to have been allocated, or in other words, the symbol to be freed was
+  ///   registered as allocated by this checker. In the following case, \c ptr
+  ///   isn't known to be allocated.
+  ///      void Haha(int *ptr) {
+  ///        ptr = realloc(ptr, 67);
+  ///        // ...
+  ///      }
+  /// \param [in] ReturnsNullOnFailure Whether the memory deallocation function
+  ///   we're modeling returns with Null on failure.
+  /// \returns The ProgramState right after deallocation.
+  ProgramStateRef FreeMemAux(CheckerContext &C, const Expr *ArgExpr,
+                             const Expr *ParentExpr, ProgramStateRef State,
+                             bool Hold, bool &IsKnownToBeAllocated,
                              bool ReturnsNullOnFailure = false) const;
 
+  // TODO: Needs some refactoring, as all other deallocation modeling
+  // functions are suffering from out parameters and messy code due to how
+  // realloc is handled.
+  //
+  /// Models memory reallocation.
+  ///
+  /// \param [in] CE The expression that reallocated memory
+  /// \param [in] ShouldFreeOnFail Whether if reallocation fails, the supplied
+  ///   memory should be freed.
+  /// \param [in] State The \c ProgramState right before reallocation.
+  /// \param [in] SuffixWithN Whether the reallocation function we're modeling
+  ///   has an '_n' suffix, such as g_realloc_n.
+  /// \returns The ProgramState right after reallocation.
   ProgramStateRef ReallocMemAux(CheckerContext &C, const CallExpr *CE,
-                                bool FreesMemOnFailure,
-                                ProgramStateRef State,
+                                bool ShouldFreeOnFail, ProgramStateRef State,
                                 bool SuffixWithN = false) const;
+
+  /// Evaluates the buffer size that needs to be allocated.
+  ///
+  /// \param [in] Blocks The amount of blocks that needs to be allocated.
+  /// \param [in] BlockBytes The size of a block.
+  /// \returns The symbolic value of \p Blocks * \p BlockBytes.
   static SVal evalMulForBufferSize(CheckerContext &C, const Expr *Blocks,
                                    const Expr *BlockBytes);
+
+  /// Models zero initialized array allocation.
+  ///
+  /// \param [in] CE The expression that reallocated memory
+  /// \param [in] State The \c ProgramState right before reallocation.
+  /// \returns The ProgramState right after allocation.
   static ProgramStateRef CallocMem(CheckerContext &C, const CallExpr *CE,
                                    ProgramStateRef State);
 
-  /// Check if the memory associated with this symbol was released.
-  bool isReleased(SymbolRef Sym, CheckerContext &C) const;
-
   /// See if deallocation happens in a suspicious context. If so, escape the
   /// pointers that otherwise would have been deallocated and return true.
   bool suppressDeallocationsInSuspiciousContexts(const CallExpr *CE,
                                                  CheckerContext &C) const;
 
+  /// If in \p S  \p Sym is used, check whether \p Sym was already freed.
   bool checkUseAfterFree(SymbolRef Sym, CheckerContext &C, const Stmt *S) const;
 
+  /// If in \p S \p Sym is used, check whether \p Sym was allocated as a zero
+  /// sized memory region.
   void checkUseZeroAllocated(SymbolRef Sym, CheckerContext &C,
                              const Stmt *S) const;
 
+  /// If in \p S \p Sym is being freed, check whether \p Sym was already freed.
   bool checkDoubleDelete(SymbolRef Sym, CheckerContext &C) const;
 
-  /// Check if the function is known free memory, or if it is
+  /// Check if the function is known to free memory, or if it is
   /// "interesting" and should be modeled explicitly.
   ///
   /// \param [out] EscapingSymbol A function might not free memory in general,
@@ -386,12 +606,12 @@ class MallocChecker : public Checker<check::DeadSymbols,
                                    ProgramStateRef State,
                                    SymbolRef &EscapingSymbol) const;
 
-  // Implementation of the checkPointerEscape callbacks.
+  /// Implementation of the checkPointerEscape callbacks.
   ProgramStateRef checkPointerEscapeAux(ProgramStateRef State,
-                                  const InvalidatedSymbols &Escaped,
-                                  const CallEvent *Call,
-                                  PointerEscapeKind Kind,
-                                  bool(*CheckRefState)(const RefState*)) const;
+                                        const InvalidatedSymbols &Escaped,
+                                        const CallEvent *Call,
+                                        PointerEscapeKind Kind,
+                                        bool IsConstPointerEscape) const;
 
   // Implementation of the checkPreStmt and checkEndFunction callbacks.
   void checkEscapeOnReturn(const ReturnStmt *S, CheckerContext &C) const;
@@ -410,6 +630,7 @@ class MallocChecker : public Checker<check::DeadSymbols,
   ///@}
   static bool SummarizeValue(raw_ostream &os, SVal V);
   static bool SummarizeRegion(raw_ostream &os, const MemRegion *MR);
+
   void ReportBadFree(CheckerContext &C, SVal ArgVal, SourceRange Range,
                      const Expr *DeallocExpr) const;
   void ReportFreeAlloca(CheckerContext &C, SVal ArgVal,
@@ -435,141 +656,142 @@ class MallocChecker : public Checker<check::DeadSymbols,
 
   /// Find the location of the allocation for Sym on the path leading to the
   /// exploded node N.
-  LeakInfo getAllocationSite(const ExplodedNode *N, SymbolRef Sym,
-                             CheckerContext &C) const;
+  static LeakInfo getAllocationSite(const ExplodedNode *N, SymbolRef Sym,
+                                    CheckerContext &C);
 
   void reportLeak(SymbolRef Sym, ExplodedNode *N, CheckerContext &C) const;
+};
 
-  /// The bug visitor which allows us to print extra diagnostics along the
-  /// BugReport path. For example, showing the allocation site of the leaked
-  /// region.
-  class MallocBugVisitor final : public BugReporterVisitor {
-  protected:
-    enum NotificationMode {
-      Normal,
-      ReallocationFailed
-    };
-
-    // The allocated region symbol tracked by the main analysis.
-    SymbolRef Sym;
-
-    // The mode we are in, i.e. what kind of diagnostics will be emitted.
-    NotificationMode Mode;
-
-    // A symbol from when the primary region should have been reallocated.
-    SymbolRef FailedReallocSymbol;
-
-    // A C++ destructor stack frame in which memory was released. Used for
-    // miscellaneous false positive suppression.
-    const StackFrameContext *ReleaseDestructorLC;
-
-    bool IsLeak;
-
-  public:
-    MallocBugVisitor(SymbolRef S, bool isLeak = false)
-        : Sym(S), Mode(Normal), FailedReallocSymbol(nullptr),
-          ReleaseDestructorLC(nullptr), IsLeak(isLeak) {}
-
-    static void *getTag() {
-      static int Tag = 0;
-      return &Tag;
-    }
+//===----------------------------------------------------------------------===//
+// Definition of MallocBugVisitor.
+//===----------------------------------------------------------------------===//
 
-    void Profile(llvm::FoldingSetNodeID &ID) const override {
-      ID.AddPointer(getTag());
-      ID.AddPointer(Sym);
-    }
+/// The bug visitor which allows us to print extra diagnostics along the
+/// BugReport path. For example, showing the allocation site of the leaked
+/// region.
+class MallocBugVisitor final : public BugReporterVisitor {
+protected:
+  enum NotificationMode { Normal, ReallocationFailed };
 
-    inline bool isAllocated(const RefState *S, const RefState *SPrev,
-                            const Stmt *Stmt) {
-      // Did not track -> allocated. Other state (released) -> allocated.
-      return (Stmt && (isa<CallExpr>(Stmt) || isa<CXXNewExpr>(Stmt)) &&
-              (S && (S->isAllocated() || S->isAllocatedOfSizeZero())) &&
-              (!SPrev || !(SPrev->isAllocated() ||
-                           SPrev->isAllocatedOfSizeZero())));
-    }
+  // The allocated region symbol tracked by the main analysis.
+  SymbolRef Sym;
 
-    inline bool isReleased(const RefState *S, const RefState *SPrev,
-                           const Stmt *Stmt) {
-      // Did not track -> released. Other state (allocated) -> released.
-      // The statement associated with the release might be missing.
-      bool IsReleased = (S && S->isReleased()) &&
-                        (!SPrev || !SPrev->isReleased());
-      assert(!IsReleased ||
-             (Stmt && (isa<CallExpr>(Stmt) || isa<CXXDeleteExpr>(Stmt))) ||
-             (!Stmt && S->getAllocationFamily() == AF_InnerBuffer));
-      return IsReleased;
-    }
+  // The mode we are in, i.e. what kind of diagnostics will be emitted.
+  NotificationMode Mode;
 
-    inline bool isRelinquished(const RefState *S, const RefState *SPrev,
-                               const Stmt *Stmt) {
-      // Did not track -> relinquished. Other state (allocated) -> relinquished.
-      return (Stmt && (isa<CallExpr>(Stmt) || isa<ObjCMessageExpr>(Stmt) ||
-                                              isa<ObjCPropertyRefExpr>(Stmt)) &&
-              (S && S->isRelinquished()) &&
-              (!SPrev || !SPrev->isRelinquished()));
-    }
+  // A symbol from when the primary region should have been reallocated.
+  SymbolRef FailedReallocSymbol;
 
-    inline bool isReallocFailedCheck(const RefState *S, const RefState *SPrev,
-                                     const Stmt *Stmt) {
-      // If the expression is not a call, and the state change is
-      // released -> allocated, it must be the realloc return value
-      // check. If we have to handle more cases here, it might be cleaner just
-      // to track this extra bit in the state itself.
-      return (
-          (!Stmt || !isa<CallExpr>(Stmt)) &&
-          (S && (S->isAllocated() || S->isAllocatedOfSizeZero())) &&
-          (SPrev && !(SPrev->isAllocated() || SPrev->isAllocatedOfSizeZero())));
-    }
+  // A C++ destructor stack frame in which memory was released. Used for
+  // miscellaneous false positive suppression.
+  const StackFrameContext *ReleaseDestructorLC;
 
-    PathDiagnosticPieceRef VisitNode(const ExplodedNode *N,
-                                     BugReporterContext &BRC,
-                                     PathSensitiveBugReport &BR) override;
+  bool IsLeak;
 
-    PathDiagnosticPieceRef getEndPath(BugReporterContext &BRC,
-                                      const ExplodedNode *EndPathNode,
-                                      PathSensitiveBugReport &BR) override {
-      if (!IsLeak)
-        return nullptr;
+public:
+  MallocBugVisitor(SymbolRef S, bool isLeak = false)
+      : Sym(S), Mode(Normal), FailedReallocSymbol(nullptr),
+        ReleaseDestructorLC(nullptr), IsLeak(isLeak) {}
+
+  static void *getTag() {
+    static int Tag = 0;
+    return &Tag;
+  }
+
+  void Profile(llvm::FoldingSetNodeID &ID) const override {
+    ID.AddPointer(getTag());
+    ID.AddPointer(Sym);
+  }
+
+  /// Did not track -> allocated. Other state (released) -> allocated.
+  static inline bool isAllocated(const RefState *RSCurr, const RefState *RSPrev,
+                                 const Stmt *Stmt) {
+    return (Stmt && (isa<CallExpr>(Stmt) || isa<CXXNewExpr>(Stmt)) &&
+            (RSCurr &&
+             (RSCurr->isAllocated() || RSCurr->isAllocatedOfSizeZero())) &&
+            (!RSPrev ||
+             !(RSPrev->isAllocated() || RSPrev->isAllocatedOfSizeZero())));
+  }
+
+  /// Did not track -> released. Other state (allocated) -> released.
+  /// The statement associated with the release might be missing.
+  static inline bool isReleased(const RefState *RSCurr, const RefState *RSPrev,
+                                const Stmt *Stmt) {
+    bool IsReleased =
+        (RSCurr && RSCurr->isReleased()) && (!RSPrev || !RSPrev->isReleased());
+    assert(!IsReleased ||
+           (Stmt && (isa<CallExpr>(Stmt) || isa<CXXDeleteExpr>(Stmt))) ||
+           (!Stmt && RSCurr->getAllocationFamily() == AF_InnerBuffer));
+    return IsReleased;
+  }
+
+  /// Did not track -> relinquished. Other state (allocated) -> relinquished.
+  static inline bool isRelinquished(const RefState *RSCurr,
+                                    const RefState *RSPrev, const Stmt *Stmt) {
+    return (Stmt &&
+            (isa<CallExpr>(Stmt) || isa<ObjCMessageExpr>(Stmt) ||
+             isa<ObjCPropertyRefExpr>(Stmt)) &&
+            (RSCurr && RSCurr->isRelinquished()) &&
+            (!RSPrev || !RSPrev->isRelinquished()));
+  }
+
+  /// If the expression is not a call, and the state change is
+  /// released -> allocated, it must be the realloc return value
+  /// check. If we have to handle more cases here, it might be cleaner just
+  /// to track this extra bit in the state itself.
+  static inline bool hasReallocFailed(const RefState *RSCurr,
+                                      const RefState *RSPrev,
+                                      const Stmt *Stmt) {
+    return ((!Stmt || !isa<CallExpr>(Stmt)) &&
+            (RSCurr &&
+             (RSCurr->isAllocated() || RSCurr->isAllocatedOfSizeZero())) &&
+            (RSPrev &&
+             !(RSPrev->isAllocated() || RSPrev->isAllocatedOfSizeZero())));
+  }
+
+  PathDiagnosticPieceRef VisitNode(const ExplodedNode *N,
+                                   BugReporterContext &BRC,
+                                   PathSensitiveBugReport &BR) override;
+
+  PathDiagnosticPieceRef getEndPath(BugReporterContext &BRC,
+                                    const ExplodedNode *EndPathNode,
+                                    PathSensitiveBugReport &BR) override {
+    if (!IsLeak)
+      return nullptr;
 
-      PathDiagnosticLocation L = BR.getLocation();
-      // Do not add the statement itself as a range in case of leak.
-      return std::make_shared<PathDiagnosticEventPiece>(L, BR.getDescription(),
-                                                         false);
-    }
+    PathDiagnosticLocation L = BR.getLocation();
+    // Do not add the statement itself as a range in case of leak.
+    return std::make_shared<PathDiagnosticEventPiece>(L, BR.getDescription(),
+                                                      false);
+  }
 
-  private:
-    class StackHintGeneratorForReallocationFailed
-        : public StackHintGeneratorForSymbol {
-    public:
-      StackHintGeneratorForReallocationFailed(SymbolRef S, StringRef M)
+private:
+  class StackHintGeneratorForReallocationFailed
+      : public StackHintGeneratorForSymbol {
+  public:
+    StackHintGeneratorForReallocationFailed(SymbolRef S, StringRef M)
         : StackHintGeneratorForSymbol(S, M) {}
 
-      std::string getMessageForArg(const Expr *ArgE,
-                                   unsigned ArgIndex) override {
-        // Printed parameters start at 1, not 0.
-        ++ArgIndex;
+    std::string getMessageForArg(const Expr *ArgE, unsigned ArgIndex) override {
+      // Printed parameters start at 1, not 0.
+      ++ArgIndex;
 
-        SmallString<200> buf;
-        llvm::raw_svector_ostream os(buf);
+      SmallString<200> buf;
+      llvm::raw_svector_ostream os(buf);
 
-        os << "Reallocation of " << ArgIndex << llvm::getOrdinalSuffix(ArgIndex)
-           << " parameter failed";
+      os << "Reallocation of " << ArgIndex << llvm::getOrdinalSuffix(ArgIndex)
+         << " parameter failed";
 
-        return os.str();
-      }
+      return os.str();
+    }
 
-      std::string getMessageForReturn(const CallExpr *CallExpr) override {
-        return "Reallocation of returned value failed";
-      }
-    };
+    std::string getMessageForReturn(const CallExpr *CallExpr) override {
+      return "Reallocation of returned value failed";
+    }
   };
 };
-} // end anonymous namespace
 
-REGISTER_MAP_WITH_PROGRAMSTATE(RegionState, SymbolRef, RefState)
-REGISTER_MAP_WITH_PROGRAMSTATE(ReallocPairs, SymbolRef, ReallocPair)
-REGISTER_SET_WITH_PROGRAMSTATE(ReallocSizeZeroSymbols, SymbolRef)
+} // end anonymous namespace
 
 // A map from the freed symbol to the symbol representing the return value of
 // the free function.
@@ -589,7 +811,11 @@ class StopTrackingCallback final : public SymbolVisitor {
 };
 } // end anonymous namespace
 
-void MallocChecker::initIdentifierInfo(ASTContext &Ctx) const {
+//===----------------------------------------------------------------------===//
+// Methods of MemFunctionInfoTy.
+//===----------------------------------------------------------------------===//
+
+void MemFunctionInfoTy::initIdentifierInfo(ASTContext &Ctx) const {
   if (II_malloc)
     return;
   II_alloca = &Ctx.Idents.get("alloca");
@@ -629,7 +855,8 @@ void MallocChecker::initIdentifierInfo(ASTContext &Ctx) const {
   II_g_try_realloc_n = &Ctx.Idents.get("g_try_realloc_n");
 }
 
-bool MallocChecker::isMemFunction(const FunctionDecl *FD, ASTContext &C) const {
+bool MemFunctionInfoTy::isMemFunction(const FunctionDecl *FD,
+                                      ASTContext &C) const {
   if (isCMemFunction(FD, C, AF_Malloc, MemoryOperationKind::MOK_Any))
     return true;
 
@@ -645,10 +872,9 @@ bool MallocChecker::isMemFunction(const FunctionDecl *FD, ASTContext &C) const {
   return false;
 }
 
-bool MallocChecker::isCMemFunction(const FunctionDecl *FD,
-                                   ASTContext &C,
-                                   AllocationFamily Family,
-                                   MemoryOperationKind MemKind) const {
+bool MemFunctionInfoTy::isCMemFunction(const FunctionDecl *FD, ASTContext &C,
+                                       AllocationFamily Family,
+                                       MemoryOperationKind MemKind) const {
   if (!FD)
     return false;
 
@@ -701,7 +927,7 @@ bool MallocChecker::isCMemFunction(const FunctionDecl *FD,
   if (Family != AF_Malloc)
     return false;
 
-  if (IsOptimistic && FD->hasAttrs()) {
+  if (ShouldIncludeOwnershipAnnotatedFunctions && FD->hasAttrs()) {
     for (const auto *I : FD->specific_attrs<OwnershipAttr>()) {
       OwnershipAttr::OwnershipKind OwnKind = I->getOwnKind();
       if(OwnKind == OwnershipAttr::Takes || OwnKind == OwnershipAttr::Holds) {
@@ -716,11 +942,8 @@ bool MallocChecker::isCMemFunction(const FunctionDecl *FD,
 
   return false;
 }
-
-// Tells if the callee is one of the builtin new/delete operators, including
-// placement operators and other standard overloads.
-bool MallocChecker::isStandardNewDelete(const FunctionDecl *FD,
-                                        ASTContext &C) const {
+bool MemFunctionInfoTy::isStandardNewDelete(const FunctionDecl *FD,
+                                            ASTContext &C) const {
   if (!FD)
     return false;
 
@@ -736,6 +959,10 @@ bool MallocChecker::isStandardNewDelete(const FunctionDecl *FD,
   return !L.isValid() || C.getSourceManager().isInSystemHeader(L);
 }
 
+//===----------------------------------------------------------------------===//
+// Methods of MallocChecker and MallocBugVisitor.
+//===----------------------------------------------------------------------===//
+
 llvm::Optional<ProgramStateRef> MallocChecker::performKernelMalloc(
   const CallExpr *CE, CheckerContext &C, const ProgramStateRef &State) const {
   // 3-argument malloc(), as commonly used in {Free,Net,Open}BSD Kernels:
@@ -834,28 +1061,35 @@ void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
     return;
 
   ProgramStateRef State = C.getState();
-  bool ReleasedAllocatedMemory = false;
+  bool IsKnownToBeAllocatedMemory = false;
 
   if (FD->getKind() == Decl::Function) {
-    initIdentifierInfo(C.getASTContext());
+    MemFunctionInfo.initIdentifierInfo(C.getASTContext());
     IdentifierInfo *FunI = FD->getIdentifier();
 
-    if (FunI == II_malloc || FunI == II_g_malloc || FunI == II_g_try_malloc) {
-      if (CE->getNumArgs() < 1)
+    if (FunI == MemFunctionInfo.II_malloc ||
+        FunI == MemFunctionInfo.II_g_malloc ||
+        FunI == MemFunctionInfo.II_g_try_malloc) {
+      switch (CE->getNumArgs()) {
+      default:
         return;
-      if (CE->getNumArgs() < 3) {
+      case 1:
+        State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
+        State = ProcessZeroAllocCheck(C, CE, 0, State);
+        break;
+      case 2:
         State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
-        if (CE->getNumArgs() == 1)
-          State = ProcessZeroAllocation(C, CE, 0, State);
-      } else if (CE->getNumArgs() == 3) {
+        break;
+      case 3:
         llvm::Optional<ProgramStateRef> MaybeState =
           performKernelMalloc(CE, C, State);
         if (MaybeState.hasValue())
           State = MaybeState.getValue();
         else
           State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
+        break;
       }
-    } else if (FunI == II_kmalloc) {
+    } else if (FunI == MemFunctionInfo.II_kmalloc) {
       if (CE->getNumArgs() < 1)
         return;
       llvm::Optional<ProgramStateRef> MaybeState =
@@ -864,100 +1098,116 @@ void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
         State = MaybeState.getValue();
       else
         State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
-    } else if (FunI == II_valloc) {
+    } else if (FunI == MemFunctionInfo.II_valloc) {
       if (CE->getNumArgs() < 1)
         return;
       State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State);
-      State = ProcessZeroAllocation(C, CE, 0, State);
-    } else if (FunI == II_realloc || FunI == II_g_realloc ||
-               FunI == II_g_try_realloc) {
-      State = ReallocMemAux(C, CE, false, State);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-    } else if (FunI == II_reallocf) {
-      State = ReallocMemAux(C, CE, true, State);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-    } else if (FunI == II_calloc) {
+      State = ProcessZeroAllocCheck(C, CE, 0, State);
+    } else if (FunI == MemFunctionInfo.II_realloc ||
+               FunI == MemFunctionInfo.II_g_realloc ||
+               FunI == MemFunctionInfo.II_g_try_realloc) {
+      State = ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ false, State);
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+    } else if (FunI == MemFunctionInfo.II_reallocf) {
+      State = ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ true, State);
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+    } else if (FunI == MemFunctionInfo.II_calloc) {
       State = CallocMem(C, CE, State);
-      State = ProcessZeroAllocation(C, CE, 0, State);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-    } else if (FunI == II_free || FunI == II_g_free || FunI == II_kfree) {
+      State = ProcessZeroAllocCheck(C, CE, 0, State);
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+    } else if (FunI == MemFunctionInfo.II_free ||
+               FunI == MemFunctionInfo.II_g_free ||
+               FunI == MemFunctionInfo.II_kfree) {
       if (suppressDeallocationsInSuspiciousContexts(CE, C))
         return;
 
-      State = FreeMemAux(C, CE, State, 0, false, ReleasedAllocatedMemory);
-    } else if (FunI == II_strdup || FunI == II_win_strdup ||
-               FunI == II_wcsdup || FunI == II_win_wcsdup) {
+      State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory);
+    } else if (FunI == MemFunctionInfo.II_strdup ||
+               FunI == MemFunctionInfo.II_win_strdup ||
+               FunI == MemFunctionInfo.II_wcsdup ||
+               FunI == MemFunctionInfo.II_win_wcsdup) {
       State = MallocUpdateRefState(C, CE, State);
-    } else if (FunI == II_strndup) {
+    } else if (FunI == MemFunctionInfo.II_strndup) {
       State = MallocUpdateRefState(C, CE, State);
-    } else if (FunI == II_alloca || FunI == II_win_alloca) {
+    } else if (FunI == MemFunctionInfo.II_alloca ||
+               FunI == MemFunctionInfo.II_win_alloca) {
       if (CE->getNumArgs() < 1)
         return;
       State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
                            AF_Alloca);
-      State = ProcessZeroAllocation(C, CE, 0, State);
-    } else if (isStandardNewDelete(FD, C.getASTContext())) {
+      State = ProcessZeroAllocCheck(C, CE, 0, State);
+    } else if (MemFunctionInfo.isStandardNewDelete(FD, C.getASTContext())) {
       // Process direct calls to operator new/new[]/delete/delete[] functions
       // as distinct from new/new[]/delete/delete[] expressions that are
       // processed by the checkPostStmt callbacks for CXXNewExpr and
       // CXXDeleteExpr.
-      OverloadedOperatorKind K = FD->getOverloadedOperator();
-      if (K == OO_New) {
+      switch (FD->getOverloadedOperator()) {
+      case OO_New:
         State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
                              AF_CXXNew);
-        State = ProcessZeroAllocation(C, CE, 0, State);
-      }
-      else if (K == OO_Array_New) {
+        State = ProcessZeroAllocCheck(C, CE, 0, State);
+        break;
+      case OO_Array_New:
         State = MallocMemAux(C, CE, CE->getArg(0), UndefinedVal(), State,
                              AF_CXXNewArray);
-        State = ProcessZeroAllocation(C, CE, 0, State);
-      }
-      else if (K == OO_Delete || K == OO_Array_Delete)
-        State = FreeMemAux(C, CE, State, 0, false, ReleasedAllocatedMemory);
-      else
+        State = ProcessZeroAllocCheck(C, CE, 0, State);
+        break;
+      case OO_Delete:
+      case OO_Array_Delete:
+        State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory);
+        break;
+      default:
         llvm_unreachable("not a new/delete operator");
-    } else if (FunI == II_if_nameindex) {
+      }
+    } else if (FunI == MemFunctionInfo.II_if_nameindex) {
       // Should we model this differently? We can allocate a fixed number of
       // elements with zeros in the last one.
       State = MallocMemAux(C, CE, UnknownVal(), UnknownVal(), State,
                            AF_IfNameIndex);
-    } else if (FunI == II_if_freenameindex) {
-      State = FreeMemAux(C, CE, State, 0, false, ReleasedAllocatedMemory);
-    } else if (FunI == II_g_malloc0 || FunI == II_g_try_malloc0) {
+    } else if (FunI == MemFunctionInfo.II_if_freenameindex) {
+      State = FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocatedMemory);
+    } else if (FunI == MemFunctionInfo.II_g_malloc0 ||
+               FunI == MemFunctionInfo.II_g_try_malloc0) {
       if (CE->getNumArgs() < 1)
         return;
       SValBuilder &svalBuilder = C.getSValBuilder();
       SVal zeroVal = svalBuilder.makeZeroVal(svalBuilder.getContext().CharTy);
       State = MallocMemAux(C, CE, CE->getArg(0), zeroVal, State);
-      State = ProcessZeroAllocation(C, CE, 0, State);
-    } else if (FunI == II_g_memdup) {
+      State = ProcessZeroAllocCheck(C, CE, 0, State);
+    } else if (FunI == MemFunctionInfo.II_g_memdup) {
       if (CE->getNumArgs() < 2)
         return;
       State = MallocMemAux(C, CE, CE->getArg(1), UndefinedVal(), State);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-    } else if (FunI == II_g_malloc_n || FunI == II_g_try_malloc_n ||
-               FunI == II_g_malloc0_n || FunI == II_g_try_malloc0_n) {
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+    } else if (FunI == MemFunctionInfo.II_g_malloc_n ||
+               FunI == MemFunctionInfo.II_g_try_malloc_n ||
+               FunI == MemFunctionInfo.II_g_malloc0_n ||
+               FunI == MemFunctionInfo.II_g_try_malloc0_n) {
       if (CE->getNumArgs() < 2)
         return;
       SVal Init = UndefinedVal();
-      if (FunI == II_g_malloc0_n || FunI == II_g_try_malloc0_n) {
+      if (FunI == MemFunctionInfo.II_g_malloc0_n ||
+          FunI == MemFunctionInfo.II_g_try_malloc0_n) {
         SValBuilder &SB = C.getSValBuilder();
         Init = SB.makeZeroVal(SB.getContext().CharTy);
       }
       SVal TotalSize = evalMulForBufferSize(C, CE->getArg(0), CE->getArg(1));
       State = MallocMemAux(C, CE, TotalSize, Init, State);
-      State = ProcessZeroAllocation(C, CE, 0, State);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-    } else if (FunI == II_g_realloc_n || FunI == II_g_try_realloc_n) {
+      State = ProcessZeroAllocCheck(C, CE, 0, State);
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+    } else if (FunI == MemFunctionInfo.II_g_realloc_n ||
+               FunI == MemFunctionInfo.II_g_try_realloc_n) {
       if (CE->getNumArgs() < 3)
         return;
-      State = ReallocMemAux(C, CE, false, State, true);
-      State = ProcessZeroAllocation(C, CE, 1, State);
-      State = ProcessZeroAllocation(C, CE, 2, State);
+      State = ReallocMemAux(C, CE, /*ShouldFreeOnFail*/ false, State,
+                            /*SuffixWithN*/ true);
+      State = ProcessZeroAllocCheck(C, CE, 1, State);
+      State = ProcessZeroAllocCheck(C, CE, 2, State);
     }
   }
 
-  if (IsOptimistic || ChecksEnabled[CK_MismatchedDeallocatorChecker]) {
+  if (MemFunctionInfo.ShouldIncludeOwnershipAnnotatedFunctions ||
+      ChecksEnabled[CK_MismatchedDeallocatorChecker]) {
     // Check all the attributes, if there are any.
     // There can be multiple of these attributes.
     if (FD->hasAttrs())
@@ -977,9 +1227,9 @@ void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
 }
 
 // Performs a 0-sized allocations check.
-ProgramStateRef MallocChecker::ProcessZeroAllocation(
-    CheckerContext &C, const Expr *E, const unsigned AllocationSizeArg,
-    ProgramStateRef State, Optional<SVal> RetVal) const {
+ProgramStateRef MallocChecker::ProcessZeroAllocCheck(
+    CheckerContext &C, const Expr *E, const unsigned IndexOfSizeArg,
+    ProgramStateRef State, Optional<SVal> RetVal) {
   if (!State)
     return nullptr;
 
@@ -989,7 +1239,7 @@ ProgramStateRef MallocChecker::ProcessZeroAllocation(
   const Expr *Arg = nullptr;
 
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
-    Arg = CE->getArg(AllocationSizeArg);
+    Arg = CE->getArg(IndexOfSizeArg);
   }
   else if (const CXXNewExpr *NE = dyn_cast<CXXNewExpr>(E)) {
     if (NE->isArray())
@@ -1051,7 +1301,9 @@ static QualType getDeepPointeeType(QualType T) {
   return Result;
 }
 
-static bool treatUnusedNewEscaped(const CXXNewExpr *NE) {
+/// \returns true if the constructor invoked by \p NE has an argument of a
+/// pointer/reference to a record type.
+static bool hasNonTrivialConstructorCall(const CXXNewExpr *NE) {
 
   const CXXConstructExpr *ConstructE = NE->getConstructExpr();
   if (!ConstructE)
@@ -1081,11 +1333,17 @@ static bool treatUnusedNewEscaped(const CXXNewExpr *NE) {
 void MallocChecker::processNewAllocation(const CXXNewExpr *NE,
                                          CheckerContext &C,
                                          SVal Target) const {
-  if (!isStandardNewDelete(NE->getOperatorNew(), C.getASTContext()))
+  if (!MemFunctionInfo.isStandardNewDelete(NE->getOperatorNew(),
+                                           C.getASTContext()))
     return;
 
   const ParentMap &PM = C.getLocationContext()->getParentMap();
-  if (!PM.isConsumedExpr(NE) && treatUnusedNewEscaped(NE))
+
+  // Non-trivial constructors have a chance to escape 'this', but marking all
+  // invocations of trivial constructors as escaped would cause too great of
+  // reduction of true positives, so let's just do that for constructors that
+  // have an argument of a pointer-to-record type.
+  if (!PM.isConsumedExpr(NE) && hasNonTrivialConstructorCall(NE))
     return;
 
   ProgramStateRef State = C.getState();
@@ -1096,7 +1354,7 @@ void MallocChecker::processNewAllocation(const CXXNewExpr *NE,
   State = MallocUpdateRefState(C, NE, State, NE->isArray() ? AF_CXXNewArray
                                                            : AF_CXXNew, Target);
   State = addExtentSize(C, NE, State, Target);
-  State = ProcessZeroAllocation(C, NE, 0, State, Target);
+  State = ProcessZeroAllocCheck(C, NE, 0, State, Target);
   C.addTransition(State);
 }
 
@@ -1164,13 +1422,14 @@ void MallocChecker::checkPreStmt(const CXXDeleteExpr *DE,
     if (SymbolRef Sym = C.getSVal(DE->getArgument()).getAsSymbol())
       checkUseAfterFree(Sym, C, DE->getArgument());
 
-  if (!isStandardNewDelete(DE->getOperatorDelete(), C.getASTContext()))
+  if (!MemFunctionInfo.isStandardNewDelete(DE->getOperatorDelete(),
+                                           C.getASTContext()))
     return;
 
   ProgramStateRef State = C.getState();
-  bool ReleasedAllocated;
+  bool IsKnownToBeAllocated;
   State = FreeMemAux(C, DE->getArgument(), DE, State,
-                     /*Hold*/false, ReleasedAllocated);
+                     /*Hold*/ false, IsKnownToBeAllocated);
 
   C.addTransition(State);
 }
@@ -1210,11 +1469,11 @@ void MallocChecker::checkPostObjCMessage(const ObjCMethodCall &Call,
     if (!*FreeWhenDone)
       return;
 
-  bool ReleasedAllocatedMemory;
-  ProgramStateRef State = FreeMemAux(C, Call.getArgExpr(0),
-                                     Call.getOriginExpr(), C.getState(),
-                                     /*Hold=*/true, ReleasedAllocatedMemory,
-                                     /*ReturnsNullOnFailure=*/true);
+  bool IsKnownToBeAllocatedMemory;
+  ProgramStateRef State =
+      FreeMemAux(C, Call.getArgExpr(0), Call.getOriginExpr(), C.getState(),
+                 /*Hold=*/true, IsKnownToBeAllocatedMemory,
+                 /*RetNullOnFailure=*/true);
 
   C.addTransition(State);
 }
@@ -1226,7 +1485,7 @@ MallocChecker::MallocMemReturnsAttr(CheckerContext &C, const CallExpr *CE,
   if (!State)
     return nullptr;
 
-  if (Att->getModule() != II_malloc)
+  if (Att->getModule() != MemFunctionInfo.II_malloc)
     return nullptr;
 
   OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
@@ -1292,11 +1551,10 @@ ProgramStateRef MallocChecker::MallocMemAux(CheckerContext &C,
   return MallocUpdateRefState(C, CE, State, Family);
 }
 
-ProgramStateRef MallocChecker::MallocUpdateRefState(CheckerContext &C,
-                                                    const Expr *E,
-                                                    ProgramStateRef State,
-                                                    AllocationFamily Family,
-                                                    Optional<SVal> RetVal) {
+static ProgramStateRef MallocUpdateRefState(CheckerContext &C, const Expr *E,
+                                            ProgramStateRef State,
+                                            AllocationFamily Family,
+                                            Optional<SVal> RetVal) {
   if (!State)
     return nullptr;
 
@@ -1324,27 +1582,24 @@ ProgramStateRef MallocChecker::FreeMemAttr(CheckerContext &C,
   if (!State)
     return nullptr;
 
-  if (Att->getModule() != II_malloc)
+  if (Att->getModule() != MemFunctionInfo.II_malloc)
     return nullptr;
 
-  bool ReleasedAllocated = false;
+  bool IsKnownToBeAllocated = false;
 
   for (const auto &Arg : Att->args()) {
     ProgramStateRef StateI = FreeMemAux(
         C, CE, State, Arg.getASTIndex(),
-        Att->getOwnKind() == OwnershipAttr::Holds, ReleasedAllocated);
+        Att->getOwnKind() == OwnershipAttr::Holds, IsKnownToBeAllocated);
     if (StateI)
       State = StateI;
   }
   return State;
 }
 
-ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
-                                          const CallExpr *CE,
-                                          ProgramStateRef State,
-                                          unsigned Num,
-                                          bool Hold,
-                                          bool &ReleasedAllocated,
+ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C, const CallExpr *CE,
+                                          ProgramStateRef State, unsigned Num,
+                                          bool Hold, bool &IsKnownToBeAllocated,
                                           bool ReturnsNullOnFailure) const {
   if (!State)
     return nullptr;
@@ -1352,8 +1607,8 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
   if (CE->getNumArgs() < (Num + 1))
     return nullptr;
 
-  return FreeMemAux(C, CE->getArg(Num), CE, State, Hold,
-                    ReleasedAllocated, ReturnsNullOnFailure);
+  return FreeMemAux(C, CE->getArg(Num), CE, State, Hold, IsKnownToBeAllocated,
+                    ReturnsNullOnFailure);
 }
 
 /// Checks if the previous call to free on the given symbol failed - if free
@@ -1371,8 +1626,10 @@ static bool didPreviousFreeFail(ProgramStateRef State,
   return false;
 }
 
-AllocationFamily MallocChecker::getAllocationFamily(CheckerContext &C,
-                                                    const Stmt *S) const {
+static AllocationFamily
+getAllocationFamily(const MemFunctionInfoTy &MemFunctionInfo, CheckerContext &C,
+                    const Stmt *S) {
+
   if (!S)
     return AF_None;
 
@@ -1384,10 +1641,11 @@ AllocationFamily MallocChecker::getAllocationFamily(CheckerContext &C,
 
     ASTContext &Ctx = C.getASTContext();
 
-    if (isCMemFunction(FD, Ctx, AF_Malloc, MemoryOperationKind::MOK_Any))
+    if (MemFunctionInfo.isCMemFunction(FD, Ctx, AF_Malloc,
+                                       MemoryOperationKind::MOK_Any))
       return AF_Malloc;
 
-    if (isStandardNewDelete(FD, Ctx)) {
+    if (MemFunctionInfo.isStandardNewDelete(FD, Ctx)) {
       OverloadedOperatorKind Kind = FD->getOverloadedOperator();
       if (Kind == OO_New || Kind == OO_Delete)
         return AF_CXXNew;
@@ -1395,10 +1653,12 @@ AllocationFamily MallocChecker::getAllocationFamily(CheckerContext &C,
         return AF_CXXNewArray;
     }
 
-    if (isCMemFunction(FD, Ctx, AF_IfNameIndex, MemoryOperationKind::MOK_Any))
+    if (MemFunctionInfo.isCMemFunction(FD, Ctx, AF_IfNameIndex,
+                                       MemoryOperationKind::MOK_Any))
       return AF_IfNameIndex;
 
-    if (isCMemFunction(FD, Ctx, AF_Alloca, MemoryOperationKind::MOK_Any))
+    if (MemFunctionInfo.isCMemFunction(FD, Ctx, AF_Alloca,
+                                       MemoryOperationKind::MOK_Any))
       return AF_Alloca;
 
     return AF_None;
@@ -1416,8 +1676,8 @@ AllocationFamily MallocChecker::getAllocationFamily(CheckerContext &C,
   return AF_None;
 }
 
-bool MallocChecker::printAllocDeallocName(raw_ostream &os, CheckerContext &C,
-                                          const Expr *E) const {
+static bool printAllocDeallocName(raw_ostream &os, CheckerContext &C,
+                                  const Expr *E) {
   if (const CallExpr *CE = dyn_cast<CallExpr>(E)) {
     // FIXME: This doesn't handle indirect calls.
     const FunctionDecl *FD = CE->getDirectCallee();
@@ -1456,9 +1716,10 @@ bool MallocChecker::printAllocDeallocName(raw_ostream &os, CheckerContext &C,
   return false;
 }
 
-void MallocChecker::printExpectedAllocName(raw_ostream &os, CheckerContext &C,
-                                           const Expr *E) const {
-  AllocationFamily Family = getAllocationFamily(C, E);
+static void printExpectedAllocName(raw_ostream &os,
+                                   const MemFunctionInfoTy &MemFunctionInfo,
+                                   CheckerContext &C, const Expr *E) {
+  AllocationFamily Family = getAllocationFamily(MemFunctionInfo, C, E);
 
   switch(Family) {
     case AF_Malloc: os << "malloc()"; return;
@@ -1471,8 +1732,7 @@ void MallocChecker::printExpectedAllocName(raw_ostream &os, CheckerContext &C,
   }
 }
 
-void MallocChecker::printExpectedDeallocName(raw_ostream &os,
-                                             AllocationFamily Family) const {
+static void printExpectedDeallocName(raw_ostream &os, AllocationFamily Family) {
   switch(Family) {
     case AF_Malloc: os << "free()"; return;
     case AF_CXXNew: os << "'delete'"; return;
@@ -1487,9 +1747,8 @@ void MallocChecker::printExpectedDeallocName(raw_ostream &os,
 ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
                                           const Expr *ArgExpr,
                                           const Expr *ParentExpr,
-                                          ProgramStateRef State,
-                                          bool Hold,
-                                          bool &ReleasedAllocated,
+                                          ProgramStateRef State, bool Hold,
+                                          bool &IsKnownToBeAllocated,
                                           bool ReturnsNullOnFailure) const {
 
   if (!State)
@@ -1563,6 +1822,9 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
   const RefState *RsBase = State->get<RegionState>(SymBase);
   SymbolRef PreviousRetStatusSymbol = nullptr;
 
+  IsKnownToBeAllocated =
+      RsBase && (RsBase->isAllocated() || RsBase->isAllocatedOfSizeZero());
+
   if (RsBase) {
 
     // Memory returned by alloca() shouldn't be freed.
@@ -1585,7 +1847,8 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
 
       // Check if an expected deallocation function matches the real one.
       bool DeallocMatchesAlloc =
-        RsBase->getAllocationFamily() == getAllocationFamily(C, ParentExpr);
+          RsBase->getAllocationFamily() ==
+          getAllocationFamily(MemFunctionInfo, C, ParentExpr);
       if (!DeallocMatchesAlloc) {
         ReportMismatchedDealloc(C, ArgExpr->getSourceRange(),
                                 ParentExpr, RsBase, SymBase, Hold);
@@ -1611,9 +1874,6 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
     return nullptr;
   }
 
-  ReleasedAllocated = (RsBase != nullptr) && (RsBase->isAllocated() ||
-                                              RsBase->isAllocatedOfSizeZero());
-
   // Clean out the info on previous call to free return info.
   State = State->remove<FreeReturnValue>(SymBase);
 
@@ -1628,8 +1888,9 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
     }
   }
 
-  AllocationFamily Family = RsBase ? RsBase->getAllocationFamily()
-                                   : getAllocationFamily(C, ParentExpr);
+  AllocationFamily Family =
+      RsBase ? RsBase->getAllocationFamily()
+             : getAllocationFamily(MemFunctionInfo, C, ParentExpr);
   // Normal free.
   if (Hold)
     return State->set<RegionState>(SymBase,
@@ -1679,8 +1940,8 @@ Optional<MallocChecker::CheckKind>
 MallocChecker::getCheckIfTracked(CheckerContext &C,
                                  const Stmt *AllocDeallocStmt,
                                  bool IsALeakCheck) const {
-  return getCheckIfTracked(getAllocationFamily(C, AllocDeallocStmt),
-                           IsALeakCheck);
+  return getCheckIfTracked(
+      getAllocationFamily(MemFunctionInfo, C, AllocDeallocStmt), IsALeakCheck);
 }
 
 Optional<MallocChecker::CheckKind>
@@ -1818,7 +2079,7 @@ void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
     else
       os << "not memory allocated by ";
 
-    printExpectedAllocName(os, C, DeallocExpr);
+    printExpectedAllocName(os, MemFunctionInfo, C, DeallocExpr);
 
     auto R = std::make_unique<PathSensitiveBugReport>(*BT_BadFree[*CheckKind],
                                                       os.str(), N);
@@ -2130,7 +2391,7 @@ void MallocChecker::ReportFunctionPointerFree(CheckerContext &C, SVal ArgVal,
 
 ProgramStateRef MallocChecker::ReallocMemAux(CheckerContext &C,
                                              const CallExpr *CE,
-                                             bool FreesOnFail,
+                                             bool ShouldFreeOnFail,
                                              ProgramStateRef State,
                                              bool SuffixWithN) const {
   if (!State)
@@ -2195,33 +2456,32 @@ ProgramStateRef MallocChecker::ReallocMemAux(CheckerContext &C,
   if (!FromPtr || !ToPtr)
     return nullptr;
 
-  bool ReleasedAllocated = false;
+  bool IsKnownToBeAllocated = false;
 
   // If the size is 0, free the memory.
   if (SizeIsZero)
-    if (ProgramStateRef stateFree = FreeMemAux(C, CE, StateSizeIsZero, 0,
-                                               false, ReleasedAllocated)){
-      // The semantics of the return value are:
-      // If size was equal to 0, either NULL or a pointer suitable to be passed
-      // to free() is returned. We just free the input pointer and do not add
-      // any constrains on the output pointer.
+    // The semantics of the return value are:
+    // If size was equal to 0, either NULL or a pointer suitable to be passed
+    // to free() is returned. We just free the input pointer and do not add
+    // any constrains on the output pointer.
+    if (ProgramStateRef stateFree =
+            FreeMemAux(C, CE, StateSizeIsZero, 0, false, IsKnownToBeAllocated))
       return stateFree;
-    }
 
   // Default behavior.
   if (ProgramStateRef stateFree =
-        FreeMemAux(C, CE, State, 0, false, ReleasedAllocated)) {
+          FreeMemAux(C, CE, State, 0, false, IsKnownToBeAllocated)) {
 
     ProgramStateRef stateRealloc = MallocMemAux(C, CE, TotalSize,
                                                 UnknownVal(), stateFree);
     if (!stateRealloc)
       return nullptr;
 
-    ReallocPairKind Kind = RPToBeFreedAfterFailure;
-    if (FreesOnFail)
-      Kind = RPIsFreeOnFailure;
-    else if (!ReleasedAllocated)
-      Kind = RPDoNotTrackAfterFailure;
+    OwnershipAfterReallocKind Kind = OAR_ToBeFreedAfterFailure;
+    if (ShouldFreeOnFail)
+      Kind = OAR_FreeOnFailure;
+    else if (!IsKnownToBeAllocated)
+      Kind = OAR_DoNotTrackAfterFailure;
 
     // Record the info about the reallocated symbol so that we could properly
     // process failed reallocation.
@@ -2249,9 +2509,9 @@ ProgramStateRef MallocChecker::CallocMem(CheckerContext &C, const CallExpr *CE,
   return MallocMemAux(C, CE, TotalSize, zeroVal, State);
 }
 
-LeakInfo
-MallocChecker::getAllocationSite(const ExplodedNode *N, SymbolRef Sym,
-                                 CheckerContext &C) const {
+MallocChecker::LeakInfo MallocChecker::getAllocationSite(const ExplodedNode *N,
+                                                         SymbolRef Sym,
+                                                         CheckerContext &C) {
   const LocationContext *LeakContext = N->getLocationContext();
   // Walk the ExplodedGraph backwards and find the first node that referred to
   // the tracked symbol.
@@ -2432,9 +2692,10 @@ void MallocChecker::checkPreCall(const CallEvent &Call,
 
     ASTContext &Ctx = C.getASTContext();
     if (ChecksEnabled[CK_MallocChecker] &&
-        (isCMemFunction(FD, Ctx, AF_Malloc, MemoryOperationKind::MOK_Free) ||
-         isCMemFunction(FD, Ctx, AF_IfNameIndex,
-                        MemoryOperationKind::MOK_Free)))
+        (MemFunctionInfo.isCMemFunction(FD, Ctx, AF_Malloc,
+                                        MemoryOperationKind::MOK_Free) ||
+         MemFunctionInfo.isCMemFunction(FD, Ctx, AF_IfNameIndex,
+                                        MemoryOperationKind::MOK_Free)))
       return;
   }
 
@@ -2537,7 +2798,7 @@ void MallocChecker::checkPostStmt(const BlockExpr *BE,
   C.addTransition(state);
 }
 
-bool MallocChecker::isReleased(SymbolRef Sym, CheckerContext &C) const {
+static bool isReleased(SymbolRef Sym, CheckerContext &C) {
   assert(Sym);
   const RefState *RS = C.getState()->get<RegionState>(Sym);
   return (RS && RS->isReleased());
@@ -2642,13 +2903,17 @@ ProgramStateRef MallocChecker::evalAssume(ProgramStateRef state,
     SymbolRef ReallocSym = I.getData().ReallocatedSym;
     if (const RefState *RS = state->get<RegionState>(ReallocSym)) {
       if (RS->isReleased()) {
-        if (I.getData().Kind == RPToBeFreedAfterFailure)
+        switch (I.getData().Kind) {
+        case OAR_ToBeFreedAfterFailure:
           state = state->set<RegionState>(ReallocSym,
               RefState::getAllocated(RS->getAllocationFamily(), RS->getStmt()));
-        else if (I.getData().Kind == RPDoNotTrackAfterFailure)
+          break;
+        case OAR_DoNotTrackAfterFailure:
           state = state->remove<RegionState>(ReallocSym);
-        else
-          assert(I.getData().Kind == RPIsFreeOnFailure);
+          break;
+        default:
+          assert(I.getData().Kind == OAR_FreeOnFailure);
+        }
       }
     }
     state = state->remove<ReallocPairs>(I.getKey());
@@ -2731,7 +2996,7 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
 
   // If it's one of the allocation functions we can reason about, we model
   // its behavior explicitly.
-  if (isMemFunction(FD, ASTC))
+  if (MemFunctionInfo.isMemFunction(FD, ASTC))
     return false;
 
   // If it's not a system call, assume it frees memory.
@@ -2823,35 +3088,32 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
   return false;
 }
 
-static bool retTrue(const RefState *RS) {
-  return true;
-}
-
-static bool checkIfNewOrNewArrayFamily(const RefState *RS) {
-  return (RS->getAllocationFamily() == AF_CXXNewArray ||
-          RS->getAllocationFamily() == AF_CXXNew);
-}
-
 ProgramStateRef MallocChecker::checkPointerEscape(ProgramStateRef State,
                                              const InvalidatedSymbols &Escaped,
                                              const CallEvent *Call,
                                              PointerEscapeKind Kind) const {
-  return checkPointerEscapeAux(State, Escaped, Call, Kind, &retTrue);
+  return checkPointerEscapeAux(State, Escaped, Call, Kind,
+                               /*IsConstPointerEscape*/ false);
 }
 
 ProgramStateRef MallocChecker::checkConstPointerEscape(ProgramStateRef State,
                                               const InvalidatedSymbols &Escaped,
                                               const CallEvent *Call,
                                               PointerEscapeKind Kind) const {
+  // If a const pointer escapes, it may not be freed(), but it could be deleted.
   return checkPointerEscapeAux(State, Escaped, Call, Kind,
-                               &checkIfNewOrNewArrayFamily);
+                               /*IsConstPointerEscape*/ true);
 }
 
-ProgramStateRef MallocChecker::checkPointerEscapeAux(ProgramStateRef State,
-                                              const InvalidatedSymbols &Escaped,
-                                              const CallEvent *Call,
-                                              PointerEscapeKind Kind,
-                                  bool(*CheckRefState)(const RefState*)) const {
+static bool checkIfNewOrNewArrayFamily(const RefState *RS) {
+  return (RS->getAllocationFamily() == AF_CXXNewArray ||
+          RS->getAllocationFamily() == AF_CXXNew);
+}
+
+ProgramStateRef MallocChecker::checkPointerEscapeAux(
+    ProgramStateRef State, const InvalidatedSymbols &Escaped,
+    const CallEvent *Call, PointerEscapeKind Kind,
+    bool IsConstPointerEscape) const {
   // If we know that the call does not free memory, or we want to process the
   // call later, keep tracking the top level arguments.
   SymbolRef EscapingSymbol = nullptr;
@@ -2870,12 +3132,10 @@ ProgramStateRef MallocChecker::checkPointerEscapeAux(ProgramStateRef State,
     if (EscapingSymbol && EscapingSymbol != sym)
       continue;
 
-    if (const RefState *RS = State->get<RegionState>(sym)) {
-      if ((RS->isAllocated() || RS->isAllocatedOfSizeZero()) &&
-          CheckRefState(RS)) {
-        State = State->set<RegionState>(sym, RefState::getEscaped(RS));
-      }
-    }
+    if (const RefState *RS = State->get<RegionState>(sym))
+      if (RS->isAllocated() || RS->isAllocatedOfSizeZero())
+        if (!IsConstPointerEscape || checkIfNewOrNewArrayFamily(RS))
+          State = State->set<RegionState>(sym, RefState::getEscaped(RS));
   }
   return State;
 }
@@ -2885,9 +3145,8 @@ static SymbolRef findFailedReallocSymbol(ProgramStateRef currState,
   ReallocPairsTy currMap = currState->get<ReallocPairs>();
   ReallocPairsTy prevMap = prevState->get<ReallocPairs>();
 
-  for (ReallocPairsTy::iterator I = prevMap.begin(), E = prevMap.end();
-       I != E; ++I) {
-    SymbolRef sym = I.getKey();
+  for (const ReallocPairsTy::value_type &Pair : prevMap) {
+    SymbolRef sym = Pair.first;
     if (!currMap.lookup(sym))
       return sym;
   }
@@ -2908,21 +3167,19 @@ static bool isReferenceCountingPointerDestructor(const CXXDestructorDecl *DD) {
   return false;
 }
 
-PathDiagnosticPieceRef
-MallocChecker::MallocBugVisitor::VisitNode(const ExplodedNode *N,
-                                           BugReporterContext &BRC,
-                                           PathSensitiveBugReport &BR) {
-
+PathDiagnosticPieceRef MallocBugVisitor::VisitNode(const ExplodedNode *N,
+                                                   BugReporterContext &BRC,
+                                                   PathSensitiveBugReport &BR) {
   ProgramStateRef state = N->getState();
   ProgramStateRef statePrev = N->getFirstPred()->getState();
 
-  const RefState *RS = state->get<RegionState>(Sym);
+  const RefState *RSCurr = state->get<RegionState>(Sym);
   const RefState *RSPrev = statePrev->get<RegionState>(Sym);
 
   const Stmt *S = N->getStmtForDiagnostics();
   // When dealing with containers, we sometimes want to give a note
   // even if the statement is missing.
-  if (!S && (!RS || RS->getAllocationFamily() != AF_InnerBuffer))
+  if (!S && (!RSCurr || RSCurr->getAllocationFamily() != AF_InnerBuffer))
     return nullptr;
 
   const LocationContext *CurrentLC = N->getLocationContext();
@@ -2957,12 +3214,12 @@ MallocChecker::MallocBugVisitor::VisitNode(const ExplodedNode *N,
   llvm::raw_svector_ostream OS(Buf);
 
   if (Mode == Normal) {
-    if (isAllocated(RS, RSPrev, S)) {
+    if (isAllocated(RSCurr, RSPrev, S)) {
       Msg = "Memory is allocated";
       StackHint = std::make_unique<StackHintGeneratorForSymbol>(
           Sym, "Returned allocated memory");
-    } else if (isReleased(RS, RSPrev, S)) {
-      const auto Family = RS->getAllocationFamily();
+    } else if (isReleased(RSCurr, RSPrev, S)) {
+      const auto Family = RSCurr->getAllocationFamily();
       switch (Family) {
         case AF_Alloca:
         case AF_Malloc:
@@ -2986,7 +3243,7 @@ MallocChecker::MallocBugVisitor::VisitNode(const ExplodedNode *N,
                 Sym, "Returning; inner buffer was deallocated");
           } else {
             OS << "reallocated by call to '";
-            const Stmt *S = RS->getStmt();
+            const Stmt *S = RSCurr->getStmt();
             if (const auto *MemCallE = dyn_cast<CXXMemberCallExpr>(S)) {
               OS << MemCallE->getMethodDecl()->getNameAsString();
             } else if (const auto *OpCallE = dyn_cast<CXXOperatorCallExpr>(S)) {
@@ -3037,10 +3294,10 @@ MallocChecker::MallocBugVisitor::VisitNode(const ExplodedNode *N,
           }
         }
       }
-    } else if (isRelinquished(RS, RSPrev, S)) {
+    } else if (isRelinquished(RSCurr, RSPrev, S)) {
       Msg = "Memory ownership is transferred";
       StackHint = std::make_unique<StackHintGeneratorForSymbol>(Sym, "");
-    } else if (isReallocFailedCheck(RS, RSPrev, S)) {
+    } else if (hasReallocFailed(RSCurr, RSPrev, S)) {
       Mode = ReallocationFailed;
       Msg = "Reallocation failed";
       StackHint = std::make_unique<StackHintGeneratorForReallocationFailed>(
@@ -3080,7 +3337,7 @@ MallocChecker::MallocBugVisitor::VisitNode(const ExplodedNode *N,
   // Generate the extra diagnostic.
   PathDiagnosticLocation Pos;
   if (!S) {
-    assert(RS->getAllocationFamily() == AF_InnerBuffer);
+    assert(RSCurr->getAllocationFamily() == AF_InnerBuffer);
     auto PostImplCall = N->getLocation().getAs<PostImplicitCall>();
     if (!PostImplCall)
       return nullptr;
@@ -3145,8 +3402,8 @@ void ento::registerInnerPointerCheckerAux(CheckerManager &mgr) {
 
 void ento::registerDynamicMemoryModeling(CheckerManager &mgr) {
   auto *checker = mgr.registerChecker<MallocChecker>();
-  checker->IsOptimistic = mgr.getAnalyzerOptions().getCheckerBooleanOption(
-                                                         checker, "Optimistic");
+  checker->MemFunctionInfo.ShouldIncludeOwnershipAnnotatedFunctions =
+      mgr.getAnalyzerOptions().getCheckerBooleanOption(checker, "Optimistic");
 }
 
 bool ento::shouldRegisterDynamicMemoryModeling(const LangOptions &LO) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
index 0734272381b8a..6ffc89745365c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NonNullParamChecker.cpp
@@ -35,11 +35,11 @@ class NonNullParamChecker
 
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
 
-  std::unique_ptr<BugReport>
+  std::unique_ptr<PathSensitiveBugReport>
   genReportNullAttrNonNull(const ExplodedNode *ErrorN,
                            const Expr *ArgE,
                            unsigned IdxOfArg) const;
-  std::unique_ptr<BugReport>
+  std::unique_ptr<PathSensitiveBugReport>
   genReportReferenceToNullPointer(const ExplodedNode *ErrorN,
                                   const Expr *ArgE) const;
 };
@@ -179,7 +179,7 @@ void NonNullParamChecker::checkPreCall(const CallEvent &Call,
   C.addTransition(state);
 }
 
-std::unique_ptr<BugReport>
+std::unique_ptr<PathSensitiveBugReport>
 NonNullParamChecker::genReportNullAttrNonNull(const ExplodedNode *ErrorNode,
                                               const Expr *ArgE,
                                               unsigned IdxOfArg) const {
@@ -204,7 +204,8 @@ NonNullParamChecker::genReportNullAttrNonNull(const ExplodedNode *ErrorNode,
   return R;
 }
 
-std::unique_ptr<BugReport> NonNullParamChecker::genReportReferenceToNullPointer(
+std::unique_ptr<PathSensitiveBugReport>
+NonNullParamChecker::genReportReferenceToNullPointer(
     const ExplodedNode *ErrorNode, const Expr *ArgE) const {
   if (!BTNullRefArg)
     BTNullRefArg.reset(new BuiltinBug(this, "Dereference of null pointer"));
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index c6b7ef041ac4e..f1592ebff669d 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -1438,6 +1438,7 @@ FindLastStoreBRVisitor::VisitNode(const ExplodedNode *Succ,
 
   if (!StoreSite)
     return nullptr;
+
   Satisfied = true;
 
   // If we have an expression that provided the value, try to track where it
@@ -1802,7 +1803,7 @@ TrackControlDependencyCondBRVisitor::VisitNode(const ExplodedNode *N,
   if (ControlDeps.isControlDependent(OriginB, NB)) {
     // We don't really want to explain for range loops. Evidence suggests that
     // the only thing that leads to is the addition of calls to operator!=.
-    if (isa<CXXForRangeStmt>(NB->getTerminator()))
+    if (llvm::isa_and_nonnull<CXXForRangeStmt>(NB->getTerminatorStmt()))
       return nullptr;
 
     if (const Expr *Condition = NB->getLastCondition()) {
diff --git a/clang/lib/StaticAnalyzer/Core/Store.cpp b/clang/lib/StaticAnalyzer/Core/Store.cpp
index 3cf616161c669..b4ab6877726ce 100644
--- a/clang/lib/StaticAnalyzer/Core/Store.cpp
+++ b/clang/lib/StaticAnalyzer/Core/Store.cpp
@@ -52,7 +52,7 @@ StoreRef StoreManager::enterStackFrame(Store OldStore,
   Call.getInitialStackFrameContents(LCtx, InitialBindings);
 
   for (const auto &I : InitialBindings)
-    Store = Bind(Store.getStore(), I.first, I.second);
+    Store = Bind(Store.getStore(), I.first.castAs<Loc>(), I.second);
 
   return Store;
 }
diff --git a/clang/lib/Tooling/ArgumentsAdjusters.cpp b/clang/lib/Tooling/ArgumentsAdjusters.cpp
index 942b35df453e9..f56d08c47b9a4 100644
--- a/clang/lib/Tooling/ArgumentsAdjusters.cpp
+++ b/clang/lib/Tooling/ArgumentsAdjusters.cpp
@@ -57,6 +57,22 @@ ArgumentsAdjuster getClangStripOutputAdjuster() {
   };
 }
 
+ArgumentsAdjuster getClangStripSerializeDiagnosticAdjuster() {
+  return [](const CommandLineArguments &Args, StringRef /*unused*/) {
+    CommandLineArguments AdjustedArgs;
+    for (size_t i = 0, e = Args.size(); i < e; ++i) {
+      StringRef Arg = Args[i];
+      if (Arg == "--serialize-diagnostics") {
+        // Skip the diagnostic output argument.
+        ++i;
+        continue;
+      }
+      AdjustedArgs.push_back(Args[i]);
+    }
+    return AdjustedArgs;
+  };
+}
+
 ArgumentsAdjuster getClangStripDependencyFileAdjuster() {
   return [](const CommandLineArguments &Args, StringRef /*unused*/) {
     CommandLineArguments AdjustedArgs;
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 7a3d189adac28..35ecbd4a7fb4c 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -193,6 +193,9 @@ class MinimizedVFSFile final : public llvm::vfs::File {
 llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
 createFile(const CachedFileSystemEntry *Entry,
            ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings) {
+  if (Entry->isDirectory())
+    return llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>(
+        std::make_error_code(std::errc::is_a_directory));
   llvm::ErrorOr<StringRef> Contents = Entry->getContents();
   if (!Contents)
     return Contents.getError();
diff --git a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
index a7f79c33bc556..e746bbb7f87b2 100644
--- a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
+++ b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
@@ -199,6 +199,20 @@ int IncludeCategoryManager::getIncludePriority(StringRef IncludeName,
   return Ret;
 }
 
+int IncludeCategoryManager::getSortIncludePriority(StringRef IncludeName,
+                                                   bool CheckMainHeader) const {
+  int Ret = INT_MAX;
+  for (unsigned i = 0, e = CategoryRegexs.size(); i != e; ++i)
+    if (CategoryRegexs[i].match(IncludeName)) {
+      Ret = Style.IncludeCategories[i].SortPriority;
+      if (Ret == 0)
+        Ret = Style.IncludeCategories[i].Priority;
+      break;
+    }
+  if (CheckMainHeader && IsMainFile && Ret > 0 && isMainHeader(IncludeName))
+    Ret = 0;
+  return Ret;
+}
 bool IncludeCategoryManager::isMainHeader(StringRef IncludeName) const {
   if (!IncludeName.startswith("\""))
     return false;
diff --git a/clang/lib/Tooling/Inclusions/IncludeStyle.cpp b/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
index c53c82c836307..26dc0b87cf9d7 100644
--- a/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
+++ b/clang/lib/Tooling/Inclusions/IncludeStyle.cpp
@@ -17,6 +17,7 @@ void MappingTraits<IncludeStyle::IncludeCategory>::mapping(
     IO &IO, IncludeStyle::IncludeCategory &Category) {
   IO.mapOptional("Regex", Category.Regex);
   IO.mapOptional("Priority", Category.Priority);
+  IO.mapOptional("SortPriority", Category.SortPriority);
 }
 
 void ScalarEnumerationTraits<IncludeStyle::IncludeBlocksStyle>::enumeration(
diff --git a/clang/lib/Tooling/Refactoring/CMakeLists.txt b/clang/lib/Tooling/Refactoring/CMakeLists.txt
index d1f092f261c9f..e3961db2841ec 100644
--- a/clang/lib/Tooling/Refactoring/CMakeLists.txt
+++ b/clang/lib/Tooling/Refactoring/CMakeLists.txt
@@ -14,6 +14,7 @@ add_clang_library(clangToolingRefactoring
   Rename/USRFindingAction.cpp
   Rename/USRLocFinder.cpp
   SourceCode.cpp
+  SourceCodeBuilders.cpp
   Stencil.cpp
   Transformer.cpp
 
diff --git a/clang/lib/Tooling/Refactoring/RangeSelector.cpp b/clang/lib/Tooling/Refactoring/RangeSelector.cpp
index 768c02e2277b3..972c7e65540a9 100644
--- a/clang/lib/Tooling/Refactoring/RangeSelector.cpp
+++ b/clang/lib/Tooling/Refactoring/RangeSelector.cpp
@@ -219,6 +219,9 @@ RangeSelector tooling::name(std::string ID) {
 }
 
 namespace {
+// FIXME: make this available in the public API for users to easily create their
+// own selectors.
+
 // Creates a selector from a range-selection function \p Func, which selects a
 // range that is relative to a bound node id.  \c T is the node type expected by
 // \p Func.
@@ -286,6 +289,19 @@ RangeSelector tooling::initListElements(std::string ID) {
   return RelativeSelector<InitListExpr, getElementsRange>(std::move(ID));
 }
 
+namespace {
+// Returns the range of the else branch, including the `else` keyword.
+CharSourceRange getElseRange(const MatchResult &Result, const IfStmt &S) {
+  return maybeExtendRange(
+      CharSourceRange::getTokenRange(S.getElseLoc(), S.getEndLoc()),
+      tok::TokenKind::semi, *Result.Context);
+}
+} // namespace
+
+RangeSelector tooling::elseBranch(std::string ID) {
+  return RelativeSelector<IfStmt, getElseRange>(std::move(ID));
+}
+
 RangeSelector tooling::expansion(RangeSelector S) {
   return [S](const MatchResult &Result) -> Expected<CharSourceRange> {
     Expected<CharSourceRange> SRange = S(Result);
diff --git a/clang/lib/Tooling/Refactoring/SourceCodeBuilders.cpp b/clang/lib/Tooling/Refactoring/SourceCodeBuilders.cpp
new file mode 100644
index 0000000000000..2499c0f1eb398
--- /dev/null
+++ b/clang/lib/Tooling/Refactoring/SourceCodeBuilders.cpp
@@ -0,0 +1,160 @@
+//===--- SourceCodeBuilder.cpp ----------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/Refactoring/SourceCodeBuilders.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/Tooling/Refactoring/SourceCode.h"
+#include "llvm/ADT/Twine.h"
+#include <string>
+
+using namespace clang;
+using namespace tooling;
+
+const Expr *tooling::reallyIgnoreImplicit(const Expr &E) {
+  const Expr *Expr = E.IgnoreImplicit();
+  if (const auto *CE = dyn_cast<CXXConstructExpr>(Expr)) {
+    if (CE->getNumArgs() > 0 &&
+        CE->getArg(0)->getSourceRange() == Expr->getSourceRange())
+      return CE->getArg(0)->IgnoreImplicit();
+  }
+  return Expr;
+}
+
+bool tooling::mayEverNeedParens(const Expr &E) {
+  const Expr *Expr = reallyIgnoreImplicit(E);
+  // We always want parens around unary, binary, and ternary operators, because
+  // they are lower precedence.
+  if (isa<UnaryOperator>(Expr) || isa<BinaryOperator>(Expr) ||
+      isa<AbstractConditionalOperator>(Expr))
+    return true;
+
+  // We need parens around calls to all overloaded operators except: function
+  // calls, subscripts, and expressions that are already part of an (implicit)
+  // call to operator->. These latter are all in the same precedence level as
+  // dot/arrow and that level is left associative, so they don't need parens
+  // when appearing on the left.
+  if (const auto *Op = dyn_cast<CXXOperatorCallExpr>(Expr))
+    return Op->getOperator() != OO_Call && Op->getOperator() != OO_Subscript &&
+           Op->getOperator() != OO_Arrow;
+
+  return false;
+}
+
+bool tooling::needParensAfterUnaryOperator(const Expr &E) {
+  const Expr *Expr = reallyIgnoreImplicit(E);
+  if (isa<BinaryOperator>(Expr) || isa<AbstractConditionalOperator>(Expr))
+    return true;
+
+  if (const auto *Op = dyn_cast<CXXOperatorCallExpr>(Expr))
+    return Op->getNumArgs() == 2 && Op->getOperator() != OO_PlusPlus &&
+           Op->getOperator() != OO_MinusMinus && Op->getOperator() != OO_Call &&
+           Op->getOperator() != OO_Subscript;
+
+  return false;
+}
+
+llvm::Optional<std::string> tooling::buildParens(const Expr &E,
+                                                 const ASTContext &Context) {
+  StringRef Text = getText(E, Context);
+  if (Text.empty())
+    return llvm::None;
+  if (mayEverNeedParens(E))
+    return ("(" + Text + ")").str();
+  return Text.str();
+}
+
+llvm::Optional<std::string>
+tooling::buildDereference(const Expr &E, const ASTContext &Context) {
+  if (const auto *Op = dyn_cast<UnaryOperator>(&E))
+    if (Op->getOpcode() == UO_AddrOf) {
+      // Strip leading '&'.
+      StringRef Text =
+          getText(*Op->getSubExpr()->IgnoreParenImpCasts(), Context);
+      if (Text.empty())
+        return llvm::None;
+      return Text.str();
+    }
+
+  StringRef Text = getText(E, Context);
+  if (Text.empty())
+    return llvm::None;
+  // Add leading '*'.
+  if (needParensAfterUnaryOperator(E))
+    return ("*(" + Text + ")").str();
+  return ("*" + Text).str();
+}
+
+llvm::Optional<std::string> tooling::buildAddressOf(const Expr &E,
+                                                    const ASTContext &Context) {
+  if (const auto *Op = dyn_cast<UnaryOperator>(&E))
+    if (Op->getOpcode() == UO_Deref) {
+      // Strip leading '*'.
+      StringRef Text =
+          getText(*Op->getSubExpr()->IgnoreParenImpCasts(), Context);
+      if (Text.empty())
+        return llvm::None;
+      return Text.str();
+    }
+  // Add leading '&'.
+  StringRef Text = getText(E, Context);
+  if (Text.empty())
+    return llvm::None;
+  if (needParensAfterUnaryOperator(E)) {
+    return ("&(" + Text + ")").str();
+  }
+  return ("&" + Text).str();
+}
+
+llvm::Optional<std::string> tooling::buildDot(const Expr &E,
+                                              const ASTContext &Context) {
+  if (const auto *Op = llvm::dyn_cast<UnaryOperator>(&E))
+    if (Op->getOpcode() == UO_Deref) {
+      // Strip leading '*', add following '->'.
+      const Expr *SubExpr = Op->getSubExpr()->IgnoreParenImpCasts();
+      StringRef DerefText = getText(*SubExpr, Context);
+      if (DerefText.empty())
+        return llvm::None;
+      if (needParensBeforeDotOrArrow(*SubExpr))
+        return ("(" + DerefText + ")->").str();
+      return (DerefText + "->").str();
+    }
+
+  // Add following '.'.
+  StringRef Text = getText(E, Context);
+  if (Text.empty())
+    return llvm::None;
+  if (needParensBeforeDotOrArrow(E)) {
+    return ("(" + Text + ").").str();
+  }
+  return (Text + ".").str();
+}
+
+llvm::Optional<std::string> tooling::buildArrow(const Expr &E,
+                                                const ASTContext &Context) {
+  if (const auto *Op = llvm::dyn_cast<UnaryOperator>(&E))
+    if (Op->getOpcode() == UO_AddrOf) {
+      // Strip leading '&', add following '.'.
+      const Expr *SubExpr = Op->getSubExpr()->IgnoreParenImpCasts();
+      StringRef DerefText = getText(*SubExpr, Context);
+      if (DerefText.empty())
+        return llvm::None;
+      if (needParensBeforeDotOrArrow(*SubExpr))
+        return ("(" + DerefText + ").").str();
+      return (DerefText + ".").str();
+    }
+
+  // Add following '->'.
+  StringRef Text = getText(E, Context);
+  if (Text.empty())
+    return llvm::None;
+  if (needParensBeforeDotOrArrow(E))
+    return ("(" + Text + ")->").str();
+  return (Text + "->").str();
+}
diff --git a/clang/lib/Tooling/Refactoring/Stencil.cpp b/clang/lib/Tooling/Refactoring/Stencil.cpp
index 09eca21c3cef1..a01b81023389f 100644
--- a/clang/lib/Tooling/Refactoring/Stencil.cpp
+++ b/clang/lib/Tooling/Refactoring/Stencil.cpp
@@ -14,6 +14,7 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Tooling/Refactoring/SourceCode.h"
+#include "clang/Tooling/Refactoring/SourceCodeBuilders.h"
 #include "llvm/Support/Errc.h"
 #include <atomic>
 #include <memory>
@@ -23,7 +24,10 @@ using namespace clang;
 using namespace tooling;
 
 using ast_matchers::MatchFinder;
+using llvm::errc;
 using llvm::Error;
+using llvm::Expected;
+using llvm::StringError;
 
 // A down_cast function to safely down cast a StencilPartInterface to a subclass
 // D. Returns nullptr if P is not an instance of D.
@@ -51,28 +55,58 @@ struct RawTextData {
 };
 
 // A debugging operation to dump the AST for a particular (bound) AST node.
-struct DebugPrintNodeOpData {
-  explicit DebugPrintNodeOpData(std::string S) : Id(std::move(S)) {}
+struct DebugPrintNodeData {
+  explicit DebugPrintNodeData(std::string S) : Id(std::move(S)) {}
   std::string Id;
 };
 
 // The fragment of code corresponding to the selected range.
-struct SelectorOpData {
-  explicit SelectorOpData(RangeSelector S) : Selector(std::move(S)) {}
+struct SelectorData {
+  explicit SelectorData(RangeSelector S) : Selector(std::move(S)) {}
   RangeSelector Selector;
 };
-} // namespace
+
+// A stencil operation to build a member access `e.m` or `e->m`, as appropriate.
+struct AccessData {
+  AccessData(StringRef BaseId, StencilPart Member)
+      : BaseId(BaseId), Member(std::move(Member)) {}
+  std::string BaseId;
+  StencilPart Member;
+};
+
+struct IfBoundData {
+  IfBoundData(StringRef Id, StencilPart TruePart, StencilPart FalsePart)
+      : Id(Id), TruePart(std::move(TruePart)), FalsePart(std::move(FalsePart)) {
+  }
+  std::string Id;
+  StencilPart TruePart;
+  StencilPart FalsePart;
+};
 
 bool isEqualData(const RawTextData &A, const RawTextData &B) {
   return A.Text == B.Text;
 }
 
-bool isEqualData(const DebugPrintNodeOpData &A, const DebugPrintNodeOpData &B) {
+bool isEqualData(const DebugPrintNodeData &A, const DebugPrintNodeData &B) {
   return A.Id == B.Id;
 }
 
 // Equality is not (yet) defined for \c RangeSelector.
-bool isEqualData(const SelectorOpData &, const SelectorOpData &) { return false; }
+bool isEqualData(const SelectorData &, const SelectorData &) { return false; }
+
+bool isEqualData(const AccessData &A, const AccessData &B) {
+  return A.BaseId == B.BaseId && A.Member == B.Member;
+}
+
+bool isEqualData(const IfBoundData &A, const IfBoundData &B) {
+  return A.Id == B.Id && A.TruePart == B.TruePart && A.FalsePart == B.FalsePart;
+}
+
+// Equality is not defined over MatchConsumers, which are opaque.
+bool isEqualData(const MatchConsumer<std::string> &A,
+                 const MatchConsumer<std::string> &B) {
+  return false;
+}
 
 // The `evalData()` overloads evaluate the given stencil data to a string, given
 // the match result, and append it to `Result`. We define an overload for each
@@ -84,7 +118,7 @@ Error evalData(const RawTextData &Data, const MatchFinder::MatchResult &,
   return Error::success();
 }
 
-Error evalData(const DebugPrintNodeOpData &Data,
+Error evalData(const DebugPrintNodeData &Data,
                const MatchFinder::MatchResult &Match, std::string *Result) {
   std::string Output;
   llvm::raw_string_ostream Os(Output);
@@ -96,7 +130,7 @@ Error evalData(const DebugPrintNodeOpData &Data,
   return Error::success();
 }
 
-Error evalData(const SelectorOpData &Data, const MatchFinder::MatchResult &Match,
+Error evalData(const SelectorData &Data, const MatchFinder::MatchResult &Match,
                std::string *Result) {
   auto Range = Data.Selector(Match);
   if (!Range)
@@ -105,6 +139,41 @@ Error evalData(const SelectorOpData &Data, const MatchFinder::MatchResult &Match
   return Error::success();
 }
 
+Error evalData(const AccessData &Data, const MatchFinder::MatchResult &Match,
+               std::string *Result) {
+  const auto *E = Match.Nodes.getNodeAs<Expr>(Data.BaseId);
+  if (E == nullptr)
+    return llvm::make_error<StringError>(errc::invalid_argument,
+                                         "Id not bound: " + Data.BaseId);
+  if (!E->isImplicitCXXThis()) {
+    if (llvm::Optional<std::string> S = E->getType()->isAnyPointerType()
+                                            ? buildArrow(*E, *Match.Context)
+                                            : buildDot(*E, *Match.Context))
+      *Result += *S;
+    else
+      return llvm::make_error<StringError>(
+          errc::invalid_argument,
+          "Could not construct object text from ID: " + Data.BaseId);
+  }
+  return Data.Member.eval(Match, Result);
+}
+
+Error evalData(const IfBoundData &Data, const MatchFinder::MatchResult &Match,
+               std::string *Result) {
+  auto &M = Match.Nodes.getMap();
+  return (M.find(Data.Id) != M.end() ? Data.TruePart : Data.FalsePart)
+      .eval(Match, Result);
+}
+
+Error evalData(const MatchConsumer<std::string> &Fn,
+               const MatchFinder::MatchResult &Match, std::string *Result) {
+  Expected<std::string> Value = Fn(Match);
+  if (!Value)
+    return Value.takeError();
+  *Result += *Value;
+  return Error::success();
+}
+
 template <typename T>
 class StencilPartImpl : public StencilPartInterface {
   T Data;
@@ -133,11 +202,6 @@ class StencilPartImpl : public StencilPartInterface {
     return false;
   }
 };
-
-namespace {
-using RawText = StencilPartImpl<RawTextData>;
-using DebugPrintNodeOp = StencilPartImpl<DebugPrintNodeOpData>;
-using SelectorOp = StencilPartImpl<SelectorOpData>;
 } // namespace
 
 StencilPart Stencil::wrap(StringRef Text) {
@@ -163,13 +227,31 @@ Stencil::eval(const MatchFinder::MatchResult &Match) const {
 }
 
 StencilPart stencil::text(StringRef Text) {
-  return StencilPart(std::make_shared<RawText>(Text));
+  return StencilPart(std::make_shared<StencilPartImpl<RawTextData>>(Text));
 }
 
 StencilPart stencil::selection(RangeSelector Selector) {
-  return StencilPart(std::make_shared<SelectorOp>(std::move(Selector)));
+  return StencilPart(
+      std::make_shared<StencilPartImpl<SelectorData>>(std::move(Selector)));
 }
 
 StencilPart stencil::dPrint(StringRef Id) {
-  return StencilPart(std::make_shared<DebugPrintNodeOp>(Id));
+  return StencilPart(std::make_shared<StencilPartImpl<DebugPrintNodeData>>(Id));
+}
+
+StencilPart stencil::access(StringRef BaseId, StencilPart Member) {
+  return StencilPart(
+      std::make_shared<StencilPartImpl<AccessData>>(BaseId, std::move(Member)));
+}
+
+StencilPart stencil::ifBound(StringRef Id, StencilPart TruePart,
+                             StencilPart FalsePart) {
+  return StencilPart(std::make_shared<StencilPartImpl<IfBoundData>>(
+      Id, std::move(TruePart), std::move(FalsePart)));
+}
+
+StencilPart stencil::run(MatchConsumer<std::string> Fn) {
+  return StencilPart(
+      std::make_shared<StencilPartImpl<MatchConsumer<std::string>>>(
+          std::move(Fn)));
 }
diff --git a/clang/lib/Tooling/Refactoring/Transformer.cpp b/clang/lib/Tooling/Refactoring/Transformer.cpp
index c27739c596267..905d5944449c9 100644
--- a/clang/lib/Tooling/Refactoring/Transformer.cpp
+++ b/clang/lib/Tooling/Refactoring/Transformer.cpp
@@ -150,6 +150,21 @@ DynTypedMatcher tooling::detail::buildMatcher(const RewriteRule &Rule) {
   return Ms[0];
 }
 
+SourceLocation tooling::detail::getRuleMatchLoc(const MatchResult &Result) {
+  auto &NodesMap = Result.Nodes.getMap();
+  auto Root = NodesMap.find(RewriteRule::RootID);
+  assert(Root != NodesMap.end() && "Transformation failed: missing root node.");
+  llvm::Optional<CharSourceRange> RootRange = getRangeForEdit(
+      CharSourceRange::getTokenRange(Root->second.getSourceRange()),
+      *Result.Context);
+  if (RootRange)
+    return RootRange->getBegin();
+  // The match doesn't have a coherent range, so fall back to the expansion
+  // location as the "beginning" of the match.
+  return Result.SourceManager->getExpansionLoc(
+      Root->second.getSourceRange().getBegin());
+}
+
 // Finds the case that was "selected" -- that is, whose matcher triggered the
 // `MatchResult`.
 const RewriteRule::Case &
@@ -178,14 +193,6 @@ void Transformer::run(const MatchResult &Result) {
   if (Result.Context->getDiagnostics().hasErrorOccurred())
     return;
 
-  // Verify the existence and validity of the AST node that roots this rule.
-  auto &NodesMap = Result.Nodes.getMap();
-  auto Root = NodesMap.find(RewriteRule::RootID);
-  assert(Root != NodesMap.end() && "Transformation failed: missing root node.");
-  SourceLocation RootLoc = Result.SourceManager->getExpansionLoc(
-      Root->second.getSourceRange().getBegin());
-  assert(RootLoc.isValid() && "Invalid location for Root node of match.");
-
   RewriteRule::Case Case = tooling::detail::findSelectedCase(Result, Rule);
   auto Transformations = tooling::detail::translateEdits(Result, Case.Edits);
   if (!Transformations) {
@@ -195,14 +202,16 @@ void Transformer::run(const MatchResult &Result) {
 
   if (Transformations->empty()) {
     // No rewrite applied (but no error encountered either).
-    RootLoc.print(llvm::errs() << "note: skipping match at loc ",
-                  *Result.SourceManager);
+    detail::getRuleMatchLoc(Result).print(
+        llvm::errs() << "note: skipping match at loc ", *Result.SourceManager);
     llvm::errs() << "\n";
     return;
   }
 
-  // Record the results in the AtomicChange.
-  AtomicChange AC(*Result.SourceManager, RootLoc);
+  // Record the results in the AtomicChange, anchored at the location of the
+  // first change.
+  AtomicChange AC(*Result.SourceManager,
+                  (*Transformations)[0].Range.getBegin());
   for (const auto &T : *Transformations) {
     if (auto Err = AC.replace(*Result.SourceManager, T.Range, T.Replacement)) {
       Consumer(std::move(Err));
diff --git a/clang/test/AST/ast-dump-color.cpp b/clang/test/AST/ast-dump-color.cpp
index f4bfdaa22942c..6651ef2652e02 100644
--- a/clang/test/AST/ast-dump-color.cpp
+++ b/clang/test/AST/ast-dump-color.cpp
@@ -49,13 +49,13 @@ struct Invalid {
 //CHECK: {{^}}[[Blue]]| |   |-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:10:11[[RESET]]> [[Green]]'int'[[RESET]][[Cyan:.\[0;36m]][[RESET]][[Cyan]][[RESET]][[CYAN]] 1[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |   `-[[RESET]][[MAGENTA]]CompoundStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:14[[RESET]], [[Yellow]]line:15:3[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |-[[RESET]][[MAGENTA]]CaseStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:3[[RESET]], [[Yellow]]line:12:27[[RESET]]>{{$}}
-//CHECK: {{^}}[[Blue]]| |     | |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 1[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |     | |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:11:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] Int: 1[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |     | | `-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 1[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |     | `-[[RESET]][[MAGENTA]]AttributedStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:12:5[[RESET]], [[Yellow]]col:27[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |   |-[[RESET]][[BLUE]]FallThroughAttr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:7[[RESET]], [[Yellow]]col:14[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     |   `-[[RESET]][[MAGENTA]]NullStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:27[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| |     `-[[RESET]][[MAGENTA]]CaseStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:3[[RESET]], [[Yellow]]line:14:5[[RESET]]>{{$}}
-//CHECK: {{^}}[[Blue]]| |       |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 2[[RESET]]{{$}}
+//CHECK: {{^}}[[Blue]]| |       |-[[RESET]][[MAGENTA]]ConstantExpr[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:13:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] Int: 2[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |       | `-[[RESET]][[MAGENTA]]IntegerLiteral[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]col:8[[RESET]]> [[Green]]'int'[[RESET]][[Cyan]][[RESET]][[Cyan]][[RESET]][[CYAN]] 2[[RESET]]{{$}}
 //CHECK: {{^}}[[Blue]]| |       `-[[RESET]][[MAGENTA]]NullStmt[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:14:5[[RESET]]>{{$}}
 //CHECK: {{^}}[[Blue]]| `-[[RESET]][[Blue]]FullComment[[RESET]][[Yellow]] 0x{{[0-9a-fA-F]*}}[[RESET]] <[[Yellow]]line:8:4[[RESET]], [[Yellow]]col:11[[RESET]]>{{$}}
diff --git a/clang/test/AST/ast-dump-decl.cpp b/clang/test/AST/ast-dump-decl.cpp
index 0d8fadc01eda1..529733475a253 100644
--- a/clang/test/AST/ast-dump-decl.cpp
+++ b/clang/test/AST/ast-dump-decl.cpp
@@ -413,13 +413,13 @@ namespace testClassTemplateDecl {
 
 // CHECK:       ClassTemplateDecl 0x{{.+}} <{{.+}}:275:3, col:68> col:68 TestTemplateTemplateDefaultType
 // CHECK-NEXT:  |-TemplateTemplateParmDecl 0x{{.+}} <col:12, col:42> col:37 depth 0 index 0 TT
-// CHECK-NEXT:  | |-TemplateTypeParmDecl 0x{{.+}} <col:21> col:21 typename depth 1 index 0
+// CHECK-NEXT:  | |-TemplateTypeParmDecl 0x{{.+}} <col:21> col:29 typename depth 1 index 0
 // CHECK-NEXT:  | `-TemplateArgument <col:42> template TestClassTemplate
 // CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} <col:61, col:68> col:68 struct TestTemplateTemplateDefaultType
 
 // CHECK:       ClassTemplateDecl 0x{{.+}} prev 0x{{.+}} <{{.+}}:276:3, col:82> col:48 TestTemplateTemplateDefaultType
 // CHECK-NEXT:  |-TemplateTemplateParmDecl 0x{{.+}} <col:12, col:37> col:37 depth 0 index 0 TT
-// CHECK-NEXT:  | |-TemplateTypeParmDecl 0x{{.+}} <col:21> col:21 typename depth 1 index 0
+// CHECK-NEXT:  | |-TemplateTypeParmDecl 0x{{.+}} <col:21> col:29 typename depth 1 index 0
 // CHECK-NEXT:  | `-TemplateArgument <line:275:42> template TestClassTemplate
 // CHECK-NEXT:  |   `-inherited from TemplateTemplateParm 0x{{.+}} 'TT'
 // CHECK-NEXT:  `-CXXRecordDecl 0x{{.+}} prev 0x{{.+}} <line:276:41, col:82> col:48 struct TestTemplateTemplateDefaultType definition
diff --git a/clang/test/AST/ast-dump-record-definition-data-json.cpp b/clang/test/AST/ast-dump-record-definition-data-json.cpp
index 4ad938683e5e7..9de52868bdc16 100644
--- a/clang/test/AST/ast-dump-record-definition-data-json.cpp
+++ b/clang/test/AST/ast-dump-record-definition-data-json.cpp
@@ -395,14 +395,8 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:       "tokLen": 4
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
-// CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       },
-// CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       }
+// CHECK-NEXT:       "begin": {},
+// CHECK-NEXT:       "end": {}
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "isImplicit": true,
 // CHECK-NEXT:      "tagUsed": "class",
@@ -497,14 +491,8 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:       "tokLen": 4
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
-// CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       },
-// CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       }
+// CHECK-NEXT:       "begin": {},
+// CHECK-NEXT:       "end": {}
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "isImplicit": true,
 // CHECK-NEXT:      "tagUsed": "class",
@@ -563,14 +551,8 @@ struct DoesNotAllowConstDefaultInit {
 // CHECK-NEXT:       "tokLen": 4
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
-// CHECK-NEXT:       "begin": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       },
-// CHECK-NEXT:       "end": {
-// CHECK-NEXT:        "col": 29,
-// CHECK-NEXT:        "tokLen": 4
-// CHECK-NEXT:       }
+// CHECK-NEXT:       "begin": {},
+// CHECK-NEXT:       "end": {}
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "isImplicit": true,
 // CHECK-NEXT:      "tagUsed": "class",
diff --git a/clang/test/AST/ast-dump-template-decls-json.cpp b/clang/test/AST/ast-dump-template-decls-json.cpp
index 21574af2c4a22..fc9aeca5b2728 100644
--- a/clang/test/AST/ast-dump-template-decls-json.cpp
+++ b/clang/test/AST/ast-dump-template-decls-json.cpp
@@ -656,8 +656,8 @@ void V<Ty>::f() {}
 // CHECK-NEXT:        "id": "0x{{.*}}",
 // CHECK-NEXT:        "kind": "TemplateTypeParmDecl",
 // CHECK-NEXT:        "loc": {
-// CHECK-NEXT:         "col": 33,
-// CHECK-NEXT:         "tokLen": 8
+// CHECK-NEXT:         "col": 41,
+// CHECK-NEXT:         "tokLen": 1
 // CHECK-NEXT:        },
 // CHECK-NEXT:        "range": {
 // CHECK-NEXT:         "begin": {
@@ -1099,8 +1099,8 @@ void V<Ty>::f() {}
 // CHECK-NEXT:      "kind": "TemplateTypeParmDecl",
 // CHECK-NEXT:      "loc": {
 // CHECK-NEXT:       "line": 27,
-// CHECK-NEXT:       "col": 11,
-// CHECK-NEXT:       "tokLen": 8
+// CHECK-NEXT:       "col": 20,
+// CHECK-NEXT:       "tokLen": 1
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "range": {
 // CHECK-NEXT:       "begin": {
diff --git a/clang/test/AST/ast-dump-template-decls.cpp b/clang/test/AST/ast-dump-template-decls.cpp
index a1f355b4da0d8..2fa50d8f549ee 100644
--- a/clang/test/AST/ast-dump-template-decls.cpp
+++ b/clang/test/AST/ast-dump-template-decls.cpp
@@ -26,7 +26,7 @@ template <typename Ty, template<typename> typename Uy>
 // CHECK: FunctionTemplateDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+4]]:18> col:6 d
 // CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <line:[[@LINE-2]]:11, col:20> col:20 referenced typename depth 0 index 0 Ty
 // CHECK-NEXT: TemplateTemplateParmDecl 0x{{[^ ]*}} <col:24, col:52> col:52 depth 0 index 1 Uy
-// CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <col:33> col:33 typename depth 1 index 0
+// CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <col:33> col:41 typename depth 1 index 0
 void d(Ty, Uy<Ty>);
 
 template <class Ty>
@@ -47,7 +47,7 @@ void g(Ty);
 
 template <typename = void>
 // CHECK: FunctionTemplateDecl 0x{{[^ ]*}} <line:[[@LINE-1]]:1, line:[[@LINE+3]]:8> col:6 h
-// CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <line:[[@LINE-2]]:11, col:22> col:11 typename depth 0 index 0
+// CHECK-NEXT: TemplateTypeParmDecl 0x{{[^ ]*}} <line:[[@LINE-2]]:11, col:22> col:20 typename depth 0 index 0
 // CHECK-NEXT: TemplateArgument type 'void'
 void h();
 
diff --git a/clang/test/AST/atomic-expr.cpp b/clang/test/AST/atomic-expr.cpp
new file mode 100644
index 0000000000000..e7fb6ba833e5e
--- /dev/null
+++ b/clang/test/AST/atomic-expr.cpp
@@ -0,0 +1,60 @@
+// RUN: %clang_cc1 -ast-dump %s | FileCheck %s
+
+template<int N = 0>
+void pr43370() {
+  int arr[2];
+  __atomic_store_n(arr, 0, 5);
+}
+
+template<int N = 0>
+void foo() {
+  int arr[2];
+  (void)__atomic_compare_exchange_n(arr, arr, 1, 0, 3, 4);
+}
+
+void useage(){
+  pr43370();
+  foo();
+}
+
+// CHECK:FunctionTemplateDecl 0x{{[0-9a-f]+}} <{{[^,]+}}, line:7:1> line:4:6 pr43370
+// CHECK: AtomicExpr
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:20> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:28> 'int' 5
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:25> 'int' 0
+// CHECK:FunctionDecl 0x{{[0-9a-f]+}} <line:4:1, line:7:1> line:4:6 used pr43370
+// CHECK: AtomicExpr
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:20> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:28> 'int' 5
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:25> 'int' 0
+
+// CHECK:FunctionTemplateDecl 0x{{[0-9a-f]+}} <line:9:1, line:13:1> line:10:6 foo
+// CHECK: AtomicExpr
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:37> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:53> 'int' 3
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:42> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:56> 'int' 4
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:47> 'int' 1
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:50> 'int' 0
+// CHECK:FunctionDecl 0x{{[0-9a-f]+}} <line:10:1, line:13:1> line:10:6 used foo
+// CHECK: AtomicExpr
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:37> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:53> 'int' 3
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-SAME: <ArrayToPointerDecay>
+// CHECK-NEXT: DeclRefExpr 0x{{[0-9a-f]+}} <{{[^:]+}}:42> 'int [2]' lvalue Var 0x{{[0-9a-f]+}} 'arr' 'int [2]'
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:56> 'int' 4
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:47> 'int' 1
+// CHECK-NEXT: ImplicitCastExpr
+// CHECK-NEXT: IntegerLiteral 0x{{[0-9a-f]+}} <{{[^:]+}}:50> 'int' 0
diff --git a/clang/test/ASTMerge/class-template/test.cpp b/clang/test/ASTMerge/class-template/test.cpp
index 27761f6983a5e..671064786d76f 100644
--- a/clang/test/ASTMerge/class-template/test.cpp
+++ b/clang/test/ASTMerge/class-template/test.cpp
@@ -9,13 +9,13 @@ static_assert(sizeof(X0<int>().getValue(1)) == sizeof(int));
 // CHECK: class-template2.cpp:9:15: note: declared here with type 'long'
 
 // CHECK: class-template1.cpp:12:14: warning: template parameter has different kinds in different translation units
-// CHECK: class-template2.cpp:12:10: note: template parameter declared here
+// CHECK: class-template2.cpp:12:18: note: template parameter declared here
 
 // CHECK: class-template1.cpp:18:23: warning: non-type template parameter declared with incompatible types in different translation units ('long' vs. 'int')
 // CHECK: class-template2.cpp:18:23: note: declared here with type 'int'
 
-// CHECK: class-template1.cpp:21:10: warning: template parameter has different kinds in different translation units
-// CHECK: class-template2.cpp:21:10: note: template parameter declared here
+// CHECK: class-template1.cpp:21:18: warning: template parameter has different kinds in different translation units
+// CHECK: class-template2.cpp:21:31: note: template parameter declared here
 
 // CHECK: class-template2.cpp:27:20: warning: external variable 'x0r' declared with incompatible types in different translation units ('X0<double> *' vs. 'X0<float> *')
 // CHECK: class-template1.cpp:26:19: note: declared here with type 'X0<float> *'
diff --git a/clang/test/Analysis/array-struct-region.cpp b/clang/test/Analysis/array-struct-region.cpp
index cfb57d39242db..1b9fa3e8db55c 100644
--- a/clang/test/Analysis/array-struct-region.cpp
+++ b/clang/test/Analysis/array-struct-region.cpp
@@ -1,20 +1,26 @@
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -x c %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -x c++ -std=c++14 %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -x c++ -std=c++17 %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -DINLINE -x c %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -DINLINE -x c++ -std=c++14 %s
 // RUN: %clang_analyze_cc1 -analyzer-checker=core,alpha.core\
 // RUN:                    -analyzer-checker=debug.ExprInspection -verify\
+// RUN:                    -Wno-tautological-compare\
 // RUN:                    -DINLINE -x c++ -std=c++17 %s
 
 void clang_analyzer_eval(int);
diff --git a/clang/test/Analysis/cfg.cpp b/clang/test/Analysis/cfg.cpp
index 9b0203e99efe9..a7d707ee20d1a 100644
--- a/clang/test/Analysis/cfg.cpp
+++ b/clang/test/Analysis/cfg.cpp
@@ -547,6 +547,27 @@ int foo() {
 }
 } // namespace statement_expression_in_return
 
+// CHECK-LABEL: int overlap_compare(int x)
+// CHECK: [B2]
+// CHECK-NEXT:   1: 1
+// CHECK-NEXT:   2: return [B2.1];
+// CHECK-NEXT:   Preds (1): B3(Unreachable)
+// CHECK-NEXT:   Succs (1): B0
+// CHECK: [B3]
+// CHECK-NEXT:   1: x
+// CHECK-NEXT:   2: [B3.1] (ImplicitCastExpr, LValueToRValue, int)
+// CHECK-NEXT:   3: 5
+// CHECK-NEXT:   4: [B3.2] > [B3.3]
+// CHECK-NEXT:   T: if [B4.5] && [B3.4]
+// CHECK-NEXT:   Preds (1): B4
+// CHECK-NEXT:   Succs (2): B2(Unreachable) B1
+int overlap_compare(int x) {
+  if (x == -1 && x > 5)
+    return 1;
+
+  return 2;
+}
+
 // CHECK-LABEL: template<> int *PR18472<int>()
 // CHECK: [B2 (ENTRY)]
 // CHECK-NEXT:   Succs (1): B1
diff --git a/clang/test/Analysis/debug-CallGraph.cpp b/clang/test/Analysis/debug-CallGraph.cpp
index 1d6844fad94fc..0f5a83b268a01 100644
--- a/clang/test/Analysis/debug-CallGraph.cpp
+++ b/clang/test/Analysis/debug-CallGraph.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCallGraph %s -fblocks 2>&1 | FileCheck %s
+// RUN: %clang_analyze_cc1 -analyzer-checker=debug.DumpCallGraph %s -fblocks -std=c++14 2>&1 | FileCheck %s
 
 int get5() {
   return 5;
@@ -68,8 +68,25 @@ void templUser() {
 }
 }
 
+namespace Lambdas {
+  void Callee(){}
+
+  void f1() {
+    [](int i) {
+      Callee();
+    }(1);
+    [](auto i) {
+      Callee();
+    }(1);
+  }
+}
+
 // CHECK:--- Call graph Dump ---
-// CHECK-NEXT: {{Function: < root > calls: get5 add test_add mmm foo aaa < > bbb ddd ccc eee fff do_nothing test_single_call SomeNS::templ SomeNS::templ SomeNS::templUser $}}
+// CHECK-NEXT: {{Function: < root > calls: get5 add test_add mmm foo aaa < > bbb ddd ccc eee fff do_nothing test_single_call SomeNS::templ SomeNS::templ SomeNS::templUser Lambdas::Callee Lambdas::f1 Lambdas::f1\(\)::\(anonymous class\)::operator\(\) Lambdas::f1\(\)::\(anonymous class\)::operator\(\) $}}
+// CHECK-NEXT: {{Function: Lambdas::f1 calls: Lambdas::f1\(\)::\(anonymous class\)::operator\(\) Lambdas::f1\(\)::\(anonymous class\)::operator\(\) $}}
+// CHECK-NEXT: {{Function: Lambdas::f1\(\)::\(anonymous class\)::operator\(\) calls: Lambdas::Callee $}}
+// CHECK-NEXT: {{Function: Lambdas::f1\(\)::\(anonymous class\)::operator\(\) calls: Lambdas::Callee $}}
+// CHECK-NEXT: {{Function: Lambdas::Callee calls: $}}
 // CHECK-NEXT: {{Function: SomeNS::templUser calls: SomeNS::templ SomeNS::templ $}}
 // CHECK-NEXT: {{Function: SomeNS::templ calls: eee $}}
 // CHECK-NEXT: {{Function: SomeNS::templ calls: ccc $}}
diff --git a/clang/test/Analysis/loop-widening.cpp b/clang/test/Analysis/loop-widening.cpp
new file mode 100644
index 0000000000000..fbcb72dee160a
--- /dev/null
+++ b/clang/test/Analysis/loop-widening.cpp
@@ -0,0 +1,27 @@
+// RUN: %clang_analyze_cc1 -verify %s \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-config widen-loops=true \
+// RUN:   -analyzer-config track-conditions=false \
+// RUN:   -analyzer-max-loop 2 -analyzer-output=text
+
+namespace pr43102 {
+class A {
+public:
+  void m_fn1();
+};
+bool g;
+void fn1() {
+  A a;
+  A *b = &a;
+
+  for (;;) { // expected-note{{Loop condition is true.  Entering loop body}}
+             // expected-note@-1{{Loop condition is true.  Entering loop body}}
+             // expected-note@-2{{Value assigned to 'b'}}
+             // no crash during bug report construction
+
+    g = !b;     // expected-note{{Assuming 'b' is null}}
+    b->m_fn1(); // expected-warning{{Called C++ object pointer is null}}
+                // expected-note@-1{{Called C++ object pointer is null}}
+  }
+}
+} // end of namespace pr43102
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp
new file mode 100644
index 0000000000000..3d45949430f99
--- /dev/null
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/dtor.cpp
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 -std=c++2a -verify %s
+
+// p3: if the function is a constructor or destructor, its class shall not have
+// any virtual base classes;
+namespace vbase {
+  struct A {};
+  struct B : virtual A { // expected-note {{virtual}}
+    constexpr ~B() {} // expected-error {{constexpr member function not allowed in struct with virtual base class}}
+  };
+}
+
+// p3: its function-body shall not enclose
+//  -- a goto statement
+//  -- an identifier label
+//  -- a variable of non-literal type or of static or thread storage duration
+namespace contents {
+  struct A {
+    constexpr ~A() {
+      goto x; // expected-error {{statement not allowed in constexpr function}}
+      x: ;
+    }
+  };
+  struct B {
+    constexpr ~B() {
+      x: ; // expected-error {{statement not allowed in constexpr function}}
+    }
+  };
+  struct Nonlit { Nonlit(); }; // expected-note {{not literal}}
+  struct C {
+    constexpr ~C() {
+      Nonlit nl; // expected-error {{non-literal}}
+    }
+  };
+  struct D {
+    constexpr ~D() {
+      static int a; // expected-error {{static variable}}
+    }
+  };
+  struct E {
+    constexpr ~E() {
+      thread_local int e; // expected-error {{thread_local variable}}
+    }
+  };
+  struct F {
+    constexpr ~F() {
+      extern int f;
+    }
+  };
+}
+
+// p5: for every subobject of class type or (possibly multi-dimensional) array
+// thereof, that class type shall have a constexpr destructor
+namespace subobject {
+  struct A {
+    ~A();
+  };
+  struct B : A { // expected-note {{here}}
+    constexpr ~B() {} // expected-error {{destructor cannot be declared constexpr because base class 'subobject::A' does not have a constexpr destructor}}
+  };
+  struct C {
+    A a; // expected-note {{here}}
+    constexpr ~C() {} // expected-error {{destructor cannot be declared constexpr because data member 'a' does not have a constexpr destructor}}
+  };
+  struct D : A {
+    A a;
+    constexpr ~D() = delete;
+  };
+}
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
index f472e9213fbed..59e3dcf6ed657 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p1.cpp
@@ -1,6 +1,7 @@
 // RUN: %clang_cc1 -triple %itanium_abi_triple -fsyntax-only -verify -std=c++11 %s
 // RUN: %clang_cc1 -triple %itanium_abi_triple -fsyntax-only -verify -std=c++14 %s
-// RUN: %clang_cc1 -triple %itanium_abi_triple -fsyntax-only -verify -std=c++1z %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -fsyntax-only -verify -std=c++17 %s
+// RUN: %clang_cc1 -triple %itanium_abi_triple -fsyntax-only -verify -std=c++2a %s
 
 // MSVC always adopted the C++17 rule that implies that constexpr variables are
 // implicitly inline, so do the test again.
@@ -75,7 +76,10 @@ template<typename T> T f6(T); // expected-note {{here}}
 template<typename T> constexpr T f6(T); // expected-error {{constexpr declaration of 'f6' follows non-constexpr declaration}}
 // destructor
 struct ConstexprDtor {
-  constexpr ~ConstexprDtor() = default; // expected-error {{destructor cannot be marked constexpr}}
+  constexpr ~ConstexprDtor() = default;
+#if __cplusplus <= 201703L
+  // expected-error@-2 {{destructor cannot be declared constexpr}}
+#endif
 };
 
 // template stuff
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
index f72984a6a1207..59c2ee7d936e3 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p3.cpp
@@ -57,7 +57,10 @@ struct T : SS, NonLiteral {
 #ifndef CXX1Y
   // expected-error@-2 {{constexpr function's return type 'void' is not a literal type}}
 #endif
-  constexpr ~T(); // expected-error {{destructor cannot be marked constexpr}}
+  constexpr ~T();
+#ifndef CXX2A
+  // expected-error@-2 {{destructor cannot be declared constexpr}}
+#endif
   typedef NonLiteral F() const;
   constexpr F NonLiteralReturn2; // ok until definition
 
@@ -183,7 +186,10 @@ constexpr int DisallowedStmtsCXX1Y_6() {
 }
 constexpr int DisallowedStmtsCXX1Y_7() {
   //  - a definition of a variable for which no initialization is performed
-  int n; // expected-error {{variables defined in a constexpr function must be initialized}}
+  int n;
+#ifndef CXX2A
+  // expected-error@-2 {{uninitialized variable in a constexpr function}}
+#endif
   return 0;
 }
 
@@ -315,7 +321,10 @@ namespace std_example {
     return value;
   }
   constexpr int uninit() {
-    int a; // expected-error {{must be initialized}}
+    int a;
+#ifndef CXX2A
+    // expected-error@-2 {{uninitialized}}
+#endif
     return a;
   }
   constexpr int prev(int x) {
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
index 17753604b8a79..39088042251f0 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p4.cpp
@@ -136,29 +136,33 @@ struct V {
 
 // - every non-static data member and base class sub-object shall be initialized
 struct W {
-  int n; // expected-note {{member not initialized by constructor}}
-  constexpr W() {} // expected-error {{constexpr constructor must initialize all members}}
+  int n;
+  constexpr W() {}
+#ifndef CXX2A
+  // expected-error@-2 {{constexpr constructor that does not initialize all members}}
+  // expected-note@-4 {{member not initialized by constructor}}
+#endif
 };
 struct AnonMembers {
-  int a; // expected-note {{member not initialized by constructor}}
-  union { // expected-note 2{{member not initialized by constructor}}
+  int a; // expected-note 0-1{{member not initialized by constructor}}
+  union { // expected-note 0-2{{member not initialized by constructor}}
     char b;
     struct {
       double c;
-      long d; // expected-note {{member not initialized by constructor}}
+      long d; // expected-note 0-1{{member not initialized by constructor}}
     };
     union {
       char e;
       void *f;
     };
   };
-  struct { // expected-note {{member not initialized by constructor}}
+  struct { // expected-note 0-1{{member not initialized by constructor}}
     long long g;
     struct {
-      int h; // expected-note {{member not initialized by constructor}}
-      double i; // expected-note {{member not initialized by constructor}}
+      int h; // expected-note 0-1{{member not initialized by constructor}}
+      double i; // expected-note 0-1{{member not initialized by constructor}}
     };
-    union { // expected-note 2{{member not initialized by constructor}}
+    union { // expected-note 0-2{{member not initialized by constructor}}
       char *j;
       AnonMembers *k;
     };
@@ -166,14 +170,26 @@ struct AnonMembers {
 
   constexpr AnonMembers(int(&)[1]) : a(), b(), g(), h(), i(), j() {} // ok
   // missing d, i, j/k union
-  constexpr AnonMembers(int(&)[2]) : a(), c(), g(), h() {} // expected-error {{constexpr constructor must initialize all members}}
+  constexpr AnonMembers(int(&)[2]) : a(), c(), g(), h() {}
+#ifndef CXX2A
+  // expected-error@-2 {{constexpr constructor that does not initialize all members}}
+#endif
   constexpr AnonMembers(int(&)[3]) : a(), e(), g(), h(), i(), k() {} // ok
   // missing h, j/k union
-  constexpr AnonMembers(int(&)[4]) : a(), c(), d(), g(), i() {} // expected-error {{constexpr constructor must initialize all members}}
+  constexpr AnonMembers(int(&)[4]) : a(), c(), d(), g(), i() {}
+#ifndef CXX2A
+  // expected-error@-2 {{constexpr constructor that does not initialize all members}}
+#endif
   // missing b/c/d/e/f union
-  constexpr AnonMembers(int(&)[5]) : a(), g(), h(), i(), k() {} // expected-error {{constexpr constructor must initialize all members}}
+  constexpr AnonMembers(int(&)[5]) : a(), g(), h(), i(), k() {}
+#ifndef CXX2A
+  // expected-error@-2 {{constexpr constructor that does not initialize all members}}
+#endif
   // missing a, b/c/d/e/f union, g/h/i/j/k struct
-  constexpr AnonMembers(int(&)[6]) {} // expected-error {{constexpr constructor must initialize all members}}
+  constexpr AnonMembers(int(&)[6]) {}
+#ifndef CXX2A
+  // expected-error@-2 {{constexpr constructor that does not initialize all members}}
+#endif
 };
 
 union Empty {
@@ -253,14 +269,20 @@ struct X {
   constexpr X(int c) : a(c) {} // ok, b initialized by 2 * c + 1
 };
 
-union XU1 { int a; constexpr XU1() = default; }; // expected-error{{not constexpr}}
+union XU1 { int a; constexpr XU1() = default; };
+#ifndef CXX2A
+// expected-error@-2{{not constexpr}}
+#endif
 union XU2 { int a = 1; constexpr XU2() = default; };
 
 struct XU3 {
   union {
     int a;
   };
-  constexpr XU3() = default; // expected-error{{not constexpr}}
+  constexpr XU3() = default;
+#ifndef CXX2A
+  // expected-error@-2{{not constexpr}}
+#endif
 };
 struct XU4 {
   union {
diff --git a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p9.cpp b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p9.cpp
index 0aaedcc07696b..8d51dbde71776 100644
--- a/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p9.cpp
+++ b/clang/test/CXX/dcl.dcl/dcl.spec/dcl.constexpr/p9.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2a %s
 
 // A constexpr specifier used in an object declaration declares the object as
 // const.
@@ -35,3 +36,19 @@ struct pixel {
 };
 constexpr pixel ur = { 1294, 1024 }; // ok
 constexpr pixel origin;              // expected-error {{default initialization of an object of const type 'const pixel' without a user-provided default constructor}}
+
+#if __cplusplus > 201702L
+// A constexpr variable shall have constant destruction.
+struct A {
+  bool ok;
+  constexpr A(bool ok) : ok(ok) {}
+  constexpr ~A() noexcept(false) {
+    void oops(); // expected-note 2{{declared here}}
+    if (!ok) oops(); // expected-note 2{{non-constexpr function}}
+  }
+};
+
+constexpr A const_dtor(true);
+constexpr A non_const_dtor(false); // expected-error {{must have constant destruction}} expected-note {{in call}}
+constexpr A arr_dtor[5] = {true, true, true, false, true}; // expected-error {{must have constant destruction}} expected-note {{in call to '&arr_dtor[3]->~A()'}}
+#endif
diff --git a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp
index 2b7886d383db8..7274ee9c542fc 100644
--- a/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp
+++ b/clang/test/CXX/dcl.decl/dcl.fct.def/dcl.fct.def.default/p2.cpp
@@ -8,7 +8,7 @@ struct S1 {
   constexpr S1(S1&&) = default;
   constexpr S1 &operator=(const S1&) const = default; // expected-error {{explicitly-defaulted copy assignment operator may not have}}
   constexpr S1 &operator=(S1&&) const = default; // expected-error {{explicitly-defaulted move assignment operator may not have}}
-  constexpr ~S1() = default; // expected-error {{destructor cannot be marked constexpr}}
+  constexpr ~S1() = default; // expected-error {{destructor cannot be declared constexpr}}
   int n;
 };
 struct NoCopyMove {
diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp
index eb086178fc06f..52129844c4188 100644
--- a/clang/test/CXX/drs/dr14xx.cpp
+++ b/clang/test/CXX/drs/dr14xx.cpp
@@ -1,7 +1,8 @@
 // RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
-// RUN: %clang_cc1 -std=c++1z %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
+// RUN: %clang_cc1 -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors
 
 #if __cplusplus < 201103L
 // expected-no-diagnostics
@@ -38,18 +39,24 @@ namespace dr1460 { // dr1460: 3.5
   }
 
   union A {};
-  union B { int n; }; // expected-note +{{here}}
+  union B { int n; }; // expected-note 0+{{here}}
   union C { int n = 0; };
   struct D { union {}; }; // expected-error {{does not declare anything}}
-  struct E { union { int n; }; }; // expected-note +{{here}}
+  struct E { union { int n; }; }; // expected-note 0+{{here}}
   struct F { union { int n = 0; }; };
 
   struct X {
     friend constexpr A::A() noexcept;
-    friend constexpr B::B() noexcept; // expected-error {{follows non-constexpr declaration}}
+    friend constexpr B::B() noexcept;
+#if __cplusplus <= 201703L
+    // expected-error@-2 {{follows non-constexpr declaration}}
+#endif
     friend constexpr C::C() noexcept;
     friend constexpr D::D() noexcept;
-    friend constexpr E::E() noexcept; // expected-error {{follows non-constexpr declaration}}
+    friend constexpr E::E() noexcept;
+#if __cplusplus <= 201703L
+    // expected-error@-2 {{follows non-constexpr declaration}}
+#endif
     friend constexpr F::F() noexcept;
   };
 
@@ -64,37 +71,61 @@ namespace dr1460 { // dr1460: 3.5
 
   namespace Defaulted {
     union A { constexpr A() = default; };
-    union B { int n; constexpr B() = default; }; // expected-error {{not constexpr}}
+    union B { int n; constexpr B() = default; };
+#if __cplusplus <= 201703L
+    // expected-error@-2 {{not constexpr}}
+#endif
     union C { int n = 0; constexpr C() = default; };
     struct D { union {}; constexpr D() = default; }; // expected-error {{does not declare anything}}
-    struct E { union { int n; }; constexpr E() = default; }; // expected-error {{not constexpr}}
+    struct E { union { int n; }; constexpr E() = default; };
+#if __cplusplus <= 201703L
+    // expected-error@-2 {{not constexpr}}
+#endif
     struct F { union { int n = 0; }; constexpr F() = default; };
 
-    struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; }; // expected-error {{not constexpr}}
+    struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; };
+#if __cplusplus <= 201703L
+    // expected-error@-2 {{not constexpr}}
+#endif
     struct H {
       union {
         int n = 0;
       };
-      union { // expected-note 2{{member not initialized}}
+      union { // expected-note 0-2{{member not initialized}}
         int m;
       };
-      constexpr H() {} // expected-error {{must initialize all members}}
+      constexpr H() {}
+#if __cplusplus <= 201703L
+      // expected-error@-2 {{initialize all members}}
+#endif
       constexpr H(bool) : m(1) {}
-      constexpr H(char) : n(1) {} // expected-error {{must initialize all members}}
+      constexpr H(char) : n(1) {}
+#if __cplusplus <= 201703L
+      // expected-error@-2 {{initialize all members}}
+#endif
       constexpr H(double) : m(1), n(1) {}
     };
   }
 
 #if __cplusplus > 201103L
   template<typename T> constexpr bool check() {
-    T t; // expected-note-re 2{{non-constexpr constructor '{{[BE]}}'}}
+    T t;
+#if __cplusplus <= 201703L
+    // expected-note-re@-2 2{{non-constexpr constructor '{{[BE]}}'}}
+#endif
     return true;
   }
   static_assert(check<A>(), "");
-  static_assert(check<B>(), ""); // expected-error {{constant}} expected-note {{in call}}
+  static_assert(check<B>(), "");
+#if __cplusplus <= 201703L
+  // expected-error@-2 {{constant}} expected-note@-2 {{in call}}
+#endif
   static_assert(check<C>(), "");
   static_assert(check<D>(), "");
-  static_assert(check<E>(), ""); // expected-error {{constant}} expected-note {{in call}}
+  static_assert(check<E>(), "");
+#if __cplusplus <= 201703L
+  // expected-error@-2 {{constant}} expected-note@-2 {{in call}}
+#endif
   static_assert(check<F>(), "");
 #endif
 
diff --git a/clang/test/CXX/drs/dr2xx.cpp b/clang/test/CXX/drs/dr2xx.cpp
index c7ffa77d30dec..89e13367a474c 100644
--- a/clang/test/CXX/drs/dr2xx.cpp
+++ b/clang/test/CXX/drs/dr2xx.cpp
@@ -751,9 +751,12 @@ namespace dr263 { // dr263: yes
 #if __cplusplus < 201103L
     friend X::X() throw();
     friend X::~X() throw();
-#else
+#elif __cplusplus <= 201703L
     friend constexpr X::X() noexcept;
     friend X::~X();
+#else
+    friend constexpr X::X() noexcept;
+    friend constexpr X::~X();
 #endif
     Y::Y(); // expected-error {{extra qualification}}
     Y::~Y(); // expected-error {{extra qualification}}
diff --git a/clang/test/CXX/expr/expr.const/p2-0x.cpp b/clang/test/CXX/expr/expr.const/p2-0x.cpp
index 091ef097255a0..e84753759c90b 100644
--- a/clang/test/CXX/expr/expr.const/p2-0x.cpp
+++ b/clang/test/CXX/expr/expr.const/p2-0x.cpp
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -fsyntax-only -std=c++11 -pedantic -verify -fcxx-exceptions %s -fconstexpr-depth 128 -triple i686-pc-linux-gnu
+// RUN: %clang_cc1 -fsyntax-only -std=c++11 -pedantic -verify=expected,cxx11 -fcxx-exceptions %s -fconstexpr-depth 128 -triple i686-pc-linux-gnu
+// RUN: %clang_cc1 -fsyntax-only -std=c++2a -pedantic -verify=expected,cxx20 -fcxx-exceptions %s -fconstexpr-depth 128 -triple i686-pc-linux-gnu
 
 // A conditional-expression is a core constant expression unless it involves one
 // of the following as a potentially evaluated subexpression [...]:
@@ -157,13 +158,13 @@ namespace UndefinedBehavior {
   constexpr int shl_unsigned_negative = unsigned(-3) << 1; // ok
   constexpr int shl_unsigned_into_sign = 1u << 31; // ok
   constexpr int shl_unsigned_overflow = 1024u << 31; // ok
-  constexpr int shl_signed_negative = (-3) << 1; // expected-error {{constant expression}} expected-note {{left shift of negative value -3}}
+  constexpr int shl_signed_negative = (-3) << 1; // cxx11-error {{constant expression}} cxx11-note {{left shift of negative value -3}}
   constexpr int shl_signed_ok = 1 << 30; // ok
   constexpr int shl_signed_into_sign = 1 << 31; // ok (DR1457)
   constexpr int shl_signed_into_sign_2 = 0x7fffffff << 1; // ok (DR1457)
-  constexpr int shl_signed_off_end = 2 << 31; // expected-error {{constant expression}} expected-note {{signed left shift discards bits}} expected-warning {{signed shift result (0x100000000) requires 34 bits to represent, but 'int' only has 32 bits}}
-  constexpr int shl_signed_off_end_2 = 0x7fffffff << 2; // expected-error {{constant expression}} expected-note {{signed left shift discards bits}} expected-warning {{signed shift result (0x1FFFFFFFC) requires 34 bits to represent, but 'int' only has 32 bits}}
-  constexpr int shl_signed_overflow = 1024 << 31; // expected-error {{constant expression}} expected-note {{signed left shift discards bits}} expected-warning {{requires 43 bits to represent}}
+  constexpr int shl_signed_off_end = 2 << 31; // cxx11-error {{constant expression}} cxx11-note {{signed left shift discards bits}} expected-warning {{signed shift result (0x100000000) requires 34 bits to represent, but 'int' only has 32 bits}}
+  constexpr int shl_signed_off_end_2 = 0x7fffffff << 2; // cxx11-error {{constant expression}} cxx11-note {{signed left shift discards bits}} expected-warning {{signed shift result (0x1FFFFFFFC) requires 34 bits to represent, but 'int' only has 32 bits}}
+  constexpr int shl_signed_overflow = 1024 << 31; // cxx11-error {{constant expression}} cxx11-note {{signed left shift discards bits}} expected-warning {{requires 43 bits to represent}}
   constexpr int shl_signed_ok2 = 1024 << 20; // ok
 
   constexpr int shr_m1 = 0 >> -1; // expected-error {{constant expression}} expected-note {{negative shift count -1}}
@@ -291,7 +292,7 @@ namespace UndefinedBehavior {
 
 // - a lambda-expression (5.1.2);
 struct Lambda {
-  int n : []{ return 1; }(); // expected-error {{constant expression}} expected-error {{integral constant expression}} expected-note {{non-literal type}}
+  int n : []{ return 1; }(); // cxx11-error {{constant expression}} cxx11-error {{integral constant expression}} cxx11-note {{non-literal type}}
 };
 
 // - an lvalue-to-rvalue conversion (4.1) unless it is applied to
@@ -360,7 +361,7 @@ namespace LValueToRValueUnion {
   extern const U pu;
   constexpr const int *pua = &pu.a;
   constexpr const int *pub = &pu.b;
-  constexpr U pu = { .b = 1 }; // expected-warning {{C++20 extension}}
+  constexpr U pu = { .b = 1 }; // cxx11-warning {{C++20 extension}}
   constexpr const int a2 = *pua; // expected-error {{constant expression}} expected-note {{read of member 'a' of union with active member 'b'}}
   constexpr const int b2 = *pub; // ok
 }
@@ -402,7 +403,7 @@ namespace DynamicCast {
   struct S { int n; };
   constexpr S s { 16 };
   struct T {
-    int n : dynamic_cast<const S*>(&s)->n; // expected-warning {{constant expression}} expected-note {{dynamic_cast}}
+    int n : dynamic_cast<const S*>(&s)->n; // cxx11-warning {{constant expression}} cxx11-note {{dynamic_cast}}
   };
 }
 
@@ -423,7 +424,7 @@ namespace PseudoDtor {
   int k;
   typedef int I;
   struct T {
-    int n : (k.~I(), 0); // expected-error {{constant expression}}
+    int n : (k.~I(), 1); // cxx11-warning {{constant expression}} cxx11-note {{pseudo-destructor}}
   };
 }
 
@@ -431,8 +432,8 @@ namespace PseudoDtor {
 namespace IncDec {
   int k = 2;
   struct T {
-    int n : ++k; // expected-error {{constant expression}}
-    int m : --k; // expected-error {{constant expression}}
+    int n : ++k; // expected-error {{constant expression}} cxx20-note {{visible outside}}
+    int m : --k; // expected-error {{constant expression}} cxx20-note {{visible outside}}
   };
 }
 
@@ -446,7 +447,7 @@ namespace std {
 namespace TypeId {
   struct S { virtual void f(); };
   constexpr S *p = 0;
-  constexpr const std::type_info &ti1 = typeid(*p); // expected-error {{must be initialized by a constant expression}} expected-note {{typeid applied to expression of polymorphic type 'TypeId::S'}}
+  constexpr const std::type_info &ti1 = typeid(*p); // expected-error {{must be initialized by a constant expression}} cxx11-note {{typeid applied to expression of polymorphic type 'TypeId::S'}} cxx20-note {{dereferenced null pointer}}
 
   struct T {} t;
   constexpr const std::type_info &ti2 = typeid(t);
@@ -455,10 +456,10 @@ namespace TypeId {
 // - a new-expression (5.3.4);
 // - a delete-expression (5.3.5);
 namespace NewDelete {
-  int *p = 0;
+  constexpr int *p = 0;
   struct T {
-    int n : *new int(4); // expected-error {{constant expression}}
-    int m : (delete p, 2); // expected-error {{constant expression}}
+    int n : *new int(4); // expected-warning {{constant expression}} cxx11-note {{until C++20}} cxx20-note {{was not deallocated}}
+    int m : (delete p, 2); // cxx11-warning {{constant expression}} cxx11-note {{until C++20}}
   };
 }
 
@@ -550,8 +551,8 @@ namespace UnspecifiedRelations {
 namespace Assignment {
   int k;
   struct T {
-    int n : (k = 9); // expected-error {{constant expression}}
-    int m : (k *= 2); // expected-error {{constant expression}}
+    int n : (k = 9); // expected-error {{constant expression}} cxx20-note {{visible outside}}
+    int m : (k *= 2); // expected-error {{constant expression}} cxx20-note {{visible outside}}
   };
 
   struct Literal {
diff --git a/clang/test/CXX/expr/expr.const/p6-2a.cpp b/clang/test/CXX/expr/expr.const/p6-2a.cpp
new file mode 100644
index 0000000000000..312c28354188d
--- /dev/null
+++ b/clang/test/CXX/expr/expr.const/p6-2a.cpp
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -std=c++2a -verify %s
+
+constexpr int non_class = 42;
+constexpr int arr_non_class[5] = {1, 2, 3};
+
+struct A {
+  int member = 1;
+  constexpr ~A() { member = member + 1; }
+};
+constexpr A class_ = {};
+constexpr A arr_class[5] = {{}, {}};
+
+struct Mutable {
+  mutable int member = 1; // expected-note {{declared here}}
+  constexpr ~Mutable() { member = member + 1; } // expected-note {{read of mutable member}}
+};
+constexpr Mutable mut_member; // expected-error {{must have constant destruction}} expected-note {{in call}}
+
+struct MutableStore {
+  mutable int member = 1; // expected-note {{declared here}}
+  constexpr ~MutableStore() { member = 2; } // expected-note {{assignment to mutable member}}
+};
+constexpr MutableStore mut_store; // expected-error {{must have constant destruction}} expected-note {{in call}}
+
+// Note: the constant destruction rules disallow this example even though hcm.n is a const object.
+struct MutableConst {
+  struct HasConstMember {
+    const int n = 4;
+  };
+  mutable HasConstMember hcm; // expected-note {{here}}
+  constexpr ~MutableConst() {
+    int q = hcm.n; // expected-note {{read of mutable}}
+  }
+};
+constexpr MutableConst mc; // expected-error {{must have constant destruction}} expected-note {{in call}}
+
+struct Temporary {
+  int &&temp;
+  constexpr ~Temporary() {
+    int n = temp; // expected-note {{outside the expression that created the temporary}}
+  }
+};
+constexpr Temporary t = {3}; // expected-error {{must have constant destruction}} expected-note {{created here}} expected-note {{in call}}
diff --git a/clang/test/ClangScanDeps/Inputs/foodir b/clang/test/ClangScanDeps/Inputs/foodir
new file mode 100644
index 0000000000000..c2e511e0cbc56
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/foodir
@@ -0,0 +1 @@
+// A C++ header with same name as that of a directory in the include path.
diff --git a/clang/test/ClangScanDeps/Inputs/headerwithdirname.json b/clang/test/ClangScanDeps/Inputs/headerwithdirname.json
new file mode 100644
index 0000000000000..2ae561935bec3
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/headerwithdirname.json
@@ -0,0 +1,7 @@
+[
+    {
+      "directory": "DIR",
+      "command": "clang -c -IDIR -IDIR/foodir -IInputs DIR/headerwithdirname_input.cpp",
+      "file": "DIR/headerwithdirname_input.cpp"
+    }
+]
diff --git a/clang/test/ClangScanDeps/Inputs/strip_diag_serialize.json b/clang/test/ClangScanDeps/Inputs/strip_diag_serialize.json
new file mode 100644
index 0000000000000..a774d95a3b023
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/strip_diag_serialize.json
@@ -0,0 +1,7 @@
+[
+{
+  "directory": "DIR",
+  "command": "clang -E -fsyntax-only DIR/strip_diag_serialize_input.cpp --serialize-diagnostics /does/not/exist",
+  "file": "DIR/strip_diag_serialize_input.cpp"
+}
+]
diff --git a/clang/test/ClangScanDeps/headerwithdirname.cpp b/clang/test/ClangScanDeps/headerwithdirname.cpp
new file mode 100644
index 0000000000000..b0f60333aa4c3
--- /dev/null
+++ b/clang/test/ClangScanDeps/headerwithdirname.cpp
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.dir/foodir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: mkdir -p %t.dir/foodir
+// RUN: cp %s %t.dir/headerwithdirname_input.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/foodir %t.dir/Inputs/foodir
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/headerwithdirname.json > %t.cdb
+//
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 | FileCheck %s
+
+#include <foodir>
+
+// CHECK: headerwithdirname_input.o
+// CHECK-NEXT: headerwithdirname_input.cpp
+// CHECK-NEXT: Inputs{{/|\\}}foodir
diff --git a/clang/test/ClangScanDeps/strip_diag_serialize.cpp b/clang/test/ClangScanDeps/strip_diag_serialize.cpp
new file mode 100644
index 0000000000000..ec62e75134814
--- /dev/null
+++ b/clang/test/ClangScanDeps/strip_diag_serialize.cpp
@@ -0,0 +1,11 @@
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/strip_diag_serialize_input.cpp
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/strip_diag_serialize.json > %t.cdb
+//
+// RUN: clang-scan-deps -compilation-database %t.cdb 2>&1 | FileCheck %s
+// CHECK-NOT: unable to open file
+// CHECK: strip_diag_serialize_input.cpp
+
+#warning "diagnostic"
diff --git a/clang/test/CodeGen/align-global-large.c b/clang/test/CodeGen/align-global-large.c
index 14f5d8d9e1442..e53323f65f527 100644
--- a/clang/test/CodeGen/align-global-large.c
+++ b/clang/test/CodeGen/align-global-large.c
@@ -1,5 +1,5 @@
 // PR13606 - Clang crashes with large alignment attribute
-// RUN: %clang -cc1 -S -emit-llvm %s -o - -triple i686-pc-gnu | FileCheck %s
+// RUN: %clang_cc1 -S -emit-llvm %s -o - -triple i686-pc-gnu | FileCheck %s
 
 // CHECK: x
 // CHECK: align
diff --git a/clang/test/CodeGen/altivec-ct.c b/clang/test/CodeGen/altivec-ct.c
index 1a3e14dc1fd52..e36f556eb950b 100644
--- a/clang/test/CodeGen/altivec-ct.c
+++ b/clang/test/CodeGen/altivec-ct.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -triple powerpc64le-linux-gnu -S -O0 -o - %s -target-feature +altivec -target-feature +vsx | FileCheck %s -check-prefix=CHECK -check-prefix=VSX
-// RUN: %clang_cc1 -triple powerpc-linux-gnu -S -O0 -o - %s -target-feature +altivec -target-feature -vsx | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -triple powerpc64le-linux-gnu -S -O0 -o - %s -target-feature +altivec -target-feature +vsx | FileCheck %s -check-prefix=CHECK -check-prefix=VSX
+// RUN: %clang_cc1 -flax-vector-conversions=none -triple powerpc-linux-gnu -S -O0 -o - %s -target-feature +altivec -target-feature -vsx | FileCheck %s
 
 // REQUIRES: powerpc-registered-target
 
diff --git a/clang/test/CodeGen/arm-target-features.c b/clang/test/CodeGen/arm-target-features.c
index a0fbafc5d0c54..6200d057ad509 100644
--- a/clang/test/CodeGen/arm-target-features.c
+++ b/clang/test/CodeGen/arm-target-features.c
@@ -1,23 +1,23 @@
 // REQUIRES: arm-registered-target
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabihf -target-cpu cortex-a8 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP3
-// CHECK-VFP3: "target-features"="+armv7-a,+d32,+dsp,+fp64,+fpregs,+neon,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp"
+// CHECK-VFP3: "target-features"="+armv7-a,+d32,+dsp,+fp64,+fpregs,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp"
 
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabihf -target-cpu cortex-a5 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4
-// CHECK-VFP4: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+neon,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
+// CHECK-VFP4: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
 
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabihf -target-cpu cortex-a7 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-a12 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV
 // RUN: %clang_cc1 -triple thumbv7s-linux-gnueabi -target-cpu swift -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV-2
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabihf -target-cpu krait -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV
-// CHECK-VFP4-DIV: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
-// CHECK-VFP4-DIV-2: "target-features"="+armv7s,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
+// CHECK-VFP4-DIV: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
+// CHECK-VFP4-DIV-2: "target-features"="+armv7s,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
 
 // RUN: %clang_cc1 -triple armv7-linux-gnueabihf -target-cpu cortex-a15 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV-ARM
 // RUN: %clang_cc1 -triple armv7-linux-gnueabihf -target-cpu cortex-a17 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-DIV-ARM
-// CHECK-VFP4-DIV-ARM: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp,-thumb-mode"
+// CHECK-VFP4-DIV-ARM: "target-features"="+armv7-a,+d32,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp,-thumb-mode"
 
 // RUN: %clang_cc1 -triple thumbv7s-apple-ios7.0 -target-cpu cyclone -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu cortex-a32 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8
@@ -28,34 +28,34 @@
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu exynos-m1 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu exynos-m2 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu exynos-m3 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8
-// CHECK-BASIC-V8: "target-features"="+armv8-a,+crc,+crypto,+d32,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
+// CHECK-BASIC-V8: "target-features"="+armv8-a,+crc,+crypto,+d32,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
 
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu exynos-m4 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V82
 // RUN: %clang_cc1 -triple thumbv8-linux-gnueabihf -target-cpu exynos-m5 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V82
-// CHECK-BASIC-V82: "target-features"="+armv8.2-a,+crc,+crypto,+d32,+dotprod,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+fullfp16,+hwdiv,+hwdiv-arm,+neon,+ras,+thumb-mode,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
+// CHECK-BASIC-V82: "target-features"="+armv8.2-a,+crc,+crypto,+d32,+dotprod,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+fullfp16,+hwdiv,+hwdiv-arm,+neon,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp"
 
 // RUN: %clang_cc1 -triple armv8-linux-gnueabi -target-cpu cortex-a53 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-BASIC-V8-ARM
-// CHECK-BASIC-V8-ARM: "target-features"="+armv8-a,+crc,+crypto,+d32,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+vfp2,+vfp2d16,+vfp2d16sp,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp,-thumb-mode"
+// CHECK-BASIC-V8-ARM: "target-features"="+armv8-a,+crc,+crypto,+d32,+dsp,+fp-armv8,+fp-armv8d16,+fp-armv8d16sp,+fp-armv8sp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+neon,+vfp2,+vfp2sp,+vfp3,+vfp3d16,+vfp3d16sp,+vfp3sp,+vfp4,+vfp4d16,+vfp4d16sp,+vfp4sp,-thumb-mode"
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-r5 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP3-D16-DIV
-// CHECK-VFP3-D16-DIV: "target-features"="+armv7-r,+dsp,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+thumb-mode,+vfp2d16,+vfp2d16sp,+vfp3d16,+vfp3d16sp"
+// CHECK-VFP3-D16-DIV: "target-features"="+armv7-r,+dsp,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp"
 
 
 // RUN: %clang_cc1 -triple armv7-linux-gnueabi -target-cpu cortex-r4f -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP3-D16-THUMB-DIV
-// CHECK-VFP3-D16-THUMB-DIV: "target-features"="+armv7-r,+dsp,+fp64,+fpregs,+hwdiv,+vfp2d16,+vfp2d16sp,+vfp3d16,+vfp3d16sp,-thumb-mode"
+// CHECK-VFP3-D16-THUMB-DIV: "target-features"="+armv7-r,+dsp,+fp64,+fpregs,+hwdiv,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,-thumb-mode"
 
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-r7 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP3-D16-FP16-DIV
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-r8 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP3-D16-FP16-DIV
-// CHECK-VFP3-D16-FP16-DIV: "target-features"="+armv7-r,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+thumb-mode,+vfp2d16,+vfp2d16sp,+vfp3d16,+vfp3d16sp"
+// CHECK-VFP3-D16-FP16-DIV: "target-features"="+armv7-r,+dsp,+fp16,+fp64,+fpregs,+hwdiv,+hwdiv-arm,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp"
 
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-m4 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP4-D16-SP-THUMB-DIV
-// CHECK-VFP4-D16-SP-THUMB-DIV: "target-features"="+armv7e-m,+dsp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2d16sp,+vfp3d16sp,+vfp4d16sp"
+// CHECK-VFP4-D16-SP-THUMB-DIV: "target-features"="+armv7e-m,+dsp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp"
 
 
 // RUN: %clang_cc1 -triple thumbv7-linux-gnueabi -target-cpu cortex-m7 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-VFP5-D16-THUMB-DIV
-// CHECK-VFP5-D16-THUMB-DIV: "target-features"="+armv7e-m,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fpregs,+hwdiv,+thumb-mode,+vfp2d16,+vfp2d16sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp"
+// CHECK-VFP5-D16-THUMB-DIV: "target-features"="+armv7e-m,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fpregs,+hwdiv,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp"
 
 
 // RUN: %clang_cc1 -triple armv7-linux-gnueabi -target-cpu cortex-r4 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-THUMB-DIV
@@ -107,6 +107,6 @@
 // CHECK-ARMV8M-M23-LINUX: "target-features"="+armv8-m.base,+hwdiv,+thumb-mode"
 
 // RUN: %clang_cc1 -triple thumb-linux-gnueabi -target-cpu cortex-m33 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARMV8M-MAIN-LINUX 
-// CHECK-ARMV8M-MAIN-LINUX: "target-features"="+armv8-m.main,+dsp,+fp-armv8d16sp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2d16sp,+vfp3d16sp,+vfp4d16sp"
+// CHECK-ARMV8M-MAIN-LINUX: "target-features"="+armv8-m.main,+dsp,+fp-armv8d16sp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp"
 
 void foo() {}
diff --git a/clang/test/CodeGen/arm_acle.c b/clang/test/CodeGen/arm_acle.c
index 2f086ee70bfd9..ce2c5fac70b2c 100644
--- a/clang/test/CodeGen/arm_acle.c
+++ b/clang/test/CodeGen/arm_acle.c
@@ -6,6 +6,8 @@
 // RUN: %clang_cc1 -ffreestanding -triple aarch64-eabi -target-cpu cortex-a57 -target-feature +v8.4a -O2 -fexperimental-new-pass-manager -S -emit-llvm -o - %s | FileCheck %s -check-prefix=AArch64-v8_3
 // RUN: %clang_cc1 -ffreestanding -triple aarch64-eabi -target-cpu cortex-a57 -target-feature +v8.5a -O2 -fexperimental-new-pass-manager -S -emit-llvm -o - %s | FileCheck %s -check-prefix=AArch64-v8_3
 
+// REQUIRES: rewrite
+
 #include <arm_acle.h>
 
 /* 8 SYNCHRONIZATION, BARRIER AND HINT INTRINSICS */
diff --git a/clang/test/CodeGen/available-externally-suppress.c b/clang/test/CodeGen/available-externally-suppress.c
index 55f85ad250d29..1e79750fb9d6d 100644
--- a/clang/test/CodeGen/available-externally-suppress.c
+++ b/clang/test/CodeGen/available-externally-suppress.c
@@ -30,10 +30,8 @@ inline int __attribute__((always_inline)) f1(int x) {
 // CHECK: @test1
 // LTO: @test1
 int test1(int x) {
-  // CHECK: br i1
   // CHECK-NOT: call {{.*}} @f1
   // CHECK: ret i32
-  // LTO: br i1
   // LTO-NOT: call {{.*}} @f1
   // LTO: ret i32
   return f1(x);
diff --git a/clang/test/CodeGen/avr-builtins.c b/clang/test/CodeGen/avr-builtins.c
index cbba6b2f2a2e5..8fa983a782393 100644
--- a/clang/test/CodeGen/avr-builtins.c
+++ b/clang/test/CodeGen/avr-builtins.c
@@ -1,5 +1,9 @@
 // RUN: %clang_cc1 -triple avr-unknown-unknown -emit-llvm -o - %s | FileCheck %s
 
+// Check that the parameter types match. This verifies pr43309.
+// RUN: %clang_cc1 -triple avr-unknown-unknown -Wconversion -verify %s
+// expected-no-diagnostics
+
 unsigned char bitrev8(unsigned char data) {
     return __builtin_bitreverse8(data);
 }
diff --git a/clang/test/CodeGen/avx-builtins.c b/clang/test/CodeGen/avx-builtins.c
index 0a09c53391e57..b31a234db2eda 100644
--- a/clang/test/CodeGen/avx-builtins.c
+++ b/clang/test/CodeGen/avx-builtins.c
@@ -1,6 +1,6 @@
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx -fno-signed-char -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 
 #include <immintrin.h>
@@ -105,7 +105,7 @@ __m256d test_mm256_broadcast_sd(double* A) {
   return _mm256_broadcast_sd(A);
 }
 
-__m128d test_mm_broadcast_ss(float* A) {
+__m128 test_mm_broadcast_ss(float* A) {
   // CHECK-LABEL: test_mm_broadcast_ss
   // CHECK: load float, float* %{{.*}}
   // CHECK: insertelement <4 x float> undef, float %{{.*}}, i32 0
@@ -115,7 +115,7 @@ __m128d test_mm_broadcast_ss(float* A) {
   return _mm_broadcast_ss(A);
 }
 
-__m256d test_mm256_broadcast_ss(float* A) {
+__m256 test_mm256_broadcast_ss(float* A) {
   // CHECK-LABEL: test_mm256_broadcast_ss
   // CHECK: load float, float* %{{.*}}
   // CHECK: insertelement <8 x float> undef, float %{{.*}}, i32 0
@@ -1278,7 +1278,7 @@ __m128 test_mm_maskload_ps(float* A, __m128i B) {
   return _mm_maskload_ps(A, B);
 }
 
-__m256d test_mm256_maskload_ps(float* A, __m256i B) {
+__m256 test_mm256_maskload_ps(float* A, __m256i B) {
   // CHECK-LABEL: test_mm256_maskload_ps
   // CHECK: call <8 x float> @llvm.x86.avx.maskload.ps.256(i8* %{{.*}}, <8 x i32> %{{.*}})
   return _mm256_maskload_ps(A, B);
diff --git a/clang/test/CodeGen/avx-cmp-builtins.c b/clang/test/CodeGen/avx-cmp-builtins.c
index 609f5964f31f2..adb8a86112cc2 100644
--- a/clang/test/CodeGen/avx-cmp-builtins.c
+++ b/clang/test/CodeGen/avx-cmp-builtins.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -ffreestanding %s -O3 -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s
+// RUN: %clang_cc1 -flax-vector-conversions=none -ffreestanding %s -O3 -triple=x86_64-apple-darwin -target-feature +avx -emit-llvm -o - | FileCheck %s
 // FIXME: The shufflevector instructions in test_cmpgt_sd are relying on O3 here.
 
 
@@ -14,7 +14,7 @@ __m128d test_cmp_sd(__m128d a, __m128d b) {
   return _mm_cmp_sd(a, b, _CMP_GE_OS);
 }
 
-__m128d test_cmp_ss(__m128 a, __m128 b) {
+__m128 test_cmp_ss(__m128 a, __m128 b) {
   // Expects that the third argument in LLVM IR is immediate expression
   // CHECK: @llvm.x86.sse.cmp.ss({{.*}}, i8 13)
   return _mm_cmp_ss(a, b, _CMP_GE_OS);
diff --git a/clang/test/CodeGen/avx512f-builtins.c b/clang/test/CodeGen/avx512f-builtins.c
index 2a083e3e5c609..73a093c54683b 100644
--- a/clang/test/CodeGen/avx512f-builtins.c
+++ b/clang/test/CodeGen/avx512f-builtins.c
@@ -1,5 +1,5 @@
-// RUN: %clang_cc1 -fexperimental-new-pass-manager -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s
-// RUN: %clang_cc1 -fexperimental-new-pass-manager -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -fexperimental-new-pass-manager -flax-vector-conversions=none -ffreestanding %s -triple=x86_64-apple-darwin -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s
+// RUN: %clang_cc1 -fexperimental-new-pass-manager -flax-vector-conversions=none -fms-extensions -fms-compatibility -ffreestanding %s -triple=x86_64-windows-msvc -target-feature +avx512f -emit-llvm -o - -Wall -Werror | FileCheck %s
 
 #include <immintrin.h>
 
@@ -9468,7 +9468,7 @@ __m256 test_mm512_mask_cvtpd_ps (__m256 __W, __mmask8 __U, __m512d __A)
   return _mm512_mask_cvtpd_ps (__W,__U,__A);
 }
 
-__m512d test_mm512_cvtpd_pslo(__m512 __A) 
+__m512 test_mm512_cvtpd_pslo(__m512d __A)
 {
   // CHECK-LABEL: @test_mm512_cvtpd_pslo
   // CHECK: @llvm.x86.avx512.mask.cvtpd2ps.512
@@ -9477,7 +9477,7 @@ __m512d test_mm512_cvtpd_pslo(__m512 __A)
   return _mm512_cvtpd_pslo(__A);
 }
 
-__m512d test_mm512_mask_cvtpd_pslo(__m512 __W, __mmask8 __U, __m512d __A) {
+__m512 test_mm512_mask_cvtpd_pslo(__m512 __W, __mmask8 __U, __m512d __A) {
   // CHECK-LABEL: @test_mm512_mask_cvtpd_pslo
   // CHECK: @llvm.x86.avx512.mask.cvtpd2ps.512
   // CHECK: zeroinitializer
@@ -10659,7 +10659,7 @@ __m512i test_mm512_setzero_epi32()
   return _mm512_setzero_epi32();
 }
 
-__m512i test_mm512_setzero()
+__m512 test_mm512_setzero()
 {
   // CHECK-LABEL: @test_mm512_setzero
   // CHECK: zeroinitializer
@@ -10673,7 +10673,7 @@ __m512i test_mm512_setzero_si512()
   return _mm512_setzero_si512();
 }
 
-__m512i test_mm512_setzero_ps()
+__m512 test_mm512_setzero_ps()
 {
   // CHECK-LABEL: @test_mm512_setzero_ps
   // CHECK: zeroinitializer
diff --git a/clang/test/CodeGen/builtin-preserve-access-index-nonptr.c b/clang/test/CodeGen/builtin-preserve-access-index-nonptr.c
new file mode 100644
index 0000000000000..bb8caf4921bef
--- /dev/null
+++ b/clang/test/CodeGen/builtin-preserve-access-index-nonptr.c
@@ -0,0 +1,18 @@
+// RUN: %clang -target x86_64 -emit-llvm -S -g %s -o - | FileCheck %s
+
+#define _(x) (__builtin_preserve_access_index(x))
+
+struct s1 {
+  char a;
+  int b[4];
+};
+
+int unit1(struct s1 *arg) {
+  return _(arg->b[2]);
+}
+// CHECK: define dso_local i32 @unit1
+// CHECK: call [4 x i32]* @llvm.preserve.struct.access.index.p0a4i32.p0s_struct.s1s(%struct.s1* %{{[0-9a-z]+}}, i32 1, i32 1), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[STRUCT_S1:[0-9]+]]
+// CHECK: call i32* @llvm.preserve.array.access.index.p0i32.p0a4i32([4 x i32]* %{{[0-9a-z]+}}, i32 1, i32 2), !dbg !{{[0-9]+}}, !llvm.preserve.access.index ![[ARRAY:[0-9]+]]
+//
+// CHECK: ![[ARRAY]] = !DICompositeType(tag: DW_TAG_array_type
+// CHECK: ![[STRUCT_S1]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s1"
diff --git a/clang/test/CodeGen/builtins-ppc-error.c b/clang/test/CodeGen/builtins-ppc-error.c
index 39a2458226433..80ca227eeb24c 100644
--- a/clang/test/CodeGen/builtins-ppc-error.c
+++ b/clang/test/CodeGen/builtins-ppc-error.c
@@ -2,10 +2,19 @@
 
 // RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector \
 // RUN:   -triple powerpc64-unknown-unknown -fsyntax-only   \
+// RUN: -flax-vector-conversions=integer \
 // RUN: -Wall -Werror -verify %s
 
 // RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector  \
 // RUN: -triple powerpc64le-unknown-unknown -fsyntax-only    \
+// RUN: -flax-vector-conversions=integer \
+// RUN: -Wall -Werror -verify %s
+
+// FIXME: Fix <altivec.h> so this test also passes under
+// -flax-vector-conversions=none (this last test exists to produce an error if
+// we change the default to that without fixing <altivec.h>).
+// RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector \
+// RUN:   -triple powerpc64-unknown-unknown -fsyntax-only   \
 // RUN: -Wall -Werror -verify %s
 
 #include <altivec.h>
diff --git a/clang/test/CodeGen/builtins-ppc-p9vector.c b/clang/test/CodeGen/builtins-ppc-p9vector.c
index bfbb815854714..e920cb76f4d91 100644
--- a/clang/test/CodeGen/builtins-ppc-p9vector.c
+++ b/clang/test/CodeGen/builtins-ppc-p9vector.c
@@ -1,12 +1,20 @@
 // REQUIRES: powerpc-registered-target
 // RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector \
 // RUN:   -triple powerpc64-unknown-unknown -emit-llvm %s \
+// RUN:   -flax-vector-conversions=integer \
 // RUN:   -o - | FileCheck %s -check-prefix=CHECK-BE
 
 // RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector \
 // RUN:   -triple powerpc64le-unknown-unknown -emit-llvm %s \
+// RUN:   -flax-vector-conversions=integer \
 // RUN:   -o - | FileCheck %s
 
+// FIXME: This last test is intended to fail if the default is changed to
+// -flax-vector-conversions=none and <altivec.h> isn't fixed first.
+// RUN: %clang_cc1 -target-feature +altivec -target-feature +power9-vector \
+// RUN:   -triple powerpc64-unknown-unknown -emit-llvm %s \
+// RUN:   -o - | FileCheck %s -check-prefix=CHECK-BE
+
 #include <altivec.h>
 
 vector signed char vsca, vscb;
@@ -919,7 +927,7 @@ vector double test80(void) {
 // CHECK: insertelement <2 x double>
   return vec_unpackl(vfa);
 }
-vector double test81(void) {
+vector float test81(void) {
   // CHECK: extractelement <2 x double>
   // CHECK: fptrunc double
   // CHECK: insertelement <4 x float>
diff --git a/clang/test/CodeGen/builtins-ppc-vsx.c b/clang/test/CodeGen/builtins-ppc-vsx.c
index 838d94cf7de9a..0c797a389b944 100644
--- a/clang/test/CodeGen/builtins-ppc-vsx.c
+++ b/clang/test/CodeGen/builtins-ppc-vsx.c
@@ -882,7 +882,7 @@ void test1() {
 // CHECK: call void @dummy()
 // CHECK-LE: call void @dummy()
 
-  res_vf = vec_sel(vd, vd, vbll);
+  res_vd = vec_sel(vd, vd, vbll);
 // CHECK: xor <2 x i64> %{{[0-9]+}}, <i64 -1, i64 -1>
 // CHECK: and <2 x i64> %{{[0-9]+}},
 // CHECK: and <2 x i64> %{{[0-9]+}}, %{{[0-9]+}}
diff --git a/clang/test/CodeGen/builtins-systemz-vector3-error.c b/clang/test/CodeGen/builtins-systemz-vector3-error.c
index 6583857c3c178..99b171c55bb55 100644
--- a/clang/test/CodeGen/builtins-systemz-vector3-error.c
+++ b/clang/test/CodeGen/builtins-systemz-vector3-error.c
@@ -1,5 +1,5 @@
 // REQUIRES: systemz-registered-target
-// RUN: %clang_cc1 -target-cpu arch13 -triple s390x-unknown-unknown \
+// RUN: %clang_cc1 -target-cpu z15 -triple s390x-unknown-unknown \
 // RUN: -Wall -Wno-unused -Werror -fsyntax-only -verify %s
 
 typedef __attribute__((vector_size(16))) signed char vec_schar;
diff --git a/clang/test/CodeGen/builtins-systemz-vector3.c b/clang/test/CodeGen/builtins-systemz-vector3.c
index e8194b642cd63..e4af1c1e54b86 100644
--- a/clang/test/CodeGen/builtins-systemz-vector3.c
+++ b/clang/test/CodeGen/builtins-systemz-vector3.c
@@ -1,5 +1,5 @@
 // REQUIRES: systemz-registered-target
-// RUN: %clang_cc1 -target-cpu arch13 -triple s390x-ibm-linux -flax-vector-conversions=none \
+// RUN: %clang_cc1 -target-cpu z15 -triple s390x-ibm-linux -flax-vector-conversions=none \
 // RUN: -Wall -Wno-unused -Werror -emit-llvm %s -o - | FileCheck %s
 
 typedef __attribute__((vector_size(16))) signed char vec_schar;
diff --git a/clang/test/CodeGen/builtins-systemz-zvector3-error.c b/clang/test/CodeGen/builtins-systemz-zvector3-error.c
index 657508b261867..557b9a3a7ec1b 100644
--- a/clang/test/CodeGen/builtins-systemz-zvector3-error.c
+++ b/clang/test/CodeGen/builtins-systemz-zvector3-error.c
@@ -1,5 +1,5 @@
 // REQUIRES: systemz-registered-target
-// RUN: %clang_cc1 -target-cpu arch13 -triple s390x-linux-gnu \
+// RUN: %clang_cc1 -target-cpu z15 -triple s390x-linux-gnu \
 // RUN: -fzvector -flax-vector-conversions=none \
 // RUN: -Wall -Wno-unused -Werror -fsyntax-only -verify %s
 
diff --git a/clang/test/CodeGen/builtins-systemz-zvector3.c b/clang/test/CodeGen/builtins-systemz-zvector3.c
index db30f41933263..a987c6113e36b 100644
--- a/clang/test/CodeGen/builtins-systemz-zvector3.c
+++ b/clang/test/CodeGen/builtins-systemz-zvector3.c
@@ -1,8 +1,8 @@
 // REQUIRES: systemz-registered-target
-// RUN: %clang_cc1 -target-cpu arch13 -triple s390x-linux-gnu \
+// RUN: %clang_cc1 -target-cpu z15 -triple s390x-linux-gnu \
 // RUN: -O -fzvector -flax-vector-conversions=none \
 // RUN: -Wall -Wno-unused -Werror -emit-llvm %s -o - | FileCheck %s
-// RUN: %clang_cc1 -target-cpu arch13 -triple s390x-linux-gnu \
+// RUN: %clang_cc1 -target-cpu z15 -triple s390x-linux-gnu \
 // RUN: -O -fzvector -flax-vector-conversions=none \
 // RUN: -Wall -Wno-unused -Werror -S %s -o - | FileCheck %s --check-prefix=CHECK-ASM
 
diff --git a/clang/test/CodeGen/builtins-wasm.c b/clang/test/CodeGen/builtins-wasm.c
index ee08983d9a1e7..ca407ebc0ed4f 100644
--- a/clang/test/CodeGen/builtins-wasm.c
+++ b/clang/test/CodeGen/builtins-wasm.c
@@ -463,3 +463,79 @@ i64x2 trunc_saturate_u_i64x2_f64x2(f64x2 f) {
   // WEBASSEMBLY: call <2 x i64> @llvm.wasm.trunc.saturate.unsigned.v2i64.v2f64(<2 x double> %f)
   // WEBASSEMBLY-NEXT: ret
 }
+
+i8x16 narrow_s_i8x16_i16x8(i16x8 low, i16x8 high) {
+  return __builtin_wasm_narrow_s_i8x16_i16x8(low, high);
+  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(
+  // WEBASSEMBLY-SAME: <8 x i16> %low, <8 x i16> %high)
+  // WEBASSEMBLY: ret
+}
+
+i8x16 narrow_u_i8x16_i16x8(i16x8 low, i16x8 high) {
+  return __builtin_wasm_narrow_u_i8x16_i16x8(low, high);
+  // WEBASSEMBLY: call <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(
+  // WEBASSEMBLY-SAME: <8 x i16> %low, <8 x i16> %high)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 narrow_s_i16x8_i32x4(i32x4 low, i32x4 high) {
+  return __builtin_wasm_narrow_s_i16x8_i32x4(low, high);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(
+  // WEBASSEMBLY-SAME: <4 x i32> %low, <4 x i32> %high)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 narrow_u_i16x8_i32x4(i32x4 low, i32x4 high) {
+  return __builtin_wasm_narrow_u_i16x8_i32x4(low, high);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(
+  // WEBASSEMBLY-SAME: <4 x i32> %low, <4 x i32> %high)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 widen_low_s_i16x8_i8x16(i8x16 v) {
+  return __builtin_wasm_widen_low_s_i16x8_i8x16(v);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8> %v)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 widen_high_s_i16x8_i8x16(i8x16 v) {
+  return __builtin_wasm_widen_high_s_i16x8_i8x16(v);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8> %v)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 widen_low_u_i16x8_i8x16(i8x16 v) {
+  return __builtin_wasm_widen_low_u_i16x8_i8x16(v);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8> %v)
+  // WEBASSEMBLY: ret
+}
+
+i16x8 widen_high_u_i16x8_i8x16(i8x16 v) {
+  return __builtin_wasm_widen_high_u_i16x8_i8x16(v);
+  // WEBASSEMBLY: call <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8> %v)
+  // WEBASSEMBLY: ret
+}
+
+i32x4 widen_low_s_i32x4_i16x8(i16x8 v) {
+  return __builtin_wasm_widen_low_s_i32x4_i16x8(v);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16> %v)
+  // WEBASSEMBLY: ret
+}
+
+i32x4 widen_high_s_i32x4_i16x8(i16x8 v) {
+  return __builtin_wasm_widen_high_s_i32x4_i16x8(v);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16> %v)
+  // WEBASSEMBLY: ret
+}
+
+i32x4 widen_low_u_i32x4_i16x8(i16x8 v) {
+  return __builtin_wasm_widen_low_u_i32x4_i16x8(v);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16> %v)
+  // WEBASSEMBLY: ret
+}
+
+i32x4 widen_high_u_i32x4_i16x8(i16x8 v) {
+  return __builtin_wasm_widen_high_u_i32x4_i16x8(v);
+  // WEBASSEMBLY: call <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16> %v)
+  // WEBASSEMBLY: ret
+}
diff --git a/clang/test/CodeGen/const-init.c b/clang/test/CodeGen/const-init.c
index 41ac8f2eb05bb..374aa00e65893 100644
--- a/clang/test/CodeGen/const-init.c
+++ b/clang/test/CodeGen/const-init.c
@@ -141,8 +141,8 @@ void g28() {
   // CHECK: @g28.b = internal global <12 x i16> <i16 0, i16 0, i16 0, i16 -32768, i16 16383, i16 0, i16 0, i16 0, i16 0, i16 -32768, i16 16384, i16 0>
   // CHECK: @g28.c = internal global <2 x x86_fp80> <x86_fp80 0xK3FFF8000000000000000, x86_fp80 0xK40008000000000000000>, align 32
   static v1i64 a = (v1i64)10LL;
-  static v12i16 b = (v2f80){1,2};
-  static v2f80 c = (v12i16){0,0,0,-32768,16383,0,0,0,0,-32768,16384,0};
+  static v12i16 b = (v12i16)(v2f80){1,2};
+  static v2f80 c = (v2f80)(v12i16){0,0,0,-32768,16383,0,0,0,0,-32768,16384,0};
 }
 
 // PR13643
diff --git a/clang/test/CodeGen/inline.c b/clang/test/CodeGen/inline.c
index ea6e9d76494b2..b511ece2813b9 100644
--- a/clang/test/CodeGen/inline.c
+++ b/clang/test/CodeGen/inline.c
@@ -52,7 +52,7 @@
 // CHECK3-LABEL: define i32 @_Z3barv()
 // CHECK3-LABEL: define linkonce_odr i32 @_Z3foov()
 // CHECK3-NOT: unreferenced
-// CHECK3-LABEL: define void @_Z10gnu_inlinev()
+// CHECK3-LABEL: define available_externally void @_Z10gnu_inlinev()
 // CHECK3-LABEL: define available_externally void @_Z13gnu_ei_inlinev()
 // CHECK3-NOT: @_Z5testCv
 // CHECK3-LABEL: define linkonce_odr i32 @_Z2eiv()
@@ -85,6 +85,7 @@ __inline void unreferenced1() {}
 extern __inline void unreferenced2() {}
 
 __inline __attribute((__gnu_inline__)) void gnu_inline() {}
+void (*P1)() = gnu_inline;
 
 // PR3988
 extern __inline __attribute__((gnu_inline)) void gnu_ei_inline() {}
diff --git a/clang/test/CodeGen/ppc-smmintrin.c b/clang/test/CodeGen/ppc-smmintrin.c
index e11656225a3d0..666bc0fc440cb 100644
--- a/clang/test/CodeGen/ppc-smmintrin.c
+++ b/clang/test/CodeGen/ppc-smmintrin.c
@@ -15,7 +15,7 @@ test_extract() {
   _mm_extract_epi8(mi, 0);
   _mm_extract_epi32(mi, 0);
   _mm_extract_epi64(mi, 0);
-  _mm_extract_ps(mi, 0);
+  _mm_extract_ps((__m128)mi, 0);
 }
 
 // CHECK-LABEL: @test_extract
diff --git a/clang/test/CodeGen/sse-builtins.c b/clang/test/CodeGen/sse-builtins.c
index b68a714b43b34..3891c076fa9b8 100644
--- a/clang/test/CodeGen/sse-builtins.c
+++ b/clang/test/CodeGen/sse-builtins.c
@@ -714,7 +714,7 @@ void test_mm_storeu_ps(float* x,  __m128 y) {
   _mm_storeu_ps(x, y);
 }
 
-void test_mm_stream_ps(float*A, __m128d B) {
+void test_mm_stream_ps(float*A, __m128 B) {
   // CHECK-LABEL: test_mm_stream_ps
   // CHECK: store <4 x float> %{{.*}}, <4 x float>* %{{.*}}, align 16, !nontemporal
   _mm_stream_ps(A, B);
diff --git a/clang/test/CodeGen/sse.c b/clang/test/CodeGen/sse.c
index 1191f55f81e7a..a75b8dc77e86e 100644
--- a/clang/test/CodeGen/sse.c
+++ b/clang/test/CodeGen/sse.c
@@ -5,37 +5,37 @@
 #include <emmintrin.h>
 
 // Byte-shifts look reversed due to xmm register layout
-__m128 test_mm_slli_si128(__m128 a) {
+__m128i test_mm_slli_si128(__m128i a) {
   // CHECK-LABEL: @test_mm_slli_si128
   // CHECK: shufflevector <16 x i8> <{{.*}}, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> {{.*}}, <16 x i32> <i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26>
   return _mm_slli_si128(a, 5);
 }
 
-__m128 test_mm_slli_si128_0(__m128 a) {
+__m128i test_mm_slli_si128_0(__m128i a) {
   // CHECK-LABEL: @test_mm_slli_si128_0
   // CHECK-NOT: shufflevector
   return _mm_slli_si128(a, 0);
 }
 
-__m128 test_mm_slli_si128_16(__m128 a) {
+__m128i test_mm_slli_si128_16(__m128i a) {
   // CHECK-LABEL: @test_mm_slli_si128_16
   // CHECK-NOT: shufflevector
   return _mm_slli_si128(a, 16);
 }
 
-__m128 test_mm_srli_si128(__m128 a) {
+__m128i test_mm_srli_si128(__m128i a) {
   // CHECK-LABEL: @test_mm_srli_si128
   // CHECK: shufflevector <16 x i8> {{.*}}, <16 x i8> <i8 0, i8 0, i8 0, i8 0, i8 0, {{.*}}>, <16 x i32> <i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20>
   return _mm_srli_si128(a, 5);
 }
 
-__m128 test_mm_srli_si128_0(__m128 a) {
+__m128i test_mm_srli_si128_0(__m128i a) {
   // CHECK-LABEL: @test_mm_srli_si128_0
   // CHECK-NOT: shufflevector
   return _mm_srli_si128(a, 0);
 }
 
-__m128 test_mm_srli_si128_16(__m128 a) {
+__m128i test_mm_srli_si128_16(__m128i a) {
   // CHECK-LABEL: @test_mm_srli_si128_16
   // CHECK-NOT: shufflevector
   return _mm_srli_si128(a, 16);
diff --git a/clang/test/CodeGen/systemz-abi-vector.c b/clang/test/CodeGen/systemz-abi-vector.c
index 46c008a8d9f8e..f2e6c13c718f5 100644
--- a/clang/test/CodeGen/systemz-abi-vector.c
+++ b/clang/test/CodeGen/systemz-abi-vector.c
@@ -10,6 +10,8 @@
 // RUN:   -emit-llvm -o - %s | FileCheck --check-prefix=CHECK-VECTOR %s
 // RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu arch12 \
 // RUN:   -emit-llvm -o - %s | FileCheck --check-prefix=CHECK-VECTOR %s
+// RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu z15 \
+// RUN:   -emit-llvm -o - %s | FileCheck --check-prefix=CHECK-VECTOR %s
 // RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu arch13 \
 // RUN:   -emit-llvm -o - %s | FileCheck --check-prefix=CHECK-VECTOR %s
 
diff --git a/clang/test/CodeGen/systemz-abi.c b/clang/test/CodeGen/systemz-abi.c
index c04a51ff6ef44..3511983e32d76 100644
--- a/clang/test/CodeGen/systemz-abi.c
+++ b/clang/test/CodeGen/systemz-abi.c
@@ -10,6 +10,8 @@
 // RUN:   -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu arch12 \
 // RUN:   -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu z15 \
+// RUN:   -emit-llvm -o - %s | FileCheck %s
 // RUN: %clang_cc1 -triple s390x-linux-gnu -target-cpu arch13 \
 // RUN:   -emit-llvm -o - %s | FileCheck %s
 
diff --git a/clang/test/CodeGen/target-builtin-error.c b/clang/test/CodeGen/target-builtin-error.c
index ee41277706aee..4e44b7208a963 100644
--- a/clang/test/CodeGen/target-builtin-error.c
+++ b/clang/test/CodeGen/target-builtin-error.c
@@ -3,6 +3,6 @@
 
 #include <x86intrin.h>
 
-__m128d foo(__m128d a, __m128d b) {
+__m128 foo(__m128 a, __m128 b) {
   return __builtin_ia32_addsubps(b, a); // expected-error {{'__builtin_ia32_addsubps' needs target feature sse3}}
 }
diff --git a/clang/test/CodeGen/target-data.c b/clang/test/CodeGen/target-data.c
index 3e4d05436327d..21a8b415dc217 100644
--- a/clang/test/CodeGen/target-data.c
+++ b/clang/test/CodeGen/target-data.c
@@ -199,6 +199,8 @@
 // RUN: FileCheck %s -check-prefix=SYSTEMZ-VECTOR
 // RUN: %clang_cc1 -triple s390x-unknown -target-cpu arch12 -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=SYSTEMZ-VECTOR
+// RUN: %clang_cc1 -triple s390x-unknown -target-cpu z15 -o - -emit-llvm %s | \
+// RUN: FileCheck %s -check-prefix=SYSTEMZ-VECTOR
 // RUN: %clang_cc1 -triple s390x-unknown -target-cpu arch13 -o - -emit-llvm %s | \
 // RUN: FileCheck %s -check-prefix=SYSTEMZ-VECTOR
 // SYSTEMZ-VECTOR: target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
diff --git a/clang/test/CodeGen/tbaa-struct.cpp b/clang/test/CodeGen/tbaa-struct.cpp
index f9eb71962f56b..0c0e4b40bfffb 100644
--- a/clang/test/CodeGen/tbaa-struct.cpp
+++ b/clang/test/CodeGen/tbaa-struct.cpp
@@ -17,7 +17,7 @@ typedef A __attribute__((may_alias)) AA;
 
 void copy(A *a1, A *a2) {
 // CHECK-LABEL: _Z4copyP1AS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(16) %{{.*}}, i8* align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(16) %{{.*}}, i8* nonnull align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS:!.*]]
 // CHECK-NEW-SAME: !tbaa [[TAG_A:![0-9]*]]
   *a1 = *a2;
@@ -31,7 +31,7 @@ struct B {
 
 void copy2(B *b1, B *b2) {
 // CHECK-LABEL: _Z5copy2P1BS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(24) %{{.*}}, i8* align 4 dereferenceable(24) %{{.*}}, i64 24, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(24) %{{.*}}, i8* nonnull align 4 dereferenceable(24) %{{.*}}, i64 24, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS2:!.*]]
 // CHECK-NEW-SAME: !tbaa [[TAG_B:![0-9]*]]
   *b1 = *b2;
@@ -49,7 +49,7 @@ union U {
 
 void copy3(U *u1, U *u2) {
 // CHECK-LABEL: _Z5copy3P1US0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(12) %{{.*}}, i8* align 4 dereferenceable(12) %{{.*}}, i64 12, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(12) %{{.*}}, i8* nonnull align 4 dereferenceable(12) %{{.*}}, i64 12, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS3:!.*]]
 // CHECK-NEW-SAME: !tbaa [[TAG_U:![0-9]*]]
   *u1 = *u2;
@@ -65,7 +65,7 @@ struct C {
 
 void copy4(C *c1, C *c2) {
 // CHECK-LABEL: _Z5copy4P1CS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(3) {{.*}}, i8* align 1 dereferenceable(3) {{.*}}, i64 3, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(3) {{.*}}, i8* nonnull align 1 dereferenceable(3) {{.*}}, i64 3, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS4:!.*]]
 // CHECK-NEW-SAME: !tbaa [[TAG_C:![0-9]*]]
   *c1 = *c2;
@@ -80,7 +80,7 @@ struct D {
 
 void copy5(D *d1, D *d2) {
 // CHECK-LABEL: _Z5copy5P1DS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(6) {{.*}}, i8* align 1 dereferenceable(6) {{.*}}, i64 6, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(6) {{.*}}, i8* nonnull align 1 dereferenceable(6) {{.*}}, i64 6, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS5:!.*]]
 // CHECK-NEW-SAME: !tbaa [[TAG_D:![0-9]*]]
   *d1 = *d2;
@@ -88,7 +88,7 @@ void copy5(D *d1, D *d2) {
 
 void copy6(AA *a1, A *a2) {
 // CHECK-LABEL: _Z5copy6P1AS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(16) %{{.*}}, i8* align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(16) %{{.*}}, i8* nonnull align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS]]
 // CHECK-NEW-SAME: !tbaa [[TAG_char:![0-9]*]]
   *a1 = *a2;
@@ -96,7 +96,7 @@ void copy6(AA *a1, A *a2) {
 
 void copy7(A *a1, AA *a2) {
 // CHECK-LABEL: _Z5copy7P1AS0_
-// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(16) %{{.*}}, i8* align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
+// CHECK: call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(16) %{{.*}}, i8* nonnull align 4 dereferenceable(16) %{{.*}}, i64 16, i1 false)
 // CHECK-OLD-SAME: !tbaa.struct [[TS]]
 // CHECK-NEW-SAME: !tbaa [[TAG_char]]
   *a1 = *a2;
diff --git a/clang/test/CodeGen/x86-builtins-vector-width.c b/clang/test/CodeGen/x86-builtins-vector-width.c
index 62f5929e4fda9..dca3e5874ea9c 100644
--- a/clang/test/CodeGen/x86-builtins-vector-width.c
+++ b/clang/test/CodeGen/x86-builtins-vector-width.c
@@ -1,7 +1,7 @@
 // RUN: %clang_cc1 -triple i686-linux-gnu -target-cpu i686 -emit-llvm %s -o - | FileCheck %s
 
-typedef signed long long V2LLi __attribute__((vector_size(16)));
-typedef signed long long V4LLi __attribute__((vector_size(32)));
+typedef double V2LLi __attribute__((vector_size(16)));
+typedef double V4LLi __attribute__((vector_size(32)));
 
 // Make sure builtin forces a min-legal-width attribute
 void foo(void) {
diff --git a/clang/test/CodeGen/x86-builtins.c b/clang/test/CodeGen/x86-builtins.c
new file mode 100644
index 0000000000000..fa2530b2cd15f
--- /dev/null
+++ b/clang/test/CodeGen/x86-builtins.c
@@ -0,0 +1,45 @@
+// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-unknown-unknown -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=CHECK-64
+// RUN: %clang_cc1 -ffreestanding %s -triple=i386-unknown-unknown -emit-llvm -o - -Wall -Werror | FileCheck %s -check-prefix=CHECK-32
+
+#include <x86intrin.h>
+
+unsigned int test_castf32_u32 (float __A){
+  // CHECK-64-LABEL: @test_castf32_u32
+  // CHECK-64: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %{{.*}}, i8* align 4 %{{.*}}, i64 4, i1 false)
+  // CHECK-64: %{{.*}} = load i32, i32* %{{.*}}, align 4
+  // CHECK-32-LABEL: @test_castf32_u32
+  // CHECK-32: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %{{.*}}, i8* align 4 %{{.*}}, i32 4, i1 false)
+  // CHECK-32: %{{.*}} = load i32, i32* %{{.*}}, align 4
+  return _castf32_u32(__A);
+}
+
+unsigned long long test_castf64_u64 (double __A){
+  // CHECK-64-LABEL: @test_castf64_u64
+  // CHECK-64: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %{{.*}}, i8* align 8 %{{.*}}, i64 8, i1 false)
+  // CHECK-64: %{{.*}} = load i64, i64* %{{.*}}, align 8
+  // CHECK-32-LABEL: @test_castf64_u64
+  // CHECK-32: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %{{.*}}, i8* align 8 %{{.*}}, i32 8, i1 false)
+  // CHECK-32: %{{.*}} = load i64, i64* %{{.*}}, align 8
+  return _castf64_u64(__A);
+}
+
+float test_castu32_f32 (unsigned int __A){
+  // CHECK-64-LABEL: @test_castu32_f32
+  // CHECK-64: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %{{.*}}, i8* align 4 %{{.*}}, i64 4, i1 false)
+  // CHECK-64: %{{.*}} = load float, float* %{{.*}}, align 4
+  // CHECK-32-LABEL: @test_castu32_f32
+  // CHECK-32: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 4 %{{.*}}, i8* align 4 %{{.*}}, i32 4, i1 false)
+  // CHECK-32: %{{.*}} = load float, float* %{{.*}}, align 4
+  return _castu32_f32(__A);
+}
+
+double test_castu64_f64 (unsigned long long __A){
+  // CHECK-64-LABEL: @test_castu64_f64
+  // CHECK-64: call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 8 %{{.*}}, i8* align 8 %{{.*}}, i64 8, i1 false)
+  // CHECK-64: %{{.*}} = load double, double* %{{.*}}, align 8
+  // CHECK-32-LABEL: @test_castu64_f64
+  // CHECK-32: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 8 %{{.*}}, i8* align 8 %{{.*}}, i32 8, i1 false)
+  // CHECK-32: %{{.*}} = load double, double* %{{.*}}, align 8
+  return _castu64_f64(__A);
+}
+
diff --git a/clang/test/CodeGen/x86_64-mno-sse2.c b/clang/test/CodeGen/x86_64-mno-sse2.c
new file mode 100644
index 0000000000000..0c8e78edb9951
--- /dev/null
+++ b/clang/test/CodeGen/x86_64-mno-sse2.c
@@ -0,0 +1,20 @@
+// RUN: %clang_cc1 -triple x86_64-linux -target-feature -sse2 -S -o /dev/null -verify %s
+// REQUIRES: x86-registered-target
+
+double f1(void) { // expected-error {{SSE2 register return with SSE2 disabled}}
+  return 1.4;
+}
+extern double g;
+void f2(void) { // expected-error {{SSE2 register return with SSE2 disabled}}
+  g = f1();
+}
+void take_double(double);
+void pass_double(void) {
+  // FIXME: Still asserts.
+  //take_double(1.5);
+}
+
+double return_double();
+void call_double(double *a) { // expected-error {{SSE2 register return with SSE2 disabled}}
+  *a = return_double();
+}
diff --git a/clang/test/CodeGenCUDA/Inputs/cuda.h b/clang/test/CodeGenCUDA/Inputs/cuda.h
index 0fd175765a205..5d73b81041ab5 100644
--- a/clang/test/CodeGenCUDA/Inputs/cuda.h
+++ b/clang/test/CodeGenCUDA/Inputs/cuda.h
@@ -14,12 +14,21 @@ struct dim3 {
   __host__ __device__ dim3(unsigned x, unsigned y = 1, unsigned z = 1) : x(x), y(y), z(z) {}
 };
 
-typedef struct cudaStream *cudaStream_t;
-typedef enum cudaError {} cudaError_t;
 #ifdef __HIP__
+typedef struct hipStream *hipStream_t;
+typedef enum hipError {} hipError_t;
 int hipConfigureCall(dim3 gridSize, dim3 blockSize, size_t sharedSize = 0,
-                     cudaStream_t stream = 0);
+                     hipStream_t stream = 0);
+extern "C" hipError_t __hipPushCallConfiguration(dim3 gridSize, dim3 blockSize,
+                                                 size_t sharedSize = 0,
+                                                 hipStream_t stream = 0);
+extern "C" hipError_t hipLaunchKernel(const void *func, dim3 gridDim,
+                                      dim3 blockDim, void **args,
+                                      size_t sharedMem,
+                                      hipStream_t stream);
 #else
+typedef struct cudaStream *cudaStream_t;
+typedef enum cudaError {} cudaError_t;
 extern "C" int cudaConfigureCall(dim3 gridSize, dim3 blockSize,
                                  size_t sharedSize = 0,
                                  cudaStream_t stream = 0);
diff --git a/clang/test/CodeGenCUDA/kernel-call.cu b/clang/test/CodeGenCUDA/kernel-call.cu
index ed48a6cc81384..b76f2c1883576 100644
--- a/clang/test/CodeGenCUDA/kernel-call.cu
+++ b/clang/test/CodeGenCUDA/kernel-call.cu
@@ -3,14 +3,17 @@
 // RUN: %clang_cc1 -target-sdk-version=9.2  -emit-llvm %s -o - \
 // RUN: | FileCheck %s --check-prefixes=CUDA-NEW,CHECK
 // RUN: %clang_cc1 -x hip -emit-llvm %s -o - \
-// RUN: | FileCheck %s --check-prefixes=HIP,CHECK
-
+// RUN: | FileCheck %s --check-prefixes=HIP-OLD,CHECK
+// RUN: %clang_cc1 -fhip-new-launch-api -x hip -emit-llvm %s -o - \
+// RUN: | FileCheck %s --check-prefixes=HIP-NEW,CHECK
 
 #include "Inputs/cuda.h"
 
 // CHECK-LABEL: define{{.*}}g1
-// HIP: call{{.*}}hipSetupArgument
-// HIP: call{{.*}}hipLaunchByPtr
+// HIP-OLD: call{{.*}}hipSetupArgument
+// HIP-OLD: call{{.*}}hipLaunchByPtr
+// HIP-NEW: call{{.*}}__hipPopCallConfiguration
+// HIP-NEW: call{{.*}}hipLaunchKernel
 // CUDA-OLD: call{{.*}}cudaSetupArgument
 // CUDA-OLD: call{{.*}}cudaLaunch
 // CUDA-NEW: call{{.*}}__cudaPopCallConfiguration
@@ -19,7 +22,8 @@ __global__ void g1(int x) {}
 
 // CHECK-LABEL: define{{.*}}main
 int main(void) {
-  // HIP: call{{.*}}hipConfigureCall
+  // HIP-OLD: call{{.*}}hipConfigureCall
+  // HIP-NEW: call{{.*}}__hipPushCallConfiguration
   // CUDA-OLD: call{{.*}}cudaConfigureCall
   // CUDA-NEW: call{{.*}}__cudaPushCallConfiguration
   // CHECK: icmp
diff --git a/clang/test/CodeGenCXX/attr-no-destroy-d54344.cpp b/clang/test/CodeGenCXX/attr-no-destroy-d54344.cpp
index 2e004d6426d94..b03791e5135df 100644
--- a/clang/test/CodeGenCXX/attr-no-destroy-d54344.cpp
+++ b/clang/test/CodeGenCXX/attr-no-destroy-d54344.cpp
@@ -14,6 +14,7 @@
 
 class a {
 public:
+  a();
   ~a();
 };
 class logger_base {
diff --git a/clang/test/CodeGenCXX/const-init-cxx2a.cpp b/clang/test/CodeGenCXX/const-init-cxx2a.cpp
new file mode 100644
index 0000000000000..1195b912c2556
--- /dev/null
+++ b/clang/test/CodeGenCXX/const-init-cxx2a.cpp
@@ -0,0 +1,58 @@
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-llvm -o - %s -std=c++2a | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
+
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -emit-pch -o %t.pch %s -std=c++2a
+// RUN: %clang_cc1 -triple x86_64-linux-gnu -include-pch %t.pch -x c++ /dev/null -emit-llvm -o - -std=c++2a | FileCheck %s --implicit-check-not=cxx_global_var_init --implicit-check-not=cxa_atexit
+
+// CHECK: @a = global i32 123,
+int a = (delete new int, 123);
+
+struct B {
+  constexpr B() {}
+  constexpr ~B() { n *= 5; }
+  int n = 123;
+};
+// CHECK: @b = global {{.*}} i32 123
+extern constexpr B b = B();
+
+// CHECK: @_ZL1c = internal global {{.*}} i32 123
+const B c;
+int use_c() { return c.n; }
+
+struct D {
+  int n;
+  constexpr ~D() {}
+};
+D d;
+// CHECK: @d = global {{.*}} zeroinitializer
+
+D d_arr[3];
+// CHECK: @d_arr = global {{.*}} zeroinitializer
+
+thread_local D d_tl;
+// CHECK: @d_tl = thread_local global {{.*}} zeroinitializer
+
+// CHECK-NOT: @llvm.global_ctors
+
+// CHECK-LABEL: define {{.*}} @_Z1fv(
+void f() {
+  // CHECK-NOT: call
+  // CHECK: call {{.*}}memcpy
+  // CHECK-NOT: call
+  // CHECK: call {{.*}}memset
+  // CHECK-NOT: call
+  // CHECK: }
+  constexpr B b;
+  D d = D();
+}
+
+// CHECK-LABEL: define {{.*}} @_Z1gv(
+void g() {
+  // CHECK-NOT: call
+  // CHECK-NOT: cxa_guard
+  // CHECK-NOT: _ZGV
+  // CHECK: }
+  static constexpr B b1;
+  static const B b2;
+  static D d;
+  thread_local D d_tl;
+}
diff --git a/clang/test/CodeGenCXX/cxx2a-thread-local-constinit.cpp b/clang/test/CodeGenCXX/cxx2a-thread-local-constinit.cpp
index 149d5b99d4628..af46a92c7dc2d 100644
--- a/clang/test/CodeGenCXX/cxx2a-thread-local-constinit.cpp
+++ b/clang/test/CodeGenCXX/cxx2a-thread-local-constinit.cpp
@@ -33,11 +33,6 @@ extern thread_local int c;
 // CHECK: }
 int h() { return c; }
 
-thread_local int c = 0;
-
-int d_init();
-thread_local int d = d_init();
-
 // Note: use of 'c' does not trigger initialization of 'd', because 'c' has a
 // constant initializer.
 // CHECK-LABEL: define weak_odr {{.*}} @_ZTW1c()
@@ -45,3 +40,30 @@ thread_local int d = d_init();
 // CHECK-NOT: call
 // CHECK: ret i32* @c
 // CHECK: }
+
+thread_local int c = 0;
+
+int d_init();
+
+// CHECK: define {{.*}}[[D_INIT:@__cxx_global_var_init[^(]*]](
+// CHECK: call {{.*}} @_Z6d_initv()
+thread_local int d = d_init();
+
+struct Destructed {
+  int n;
+  ~Destructed();
+};
+
+extern thread_local constinit Destructed e;
+// CHECK-LABEL: define i32 @_Z1iv()
+// CHECK: call {{.*}}* @_ZTW1e()
+// CHECK: }
+int i() { return e.n; }
+
+// CHECK: define {{.*}}[[E2_INIT:@__cxx_global_var_init[^(]*]](
+// CHECK: call {{.*}} @__cxa_thread_atexit({{.*}} @_ZN10DestructedD1Ev {{.*}} @e2
+thread_local constinit Destructed e2;
+
+// CHECK-LABEL: define {{.*}}__tls_init
+// CHECK: call {{.*}} [[D_INIT]]
+// CHECK: call {{.*}} [[E2_INIT]]
diff --git a/clang/test/CodeGenCXX/ms-constexpr-var-template.cpp b/clang/test/CodeGenCXX/ms-constexpr-var-template.cpp
deleted file mode 100644
index a40f7aacac03b..0000000000000
--- a/clang/test/CodeGenCXX/ms-constexpr-var-template.cpp
+++ /dev/null
@@ -1,11 +0,0 @@
-// RUN: %clang_cc1 -emit-llvm -triple=x86_64-windows-msvc -fms-compatibility %s -o - | FileCheck %s
-
-template <typename> constexpr bool _Is_integer = false;
-template <> constexpr bool _Is_integer<int> = true;
-template <> constexpr bool _Is_integer<char> = false;
-extern "C" const bool *escape = &_Is_integer<int>;
-
-// CHECK: @"??$_Is_integer@H@@3_NB" = linkonce_odr dso_local constant i8 1, comdat, align 1
-//   Should not emit _Is_integer<char>, since it's not referenced.
-// CHECK-NOT: @"??$_Is_integer@D@@3_NB"
-// CHECK: @escape = dso_local global i8* @"??$_Is_integer@H@@3_NB", align 8
diff --git a/clang/test/CodeGenCXX/no_destroy.cpp b/clang/test/CodeGenCXX/no_destroy.cpp
index 3400b6080b53f..607cbfb3a1fca 100644
--- a/clang/test/CodeGenCXX/no_destroy.cpp
+++ b/clang/test/CodeGenCXX/no_destroy.cpp
@@ -5,10 +5,8 @@ struct NonTrivial {
   ~NonTrivial();
 };
 
-// CHECK-LABEL: define internal void @__cxx_global_var_init
 // CHECK-NOT: __cxa_atexit{{.*}}_ZN10NonTrivialD1Ev
 [[clang::no_destroy]] NonTrivial nt1;
-// CHECK-LABEL: define internal void @__cxx_global_var_init
 // CHECK-NOT: _tlv_atexit{{.*}}_ZN10NonTrivialD1Ev
 [[clang::no_destroy]] thread_local NonTrivial nt2;
 
@@ -16,11 +14,9 @@ struct NonTrivial2 {
   ~NonTrivial2();
 };
 
-// CHECK-LABEL: define internal void @__cxx_global_var_init
-// CHECK: __cxa_atexit{{.*}}_ZN11NonTrivial2D1Ev
+// CHECK: __cxa_atexit{{.*}}_ZN11NonTrivial2D1Ev{{.*}}nt21
 NonTrivial2 nt21;
-// CHECK-LABEL: define internal void @__cxx_global_var_init
-// CHECK: _tlv_atexit{{.*}}_ZN11NonTrivial2D1Ev
+// CHECK: _tlv_atexit{{.*}}_ZN11NonTrivial2D1Ev{{.*}}nt22
 thread_local NonTrivial2 nt22;
 
 // CHECK-LABEL: define void @_Z1fv
diff --git a/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp b/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp
new file mode 100644
index 0000000000000..120b32090fb4a
--- /dev/null
+++ b/clang/test/CodeGenCXX/non-const-init-cxx2a.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-llvm -o - %s -std=c++2a | FileCheck %s
+
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -emit-pch -o %t.pch %s -std=c++2a
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -include-pch %t.pch -x c++ /dev/null -emit-llvm -o - -std=c++2a | FileCheck %s
+
+struct B {
+  constexpr B() {}
+  constexpr ~B() { n *= 5; }
+  int n = 123;
+};
+
+// We emit a dynamic destructor here because b.n might have been modified
+// before b is destroyed.
+//
+// CHECK: @b = global {{.*}} i32 123
+B b = B();
+
+// CHECK: define {{.*}}cxx_global_var_init
+// CHECK: call {{.*}} @__cxa_atexit({{.*}} @_ZN1BD1Ev {{.*}} @b
diff --git a/clang/test/CodeGenCXX/wasm-args-returns.cpp b/clang/test/CodeGenCXX/wasm-args-returns.cpp
index 50ebdb96848f5..5718223f9f740 100644
--- a/clang/test/CodeGenCXX/wasm-args-returns.cpp
+++ b/clang/test/CodeGenCXX/wasm-args-returns.cpp
@@ -46,7 +46,7 @@ struct copy_ctor {
   copy_ctor(copy_ctor const &);
 };
 test(copy_ctor);
-// CHECK: define void @_Z7forward9copy_ctor(%struct.copy_ctor* noalias sret %{{.*}}, %struct.copy_ctor* %{{.*}})
+// CHECK: define void @_Z7forward9copy_ctor(%struct.copy_ctor* noalias sret %{{.*}}, %struct.copy_ctor* nonnull %{{.*}})
 //
 // CHECK: declare %struct.copy_ctor* @_ZN9copy_ctorC1ERKS_(%struct.copy_ctor* returned, %struct.copy_ctor* dereferenceable(8))
 //
@@ -64,7 +64,7 @@ struct __attribute__((aligned(16))) aligned_copy_ctor {
   aligned_copy_ctor(aligned_copy_ctor const &);
 };
 test(aligned_copy_ctor);
-// CHECK: define void @_Z7forward17aligned_copy_ctor(%struct.aligned_copy_ctor* noalias sret %{{.*}}, %struct.aligned_copy_ctor* %{{.*}})
+// CHECK: define void @_Z7forward17aligned_copy_ctor(%struct.aligned_copy_ctor* noalias sret %{{.*}}, %struct.aligned_copy_ctor* nonnull %{{.*}})
 //
 // CHECK: declare %struct.aligned_copy_ctor* @_ZN17aligned_copy_ctorC1ERKS_(%struct.aligned_copy_ctor* returned, %struct.aligned_copy_ctor* dereferenceable(16))
 //
diff --git a/clang/test/CodeGenObjC/protocol-comdat.m b/clang/test/CodeGenObjC/protocol-comdat.m
index 1402ad0a045a1..79a1d5535576b 100644
--- a/clang/test/CodeGenObjC/protocol-comdat.m
+++ b/clang/test/CodeGenObjC/protocol-comdat.m
@@ -1,4 +1,4 @@
-// RUN: %clang -cc1 -triple thumbv7--windows-itanium -fobjc-runtime=ios -emit-llvm -o - %s -Wno-objc-root-class | FileCheck %s
+// RUN: %clang_cc1 -triple thumbv7--windows-itanium -fobjc-runtime=ios -emit-llvm -o - %s -Wno-objc-root-class | FileCheck %s
 
 @protocol P
 - (void) method;
diff --git a/clang/test/Driver/arm-mfpu.c b/clang/test/Driver/arm-mfpu.c
index 4b0289b73b93a..9a87b63856794 100644
--- a/clang/test/Driver/arm-mfpu.c
+++ b/clang/test/Driver/arm-mfpu.c
@@ -35,7 +35,7 @@
 // CHECK-SOFT-ABI-FP-2-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-2-DAG: "-target-feature" "-neon"
 // CHECK-SOFT-ABI-FP-2-DAG: "-target-feature" "-crypto"
-// CHECK-SOFT-ABI-FP-2-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-2-DAG: "-target-feature" "-vfp2sp"
 
 // RUN: %clang -target arm-linux-eabi -mfpu=vfp3 %s -### -o %t.o 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-VFP3 %s
@@ -50,7 +50,7 @@
 // CHECK-VFP3-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-VFP3-DAG: "-target-feature" "-neon"
 // CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-3-DAG: "-target-feature" "-neon"
@@ -144,7 +144,7 @@
 // CHECK-VFP4-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-VFP4-DAG: "-target-feature" "-neon"
 // CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "-vfp3d16sp"
 // CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-4-DAG: "-target-feature" "-neon"
@@ -208,7 +208,7 @@
 // CHECK-FP5-DP-D16-DAG: "-target-feature" "-crypto"
 // CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "+soft-float"
 // CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "-vfp3d16sp"
 // CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-SOFT-ABI-FP-5-DAG: "-target-feature" "-neon"
@@ -222,7 +222,7 @@
 // CHECK-NEON-NOT: "-target-feature" "+soft-float"
 // CHECK-NEON-DAG: "-target-feature" "+neon"
 // CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-6-DAG: "-target-feature" "-crypto"
@@ -262,7 +262,7 @@
 // CHECK-NEON-VFPV4-DAG: "-target-feature" "+vfp4"
 // CHECK-NEON-VFPV4-DAG: "-target-feature" "+neon"
 // CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "-vfp3d16sp"
 // CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-7-DAG: "-target-feature" "-crypto"
@@ -276,7 +276,7 @@
 // RUN: %clang -target armv8a -mfpu=neon %s -### -c 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-SOFT-ABI-FP-8 %s
 // CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "-fp-armv8d16sp"
 // CHECK-SOFT-ABI-FP-8-DAG: "-target-feature" "-crypto"
@@ -319,7 +319,7 @@
 // CHECK-NO-FP-NOT: "-target-feature" "+soft-float"
 // CHECK-NO-FP-DAG: "-target-feature" "+soft-float-abi"
 // CHECK-NO-FP-DAG: "-target-feature" "-fpregs"
-// CHECK-NO-FP-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-NO-FP-DAG: "-target-feature" "-vfp2sp"
 // CHECK-NO-FP-DAG: "-target-feature" "-vfp3d16sp"
 // CHECK-NO-FP-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-NO-FP-DAG: "-target-feature" "-fp-armv8d16sp"
@@ -358,7 +358,7 @@
 // RUN:   | FileCheck --check-prefix=CHECK-SOFT-ABI-FP %s
 // CHECK-SOFT-ABI-FP-DAG: "-target-feature" "+soft-float"
 // CHECK-SOFT-ABI-FP-DAG: "-target-feature" "+soft-float-abi"
-// CHECK-SOFT-ABI-FP-DAG: "-target-feature" "-vfp2d16sp"
+// CHECK-SOFT-ABI-FP-DAG: "-target-feature" "-vfp2sp"
 // CHECK-SOFT-ABI-FP-DAG: "-target-feature" "-vfp3d16sp"
 // CHECK-SOFT-ABI-FP-DAG: "-target-feature" "-vfp4d16sp"
 // CHECK-SOFT-ABI-FP-DAG: "-target-feature" "-fp-armv8d16sp"
diff --git a/clang/test/Driver/compilation_database.c b/clang/test/Driver/compilation_database.c
index 017178d60cde5..343b76aa44f3a 100644
--- a/clang/test/Driver/compilation_database.c
+++ b/clang/test/Driver/compilation_database.c
@@ -1,8 +1,8 @@
-// RUN: mkdir -p %t && cd %t
+// RUN: mkdir -p %t.workdir && cd %t.workdir
 // RUN: %clang -MD -MP --sysroot=somewhere -c -x c %s -xc++ %s -Wall -MJ - -no-canonical-prefixes 2>&1 | FileCheck %s
 // RUN: not %clang -c -x c %s -MJ %s/non-existant -no-canonical-prefixes 2>&1 | FileCheck --check-prefix=ERROR %s
 
-// CHECK: { "directory": "{{.*}}",  "file": "[[SRC:[^"]+[/|\\]compilation_database.c]]", "output": "compilation_database.o", "arguments": ["{{[^"]*}}clang{{[^"]*}}", "-xc", "[[SRC]]", "--sysroot=somewhere", "-c", "-Wall",{{.*}} "--target={{[^"]+}}"]},
+// CHECK: { "directory": "{{[^"]*}}workdir",  "file": "[[SRC:[^"]+[/|\\]compilation_database.c]]", "output": "compilation_database.o", "arguments": ["{{[^"]*}}clang{{[^"]*}}", "-xc", "[[SRC]]", "--sysroot=somewhere", "-c", "-Wall",{{.*}} "--target={{[^"]+}}"]},
 // CHECK: { "directory": "{{.*}}",  "file": "[[SRC:[^"]+[/|\\]compilation_database.c]]", "output": "compilation_database.o", "arguments": ["{{[^"]*}}clang{{[^"]*}}", "-xc++", "[[SRC]]", "--sysroot=somewhere", "-c", "-Wall",{{.*}} "--target={{[^"]+}}"]},
 // ERROR: error: compilation database '{{.*}}/non-existant' could not be opened:
 
diff --git a/clang/test/Driver/coverage_no_integrated_as.c b/clang/test/Driver/coverage_no_integrated_as.c
index d7689e591ff00..fc9af6bd9458b 100644
--- a/clang/test/Driver/coverage_no_integrated_as.c
+++ b/clang/test/Driver/coverage_no_integrated_as.c
@@ -1,21 +1,26 @@
 // REQUIRES: clang-driver
-// XFAIL: windows-msvc
+
+// RUN: %clang -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NO-COV %s
 
 // RUN: %clang -### -S -fprofile-arcs %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
-// RUN: %clang -### -S -fprofile-arcs -no-integrated-as %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
+// RUN: %clang -### -S -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
 // RUN: %clang -### -c -fprofile-arcs %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
-// RUN: %clang -### -c -fprofile-arcs -no-integrated-as %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
+// RUN: %clang -### -c -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s 2>&1 | FileCheck -check-prefix=CHECK-GCNO-DEFAULT-LOCATION %s
 
 // RUN: %clang -### -S -fprofile-arcs %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
-// RUN: %clang -### -S -fprofile-arcs -no-integrated-as %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
+// RUN: %clang -### -S -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
 // RUN: %clang -### -c -fprofile-arcs %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
-// RUN: %clang -### -c -fprofile-arcs -no-integrated-as %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
+// RUN: %clang -### -c -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s -o /foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION %s
 
 // RUN: %clang -### -S -fprofile-arcs %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
-// RUN: %clang -### -S -fprofile-arcs -no-integrated-as %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
+// RUN: %clang -### -S -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
 // RUN: %clang -### -c -fprofile-arcs %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
-// RUN: %clang -### -c -fprofile-arcs -no-integrated-as %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
+// RUN: %clang -### -c -fprofile-arcs --target=%itanium_abi_triple -no-integrated-as %s -o foo/bar.o 2>&1 | FileCheck -check-prefix=CHECK-GCNO-LOCATION-REL-PATH %s
 
+// These should only get passed if any of --coverage, -ftest-coverage, or
+// -fprofile-arcs is passed.
+// CHECK-NO-COV-NOT: "-coverage-notes-file"
+// CHECK-NO-COV-NOT: "-coverage-data-file"
 
 // CHECK-GCNO-DEFAULT-LOCATION: "-coverage-notes-file" "{{.*}}{{/|\\\\}}coverage_no_integrated_as.c"
 // CHECK-GCNO-DEFAULT-LOCATION-NOT: "-coverage-notes-file" "/tmp/{{.*}}/coverage_no_integrated_as.c"
diff --git a/clang/test/Driver/darwin-sdkroot.c b/clang/test/Driver/darwin-sdkroot.c
index 8b3782ea22a71..700993fa96a66 100644
--- a/clang/test/Driver/darwin-sdkroot.c
+++ b/clang/test/Driver/darwin-sdkroot.c
@@ -2,7 +2,7 @@
 //
 // RUN: rm -rf %t.tmpdir
 // RUN: mkdir -p %t.tmpdir
-// RUN: env SDKROOT=%t.tmpdir %clang -target x86_64-apple-darwin10 \
+// RUN: env SDKROOT=%t.tmpdir %clang -target x86_64-apple-darwin10 --sysroot="" \
 // RUN:   -c %s -### 2> %t.log
 // RUN: FileCheck --check-prefix=CHECK-BASIC < %t.log %s
 //
@@ -13,7 +13,7 @@
 // Check that we don't use SDKROOT as the default if it is not a valid path.
 //
 // RUN: rm -rf %t.nonpath
-// RUN: env SDKROOT=%t.nonpath %clang -target x86_64-apple-darwin10 \
+// RUN: env SDKROOT=%t.nonpath %clang -target x86_64-apple-darwin10 --sysroot="" \
 // RUN:   -c %s -### 2> %t.log
 // RUN: FileCheck --check-prefix=CHECK-NONPATH < %t.log %s
 //
@@ -23,7 +23,7 @@
 
 // Check that we don't use SDKROOT as the default if it is just "/"
 //
-// RUN: env SDKROOT=/ %clang -target x86_64-apple-darwin10 \
+// RUN: env SDKROOT=/ %clang -target x86_64-apple-darwin10 --sysroot="" \
 // RUN:   -c %s -### 2> %t.log
 // RUN: FileCheck --check-prefix=CHECK-NONROOT < %t.log %s
 //
@@ -43,7 +43,7 @@
 //
 // RUN: rm -rf %t/SDKs/iPhoneOS8.0.0.sdk
 // RUN: mkdir -p %t/SDKs/iPhoneOS8.0.0.sdk
-// RUN: env SDKROOT=%t/SDKs/iPhoneOS8.0.0.sdk %clang -target arm64-apple-darwin %s -### 2>&1 \
+// RUN: env SDKROOT=%t/SDKs/iPhoneOS8.0.0.sdk %clang -target arm64-apple-darwin --sysroot="" %s -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-IPHONE %s
 //
 // CHECK-IPHONE: clang
@@ -55,7 +55,7 @@
 //
 // RUN: rm -rf %t/SDKs/iPhoneSimulator8.0.sdk
 // RUN: mkdir -p %t/SDKs/iPhoneSimulator8.0.sdk
-// RUN: env SDKROOT=%t/SDKs/iPhoneSimulator8.0.sdk %clang -target x86_64-apple-darwin %s -### 2>&1 \
+// RUN: env SDKROOT=%t/SDKs/iPhoneSimulator8.0.sdk %clang -target x86_64-apple-darwin --sysroot="" %s -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-SIMULATOR %s
 //
 // CHECK-SIMULATOR: clang
@@ -66,7 +66,7 @@
 //
 // RUN: rm -rf %t/SDKs/MacOSX10.10.0.sdk
 // RUN: mkdir -p %t/SDKs/MacOSX10.10.0.sdk
-// RUN: env SDKROOT=%t/SDKs/MacOSX10.10.0.sdk %clang -target x86_64-apple-darwin %s -### 2>&1 \
+// RUN: env SDKROOT=%t/SDKs/MacOSX10.10.0.sdk %clang -target x86_64-apple-darwin --sysroot="" %s -### 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-MACOSX %s
 //
 // CHECK-MACOSX: clang
diff --git a/clang/test/Driver/fuchsia.c b/clang/test/Driver/fuchsia.c
index ec664fb3b5a8d..bf8e5a04dc91b 100644
--- a/clang/test/Driver/fuchsia.c
+++ b/clang/test/Driver/fuchsia.c
@@ -16,7 +16,7 @@
 // CHECK: "-fsanitize=safe-stack"
 // CHECK: "-stack-protector" "2"
 // CHECK: "-fno-common"
-// CHECK: {{.*}}ld.lld{{.*}}" "-z" "rodynamic" "-z" "separate-code"
+// CHECK: {{.*}}ld.lld{{.*}}" "-z" "rodynamic" "-z" "separate-loadable-segments"
 // CHECK: "--sysroot=[[SYSROOT]]"
 // CHECK: "-pie"
 // CHECK: "--build-id"
diff --git a/clang/test/Driver/gcc-toolchain.cpp b/clang/test/Driver/gcc-toolchain.cpp
index ca96757a2bbcf..6c872f4255c34 100644
--- a/clang/test/Driver/gcc-toolchain.cpp
+++ b/clang/test/Driver/gcc-toolchain.cpp
@@ -3,12 +3,14 @@
 // RUN: %clangxx -no-canonical-prefixes %s -### -o %t 2>&1 \
 // RUN:     --target=i386-unknown-linux -stdlib=libstdc++ \
 // RUN:     --gcc-toolchain=%S/Inputs/ubuntu_11.04_multiarch_tree/usr \
+// RUN:     --sysroot="" \
 // RUN:   | FileCheck %s
 //
 // Additionally check that the legacy spelling of the flag works.
 // RUN: %clangxx -no-canonical-prefixes %s -### -o %t 2>&1 \
 // RUN:     --target=i386-unknown-linux -stdlib=libstdc++ \
 // RUN:     -gcc-toolchain %S/Inputs/ubuntu_11.04_multiarch_tree/usr \
+// RUN:     --sysroot="" \
 // RUN:   | FileCheck %s
 //
 // Test for header search toolchain detection.
diff --git a/clang/test/Driver/m-and-mm.c b/clang/test/Driver/m-and-mm.c
new file mode 100644
index 0000000000000..cb719a6c97fe8
--- /dev/null
+++ b/clang/test/Driver/m-and-mm.c
@@ -0,0 +1,35 @@
+// RUN: %clang -M %s 2>&1 | FileCheck %s --implicit-check-not=warning
+// RUN: %clang -MM %s 2>&1 | FileCheck %s --implicit-check-not=warning
+
+// CHECK: m-and-mm.o:
+// TEST-I: {{.*}}test.i:
+// TEST: {{.*}}test:
+
+// RUN: mkdir -p %t.dir
+
+/// if -MD and -MMD are not specified, -o specifies the dependency file name.
+// RUN: rm -f %t.dir/test.i
+// RUN: %clang -M %s -o %t.dir/test.i
+// RUN: FileCheck %s < %t.dir/test.i
+// RUN: rm -f %t.dir/test.i
+// RUN: %clang -MM %s -o %t.dir/test.i
+// RUN: FileCheck %s < %t.dir/test.i
+
+// RUN: rm -f %t.dir/test.d
+// RUN: %clang -fsyntax-only -MD %s -o %t.dir/test.i
+// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d
+
+// RUN: rm -f %t.dir/test.d
+// RUN: %clang -M -MD %s -o %t.dir/test.i
+// RUN: FileCheck --check-prefix=TEST-I %s < %t.dir/test.d
+
+/// If the output file name does not have a suffix, just append `.d`.
+// RUN: rm -f %t.dir/test.d
+// RUN: %clang -fsyntax-only -MD %s -o %t.dir/test
+// RUN: FileCheck --check-prefix=TEST %s < %t.dir/test.d
+
+#warning "-M and -MM suppresses warnings, thus this warning shouldn't show up"
+int main(void)
+{
+    return 0;
+}
diff --git a/clang/test/Driver/m_and_mm.c b/clang/test/Driver/m_and_mm.c
deleted file mode 100644
index 6e40c82cb16cf..0000000000000
--- a/clang/test/Driver/m_and_mm.c
+++ /dev/null
@@ -1,15 +0,0 @@
-// RUN: %clang -### \
-// RUN:   -M -MM %s 2> %t
-// RUN: not grep '"-sys-header-deps"' %t
-
-// RUN: %clang -M -MM %s 2> %t
-// RUN: not grep "warning" %t
-
-// RUN: %clang -MMD -MD %s 2> %t || true
-// RUN: grep "warning" %t
-
-#warning "This warning shouldn't show up with -M and -MM"
-int main (void)
-{
-    return 0;
-}
diff --git a/clang/test/Driver/metadata-with-dots.c b/clang/test/Driver/metadata-with-dots.c
deleted file mode 100644
index 371b48bd2cc6e..0000000000000
--- a/clang/test/Driver/metadata-with-dots.c
+++ /dev/null
@@ -1,11 +0,0 @@
-// REQUIRES: shell
-// RUN: mkdir -p out.dir
-// RUN: cat %s > out.dir/test.c
-// RUN: %clang -E -MMD %s -o out.dir/test
-// RUN: test ! -f %out.d
-// RUN: test -f out.dir/test.d
-// RUN: rm -rf out.dir/test.d out.dir/ out.d
-int main (void)
-{
-    return 0;
-}
diff --git a/clang/test/Driver/mips-features.c b/clang/test/Driver/mips-features.c
index 19725bc096b5d..5ae566774959f 100644
--- a/clang/test/Driver/mips-features.c
+++ b/clang/test/Driver/mips-features.c
@@ -268,13 +268,13 @@
 // RUN: %clang -target mips-linux-gnu -### -c %s \
 // RUN:     -mno-xgot -mxgot 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-XGOT %s
-// CHECK-XGOT: "-mllvm" "-mxgot"
+// CHECK-XGOT: "-target-feature" "+xgot"
 //
 // -mno-xgot
 // RUN: %clang -target mips-linux-gnu -### -c %s \
 // RUN:     -mxgot -mno-xgot 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-NOXGOT %s
-// CHECK-NOXGOT-NOT: "-mllvm" "-mxgot"
+// CHECK-NOXGOT: "-target-feature" "-xgot"
 //
 // -mldc1-sdc1
 // RUN: %clang -target mips-linux-gnu -### -c %s \
diff --git a/clang/test/Driver/mips-integrated-as.s b/clang/test/Driver/mips-integrated-as.s
index 0e128770678fd..46ce5b6871f4e 100644
--- a/clang/test/Driver/mips-integrated-as.s
+++ b/clang/test/Driver/mips-integrated-as.s
@@ -293,3 +293,13 @@
 // IMG-SINGLEFLOAT-EXPLICIT-FPXX: "-target-feature" "+single-float"
 // IMG-SINGLEFLOAT-EXPLICIT-FPXX: "-target-feature" "+fpxx"
 // IMG-SINGLEFLOAT-EXPLICIT-FPXX: "-target-feature" "+nooddspreg"
+
+// RUN: %clang -target mips-linux-gnu -### -fintegrated-as -mxgot -c %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=XGOT %s
+// XGOT: -cc1as
+// XGOT: "-target-feature" "+xgot"
+
+// RUN: %clang -target mips-linux-gnu -### -fintegrated-as -mno-xgot -c %s 2>&1 | \
+// RUN:   FileCheck -check-prefix=NOXGOT %s
+// NOXGOT: -cc1as
+// NOXGOT: "-target-feature" "-xgot"
diff --git a/clang/test/Driver/mips-mti.cpp b/clang/test/Driver/mips-mti.cpp
index 147239cf3759e..94c3c5568e332 100644
--- a/clang/test/Driver/mips-mti.cpp
+++ b/clang/test/Driver/mips-mti.cpp
@@ -4,6 +4,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -mabi=32 \
 // RUN:   | FileCheck --check-prefix=EB-HARD-O32 %s
@@ -32,6 +33,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -mabi=n32 \
 // RUN:   | FileCheck --check-prefix=EB-HARD-N32 %s
@@ -60,6 +62,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips64-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -mabi=64 \
 // RUN:   | FileCheck --check-prefix=EB-HARD-N64 %s
@@ -88,6 +91,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -mabi=32 \
 // RUN:   | FileCheck --check-prefix=EL-HARD-O32 %s
@@ -116,6 +120,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -mabi=n32 \
 // RUN:   | FileCheck --check-prefix=EL-HARD-N32 %s
@@ -144,6 +149,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips64-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -mabi=64 \
 // RUN:   | FileCheck --check-prefix=EL-HARD-N64 %s
@@ -172,6 +178,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -msoft-float \
 // RUN:   | FileCheck --check-prefix=EB-SOFT %s
@@ -200,6 +207,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -msoft-float \
 // RUN:   | FileCheck --check-prefix=EL-SOFT %s
@@ -228,6 +236,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -muclibc \
 // RUN:   | FileCheck --check-prefix=EB-HARD-UCLIBC %s
@@ -256,6 +265,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -muclibc \
 // RUN:   | FileCheck --check-prefix=EL-HARD-UCLIBC %s
@@ -284,6 +294,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -mnan=2008 \
 // RUN:   | FileCheck --check-prefix=EB-HARD-NAN2008 %s
@@ -312,6 +323,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -mnan=2008 \
 // RUN:   | FileCheck --check-prefix=EL-HARD-NAN2008 %s
@@ -340,6 +352,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EB -mhard-float -muclibc -mnan=2008 \
 // RUN:   | FileCheck --check-prefix=EB-HARD-UCLIBC-NAN2008 %s
@@ -368,6 +381,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -muclibc -mnan=2008 \
 // RUN:   | FileCheck --check-prefix=EL-HARD-UCLIBC-NAN2008 %s
@@ -396,6 +410,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -msoft-float -mmicromips \
 // RUN:   | FileCheck --check-prefix=EL-SOFT-MICRO %s
@@ -424,6 +439,7 @@
 // RUN: %clang -no-canonical-prefixes %s -### -o %t.o 2>&1 \
 // RUN:        --target=mips-mti-linux-gnu \
 // RUN:        --gcc-toolchain=%S/Inputs/mips_mti_tree \
+// RUN:        --sysroot="" \
 // RUN:        -stdlib=libstdc++ \
 // RUN:        -EL -mhard-float -mmicromips -mnan=2008 \
 // RUN:   | FileCheck --check-prefix=EL-SOFT-MICRO-NAN2008 %s
diff --git a/clang/test/Driver/modules.cpp b/clang/test/Driver/modules.cpp
index 7c549c1300ff3..4f4e3a414002d 100644
--- a/clang/test/Driver/modules.cpp
+++ b/clang/test/Driver/modules.cpp
@@ -15,7 +15,7 @@
 // RUN: %clang -std=c++2a %t/module.pcm -S -o %t/module.pcm.o -v 2>&1 | FileCheck %s --check-prefix=CHECK-COMPILE
 //
 // CHECK-COMPILE: -cc1 {{.*}} {{-emit-obj|-S}}
-// CHECK-COMPILE-SAME: -o {{.*}}.{{pcm.o|s}}
+// CHECK-COMPILE-SAME: -o {{.*}}module{{2*}}.pcm.o
 // CHECK-COMPILE-SAME: -x pcm
 // CHECK-COMPILE-SAME: {{.*}}.pcm
 
diff --git a/clang/test/Driver/msp430-toolchain.c b/clang/test/Driver/msp430-toolchain.c
index 62ef1c0c1f150..f741491c94e8c 100644
--- a/clang/test/Driver/msp430-toolchain.c
+++ b/clang/test/Driver/msp430-toolchain.c
@@ -1,11 +1,11 @@
 // A basic clang -cc1 command-line, and simple environment check.
 
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=CC1 %s
 // CC1: clang{{.*}} "-cc1" "-triple" "msp430"
 
 // RUN: %clang %s -### -no-canonical-prefixes -target msp430 \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree 2>&1 \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430 %s
 
 // MSP430: "{{.*}}Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}bin{{/|\\\\}}msp430-elf-ld"
@@ -18,7 +18,7 @@
 // MSP430: "{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}msp430-elf{{/|\\\\}}lib/430{{/|\\\\}}crtn.o"
 
 // RUN: %clang %s -### -no-canonical-prefixes -target msp430 -nodefaultlibs \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree 2>&1 \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-NO-DFT-LIB %s
 
 // MSP430-NO-DFT-LIB: "{{.*}}Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}bin{{/|\\\\}}msp430-elf-ld"
@@ -31,7 +31,7 @@
 // MSP430-NO-DFT-LIB: "{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}msp430-elf{{/|\\\\}}lib/430{{/|\\\\}}crtn.o"
 
 // RUN: %clang %s -### -no-canonical-prefixes -target msp430 -nostartfiles \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree 2>&1 \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-NO-START %s
 
 // MSP430-NO-START: "{{.*}}Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}bin{{/|\\\\}}msp430-elf-ld"
@@ -40,7 +40,7 @@
 // MSP430-NO-START: "--start-group" "-lmul_none" "-lgcc" "-lc" "-lcrt" "-lnosys" "--end-group"
 
 // RUN: %clang %s -### -no-canonical-prefixes -target msp430 -nostdlib \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree 2>&1 \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_msp430_tree --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-NO-STD-LIB %s
 
 // MSP430-NO-STD-LIB: "{{.*}}Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}bin{{/|\\\\}}msp430-elf-ld"
@@ -48,31 +48,31 @@
 // MSP430-NO-STD-LIB: "-L{{.*}}/Inputs/basic_msp430_tree/lib/gcc/msp430-elf/7.3.1/../../..{{/|\\\\}}..{{/|\\\\}}msp430-elf{{/|\\\\}}lib/430"
 // MSP430-NO-STD-LIB: "--start-group" "-lmul_none" "-lgcc" "--end-group"
 
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f147 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f147 --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-16BIT %s
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f147 -mhwmult=auto 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f147 -mhwmult=auto --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-16BIT %s
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=16bit 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=16bit --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-16BIT %s
 
 // MSP430-HWMult-16BIT: "--start-group" "-lmul_16"
 
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f4783 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f4783 --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-32BIT %s
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f4783 -mhwmult=auto 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mmcu=msp430f4783 -mhwmult=auto --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-32BIT %s
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=32bit 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=32bit --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-32BIT %s
 
 // MSP430-HWMult-32BIT: "--start-group" "-lmul_32"
 
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=f5series 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=f5series --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-F5 %s
 // MSP430-HWMult-F5: "--start-group" "-lmul_f5"
 
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=none 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=none --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-NONE %s
-// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=none -mmcu=msp430f4783 2>&1 \
+// RUN: %clang %s -### -no-canonical-prefixes -target msp430 -mhwmult=none -mmcu=msp430f4783 --sysroot="" 2>&1 \
 // RUN:   | FileCheck -check-prefix=MSP430-HWMult-NONE %s
 
 // MSP430-HWMult-NONE: "--start-group" "-lmul_none"
diff --git a/clang/test/Driver/openmp-offload.c b/clang/test/Driver/openmp-offload.c
index 2f8992c594015..86121d2f75369 100644
--- a/clang/test/Driver/openmp-offload.c
+++ b/clang/test/Driver/openmp-offload.c
@@ -280,14 +280,6 @@
 // CHK-LKS:     [[T2BIN]]
 // CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.img_end.x86_64-pc-linux-gnu = .);
 // CHK-LKS:   }
-// CHK-LKS:   .omp_offloading.entries :
-// CHK-LKS:   ALIGN(0x10)
-// CHK-LKS:   SUBALIGN(0x01)
-// CHK-LKS:   {
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.entries_begin = .);
-// CHK-LKS:     *(.omp_offloading.entries)
-// CHK-LKS:     PROVIDE_HIDDEN(.omp_offloading.entries_end = .);
-// CHK-LKS:   }
 // CHK-LKS: }
 // CHK-LKS: INSERT BEFORE .data
 
diff --git a/clang/test/Driver/split-lto-unit.c b/clang/test/Driver/split-lto-unit.c
index fab5790c26b95..d2ed253ca201a 100644
--- a/clang/test/Driver/split-lto-unit.c
+++ b/clang/test/Driver/split-lto-unit.c
@@ -6,5 +6,5 @@
 
 // UNIT: "-fsplit-lto-unit"
 // NOUNIT-NOT: "-fsplit-lto-unit"
-// ERROR1: error: invalid argument '-fno-split-lto-unit' not allowed with '-fwhole-program-vtables'
+// ERROR1-NOT: error: invalid argument
 // ERROR2: error: invalid argument '-fno-split-lto-unit' not allowed with '-fsanitize=cfi'
diff --git a/clang/test/Driver/systemz-march.c b/clang/test/Driver/systemz-march.c
index 6b751523c34f9..f07a2c50131ad 100644
--- a/clang/test/Driver/systemz-march.c
+++ b/clang/test/Driver/systemz-march.c
@@ -11,6 +11,7 @@
 // RUN: %clang -target s390x -### -S -emit-llvm -march=arch11 %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH11 %s
 // RUN: %clang -target s390x -### -S -emit-llvm -march=z14 %s 2>&1 | FileCheck --check-prefix=CHECK-Z14 %s
 // RUN: %clang -target s390x -### -S -emit-llvm -march=arch12 %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH12 %s
+// RUN: %clang -target s390x -### -S -emit-llvm -march=z15 %s 2>&1 | FileCheck --check-prefix=CHECK-Z15 %s
 // RUN: %clang -target s390x -### -S -emit-llvm -march=arch13 %s 2>&1 | FileCheck --check-prefix=CHECK-ARCH13 %s
 
 // CHECK-Z9: error: unknown target CPU 'z9'
@@ -24,6 +25,7 @@
 // CHECK-ARCH11: "-target-cpu" "arch11"
 // CHECK-Z14: "-target-cpu" "z14"
 // CHECK-ARCH12: "-target-cpu" "arch12"
+// CHECK-Z15: "-target-cpu" "z15"
 // CHECK-ARCH13: "-target-cpu" "arch13"
 
 int x;
diff --git a/clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h b/clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h
index b074bd1a4468c..75e4c963fd40a 100644
--- a/clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h
+++ b/clang/test/Frontend/Inputs/NextIncludes/rewrite-includes9.h
@@ -1 +1 @@
-included_line9
+int included_line9;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes1.h b/clang/test/Frontend/Inputs/rewrite-includes1.h
index 1b6c80d5f4358..3312f9ff9025e 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes1.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes1.h
@@ -1,3 +1,3 @@
 #pragma clang system_header
-included_line1
+int included_line1;
 #include "rewrite-includes2.h"
diff --git a/clang/test/Frontend/Inputs/rewrite-includes2.h b/clang/test/Frontend/Inputs/rewrite-includes2.h
index 1114e51cc1351..d1097c8e17a94 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes2.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes2.h
@@ -1 +1 @@
-included_line2
+int included_line2;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes3.h b/clang/test/Frontend/Inputs/rewrite-includes3.h
index 3757bc8b3416d..2abf780590a44 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes3.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes3.h
@@ -1 +1 @@
-included_line3
+unsigned int included_line3 = -10;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes4.h b/clang/test/Frontend/Inputs/rewrite-includes4.h
index b4e25d26f62df..7a442013f7227 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes4.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes4.h
@@ -1 +1 @@
-included_line4
+int included_line4;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes5.h b/clang/test/Frontend/Inputs/rewrite-includes5.h
index 934bf413e50ff..0de12113cb647 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes5.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes5.h
@@ -1 +1 @@
-included_line5
+int included_line5;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes6.h b/clang/test/Frontend/Inputs/rewrite-includes6.h
index c18e5013affc0..5d891446234e6 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes6.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes6.h
@@ -1,2 +1,2 @@
 #pragma once
-included_line6
+int included_line6;
diff --git a/clang/test/Frontend/Inputs/rewrite-includes7.h b/clang/test/Frontend/Inputs/rewrite-includes7.h
index da00d4fb97d82..221bab82df304 100644
--- a/clang/test/Frontend/Inputs/rewrite-includes7.h
+++ b/clang/test/Frontend/Inputs/rewrite-includes7.h
@@ -1,4 +1,4 @@
 #ifndef REWRITE_INCLUDES_7
 #define REWRITE_INCLUDES_7
-included_line7
+int included_line7;
 #endif
diff --git a/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/lib/.keep b/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/lib/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/include/c++/.keep b/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/include/c++/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/lib/gcc/.keep b/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/lib/gcc/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/local/include/.keep b/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/local/include/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/local/lib/.keep b/clang/test/Frontend/Inputs/sysroot_x86_64_cross_linux_tree/usr/local/lib/.keep
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/clang/test/Frontend/cc1-return-codes.c b/clang/test/Frontend/cc1-return-codes.c
index da329b9342654..fde1b4adf2ad5 100644
--- a/clang/test/Frontend/cc1-return-codes.c
+++ b/clang/test/Frontend/cc1-return-codes.c
@@ -1,4 +1,4 @@
 // cc1 immediate arguments (arguments which displays information and exits)
 // shall exit indicating success (return code 0)
-// RUN: %clang -cc1 -help
-// RUN: %clang -cc1 -version
+// RUN: %clang_cc1 -help
+// RUN: %clang_cc1 -version
diff --git a/clang/test/Frontend/nostdlib-for-asmpp.s b/clang/test/Frontend/nostdlib-for-asmpp.s
index 330fee32a361d..c1be5a2777907 100644
--- a/clang/test/Frontend/nostdlib-for-asmpp.s
+++ b/clang/test/Frontend/nostdlib-for-asmpp.s
@@ -1,4 +1,4 @@
-// RUN: %clang -cc1 -x assembler-with-cpp -triple arm64-apple-ios6.0.0 -isysroot %S/doesnotexist -std=c++11 -v %s 2>&1 | FileCheck %s
+// RUN: %clang_cc1 -x assembler-with-cpp -triple arm64-apple-ios6.0.0 -isysroot %S/doesnotexist -std=c++11 -v %s 2>&1 | FileCheck %s
 // The C++ stdlib path should not be included for an assembly source.
 
 // CHECK-NOT: usr/include/c++/
diff --git a/clang/test/Frontend/rewrite-includes-cli-include.c b/clang/test/Frontend/rewrite-includes-cli-include.c
index ba96039bdd5ad..d63f966f79ca6 100644
--- a/clang/test/Frontend/rewrite-includes-cli-include.c
+++ b/clang/test/Frontend/rewrite-includes-cli-include.c
@@ -2,7 +2,7 @@
 main_file_line
 // CHECK: {{^}}# 1 "<built-in>"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes2.h" 1{{$}}
-// CHECK-NEXT: {{^}}included_line2{{$}}
+// CHECK-NEXT: {{^}}int included_line2;{{$}}
 // CHECK-NEXT: {{^}}# 1 "<built-in>" 2{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*}}rewrite-includes-cli-include.c"{{$}}
 // CHECK-NEXT: FileCheck
diff --git a/clang/test/Frontend/rewrite-includes-conditions.c b/clang/test/Frontend/rewrite-includes-conditions.c
new file mode 100644
index 0000000000000..37a55f31de9ec
--- /dev/null
+++ b/clang/test/Frontend/rewrite-includes-conditions.c
@@ -0,0 +1,113 @@
+// RUN: %clang_cc1 -E -frewrite-includes -I %S/Inputs %s -o - | FileCheck -strict-whitespace %s
+// RUN: %clang_cc1 -E -frewrite-includes -I %S/Inputs %s -o - | %clang_cc1 -Wall -Wextra -Wconversion -x c -fsyntax-only 2>&1 | FileCheck -check-prefix=COMPILE --implicit-check-not warning: %s
+
+#define value1 1
+#if value1
+int line1;
+#else
+int line2;
+#endif
+
+#define value2 2
+
+#if value1 == value2
+int line3;
+#elif value1 > value2
+int line4;
+#elif value1 < value2
+int line5;
+#else
+int line6;
+#endif
+
+#if __has_include(<rewrite-includes3.h>)
+#include <rewrite-includes3.h>
+#endif
+
+#define HAS_INCLUDE(x) __has_include(x)
+
+#if HAS_INCLUDE(<rewrite-includes1.h>)
+#endif
+
+/*
+#if value1
+commented out
+*/
+
+#if value1 < value2 \
+|| value1 != value2
+int line7;
+#endif
+
+#if value1 /*
+*/
+#endif
+
+static int unused;
+
+// ENDCOMPARE
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if value1
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 6 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if value1 == value2
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 0 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 14 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 0
+// CHECK-NEXT: #elif value1 > value2
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #elif 0 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 16 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 0
+// CHECK-NEXT: #elif value1 < value2
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #elif 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 18 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if __has_include(<rewrite-includes3.h>)
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 24 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if HAS_INCLUDE(<rewrite-includes1.h>)
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 30 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if value1 < value2 \
+// CHECK-NEXT: || value1 != value2
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 39 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: #if 0 /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if value1 /*
+// CHECK-NEXT: */
+// CHECK-NEXT: #endif
+// CHECK-NEXT: #endif /* disabled by -frewrite-includes */
+// CHECK-NEXT: #if 1 /* evaluated by -frewrite-includes */
+// CHECK-NEXT: # 44 "{{.*}}rewrite-includes-conditions.c"
+
+// CHECK: {{^}}// ENDCOMPARE{{$}}
+
+// COMPILE: Inputs{{[/\\]}}rewrite-includes3.h:1:31: warning: implicit conversion changes signedness:
+// COMPILE: rewrite-includes-conditions.c:46:12: warning: unused variable 'unused'
diff --git a/clang/test/Frontend/rewrite-includes-warnings.c b/clang/test/Frontend/rewrite-includes-warnings.c
index 1fb98db30147e..d955f86f826a8 100644
--- a/clang/test/Frontend/rewrite-includes-warnings.c
+++ b/clang/test/Frontend/rewrite-includes-warnings.c
@@ -1,4 +1,7 @@
-// RUN: %clang_cc1 -verify -Wall -Wextra -E -frewrite-includes %s
+// RUN: %clang_cc1 -verify -Wall -Wextra -Wunused-macros -E -frewrite-includes %s
 // expected-no-diagnostics
 
 #pragma GCC visibility push (default)
+
+#define USED_MACRO 1
+int test() { return USED_MACRO; }
diff --git a/clang/test/Frontend/rewrite-includes.c b/clang/test/Frontend/rewrite-includes.c
index 630e761cab2ff..15254931250c1 100644
--- a/clang/test/Frontend/rewrite-includes.c
+++ b/clang/test/Frontend/rewrite-includes.c
@@ -1,8 +1,9 @@
-// RUN: not %clang_cc1 -verify -E -frewrite-includes -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -strict-whitespace %s
-// RUN: not %clang_cc1 -verify -E -frewrite-includes -P -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s
+// RUN: %clang_cc1 -E -frewrite-includes -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -strict-whitespace %s
+// RUN: %clang_cc1 -E -frewrite-includes -P -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | FileCheck -check-prefix=CHECKNL -strict-whitespace %s
+// RUN: %clang_cc1 -E -frewrite-includes -DFIRST -I %S/Inputs -I %S/Inputs/NextIncludes %s -o - | %clang_cc1 -Wall -Wextra -Wconversion -DFIRST -x c -fsyntax-only 2>&1 | FileCheck -check-prefix=COMPILE --implicit-check-not warning: %s
 // STARTCOMPARE
 #define A(a,b) a ## b
-A(1,2)
+A(in,t) a;
 #include "rewrite-includes1.h"
 #ifdef FIRST
 #define HEADER "rewrite-includes3.h"
@@ -21,142 +22,164 @@ A(1,2)
 #include "rewrite-includes7.h"
 #include "rewrite-includes8.h"
 #include "rewrite-includes9.h"
+static int unused;
 // ENDCOMPARE
 // CHECK: {{^}}# 1 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK: {{^}}// STARTCOMPARE{{$}}
 // CHECK-NEXT: {{^}}#define A(a,b) a ## b{{$}}
-// CHECK-NEXT: {{^}}A(1,2){{$}}
+// CHECK-NEXT: {{^}}A(in,t) a;{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 6 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 7 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes1.h" 1{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#pragma clang system_header{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes1.h" 3{{$}}
-// CHECK-NEXT: {{^}}included_line1{{$}}
+// CHECK-NEXT: {{^}}int included_line1;{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes2.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 3 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes1.h" 3{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes2.h" 1 3{{$}}
-// CHECK-NEXT: {{^}}included_line2{{$}}
+// CHECK-NEXT: {{^}}int included_line2;{{$}}
 // CHECK-NEXT: {{^}}# 4 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes1.h" 2 3{{$}}
-// CHECK-NEXT: {{^}}# 7 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}# 8 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}#ifdef FIRST{{$}}
 // CHECK-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include HEADER{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 9 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 10 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes3.h" 1{{$}}
-// CHECK-NEXT: {{^}}included_line3{{$}}
-// CHECK-NEXT: {{^}}# 10 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}unsigned int included_line3 = -10;{{$}}
+// CHECK-NEXT: {{^}}# 11 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}#else{{$}}
-// CHECK-NEXT: {{^}}# 11 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 12 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes4.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 11 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 12 "{{.*}}rewrite-includes.c"{{$}}
-// CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 13 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}# 14 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}  // indented{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}}
 // CHECK-NEXT: {{^}} {{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 15 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 16 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes5.h" 1{{$}}
-// CHECK-NEXT: {{^}}included_line5{{$}}
-// CHECK-NEXT: {{^}}# 16 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}int included_line5;{{$}}
+// CHECK-NEXT: {{^}}# 17 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 16 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 17 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes6.h" 1{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#pragma once{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes6.h"{{$}}
-// CHECK-NEXT: {{^}}included_line6{{$}}
-// CHECK-NEXT: {{^}}# 17 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}int included_line6;{{$}}
+// CHECK-NEXT: {{^}}# 18 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}} {{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}}
 // CHECK-NEXT: {{^}}                                  continues */{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 19 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 20 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 20 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes7.h" 1{{$}}
 // CHECK-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}}
 // CHECK-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}}
-// CHECK-NEXT: {{^}}included_line7{{$}}
+// CHECK-NEXT: {{^}}int included_line7;{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 5 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes7.h"{{$}}
-// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}# 22 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 21 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 22 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 22 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h" 1{{$}}
-// CHECK-NEXT: {{^}}#if (0)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
-// CHECK-NEXT: {{^}}#elif (0)/*__has_include(<rewrite-includes8.hfail>)*/{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if __has_include_next(<rewrite-includes8.h>){{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* evaluated by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if 0{{$}}
+// CHECK-NEXT: {{^}}#elif __has_include(<rewrite-includes8.hfail>){{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#elif 0 /* evaluated by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 3 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 4 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
-// CHECK-NEXT: {{^}}#if !(1)/*__has_include("rewrite-includes8.h")*/{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if !__has_include("rewrite-includes8.h"){{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* evaluated by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 5 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 6 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes8.h"{{$}}
-// CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}# 24 "{{.*}}rewrite-includes.c" 2{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include "rewrite-includes9.h"{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECK-NEXT: {{^}}# 23 "{{.*}}rewrite-includes.c"{{$}}
+// CHECK-NEXT: {{^}}# 24 "{{.*}}rewrite-includes.c"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h" 1{{$}}
-// CHECK-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes9.h>)*/{{$}}
+// CHECK-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if __has_include_next(<rewrite-includes9.h>){{$}}
+// CHECK-NEXT: {{^}}#endif{{$}}
+// CHECK-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}#if 1 /* evaluated by -frewrite-includes */{{$}}
+// CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h"{{$}}
 // CHECK-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}#include_next <rewrite-includes9.h>{{$}}
 // CHECK-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECK-NEXT: {{^}}# 2 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h"{{$}}
 // CHECK-NEXT: {{^}}# 1 "{{.*[/\\]Inputs(/|\\\\)NextIncludes(/|\\\\)}}rewrite-includes9.h" 1{{$}}
-// CHECK-NEXT: {{^}}included_line9{{$}}
+// CHECK-NEXT: {{^}}int included_line9;{{$}}
 // CHECK-NEXT: {{^}}# 3 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h" 2{{$}}
 // CHECK-NEXT: {{^}}#endif{{$}}
 // CHECK-NEXT: {{^}}# 4 "{{.*[/\\]Inputs(/|\\\\)}}rewrite-includes9.h"{{$}}
-// CHECK-NEXT: {{^}}# 24 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}# 25 "{{.*}}rewrite-includes.c" 2{{$}}
+// CHECK-NEXT: {{^}}static int unused;{{$}}
 // CHECK-NEXT: {{^}}// ENDCOMPARE{{$}}
 
 // CHECKNL: {{^}}// STARTCOMPARE{{$}}
 // CHECKNL-NEXT: {{^}}#define A(a,b) a ## b{{$}}
-// CHECKNL-NEXT: {{^}}A(1,2){{$}}
+// CHECKNL-NEXT: {{^}}A(in,t) a;{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes1.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#pragma clang system_header{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line1{{$}}
+// CHECKNL-NEXT: {{^}}int included_line1;{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes2.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line2{{$}}
+// CHECKNL-NEXT: {{^}}int included_line2;{{$}}
 // CHECKNL-NEXT: {{^}}#ifdef FIRST{{$}}
 // CHECKNL-NEXT: {{^}}#define HEADER "rewrite-includes3.h"{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include HEADER{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line3{{$}}
+// CHECKNL-NEXT: {{^}}unsigned int included_line3 = -10;{{$}}
 // CHECKNL-NEXT: {{^}}#else{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes4.h"{{$}}
@@ -167,14 +190,14 @@ A(1,2)
 // CHECKNL-NEXT: {{^}}#/**/include /**/ "rewrite-includes5.h" /**/ {{\\}}{{$}}
 // CHECKNL-NEXT: {{^}} {{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line5{{$}}
+// CHECKNL-NEXT: {{^}}int included_line5;{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" // comment{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#pragma once{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line6{{$}}
+// CHECKNL-NEXT: {{^}}int included_line6;{{$}}
 // CHECKNL-NEXT: {{^}} {{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes6.h" /* comment{{$}}
@@ -185,7 +208,7 @@ A(1,2)
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#ifndef REWRITE_INCLUDES_7{{$}}
 // CHECKNL-NEXT: {{^}}#define REWRITE_INCLUDES_7{{$}}
-// CHECKNL-NEXT: {{^}}included_line7{{$}}
+// CHECKNL-NEXT: {{^}}int included_line7;{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes7.h"{{$}}
@@ -193,18 +216,39 @@ A(1,2)
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes8.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}#if (0)/*__has_include_next(<rewrite-includes8.h>)*/{{$}}
-// CHECKNL-NEXT: {{^}}#elif (0)/*__has_include(<rewrite-includes8.hfail>)*/{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if __has_include_next(<rewrite-includes8.h>){{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* evaluated by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0{{$}}
+// CHECKNL-NEXT: {{^}}#elif __has_include(<rewrite-includes8.hfail>){{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#elif 0 /* evaluated by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
-// CHECKNL-NEXT: {{^}}#if !(1)/*__has_include("rewrite-includes8.h")*/{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if !__has_include("rewrite-includes8.h"){{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* evaluated by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include "rewrite-includes9.h"{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}#if (1)/*__has_include_next(<rewrite-includes9.h>)*/{{$}}
+// CHECKNL-NEXT: {{^}}#if 0 /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if __has_include_next(<rewrite-includes9.h>){{$}}
+// CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}#endif /* disabled by -frewrite-includes */{{$}}
+// CHECKNL-NEXT: {{^}}#if 1 /* evaluated by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#if 0 /* expanded by -frewrite-includes */{{$}}
 // CHECKNL-NEXT: {{^}}#include_next <rewrite-includes9.h>{{$}}
 // CHECKNL-NEXT: {{^}}#endif /* expanded by -frewrite-includes */{{$}}
-// CHECKNL-NEXT: {{^}}included_line9{{$}}
+// CHECKNL-NEXT: {{^}}int included_line9;{{$}}
 // CHECKNL-NEXT: {{^}}#endif{{$}}
+// CHECKNL-NEXT: {{^}}static int unused;{{$}}
 // CHECKNL-NEXT: {{^}}// ENDCOMPARE{{$}}
+
+// COMPILE: Inputs{{[/\\]}}rewrite-includes3.h:1:31: warning: implicit conversion changes signedness:
+// COMPILE: rewrite-includes.c:25:12: warning: unused variable 'unused'
diff --git a/clang/test/Frontend/stdin-input.c b/clang/test/Frontend/stdin-input.c
new file mode 100644
index 0000000000000..7b15482b8c513
--- /dev/null
+++ b/clang/test/Frontend/stdin-input.c
@@ -0,0 +1,7 @@
+// RUN: cat %s | %clang -emit-llvm -g -S \
+// RUN: -Xclang -main-file-name -Xclang test/foo.c -x c - -o - | FileCheck %s
+// CHECK: ; ModuleID = 'test/foo.c'
+// CHECK: source_filename = "test/foo.c"
+// CHECK: !DIFile(filename: "test/foo.c"
+
+int main() {}
diff --git a/clang/test/Frontend/warning-poison-system-directories.c b/clang/test/Frontend/warning-poison-system-directories.c
new file mode 100644
index 0000000000000..ba4c0e1990ed3
--- /dev/null
+++ b/clang/test/Frontend/warning-poison-system-directories.c
@@ -0,0 +1,29 @@
+// REQUIRES: x86-registered-target
+
+// System directory and sysroot option causes warning.
+// RUN: %clang -Wpoison-system-directories -target x86_64 -I/usr/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.1.stderr
+// RUN: FileCheck -check-prefix=WARN < %t.1.stderr %s
+// RUN: %clang -Wpoison-system-directories -target x86_64 -cxx-isystem/usr/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.1.stderr
+// RUN: FileCheck -check-prefix=WARN < %t.1.stderr %s
+// RUN: %clang -Wpoison-system-directories -target x86_64 -iquote/usr/local/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.1.stderr
+// RUN: FileCheck -check-prefix=WARN < %t.1.stderr %s
+// RUN: %clang -Wpoison-system-directories -target x86_64 -isystem/usr/local/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.1.stderr
+// RUN: FileCheck -check-prefix=WARN < %t.1.stderr %s
+
+// Missing target but included sysroot still causes the warning.
+// RUN: %clang -Wpoison-system-directories -I/usr/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.2.stderr
+// RUN: FileCheck -check-prefix=WARN < %t.2.stderr %s
+
+// With -Werror the warning causes the failure.
+// RUN: not %clang -Werror=poison-system-directories -target x86_64 -I/usr/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s 2> %t.3.stderr
+// RUN: FileCheck -check-prefix=ERROR < %t.3.stderr %s
+
+// Cros target without sysroot causes no warning.
+// RUN: %clang -Wpoison-system-directories -Werror -target x86_64 -I/usr/include -c -o - %s
+
+// By default the warning is off.
+// RUN: %clang -Werror -target x86_64 -I/usr/include --sysroot %S/Inputs/sysroot_x86_64_cross_linux_tree -c -o - %s
+
+// WARN: warning: include location {{[^ ]+}} is unsafe for cross-compilation [-Wpoison-system-directories]
+
+// ERROR: error: include location {{[^ ]+}} is unsafe for cross-compilation [-Werror,-Wpoison-system-directories]
diff --git a/clang/test/Headers/arm-fp16-header.c b/clang/test/Headers/arm-fp16-header.c
index 781113debda0b..85068bb892779 100644
--- a/clang/test/Headers/arm-fp16-header.c
+++ b/clang/test/Headers/arm-fp16-header.c
@@ -6,14 +6,14 @@
 // RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c99 -xc %s
 // RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c11 -xc %s
 
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
 
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
 
 #include <arm_fp16.h>
diff --git a/clang/test/Headers/arm-neon-header.c b/clang/test/Headers/arm-neon-header.c
index 1da5ed052c329..e8ab33d7702c9 100644
--- a/clang/test/Headers/arm-neon-header.c
+++ b/clang/test/Headers/arm-neon-header.c
@@ -10,14 +10,14 @@
 // RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c99 -xc %s
 // RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c11 -xc %s
 
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
 
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
-// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++98 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++11 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++14 -xc++ %s
+// RUN: %clang -fsyntax-only -Wall -Werror -ffreestanding -nostdinc++ --target=aarch64_be-none-eabi -march=armv8.2-a+fp16 -std=c++17 -xc++ %s
 
 #include <arm_neon.h>
diff --git a/clang/test/Headers/xmmintrin.c b/clang/test/Headers/xmmintrin.c
index c617504b85da5..9b8993e6bb679 100644
--- a/clang/test/Headers/xmmintrin.c
+++ b/clang/test/Headers/xmmintrin.c
@@ -22,7 +22,7 @@ __m64 test_mm_cvtps_pi16(__m128 a) {
 
 // Make sure that including <xmmintrin.h> also makes <emmintrin.h>'s content available.
 // This is an ugly hack for GCC compatibility.
-__m128 test_xmmintrin_provides_emmintrin(__m128d __a, __m128d __b) {
+__m128d test_xmmintrin_provides_emmintrin(__m128d __a, __m128d __b) {
   return _mm_add_sd(__a, __b);
 }
 
diff --git a/clang/test/Import/cxx-anon-namespace/test.cpp b/clang/test/Import/cxx-anon-namespace/test.cpp
index 0cbf08c181eea..7331a6bc4315e 100644
--- a/clang/test/Import/cxx-anon-namespace/test.cpp
+++ b/clang/test/Import/cxx-anon-namespace/test.cpp
@@ -2,9 +2,13 @@
 
 // The implicit UsingDirectiveDecls for the anonymous namespaces are created by the Sema.
 
-// CHECK: NamespaceDecl
+// There might be another builtin namespace before our first namespace, so we can't
+// just look for NamespaceDecl. Instead look for the first line of F.cpp (which only
+// contains the namespace we are looking for but no other decl).
+// CHECK: F.cpp:1:1
 // The nested anonymous namespace.
 // CHECK-NEXT: NamespaceDecl
+// CHECK-SAME: <invalid sloc>
 // CHECK: FunctionDecl
 // CHECK-SAME: func4
 // CHECK-NEXT: CompoundStmt
diff --git a/clang/test/Index/Core/index-source.cpp b/clang/test/Index/Core/index-source.cpp
index b57b156072828..f159b1c884304 100644
--- a/clang/test/Index/Core/index-source.cpp
+++ b/clang/test/Index/Core/index-source.cpp
@@ -461,12 +461,12 @@ struct StaticAssertRef {
 };
 
 static_assert(StaticAssertRef::constVar, "index static asserts");
-// CHECK: [[@LINE-1]]:32 | static-property/C++ | constVar | c:@S@StaticAssertRef@constVar | __ZN15StaticAssertRef8constVarE | Ref | rel: 0
+// CHECK: [[@LINE-1]]:32 | static-property/C++ | constVar | c:@S@StaticAssertRef@constVar | __ZN15StaticAssertRef8constVarE | Ref,Read | rel: 0
 // CHECK: [[@LINE-2]]:15 | struct/C++ | StaticAssertRef | c:@S@StaticAssertRef | <no-cgname> | Ref | rel: 0
 
 void staticAssertInFn() {
   static_assert(StaticAssertRef::constVar, "index static asserts");
-// CHECK: [[@LINE-1]]:34 | static-property/C++ | constVar | c:@S@StaticAssertRef@constVar | __ZN15StaticAssertRef8constVarE | Ref,RelCont | rel: 1
+// CHECK: [[@LINE-1]]:34 | static-property/C++ | constVar | c:@S@StaticAssertRef@constVar | __ZN15StaticAssertRef8constVarE | Ref,Read,RelCont | rel: 1
 // CHECK-NEXT: RelCont | staticAssertInFn | c:@F@staticAssertInFn#
 // CHECK: [[@LINE-3]]:17 | struct/C++ | StaticAssertRef | c:@S@StaticAssertRef | <no-cgname> | Ref,RelCont | rel: 1
 // CHECK-NEXT: RelCont | staticAssertInFn | c:@F@staticAssertInFn#
diff --git a/clang/test/Index/crash-recovery-modules.m b/clang/test/Index/crash-recovery-modules.m
index 85b39cf58df74..82979f794bea8 100644
--- a/clang/test/Index/crash-recovery-modules.m
+++ b/clang/test/Index/crash-recovery-modules.m
@@ -31,5 +31,5 @@ void test() {
 // ...and with module building successful.
 // RUN: env CINDEXTEST_FAILONERROR=1 not c-index-test -test-load-source all -fmodules -fmodules-cache-path=%t -Xclang -fdisable-module-hash -I %S/Inputs/Headers -DLIBCLANG_CRASH %s > /dev/null 2> %t.err
 // RUN: FileCheck < %t.err -check-prefix=CHECK-LIBCLANG-CRASH %s
-// CHECK-LIBCLANG-CRASH: libclang: crash detected during parsing
-// CHECK-LIBCLANG-CRASH: Unable to load translation unit!
+// CHECK-LIBCLANG-CRASH-DAG: libclang: crash detected during parsing
+// CHECK-LIBCLANG-CRASH-DAG: Unable to load translation unit!
diff --git a/clang/test/Index/index-templates.cpp b/clang/test/Index/index-templates.cpp
index 424a638ffbb8f..b06ea7fed7a84 100644
--- a/clang/test/Index/index-templates.cpp
+++ b/clang/test/Index/index-templates.cpp
@@ -155,7 +155,7 @@ using alias = T;
 // CHECK-LOAD: index-templates.cpp:36:44: DeclRefExpr=OneDimension:35:16 Extent=[36:44 - 36:56]
 // CHECK-LOAD: index-templates.cpp:40:8: ClassTemplate=storage:40:8 (Definition) Extent=[39:1 - 40:19]
 // CHECK-LOAD: index-templates.cpp:39:45: TemplateTemplateParameter=DataStructure:39:45 (Definition) Extent=[39:10 - 39:66]
-// CHECK-LOAD: index-templates.cpp:39:19: TemplateTypeParameter=:39:19 (Definition) Extent=[39:19 - 39:27]
+// CHECK-LOAD: index-templates.cpp:39:27: TemplateTypeParameter=:39:27 (Definition) Extent=[39:19 - 39:27]
 // CHECK-LOAD: index-templates.cpp:39:37: NonTypeTemplateParameter=:39:37 (Definition) Extent=[39:29 - 39:37]
 // CHECK-LOAD: index-templates.cpp:39:61: TemplateRef=array:37:8 Extent=[39:61 - 39:66]
 // CHECK-LOAD: index-templates.cpp:42:18: TypedefDecl=Unsigned:42:18 (Definition) Extent=[42:1 - 42:26]
diff --git a/clang/test/Lexer/cross-windows-on-linux-default.cpp b/clang/test/Lexer/cross-windows-on-linux-default.cpp
index 520b419a71003..bcd4999a663a1 100644
--- a/clang/test/Lexer/cross-windows-on-linux-default.cpp
+++ b/clang/test/Lexer/cross-windows-on-linux-default.cpp
@@ -1,4 +1,4 @@
-// RUN: not %clang_cc1 -fsyntax-only -fms-compatibility -triple i686-win32 %s 2>&1 \
+// RUN: not %clang_cc1 -fsyntax-only -fms-extensions -triple i686-win32 %s 2>&1 \
 // RUN:   | FileCheck %s
 
 #include "Inputs\success.h"
diff --git a/clang/test/Lexer/cross-windows-on-linux.cpp b/clang/test/Lexer/cross-windows-on-linux.cpp
index c6dcbca540561..3932ffcb33c82 100644
--- a/clang/test/Lexer/cross-windows-on-linux.cpp
+++ b/clang/test/Lexer/cross-windows-on-linux.cpp
@@ -6,10 +6,8 @@
 // CHECK: #include "Inputs\success.h"
 // CHECK:          ^
 
-// expected to fail on windows as the inclusion would succeed and the
-// compilation will fail due to the '#error success'.
-// XFAIL: windows-msvc
-
-// This test may or may not fail since 'Inputs\success.h' is passed
-// to Win32 APIs on Windows.
-// REQUIRES: disabled
+// This test is really checking that we *don't* replace backslashes with slashes
+// on non-Windows unless -fms-extensions is passed. It won't fail in this way on
+// Windows because the filesystem will interpret the backslash as a directory
+// separator.
+// UNSUPPORTED: system-windows
diff --git a/clang/test/Lexer/cxx-features.cpp b/clang/test/Lexer/cxx-features.cpp
index cda6f888cbce5..868ec24a1d9f7 100644
--- a/clang/test/Lexer/cxx-features.cpp
+++ b/clang/test/Lexer/cxx-features.cpp
@@ -34,6 +34,10 @@
 #error "wrong value for __cpp_char8_t"
 #endif
 
+#if check(constexpr_dynamic_alloc, 0, 0, 0, 0, 201907)
+#error "wrong value for __cpp_constexpr_dynamic_alloc"
+#endif
+
 #if check(constinit, 0, 0, 0, 0, 201907)
 #error "wrong value for __cpp_constinit"
 #endif
diff --git a/clang/test/Lexer/has_feature_leak_sanitizer.cpp b/clang/test/Lexer/has_feature_leak_sanitizer.cpp
new file mode 100644
index 0000000000000..00ca96f4ba73f
--- /dev/null
+++ b/clang/test/Lexer/has_feature_leak_sanitizer.cpp
@@ -0,0 +1,11 @@
+// RUN: %clang_cc1 -E -fsanitize=leak %s -o - | FileCheck --check-prefix=CHECK-LSAN %s
+// RUN: %clang_cc1 -E %s -o - | FileCheck --check-prefix=CHECK-NO-LSAN %s
+
+#if __has_feature(leak_sanitizer)
+int LeakSanitizerEnabled();
+#else
+int LeakSanitizerDisabled();
+#endif
+
+// CHECK-LSAN: LeakSanitizerEnabled
+// CHECK-NO-LSAN: LeakSanitizerDisabled
diff --git a/clang/test/Misc/diag-macro-backtrace2.c b/clang/test/Misc/diag-macro-backtrace2.c
index 64fc5f6307278..ca13dca074d83 100644
--- a/clang/test/Misc/diag-macro-backtrace2.c
+++ b/clang/test/Misc/diag-macro-backtrace2.c
@@ -1,4 +1,4 @@
-// RUN: not %clang -cc1 -fsyntax-only %s 2>&1 | FileCheck %s
+// RUN: not %clang_cc1 -fsyntax-only %s 2>&1 | FileCheck %s
 
 #define a b
 #define b c
diff --git a/clang/test/Misc/driver-verify.c b/clang/test/Misc/driver-verify.c
index fa31f820b1087..f858a3db85cb5 100644
--- a/clang/test/Misc/driver-verify.c
+++ b/clang/test/Misc/driver-verify.c
@@ -1,5 +1,5 @@
 // RUN: not %clang -verify %s 2>&1 | FileCheck %s
-// RUN: %clang -cc1 -verify %s
+// RUN: %clang_cc1 -verify %s
 // expected-no-diagnostics
 
 // Test that -verify is strictly rejected as unknown by the driver.
diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c
index e7dc6ed478b6d..4f8158e50229e 100644
--- a/clang/test/Misc/target-invalid-cpu-note.c
+++ b/clang/test/Misc/target-invalid-cpu-note.c
@@ -60,7 +60,7 @@
 // RUN: not %clang_cc1 -triple systemz--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix SYSTEMZ
 // SYSTEMZ: error: unknown target CPU 'not-a-cpu'
 // SYSTEMZ: note: valid target CPU values are: arch8, z10, arch9, z196, arch10,
-// SYSTEMZ-SAME: zEC12, arch11, z13, arch12, z14, arch13
+// SYSTEMZ-SAME: zEC12, arch11, z13, arch12, z14, arch13, z15
 
 // RUN: not %clang_cc1 -triple sparc--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix SPARC
 // SPARC: error: unknown target CPU 'not-a-cpu'
diff --git a/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Bar.h b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Bar.h
new file mode 100644
index 0000000000000..3ce482e6f243a
--- /dev/null
+++ b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Bar.h
@@ -0,0 +1 @@
+// interface-diagnose-missing-import/Foo.framework/Headers/Bar.h
diff --git a/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Foo.h b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Foo.h
new file mode 100644
index 0000000000000..c9c40986d9ed3
--- /dev/null
+++ b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Headers/Foo.h
@@ -0,0 +1,2 @@
+#import <Foo/RandoPriv.h>
+#import <Foo/Bar.h>
diff --git a/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Modules/module.modulemap b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Modules/module.modulemap
new file mode 100644
index 0000000000000..ebb4fa6e90248
--- /dev/null
+++ b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/Modules/module.modulemap
@@ -0,0 +1,6 @@
+// interface-diagnose-missing-import/Foo.framework/Modules/module.modulemap
+framework module Foo {
+  umbrella header "Foo.h"
+  export *
+  module * { export * }
+}
diff --git a/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/PrivateHeaders/RandoPriv.h b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/PrivateHeaders/RandoPriv.h
new file mode 100644
index 0000000000000..3e195fd85b36e
--- /dev/null
+++ b/clang/test/Modules/Inputs/interface-diagnose-missing-import/Foo.framework/PrivateHeaders/RandoPriv.h
@@ -0,0 +1,4 @@
+@interface NSObject
+@end
+@interface Buggy : NSObject
+@end
diff --git a/clang/test/Modules/ModuleDebugInfo.cpp b/clang/test/Modules/ModuleDebugInfo.cpp
index 6fe546f7013e5..40136978017cb 100644
--- a/clang/test/Modules/ModuleDebugInfo.cpp
+++ b/clang/test/Modules/ModuleDebugInfo.cpp
@@ -12,7 +12,7 @@
 
 // PCH:
 // RUN: %clang_cc1 -triple %itanium_abi_triple -x c++ -std=c++11  -debugger-tuning=lldb -emit-pch -fmodule-format=obj -I %S/Inputs -o %t.pch %S/Inputs/DebugCXX.h -mllvm -debug-only=pchcontainer &>%t-pch.ll
-// RUN: cat %t-pch.ll | FileCheck %s
+// RUN: cat %t-pch.ll | FileCheck --check-prefix=CHECK-CXX %s
 // RUN: cat %t-pch.ll | FileCheck --check-prefix=CHECK-NEG %s
 
 #ifdef MODULES
@@ -23,6 +23,7 @@
 // CHECK-MOD: distinct !DICompileUnit(language: DW_LANG_{{.*}}C_plus_plus,
 
 // CHECK: distinct !DICompileUnit(language: DW_LANG_{{.*}}C_plus_plus,
+// CHECK-CXX: distinct !DICompileUnit(language: DW_LANG_C_plus_plus_11,
 // CHECK-SAME:                    isOptimized: false,
 // CHECK-NOT:                     splitDebugFilename:
 // CHECK-SAME:                    dwoId:
diff --git a/clang/test/Modules/builtin-import.mm b/clang/test/Modules/builtin-import.mm
index cbc312b059042..8a27cb358484c 100644
--- a/clang/test/Modules/builtin-import.mm
+++ b/clang/test/Modules/builtin-import.mm
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t
-// RUN: %clang -cc1 -fsyntax-only -nobuiltininc -nostdinc++ -isysroot %S/Inputs/libc-libcxx/sysroot -isystem %S/Inputs/libc-libcxx/sysroot/usr/include/c++/v1 -isystem %S/Inputs/libc-libcxx/sysroot/usr/include -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x objective-c++ -fmodules-local-submodule-visibility %s
+// RUN: %clang_cc1 -fsyntax-only -nobuiltininc -nostdinc++ -isysroot %S/Inputs/libc-libcxx/sysroot -isystem %S/Inputs/libc-libcxx/sysroot/usr/include/c++/v1 -isystem %S/Inputs/libc-libcxx/sysroot/usr/include -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x objective-c++ -fmodules-local-submodule-visibility %s
 
 #include <stdio.h>
 #include <stddef.h>
diff --git a/clang/test/Modules/interface-diagnose-missing-import.m b/clang/test/Modules/interface-diagnose-missing-import.m
new file mode 100644
index 0000000000000..5bbac36423006
--- /dev/null
+++ b/clang/test/Modules/interface-diagnose-missing-import.m
@@ -0,0 +1,11 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 %s -fsyntax-only -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -F%S/Inputs/interface-diagnose-missing-import -verify
+@interface Buggy
+@end
+
+@import Foo.Bar;
+
+@interface Buggy (MyExt) // expected-error {{definition of 'Buggy' must be imported from module 'Foo' before it is required}}
+@end
+
+// expected-note@Foo/RandoPriv.h:3{{previous definition is here}}
diff --git a/clang/test/Modules/preprocess-module.cpp b/clang/test/Modules/preprocess-module.cpp
index b0cbac18e7807..ee909a811836f 100644
--- a/clang/test/Modules/preprocess-module.cpp
+++ b/clang/test/Modules/preprocess-module.cpp
@@ -51,7 +51,7 @@
 // RUN: %clang_cc1 -fmodules -fmodule-file=%t/file.rewrite.pcm %s -I%t -verify -fno-modules-error-recovery -DFILE_REWRITE -DINCLUDE -I%S/Inputs/preprocess
 //
 // Check that we can preprocess this user of the .pcm file.
-// RUN: %clang_cc1 -fmodules -fmodule-file=%t/file.pcm %s -I%t -E -frewrite-imports -o %t/preprocess-module.ii
+// RUN: %clang_cc1 -fmodules -fmodule-file=%t/file.pcm %s -I%t -E -frewrite-imports -DFILE_REWRITE_FULL -o %t/preprocess-module.ii
 // RUN: %clang_cc1 -fmodules %t/preprocess-module.ii -verify -fno-modules-error-recovery -DFILE_REWRITE_FULL
 //
 // Check that language / header search options are ignored when preprocessing from a .pcm file.
diff --git a/clang/test/Modules/umbrella-header-include-builtin.mm b/clang/test/Modules/umbrella-header-include-builtin.mm
index b14c73705ca86..9f8a45e1790e7 100644
--- a/clang/test/Modules/umbrella-header-include-builtin.mm
+++ b/clang/test/Modules/umbrella-header-include-builtin.mm
@@ -1,5 +1,5 @@
 // RUN: rm -rf %t
-// RUN: %clang -cc1 -fsyntax-only -nobuiltininc -nostdinc++ -isysroot %S/Inputs/libc-libcxx/sysroot -isystem %S/Inputs/libc-libcxx/sysroot/usr/include/c++/v1 -isystem %S/Inputs/libc-libcxx/sysroot/usr/include -F%S/Inputs/libc-libcxx/sysroot/Frameworks -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x objective-c++ %s
+// RUN: %clang_cc1 -fsyntax-only -nobuiltininc -nostdinc++ -isysroot %S/Inputs/libc-libcxx/sysroot -isystem %S/Inputs/libc-libcxx/sysroot/usr/include/c++/v1 -isystem %S/Inputs/libc-libcxx/sysroot/usr/include -F%S/Inputs/libc-libcxx/sysroot/Frameworks -std=c++11 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -x objective-c++ %s
 
 #include <A/A.h>
 
diff --git a/clang/test/OpenMP/declare_mapper_messages.c b/clang/test/OpenMP/declare_mapper_messages.c
index 51b761e224204..a7400b682f8ae 100644
--- a/clang/test/OpenMP/declare_mapper_messages.c
+++ b/clang/test/OpenMP/declare_mapper_messages.c
@@ -36,6 +36,8 @@ int fun(int arg) {
     {
 #pragma omp declare mapper(id: struct vec v) map(v.len) allocate(v)   // expected-error {{unexpected OpenMP clause 'allocate' in directive '#pragma omp declare mapper'}}
       struct vec vv, v1;
+      struct vec arr[10];
+      double d;
 #pragma omp target map(mapper)                                          // expected-error {{use of undeclared identifier 'mapper'}}
       {}
 #pragma omp target map(mapper:vv)                                       // expected-error {{expected '(' after 'mapper'}}
@@ -46,9 +48,13 @@ int fun(int arg) {
       {}
 #pragma omp target map(mapper(ab) :vv)                                  // expected-error {{missing map type}} expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}}
       {}
+#pragma omp target map(mapper(ab) :arr[0:2])                            // expected-error {{missing map type}} expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}}
+      {}
 #pragma omp target map(mapper(aa) :vv)                                  // expected-error {{missing map type}}
       {}
-#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1)
+#pragma omp target map(mapper(aa) to:d)                                 // expected-error {{mapper type must be of struct, union or class type}}
+      {}
+#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1) map(mapper(aa) to:arr[0])
       {}
 
 #pragma omp target update to(mapper)                                    // expected-error {{expected '(' after 'mapper'}} expected-error {{expected expression}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -57,7 +63,10 @@ int fun(int arg) {
 #pragma omp target update to(mapper(:vv)                                // expected-error {{illegal OpenMP user-defined mapper identifier}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(mapper(aa :vv)                             // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(mapper(ab):vv)                             // expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
-#pragma omp target update to(mapper(aa):vv)
+#pragma omp target update to(mapper(ab):arr[0:2])                       // expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(mapper(aa) a:vv)                           // expected-warning {{missing ':' after ) - ignoring}}
+#pragma omp target update to(mapper(aa):d)                              // expected-error {{mapper type must be of struct, union or class type}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(mapper(aa):vv) to(mapper(aa):arr[0])
 
 #pragma omp target update from(mapper)                                  // expected-error {{expected '(' after 'mapper'}} expected-error {{expected expression}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper()                                 // expected-error {{illegal OpenMP user-defined mapper identifier}} expected-error {{expected expression}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -65,8 +74,10 @@ int fun(int arg) {
 #pragma omp target update from(mapper(:vv)                              // expected-error {{illegal OpenMP user-defined mapper identifier}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(aa :vv)                           // expected-error {{expected ')'}} expected-note {{to match this '('}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(ab):vv)                           // expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(mapper(ab):arr[0:2])                     // expected-error {{cannot find a valid user-defined mapper for type 'struct vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(aa) a:vv)                         // expected-warning {{missing ':' after ) - ignoring}}
-#pragma omp target update from(mapper(aa):vv)
+#pragma omp target update from(mapper(aa):d)                            // expected-error {{mapper type must be of struct, union or class type}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(mapper(aa):vv) from(mapper(aa):arr[0])
     }
   }
   return arg;
diff --git a/clang/test/OpenMP/declare_mapper_messages.cpp b/clang/test/OpenMP/declare_mapper_messages.cpp
index bcb5ac463a0db..7b50fb2a52c27 100644
--- a/clang/test/OpenMP/declare_mapper_messages.cpp
+++ b/clang/test/OpenMP/declare_mapper_messages.cpp
@@ -64,6 +64,7 @@ int fun(int arg) {
     {
 #pragma omp declare mapper(id: vec v) map(v.len)
       vec vv, v1;
+      vec arr[10];
 #pragma omp target map(mapper)                                          // expected-error {{use of undeclared identifier 'mapper'}}
       {}
 #pragma omp target map(mapper:vv)                                       // expected-error {{expected '(' after 'mapper'}}
@@ -82,7 +83,9 @@ int fun(int arg) {
       {}
 #pragma omp target map(mapper(N1::aa) alloc:vv)                         // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'aa'}}
       {}
-#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1)
+#pragma omp target map(mapper(N1::aa) alloc:arr[0:2])                   // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'aa'}}
+      {}
+#pragma omp target map(mapper(aa) to:vv) map(close mapper(aa) from:v1) map(mapper(aa) to:arr[0])
       {}
 #pragma omp target map(mapper(N1::stack<int>::id) to:vv)
       {}
@@ -96,8 +99,9 @@ int fun(int arg) {
 #pragma omp target update to(mapper(N1:: :vv)                           // expected-error {{illegal OpenMP user-defined mapper identifier}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(mapper(N1::aa) :vv)                        // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'aa'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(mapper(ab):vv)                             // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update to(mapper(ab):arr[0:2])                       // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update to(mapper(aa) a:vv)                           // expected-warning {{missing ':' after ) - ignoring}}
-#pragma omp target update to(mapper(aa):vv)
+#pragma omp target update to(mapper(aa):vv) to(mapper(aa):arr[0])
 #pragma omp target update to(mapper(N1::stack<int>::id) :vv)
 
 #pragma omp target update from(mapper)                                  // expected-error {{expected '(' after 'mapper'}} expected-error {{expected expression}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
@@ -109,8 +113,9 @@ int fun(int arg) {
 #pragma omp target update from(mapper(N1:: :vv)                         // expected-error {{illegal OpenMP user-defined mapper identifier}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(N1::aa) :vv)                      // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'aa'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(ab):vv)                           // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
+#pragma omp target update from(mapper(ab):arr[0:2])                     // expected-error {{cannot find a valid user-defined mapper for type 'vec' with name 'ab'}} expected-error {{expected at least one 'to' clause or 'from' clause specified to '#pragma omp target update'}}
 #pragma omp target update from(mapper(aa) a:vv)                         // expected-warning {{missing ':' after ) - ignoring}}
-#pragma omp target update from(mapper(aa):vv)
+#pragma omp target update from(mapper(aa):vv) from(mapper(aa):arr[0])
 #pragma omp target update from(mapper(N1::stack<int>::id) :vv)
     }
 #pragma omp declare mapper(id: vec v) map(v.len)                        // expected-error {{redefinition of user-defined mapper for type 'vec' with name 'id'}}
diff --git a/clang/test/OpenMP/declare_simd_ast_print.cpp b/clang/test/OpenMP/declare_simd_ast_print.cpp
index c09f8b42e6d53..565dc2dfc04d1 100644
--- a/clang/test/OpenMP/declare_simd_ast_print.cpp
+++ b/clang/test/OpenMP/declare_simd_ast_print.cpp
@@ -21,6 +21,15 @@ void add_1(float *d) __attribute__((cold));
 // CHECK-NEXT: void add_1(float *d) __attribute__((cold));
 //
 
+#pragma omp declare simd aligned(hp, hp2:V)
+#pragma omp declare simd aligned(hp, hp2:V)
+template <class C, int V> void h(C *hp, C *hp2, C *hq, C *lin) {
+}
+// CHECK-NEXT: #pragma omp declare simd aligned(hp: V) aligned(hp2: V)
+// CHECK-NEXT: #pragma omp declare simd aligned(hp: V) aligned(hp2: V)
+// CHECK-NEXT: template <class C, int V> void h(C *hp, C *hp2, C *hq, C *lin) {
+// CHECK-NEXT: }
+
 #pragma omp declare simd aligned(hp, hp2)
 template <class C> void h(C *hp, C *hp2, C *hq, C *lin) {
 }
diff --git a/clang/test/OpenMP/declare_target_codegen.cpp b/clang/test/OpenMP/declare_target_codegen.cpp
index e24e58e10aa24..dd4ab3c8248e6 100644
--- a/clang/test/OpenMP/declare_target_codegen.cpp
+++ b/clang/test/OpenMP/declare_target_codegen.cpp
@@ -7,6 +7,15 @@
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DOMP5
 // RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix DEV5
 
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix KMPC-ONLY
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DOMP5
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix KMPC-ONLY
+
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm-bc %s -o %t-ppc-host.bc
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o -| FileCheck %s --check-prefix KMPC-ONLY
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -emit-pch -o %t
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple powerpc64le-unknown-unknown -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -include-pch %t -verify -o - | FileCheck %s --check-prefix KMPC-ONLY
+
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix SIMD-ONLY
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm-bc %s -o %t-ppc-host.bc -fopenmp-version=50 -DOMP5
 // RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -triple powerpc64le-unknown-unknown -fopenmp-targets=powerpc64le-ibm-linux-gnu -emit-llvm %s -fopenmp-is-device -fopenmp-host-ir-file-path %t-ppc-host.bc -o - -fopenmp-version=50 -DOMP5 | FileCheck %s --check-prefix SIMD-ONLY
@@ -19,6 +28,7 @@
 // expected-no-diagnostics
 
 // SIMD-ONLY-NOT: {{__kmpc|__tgt}}
+// KMPC-ONLY-NOT: __tgt
 
 // CHECK-NOT: define {{.*}}{{baz1|baz4|maini1|Base|virtual_}}
 // CHECK-DAG: Bake
diff --git a/clang/test/OpenMP/declare_target_link_codegen.cpp b/clang/test/OpenMP/declare_target_link_codegen.cpp
index 041dc6d02cebe..613c837afad09 100644
--- a/clang/test/OpenMP/declare_target_link_codegen.cpp
+++ b/clang/test/OpenMP/declare_target_link_codegen.cpp
@@ -26,7 +26,7 @@
 // HOST: [[SIZES:@.+]] = private unnamed_addr constant [3 x i64] [i64 4, i64 4, i64 4]
 // HOST: [[MAPTYPES:@.+]] = private unnamed_addr constant [3 x i64] [i64 35, i64 531, i64 531]
 // HOST: @.omp_offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"c_decl_tgt_ref_ptr\00"
-// HOST: @.omp_offloading.entry.c_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i8* bitcast (i32** @c_decl_tgt_ref_ptr to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name, i32 0, i32 0), i64 8, i32 1, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST: @.omp_offloading.entry.c_decl_tgt_ref_ptr = weak constant %struct.__tgt_offload_entry { i8* bitcast (i32** @c_decl_tgt_ref_ptr to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name, i32 0, i32 0), i64 8, i32 1, i32 0 }, section "omp_offloading_entries", align 1
 // DEVICE-NOT: internal unnamed_addr constant [{{[0-9]+}} x i8] c"c_{{.*}}_decl_tgt_ref_ptr\00"
 // HOST: @.omp_offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"_{{.*}}d_{{.*}}_decl_tgt_ref_ptr\00"
 // HOST: @.omp_offloading.entry.[[D_PTR]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (i32** @[[D_PTR]] to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0
diff --git a/clang/test/OpenMP/declare_variant_ast_print.c b/clang/test/OpenMP/declare_variant_ast_print.c
new file mode 100644
index 0000000000000..ad1ef91eca14c
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_ast_print.c
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c -std=c99 -ast-print %s -o - | FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c -std=c99 -ast-print %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+int foo(void);
+
+#pragma omp declare variant(foo) match(xxx={}, yyy={ccc})
+#pragma omp declare variant(foo) match(xxx={vvv})
+#pragma omp declare variant(foo) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(foo) match(implementation={vendor(unknown)})
+#pragma omp declare variant(foo) match(implementation={vendor(score(5): ibm)})
+int bar(void);
+
+// CHECK:      int foo();
+// CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={vendor(score(5):ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foo) match(implementation={vendor(llvm)})
+// CHECK-NEXT: int bar();
diff --git a/clang/test/OpenMP/declare_variant_ast_print.cpp b/clang/test/OpenMP/declare_variant_ast_print.cpp
new file mode 100644
index 0000000000000..77f03f518552f
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_ast_print.cpp
@@ -0,0 +1,201 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -std=c++14 -fexceptions -fcxx-exceptions %s -ast-print -o - -Wno-source-uses-openmp | FileCheck %s
+
+// RUN: %clang_cc1 -verify -fopenmp-simd -x c++ -std=c++14 -fexceptions -fcxx-exceptions %s -ast-print -o - -Wno-source-uses-openmp | FileCheck %s
+
+// expected-no-diagnostics
+
+// CHECK: int foo();
+int foo();
+
+// CHECK:      template <typename T> T foofoo() {
+// CHECK-NEXT: return T();
+// CHECK-NEXT: }
+template <typename T>
+T foofoo() { return T(); }
+
+// CHECK:      template<> int foofoo<int>() {
+// CHECK-NEXT: return int();
+// CHECK-NEXT: }
+
+// CHECK:      #pragma omp declare variant(foofoo<int>) match(implementation={vendor(score(5):ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(llvm)})
+// CHECK-NEXT: int bar();
+#pragma omp declare variant(foofoo <int>) match(xxx = {})
+#pragma omp declare variant(foofoo <int>) match(xxx = {vvv})
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(unknown)})
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(5): ibm)})
+int bar();
+
+// CHECK:      #pragma omp declare variant(foofoo<T>) match(implementation={vendor(score(C + 5):ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<T>) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<T>) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<T>) match(implementation={vendor(llvm)})
+// CHECK-NEXT: template <typename T, int C> T barbar();
+#pragma omp declare variant(foofoo <T>) match(xxx = {})
+#pragma omp declare variant(foofoo <T>) match(xxx = {vvv})
+#pragma omp declare variant(foofoo <T>) match(user = {score(<expr>) : condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {score(<expr>) : condition(<expr>)}, user = {condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(foofoo <T>) match(implementation={vendor(unknown)})
+#pragma omp declare variant(foofoo <T>) match(implementation={vendor(score(C+5): ibm)})
+template <typename T, int C>
+T barbar();
+
+// CHECK:      #pragma omp declare variant(foofoo<int>) match(implementation={vendor(score(3 + 5):ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(foofoo<int>) match(implementation={vendor(llvm)})
+// CHECK-NEXT: template<> int barbar<int, 3>();
+
+// CHECK-NEXT: int baz() {
+// CHECK-NEXT: return barbar<int, 3>();
+// CHECK-NEXT: }
+int baz() {
+  return barbar<int, 3>();
+}
+
+// CHECK:      template <class C> void h_ref(C *hp, C *hp2, C *hq, C *lin) {
+// CHECK-NEXT: }
+// CHECK-NEXT: template<> void h_ref<double>(double *hp, double *hp2, double *hq, double *lin) {
+// CHECK-NEXT: }
+// CHECK-NEXT: template<> void h_ref<float>(float *hp, float *hp2, float *hq, float *lin) {
+// CHECK-NEXT: }
+template <class C>
+void h_ref(C *hp, C *hp2, C *hq, C *lin) {
+}
+
+// CHECK:      #pragma omp declare variant(h_ref<C>) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(h_ref<C>) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(h_ref<C>) match(implementation={vendor(llvm)})
+// CHECK-NEXT: template <class C> void h(C *hp, C *hp2, C *hq, C *lin) {
+// CHECK-NEXT: }
+#pragma omp declare variant(h_ref <C>) match(xxx = {})
+#pragma omp declare variant(h_ref <C>) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(h_ref <C>) match(implementation={vendor(unknown)})
+template <class C>
+void h(C *hp, C *hp2, C *hq, C *lin) {
+}
+
+// CHECK:      #pragma omp declare variant(h_ref<float>) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(h_ref<float>) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(h_ref<float>) match(implementation={vendor(llvm)})
+// CHECK-NEXT: template<> void h<float>(float *hp, float *hp2, float *hq, float *lin) {
+// CHECK-NEXT: }
+
+// CHECK-NEXT: template<> void h<double>(double *hp, double *hp2, double *hq, double *lin) {
+// CHECK-NEXT:   h((float *)hp, (float *)hp2, (float *)hq, (float *)lin);
+// CHECK-NEXT: }
+#pragma omp declare variant(h_ref <double>) match(xxx = {})
+#pragma omp declare variant(h_ref <double>) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(h_ref <double>) match(implementation={vendor(unknown)})
+template <>
+void h(double *hp, double *hp2, double *hq, double *lin) {
+  h((float *)hp, (float *)hp2, (float *)hq, (float *)lin);
+}
+
+// CHECK: int fn();
+int fn();
+// CHECK: int fn(int);
+int fn(int);
+// CHECK:      #pragma omp declare variant(fn) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(fn) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(fn) match(implementation={vendor(llvm)})
+// CHECK-NEXT: int overload();
+#pragma omp declare variant(fn) match(xxx = {})
+#pragma omp declare variant(fn) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(fn) match(implementation={vendor(unknown)})
+int overload(void);
+
+// CHECK:      int fn_deduced_variant() {
+// CHECK-NEXT: return 0;
+// CHECK-NEXT: }
+auto fn_deduced_variant() { return 0; }
+// CHECK:      #pragma omp declare variant(fn_deduced_variant) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(fn_deduced_variant) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(fn_deduced_variant) match(implementation={vendor(llvm)})
+// CHECK-NEXT: int fn_deduced();
+#pragma omp declare variant(fn_deduced_variant) match(xxx = {})
+#pragma omp declare variant(fn_deduced_variant) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(fn_deduced_variant) match(implementation={vendor(unknown)})
+int fn_deduced();
+
+// CHECK: int fn_deduced_variant1();
+int fn_deduced_variant1();
+// CHECK:      #pragma omp declare variant(fn_deduced_variant1) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(fn_deduced_variant1) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(fn_deduced_variant1) match(implementation={vendor(llvm)})
+// CHECK-NEXT: int fn_deduced1() {
+// CHECK-NEXT: return 0;
+// CHECK-NEXT: }
+#pragma omp declare variant(fn_deduced_variant1) match(xxx = {})
+#pragma omp declare variant(fn_deduced_variant1) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(fn_deduced_variant1) match(implementation={vendor(unknown)})
+auto fn_deduced1() { return 0; }
+
+// CHECK:      struct SpecialFuncs {
+// CHECK-NEXT: void vd() {
+// CHECK-NEXT: }
+// CHECK-NEXT: SpecialFuncs();
+// CHECK-NEXT: ~SpecialFuncs() noexcept;
+// CHECK-NEXT: void baz() {
+// CHECK-NEXT: }
+// CHECK-NEXT: void bar() {
+// CHECK-NEXT: }
+// CHECK-NEXT: void bar(int) {
+// CHECK-NEXT: }
+// CHECK-NEXT: #pragma omp declare variant(SpecialFuncs::baz) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(SpecialFuncs::bar) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(SpecialFuncs::bar) match(implementation={vendor(llvm)})
+// CHECK-NEXT: void foo1() {
+// CHECK-NEXT: }
+// CHECK-NEXT: #pragma omp declare variant(SpecialFuncs::baz) match(implementation={vendor(unknown)})
+// CHECK-NEXT: void xxx();
+// CHECK-NEXT: } s;
+struct SpecialFuncs {
+  void vd() {}
+  SpecialFuncs();
+  ~SpecialFuncs();
+
+  void baz() {}
+  void bar() {}
+  void bar(int) {}
+#pragma omp declare variant(SpecialFuncs::baz) match(xxx = {})
+#pragma omp declare variant(SpecialFuncs::bar) match(xxx = {})
+#pragma omp declare variant(SpecialFuncs::bar) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(SpecialFuncs::baz) match(implementation={vendor(unknown)})
+  void foo1() {}
+#pragma omp declare variant(SpecialFuncs::baz) match(implementation={vendor(unknown)})
+  void xxx();
+} s;
+
+// CHECK:      #pragma omp declare variant(SpecialFuncs::baz) match(implementation={vendor(unknown)})
+// CHECK-NEXT: void SpecialFuncs::xxx() {
+// CHECK-NEXT: }
+void SpecialFuncs::xxx() {}
+
+// CHECK:      static void static_f_variant() {
+// CHECK-NEXT: }
+static void static_f_variant() {}
+// CHECK:      #pragma omp declare variant(static_f_variant) match(implementation={vendor(unknown)})
+// CHECK-NEXT: #pragma omp declare variant(static_f_variant) match(implementation={vendor(ibm)})
+// CHECK-NEXT: #pragma omp declare variant(static_f_variant) match(implementation={vendor(llvm)})
+// CHECK-NEXT: static void static_f() {
+// CHECK-NEXT: }
+#pragma omp declare variant(static_f_variant) match(xxx = {})
+#pragma omp declare variant(static_f_variant) match(implementation={vendor(ibm)}, implementation={vendor(llvm)})
+#pragma omp declare variant(static_f_variant) match(implementation={vendor(unknown)})
+static void static_f() {}
+
+// CHECK: void bazzzz() {
+// CHECK-NEXT: s.foo1();
+// CHECK-NEXT: static_f();
+// CHECK-NEXT: }
+void bazzzz() {
+  s.foo1();
+  static_f();
+}
diff --git a/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
new file mode 100644
index 0000000000000..ed814b7b28950
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_implementation_vendor_codegen.cpp
@@ -0,0 +1,91 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple %itanium_abi_triple -emit-llvm %s -fexceptions -fcxx-exceptions -o - -fsanitize-address-use-after-scope | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -x c++ -std=c++11 -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -emit-pch -o %t -fopenmp-version=50 %s
+// RUN: %clang_cc1 -fopenmp -x c++ -triple %itanium_abi_triple -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - -fopenmp-version=50 | FileCheck %s
+// expected-no-diagnostics
+
+// CHECK-NOT: ret i32 {{1|4}}
+// CHECK-DAG: @_Z3barv = {{.*}}alias i32 (), i32 ()* @_Z3foov
+// CHECK-DAG: @_ZN16SpecSpecialFuncs6MethodEv = {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
+// CHECK-DAG: @_ZN16SpecSpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecSpecialFuncs*), i32 (%struct.SpecSpecialFuncs*)* @_ZN16SpecSpecialFuncs7method_Ev
+// CHECK-DAG: @_ZN12SpecialFuncs6methodEv = linkonce_odr {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
+// CHECK-DAG: @_Z4callv = {{.*}}alias i32 (), i32 ()* @_Z4testv
+// CHECK-DAG: @_ZL9stat_usedv = internal alias i32 (), i32 ()* @_ZL10stat_used_v
+// CHECK-DAG: @_ZN12SpecialFuncs6MethodEv =  {{.*}}alias i32 (%struct.SpecialFuncs*), i32 (%struct.SpecialFuncs*)* @_ZN12SpecialFuncs7method_Ev
+// CHECK-DAG: declare {{.*}}i32 @_Z5bazzzv()
+// CHECK-DAG: declare {{.*}}i32 @_Z3bazv()
+// CHECK-DAG: ret i32 2
+// CHECK-DAG: ret i32 3
+// CHECK-DAG: ret i32 5
+// CHECK-DAG: ret i32 6
+// CHECK-DAG: ret i32 7
+// CHECK-NOT: ret i32 {{1|4}}
+
+#ifndef HEADER
+#define HEADER
+
+int foo() { return 2; }
+
+#pragma omp declare variant(foo) match(implementation = {vendor(llvm)})
+int bar() { return 1; }
+
+int bazzz();
+#pragma omp declare variant(bazzz) match(implementation = {vendor(llvm)})
+int baz() { return 1; }
+
+int test();
+#pragma omp declare variant(test) match(implementation = {vendor(llvm)})
+int call() { return 1; }
+
+static int stat_unused_();
+#pragma omp declare variant(stat_unused_) match(implementation = {vendor(llvm)})
+static int stat_unused() { return 1; }
+
+static int stat_used_();
+#pragma omp declare variant(stat_used_) match(implementation = {vendor(llvm)})
+static int stat_used() { return 1; }
+
+int main() { return bar() + baz() + call() + stat_used(); }
+
+int test() { return 3; }
+static int stat_unused_() { return 4; }
+static int stat_used_() { return 5; }
+
+struct SpecialFuncs {
+  void vd() {}
+  SpecialFuncs();
+  ~SpecialFuncs();
+
+  int method_() { return 6; }
+#pragma omp declare variant(SpecialFuncs::method_)                             \
+    match(implementation = {vendor(llvm)})
+  int method() { return 1; }
+#pragma omp declare variant(SpecialFuncs::method_)                             \
+    match(implementation = {vendor(llvm)})
+  int Method();
+} s;
+
+int SpecialFuncs::Method() { return 1; }
+
+struct SpecSpecialFuncs {
+  void vd() {}
+  SpecSpecialFuncs();
+  ~SpecSpecialFuncs();
+
+  int method_();
+#pragma omp declare variant(SpecSpecialFuncs::method_)                         \
+    match(implementation = {vendor(llvm)})
+  int method() { return 1; }
+#pragma omp declare variant(SpecSpecialFuncs::method_)                         \
+    match(implementation = {vendor(llvm)})
+  int Method();
+} s1;
+
+int SpecSpecialFuncs::method_() { return 7; }
+int SpecSpecialFuncs::Method() { return 1; }
+
+void xxx() {
+  (void)s.method();
+  (void)s1.method();
+}
+
+#endif // HEADER
diff --git a/clang/test/OpenMP/declare_variant_messages.c b/clang/test/OpenMP/declare_variant_messages.c
new file mode 100644
index 0000000000000..7e69cec1f1b0c
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_messages.c
@@ -0,0 +1,117 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c -std=c99 -fms-extensions -Wno-pragma-pack %s
+
+// expected-error@+1 {{expected an OpenMP directive}}
+#pragma omp declare
+
+int foo(void);
+
+#pragma omp declare variant // expected-error {{expected '(' after 'declare variant'}}
+#pragma omp declare variant(  // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo // expected-error {{expected ')'}} expected-error {{expected 'match' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
+#pragma omp declare variant(x) // expected-error {{use of undeclared identifier 'x'}}
+#pragma omp declare variant(foo) // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) xxx // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match // expected-error {{expected '(' after 'match'}}
+#pragma omp declare variant(foo) match( // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match() // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx) // expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx=) // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foo) match(xxx=yyy) // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foo) match(xxx=yyy}) // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foo) match(xxx={) // expected-error {{expected '}'}} expected-note {{to match this '{'}}
+#pragma omp declare variant(foo) match(xxx={})
+#pragma omp declare variant(foo) match(xxx={vvv})
+#pragma omp declare variant(foo) match(xxx={vvv} xxx) // expected-error {{expected ','}} expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={vvv}) xxx // expected-warning {{extra tokens at the end of '#pragma omp declare variant' are ignored}}
+#pragma omp declare variant(foo) match(implementation={xxx}) // expected-warning {{unknown context selector in 'implementation' context selector set of 'omp declare variant' directive, ignored}}
+#pragma omp declare variant(foo) match(implementation={vendor}) // expected-error {{expected '(' after 'vendor'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(implementation={vendor(}) // expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(implementation={vendor()}) // expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}}
+#pragma omp declare variant(foo) match(implementation={vendor(score ibm)}) // expected-error {{expected '(' after 'score'}} expected-warning {{missing ':' after context selector score clause - ignoring}}
+#pragma omp declare variant(foo) match(implementation={vendor(score( ibm)}) // expected-error {{expected ')'}} expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo) match(implementation={vendor(score(2 ibm)}) // expected-error 2 {{expected ')'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note 2 {{to match this '('}}
+#pragma omp declare variant(foo) match(implementation={vendor(score(foo()) ibm)}) // expected-warning {{missing ':' after context selector score clause - ignoring}} expected-error {{expression is not an integer constant expression}}
+#pragma omp declare variant(foo) match(implementation={vendor(score(5): ibm)})
+int bar(void);
+
+// expected-error@+2 {{'#pragma omp declare variant' can only be applied to functions}}
+#pragma omp declare variant(foo) match(xxx={})
+int a;
+// expected-error@+2 {{'#pragma omp declare variant' can only be applied to functions}}
+#pragma omp declare variant(foo) match(xxx={})
+#pragma omp threadprivate(a)
+int var;
+#pragma omp threadprivate(var)
+
+// expected-error@+2 {{expected an OpenMP directive}} expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={})
+#pragma omp declare
+
+// expected-error@+3 {{function declaration is expected after 'declare variant' directive}}
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={})
+#pragma omp declare variant(foo) match(xxx={})
+#pragma options align=packed
+int main();
+
+// expected-error@+3 {{function declaration is expected after 'declare variant' directive}}
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={})
+#pragma omp declare variant(foo) match(xxx={})
+#pragma init_seg(compiler)
+int main();
+
+// expected-error@+1 {{single declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(foo) match(xxx={})
+int b, c;
+
+int no_proto();
+
+// expected-error@+3 {{function with '#pragma omp declare variant' must have a prototype}}
+// expected-note@+1 {{'#pragma omp declare variant' for function specified here}}
+#pragma omp declare variant(no_proto) match(xxx={})
+int no_proto_too();
+
+int after_use_variant(void);
+int after_use();
+int bar() {
+  return after_use();
+}
+
+// expected-warning@+1 {{'#pragma omp declare variant' cannot be applied for function after first usage; the original function might be used}}
+#pragma omp declare variant(after_use_variant) match(xxx={})
+int after_use(void);
+#pragma omp declare variant(after_use_variant) match(xxx={})
+int defined(void) { return 0; }
+int defined1(void) { return 0; }
+// expected-warning@+1 {{#pragma omp declare variant' cannot be applied to the function that was defined already; the original function might be used}}
+#pragma omp declare variant(after_use_variant) match(xxx={})
+int defined1(void);
+
+
+int diff_cc_variant(void);
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different calling convention}}
+#pragma omp declare variant(diff_cc_variant) match(xxx={})
+__vectorcall int diff_cc(void);
+
+int diff_ret_variant(void);
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different return type}}
+#pragma omp declare variant(diff_ret_variant) match(xxx={})
+void diff_ret(void);
+
+void marked(void);
+void not_marked(void);
+// expected-note@+1 {{marked as 'declare variant' here}}
+#pragma omp declare variant(not_marked) match(implementation={vendor(unknown)})
+void marked_variant(void);
+// expected-warning@+1 {{variant function in '#pragma omp declare variant' is itself marked as '#pragma omp declare variant'}}
+#pragma omp declare variant(marked_variant) match(xxx={})
+void marked(void);
+
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant
diff --git a/clang/test/OpenMP/declare_variant_messages.cpp b/clang/test/OpenMP/declare_variant_messages.cpp
new file mode 100644
index 0000000000000..88e724d12470d
--- /dev/null
+++ b/clang/test/OpenMP/declare_variant_messages.cpp
@@ -0,0 +1,242 @@
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp -x c++ -std=c++14 -fms-extensions -Wno-pragma-pack -fexceptions -fcxx-exceptions %s
+
+// RUN: %clang_cc1 -triple=x86_64-pc-win32 -verify -fopenmp-simd -x c++ -std=c++14 -fms-extensions -Wno-pragma-pack -fexceptions -fcxx-exceptions %s
+
+// expected-error@+1 {{expected an OpenMP directive}}
+#pragma omp declare
+
+int foo();
+
+template <typename T>
+T foofoo(); // expected-note 2 {{declared here}}
+
+#pragma omp declare variant                                  // expected-error {{expected '(' after 'declare variant'}}
+#pragma omp declare variant(                                 // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foo                              // expected-error {{expected ')'}} expected-error {{expected 'match' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
+#pragma omp declare variant(x)                               // expected-error {{use of undeclared identifier 'x'}}
+#pragma omp declare variant(foo)                             // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>)                    // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) xxx                // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match              // expected-error {{expected '(' after 'match'}}
+#pragma omp declare variant(foofoo <int>) match(             // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match()            // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match(xxx)         // expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match(xxx =)       // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foofoo <int>) match(xxx = yyy)   // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foofoo <int>) match(xxx = yyy }) // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foofoo <int>) match(xxx = {)     // expected-error {{expected '}'}} expected-note {{to match this '{'}}
+#pragma omp declare variant(foofoo <int>) match(xxx = {})
+#pragma omp declare variant(foofoo <int>) match(xxx = {vvv})
+#pragma omp declare variant(foofoo <int>) match(xxx = {vvv} xxx) // expected-error {{expected ','}} expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match(xxx = {vvv}) xxx // expected-warning {{extra tokens at the end of '#pragma omp declare variant' are ignored}}
+#pragma omp declare variant(foofoo <int>) match(implementation={xxx}) // expected-warning {{unknown context selector in 'implementation' context selector set of 'omp declare variant' directive, ignored}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor}) // expected-error {{expected '(' after 'vendor'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(}) // expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor()}) // expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score ibm)}) // expected-error {{expected '(' after 'score'}} expected-warning {{missing ':' after context selector score clause - ignoring}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score( ibm)}) // expected-error {{expected ')'}} expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(2 ibm)}) // expected-error 2 {{expected ')'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note 2 {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(foofoo <int>()) ibm)}) // expected-warning {{missing ':' after context selector score clause - ignoring}} expected-error {{expression is not an integral constant expression}} expected-note {{non-constexpr function 'foofoo<int>' cannot be used in a constant expression}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(5): ibm)})
+int bar();
+
+#pragma omp declare variant                            // expected-error {{expected '(' after 'declare variant'}}
+#pragma omp declare variant(                           // expected-error {{expected expression}} expected-error {{expected ')'}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <T>                 // expected-error {{expected ')'}} expected-error {{expected 'match' clause on 'omp declare variant' directive}} expected-note {{to match this '('}}
+#pragma omp declare variant(x)                         // expected-error {{use of undeclared identifier 'x'}}
+#pragma omp declare variant(foo)                       // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo)                    // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>)                // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) xxx            // expected-error {{expected 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) match          // expected-error {{expected '(' after 'match'}}
+#pragma omp declare variant(foofoo <T>) match(         // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) match()        // expected-error {{expected context selector in 'match' clause on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) match(xxx)     // expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) match(xxx =)   // expected-error {{expected '{' after '='}}
+#pragma omp declare variant(foofoo <T>) match(xxx = {) // expected-error {{expected '}'}} expected-note {{to match this '{'}}
+#pragma omp declare variant(foofoo <T>) match(xxx = {})
+#pragma omp declare variant(foofoo <T>) match(xxx = {vvv})
+#pragma omp declare variant(foofoo <T>) match(user = {score(<expr>) : condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {score(<expr>) : condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(user = {condition(<expr>)})
+#pragma omp declare variant(foofoo <T>) match(xxx = {vvv} xxx) // expected-error {{expected ','}} expected-error {{expected '=' after 'xxx' context selector set name on 'omp declare variant' directive}}
+#pragma omp declare variant(foofoo <T>) match(xxx = {vvv}) xxx // expected-warning {{extra tokens at the end of '#pragma omp declare variant' are ignored}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score ibm)}) // expected-error {{expected '(' after 'score'}} expected-warning {{missing ':' after context selector score clause - ignoring}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score( ibm)}) // expected-error {{expected ')'}} expected-error {{use of undeclared identifier 'ibm'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(C ibm)}) // expected-error 2 {{expected ')'}} expected-error {{expected vendor identifier in 'vendor' context selector of 'implementation' selector set of 'omp declare variant' directive}} expected-warning {{missing ':' after context selector score clause - ignoring}} expected-note 2 {{to match this '('}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(foofoo <int>()) ibm)}) // expected-warning {{missing ':' after context selector score clause - ignoring}} expected-error {{expression is not an integral constant expression}} expected-note {{non-constexpr function 'foofoo<int>' cannot be used in a constant expression}}
+#pragma omp declare variant(foofoo <int>) match(implementation={vendor(score(C+5): ibm)})
+template <typename T, int C>
+T barbar();
+
+// expected-error@+2 {{'#pragma omp declare variant' can only be applied to functions}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+int a;
+// expected-error@+2 {{'#pragma omp declare variant' can only be applied to functions}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma omp threadprivate(a)
+int var;
+#pragma omp threadprivate(var)
+
+// expected-error@+2 {{expected an OpenMP directive}} expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma omp declare
+
+// expected-error@+3 {{function declaration is expected after 'declare variant' directive}}
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma options align = packed
+int main();
+
+// expected-error@+3 {{function declaration is expected after 'declare variant' directive}}
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+#pragma init_seg(compiler)
+int main();
+
+// expected-error@+1 {{single declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+int b, c;
+
+// expected-error@+1 {{'C' does not refer to a value}}
+#pragma omp declare variant(C) match(xxx = {})
+// expected-note@+1 {{declared here}}
+template <class C>
+void h(C *hp, C *hp2, C *hq, C *lin) {
+  b = 0;
+}
+
+// expected-error@+1 {{variant in '#pragma omp declare variant' with type '<overloaded function type>' is incompatible with type 'void (*)(int *, int *, int *, int *)'}}
+#pragma omp declare variant(barbar <int>) match(xxx = {})
+template <>
+void h(int *hp, int *hp2, int *hq, int *lin);
+
+int after_use_variant(void);
+int after_use();
+int bar() {
+  return after_use();
+}
+
+// expected-warning@+1 {{'#pragma omp declare variant' cannot be applied for function after first usage; the original function might be used}}
+#pragma omp declare variant(after_use_variant) match(xxx = {})
+int after_use(void);
+
+int fn();
+int fn(int);
+#pragma omp declare variant(fn) match(xxx = {})
+int overload(void);
+
+int fn1();
+int fn1(int);
+// expected-error@+1 {{variant in '#pragma omp declare variant' with type '<overloaded function type>' is incompatible with type 'int (*)(float)'}}
+#pragma omp declare variant(fn1) match(xxx = {})
+int overload1(float);
+
+int fn_constexpr_variant();
+// expected-error@+2 {{'#pragma omp declare variant' does not support constexpr functions}}
+#pragma omp declare variant(fn_constexpr_variant) match(xxx = {})
+constexpr int fn_constexpr();
+
+constexpr int fn_constexpr_variant1();
+// expected-error@+1 {{'#pragma omp declare variant' does not support constexpr functions}}
+#pragma omp declare variant(fn_constexpr_variant1) match(xxx = {})
+int fn_constexpr1();
+
+int fn_sc_variant();
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different storage class}}
+#pragma omp declare variant(fn_sc_variant) match(xxx = {})
+static int fn_sc();
+
+static int fn_sc_variant1();
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different storage class}}
+#pragma omp declare variant(fn_sc_variant1) match(xxx = {})
+int fn_sc1();
+
+int fn_inline_variant();
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different inline specification}}
+#pragma omp declare variant(fn_inline_variant) match(xxx = {})
+inline int fn_inline();
+
+inline int fn_inline_variant1();
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different inline specification}}
+#pragma omp declare variant(fn_inline_variant1) match(xxx = {})
+int fn_inline1();
+
+int fn_linkage_variant();
+extern "C" {
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different linkage}}
+#pragma omp declare variant(fn_linkage_variant) match(xxx = {})
+int fn_linkage();
+}
+
+extern "C" int fn_linkage_variant1();
+// expected-error@+1 {{function with '#pragma omp declare variant' has a different linkage}}
+#pragma omp declare variant(fn_linkage_variant1) match(xxx = {})
+int fn_linkage1();
+
+auto fn_deduced_variant() { return 0; }
+#pragma omp declare variant(fn_deduced_variant) match(xxx = {})
+int fn_deduced();
+
+int fn_deduced_variant1();
+#pragma omp declare variant(fn_deduced_variant1) match(xxx = {})
+auto fn_deduced1() { return 0; }
+
+auto fn_deduced3() { return 0; }
+// expected-warning@+1 {{'#pragma omp declare variant' cannot be applied to the function that was defined already; the original function might be used}}
+#pragma omp declare variant(fn_deduced_variant1) match(xxx = {})
+auto fn_deduced3();
+
+auto fn_deduced_variant2() { return 0; }
+// expected-error@+1 {{variant in '#pragma omp declare variant' with type 'int ()' is incompatible with type 'float (*)()'}}
+#pragma omp declare variant(fn_deduced_variant2) match(xxx = {})
+float fn_deduced2();
+
+// expected-error@+1 {{exception specification in declaration does not match previous declaration}}
+int fn_except_variant() noexcept(true);
+// expected-note@+2 {{previous declaration is here}}
+#pragma omp declare variant(fn_except_variant) match(xxx = {})
+int fn_except() noexcept(false);
+
+// expected-error@+1 {{exception specification in declaration does not match previous declaration}}
+int fn_except_variant1() noexcept(false);
+// expected-note@+2 {{previous declaration is here}}
+#pragma omp declare variant(fn_except_variant1) match(xxx = {})
+int fn_except1() noexcept(true);
+
+struct SpecialFuncs {
+  void vd();
+  // expected-error@+2 {{'#pragma omp declare variant' does not support constructors}}
+#pragma omp declare variant(SpecialFuncs::vd) match(xxx = {})
+  SpecialFuncs();
+  // expected-error@+2 {{'#pragma omp declare variant' does not support destructors}}
+#pragma omp declare variant(SpecialFuncs::vd) match(xxx = {})
+  ~SpecialFuncs();
+
+  void baz();
+  void bar();
+  void bar(int);
+#pragma omp declare variant(SpecialFuncs::baz) match(xxx = {})
+#pragma omp declare variant(SpecialFuncs::bar) match(xxx = {})
+  void foo1();
+  SpecialFuncs& foo(const SpecialFuncs&);
+  SpecialFuncs& bar(SpecialFuncs&&);
+  // expected-error@+2 {{'#pragma omp declare variant' does not support defaulted functions}}
+#pragma omp declare variant(SpecialFuncs::foo) match(xxx = {})
+  SpecialFuncs& operator=(const SpecialFuncs&) = default;
+  // expected-error@+2 {{'#pragma omp declare variant' does not support deleted functions}}
+#pragma omp declare variant(SpecialFuncs::bar) match(xxx = {})
+  SpecialFuncs& operator=(SpecialFuncs&&) = delete;
+};
+
+namespace N {
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant
+} // namespace N
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant
+// expected-error@+1 {{function declaration is expected after 'declare variant' directive}}
+#pragma omp declare variant
diff --git a/clang/test/OpenMP/for_codegen.cpp b/clang/test/OpenMP/for_codegen.cpp
index 756d6ccebf424..925235ef960e5 100644
--- a/clang/test/OpenMP/for_codegen.cpp
+++ b/clang/test/OpenMP/for_codegen.cpp
@@ -89,23 +89,6 @@ void loop_with_counter_collapse() {
   // CHECK: [[NUM_ITERS_VAL:%.+]] = sub nsw i64 [[MUL]], 1
   // CHECK: store i64 [[NUM_ITERS_VAL]], i64* [[NUM_ITERS:%.+]],
 
-  // Initialization
-  // CHECK: store i32 0, i32* [[I:%.+]],
-  // CHECK: [[I_INIT:%.+]] = load i32, i32* [[I]],
-  // CHECK: store i32 [[I_INIT]], i32* [[J:%.+]],
-
-  // LIFETIME: call void @llvm.lifetime.end
-  // LIFETIME: call void @llvm.lifetime.end
-
-  // Precondition for j counter
-  // CHECK: store i32 0, i32* [[TMP_I:%.+]],
-  // CHECK: [[J_LB_VAL:%.+]] = load i32, i32* [[TMP_I]],
-  // CHECK: [[I_VAL:%.+]] = load i32, i32* [[TMP_I]],
-  // CHECK: [[J_UB_VAL:%.+]] = add nsw i32 4, [[I_VAL]]
-  // CHECK: [[CMP:%.+]] = icmp slt i32 [[J_LB_VAL]], [[J_UB_VAL]]
-  // CHECK: br i1 [[CMP]], label %[[THEN:[^,]+]], label %[[ELSE:[^,]+]]
-
-  // CHECK: [[THEN]]:
   // CHECK: store i64 0, i64* [[LB:%.+]],
   // CHECK: [[NUM_ITERS_VAL:%.+]] = load i64, i64* [[NUM_ITERS]],
   // CHECK: store i64 [[NUM_ITERS_VAL]], i64* [[UB:%.+]],
@@ -633,6 +616,22 @@ void for_with_references() {
     k = cnt;
 }
 
+// CHECK-LABEL: for_with_references_dep_cond
+void for_with_references_dep_cond() {
+// CHECK: [[I:%.+]] = alloca i8,
+// CHECK: [[CNT:%.+]] = alloca i8*,
+// CHECK: [[CNT_PRIV:%.+]] = alloca i8,
+// CHECK: call void @__kmpc_for_static_init_8(
+// CHECK-NOT: load i8, i8* [[CNT]],
+// CHECK: call void @__kmpc_for_static_fini(
+  char i = 0;
+  char &cnt = i;
+#pragma omp for collapse(2)
+  for (cnt = 0; cnt < 2; ++cnt)
+    for (int j = 0; j < 4 + cnt; j++)
+    k = cnt;
+}
+
 struct Bool {
   Bool(bool b) : b(b) {}
   operator bool() const { return b; }
diff --git a/clang/test/OpenMP/nvptx_declare_target_var_ctor_dtor_codegen.cpp b/clang/test/OpenMP/nvptx_declare_target_var_ctor_dtor_codegen.cpp
index 03520e17a614c..48039469489c1 100644
--- a/clang/test/OpenMP/nvptx_declare_target_var_ctor_dtor_codegen.cpp
+++ b/clang/test/OpenMP/nvptx_declare_target_var_ctor_dtor_codegen.cpp
@@ -73,15 +73,15 @@ S cd = doo() + car() + caz() + baz();
 // DEVICE-DAG: ret void
 
 // HOST-DAG: @.omp_offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[C_ADDR]]\00"
-// HOST-DAG: @.omp_offloading.entry.[[C_ADDR]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (i32* @[[C_ADDR]] to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 4, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST-DAG: @.omp_offloading.entry.[[C_ADDR]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (i32* @[[C_ADDR]] to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // HOST-DAG: @.omp_offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[CD_ADDR]]\00"
-// HOST-DAG: @.omp_offloading.entry.[[CD_ADDR]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (%struct.S* @[[CD_ADDR]] to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 4, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST-DAG: @.omp_offloading.entry.[[CD_ADDR]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (%struct.S* @[[CD_ADDR]] to i8*), i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 4, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // HOST-DAG: @.omp_offloading.entry_name{{.*}} = internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[C_CTOR]]\00"
-// HOST-DAG: @.omp_offloading.entry.[[C_CTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[C_CTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 2, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST-DAG: @.omp_offloading.entry.[[C_CTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[C_CTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 2, i32 0 }, section "omp_offloading_entries", align 1
 // HOST-DAG: @.omp_offloading.entry_name{{.*}}= internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[CD_CTOR]]\00"
-// HOST-DAG: @.omp_offloading.entry.[[CD_CTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[CD_CTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 2, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST-DAG: @.omp_offloading.entry.[[CD_CTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[CD_CTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 2, i32 0 }, section "omp_offloading_entries", align 1
 // HOST-DAG: @.omp_offloading.entry_name{{.*}}= internal unnamed_addr constant [{{[0-9]+}} x i8] c"[[CD_DTOR]]\00"
-// HOST-DAG: @.omp_offloading.entry.[[CD_DTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[CD_DTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 4, i32 0 }, section ".omp_offloading.entries", align 1
+// HOST-DAG: @.omp_offloading.entry.[[CD_DTOR]] = weak constant %struct.__tgt_offload_entry { i8* @[[CD_DTOR]], i8* getelementptr inbounds ([{{[0-9]+}} x i8], [{{[0-9]+}} x i8]* @.omp_offloading.entry_name{{.*}}, i32 0, i32 0), i64 0, i32 4, i32 0 }, section "omp_offloading_entries", align 1
 int maini1() {
   int a;
 #pragma omp target map(tofrom : a)
diff --git a/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp b/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
index c676f77a0bc63..877aa7ab0b622 100644
--- a/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
+++ b/clang/test/OpenMP/nvptx_target_requires_unified_shared_memory.cpp
@@ -40,10 +40,10 @@ int bar(int n){
 // CHECK-HOST: [[OFFLOAD_MAPTYPES:@.+]] = private unnamed_addr constant [2 x i64] [i64 800, i64 800]
 
 // CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME:@.+]] = internal unnamed_addr constant [21 x i8]
-// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (double** [[VAR_DECL_TGT_LINK_PTR]] to i8*), i8* getelementptr inbounds ([21 x i8], [21 x i8]* [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME]], i32 0, i32 0), i64 8, i32 1, i32 0 }, section ".omp_offloading.entries"
+// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (double** [[VAR_DECL_TGT_LINK_PTR]] to i8*), i8* getelementptr inbounds ([21 x i8], [21 x i8]* [[OMP_OFFLOAD_ENTRY_LINK_VAR_PTR_NAME]], i32 0, i32 0), i64 8, i32 1, i32 0 }, section "omp_offloading_entries"
 
 // CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR_NAME:@.+]] = internal unnamed_addr constant [24 x i8]
-// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (double** [[VAR_DECL_TGT_TO_PTR]] to i8*), i8* getelementptr inbounds ([24 x i8], [24 x i8]* [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR_NAME]], i32 0, i32 0), i64 8, i32 0, i32 0 }, section ".omp_offloading.entries"
+// CHECK-HOST: [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR:@.+]] = weak constant %struct.__tgt_offload_entry { i8* bitcast (double** [[VAR_DECL_TGT_TO_PTR]] to i8*), i8* getelementptr inbounds ([24 x i8], [24 x i8]* [[OMP_OFFLOAD_ENTRY_TO_VAR_PTR_NAME]], i32 0, i32 0), i64 8, i32 0, i32 0 }, section "omp_offloading_entries"
 
 // CHECK-HOST: [[N_CASTED:%.+]] = alloca i64
 // CHECK-HOST: [[SUM_CASTED:%.+]] = alloca i64
diff --git a/clang/test/OpenMP/openmp_offload_registration.cpp b/clang/test/OpenMP/openmp_offload_registration.cpp
index 4705cb6c0dc3d..4b2e4830dc1fb 100644
--- a/clang/test/OpenMP/openmp_offload_registration.cpp
+++ b/clang/test/OpenMP/openmp_offload_registration.cpp
@@ -16,8 +16,8 @@ void foo() {
 // CHECK-DAG: $[[REGFN:\.omp_offloading\..+\.powerpc64le-ibm-linux-gnu\.x86_64-pc-linux-gnu+]] = comdat any
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEV1BEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEV1END:@.+]] = extern_weak constant i8
 // CHECK: [[DEV2BEGIN:@.+]] = extern_weak constant i8
diff --git a/clang/test/OpenMP/parallel_for_codegen.cpp b/clang/test/OpenMP/parallel_for_codegen.cpp
index e4e76060e7073..9e3390214d78d 100644
--- a/clang/test/OpenMP/parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/parallel_for_codegen.cpp
@@ -15,10 +15,13 @@
 // RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=50 -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix=OMP5 %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -DOMP5 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
 // RUN: %clang_cc1 -fopenmp -fopenmp-version=50 -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix=OMP5 %s
-// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -fopenmp-simd -fopenmp-version=50 -DOMP5 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck --check-prefix SIMD-ONLY0 %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -DOMP5 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp-simd -fopenmp-version=50 -DOMP5 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck --check-prefix SIMD-ONLY0 %s
 #ifndef HEADER
 #define HEADER
 
+#ifndef OMP5
 // CHECK-DAG: [[IDENT_T_TY:%.+]] = type { i32, i32, i32, i32, i8* }
 // CHECK-DAG: [[LOOP_LOC:@.+]] = private unnamed_addr global [[IDENT_T_TY]] { i32 0, i32 514, i32 0, i32 0, i8*
 
@@ -32,12 +35,14 @@ void with_var_schedule() {
 // CHECK: [[CHUNK:%.+]] = load i64, i64* %
 // CHECK: call void {{.+}} @__kmpc_fork_call({{.+}}, i64 [[CHUNK]])
 
+// CHECK: [[UNDEF_A:%.+]] = load double, double* undef
+// CHECK: fadd double 2.000000e+00, [[UNDEF_A]]
 // CHECK: [[CHUNK_VAL:%.+]] = load i8, i8* %
 // CHECK: [[CHUNK_SIZE:%.+]] = sext i8 [[CHUNK_VAL]] to i64
 // CHECK: call void @__kmpc_for_static_init_8u([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID:%[^,]+]], i32 33, i32* [[IS_LAST:%[^,]+]], i64* [[OMP_LB:%[^,]+]], i64* [[OMP_UB:%[^,]+]], i64* [[OMP_ST:%[^,]+]], i64 1, i64 [[CHUNK_SIZE]])
 // CHECK: call void @__kmpc_for_static_fini([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID]])
-#pragma omp parallel for schedule(static, char(a))
-  for (unsigned long long i = 1; i < 2; ++i) {
+#pragma omp parallel for schedule(static, char(a)) private(a)
+  for (unsigned long long i = 1; i < 2 + a; ++i) {
   }
 }
 
@@ -383,16 +388,13 @@ void parallel_for(float *a, const int n) {
 // TERM_DEBUG-DAG: [[DBG_LOC_START]] = !DILocation(line: [[@LINE-4]],
 // TERM_DEBUG-DAG: [[DBG_LOC_END]] = !DILocation(line: [[@LINE-18]],
 
-#ifdef OMP5
-// OMP5-DAG: [[IDENT_T_TY:%.+]] = type { i32, i32, i32, i32, i8* }
-// OMP5-DAG: [[LOOP_LOC:@.+]] = private unnamed_addr global [[IDENT_T_TY]] { i32 0, i32 514, i32 0, i32 0, i8*
-
+#else // OMP5
 // OMP5-LABEL: increment
 int increment () {
-// OMP5: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]])
+// OMP5: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* [[DEFAULT_LOC:[@%].+]])
   #pragma omp for
 // Determine UB = min(UB, GlobalUB)
-// OMP5: call void @__kmpc_for_static_init_4([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID]], i32 34, i32* [[IS_LAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]], i32 1, i32 1)
+// OMP5: call void @__kmpc_for_static_init_4(%struct.ident_t* [[LOOP_LOC:[@%].+]], i32 [[GTID]], i32 34, i32* [[IS_LAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]], i32 1, i32 1)
 // OMP5-NEXT: [[UB:%.+]] = load i32, i32* [[OMP_UB]]
 // OMP5-NEXT: [[UBCMP:%.+]] = icmp sgt i32 [[UB]], 4
 // OMP5-NEXT: br i1 [[UBCMP]], label [[UB_TRUE:%[^,]+]], label [[UB_FALSE:%[^,]+]]
@@ -422,7 +424,7 @@ int increment () {
 // OMP5-NEXT: br label %[[LOOP1_HEAD]]
     ;
 // OMP5: [[LOOP1_END]]
-// OMP5: call void @__kmpc_for_static_fini([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID]])
+// OMP5: call void @__kmpc_for_static_fini(%struct.ident_t* [[LOOP_LOC]], i32 [[GTID]])
 // OMP5: __kmpc_barrier
   return 0;
 // OMP5: ret i32 0
@@ -430,10 +432,10 @@ int increment () {
 
 // OMP5-LABEL: decrement_nowait
 int decrement_nowait () {
-// OMP5: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num([[IDENT_T_TY]]* [[DEFAULT_LOC:[@%].+]])
+// OMP5: [[GTID:%.+]] = call i32 @__kmpc_global_thread_num(%struct.ident_t* [[DEFAULT_LOC:[@%].+]])
   #pragma omp for nowait
 // Determine UB = min(UB, GlobalUB)
-// OMP5: call void @__kmpc_for_static_init_4([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID]], i32 34, i32* [[IS_LAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]], i32 1, i32 1)
+// OMP5: call void @__kmpc_for_static_init_4(%struct.ident_t* [[LOOP_LOC]], i32 [[GTID]], i32 34, i32* [[IS_LAST:%[^,]+]], i32* [[OMP_LB:%[^,]+]], i32* [[OMP_UB:%[^,]+]], i32* [[OMP_ST:%[^,]+]], i32 1, i32 1)
 // OMP5-NEXT: [[UB:%.+]] = load i32, i32* [[OMP_UB]]
 // OMP5-NEXT: [[UBCMP:%.+]] = icmp sgt i32 [[UB]], 4
 // OMP5-NEXT: br i1 [[UBCMP]], label [[UB_TRUE:%[^,]+]], label [[UB_FALSE:%[^,]+]]
@@ -462,12 +464,12 @@ int decrement_nowait () {
 // OMP5-NEXT: br label %[[LOOP1_HEAD]]
     ;
 // OMP5: [[LOOP1_END]]
-// OMP5: call void @__kmpc_for_static_fini([[IDENT_T_TY]]* [[LOOP_LOC]], i32 [[GTID]])
+// OMP5: call void @__kmpc_for_static_fini(%struct.ident_t* [[LOOP_LOC]], i32 [[GTID]])
 // OMP5-NOT: __kmpc_barrier
   return 0;
 // OMP5: ret i32 0
 }
-#endif
+#endif // OMP5
 
 #endif // HEADER
 
diff --git a/clang/test/OpenMP/target_codegen.cpp b/clang/test/OpenMP/target_codegen.cpp
index a6f0da171b2a6..e1db9c9d49a6f 100644
--- a/clang/test/OpenMP/target_codegen.cpp
+++ b/clang/test/OpenMP/target_codegen.cpp
@@ -88,8 +88,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_codegen_registration.cpp b/clang/test/OpenMP/target_codegen_registration.cpp
index 31a1c004a8306..cc56f7a8dc57a 100644
--- a/clang/test/OpenMP/target_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME1]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME1]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME2]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME2]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME3]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME3]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME4]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME4]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME5]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME5]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME6]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME6]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME7]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME7]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME8]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME8]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME9]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME9]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME10]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME10]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME11]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME11]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: @.omp_offloading.entry.[[NAME12]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: @.omp_offloading.entry.[[NAME12]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME1]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME1]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME2]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME2]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME3]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME3]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME4]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME4]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME5]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME5]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME6]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME6]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME7]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME7]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME8]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME8]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME9]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME9]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME10]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME10]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME11]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME11]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: @.omp_offloading.entry.[[NAME12]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: @.omp_offloading.entry.[[NAME12]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_depend_codegen.cpp b/clang/test/OpenMP/target_depend_codegen.cpp
index a5b839047f408..735fb763c762c 100644
--- a/clang/test/OpenMP/target_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_map_codegen.cpp b/clang/test/OpenMP/target_map_codegen.cpp
index ca6680237ad04..0a8198a904178 100644
--- a/clang/test/OpenMP/target_map_codegen.cpp
+++ b/clang/test/OpenMP/target_map_codegen.cpp
@@ -1323,172 +1323,176 @@ void implicit_maps_template_type_capture (int a){
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK19
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1510.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1514.region_id = weak constant i8 0
 // CK19: [[SIZE00:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1531.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1535.region_id = weak constant i8 0
 // CK19: [[SIZE00n:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE00n:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1553.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1557.region_id = weak constant i8 0
 // CK19: [[SIZE01:@.+]] = private {{.*}}constant [1 x i64] [i64 400]
 // CK19: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1572.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1576.region_id = weak constant i8 0
 // CK19: [[SIZE02:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE02:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1591.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1595.region_id = weak constant i8 0
 // CK19: [[SIZE03:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE03:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1610.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1614.region_id = weak constant i8 0
 // CK19: [[SIZE04:@.+]] = private {{.*}}constant [1 x i64] [i64 400]
 // CK19: [[MTYPE04:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1629.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1633.region_id = weak constant i8 0
 // CK19: [[SIZE05:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE05:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1652.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1656.region_id = weak constant i8 0
 // CK19: [[MTYPE06:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1675.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1679.region_id = weak constant i8 0
 // CK19: [[MTYPE07:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1694.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1698.region_id = weak constant i8 0
 // CK19: [[SIZE08:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE08:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1715.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1719.region_id = weak constant i8 0
 // CK19: [[SIZE09:@.+]] = private {{.*}}constant [1 x i64] [i64 {{8|4}}]
 // CK19: [[MTYPE09:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1736.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1740.region_id = weak constant i8 0
 // CK19: [[SIZE10:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE10:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1757.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1761.region_id = weak constant i8 0
 // CK19: [[SIZE11:@.+]] = private {{.*}}constant [1 x i64] [i64 240]
 // CK19: [[MTYPE11:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1778.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1782.region_id = weak constant i8 0
 // CK19: [[SIZE12:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE12:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1803.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1807.region_id = weak constant i8 0
 // CK19: [[MTYPE13:@.+]] = private {{.*}}constant [1 x i64] [i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1828.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1832.region_id = weak constant i8 0
 // CK19: [[MTYPE14:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1849.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1853.region_id = weak constant i8 0
 // CK19: [[SIZE15:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE15:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1883.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1887.region_id = weak constant i8 0
 // CK19: [[MTYPE16:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1909.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1913.region_id = weak constant i8 0
 // CK19: [[SIZE17:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 240]
 // CK19: [[MTYPE17:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 34]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1935.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1939.region_id = weak constant i8 0
 // CK19: [[SIZE18:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 240]
 // CK19: [[MTYPE18:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1967.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1971.region_id = weak constant i8 0
 // CK19: [[MTYPE19:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 32]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1993.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l1997.region_id = weak constant i8 0
 // CK19: [[SIZE20:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE20:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 33]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2025.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2029.region_id = weak constant i8 0
 // CK19: [[MTYPE21:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2051.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2055.region_id = weak constant i8 0
 // CK19: [[SIZE22:@.+]] = private {{.*}}constant [2 x i64] [i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE22:@.+]] = private {{.*}}constant [2 x i64] [i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2070.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2074.region_id = weak constant i8 0
 // CK19: [[SIZE23:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE23:@.+]] = private {{.*}}constant [1 x i64] [i64 39]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2092.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2096.region_id = weak constant i8 0
 // CK19: [[SIZE24:@.+]] = private {{.*}}constant [1 x i64] [i64 480]
 // CK19: [[MTYPE24:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2113.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2117.region_id = weak constant i8 0
 // CK19: [[SIZE25:@.+]] = private {{.*}}constant [1 x i64] [i64 16]
 // CK19: [[MTYPE25:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2134.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2138.region_id = weak constant i8 0
 // CK19: [[SIZE26:@.+]] = private {{.*}}constant [1 x i64] [i64 24]
 // CK19: [[MTYPE26:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2155.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2159.region_id = weak constant i8 0
 // CK19: [[SIZE27:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK19: [[MTYPE27:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2200.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2204.region_id = weak constant i8 0
 // CK19: [[SIZE28:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 16]
 // CK19: [[MTYPE28:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2245.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2249.region_id = weak constant i8 0
 // CK19: [[SIZE29:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 4]
 // CK19: [[MTYPE29:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2301.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2305.region_id = weak constant i8 0
 // CK19: [[MTYPE30:@.+]] = private {{.*}}constant [4 x i64] [i64 800, i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2345.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2349.region_id = weak constant i8 0
 // CK19: [[SIZE31:@.+]] = private {{.*}}constant [4 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 {{8|4}}, i64 40]
 // CK19: [[MTYPE31:@.+]] = private {{.*}}constant [4 x i64] [i64 800, i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2368.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2372.region_id = weak constant i8 0
 // CK19: [[SIZE32:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE32:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2387.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2391.region_id = weak constant i8 0
 // CK19: [[SIZE33:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE33:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2406.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2410.region_id = weak constant i8 0
 // CK19: [[SIZE34:@.+]] = private {{.*}}constant [1 x i64] [i64 13728]
 // CK19: [[MTYPE34:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2431.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2435.region_id = weak constant i8 0
 // CK19: [[MTYPE35:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2452.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2456.region_id = weak constant i8 0
 // CK19: [[SIZE36:@.+]] = private {{.*}}constant [1 x i64] [i64 208]
 // CK19: [[MTYPE36:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2492.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2496.region_id = weak constant i8 0
 // CK19: [[MTYPE37:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2534.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2538.region_id = weak constant i8 0
 // CK19: [[MTYPE38:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2576.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2580.region_id = weak constant i8 0
 // CK19: [[MTYPE39:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2618.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2622.region_id = weak constant i8 0
 // CK19: [[MTYPE40:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2653.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2657.region_id = weak constant i8 0
 // CK19: [[SIZE41:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 208]
 // CK19: [[MTYPE41:@.+]] = private {{.*}}constant [3 x i64] [i64 800, i64 800, i64 35]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2698.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2702.region_id = weak constant i8 0
 // CK19: [[SIZE42:@.+]] = private {{.*}}constant [3 x i64] [i64 {{8|4}}, i64 {{8|4}}, i64 104]
 // CK19: [[MTYPE42:@.+]] = private {{.*}}constant [3 x i64] [i64 35, i64 16, i64 19]
 
-// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2723.region_id = weak constant i8 0
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2727.region_id = weak constant i8 0
 // CK19: [[MTYPE43:@.+]] = private {{.*}}constant [1 x i64] [i64 35]
 
+// CK19-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l2746.region_id = weak constant i8 0
+// CK19: [[SIZE44:@.+]] = private {{.*}}constant [1 x i64] [i64 320]
+// CK19: [[MTYPE44:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
+
 // CK19-LABEL: explicit_maps_single{{.*}}(
 void explicit_maps_single (int ii){
   // Map of a scalar.
@@ -2725,6 +2729,25 @@ void explicit_maps_single (int ii){
     marras[1][2][3]++;
   }
 
+  // Region 44
+  // CK19-DAG: call i32 @__tgt_target(i64 {{[^,]+}}, i8* {{[^,]+}}, i32 1, i8** [[GEPBP:%.+]], i8** [[GEPP:%.+]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[SIZE44]], {{.+}}getelementptr {{.+}}[1 x i{{.+}}]* [[MTYPE44]]{{.+}})
+  // CK19-DAG: [[GEPBP]] = getelementptr inbounds {{.+}}[[BP:%[^,]+]]
+  // CK19-DAG: [[GEPP]] = getelementptr inbounds {{.+}}[[P:%[^,]+]]
+
+  // CK19-DAG: [[BP0:%.+]] = getelementptr inbounds {{.+}}[[BP]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[P0:%.+]] = getelementptr inbounds {{.+}}[[P]], i{{.+}} 0, i{{.+}} 0
+  // CK19-DAG: [[CBP0:%.+]] = bitcast i8** [[BP0]] to [100 x i32]**
+  // CK19-DAG: [[CP0:%.+]] = bitcast i8** [[P0]] to i32**
+  // CK19-DAG: store [100 x i32]* [[VAR0:%.+]], [100 x i32]** [[CBP0]]
+  // CK19-DAG: store i32* [[SEC0:%[^,]+]], i32** [[CP0]]
+  // CK19-DAG: [[SEC0]] = getelementptr {{.*}}[100 x i32]* [[VAR0]], i{{.+}} 0, i{{.+}} 20
+
+  // CK19: call void [[CALL44:@.+]]([100 x i32]* {{[^,]+}})
+  #pragma omp target map(from:arra[20:])
+  {
+    arra[50]++;
+  }
+
 }
 
 // CK19: define {{.+}}[[CALL00]]
@@ -2771,6 +2794,7 @@ void explicit_maps_single (int ii){
 // CK19: define {{.+}}[[CALL41]]
 // CK19: define {{.+}}[[CALL42]]
 // CK19: define {{.+}}[[CALL43]]
+// CK19: define {{.+}}[[CALL44]]
 
 #endif
 ///==========================================================================///
@@ -2790,19 +2814,19 @@ void explicit_maps_single (int ii){
 // SIMD-ONLY19-NOT: {{__kmpc|__tgt}}
 #ifdef CK20
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2832.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2856.region_id = weak constant i8 0
 // CK20: [[SIZE00:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK20: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2853.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2877.region_id = weak constant i8 0
 // CK20: [[SIZE01:@.+]] = private {{.*}}constant [1 x i64] [i64 20]
 // CK20: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 33]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2871.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2895.region_id = weak constant i8 0
 // CK20: [[SIZE02:@.+]] = private {{.*}}constant [1 x i64] [i64 4]
 // CK20: [[MTYPE02:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
-// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2892.region_id = weak constant i8 0
+// CK20-LABEL: @.__omp_offloading_{{.*}}explicit_maps_references_and_function_args{{.*}}_l2916.region_id = weak constant i8 0
 // CK20: [[SIZE03:@.+]] = private {{.*}}constant [1 x i64] [i64 12]
 // CK20: [[MTYPE03:@.+]] = private {{.*}}constant [1 x i64] [i64 34]
 
@@ -5275,11 +5299,11 @@ void map_with_deep_copy() {
 // SIMD-ONLY18-NOT: {{__kmpc|__tgt}}
 #ifdef CK31
 
-// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5305.region_id = weak constant i8 0
+// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5329.region_id = weak constant i8 0
 // CK31: [[SIZE00:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK31: [[MTYPE00:@.+]] = private {{.*}}constant [1 x i64] [i64 1059]
 
-// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5324.region_id = weak constant i8 0
+// CK31-LABEL: @.__omp_offloading_{{.*}}explicit_maps_single{{.*}}_l5348.region_id = weak constant i8 0
 // CK31: [[SIZE01:@.+]] = private {{.*}}constant [1 x i[[Z:64|32]]] [i[[Z:64|32]] 4]
 // CK31: [[MTYPE01:@.+]] = private {{.*}}constant [1 x i64] [i64 1063]
 
diff --git a/clang/test/OpenMP/target_parallel_codegen.cpp b/clang/test/OpenMP/target_parallel_codegen.cpp
index ca12a4c30ba85..48cea928ce972 100644
--- a/clang/test/OpenMP/target_parallel_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_codegen.cpp
@@ -83,8 +83,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_codegen_registration.cpp
index a1aa66eb5697b..380e00e55b422 100644
--- a/clang/test/OpenMP/target_parallel_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
index b7d338c1f272e..6217de39c0933 100644
--- a/clang/test/OpenMP/target_parallel_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_codegen.cpp b/clang/test/OpenMP/target_parallel_for_codegen.cpp
index 66d2a233a83b2..3d80d990e87a5 100644
--- a/clang/test/OpenMP/target_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_codegen.cpp
@@ -84,8 +84,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
index aac7795c046ce..a2625c3447068 100644
--- a/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_for_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
index f05bb6e66905e..a6b434e137a92 100644
--- a/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
index a6087c0cbbe8f..89fcf0a1f977f 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_codegen.cpp
@@ -83,8 +83,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp b/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
index 26f6b1ec522d4..072fb44d7b474 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
index 57b91f2a0c82b..7c1f2bc7bcc74 100644
--- a/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_for_simd_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_if_codegen.cpp b/clang/test/OpenMP/target_parallel_if_codegen.cpp
index ae862320f8d03..e3ffe58aac862 100644
--- a/clang/test/OpenMP/target_parallel_if_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_if_codegen.cpp
@@ -68,8 +68,8 @@
 // TCHECK: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_parallel_num_threads_codegen.cpp b/clang/test/OpenMP/target_parallel_num_threads_codegen.cpp
index f3bc5a441a932..8a3d45054873d 100644
--- a/clang/test/OpenMP/target_parallel_num_threads_codegen.cpp
+++ b/clang/test/OpenMP/target_parallel_num_threads_codegen.cpp
@@ -68,8 +68,8 @@
 // TCHECK: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_simd_codegen.cpp b/clang/test/OpenMP/target_simd_codegen.cpp
index b68745e1ade96..f561e545f6590 100644
--- a/clang/test/OpenMP/target_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_codegen.cpp
@@ -80,8 +80,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_simd_codegen_registration.cpp b/clang/test/OpenMP/target_simd_codegen_registration.cpp
index b27fb106c0668..61c0ae5bce562 100644
--- a/clang/test/OpenMP/target_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_simd_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_simd_depend_codegen.cpp b/clang/test/OpenMP/target_simd_depend_codegen.cpp
index 5845daee495cb..f07de8f9d671d 100644
--- a/clang/test/OpenMP/target_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_simd_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_codegen.cpp b/clang/test/OpenMP/target_teams_codegen.cpp
index 4713f0766dc78..c788a20966fe6 100644
--- a/clang/test/OpenMP/target_teams_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_codegen.cpp
@@ -89,8 +89,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_codegen_registration.cpp b/clang/test/OpenMP/target_teams_codegen_registration.cpp
index 7c6428b154be4..bc0eecae4807b 100644
--- a/clang/test/OpenMP/target_teams_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_depend_codegen.cpp b/clang/test/OpenMP/target_teams_depend_codegen.cpp
index f47122eab4e27..8591d65f29dba 100644
--- a/clang/test/OpenMP/target_teams_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_codegen.cpp
index 28ea08ff1bd33..feb6b46d187c9 100644
--- a/clang/test/OpenMP/target_teams_distribute_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_codegen.cpp
@@ -86,8 +86,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
index 4db52b21598ce..b10823aa04333 100644
--- a/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
index e6aee65e20710..e10f8002a97c6 100644
--- a/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_codegen.cpp
index 4c7ac2441cc5c..1617f8510a379 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_codegen.cpp
@@ -54,17 +54,14 @@ int target_teams_fun(int *g){
   // discard capture expressions for te and th
   // HCK1: = alloca i32,
   // HCK1: = alloca i32,
-  // HCK1: = alloca i32,
-  // HCK1: = alloca i32,
-  // HCK1: = alloca i32,
   // HCK1: [[N_CAST:%.+]] = alloca i{{32|64}},
   // HCK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
   // HCK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-  // HCK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // HCK1: [[N_PAR:%.+]] = load{{.+}}, {{.+}} [[N_CAST]],
   // HCK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
   // HCK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
-  // HCK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 4, i8** %{{[^,]+}}, i8** %{{[^,]+}}, 
+  // HCK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
+  // HCK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 4, i8** %{{[^,]+}}, i8** %{{[^,]+}},
 
   // HCK1: call void @[[OFFL1:.+]](i{{32|64}} [[N_PAR]], {{.+}}, i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]])
   #pragma omp target teams distribute parallel for num_teams(te), thread_limit(th)
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
index c8615a9fc5b55..e2cdd39145d03 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen.cpp
index 6846aaf434d2e..2c74b18fbd00c 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen.cpp
@@ -52,18 +52,15 @@ int target_teams_fun(int *g){
 // discard capture expressions for te and th
 // HCK1: = alloca i32,
 // HCK1: = alloca i32,
-// HCK1: = alloca i32,
-// HCK1: = alloca i32,
-// HCK1: = alloca i32,
 // HCK1: [[I_CAST:%.+]] = alloca i{{32|64}},
 // HCK1: [[N_CAST:%.+]] = alloca i{{32|64}},
 // HCK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
 // HCK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-// HCK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
 // HCK1: [[I_PAR:%.+]] = load{{.+}}, {{.+}} [[I_CAST]],
 // HCK1: [[N_PAR:%.+]] = load{{.+}}, {{.+}} [[N_CAST]],
 // HCK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
 // HCK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
+// HCK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
 // HCK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 5, i8** %{{[^,]+}}, i8** %{{[^,]+}},
 
 // HCK1: call void @[[OFFL1:.+]](i{{32|64}} [[I_PAR]], i{{32|64}} [[N_PAR]], {{.+}}, i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]])
@@ -77,8 +74,7 @@ int target_teams_fun(int *g){
   // HCK1: call void @[[OFFL2:.+]](i{{64|32}} %{{.+}})
   {{{
   #pragma omp target teams distribute parallel for simd is_device_ptr(g) simdlen(8)
-  for(
-    int i = 0; i < n; i++) {
+  for(int i = 0; i < n; i++) {
     a[i] = g[0];
   }
   }}}
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
index 147ead404e1f3..3d12ad3b85a33 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
index 8d46979a1207b..1b2800606a270 100644
--- a/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_parallel_for_simd_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
index 37615f0f032c9..de05d82db4a58 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_codegen.cpp
@@ -86,8 +86,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp b/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
index 843f377f6e9da..36c3aaec2b096 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_codegen_registration.cpp
@@ -123,57 +123,57 @@
 // CHECK-NTARGET-NOT: private unnamed_addr constant [1 x i
 
 // CHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // CHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// CHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* @{{.*}}, i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
 // TCHECK-DAG: [[NAMEPTR1:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME1:__omp_offloading_[0-9a-f]+_[0-9a-f]+__Z.+_l[0-9]+]]\00"
-// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY1:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR1]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR2:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME2:.+]]\00"
-// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY2:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR2]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR3:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME3:.+]]\00"
-// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY3:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR3]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR4:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME4:.+]]\00"
-// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY4:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR4]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR5:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME5:.+]]\00"
-// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY5:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR5]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR6:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME6:.+]]\00"
-// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY6:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR6]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR7:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME7:.+]]\00"
-// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY7:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR7]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR8:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME8:.+]]\00"
-// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY8:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR8]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR9:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME9:.+]]\00"
-// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY9:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR9]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR10:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME10:.+]]\00"
-// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY10:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR10]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR11:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME11:.+]]\00"
-// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY11:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR11]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 // TCHECK-DAG: [[NAMEPTR12:@.+]] = internal unnamed_addr constant [{{.*}} x i8] c"[[NAME12:.+]]\00"
-// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section ".omp_offloading.entries", align 1
+// TCHECK-DAG: [[ENTRY12:@.+]] = weak constant [[ENTTY]] { i8* bitcast (void (i[[SZ]])* @{{.*}} to i8*), i8* getelementptr inbounds ([{{.*}} x i8], [{{.*}} x i8]* [[NAMEPTR12]], i32 0, i32 0), i[[SZ]] 0, i32 0, i32 0 }, section "omp_offloading_entries", align 1
 
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
index b7bb85df77bba..5789d1cd394b1 100644
--- a/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_distribute_simd_depend_codegen.cpp
@@ -56,8 +56,8 @@
 // TCHECK-NOT: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_num_teams_codegen.cpp b/clang/test/OpenMP/target_teams_num_teams_codegen.cpp
index 3d2eb55f1d5ff..35e8bbc11ca6c 100644
--- a/clang/test/OpenMP/target_teams_num_teams_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_num_teams_codegen.cpp
@@ -68,8 +68,8 @@
 // TCHECK: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/target_teams_thread_limit_codegen.cpp b/clang/test/OpenMP/target_teams_thread_limit_codegen.cpp
index 3dfd7532190a4..6f0e27e9e6340 100644
--- a/clang/test/OpenMP/target_teams_thread_limit_codegen.cpp
+++ b/clang/test/OpenMP/target_teams_thread_limit_codegen.cpp
@@ -68,8 +68,8 @@
 // TCHECK: @{{.+}} = weak constant [[ENTTY]]
 
 // Check if offloading descriptor is created.
-// CHECK: [[ENTBEGIN:@.+]] = external constant [[ENTTY]]
-// CHECK: [[ENTEND:@.+]] = external constant [[ENTTY]]
+// CHECK: [[ENTBEGIN:@.+]] = external hidden constant [[ENTTY]]
+// CHECK: [[ENTEND:@.+]] = external hidden constant [[ENTTY]]
 // CHECK: [[DEVBEGIN:@.+]] = extern_weak constant i8
 // CHECK: [[DEVEND:@.+]] = extern_weak constant i8
 // CHECK: [[IMAGES:@.+]] = internal unnamed_addr constant [1 x [[DEVTY]]] [{{.+}} { i8* [[DEVBEGIN]], i8* [[DEVEND]], [[ENTTY]]* [[ENTBEGIN]], [[ENTTY]]* [[ENTEND]] }], comdat($[[REGFN]])
diff --git a/clang/test/OpenMP/teams_distribute_codegen.cpp b/clang/test/OpenMP/teams_distribute_codegen.cpp
index ea299bc346a66..0065c6cbd51cb 100644
--- a/clang/test/OpenMP/teams_distribute_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_codegen.cpp
@@ -21,22 +21,19 @@
 int a[100];
 
 // CK1: define {{.*}}i32 @{{.+}}teams_argument_globali(
-int teams_argument_global(int n){
+int teams_argument_global(int n) {
   int te = n / 128;
   int th = 128;
   // discard n_addr
   // CK1: alloca i32,
   // CK1: [[TE:%.+]] = alloca i32,
   // CK1: [[TH:%.+]] = alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
   // CK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
   // CK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
   // CK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
 
+  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 4, i8** %{{[^,]+}}, i8** %{{[^,]+}}, i{{64|32}}* {{.+}}@{{[^,]+}}, i32 0, i32 0), i64* {{.+}}@{{[^,]+}}, i32 0, i32 0), i32 {{.+}}, i32 {{.+}})
 
   // CK1: call void @[[OFFL1:.+]](i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]],
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
index fa425b3ecab0c..bc42f740ad741 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_codegen.cpp
@@ -28,14 +28,11 @@ int teams_argument_global(int n){
   // CK1: alloca i32,
   // CK1: [[TE:%.+]] = alloca i32,
   // CK1: [[TH:%.+]] = alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
   // CK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
   // CK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
   // CK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
+  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 4, i8** %{{[^,]+}}, i8** %{{[^,]+}}, i{{64|32}}* {{.+}}@{{[^,]+}}, i32 0, i32 0), i64* {{.+}}@{{[^,]+}}, i32 0, i32 0), i32 {{.+}}, i32 {{.+}})
 
   // CK1: call void @[[OFFL1:.+]](i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]],
diff --git a/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp b/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
index 45793419d7c4b..fd2b2da5333ba 100644
--- a/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_parallel_for_simd_codegen.cpp
@@ -28,15 +28,12 @@ int teams_argument_global(int n){
   // CK1: alloca i32,
   // CK1: [[TE:%.+]] = alloca i32,
   // CK1: [[TH:%.+]] = alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
   // CK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
   // CK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
   // CK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
 
+  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 4, i8** %{{[^,]+}}, i8** %{{[^,]+}}, i{{64|32}}* {{.+}}@{{[^,]+}}, i32 0, i32 0), i64* {{.+}}@{{[^,]+}}, i32 0, i32 0)
 
   // CK1: call void @[[OFFL1:.+]](i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]],
diff --git a/clang/test/OpenMP/teams_distribute_simd_codegen.cpp b/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
index ab1482855e750..ff2a752cfa348 100644
--- a/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
+++ b/clang/test/OpenMP/teams_distribute_simd_codegen.cpp
@@ -30,15 +30,12 @@ int teams_argument_global(int n) {
   // CK1: alloca i32,
   // CK1: [[TE:%.+]] = alloca i32,
   // CK1: [[TH:%.+]] = alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
-  // CK1: alloca i32,
   // CK1: [[TE_CAST:%.+]] = alloca i{{32|64}},
   // CK1: [[TH_CAST:%.+]] = alloca i{{32|64}},
-  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: [[TE_PAR:%.+]] = load{{.+}}, {{.+}} [[TE_CAST]],
   // CK1: [[TH_PAR:%.+]] = load{{.+}}, {{.+}} [[TH_CAST]],
 
+  // CK1: call void @__kmpc_push_target_tripcount(i64 -1, i64 %{{.+}})
   // CK1: call i32 @__tgt_target_teams(i64 -1, i8* @{{[^,]+}}, i32 5, i8** %{{[^,]+}}, i8** %{{[^,]+}}, i{{64|32}}* {{.+}}@{{[^,]+}}, i32 0, i32 0), i64* {{.+}}@{{[^,]+}}, i32 0, i32 0), i32 {{.+}}, i32 1)
 
   // CK1: call void @[[OFFL1:.+]](i{{32|64}} [[TE_PAR]], i{{32|64}} [[TH_PAR]],
diff --git a/clang/test/Preprocessor/pragma_module.c b/clang/test/Preprocessor/pragma_module.c
index 90aa9481fb13c..c52d42537573f 100644
--- a/clang/test/Preprocessor/pragma_module.c
+++ b/clang/test/Preprocessor/pragma_module.c
@@ -1,9 +1,9 @@
 // RUN: rm -rf %t
 // RUN: mkdir %t
 // RUN: echo 'module foo { module a {} module b {} } module bar {} module if {}' > %t/module.map
-// RUN: %clang -cc1 -fmodules -fmodule-name=if -x c %t/module.map -emit-module -o %t/if.pcm
-// RUN: %clang -cc1 -E -fmodules %s -fmodule-file=%t/if.pcm -verify -fmodule-name=foo -fmodule-map-file=%t/module.map
-// RUN: %clang -cc1 -E -fmodules %s -fmodule-file=%t/if.pcm -verify -fmodule-name=foo -fmodule-map-file=%t/module.map -fmodules-local-submodule-visibility -DLOCAL_VIS
+// RUN: %clang_cc1 -fmodules -fmodule-name=if -x c %t/module.map -emit-module -o %t/if.pcm
+// RUN: %clang_cc1 -E -fmodules %s -fmodule-file=%t/if.pcm -verify -fmodule-name=foo -fmodule-map-file=%t/module.map
+// RUN: %clang_cc1 -E -fmodules %s -fmodule-file=%t/if.pcm -verify -fmodule-name=foo -fmodule-map-file=%t/module.map -fmodules-local-submodule-visibility -DLOCAL_VIS
 
 // Just checking the syntax here; the semantics are tested elsewhere.
 #pragma clang module import // expected-error {{expected module name}}
diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index 5981c80ae3a1a..95bea654fe220 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -3242,6 +3242,9 @@
 // RUN: %clang -march=arch13 -E -dM %s -o - 2>&1 \
 // RUN:     -target s390x-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SYSTEMZ_ARCH13
+// RUN: %clang -march=z15 -E -dM %s -o - 2>&1 \
+// RUN:     -target s390x-unknown-linux \
+// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SYSTEMZ_ARCH13
 // CHECK_SYSTEMZ_ARCH13: #define __ARCH__ 13
 // CHECK_SYSTEMZ_ARCH13: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_1 1
 // CHECK_SYSTEMZ_ARCH13: #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_2 1
diff --git a/clang/test/Preprocessor/riscv-cmodel.c b/clang/test/Preprocessor/riscv-cmodel.c
new file mode 100644
index 0000000000000..c0b85eba16bda
--- /dev/null
+++ b/clang/test/Preprocessor/riscv-cmodel.c
@@ -0,0 +1,30 @@
+// RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64i -x c -E -dM %s \
+// RUN: -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
+// RUN: -mcmodel=small -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64i -x c -E -dM %s \
+// RUN: -mcmodel=small -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
+// RUN: -mcmodel=medlow -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64i -x c -E -dM %s \
+// RUN: -mcmodel=medlow -o - | FileCheck --check-prefix=CHECK-MEDLOW %s
+
+// CHECK-MEDLOW: #define __riscv_cmodel_medlow 1
+// CHECK-MEDLOW-NOT: __riscv_cmodel_medany
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
+// RUN: -mcmodel=medium -o - | FileCheck --check-prefix=CHECK-MEDANY %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64i -x c -E -dM %s \
+// RUN: -mcmodel=medium -o - | FileCheck --check-prefix=CHECK-MEDANY %s
+
+// RUN: %clang -target riscv32-unknown-linux-gnu -march=rv32i -x c -E -dM %s \
+// RUN: -mcmodel=medany -o - | FileCheck --check-prefix=CHECK-MEDANY %s
+// RUN: %clang -target riscv64-unknown-linux-gnu -march=rv64i -x c -E -dM %s \
+// RUN: -mcmodel=medany -o - | FileCheck --check-prefix=CHECK-MEDANY %s
+
+// CHECK-MEDANY: #define __riscv_cmodel_medany 1
+// CHECK-MEDANY-NOT: __riscv_cmodel_medlow
diff --git a/clang/test/Sema/builtin-preserve-access-index.c b/clang/test/Sema/builtin-preserve-access-index.c
index c10ceb5145b8c..8cd829da200b1 100644
--- a/clang/test/Sema/builtin-preserve-access-index.c
+++ b/clang/test/Sema/builtin-preserve-access-index.c
@@ -4,10 +4,36 @@ const void *invalid1(const int *arg) {
   return __builtin_preserve_access_index(&arg[1], 1); // expected-error {{too many arguments to function call, expected 1, have 2}}
 }
 
-void *invalid2(const int *arg) {
-  return __builtin_preserve_access_index(&arg[1]); // expected-warning {{returning 'const void *' from a function with result type 'void *' discards qualifiers}}
+int valid2(void) {
+  return __builtin_preserve_access_index(1);
 }
 
-const void *invalid3(const int *arg) {
-  return __builtin_preserve_access_index(1); // expected-warning {{incompatible integer to pointer conversion passing 'int' to parameter of type 'const void *'}}
+void *invalid3(const int *arg) {
+  return __builtin_preserve_access_index(&arg[1]); // expected-warning {{returning 'const int *' from a function with result type 'void *' discards qualifiers}}
+}
+
+const void *invalid4(volatile const int *arg) {
+  return __builtin_preserve_access_index(arg); // expected-warning {{returning 'const volatile int *' from a function with result type 'const void *' discards qualifiers}}
+}
+
+int *valid5(int *arg) {
+  return __builtin_preserve_access_index(arg);
+}
+
+int valid6(const volatile int *arg) {
+  return *__builtin_preserve_access_index(arg);
+}
+
+struct s { int a; int b; };
+
+int valid7(struct s *arg) {
+  return *__builtin_preserve_access_index(&arg->b);
+}
+
+int valid8(struct s *arg) {
+  return __builtin_preserve_access_index(arg->a + arg->b);
+}
+
+int valid9(struct s *arg) {
+  return __builtin_preserve_access_index(({arg->a = 2; arg->b = 3; }));
 }
diff --git a/clang/test/Sema/builtins-x86.c b/clang/test/Sema/builtins-x86.c
index dca0bdc720a0e..cbaf7bcde871e 100644
--- a/clang/test/Sema/builtins-x86.c
+++ b/clang/test/Sema/builtins-x86.c
@@ -37,7 +37,7 @@ __m128d test__builtin_ia32_cmpsd(__m128d __a, __m128d __b) {
   return __builtin_ia32_cmpsd(__a, __b, 32); // expected-error {{argument value 32 is outside the valid range [0, 31]}}
 }
 
-__mmask16 test__builtin_ia32_cmpps512_mask(__m512d __a, __m512d __b) {
+__mmask16 test__builtin_ia32_cmpps512_mask(__m512 __a, __m512 __b) {
   return __builtin_ia32_cmpps512_mask(__a, __b, 32, -1, 4); // expected-error {{argument value 32 is outside the valid range [0, 31]}}
 }
 
@@ -185,3 +185,19 @@ __m256i test_mm256_shrdi_epi16(__m256i __A, __m256i __B) {
 __m128i test_mm128_shrdi_epi16(__m128i __A, __m128i __B) {
   return __builtin_ia32_vpshrdw128(__A, __B, 1024); // expected-error {{argument value 1024 is outside the valid range [0, 255]}}
 }
+
+unsigned char test_lwpins32(unsigned int data2, unsigned int data1, unsigned int flags) {
+  return __builtin_ia32_lwpins32(data2, data1, flags); // expected-error {{argument to '__builtin_ia32_lwpins32' must be a constant integer}}
+}
+
+void test_lwpval32(unsigned int data2, unsigned int data1, unsigned int flags) {
+  __builtin_ia32_lwpval32(data2, data1, flags); // expected-error {{argument to '__builtin_ia32_lwpval32' must be a constant integer}}
+}
+
+unsigned char test_lwpins64(unsigned long long data2, unsigned long long data1, unsigned int flags) {
+  return __builtin_ia32_lwpins64(data2, data1, flags); // expected-error {{argument to '__builtin_ia32_lwpins64' must be a constant integer}}
+}
+
+void test_lwpval64(unsigned long long data2, unsigned long long data1, unsigned int flags) {
+  __builtin_ia32_lwpval64(data2, data1, flags); // expected-error {{argument to '__builtin_ia32_lwpval64' must be a constant integer}}
+}
diff --git a/clang/test/Sema/div-sizeof-array.cpp b/clang/test/Sema/div-sizeof-array.cpp
index 15da1da0f9377..7c76a5265b85e 100644
--- a/clang/test/Sema/div-sizeof-array.cpp
+++ b/clang/test/Sema/div-sizeof-array.cpp
@@ -9,21 +9,35 @@ int f(Ty (&Array)[N]) {
 typedef int int32;
 
 void test(void) {
-  int arr[12];                // expected-note 2 {{array 'arr' declared here}}
+  int arr[12]; // expected-note 2 {{array 'arr' declared here}}
   unsigned long long arr2[4];
   int *p = &arr[0];
   int a1 = sizeof(arr) / sizeof(*arr);
   int a2 = sizeof arr / sizeof p; // expected-warning {{expression does not compute the number of elements in this array; element type is 'int', not 'int *'}}
+  // expected-note@-1 {{place parentheses around the 'sizeof p' expression to silence this warning}}
   int a4 = sizeof arr2 / sizeof p;
   int a5 = sizeof(arr) / sizeof(short); // expected-warning {{expression does not compute the number of elements in this array; element type is 'int', not 'short'}}
+  // expected-note@-1 {{place parentheses around the 'sizeof(short)' expression to silence this warning}}
   int a6 = sizeof(arr) / sizeof(int32);
   int a7 = sizeof(arr) / sizeof(int);
   int a9 = sizeof(arr) / sizeof(unsigned int);
   const char arr3[2] = "A";
   int a10 = sizeof(arr3) / sizeof(char);
+  int a11 = sizeof(arr2) / (sizeof(unsigned));
+  int a12 = sizeof(arr) / (sizeof(short));
 
-  int arr4[10][12];                         // expected-note 3 {{array 'arr4' declared here}}
-  int b1 = sizeof(arr4) / sizeof(arr2[12]); // expected-warning {{expression does not compute the number of elements in this array; element type is 'int [12]', not 'unsigned long long'}}
-  int b2 = sizeof(arr4) / sizeof(int *);    // expected-warning {{expression does not compute the number of elements in this array; element type is 'int [12]', not 'int *'}}
-  int b3 = sizeof(arr4) / sizeof(short *);  // expected-warning {{expression does not compute the number of elements in this array; element type is 'int [12]', not 'short *'}}
+  int arr4[10][12];
+  int b1 = sizeof(arr4) / sizeof(arr2[12]);
+  int b2 = sizeof(arr4) / sizeof(int *);
+  int b3 = sizeof(arr4) / sizeof(short *);
+  int arr5[][5] = {
+      {1, 2, 3, 4, 5},
+      {6, 7, 8, 9, 0},
+  };
+  int c1 = sizeof(arr5) / sizeof(*arr5);
+  int c2 = sizeof(arr5) / sizeof(**arr5);
+  int c3 = sizeof(arr5) / sizeof(int);
+
+  float m[4][4];
+  int d1 = sizeof(m) / sizeof(**m);
 }
diff --git a/clang/test/Sema/format-bool.c b/clang/test/Sema/format-bool.c
new file mode 100644
index 0000000000000..53c2c7fd126a6
--- /dev/null
+++ b/clang/test/Sema/format-bool.c
@@ -0,0 +1,46 @@
+// RUN: %clang_cc1 -xc %s -verify -DBOOL=_Bool
+// RUN: %clang_cc1 -xc++ %s -verify -DBOOL=bool
+// RUN: %clang_cc1 -xobjective-c %s -verify -DBOOL=_Bool
+// RUN: %clang_cc1 -xc %s -verify -DBOOL=_Bool -Wformat-pedantic -DPEDANTIC
+// RUN: %clang_cc1 -xc++ %s -verify -DBOOL=bool -Wformat-pedantic -DPEDANTIC
+
+__attribute__((format(__printf__, 1, 2)))
+int p(const char *fmt, ...);
+
+BOOL b;
+
+#ifdef __OBJC__
+@interface NSString
++(NSString *)stringWithFormat:(NSString *)fmt, ...
+    __attribute__((format(__NSString__, 1, 2)));
+@end
+
+#define YES __objc_yes
+#define NO __objc_no
+#endif
+
+int main() {
+  p("%d", b);
+  p("%hd", b);
+#ifdef PEDANTIC
+  // expected-warning@-2 {{format specifies type 'short' but the argument has type}}
+#endif
+  p("%hhd", b);
+  p("%u", b);
+  p("%hu", b);
+#ifdef PEDANTIC
+  // expected-warning@-2 {{format specifies type 'unsigned short' but the argument has type}}
+#endif
+  p("%hhu", b);
+  p("%c", b); // expected-warning {{using '%c' format specifier, but argument has boolean value}}
+  p("%lc", b); // expected-warning {{using '%lc' format specifier, but argument has boolean value}}
+  p("%c", 1 == 1); // expected-warning {{using '%c' format specifier, but argument has boolean value}}
+  p("%f", b); // expected-warning{{format specifies type 'double' but the argument has type}}
+  p("%ld", b); // expected-warning{{format specifies type 'long' but the argument has type}}
+  p("%lld", b); // expected-warning{{format specifies type 'long long' but the argument has type}}
+
+#ifdef __OBJC__
+  [NSString stringWithFormat: @"%c", 0]; // probably fine?
+  [NSString stringWithFormat: @"%c", NO]; // expected-warning {{using '%c' format specifier, but argument has boolean value}}
+#endif
+}
diff --git a/clang/test/Sema/objc-bool-constant-conversion-fixit.m b/clang/test/Sema/objc-bool-constant-conversion-fixit.m
index 57f575222ee87..08325c1441961 100644
--- a/clang/test/Sema/objc-bool-constant-conversion-fixit.m
+++ b/clang/test/Sema/objc-bool-constant-conversion-fixit.m
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -Werror=constant-conversion %s -fixit-recompile -fixit-to-temporary -E -o - | FileCheck %s
+// RUN: %clang_cc1 -Werror=objc-signed-char-bool %s -fixit-recompile -fixit-to-temporary -E -o - | FileCheck %s
 
 typedef signed char BOOL;
 
@@ -25,6 +25,17 @@ int main() {
 
   b = 1 << 1;
   // CHECK: b = (1 << 1) ? YES : NO;
+
+  int i;
+
+  b = i;
+  // CHECK: b = i ? YES : NO;
+
+  b = i * 2;
+  // CHECK b = (i * 2) ? YES : NO;
+
+  b = 1 ? 2 : 3;
+  // CHECK: b = 1 ? 2 ? YES : NO : 3 ? YES : NO;
 }
 
 @interface BoolProp
@@ -37,4 +48,12 @@ void f(BoolProp *bp) {
 
   [bp setB:43];
   // CHECK: [bp setB:43 ? YES : NO];
+
+  int i;
+
+  bp.b = i;
+  // CHECK: bp.b = i ? YES : NO;
+
+  bp.b = i + 1;
+  // CHECK: bp.b = (i + 1) ? YES : NO;
 }
diff --git a/clang/test/Sema/objc-bool-constant-conversion.m b/clang/test/Sema/objc-bool-constant-conversion.m
index 3638e2f8a9dcf..25546ab0a647d 100644
--- a/clang/test/Sema/objc-bool-constant-conversion.m
+++ b/clang/test/Sema/objc-bool-constant-conversion.m
@@ -12,20 +12,20 @@ int main() {
   B = YES;
   B = NO;
 
-  B = -1; // expected-warning{{implicit conversion from constant value -1 to BOOL; the only well defined values for BOOL are YES and NO}}
-  B = 0 - 1; // expected-warning{{implicit conversion from constant value -1 to BOOL; the only well defined values for BOOL are YES and NO}}
-  B = YES + YES; // expected-warning {{implicit conversion from constant value 2 to BOOL; the only well defined values for BOOL are YES and NO}}
+  B = -1; // expected-warning{{implicit conversion from constant value -1 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+  B = 0 - 1; // expected-warning{{implicit conversion from constant value -1 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+  B = YES + YES; // expected-warning {{implicit conversion from constant value 2 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
   B = YES | YES;
 
-  B = B ? 2 : 2; // expected-warning 2 {{implicit conversion from constant value 2 to BOOL; the only well defined values for BOOL are YES and NO}}
+  B = B ? 2 : 2; // expected-warning 2 {{implicit conversion from constant value 2 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
 
-  BOOL Init = -1; // expected-warning{{implicit conversion from constant value -1 to BOOL; the only well defined values for BOOL are YES and NO}}
-  BOOL Init2 = B ? 2 : 2; // expected-warning 2 {{implicit conversion from constant value 2 to BOOL; the only well defined values for BOOL are YES and NO}}
+  BOOL Init = -1; // expected-warning{{implicit conversion from constant value -1 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+  BOOL Init2 = B ? 2 : 2; // expected-warning 2 {{implicit conversion from constant value 2 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
 
   void takesbool(BOOL);
-  takesbool(43); // expected-warning {{implicit conversion from constant value 43 to BOOL; the only well defined values for BOOL are YES and NO}}
+  takesbool(43); // expected-warning {{implicit conversion from constant value 43 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
 
-  BOOL OutOfRange = 400; // expected-warning{{implicit conversion from constant value 400 to BOOL; the only well defined values for BOOL are YES and NO}}
+  BOOL OutOfRange = 400; // expected-warning{{implicit conversion from constant value 400 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
 }
 
 @interface BoolProp
@@ -33,6 +33,6 @@ @interface BoolProp
 @end
 
 void f(BoolProp *bp) {
-  bp.b = 43; // expected-warning {{implicit conversion from constant value 43 to BOOL; the only well defined values for BOOL are YES and NO}}
-  [bp setB:43]; // expected-warning {{implicit conversion from constant value 43 to BOOL; the only well defined values for BOOL are YES and NO}}
+  bp.b = 43; // expected-warning {{implicit conversion from constant value 43 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+  [bp setB:43]; // expected-warning {{implicit conversion from constant value 43 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
 }
diff --git a/clang/test/Sema/tautological-objc-bool-compare.m b/clang/test/Sema/tautological-objc-bool-compare.m
index 525862ed9ea28..5fd7b9e1289a6 100644
--- a/clang/test/Sema/tautological-objc-bool-compare.m
+++ b/clang/test/Sema/tautological-objc-bool-compare.m
@@ -9,16 +9,16 @@
 void test() {
   int r;
   r = B > 0;
-  r = B > 1; // expected-warning {{result of comparison of constant 1 with expression of type BOOL is always false, as the only well defined values for BOOL are YES and NO}}
+  r = B > 1; // expected-warning {{result of comparison of constant 1 with expression of type 'BOOL' is always false, as the only well defined values for 'BOOL' are YES and NO}}
   r = B < 1;
-  r = B < 0; // expected-warning {{result of comparison of constant 0 with expression of type BOOL is always false, as the only well defined values for BOOL are YES and NO}}
-  r = B >= 0; // expected-warning {{result of comparison of constant 0 with expression of type BOOL is always true, as the only well defined values for BOOL are YES and NO}}
+  r = B < 0; // expected-warning {{result of comparison of constant 0 with expression of type 'BOOL' is always false, as the only well defined values for 'BOOL' are YES and NO}}
+  r = B >= 0; // expected-warning {{result of comparison of constant 0 with expression of type 'BOOL' is always true, as the only well defined values for 'BOOL' are YES and NO}}
   r = B <= 0;
 
-  r = B > YES; // expected-warning {{result of comparison of constant YES with expression of type BOOL is always false, as the only well defined values for BOOL are YES and NO}}
+  r = B > YES; // expected-warning {{result of comparison of constant YES with expression of type 'BOOL' is always false, as the only well defined values for 'BOOL' are YES and NO}}
   r = B > NO;
-  r = B < NO; // expected-warning {{result of comparison of constant NO with expression of type BOOL is always false, as the only well defined values for BOOL are YES and NO}}
+  r = B < NO; // expected-warning {{result of comparison of constant NO with expression of type 'BOOL' is always false, as the only well defined values for 'BOOL' are YES and NO}}
   r = B < YES;
-  r = B >= NO; // expected-warning {{result of comparison of constant NO with expression of type BOOL is always true, as the only well defined values for BOOL are YES and NO}}
+  r = B >= NO; // expected-warning {{result of comparison of constant NO with expression of type 'BOOL' is always true, as the only well defined values for 'BOOL' are YES and NO}}
   r = B <= NO;
 }
diff --git a/clang/test/Sema/warn-conditional-emum-types-mismatch.c b/clang/test/Sema/warn-conditional-emum-types-mismatch.c
new file mode 100644
index 0000000000000..12264ff89270d
--- /dev/null
+++ b/clang/test/Sema/warn-conditional-emum-types-mismatch.c
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -x c -fsyntax-only -verify -Wenum-compare-conditional %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify -Wenum-compare-conditional %s
+
+enum ro { A = 0x10 };
+enum rw { B = 0xFF };
+enum { C = 0x1A};
+
+enum {
+  STATUS_SUCCESS,
+  STATUS_FAILURE,
+  MAX_BASE_STATUS_CODE
+};
+
+enum ExtendedStatusCodes {
+  STATUS_SOMETHING_INTERESTING = MAX_BASE_STATUS_CODE + 1000,
+};
+
+
+int get_flag(int cond) {
+  return cond ? A : B; 
+  #ifdef __cplusplus
+  // expected-warning@-2 {{enumeration type mismatch in conditional expression ('ro' and 'rw')}}
+  #else 
+  // expected-no-diagnostics
+  #endif
+}
+
+// In the following cases we purposefully differ from GCC and dont warn because
+// this code pattern is quite sensitive and we dont want to produce so many false positives.
+
+int get_flag_anon_enum(int cond) {
+  return cond ? A : C;
+}
+
+int foo(int c) {
+  return c ? STATUS_SOMETHING_INTERESTING : STATUS_SUCCESS;
+}
diff --git a/clang/test/Sema/warn-int-in-bool-context.c b/clang/test/Sema/warn-int-in-bool-context.c
new file mode 100644
index 0000000000000..0c94ebb391f3c
--- /dev/null
+++ b/clang/test/Sema/warn-int-in-bool-context.c
@@ -0,0 +1,74 @@
+// RUN: %clang_cc1 -x c -fsyntax-only -verify -Wint-in-bool-context %s
+// RUN: %clang_cc1 -x c -fsyntax-only -verify -Wall %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify -Wint-in-bool-context %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify -Wall %s
+
+#define ONE 1
+#define TWO 2
+
+#define SHIFT(l, r) l << r
+#define MM a << a
+#define AF 1 << 7
+
+#ifdef __cplusplus
+typedef bool boolean;
+#else
+typedef _Bool boolean;
+#endif
+
+enum num {
+  zero,
+  one,
+  two,
+};
+
+int test(int a, unsigned b, enum num n) {
+  boolean r;
+  r = a << a;    // expected-warning {{converting the result of '<<' to a boolean; did you mean '(a << a) != 0'?}}
+  r = MM;        // expected-warning {{converting the result of '<<' to a boolean; did you mean '(a << a) != 0'?}}
+  r = (1 << 7);  // expected-warning {{converting the result of '<<' to a boolean always evaluates to true}}
+  r = 2UL << 2;  // expected-warning {{converting the result of '<<' to a boolean always evaluates to true}}
+  r = 0 << a;    // expected-warning {{converting the result of '<<' to a boolean always evaluates to false}}
+  r = 0 << 2;    // expected-warning {{converting the result of '<<' to a boolean always evaluates to false}}
+  r = 1 << 0;    // expected-warning {{converting the result of '<<' to a boolean always evaluates to true}}
+  r = 1 << 2;    // expected-warning {{converting the result of '<<' to a boolean always evaluates to true}}
+  r = 1ULL << 2; // expected-warning {{converting the result of '<<' to a boolean always evaluates to true}}
+  r = 2 << b;    // expected-warning {{converting the result of '<<' to a boolean; did you mean '(2 << b) != 0'?}}
+  r = (unsigned)(2 << b);
+  r = b << 7;
+  r = (1 << a); // expected-warning {{converting the result of '<<' to a boolean; did you mean '(1 << a) != 0'?}}
+  r = TWO << a; // expected-warning {{converting the result of '<<' to a boolean; did you mean '(2 << a) != 0'?}}
+  r = a << 7;   // expected-warning {{converting the result of '<<' to a boolean; did you mean '(a << 7) != 0'?}}
+  r = ONE << a; // expected-warning {{converting the result of '<<' to a boolean; did you mean '(1 << a) != 0'?}}
+  if (TWO << a) // expected-warning {{converting the result of '<<' to a boolean; did you mean '(2 << a) != 0'?}}
+    return a;
+
+  for (a = 0; 1 << a; a++) // expected-warning {{converting the result of '<<' to a boolean; did you mean '(1 << a) != 0'?}}
+    ;
+
+  if (a << TWO) // expected-warning {{converting the result of '<<' to a boolean; did you mean '(a << 2) != 0'?}}
+    return a;
+
+  if (n || two)
+    // expected-warning@-1 {{converting the enum constant to a boolean}}
+    return a;
+
+  if (n == one || two)
+    // expected-warning@-1 {{converting the enum constant to a boolean}}
+    return a;
+
+  if (r && two)
+    // expected-warning@-1 {{converting the enum constant to a boolean}}
+    return a;
+
+  if (two && r)
+    // expected-warning@-1 {{converting the enum constant to a boolean}}
+    return a;
+
+  if (n == one && two)
+    // expected-warning@-1 {{converting the enum constant to a boolean}}
+    return a;
+
+  // Don't warn in macros.
+  return SHIFT(1, a);
+}
diff --git a/clang/test/Sema/warn-integer-constants-in-ternary.c b/clang/test/Sema/warn-integer-constants-in-ternary.c
new file mode 100644
index 0000000000000..287b91ecdb723
--- /dev/null
+++ b/clang/test/Sema/warn-integer-constants-in-ternary.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -x c -fsyntax-only -verify -Wtautological-constant-compare %s
+// RUN: %clang_cc1 -x c -fsyntax-only -verify %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify -Wtautological-constant-compare %s
+// RUN: %clang_cc1 -x c++ -fsyntax-only -verify %s
+
+#define ONE 1
+#define TWO 2
+
+#define TERN(c, l, r) c ? l : r
+
+#ifdef __cplusplus
+typedef bool boolean;
+#else
+typedef _Bool boolean;
+#endif
+
+void test(boolean a) {
+  boolean r;
+  r = a ? (1) : TWO;
+  r = a ? 3 : TWO; // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+  r = a ? -2 : 0;  // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+  r = a ? 3 : -2;  // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+  r = a ? 0 : TWO;
+  r = a ? 3 : ONE; // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+  r = a ? ONE : 0;
+  r = a ? 0 : -0;
+  r = a ? 1 : 0;
+  r = a ? ONE : 0;
+  r = a ? ONE : ONE;
+  r = TERN(a, 4, 8);   // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+  r = TERN(a, -1, -8); // expected-warning {{converting the result of '?:' with integer constants to a boolean always evaluates to 'true'}}
+}
diff --git a/clang/test/Sema/warn-overlap.c b/clang/test/Sema/warn-overlap.c
index 6299c511fe230..066312591c1eb 100644
--- a/clang/test/Sema/warn-overlap.c
+++ b/clang/test/Sema/warn-overlap.c
@@ -141,3 +141,34 @@ int returns(int x) {
   return x < 1 || x != 0;
   // expected-warning@-1{{overlapping comparisons always evaluate to true}}
 }
+
+int integer_conversion(unsigned x, int y) {
+  return x > 4 || x < 10;
+  // expected-warning@-1{{overlapping comparisons always evaluate to true}}
+  return y > 4u || y < 10u;
+  // expected-warning@-1{{overlapping comparisons always evaluate to true}}
+}
+
+int negative_compare(int x) {
+  return x > -1 || x < 1;
+  // expected-warning@-1{{overlapping comparisons always evaluate to true}}
+}
+
+int no_warning(unsigned x) {
+  return x >= 0 || x == 1;
+  // no warning since "x >= 0" is caught by a different tautological warning.
+}
+
+struct A {
+  int x;
+  int y;
+};
+
+int struct_test(struct A a) {
+  return a.x > 5 && a.y < 1;  // no warning, different variables
+
+  return a.x > 5 && a.x < 1;
+  // expected-warning@-1{{overlapping comparisons always evaluate to false}}
+  return a.y == 1 || a.y != 1;
+  // expected-warning@-1{{overlapping comparisons always evaluate to true}}
+}
diff --git a/clang/test/Sema/warn-unreachable.c b/clang/test/Sema/warn-unreachable.c
index aec3b070213c7..cfac36a131869 100644
--- a/clang/test/Sema/warn-unreachable.c
+++ b/clang/test/Sema/warn-unreachable.c
@@ -433,7 +433,7 @@ void wrapOneInFixit(struct StructWithPointer *s) {
 }
 
 void unaryOpNoFixit() {
-  if (- 1)
+  if (~ 1)
     return; // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]
   unaryOpNoFixit(); // expected-warning {{code will never be executed}}
 }
diff --git a/clang/test/SemaCUDA/autoret-global.cu b/clang/test/SemaCUDA/autoret-global.cu
new file mode 100644
index 0000000000000..ae2f1ca7598bb
--- /dev/null
+++ b/clang/test/SemaCUDA/autoret-global.cu
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -std=c++14 -fsyntax-only -verify %s
+
+#include "Inputs/cuda.h"
+
+template <typename T>
+__global__ T foo() {
+  // expected-note@-1 {{kernel function type 'T ()' must have void return type}}
+}
+
+void f0() {
+  foo<void><<<0, 0>>>();
+  foo<int><<<0, 0>>>();
+  // expected-error@-1 {{no matching function for call to 'foo'}}
+}
+
+__global__ auto f1() {
+}
+
+__global__ auto f2(int x) {
+  return x + 1;
+  // expected-error@-2 {{kernel function type 'auto (int)' must have void return type}}
+}
+
+template <bool Cond, typename T = void> struct enable_if { typedef T type; };
+template <typename T> struct enable_if<false, T> {};
+
+template <int N>
+__global__
+auto bar() -> typename enable_if<N == 1>::type {
+  // expected-note@-1 {{requirement '3 == 1' was not satisfied [with N = 3]}}
+}
+
+template <int N>
+__global__
+auto bar() -> typename enable_if<N == 2>::type {
+  // expected-note@-1 {{requirement '3 == 2' was not satisfied [with N = 3]}}
+}
+
+void f3() {
+  bar<1><<<0, 0>>>();
+  bar<2><<<0, 0>>>();
+  bar<3><<<0, 0>>>();
+  // expected-error@-1 {{no matching function for call to 'bar'}}
+}
diff --git a/clang/test/SemaCUDA/default-ctor.cu b/clang/test/SemaCUDA/default-ctor.cu
new file mode 100644
index 0000000000000..cbad7a1774c15
--- /dev/null
+++ b/clang/test/SemaCUDA/default-ctor.cu
@@ -0,0 +1,43 @@
+// RUN: %clang_cc1 -std=c++11 -triple nvptx64-nvidia-cuda -fsyntax-only \
+// RUN:            -fcuda-is-device -verify -verify-ignore-unexpected=note %s
+// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-linux-gnu -fsyntax-only \
+// RUN:            -verify -verify-ignore-unexpected=note %s
+
+#include "Inputs/cuda.h"
+
+struct In { In() = default; };
+struct InD { __device__ InD() = default; };
+struct InH { __host__ InH() = default; };
+struct InHD { __host__ __device__ InHD() = default; };
+
+struct Out { Out(); };
+struct OutD { __device__ OutD(); };
+struct OutH { __host__ OutH(); };
+struct OutHD { __host__ __device__ OutHD(); };
+
+Out::Out() = default;
+__device__ OutD::OutD() = default;
+__host__ OutH::OutH() = default;
+__host__ __device__ OutHD::OutHD() = default;
+
+__device__ void fd() {
+  In in;
+  InD ind;
+  InH inh; // expected-error{{no matching constructor for initialization of 'InH'}}
+  InHD inhd;
+  Out out; // expected-error{{no matching constructor for initialization of 'Out'}}
+  OutD outd;
+  OutH outh; // expected-error{{no matching constructor for initialization of 'OutH'}}
+  OutHD outhd;
+}
+
+__host__ void fh() {
+  In in;
+  InD ind; // expected-error{{no matching constructor for initialization of 'InD'}}
+  InH inh;
+  InHD inhd;
+  Out out;
+  OutD outd; // expected-error{{no matching constructor for initialization of 'OutD'}}
+  OutH outh;
+  OutHD outhd;
+}
diff --git a/clang/test/SemaCUDA/function-overload.cu b/clang/test/SemaCUDA/function-overload.cu
index 1d78636a7089b..b9efd1c09e699 100644
--- a/clang/test/SemaCUDA/function-overload.cu
+++ b/clang/test/SemaCUDA/function-overload.cu
@@ -402,3 +402,20 @@ __host__ void test_host_template_overload() {
 __device__ void test_device_template_overload() {
   template_overload(1); // OK. Attribute-based overloading picks __device__ variant.
 }
+
+// Two classes with `operator-` defined. One of them is device only.
+struct C1;
+struct C2;
+__device__
+int operator-(const C1 &x, const C1 &y);
+int operator-(const C2 &x, const C2 &y);
+
+template <typename T>
+__host__ __device__ int constexpr_overload(const T &x, const T &y) {
+  return x - y;
+}
+
+// Verify that function overloading doesn't prune candidate wrongly.
+int test_constexpr_overload(C2 &x, C2 &y) {
+  return constexpr_overload(x, y);
+}
diff --git a/clang/test/SemaCUDA/gnu-inline.cu b/clang/test/SemaCUDA/gnu-inline.cu
index 0ed543326be80..e383082f2a773 100644
--- a/clang/test/SemaCUDA/gnu-inline.cu
+++ b/clang/test/SemaCUDA/gnu-inline.cu
@@ -7,4 +7,4 @@
 // Check that we can handle gnu_inline functions when compiling in CUDA mode.
 
 void foo();
-inline __attribute__((gnu_inline)) void bar() { foo(); }
+extern inline __attribute__((gnu_inline)) void bar() { foo(); }
diff --git a/clang/test/SemaCUDA/implicit-member-target-collision-cxx11.cu b/clang/test/SemaCUDA/implicit-member-target-collision-cxx11.cu
index 7aa1dd3f20880..06015ed0d6d8e 100644
--- a/clang/test/SemaCUDA/implicit-member-target-collision-cxx11.cu
+++ b/clang/test/SemaCUDA/implicit-member-target-collision-cxx11.cu
@@ -74,11 +74,13 @@ struct B4_with_device_copy_ctor {
 struct C4_with_collision : A4_with_host_copy_ctor, B4_with_device_copy_ctor {
 };
 
-// expected-note@-3 {{copy constructor of 'C4_with_collision' is implicitly deleted because base class 'B4_with_device_copy_ctor' has no copy constructor}}
+// expected-note@-3 {{candidate constructor (the implicit copy constructor) not viable: call to invalid function from __host__ function}}
+// expected-note@-4 {{implicit copy constructor inferred target collision: call to both __host__ and __device__ members}}
+// expected-note@-5 {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 1 was provided}}
 
 void hostfoo4() {
   C4_with_collision c;
-  C4_with_collision c2 = c; // expected-error {{call to implicitly-deleted copy constructor of 'C4_with_collision'}}
+  C4_with_collision c2 = c; // expected-error {{no matching constructor for initialization of 'C4_with_collision'}}
 }
 
 //------------------------------------------------------------------------------
diff --git a/clang/test/SemaCXX/attr-require-constant-initialization.cpp b/clang/test/SemaCXX/attr-require-constant-initialization.cpp
index 12bae81fd2a47..5584b4b55614b 100644
--- a/clang/test/SemaCXX/attr-require-constant-initialization.cpp
+++ b/clang/test/SemaCXX/attr-require-constant-initialization.cpp
@@ -185,7 +185,7 @@ struct TT2 {
   ATTR static constexpr LitType lit = {};
   ATTR static const NonLit non_lit;
   ATTR static const NonLit non_lit_list_init;
-  ATTR static const NonLit non_lit_copy_init; // expected-note {{required by 'require_constant_initialization' attribute here}}
+  ATTR static const NonLit non_lit_copy_init;
 #endif
 };
 PODType TT2::pod_noinit; // expected-note 0+ {{declared here}}
@@ -203,8 +203,9 @@ PODType TT2::pod_copy_init(TT2::pod_noinit); // expected-error {{variable does n
 #if __cplusplus >= 201402L
 const NonLit TT2::non_lit(42);
 const NonLit TT2::non_lit_list_init = {42};
-const NonLit TT2::non_lit_copy_init = 42; // expected-error {{variable does not have a constant initializer}}
-// expected-note@-1 {{subexpression not valid in a constant expression}}
+// FIXME: This is invalid, but we incorrectly elide the copy. It's OK if we
+// start diagnosing this.
+const NonLit TT2::non_lit_copy_init = 42;
 #endif
 
 #if __cplusplus >= 201103L
diff --git a/clang/test/SemaCXX/builtin-object-size-cxx14.cpp b/clang/test/SemaCXX/builtin-object-size-cxx14.cpp
index e0b05883bb24f..a57bb6b53787a 100644
--- a/clang/test/SemaCXX/builtin-object-size-cxx14.cpp
+++ b/clang/test/SemaCXX/builtin-object-size-cxx14.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -std=c++14 %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++2a %s
 
 typedef __SIZE_TYPE__ size_t;
 
@@ -105,4 +106,10 @@ namespace InvalidBase {
   struct S { const char *name; };
   S invalid_base();
   constexpr size_t bos_name = __builtin_object_size(invalid_base().name, 1);
+  static_assert(bos_name == -1, "");
+
+  struct T { ~T(); };
+  T invalid_base_2();
+  constexpr size_t bos_dtor = __builtin_object_size(&(T&)(T&&)invalid_base_2(), 0);
+  static_assert(bos_dtor == -1, "");
 }
diff --git a/clang/test/SemaCXX/compare-cxx2a.cpp b/clang/test/SemaCXX/compare-cxx2a.cpp
index c68a0ae9133f3..b6e7fc806114b 100644
--- a/clang/test/SemaCXX/compare-cxx2a.cpp
+++ b/clang/test/SemaCXX/compare-cxx2a.cpp
@@ -8,12 +8,18 @@
 #define ASSERT_TYPE(...) static_assert(__is_same(__VA_ARGS__))
 #define ASSERT_EXPR_TYPE(Expr, Expect) static_assert(__is_same(decltype(Expr), Expect));
 
+struct S {
+  static int x[5];
+};
+
 void self_compare() {
   int a;
   int *b = nullptr;
+  S s;
 
   (void)(a <=> a); // expected-warning {{self-comparison always evaluates to 'std::strong_ordering::equal'}}
   (void)(b <=> b); // expected-warning {{self-comparison always evaluates to 'std::strong_ordering::equal'}}
+  (void)(s.x[a] <=> S::x[a]); // expected-warning {{self-comparison always evaluates to 'std::strong_ordering::equal'}}
 }
 
 void test0(long a, unsigned long b) {
diff --git a/clang/test/SemaCXX/constant-expression-cxx11.cpp b/clang/test/SemaCXX/constant-expression-cxx11.cpp
index bcfb3ac5e9550..27ad0f654f566 100644
--- a/clang/test/SemaCXX/constant-expression-cxx11.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx11.cpp
@@ -1108,6 +1108,11 @@ namespace MemberPointer {
   static_assert((int Derived::*)(int Mid<0>::*)&Mid<0>::n !=
                 (int Derived::*)(int Mid<1>::*)&Mid<1>::n, "");
   static_assert(&Mid<0>::n == (int Mid<0>::*)&Base::n, "");
+
+  constexpr int apply(const A &a, int (A::*f)() const) {
+    return (a.*f)();
+  }
+  static_assert(apply(A(2), &A::f) == 5, "");
 }
 
 namespace ArrayBaseDerived {
@@ -1889,9 +1894,10 @@ namespace ConstexprConstructorRecovery {
       };
       constexpr X() noexcept {};
   protected:
-      E val{0}; // expected-error {{cannot initialize a member subobject of type 'ConstexprConstructorRecovery::X::E' with an rvalue of type 'int'}}
+      E val{0}; // expected-error {{cannot initialize a member subobject of type 'ConstexprConstructorRecovery::X::E' with an rvalue of type 'int'}} expected-note {{here}}
   };
-  constexpr X x{};
+  // FIXME: We should avoid issuing this follow-on diagnostic.
+  constexpr X x{}; // expected-error {{constant expression}} expected-note {{not initialized}}
 }
 
 namespace Lifetime {
@@ -2036,7 +2042,7 @@ namespace BadDefaultInit {
   // here is bogus (we discard the k(k) initializer because the parameter 'k'
   // has been marked invalid).
   struct B { // expected-note 2{{candidate}}
-    constexpr B( // expected-error {{must initialize all members}} expected-note {{candidate}}
+    constexpr B( // expected-warning {{initialize all members}} expected-note {{candidate}}
         int k = X<B().k>::n) : // expected-error {{no matching constructor}}
       k(k) {}
     int k; // expected-note {{not initialized}}
diff --git a/clang/test/SemaCXX/constant-expression-cxx1y.cpp b/clang/test/SemaCXX/constant-expression-cxx1y.cpp
index 1f9cb57492584..2a8304ebda641 100644
--- a/clang/test/SemaCXX/constant-expression-cxx1y.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx1y.cpp
@@ -810,9 +810,10 @@ namespace StmtExpr {
   }
   static_assert(f(1) == 1, ""); // expected-error {{constant expression}} expected-note {{in call}}
 
-  constexpr int g() { // expected-error {{never produces a constant}}
-    return ({ int n; n; }); // expected-note {{object of type 'int' is not initialized}}
+  constexpr int g() {
+    return ({ int n; n; }); // expected-note {{read of uninitialized object}}
   }
+  static_assert(g() == 0, ""); // expected-error {{constant expression}} expected-note {{in call}}
 
   // FIXME: We should handle the void statement expression case.
   constexpr int h() { // expected-error {{never produces a constant}}
@@ -1222,3 +1223,22 @@ namespace PR39728 {
     ~Comment1() = default;
   };
 }
+
+namespace TemporaryWithBadPointer {
+  constexpr int *get_bad_pointer() {
+    int n = 0; // expected-note 2{{here}}
+    return &n; // expected-warning {{stack}}
+  }
+  constexpr int *bad_pointer = get_bad_pointer(); // expected-error {{constant expression}} expected-note {{pointer to 'n' is not a constant expression}}
+
+  struct DoBadThings { int *&&wp; int n; };
+  constexpr DoBadThings dbt = { // expected-error {{constant expression}}
+    nullptr, // expected-note {{pointer to 'n' is not a constant expression}}
+    (dbt.wp = get_bad_pointer(), 0)
+  };
+
+  constexpr DoBadThings dbt2 = { // ok
+    get_bad_pointer(),
+    (dbt2.wp = nullptr, 0)
+  };
+}
diff --git a/clang/test/SemaCXX/constant-expression-cxx2a.cpp b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
index dba877ff2111a..cb8f16a8b0f57 100644
--- a/clang/test/SemaCXX/constant-expression-cxx2a.cpp
+++ b/clang/test/SemaCXX/constant-expression-cxx2a.cpp
@@ -1,11 +1,32 @@
-// RUN: %clang_cc1 -std=c++2a -verify %s -fcxx-exceptions -triple=x86_64-linux-gnu
+// RUN: %clang_cc1 -std=c++2a -verify %s -fcxx-exceptions -triple=x86_64-linux-gnu -Wno-mismatched-new-delete
 
 #include "Inputs/std-compare.h"
 
 namespace std {
   struct type_info;
+  struct destroying_delete_t {
+    explicit destroying_delete_t() = default;
+  } inline constexpr destroying_delete{};
+  struct nothrow_t {
+    explicit nothrow_t() = default;
+  } inline constexpr nothrow{};
+  using size_t = decltype(sizeof(0));
+  enum class align_val_t : size_t {};
 };
 
+[[nodiscard]] void *operator new(std::size_t, const std::nothrow_t&) noexcept;
+[[nodiscard]] void *operator new(std::size_t, std::align_val_t, const std::nothrow_t&) noexcept;
+[[nodiscard]] void *operator new[](std::size_t, const std::nothrow_t&) noexcept;
+[[nodiscard]] void *operator new[](std::size_t, std::align_val_t, const std::nothrow_t&) noexcept;
+void operator delete(void*, const std::nothrow_t&) noexcept;
+void operator delete(void*, std::align_val_t, const std::nothrow_t&) noexcept;
+void operator delete[](void*, const std::nothrow_t&) noexcept;
+void operator delete[](void*, std::align_val_t, const std::nothrow_t&) noexcept;
+
+// Helper to print out values for debugging.
+constexpr void not_defined();
+template<typename T> constexpr void print(T) { not_defined(); }
+
 namespace ThreeWayComparison {
   struct A {
     int n;
@@ -219,15 +240,19 @@ static_assert(for_range_init());
 namespace Virtual {
   struct NonZeroOffset { int padding = 123; };
 
+  constexpr void assert(bool b) { if (!b) throw 0; }
+
   // Ensure that we pick the right final overrider during construction.
   struct A {
     virtual constexpr char f() const { return 'A'; }
     char a = f();
+    constexpr ~A() { assert(f() == 'A'); }
   };
   struct NoOverrideA : A {};
   struct B : NonZeroOffset, NoOverrideA {
     virtual constexpr char f() const { return 'B'; }
     char b = f();
+    constexpr ~B() { assert(f() == 'B'); }
   };
   struct NoOverrideB : B {};
   struct C : NonZeroOffset, A {
@@ -236,12 +261,15 @@ namespace Virtual {
     char c = ((A*)this)->f();
     char ba = pba->f();
     constexpr C(A *pba) : pba(pba) {}
+    constexpr ~C() { assert(f() == 'C'); }
   };
   struct D : NonZeroOffset, NoOverrideB, C { // expected-warning {{inaccessible}}
     virtual constexpr char f() const { return 'D'; }
     char d = f();
     constexpr D() : C((B*)this) {}
+    constexpr ~D() { assert(f() == 'D'); }
   };
+  constexpr int n = (D(), 0);
   constexpr D d;
   static_assert(((B&)d).a == 'A');
   static_assert(((C&)d).a == 'A');
@@ -416,7 +444,7 @@ namespace TypeId {
 
 namespace Union {
   struct Base {
-    int y; // expected-note {{here}}
+    int y; // expected-note 2{{here}}
   };
   struct A : Base {
     int x;
@@ -489,13 +517,15 @@ namespace Union {
   constexpr A return_uninit_struct() {
     B b = {.b = 1};
     b.a.x = 2;
-    return b.a;
+    return b.a; // expected-note {{in call to 'A(b.a)'}} expected-note {{subobject of type 'int' is not initialized}}
   }
-  // FIXME: It's unclear that this should be valid. Copying a B involves
-  // copying the object representation of the union, but copying an A invokes a
-  // copy constructor that copies the object elementwise, and reading from
-  // b.a.y is undefined.
-  static_assert(return_uninit_struct().x == 2);
+  // Note that this is rejected even though return_uninit() is accepted, and
+  // return_uninit() copies the same stuff wrapped in a union.
+  //
+  // Copying a B involves copying the object representation of the union, but
+  // copying an A invokes a copy constructor that copies the object
+  // elementwise, and reading from b.a.y is undefined.
+  static_assert(return_uninit_struct().x == 2); // expected-error {{constant expression}} expected-note {{in call}}
   constexpr B return_init_all() {
     B b = {.b = 1};
     b.a.x = 2;
@@ -531,6 +561,29 @@ namespace Union {
     S3 s;
     s.n = 0;
   }
+
+  union ref_member_1 {
+    int a;
+    int b;
+  };
+  struct ref_member_2 {
+    ref_member_1 &&r;
+  };
+  union ref_member_3 {
+    ref_member_2 a, b;
+  };
+  constexpr int ref_member_test_1() {
+    ref_member_3 r = {.a = {.r = {.a = 1}}};
+    r.a.r.b = 2;
+    return r.a.r.b;
+  }
+  static_assert(ref_member_test_1() == 2);
+  constexpr int ref_member_test_2() { // expected-error {{never produces a constant}}
+    ref_member_3 r = {.a = {.r = {.a = 1}}};
+    // FIXME: This note isn't great. The 'read' here is reading the referent of the reference.
+    r.b.r.b = 2; // expected-note {{read of member 'b' of union with active member 'a'}}
+    return r.b.r.b;
+  }
 }
 
 namespace TwosComplementShifts {
@@ -548,3 +601,723 @@ namespace TwosComplementShifts {
   static_assert(-3 >> 1 == -2);
   static_assert(-4 >> 1 == -2);
 }
+
+namespace Uninit {
+  constexpr int f(bool init) {
+    int a;
+    if (init)
+      a = 1;
+    return a; // expected-note {{read of uninitialized object}}
+  }
+  static_assert(f(true) == 1);
+  static_assert(f(false) == 1); // expected-error {{constant expression}} expected-note {{in call}}
+
+  struct X {
+    int n; // expected-note {{declared here}}
+    constexpr X(bool init) {
+      if (init) n = 123;
+    }
+  };
+  constinit X x1(true);
+  constinit X x2(false); // expected-error {{constant initializer}} expected-note {{constinit}} expected-note {{subobject of type 'int' is not initialized}}
+
+  struct Y {
+    struct Z { int n; }; // expected-note {{here}}
+    Z z1;
+    Z z2;
+    Z z3;
+    // OK: the lifetime of z1 (and its members) start before the initializer of
+    // z2 runs.
+    constexpr Y() : z2{ (z1.n = 1, z1.n + 1) } { z3.n = 3; }
+    // Not OK: z3 is not in its lifetime when the initializer of z2 runs.
+    constexpr Y(int) : z2{
+      (z3.n = 1, // expected-note {{assignment to object outside its lifetime}}
+       z3.n + 1) // expected-warning {{uninitialized}}
+    } { z1.n = 3; }
+    constexpr Y(int, int) : z2{} {}
+  };
+  // FIXME: This is working around clang not implementing DR2026. With that
+  // fixed, we should be able to test this without the injected copy.
+  constexpr Y copy(Y y) { return y; } // expected-note {{in call to 'Y(y)'}} expected-note {{subobject of type 'int' is not initialized}}
+  constexpr Y y1 = copy(Y());
+  static_assert(y1.z1.n == 1 && y1.z2.n == 2 && y1.z3.n == 3);
+
+  constexpr Y y2 = copy(Y(0)); // expected-error {{constant expression}} expected-note {{in call}}
+
+  static_assert(Y(0,0).z2.n == 0);
+  static_assert(Y(0,0).z1.n == 0); // expected-error {{constant expression}} expected-note {{read of uninitialized object}}
+  static_assert(Y(0,0).z3.n == 0); // expected-error {{constant expression}} expected-note {{read of uninitialized object}}
+
+  static_assert(copy(Y(0,0)).z2.n == 0); // expected-error {{constant expression}} expected-note {{in call}}
+
+  constexpr unsigned char not_even_unsigned_char() {
+    unsigned char c;
+    return c; // expected-note {{read of uninitialized object}}
+  }
+  constexpr unsigned char x = not_even_unsigned_char(); // expected-error {{constant expression}} expected-note {{in call}}
+
+  constexpr int switch_var(int n) {
+    switch (n) {
+    case 1:
+      int a;
+      a = n;
+      return a;
+
+    case 2:
+      a = n;
+      return a;
+    }
+  }
+  constexpr int s1 = switch_var(1);
+  constexpr int s2 = switch_var(2);
+  static_assert(s1 == 1 && s2 == 2);
+
+  constexpr bool switch_into_init_stmt() {
+    switch (1) {
+      if (int n; false) {
+        for (int m; false;) {
+        case 1:
+          n = m = 1;
+          return n == 1 && m == 1;
+        }
+      }
+    }
+  }
+  static_assert(switch_into_init_stmt());
+}
+
+namespace dtor {
+  void lifetime_extension() {
+    struct X { constexpr ~X() {} };
+    X &&a = X();
+  }
+
+  template<typename T> constexpr T &&ref(T &&t) { return (T&&)t; }
+
+  struct Buf {
+    char buf[64];
+    int n = 0;
+    constexpr void operator+=(char c) { buf[n++] = c; }
+    constexpr bool operator==(const char *str) const {
+      return str[n] == 0 && __builtin_memcmp(str, buf, n) == 0;
+    }
+    constexpr bool operator!=(const char *str) const { return !operator==(str); }
+  };
+
+  struct A {
+    constexpr A(Buf &buf, char c) : buf(buf), c(c) { buf += c; }
+    constexpr ~A() { buf += c; }
+    constexpr operator bool() const { return true; }
+    Buf &buf;
+    char c;
+  };
+
+  constexpr bool dtor_calls_dtor() {
+    union U {
+      constexpr U(Buf &buf) : u(buf, 'u') { buf += 'U'; }
+      constexpr ~U() { u.buf += 'U'; }
+      A u, v;
+    };
+
+    struct B : A {
+      A c, &&d, e;
+      union {
+        A f;
+      };
+      U u;
+      constexpr B(Buf &buf)
+          : A(buf, 'a'), c(buf, 'c'), d(ref(A(buf, 'd'))), e(A(buf, 'e')), f(buf, 'f'), u(buf) {
+        buf += 'b';
+      }
+      constexpr ~B() {
+        buf += 'b';
+      }
+    };
+
+    Buf buf;
+    {
+      B b(buf);
+      if (buf != "acddefuUb")
+        return false;
+    }
+    if (buf != "acddefuUbbUeca")
+      return false;
+    return true;
+  }
+  static_assert(dtor_calls_dtor());
+
+  constexpr void abnormal_termination(Buf &buf) {
+    struct Indestructible {
+      constexpr ~Indestructible(); // not defined
+    };
+
+    A a(buf, 'a');
+    A(buf, 'b');
+    int n = 0;
+    for (A &&c = A(buf, 'c'); A d = A(buf, 'd'); A(buf, 'e')) {
+      switch (A f(buf, 'f'); A g = A(buf, 'g')) { // expected-warning {{boolean}}
+      case false: {
+        A x(buf, 'x');
+      }
+
+      case true: {
+        A h(buf, 'h');
+        switch (n++) {
+        case 0:
+          break;
+        case 1:
+          continue;
+        case 2:
+          return;
+        }
+        break;
+      }
+
+      default:
+        Indestructible indest;
+      }
+
+      A j = (A(buf, 'i'), A(buf, 'j'));
+    }
+  }
+
+  constexpr bool check_abnormal_termination() {
+    Buf buf = {};
+    abnormal_termination(buf);
+    return buf ==
+      "abbc"
+        "dfgh" /*break*/ "hgfijijeed"
+        "dfgh" /*continue*/ "hgfeed"
+        "dfgh" /*return*/ "hgfd"
+      "ca";
+  }
+  static_assert(check_abnormal_termination());
+
+  constexpr bool run_dtors_on_array_filler() {
+    struct S {
+      int times_destroyed = 0;
+      constexpr ~S() { if (++times_destroyed != 1) throw "oops"; }
+    };
+    S s[3];
+    return true;
+  }
+  static_assert(run_dtors_on_array_filler());
+
+  // Ensure that we can handle temporary cleanups for array temporaries.
+  struct ArrElem { constexpr ~ArrElem() {} };
+  using Arr = ArrElem[3];
+  static_assert((Arr{}, true));
+}
+
+namespace dynamic_alloc {
+  constexpr int *p = // expected-error {{constant}} expected-note {{pointer to heap-allocated object is not a constant expression}}
+    new int; // expected-note {{heap allocation performed here}}
+
+  constexpr int f(int n) {
+    int *p = new int[n];
+    for (int i = 0; i != n; ++i) {
+      p[i] = i;
+    }
+    int k = 0;
+    for (int i = 0; i != n; ++i) {
+      k += p[i];
+    }
+    delete[] p;
+    return k;
+  }
+  static_assert(f(123) == 123 * 122 / 2);
+
+  constexpr bool nvdtor() { // expected-error {{never produces a constant expression}}
+    struct S {
+      constexpr ~S() {}
+    };
+    struct T : S {};
+    delete (S*)new T; // expected-note {{delete of object with dynamic type 'T' through pointer to base class type 'S' with non-virtual destructor}}
+    return true;
+  }
+
+  constexpr int vdtor_1() {
+    int a;
+    struct S {
+      constexpr S(int *p) : p(p) {}
+      constexpr virtual ~S() { *p = 1; }
+      int *p;
+    };
+    struct T : S {
+      // implicit destructor defined eagerly because it is constexpr and virtual
+      using S::S;
+    };
+    delete (S*)new T(&a);
+    return a;
+  }
+  static_assert(vdtor_1() == 1);
+
+  constexpr int vdtor_2() {
+    int a = 0;
+    struct S { constexpr virtual ~S() {} };
+    struct T : S {
+      constexpr T(int *p) : p(p) {}
+      constexpr ~T() { ++*p; }
+      int *p;
+    };
+    S *p = new T{&a};
+    delete p;
+    return a;
+  }
+  static_assert(vdtor_2() == 1);
+
+  constexpr int vdtor_3(int mode) {
+    int a = 0;
+    struct S { constexpr virtual ~S() {} };
+    struct T : S {
+      constexpr T(int *p) : p(p) {}
+      constexpr ~T() { ++*p; }
+      int *p;
+    };
+    S *p = new T[3]{&a, &a, &a}; // expected-note 2{{heap allocation}}
+    switch (mode) {
+    case 0:
+      delete p; // expected-note {{non-array delete used to delete pointer to array object of type 'T [3]'}}
+      break;
+    case 1:
+      // FIXME: This diagnosic isn't great; we should mention the cast to S*
+      // somewhere in here.
+      delete[] p; // expected-note {{delete of pointer to subobject '&{*new T [3]#0}[0]'}}
+      break;
+    case 2:
+      delete (T*)p; // expected-note {{non-array delete used to delete pointer to array object of type 'T [3]'}}
+      break;
+    case 3:
+      delete[] (T*)p;
+      break;
+    }
+    return a;
+  }
+  static_assert(vdtor_3(0) == 3); // expected-error {{}} expected-note {{in call}}
+  static_assert(vdtor_3(1) == 3); // expected-error {{}} expected-note {{in call}}
+  static_assert(vdtor_3(2) == 3); // expected-error {{}} expected-note {{in call}}
+  static_assert(vdtor_3(3) == 3);
+
+  constexpr void delete_mismatch() { // expected-error {{never produces a constant expression}}
+    delete[] // expected-note {{array delete used to delete pointer to non-array object of type 'int'}}
+      new int; // expected-note {{allocation}}
+  }
+
+  template<typename T>
+  constexpr T dynarray(int elems, int i) {
+    T *p;
+    if constexpr (sizeof(T) == 1)
+      p = new T[elems]{"fox"}; // expected-note {{evaluated array bound 3 is too small to hold 4 explicitly initialized elements}}
+    else
+      p = new T[elems]{1, 2, 3}; // expected-note {{evaluated array bound 2 is too small to hold 3 explicitly initialized elements}}
+    T n = p[i]; // expected-note 4{{past-the-end}}
+    delete [] p;
+    return n;
+  }
+  static_assert(dynarray<int>(4, 0) == 1);
+  static_assert(dynarray<int>(4, 1) == 2);
+  static_assert(dynarray<int>(4, 2) == 3);
+  static_assert(dynarray<int>(4, 3) == 0);
+  static_assert(dynarray<int>(4, 4) == 0); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(dynarray<int>(3, 2) == 3);
+  static_assert(dynarray<int>(3, 3) == 0); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(dynarray<int>(2, 1) == 0); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(dynarray<char>(5, 0) == 'f');
+  static_assert(dynarray<char>(5, 1) == 'o');
+  static_assert(dynarray<char>(5, 2) == 'x');
+  static_assert(dynarray<char>(5, 3) == 0); // (from string)
+  static_assert(dynarray<char>(5, 4) == 0); // (from filler)
+  static_assert(dynarray<char>(5, 5) == 0); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(dynarray<char>(4, 0) == 'f');
+  static_assert(dynarray<char>(4, 1) == 'o');
+  static_assert(dynarray<char>(4, 2) == 'x');
+  static_assert(dynarray<char>(4, 3) == 0);
+  static_assert(dynarray<char>(4, 4) == 0); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(dynarray<char>(3, 2) == 'x'); // expected-error {{constant expression}} expected-note {{in call}}
+
+  constexpr bool run_dtors_on_array_filler() {
+    struct S {
+      int times_destroyed = 0;
+      constexpr ~S() { if (++times_destroyed != 1) throw "oops"; }
+    };
+    delete[] new S[3];
+    return true;
+  }
+  static_assert(run_dtors_on_array_filler());
+
+  constexpr bool erroneous_array_bound(long long n) {
+    delete[] new int[n]; // expected-note {{array bound -1 is negative}} expected-note {{array bound 4611686018427387904 is too large}}
+    return true;
+  }
+  static_assert(erroneous_array_bound(3));
+  static_assert(erroneous_array_bound(0));
+  static_assert(erroneous_array_bound(-1)); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(erroneous_array_bound(1LL << 62)); // expected-error {{constant expression}} expected-note {{in call}}
+
+  constexpr bool erroneous_array_bound_nothrow(long long n) {
+    int *p = new (std::nothrow) int[n];
+    bool result = p != 0;
+    delete[] p;
+    return result;
+  }
+  static_assert(erroneous_array_bound_nothrow(3));
+  static_assert(erroneous_array_bound_nothrow(0));
+  static_assert(!erroneous_array_bound_nothrow(-1));
+  static_assert(!erroneous_array_bound_nothrow(1LL << 62));
+
+  constexpr bool evaluate_nothrow_arg() {
+    bool ok = false;
+    delete new ((ok = true, std::nothrow)) int;
+    return ok;
+  }
+  static_assert(evaluate_nothrow_arg());
+
+  constexpr void double_delete() { // expected-error {{never produces a constant expression}}
+    int *p = new int;
+    delete p;
+    delete p; // expected-note {{delete of pointer that has already been deleted}}
+  }
+  constexpr bool super_secret_double_delete() {
+    struct A {
+      constexpr ~A() { delete this; } // expected-note {{destruction of object that is already being destroyed}} expected-note {{in call}}
+    };
+    delete new A; // expected-note {{in call}}
+    return true;
+  }
+  static_assert(super_secret_double_delete()); // expected-error {{constant expression}} expected-note {{in call}}
+
+  constexpr void use_after_free() { // expected-error {{never produces a constant expression}}
+    int *p = new int;
+    delete p;
+    *p = 1; // expected-note {{assignment to heap allocated object that has been deleted}}
+  }
+  constexpr void use_after_free_2() { // expected-error {{never produces a constant expression}}
+    struct X { constexpr void f() {} };
+    X *p = new X;
+    delete p;
+    p->f(); // expected-note {{member call on heap allocated object that has been deleted}}
+  }
+
+  template<typename T> struct X {
+    std::size_t n;
+    char *p;
+    void dependent();
+  };
+  template<typename T> void X<T>::dependent() {
+    char *p;
+    // Ensure that we don't try to evaluate these for overflow and crash. These
+    // are all value-dependent expressions.
+    p = new char[n];
+    p = new (n) char[n];
+    p = new char(n);
+  }
+}
+
+struct placement_new_arg {};
+void *operator new(std::size_t, placement_new_arg);
+void operator delete(void*, placement_new_arg);
+
+namespace placement_new_delete {
+  struct ClassSpecificNew {
+    void *operator new(std::size_t);
+  };
+  struct ClassSpecificDelete {
+    void operator delete(void*);
+  };
+  struct DestroyingDelete {
+    void operator delete(DestroyingDelete*, std::destroying_delete_t);
+  };
+  struct alignas(64) Overaligned {};
+
+  constexpr bool ok() {
+    delete new Overaligned;
+    delete ::new ClassSpecificNew;
+    ::delete new ClassSpecificDelete;
+    ::delete new DestroyingDelete;
+    return true;
+  }
+  static_assert(ok());
+
+  constexpr bool bad(int which) {
+    switch (which) {
+    case 0:
+      delete new (placement_new_arg{}) int; // expected-note {{call to placement 'operator new'}}
+      break;
+
+    case 1:
+      delete new ClassSpecificNew; // expected-note {{call to class-specific 'operator new'}}
+      break;
+
+    case 2:
+      delete new ClassSpecificDelete; // expected-note {{call to class-specific 'operator delete'}}
+      break;
+
+    case 3:
+      delete new DestroyingDelete; // expected-note {{call to class-specific 'operator delete'}}
+      break;
+
+    case 4:
+      // FIXME: This technically follows the standard's rules, but it seems
+      // unreasonable to expect implementations to support this.
+      delete new (std::align_val_t{64}) Overaligned; // expected-note {{placement new expression is not yet supported}}
+      break;
+    }
+
+    return true;
+  }
+  static_assert(bad(0)); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(bad(1)); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(bad(2)); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(bad(3)); // expected-error {{constant expression}} expected-note {{in call}}
+  static_assert(bad(4)); // expected-error {{constant expression}} expected-note {{in call}}
+}
+
+namespace delete_random_things {
+  static_assert((delete new int, true));
+  static_assert((delete (int*)0, true));
+  int n; // expected-note {{declared here}}
+  static_assert((delete &n, true)); // expected-error {{}} expected-note {{delete of pointer '&n' that does not point to a heap-allocated object}}
+  struct A { int n; };
+  static_assert((delete &(new A)->n, true)); // expected-error {{}} expected-note {{delete of pointer to subobject '&{*new delete_random_things::A#0}.n'}}
+  static_assert((delete (new int + 1), true)); // expected-error {{}} expected-note {{delete of pointer '&{*new int#0} + 1' that does not point to complete object}}
+  static_assert((delete[] (new int[3] + 1), true)); // expected-error {{}} expected-note {{delete of pointer to subobject '&{*new int [3]#0}[1]'}}
+  static_assert((delete &(int&)(int&&)0, true)); // expected-error {{}} expected-note {{delete of pointer '&0' that does not point to a heap-allocated object}} expected-note {{temporary created here}}
+}
+
+namespace value_dependent_delete {
+  template<typename T> void f(T *p) {
+    int arr[(delete p, 0)];
+  }
+}
+
+namespace memory_leaks {
+  static_assert(*new bool(true)); // expected-error {{}} expected-note {{allocation performed here was not deallocated}}
+
+  constexpr bool *f() { return new bool(true); } // expected-note {{allocation performed here was not deallocated}}
+  static_assert(*f()); // expected-error {{}}
+
+  struct UP {
+    bool *p;
+    constexpr ~UP() { delete p; }
+    constexpr bool &operator*() { return *p; }
+  };
+  constexpr UP g() { return {new bool(true)}; }
+  static_assert(*g()); // ok
+
+  constexpr bool h(UP p) { return *p; }
+  static_assert(h({new bool(true)})); // ok
+}
+
+namespace dtor_call {
+  struct A { int n; };
+  constexpr void f() { // expected-error {{never produces a constant expression}}
+    A a; // expected-note {{destroying object 'a' whose lifetime has already ended}}
+    a.~A();
+  }
+  union U { A a; };
+  constexpr void g() {
+    U u;
+    u.a.n = 3;
+    u.a.~A();
+    // There's now effectively no active union member, but we model it as if
+    // 'a' is still the active union member (but its lifetime has ended).
+    u.a.n = 4; // Start lifetime of 'a' again.
+    u.a.~A();
+  }
+  static_assert((g(), true));
+
+  constexpr bool pseudo() {
+    using T = bool;
+    bool b = false;
+    // This does evaluate the store to 'b'...
+    (b = true).~T();
+    // ... but does not end the lifetime of the object.
+    return b;
+  }
+  static_assert(pseudo());
+
+  constexpr void use_after_destroy() {
+    A a;
+    a.~A();
+    A b = a; // expected-note {{in call}} expected-note {{read of object outside its lifetime}}
+  }
+  static_assert((use_after_destroy(), true)); // expected-error {{}} expected-note {{in call}}
+
+  constexpr void double_destroy() {
+    A a;
+    a.~A();
+    a.~A(); // expected-note {{destruction of object outside its lifetime}}
+  }
+  static_assert((double_destroy(), true)); // expected-error {{}} expected-note {{in call}}
+
+  struct X { char *p; constexpr ~X() { *p++ = 'X'; } };
+  struct Y : X { int y; virtual constexpr ~Y() { *p++ = 'Y'; } };
+  struct Z : Y { int z; constexpr ~Z() override { *p++ = 'Z'; } };
+  union VU {
+    constexpr VU() : z() {}
+    constexpr ~VU() {}
+    Z z;
+  };
+
+  constexpr bool virt_dtor(int mode, const char *expected) {
+    char buff[4] = {};
+    VU vu;
+    vu.z.p = buff;
+    switch (mode) {
+    case 0:
+      vu.z.~Z();
+      break;
+    case 1:
+      ((Y&)vu.z).~Y();
+      break;
+    case 2:
+      ((X&)vu.z).~X();
+      break;
+    case 3:
+      ((Y&)vu.z).Y::~Y();
+      vu.z.z = 1; // ok, still have a Z (with no Y base class!)
+      break;
+    case 4:
+      ((X&)vu.z).X::~X();
+      vu.z.y = 1; // ok, still have a Z and a Y (with no X base class!)
+      break;
+    }
+    return __builtin_strcmp(expected, buff) == 0;
+  }
+  static_assert(virt_dtor(0, "ZYX"));
+  static_assert(virt_dtor(1, "ZYX"));
+  static_assert(virt_dtor(2, "X"));
+  static_assert(virt_dtor(3, "YX"));
+  static_assert(virt_dtor(4, "X"));
+
+  constexpr void use_after_virt_destroy() {
+    char buff[4] = {};
+    VU vu;
+    vu.z.p = buff;
+    ((Y&)vu.z).~Y();
+    ((Z&)vu.z).z = 1; // expected-note {{assignment to object outside its lifetime}}
+  }
+  static_assert((use_after_virt_destroy(), true)); // expected-error {{}} expected-note {{in call}}
+
+  constexpr void destroy_after_lifetime() {
+    A *p;
+    {
+      A a;
+      p = &a;
+    }
+    p->~A(); // expected-note {{destruction of object outside its lifetime}}
+  }
+  static_assert((destroy_after_lifetime(), true)); // expected-error {{}} expected-note {{in call}}
+
+  constexpr void destroy_after_lifetime2() {
+    A *p = []{ A a; return &a; }(); // expected-warning {{}} expected-note {{declared here}}
+    p->~A(); // expected-note {{destruction of variable whose lifetime has ended}}
+  }
+  static_assert((destroy_after_lifetime2(), true)); // expected-error {{}} expected-note {{in call}}
+
+  constexpr void destroy_after_lifetime3() {
+    A *p = []{ return &(A&)(A&&)A(); }(); // expected-warning {{}} expected-note {{temporary created here}}
+    p->~A(); // expected-note {{destruction of temporary whose lifetime has ended}}
+  }
+  static_assert((destroy_after_lifetime3(), true)); // expected-error {{}} expected-note {{in call}}
+
+  constexpr void destroy_after_lifetime4() { // expected-error {{never produces a constant expression}}
+    A *p = new A;
+    delete p;
+    p->~A(); // expected-note {{destruction of heap allocated object that has been deleted}}
+  }
+
+  struct Extern { constexpr ~Extern() {} } extern e;
+  constexpr void destroy_extern() { // expected-error {{never produces a constant expression}}
+    e.~Extern(); // expected-note {{cannot modify an object that is visible outside}}
+  }
+
+  constexpr A &&a_ref = A(); // expected-note {{temporary created here}}
+  constexpr void destroy_extern_2() { // expected-error {{never produces a constant expression}}
+    a_ref.~A(); // expected-note {{destruction of temporary is not allowed in a constant expression outside the expression that created the temporary}}
+  }
+
+  struct S {
+    constexpr S() { n = 1; }
+    constexpr ~S() { n = 0; }
+    int n;
+  };
+  constexpr void destroy_volatile() {
+    volatile S s;
+  }
+  static_assert((destroy_volatile(), true)); // ok, not volatile during construction and destruction
+
+  constexpr void destroy_null() { // expected-error {{never produces a constant expression}}
+    ((A*)nullptr)->~A(); // expected-note {{destruction of dereferenced null pointer}}
+  }
+
+  constexpr void destroy_past_end() { // expected-error {{never produces a constant expression}}
+    A a;
+    (&a+1)->~A(); // expected-note {{destruction of dereferenced one-past-the-end pointer}}
+  }
+
+  constexpr void destroy_past_end_array() { // expected-error {{never produces a constant expression}}
+    A a[2];
+    a[2].~A(); // expected-note {{destruction of dereferenced one-past-the-end pointer}}
+  }
+
+  union As {
+    A a, b;
+  };
+
+  constexpr void destroy_no_active() { // expected-error {{never produces a constant expression}}
+    As as;
+    as.b.~A(); // expected-note {{destruction of member 'b' of union with no active member}}
+  }
+
+  constexpr void destroy_inactive() { // expected-error {{never produces a constant expression}}
+    As as;
+    as.a.n = 1;
+    as.b.~A(); // expected-note {{destruction of member 'b' of union with active member 'a'}}
+  }
+
+  constexpr void destroy_no_active_2() { // expected-error {{never produces a constant expression}}
+    As as;
+    as.a.n = 1;
+    as.a.~A();
+    // FIXME: This diagnostic is wrong; the union has no active member now.
+    as.b.~A(); // expected-note {{destruction of member 'b' of union with active member 'a'}}
+  }
+
+  constexpr void destroy_pointer() {
+    using T = int*;
+    T p;
+    // We used to think this was an -> member access because its left-hand side
+    // is a pointer. Ensure we don't crash.
+    p.~T();
+  }
+  static_assert((destroy_pointer(), true));
+}
+
+namespace temp_dtor {
+  void f();
+  struct A {
+    bool b;
+    constexpr ~A() { if (b) f(); }
+  };
+
+  // We can't accept either of these unless we start actually registering the
+  // destructors of the A temporaries to run on shutdown. It's unclear what the
+  // intended standard behavior is so we reject this for now.
+  constexpr A &&a = A{false}; // expected-error {{constant}} expected-note {{non-trivial destruction of lifetime-extended temporary}}
+  void f() { a.b = true; }
+
+  constexpr A &&b = A{true}; // expected-error {{constant}} expected-note {{non-trivial destruction of lifetime-extended temporary}}
+
+  // FIXME: We could in prinicple accept this.
+  constexpr const A &c = A{false}; // expected-error {{constant}} expected-note {{non-trivial destruction of lifetime-extended temporary}}
+}
+
+namespace value_dependent_init {
+  struct A {
+    constexpr ~A() {}
+  };
+  template<typename T> void f() {
+    A a = T();
+  }
+}
diff --git a/clang/test/SemaCXX/constexpr-builtin-bit-cast.cpp b/clang/test/SemaCXX/constexpr-builtin-bit-cast.cpp
index d48ee03dccebb..0a12e7eebe45e 100644
--- a/clang/test/SemaCXX/constexpr-builtin-bit-cast.cpp
+++ b/clang/test/SemaCXX/constexpr-builtin-bit-cast.cpp
@@ -61,13 +61,13 @@ void test_record() {
 
   constexpr int_splicer splice{0x0C05FEFE, 0xCAFEBABE};
 
-  static_assert(bit_cast<unsigned long long>(splice) == LITTLE_END
-                ? 0xCAFEBABE0C05FEFE
-                : 0x0C05FEFECAFEBABE);
+  static_assert(bit_cast<unsigned long long>(splice) == (LITTLE_END
+                                                             ? 0xCAFEBABE0C05FEFE
+                                                             : 0x0C05FEFECAFEBABE));
 
-  static_assert(bit_cast<int_splicer>(0xCAFEBABE0C05FEFE).x == LITTLE_END
-                ? 0x0C05FEFE
-                : 0xCAFEBABE);
+  static_assert(bit_cast<int_splicer>(0xCAFEBABE0C05FEFE).x == (LITTLE_END
+                                                                    ? 0x0C05FEFE
+                                                                    : 0xCAFEBABE));
 
   static_assert(round_trip<unsigned long long>(splice));
   static_assert(round_trip<long long>(splice));
@@ -220,7 +220,7 @@ void backtrace() {
 void test_array_fill() {
   constexpr unsigned char a[4] = {1, 2};
   constexpr unsigned int i = bit_cast<unsigned int>(a);
-  static_assert(i == LITTLE_END ? 0x00000201 : 0x01020000, "");
+  static_assert(i == (LITTLE_END ? 0x00000201 : 0x01020000));
 }
 
 typedef decltype(nullptr) nullptr_t;
diff --git a/clang/test/SemaCXX/constexpr-turing-cxx2a.cpp b/clang/test/SemaCXX/constexpr-turing-cxx2a.cpp
new file mode 100644
index 0000000000000..0941cf408f2e3
--- /dev/null
+++ b/clang/test/SemaCXX/constexpr-turing-cxx2a.cpp
@@ -0,0 +1,66 @@
+// RUN: %clang_cc1 -verify -std=c++2a %s
+// expected-no-diagnostics
+
+const unsigned halt = (unsigned)-1;
+
+enum Dir { L, R };
+struct Action {
+  bool tape;
+  Dir dir;
+  unsigned next;
+};
+using State = Action[2];
+
+// An infinite tape!
+struct Tape {
+  constexpr Tape() = default;
+  constexpr ~Tape() {
+    if (l) { l->r = nullptr; delete l; }
+    if (r) { r->l = nullptr; delete r; }
+  }
+  constexpr Tape *left() {
+    if (!l) { l = new Tape; l->r = this; }
+    return l;
+  }
+  constexpr Tape *right() {
+    if (!r) { r = new Tape; r->l = this; }
+    return r;
+  }
+  Tape *l = nullptr;
+  bool val = false;
+  Tape *r = nullptr;
+};
+
+// Run turing machine 'tm' on tape 'tape' from state 'state'. Return number of
+// steps taken until halt.
+constexpr unsigned run(const State *tm) {
+  Tape *tape = new Tape;
+  unsigned state = 0;
+  unsigned steps = 0;
+
+  for (state = 0; state != halt; ++steps) {
+    auto [val, dir, next_state] = tm[state][tape->val];
+    tape->val = val;
+    tape = (dir == L ? tape->left() : tape->right());
+    state = next_state;
+  }
+
+  delete tape;
+  return steps;
+}
+
+// 3-state busy beaver. S(bb3) = 21.
+constexpr State bb3[] = {
+  { { true, R, 1 }, { true, R, halt } },
+  { { true, L, 1 }, { false, R, 2 } },
+  { { true, L, 2 }, { true, L, 0 } }
+};
+static_assert(run(bb3) == 21, "");
+
+// 4-state busy beaver. S(bb4) = 107.
+constexpr State bb4[] = {
+  { { true, R, 1 }, { true, L, 1 } },
+  { { true, L, 0 }, { false, L, 2 } },
+  { { true, R, halt }, { true, L, 3 } },
+  { { true, R, 3 }, { false, R, 0 } } };
+static_assert(run(bb4) == 107, "");
diff --git a/clang/test/SemaCXX/cxx2a-consteval.cpp b/clang/test/SemaCXX/cxx2a-consteval.cpp
index c531b92033d30..3b52ed4409ee1 100644
--- a/clang/test/SemaCXX/cxx2a-consteval.cpp
+++ b/clang/test/SemaCXX/cxx2a-consteval.cpp
@@ -27,7 +27,7 @@ struct A {
   }
   consteval A(int i);
   consteval A() = default;
-  consteval ~A() = default; // expected-error {{destructor cannot be marked consteval}}
+  consteval ~A() = default;
 };
 
 consteval struct B {}; // expected-error {{struct cannot be marked consteval}}
@@ -45,11 +45,17 @@ consteval int f1() {} // expected-error {{no return statement in consteval funct
 
 struct C {
   C() {}
+  ~C() {}
 };
 
 struct D {
   C c;
   consteval D() = default; // expected-error {{cannot be consteval}}
+  consteval ~D() = default; // expected-error {{cannot be consteval}}
+};
+
+struct E : C { // expected-note {{here}}
+  consteval ~E() {} // expected-error {{cannot be declared consteval because base class 'basic_sema::C' does not have a constexpr destructor}}
 };
 }
 
diff --git a/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
new file mode 100644
index 0000000000000..23582f2e3026e
--- /dev/null
+++ b/clang/test/SemaCXX/cxx2a-constexpr-dynalloc.cpp
@@ -0,0 +1,168 @@
+// RUN: %clang_cc1 -std=c++2a -verify %s -DNEW=__builtin_operator_new -DDELETE=__builtin_operator_delete
+// RUN: %clang_cc1 -std=c++2a -verify %s "-DNEW=operator new" "-DDELETE=operator delete"
+// RUN: %clang_cc1 -std=c++2a -verify %s "-DNEW=::operator new" "-DDELETE=::operator delete"
+
+constexpr bool alloc_from_user_code() {
+  void *p = NEW(sizeof(int)); // expected-note {{cannot allocate untyped memory in a constant expression; use 'std::allocator<T>::allocate'}}
+  DELETE(p);
+  return true;
+}
+static_assert(alloc_from_user_code()); // expected-error {{constant expression}} expected-note {{in call}}
+
+namespace std {
+  using size_t = decltype(sizeof(0));
+  // FIXME: It would be preferable to point these notes at the location of the call to allocator<...>::[de]allocate instead
+  template<typename T> struct allocator {
+    constexpr T *allocate(size_t N) {
+      return (T*)NEW(sizeof(T) * N); // expected-note 3{{heap allocation}} expected-note {{not deallocated}}
+    }
+    constexpr void deallocate(void *p) {
+      DELETE(p); // expected-note 2{{'std::allocator<...>::deallocate' used to delete pointer to object allocated with 'new'}}
+    }
+  };
+}
+
+constexpr bool alloc_via_std_allocator() {
+  std::allocator<int> alloc;
+  int *p = alloc.allocate(1);
+  alloc.deallocate(p);
+  return true;
+}
+static_assert(alloc_via_std_allocator());
+
+template<> struct std::allocator<void()> {
+  constexpr void *allocate() { return NEW(8); } // expected-note {{cannot allocate memory of function type 'void ()'}}
+};
+constexpr void *fn = std::allocator<void()>().allocate(); // expected-error {{constant expression}} expected-note {{in call}}
+
+struct Incomplete;
+template<> struct std::allocator<Incomplete> {
+  constexpr void *allocate() { return NEW(8); } // expected-note {{cannot allocate memory of incomplete type 'Incomplete'}}
+};
+constexpr void *incomplete = std::allocator<Incomplete>().allocate(); // expected-error {{constant expression}} expected-note {{in call}}
+
+struct WrongSize { char x[5]; };
+static_assert(sizeof(WrongSize) == 5);
+template<> struct std::allocator<WrongSize> {
+  constexpr void *allocate() { return NEW(7); } // expected-note {{allocated size 7 is not a multiple of size 5 of element type 'WrongSize'}}
+};
+constexpr void *wrong_size = std::allocator<WrongSize>().allocate(); // expected-error {{constant expression}} expected-note {{in call}}
+
+constexpr bool mismatched(int alloc_kind, int dealloc_kind) {
+  int *p;
+  switch (alloc_kind) {
+  case 0:
+    p = new int; // expected-note {{heap allocation}}
+    break;
+  case 1:
+    p = new int[1]; // expected-note {{heap allocation}}
+    break;
+  case 2:
+    p = std::allocator<int>().allocate(1);
+    break;
+  }
+  switch (dealloc_kind) {
+  case 0:
+    delete p; // expected-note {{'delete' used to delete pointer to object allocated with 'std::allocator<...>::allocate'}}
+    break;
+  case 1:
+    delete[] p; // expected-note {{'delete' used to delete pointer to object allocated with 'std::allocator<...>::allocate'}}
+    break;
+  case 2:
+    std::allocator<int>().deallocate(p); // expected-note 2{{in call}}
+    break;
+  }
+  return true;
+}
+static_assert(mismatched(0, 2)); // expected-error {{constant expression}} expected-note {{in call}}
+static_assert(mismatched(1, 2)); // expected-error {{constant expression}} expected-note {{in call}}
+static_assert(mismatched(2, 0)); // expected-error {{constant expression}} expected-note {{in call}}
+static_assert(mismatched(2, 1)); // expected-error {{constant expression}} expected-note {{in call}}
+static_assert(mismatched(2, 2));
+
+constexpr int *escape = std::allocator<int>().allocate(3); // expected-error {{constant expression}} expected-note {{pointer to subobject of heap-allocated}}
+constexpr int leak = (std::allocator<int>().allocate(3), 0); // expected-error {{constant expression}}
+constexpr int no_lifetime_start = (*std::allocator<int>().allocate(1) = 1); // expected-error {{constant expression}} expected-note {{assignment to object outside its lifetime}}
+
+void *operator new(std::size_t, void *p) { return p; }
+constexpr bool no_placement_new_in_user_code() { // expected-error {{never produces a constant expression}}
+  int a;
+  new (&a) int(42); // expected-note {{call to placement 'operator new'}}
+  return a == 42;
+}
+
+namespace std {
+  constexpr bool placement_new_in_stdlib() {
+    int a;
+    new (&a) int(42);
+    return a == 42;
+  }
+}
+static_assert(std::placement_new_in_stdlib());
+
+namespace std {
+  template<typename T, typename ...Args>
+  constexpr void construct_at(void *p, Args &&...args) {
+    new (p) T((Args&&)args...); // #new
+  }
+}
+
+constexpr bool call_std_construct_at() {
+  int *p = std::allocator<int>().allocate(3);
+  std::construct_at<int>(p, 1);
+  std::construct_at<int>(p + 1, 2);
+  std::construct_at<int>(p + 2, 3);
+  bool good = p[0] + p[1] + p[2] == 6;
+  std::allocator<int>().deallocate(p);
+  return good;
+}
+static_assert(call_std_construct_at());
+
+constexpr bool bad_construct_at_type() {
+  int a;
+  // expected-note@#new {{placement new would change type of storage from 'int' to 'float'}}
+  std::construct_at<float>(&a, 1.0f); // expected-note {{in call}}
+  return true;
+}
+static_assert(bad_construct_at_type()); // expected-error{{}} expected-note {{in call}}
+
+constexpr bool bad_construct_at_subobject() {
+  struct X { int a, b; };
+  union A {
+    int a;
+    X x;
+  };
+  A a = {1};
+  // expected-note@#new {{construction of subobject of member 'x' of union with active member 'a' is not allowed in a constant expression}}
+  std::construct_at<int>(&a.x.a, 1); // expected-note {{in call}}
+  return true;
+}
+static_assert(bad_construct_at_subobject()); // expected-error{{}} expected-note {{in call}}
+
+constexpr bool change_union_member() {
+  union U {
+    int a;
+    int b;
+  };
+  U u = {.a = 1};
+  std::construct_at<int>(&u.b, 2);
+  return u.b == 2;
+}
+static_assert(change_union_member());
+
+int external;
+// expected-note@#new {{visible outside}}
+static_assert((std::construct_at<int>(&external, 1), true)); // expected-error{{}} expected-note {{in call}}
+
+constexpr int &&temporary = 0; // expected-note {{created here}}
+// expected-note@#new {{construction of temporary is not allowed in a constant expression outside the expression that created the temporary}}
+static_assert((std::construct_at<int>(&temporary, 1), true)); // expected-error{{}} expected-note {{in call}}
+
+constexpr bool construct_after_lifetime() {
+  int *p = new int;
+  delete p;
+  // expected-note@#new {{construction of heap allocated object that has been deleted}}
+  std::construct_at<int>(p); // expected-note {{in call}}
+  return true;
+}
+static_assert(construct_after_lifetime()); // expected-error {{}} expected-note {{in call}}
diff --git a/clang/test/SemaCXX/cxx2a-explicit-bool.cpp b/clang/test/SemaCXX/cxx2a-explicit-bool.cpp
index 1c532cfb59766..56fa76f0a8bd5 100644
--- a/clang/test/SemaCXX/cxx2a-explicit-bool.cpp
+++ b/clang/test/SemaCXX/cxx2a-explicit-bool.cpp
@@ -717,3 +717,21 @@ A d2{0, 0};
 A d3 = {0.0, 0.0};// expected-error {{explicit deduction guide}}
 
 }
+
+namespace PR42980 {
+using size_t = decltype(sizeof(0));
+
+struct Str {// expected-note+ {{candidate constructor}}
+  template <size_t N>
+  explicit(N > 7)// expected-note {{resolved to true}}
+  Str(char const (&str)[N]);
+};
+
+template <size_t N>
+Str::Str(char const(&str)[N]) { }
+// expected-note@-1 {{candidate constructor}}
+
+Str a = "short";
+Str b = "not so short";// expected-error {{no viable conversion}}
+
+}
\ No newline at end of file
diff --git a/clang/test/SemaCXX/gnu_inline.cpp b/clang/test/SemaCXX/gnu_inline.cpp
new file mode 100644
index 0000000000000..43a4bae48e95c
--- /dev/null
+++ b/clang/test/SemaCXX/gnu_inline.cpp
@@ -0,0 +1,9 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+
+extern inline
+__attribute__((__gnu_inline__))
+void gnu_inline1() {}
+
+inline
+__attribute__((__gnu_inline__)) // expected-warning {{'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10}}
+void gnu_inline2() {}
diff --git a/clang/test/SemaCXX/ms-exception-spec.cpp b/clang/test/SemaCXX/ms-exception-spec.cpp
index 60bfeba037ad4..cf460356c9789 100644
--- a/clang/test/SemaCXX/ms-exception-spec.cpp
+++ b/clang/test/SemaCXX/ms-exception-spec.cpp
@@ -1,9 +1,36 @@
-// RUN: %clang_cc1 %s -fsyntax-only -verify -fms-compatibility -fexceptions -fcxx-exceptions
+// RUN: %clang_cc1 -std=c++11 %s -fsyntax-only -verify -fms-compatibility -fexceptions -fcxx-exceptions
+// RUN: %clang_cc1 -std=c++17 %s -fsyntax-only -verify -fms-compatibility -fexceptions -fcxx-exceptions
 
+// FIXME: Should -fms-compatibility soften these errors into warnings to match
+// MSVC? In practice, MSVC never implemented dynamic exception specifiers, so
+// there isn't much Windows code in the wild that uses them.
+#if __cplusplus >= 201703L
+// expected-error@+3 {{ISO C++17 does not allow dynamic exception specifications}}
+// expected-note@+2 {{use 'noexcept(false)' instead}}
+#endif
 void f() throw(...) { }
 
 namespace PR28080 {
 struct S;           // expected-note {{forward declaration}}
+#if __cplusplus >= 201703L
+// expected-error@+3 {{ISO C++17 does not allow dynamic exception specifications}}
+// expected-note@+2 {{use 'noexcept(false)' instead}}
+#endif
 void fn() throw(S); // expected-warning {{incomplete type}} expected-note{{previous declaration}}
 void fn() throw();  // expected-warning {{does not match previous declaration}}
 }
+
+template <typename T> struct FooPtr {
+  template <typename U> FooPtr(U *p) : m_pT(nullptr) {}
+
+  template <>
+      // FIXME: It would be better if this note pointed at the primary template
+      // above.
+      // expected-note@+1 {{previous declaration is here}}
+  FooPtr(T *pInterface) throw() // expected-warning {{exception specification in declaration does not match previous declaration}}
+      : m_pT(pInterface) {}
+
+  T *m_pT;
+};
+struct Bar {};
+template struct FooPtr<Bar>; // expected-note {{requested here}}
diff --git a/clang/test/SemaCXX/self-comparison.cpp b/clang/test/SemaCXX/self-comparison.cpp
index ac129b68a67a9..e20706a8b6cc9 100644
--- a/clang/test/SemaCXX/self-comparison.cpp
+++ b/clang/test/SemaCXX/self-comparison.cpp
@@ -40,3 +40,85 @@ bool g() {
     Y<T>::n == Y<U>::n;
 }
 template bool g<int, int>(); // should not produce any warnings
+
+namespace member_tests {
+struct B {
+  int field;
+  static int static_field;
+  int test(B b) {
+    return field == field;  // expected-warning {{self-comparison always evaluates to true}}
+    return static_field == static_field;  // expected-warning {{self-comparison always evaluates to true}}
+    return static_field == b.static_field;  // expected-warning {{self-comparison always evaluates to true}}
+    return B::static_field == this->static_field;  // expected-warning {{self-comparison always evaluates to true}}
+    return this == this;  // expected-warning {{self-comparison always evaluates to true}}
+
+    return field == b.field;
+    return this->field == b.field;
+  }
+};
+
+enum {
+  I0,
+  I1,
+  I2,
+};
+
+struct S {
+  int field;
+  static int static_field;
+  int array[4];
+};
+
+struct T {
+  int field;
+  static int static_field;
+  int array[4];
+  S s;
+};
+
+int struct_test(S s1, S s2, S *s3, T t) {
+  return s1.field == s1.field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s2.field == s2.field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.static_field == s2.static_field;  // expected-warning {{self-comparison always evaluates to true}}
+  return S::static_field == s1.static_field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.array == s1.array;  // expected-warning {{self-comparison always evaluates to true}}
+  return t.s.static_field == S::static_field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s3->field == s3->field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s3->static_field == S::static_field;  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.array[0] == s1.array[0];  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.array[0] == s1.array[0ull];  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.array[I1] == s1.array[I1];  // expected-warning {{self-comparison always evaluates to true}}
+  return s1.array[s2.array[0]] == s1.array[s2.array[0]];  // expected-warning {{self-comparison always evaluates to true}}
+  return s3->array[t.field] == s3->array[t.field];  // expected-warning {{self-comparison always evaluates to true}}
+
+  // Try all operators
+  return t.field == t.field;  // expected-warning {{self-comparison always evaluates to true}}
+  return t.field <= t.field;  // expected-warning {{self-comparison always evaluates to true}}
+  return t.field >= t.field;  // expected-warning {{self-comparison always evaluates to true}}
+
+  return t.field != t.field;  // expected-warning {{self-comparison always evaluates to false}}
+  return t.field < t.field;  // expected-warning {{self-comparison always evaluates to false}}
+  return t.field > t.field;  // expected-warning {{self-comparison always evaluates to false}}
+
+  // no warning
+  return s1.field == s2.field;
+  return s2.array == s1.array;
+  return s2.array[0] == s1.array[0];
+  return s1.array[I1] == s1.array[I2];
+
+  return s1.static_field == t.static_field;
+};
+
+struct U {
+  bool operator!=(const U&);
+};
+
+bool operator==(const U&, const U&);
+
+// May want to warn on this in the future.
+int user_defined(U u) {
+  return u == u;
+  return u != u;
+}
+
+} // namespace member_tests
diff --git a/clang/test/SemaCXX/type-traits.cpp b/clang/test/SemaCXX/type-traits.cpp
index 9bc238e678954..99270f19e68b1 100644
--- a/clang/test/SemaCXX/type-traits.cpp
+++ b/clang/test/SemaCXX/type-traits.cpp
@@ -12,6 +12,7 @@ typedef NonPOD NonPODArMB[10][2];
 
 // PODs
 enum Enum { EV };
+enum SignedEnum : signed int { };
 struct POD { Enum e; int i; float f; NonPOD* p; };
 struct Empty {};
 struct IncompleteStruct;
@@ -56,14 +57,14 @@ struct HasInheritedCons : HasDefaultCons { using HasDefaultCons::HasDefaultCons;
 struct HasNoInheritedCons : HasCons {};
 struct HasCopyAssign { HasCopyAssign operator =(const HasCopyAssign&); };
 struct HasMoveAssign { HasMoveAssign operator =(const HasMoveAssign&&); };
-struct HasNoThrowMoveAssign { 
+struct HasNoThrowMoveAssign {
   HasNoThrowMoveAssign& operator=(
     const HasNoThrowMoveAssign&&) throw(); };
-struct HasNoExceptNoThrowMoveAssign { 
+struct HasNoExceptNoThrowMoveAssign {
   HasNoExceptNoThrowMoveAssign& operator=(
-    const HasNoExceptNoThrowMoveAssign&&) noexcept; 
+    const HasNoExceptNoThrowMoveAssign&&) noexcept;
 };
-struct HasThrowMoveAssign { 
+struct HasThrowMoveAssign {
   HasThrowMoveAssign& operator=(const HasThrowMoveAssign&&)
 #if __cplusplus <= 201402L
   throw(POD);
@@ -73,7 +74,7 @@ struct HasThrowMoveAssign {
 };
 
 
-struct HasNoExceptFalseMoveAssign { 
+struct HasNoExceptFalseMoveAssign {
   HasNoExceptFalseMoveAssign& operator=(
     const HasNoExceptFalseMoveAssign&&) noexcept(false); };
 struct HasMoveCtor { HasMoveCtor(const HasMoveCtor&&); };
@@ -82,17 +83,17 @@ struct HasMemberMoveAssign { HasMoveAssign member; };
 struct HasStaticMemberMoveCtor { static HasMoveCtor member; };
 struct HasStaticMemberMoveAssign { static HasMoveAssign member; };
 struct HasMemberThrowMoveAssign { HasThrowMoveAssign member; };
-struct HasMemberNoExceptFalseMoveAssign { 
+struct HasMemberNoExceptFalseMoveAssign {
   HasNoExceptFalseMoveAssign member; };
 struct HasMemberNoThrowMoveAssign { HasNoThrowMoveAssign member; };
-struct HasMemberNoExceptNoThrowMoveAssign { 
+struct HasMemberNoExceptNoThrowMoveAssign {
   HasNoExceptNoThrowMoveAssign member; };
 
-struct HasDefaultTrivialCopyAssign { 
+struct HasDefaultTrivialCopyAssign {
   HasDefaultTrivialCopyAssign &operator=(
-    const HasDefaultTrivialCopyAssign&) = default; 
+    const HasDefaultTrivialCopyAssign&) = default;
 };
-struct TrivialMoveButNotCopy { 
+struct TrivialMoveButNotCopy {
   TrivialMoveButNotCopy &operator=(TrivialMoveButNotCopy&&) = default;
   TrivialMoveButNotCopy &operator=(const TrivialMoveButNotCopy&);
 };
@@ -361,7 +362,7 @@ void is_enum()
 struct FinalClass final {
 };
 
-template<typename T> 
+template<typename T>
 struct PotentiallyFinal { };
 
 template<typename T>
@@ -801,6 +802,7 @@ void is_fundamental()
   int t23[T(__is_fundamental(unsigned long))];
   int t24[T(__is_fundamental(void))];
   int t25[T(__is_fundamental(cvoid))];
+  int t26[T(__is_fundamental(decltype(nullptr)))];
 
   int t30[F(__is_fundamental(Union))];
   int t31[F(__is_fundamental(UnionAr))];
@@ -1419,12 +1421,12 @@ void is_signed()
   int t04[T(__is_signed(short))];
   int t05[T(__is_signed(signed char))];
   int t06[T(__is_signed(wchar_t))];
+  int t07[T(__is_signed(float))];
+  int t08[T(__is_signed(double))];
+  int t09[T(__is_signed(long double))];
 
-  int t10[F(__is_signed(bool))];
-  int t11[F(__is_signed(cvoid))];
-  int t12[F(__is_signed(float))];
-  int t13[F(__is_signed(double))];
-  int t14[F(__is_signed(long double))];
+  int t13[F(__is_signed(bool))];
+  int t14[F(__is_signed(cvoid))];
   int t15[F(__is_signed(unsigned char))];
   int t16[F(__is_signed(unsigned int))];
   int t17[F(__is_signed(unsigned long long))];
@@ -1434,9 +1436,10 @@ void is_signed()
   int t21[F(__is_signed(ClassType))];
   int t22[F(__is_signed(Derives))];
   int t23[F(__is_signed(Enum))];
-  int t24[F(__is_signed(IntArNB))];
-  int t25[F(__is_signed(Union))];
-  int t26[F(__is_signed(UnionAr))];
+  int t24[F(__is_signed(SignedEnum))];
+  int t25[F(__is_signed(IntArNB))];
+  int t26[F(__is_signed(Union))];
+  int t27[F(__is_signed(UnionAr))];
 }
 
 void is_unsigned()
@@ -2005,7 +2008,7 @@ class PrivateCopy {
 };
 
 template<typename T>
-struct X0 { 
+struct X0 {
   template<typename U> X0(const X0<U>&);
 };
 
@@ -2766,7 +2769,7 @@ struct CanBeUniqueIfNoPadding : NotUniqueBecauseTailPadding {
   char b[7];
 };
 
-static_assert(!has_unique_object_representations<NotUniqueBecauseTailPadding>::value, 
+static_assert(!has_unique_object_representations<NotUniqueBecauseTailPadding>::value,
               "non trivial");
 // Can be unique on Itanium, since the is child class' data is 'folded' into the
 // parent's tail padding.
diff --git a/clang/test/SemaCXX/undefined-inline.cpp b/clang/test/SemaCXX/undefined-inline.cpp
index feb12f4552cab..21c6b5de79de3 100644
--- a/clang/test/SemaCXX/undefined-inline.cpp
+++ b/clang/test/SemaCXX/undefined-inline.cpp
@@ -40,20 +40,20 @@ namespace test7 {
 }
 
 namespace test8 {
-  inline void foo() __attribute__((gnu_inline));
+  inline void foo() __attribute__((gnu_inline)); // expected-warning {{'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10}}
   void test() { foo(); }
 }
 
 namespace test9 {
   void foo();
   void test() { foo(); }
-  inline void foo() __attribute__((gnu_inline));
+  inline void foo() __attribute__((gnu_inline)); // expected-warning {{'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10}}
 }
 
 namespace test10 {
   inline void foo();
   void test() { foo(); }
-  inline void foo() __attribute__((gnu_inline));
+  inline void foo() __attribute__((gnu_inline)); // expected-warning {{'gnu_inline' attribute without 'extern' in C++ treated as externally available, this changed in Clang 10}}
 }
 
 namespace test11 {
diff --git a/clang/test/SemaCXX/warn-consumed-analysis.cpp b/clang/test/SemaCXX/warn-consumed-analysis.cpp
index 0a6aed6186265..b4dddb6763724 100644
--- a/clang/test/SemaCXX/warn-consumed-analysis.cpp
+++ b/clang/test/SemaCXX/warn-consumed-analysis.cpp
@@ -53,12 +53,18 @@ class CONSUMABLE(unconsumed) DestructorTester {
 public:
   DestructorTester();
   DestructorTester(int);
+  DestructorTester(nullptr_t) RETURN_TYPESTATE(unconsumed);
+  DestructorTester(DestructorTester &&);
   
   void operator*() CALLABLE_WHEN("unconsumed");
   
   ~DestructorTester() CALLABLE_WHEN("consumed");
+
 };
 
+void dtByVal(DestructorTester);
+void dtByValMarkUnconsumed(DestructorTester RETURN_TYPESTATE(unconsumed));
+
 void baf0(const ConsumableClass<int>  var);
 void baf1(const ConsumableClass<int> &var);
 void baf2(const ConsumableClass<int> *var);
@@ -120,6 +126,19 @@ void testDestruction() {
              expected-warning {{invalid invocation of method '~DestructorTester' on object 'D1' while it is in the 'unconsumed' state}}
 }
 
+void testDestructionByVal() {
+  {
+    // both the var and the temporary are consumed:
+    DestructorTester D0(nullptr);
+    dtByVal((DestructorTester &&)D0);
+  }
+  {
+    // the var is consumed but the temporary isn't:
+    DestructorTester D1(nullptr);
+    dtByValMarkUnconsumed((DestructorTester &&)D1); // expected-warning {{invalid invocation of method '~DestructorTester' on a temporary object while it is in the 'unconsumed' state}}
+  }
+}
+
 void testTempValue() {
   *ConsumableClass<int>(); // expected-warning {{invalid invocation of method 'operator*' on a temporary object while it is in the 'consumed' state}}
 }
@@ -413,10 +432,15 @@ void testParamReturnTypestateCallee(bool cond, ConsumableClass<int> &Param RETUR
   Param.consume();
 }
 
+void testRvalueRefParamReturnTypestateCallee(ConsumableClass<int> &&Param RETURN_TYPESTATE(unconsumed)) {
+  Param.unconsume();
+}
+
 void testParamReturnTypestateCaller() {
   ConsumableClass<int> var;
   
   testParamReturnTypestateCallee(true, var);
+  testRvalueRefParamReturnTypestateCallee((ConsumableClass<int> &&)var);
   
   *var;
 }
@@ -480,6 +504,9 @@ void testCallingConventions() {
   
   baf2(&var);  
   *var;
+
+  baf3(var);
+  *var;
   
   baf4(var);  
   *var; // expected-warning {{invalid invocation of method 'operator*' on object 'var' while it is in the 'unknown' state}}
diff --git a/clang/test/SemaCXX/warn-pessmizing-move.cpp b/clang/test/SemaCXX/warn-pessmizing-move.cpp
index deedec41bd766..2c27cd7f95f74 100644
--- a/clang/test/SemaCXX/warn-pessmizing-move.cpp
+++ b/clang/test/SemaCXX/warn-pessmizing-move.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -Wpessimizing-move -std=c++11 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -Wpessimizing-move -std=c++11 -verify %s -DUSER_DEFINED
 // RUN: %clang_cc1 -fsyntax-only -Wpessimizing-move -std=c++11 -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s
 
 // definitions for std::move
@@ -12,7 +13,15 @@ template <class T> typename remove_reference<T>::type &&move(T &&t);
 }
 }
 
-struct A {};
+struct A {
+#ifdef USER_DEFINED
+  A() {}
+  A(const A &) {}
+  A(A &&) {}
+  A &operator=(const A &) { return *this; }
+  A &operator=(A &&) { return *this; }
+#endif
+};
 struct B {
   B() {}
   B(A) {}
@@ -47,6 +56,19 @@ B test2(A a1, B b1) {
   // expected-note@-2{{remove std::move call}}
   // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:20}:""
   // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:22-[[@LINE-4]]:23}:""
+
+  return A();
+  return test1(a2);
+  return std::move(A());
+  // expected-warning@-1{{prevents copy elision}}
+  // expected-note@-2{{remove std::move call}}
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:20}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:23-[[@LINE-4]]:24}:""
+  return std::move(test1(a2));
+  // expected-warning@-1{{prevents copy elision}}
+  // expected-note@-2{{remove std::move call}}
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:20}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:29-[[@LINE-4]]:30}:""
 }
 
 A global_a;
@@ -101,11 +123,24 @@ void test6() {
   // expected-note@-2{{remove std::move call}}
   // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:20}:""
   // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:23-[[@LINE-4]]:24}:""
+
+  a3 = std::move(A());
+  // expected-warning@-1{{prevents copy elision}}
+  // expected-note@-2{{remove std::move call}}
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:8-[[@LINE-3]]:18}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:21-[[@LINE-4]]:22}:""
+
   A a4 = std::move(test3());
   // expected-warning@-1{{prevents copy elision}}
   // expected-note@-2{{remove std::move call}}
   // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:20}:""
   // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:27-[[@LINE-4]]:28}:""
+
+  a4 = std::move(test3());
+  // expected-warning@-1{{prevents copy elision}}
+  // expected-note@-2{{remove std::move call}}
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:8-[[@LINE-3]]:18}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:25-[[@LINE-4]]:26}:""
 }
 
 A test7() {
@@ -122,13 +157,13 @@ A test7() {
   A a3 = (std::move(A()));
   // expected-warning@-1{{prevents copy elision}}
   // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:21}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:24-[[@LINE-4]]:26}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:11-[[@LINE-3]]:21}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:24-[[@LINE-4]]:25}:""
   A a4 = (std::move((A())));
   // expected-warning@-1{{prevents copy elision}}
   // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:21}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:26-[[@LINE-4]]:28}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:11-[[@LINE-3]]:21}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:26-[[@LINE-4]]:27}:""
 
   return std::move(a1);
   // expected-warning@-1{{prevents copy elision}}
@@ -143,13 +178,13 @@ A test7() {
   return (std::move(a1));
   // expected-warning@-1{{prevents copy elision}}
   // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:21}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:23-[[@LINE-4]]:25}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:11-[[@LINE-3]]:21}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:23-[[@LINE-4]]:24}:""
   return (std::move((a1)));
   // expected-warning@-1{{prevents copy elision}}
   // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:21}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:25-[[@LINE-4]]:27}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:11-[[@LINE-3]]:21}:""
+  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:25-[[@LINE-4]]:26}:""
 }
 
 #define wrap1(x) x
@@ -227,30 +262,3 @@ namespace templates {
     test2<B, A>();
   }
 }
-
-A init_list() {
-  A a1;
-  return { std::move(a1) };
-  // expected-warning@-1{{prevents copy elision}}
-  // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:22}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:24-[[@LINE-4]]:27}:""
-
-  return { (std::move(a1)) };
-  // expected-warning@-1{{prevents copy elision}}
-  // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:23}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:25-[[@LINE-4]]:29}:""
-
-  A a2 = { std::move(A()) };
-  // expected-warning@-1{{prevents copy elision}}
-  // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:22}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:25-[[@LINE-4]]:28}:""
-
-  A a3 = { (std::move(A())) };
-  // expected-warning@-1{{prevents copy elision}}
-  // expected-note@-2{{remove std::move call}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:23}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:26-[[@LINE-4]]:30}:""
-}
diff --git a/clang/test/SemaCXX/warn-redundant-move.cpp b/clang/test/SemaCXX/warn-redundant-move.cpp
index e04a525641738..2bfc8c9312f02 100644
--- a/clang/test/SemaCXX/warn-redundant-move.cpp
+++ b/clang/test/SemaCXX/warn-redundant-move.cpp
@@ -114,17 +114,3 @@ namespace templates {
     test2<B, A>(A());
   }
 }
-
-A init_list(A a) {
-  return { std::move(a) };
-  // expected-warning@-1{{redundant move in return statement}}
-  // expected-note@-2{{remove std::move call here}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:22}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:23-[[@LINE-4]]:26}:""
-
-  return { (std::move(a)) };
-  // expected-warning@-1{{redundant move in return statement}}
-  // expected-note@-2{{remove std::move call here}}
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:10-[[@LINE-3]]:23}:""
-  // CHECK: fix-it:"{{.*}}":{[[@LINE-4]]:24-[[@LINE-4]]:28}:""
-}
diff --git a/clang/test/SemaCXX/warn-unreachable.cpp b/clang/test/SemaCXX/warn-unreachable.cpp
index a14e4226861c1..c664c1912899d 100644
--- a/clang/test/SemaCXX/warn-unreachable.cpp
+++ b/clang/test/SemaCXX/warn-unreachable.cpp
@@ -393,6 +393,9 @@ void tautological_compare(bool x, int y) {
   else
     calledFun();        // expected-warning {{will never be executed}}
 
+  if (y == -1 && y != -1)  // expected-note {{silence}}
+    calledFun();        // expected-warning {{will never be executed}}
+
   // TODO: Extend warning to the following code:
   if (x < -1)
     calledFun();
@@ -408,6 +411,4 @@ void tautological_compare(bool x, int y) {
   else
     calledFun();
 
-  if (y == -1 && y != -1)
-    calledFun();
 }
diff --git a/clang/test/SemaObjC/signed-char-bool-conversion.m b/clang/test/SemaObjC/signed-char-bool-conversion.m
new file mode 100644
index 0000000000000..476ecc6a0603c
--- /dev/null
+++ b/clang/test/SemaObjC/signed-char-bool-conversion.m
@@ -0,0 +1,49 @@
+// RUN: %clang_cc1 %s -verify -Wobjc-signed-char-bool
+// RUN: %clang_cc1 -xobjective-c++ %s -verify -Wobjc-signed-char-bool
+
+typedef signed char BOOL;
+#define YES __objc_yes
+#define NO __objc_no
+
+typedef unsigned char Boolean;
+
+BOOL b;
+Boolean boolean;
+float fl;
+int i;
+int *ptr;
+
+void t1() {
+  b = boolean;
+  b = fl; // expected-warning {{implicit conversion from floating-point type 'float' to 'BOOL'}}
+  b = i; // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+
+  b = 1.0;
+  b = 0.0;
+  b = 1.1; // expected-warning {{implicit conversion from 'double' to 'BOOL' (aka 'signed char') changes value from 1.1 to 1}}
+  b = 2.1; // expected-warning {{implicit conversion from constant value 2.1 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+
+  b = YES;
+#ifndef __cplusplus
+  b = ptr; // expected-warning {{incompatible pointer to integer conversion assigning to 'BOOL' (aka 'signed char') from 'int *'}}
+#endif
+}
+
+@interface BoolProp
+@property BOOL p;
+@end
+
+void t2(BoolProp *bp) {
+  bp.p = YES;
+  bp.p = NO;
+  bp.p = boolean;
+  bp.p = fl; // expected-warning {{implicit conversion from floating-point type 'float' to 'BOOL'}}
+  bp.p = i; // expected-warning {{implicit conversion from integral type 'int' to 'BOOL'}}
+  bp.p = b;
+  bp.p = bp.p;
+#ifndef __cplusplus
+  bp.p = ptr; // expected-warning {{incompatible pointer to integer conversion assigning to 'BOOL' (aka 'signed char') from 'int *'}}
+#endif
+  bp.p = 1;
+  bp.p = 2; // expected-warning {{implicit conversion from constant value 2 to 'BOOL'; the only well defined values for 'BOOL' are YES and NO}}
+}
diff --git a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
index 590f27353dd4c..250a3f008c942 100644
--- a/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
+++ b/clang/test/SemaOpenCL/fdeclare-opencl-builtins.cl
@@ -1,10 +1,20 @@
-// RUN: %clang_cc1 %s -triple spir -verify -pedantic -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -DNO_HEADER
-// RUN: %clang_cc1 %s -triple spir -verify -pedantic -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -finclude-default-header
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -fdeclare-opencl-builtins -DNO_HEADER
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL -fdeclare-opencl-builtins -finclude-default-header
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL1.2 -fdeclare-opencl-builtins -DNO_HEADER
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL1.2 -fdeclare-opencl-builtins -finclude-default-header
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -DNO_HEADER
+// RUN: %clang_cc1 %s -triple spir -verify -pedantic -Wconversion -Werror -fsyntax-only -cl-std=CL2.0 -fdeclare-opencl-builtins -finclude-default-header
+
+#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 // expected-no-diagnostics
+#endif
 
 // Test the -fdeclare-opencl-builtins option.
 
 #pragma OPENCL EXTENSION cl_khr_fp16 : enable
+#if __OPENCL_C_VERSION__ < CL_VERSION_1_2
+#pragma OPENCL EXTENSION cl_khr_fp64 : enable
+#endif
 
 // Provide typedefs when invoking clang without -finclude-default-header.
 #ifdef NO_HEADER
@@ -15,7 +25,10 @@ typedef half half4 __attribute__((ext_vector_type(4)));
 typedef int int2 __attribute__((ext_vector_type(2)));
 typedef int int4 __attribute__((ext_vector_type(4)));
 typedef long long2 __attribute__((ext_vector_type(2)));
+typedef unsigned char uchar;
 typedef unsigned int uint;
+typedef unsigned long ulong;
+typedef unsigned short ushort;
 typedef __SIZE_TYPE__ size_t;
 #endif
 
@@ -58,14 +71,20 @@ kernel void basic_image_readonly(read_only image2d_t image_read_only_image2d) {
   resf = read_imagef(image_read_only_image2d, i2);
   res = read_imageh(image_read_only_image2d, i2);
   res = read_imageh(image_read_only_image2d, sampler, i2);
+
+  int imgWidth = get_image_width(image_read_only_image2d);
 }
 
+#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 kernel void basic_image_readwrite(read_write image3d_t image_read_write_image3d) {
   half4 h4;
   int4 i4;
 
   write_imageh(image_read_write_image3d, i4, h4);
+
+  int imgDepth = get_image_depth(image_read_write_image3d);
 }
+#endif // __OPENCL_C_VERSION__ >= CL_VERSION_2_0
 
 kernel void basic_image_writeonly(write_only image1d_buffer_t image_write_only_image1d_buffer) {
   half4 h4;
@@ -78,4 +97,40 @@ kernel void basic_image_writeonly(write_only image1d_buffer_t image_write_only_i
 
 kernel void basic_subgroup(global uint *out) {
   out[0] = get_sub_group_size();
+#if __OPENCL_C_VERSION__ < CL_VERSION_2_0
+// expected-error@-2{{implicit declaration of function 'get_sub_group_size' is invalid in OpenCL}}
+// expected-error@-3{{implicit conversion changes signedness: 'int' to 'uint' (aka 'unsigned int')}}
+#endif
+}
+
+kernel void basic_vector_data() {
+#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
+  generic void *generic_p;
+#endif
+  constant void *constant_p;
+  local void *local_p;
+  global void *global_p;
+  private void *private_p;
+  size_t s;
+
+  vload4(s, (const __constant ulong *) constant_p);
+  vload16(s, (const __constant short *) constant_p);
+
+#if __OPENCL_C_VERSION__ >= CL_VERSION_2_0
+  vload3(s, (const __generic ushort *) generic_p);
+  vload16(s, (const __generic uchar *) generic_p);
+#endif
+
+  vload8(s, (const __global long *) global_p);
+  vload2(s, (const __local uint *) local_p);
+  vload16(s, (const __private float *) private_p);
+}
+
+kernel void basic_work_item() {
+  uint ui;
+
+  get_enqueued_local_size(ui);
+#if __OPENCL_C_VERSION__ < CL_VERSION_2_0
+// expected-error@-2{{implicit declaration of function 'get_enqueued_local_size' is invalid in OpenCL}}
+#endif
 }
diff --git a/clang/test/SemaTemplate/default-arguments-cxx0x.cpp b/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
index c24ed12a0248c..2114cc94e6c60 100644
--- a/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
+++ b/clang/test/SemaTemplate/default-arguments-cxx0x.cpp
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -fsyntax-only -std=c++11 -verify %s
+// RUN: %clang_cc1 -fsyntax-only -std=c++14 -verify %s
 // expected-no-diagnostics
 
 // Test default template arguments for function templates.
@@ -114,3 +115,17 @@ namespace rdar34167492 {
     S<int> _a{};
   };
 }
+
+#if __cplusplus >= 201402L
+namespace lambda {
+  // Verify that a default argument in a lambda can refer to the type of a
+  // previous `auto` argument without crashing.
+  template <class T>
+  void bar() {
+    (void) [](auto c, int x = sizeof(decltype(c))) {};
+  }
+  void foo() {
+    bar<int>();
+  }
+} // namespace lambda
+#endif
diff --git a/clang/test/SemaTemplate/using-decl.cpp b/clang/test/SemaTemplate/using-decl.cpp
new file mode 100644
index 0000000000000..1ef2a2dfaa019
--- /dev/null
+++ b/clang/test/SemaTemplate/using-decl.cpp
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -std=c++2a -verify %s
+// expected-no-diagnostics
+
+namespace UsingInGenericLambda {
+  namespace a {
+    enum { b };
+  }
+  template<typename> void c() {
+    auto d = [](auto) {
+      using a::b;
+      (void)b;
+    };
+    d(0);
+  }
+  void e() { c<int>(); }
+}
diff --git a/clang/test/clang-rename/Typedef.cpp b/clang/test/clang-rename/Typedef.cpp
new file mode 100644
index 0000000000000..64d337fae22c7
--- /dev/null
+++ b/clang/test/clang-rename/Typedef.cpp
@@ -0,0 +1,8 @@
+namespace std {
+class basic_string {};
+typedef basic_string string;
+} // namespace std
+
+std::string foo(); //  // CHECK: std::new_string foo();
+
+// RUN: clang-rename -offset=93 -new-name=new_string %s --  | sed 's,//.*,,' | FileCheck %s
diff --git a/clang/tools/clang-format/clang-format-diff.py b/clang/tools/clang-format/clang-format-diff.py
index a4f22ca029366..122db49ff73c7 100755
--- a/clang/tools/clang-format/clang-format-diff.py
+++ b/clang/tools/clang-format/clang-format-diff.py
@@ -43,8 +43,8 @@ def main():
                       help='custom pattern selecting file paths to reformat '
                       '(case sensitive, overrides -iregex)')
   parser.add_argument('-iregex', metavar='PATTERN', default=
-                      r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc|js|ts|proto'
-                      r'|protodevel|java)',
+                      r'.*\.(cpp|cc|c\+\+|cxx|c|cl|h|hh|hpp|m|mm|inc|js|ts|proto'
+                      r'|protodevel|java|cs)',
                       help='custom pattern selecting file paths to reformat '
                       '(case insensitive, overridden by -regex)')
   parser.add_argument('-sort-includes', action='store_true', default=False,
diff --git a/clang/tools/clang-format/git-clang-format b/clang/tools/clang-format/git-clang-format
index ef7f22d0f873e..bf05d9143fa18 100755
--- a/clang/tools/clang-format/git-clang-format
+++ b/clang/tools/clang-format/git-clang-format
@@ -77,13 +77,14 @@ def main():
       'c', 'h',  # C
       'm',  # ObjC
       'mm',  # ObjC++
-      'cc', 'cp', 'cpp', 'c++', 'cxx', 'hpp', 'hxx',  # C++
+      'cc', 'cp', 'cpp', 'c++', 'cxx', 'hh', 'hpp', 'hxx',  # C++
       'cu',  # CUDA
       # Other languages that clang-format supports
       'proto', 'protodevel',  # Protocol Buffers
       'java',  # Java
       'js',  # JavaScript
       'ts',  # TypeScript
+      'cs',  # C Sharp
       ])
 
   p = argparse.ArgumentParser(
diff --git a/clang/tools/clang-import-test/clang-import-test.cpp b/clang/tools/clang-import-test/clang-import-test.cpp
index e8124d624267c..b42c93e8d3819 100644
--- a/clang/tools/clang-import-test/clang-import-test.cpp
+++ b/clang/tools/clang-import-test/clang-import-test.cpp
@@ -263,8 +263,8 @@ void AddExternalSource(CIAndOrigins &CI,
       {CI.getASTContext(), CI.getFileManager()});
   llvm::SmallVector<ExternalASTMerger::ImporterSource, 3> Sources;
   for (CIAndOrigins &Import : Imports)
-    Sources.push_back({Import.getASTContext(), Import.getFileManager(),
-                       Import.getOriginMap()});
+    Sources.emplace_back(Import.getASTContext(), Import.getFileManager(),
+                         Import.getOriginMap());
   auto ES = std::make_unique<ExternalASTMerger>(Target, Sources);
   CI.getASTContext().setExternalSource(ES.release());
   CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage();
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 4bd61fb9c893d..d44e3b9ff3558 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -177,6 +177,11 @@ llvm::cl::opt<bool> SkipExcludedPPRanges(
         "until reaching the end directive."),
     llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
 
+llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
+                            llvm::cl::desc("Use verbose output."),
+                            llvm::cl::init(false),
+                            llvm::cl::cat(DependencyScannerCategory));
+
 } // end anonymous namespace
 
 /// \returns object-file path derived from source-file path.
@@ -242,6 +247,7 @@ int main(int argc, const char **argv) {
         AdjustedArgs.push_back("-o");
         AdjustedArgs.push_back("/dev/null");
         if (!HasMT && !HasMQ) {
+          AdjustedArgs.push_back("-M");
           AdjustedArgs.push_back("-MT");
           // We're interested in source dependencies of an object file.
           if (!HasMD) {
@@ -260,6 +266,8 @@ int main(int argc, const char **argv) {
         AdjustedArgs.push_back("-Wno-error");
         return AdjustedArgs;
       });
+  AdjustingCompilations->appendArgumentsAdjuster(
+      tooling::getClangStripSerializeDiagnosticAdjuster());
 
   SharedStream Errs(llvm::errs());
   // Print out the dependency results to STDOUT by default.
@@ -283,8 +291,10 @@ int main(int argc, const char **argv) {
   std::mutex Lock;
   size_t Index = 0;
 
-  llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
-               << " files using " << NumWorkers << " workers\n";
+  if (Verbose) {
+    llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
+                 << " files using " << NumWorkers << " workers\n";
+  }
   for (unsigned I = 0; I < NumWorkers; ++I) {
     auto Worker = [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools]() {
       while (true) {
diff --git a/clang/tools/driver/cc1_main.cpp b/clang/tools/driver/cc1_main.cpp
index 29767b880b62d..8ab713fd2954a 100644
--- a/clang/tools/driver/cc1_main.cpp
+++ b/clang/tools/driver/cc1_main.cpp
@@ -253,6 +253,7 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
   // If any timers were active but haven't been destroyed yet, print their
   // results now.  This happens in -disable-free mode.
   llvm::TimerGroup::printAll(llvm::errs());
+  llvm::TimerGroup::clearAll();
 
   if (llvm::timeTraceProfilerEnabled()) {
     SmallString<128> Path(Clang->getFrontendOpts().OutputFile);
@@ -270,9 +271,6 @@ int cc1_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
     llvm::timeTraceProfilerCleanup();
 
     llvm::errs() << "Time trace json-file dumped to " << Path.str() << "\n";
-    llvm::errs()
-        << "Use chrome://tracing or Speedscope App "
-           "(https://www.speedscope.app) for flamegraph visualization\n";
   }
 
   // Our error handler depends on the Diagnostics object, which we're
diff --git a/clang/tools/driver/cc1as_main.cpp b/clang/tools/driver/cc1as_main.cpp
index fbeea0dd3b02a..ae58a95f36f5b 100644
--- a/clang/tools/driver/cc1as_main.cpp
+++ b/clang/tools/driver/cc1as_main.cpp
@@ -609,6 +609,7 @@ int cc1as_main(ArrayRef<const char *> Argv, const char *Argv0, void *MainAddr) {
   // If any timers were active but haven't been destroyed yet, print their
   // results now.
   TimerGroup::printAll(errs());
+  TimerGroup::clearAll();
 
   return !!Failed;
 }
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index eebc8920fe0ab..f1600490017d7 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -499,6 +499,7 @@ int main(int argc_, const char **argv_) {
   // If any timers were active but haven't been destroyed yet, print their
   // results now.  This happens in -disable-free mode.
   llvm::TimerGroup::printAll(llvm::errs());
+  llvm::TimerGroup::clearAll();
 
 #ifdef _WIN32
   // Exit status should not be negative on Win32, unless abnormal termination.
diff --git a/clang/tools/libclang/FatalErrorHandler.cpp b/clang/tools/libclang/FatalErrorHandler.cpp
index e9a0d41bab3f5..eab17f3dfac73 100644
--- a/clang/tools/libclang/FatalErrorHandler.cpp
+++ b/clang/tools/libclang/FatalErrorHandler.cpp
@@ -18,11 +18,13 @@ static void aborting_fatal_error_handler(void *, const std::string &reason,
   ::abort();
 }
 
-void clang_install_aborting_llvm_fatal_error_handler() {
+extern "C" {
+void clang_install_aborting_llvm_fatal_error_handler(void) {
   llvm::remove_fatal_error_handler();
   llvm::install_fatal_error_handler(aborting_fatal_error_handler, nullptr);
 }
 
-void clang_uninstall_llvm_fatal_error_handler() {
+void clang_uninstall_llvm_fatal_error_handler(void) {
   llvm::remove_fatal_error_handler();
 }
+}
\ No newline at end of file
diff --git a/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp b/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp
new file mode 100644
index 0000000000000..6a2aa2bfc328b
--- /dev/null
+++ b/clang/unittests/AST/ASTImporterODRStrategiesTest.cpp
@@ -0,0 +1,676 @@
+//===- unittest/AST/ASTImporterODRStrategiesTest.cpp -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Type-parameterized tests to verify the import behaviour in case of ODR
+// violation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ASTImporterFixtures.h"
+
+namespace clang {
+namespace ast_matchers {
+
+using internal::BindableMatcher;
+
+// DeclTy: Type of the Decl to check.
+// Prototype: "Prototype" (forward declaration) of the Decl.
+// Definition: A definition for the Prototype.
+// ConflictingPrototype: A prototype with the same name but different
+// declaration.
+// ConflictingDefinition: A different definition for Prototype.
+// ConflictingProtoDef: A definition for ConflictingPrototype.
+// getPattern: Return a matcher that matches any of Prototype, Definition,
+// ConflictingPrototype, ConflictingDefinition, ConflictingProtoDef.
+
+struct Function {
+  using DeclTy = FunctionDecl;
+  static constexpr auto *Prototype = "void X(int);";
+  static constexpr auto *ConflictingPrototype = "void X(double);";
+  static constexpr auto *Definition = "void X(int a) {}";
+  static constexpr auto *ConflictingDefinition = "void X(double a) {}";
+  BindableMatcher<Decl> getPattern() {
+    return functionDecl(hasName("X"), unless(isImplicit()));
+  }
+  Language getLang() { return Lang_C; }
+};
+
+struct Typedef {
+  using DeclTy = TypedefNameDecl;
+  static constexpr auto *Definition = "typedef int X;";
+  static constexpr auto *ConflictingDefinition = "typedef double X;";
+  BindableMatcher<Decl> getPattern() { return typedefNameDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct TypedefAlias {
+  using DeclTy = TypedefNameDecl;
+  static constexpr auto *Definition = "using X = int;";
+  static constexpr auto *ConflictingDefinition = "using X = double;";
+  BindableMatcher<Decl> getPattern() { return typedefNameDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX11; }
+};
+
+struct Enum {
+  using DeclTy = EnumDecl;
+  static constexpr auto *Definition = "enum X { a, b };";
+  static constexpr auto *ConflictingDefinition = "enum X { a, b, c };";
+  BindableMatcher<Decl> getPattern() { return enumDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct EnumConstant {
+  using DeclTy = EnumConstantDecl;
+  static constexpr auto *Definition = "enum E { X = 0 };";
+  static constexpr auto *ConflictingDefinition = "enum E { X = 1 };";
+  BindableMatcher<Decl> getPattern() { return enumConstantDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct Class {
+  using DeclTy = CXXRecordDecl;
+  static constexpr auto *Prototype = "class X;";
+  static constexpr auto *Definition = "class X {};";
+  static constexpr auto *ConflictingDefinition = "class X { int A; };";
+  BindableMatcher<Decl> getPattern() {
+    return cxxRecordDecl(hasName("X"), unless(isImplicit()));
+  }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct Variable {
+  using DeclTy = VarDecl;
+  static constexpr auto *Prototype = "extern int X;";
+  static constexpr auto *ConflictingPrototype = "extern float X;";
+  static constexpr auto *Definition = "int X;";
+  static constexpr auto *ConflictingDefinition = "float X;";
+  BindableMatcher<Decl> getPattern() { return varDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct ClassTemplate {
+  using DeclTy = ClassTemplateDecl;
+  static constexpr auto *Prototype = "template <class> class X;";
+  static constexpr auto *ConflictingPrototype = "template <int> class X;";
+  static constexpr auto *Definition = "template <class> class X {};";
+  static constexpr auto *ConflictingDefinition =
+      "template <class> class X { int A; };";
+  static constexpr auto *ConflictingProtoDef = "template <int> class X { };";
+  BindableMatcher<Decl> getPattern() {
+    return classTemplateDecl(hasName("X"), unless(isImplicit()));
+  }
+  Language getLang() { return Lang_CXX; }
+};
+
+struct FunctionTemplate {
+  using DeclTy = FunctionTemplateDecl;
+  static constexpr auto *Definition0 =
+      R"(
+      template <class T>
+      void X(T a) {};
+      )";
+  // This is actually not a conflicting definition, but another primary template.
+  static constexpr auto *Definition1 =
+      R"(
+      template <class T>
+      void X(T* a) {};
+      )";
+  BindableMatcher<Decl> getPattern() {
+    return functionTemplateDecl(hasName("X"), unless(isImplicit()));
+  }
+  static std::string getDef0() { return Definition0; }
+  static std::string getDef1() { return Definition1; }
+  Language getLang() { return Lang_CXX; }
+};
+
+static const internal::VariadicDynCastAllOfMatcher<Decl, VarTemplateDecl>
+    varTemplateDecl;
+
+struct VarTemplate {
+  using DeclTy = VarTemplateDecl;
+  static constexpr auto *Definition =
+      R"(
+      template <class T>
+      constexpr T X = 0;
+      )";
+  static constexpr auto *ConflictingDefinition =
+      R"(
+      template <int>
+      constexpr int X = 0;
+      )";
+  BindableMatcher<Decl> getPattern() { return varTemplateDecl(hasName("X")); }
+  Language getLang() { return Lang_CXX14; }
+};
+
+struct ClassTemplateSpec {
+  using DeclTy = ClassTemplateSpecializationDecl;
+  static constexpr auto *Prototype =
+      R"(
+      template <class T> class X;
+      template <> class X<int>;
+      )";
+  static constexpr auto *Definition =
+      R"(
+      template <class T> class X;
+      template <> class X<int> {};
+      )";
+  static constexpr auto *ConflictingDefinition =
+      R"(
+      template <class T> class X;
+      template <> class X<int> { int A; };
+      )";
+  BindableMatcher<Decl> getPattern() {
+    return classTemplateSpecializationDecl(hasName("X"), unless(isImplicit()));
+  }
+  Language getLang() { return Lang_CXX; }
+};
+
+// Function template specializations are all "full" specializations.
+// Structural equivalency does not check the body of functions, so we cannot
+// create conflicting function template specializations.
+struct FunctionTemplateSpec {
+  using DeclTy = FunctionDecl;
+
+  static constexpr auto *Definition0 =
+      R"(
+      template <class T>
+      void X(T a);
+      template <> void X(int a) {};
+      )";
+
+  // This is actually not a conflicting definition, but another full
+  // specialization.
+  // Thus, during the import we would create a new specialization with a
+  // different type argument.
+  static constexpr auto *Definition1 =
+      R"(
+      template <class T>
+      void X(T a);
+      template <> void X(double a) {};
+      )";
+
+  BindableMatcher<Decl> getPattern() {
+    return functionDecl(hasName("X"), isExplicitTemplateSpecialization(),
+                        unless(isImplicit()));
+  }
+  static std::string getDef0() { return Definition0; }
+  static std::string getDef1() { return Definition1; }
+  Language getLang() { return Lang_CXX; }
+};
+
+static const internal::VariadicDynCastAllOfMatcher<
+    Decl, VarTemplateSpecializationDecl>
+    varTemplateSpecializationDecl;
+
+struct VarTemplateSpec {
+  using DeclTy = VarTemplateSpecializationDecl;
+  static constexpr auto *Definition =
+      R"(
+      template <class T> T X = 0;
+      template <> int X<int> = 0;
+      )";
+  static constexpr auto *ConflictingDefinition =
+      R"(
+      template <class T> T X = 0;
+      template <> float X<int> = 1.0;
+      )";
+  BindableMatcher<Decl> getPattern() {
+    return varTemplateSpecializationDecl(hasName("X"), unless(isImplicit()));
+  }
+  Language getLang() { return Lang_CXX14; }
+};
+
+template <typename TypeParam, ASTImporter::ODRHandlingType ODRHandlingParam>
+struct ODRViolation : ASTImporterOptionSpecificTestBase {
+
+  using DeclTy = typename TypeParam::DeclTy;
+
+  ODRViolation() { ODRHandling = ODRHandlingParam; }
+
+  static std::string getPrototype() { return TypeParam::Prototype; }
+  static std::string getConflictingPrototype() {
+    return TypeParam::ConflictingPrototype;
+  }
+  static std::string getDefinition() { return TypeParam::Definition; }
+  static std::string getConflictingDefinition() {
+    return TypeParam::ConflictingDefinition;
+  }
+  static std::string getConflictingProtoDef() {
+    return TypeParam::ConflictingProtoDef;
+  }
+  static BindableMatcher<Decl> getPattern() { return TypeParam().getPattern(); }
+  static Language getLang() { return TypeParam().getLang(); }
+
+  template <std::string (*ToTUContent)(), std::string (*FromTUContent)(),
+            void (*ResultChecker)(llvm::Expected<Decl *> &, Decl *, Decl *)>
+  void TypedTest_ImportAfter() {
+    Decl *ToTU = getToTuDecl(ToTUContent(), getLang());
+    auto *ToD = FirstDeclMatcher<DeclTy>().match(ToTU, getPattern());
+
+    Decl *FromTU = getTuDecl(FromTUContent(), getLang());
+    auto *FromD = FirstDeclMatcher<DeclTy>().match(FromTU, getPattern());
+
+    auto Result = importOrError(FromD, getLang());
+
+    ResultChecker(Result, ToTU, ToD);
+  }
+
+  // Check that a Decl has been successfully imported into a standalone redecl
+  // chain.
+  static void CheckImportedAsNew(llvm::Expected<Decl *> &Result, Decl *ToTU,
+                                 Decl *ToD) {
+    ASSERT_TRUE(isSuccess(Result));
+    Decl *ImportedD = *Result;
+    ASSERT_TRUE(ImportedD);
+    EXPECT_NE(ImportedD, ToD);
+    EXPECT_EQ(DeclCounter<DeclTy>().match(ToTU, getPattern()), 2u);
+
+    // There may be a hidden fwd spec decl before a function spec decl.
+    if (auto *ImportedF = dyn_cast<FunctionDecl>(ImportedD))
+      if (ImportedF->getTemplatedKind() ==
+          FunctionDecl::TK_FunctionTemplateSpecialization)
+        return;
+
+    EXPECT_FALSE(ImportedD->getPreviousDecl());
+  }
+
+  // Check that a Decl was not imported because of NameConflict.
+  static void CheckImportNameConflict(llvm::Expected<Decl *> &Result,
+                                      Decl *ToTU, Decl *ToD) {
+    EXPECT_TRUE(isImportError(Result, ImportError::NameConflict));
+    EXPECT_EQ(DeclCounter<DeclTy>().match(ToTU, getPattern()), 1u);
+  }
+
+  // Check that a Decl was not imported because lookup found the same decl.
+  static void CheckImportFoundExisting(llvm::Expected<Decl *> &Result,
+                                      Decl *ToTU, Decl *ToD) {
+    ASSERT_TRUE(isSuccess(Result));
+    EXPECT_EQ(DeclCounter<DeclTy>().match(ToTU, getPattern()), 1u);
+  }
+
+  void TypedTest_ImportConflictingDefAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingDefinition,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingProtoAfterProto() {
+    TypedTest_ImportAfter<getPrototype, getConflictingPrototype,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingProtoAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingPrototype,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingDefAfterProto() {
+    TypedTest_ImportAfter<getConflictingPrototype, getDefinition,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingProtoDefAfterProto() {
+    TypedTest_ImportAfter<getPrototype, getConflictingProtoDef,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingProtoAfterProtoDef() {
+    TypedTest_ImportAfter<getConflictingProtoDef, getPrototype,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingProtoDefAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingProtoDef,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_ImportConflictingDefAfterProtoDef() {
+    TypedTest_ImportAfter<getConflictingProtoDef, getDefinition,
+                          CheckImportedAsNew>();
+  }
+
+  void TypedTest_DontImportConflictingProtoAfterProto() {
+    TypedTest_ImportAfter<getPrototype, getConflictingPrototype,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingDefAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingDefinition,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingProtoAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingPrototype,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingDefAfterProto() {
+    TypedTest_ImportAfter<getConflictingPrototype, getDefinition,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingProtoDefAfterProto() {
+    TypedTest_ImportAfter<getPrototype, getConflictingProtoDef,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingProtoAfterProtoDef() {
+    TypedTest_ImportAfter<getConflictingProtoDef, getPrototype,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingProtoDefAfterDef() {
+    TypedTest_ImportAfter<getDefinition, getConflictingProtoDef,
+                          CheckImportNameConflict>();
+  }
+  void TypedTest_DontImportConflictingDefAfterProtoDef() {
+    TypedTest_ImportAfter<getConflictingProtoDef, getDefinition,
+                          CheckImportNameConflict>();
+  }
+
+  // Used for function templates and function template specializations.
+  void TypedTest_ImportDifferentDefAfterDef() {
+    TypedTest_ImportAfter<TypeParam::getDef0, TypeParam::getDef1,
+                          CheckImportedAsNew>();
+  }
+  void TypedTest_DontImportSameDefAfterDef() {
+    TypedTest_ImportAfter<TypeParam::getDef0, TypeParam::getDef0,
+                          CheckImportFoundExisting>();
+  }
+};
+
+// ==============================
+// Define the parametrized tests.
+// ==============================
+
+#define ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(                           \
+    TypeParam, ODRHandlingParam, NamePrefix, TestCase)                         \
+  using TypeParam##ODRHandlingParam =                                          \
+      ODRViolation<TypeParam, ASTImporter::ODRHandlingType::ODRHandlingParam>; \
+  TEST_P(TypeParam##ODRHandlingParam, NamePrefix##TestCase) {                  \
+    TypedTest_##TestCase();                                                    \
+  }
+
+// clang-format off
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Typedef, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    TypedefAlias, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Enum, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    EnumConstant, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Class, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    VarTemplate, Liberal, ,
+    ImportConflictingDefAfterDef)
+// Class and variable template specializations/instantiatons are always
+// imported conservatively, because the AST holds the specializations in a set,
+// and the key within the set is a hash calculated from the arguments of the
+// specialization.
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplateSpec, Liberal, ,
+    DontImportConflictingDefAfterDef) // Don't import !!!
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    VarTemplateSpec, Liberal, ,
+    DontImportConflictingDefAfterDef) // Don't import !!!
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Typedef, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    TypedefAlias, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Enum, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    EnumConstant, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Class, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    VarTemplate, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplateSpec, Conservative, ,
+    DontImportConflictingDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    VarTemplateSpec, Conservative, ,
+    DontImportConflictingDefAfterDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Liberal, ,
+    ImportConflictingProtoAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Liberal, ,
+    ImportConflictingProtoAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingProtoAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Conservative, ,
+    DontImportConflictingProtoAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Conservative, ,
+    DontImportConflictingProtoAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingProtoAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Liberal, ,
+    ImportConflictingProtoAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingProtoAfterDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Conservative, ,
+    DontImportConflictingProtoAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingProtoAfterDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Liberal, ,
+    ImportConflictingDefAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Liberal, ,
+    ImportConflictingDefAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingDefAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Function, Conservative, ,
+    DontImportConflictingDefAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    Variable, Conservative, ,
+    DontImportConflictingDefAfterProto)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingDefAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingProtoDefAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingProtoDefAfterProto)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingProtoAfterProtoDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingProtoAfterProtoDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingProtoDefAfterDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingProtoDefAfterDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Liberal, ,
+    ImportConflictingDefAfterProtoDef)
+
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    ClassTemplate, Conservative, ,
+    DontImportConflictingDefAfterProtoDef)
+
+// FunctionTemplate decls overload with each other. Thus, they are imported
+// always as a new node, independently from any ODRHandling strategy.
+//
+// Function template specializations are "full" specializations. Structural
+// equivalency does not check the body of functions, so we cannot create
+// conflicting function template specializations. Thus, ODR handling strategies
+// has nothing to do with function template specializations. Fully specialized
+// function templates are imported as new nodes if their template arguments are
+// different.
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplate, Liberal, ,
+    ImportDifferentDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplateSpec, Liberal, ,
+    ImportDifferentDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplate, Conservative, ,
+    ImportDifferentDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplateSpec, Conservative, ,
+    ImportDifferentDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplate, Liberal, ,
+    DontImportSameDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplateSpec, Liberal, ,
+    DontImportSameDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplate, Conservative, ,
+    DontImportSameDefAfterDef)
+ASTIMPORTER_ODR_INSTANTIATE_TYPED_TEST_CASE(
+    FunctionTemplateSpec, Conservative, ,
+    DontImportSameDefAfterDef)
+
+// ======================
+// Instantiate the tests.
+// ======================
+
+// FIXME: These fail on Windows.
+#if !defined(_WIN32)
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionConservative,
+    DefaultTestValuesForRunOptions, );
+#endif
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, TypedefConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, TypedefAliasConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, EnumConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, EnumConstantConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, VariableConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassTemplateConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionTemplateConservative,
+    DefaultTestValuesForRunOptions, );
+// FIXME: Make VarTemplate tests work.
+//INSTANTIATE_TEST_CASE_P(
+    //ODRViolationTests, VarTemplateConservative,
+    //DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionTemplateSpecConservative,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassTemplateSpecConservative,
+    DefaultTestValuesForRunOptions, );
+// FIXME: Make VarTemplateSpec tests work.
+//INSTANTIATE_TEST_CASE_P(
+    //ODRViolationTests, VarTemplateSpecConservative,
+    //DefaultTestValuesForRunOptions, );
+
+// FIXME: These fail on Windows.
+#if !defined(_WIN32)
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionLiberal,
+    DefaultTestValuesForRunOptions, );
+#endif
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, TypedefLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, TypedefAliasLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, EnumLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, EnumConstantLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, VariableLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassTemplateLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionTemplateLiberal,
+    DefaultTestValuesForRunOptions, );
+// FIXME: Make VarTemplate tests work.
+// INSTANTIATE_TEST_CASE_P(
+//     ODRViolationTests, VarTemplateLiberal,
+//     DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, ClassTemplateSpecLiberal,
+    DefaultTestValuesForRunOptions, );
+INSTANTIATE_TEST_CASE_P(
+    ODRViolationTests, FunctionTemplateSpecLiberal,
+    DefaultTestValuesForRunOptions, );
+// FIXME: Make VarTemplateSpec tests work.
+//INSTANTIATE_TEST_CASE_P(
+    //ODRViolationTests, VarTemplateSpecLiberal,
+    //DefaultTestValuesForRunOptions, );
+
+// clang-format on
+
+} // end namespace ast_matchers
+} // end namespace clang
diff --git a/clang/unittests/AST/ASTImporterTest.cpp b/clang/unittests/AST/ASTImporterTest.cpp
index 17696a38e94f6..1514313231bf3 100644
--- a/clang/unittests/AST/ASTImporterTest.cpp
+++ b/clang/unittests/AST/ASTImporterTest.cpp
@@ -5481,187 +5481,6 @@ TEST_P(ASTImporterOptionSpecificTestBase,
   EXPECT_EQ(ImportedX->isAggregate(), FromX->isAggregate());
 }
 
-struct ConflictingDeclsWithLiberalStrategy : ASTImporterOptionSpecificTestBase {
-  ConflictingDeclsWithLiberalStrategy() {
-    this->ODRHandling = ASTImporter::ODRHandlingType::Liberal;
-  }
-};
-
-// Check that a Decl has been successfully imported into a standalone redecl
-// chain.
-template <typename DeclTy, typename PatternTy>
-static void CheckImportedAsNew(llvm::Expected<Decl *> &Result, Decl *ToTU,
-                               PatternTy Pattern) {
-  ASSERT_TRUE(isSuccess(Result));
-  Decl *ImportedD = *Result;
-  ASSERT_TRUE(ImportedD);
-  auto *ToD = FirstDeclMatcher<DeclTy>().match(ToTU, Pattern);
-  EXPECT_NE(ImportedD, ToD);
-  EXPECT_FALSE(ImportedD->getPreviousDecl());
-  EXPECT_EQ(DeclCounter<DeclTy>().match(ToTU, Pattern), 2u);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, Typedef) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      typedef int X;
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      typedef double X;
-      )",
-      Lang_CXX11);
-  auto Pattern = typedefNameDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<TypedefNameDecl>().match(FromTU, Pattern);
-
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<TypedefNameDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, TypeAlias) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      using X = int;
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      using X = double;
-      )",
-      Lang_CXX11);
-  auto Pattern = typedefNameDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<TypedefNameDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<TypedefNameDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, EnumDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      enum X { a, b };
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      enum X { a, b, c };
-      )",
-      Lang_CXX11);
-  auto Pattern = enumDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<EnumDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<EnumDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, EnumConstantDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      enum E { X = 0 };
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      enum E { X = 1 };
-      )",
-      Lang_CXX11);
-  auto Pattern = enumConstantDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<EnumConstantDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<EnumConstantDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, RecordDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      class X { int a; };
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      class X { int b; };
-      )",
-      Lang_CXX11);
-  auto Pattern = cxxRecordDecl(hasName("X"), unless(isImplicit()));
-  auto *FromX = FirstDeclMatcher<RecordDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<RecordDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, VarDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      int X;
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      double X;
-      )",
-      Lang_CXX11);
-  auto Pattern = varDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<VarDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<VarDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, FunctionDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      void X(int);
-      )",
-      Lang_C); // C, no overloading!
-  Decl *FromTU = getTuDecl(
-      R"(
-      void X(double);
-      )",
-      Lang_C);
-  auto Pattern = functionDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<FunctionDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<FunctionDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, ClassTemplateDecl) {
-  Decl *ToTU = getToTuDecl(
-      R"(
-      template <class>
-      struct X;
-      )",
-      Lang_CXX11);
-  Decl *FromTU = getTuDecl(
-      R"(
-      template <int>
-      struct X;
-      )",
-      Lang_CXX11);
-  auto Pattern = classTemplateDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<ClassTemplateDecl>().match(FromTU, Pattern);
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<ClassTemplateDecl>(Result, ToTU, Pattern);
-}
-
-TEST_P(ConflictingDeclsWithLiberalStrategy, DISABLED_VarTemplateDecl) {
-  const internal::VariadicDynCastAllOfMatcher<Decl, VarTemplateDecl>
-      varTemplateDecl;
-  Decl *ToTU = getToTuDecl(
-      R"(
-      template <class T>
-      constexpr T X;
-      )",
-      Lang_CXX14);
-  Decl *FromTU = getTuDecl(
-      R"(
-      template <int>
-      constexpr int X = 0;
-      )",
-      Lang_CXX14);
-  auto Pattern = varTemplateDecl(hasName("X"));
-  auto *FromX = FirstDeclMatcher<VarTemplateDecl>().match(
-      FromTU, varTemplateDecl(hasName("X")));
-  Expected<Decl *> Result = importOrError(FromX, Lang_CXX11);
-  CheckImportedAsNew<VarTemplateDecl>(Result, ToTU, Pattern);
-}
-
 INSTANTIATE_TEST_CASE_P(ParameterizedTests, SVEBuiltins,
                         ::testing::Values(ArgVector{"-target",
                                                     "aarch64-linux-gnu"}), );
@@ -5838,8 +5657,5 @@ INSTANTIATE_TEST_CASE_P(ParameterizedTests, ImportVariables,
 INSTANTIATE_TEST_CASE_P(ParameterizedTests, LLDBLookupTest,
                         DefaultTestValuesForRunOptions, );
 
-INSTANTIATE_TEST_CASE_P(ParameterizedTests, ConflictingDeclsWithLiberalStrategy,
-                        DefaultTestValuesForRunOptions, );
-
 } // end namespace ast_matchers
 } // end namespace clang
diff --git a/clang/unittests/AST/CMakeLists.txt b/clang/unittests/AST/CMakeLists.txt
index c4940e8acea45..4d463d1cc3095 100644
--- a/clang/unittests/AST/CMakeLists.txt
+++ b/clang/unittests/AST/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_unittest(ASTTests
   ASTImporterFixtures.cpp
   ASTImporterTest.cpp
   ASTImporterGenericRedeclTest.cpp
+  ASTImporterODRStrategiesTest.cpp
   ASTImporterVisibilityTest.cpp
   ASTTraverserTest.cpp
   ASTTypeTraitsTest.cpp
diff --git a/clang/unittests/AST/DeclTest.cpp b/clang/unittests/AST/DeclTest.cpp
index 6691952b2f2b9..7a662fb4fbae4 100644
--- a/clang/unittests/AST/DeclTest.cpp
+++ b/clang/unittests/AST/DeclTest.cpp
@@ -10,12 +10,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Mangle.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Tooling/Tooling.h"
 #include "gtest/gtest.h"
 
 using namespace clang::ast_matchers;
 using namespace clang::tooling;
+using namespace clang;
 
 TEST(Decl, CleansUpAPValues) {
   MatchFinder Finder;
@@ -56,3 +60,49 @@ TEST(Decl, CleansUpAPValues) {
       "constexpr _Complex __uint128_t c = 0xffffffffffffffff;",
       Args));
 }
+
+TEST(Decl, AsmLabelAttr) {
+  // Create two method decls: `f` and `g`.
+  StringRef Code = R"(
+    struct S {
+      void f() {}
+      void g() {}
+    };
+  )";
+  auto AST =
+      tooling::buildASTFromCodeWithArgs(Code, {"-target", "i386-apple-darwin"});
+  ASTContext &Ctx = AST->getASTContext();
+  assert(Ctx.getTargetInfo().getDataLayout().getGlobalPrefix() &&
+         "Expected target to have a global prefix");
+  DiagnosticsEngine &Diags = AST->getDiagnostics();
+  SourceManager &SM = AST->getSourceManager();
+  FileID MainFileID = SM.getMainFileID();
+
+  // Find the method decls within the AST.
+  SmallVector<Decl *, 1> Decls;
+  AST->findFileRegionDecls(MainFileID, Code.find('{'), 0, Decls);
+  ASSERT_TRUE(Decls.size() == 1);
+  CXXRecordDecl *DeclS = cast<CXXRecordDecl>(Decls[0]);
+  NamedDecl *DeclF = *DeclS->method_begin();
+  NamedDecl *DeclG = *(++DeclS->method_begin());
+
+  // Attach asm labels to the decls: one literal, and one not.
+  DeclF->addAttr(::new (Ctx) AsmLabelAttr(Ctx, SourceLocation(), "foo",
+                                          /*LiteralLabel=*/true));
+  DeclG->addAttr(::new (Ctx) AsmLabelAttr(Ctx, SourceLocation(), "goo",
+                                          /*LiteralLabel=*/false));
+
+  // Mangle the decl names.
+  std::string MangleF, MangleG;
+  std::unique_ptr<ItaniumMangleContext> MC(
+      ItaniumMangleContext::create(Ctx, Diags));
+  {
+    llvm::raw_string_ostream OS_F(MangleF);
+    llvm::raw_string_ostream OS_G(MangleG);
+    MC->mangleName(DeclF, OS_F);
+    MC->mangleName(DeclG, OS_G);
+  }
+
+  ASSERT_TRUE(0 == MangleF.compare("\x01" "foo"));
+  ASSERT_TRUE(0 == MangleG.compare("goo"));
+}
diff --git a/clang/unittests/AST/NamedDeclPrinterTest.cpp b/clang/unittests/AST/NamedDeclPrinterTest.cpp
index a50626517f619..a5c3e19055f43 100644
--- a/clang/unittests/AST/NamedDeclPrinterTest.cpp
+++ b/clang/unittests/AST/NamedDeclPrinterTest.cpp
@@ -16,9 +16,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/PrettyPrinter.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 
 using namespace clang;
@@ -30,11 +33,12 @@ namespace {
 class PrintMatch : public MatchFinder::MatchCallback {
   SmallString<1024> Printed;
   unsigned NumFoundDecls;
-  bool SuppressUnwrittenScope;
+  std::function<void(llvm::raw_ostream &OS, const NamedDecl *)> Printer;
 
 public:
-  explicit PrintMatch(bool suppressUnwrittenScope)
-    : NumFoundDecls(0), SuppressUnwrittenScope(suppressUnwrittenScope) {}
+  explicit PrintMatch(
+      std::function<void(llvm::raw_ostream &OS, const NamedDecl *)> Printer)
+      : NumFoundDecls(0), Printer(std::move(Printer)) {}
 
   void run(const MatchFinder::MatchResult &Result) override {
     const NamedDecl *ND = Result.Nodes.getNodeAs<NamedDecl>("id");
@@ -45,9 +49,7 @@ class PrintMatch : public MatchFinder::MatchCallback {
       return;
 
     llvm::raw_svector_ostream Out(Printed);
-    PrintingPolicy Policy = Result.Context->getPrintingPolicy();
-    Policy.SuppressUnwrittenScope = SuppressUnwrittenScope;
-    ND->printQualifiedName(Out, Policy);
+    Printer(Out, ND);
   }
 
   StringRef getPrinted() const {
@@ -59,12 +61,12 @@ class PrintMatch : public MatchFinder::MatchCallback {
   }
 };
 
-::testing::AssertionResult
-PrintedNamedDeclMatches(StringRef Code, const std::vector<std::string> &Args,
-                        bool SuppressUnwrittenScope,
-                        const DeclarationMatcher &NodeMatch,
-                        StringRef ExpectedPrinted, StringRef FileName) {
-  PrintMatch Printer(SuppressUnwrittenScope);
+::testing::AssertionResult PrintedDeclMatches(
+    StringRef Code, const std::vector<std::string> &Args,
+    const DeclarationMatcher &NodeMatch, StringRef ExpectedPrinted,
+    StringRef FileName,
+    std::function<void(llvm::raw_ostream &, const NamedDecl *)> Print) {
+  PrintMatch Printer(std::move(Print));
   MatchFinder Finder;
   Finder.addMatcher(NodeMatch, &Printer);
   std::unique_ptr<FrontendActionFactory> Factory =
@@ -91,6 +93,21 @@ PrintedNamedDeclMatches(StringRef Code, const std::vector<std::string> &Args,
   return ::testing::AssertionSuccess();
 }
 
+::testing::AssertionResult
+PrintedNamedDeclMatches(StringRef Code, const std::vector<std::string> &Args,
+                        bool SuppressUnwrittenScope,
+                        const DeclarationMatcher &NodeMatch,
+                        StringRef ExpectedPrinted, StringRef FileName) {
+  return PrintedDeclMatches(Code, Args, NodeMatch, ExpectedPrinted, FileName,
+                            [=](llvm::raw_ostream &Out, const NamedDecl *ND) {
+                              auto Policy =
+                                  ND->getASTContext().getPrintingPolicy();
+                              Policy.SuppressUnwrittenScope =
+                                  SuppressUnwrittenScope;
+                              ND->printQualifiedName(Out, Policy);
+                            });
+}
+
 ::testing::AssertionResult
 PrintedNamedDeclCXX98Matches(StringRef Code, StringRef DeclName,
                              StringRef ExpectedPrinted) {
@@ -127,6 +144,17 @@ PrintedWrittenPropertyDeclObjCMatches(StringRef Code, StringRef DeclName,
                                  "input.m");
 }
 
+::testing::AssertionResult
+PrintedNestedNameSpecifierMatches(StringRef Code, StringRef DeclName,
+                                  StringRef ExpectedPrinted) {
+  std::vector<std::string> Args{"-std=c++11"};
+  return PrintedDeclMatches(Code, Args, namedDecl(hasName(DeclName)).bind("id"),
+                            ExpectedPrinted, "input.cc",
+                            [](llvm::raw_ostream &Out, const NamedDecl *D) {
+                              D->printNestedNameSpecifier(Out);
+                            });
+}
+
 } // unnamed namespace
 
 TEST(NamedDeclPrinter, TestNamespace1) {
@@ -223,3 +251,21 @@ R"(
     "property",
     "Obj::property"));
 }
+
+TEST(NamedDeclPrinter, NestedNameSpecifierSimple) {
+  const char *Code =
+      R"(
+  namespace foo { namespace bar { void func(); }  }
+)";
+  ASSERT_TRUE(PrintedNestedNameSpecifierMatches(Code, "func", "foo::bar::"));
+}
+
+TEST(NamedDeclPrinter, NestedNameSpecifierTemplateArgs) {
+  const char *Code =
+      R"(
+        template <class T> struct vector;
+        template <> struct vector<int> { int method(); };
+)";
+  ASSERT_TRUE(
+      PrintedNestedNameSpecifierMatches(Code, "method", "vector<int>::"));
+}
diff --git a/clang/unittests/Driver/ToolChainTest.cpp b/clang/unittests/Driver/ToolChainTest.cpp
index 80938c83f8247..f84e508b6cbdb 100644
--- a/clang/unittests/Driver/ToolChainTest.cpp
+++ b/clang/unittests/Driver/ToolChainTest.cpp
@@ -60,7 +60,7 @@ TEST(ToolChainTest, VFSGCCInstallation) {
                                 llvm::MemoryBuffer::getMemBuffer("\n"));
 
   std::unique_ptr<Compilation> C(TheDriver.BuildCompilation(
-      {"-fsyntax-only", "--gcc-toolchain=", "foo.cpp"}));
+      {"-fsyntax-only", "--gcc-toolchain=", "--sysroot=", "foo.cpp"}));
   EXPECT_TRUE(C);
 
   std::string S;
diff --git a/clang/unittests/Format/CleanupTest.cpp b/clang/unittests/Format/CleanupTest.cpp
index 0628c38a2bd6b..b0c81b509d2ae 100644
--- a/clang/unittests/Format/CleanupTest.cpp
+++ b/clang/unittests/Format/CleanupTest.cpp
@@ -183,10 +183,15 @@ TEST_F(CleanupTest, TrailingCommaInParens) {
   std::string Code = "int main() { f(,1,,2,3,f(1,2,),4,,);}";
   std::string Expected = "int main() { f(1,2,3,f(1,2),4);}";
   EXPECT_EQ(Expected, cleanupAroundOffsets({15, 18, 29, 33}, Code));
+
+  // Lambda contents are also checked for trailing commas.
+  Code = "int main() { [](){f(,1,,2,3,f(1,2,),4,,);}();}";
+  Expected = "int main() { [](){f(1,2,3,f(1,2),4);}();}";
+  EXPECT_EQ(Expected, cleanupAroundOffsets({20, 23, 34, 38}, Code));
 }
 
 TEST_F(CleanupTest, TrailingCommaInBraces) {
-  // Trainling comma is allowed in brace list.
+  // Trailing comma is allowed in brace list.
   // If there was trailing comma in the original code, then trailing comma is
   // preserved. In this example, element between the last two commas is deleted
   // causing the second-last comma to be redundant.
@@ -194,7 +199,7 @@ TEST_F(CleanupTest, TrailingCommaInBraces) {
   std::string Expected = "void f() { std::vector<int> v = {1,2,3,}; }";
   EXPECT_EQ(Expected, cleanupAroundOffsets({39}, Code));
 
-  // If there was no trailing comma in the original code, then trainling comma
+  // If there was no trailing comma in the original code, then trailing comma
   // introduced by replacements should be cleaned up. In this example, the
   // element after the last comma is deleted causing the last comma to be
   // redundant.
diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index b1fcd39a49aed..d7759bc6df3b3 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -29,11 +29,7 @@ FormatStyle getGoogleStyle() { return getGoogleStyle(FormatStyle::LK_Cpp); }
 
 class FormatTest : public ::testing::Test {
 protected:
-  enum StatusCheck {
-    SC_ExpectComplete,
-    SC_ExpectIncomplete,
-    SC_DoNotCheck
-  };
+  enum StatusCheck { SC_ExpectComplete, SC_ExpectIncomplete, SC_DoNotCheck };
 
   std::string format(llvm::StringRef Code,
                      const FormatStyle &Style = getLLVMStyle(),
@@ -332,13 +328,15 @@ TEST_F(FormatTest, RemovesEmptyLines) {
             format("namespace {\n"
                    "int i;\n"
                    "\n"
-                   "}", LLVMWithNoNamespaceFix));
+                   "}",
+                   LLVMWithNoNamespaceFix));
   EXPECT_EQ("namespace {\n"
             "int i;\n"
             "}",
             format("namespace {\n"
                    "int i;\n"
-                   "}", LLVMWithNoNamespaceFix));
+                   "}",
+                   LLVMWithNoNamespaceFix));
   EXPECT_EQ("namespace {\n"
             "int i;\n"
             "\n"
@@ -346,13 +344,15 @@ TEST_F(FormatTest, RemovesEmptyLines) {
             format("namespace {\n"
                    "int i;\n"
                    "\n"
-                   "};", LLVMWithNoNamespaceFix));
+                   "};",
+                   LLVMWithNoNamespaceFix));
   EXPECT_EQ("namespace {\n"
             "int i;\n"
             "};",
             format("namespace {\n"
                    "int i;\n"
-                   "};", LLVMWithNoNamespaceFix));
+                   "};",
+                   LLVMWithNoNamespaceFix));
   EXPECT_EQ("namespace {\n"
             "int i;\n"
             "\n"
@@ -903,7 +903,8 @@ TEST_F(FormatTest, FormatsForLoop) {
   verifyFormat("for (Foo *x = 0; x != in; x++) {\n}");
   verifyFormat("Foo *x;\nfor (x = 0; x != in; x++) {\n}");
   verifyFormat("Foo *x;\nfor (x in y) {\n}");
-  verifyFormat("for (const Foo<Bar> &baz = in.value(); !baz.at_end(); ++baz) {\n}");
+  verifyFormat(
+      "for (const Foo<Bar> &baz = in.value(); !baz.at_end(); ++baz) {\n}");
 
   FormatStyle NoBinPacking = getLLVMStyle();
   NoBinPacking.BinPackParameters = false;
@@ -1295,20 +1296,26 @@ TEST_F(FormatTest, ShortCaseLabels) {
                    "#endif\n"
                    "}",
                    Style));
-  EXPECT_EQ("switch (a) {\n" "case 0:\n"
-            "  return; // long long long long long long long long long long long long comment\n"
-            "          // line\n" "}",
+  EXPECT_EQ("switch (a) {\n"
+            "case 0:\n"
+            "  return; // long long long long long long long long long long "
+            "long long comment\n"
+            "          // line\n"
+            "}",
             format("switch (a) {\n"
-                   "case 0: return; // long long long long long long long long long long long long comment line\n"
+                   "case 0: return; // long long long long long long long long "
+                   "long long long long comment line\n"
                    "}",
                    Style));
   EXPECT_EQ("switch (a) {\n"
             "case 0:\n"
-            "  return; /* long long long long long long long long long long long long comment\n"
+            "  return; /* long long long long long long long long long long "
+            "long long comment\n"
             "             line */\n"
             "}",
             format("switch (a) {\n"
-                   "case 0: return; /* long long long long long long long long long long long long comment line */\n"
+                   "case 0: return; /* long long long long long long long long "
+                   "long long long long comment line */\n"
                    "}",
                    Style));
   verifyFormat("switch (a) {\n"
@@ -1596,7 +1603,7 @@ TEST_F(FormatTest, FormatsClasses) {
 TEST_F(FormatTest, BreakInheritanceStyle) {
   FormatStyle StyleWithInheritanceBreakBeforeComma = getLLVMStyle();
   StyleWithInheritanceBreakBeforeComma.BreakInheritanceList =
-          FormatStyle::BILS_BeforeComma;
+      FormatStyle::BILS_BeforeComma;
   verifyFormat("class MyClass : public X {};",
                StyleWithInheritanceBreakBeforeComma);
   verifyFormat("class MyClass\n"
@@ -1614,7 +1621,7 @@ TEST_F(FormatTest, BreakInheritanceStyle) {
 
   FormatStyle StyleWithInheritanceBreakAfterColon = getLLVMStyle();
   StyleWithInheritanceBreakAfterColon.BreakInheritanceList =
-          FormatStyle::BILS_AfterColon;
+      FormatStyle::BILS_AfterColon;
   verifyFormat("class MyClass : public X {};",
                StyleWithInheritanceBreakAfterColon);
   verifyFormat("class MyClass : public X, public Y {};",
@@ -1908,8 +1915,7 @@ TEST_F(FormatTest, FormatsNamespaces) {
   // This code is more common than we thought; if we
   // layout this correctly the semicolon will go into
   // its own line, which is undesirable.
-  verifyFormat("namespace {};",
-               LLVMWithNoNamespaceFix);
+  verifyFormat("namespace {};", LLVMWithNoNamespaceFix);
   verifyFormat("namespace {\n"
                "class A {};\n"
                "};",
@@ -2086,8 +2092,8 @@ TEST_F(FormatTest, FormatsCompactNamespaces) {
   Style.NamespaceMacros.push_back("TESTSUITE");
 
   verifyFormat("namespace A { namespace B {\n"
-			   "}} // namespace A::B",
-			   Style);
+               "}} // namespace A::B",
+               Style);
 
   EXPECT_EQ("namespace out { namespace in {\n"
             "}} // namespace out::in",
@@ -2401,6 +2407,16 @@ TEST_F(FormatTest, FormatTryCatchBraceStyles) {
                "  // something\n"
                "}",
                Style);
+  Style.BreakBeforeBraces = FormatStyle::BS_Whitesmiths;
+  verifyFormat("try\n"
+               "  {\n"
+               "  // something white\n"
+               "  }\n"
+               "catch (...)\n"
+               "  {\n"
+               "  // something white\n"
+               "  }",
+               Style);
   Style.BreakBeforeBraces = FormatStyle::BS_GNU;
   verifyFormat("try\n"
                "  {\n"
@@ -2931,7 +2947,8 @@ TEST_F(FormatTest, MacroCallsWithoutTrailingSemicolon) {
             "{\n"
             "}",
             format("void f() try {\n"
-                   "}", Style));
+                   "}",
+                   Style));
   EXPECT_EQ("class SomeClass {\n"
             "public:\n"
             "  SomeClass() EXCLUSIVE_LOCK_FUNCTION(mu_);\n"
@@ -3730,16 +3747,19 @@ TEST_F(FormatTest, FormatBeginBlockEndMacros) {
   Style.MacroBlockEnd = "^[A-Z_]+_END$";
   verifyFormat("FOO_BEGIN\n"
                "  FOO_ENTRY\n"
-               "FOO_END", Style);
+               "FOO_END",
+               Style);
   verifyFormat("FOO_BEGIN\n"
                "  NESTED_FOO_BEGIN\n"
                "    NESTED_FOO_ENTRY\n"
                "  NESTED_FOO_END\n"
-               "FOO_END", Style);
+               "FOO_END",
+               Style);
   verifyFormat("FOO_BEGIN(Foo, Bar)\n"
                "  int x;\n"
                "  x = 1;\n"
-               "FOO_END(Baz)", Style);
+               "FOO_END(Baz)",
+               Style);
 }
 
 //===----------------------------------------------------------------------===//
@@ -4006,7 +4026,9 @@ TEST_F(FormatTest, ExpressionIndentationBreakingBeforeOperators) {
   EXPECT_EQ("return someVeryVeryLongConditionThatBarelyFitsOnALine\n"
             "\t&& (someOtherLongishConditionPart1\n"
             "\t\t|| someOtherEvenLongerNestedConditionPart2);",
-            format("return someVeryVeryLongConditionThatBarelyFitsOnALine && (someOtherLongishConditionPart1 || someOtherEvenLongerNestedConditionPart2);",
+            format("return someVeryVeryLongConditionThatBarelyFitsOnALine && "
+                   "(someOtherLongishConditionPart1 || "
+                   "someOtherEvenLongerNestedConditionPart2);",
                    Style));
 }
 
@@ -4450,28 +4472,28 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
       "SomeClass::Constructor() :\n"
       "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa),\n"
       "    aaaaaaaaaaaaaaa(aaaaaaaaaaaa) {}",
-	  Style);
+      Style);
   verifyFormat("Constructor(aaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
                "            aaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) :\n"
                "    aaaaaaaaaa(aaaaaa) {}",
-			   Style);
+               Style);
 
   verifyFormat("Constructor() :\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa),\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
                "                             aaaaaaaaaaaaaaaaaaaaaaaaaaa),\n"
                "    aaaaaaaaaaaaaaaaaaaaaaa() {}",
-			   Style);
+               Style);
 
   verifyFormat("Constructor() :\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
                "        aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}",
-			   Style);
+               Style);
 
   verifyFormat("Constructor(int Parameter = 0) :\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaa),\n"
                "    aaaaaaaaaaaa(aaaaaaaaaaaaaaaaa) {}",
-			   Style);
+               Style);
   verifyFormat("Constructor() :\n"
                "    aaaaaaaaaaaaaaaaaaaaaa(a), bbbbbbbbbbbbbbbbbbbbbbbb(b) {\n"
                "}",
@@ -4479,7 +4501,7 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
   verifyFormat("Constructor() :\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
                "        aaaaaaaaaaaaaaaaaaaaaaaaa(aaaa, aaaa)) {}",
-			   Style);
+               Style);
 
   // Here a line could be saved by splitting the second initializer onto two
   // lines, but that is not desirable.
@@ -4487,7 +4509,7 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
                "    aaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaa),\n"
                "    aaaaaaaaaaa(aaaaaaaaaaa),\n"
                "    aaaaaaaaaaaaaaaaaaaaat(aaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}",
-			   Style);
+               Style);
 
   FormatStyle OnePerLine = Style;
   OnePerLine.ConstructorInitializerAllOnOneLineOrOnePerLine = true;
@@ -4519,12 +4541,11 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
                "          aaaaaaaaaaaaaaaaaaaaaa) {}",
                OnePerLine);
   OnePerLine.BinPackParameters = false;
-  verifyFormat(
-      "Constructor() :\n"
-      "    aaaaaaaaaaaaaaaaaaaaaaaa(\n"
-      "        aaaaaaaaaaa().aaa(),\n"
-      "        aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}",
-      OnePerLine);
+  verifyFormat("Constructor() :\n"
+               "    aaaaaaaaaaaaaaaaaaaaaaaa(\n"
+               "        aaaaaaaaaaa().aaa(),\n"
+               "        aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) {}",
+               OnePerLine);
   OnePerLine.ColumnLimit = 60;
   verifyFormat("Constructor() :\n"
                "    aaaaaaaaaaaaaaaaaaaa(a),\n"
@@ -4537,7 +4558,7 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
             format("Constructor() :\n"
                    "    // Comment forcing unwanted break.\n"
                    "    aaaa(aaaa) {}",
-				   Style));
+                   Style));
 
   Style.ColumnLimit = 0;
   verifyFormat("SomeClass::Constructor() :\n"
@@ -4547,7 +4568,7 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
                "    a(a) {}",
                Style);
   verifyFormat("SomeClass::Constructor() :\n"
-			   "    a(a), b(b), c(c) {}",
+               "    a(a), b(b), c(c) {}",
                Style);
   verifyFormat("SomeClass::Constructor() :\n"
                "    a(a) {\n"
@@ -4558,40 +4579,43 @@ TEST_F(FormatTest, BreakConstructorInitializersAfterColon) {
 
   Style.AllowShortFunctionsOnASingleLine = FormatStyle::SFS_None;
   verifyFormat("SomeClass::Constructor() :\n"
-			   "    a(a), b(b), c(c) {\n"
-			   "}",
+               "    a(a), b(b), c(c) {\n"
+               "}",
                Style);
   verifyFormat("SomeClass::Constructor() :\n"
                "    a(a) {\n"
-			   "}",
+               "}",
                Style);
 
   Style.ColumnLimit = 80;
   Style.AllowShortFunctionsOnASingleLine = FormatStyle::SFS_All;
   Style.ConstructorInitializerIndentWidth = 2;
-  verifyFormat("SomeClass::Constructor() : a(a), b(b), c(c) {}",
-               Style);
+  verifyFormat("SomeClass::Constructor() : a(a), b(b), c(c) {}", Style);
   verifyFormat("SomeClass::Constructor() :\n"
                "  aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
                "  bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {}",
                Style);
 
-  // `ConstructorInitializerIndentWidth` actually applies to InheritanceList as well
+  // `ConstructorInitializerIndentWidth` actually applies to InheritanceList as
+  // well
   Style.BreakInheritanceList = FormatStyle::BILS_BeforeColon;
-  verifyFormat("class SomeClass\n"
-               "  : public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
-               "    public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
-               Style);
+  verifyFormat(
+      "class SomeClass\n"
+      "  : public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
+      "    public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
+      Style);
   Style.BreakInheritanceList = FormatStyle::BILS_BeforeComma;
-  verifyFormat("class SomeClass\n"
-               "  : public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
-               "  , public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
-               Style);
+  verifyFormat(
+      "class SomeClass\n"
+      "  : public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
+      "  , public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
+      Style);
   Style.BreakInheritanceList = FormatStyle::BILS_AfterColon;
-  verifyFormat("class SomeClass :\n"
-               "  public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
-               "  public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
-               Style);
+  verifyFormat(
+      "class SomeClass :\n"
+      "  public aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
+      "  public bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb {};",
+      Style);
 }
 
 #ifndef EXPENSIVE_CHECKS
@@ -4777,11 +4801,10 @@ TEST_F(FormatTest, BreaksFunctionDeclarations) {
       "typename aaaaaaaaaa<aaaaaa>::aaaaaaaaaaa\n"
       "aaaaaaaaaa<aaaaaa>::aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
       "    bool *aaaaaaaaaaaaaaaaaa, bool *aa) {}");
-  verifyGoogleFormat(
-      "template <typename T>\n"
-      "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
-      "aaaaaaaaaaaaaaaaaaaaaaa<T>::aaaaaaaaaaaaa(\n"
-      "    aaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa);");
+  verifyGoogleFormat("template <typename T>\n"
+                     "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
+                     "aaaaaaaaaaaaaaaaaaaaaaa<T>::aaaaaaaaaaaaa(\n"
+                     "    aaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaa);");
 
   FormatStyle Style = getLLVMStyle();
   Style.PointerAlignment = FormatStyle::PAS_Left;
@@ -4880,6 +4903,13 @@ TEST_F(FormatTest, BreaksFunctionDeclarationsWithTrailingTokens) {
                "}",
                Style);
 
+  Style.BreakBeforeBraces = FormatStyle::BS_Whitesmiths;
+  verifyFormat("void someLongFunction(\n"
+               "    int someLongParameter) const\n"
+               "  {\n"
+               "  }",
+               Style);
+
   // Unless these are unknown annotations.
   verifyFormat("void SomeFunction(aaaaaaaaaa aaaaaaaaaaaaaaa,\n"
                "                  aaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)\n"
@@ -5154,11 +5184,10 @@ TEST_F(FormatTest, FormatsBuilderPattern) {
 
   verifyFormat("return aaaaaaaaaaaaaaaaa->aaaaa().aaaaaaaaaaaaa().aaaaaa() <\n"
                "       aaaaaaaaaaaaaaa->aaaaa().aaaaaaaaaaaaa().aaaaaa();");
-  verifyFormat(
-      "aaaaaaa->aaaaaaa\n"
-      "    ->aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
-      "        aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)\n"
-      "    ->aaaaaaaa(aaaaaaaaaaaaaaa);");
+  verifyFormat("aaaaaaa->aaaaaaa\n"
+               "    ->aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
+               "        aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)\n"
+               "    ->aaaaaaaa(aaaaaaaaaaaaaaa);");
   verifyFormat(
       "aaaaaaa->aaaaaaa\n"
       "    ->aaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa)\n"
@@ -5515,10 +5544,9 @@ TEST_F(FormatTest, BreaksConditionalExpressions) {
   verifyFormat(
       "aaaa(aaaaaaaaaaaaaaaaaaaa, aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
       "                                   : aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);");
-  verifyFormat(
-      "aaaa(aaaaaaaaa, aaaaaaaaa,\n"
-      "     aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
-      "             : aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);");
+  verifyFormat("aaaa(aaaaaaaaa, aaaaaaaaa,\n"
+               "     aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
+               "             : aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);");
   verifyFormat(
       "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa ? aaaa(aaaaaa)\n"
       "                                                    : aaaaaaaaaaaaa);");
@@ -5660,11 +5688,10 @@ TEST_F(FormatTest, BreaksConditionalExpressionsAfterOperator) {
       "aaaa(aaaaaaaaaaaaaaaaaaaa, aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa :\n"
       "                                     aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);",
       Style);
-  verifyFormat(
-      "aaaa(aaaaaaaa, aaaaaaaaaa,\n"
-      "     aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa :\n"
-      "               aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);",
-      Style);
+  verifyFormat("aaaa(aaaaaaaa, aaaaaaaaaa,\n"
+               "     aaaaaaa ? aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa :\n"
+               "               aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);",
+               Style);
   verifyFormat(
       "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa ? aaaa(aaaaaa) :\n"
       "                                                      aaaaaaaaaaaaa);",
@@ -6467,7 +6494,8 @@ TEST_F(FormatTest, WrapsTemplateDeclarations) {
   verifyFormat("template <typename T> class C {};", NeverBreak);
   verifyFormat("template <typename T> void f();", NeverBreak);
   verifyFormat("template <typename T> void f() {}", NeverBreak);
-  verifyFormat("template <typename T>\nvoid foo(aaaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbb) {}",
+  verifyFormat("template <typename T>\nvoid foo(aaaaaaaaaaaaaaaaaaaaaaaaaa "
+               "bbbbbbbbbbbbbbbbbbbb) {}",
                NeverBreak);
   verifyFormat("void aaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
                "                         bbbbbbbbbbbbbbbbbbbbbbbbbbbb>(\n"
@@ -6483,9 +6511,11 @@ TEST_F(FormatTest, WrapsTemplateDeclarations) {
   verifyFormat("template <enum E> class A {\n"
                "public:\n"
                "  E *f();\n"
-               "};", NeverBreak);
+               "};",
+               NeverBreak);
   NeverBreak.PenaltyBreakTemplateDeclaration = 100;
-  verifyFormat("template <typename T> void\nfoo(aaaaaaaaaaaaaaaaaaaaaaaaaa bbbbbbbbbbbbbbbbbbbb) {}",
+  verifyFormat("template <typename T> void\nfoo(aaaaaaaaaaaaaaaaaaaaaaaaaa "
+               "bbbbbbbbbbbbbbbbbbbb) {}",
                NeverBreak);
 }
 
@@ -6560,22 +6590,24 @@ TEST_F(FormatTest, WrapsTemplateParameters) {
       Style);
   Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
   Style.BreakBeforeBinaryOperators = FormatStyle::BOS_None;
-  verifyFormat(
-      "template <typename... a> struct s {};\n"
-      "extern s<\n"
-      "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa,\n"
-      "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa>\n"
-      "    y;",
-      Style);
+  verifyFormat("template <typename... a> struct s {};\n"
+               "extern s<\n"
+               "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, "
+               "aaaaaaaaaaaaaaaaaaaaaa,\n"
+               "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, "
+               "aaaaaaaaaaaaaaaaaaaaaa>\n"
+               "    y;",
+               Style);
   Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
   Style.BreakBeforeBinaryOperators = FormatStyle::BOS_All;
-  verifyFormat(
-      "template <typename... a> struct t {};\n"
-      "extern t<\n"
-      "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa,\n"
-      "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa>\n"
-      "    y;",
-      Style);
+  verifyFormat("template <typename... a> struct t {};\n"
+               "extern t<\n"
+               "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, "
+               "aaaaaaaaaaaaaaaaaaaaaa,\n"
+               "    aaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaa, "
+               "aaaaaaaaaaaaaaaaaaaaaa>\n"
+               "    y;",
+               Style);
 }
 
 TEST_F(FormatTest, WrapsAtNestedNameSpecifiers) {
@@ -6602,12 +6634,11 @@ TEST_F(FormatTest, WrapsAtNestedNameSpecifiers) {
       "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa::aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n"
       "    aaaaaaaaaaaaaaaaaaaaaaa);");
 
-  verifyFormat(
-      "aaaaaaaaaaaaaaaaaa(aaaaaaaa,\n"
-      "                   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa::\n"
-      "                       aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
-      "                   aaaaaaaaaaaaaaaaaaaaa);",
-      getLLVMStyleWithColumns(74));
+  verifyFormat("aaaaaaaaaaaaaaaaaa(aaaaaaaa,\n"
+               "                   aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa::\n"
+               "                       aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa,\n"
+               "                   aaaaaaaaaaaaaaaaaaaaa);",
+               getLLVMStyleWithColumns(74));
 
   verifyFormat("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa::\n"
                "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
@@ -6866,9 +6897,78 @@ TEST_F(FormatTest, UnderstandsFunctionRefQualification) {
   Spaces.SpacesInCStyleCastParentheses = false;
   Spaces.SpacesInParentheses = true;
   verifyFormat("Deleted &operator=( const Deleted & ) & = default;", Spaces);
-  verifyFormat("SomeType MemberFunction( const Deleted & ) & = delete;", Spaces);
+  verifyFormat("SomeType MemberFunction( const Deleted & ) & = delete;",
+               Spaces);
   verifyFormat("Deleted &operator=( const Deleted & ) &;", Spaces);
   verifyFormat("SomeType MemberFunction( const Deleted & ) &;", Spaces);
+
+  FormatStyle BreakTemplate = getLLVMStyle();
+  BreakTemplate.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int &foo(const std::string &str) & noexcept {}\n"
+               "};",
+               BreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int &foo(const std::string &str) && noexcept {}\n"
+               "};",
+               BreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int &foo(const std::string &str) const & noexcept {}\n"
+               "};",
+               BreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int &foo(const std::string &str) const & noexcept {}\n"
+               "};",
+               BreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  auto foo(const std::string &str) && noexcept -> int & {}\n"
+               "};",
+               BreakTemplate);
+
+  FormatStyle AlignLeftBreakTemplate = getLLVMStyle();
+  AlignLeftBreakTemplate.AlwaysBreakTemplateDeclarations =
+      FormatStyle::BTDS_Yes;
+  AlignLeftBreakTemplate.PointerAlignment = FormatStyle::PAS_Left;
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int& foo(const std::string& str) & noexcept {}\n"
+               "};",
+               AlignLeftBreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int& foo(const std::string& str) && noexcept {}\n"
+               "};",
+               AlignLeftBreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int& foo(const std::string& str) const & noexcept {}\n"
+               "};",
+               AlignLeftBreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  int& foo(const std::string& str) const & noexcept {}\n"
+               "};",
+               AlignLeftBreakTemplate);
+
+  verifyFormat("struct f {\n"
+               "  template <class T>\n"
+               "  auto foo(const std::string& str) && noexcept -> int& {}\n"
+               "};",
+               AlignLeftBreakTemplate);
 }
 
 TEST_F(FormatTest, UnderstandsNewAndDelete) {
@@ -7075,14 +7175,13 @@ TEST_F(FormatTest, UnderstandsUsesOfStarAndAmp) {
       "                       (sizeof(T) > 1 || sizeof(T) < 8)>::type>\n"
       "void F();",
       getLLVMStyleWithColumns(70));
-  verifyFormat(
-      "template <class T,\n"
-      "          class = typename std::enable_if<\n"
-      "              std::is_integral<T>::value &&\n"
-      "              (sizeof(T) > 1 || sizeof(T) < 8)>::type,\n"
-      "          class U>\n"
-      "void F();",
-      getLLVMStyleWithColumns(70));
+  verifyFormat("template <class T,\n"
+               "          class = typename std::enable_if<\n"
+               "              std::is_integral<T>::value &&\n"
+               "              (sizeof(T) > 1 || sizeof(T) < 8)>::type,\n"
+               "          class U>\n"
+               "void F();",
+               getLLVMStyleWithColumns(70));
   verifyFormat(
       "template <class T,\n"
       "          class = typename ::std::enable_if<\n"
@@ -7801,13 +7900,14 @@ TEST_F(FormatTest, LayoutCxx11BraceInitializers) {
   // Binpacking only if there is no trailing comma
   verifyFormat("const Aaaaaa aaaaa = {aaaaaaaaaa, bbbbbbbbbb,\n"
                "                      cccccccccc, dddddddddd};",
-			   getLLVMStyleWithColumns(50));
+               getLLVMStyleWithColumns(50));
   verifyFormat("const Aaaaaa aaaaa = {\n"
                "    aaaaaaaaaaa,\n"
                "    bbbbbbbbbbb,\n"
                "    ccccccccccc,\n"
                "    ddddddddddd,\n"
-               "};", getLLVMStyleWithColumns(50));
+               "};",
+               getLLVMStyleWithColumns(50));
 
   // Cases where distinguising braced lists and blocks is hard.
   verifyFormat("vector<int> v{12} GUARDED_BY(mutex);");
@@ -8475,7 +8575,7 @@ TEST_F(FormatTest, SplitEmptyStruct) {
                "{\n"
                "} Foo_t;",
                Style);
-  //typedef struct Bar {} Bar_t;
+  // typedef struct Bar {} Bar_t;
 }
 
 TEST_F(FormatTest, SplitEmptyUnion) {
@@ -8866,7 +8966,6 @@ TEST_F(FormatTest, FormatForObjectiveCMethodDecls) {
   verifyGoogleFormat("- foo:(int)foo;");
 }
 
-
 TEST_F(FormatTest, BreaksStringLiterals) {
   EXPECT_EQ("\"some text \"\n"
             "\"other\";",
@@ -8989,14 +9088,13 @@ TEST_F(FormatTest, BreaksStringLiterals) {
 
   // Verify that splitting the strings understands
   // Style::AlwaysBreakBeforeMultilineStrings.
-  EXPECT_EQ(
-      "aaaaaaaaaaaa(\n"
-      "    \"aaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa \"\n"
-      "    \"aaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa\");",
-      format("aaaaaaaaaaaa(\"aaaaaaaaaaaaaaaaaaaaaaaaaa "
-             "aaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaa "
-             "aaaaaaaaaaaaaaaaaaaaaa\");",
-             getGoogleStyle()));
+  EXPECT_EQ("aaaaaaaaaaaa(\n"
+            "    \"aaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa \"\n"
+            "    \"aaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa\");",
+            format("aaaaaaaaaaaa(\"aaaaaaaaaaaaaaaaaaaaaaaaaa "
+                   "aaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaa "
+                   "aaaaaaaaaaaaaaaaaaaaaa\");",
+                   getGoogleStyle()));
   EXPECT_EQ("return \"aaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaaaaaa \"\n"
             "       \"aaaaaaaaaaaaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaaaaaaaa\";",
             format("return \"aaaaaaaaaaaaaaaaaaaaaa "
@@ -9133,10 +9231,10 @@ TEST_F(FormatTest, BreaksStringLiteralsWithin_TMacro) {
 TEST_F(FormatTest, BreaksStringLiteralOperands) {
   // In a function call with two operands, the second can be broken with no line
   // break before it.
-  EXPECT_EQ("func(a, \"long long \"\n"
-            "        \"long long\");",
-            format("func(a, \"long long long long\");",
-                   getLLVMStyleWithColumns(24)));
+  EXPECT_EQ(
+      "func(a, \"long long \"\n"
+      "        \"long long\");",
+      format("func(a, \"long long long long\");", getLLVMStyleWithColumns(24)));
   // In a function call with three operands, the second must be broken with a
   // line break before it.
   EXPECT_EQ("func(a,\n"
@@ -9165,22 +9263,21 @@ TEST_F(FormatTest, BreaksStringLiteralOperands) {
   // break before it.
   EXPECT_EQ("a << \"line line \"\n"
             "     \"line\";",
-            format("a << \"line line line\";",
-                   getLLVMStyleWithColumns(20)));
+            format("a << \"line line line\";", getLLVMStyleWithColumns(20)));
   // In a chain of << with three operands, the second can be broken with no line
   // break before it.
-  EXPECT_EQ("abcde << \"line \"\n"
-            "         \"line line\"\n"
-            "      << c;",
-            format("abcde << \"line line line\" << c;",
-                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ(
+      "abcde << \"line \"\n"
+      "         \"line line\"\n"
+      "      << c;",
+      format("abcde << \"line line line\" << c;", getLLVMStyleWithColumns(20)));
   // In a chain of << with three operands, the third must be broken with a line
   // break before it.
-  EXPECT_EQ("a << b\n"
-            "  << \"line line \"\n"
-            "     \"line\";",
-            format("a << b << \"line line line\";",
-                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ(
+      "a << b\n"
+      "  << \"line line \"\n"
+      "     \"line\";",
+      format("a << b << \"line line line\";", getLLVMStyleWithColumns(20)));
   // In a chain of << with three operands, the second can be broken with no line
   // break before it and the third must be broken with a line break before it.
   EXPECT_EQ("abcd << \"line line \"\n"
@@ -9191,10 +9288,10 @@ TEST_F(FormatTest, BreaksStringLiteralOperands) {
                    getLLVMStyleWithColumns(20)));
   // In a chain of binary operators with two operands, the second can be broken
   // with no line break before it.
-  EXPECT_EQ("abcd + \"line line \"\n"
-            "       \"line line\";",
-            format("abcd + \"line line line line\";",
-                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ(
+      "abcd + \"line line \"\n"
+      "       \"line line\";",
+      format("abcd + \"line line line line\";", getLLVMStyleWithColumns(20)));
   // In a chain of binary operators with three operands, the second must be
   // broken with a line break before it.
   EXPECT_EQ("abcd +\n"
@@ -9291,6 +9388,19 @@ TEST_F(FormatTest, DoesNotTryToParseUDLiteralsInPreCpp11Code) {
             format("#define x(_a) printf(\"foo\"_a);", Style));
 }
 
+TEST_F(FormatTest, CppLexVersion) {
+  FormatStyle Style = getLLVMStyle();
+  // Formatting of x * y differs if x is a type.
+  verifyFormat("void foo() { MACRO(a * b); }", Style);
+  verifyFormat("void foo() { MACRO(int *b); }", Style);
+
+  // LLVM style uses latest lexer.
+  verifyFormat("void foo() { MACRO(char8_t *b); }", Style);
+  Style.Standard = FormatStyle::LS_Cpp17;
+  // But in c++17, char8_t isn't a keyword.
+  verifyFormat("void foo() { MACRO(char8_t * b); }", Style);
+}
+
 TEST_F(FormatTest, UnderstandsCpp1y) { verifyFormat("int bi{1'000'000};"); }
 
 TEST_F(FormatTest, BreakStringLiteralsBeforeUnbreakableTokenSequence) {
@@ -9877,6 +9987,79 @@ TEST_F(FormatTest, ConfigurableUseOfTab) {
                Tab);
 }
 
+TEST_F(FormatTest, ZeroTabWidth) {
+  FormatStyle Tab = getLLVMStyleWithColumns(42);
+  Tab.IndentWidth = 8;
+  Tab.UseTab = FormatStyle::UT_Never;
+  Tab.TabWidth = 0;
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  Tab.UseTab = FormatStyle::UT_ForIndentation;
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  Tab.UseTab = FormatStyle::UT_ForContinuationAndIndentation;
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  EXPECT_EQ("void a(){\n"
+            "    // line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  Tab.UseTab = FormatStyle::UT_Always;
+  EXPECT_EQ("void a(){\n"
+            "// line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+
+  EXPECT_EQ("void a(){\n"
+            "// line starts with '\t'\n"
+            "};",
+            format("void a(){\n"
+                   "\t\t// line starts with '\t'\n"
+                   "};",
+                   Tab));
+}
+
 TEST_F(FormatTest, CalculatesOriginalColumn) {
   EXPECT_EQ("\"qqqqqqqqqqqqqqqqqqqqqqqqqq\\\n"
             "q\"; /* some\n"
@@ -11347,6 +11530,253 @@ TEST_F(FormatTest, AllmanBraceBreaking) {
                BreakBeforeBraceShortIfs);
 }
 
+TEST_F(FormatTest, WhitesmithsBraceBreaking) {
+  FormatStyle WhitesmithsBraceStyle = getLLVMStyle();
+  WhitesmithsBraceStyle.BreakBeforeBraces = FormatStyle::BS_Whitesmiths;
+
+  // Make a few changes to the style for testing purposes
+  WhitesmithsBraceStyle.AllowShortFunctionsOnASingleLine =
+      FormatStyle::SFS_Empty;
+  WhitesmithsBraceStyle.AllowShortLambdasOnASingleLine = FormatStyle::SLS_None;
+  WhitesmithsBraceStyle.ColumnLimit = 0;
+
+  // FIXME: this test case can't decide whether there should be a blank line
+  // after the ~D() line or not. It adds one if one doesn't exist in the test
+  // and it removes the line if one exists.
+  /*
+  verifyFormat("class A;\n"
+               "namespace B\n"
+               "  {\n"
+               "class C;\n"
+               "// Comment\n"
+               "class D\n"
+               "  {\n"
+               "public:\n"
+               "  D();\n"
+               "  ~D() {}\n"
+               "private:\n"
+               "  enum E\n"
+               "    {\n"
+               "    F\n"
+               "    }\n"
+               "  };\n"
+               "  } // namespace B\n",
+               WhitesmithsBraceStyle);
+  */
+
+  verifyFormat("namespace a\n"
+               "  {\n"
+               "class A\n"
+               "  {\n"
+               "  void f()\n"
+               "    {\n"
+               "    if (true)\n"
+               "      {\n"
+               "      a();\n"
+               "      b();\n"
+               "      }\n"
+               "    }\n"
+               "  void g()\n"
+               "    {\n"
+               "    return;\n"
+               "    }\n"
+               "  };\n"
+               "struct B\n"
+               "  {\n"
+               "  int x;\n"
+               "  };\n"
+               "  } // namespace a",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void f()\n"
+               "  {\n"
+               "  if (true)\n"
+               "    {\n"
+               "    a();\n"
+               "    }\n"
+               "  else if (false)\n"
+               "    {\n"
+               "    b();\n"
+               "    }\n"
+               "  else\n"
+               "    {\n"
+               "    c();\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void f()\n"
+               "  {\n"
+               "  for (int i = 0; i < 10; ++i)\n"
+               "    {\n"
+               "    a();\n"
+               "    }\n"
+               "  while (false)\n"
+               "    {\n"
+               "    b();\n"
+               "    }\n"
+               "  do\n"
+               "    {\n"
+               "    c();\n"
+               "    } while (false)\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  // FIXME: the block and the break under case 2 in this test don't get indented
+  // correctly
+  /*
+  verifyFormat("void switchTest1(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
+               "    {\n"
+               "    case 2:\n"
+               "      {\n"
+               "      }\n"
+               "      break;\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+  */
+
+  // FIXME: the block and the break under case 2 in this test don't get indented
+  // correctly
+  /*
+  verifyFormat("void switchTest2(int a)\n"
+               "  {\n"
+               "  switch (a)\n"
+               "    {\n"
+               "  case 0:\n"
+               "    break;\n"
+               "  case 1:\n"
+               "    {\n"
+               "    break;\n"
+               "    }\n"
+               "  case 2:\n"
+               "    {\n"
+               "    }\n"
+               "    break;\n"
+               "  default:\n"
+               "    break;\n"
+               "    }\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+  */
+
+  verifyFormat("enum X\n"
+               "  {\n"
+               "  Y = 0, // testing\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("enum X\n"
+               "  {\n"
+               "  Y = 0\n"
+               "  }\n",
+               WhitesmithsBraceStyle);
+  verifyFormat("enum X\n"
+               "  {\n"
+               "  Y = 0,\n"
+               "  Z = 1\n"
+               "  };\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("@interface BSApplicationController ()\n"
+               "  {\n"
+               "@private\n"
+               "  id _extraIvar;\n"
+               "  }\n"
+               "@end\n",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("#ifdef _DEBUG\n"
+               "int foo(int i = 0)\n"
+               "#else\n"
+               "int foo(int i = 5)\n"
+               "#endif\n"
+               "  {\n"
+               "  return i;\n"
+               "  }",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void foo() {}\n"
+               "void bar()\n"
+               "#ifdef _DEBUG\n"
+               "  {\n"
+               "  foo();\n"
+               "  }\n"
+               "#else\n"
+               "  {\n"
+               "  }\n"
+               "#endif",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("void foobar()\n"
+               "  {\n"
+               "  int i = 5;\n"
+               "  }\n"
+               "#ifdef _DEBUG\n"
+               "void bar()\n"
+               "  {\n"
+               "  }\n"
+               "#else\n"
+               "void bar()\n"
+               "  {\n"
+               "  foobar();\n"
+               "  }\n"
+               "#endif",
+               WhitesmithsBraceStyle);
+
+  // This shouldn't affect ObjC blocks..
+  verifyFormat("[self doSomeThingWithACompletionHandler:^{\n"
+               "  // ...\n"
+               "  int i;\n"
+               "}];",
+               WhitesmithsBraceStyle);
+  verifyFormat("void (^block)(void) = ^{\n"
+               "  // ...\n"
+               "  int i;\n"
+               "};",
+               WhitesmithsBraceStyle);
+  // .. or dict literals.
+  verifyFormat("void f()\n"
+               "  {\n"
+               "  [object someMethod:@{@\"a\" : @\"b\"}];\n"
+               "  }",
+               WhitesmithsBraceStyle);
+
+  verifyFormat("int f()\n"
+               "  { // comment\n"
+               "  return 42;\n"
+               "  }",
+               WhitesmithsBraceStyle);
+
+  FormatStyle BreakBeforeBraceShortIfs = WhitesmithsBraceStyle;
+  BreakBeforeBraceShortIfs.AllowShortIfStatementsOnASingleLine =
+      FormatStyle::SIS_Always;
+  BreakBeforeBraceShortIfs.AllowShortLoopsOnASingleLine = true;
+  verifyFormat("void f(bool b)\n"
+               "  {\n"
+               "  if (b)\n"
+               "    {\n"
+               "    return;\n"
+               "    }\n"
+               "  }\n",
+               BreakBeforeBraceShortIfs);
+  verifyFormat("void f(bool b)\n"
+               "  {\n"
+               "  if (b) return;\n"
+               "  }\n",
+               BreakBeforeBraceShortIfs);
+  verifyFormat("void f(bool b)\n"
+               "  {\n"
+               "  while (b)\n"
+               "    {\n"
+               "    return;\n"
+               "    }\n"
+               "  }\n",
+               BreakBeforeBraceShortIfs);
+}
+
 TEST_F(FormatTest, GNUBraceBreaking) {
   FormatStyle GNUBraceStyle = getLLVMStyle();
   GNUBraceStyle.BreakBeforeBraces = FormatStyle::BS_GNU;
@@ -11558,7 +11988,8 @@ TEST_F(FormatTest, OptimizeBreakPenaltyVsExcess) {
                    Style));
 
   verifyFormat("int a; // the\n"
-               "       // comment", Style);
+               "       // comment",
+               Style);
   EXPECT_EQ("int a; /* first line\n"
             "        * second\n"
             "        * line third\n"
@@ -11648,11 +12079,11 @@ TEST_F(FormatTest, OptimizeBreakPenaltyVsExcess) {
   // Make sure we do not keep protruding characters if strict mode reflow is
   // cheaper than keeping protruding characters.
   Style.ColumnLimit = 21;
-  EXPECT_EQ("// foo foo foo foo\n"
-            "// foo foo foo foo\n"
-            "// foo foo foo foo\n",
-            format("// foo foo foo foo foo foo foo foo foo foo foo foo\n",
-                           Style));
+  EXPECT_EQ(
+      "// foo foo foo foo\n"
+      "// foo foo foo foo\n"
+      "// foo foo foo foo\n",
+      format("// foo foo foo foo foo foo foo foo foo foo foo foo\n", Style));
 
   EXPECT_EQ("int a = /* long block\n"
             "           comment */\n"
@@ -11662,8 +12093,8 @@ TEST_F(FormatTest, OptimizeBreakPenaltyVsExcess) {
 
 #define EXPECT_ALL_STYLES_EQUAL(Styles)                                        \
   for (size_t i = 1; i < Styles.size(); ++i)                                   \
-  EXPECT_EQ(Styles[0], Styles[i]) << "Style #" << i << " of " << Styles.size() \
-                                  << " differs from Style #0"
+  EXPECT_EQ(Styles[0], Styles[i])                                              \
+      << "Style #" << i << " of " << Styles.size() << " differs from Style #0"
 
 TEST_F(FormatTest, GetsPredefinedStyleByName) {
   SmallVector<FormatStyle, 3> Styles;
@@ -11857,8 +12288,7 @@ TEST_F(FormatTest, ParsesConfiguration) {
   CHECK_PARSE("ObjCBlockIndentWidth: 1234", ObjCBlockIndentWidth, 1234u);
   CHECK_PARSE("ColumnLimit: 1234", ColumnLimit, 1234u);
   CHECK_PARSE("MaxEmptyLinesToKeep: 1234", MaxEmptyLinesToKeep, 1234u);
-  CHECK_PARSE("PenaltyBreakAssignment: 1234",
-              PenaltyBreakAssignment, 1234u);
+  CHECK_PARSE("PenaltyBreakAssignment: 1234", PenaltyBreakAssignment, 1234u);
   CHECK_PARSE("PenaltyBreakBeforeFirstCallParameter: 1234",
               PenaltyBreakBeforeFirstCallParameter, 1234u);
   CHECK_PARSE("PenaltyBreakTemplateDeclaration: 1234",
@@ -11888,11 +12318,18 @@ TEST_F(FormatTest, ParsesConfiguration) {
               FormatStyle::PAS_Middle);
 
   Style.Standard = FormatStyle::LS_Auto;
+  CHECK_PARSE("Standard: c++03", Standard, FormatStyle::LS_Cpp03);
+  CHECK_PARSE("Standard: c++11", Standard, FormatStyle::LS_Cpp11);
+  CHECK_PARSE("Standard: c++14", Standard, FormatStyle::LS_Cpp14);
+  CHECK_PARSE("Standard: c++17", Standard, FormatStyle::LS_Cpp17);
+  CHECK_PARSE("Standard: c++20", Standard, FormatStyle::LS_Cpp20);
+  CHECK_PARSE("Standard: Auto", Standard, FormatStyle::LS_Auto);
+  CHECK_PARSE("Standard: Latest", Standard, FormatStyle::LS_Latest);
+  // Legacy aliases:
   CHECK_PARSE("Standard: Cpp03", Standard, FormatStyle::LS_Cpp03);
-  CHECK_PARSE("Standard: Cpp11", Standard, FormatStyle::LS_Cpp11);
+  CHECK_PARSE("Standard: Cpp11", Standard, FormatStyle::LS_Latest);
   CHECK_PARSE("Standard: C++03", Standard, FormatStyle::LS_Cpp03);
   CHECK_PARSE("Standard: C++11", Standard, FormatStyle::LS_Cpp11);
-  CHECK_PARSE("Standard: Auto", Standard, FormatStyle::LS_Auto);
 
   Style.BreakBeforeBinaryOperators = FormatStyle::BOS_All;
   CHECK_PARSE("BreakBeforeBinaryOperators: NonAssignment",
@@ -11919,15 +12356,15 @@ TEST_F(FormatTest, ParsesConfiguration) {
               BreakConstructorInitializers, FormatStyle::BCIS_BeforeComma);
 
   Style.BreakInheritanceList = FormatStyle::BILS_BeforeColon;
-  CHECK_PARSE("BreakInheritanceList: BeforeComma",
-              BreakInheritanceList, FormatStyle::BILS_BeforeComma);
-  CHECK_PARSE("BreakInheritanceList: AfterColon",
-              BreakInheritanceList, FormatStyle::BILS_AfterColon);
-  CHECK_PARSE("BreakInheritanceList: BeforeColon",
-              BreakInheritanceList, FormatStyle::BILS_BeforeColon);
+  CHECK_PARSE("BreakInheritanceList: BeforeComma", BreakInheritanceList,
+              FormatStyle::BILS_BeforeComma);
+  CHECK_PARSE("BreakInheritanceList: AfterColon", BreakInheritanceList,
+              FormatStyle::BILS_AfterColon);
+  CHECK_PARSE("BreakInheritanceList: BeforeColon", BreakInheritanceList,
+              FormatStyle::BILS_BeforeColon);
   // For backward compatibility:
-  CHECK_PARSE("BreakBeforeInheritanceComma: true",
-              BreakInheritanceList, FormatStyle::BILS_BeforeComma);
+  CHECK_PARSE("BreakBeforeInheritanceComma: true", BreakInheritanceList,
+              FormatStyle::BILS_BeforeComma);
 
   Style.AlignAfterOpenBracket = FormatStyle::BAS_AlwaysBreak;
   CHECK_PARSE("AlignAfterOpenBracket: Align", AlignAfterOpenBracket,
@@ -12024,6 +12461,8 @@ TEST_F(FormatTest, ParsesConfiguration) {
               FormatStyle::BS_Stroustrup);
   CHECK_PARSE("BreakBeforeBraces: Allman", BreakBeforeBraces,
               FormatStyle::BS_Allman);
+  CHECK_PARSE("BreakBeforeBraces: Whitesmiths", BreakBeforeBraces,
+              FormatStyle::BS_Whitesmiths);
   CHECK_PARSE("BreakBeforeBraces: GNU", BreakBeforeBraces, FormatStyle::BS_GNU);
   CHECK_PARSE("BreakBeforeBraces: WebKit", BreakBeforeBraces,
               FormatStyle::BS_WebKit);
@@ -12044,16 +12483,16 @@ TEST_F(FormatTest, ParsesConfiguration) {
               FormatStyle::RTBS_TopLevelDefinitions);
 
   Style.AlwaysBreakTemplateDeclarations = FormatStyle::BTDS_Yes;
-  CHECK_PARSE("AlwaysBreakTemplateDeclarations: No", AlwaysBreakTemplateDeclarations,
-              FormatStyle::BTDS_No);
-  CHECK_PARSE("AlwaysBreakTemplateDeclarations: MultiLine", AlwaysBreakTemplateDeclarations,
-              FormatStyle::BTDS_MultiLine);
-  CHECK_PARSE("AlwaysBreakTemplateDeclarations: Yes", AlwaysBreakTemplateDeclarations,
-              FormatStyle::BTDS_Yes);
-  CHECK_PARSE("AlwaysBreakTemplateDeclarations: false", AlwaysBreakTemplateDeclarations,
-              FormatStyle::BTDS_MultiLine);
-  CHECK_PARSE("AlwaysBreakTemplateDeclarations: true", AlwaysBreakTemplateDeclarations,
-              FormatStyle::BTDS_Yes);
+  CHECK_PARSE("AlwaysBreakTemplateDeclarations: No",
+              AlwaysBreakTemplateDeclarations, FormatStyle::BTDS_No);
+  CHECK_PARSE("AlwaysBreakTemplateDeclarations: MultiLine",
+              AlwaysBreakTemplateDeclarations, FormatStyle::BTDS_MultiLine);
+  CHECK_PARSE("AlwaysBreakTemplateDeclarations: Yes",
+              AlwaysBreakTemplateDeclarations, FormatStyle::BTDS_Yes);
+  CHECK_PARSE("AlwaysBreakTemplateDeclarations: false",
+              AlwaysBreakTemplateDeclarations, FormatStyle::BTDS_MultiLine);
+  CHECK_PARSE("AlwaysBreakTemplateDeclarations: true",
+              AlwaysBreakTemplateDeclarations, FormatStyle::BTDS_Yes);
 
   Style.AlwaysBreakAfterDefinitionReturnType = FormatStyle::DRTBS_All;
   CHECK_PARSE("AlwaysBreakAfterDefinitionReturnType: None",
@@ -12112,7 +12551,7 @@ TEST_F(FormatTest, ParsesConfiguration) {
 
   Style.IncludeStyle.IncludeCategories.clear();
   std::vector<tooling::IncludeStyle::IncludeCategory> ExpectedCategories = {
-      {"abc/.*", 2}, {".*", 1}};
+      {"abc/.*", 2, 0}, {".*", 1, 0}};
   CHECK_PARSE("IncludeCategories:\n"
               "  - Regex: abc/.*\n"
               "    Priority: 2\n"
@@ -12445,17 +12884,17 @@ TEST_F(FormatTest, ConstructorInitializerIndentWidth) {
       "SomeLongTemplateVariableName<\n"
       "    aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>",
       Style);
-  verifyFormat(
-      "bool smaller = 1 < bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb(\n"
-      "                       aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);",
-      Style);
+  verifyFormat("bool smaller = 1 < "
+               "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb(\n"
+               "                       "
+               "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);",
+               Style);
 
   Style.BreakConstructorInitializers = FormatStyle::BCIS_AfterColon;
-  verifyFormat(
-      "SomeClass::Constructor() :\n"
-      "aaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa),\n"
-      "aaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa) {}",
-      Style);
+  verifyFormat("SomeClass::Constructor() :\n"
+               "aaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa),\n"
+               "aaaaaaaaaaaaaaaaaaaa(aaaaaaaaaaaaaaaaaaaa) {}",
+               Style);
 }
 
 TEST_F(FormatTest, BreakConstructorInitializersBeforeComma) {
@@ -12591,7 +13030,8 @@ TEST_F(FormatTest, FormatsWithWebKitStyle) {
   // However, don't merge non-empty short loops.
   EXPECT_EQ("while (true) {\n"
             "    continue;\n"
-            "}", format("while (true) { continue; }", Style));
+            "}",
+            format("while (true) { continue; }", Style));
 
   // Constructor initializers are formatted one per line with the "," on the
   // new line.
@@ -12729,15 +13169,17 @@ TEST_F(FormatTest, FormatsLambdas) {
   verifyFormat("void f() {\n"
                "  other.other.other.other.other(\n"
                "      x.begin(), x.end(),\n"
-               "      [something, rather](int, int, int, int, int, int, int) { return 1; });\n"
-               "}\n");
-  verifyFormat("void f() {\n"
-               "  other.other.other.other.other(\n"
-               "      x.begin(), x.end(),\n"
-               "      [something, rather](int, int, int, int, int, int, int) {\n"
-               "        //\n"
-               "      });\n"
+               "      [something, rather](int, int, int, int, int, int, int) { "
+               "return 1; });\n"
                "}\n");
+  verifyFormat(
+      "void f() {\n"
+      "  other.other.other.other.other(\n"
+      "      x.begin(), x.end(),\n"
+      "      [something, rather](int, int, int, int, int, int, int) {\n"
+      "        //\n"
+      "      });\n"
+      "}\n");
   verifyFormat("SomeFunction([]() { // A cool function...\n"
                "  return 43;\n"
                "});");
@@ -12789,9 +13231,10 @@ TEST_F(FormatTest, FormatsLambdas) {
   verifyFormat("SomeFunction({[&] {\n"
                "  // comment\n"
                "}});");
-  verifyFormat("virtual aaaaaaaaaaaaaaaa(\n"
-               "    std::function<bool()> bbbbbbbbbbbb = [&]() { return true; },\n"
-               "    aaaaa aaaaaaaaa);");
+  verifyFormat(
+      "virtual aaaaaaaaaaaaaaaa(\n"
+      "    std::function<bool()> bbbbbbbbbbbb = [&]() { return true; },\n"
+      "    aaaaa aaaaaaaaa);");
 
   // Lambdas with return types.
   verifyFormat("int c = []() -> int { return 2; }();\n");
@@ -12822,77 +13265,77 @@ TEST_F(FormatTest, FormatsLambdas) {
   verifyFormat("[]() -> foo<5 < 2> { return {}; };");
   verifyFormat("[]() -> foo<2 ? 1 : 0> { return {}; };");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 + 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 + 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 - 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 - 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 / 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 / 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 * 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 * 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 % 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 % 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 << 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 << 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<!5> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<!5> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<~5> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<~5> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 | 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 | 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 || 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 || 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 & 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 & 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 && 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 && 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 == 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 == 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 != 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 != 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 >= 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 >= 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 <= 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 <= 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<5 < 2> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<5 < 2> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("namespace bar {\n"
-              "// broken:\n"
-              "auto foo{[]() -> foo<2 ? 1 : 0> { return {}; }};\n"
-              "} // namespace bar");
+               "// broken:\n"
+               "auto foo{[]() -> foo<2 ? 1 : 0> { return {}; }};\n"
+               "} // namespace bar");
   verifyFormat("[]() -> a<1> {};");
   verifyFormat("[]() -> a<1> { ; };");
   verifyFormat("[]() -> a<1> { ; }();");
@@ -12964,23 +13407,25 @@ TEST_F(FormatTest, FormatsLambdas) {
 
   // A lambda with a very long line forces arg0 to be pushed out irrespective of
   // the BinPackArguments value (as long as the code is wide enough).
-  verifyFormat("something->SomeFunction(\n"
-               "    a,\n"
-               "    [this] {\n"
-               "      D0000000000000000000000000000000000000000000000000000000000001();\n"
-               "    },\n"
-               "    b);\n");
+  verifyFormat(
+      "something->SomeFunction(\n"
+      "    a,\n"
+      "    [this] {\n"
+      "      "
+      "D0000000000000000000000000000000000000000000000000000000000001();\n"
+      "    },\n"
+      "    b);\n");
 
-  // A multi-line lambda is pulled up as long as the introducer fits on the previous
-  // line and there are no further args.
+  // A multi-line lambda is pulled up as long as the introducer fits on the
+  // previous line and there are no further args.
   verifyFormat("function(1, [this, that] {\n"
                "  //\n"
                "});\n");
   verifyFormat("function([this, that] {\n"
                "  //\n"
                "});\n");
-  // FIXME: this format is not ideal and we should consider forcing the first arg
-  // onto its own line.
+  // FIXME: this format is not ideal and we should consider forcing the first
+  // arg onto its own line.
   verifyFormat("function(a, b, c, //\n"
                "         d, [this, that] {\n"
                "           //\n"
@@ -13550,24 +13995,23 @@ TEST_F(FormatTest, DisableRegions) {
                    "   int   k;"));
 
   // Don't reflow comments within disabled regions.
-  EXPECT_EQ(
-      "// clang-format off\n"
-      "// long long long long long long line\n"
-      "/* clang-format on */\n"
-      "/* long long long\n"
-      " * long long long\n"
-      " * line */\n"
-      "int i;\n"
-      "/* clang-format off */\n"
-      "/* long long long long long long line */\n",
-      format("// clang-format off\n"
-             "// long long long long long long line\n"
-             "/* clang-format on */\n"
-             "/* long long long long long long line */\n"
-             "int i;\n"
-             "/* clang-format off */\n"
-             "/* long long long long long long line */\n",
-             getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("// clang-format off\n"
+            "// long long long long long long line\n"
+            "/* clang-format on */\n"
+            "/* long long long\n"
+            " * long long long\n"
+            " * line */\n"
+            "int i;\n"
+            "/* clang-format off */\n"
+            "/* long long long long long long line */\n",
+            format("// clang-format off\n"
+                   "// long long long long long long line\n"
+                   "/* clang-format on */\n"
+                   "/* long long long long long long line */\n"
+                   "int i;\n"
+                   "/* clang-format off */\n"
+                   "/* long long long long long long line */\n",
+                   getLLVMStyleWithColumns(20)));
 }
 
 TEST_F(FormatTest, DoNotCrashOnInvalidInput) {
@@ -13601,9 +14045,7 @@ TEST_F(FormatTest, ArrayAsTemplateType) {
             format("auto a = unique_ptr < Foo < Bar>[10]> ;", Spaces));
 }
 
-TEST_F(FormatTest, NoSpaceAfterSuper) {
-    verifyFormat("__super::FooBar();");
-}
+TEST_F(FormatTest, NoSpaceAfterSuper) { verifyFormat("__super::FooBar();"); }
 
 TEST(FormatStyle, GetStyleWithEmptyFileName) {
   llvm::vfs::InMemoryFileSystem FS;
@@ -13758,7 +14200,8 @@ TEST_F(FormatTest, FormatSortsUsingDeclarations) {
   EXPECT_EQ("using std::cin;\n"
             "using std::cout;",
             format("using std::cout;\n"
-                   "using std::cin;", getGoogleStyle()));
+                   "using std::cin;",
+                   getGoogleStyle()));
 }
 
 TEST_F(FormatTest, UTF8CharacterLiteralCpp03) {
@@ -13775,10 +14218,8 @@ TEST_F(FormatTest, UTF8CharacterLiteralCpp11) {
 }
 
 TEST_F(FormatTest, DoNotFormatLikelyXml) {
-  EXPECT_EQ("<!-- ;> -->",
-            format("<!-- ;> -->", getGoogleStyle()));
-  EXPECT_EQ(" <!-- >; -->",
-            format(" <!-- >; -->", getGoogleStyle()));
+  EXPECT_EQ("<!-- ;> -->", format("<!-- ;> -->", getGoogleStyle()));
+  EXPECT_EQ(" <!-- >; -->", format(" <!-- >; -->", getGoogleStyle()));
 }
 
 TEST_F(FormatTest, StructuredBindings) {
@@ -13798,13 +14239,16 @@ TEST_F(FormatTest, StructuredBindings) {
   EXPECT_EQ("auto const &[a, b] = f();", format("auto  const&[a, b] = f();"));
   EXPECT_EQ("auto const volatile &&[a, b] = f();",
             format("auto  const  volatile  &&[a, b] = f();"));
-  EXPECT_EQ("auto const &&[a, b] = f();", format("auto  const   &&  [a, b] = f();"));
-  EXPECT_EQ("const auto &[a, b] = f();", format("const  auto  &  [a, b] = f();"));
+  EXPECT_EQ("auto const &&[a, b] = f();",
+            format("auto  const   &&  [a, b] = f();"));
+  EXPECT_EQ("const auto &[a, b] = f();",
+            format("const  auto  &  [a, b] = f();"));
   EXPECT_EQ("const auto volatile &&[a, b] = f();",
             format("const  auto   volatile  &&[a, b] = f();"));
   EXPECT_EQ("volatile const auto &&[a, b] = f();",
             format("volatile  const  auto   &&[a, b] = f();"));
-  EXPECT_EQ("const auto &&[a, b] = f();", format("const  auto  &&  [a, b] = f();"));
+  EXPECT_EQ("const auto &&[a, b] = f();",
+            format("const  auto  &&  [a, b] = f();"));
 
   // Make sure we don't mistake structured bindings for lambdas.
   FormatStyle PointerMiddle = getLLVMStyle();
@@ -13831,11 +14275,15 @@ TEST_F(FormatTest, StructuredBindings) {
   EXPECT_EQ("auto [x, y](expr);", format("auto[x,y]  (expr);"));
   EXPECT_EQ("auto &[x, y](expr);", format("auto  &  [x,y]  (expr);"));
   EXPECT_EQ("auto &&[x, y](expr);", format("auto  &&  [x,y]  (expr);"));
-  EXPECT_EQ("auto const &[x, y](expr);", format("auto  const  &  [x,y]  (expr);"));
-  EXPECT_EQ("auto const &&[x, y](expr);", format("auto  const  &&  [x,y]  (expr);"));
+  EXPECT_EQ("auto const &[x, y](expr);",
+            format("auto  const  &  [x,y]  (expr);"));
+  EXPECT_EQ("auto const &&[x, y](expr);",
+            format("auto  const  &&  [x,y]  (expr);"));
   EXPECT_EQ("auto [x, y]{expr};", format("auto[x,y]     {expr};"));
-  EXPECT_EQ("auto const &[x, y]{expr};", format("auto  const  &  [x,y]  {expr};"));
-  EXPECT_EQ("auto const &&[x, y]{expr};", format("auto  const  &&  [x,y]  {expr};"));
+  EXPECT_EQ("auto const &[x, y]{expr};",
+            format("auto  const  &  [x,y]  {expr};"));
+  EXPECT_EQ("auto const &&[x, y]{expr};",
+            format("auto  const  &&  [x,y]  {expr};"));
 
   format::FormatStyle Spaces = format::getLLVMStyle();
   Spaces.SpacesInSquareBrackets = true;
@@ -13851,7 +14299,8 @@ TEST_F(FormatTest, FileAndCode) {
   EXPECT_EQ(FormatStyle::LK_ObjC, guessLanguage("foo.m", ""));
   EXPECT_EQ(FormatStyle::LK_ObjC, guessLanguage("foo.mm", ""));
   EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h", ""));
-  EXPECT_EQ(FormatStyle::LK_ObjC, guessLanguage("foo.h", "@interface Foo\n@end\n"));
+  EXPECT_EQ(FormatStyle::LK_ObjC,
+            guessLanguage("foo.h", "@interface Foo\n@end\n"));
   EXPECT_EQ(
       FormatStyle::LK_ObjC,
       guessLanguage("foo.h", "#define TRY(x, y) @try { x; } @finally { y; }"));
@@ -13859,7 +14308,8 @@ TEST_F(FormatTest, FileAndCode) {
             guessLanguage("foo.h", "#define AVAIL(x) @available(x, *))"));
   EXPECT_EQ(FormatStyle::LK_ObjC, guessLanguage("foo.h", "@class Foo;"));
   EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo", ""));
-  EXPECT_EQ(FormatStyle::LK_ObjC, guessLanguage("foo", "@interface Foo\n@end\n"));
+  EXPECT_EQ(FormatStyle::LK_ObjC,
+            guessLanguage("foo", "@interface Foo\n@end\n"));
   EXPECT_EQ(FormatStyle::LK_ObjC,
             guessLanguage("foo.h", "int DoStuff(CGRect rect);\n"));
   EXPECT_EQ(
@@ -13921,36 +14371,36 @@ TEST_F(FormatTest, GuessLanguageWithCaret) {
 }
 
 TEST_F(FormatTest, GuessedLanguageWithInlineAsmClobbers) {
-  EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h",
-                                               "void f() {\n"
-                                               "  asm (\"mov %[e], %[d]\"\n"
-                                               "     : [d] \"=rm\" (d)\n"
-                                               "       [e] \"rm\" (*e));\n"
-                                               "}"));
-  EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h",
-                                               "void f() {\n"
-                                               "  _asm (\"mov %[e], %[d]\"\n"
-                                               "     : [d] \"=rm\" (d)\n"
-                                               "       [e] \"rm\" (*e));\n"
-                                               "}"));
-  EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h",
-                                               "void f() {\n"
-                                               "  __asm (\"mov %[e], %[d]\"\n"
-                                               "     : [d] \"=rm\" (d)\n"
-                                               "       [e] \"rm\" (*e));\n"
-                                               "}"));
-  EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h",
-                                               "void f() {\n"
-                                               "  __asm__ (\"mov %[e], %[d]\"\n"
-                                               "     : [d] \"=rm\" (d)\n"
-                                               "       [e] \"rm\" (*e));\n"
-                                               "}"));
-  EXPECT_EQ(FormatStyle::LK_Cpp, guessLanguage("foo.h",
-                                               "void f() {\n"
-                                               "  asm (\"mov %[e], %[d]\"\n"
-                                               "     : [d] \"=rm\" (d),\n"
-                                               "       [e] \"rm\" (*e));\n"
-                                               "}"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "void f() {\n"
+                                   "  asm (\"mov %[e], %[d]\"\n"
+                                   "     : [d] \"=rm\" (d)\n"
+                                   "       [e] \"rm\" (*e));\n"
+                                   "}"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "void f() {\n"
+                                   "  _asm (\"mov %[e], %[d]\"\n"
+                                   "     : [d] \"=rm\" (d)\n"
+                                   "       [e] \"rm\" (*e));\n"
+                                   "}"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "void f() {\n"
+                                   "  __asm (\"mov %[e], %[d]\"\n"
+                                   "     : [d] \"=rm\" (d)\n"
+                                   "       [e] \"rm\" (*e));\n"
+                                   "}"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "void f() {\n"
+                                   "  __asm__ (\"mov %[e], %[d]\"\n"
+                                   "     : [d] \"=rm\" (d)\n"
+                                   "       [e] \"rm\" (*e));\n"
+                                   "}"));
+  EXPECT_EQ(FormatStyle::LK_Cpp,
+            guessLanguage("foo.h", "void f() {\n"
+                                   "  asm (\"mov %[e], %[d]\"\n"
+                                   "     : [d] \"=rm\" (d),\n"
+                                   "       [e] \"rm\" (*e));\n"
+                                   "}"));
   EXPECT_EQ(FormatStyle::LK_Cpp,
             guessLanguage("foo.h", "void f() {\n"
                                    "  asm volatile (\"mov %[e], %[d]\"\n"
@@ -13981,7 +14431,8 @@ TEST_F(FormatTest, TypenameMacros) {
   verifyFormat("struct foo {\n"
                "  int bar;\n"
                "  TAILQ_ENTRY(a) bleh;\n"
-               "};", Google);
+               "};",
+               Google);
 
   FormatStyle Macros = getLLVMStyle();
   Macros.TypenameMacros = TypenameMacros;
@@ -14001,6 +14452,15 @@ TEST_F(FormatTest, TypenameMacros) {
   verifyFormat("STACK_OF(int*)* a;", Macros);
 }
 
+TEST_F(FormatTest, AmbersandInLamda) {
+  // Test case reported in https://bugs.llvm.org/show_bug.cgi?id=41899
+  FormatStyle AlignStyle = getLLVMStyle();
+  AlignStyle.PointerAlignment = FormatStyle::PAS_Left;
+  verifyFormat("auto lambda = [&a = a]() { a = 2; };", AlignStyle);
+  AlignStyle.PointerAlignment = FormatStyle::PAS_Right;
+  verifyFormat("auto lambda = [&a = a]() { a = 2; };", AlignStyle);
+}
+
 } // end namespace
 } // end namespace format
 } // end namespace clang
diff --git a/clang/unittests/Format/SortImportsTestJava.cpp b/clang/unittests/Format/SortImportsTestJava.cpp
index d2826a2107cda..12869eeb54eff 100644
--- a/clang/unittests/Format/SortImportsTestJava.cpp
+++ b/clang/unittests/Format/SortImportsTestJava.cpp
@@ -285,6 +285,13 @@ TEST_F(SortImportsTestJava, NoReplacementsForValidImports) {
       sortIncludes(FmtStyle, Code, GetCodeRange(Code), "input.java").empty());
 }
 
+TEST_F(SortImportsTestJava, NoReplacementsForValidImportsWindows) {
+  std::string Code = "import org.a;\r\n"
+                     "import org.b;\r\n";
+  EXPECT_TRUE(
+      sortIncludes(FmtStyle, Code, GetCodeRange(Code), "input.java").empty());
+}
+
 } // end namespace
 } // end namespace format
 } // end namespace clang
diff --git a/clang/unittests/Format/SortIncludesTest.cpp b/clang/unittests/Format/SortIncludesTest.cpp
index c00d3cb747d65..0c3a95a10630e 100644
--- a/clang/unittests/Format/SortIncludesTest.cpp
+++ b/clang/unittests/Format/SortIncludesTest.cpp
@@ -70,6 +70,77 @@ TEST_F(SortIncludesTest, BasicSorting) {
                  {tooling::Range(25, 1)}));
 }
 
+TEST_F(SortIncludesTest, SortedIncludesUsingSortPriorityAttribute) {
+  FmtStyle.IncludeStyle.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
+  FmtStyle.IncludeStyle.IncludeCategories = {
+      {"^<sys/param\\.h>", 1, 0},
+      {"^<sys/types\\.h>", 1, 1},
+      {"^<sys.*/", 1, 2},
+      {"^<uvm/", 2, 3},
+      {"^<machine/", 3, 4},
+      {"^<dev/", 4, 5},
+      {"^<net.*/", 5, 6},
+      {"^<protocols/", 5, 7},
+      {"^<(fs|miscfs|msdosfs|nfs|ntfs|ufs)/", 6, 8},
+      {"^<(x86|amd64|i386|xen)/", 7, 8},
+      {"<path", 9, 11},
+      {"^<[^/].*\\.h>", 8, 10},
+      {"^\".*\\.h\"", 10, 12}};
+  EXPECT_EQ("#include <sys/param.h>\n"
+            "#include <sys/types.h>\n"
+            "#include <sys/ioctl.h>\n"
+            "#include <sys/socket.h>\n"
+            "#include <sys/stat.h>\n"
+            "#include <sys/wait.h>\n"
+            "\n"
+            "#include <net/if.h>\n"
+            "#include <net/if_dl.h>\n"
+            "#include <net/route.h>\n"
+            "#include <netinet/in.h>\n"
+            "#include <protocols/rwhod.h>\n"
+            "\n"
+            "#include <assert.h>\n"
+            "#include <errno.h>\n"
+            "#include <inttypes.h>\n"
+            "#include <stdio.h>\n"
+            "#include <stdlib.h>\n"
+            "\n"
+            "#include <paths.h>\n"
+            "\n"
+            "#include \"pathnames.h\"\n",
+            sort("#include <sys/param.h>\n"
+                 "#include <sys/types.h>\n"
+                 "#include <sys/ioctl.h>\n"
+                 "#include <net/if_dl.h>\n"
+                 "#include <net/route.h>\n"
+                 "#include <netinet/in.h>\n"
+                 "#include <sys/socket.h>\n"
+                 "#include <sys/stat.h>\n"
+                 "#include <sys/wait.h>\n"
+                 "#include <net/if.h>\n"
+                 "#include <protocols/rwhod.h>\n"
+                 "#include <assert.h>\n"
+                 "#include <paths.h>\n"
+                 "#include \"pathnames.h\"\n"
+                 "#include <errno.h>\n"
+                 "#include <inttypes.h>\n"
+                 "#include <stdio.h>\n"
+                 "#include <stdlib.h>\n"));
+}
+TEST_F(SortIncludesTest, SortPriorityNotDefined) {
+  FmtStyle = getLLVMStyle();
+  EXPECT_EQ("#include \"FormatTestUtils.h\"\n"
+            "#include \"clang/Format/Format.h\"\n"
+            "#include \"llvm/ADT/None.h\"\n"
+            "#include \"llvm/Support/Debug.h\"\n"
+            "#include \"gtest/gtest.h\"\n",
+            sort("#include \"clang/Format/Format.h\"\n"
+                 "#include \"llvm/ADT/None.h\"\n"
+                 "#include \"FormatTestUtils.h\"\n"
+                 "#include \"gtest/gtest.h\"\n"
+                 "#include \"llvm/Support/Debug.h\"\n"));
+}
+
 TEST_F(SortIncludesTest, NoReplacementsForValidIncludes) {
   // Identical #includes have led to a failure with an unstable sort.
   std::string Code = "#include <a>\n"
@@ -448,7 +519,7 @@ TEST_F(SortIncludesTest, SupportCaseInsensitiveMatching) {
 }
 
 TEST_F(SortIncludesTest, NegativePriorities) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1}, {".*", 1}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
   EXPECT_EQ("#include \"important_os_header.h\"\n"
             "#include \"c_main.h\"\n"
             "#include \"a_other.h\"\n",
@@ -468,7 +539,7 @@ TEST_F(SortIncludesTest, NegativePriorities) {
 }
 
 TEST_F(SortIncludesTest, PriorityGroupsAreSeparatedWhenRegroupping) {
-  Style.IncludeCategories = {{".*important_os_header.*", -1}, {".*", 1}};
+  Style.IncludeCategories = {{".*important_os_header.*", -1, 0}, {".*", 1, 0}};
   Style.IncludeBlocks = tooling::IncludeStyle::IBS_Regroup;
 
   EXPECT_EQ("#include \"important_os_header.h\"\n"
@@ -653,6 +724,14 @@ TEST_F(SortIncludesTest, DoNotOutputReplacementsForSortedBlocksWithRegrouping) {
   EXPECT_EQ(Code, sort(Code, "input.h", 0));
 }
 
+TEST_F(SortIncludesTest,
+       DoNotOutputReplacementsForSortedBlocksWithRegroupingWindows) {
+  Style.IncludeBlocks = Style.IBS_Regroup;
+  std::string Code = "#include \"b.h\"\r\n"
+                     "\r\n"
+                     "#include <a.h>\r\n";
+  EXPECT_EQ(Code, sort(Code, "input.h", 0));
+}
 
 TEST_F(SortIncludesTest, DoNotRegroupGroupsInGoogleObjCStyle) {
   FmtStyle = getGoogleStyle(FormatStyle::LK_ObjC);
diff --git a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp b/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
index 530519573a012..5eb7d256a367a 100644
--- a/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
+++ b/clang/unittests/Lex/DependencyDirectivesSourceMinimizerTest.cpp
@@ -187,7 +187,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest, DefineMultilineArgs) {
                                            "        call((a),      \\\n"
                                            "             (b))",
                                            Out));
-  EXPECT_STREQ("#define MACRO(a,b) call((a),(b))\n", Out.data());
+  EXPECT_STREQ("#define MACRO(a,b) call((a), (b))\n", Out.data());
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest,
@@ -200,7 +200,17 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
                                            "        call((a),      \\\r"
                                            "             (b))",
                                            Out));
-  EXPECT_STREQ("#define MACRO(a,b) call((a),(b))\n", Out.data());
+  EXPECT_STREQ("#define MACRO(a,b) call((a), (b))\n", Out.data());
+}
+
+TEST(MinimizeSourceToDependencyDirectivesTest, DefineMultilineArgsStringize) {
+  SmallVector<char, 128> Out;
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("#define MACRO(a,b) \\\n"
+                                                    "                #a \\\n"
+                                                    "                #b",
+                                                    Out));
+  EXPECT_STREQ("#define MACRO(a,b) #a #b\n", Out.data());
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest,
@@ -213,7 +223,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
                                            "        call((a),      \\\r\n"
                                            "             (b))",
                                            Out));
-  EXPECT_STREQ("#define MACRO(a,b) call((a),(b))\n", Out.data());
+  EXPECT_STREQ("#define MACRO(a,b) call((a), (b))\n", Out.data());
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest,
@@ -226,7 +236,7 @@ TEST(MinimizeSourceToDependencyDirectivesTest,
                                            "        call((a),      \\\n\r"
                                            "             (b))",
                                            Out));
-  EXPECT_STREQ("#define MACRO(a,b) call((a),(b))\n", Out.data());
+  EXPECT_STREQ("#define MACRO(a,b) call((a), (b))\n", Out.data());
 }
 
 TEST(MinimizeSourceToDependencyDirectivesTest, DefineNumber) {
@@ -466,6 +476,17 @@ TEST(MinimizeSourceToDependencyDirectivesTest, SplitIdentifier) {
   EXPECT_STREQ("#define GUA RD\n", Out.data());
 }
 
+TEST(MinimizeSourceToDependencyDirectivesTest,
+     WhitespaceAfterLineContinuationSlash) {
+  SmallVector<char, 128> Out;
+
+  ASSERT_FALSE(minimizeSourceToDependencyDirectives("#define A 1 + \\  \n"
+                                                    "2 + \\\t\n"
+                                                    "3\n",
+                                                    Out));
+  EXPECT_STREQ("#define A 1 + 2 + 3\n", Out.data());
+}
+
 TEST(MinimizeSourceToDependencyDirectivesTest, PoundWarningAndError) {
   SmallVector<char, 128> Out;
 
diff --git a/clang/unittests/Tooling/CMakeLists.txt b/clang/unittests/Tooling/CMakeLists.txt
index 3a4094a8b4051..2b35302c7b1fa 100644
--- a/clang/unittests/Tooling/CMakeLists.txt
+++ b/clang/unittests/Tooling/CMakeLists.txt
@@ -54,6 +54,7 @@ add_clang_unittest(ToolingTests
   RefactoringTest.cpp
   ReplacementsYamlTest.cpp
   RewriterTest.cpp
+  SourceCodeBuildersTest.cpp
   SourceCodeTest.cpp
   StencilTest.cpp
   ToolingTest.cpp
diff --git a/clang/unittests/Tooling/RangeSelectorTest.cpp b/clang/unittests/Tooling/RangeSelectorTest.cpp
index 38c15be00cd06..58ce63cbd750f 100644
--- a/clang/unittests/Tooling/RangeSelectorTest.cpp
+++ b/clang/unittests/Tooling/RangeSelectorTest.cpp
@@ -520,6 +520,35 @@ TEST(RangeSelectorTest, ElementsOpErrors) {
                        Failed<StringError>(withTypeErrorMessage("stmt")));
 }
 
+TEST(RangeSelectorTest, ElseBranchOpSingleStatement) {
+  StringRef Code = R"cc(
+    int f() {
+      int x = 0;
+      if (true) x = 3;
+      else x = 4;
+      return x + 5;
+    }
+  )cc";
+  StringRef ID = "id";
+  TestMatch Match = matchCode(Code, ifStmt().bind(ID));
+  EXPECT_THAT_EXPECTED(select(elseBranch(ID), Match), HasValue("else x = 4;"));
+}
+
+TEST(RangeSelectorTest, ElseBranchOpCompoundStatement) {
+  StringRef Code = R"cc(
+    int f() {
+      int x = 0;
+      if (true) x = 3;
+      else { x = 4; }
+      return x + 5;
+    }
+  )cc";
+  StringRef ID = "id";
+  TestMatch Match = matchCode(Code, ifStmt().bind(ID));
+  EXPECT_THAT_EXPECTED(select(elseBranch(ID), Match),
+                       HasValue("else { x = 4; }"));
+}
+
 // Tests case where the matched node is the complete expanded text.
 TEST(RangeSelectorTest, ExpansionOp) {
   StringRef Code = R"cc(
@@ -546,4 +575,29 @@ TEST(RangeSelectorTest, ExpansionOpPartial) {
                        HasValue("BADDECL(x * x)"));
 }
 
+TEST(RangeSelectorTest, IfBoundOpBound) {
+  StringRef Code = R"cc(
+    int f() {
+      return 3 + 5;
+    }
+  )cc";
+  StringRef ID = "id", Op = "op";
+  TestMatch Match =
+      matchCode(Code, binaryOperator(hasLHS(expr().bind(ID))).bind(Op));
+  EXPECT_THAT_EXPECTED(select(ifBound(ID, node(ID), node(Op)), Match),
+                       HasValue("3"));
+}
+
+TEST(RangeSelectorTest, IfBoundOpUnbound) {
+  StringRef Code = R"cc(
+    int f() {
+      return 3 + 5;
+    }
+  )cc";
+  StringRef ID = "id", Op = "op";
+  TestMatch Match = matchCode(Code, binaryOperator().bind(Op));
+  EXPECT_THAT_EXPECTED(select(ifBound(ID, node(ID), node(Op)), Match),
+                       HasValue("3 + 5"));
+}
+
 } // namespace
diff --git a/clang/unittests/Tooling/SourceCodeBuildersTest.cpp b/clang/unittests/Tooling/SourceCodeBuildersTest.cpp
new file mode 100644
index 0000000000000..2bf50ffad5850
--- /dev/null
+++ b/clang/unittests/Tooling/SourceCodeBuildersTest.cpp
@@ -0,0 +1,230 @@
+//===- unittest/Tooling/SourceCodeBuildersTest.cpp ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/Refactoring/SourceCodeBuilders.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+#include "clang/ASTMatchers/ASTMatchers.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+using namespace tooling;
+using namespace ast_matchers;
+
+namespace {
+using MatchResult = MatchFinder::MatchResult;
+using llvm::ValueIs;
+
+// Create a valid translation unit from a statement.
+static std::string wrapSnippet(StringRef StatementCode) {
+  return ("struct S { S(); S(int); int field; };\n"
+          "S operator+(const S &a, const S &b);\n"
+          "auto test_snippet = []{" +
+          StatementCode + "};")
+      .str();
+}
+
+static DeclarationMatcher wrapMatcher(const StatementMatcher &Matcher) {
+  return varDecl(hasName("test_snippet"),
+                 hasDescendant(compoundStmt(hasAnySubstatement(Matcher))));
+}
+
+struct TestMatch {
+  // The AST unit from which `result` is built. We bundle it because it backs
+  // the result. Users are not expected to access it.
+  std::unique_ptr<ASTUnit> AstUnit;
+  // The result to use in the test. References `ast_unit`.
+  MatchResult Result;
+};
+
+// Matches `Matcher` against the statement `StatementCode` and returns the
+// result. Handles putting the statement inside a function and modifying the
+// matcher correspondingly. `Matcher` should match one of the statements in
+// `StatementCode` exactly -- that is, produce exactly one match. However,
+// `StatementCode` may contain other statements not described by `Matcher`.
+static llvm::Optional<TestMatch> matchStmt(StringRef StatementCode,
+                                           StatementMatcher Matcher) {
+  auto AstUnit = buildASTFromCode(wrapSnippet(StatementCode));
+  if (AstUnit == nullptr) {
+    ADD_FAILURE() << "AST construction failed";
+    return llvm::None;
+  }
+  ASTContext &Context = AstUnit->getASTContext();
+  auto Matches = ast_matchers::match(wrapMatcher(Matcher), Context);
+  // We expect a single, exact match for the statement.
+  if (Matches.size() != 1) {
+    ADD_FAILURE() << "Wrong number of matches: " << Matches.size();
+    return llvm::None;
+  }
+  return TestMatch{std::move(AstUnit), MatchResult(Matches[0], &Context)};
+}
+
+static void testPredicate(bool (*Pred)(const Expr &), StringRef Snippet,
+                          bool Expected) {
+  auto StmtMatch = matchStmt(Snippet, expr().bind("expr"));
+  ASSERT_TRUE(StmtMatch) << "Snippet: " << Snippet;
+  EXPECT_EQ(Expected, Pred(*StmtMatch->Result.Nodes.getNodeAs<Expr>("expr")))
+      << "Snippet: " << Snippet;
+}
+
+// Tests the predicate on the call argument, assuming `Snippet` is a function
+// call.
+static void testPredicateOnArg(bool (*Pred)(const Expr &), StringRef Snippet,
+                               bool Expected) {
+  auto StmtMatch = matchStmt(
+      Snippet, expr(ignoringImplicit(callExpr(hasArgument(
+                   0, ignoringElidableConstructorCall(expr().bind("arg")))))));
+  ASSERT_TRUE(StmtMatch) << "Snippet: " << Snippet;
+  EXPECT_EQ(Expected, Pred(*StmtMatch->Result.Nodes.getNodeAs<Expr>("arg")))
+      << "Snippet: " << Snippet;
+}
+
+TEST(SourceCodeBuildersTest, needParensAfterUnaryOperator) {
+  testPredicate(needParensAfterUnaryOperator, "3 + 5;", true);
+  testPredicate(needParensAfterUnaryOperator, "true ? 3 : 5;", true);
+  testPredicate(needParensAfterUnaryOperator, "S(3) + S(5);", true);
+
+  testPredicate(needParensAfterUnaryOperator, "int x; x;", false);
+  testPredicate(needParensAfterUnaryOperator, "int(3.0);", false);
+  testPredicate(needParensAfterUnaryOperator, "void f(); f();", false);
+  testPredicate(needParensAfterUnaryOperator, "int a[3]; a[0];", false);
+  testPredicate(needParensAfterUnaryOperator, "S x; x.field;", false);
+  testPredicate(needParensAfterUnaryOperator, "int x = 1; --x;", false);
+  testPredicate(needParensAfterUnaryOperator, "int x = 1; -x;", false);
+}
+
+TEST(SourceCodeBuildersTest, needParensAfterUnaryOperatorInImplicitConversion) {
+  // The binary operation will be embedded in various implicit
+  // expressions. Verify they are ignored.
+  testPredicateOnArg(needParensAfterUnaryOperator, "void f(S); f(3 + 5);",
+                     true);
+}
+
+TEST(SourceCodeBuildersTest, mayEverNeedParens) {
+  testPredicate(mayEverNeedParens, "3 + 5;", true);
+  testPredicate(mayEverNeedParens, "true ? 3 : 5;", true);
+  testPredicate(mayEverNeedParens, "int x = 1; --x;", true);
+  testPredicate(mayEverNeedParens, "int x = 1; -x;", true);
+
+  testPredicate(mayEverNeedParens, "int x; x;", false);
+  testPredicate(mayEverNeedParens, "int(3.0);", false);
+  testPredicate(mayEverNeedParens, "void f(); f();", false);
+  testPredicate(mayEverNeedParens, "int a[3]; a[0];", false);
+  testPredicate(mayEverNeedParens, "S x; x.field;", false);
+}
+
+TEST(SourceCodeBuildersTest, mayEverNeedParensInImplictConversion) {
+  // The binary operation will be embedded in various implicit
+  // expressions. Verify they are ignored.
+  testPredicateOnArg(mayEverNeedParens, "void f(S); f(3 + 5);", true);
+}
+
+static void testBuilder(
+    llvm::Optional<std::string> (*Builder)(const Expr &, const ASTContext &),
+    StringRef Snippet, StringRef Expected) {
+  auto StmtMatch = matchStmt(Snippet, expr().bind("expr"));
+  ASSERT_TRUE(StmtMatch);
+  EXPECT_THAT(Builder(*StmtMatch->Result.Nodes.getNodeAs<Expr>("expr"),
+                      *StmtMatch->Result.Context),
+              ValueIs(Expected));
+}
+
+TEST(SourceCodeBuildersTest, BuildParensUnaryOp) {
+  testBuilder(buildParens, "-4;", "(-4)");
+}
+
+TEST(SourceCodeBuildersTest, BuildParensBinOp) {
+  testBuilder(buildParens, "4 + 4;", "(4 + 4)");
+}
+
+TEST(SourceCodeBuildersTest, BuildParensValue) {
+  testBuilder(buildParens, "4;", "4");
+}
+
+TEST(SourceCodeBuildersTest, BuildParensSubscript) {
+  testBuilder(buildParens, "int a[3]; a[0];", "a[0]");
+}
+
+TEST(SourceCodeBuildersTest, BuildParensCall) {
+  testBuilder(buildParens, "int f(int); f(4);", "f(4)");
+}
+
+TEST(SourceCodeBuildersTest, BuildAddressOfValue) {
+  testBuilder(buildAddressOf, "S x; x;", "&x");
+}
+
+TEST(SourceCodeBuildersTest, BuildAddressOfPointerDereference) {
+  testBuilder(buildAddressOf, "S *x; *x;", "x");
+}
+
+TEST(SourceCodeBuildersTest, BuildAddressOfPointerDereferenceIgnoresParens) {
+  testBuilder(buildAddressOf, "S *x; *(x);", "x");
+}
+
+TEST(SourceCodeBuildersTest, BuildAddressOfBinaryOperation) {
+  testBuilder(buildAddressOf, "S x; x + x;", "&(x + x)");
+}
+
+TEST(SourceCodeBuildersTest, BuildDereferencePointer) {
+  testBuilder(buildDereference, "S *x; x;", "*x");
+}
+
+TEST(SourceCodeBuildersTest, BuildDereferenceValueAddress) {
+  testBuilder(buildDereference, "S x; &x;", "x");
+}
+
+TEST(SourceCodeBuildersTest, BuildDereferenceValueAddressIgnoresParens) {
+  testBuilder(buildDereference, "S x; &(x);", "x");
+}
+
+TEST(SourceCodeBuildersTest, BuildDereferenceBinaryOperation) {
+  testBuilder(buildDereference, "S *x; x + 1;", "*(x + 1)");
+}
+
+TEST(SourceCodeBuildersTest, BuildDotValue) {
+  testBuilder(buildDot, "S x; x;", "x.");
+}
+
+TEST(SourceCodeBuildersTest, BuildDotPointerDereference) {
+  testBuilder(buildDot, "S *x; *x;", "x->");
+}
+
+TEST(SourceCodeBuildersTest, BuildDotPointerDereferenceIgnoresParens) {
+  testBuilder(buildDot, "S *x; *(x);", "x->");
+}
+
+TEST(SourceCodeBuildersTest, BuildDotBinaryOperation) {
+  testBuilder(buildDot, "S x; x + x;", "(x + x).");
+}
+
+TEST(SourceCodeBuildersTest, BuildDotPointerDereferenceExprWithParens) {
+  testBuilder(buildDot, "S *x; *(x + 1);", "(x + 1)->");
+}
+
+TEST(SourceCodeBuildersTest, BuildArrowPointer) {
+  testBuilder(buildArrow, "S *x; x;", "x->");
+}
+
+TEST(SourceCodeBuildersTest, BuildArrowValueAddress) {
+  testBuilder(buildArrow, "S x; &x;", "x.");
+}
+
+TEST(SourceCodeBuildersTest, BuildArrowValueAddressIgnoresParens) {
+  testBuilder(buildArrow, "S x; &(x);", "x.");
+}
+
+TEST(SourceCodeBuildersTest, BuildArrowBinaryOperation) {
+  testBuilder(buildArrow, "S *x; x + 1;", "(x + 1)->");
+}
+
+TEST(SourceCodeBuildersTest, BuildArrowValueAddressWithParens) {
+  testBuilder(buildArrow, "S x; &(true ? x : x);", "(true ? x : x).");
+}
+} // namespace
diff --git a/clang/unittests/Tooling/StencilTest.cpp b/clang/unittests/Tooling/StencilTest.cpp
index e5063e15af3e2..2202693f023db 100644
--- a/clang/unittests/Tooling/StencilTest.cpp
+++ b/clang/unittests/Tooling/StencilTest.cpp
@@ -10,6 +10,8 @@
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Tooling/FixIt.h"
 #include "clang/Tooling/Tooling.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Testing/Support/Error.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
@@ -18,34 +20,23 @@ using namespace tooling;
 using namespace ast_matchers;
 
 namespace {
+using ::llvm::HasValue;
 using ::testing::AllOf;
 using ::testing::Eq;
 using ::testing::HasSubstr;
 using MatchResult = MatchFinder::MatchResult;
+using stencil::access;
 using stencil::cat;
 using stencil::dPrint;
+using stencil::ifBound;
+using stencil::run;
 using stencil::text;
 
-// In tests, we can't directly match on llvm::Expected since its accessors
-// mutate the object. So, we collapse it to an Optional.
-static llvm::Optional<std::string> toOptional(llvm::Expected<std::string> V) {
-  if (V)
-    return *V;
-  ADD_FAILURE() << "Losing error in conversion to IsSomething: "
-                << llvm::toString(V.takeError());
-  return llvm::None;
-}
-
-// A very simple matcher for llvm::Optional values.
-MATCHER_P(IsSomething, ValueMatcher, "") {
-  if (!arg)
-    return false;
-  return ::testing::ExplainMatchResult(ValueMatcher, *arg, result_listener);
-}
-
 // Create a valid translation-unit from a statement.
-static std::string wrapSnippet(llvm::Twine StatementCode) {
-  return ("auto stencil_test_snippet = []{" + StatementCode + "};").str();
+static std::string wrapSnippet(StringRef StatementCode) {
+  return ("struct S { int field; }; auto stencil_test_snippet = []{" +
+          StatementCode + "};")
+      .str();
 }
 
 static DeclarationMatcher wrapMatcher(const StatementMatcher &Matcher) {
@@ -63,9 +54,10 @@ struct TestMatch {
 
 // Matches `Matcher` against the statement `StatementCode` and returns the
 // result. Handles putting the statement inside a function and modifying the
-// matcher correspondingly. `Matcher` should match `StatementCode` exactly --
-// that is, produce exactly one match.
-static llvm::Optional<TestMatch> matchStmt(llvm::Twine StatementCode,
+// matcher correspondingly. `Matcher` should match one of the statements in
+// `StatementCode` exactly -- that is, produce exactly one match. However,
+// `StatementCode` may contain other statements not described by `Matcher`.
+static llvm::Optional<TestMatch> matchStmt(StringRef StatementCode,
                                            StatementMatcher Matcher) {
   auto AstUnit = buildASTFromCode(wrapSnippet(StatementCode));
   if (AstUnit == nullptr) {
@@ -123,7 +115,7 @@ class StencilTest : public ::testing::Test {
 
   // Tests failures caused by references to unbound nodes. `unbound_id` is the
   // id that will cause the failure.
-  void testUnboundNodeError(const Stencil &Stencil, llvm::StringRef UnboundId) {
+  void testUnboundNodeError(const Stencil &Stencil, StringRef UnboundId) {
     testError(Stencil, AllOf(HasSubstr(UnboundId), HasSubstr("not bound")));
   }
 };
@@ -143,8 +135,8 @@ TEST_F(StencilTest, SingleStatement) {
   // Invert the if-then-else.
   auto Stencil = cat("if (!", node(Condition), ") ", statement(Else), " else ",
                      statement(Then));
-  EXPECT_THAT(toOptional(Stencil.eval(StmtMatch->Result)),
-              IsSomething(Eq("if (!true) return 0; else return 1;")));
+  EXPECT_THAT_EXPECTED(Stencil.eval(StmtMatch->Result),
+                       HasValue("if (!true) return 0; else return 1;"));
 }
 
 TEST_F(StencilTest, SingleStatementCallOperator) {
@@ -162,8 +154,8 @@ TEST_F(StencilTest, SingleStatementCallOperator) {
   // Invert the if-then-else.
   Stencil S = cat("if (!", node(Condition), ") ", statement(Else), " else ",
                   statement(Then));
-  EXPECT_THAT(toOptional(S(StmtMatch->Result)),
-              IsSomething(Eq("if (!true) return 0; else return 1;")));
+  EXPECT_THAT_EXPECTED(S(StmtMatch->Result),
+                       HasValue("if (!true) return 0; else return 1;"));
 }
 
 TEST_F(StencilTest, UnboundNode) {
@@ -188,8 +180,7 @@ void testExpr(StringRef Id, StringRef Snippet, const Stencil &Stencil,
               StringRef Expected) {
   auto StmtMatch = matchStmt(Snippet, expr().bind(Id));
   ASSERT_TRUE(StmtMatch);
-  EXPECT_THAT(toOptional(Stencil.eval(StmtMatch->Result)),
-              IsSomething(Expected));
+  EXPECT_THAT_EXPECTED(Stencil.eval(StmtMatch->Result), HasValue(Expected));
 }
 
 TEST_F(StencilTest, SelectionOp) {
@@ -197,6 +188,111 @@ TEST_F(StencilTest, SelectionOp) {
   testExpr(Id, "3;", cat(node(Id)), "3");
 }
 
+TEST_F(StencilTest, IfBoundOpBound) {
+  StringRef Id = "id";
+  testExpr(Id, "3;", cat(ifBound(Id, text("5"), text("7"))), "5");
+}
+
+TEST_F(StencilTest, IfBoundOpUnbound) {
+  StringRef Id = "id";
+  testExpr(Id, "3;", cat(ifBound("other", text("5"), text("7"))), "7");
+}
+
+TEST_F(StencilTest, AccessOpValue) {
+  StringRef Snippet = R"cc(
+    S x;
+    x;
+  )cc";
+  StringRef Id = "id";
+  testExpr(Id, Snippet, cat(access(Id, "field")), "x.field");
+}
+
+TEST_F(StencilTest, AccessOpValueExplicitText) {
+  StringRef Snippet = R"cc(
+    S x;
+    x;
+  )cc";
+  StringRef Id = "id";
+  testExpr(Id, Snippet, cat(access(Id, text("field"))), "x.field");
+}
+
+TEST_F(StencilTest, AccessOpValueAddress) {
+  StringRef Snippet = R"cc(
+    S x;
+    &x;
+  )cc";
+  StringRef Id = "id";
+  testExpr(Id, Snippet, cat(access(Id, "field")), "x.field");
+}
+
+TEST_F(StencilTest, AccessOpPointer) {
+  StringRef Snippet = R"cc(
+    S *x;
+    x;
+  )cc";
+  StringRef Id = "id";
+  testExpr(Id, Snippet, cat(access(Id, "field")), "x->field");
+}
+
+TEST_F(StencilTest, AccessOpPointerDereference) {
+  StringRef Snippet = R"cc(
+    S *x;
+    *x;
+  )cc";
+  StringRef Id = "id";
+  testExpr(Id, Snippet, cat(access(Id, "field")), "x->field");
+}
+
+TEST_F(StencilTest, AccessOpExplicitThis) {
+  using clang::ast_matchers::hasObjectExpression;
+  using clang::ast_matchers::memberExpr;
+
+  // Set up the code so we can bind to a use of this.
+  StringRef Snippet = R"cc(
+    class C {
+     public:
+      int x;
+      int foo() { return this->x; }
+    };
+  )cc";
+  auto StmtMatch =
+      matchStmt(Snippet, returnStmt(hasReturnValue(ignoringImplicit(memberExpr(
+                             hasObjectExpression(expr().bind("obj")))))));
+  ASSERT_TRUE(StmtMatch);
+  const Stencil Stencil = cat(access("obj", "field"));
+  EXPECT_THAT_EXPECTED(Stencil.eval(StmtMatch->Result),
+                       HasValue("this->field"));
+}
+
+TEST_F(StencilTest, AccessOpImplicitThis) {
+  using clang::ast_matchers::hasObjectExpression;
+  using clang::ast_matchers::memberExpr;
+
+  // Set up the code so we can bind to a use of (implicit) this.
+  StringRef Snippet = R"cc(
+    class C {
+     public:
+      int x;
+      int foo() { return x; }
+    };
+  )cc";
+  auto StmtMatch =
+      matchStmt(Snippet, returnStmt(hasReturnValue(ignoringImplicit(memberExpr(
+                             hasObjectExpression(expr().bind("obj")))))));
+  ASSERT_TRUE(StmtMatch);
+  const Stencil Stencil = cat(access("obj", "field"));
+  EXPECT_THAT_EXPECTED(Stencil.eval(StmtMatch->Result), HasValue("field"));
+}
+
+TEST_F(StencilTest, RunOp) {
+  StringRef Id = "id";
+  auto SimpleFn = [Id](const MatchResult &R) {
+    return std::string(R.Nodes.getNodeAs<Stmt>(Id) != nullptr ? "Bound"
+                                                              : "Unbound");
+  };
+  testExpr(Id, "3;", cat(run(SimpleFn)), "Bound");
+}
+
 TEST(StencilEqualityTest, Equality) {
   auto Lhs = cat("foo", dPrint("dprint_id"));
   auto Rhs = cat("foo", dPrint("dprint_id"));
@@ -221,4 +317,12 @@ TEST(StencilEqualityTest, InEqualitySelection) {
   auto S2 = cat(node("node"));
   EXPECT_NE(S1, S2);
 }
+
+// `run` is opaque.
+TEST(StencilEqualityTest, InEqualityRun) {
+  auto F = [](const MatchResult &R) { return "foo"; };
+  auto S1 = cat(run(F));
+  auto S2 = cat(run(F));
+  EXPECT_NE(S1, S2);
+}
 } // namespace
diff --git a/clang/unittests/Tooling/TransformerTest.cpp b/clang/unittests/Tooling/TransformerTest.cpp
index 554e586c3c484..5d55182f8273b 100644
--- a/clang/unittests/Tooling/TransformerTest.cpp
+++ b/clang/unittests/Tooling/TransformerTest.cpp
@@ -710,6 +710,57 @@ TEST_F(TransformerTest, IdentityMacro) {
   testRule(ruleStrlenSize(), Input, Expected);
 }
 
+// Tests that two changes in a single macro expansion do not lead to conflicts
+// in applying the changes.
+TEST_F(TransformerTest, TwoChangesInOneMacroExpansion) {
+  std::string Input = R"cc(
+#define PLUS(a,b) (a) + (b)
+    int f() { return PLUS(3, 4); }
+  )cc";
+  std::string Expected = R"cc(
+#define PLUS(a,b) (a) + (b)
+    int f() { return PLUS(LIT, LIT); }
+  )cc";
+
+  testRule(makeRule(integerLiteral(), change(text("LIT"))), Input, Expected);
+}
+
+// Tests case where the rule's match spans both source from the macro and its
+// arg, with the begin location (the "anchor") being the arg.
+TEST_F(TransformerTest, MatchSpansMacroTextButChangeDoesNot) {
+  std::string Input = R"cc(
+#define PLUS_ONE(a) a + 1
+    int f() { return PLUS_ONE(3); }
+  )cc";
+  std::string Expected = R"cc(
+#define PLUS_ONE(a) a + 1
+    int f() { return PLUS_ONE(LIT); }
+  )cc";
+
+  StringRef E = "expr";
+  testRule(makeRule(binaryOperator(hasLHS(expr().bind(E))),
+                    change(node(E), text("LIT"))),
+           Input, Expected);
+}
+
+// Tests case where the rule's match spans both source from the macro and its
+// arg, with the begin location (the "anchor") being inside the macro.
+TEST_F(TransformerTest, MatchSpansMacroTextButChangeDoesNotAnchoredInMacro) {
+  std::string Input = R"cc(
+#define PLUS_ONE(a) 1 + a
+    int f() { return PLUS_ONE(3); }
+  )cc";
+  std::string Expected = R"cc(
+#define PLUS_ONE(a) 1 + a
+    int f() { return PLUS_ONE(LIT); }
+  )cc";
+
+  StringRef E = "expr";
+  testRule(makeRule(binaryOperator(hasRHS(expr().bind(E))),
+                    change(node(E), text("LIT"))),
+           Input, Expected);
+}
+
 // No rewrite is applied when the changed text does not encompass the entirety
 // of the expanded text. That is, the edit would have to be applied to the
 // macro's definition to succeed and editing the expansion point would not
diff --git a/clang/utils/TableGen/CMakeLists.txt b/clang/utils/TableGen/CMakeLists.txt
index d33ede2a75a29..96b97a7483833 100644
--- a/clang/utils/TableGen/CMakeLists.txt
+++ b/clang/utils/TableGen/CMakeLists.txt
@@ -12,6 +12,7 @@ add_tablegen(clang-tblgen CLANG
   ClangOpenCLBuiltinEmitter.cpp
   ClangOptionDocEmitter.cpp
   ClangSACheckersEmitter.cpp
+  ClangTypeNodesEmitter.cpp
   NeonEmitter.cpp
   TableGen.cpp
   )
diff --git a/clang/utils/TableGen/ClangASTNodesEmitter.cpp b/clang/utils/TableGen/ClangASTNodesEmitter.cpp
index a0bbdbab33542..3ece9be6a5c77 100644
--- a/clang/utils/TableGen/ClangASTNodesEmitter.cpp
+++ b/clang/utils/TableGen/ClangASTNodesEmitter.cpp
@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
+
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 #include <cctype>
@@ -173,15 +175,14 @@ void ClangASTNodesEmitter::run(raw_ostream &OS) {
   OS << "#undef ABSTRACT_" << macroName(Root.getName()) << "\n";
 }
 
-namespace clang {
-void EmitClangASTNodes(RecordKeeper &RK, raw_ostream &OS,
-                       const std::string &N, const std::string &S) {
+void clang::EmitClangASTNodes(RecordKeeper &RK, raw_ostream &OS,
+                              const std::string &N, const std::string &S) {
   ClangASTNodesEmitter(RK, N, S).run(OS);
 }
 
 // Emits and addendum to a .inc file to enumerate the clang declaration
 // contexts.
-void EmitClangDeclContext(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangDeclContext(RecordKeeper &Records, raw_ostream &OS) {
   // FIXME: Find a .td file format to allow for this to be represented better.
 
   emitSourceFileHeader("List of AST Decl nodes", OS);
@@ -225,4 +226,3 @@ void EmitClangDeclContext(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#undef DECL_CONTEXT\n";
   OS << "#undef DECL_CONTEXT_BASE\n";
 }
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index 1020492bc2a89..0d92a321c7475 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
+
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -1596,6 +1598,13 @@ CreateSemanticSpellings(const std::vector<FlattenedSpelling> &Spellings,
   std::string Ret("  enum Spelling {\n");
   std::set<std::string> Uniques;
   unsigned Idx = 0;
+
+  // If we have a need to have this many spellings we likely need to add an
+  // extra bit to the SpellingIndex in AttributeCommonInfo, then increase the
+  // value of SpellingNotCalculated there and here.
+  assert(Spellings.size() < 15 &&
+         "Too many spellings, would step on SpellingNotCalculated in "
+         "AttributeCommonInfo");
   for (auto I = Spellings.begin(), E = Spellings.end(); I != E; ++I, ++Idx) {
     const FlattenedSpelling &S = *I;
     const std::string &Variety = S.variety();
@@ -1629,6 +1638,7 @@ CreateSemanticSpellings(const std::vector<FlattenedSpelling> &Spellings,
     // enumerator.
     Ret += "    " + EnumName + " = " + llvm::utostr(Idx);
   }
+  Ret += ",\n  SpellingNotCalculated = 15\n";
   Ret += "\n  };\n\n";
   return Ret;
 }
@@ -2211,10 +2221,8 @@ static void emitClangAttrThisIsaIdentifierArgList(RecordKeeper &Records,
   OS << "#endif // CLANG_ATTR_THIS_ISA_IDENTIFIER_ARG_LIST\n\n";
 }
 
-namespace clang {
-
 // Emits the class definitions for attributes.
-void EmitClangAttrClass(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangAttrClass(RecordKeeper &Records, raw_ostream &OS) {
   emitSourceFileHeader("Attribute classes' definitions", OS);
 
   OS << "#ifndef LLVM_CLANG_ATTR_CLASSES_INC\n";
@@ -2341,7 +2349,7 @@ void EmitClangAttrClass(RecordKeeper &Records, raw_ostream &OS) {
       OS << ", SourceRange Range, AttributeCommonInfo::Syntax Syntax";
       if (!ElideSpelling)
         OS << ", " << R.getName()
-           << "Attr::Spelling Spelling = "
+           << "Attr::Spelling S = "
               "static_cast<Spelling>(SpellingNotCalculated)";
       OS << ") {\n";
       OS << "    AttributeCommonInfo I(Range, ";
@@ -2353,7 +2361,7 @@ void EmitClangAttrClass(RecordKeeper &Records, raw_ostream &OS) {
 
       OS << ", Syntax";
       if (!ElideSpelling)
-        OS << ", Spelling";
+        OS << ", S";
       OS << ");\n";
       OS << "    return Create";
       if (Implicit)
@@ -2483,7 +2491,7 @@ void EmitClangAttrClass(RecordKeeper &Records, raw_ostream &OS) {
 }
 
 // Emits the class method definitions for attributes.
-void EmitClangAttrImpl(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangAttrImpl(RecordKeeper &Records, raw_ostream &OS) {
   emitSourceFileHeader("Attribute classes' member function definitions", OS);
 
   std::vector<Record*> Attrs = Records.getAllDerivedDefinitions("Attr");
@@ -2548,8 +2556,6 @@ void EmitClangAttrImpl(RecordKeeper &Records, raw_ostream &OS) {
   EmitFunc("printPretty(OS, Policy)");
 }
 
-} // end namespace clang
-
 static void emitAttrList(raw_ostream &OS, StringRef Class,
                          const std::vector<Record*> &AttrList) {
   for (auto Cur : AttrList) {
diff --git a/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp b/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
index c0dd70281a4d9..fc79d59713d69 100644
--- a/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
+++ b/clang/utils/TableGen/ClangCommentCommandInfoEmitter.cpp
@@ -11,6 +11,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
+
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/StringMatcher.h"
 #include "llvm/TableGen/TableGenBackend.h"
@@ -18,8 +20,7 @@
 
 using namespace llvm;
 
-namespace clang {
-void EmitClangCommentCommandInfo(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangCommentCommandInfo(RecordKeeper &Records, raw_ostream &OS) {
   emitSourceFileHeader("A list of commands useable in documentation "
                        "comments", OS);
 
@@ -105,7 +106,7 @@ static std::string MangleName(StringRef Str) {
   return Mangled;
 }
 
-void EmitClangCommentCommandList(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangCommentCommandList(RecordKeeper &Records, raw_ostream &OS) {
   emitSourceFileHeader("A list of commands useable in documentation "
                        "comments", OS);
 
@@ -121,4 +122,3 @@ void EmitClangCommentCommandList(RecordKeeper &Records, raw_ostream &OS) {
     OS << "COMMENT_COMMAND(" << MangledName << ")\n";
   }
 }
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp b/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
index 81af5b4b95f19..ed3f4bd6ef6c1 100644
--- a/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
+++ b/clang/utils/TableGen/ClangCommentHTMLNamedCharacterReferenceEmitter.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/TableGen/Error.h"
@@ -45,9 +46,8 @@ static bool translateCodePointToUTF8(unsigned CodePoint,
   return true;
 }
 
-namespace clang {
-void EmitClangCommentHTMLNamedCharacterReferences(RecordKeeper &Records,
-                                                  raw_ostream &OS) {
+void clang::EmitClangCommentHTMLNamedCharacterReferences(RecordKeeper &Records,
+                                                         raw_ostream &OS) {
   std::vector<Record *> Tags = Records.getAllDerivedDefinitions("NCR");
   std::vector<StringMatcher::StringPair> NameToUTF8;
   SmallString<32> CLiteral;
@@ -79,6 +79,3 @@ void EmitClangCommentHTMLNamedCharacterReferences(RecordKeeper &Records,
   OS << "  return StringRef();\n"
      << "}\n\n";
 }
-
-} // end namespace clang
-
diff --git a/clang/utils/TableGen/ClangDataCollectorsEmitter.cpp b/clang/utils/TableGen/ClangDataCollectorsEmitter.cpp
index 4079efc808231..45082935c1f79 100644
--- a/clang/utils/TableGen/ClangDataCollectorsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDataCollectorsEmitter.cpp
@@ -1,10 +1,10 @@
+#include "TableGenBackends.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
 
 using namespace llvm;
 
-namespace clang {
-void EmitClangDataCollectors(RecordKeeper &RK, raw_ostream &OS) {
+void clang::EmitClangDataCollectors(RecordKeeper &RK, raw_ostream &OS) {
   const auto &Defs = RK.getClasses();
   for (const auto &Entry : Defs) {
     Record &R = *Entry.second;
@@ -15,4 +15,3 @@ void EmitClangDataCollectors(RecordKeeper &RK, raw_ostream &OS) {
   }
   OS << "#undef DEF_ADD_DATA\n";
 }
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index 13e564e130ce4..778375010041d 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/PointerUnion.h"
@@ -1187,9 +1188,8 @@ static bool isRemark(const Record &Diag) {
 
 /// ClangDiagsDefsEmitter - The top-level class emits .def files containing
 /// declarations of Clang diagnostics.
-namespace clang {
-void EmitClangDiagsDefs(RecordKeeper &Records, raw_ostream &OS,
-                        const std::string &Component) {
+void clang::EmitClangDiagsDefs(RecordKeeper &Records, raw_ostream &OS,
+                               const std::string &Component) {
   // Write the #if guard
   if (!Component.empty()) {
     std::string ComponentName = StringRef(Component).upper();
@@ -1288,7 +1288,6 @@ void EmitClangDiagsDefs(RecordKeeper &Records, raw_ostream &OS,
     OS << ")\n";
   }
 }
-} // end namespace clang
 
 //===----------------------------------------------------------------------===//
 // Warning Group Tables generation
@@ -1528,8 +1527,7 @@ static void emitCategoryTable(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#endif // GET_CATEGORY_TABLE\n\n";
 }
 
-namespace clang {
-void EmitClangDiagGroups(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangDiagGroups(RecordKeeper &Records, raw_ostream &OS) {
   // Compute a mapping from a DiagGroup to all of its parents.
   DiagGroupParentMap DGParentMap(Records);
 
@@ -1565,7 +1563,6 @@ void EmitClangDiagGroups(RecordKeeper &Records, raw_ostream &OS) {
                 OS);
   emitCategoryTable(Records, OS);
 }
-} // end namespace clang
 
 //===----------------------------------------------------------------------===//
 // Diagnostic name index generation
@@ -1582,8 +1579,7 @@ struct RecordIndexElement
 };
 } // end anonymous namespace.
 
-namespace clang {
-void EmitClangDiagsIndexName(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangDiagsIndexName(RecordKeeper &Records, raw_ostream &OS) {
   const std::vector<Record*> &Diags =
     Records.getAllDerivedDefinitions("Diagnostic");
 
@@ -1673,7 +1669,7 @@ void writeDiagnosticText(DiagnosticTextBuilder &Builder, const Record *R,
 }  // namespace
 }  // namespace docs
 
-void EmitClangDiagDocs(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangDiagDocs(RecordKeeper &Records, raw_ostream &OS) {
   using namespace docs;
 
   // Get the documentation introduction paragraph.
@@ -1792,5 +1788,3 @@ void EmitClangDiagDocs(RecordKeeper &Records, raw_ostream &OS) {
     OS << "\n";
   }
 }
-
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangOpcodesEmitter.cpp b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
index 1ff20f0f44943..e5bfac5ba1b6f 100644
--- a/clang/utils/TableGen/ClangOpcodesEmitter.cpp
+++ b/clang/utils/TableGen/ClangOpcodesEmitter.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/StringMatcher.h"
@@ -351,10 +352,6 @@ void ClangOpcodesEmitter::PrintTypes(raw_ostream &OS, ArrayRef<Record *> Types)
   OS << ">";
 }
 
-namespace clang {
-
-void EmitClangOpcodes(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangOpcodes(RecordKeeper &Records, raw_ostream &OS) {
   ClangOpcodesEmitter(Records).run(OS);
 }
-
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp b/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
index b93a68fb30946..c8975d7bf615d 100644
--- a/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
+++ b/clang/utils/TableGen/ClangOpenCLBuiltinEmitter.cpp
@@ -50,6 +50,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
@@ -121,11 +122,13 @@ class BuiltinNameEmitter {
   // Emit the BuiltinTable table. This table contains all the overloads of
   // each function, and is a struct OpenCLBuiltinDecl.
   // E.g.:
-  // // convert_float2_rtn
-  //   { 58, 2 },
+  // // 891 convert_float2_rtn
+  //   { 58, 2, 100, 0 },
   // This means that the signature of this convert_float2_rtn overload has
   // 1 argument (+1 for the return type), stored at index 58 in
-  // the SignatureTable.
+  // the SignatureTable.  The last two values represent the minimum (1.0) and
+  // maximum (0, meaning no max version) OpenCL version in which this overload
+  // is supported.
   void EmitBuiltinTable();
 
   // Emit a StringMatcher function to check whether a function name is an
@@ -268,6 +271,10 @@ struct OpenCLBuiltinStruct {
   // the SignatureTable represent the complete signature.  The first type at
   // index SigTableIndex is the return type.
   const unsigned NumTypes;
+  // First OpenCL version in which this overload was introduced (e.g. CL20).
+  const unsigned short MinVersion;
+  // First OpenCL version in which this overload was removed (e.g. CL20).
+  const unsigned short MaxVersion;
 };
 
 )";
@@ -400,11 +407,13 @@ void BuiltinNameEmitter::EmitBuiltinTable() {
     OS << "  // " << (Index + 1) << ": " << FOM.first << "\n";
 
     for (const auto &Overload : FOM.second) {
-      OS << "  { "
-         << Overload.second << ", "
-         << Overload.first->getValueAsListOfDefs("Signature").size()
+      OS << "  { " << Overload.second << ", "
+         << Overload.first->getValueAsListOfDefs("Signature").size() << ", "
+         << Overload.first->getValueAsDef("MinVersion")->getValueAsInt("ID")
+         << ", "
+         << Overload.first->getValueAsDef("MaxVersion")->getValueAsInt("ID")
          << " },\n";
-         Index++;
+      Index++;
     }
   }
   OS << "};\n\n";
@@ -631,11 +640,7 @@ static void OCL2Qual(ASTContext &Context, const OpenCLTypeStruct &Ty,
   OS << "\n} // OCL2Qual\n";
 }
 
-namespace clang {
-
-void EmitClangOpenCLBuiltins(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangOpenCLBuiltins(RecordKeeper &Records, raw_ostream &OS) {
   BuiltinNameEmitter NameChecker(Records, OS);
   NameChecker.Emit();
 }
-
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangOptionDocEmitter.cpp b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
index 7027113c4fa81..b944ad9608f5e 100644
--- a/clang/utils/TableGen/ClangOptionDocEmitter.cpp
+++ b/clang/utils/TableGen/ClangOptionDocEmitter.cpp
@@ -8,6 +8,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
@@ -21,8 +22,6 @@
 
 using namespace llvm;
 
-namespace clang {
-namespace docs {
 namespace {
 struct DocumentedOption {
   Record *Option;
@@ -380,11 +379,8 @@ void emitDocumentation(int Depth, const Documentation &Doc,
 }
 
 }  // namespace
-}  // namespace docs
-
-void EmitClangOptDocs(RecordKeeper &Records, raw_ostream &OS) {
-  using namespace docs;
 
+void clang::EmitClangOptDocs(RecordKeeper &Records, raw_ostream &OS) {
   const Record *DocInfo = Records.getDef("GlobalDocumentation");
   if (!DocInfo) {
     PrintFatalError("The GlobalDocumentation top-level definition is missing, "
@@ -396,4 +392,3 @@ void EmitClangOptDocs(RecordKeeper &Records, raw_ostream &OS) {
 
   emitDocumentation(0, extractDocumentation(Records), DocInfo, OS);
 }
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangSACheckersEmitter.cpp b/clang/utils/TableGen/ClangSACheckersEmitter.cpp
index 7dd0895b76d45..feefbeb411387 100644
--- a/clang/utils/TableGen/ClangSACheckersEmitter.cpp
+++ b/clang/utils/TableGen/ClangSACheckersEmitter.cpp
@@ -10,6 +10,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
@@ -174,8 +175,7 @@ static void printOption(llvm::raw_ostream &OS, StringRef FullName,
     OS << "true";
 }
 
-namespace clang {
-void EmitClangSACheckers(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitClangSACheckers(RecordKeeper &Records, raw_ostream &OS) {
   std::vector<Record*> checkers = Records.getAllDerivedDefinitions("Checker");
   std::vector<Record*> packages = Records.getAllDerivedDefinitions("Package");
 
@@ -315,4 +315,3 @@ void EmitClangSACheckers(RecordKeeper &Records, raw_ostream &OS) {
   OS << "#endif // GET_CHECKER_OPTIONS\n"
         "\n";
 }
-} // end namespace clang
diff --git a/clang/utils/TableGen/ClangTypeNodesEmitter.cpp b/clang/utils/TableGen/ClangTypeNodesEmitter.cpp
new file mode 100644
index 0000000000000..c9986c8fa496b
--- /dev/null
+++ b/clang/utils/TableGen/ClangTypeNodesEmitter.cpp
@@ -0,0 +1,220 @@
+//=== ClangTypeNodesEmitter.cpp - Generate type node tables -----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This tblgen backend emits the node table (the .def file) for Clang
+// type nodes.
+//
+// This file defines the AST type info database. Each type node is
+// enumerated by providing its name (e.g., "Builtin" or "Enum") and
+// base class (e.g., "Type" or "TagType"). Depending on where in the
+// abstract syntax tree the type will show up, the enumeration uses
+// one of five different macros:
+//
+//    TYPE(Class, Base) - A type that can show up anywhere in the AST,
+//    and might be dependent, canonical, or non-canonical. All clients
+//    will need to understand these types.
+//
+//    ABSTRACT_TYPE(Class, Base) - An abstract class that shows up in
+//    the type hierarchy but has no concrete instances.
+//
+//    NON_CANONICAL_TYPE(Class, Base) - A type that can show up
+//    anywhere in the AST but will never be a part of a canonical
+//    type. Clients that only need to deal with canonical types
+//    (ignoring, e.g., typedefs and other type aliases used for
+//    pretty-printing) can ignore these types.
+//
+//    DEPENDENT_TYPE(Class, Base) - A type that will only show up
+//    within a C++ template that has not been instantiated, e.g., a
+//    type that is always dependent. Clients that do not need to deal
+//    with uninstantiated C++ templates can ignore these types.
+//
+//    NON_CANONICAL_UNLESS_DEPENDENT_TYPE(Class, Base) - A type that
+//    is non-canonical unless it is dependent.  Defaults to TYPE because
+//    it is neither reliably dependent nor reliably non-canonical.
+//
+// There is a sixth macro, independent of the others.  Most clients
+// will not need to use it.
+//
+//    LEAF_TYPE(Class) - A type that never has inner types.  Clients
+//    which can operate on such types more efficiently may wish to do so.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Record.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include <set>
+#include <string>
+#include <vector>
+#include "TableGenBackends.h"
+
+using namespace llvm;
+
+// These are spellings in the generated output.
+#define TypeMacroName "TYPE"
+#define AbstractTypeMacroName "ABSTRACT_TYPE"
+#define DependentTypeMacroName "DEPENDENT_TYPE"
+#define NonCanonicalTypeMacroName "NON_CANONICAL_TYPE"
+#define NonCanonicalUnlessDependentTypeMacroName "NON_CANONICAL_UNLESS_DEPENDENT_TYPE"
+#define TypeMacroArgs "(Class, Base)"
+#define LastTypeMacroName "LAST_TYPE"
+#define LeafTypeMacroName "LEAF_TYPE"
+
+// These are spellings in the tblgen file.
+// (Type is also used for the spelling of the AST class.)
+#define TypeClassName "Type"
+#define DerivedTypeClassName "DerivedType"
+#define AlwaysDependentClassName "AlwaysDependent"
+#define NeverCanonicalClassName "NeverCanonical"
+#define NeverCanonicalUnlessDependentClassName "NeverCanonicalUnlessDependent"
+#define LeafTypeClassName "LeafType"
+#define AbstractFieldName "Abstract"
+#define BaseFieldName "Base"
+
+static StringRef getIdForType(Record *type) {
+	// The record name is expected to be the full C++ class name,
+	// including "Type".  Check for that and strip it off.
+	auto fullName = type->getName();
+	if (!fullName.endswith("Type"))
+		PrintFatalError(type->getLoc(), "name of Type node doesn't end in Type");
+	return fullName.drop_back(4);
+}
+
+namespace {
+class TypeNodeEmitter {
+	RecordKeeper &Records;
+	raw_ostream &Out;
+	const std::vector<Record*> Types;
+	std::vector<StringRef> MacrosToUndef;
+
+public:
+	TypeNodeEmitter(RecordKeeper &records, raw_ostream &out)
+		: Records(records), Out(out),
+			Types(Records.getAllDerivedDefinitions("Type")) {
+	}
+
+	void emit();
+
+private:
+	void emitFallbackDefine(StringRef macroName, StringRef fallbackMacroName,
+													StringRef args);
+
+	void emitNodeInvocations();
+	void emitLastNodeInvocation();
+	void emitLeafNodeInvocations();
+
+	void addMacroToUndef(StringRef macroName);
+	void emitUndefs();
+};
+}
+
+void TypeNodeEmitter::emit() {
+	if (Types.empty())
+		PrintFatalError("no Type records in input!");
+
+	emitSourceFileHeader("An x-macro database of Clang type nodes", Out);
+
+	// Preamble
+	addMacroToUndef(TypeMacroName);
+	addMacroToUndef(AbstractTypeMacroName);
+	emitFallbackDefine(AbstractTypeMacroName, TypeMacroName, TypeMacroArgs);
+	emitFallbackDefine(NonCanonicalTypeMacroName, TypeMacroName, TypeMacroArgs);
+	emitFallbackDefine(DependentTypeMacroName, TypeMacroName, TypeMacroArgs);
+	emitFallbackDefine(NonCanonicalUnlessDependentTypeMacroName, TypeMacroName, 
+										 TypeMacroArgs);
+
+	// Invocations.
+	emitNodeInvocations();
+	emitLastNodeInvocation();
+	emitLeafNodeInvocations();
+
+	// Postmatter
+	emitUndefs();
+}
+
+void TypeNodeEmitter::emitFallbackDefine(StringRef macroName,
+																				 StringRef fallbackMacroName,
+																				 StringRef args) {
+  Out << "#ifndef " << macroName << "\n";
+  Out << "#  define " << macroName << args
+  	  << " " << fallbackMacroName << args << "\n";
+  Out << "#endif\n";
+
+  addMacroToUndef(macroName);
+}
+
+void TypeNodeEmitter::emitNodeInvocations() {
+	for (auto type : Types) {
+		// The name with the Type suffix.
+		StringRef id = getIdForType(type);
+
+		// Figure out which macro to use.
+		StringRef macroName;
+		auto setMacroName = [&](StringRef newName) {
+			if (!macroName.empty())
+				PrintFatalError(type->getLoc(),
+												Twine("conflict when computing macro name for "
+															"Type node: trying to use both \"")
+													+ macroName + "\" and \"" + newName + "\"");
+			macroName = newName;
+		};
+		if (type->isSubClassOf(AlwaysDependentClassName))
+			setMacroName(DependentTypeMacroName);
+		if (type->isSubClassOf(NeverCanonicalClassName))
+			setMacroName(NonCanonicalTypeMacroName);
+		if (type->isSubClassOf(NeverCanonicalUnlessDependentClassName))
+			setMacroName(NonCanonicalUnlessDependentTypeMacroName);
+		if (type->getValueAsBit(AbstractFieldName))
+			setMacroName(AbstractTypeMacroName);
+		if (macroName.empty())
+			macroName = TypeMacroName;
+
+		// Compute the base class.
+		StringRef baseName = TypeClassName;
+		if (type->isSubClassOf(DerivedTypeClassName))
+			baseName = type->getValueAsDef(BaseFieldName)->getName();
+
+		// Generate the invocation line.
+		Out << macroName << "(" << id << ", " << baseName << ")\n";
+	}
+}
+
+void TypeNodeEmitter::emitLastNodeInvocation() {
+	// We check that this is non-empty earlier.
+	Out << "#ifdef " LastTypeMacroName "\n"
+	       LastTypeMacroName "(" << getIdForType(Types.back()) << ")\n"
+				 "#undef " LastTypeMacroName "\n"
+				 "#endif\n";
+}
+
+void TypeNodeEmitter::emitLeafNodeInvocations() {
+	Out << "#ifdef " LeafTypeMacroName "\n";
+
+	for (auto type : Types) {
+		if (!type->isSubClassOf(LeafTypeClassName)) continue;
+		Out << LeafTypeMacroName "(" << getIdForType(type) << ")\n";
+	}
+
+	Out << "#undef " LeafTypeMacroName "\n"
+				 "#endif\n";
+}
+
+void TypeNodeEmitter::addMacroToUndef(StringRef macroName) {
+	MacrosToUndef.push_back(macroName);
+}
+
+void TypeNodeEmitter::emitUndefs() {
+	for (auto &macroName : MacrosToUndef) {
+		Out << "#undef " << macroName << "\n";
+	}
+}
+
+void clang::EmitClangTypeNodes(RecordKeeper &records, raw_ostream &out) {
+	TypeNodeEmitter(records, out).emit();
+}
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index d5a9744b7d40d..f420fa6a8c893 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -23,6 +23,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "TableGenBackends.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/None.h"
@@ -2629,22 +2630,18 @@ void NeonEmitter::runFP16(raw_ostream &OS) {
   OS << "#endif /* __ARM_FP16_H */\n";
 }
 
-namespace clang {
-
-void EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitNeon(RecordKeeper &Records, raw_ostream &OS) {
   NeonEmitter(Records).run(OS);
 }
 
-void EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitFP16(RecordKeeper &Records, raw_ostream &OS) {
   NeonEmitter(Records).runFP16(OS);
 }
 
-void EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
   NeonEmitter(Records).runHeader(OS);
 }
 
-void EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
+void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
   llvm_unreachable("Neon test generation no longer implemented!");
 }
-
-} // end namespace clang
diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp
index a92370019338a..d0f8a7564496b 100644
--- a/clang/utils/TableGen/TableGen.cpp
+++ b/clang/utils/TableGen/TableGen.cpp
@@ -47,6 +47,7 @@ enum ActionType {
   GenClangCommentNodes,
   GenClangDeclNodes,
   GenClangStmtNodes,
+  GenClangTypeNodes,
   GenClangOpcodes,
   GenClangSACheckers,
   GenClangCommentHTMLTags,
@@ -130,6 +131,8 @@ cl::opt<ActionType> Action(
                    "Generate Clang AST declaration nodes"),
         clEnumValN(GenClangStmtNodes, "gen-clang-stmt-nodes",
                    "Generate Clang AST statement nodes"),
+        clEnumValN(GenClangTypeNodes, "gen-clang-type-nodes",
+                   "Generate Clang AST type nodes"),
         clEnumValN(GenClangOpcodes, "gen-clang-opcodes",
                    "Generate Clang constexpr interpreter opcodes"),
         clEnumValN(GenClangSACheckers, "gen-clang-sa-checkers",
@@ -254,6 +257,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenClangStmtNodes:
     EmitClangASTNodes(Records, OS, "Stmt", "");
     break;
+  case GenClangTypeNodes:
+    EmitClangTypeNodes(Records, OS);
+    break;
   case GenClangOpcodes:
     EmitClangOpcodes(Records, OS);
     break;
diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h
index f9ee477efde09..cdd492b4e558d 100644
--- a/clang/utils/TableGen/TableGenBackends.h
+++ b/clang/utils/TableGen/TableGenBackends.h
@@ -27,6 +27,7 @@ namespace clang {
 void EmitClangDeclContext(llvm::RecordKeeper &RK, llvm::raw_ostream &OS);
 void EmitClangASTNodes(llvm::RecordKeeper &RK, llvm::raw_ostream &OS,
                        const std::string &N, const std::string &S);
+void EmitClangTypeNodes(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 
 void EmitClangAttrParserStringSwitches(llvm::RecordKeeper &Records,
                                        llvm::raw_ostream &OS);
diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html
index 487d6fb2d93e5..40af4732ef600 100755
--- a/clang/www/cxx_dr_status.html
+++ b/clang/www/cxx_dr_status.html
@@ -9955,19 +9955,19 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg1690">1690</a></td>
     <td>C++14</td>
     <td>Associated namespace for local type</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="1691">
     <td><a href="http://wg21.link/cwg1691">1691</a></td>
     <td>C++14</td>
     <td>Argument-dependent lookup and opaque enumerations</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="1692">
     <td><a href="http://wg21.link/cwg1692">1692</a></td>
     <td>C++14</td>
     <td>Associated namespaces of doubly-nested classes</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="1693">
     <td><a href="http://wg21.link/cwg1693">1693</a></td>
@@ -10147,7 +10147,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg1722">1722</a></td>
     <td>CD4</td>
     <td>Should lambda to function pointer conversion function be <TT>noexcept</TT>?</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr class="open" id="1723">
     <td><a href="http://wg21.link/cwg1723">1723</a></td>
@@ -10483,7 +10483,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg1778">1778</a></td>
     <td>C++14</td>
     <td><I>exception-specification</I> in explicitly-defaulted functions</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="1779">
     <td><a href="http://wg21.link/cwg1779">1779</a></td>
@@ -11047,7 +11047,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg1872">1872</a></td>
     <td>CD4</td>
     <td>Instantiations of <TT>constexpr</TT> templates that cannot appear in constant expressions</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="1873">
     <td><a href="http://wg21.link/cwg1873">1873</a></td>
@@ -12655,7 +12655,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg2140">2140</a></td>
     <td>CD4</td>
     <td>Lvalue-to-rvalue conversion of <TT>std::nullptr_t</TT></td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2141">
     <td><a href="http://wg21.link/cwg2141">2141</a></td>
@@ -12835,7 +12835,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg2170">2170</a></td>
     <td>DR</td>
     <td>Unclear definition of odr-use for arrays</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2171">
     <td><a href="http://wg21.link/cwg2171">2171</a></td>
@@ -13567,7 +13567,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg2292">2292</a></td>
     <td>DRWP</td>
     <td><I>simple-template-id</I> is ambiguous between <I>class-name</I> and <I>type-name</I></td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2293">
     <td><a href="http://wg21.link/cwg2293">2293</a></td>
@@ -13933,7 +13933,7 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg2353">2353</a></td>
     <td>DR</td>
     <td>Potential results of a member access expression for a static data member</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2354">
     <td><a href="http://wg21.link/cwg2354">2354</a></td>
@@ -14131,13 +14131,13 @@ <h2 id="cxxdr">C++ defect report implementation status</h2>
     <td><a href="http://wg21.link/cwg2386">2386</a></td>
     <td>DR</td>
     <td><T>tuple_size</T> requirements for structured binding</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2387">
     <td><a href="http://wg21.link/cwg2387">2387</a></td>
     <td>DR</td>
     <td>Linkage of const-qualified variable template</td>
-    <td class="svn" align="center">Clang 9</td>
+    <td class="full" align="center">Clang 9</td>
   </tr>
   <tr id="2388">
     <td><a href="http://wg21.link/cwg2388">2388</a></td>
diff --git a/clang/www/cxx_status.html b/clang/www/cxx_status.html
index 151192b2ff28c..041d6aa242c7b 100755
--- a/clang/www/cxx_status.html
+++ b/clang/www/cxx_status.html
@@ -114,7 +114,7 @@ <h2 id="cxx11">C++11 implementation status</h2>
     </tr>
       <tr> <!-- from Kona 2019-->
         <td><a href="http://wg21.link/p1009r2">P1009R2</a> (<a href="#dr">DR</a>)</td>
-        <td class="svn" align="center">Clang 9</td>
+        <td class="full" align="center">Clang 9</td>
       </tr>
     <tr>
       <td>Static assertions</td>
@@ -289,7 +289,7 @@ <h2 id="cxx11">C++11 implementation status</h2>
     </tr>
       <tr> <!-- from Kona 2019-->
         <td><a href="http://wg21.link/p1286r2">P1286R2</a> (<a href="#dr">DR</a>)</td>
-        <td class="svn" align="center">Clang 9</td>
+        <td class="full" align="center">Clang 9</td>
       </tr>
     <tr>
       <td>Deleted functions</td>
@@ -664,7 +664,7 @@ <h2 id="cxx17">C++17 implementation status</h2>
     </tr>
       <tr> <!-- from Cologne 2019 -->
         <td><a href="http://wg21.link/p1771r1">P1771R1</a> (<a href="#dr">DR</a>)</td>
-        <td class="svn" align="center">Clang 9</td>
+        <td class="full" align="center">Clang 9</td>
       </tr>
     <tr>
       <td><tt>[[maybe_unused]]</tt> attribute</td>
@@ -866,7 +866,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     </tr>
       <tr> <!-- from Rapperswil -->
         <td><a href="http://wg21.link/p1042r1">P1042R1</a></td>
-        <td class="svn" align="center">Clang 9</td>
+        <td class="full" align="center">Clang 9</td>
       </tr>
     <tr>
       <td>Designated initializers</td>
@@ -876,7 +876,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td><i>template-parameter-list</i> for generic lambdas</td>
       <td><a href="http://wg21.link/p0428r2">P0428R2</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr id="p0734">
       <td rowspan="7">Concepts</td>
@@ -910,7 +910,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td>ADL and function templates that are not visible</td>
       <td><a href="http://wg21.link/p0846r0">P0846R0</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td><tt>const</tt> mismatch with defaulted copy constructor</td>
@@ -957,7 +957,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td><tt>[[no_unique_address]]</tt> attribute</td>
       <td><a href="http://wg21.link/p0840r2">P0840R2</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td><tt>[[likely]]</tt> and <tt>[[unlikely]]</tt> attributes</td>
@@ -972,7 +972,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td>Pack expansion in lambda <i>init-capture</i></td>
       <td><a href="http://wg21.link/p0780r2">P0780R2</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <!-- Rapperswil papers -->
     <tr>
@@ -988,7 +988,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td rowspan="7">Relaxations of <tt>constexpr</tt> restrictions</td>
       <td><a href="http://wg21.link/p1064r0">P1064R0</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
       <tr> <!-- from San Diego -->
         <td><a href="http://wg21.link/p1002r1">P1002R1</a></td>
@@ -996,22 +996,20 @@ <h2 id="cxx20">C++2a implementation status</h2>
       </tr>
       <tr>
         <td><a href="http://wg21.link/p1327r1">P1327R1</a></td>
-        <td rowspan="2" class="svn" align="center">Clang 9</td>
+        <td rowspan="2" class="full" align="center">Clang 9</td>
       </tr>
       <tr>
         <td><a href="http://wg21.link/p1330r0">P1330R0</a></td>
       </tr>
       <tr> <!-- from Cologne -->
         <td><a href="http://wg21.link/p1331r2">P1331R2</a></td>
-        <td class="none" align="center">No</td>
+        <td rowspan="3" class="svn" align="center">SVN</td>
       </tr>
       <tr>
         <td><a href="http://wg21.link/p1668r1">P1668R1</a></td>
-        <td class="svn" align="center">Clang 9</td>
       </tr>
       <tr>
         <td><a href="http://wg21.link/p0784r7">P0784R7</a></td>
-        <td class="none" align="center">No</td>
       </tr>
     <tr>
       <td>Prohibit aggregates with user-declared constructors</td>
@@ -1026,13 +1024,13 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td><tt>explicit(bool)</tt></td>
       <td><a href="http://wg21.link/p0892r2">P0892R2</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <!-- San Diego papers -->
     <tr>
       <td>Signed integers are two's complement</td>
       <td><a href="http://wg21.link/p1236r1">P1236R1</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td><tt>char8_t</tt></td>
@@ -1047,7 +1045,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td><tt>std::is_constant_evaluated</tt></td>
       <td><a href="http://wg21.link/p0595r2">P0595R2</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td>Nested inline namespaces</td>
@@ -1100,7 +1098,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td>Deprecate <tt>a[b,c]</tt></td>
       <td><a href="http://wg21.link/p1161r3">P1161R3</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td>Deprecate some problematic uses of <tt>volatile</tt></td>
@@ -1110,7 +1108,7 @@ <h2 id="cxx20">C++2a implementation status</h2>
     <tr>
       <td><tt>[[nodiscard("with reason")]]</tt></td>
       <td><a href="http://wg21.link/p1301r4">P1301R4</a></td>
-      <td class="svn" align="center">Clang 9</td>
+      <td class="full" align="center">Clang 9</td>
     </tr>
     <tr>
       <td><tt>using enum</tt></td>
diff --git a/clang/www/make_cxx_dr_status b/clang/www/make_cxx_dr_status
index ff4a3f5c1b471..4351d659e41ae 100755
--- a/clang/www/make_cxx_dr_status
+++ b/clang/www/make_cxx_dr_status
@@ -111,9 +111,6 @@ def availability(issue):
   elif status == '10':
     avail = 'SVN'
     avail_style = ' class="svn"'
-  elif status == '9':
-    avail = 'Clang 9'
-    avail_style = ' class="svn"'
   elif re.match('^[0-9]+\.?[0-9]*', status):
     avail = 'Clang %s' % status
     avail_style = ' class="full"'
diff --git a/compiler-rt/cmake/Modules/BuiltinTests.cmake b/compiler-rt/cmake/Modules/BuiltinTests.cmake
index eee77ad020091..4a123638c8bcf 100644
--- a/compiler-rt/cmake/Modules/BuiltinTests.cmake
+++ b/compiler-rt/cmake/Modules/BuiltinTests.cmake
@@ -72,6 +72,12 @@ function(try_compile_only output)
     endif()
   endforeach()
 
+  # Strip quotes from the compile command, as the compiler is not expecting
+  # quoted arguments (see discussion on D62063 for when this can come up). If
+  # the quotes were there for arugments with spaces in them, the quotes were
+  # not going to help since the string gets split on spaces below.
+  string(REPLACE "\"" "" test_compile_command "${test_compile_command}")
+
   string(REPLACE " " ";" test_compile_command "${test_compile_command}")
 
   execute_process(
diff --git a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
index 042c8a08dec41..15d34afe5368a 100644
--- a/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTDarwinUtils.cmake
@@ -1,6 +1,8 @@
 include(CMakeParseArguments)
 include(CompilerRTUtils)
 
+set(CMAKE_LIPO "lipo" CACHE PATH "path to the lipo tool")
+
 # On OS X SDKs can be installed anywhere on the base system and xcode-select can
 # set the default Xcode to use. This function finds the SDKs that are present in
 # the current Xcode.
@@ -244,7 +246,7 @@ function(darwin_lipo_libs name)
   if(LIB_DEPENDS AND LIB_LIPO_FLAGS)
     add_custom_command(OUTPUT ${LIB_OUTPUT_DIR}/lib${name}.a
       COMMAND ${CMAKE_COMMAND} -E make_directory ${LIB_OUTPUT_DIR}
-      COMMAND lipo -output
+      COMMAND ${CMAKE_LIPO} -output
               ${LIB_OUTPUT_DIR}/lib${name}.a
               -create ${LIB_LIPO_FLAGS}
       DEPENDS ${LIB_DEPENDS}
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index d2da0496b97fc..875acc83be4e3 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -208,6 +208,32 @@ macro(get_test_cc_for_arch arch cc_out cflags_out)
   endif()
 endmacro()
 
+# Returns CFLAGS that should be used to run tests for the
+# specific apple platform and architecture.
+function(get_test_cflags_for_apple_platform platform arch cflags_out)
+  is_valid_apple_platform("${platform}" is_valid_platform)
+  if (NOT is_valid_platform)
+    message(FATAL_ERROR "\"${platform}\" is not a valid apple platform")
+  endif()
+  set(test_cflags "")
+  get_target_flags_for_arch(${arch} test_cflags)
+  list(APPEND test_cflags ${DARWIN_${platform}_CFLAGS})
+  string(REPLACE ";" " " test_cflags_str "${test_cflags}")
+  string(APPEND test_cflags_str "${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+  set(${cflags_out} "${test_cflags_str}" PARENT_SCOPE)
+endfunction()
+
+function(is_valid_apple_platform platform is_valid_out)
+  set(is_valid FALSE)
+  if ("${platform}" STREQUAL "")
+    message(FATAL_ERROR "platform cannot be empty")
+  endif()
+  if ("${platform}" MATCHES "^(osx|((ios|watchos|tvos)(sim)?))$")
+    set(is_valid TRUE)
+  endif()
+  set(${is_valid_out} ${is_valid} PARENT_SCOPE)
+endfunction()
+
 set(ARM64 aarch64)
 set(ARM32 arm armhf)
 set(HEXAGON hexagon)
diff --git a/compiler-rt/lib/asan/asan_errors.cpp b/compiler-rt/lib/asan/asan_errors.cpp
index 0ef97794be3cb..541c6e0353b57 100644
--- a/compiler-rt/lib/asan/asan_errors.cpp
+++ b/compiler-rt/lib/asan/asan_errors.cpp
@@ -35,7 +35,8 @@ static void OnStackUnwind(const SignalContext &sig,
   // corresponding code in the sanitizer_common and we use this callback to
   // print it.
   static_cast<const ScarinessScoreBase *>(callback_context)->Print();
-  stack->Unwind(sig.pc, sig.bp, sig.context, fast);
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
+                fast);
 }
 
 void ErrorDeadlySignal::Print() {
diff --git a/compiler-rt/lib/asan/asan_interceptors.cpp b/compiler-rt/lib/asan/asan_interceptors.cpp
index cea4d63e64d55..b19cf25c7cd00 100644
--- a/compiler-rt/lib/asan/asan_interceptors.cpp
+++ b/compiler-rt/lib/asan/asan_interceptors.cpp
@@ -164,6 +164,11 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *)
     ASAN_MEMSET_IMPL(ctx, block, c, size);                  \
   } while (false)
 
+#if CAN_SANITIZE_LEAKS
+#define COMMON_INTERCEPTOR_STRERROR()                       \
+  __lsan::ScopedInterceptorDisabler disabler
+#endif
+
 #include "sanitizer_common/sanitizer_common_interceptors.inc"
 #include "sanitizer_common/sanitizer_signal_interceptors.inc"
 
@@ -560,24 +565,59 @@ INTERCEPTOR(long long, atoll, const char *nptr) {
 }
 #endif  // ASAN_INTERCEPT_ATOLL_AND_STRTOLL
 
-#if ASAN_INTERCEPT___CXA_ATEXIT
+#if ASAN_INTERCEPT___CXA_ATEXIT || ASAN_INTERCEPT_ATEXIT
 static void AtCxaAtexit(void *unused) {
   (void)unused;
   StopInitOrderChecking();
 }
+#endif
 
+#if ASAN_INTERCEPT___CXA_ATEXIT
 INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
             void *dso_handle) {
 #if SANITIZER_MAC
   if (UNLIKELY(!asan_inited)) return REAL(__cxa_atexit)(func, arg, dso_handle);
 #endif
   ENSURE_ASAN_INITED();
+#if CAN_SANITIZE_LEAKS
+  __lsan::ScopedInterceptorDisabler disabler;
+#endif
   int res = REAL(__cxa_atexit)(func, arg, dso_handle);
   REAL(__cxa_atexit)(AtCxaAtexit, nullptr, nullptr);
   return res;
 }
 #endif  // ASAN_INTERCEPT___CXA_ATEXIT
 
+#if ASAN_INTERCEPT_ATEXIT
+INTERCEPTOR(int, atexit, void (*func)()) {
+  ENSURE_ASAN_INITED();
+#if CAN_SANITIZE_LEAKS
+  __lsan::ScopedInterceptorDisabler disabler;
+#endif
+  // Avoid calling real atexit as it is unrechable on at least on Linux.
+  int res = REAL(__cxa_atexit)((void (*)(void *a))func, nullptr, nullptr);
+  REAL(__cxa_atexit)(AtCxaAtexit, nullptr, nullptr);
+  return res;
+}
+#endif
+
+#if ASAN_INTERCEPT_PTHREAD_ATFORK
+extern "C" {
+extern int _pthread_atfork(void (*prepare)(), void (*parent)(),
+                           void (*child)());
+};
+
+INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(),
+            void (*child)()) {
+#if CAN_SANITIZE_LEAKS
+  __lsan::ScopedInterceptorDisabler disabler;
+#endif
+  // REAL(pthread_atfork) cannot be called due to symbol indirections at least
+  // on NetBSD
+  return _pthread_atfork(prepare, parent, child);
+}
+#endif
+
 #if ASAN_INTERCEPT_VFORK
 DEFINE_REAL(int, vfork)
 DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(int, vfork)
@@ -660,6 +700,14 @@ void InitializeAsanInterceptors() {
   ASAN_INTERCEPT_FUNC(__cxa_atexit);
 #endif
 
+#if ASAN_INTERCEPT_ATEXIT
+  ASAN_INTERCEPT_FUNC(atexit);
+#endif
+
+#if ASAN_INTERCEPT_PTHREAD_ATFORK
+  ASAN_INTERCEPT_FUNC(pthread_atfork);
+#endif
+
 #if ASAN_INTERCEPT_VFORK
   ASAN_INTERCEPT_FUNC(vfork);
 #endif
diff --git a/compiler-rt/lib/asan/asan_interceptors.h b/compiler-rt/lib/asan/asan_interceptors.h
index 155ea4156abbe..344a64bd83d33 100644
--- a/compiler-rt/lib/asan/asan_interceptors.h
+++ b/compiler-rt/lib/asan/asan_interceptors.h
@@ -99,6 +99,12 @@ void InitializePlatformInterceptors();
 # define ASAN_INTERCEPT___CXA_ATEXIT 0
 #endif
 
+#if SANITIZER_NETBSD
+# define ASAN_INTERCEPT_ATEXIT 1
+#else
+# define ASAN_INTERCEPT_ATEXIT 0
+#endif
+
 #if SANITIZER_LINUX && !SANITIZER_ANDROID
 # define ASAN_INTERCEPT___STRDUP 1
 #else
@@ -112,6 +118,12 @@ void InitializePlatformInterceptors();
 # define ASAN_INTERCEPT_VFORK 0
 #endif
 
+#if SANITIZER_NETBSD
+# define ASAN_INTERCEPT_PTHREAD_ATFORK 1
+#else
+# define ASAN_INTERCEPT_PTHREAD_ATFORK 0
+#endif
+
 DECLARE_REAL(int, memcmp, const void *a1, const void *a2, uptr size)
 DECLARE_REAL(char*, strchr, const char *str, int c)
 DECLARE_REAL(SIZE_T, strlen, const char *s)
diff --git a/compiler-rt/lib/asan/tests/asan_str_test.cpp b/compiler-rt/lib/asan/tests/asan_str_test.cpp
index cc5bd2b4446e3..33a38a81567fc 100644
--- a/compiler-rt/lib/asan/tests/asan_str_test.cpp
+++ b/compiler-rt/lib/asan/tests/asan_str_test.cpp
@@ -458,8 +458,6 @@ TEST(AddressSanitizer, StrNCatOOBTest) {
   strncat(to, from, from_size);
   from[from_size - 1] = '\0';
   strncat(to, from, 2 * from_size);
-  // Catenating empty string with an invalid string is still an error.
-  EXPECT_DEATH(strncat(to - 1, from, 0), LeftOOBAccessMessage(1));
   strncat(to, from + from_size - 1, 10);
   // One of arguments points to not allocated memory.
   EXPECT_DEATH(strncat(to - 1, from, 2), LeftOOBAccessMessage(1));
diff --git a/compiler-rt/lib/builtins/fp_add_impl.inc b/compiler-rt/lib/builtins/fp_add_impl.inc
index 582324746fe52..ab63213490324 100644
--- a/compiler-rt/lib/builtins/fp_add_impl.inc
+++ b/compiler-rt/lib/builtins/fp_add_impl.inc
@@ -94,7 +94,7 @@ static __inline fp_t __addXf3__(fp_t a, fp_t b) {
   const unsigned int align = aExponent - bExponent;
   if (align) {
     if (align < typeWidth) {
-      const bool sticky = bSignificand << (typeWidth - align);
+      const bool sticky = (bSignificand << (typeWidth - align)) != 0;
       bSignificand = bSignificand >> align | sticky;
     } else {
       bSignificand = 1; // Set the sticky bit.  b is known to be non-zero.
@@ -133,7 +133,7 @@ static __inline fp_t __addXf3__(fp_t a, fp_t b) {
     // The result is denormal before rounding.  The exponent is zero and we
     // need to shift the significand.
     const int shift = 1 - aExponent;
-    const bool sticky = aSignificand << (typeWidth - shift);
+    const bool sticky = (aSignificand << (typeWidth - shift)) != 0;
     aSignificand = aSignificand >> shift | sticky;
     aExponent = 0;
   }
diff --git a/compiler-rt/lib/builtins/fp_lib.h b/compiler-rt/lib/builtins/fp_lib.h
index d1a988ea47133..e2a906681c46d 100644
--- a/compiler-rt/lib/builtins/fp_lib.h
+++ b/compiler-rt/lib/builtins/fp_lib.h
@@ -245,7 +245,7 @@ static __inline void wideLeftShift(rep_t *hi, rep_t *lo, int count) {
 static __inline void wideRightShiftWithSticky(rep_t *hi, rep_t *lo,
                                               unsigned int count) {
   if (count < typeWidth) {
-    const bool sticky = *lo << (typeWidth - count);
+    const bool sticky = (*lo << (typeWidth - count)) != 0;
     *lo = *hi << (typeWidth - count) | *lo >> count | sticky;
     *hi = *hi >> count;
   } else if (count < 2 * typeWidth) {
diff --git a/compiler-rt/lib/builtins/fp_trunc_impl.inc b/compiler-rt/lib/builtins/fp_trunc_impl.inc
index 133c8bbe5c2fe..6662be7607e70 100644
--- a/compiler-rt/lib/builtins/fp_trunc_impl.inc
+++ b/compiler-rt/lib/builtins/fp_trunc_impl.inc
@@ -113,7 +113,7 @@ static __inline dst_t __truncXfYf2__(src_t a) {
     if (shift > srcSigBits) {
       absResult = 0;
     } else {
-      const bool sticky = significand << (srcBits - shift);
+      const bool sticky = (significand << (srcBits - shift)) != 0;
       src_rep_t denormalizedSignificand = significand >> shift | sticky;
       absResult = denormalizedSignificand >> (srcSigBits - dstSigBits);
       const src_rep_t roundBits = denormalizedSignificand & roundMask;
diff --git a/compiler-rt/lib/crt/CMakeLists.txt b/compiler-rt/lib/crt/CMakeLists.txt
index 34c368f7ca444..90e94b93db477 100644
--- a/compiler-rt/lib/crt/CMakeLists.txt
+++ b/compiler-rt/lib/crt/CMakeLists.txt
@@ -70,7 +70,7 @@ function(check_cxx_section_exists section output)
 endfunction()
 
 check_cxx_section_exists(".init_array" COMPILER_RT_HAS_INITFINI_ARRAY
-  SOURCE "__attribute__((constructor)) void f() {}\nint main() { return 0; }\n")
+  SOURCE "volatile int x;\n__attribute__((constructor)) void f() {x = 0;}\nint main() { return 0; }\n")
 
 append_list_if(COMPILER_RT_HAS_STD_C11_FLAG -std=c11 CRT_CFLAGS)
 append_list_if(COMPILER_RT_HAS_INITFINI_ARRAY -DCRT_HAS_INITFINI_ARRAY CRT_CFLAGS)
diff --git a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
index 54c7ff0795857..44c90655b9328 100644
--- a/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerDriver.cpp
@@ -708,7 +708,6 @@ int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) {
     Options.FeaturesDir = Flags.features_dir;
   if (Flags.collect_data_flow)
     Options.CollectDataFlow = Flags.collect_data_flow;
-  Options.LazyCounters = Flags.lazy_counters;
   if (Flags.stop_file)
     Options.StopFile = Flags.stop_file;
 
diff --git a/compiler-rt/lib/fuzzer/FuzzerExtFunctions.def b/compiler-rt/lib/fuzzer/FuzzerExtFunctions.def
index 7b53b085560b7..51edf8444e94d 100644
--- a/compiler-rt/lib/fuzzer/FuzzerExtFunctions.def
+++ b/compiler-rt/lib/fuzzer/FuzzerExtFunctions.def
@@ -33,6 +33,7 @@ EXT_FUNC(__sanitizer_install_malloc_and_free_hooks, int,
          (void (*malloc_hook)(const volatile void *, size_t),
           void (*free_hook)(const volatile void *)),
          false);
+EXT_FUNC(__sanitizer_log_write, void, (const char *buf, size_t len), false);
 EXT_FUNC(__sanitizer_purge_allocator, void, (), false);
 EXT_FUNC(__sanitizer_print_memory_profile, void, (size_t, size_t), false);
 EXT_FUNC(__sanitizer_print_stack_trace, void, (), true);
diff --git a/compiler-rt/lib/fuzzer/FuzzerFlags.def b/compiler-rt/lib/fuzzer/FuzzerFlags.def
index a11cfe4405f31..0e19a9cde6ca3 100644
--- a/compiler-rt/lib/fuzzer/FuzzerFlags.def
+++ b/compiler-rt/lib/fuzzer/FuzzerFlags.def
@@ -123,9 +123,6 @@ FUZZER_FLAG_INT(handle_term, 1, "If 1, try to intercept SIGTERM.")
 FUZZER_FLAG_INT(handle_xfsz, 1, "If 1, try to intercept SIGXFSZ.")
 FUZZER_FLAG_INT(handle_usr1, 1, "If 1, try to intercept SIGUSR1.")
 FUZZER_FLAG_INT(handle_usr2, 1, "If 1, try to intercept SIGUSR2.")
-FUZZER_FLAG_INT(lazy_counters, 0, "If 1, a performance optimization is"
-    "enabled for the 8bit inline counters. "
-    "Requires that libFuzzer successfully installs its SEGV handler")
 FUZZER_FLAG_INT(close_fd_mask, 0, "If 1, close stdout at startup; "
     "if 2, close stderr; if 3, close both. "
     "Be careful, this will also close e.g. stderr of asan.")
diff --git a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
index 9c266739306b0..df533e877d37a 100644
--- a/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerLoop.cpp
@@ -513,10 +513,12 @@ size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
 }
 
 void Fuzzer::CrashOnOverwrittenData() {
-  Printf("==%d== ERROR: libFuzzer: fuzz target overwrites it's const input\n",
+  Printf("==%d== ERROR: libFuzzer: fuzz target overwrites its const input\n",
          GetPid());
+  PrintStackTrace();
+  Printf("SUMMARY: libFuzzer: overwrites-const-input\n");
   DumpCurrentUnit("crash-");
-  Printf("SUMMARY: libFuzzer: out-of-memory\n");
+  PrintFinalStats();
   _Exit(Options.ErrorExitCode); // Stop right now.
 }
 
@@ -740,10 +742,6 @@ void Fuzzer::ReadAndExecuteSeedCorpora(Vector<SizedFile> &CorporaFiles) {
   uint8_t dummy = 0;
   ExecuteCallback(&dummy, 0);
 
-  // Protect lazy counters here, after the once-init code has been executed.
-  if (Options.LazyCounters)
-    TPC.ProtectLazyCounters();
-
   if (CorporaFiles.empty()) {
     Printf("INFO: A corpus is not provided, starting from an empty corpus\n");
     Unit U({'\n'}); // Valid ASCII input.
diff --git a/compiler-rt/lib/fuzzer/FuzzerOptions.h b/compiler-rt/lib/fuzzer/FuzzerOptions.h
index ad3df015bc778..beecc980380b4 100644
--- a/compiler-rt/lib/fuzzer/FuzzerOptions.h
+++ b/compiler-rt/lib/fuzzer/FuzzerOptions.h
@@ -75,7 +75,6 @@ struct FuzzingOptions {
   bool HandleXfsz = false;
   bool HandleUsr1 = false;
   bool HandleUsr2 = false;
-  bool LazyCounters = false;
 };
 
 }  // namespace fuzzer
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
index c47357703b953..f03be7a39502f 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.cpp
@@ -67,45 +67,6 @@ void TracePC::HandleInline8bitCountersInit(uint8_t *Start, uint8_t *Stop) {
   NumInline8bitCounters += M.Size();
 }
 
-// Mark all full page counter regions as PROT_NONE and set Enabled=false.
-// The first time the instrumented code hits such a protected/disabled
-// counter region we should catch a SEGV and call UnprotectLazyCounters,
-// which will mark the page as PROT_READ|PROT_WRITE and set Enabled=true.
-//
-// Whenever other functions iterate over the counters they should ignore
-// regions with Enabled=false.
-void TracePC::ProtectLazyCounters() {
-  size_t NumPagesProtected = 0;
-  IterateCounterRegions([&](Module::Region &R) {
-    if (!R.OneFullPage) return;
-    if (Mprotect(R.Start, R.Stop - R.Start, false)) {
-      R.Enabled = false;
-      NumPagesProtected++;
-    }
-  });
-  if (NumPagesProtected)
-    Printf("INFO: %zd pages of counters where protected;"
-           " libFuzzer's SEGV handler must be installed\n",
-           NumPagesProtected);
-}
-
-bool TracePC::UnprotectLazyCounters(void *CounterPtr) {
-  // Printf("UnprotectLazyCounters: %p\n", CounterPtr);
-  if (!CounterPtr)
-    return false;
-  bool Done = false;
-  uint8_t *Addr = reinterpret_cast<uint8_t *>(CounterPtr);
-  IterateCounterRegions([&](Module::Region &R) {
-    if (!R.OneFullPage || R.Enabled || Done) return;
-    if (Addr >= R.Start && Addr < R.Stop)
-      if (Mprotect(R.Start, R.Stop - R.Start, true)) {
-        R.Enabled = true;
-        Done = true;
-      }
-  });
-  return Done;
-}
-
 void TracePC::HandlePCsInit(const uintptr_t *Start, const uintptr_t *Stop) {
   const PCTableEntry *B = reinterpret_cast<const PCTableEntry *>(Start);
   const PCTableEntry *E = reinterpret_cast<const PCTableEntry *>(Stop);
diff --git a/compiler-rt/lib/fuzzer/FuzzerTracePC.h b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
index 4f5ebeb047a16..501f3b544971f 100644
--- a/compiler-rt/lib/fuzzer/FuzzerTracePC.h
+++ b/compiler-rt/lib/fuzzer/FuzzerTracePC.h
@@ -119,9 +119,6 @@ class TracePC {
   void SetFocusFunction(const std::string &FuncName);
   bool ObservedFocusFunction();
 
-  void ProtectLazyCounters();
-  bool UnprotectLazyCounters(void *CounterPtr);
-
   struct PCTableEntry {
     uintptr_t PC, PCFlags;
   };
diff --git a/compiler-rt/lib/fuzzer/FuzzerUtil.h b/compiler-rt/lib/fuzzer/FuzzerUtil.h
index 0a127911df3cc..85c5571d684f5 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtil.h
+++ b/compiler-rt/lib/fuzzer/FuzzerUtil.h
@@ -52,8 +52,6 @@ void SetSignalHandler(const FuzzingOptions& Options);
 
 void SleepSeconds(int Seconds);
 
-bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite);
-
 unsigned long GetPid();
 
 size_t GetPeakRSSMb();
diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
index 1f04b33c154e4..50071a7e5c8f3 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerUtilFuchsia.cpp
@@ -305,12 +305,19 @@ void CrashHandler(zx_handle_t *Event) {
 
 } // namespace
 
-bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) {
-  return false;  // UNIMPLEMENTED
-}
-
 // Platform specific functions.
 void SetSignalHandler(const FuzzingOptions &Options) {
+  // Make sure information from libFuzzer and the sanitizers are easy to
+  // reassemble. `__sanitizer_log_write` has the added benefit of ensuring the
+  // DSO map is always available for the symbolizer.
+  // A uint64_t fits in 20 chars, so 64 is plenty.
+  char Buf[64];
+  memset(Buf, 0, sizeof(Buf));
+  snprintf(Buf, sizeof(Buf), "==%lu== INFO: libFuzzer starting.\n", GetPid());
+  if (EF->__sanitizer_log_write)
+    __sanitizer_log_write(Buf, sizeof(Buf));
+  Printf("%s", Buf);
+
   // Set up alarm handler if needed.
   if (Options.UnitTimeoutSec > 0) {
     std::thread T(AlarmHandler, Options.UnitTimeoutSec / 2 + 1);
diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp
index 110785d874137..cefe7ae181e72 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerUtilPosix.cpp
@@ -37,7 +37,6 @@ static void (*upstream_segv_handler)(int, siginfo_t *, void *);
 
 static void SegvHandler(int sig, siginfo_t *si, void *ucontext) {
   assert(si->si_signo == SIGSEGV);
-  if (TPC.UnprotectLazyCounters(si->si_addr)) return;
   if (upstream_segv_handler)
     return upstream_segv_handler(sig, si, ucontext);
   Fuzzer::StaticCrashSignalCallback();
@@ -98,11 +97,6 @@ void SetTimer(int Seconds) {
   SetSigaction(SIGALRM, AlarmHandler);
 }
 
-bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) {
-  return 0 == mprotect(Ptr, Size,
-                       AllowReadWrite ? (PROT_READ | PROT_WRITE) : PROT_NONE);
-}
-
 void SetSignalHandler(const FuzzingOptions& Options) {
   if (Options.UnitTimeoutSec > 0)
     SetTimer(Options.UnitTimeoutSec / 2 + 1);
diff --git a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp
index 074e1eb423094..ed90044c3f835 100644
--- a/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp
+++ b/compiler-rt/lib/fuzzer/FuzzerUtilWindows.cpp
@@ -111,10 +111,6 @@ static TimerQ Timer;
 
 static void CrashHandler(int) { Fuzzer::StaticCrashSignalCallback(); }
 
-bool Mprotect(void *Ptr, size_t Size, bool AllowReadWrite) {
-  return false;  // UNIMPLEMENTED
-}
-
 void SetSignalHandler(const FuzzingOptions& Options) {
   HandlerOpt = &Options;
 
diff --git a/compiler-rt/lib/fuzzer/utils/FuzzedDataProvider.h b/compiler-rt/lib/fuzzer/utils/FuzzedDataProvider.h
deleted file mode 100644
index 5692060c7f7eb..0000000000000
--- a/compiler-rt/lib/fuzzer/utils/FuzzedDataProvider.h
+++ /dev/null
@@ -1,247 +0,0 @@
-//===- FuzzedDataProvider.h - Utility header for fuzz targets ---*- C++ -* ===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// This a temporary copy of compiler-rt/include/fuzzer/FuzzedDataProvider.h.
-// TODO(mmoroz@chromium.org): delete this copy.
-// A single header library providing an utility class to break up an array of
-// bytes. Whenever run on the same input, provides the same output, as long as
-// its methods are called in the same order, with the same arguments.
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
-#define LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
-
-#include <limits.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#include <algorithm>
-#include <cstring>
-#include <initializer_list>
-#include <string>
-#include <type_traits>
-#include <utility>
-#include <vector>
-
-class FuzzedDataProvider {
- public:
-  // |data| is an array of length |size| that the FuzzedDataProvider wraps to
-  // provide more granular access. |data| must outlive the FuzzedDataProvider.
-  FuzzedDataProvider(const uint8_t *data, size_t size)
-      : data_ptr_(data), remaining_bytes_(size) {}
-  ~FuzzedDataProvider() = default;
-
-  // Returns a std::vector containing |num_bytes| of input data. If fewer than
-  // |num_bytes| of data remain, returns a shorter std::vector containing all
-  // of the data that's left. Can be used with any byte sized type, such as
-  // char, unsigned char, uint8_t, etc.
-  template <typename T> std::vector<T> ConsumeBytes(size_t num_bytes) {
-    num_bytes = std::min(num_bytes, remaining_bytes_);
-    return ConsumeBytes<T>(num_bytes, num_bytes);
-  }
-
-  // Similar to |ConsumeBytes|, but also appends the terminator value at the end
-  // of the resulting vector. Useful, when a mutable null-terminated C-string is
-  // needed, for example. But that is a rare case. Better avoid it, if possible,
-  // and prefer using |ConsumeBytes| or |ConsumeBytesAsString| methods.
-  template <typename T>
-  std::vector<T> ConsumeBytesWithTerminator(size_t num_bytes,
-                                            T terminator = 0) {
-    num_bytes = std::min(num_bytes, remaining_bytes_);
-    std::vector<T> result = ConsumeBytes<T>(num_bytes + 1, num_bytes);
-    result.back() = terminator;
-    return result;
-  }
-
-  // Returns a std::string containing |num_bytes| of input data. Using this and
-  // |.c_str()| on the resulting string is the best way to get an immutable
-  // null-terminated C string. If fewer than |num_bytes| of data remain, returns
-  // a shorter std::string containing all of the data that's left.
-  std::string ConsumeBytesAsString(size_t num_bytes) {
-    static_assert(sizeof(std::string::value_type) == sizeof(uint8_t),
-                  "ConsumeBytesAsString cannot convert the data to a string.");
-
-    num_bytes = std::min(num_bytes, remaining_bytes_);
-    std::string result(
-        reinterpret_cast<const std::string::value_type *>(data_ptr_),
-        num_bytes);
-    Advance(num_bytes);
-    return result;
-  }
-
-  // Returns a number in the range [min, max] by consuming bytes from the
-  // input data. The value might not be uniformly distributed in the given
-  // range. If there's no input data left, always returns |min|. |min| must
-  // be less than or equal to |max|.
-  template <typename T> T ConsumeIntegralInRange(T min, T max) {
-    static_assert(std::is_integral<T>::value, "An integral type is required.");
-    static_assert(sizeof(T) <= sizeof(uint64_t), "Unsupported integral type.");
-
-    if (min > max)
-      abort();
-
-    // Use the biggest type possible to hold the range and the result.
-    uint64_t range = static_cast<uint64_t>(max) - min;
-    uint64_t result = 0;
-    size_t offset = 0;
-
-    while (offset < sizeof(T) * CHAR_BIT && (range >> offset) > 0 &&
-           remaining_bytes_ != 0) {
-      // Pull bytes off the end of the seed data. Experimentally, this seems to
-      // allow the fuzzer to more easily explore the input space. This makes
-      // sense, since it works by modifying inputs that caused new code to run,
-      // and this data is often used to encode length of data read by
-      // |ConsumeBytes|. Separating out read lengths makes it easier modify the
-      // contents of the data that is actually read.
-      --remaining_bytes_;
-      result = (result << CHAR_BIT) | data_ptr_[remaining_bytes_];
-      offset += CHAR_BIT;
-    }
-
-    // Avoid division by 0, in case |range + 1| results in overflow.
-    if (range != std::numeric_limits<decltype(range)>::max())
-      result = result % (range + 1);
-
-    return static_cast<T>(min + result);
-  }
-
-  // Returns a std::string of length from 0 to |max_length|. When it runs out of
-  // input data, returns what remains of the input. Designed to be more stable
-  // with respect to a fuzzer inserting characters than just picking a random
-  // length and then consuming that many bytes with |ConsumeBytes|.
-  std::string ConsumeRandomLengthString(size_t max_length) {
-    // Reads bytes from the start of |data_ptr_|. Maps "\\" to "\", and maps "\"
-    // followed by anything else to the end of the string. As a result of this
-    // logic, a fuzzer can insert characters into the string, and the string
-    // will be lengthened to include those new characters, resulting in a more
-    // stable fuzzer than picking the length of a string independently from
-    // picking its contents.
-    std::string result;
-
-    // Reserve the anticipated capaticity to prevent several reallocations.
-    result.reserve(std::min(max_length, remaining_bytes_));
-    for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) {
-      char next = ConvertUnsignedToSigned<char>(data_ptr_[0]);
-      Advance(1);
-      if (next == '\\' && remaining_bytes_ != 0) {
-        next = ConvertUnsignedToSigned<char>(data_ptr_[0]);
-        Advance(1);
-        if (next != '\\')
-          break;
-      }
-      result += next;
-    }
-
-    result.shrink_to_fit();
-    return result;
-  }
-
-  // Returns a std::vector containing all remaining bytes of the input data.
-  template <typename T> std::vector<T> ConsumeRemainingBytes() {
-    return ConsumeBytes<T>(remaining_bytes_);
-  }
-
-  // Prefer using |ConsumeRemainingBytes| unless you actually need a std::string
-  // object.
-  // Returns a std::vector containing all remaining bytes of the input data.
-  std::string ConsumeRemainingBytesAsString() {
-    return ConsumeBytesAsString(remaining_bytes_);
-  }
-
-  // Returns a number in the range [Type's min, Type's max]. The value might
-  // not be uniformly distributed in the given range. If there's no input data
-  // left, always returns |min|.
-  template <typename T> T ConsumeIntegral() {
-    return ConsumeIntegralInRange(std::numeric_limits<T>::min(),
-                                  std::numeric_limits<T>::max());
-  }
-
-  // Reads one byte and returns a bool, or false when no data remains.
-  bool ConsumeBool() { return 1 & ConsumeIntegral<uint8_t>(); }
-
-  // Returns a copy of a value selected from a fixed-size |array|.
-  template <typename T, size_t size>
-  T PickValueInArray(const T (&array)[size]) {
-    static_assert(size > 0, "The array must be non empty.");
-    return array[ConsumeIntegralInRange<size_t>(0, size - 1)];
-  }
-
-  template <typename T>
-  T PickValueInArray(std::initializer_list<const T> list) {
-    // static_assert(list.size() > 0, "The array must be non empty.");
-    return *(list.begin() + ConsumeIntegralInRange<size_t>(0, list.size() - 1));
-  }
-
-  // Return an enum value. The enum must start at 0 and be contiguous. It must
-  // also contain |kMaxValue| aliased to its largest (inclusive) value. Such as:
-  // enum class Foo { SomeValue, OtherValue, kMaxValue = OtherValue };
-  template <typename T> T ConsumeEnum() {
-    static_assert(std::is_enum<T>::value, "|T| must be an enum type.");
-    return static_cast<T>(ConsumeIntegralInRange<uint32_t>(
-        0, static_cast<uint32_t>(T::kMaxValue)));
-  }
-
-  // Reports the remaining bytes available for fuzzed input.
-  size_t remaining_bytes() { return remaining_bytes_; }
-
- private:
-  FuzzedDataProvider(const FuzzedDataProvider &) = delete;
-  FuzzedDataProvider &operator=(const FuzzedDataProvider &) = delete;
-
-  void Advance(size_t num_bytes) {
-    if (num_bytes > remaining_bytes_)
-      abort();
-
-    data_ptr_ += num_bytes;
-    remaining_bytes_ -= num_bytes;
-  }
-
-  template <typename T>
-  std::vector<T> ConsumeBytes(size_t size, size_t num_bytes_to_consume) {
-    static_assert(sizeof(T) == sizeof(uint8_t), "Incompatible data type.");
-
-    // The point of using the size-based constructor below is to increase the
-    // odds of having a vector object with capacity being equal to the length.
-    // That part is always implementation specific, but at least both libc++ and
-    // libstdc++ allocate the requested number of bytes in that constructor,
-    // which seems to be a natural choice for other implementations as well.
-    // To increase the odds even more, we also call |shrink_to_fit| below.
-    std::vector<T> result(size);
-    std::memcpy(result.data(), data_ptr_, num_bytes_to_consume);
-    Advance(num_bytes_to_consume);
-
-    // Even though |shrink_to_fit| is also implementation specific, we expect it
-    // to provide an additional assurance in case vector's constructor allocated
-    // a buffer which is larger than the actual amount of data we put inside it.
-    result.shrink_to_fit();
-    return result;
-  }
-
-  template <typename TS, typename TU> TS ConvertUnsignedToSigned(TU value) {
-    static_assert(sizeof(TS) == sizeof(TU), "Incompatible data types.");
-    static_assert(!std::numeric_limits<TU>::is_signed,
-                  "Source type must be unsigned.");
-
-    // TODO(Dor1s): change to `if constexpr` once C++17 becomes mainstream.
-    if (std::numeric_limits<TS>::is_modulo)
-      return static_cast<TS>(value);
-
-    // Avoid using implementation-defined unsigned to signer conversions.
-    // To learn more, see https://stackoverflow.com/questions/13150449.
-    if (value <= std::numeric_limits<TS>::max())
-      return static_cast<TS>(value);
-    else {
-      constexpr auto TS_min = std::numeric_limits<TS>::min();
-      return TS_min + static_cast<char>(value - TS_min);
-    }
-  }
-
-  const uint8_t *data_ptr_;
-  size_t remaining_bytes_;
-};
-
-#endif // LLVM_FUZZER_FUZZED_DATA_PROVIDER_H_
diff --git a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
index 92f6274804868..4c060a61e98ed 100644
--- a/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
+++ b/compiler-rt/lib/hwasan/hwasan_tag_mismatch_aarch64.S
@@ -51,14 +51,60 @@
 // +---------------------------------+ <-- [x30 / SP]
 
 // This function takes two arguments:
-//   * x0: The address of read/write instruction that caused HWASan check fail.
-//   * x1: The tag size.
+//   * x0: The data address.
+//   * x1: The encoded access info for the failing access.
 
+// This function has two entry points. The first, __hwasan_tag_mismatch, is used
+// by clients that were compiled without short tag checks (i.e. binaries built
+// by older compilers and binaries targeting older runtimes). In this case the
+// outlined tag check will be missing the code handling short tags (which won't
+// be used in the binary's own stack variables but may be used on the heap
+// or stack variables in other binaries), so the check needs to be done here.
+//
+// The second, __hwasan_tag_mismatch_v2, is used by binaries targeting newer
+// runtimes. This entry point bypasses the short tag check since it will have
+// already been done as part of the outlined tag check. Since tag mismatches are
+// uncommon, there isn't a significant performance benefit to being able to
+// bypass the check; the main benefits are that we can sometimes avoid
+// clobbering the x17 register in error reports, and that the program will have
+// a runtime dependency on the __hwasan_tag_mismatch_v2 symbol therefore it will
+// fail to start up given an older (i.e. incompatible) runtime.
 .section .text
 .file "hwasan_tag_mismatch_aarch64.S"
 .global __hwasan_tag_mismatch
 .type __hwasan_tag_mismatch, %function
 __hwasan_tag_mismatch:
+  // Compute the granule position one past the end of the access.
+  mov x16, #1
+  and x17, x1, #0xf
+  lsl x16, x16, x17
+  and x17, x0, #0xf
+  add x17, x16, x17
+
+  // Load the shadow byte again and check whether it is a short tag within the
+  // range of the granule position computed above.
+  ubfx x16, x0, #4, #52
+  ldrb w16, [x9, x16]
+  cmp w16, #0xf
+  b.hi __hwasan_tag_mismatch_v2
+  cmp w16, w17
+  b.lo __hwasan_tag_mismatch_v2
+
+  // Load the real tag from the last byte of the granule and compare against
+  // the pointer tag.
+  orr x16, x0, #0xf
+  ldrb w16, [x16]
+  cmp x16, x0, lsr #56
+  b.ne __hwasan_tag_mismatch_v2
+
+  // Restore x0, x1 and sp to their values from before the __hwasan_tag_mismatch
+  // call and resume execution.
+  ldp x0, x1, [sp], #256
+  ret
+
+.global __hwasan_tag_mismatch_v2
+.type __hwasan_tag_mismatch_v2, %function
+__hwasan_tag_mismatch_v2:
   CFI_STARTPROC
 
   // Set the CFA to be the return address for caller of __hwasan_check_*. Note
diff --git a/compiler-rt/lib/lsan/lsan.cpp b/compiler-rt/lib/lsan/lsan.cpp
index deabf044afc4d..4ce03046ffb7f 100644
--- a/compiler-rt/lib/lsan/lsan.cpp
+++ b/compiler-rt/lib/lsan/lsan.cpp
@@ -89,7 +89,7 @@ static void InitializeFlags() {
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
-  stack->Unwind(sig.pc, sig.bp, sig.context,
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }
 
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index 86bd120e776a6..9ff9f4c5d1c97 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -570,11 +570,7 @@ static bool CheckForLeaks() {
   EnsureMainThreadIDIsCorrect();
   CheckForLeaksParam param;
   param.success = false;
-  LockThreadRegistry();
-  LockAllocator();
-  DoStopTheWorld(CheckForLeaksCallback, &param);
-  UnlockAllocator();
-  UnlockThreadRegistry();
+  LockStuffAndStopTheWorld(CheckForLeaksCallback, &param);
 
   if (!param.success) {
     Report("LeakSanitizer has encountered a fatal error.\n");
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index 58dc00faaee5c..d24abe31b71b5 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -129,8 +129,9 @@ struct RootRegion {
 InternalMmapVector<RootRegion> const *GetRootRegions();
 void ScanRootRegion(Frontier *frontier, RootRegion const &region,
                     uptr region_begin, uptr region_end, bool is_readable);
-// Run stoptheworld while holding any platform-specific locks.
-void DoStopTheWorld(StopTheWorldCallback callback, void* argument);
+// Run stoptheworld while holding any platform-specific locks, as well as the
+// allocator and thread registry locks.
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void* argument);
 
 void ScanRangeForPointers(uptr begin, uptr end,
                           Frontier *frontier,
diff --git a/compiler-rt/lib/lsan/lsan_common_linux.cpp b/compiler-rt/lib/lsan/lsan_common_linux.cpp
index 9ce27a983b502..ea1a4a2f569d0 100644
--- a/compiler-rt/lib/lsan/lsan_common_linux.cpp
+++ b/compiler-rt/lib/lsan/lsan_common_linux.cpp
@@ -115,10 +115,14 @@ void HandleLeaks() {
   if (common_flags()->exitcode) Die();
 }
 
-static int DoStopTheWorldCallback(struct dl_phdr_info *info, size_t size,
-                                  void *data) {
+static int LockStuffAndStopTheWorldCallback(struct dl_phdr_info *info,
+                                            size_t size, void *data) {
+  LockThreadRegistry();
+  LockAllocator();
   DoStopTheWorldParam *param = reinterpret_cast<DoStopTheWorldParam *>(data);
   StopTheWorld(param->callback, param->argument);
+  UnlockAllocator();
+  UnlockThreadRegistry();
   return 1;
 }
 
@@ -130,9 +134,9 @@ static int DoStopTheWorldCallback(struct dl_phdr_info *info, size_t size,
 // while holding the libdl lock in the parent thread, we can safely reenter it
 // in the tracer. The solution is to run stoptheworld from a dl_iterate_phdr()
 // callback in the parent thread.
-void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void *argument) {
   DoStopTheWorldParam param = {callback, argument};
-  dl_iterate_phdr(DoStopTheWorldCallback, &param);
+  dl_iterate_phdr(LockStuffAndStopTheWorldCallback, &param);
 }
 
 } // namespace __lsan
diff --git a/compiler-rt/lib/lsan/lsan_common_mac.cpp b/compiler-rt/lib/lsan/lsan_common_mac.cpp
index 5204a6624edcc..c1804e93c11db 100644
--- a/compiler-rt/lib/lsan/lsan_common_mac.cpp
+++ b/compiler-rt/lib/lsan/lsan_common_mac.cpp
@@ -193,8 +193,12 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) {
 // causes rare race conditions.
 void HandleLeaks() {}
 
-void DoStopTheWorld(StopTheWorldCallback callback, void *argument) {
+void LockStuffAndStopTheWorld(StopTheWorldCallback callback, void *argument) {
+  LockThreadRegistry();
+  LockAllocator();
   StopTheWorld(callback, argument);
+  UnlockAllocator();
+  UnlockThreadRegistry();
 }
 
 } // namespace __lsan
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index f06d5fff70635..f642bb807bc88 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -345,6 +345,55 @@ INTERCEPTOR(void, thr_exit, tid_t *state) {
 #define LSAN_MAYBE_INTERCEPT_THR_EXIT
 #endif
 
+#if SANITIZER_INTERCEPT___CXA_ATEXIT
+INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
+            void *dso_handle) {
+  __lsan::ScopedInterceptorDisabler disabler;
+  return REAL(__cxa_atexit)(func, arg, dso_handle);
+}
+#define LSAN_MAYBE_INTERCEPT___CXA_ATEXIT INTERCEPT_FUNCTION(__cxa_atexit)
+#else
+#define LSAN_MAYBE_INTERCEPT___CXA_ATEXIT
+#endif
+
+#if SANITIZER_INTERCEPT_ATEXIT
+INTERCEPTOR(int, atexit, void (*f)()) {
+  __lsan::ScopedInterceptorDisabler disabler;
+  return REAL(__cxa_atexit)((void (*)(void *a))f, 0, 0);
+}
+#define LSAN_MAYBE_INTERCEPT_ATEXIT INTERCEPT_FUNCTION(atexit)
+#else
+#define LSAN_MAYBE_INTERCEPT_ATEXIT
+#endif
+
+#if SANITIZER_INTERCEPT_PTHREAD_ATFORK
+extern "C" {
+extern int _pthread_atfork(void (*prepare)(), void (*parent)(),
+                           void (*child)());
+};
+
+INTERCEPTOR(int, pthread_atfork, void (*prepare)(), void (*parent)(),
+            void (*child)()) {
+  __lsan::ScopedInterceptorDisabler disabler;
+  // REAL(pthread_atfork) cannot be called due to symbol indirections at least
+  // on NetBSD
+  return _pthread_atfork(prepare, parent, child);
+}
+#define LSAN_MAYBE_INTERCEPT_PTHREAD_ATFORK INTERCEPT_FUNCTION(pthread_atfork)
+#else
+#define LSAN_MAYBE_INTERCEPT_PTHREAD_ATFORK
+#endif
+
+#if SANITIZER_INTERCEPT_STRERROR
+INTERCEPTOR(char *, strerror, int errnum) {
+  __lsan::ScopedInterceptorDisabler disabler;
+  return REAL(strerror)(errnum);
+}
+#define LSAN_MAYBE_INTERCEPT_STRERROR INTERCEPT_FUNCTION(strerror)
+#else
+#define LSAN_MAYBE_INTERCEPT_STRERROR
+#endif
+
 struct ThreadParam {
   void *(*callback)(void *arg);
   void *param;
@@ -454,6 +503,12 @@ void InitializeInterceptors() {
   LSAN_MAYBE_INTERCEPT__LWP_EXIT;
   LSAN_MAYBE_INTERCEPT_THR_EXIT;
 
+  LSAN_MAYBE_INTERCEPT___CXA_ATEXIT;
+  LSAN_MAYBE_INTERCEPT_ATEXIT;
+  LSAN_MAYBE_INTERCEPT_PTHREAD_ATFORK;
+
+  LSAN_MAYBE_INTERCEPT_STRERROR;
+
 #if !SANITIZER_NETBSD && !SANITIZER_FREEBSD
   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
     Report("LeakSanitizer: failed to create thread key.\n");
diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp
index da3534bc53f71..6ea63cb2c48ff 100644
--- a/compiler-rt/lib/msan/msan.cpp
+++ b/compiler-rt/lib/msan/msan.cpp
@@ -378,7 +378,7 @@ void __msan_warning_noreturn() {
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
-  stack->Unwind(sig.pc, sig.bp, sig.context,
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }
 
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index ff760e43d6343..1d9d9f7986d76 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -765,17 +765,24 @@ INTERCEPTOR(char *, fgets_unlocked, char *s, int size, void *stream) {
 #define MSAN_MAYBE_INTERCEPT_FGETS_UNLOCKED
 #endif
 
+#define INTERCEPTOR_GETRLIMIT_BODY(func, resource, rlim)  \
+  if (msan_init_is_running)                               \
+    return REAL(getrlimit)(resource, rlim);               \
+  ENSURE_MSAN_INITED();                                   \
+  int res = REAL(func)(resource, rlim);                   \
+  if (!res)                                               \
+    __msan_unpoison(rlim, __sanitizer::struct_rlimit_sz); \
+  return res
+
 INTERCEPTOR(int, getrlimit, int resource, void *rlim) {
-  if (msan_init_is_running)
-    return REAL(getrlimit)(resource, rlim);
-  ENSURE_MSAN_INITED();
-  int res = REAL(getrlimit)(resource, rlim);
-  if (!res)
-    __msan_unpoison(rlim, __sanitizer::struct_rlimit_sz);
-  return res;
+  INTERCEPTOR_GETRLIMIT_BODY(getrlimit, resource, rlim);
 }
 
 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
+INTERCEPTOR(int, __getrlimit, int resource, void *rlim) {
+  INTERCEPTOR_GETRLIMIT_BODY(__getrlimit, resource, rlim);
+}
+
 INTERCEPTOR(int, getrlimit64, int resource, void *rlim) {
   if (msan_init_is_running) return REAL(getrlimit64)(resource, rlim);
   ENSURE_MSAN_INITED();
@@ -806,10 +813,12 @@ INTERCEPTOR(int, prlimit64, int pid, int resource, void *new_rlimit,
   return res;
 }
 
+#define MSAN_MAYBE_INTERCEPT___GETRLIMIT INTERCEPT_FUNCTION(__getrlimit)
 #define MSAN_MAYBE_INTERCEPT_GETRLIMIT64 INTERCEPT_FUNCTION(getrlimit64)
 #define MSAN_MAYBE_INTERCEPT_PRLIMIT INTERCEPT_FUNCTION(prlimit)
 #define MSAN_MAYBE_INTERCEPT_PRLIMIT64 INTERCEPT_FUNCTION(prlimit64)
 #else
+#define MSAN_MAYBE_INTERCEPT___GETRLIMIT
 #define MSAN_MAYBE_INTERCEPT_GETRLIMIT64
 #define MSAN_MAYBE_INTERCEPT_PRLIMIT
 #define MSAN_MAYBE_INTERCEPT_PRLIMIT64
@@ -1678,6 +1687,7 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(socketpair);
   MSAN_MAYBE_INTERCEPT_FGETS_UNLOCKED;
   INTERCEPT_FUNCTION(getrlimit);
+  MSAN_MAYBE_INTERCEPT___GETRLIMIT;
   MSAN_MAYBE_INTERCEPT_GETRLIMIT64;
   MSAN_MAYBE_INTERCEPT_PRLIMIT;
   MSAN_MAYBE_INTERCEPT_PRLIMIT64;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
index 7ea896ef2d441..587b1ea227226 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc
@@ -36,6 +36,7 @@
 //   COMMON_INTERCEPTOR_MMAP_IMPL
 //   COMMON_INTERCEPTOR_COPY_STRING
 //   COMMON_INTERCEPTOR_STRNDUP_IMPL
+//   COMMON_INTERCEPTOR_STRERROR
 //===----------------------------------------------------------------------===//
 
 #include "interception/interception.h"
@@ -301,6 +302,10 @@ bool PlatformHasDifferentMemcpyAndMemmove();
   return new_mem;
 #endif
 
+#ifndef COMMON_INTERCEPTOR_STRERROR
+#define COMMON_INTERCEPTOR_STRERROR() {}
+#endif
+
 struct FileMetadata {
   // For open_memstream().
   char **addr;
@@ -3677,6 +3682,7 @@ INTERCEPTOR(int, sched_getparam, int pid, void *param) {
 INTERCEPTOR(char *, strerror, int errnum) {
   void *ctx;
   COMMON_INTERCEPTOR_ENTER(ctx, strerror, errnum);
+  COMMON_INTERCEPTOR_STRERROR();
   char *res = REAL(strerror)(errnum);
   if (res) COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, REAL(strlen)(res) + 1);
   return res;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
index c72554973b05e..c78b6e10b6895 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc
@@ -14,6 +14,7 @@ INTERFACE_FUNCTION(__sanitizer_set_death_callback)
 INTERFACE_FUNCTION(__sanitizer_set_report_path)
 INTERFACE_FUNCTION(__sanitizer_set_report_fd)
 INTERFACE_FUNCTION(__sanitizer_verify_contiguous_container)
+INTERFACE_WEAK_FUNCTION(__sanitizer_on_print)
 INTERFACE_WEAK_FUNCTION(__sanitizer_report_error_summary)
 INTERFACE_WEAK_FUNCTION(__sanitizer_sandbox_on_notify)
 // Sanitizer weak hooks
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index cf079c4355455..c3b6f285569a8 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -567,5 +567,9 @@
 #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD)
 
 #define SANITIZER_INTERCEPT_GETRANDOM SI_LINUX
+#define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD
+#define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD
+#define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD
+
 
 #endif  // #ifndef SANITIZER_PLATFORM_INTERCEPTORS_H
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
index 69919ae74fab1..304b3a01a08b6 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp
@@ -328,7 +328,6 @@ bool MmapFixedNoReserve(uptr fixed_addr, uptr size, const char *name) {
 
 bool MmapFixedSuperNoReserve(uptr fixed_addr, uptr size, const char *name) {
 #if SANITIZER_FREEBSD
-  int flags = 0;
   if (common_flags()->no_huge_pages_for_shadow)
     return MmapFixedNoReserve(fixed_addr, size, name);
   // MAP_NORESERVE is implicit with FreeBSD
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
index 9d1c544786d34..7063de257a9f0 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_printf.cpp
@@ -229,15 +229,21 @@ void SetPrintfAndReportCallback(void (*callback)(const char *)) {
 // Can be overriden in frontend.
 #if SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS)
 // Implementation must be defined in frontend.
+// TODO(morehouse): Remove OnPrint after migrating Go to __sanitizer_on_print.
 extern "C" void OnPrint(const char *str);
+extern "C" void __sanitizer_on_print(const char *str);
 #else
-SANITIZER_INTERFACE_WEAK_DEF(void, OnPrint, const char *str) {
+SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_on_print, const char *str) {
   (void)str;
 }
 #endif
 
 static void CallPrintfAndReportCallback(const char *str) {
+#if SANITIZER_GO && defined(TSAN_EXTERNAL_HOOKS)
+  // TODO(morehouse): Remove OnPrint after migrating Go to __sanitizer_on_print.
   OnPrint(str);
+#endif
+  __sanitizer_on_print(str);
   if (PrintfAndReportCallback)
     PrintfAndReportCallback(str);
 }
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
index 1a43759e38ae1..b2628dcc4dc1f 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp
@@ -27,6 +27,8 @@
 
 namespace __sanitizer {
 
+namespace {
+
 //---------------------------- UnwindSlow --------------------------------------
 
 typedef struct {
@@ -46,38 +48,6 @@ release_my_map_info_list_func release_my_map_info_list;
 unwind_backtrace_signal_arch_func unwind_backtrace_signal_arch;
 } // extern "C"
 
-#if SANITIZER_ANDROID
-void SanitizerInitializeUnwinder() {
-  if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
-
-  // Pre-lollipop Android can not unwind through signal handler frames with
-  // libgcc unwinder, but it has a libcorkscrew.so library with the necessary
-  // workarounds.
-  void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
-  if (!p) {
-    VReport(1,
-            "Failed to open libcorkscrew.so. You may see broken stack traces "
-            "in SEGV reports.");
-    return;
-  }
-  acquire_my_map_info_list =
-      (acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
-  release_my_map_info_list =
-      (release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
-  unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
-      p, "unwind_backtrace_signal_arch");
-  if (!acquire_my_map_info_list || !release_my_map_info_list ||
-      !unwind_backtrace_signal_arch) {
-    VReport(1,
-            "Failed to find one of the required symbols in libcorkscrew.so. "
-            "You may see broken stack traces in SEGV reports.");
-    acquire_my_map_info_list = 0;
-    unwind_backtrace_signal_arch = 0;
-    release_my_map_info_list = 0;
-  }
-}
-#endif
-
 #if defined(__arm__) && !SANITIZER_NETBSD
 // NetBSD uses dwarf EH
 #define UNWIND_STOP _URC_END_OF_STACK
@@ -119,6 +89,40 @@ _Unwind_Reason_Code Unwind_Trace(struct _Unwind_Context *ctx, void *param) {
   return UNWIND_CONTINUE;
 }
 
+}  // namespace
+
+#if SANITIZER_ANDROID
+void SanitizerInitializeUnwinder() {
+  if (AndroidGetApiLevel() >= ANDROID_LOLLIPOP_MR1) return;
+
+  // Pre-lollipop Android can not unwind through signal handler frames with
+  // libgcc unwinder, but it has a libcorkscrew.so library with the necessary
+  // workarounds.
+  void *p = dlopen("libcorkscrew.so", RTLD_LAZY);
+  if (!p) {
+    VReport(1,
+            "Failed to open libcorkscrew.so. You may see broken stack traces "
+            "in SEGV reports.");
+    return;
+  }
+  acquire_my_map_info_list =
+      (acquire_my_map_info_list_func)(uptr)dlsym(p, "acquire_my_map_info_list");
+  release_my_map_info_list =
+      (release_my_map_info_list_func)(uptr)dlsym(p, "release_my_map_info_list");
+  unwind_backtrace_signal_arch = (unwind_backtrace_signal_arch_func)(uptr)dlsym(
+      p, "unwind_backtrace_signal_arch");
+  if (!acquire_my_map_info_list || !release_my_map_info_list ||
+      !unwind_backtrace_signal_arch) {
+    VReport(1,
+            "Failed to find one of the required symbols in libcorkscrew.so. "
+            "You may see broken stack traces in SEGV reports.");
+    acquire_my_map_info_list = 0;
+    unwind_backtrace_signal_arch = 0;
+    release_my_map_info_list = 0;
+  }
+}
+#endif
+
 void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) {
   CHECK_GE(max_depth, 2);
   size = 0;
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cpp b/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cpp
index a6081a8931ab9..b14bbf76d9a76 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cpp
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_win_weak_interception.cpp
@@ -38,6 +38,7 @@ int interceptWhenPossible(uptr dll_function, const char *real_function) {
 
 // Declare weak hooks.
 extern "C" {
+void __sanitizer_on_print(const char *str);
 void __sanitizer_weak_hook_memcmp(uptr called_pc, const void *s1,
                                   const void *s2, uptr n, int result);
 void __sanitizer_weak_hook_strcmp(uptr called_pc, const char *s1,
diff --git a/compiler-rt/lib/tsan/CMakeLists.txt b/compiler-rt/lib/tsan/CMakeLists.txt
index fcadacfe1bc56..9fd3e2d779249 100644
--- a/compiler-rt/lib/tsan/CMakeLists.txt
+++ b/compiler-rt/lib/tsan/CMakeLists.txt
@@ -32,7 +32,7 @@ set(TSAN_SOURCES
   rtl/tsan_fd.cpp
   rtl/tsan_flags.cpp
   rtl/tsan_ignoreset.cpp
-  rtl/tsan_interceptors.cpp
+  rtl/tsan_interceptors_posix.cpp
   rtl/tsan_interface.cpp
   rtl/tsan_interface_ann.cpp
   rtl/tsan_interface_atomic.cpp
diff --git a/compiler-rt/lib/tsan/go/tsan_go.cpp b/compiler-rt/lib/tsan/go/tsan_go.cpp
index 271f6bc00cd08..f5998c0c78166 100644
--- a/compiler-rt/lib/tsan/go/tsan_go.cpp
+++ b/compiler-rt/lib/tsan/go/tsan_go.cpp
@@ -54,20 +54,31 @@ struct SymbolizeCodeContext {
 };
 
 SymbolizedStack *SymbolizeCode(uptr addr) {
-  SymbolizedStack *s = SymbolizedStack::New(addr);
-  SymbolizeCodeContext cbctx;
-  internal_memset(&cbctx, 0, sizeof(cbctx));
-  cbctx.pc = addr;
-  go_runtime_cb(CallbackSymbolizeCode, &cbctx);
-  if (cbctx.res) {
+  SymbolizedStack *first = SymbolizedStack::New(addr);
+  SymbolizedStack *s = first;
+  for (;;) {
+    SymbolizeCodeContext cbctx;
+    internal_memset(&cbctx, 0, sizeof(cbctx));
+    cbctx.pc = addr;
+    go_runtime_cb(CallbackSymbolizeCode, &cbctx);
+    if (cbctx.res == 0)
+      break;
     AddressInfo &info = s->info;
     info.module_offset = cbctx.off;
     info.function = internal_strdup(cbctx.func ? cbctx.func : "??");
     info.file = internal_strdup(cbctx.file ? cbctx.file : "-");
     info.line = cbctx.line;
     info.column = 0;
+
+    if (cbctx.pc == addr) // outermost (non-inlined) function
+      break;
+    addr = cbctx.pc;
+    // Allocate a stack entry for the parent of the inlined function.
+    SymbolizedStack *s2 = SymbolizedStack::New(addr);
+    s->next = s2;
+    s = s2;
   }
-  return s;
+  return first;
 }
 
 struct SymbolizeDataContext {
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cpp b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
similarity index 99%
rename from compiler-rt/lib/tsan/rtl/tsan_interceptors.cpp
rename to compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
index 4aa9086a2156f..d1d83e23d5585 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interceptors.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp
@@ -1,4 +1,4 @@
-//===-- tsan_interceptors.cpp ---------------------------------------------===//
+//===-- tsan_interceptors_posix.cpp ---------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface.h b/compiler-rt/lib/tsan/rtl/tsan_interface.h
index b2f0f30c47ea6..6d7286ca5b8af 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface.h
@@ -94,6 +94,11 @@ void __tsan_read_range(void *addr, unsigned long size);
 SANITIZER_INTERFACE_ATTRIBUTE
 void __tsan_write_range(void *addr, unsigned long size);
 
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_read_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_write_range_pc(void *addr, unsigned long size, void *pc);  // NOLINT
+
 // User may provide function that would be called right when TSan detects
 // an error. The argument 'report' is an opaque pointer that can be used to
 // gather additional information using other TSan report API functions.
diff --git a/compiler-rt/lib/tsan/rtl/tsan_interface_inl.h b/compiler-rt/lib/tsan/rtl/tsan_interface_inl.h
index 193b91b2ec2aa..f955ddf99247c 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_interface_inl.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_interface_inl.h
@@ -122,3 +122,11 @@ void __tsan_read_range(void *addr, uptr size) {
 void __tsan_write_range(void *addr, uptr size) {
   MemoryAccessRange(cur_thread(), CALLERPC, (uptr)addr, size, true);
 }
+
+void __tsan_read_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, false);
+}
+
+void __tsan_write_range_pc(void *addr, uptr size, void *pc) {
+  MemoryAccessRange(cur_thread(), (uptr)pc, (uptr)addr, size, true);
+}
diff --git a/compiler-rt/lib/tsan/rtl/tsan_report.cpp b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
index 655aa5f9123b2..368f1ca8adf2c 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_report.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_report.cpp
@@ -298,7 +298,7 @@ static bool FrameIsInternal(const SymbolizedStack *frame) {
   const char *file = frame->info.file;
   const char *module = frame->info.module;
   if (file != 0 &&
-      (internal_strstr(file, "tsan_interceptors.cpp") ||
+      (internal_strstr(file, "tsan_interceptors_posix.cpp") ||
        internal_strstr(file, "sanitizer_common_interceptors.inc") ||
        internal_strstr(file, "tsan_interface_")))
     return true;
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
index 39b9cea52f744..3f3c0cce119c1 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl.cpp
@@ -338,7 +338,7 @@ static void CheckShadowMapping() {
 #if !SANITIZER_GO
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
-  stack->Unwind(sig.pc, sig.bp, sig.context,
+  stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
                 common_flags()->fast_unwind_on_fatal);
 }
 
diff --git a/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp b/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
index 627b3c4d89bb0..2c91db8ca3974 100644
--- a/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
+++ b/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp
@@ -45,8 +45,9 @@ namespace __ubsan {
 
 static void OnStackUnwind(const SignalContext &sig, const void *,
                           BufferedStackTrace *stack) {
-  ubsan_GetStackTrace(stack, kStackTraceMax, sig.pc, sig.bp, sig.context,
-                common_flags()->fast_unwind_on_fatal);
+  ubsan_GetStackTrace(stack, kStackTraceMax,
+                      StackTrace::GetNextInstructionPc(sig.pc), sig.bp,
+                      sig.context, common_flags()->fast_unwind_on_fatal);
 }
 
 static void UBsanOnDeadlySignal(int signo, void *siginfo, void *context) {
diff --git a/compiler-rt/test/asan/CMakeLists.txt b/compiler-rt/test/asan/CMakeLists.txt
index 1892c881849a1..79e34a01619a9 100644
--- a/compiler-rt/test/asan/CMakeLists.txt
+++ b/compiler-rt/test/asan/CMakeLists.txt
@@ -82,15 +82,22 @@ endforeach()
 # variable to select which iOS device or simulator to use, e.g.:
 # SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
 if(APPLE)
+  # FIXME(dliew): This logic should be refactored to the way UBSan Darwin
+  # testing is done.
   set(EXCLUDE_FROM_ALL ON)
 
   set(ASAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})
   set(ASAN_TEST_DYNAMIC True)
 
-  foreach(arch ${DARWIN_iossim_ARCHS})
+  list_intersect(ASAN_TEST_IOSSIM_ARCHS ASAN_SUPPORTED_ARCH DARWIN_iossim_ARCHS)
+  foreach(arch ${ASAN_TEST_IOSSIM_ARCHS})
     set(ASAN_TEST_APPLE_PLATFORM "iossim")
     set(ASAN_TEST_TARGET_ARCH ${arch})
-    set(ASAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_iossim_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+    get_test_cflags_for_apple_platform(
+      "${ASAN_TEST_APPLE_PLATFORM}"
+      "${ASAN_TEST_TARGET_ARCH}"
+      ASAN_TEST_TARGET_CFLAGS
+      )
     set(ASAN_TEST_CONFIG_SUFFIX "-${arch}-${ASAN_TEST_APPLE_PLATFORM}")
     get_bits_for_arch(${arch} ASAN_TEST_BITS)
     string(TOUPPER ${arch} ARCH_UPPER_CASE)
@@ -104,10 +111,14 @@ if(APPLE)
       DEPENDS ${ASAN_TEST_DEPS})
   endforeach()
 
-  foreach (arch ${DARWIN_ios_ARCHS})
+  list_intersect(ASAN_TEST_IOS_ARCHS ASAN_SUPPORTED_ARCH DARWIN_ios_ARCHS)
+  foreach (arch ${ASAN_TEST_IOS_ARCHS})
     set(ASAN_TEST_APPLE_PLATFORM "ios")
     set(ASAN_TEST_TARGET_ARCH ${arch})
-    set(ASAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_ios_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+    get_test_cflags_for_apple_platform(
+      "${ASAN_TEST_APPLE_PLATFORM}"
+      "${arch}"
+      ASAN_TEST_TARGET_CFLAGS)
     set(ASAN_TEST_CONFIG_SUFFIX "-${arch}-${ASAN_TEST_APPLE_PLATFORM}")
     get_bits_for_arch(${arch} ASAN_TEST_BITS)
     string(TOUPPER ${arch} ARCH_UPPER_CASE)
diff --git a/compiler-rt/test/asan/TestCases/inline.cpp b/compiler-rt/test/asan/TestCases/inline.cpp
index daeb7b49eb22a..12bd27e675844 100644
--- a/compiler-rt/test/asan/TestCases/inline.cpp
+++ b/compiler-rt/test/asan/TestCases/inline.cpp
@@ -13,6 +13,7 @@ int f(int *p) {
 int main(int argc, char **argv) {
   int * volatile x = (int*)malloc(2*sizeof(int) + 2);
   int res = f(x + 2);
+  free(x);
   if (res)
     exit(0);
   return 0;
diff --git a/compiler-rt/test/builtins/CMakeLists.txt b/compiler-rt/test/builtins/CMakeLists.txt
index 43b7e295acbd0..b80b8a112b5b7 100644
--- a/compiler-rt/test/builtins/CMakeLists.txt
+++ b/compiler-rt/test/builtins/CMakeLists.txt
@@ -25,9 +25,12 @@ if (MSVC AND NOT "${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
 endif()
 pythonize_bool(BUILTINS_IS_MSVC)
 
-#TODO: Add support for Apple.
-if (NOT APPLE)
-foreach(arch ${BUILTIN_SUPPORTED_ARCH})
+set(BUILTIN_TEST_ARCH ${BUILTIN_SUPPORTED_ARCH})
+if(APPLE)
+  darwin_filter_host_archs(BUILTIN_SUPPORTED_ARCH BUILTIN_TEST_ARCH)
+endif()
+
+foreach(arch ${BUILTIN_TEST_ARCH})
   set(BUILTINS_TEST_TARGET_ARCH ${arch})
   string(TOLOWER "-${arch}-${OS_NAME}" BUILTINS_TEST_CONFIG_SUFFIX)
   get_test_cc_for_arch(${arch} BUILTINS_TEST_TARGET_CC BUILTINS_TEST_TARGET_CFLAGS)
@@ -49,7 +52,8 @@ foreach(arch ${BUILTIN_SUPPORTED_ARCH})
     )
   list(APPEND BUILTINS_TESTSUITES ${CMAKE_CURRENT_BINARY_DIR}/Unit/${CONFIG_NAME})
 endforeach()
-endif()
+
+# TODO: Add support for running tests on iOS and iOS simulator.
 
 add_lit_testsuite(check-builtins "Running the Builtins tests"
   ${BUILTINS_TESTSUITES}
diff --git a/compiler-rt/test/builtins/Unit/clear_cache_test.c b/compiler-rt/test/builtins/Unit/clear_cache_test.c
index e85cc30271373..4220ba65190f9 100644
--- a/compiler-rt/test/builtins/Unit/clear_cache_test.c
+++ b/compiler-rt/test/builtins/Unit/clear_cache_test.c
@@ -13,25 +13,16 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdint.h>
+
 #if defined(_WIN32)
 #include <windows.h>
-static uintptr_t get_page_size() {
-    SYSTEM_INFO si;
-    GetSystemInfo(&si);
-    return si.dwPageSize;
-}
 #else
 #include <unistd.h>
 #include <sys/mman.h>
-
-static uintptr_t get_page_size() {
-    return sysconf(_SC_PAGE_SIZE);
-}
 #endif
 
 extern void __clear_cache(void* start, void* end);
 
-
 typedef int (*pfunc)(void);
 
 // Make these static to avoid ILT jumps for incremental linking on Windows.
@@ -51,34 +42,36 @@ memcpy_f(void *dst, const void *src, size_t n) {
 #endif
 }
 
-unsigned char execution_buffer[128] __attribute__((aligned(8)));
-
 int main()
 {
-    // make executable the page containing execution_buffer 
-    uintptr_t page_size = get_page_size();
-    char* start = (char*)((uintptr_t)execution_buffer & (-page_size));
-    char* end = (char*)((uintptr_t)(&execution_buffer[128+page_size]) & (-page_size));
-#if defined(_WIN32)
-    DWORD dummy_oldProt;
-    MEMORY_BASIC_INFORMATION b;
-    if (!VirtualQuery(start, &b, sizeof(b)))
-        return 1;
-    if (!VirtualProtect(b.BaseAddress, b.RegionSize, PAGE_EXECUTE_READWRITE, &b.Protect))
+    const int kSize = 128;
+#if !defined(_WIN32)
+    uint8_t *execution_buffer = mmap(0, kSize,
+                                     PROT_READ | PROT_WRITE | PROT_EXEC,
+                                     MAP_ANON | MAP_PRIVATE, 0, 0);
+    if (execution_buffer == MAP_FAILED)
+      return 1;
 #else
-    if (mprotect(start, end-start, PROT_READ|PROT_WRITE|PROT_EXEC) != 0)
-#endif
+    HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE, NULL,
+                                       PAGE_EXECUTE_READWRITE, 0, kSize, NULL);
+    if (mapping == NULL)
         return 1;
 
+    uint8_t* execution_buffer = MapViewOfFile(
+        mapping, FILE_MAP_ALL_ACCESS | FILE_MAP_EXECUTE, 0, 0, 0);
+    if (execution_buffer == NULL)
+        return 1;
+#endif
+
     // verify you can copy and execute a function
-    pfunc f1 = (pfunc)memcpy_f(execution_buffer, func1, 128);
-    __clear_cache(execution_buffer, &execution_buffer[128]);
+    pfunc f1 = (pfunc)memcpy_f(execution_buffer, func1, kSize);
+    __clear_cache(execution_buffer, execution_buffer + kSize);
     if ((*f1)() != 1)
         return 1;
 
     // verify you can overwrite a function with another
-    pfunc f2 = (pfunc)memcpy_f(execution_buffer, func2, 128);
-    __clear_cache(execution_buffer, &execution_buffer[128]);
+    pfunc f2 = (pfunc)memcpy_f(execution_buffer, func2, kSize);
+    __clear_cache(execution_buffer, execution_buffer + kSize);
     if ((*f2)() != 2)
         return 1;
 
diff --git a/compiler-rt/test/builtins/Unit/lit.cfg.py b/compiler-rt/test/builtins/Unit/lit.cfg.py
index e290ce107169b..44b86f0d63995 100644
--- a/compiler-rt/test/builtins/Unit/lit.cfg.py
+++ b/compiler-rt/test/builtins/Unit/lit.cfg.py
@@ -29,6 +29,9 @@ def get_required_attr(config, attr_name):
   base_lib = os.path.join(config.compiler_rt_libdir, "clang_rt.builtins%s.lib "
                           % config.target_suffix)
   config.substitutions.append( ("%librt ", base_lib) )
+elif config.host_os  == 'Darwin':
+  base_lib = os.path.join(config.compiler_rt_libdir, "libclang_rt.osx.a ")
+  config.substitutions.append( ("%librt ", base_lib + ' -lSystem ') )
 else:
   base_lib = os.path.join(config.compiler_rt_libdir, "libclang_rt.builtins%s.a"
                           % config.target_suffix)
diff --git a/compiler-rt/test/fuzzer/CMakeLists.txt b/compiler-rt/test/fuzzer/CMakeLists.txt
index 62161d0019bbe..25bdcd2f0d97f 100644
--- a/compiler-rt/test/fuzzer/CMakeLists.txt
+++ b/compiler-rt/test/fuzzer/CMakeLists.txt
@@ -89,12 +89,19 @@ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
 endif()
 
 if (APPLE)
+  # FIXME(dliew): This logic should be refactored to the way UBSan Darwin
+  # testing is done.
   set(EXCLUDE_FROM_ALL ON)
 
-  foreach(arch ${DARWIN_ios_ARCHS})
+  list_intersect(FUZZER_TEST_IOS_ARCHS FUZZER_SUPPORTED_ARCH DARWIN_ios_ARCHS)
+  foreach(arch ${FUZZER_TEST_IOS_ARCHS})
     set(LIBFUZZER_TEST_APPLE_PLATFORM "ios")
     set(LIBFUZZER_TEST_TARGET_ARCH ${arch})
-    set(LIBFUZZER_TEST_FLAGS "-arch ${arch} -isysroot ${DARWIN_ios_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
+    get_test_cflags_for_apple_platform(
+      "${LIBFUZZER_TEST_APPLE_PLATFORM}"
+      "${LIBFUZZER_TEST_TARGET_ARCH}"
+      LIBFUZZER_TEST_FLAGS
+      )
     set(LIBFUZZER_TEST_CONFIG_SUFFIX "-${arch}-${LIBFUZZER_TEST_APPLE_PLATFORM}")
     string(TOUPPER ${arch} ARCH_UPPER_CASE)
     set(CONFIG_NAME "IOS${ARCH_UPPER_CASE}Config")
diff --git a/compiler-rt/test/fuzzer/large.test b/compiler-rt/test/fuzzer/large.test
index 99ebbbe4e7da0..b03b60fdb6503 100644
--- a/compiler-rt/test/fuzzer/large.test
+++ b/compiler-rt/test/fuzzer/large.test
@@ -1,7 +1,4 @@
-REQUIRES: linux
 RUN: %cpp_compiler %S/LargeTest.cpp -o %t-LargeTest
 RUN: %run %t-LargeTest -runs=10000
-RUN: %env_asan_opts=handle_segv=0 %run %t-LargeTest -runs=10000 -lazy_counters=1 2>&1 | FileCheck %s
-RUN:                              %run %t-LargeTest -runs=10000 -lazy_counters=1 2>&1 | FileCheck %s
 
 CHECK: pages of counters where protected; libFuzzer's SEGV handler must be installed
diff --git a/compiler-rt/test/fuzzer/overwrite-input.test b/compiler-rt/test/fuzzer/overwrite-input.test
index e5ddd62cbdc70..eef1b181ad759 100644
--- a/compiler-rt/test/fuzzer/overwrite-input.test
+++ b/compiler-rt/test/fuzzer/overwrite-input.test
@@ -1,3 +1,3 @@
 RUN: %cpp_compiler %S/OverwriteInputTest.cpp -o %t-OverwriteInputTest
 RUN: not %run %t-OverwriteInputTest 2>&1 | FileCheck %s
-CHECK: ERROR: libFuzzer: fuzz target overwrites it's const input
+CHECK: ERROR: libFuzzer: fuzz target overwrites its const input
diff --git a/compiler-rt/test/hwasan/TestCases/stack-oob.c b/compiler-rt/test/hwasan/TestCases/stack-oob.c
index ba74930921ab5..8c8c110554928 100644
--- a/compiler-rt/test/hwasan/TestCases/stack-oob.c
+++ b/compiler-rt/test/hwasan/TestCases/stack-oob.c
@@ -1,4 +1,8 @@
+// RUN: %clang_hwasan_oldrt -DSIZE=2 -O0 %s -o %t && %run %t
+// RUN: %clang_hwasan -DSIZE=2 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clang_hwasan_oldrt -DSIZE=15 -O0 %s -o %t && %run %t
 // RUN: %clang_hwasan -DSIZE=15 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
+// RUN: %clang_hwasan_oldrt -DSIZE=16 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clang_hwasan -DSIZE=16 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clang_hwasan -DSIZE=64 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
 // RUN: %clang_hwasan -DSIZE=0x1000 -O0 %s -o %t && not %run %t 2>&1 | FileCheck %s
@@ -16,9 +20,9 @@ int f() {
 }
 
 int main() {
-  return f();
+  f();
   // CHECK: READ of size 1 at
-  // CHECK: #0 {{.*}} in f{{.*}}stack-oob.c:15
+  // CHECK: #0 {{.*}} in f{{.*}}stack-oob.c:19
 
   // CHECK: is located in stack of threa
 
diff --git a/compiler-rt/test/hwasan/lit.cfg.py b/compiler-rt/test/hwasan/lit.cfg.py
index eead832197259..ee67a261d9199 100644
--- a/compiler-rt/test/hwasan/lit.cfg.py
+++ b/compiler-rt/test/hwasan/lit.cfg.py
@@ -18,9 +18,11 @@
   # equivalent target feature implemented on x86_64.
   clang_hwasan_common_cflags += ["-mcmodel=large"]
 clang_hwasan_cflags = clang_hwasan_common_cflags + ["-mllvm", "-hwasan-globals",
+                                                   "-mllvm", "-hwasan-use-short-granules",
                                                    "-mllvm", "-hwasan-instrument-landing-pads=0",
                                                    "-mllvm", "-hwasan-instrument-personality-functions"]
-clang_hwasan_oldrt_cflags = clang_hwasan_common_cflags + ["-mllvm", "-hwasan-instrument-landing-pads=1",
+clang_hwasan_oldrt_cflags = clang_hwasan_common_cflags + ["-mllvm", "-hwasan-use-short-granules=0",
+                                                          "-mllvm", "-hwasan-instrument-landing-pads=1",
                                                           "-mllvm", "-hwasan-instrument-personality-functions=0"]
 
 clang_hwasan_cxxflags = config.cxx_mode_flags + clang_hwasan_cflags
@@ -31,6 +33,7 @@ def build_invocation(compile_flags):
 
 config.substitutions.append( ("%clangxx ", build_invocation(clang_cxxflags)) )
 config.substitutions.append( ("%clang_hwasan ", build_invocation(clang_hwasan_cflags)) )
+config.substitutions.append( ("%clang_hwasan_oldrt ", build_invocation(clang_hwasan_oldrt_cflags)) )
 config.substitutions.append( ("%clangxx_hwasan ", build_invocation(clang_hwasan_cxxflags)) )
 config.substitutions.append( ("%clangxx_hwasan_oldrt ", build_invocation(clang_hwasan_oldrt_cxxflags)) )
 config.substitutions.append( ("%compiler_rt_libdir", config.compiler_rt_libdir) )
diff --git a/compiler-rt/test/lsan/TestCases/Linux/libdl_deadlock.cpp b/compiler-rt/test/lsan/TestCases/Linux/libdl_deadlock.cpp
new file mode 100644
index 0000000000000..7c1f82f10beef
--- /dev/null
+++ b/compiler-rt/test/lsan/TestCases/Linux/libdl_deadlock.cpp
@@ -0,0 +1,52 @@
+// Regression test for a deadlock in leak detection,
+// where lsan would call dl_iterate_phdr while holding the allocator lock.
+// RUN: %clangxx_lsan %s -o %t && %run %t
+
+#include <link.h>
+#include <mutex>
+#include <stdlib.h>
+#include <thread>
+#include <unistd.h>
+
+std::mutex in, out;
+
+int Callback(struct dl_phdr_info *info, size_t size, void *data) {
+  for (int step = 0; step < 50; ++step) {
+    void *p[1000];
+    for (int i = 0; i < 1000; ++i)
+      p[i] = malloc(10 * i);
+
+    if (step == 0)
+      in.unlock();
+
+    for (int i = 0; i < 1000; ++i)
+      free(p[i]);
+  }
+  out.unlock();
+  return 1; // just once
+}
+
+void Watchdog() {
+  // This is just a fail-safe to turn a deadlock (in case the bug reappears)
+  // into a (slow) test failure.
+  usleep(10000000);
+  if (!out.try_lock()) {
+    write(2, "DEADLOCK\n", 9);
+    exit(1);
+  }
+}
+
+int main() {
+  in.lock();
+  out.lock();
+
+  std::thread t([] { dl_iterate_phdr(Callback, nullptr); });
+  t.detach();
+
+  std::thread w(Watchdog);
+  w.detach();
+
+  // Wait for the malloc thread to preheat, then start leak detection (on exit)
+  in.lock();
+  return 0;
+}
diff --git a/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp b/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
index 5b5d692a59010..ecc577ac4c684 100644
--- a/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
+++ b/compiler-rt/test/lsan/TestCases/many_tls_keys_pthread.cpp
@@ -1,17 +1,16 @@
 // Test that lsan handles tls correctly for many threads
 // RUN: LSAN_BASE="report_objects=1:use_stacks=0:use_registers=0"
-// RUN: %clangxx_lsan %s -DUSE_THREAD -o %t-thread
-// RUN: %clangxx_lsan %s -DUSE_PTHREAD -o %t-pthread
-// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=0" not %run %t-thread 2>&1 | FileCheck %s
-// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t-thread 2>&1
-// RUN: %env_lsan_opts="" %run %t-thread 2>&1
-// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=0" not %run %t-pthread 2>&1 | FileCheck %s
-// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t-pthread 2>&1
-// RUN: %env_lsan_opts="" %run %t-pthread 2>&1
+// RUN: %clangxx_lsan %s -o %t
+// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=0" not %run %t 2>&1 | FileCheck %s
+// RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t 2>&1
+// RUN: %env_lsan_opts="" %run %t 2>&1
 
 // Patch r303906 did not fix all the problems.
 // UNSUPPORTED: arm-linux,armhf-linux
 
+// TSD on NetBSD does not use TLS
+// UNSUPPORTED: netbsd
+
 #include <assert.h>
 #include <limits.h>
 #include <pthread.h>
@@ -24,22 +23,6 @@ pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
 pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 int finished = 0;
 
-#if USE_THREAD
-__thread void *ptr1;
-__thread void *ptr2;
-__thread void *ptr3;
-__thread void *ptr4;
-__thread void *ptr5;
-
-void alloc() {
-  ptr1 = malloc(1111);
-  ptr2 = malloc(2222);
-  ptr3 = malloc(3333);
-  ptr4 = malloc(4444);
-  ptr5 = malloc(5555);
-}
-
-#elif USE_PTHREAD
 // We won't be able to create the maximum number of keys, due to other users
 // of the tls, but we'll use as many keys as we can before failing to create
 // a new key.
@@ -59,7 +42,6 @@ void alloc() {
 void pthread_destructor(void *arg) {
   assert(0 && "pthread destructors shouldn't be called");
 }
-#endif
 
 void *thread_start(void *arg) {
   alloc();
@@ -74,14 +56,12 @@ void *thread_start(void *arg) {
 }
 
 int main() {
-#if USE_PTHREAD
   for (int i = 0; i < PTHREAD_KEYS_MAX; ++i) {
     if (pthread_key_create(&keys[i], pthread_destructor)) {
       keys[i] = PTHREAD_KEY_INVALID;
       break;
     }
   }
-#endif
 
   pthread_t thread[NUM_THREADS];
   for (int i = 0; i < NUM_THREADS; ++i) {
diff --git a/compiler-rt/test/lsan/TestCases/many_tls_keys_thread.cpp b/compiler-rt/test/lsan/TestCases/many_tls_keys_thread.cpp
index 7e7196fc1e924..cdba0d1781d9a 100644
--- a/compiler-rt/test/lsan/TestCases/many_tls_keys_thread.cpp
+++ b/compiler-rt/test/lsan/TestCases/many_tls_keys_thread.cpp
@@ -1,6 +1,6 @@
 // Test that lsan handles tls correctly for many threads
 // RUN: LSAN_BASE="report_objects=1:use_stacks=0:use_registers=0"
-// RUN: %clangxx_lsan %s -DUSE_THREAD -o %t
+// RUN: %clangxx_lsan %s -o %t
 // RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=0" not %run %t 2>&1 | FileCheck %s
 // RUN: %env_lsan_opts=$LSAN_BASE:"use_tls=1" %run %t 2>&1
 // RUN: %env_lsan_opts="" %run %t 2>&1
diff --git a/compiler-rt/test/msan/preinit_array.cpp b/compiler-rt/test/msan/preinit_array.cpp
new file mode 100644
index 0000000000000..6f877bac0b1c3
--- /dev/null
+++ b/compiler-rt/test/msan/preinit_array.cpp
@@ -0,0 +1,16 @@
+// RUN: %clangxx_msan -O0 %s -o %t && %run %t
+
+#include <sanitizer/msan_interface.h>
+
+volatile int global;
+static void pre_ctor() {
+  volatile int local;
+  global = 42;
+  local = 42;
+}
+
+__attribute__((section(".preinit_array"), used)) void(*__local_pre_ctor)(void) = pre_ctor;
+
+int main(void) {
+  return 0;
+}
diff --git a/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test b/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test
index 5a406fda4b671..31b6a23be49cd 100644
--- a/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test
+++ b/compiler-rt/test/profile/Posix/instrprof-gcov-fork.test
@@ -1,4 +1,5 @@
 UNSUPPORTED: linux
+UNSUPPORTED: darwin
 
 RUN: mkdir -p %t.d
 RUN: cd %t.d
diff --git a/compiler-rt/test/profile/instrprof-merging.cpp b/compiler-rt/test/profile/instrprof-merging.cpp
index 06f05ce612aa8..692b049ec45c3 100644
--- a/compiler-rt/test/profile/instrprof-merging.cpp
+++ b/compiler-rt/test/profile/instrprof-merging.cpp
@@ -1,8 +1,11 @@
 // 1) Compile shared code into different object files and into an executable.
 
-// RUN: %clangxx_profgen -fcoverage-mapping %s -c -o %t.v1.o -D_VERSION_1
-// RUN: %clangxx_profgen -fcoverage-mapping %s -c -o %t.v2.o -D_VERSION_2
-// RUN: %clangxx_profgen -fcoverage-mapping %t.v1.o %t.v2.o -o %t.exe
+// RUN: %clangxx_profgen -std=c++14 -fcoverage-mapping %s -c -o %t.v1.o \
+// RUN:                  -D_VERSION_1
+// RUN: %clangxx_profgen -std=c++14 -fcoverage-mapping %s -c -o %t.v2.o \
+// RUN:                  -D_VERSION_2
+// RUN: %clangxx_profgen -std=c++14 -fcoverage-mapping %t.v1.o %t.v2.o \
+// RUN:                  -o %t.exe
 
 // 2) Collect profile data.
 
diff --git a/compiler-rt/test/profile/instrprof-set-file-object-merging.c b/compiler-rt/test/profile/instrprof-set-file-object-merging.c
index 3f71a8103561a..0ca5f6ff9ed95 100644
--- a/compiler-rt/test/profile/instrprof-set-file-object-merging.c
+++ b/compiler-rt/test/profile/instrprof-set-file-object-merging.c
@@ -1,12 +1,12 @@
 // Test that the specified output merges the profiling data.
 // Run the program twice so that the counters accumulate.
 // RUN: %clang -fprofile-instr-generate -fcoverage-mapping -o %t %s
+// RUN: rm -f %t.merging.profraw %t.merging.profdata
 // RUN: %run %t %t.merging.profraw
 // RUN: %run %t %t.merging.profraw
 // RUN: test -f %t.merging.profraw
 // RUN: llvm-profdata merge -o %t.merging.profdata %t.merging.profraw
 // RUN: llvm-cov show -instr-profile %t.merging.profdata %t | FileCheck %s --match-full-lines
-// RUN: rm %t.merging.profdata %t.merging.profraw
 #include <stdio.h>
 
 extern void __llvm_profile_set_file_object(FILE *, int);
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp
new file mode 100644
index 0000000000000..57175effe55d9
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/signal_line.cpp
@@ -0,0 +1,36 @@
+// Test line numbers in signal handlers
+
+// RUN: %clangxx %s -o %t -O0
+// RUN: %env_tool_opts=handle_segv=1:print_stacktrace=1 not %run %t 1 2>&1 | FileCheck --check-prefixes=CHECK1,CHECK %s
+// RUN: %env_tool_opts=handle_segv=1:print_stacktrace=1 not %run %t 2 2>&1 | FileCheck --check-prefixes=CHECK2,CHECK %s
+// RUN: %env_tool_opts=handle_segv=1:print_stacktrace=1 not %run %t 3 2>&1 | FileCheck --check-prefixes=CHECK3,CHECK %s
+// RUN: %env_tool_opts=handle_segv=1:print_stacktrace=1 not %run %t 4 2>&1 | FileCheck --check-prefixes=CHECK4,CHECK %s
+
+#include <cstdio>
+#include <string>
+
+// CHECK: [[SAN:.*Sanitizer]]:DEADLYSIGNAL
+// CHECK: ERROR: [[SAN]]: SEGV on unknown address {{0x[^ ]*}} (pc
+int main(int argc, char **argv) {
+  int n = atoi(argv[1]);
+
+  if (n == 1)
+    *((volatile int *)0x0) = __LINE__;
+  // CHECK1: #{{[0-9]+ .*}}main {{.*}}signal_line.cpp:[[@LINE-1]]:[[TAB:[0-9]+]]
+  // CHECK1: SUMMARY: [[SAN]]: SEGV {{.*}}signal_line.cpp:[[@LINE-2]]:[[TAB]] in main
+
+  if (n == 2)
+    *((volatile int *)0x0) = __LINE__;
+  // CHECK2: #{{[0-9]+ .*}}main {{.*}}signal_line.cpp:[[@LINE-1]]:[[TAB:[0-9]+]]
+  // CHECK2: SUMMARY: [[SAN]]: SEGV {{.*}}signal_line.cpp:[[@LINE-2]]:[[TAB]] in main
+
+  if (n == 3)
+    *((volatile int *)0x0) = __LINE__;
+  // CHECK3: #{{[0-9]+ .*}}main {{.*}}signal_line.cpp:[[@LINE-1]]:[[TAB:[0-9]+]]
+  // CHECK3: SUMMARY: [[SAN]]: SEGV {{.*}}signal_line.cpp:[[@LINE-2]]:[[TAB]] in main
+
+  if (n == 4)
+    *((volatile int *)0x0) = __LINE__;
+  // CHECK4: #{{[0-9]+ .*}}main {{.*}}signal_line.cpp:[[@LINE-1]]:[[TAB:[0-9]+]]
+  // CHECK4: SUMMARY: [[SAN]]: SEGV {{.*}}signal_line.cpp:[[@LINE-2]]:[[TAB]] in main
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/onprint.cpp b/compiler-rt/test/sanitizer_common/TestCases/onprint.cpp
new file mode 100644
index 0000000000000..c5afb824714b8
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/onprint.cpp
@@ -0,0 +1,33 @@
+// Checks that the __sanitizer_on_print hook gets the exact same sanitizer
+// report as what is printed to stderr.
+//
+// RUN: %clangxx %s -o %t
+// RUN: %run %t %t-onprint.txt 2>%t-stderr.txt || true
+// RUN: diff %t-onprint.txt %t-stderr.txt
+//
+// UNSUPPORTED: android
+
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+
+FILE *f;
+volatile void *buf;
+volatile char sink;
+
+extern "C" void __sanitizer_on_print(const char *str) {
+  fprintf(f, "%s", str);
+  fflush(f);
+}
+
+int main(int argc, char *argv[]) {
+  assert(argc >= 2);
+  f = fopen(argv[1], "w");
+
+  // Use-after-free to trigger ASan/TSan reports.
+  void *ptr = malloc(1);
+  buf = ptr;
+  free(ptr);
+  sink = *static_cast<char *>(ptr);
+  return 0;
+}
diff --git a/compiler-rt/test/tsan/CMakeLists.txt b/compiler-rt/test/tsan/CMakeLists.txt
index f21fc2a3beec8..9159e75129b09 100644
--- a/compiler-rt/test/tsan/CMakeLists.txt
+++ b/compiler-rt/test/tsan/CMakeLists.txt
@@ -49,39 +49,52 @@ endforeach()
 # variable to select which iOS device or simulator to use, e.g.:
 # SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
 if(APPLE)
+  # FIXME(dliew): This logic should be refactored to the way UBSan Darwin
+  # testing is done.
   set(EXCLUDE_FROM_ALL ON)
-
   set(TSAN_TEST_TARGET_CC ${COMPILER_RT_TEST_COMPILER})
 
-  set(TSAN_TEST_APPLE_PLATFORM "iossim")
-  set(arch "x86_64")
-  set(TSAN_TEST_TARGET_ARCH ${arch})
-  set(TSAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_iossim_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
-  set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-${TSAN_TEST_APPLE_PLATFORM}")
-  string(TOUPPER ${arch} ARCH_UPPER_CASE)
-  set(CONFIG_NAME "IOSSim${ARCH_UPPER_CASE}Config")
-  configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
-    )
-  add_lit_testsuite(check-tsan-iossim-${arch} "ThreadSanitizer iOS Simulator ${arch} tests"
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
-    DEPENDS ${TSAN_TEST_DEPS})
+  list_intersect(TSAN_TEST_IOSSIM_ARCHS TSAN_SUPPORTED_ARCH DARWIN_iossim_ARCHS)
+  foreach(arch ${TSAN_TEST_IOSSIM_ARCHS})
+    set(TSAN_TEST_APPLE_PLATFORM "iossim")
+    set(TSAN_TEST_TARGET_ARCH ${arch})
+    get_test_cflags_for_apple_platform(
+      "${TSAN_TEST_APPLE_PLATFORM}"
+      "${TSAN_TEST_TARGET_ARCH}"
+      TSAN_TEST_TARGET_CFLAGS
+      )
+    set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-${TSAN_TEST_APPLE_PLATFORM}")
+    string(TOUPPER ${arch} ARCH_UPPER_CASE)
+    set(CONFIG_NAME "IOSSim${ARCH_UPPER_CASE}Config")
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+      )
+    add_lit_testsuite(check-tsan-iossim-${arch} "ThreadSanitizer iOS Simulator ${arch} tests"
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      DEPENDS ${TSAN_TEST_DEPS})
+  endforeach()
 
-  set(TSAN_TEST_APPLE_PLATFORM "ios")
-  set(arch "arm64")
-  set(TSAN_TEST_TARGET_ARCH ${arch})
-  set(TSAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_ios_SYSROOT} ${COMPILER_RT_TEST_COMPILER_CFLAGS}")
-  set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-${TSAN_TEST_APPLE_PLATFORM}")
-  string(TOUPPER ${arch} ARCH_UPPER_CASE)
-  set(CONFIG_NAME "IOS${ARCH_UPPER_CASE}Config")
-  configure_lit_site_cfg(
-    ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
-    )
-  add_lit_testsuite(check-tsan-ios-${arch} "ThreadSanitizer iOS Simulator ${arch} tests"
-    ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
-    DEPENDS ${TSAN_TEST_DEPS})
+  list_intersect(TSAN_TEST_IOS_ARCHS TSAN_SUPPORTED_ARCH DARWIN_ios_ARCHS)
+  foreach(arch ${TSAN_TEST_IOS_ARCHS})
+    set(TSAN_TEST_APPLE_PLATFORM "ios")
+    set(TSAN_TEST_TARGET_ARCH ${arch})
+    get_test_cflags_for_apple_platform(
+      "${TSAN_TEST_APPLE_PLATFORM}"
+      "${TSAN_TEST_TARGET_ARCH}"
+      TSAN_TEST_TARGET_CFLAGS
+      )
+    set(TSAN_TEST_CONFIG_SUFFIX "-${arch}-${TSAN_TEST_APPLE_PLATFORM}")
+    string(TOUPPER ${arch} ARCH_UPPER_CASE)
+    set(CONFIG_NAME "IOS${ARCH_UPPER_CASE}Config")
+    configure_lit_site_cfg(
+      ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/lit.site.cfg.py
+      )
+    add_lit_testsuite(check-tsan-ios-${arch} "ThreadSanitizer iOS ${arch} tests"
+      ${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_NAME}/
+      DEPENDS ${TSAN_TEST_DEPS})
+  endforeach()
 
   set(EXCLUDE_FROM_ALL OFF)
 endif()
diff --git a/compiler-rt/test/tsan/race_range_pc.cc b/compiler-rt/test/tsan/race_range_pc.cc
new file mode 100644
index 0000000000000..a4689bd93ff49
--- /dev/null
+++ b/compiler-rt/test/tsan/race_range_pc.cc
@@ -0,0 +1,40 @@
+// RUN: %clangxx_tsan -O1 %s -o %t && %deflake %run %t | FileCheck %s
+// This test fails on powerpc64 big endian.
+// The Tsan report is returning wrong information about
+// the location of the race.
+// XFAIL: powerpc64-unknown-linux-gnu
+
+#include "test.h"
+
+typedef unsigned long uptr;
+extern "C" void __tsan_read_range_pc(uptr addr, uptr size, uptr pc);
+extern "C" void __tsan_write_range_pc(uptr addr, uptr size, uptr pc);
+
+void foobar() {
+}
+
+void barbaz() {
+}
+
+void *Thread(void *p) {
+  barrier_wait(&barrier);
+  __tsan_read_range_pc((uptr)p, 32, (uptr)foobar + kPCInc);
+  return 0;
+}
+
+int main() {
+  barrier_init(&barrier, 2);
+  int a[128];
+  pthread_t th;
+  pthread_create(&th, 0, Thread, (void*)a);
+  __tsan_write_range_pc((uptr)(a+2), 32, (uptr)barbaz + kPCInc);
+  barrier_wait(&barrier);
+  pthread_join(th, 0);
+  fprintf(stderr, "DONE\n");
+  return 0;
+}
+
+// CHECK: WARNING: ThreadSanitizer: data race
+// CHECK:     #0 foobar
+// CHECK:     #0 barbaz
+// CHECK: DONE
diff --git a/compiler-rt/test/ubsan/CMakeLists.txt b/compiler-rt/test/ubsan/CMakeLists.txt
index ee7032a915e97..9f30718dd1cdf 100644
--- a/compiler-rt/test/ubsan/CMakeLists.txt
+++ b/compiler-rt/test/ubsan/CMakeLists.txt
@@ -101,22 +101,20 @@ if(APPLE)
   # variable to select which iOS device or simulator to use, e.g.:
   # SANITIZER_IOSSIM_TEST_DEVICE_IDENTIFIER="iPhone 6"
   set(EXCLUDE_FROM_ALL ON)
-  set(UBSAN_APPLE_PLATFORMS "")
-  if (COMPILER_RT_ENABLE_IOS)
-    list(APPEND UBSAN_APPLE_PLATFORMS ios iossim)
-  endif()
-  if (COMPILER_RT_ENABLE_WATCHOS)
-    list(APPEND UBSAN_APPLE_PLATFORMS watchos watchossim)
-  endif()
-  if (COMPILER_RT_ENABLE_TVOS)
-    list(APPEND UBSAN_APPLE_PLATFORMS tvos tvossim)
-  endif()
+  set(UBSAN_APPLE_PLATFORMS ${SANITIZER_COMMON_SUPPORTED_OS})
   foreach(platform ${UBSAN_APPLE_PLATFORMS})
-    foreach(arch ${DARWIN_${platform}_ARCHS})
-      set(UBSAN_TEST_TARGET_CFLAGS "-arch ${arch} -isysroot ${DARWIN_${platform}_SYSROOT}")
-      if (";${UBSAN_SUPPORTED_ARCH};" MATCHES ";${arch};")
-        add_ubsan_device_testsuite("Standalone" ubsan ${platform} ${arch})
-      endif()
+    list_intersect(
+      UBSAN_TEST_${platform}_ARCHS
+      UBSAN_SUPPORTED_ARCH
+      DARWIN_${platform}_ARCHS
+      )
+    foreach(arch ${UBSAN_TEST_${platform}_ARCHS})
+      get_test_cflags_for_apple_platform(
+        "${platform}"
+        "${arch}"
+        UBSAN_TEST_TARGET_CFLAGS
+      )
+      add_ubsan_device_testsuite("Standalone" ubsan ${platform} ${arch})
 
       if(COMPILER_RT_HAS_ASAN AND ";${ASAN_SUPPORTED_ARCH};" MATCHES ";${arch};")
         add_ubsan_device_testsuite("AddressSanitizer" asan ${platform} ${arch})
diff --git a/libc/LICENSE.txt b/libc/LICENSE.txt
new file mode 100644
index 0000000000000..24806ab4c9eb2
--- /dev/null
+++ b/libc/LICENSE.txt
@@ -0,0 +1,278 @@
+==============================================================================
+The LLVM Project is under the Apache License v2.0 with LLVM Exceptions:
+==============================================================================
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+    1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+    2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+    3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+    4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+    5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+    6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+    7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+    8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+    9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+    END OF TERMS AND CONDITIONS
+
+    APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+    Copyright [yyyy] [name of copyright owner]
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+
+
+---- LLVM Exceptions to the Apache 2.0 License ----
+
+As an exception, if, as a result of your compiling your source code, portions
+of this Software are embedded into an Object form of such source code, you
+may redistribute such embedded portions in such Object form without complying
+with the conditions of Sections 4(a), 4(b) and 4(d) of the License.
+
+In addition, if you combine or link compiled forms of this Software with
+software that is licensed under the GPLv2 ("Combined Software") and if a
+court of competent jurisdiction determines that the patent provision (Section
+3), the indemnity provision (Section 9) or other Section of the License
+conflicts with the conditions of the GPLv2, you may retroactively and
+prospectively choose to deem waived or otherwise exclude such Section(s) of
+the License, but only in their entirety and only with respect to the Combined
+Software.
+
+==============================================================================
+Software from third parties included in the LLVM Project:
+==============================================================================
+The LLVM Project contains third party software which is under different license
+terms. All such code will be identified clearly using at least one of two
+mechanisms:
+1) It will be in a separate directory tree with its own `LICENSE.txt` or
+   `LICENSE` file at the top containing the specific license and restrictions
+   which apply to that software, or
+2) It will contain specific license and restriction terms at the top of every
+   file.
+
+==============================================================================
+Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy):
+==============================================================================
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2007-2019 University of Illinois at Urbana-Champaign.
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
diff --git a/libc/README.txt b/libc/README.txt
new file mode 100644
index 0000000000000..2eb8591cdacb4
--- /dev/null
+++ b/libc/README.txt
@@ -0,0 +1,8 @@
+LLVM libc
+=========
+
+This directory and its subdirectories contain source code for llvm-libc,
+a retargetable implementation of the C standard library.
+
+LLVM is open source software. You may freely distribute it under the terms of
+the license agreement found in LICENSE.txt.
diff --git a/libclc/.travis.yml b/libclc/.travis.yml
index 0c9992067bf06..e8a41d98d83a2 100644
--- a/libclc/.travis.yml
+++ b/libclc/.travis.yml
@@ -1,11 +1,6 @@
 language: cpp
 
-sudo: false
-dist: trusty
-
-cache:
-  apt: true
-
+dist: xenial
 
 matrix:
   include:
@@ -15,12 +10,7 @@ matrix:
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-3.9
           packages:
-            - libedit-dev
-            - g++-4.8
-            # From sources above
             - llvm-3.9-dev
             - clang-3.9
     - env:
@@ -29,12 +19,7 @@ matrix:
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-4.0
           packages:
-            - libedit-dev
-            - g++-4.8
-            # From sources above
             - llvm-4.0-dev
             - clang-4.0
     - env:
@@ -43,41 +28,28 @@ matrix:
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-5.0
           packages:
-            - libedit-dev
-            - g++-4.8
-            # From sources above
             - llvm-5.0-dev
             - clang-5.0
     - env:
         - LABEL="make gcc LLVM-6.0"
         - LLVM_VERSION=6.0
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
-        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
-        - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-6.0
-            - ubuntu-toolchain-r-test
           packages:
-            - libedit-dev
-            # LLVM-6 needs libstdc++4.9
-            - g++-4.9
-            # From sources above
             - llvm-6.0-dev
             - clang-6.0
     - env:
         - LABEL="make gcc LLVM-7"
         - LLVM_VERSION=7
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
         - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
       addons:
         apt:
           sources:
-            - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main'
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main'
               key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
             - ubuntu-toolchain-r-test
           packages:
@@ -90,11 +62,12 @@ matrix:
         - LABEL="make gcc LLVM-8"
         - LLVM_VERSION=8
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
         - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
       addons:
         apt:
           sources:
-            - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-8 main'
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main'
               key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
             - ubuntu-toolchain-r-test
           packages:
@@ -104,17 +77,29 @@ matrix:
             - llvm-8-dev
             - clang-8
     - env:
-        - LABEL="cmake gcc LLVM-3.9"
-        - LLVM_VERSION=3.9
-        - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        - LABEL="make gcc LLVM-9"
+        - LLVM_VERSION=9
+        - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
       addons:
         apt:
           sources:
-            - llvm-toolchain-trusty-3.9
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main'
+              key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
+            - ubuntu-toolchain-r-test
           packages:
             - libedit-dev
-            - g++-4.8
+            - g++-6
             # From sources above
+            - llvm-9-dev
+            - clang-9
+    - env:
+        - LABEL="cmake gcc LLVM-3.9"
+        - LLVM_VERSION=3.9
+        - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+      addons:
+        apt:
+          packages:
             - llvm-3.9-dev
             - clang-3.9
     - env:
@@ -123,12 +108,7 @@ matrix:
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-4.0
           packages:
-            - libedit-dev
-            - g++-4.8
-            # From sources above
             - llvm-4.0-dev
             - clang-4.0
     - env:
@@ -137,41 +117,28 @@ matrix:
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-5.0
           packages:
-            - libedit-dev
-            - g++-4.8
-            # From sources above
             - llvm-5.0-dev
             - clang-5.0
     - env:
         - LABEL="cmake gcc LLVM-6.0"
         - LLVM_VERSION=6.0
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
-        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
-        - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9"
       addons:
         apt:
-          sources:
-            - llvm-toolchain-trusty-6.0
-            - ubuntu-toolchain-r-test
           packages:
-            - libedit-dev
-            # LLVM-6 needs libstdc++4.9
-            - g++-4.9
-            # From sources above
             - llvm-6.0-dev
             - clang-6.0
     - env:
         - LABEL="cmake gcc LLVM-7"
         - LLVM_VERSION=7
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
         - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
       addons:
         apt:
           sources:
-            - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-7 main'
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main'
               key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
             - ubuntu-toolchain-r-test
           packages:
@@ -184,11 +151,12 @@ matrix:
         - LABEL="cmake gcc LLVM-8"
         - LLVM_VERSION=8
         - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        # llvm passes -Werror=date-time which is only supported in gcc-4.9+
         - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
       addons:
         apt:
           sources:
-            - sourceline: 'deb http://apt.llvm.org/trusty/ llvm-toolchain-trusty-8 main'
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main'
               key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
             - ubuntu-toolchain-r-test
           packages:
@@ -197,6 +165,23 @@ matrix:
             # From sources above
             - llvm-8-dev
             - clang-8
+    - env:
+        - LABEL="cmake gcc LLVM-9"
+        - LLVM_VERSION=9
+        - CHECK_FILES="barts-r600--.bc cayman-r600--.bc cedar-r600--.bc cypress-r600--.bc tahiti-amdgcn--.bc amdgcn--amdhsa.bc tahiti-amdgcn-mesa-mesa3d.bc nvptx--nvidiacl.bc nvptx64--nvidiacl.bc"
+        - MATRIX_EVAL="CC=gcc-6 && CXX=g++-6"
+      addons:
+        apt:
+          sources:
+            - sourceline: 'deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-9 main'
+              key_url: https://apt.llvm.org/llvm-snapshot.gpg.key
+            - ubuntu-toolchain-r-test
+          packages:
+            - libedit-dev
+            - g++-6
+            # From sources above
+            - llvm-9-dev
+            - clang-9
 
 before_install:
     - eval "${MATRIX_EVAL}"
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 37a1e3fc0821e..54265633ff91e 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -522,65 +522,75 @@ remove_flags("/D_DEBUG" "/MTd" "/MDd" "/MT" "/Md")
 remove_flags(-Wno-pedantic -pedantic-errors -pedantic)
 
 # Required flags ==============================================================
-if (LIBCXX_HAS_MUSL_LIBC OR LIBCXX_TARGETING_CLANG_CL)
-  # musl's pthread implementations uses volatile types in their structs which is
-  # not a constexpr in C++11 but is in C++14, so we use C++14 with musl.
-  set(LIBCXX_STANDARD_VER c++14 CACHE STRING "internal option to change build dialect")
-else()
-  set(LIBCXX_STANDARD_VER c++11 CACHE STRING "internal option to change build dialect")
-endif()
-add_compile_flags_if_supported(-std=${LIBCXX_STANDARD_VER})
-add_compile_flags_if_supported("/std:${LIBCXX_STANDARD_VER}")
-mangle_name("LIBCXX_SUPPORTS_STD_EQ_${LIBCXX_STANDARD_VER}_FLAG" SUPPORTS_DIALECT_NAME)
-mangle_name("LIBCXX_SUPPORTS_STD_COLON_${LIBCXX_STANDARD_VER}_FLAG" SUPPORTS_DIALECT_NAME_MSVC)
-if(NOT ${SUPPORTS_DIALECT_NAME} AND NOT ${SUPPORTS_DIALECT_NAME_MSVC})
-  if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" AND NOT "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
-    message(FATAL_ERROR "C++11 or greater is required but the compiler does not support ${LIBCXX_STANDARD_VER}")
+function(cxx_add_basic_build_flags target)
+  if (LIBCXX_HAS_MUSL_LIBC OR LIBCXX_TARGETING_CLANG_CL)
+    # musl's pthread implementations uses volatile types in their structs which is
+    # not a constexpr in C++11 but is in C++14, so we use C++14 with musl.
+    set(LIBCXX_STANDARD_VER c++14 CACHE STRING "internal option to change build dialect")
+  else()
+    set(LIBCXX_STANDARD_VER c++11 CACHE STRING "internal option to change build dialect")
+  endif()
+  target_add_compile_flags_if_supported(${target} PRIVATE -std=${LIBCXX_STANDARD_VER})
+  target_add_compile_flags_if_supported(${target} PRIVATE "/std:${LIBCXX_STANDARD_VER}")
+  mangle_name("LIBCXX_SUPPORTS_STD_EQ_${LIBCXX_STANDARD_VER}_FLAG" SUPPORTS_DIALECT_NAME)
+  mangle_name("LIBCXX_SUPPORTS_STD_COLON_${LIBCXX_STANDARD_VER}_FLAG" SUPPORTS_DIALECT_NAME_MSVC)
+  if(NOT ${SUPPORTS_DIALECT_NAME} AND NOT ${SUPPORTS_DIALECT_NAME_MSVC})
+    if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC" AND NOT "${CMAKE_CXX_SIMULATE_ID}" STREQUAL "MSVC")
+      message(FATAL_ERROR "C++11 or greater is required but the compiler does not support ${LIBCXX_STANDARD_VER}")
+    endif()
   endif()
-endif()
-
-# On all systems the system c++ standard library headers need to be excluded.
-# MSVC only has -X, which disables all default includes; including the crt.
-# Thus, we do nothing and hope we don't accidentally include any of the C++
-# headers
-add_compile_flags_if_supported(-nostdinc++)
 
-# Hide all inline function definitions which have not explicitly been marked
-# visible. This prevents new definitions for inline functions from appearing in
-# the dylib when get ODR used by another function.
-add_compile_flags_if_supported(-fvisibility-inlines-hidden)
+  # On all systems the system c++ standard library headers need to be excluded.
+  # MSVC only has -X, which disables all default includes; including the crt.
+  # Thus, we do nothing and hope we don't accidentally include any of the C++
+  # headers
+  target_add_compile_flags_if_supported(${target} PUBLIC -nostdinc++)
+
+  # Hide all inline function definitions which have not explicitly been marked
+  # visible. This prevents new definitions for inline functions from appearing in
+  # the dylib when get ODR used by another function.
+  target_add_compile_flags_if_supported(${target} PRIVATE -fvisibility-inlines-hidden)
+
+  # Our visibility annotations are not quite right for non-Clang compilers,
+  # so we end up not exporting all the symbols we should. In the future, we
+  # can improve the situation by providing an explicit list of exported
+  # symbols on all compilers.
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+    target_add_compile_flags_if_supported(${target} PRIVATE -fvisibility=hidden)
+  endif()
 
-if (LIBCXX_CONFIGURE_IDE)
-  # This simply allows IDE to process <experimental/coroutine>
-  add_compile_flags_if_supported(-fcoroutines-ts)
-endif()
+  if (LIBCXX_CONFIGURE_IDE)
+    # This simply allows IDE to process <experimental/coroutine>
+    target_add_compile_flags_if_supported(${target} PRIVATE -fcoroutines-ts)
+  endif()
 
-# Let the library headers know they are currently being used to build the
-# library.
-add_definitions(-D_LIBCPP_BUILDING_LIBRARY)
+  # Let the library headers know they are currently being used to build the
+  # library.
+  target_compile_definitions(${target} PRIVATE -D_LIBCPP_BUILDING_LIBRARY)
 
-if (NOT LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS)
-  add_definitions(-D_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS)
-endif()
+  if (NOT LIBCXX_ENABLE_NEW_DELETE_DEFINITIONS)
+    target_compile_definitions(${target} PRIVATE -D_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS)
+  endif()
 
-if (LIBCXX_HAS_COMMENT_LIB_PRAGMA)
-  add_definitions(-D_LIBCPP_HAS_COMMENT_LIB_PRAGMA)
-endif()
+  if (LIBCXX_HAS_COMMENT_LIB_PRAGMA)
+    target_compile_definitions(${target} PRIVATE -D_LIBCPP_HAS_COMMENT_LIB_PRAGMA)
+  endif()
+endfunction()
 
 # Warning flags ===============================================================
-add_definitions(-D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-add_compile_flags_if_supported(
-    -Wall -Wextra -W -Wwrite-strings
-    -Wno-unused-parameter -Wno-long-long
-    -Werror=return-type -Wextra-semi)
-if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
-    add_compile_flags_if_supported(
-        -Wno-user-defined-literals
-        -Wno-covered-switch-default
-        -Wno-ignored-attributes # FIXME: Caused by _LIBCPP_NODEBUG_TYPE not being supported on older clangs
-        )
+function(cxx_add_warning_flags target)
+  target_compile_definitions(${target} PUBLIC -D_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+  target_add_compile_flags_if_supported(${target} PRIVATE -Wall -Wextra -W -Wwrite-strings
+                                                          -Wno-unused-parameter -Wno-long-long
+                                                          -Werror=return-type -Wextra-semi)
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
+    target_add_compile_flags_if_supported(${target} PRIVATE
+      -Wno-user-defined-literals
+      -Wno-covered-switch-default
+      -Wno-ignored-attributes # FIXME: Caused by _LIBCPP_NODEBUG_TYPE not being supported on older clangs
+    )
     if (LIBCXX_TARGETING_CLANG_CL)
-      add_compile_flags_if_supported(
+      target_add_compile_flags_if_supported(${target} PRIVATE
         -Wno-c++98-compat
         -Wno-c++98-compat-pedantic
         -Wno-c++11-compat
@@ -597,26 +607,27 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang")
         -Wno-double-promotion # FIXME: remove me
       )
     endif()
-elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
-    add_compile_flags_if_supported(
-        -Wno-literal-suffix
-        -Wno-c++14-compat
-        -Wno-noexcept-type)
-endif()
-if (LIBCXX_ENABLE_WERROR)
-  add_compile_flags_if_supported(-Werror)
-  add_compile_flags_if_supported(-WX)
-else()
-  # TODO(EricWF) Remove this. We shouldn't be suppressing errors when -Werror is
-  # added elsewhere.
-  add_compile_flags_if_supported(-Wno-error)
-endif()
-if (LIBCXX_ENABLE_PEDANTIC)
-  add_compile_flags_if_supported(-pedantic)
-endif()
-if (LIBCXX_DISABLE_MACRO_CONFLICT_WARNINGS)
-  add_definitions(-D_LIBCPP_DISABLE_MACRO_CONFLICT_WARNINGS)
-endif()
+  elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "GNU")
+    target_add_compile_flags_if_supported(${target} PRIVATE
+      -Wno-literal-suffix
+      -Wno-c++14-compat
+      -Wno-noexcept-type)
+  endif()
+  if (LIBCXX_ENABLE_WERROR)
+    target_add_compile_flags_if_supported(${target} PRIVATE -Werror)
+    target_add_compile_flags_if_supported(${target} PRIVATE -WX)
+  else()
+    # TODO(EricWF) Remove this. We shouldn't be suppressing errors when -Werror is
+    # added elsewhere.
+    target_add_compile_flags_if_supported(${target} PRIVATE -Wno-error)
+  endif()
+  if (LIBCXX_ENABLE_PEDANTIC)
+    target_add_compile_flags_if_supported(${target} PRIVATE -pedantic)
+  endif()
+  if (LIBCXX_DISABLE_MACRO_CONFLICT_WARNINGS)
+    target_compile_definitions(${target} PRIVATE -D_LIBCPP_DISABLE_MACRO_CONFLICT_WARNINGS)
+  endif()
+endfunction()
 
 # Exception flags =============================================================
 if (LIBCXX_ENABLE_EXCEPTIONS)
@@ -713,6 +724,31 @@ if (LIBCXX_STANDALONE_BUILD AND SANITIZER_FLAGS)
   add_flags(${SANITIZER_FLAGS})
 endif()
 
+# Windows-related flags =======================================================
+function(cxx_add_windows_flags target)
+  if(WIN32 AND NOT MINGW)
+    target_compile_definitions(${target} PRIVATE
+                                 # Ignore the -MSC_VER mismatch, as we may build
+                                 # with a different compatibility version.
+                                 _ALLOW_MSC_VER_MISMATCH
+                                 # Don't check the msvcprt iterator debug levels
+                                 # as we will define the iterator types; libc++
+                                 # uses a different macro to identify the debug
+                                 # level.
+                                 _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH
+                                 # We are building the c++ runtime, don't pull in
+                                 # msvcprt.
+                                 _CRTBLD
+                                 # Don't warn on the use of "deprecated"
+                                 # "insecure" functions which are standards
+                                 # specified.
+                                 _CRT_SECURE_NO_WARNINGS
+                                 # Use the ISO conforming behaviour for conversion
+                                 # in printf, scanf.
+                                 _CRT_STDIO_ISO_WIDE_SPECIFIERS)
+  endif()
+endfunction()
+
 # Configuration file flags =====================================================
 if (NOT LIBCXX_ABI_VERSION EQUAL 1)
   config_define(${LIBCXX_ABI_VERSION} _LIBCPP_ABI_VERSION)
diff --git a/libcxx/docs/BuildingLibcxx.rst b/libcxx/docs/BuildingLibcxx.rst
index 76aa18a047e51..e6a21a930f014 100644
--- a/libcxx/docs/BuildingLibcxx.rst
+++ b/libcxx/docs/BuildingLibcxx.rst
@@ -317,7 +317,7 @@ libc++ Feature Options
   **Values**:: ``libc++``, ``libstdc++``
 
   Build the libc++ benchmark tests and Google Benchmark library against the
-  specified standard library on the platform. On linux this can be used to
+  specified standard library on the platform. On Linux this can be used to
   compare libc++ to libstdc++ by building the benchmark tests against both
   standard libraries.
 
@@ -404,7 +404,7 @@ LLVM-specific options
 .. option:: LLVM_BUILD_32_BITS:BOOL
 
   Build 32-bits executables and libraries on 64-bits systems. This option is
-  available only on some 64-bits unix systems. Defaults to OFF.
+  available only on some 64-bits Unix systems. Defaults to OFF.
 
 .. option:: LLVM_LIT_ARGS:STRING
 
@@ -527,7 +527,7 @@ These instructions should only be used when you can't install your ABI library.
 
 Normally you must link libc++ against a ABI shared library that the
 linker can find.  If you want to build and test libc++ against an ABI
-library not in the linker's path you needq to set
+library not in the linker's path you need to set
 ``-DLIBCXX_CXX_ABI_LIBRARY_PATH=/path/to/abi/lib`` when configuring CMake.
 
 An example build using libc++abi would look like:
diff --git a/libcxx/include/__config b/libcxx/include/__config
index f631faa00345e..ee7351e9313d5 100644
--- a/libcxx/include/__config
+++ b/libcxx/include/__config
@@ -1087,6 +1087,16 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
 #  endif // _LIBCPP_HAS_THREAD_API
 #endif // _LIBCPP_HAS_NO_THREADS
 
+#if defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
+#if defined(__ANDROID__) && __ANDROID_API__ >= 30
+#define _LIBCPP_HAS_COND_CLOCKWAIT
+#elif defined(_LIBCPP_GLIBC_PREREQ)
+#if _LIBCPP_GLIBC_PREREQ(2, 30)
+#define _LIBCPP_HAS_COND_CLOCKWAIT
+#endif
+#endif
+#endif
+
 #if defined(_LIBCPP_HAS_NO_THREADS) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
 #error _LIBCPP_HAS_THREAD_API_PTHREAD may only be defined when \
        _LIBCPP_HAS_NO_THREADS is not defined.
@@ -1441,6 +1451,17 @@ _LIBCPP_FUNC_VIS extern "C" void __sanitizer_annotate_contiguous_container(
 
 #define _LIBCPP_UNUSED_VAR(x) ((void)(x))
 
+// Configures the fopen close-on-exec mode character, if any. This string will
+// be appended to any mode string used by fstream for fopen/fdopen.
+//
+// Not all platforms support this, but it helps avoid fd-leaks on platforms that
+// do.
+#if defined(__BIONIC__)
+#  define _LIBCPP_FOPEN_CLOEXEC_MODE "e"
+#else
+#  define _LIBCPP_FOPEN_CLOEXEC_MODE
+#endif
+
 #endif // __cplusplus
 
 #endif // _LIBCPP_CONFIG
diff --git a/libcxx/include/__functional_base b/libcxx/include/__functional_base
index 9587d7ab15a1d..ca761c409b627 100644
--- a/libcxx/include/__functional_base
+++ b/libcxx/include/__functional_base
@@ -558,7 +558,7 @@ struct __is_transparent<_Tp, _Up,
 
 // allocator_arg_t
 
-struct _LIBCPP_TEMPLATE_VIS allocator_arg_t { };
+struct _LIBCPP_TEMPLATE_VIS allocator_arg_t { explicit allocator_arg_t() = default; };
 
 #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
 extern _LIBCPP_EXPORTED_FROM_ABI const allocator_arg_t allocator_arg;
diff --git a/libcxx/include/__mutex_base b/libcxx/include/__mutex_base
index a85ded8341e00..ed75c82380a6c 100644
--- a/libcxx/include/__mutex_base
+++ b/libcxx/include/__mutex_base
@@ -15,6 +15,7 @@
 #include <system_error>
 #include <__threading_support>
 
+#include <time.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #pragma GCC system_header
@@ -65,9 +66,9 @@ public:
 static_assert(is_nothrow_default_constructible<mutex>::value,
               "the default constructor for std::mutex must be nothrow");
 
-struct _LIBCPP_TYPE_VIS defer_lock_t {};
-struct _LIBCPP_TYPE_VIS try_to_lock_t {};
-struct _LIBCPP_TYPE_VIS adopt_lock_t {};
+struct _LIBCPP_TYPE_VIS defer_lock_t { explicit defer_lock_t() = default; };
+struct _LIBCPP_TYPE_VIS try_to_lock_t { explicit try_to_lock_t() = default; };
+struct _LIBCPP_TYPE_VIS adopt_lock_t { explicit adopt_lock_t() = default; };
 
 #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
 
@@ -337,23 +338,75 @@ public:
 private:
     void __do_timed_wait(unique_lock<mutex>& __lk,
        chrono::time_point<chrono::system_clock, chrono::nanoseconds>) _NOEXCEPT;
+#if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+    void __do_timed_wait(unique_lock<mutex>& __lk,
+       chrono::time_point<chrono::steady_clock, chrono::nanoseconds>) _NOEXCEPT;
+#endif
+    template <class _Clock>
+    void __do_timed_wait(unique_lock<mutex>& __lk,
+       chrono::time_point<_Clock, chrono::nanoseconds>) _NOEXCEPT;
 };
 #endif // !_LIBCPP_HAS_NO_THREADS
 
-template <class _To, class _Rep, class _Period>
+template <class _Rep, class _Period>
 inline _LIBCPP_INLINE_VISIBILITY
 typename enable_if
 <
-    chrono::__is_duration<_To>::value,
-    _To
+    is_floating_point<_Rep>::value,
+    chrono::nanoseconds
 >::type
-__ceil(chrono::duration<_Rep, _Period> __d)
+__safe_nanosecond_cast(chrono::duration<_Rep, _Period> __d)
 {
     using namespace chrono;
-    _To __r = duration_cast<_To>(__d);
-    if (__r < __d)
-        ++__r;
-    return __r;
+    using __ratio = ratio_divide<_Period, nano>;
+    using __ns_rep = nanoseconds::rep;
+    _Rep __result_float = __d.count() * __ratio::num / __ratio::den;
+
+    _Rep __result_max = numeric_limits<__ns_rep>::max();
+    if (__result_float >= __result_max) {
+        return nanoseconds::max();
+    }
+
+    _Rep __result_min = numeric_limits<__ns_rep>::min();
+    if (__result_float <= __result_min) {
+        return nanoseconds::min();
+    }
+
+    return nanoseconds(static_cast<__ns_rep>(__result_float));
+}
+
+template <class _Rep, class _Period>
+inline _LIBCPP_INLINE_VISIBILITY
+typename enable_if
+<
+    !is_floating_point<_Rep>::value,
+    chrono::nanoseconds
+>::type
+__safe_nanosecond_cast(chrono::duration<_Rep, _Period> __d)
+{
+    using namespace chrono;
+    if (__d.count() == 0) {
+        return nanoseconds(0);
+    }
+
+    using __ratio = ratio_divide<_Period, nano>;
+    using __ns_rep = nanoseconds::rep;
+    __ns_rep __result_max = std::numeric_limits<__ns_rep>::max();
+    if (__d.count() > 0 && __d.count() > __result_max / __ratio::num) {
+        return nanoseconds::max();
+    }
+
+    __ns_rep __result_min = std::numeric_limits<__ns_rep>::min();
+    if (__d.count() < 0 && __d.count() < __result_min / __ratio::num) {
+        return nanoseconds::min();
+    }
+
+    __ns_rep __result = __d.count() * __ratio::num / __ratio::den;
+    if (__result == 0) {
+        return nanoseconds(1);
+    }
+
+    return nanoseconds(__result);
 }
 
 #ifndef _LIBCPP_HAS_NO_THREADS
@@ -371,7 +424,15 @@ condition_variable::wait_until(unique_lock<mutex>& __lk,
                                const chrono::time_point<_Clock, _Duration>& __t)
 {
     using namespace chrono;
-    wait_for(__lk, __t - _Clock::now());
+    using __clock_tp_ns = time_point<_Clock, nanoseconds>;
+
+    typename _Clock::time_point __now = _Clock::now();
+    if (__t <= __now)
+        return cv_status::timeout;
+
+    __clock_tp_ns __t_ns = __clock_tp_ns(__safe_nanosecond_cast(__t.time_since_epoch()));
+
+    __do_timed_wait(__lk, __t_ns);
     return _Clock::now() < __t ? cv_status::no_timeout : cv_status::timeout;
 }
 
@@ -397,15 +458,25 @@ condition_variable::wait_for(unique_lock<mutex>& __lk,
     using namespace chrono;
     if (__d <= __d.zero())
         return cv_status::timeout;
-    typedef time_point<system_clock, duration<long double, nano> > __sys_tpf;
-    typedef time_point<system_clock, nanoseconds> __sys_tpi;
-    __sys_tpf _Max = __sys_tpi::max();
+    using __ns_rep = nanoseconds::rep;
     steady_clock::time_point __c_now = steady_clock::now();
-    system_clock::time_point __s_now = system_clock::now();
-    if (_Max - __d > __s_now)
-        __do_timed_wait(__lk, __s_now + __ceil<nanoseconds>(__d));
-    else
-        __do_timed_wait(__lk, __sys_tpi::max());
+
+#if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+    using __clock_tp_ns = time_point<steady_clock, nanoseconds>;
+    __ns_rep __now_count_ns = __safe_nanosecond_cast(__c_now.time_since_epoch()).count();
+#else
+    using __clock_tp_ns = time_point<system_clock, nanoseconds>;
+    __ns_rep __now_count_ns = __safe_nanosecond_cast(system_clock::now().time_since_epoch()).count();
+#endif
+
+    __ns_rep __d_ns_count = __safe_nanosecond_cast(__d).count();
+
+    if (__now_count_ns > numeric_limits<__ns_rep>::max() - __d_ns_count) {
+        __do_timed_wait(__lk, __clock_tp_ns::max());
+    } else {
+        __do_timed_wait(__lk, __clock_tp_ns(nanoseconds(__now_count_ns + __d_ns_count)));
+    }
+
     return steady_clock::now() - __c_now < __d ? cv_status::no_timeout :
                                                  cv_status::timeout;
 }
@@ -421,6 +492,46 @@ condition_variable::wait_for(unique_lock<mutex>& __lk,
                       _VSTD::move(__pred));
 }
 
+#if defined(_LIBCPP_HAS_COND_CLOCKWAIT)
+inline
+void
+condition_variable::__do_timed_wait(unique_lock<mutex>& __lk,
+     chrono::time_point<chrono::steady_clock, chrono::nanoseconds> __tp) _NOEXCEPT
+{
+    using namespace chrono;
+    if (!__lk.owns_lock())
+        __throw_system_error(EPERM,
+                            "condition_variable::timed wait: mutex not locked");
+    nanoseconds __d = __tp.time_since_epoch();
+    timespec __ts;
+    seconds __s = duration_cast<seconds>(__d);
+    using __ts_sec = decltype(__ts.tv_sec);
+    const __ts_sec __ts_sec_max = numeric_limits<__ts_sec>::max();
+    if (__s.count() < __ts_sec_max)
+    {
+        __ts.tv_sec = static_cast<__ts_sec>(__s.count());
+        __ts.tv_nsec = (__d - __s).count();
+    }
+    else
+    {
+        __ts.tv_sec = __ts_sec_max;
+        __ts.tv_nsec = giga::num - 1;
+    }
+    int __ec = pthread_cond_clockwait(&__cv_, __lk.mutex()->native_handle(), CLOCK_MONOTONIC, &__ts);
+    if (__ec != 0 && __ec != ETIMEDOUT)
+        __throw_system_error(__ec, "condition_variable timed_wait failed");
+}
+#endif // _LIBCPP_HAS_COND_CLOCKWAIT
+
+template <class _Clock>
+inline
+void
+condition_variable::__do_timed_wait(unique_lock<mutex>& __lk,
+     chrono::time_point<_Clock, chrono::nanoseconds> __tp) _NOEXCEPT
+{
+    wait_for(__lk, __tp - _Clock::now());
+}
+
 #endif // !_LIBCPP_HAS_NO_THREADS
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__string b/libcxx/include/__string
index a88b976be3c9b..b4c8815f72d21 100644
--- a/libcxx/include/__string
+++ b/libcxx/include/__string
@@ -351,6 +351,18 @@ char_traits<wchar_t>::compare(const char_type* __s1, const char_type* __s2, size
 #endif
 }
 
+
+template <class _Traits>
+_LIBCPP_INLINE_VISIBILITY
+_LIBCPP_CONSTEXPR
+inline size_t __char_traits_length_checked(const typename _Traits::char_type* __s) _NOEXCEPT {
+#if _LIBCPP_DEBUG_LEVEL >= 1
+  return __s ? _Traits::length(__s) : (_VSTD::__libcpp_debug_function(_VSTD::__libcpp_debug_info(__FILE__, __LINE__, "p == nullptr", "null pointer pass to non-null argument of char_traits<...>::length")), 0);
+#else
+  return _Traits::length(__s);
+#endif
+}
+
 inline _LIBCPP_CONSTEXPR_AFTER_CXX14
 size_t
 char_traits<wchar_t>::length(const char_type* __s) _NOEXCEPT
diff --git a/libcxx/include/__tuple b/libcxx/include/__tuple
index 196f3c2b5aa52..4da9ec55f3548 100644
--- a/libcxx/include/__tuple
+++ b/libcxx/include/__tuple
@@ -477,8 +477,9 @@ using __tuple_like_with_size _LIBCPP_NODEBUG_TYPE = __tuple_like_with_size_imp<
                               >;
 
 struct _LIBCPP_TYPE_VIS __check_tuple_constructor_fail {
-    template <class ...>
-    static constexpr bool __enable_default() { return false; }
+
+    static constexpr bool __enable_explicit_default() { return false; }
+    static constexpr bool __enable_implicit_default() { return false; }
     template <class ...>
     static constexpr bool __enable_explicit() { return false; }
     template <class ...>
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 36d26b38760f5..0f06486fb34f3 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -644,8 +644,8 @@ _LIBCPP_CONSTEXPR _IntT __max_representable_int_for_float() _NOEXCEPT {
   static_assert(is_floating_point<_FloatT>::value, "must be a floating point type");
   static_assert(is_integral<_IntT>::value, "must be an integral type");
   static_assert(numeric_limits<_FloatT>::radix == 2, "FloatT has incorrect radix");
-  static_assert(_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value
-               || _IsSame<_FloatT,long double>::value, "unsupported floating point type");
+  static_assert((_IsSame<_FloatT, float>::value || _IsSame<_FloatT, double>::value
+                 || _IsSame<_FloatT,long double>::value), "unsupported floating point type");
   return _FloatBigger ? numeric_limits<_IntT>::max() :  (numeric_limits<_IntT>::max() >> _Bits << _Bits);
 }
 
diff --git a/libcxx/include/experimental/coroutine b/libcxx/include/experimental/coroutine
index e2f0a25f77edc..54ec74b9f9844 100644
--- a/libcxx/include/experimental/coroutine
+++ b/libcxx/include/experimental/coroutine
@@ -51,7 +51,6 @@ template <class P> struct hash<coroutine_handle<P>>;
 #include <functional>
 #include <memory> // for hash<T*>
 #include <cstddef>
-#include <cassert>
 #include <__debug>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
diff --git a/libcxx/include/fstream b/libcxx/include/fstream
index 7db9017aca9e7..e9138998bf11b 100644
--- a/libcxx/include/fstream
+++ b/libcxx/include/fstream
@@ -508,34 +508,34 @@ const char* basic_filebuf<_CharT, _Traits>::__make_mdstring(
   switch (__mode & ~ios_base::ate) {
   case ios_base::out:
   case ios_base::out | ios_base::trunc:
-    return "w";
+    return "w" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::out | ios_base::app:
   case ios_base::app:
-    return "a";
+    return "a" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in:
-    return "r";
+    return "r" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out:
-    return "r+";
+    return "r+" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::trunc:
-    return "w+";
+    return "w+" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::app:
   case ios_base::in | ios_base::app:
-    return "a+";
+    return "a+" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::out | ios_base::binary:
   case ios_base::out | ios_base::trunc | ios_base::binary:
-    return "wb";
+    return "wb" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::out | ios_base::app | ios_base::binary:
   case ios_base::app | ios_base::binary:
-    return "ab";
+    return "ab" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::binary:
-    return "rb";
+    return "rb" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::binary:
-    return "r+b";
+    return "r+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::trunc | ios_base::binary:
-    return "w+b";
+    return "w+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
   case ios_base::in | ios_base::out | ios_base::app | ios_base::binary:
   case ios_base::in | ios_base::app | ios_base::binary:
-    return "a+b";
+    return "a+b" _LIBCPP_FOPEN_CLOEXEC_MODE;
   default:
     return nullptr;
   }
diff --git a/libcxx/include/memory b/libcxx/include/memory
index 12573ca801161..0336a4d5d02d1 100644
--- a/libcxx/include/memory
+++ b/libcxx/include/memory
@@ -662,7 +662,6 @@ void* align(size_t alignment, size_t size, void*& ptr, size_t& space);
 #include <tuple>
 #include <stdexcept>
 #include <cstring>
-#include <cassert>
 #if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER)
 #  include <atomic>
 #endif
@@ -3872,8 +3871,6 @@ public:
                       : nullptr);}
 #endif  // _LIBCPP_NO_RTTI
 
-#ifndef _LIBCPP_HAS_NO_VARIADICS
-
     template<class ..._Args>
         static
         shared_ptr<_Tp>
@@ -3884,37 +3881,6 @@ public:
         shared_ptr<_Tp>
         allocate_shared(const _Alloc& __a, _Args&& ...__args);
 
-#else  // _LIBCPP_HAS_NO_VARIADICS
-
-    static shared_ptr<_Tp> make_shared();
-
-    template<class _A0>
-        static shared_ptr<_Tp> make_shared(_A0&);
-
-    template<class _A0, class _A1>
-        static shared_ptr<_Tp> make_shared(_A0&, _A1&);
-
-    template<class _A0, class _A1, class _A2>
-        static shared_ptr<_Tp> make_shared(_A0&, _A1&, _A2&);
-
-    template<class _Alloc>
-        static shared_ptr<_Tp>
-        allocate_shared(const _Alloc& __a);
-
-    template<class _Alloc, class _A0>
-        static shared_ptr<_Tp>
-        allocate_shared(const _Alloc& __a, _A0& __a0);
-
-    template<class _Alloc, class _A0, class _A1>
-        static shared_ptr<_Tp>
-        allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1);
-
-    template<class _Alloc, class _A0, class _A1, class _A2>
-        static shared_ptr<_Tp>
-        allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1, _A2& __a2);
-
-#endif  // _LIBCPP_HAS_NO_VARIADICS
-
 private:
     template <class _Yp, bool = is_function<_Yp>::value>
         struct __shared_ptr_default_allocator
@@ -4227,8 +4193,6 @@ shared_ptr<_Tp>::shared_ptr(unique_ptr<_Yp, _Dp> __r,
     __r.release();
 }
 
-#ifndef _LIBCPP_HAS_NO_VARIADICS
-
 template<class _Tp>
 template<class ..._Args>
 shared_ptr<_Tp>
@@ -4268,165 +4232,6 @@ shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _Args&& ...__args)
     return __r;
 }
 
-#else  // _LIBCPP_HAS_NO_VARIADICS
-
-template<class _Tp>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::make_shared()
-{
-    static_assert((is_constructible<_Tp>::value), "Can't construct object in make_shared" );
-    typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk;
-    typedef allocator<_CntrlBlk> _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2;
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(__hold2.get()) _CntrlBlk(__alloc2);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = __hold2.release();
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _A0>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::make_shared(_A0& __a0)
-{
-    static_assert((is_constructible<_Tp, _A0>::value), "Can't construct object in make_shared" );
-    typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk;
-    typedef allocator<_CntrlBlk> _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2;
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(__hold2.get()) _CntrlBlk(__alloc2, __a0);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = __hold2.release();
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _A0, class _A1>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::make_shared(_A0& __a0, _A1& __a1)
-{
-    static_assert((is_constructible<_Tp, _A0, _A1>::value), "Can't construct object in make_shared" );
-    typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk;
-    typedef allocator<_CntrlBlk> _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2;
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(__hold2.get()) _CntrlBlk(__alloc2, __a0, __a1);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = __hold2.release();
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _A0, class _A1, class _A2>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::make_shared(_A0& __a0, _A1& __a1, _A2& __a2)
-{
-    static_assert((is_constructible<_Tp, _A0, _A1, _A2>::value), "Can't construct object in make_shared" );
-    typedef __shared_ptr_emplace<_Tp, allocator<_Tp> > _CntrlBlk;
-    typedef allocator<_CntrlBlk> _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2;
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(__hold2.get()) _CntrlBlk(__alloc2, __a0, __a1, __a2);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = __hold2.release();
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _Alloc>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::allocate_shared(const _Alloc& __a)
-{
-    static_assert((is_constructible<_Tp>::value), "Can't construct object in allocate_shared" );
-    typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk;
-    typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2(__a);
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(static_cast<void*>(_VSTD::addressof(*__hold2.get())))
-        _CntrlBlk(__a);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = _VSTD::addressof(*__hold2.release());
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _Alloc, class _A0>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0)
-{
-    static_assert((is_constructible<_Tp, _A0>::value), "Can't construct object in allocate_shared" );
-    typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk;
-    typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2(__a);
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(static_cast<void*>(_VSTD::addressof(*__hold2.get())))
-        _CntrlBlk(__a, __a0);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = _VSTD::addressof(*__hold2.release());
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _Alloc, class _A0, class _A1>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1)
-{
-    static_assert((is_constructible<_Tp, _A0, _A1>::value), "Can't construct object in allocate_shared" );
-    typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk;
-    typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2(__a);
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(static_cast<void*>(_VSTD::addressof(*__hold2.get())))
-        _CntrlBlk(__a, __a0, __a1);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = _VSTD::addressof(*__hold2.release());
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-template<class _Tp>
-template<class _Alloc, class _A0, class _A1, class _A2>
-shared_ptr<_Tp>
-shared_ptr<_Tp>::allocate_shared(const _Alloc& __a, _A0& __a0, _A1& __a1, _A2& __a2)
-{
-    static_assert((is_constructible<_Tp, _A0, _A1, _A2>::value), "Can't construct object in allocate_shared" );
-    typedef __shared_ptr_emplace<_Tp, _Alloc> _CntrlBlk;
-    typedef typename __allocator_traits_rebind<_Alloc, _CntrlBlk>::type _Alloc2;
-    typedef __allocator_destructor<_Alloc2> _D2;
-    _Alloc2 __alloc2(__a);
-    unique_ptr<_CntrlBlk, _D2> __hold2(__alloc2.allocate(1), _D2(__alloc2, 1));
-    ::new(static_cast<void*>(_VSTD::addressof(*__hold2.get())))
-        _CntrlBlk(__a, __a0, __a1, __a2);
-    shared_ptr<_Tp> __r;
-    __r.__ptr_ = __hold2.get()->get();
-    __r.__cntrl_ = _VSTD::addressof(*__hold2.release());
-    __r.__enable_weak_this(__r.__ptr_, __r.__ptr_);
-    return __r;
-}
-
-#endif  // _LIBCPP_HAS_NO_VARIADICS
-
 template<class _Tp>
 shared_ptr<_Tp>::~shared_ptr()
 {
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index dca62202db114..8fc3c61221b57 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -86,9 +86,9 @@ public:
     void unlock();
 };
 
-struct defer_lock_t {};
-struct try_to_lock_t {};
-struct adopt_lock_t {};
+struct defer_lock_t { explicit defer_lock_t() = default; };
+struct try_to_lock_t { explicit try_to_lock_t() = default; };
+struct adopt_lock_t { explicit adopt_lock_t() = default; };
 
 inline constexpr defer_lock_t  defer_lock{};
 inline constexpr try_to_lock_t try_to_lock{};
diff --git a/libcxx/include/new b/libcxx/include/new
index 85e4c4b3fcf2a..40d351e9b7705 100644
--- a/libcxx/include/new
+++ b/libcxx/include/new
@@ -39,7 +39,7 @@ struct destroying_delete_t { // C++20
 };
 inline constexpr destroying_delete_t destroying_delete{}; // C++20
 
-struct nothrow_t {};
+struct nothrow_t { explicit nothrow_t() = default; };
 extern const nothrow_t nothrow;
 typedef void (*new_handler)();
 new_handler set_new_handler(new_handler new_p) noexcept;
@@ -126,7 +126,7 @@ namespace std  // purposefully not using versioning namespace
 {
 
 #if !defined(_LIBCPP_ABI_VCRUNTIME)
-struct _LIBCPP_TYPE_VIS nothrow_t {};
+struct _LIBCPP_TYPE_VIS nothrow_t { explicit nothrow_t() = default; };
 extern _LIBCPP_FUNC_VIS const nothrow_t nothrow;
 
 class _LIBCPP_EXCEPTION_ABI bad_alloc
diff --git a/libcxx/include/ostream b/libcxx/include/ostream
index e6cf9c970f722..ea3870532f329 100644
--- a/libcxx/include/ostream
+++ b/libcxx/include/ostream
@@ -1055,7 +1055,7 @@ operator<<(basic_ostream<_CharT, _Traits>& __os,
 template<class _CharT, class _Traits>
 basic_ostream<_CharT, _Traits>&
 operator<<(basic_ostream<_CharT, _Traits>& __os,
-           const basic_string_view<_CharT, _Traits> __sv)
+           basic_string_view<_CharT, _Traits> __sv)
 {
     return _VSTD::__put_character_sequence(__os, __sv.data(), __sv.size());
 }
diff --git a/libcxx/include/regex b/libcxx/include/regex
index 26efac1c620dd..d13f9addb70a7 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -169,15 +169,15 @@ public:
     // assign:
     basic_regex& assign(const basic_regex& that);
     basic_regex& assign(basic_regex&& that) noexcept;
-    basic_regex& assign(const charT* ptr, flag_type f = regex_constants::ECMAScript);
-    basic_regex& assign(const charT* p, size_t len, flag_type f);
+    basic_regex& assign(const charT* ptr,           flag_type f = regex_constants::ECMAScript);
+    basic_regex& assign(const charT* p, size_t len, flag_type f = regex_constants::ECMAScript);
     template <class string_traits, class A>
         basic_regex& assign(const basic_string<charT, string_traits, A>& s,
-                            flag_type f = regex_constants::ECMAScript);
+                                                    flag_type f = regex_constants::ECMAScript);
     template <class InputIterator>
         basic_regex& assign(InputIterator first, InputIterator last,
-                            flag_type f = regex_constants::ECMAScript);
-    basic_regex& assign(initializer_list<charT>, flag_type = regex_constants::ECMAScript);
+                                                    flag_type f = regex_constants::ECMAScript);
+    basic_regex& assign(initializer_list<charT>,    flag_type f = regex_constants::ECMAScript);
 
     // const operations:
     unsigned mark_count() const;
@@ -2617,7 +2617,7 @@ public:
     basic_regex& assign(const value_type* __p, flag_type __f = regex_constants::ECMAScript)
         {return assign(__p, __p + __traits_.length(__p), __f);}
     _LIBCPP_INLINE_VISIBILITY
-    basic_regex& assign(const value_type* __p, size_t __len, flag_type __f)
+    basic_regex& assign(const value_type* __p, size_t __len, flag_type __f = regex_constants::ECMAScript)
         {return assign(__p, __p + __len, __f);}
     template <class _ST, class _SA>
         _LIBCPP_INLINE_VISIBILITY
diff --git a/libcxx/include/string_view b/libcxx/include/string_view
index 04448312ff338..8a684a8f966ce 100644
--- a/libcxx/include/string_view
+++ b/libcxx/include/string_view
@@ -173,6 +173,7 @@ namespace std {
 
 #include <__config>
 #include <__string>
+#include <iosfwd>
 #include <algorithm>
 #include <iterator>
 #include <limits>
@@ -235,7 +236,7 @@ public:
 
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
     basic_string_view(const _CharT* __s)
-        : __data(__s), __size(_Traits::length(__s)) {}
+        : __data(__s), __size(std::__char_traits_length_checked<_Traits>(__s)) {}
 
     // [string.view.iterators], iterators
     _LIBCPP_CONSTEXPR _LIBCPP_INLINE_VISIBILITY
@@ -767,6 +768,12 @@ bool operator>=(typename common_type<basic_string_view<_CharT, _Traits> >::type
     return __lhs.compare(__rhs) >= 0;
 }
 
+
+template<class _CharT, class _Traits>
+basic_ostream<_CharT, _Traits>&
+operator<<(basic_ostream<_CharT, _Traits>& __os,
+           basic_string_view<_CharT, _Traits> __str);
+
 typedef basic_string_view<char>     string_view;
 #ifndef _LIBCPP_NO_HAS_CHAR8_T
 typedef basic_string_view<char8_t>  u8string_view;
diff --git a/libcxx/include/tuple b/libcxx/include/tuple
index 32bc86913aaa8..e93824f0aa7f4 100644
--- a/libcxx/include/tuple
+++ b/libcxx/include/tuple
@@ -19,7 +19,7 @@ namespace std
 template <class... T>
 class tuple {
 public:
-    constexpr tuple();
+    explicit(see-below) constexpr tuple();
     explicit(see-below) tuple(const T&...);  // constexpr in C++14
     template <class... U>
         explicit(see-below) tuple(U&&...);  // constexpr in C++14
@@ -499,11 +499,19 @@ class _LIBCPP_TEMPLATE_VIS tuple
     template <class _Dummy>
     struct _CheckArgsConstructor<true, _Dummy>
     {
-        template <class ..._Args>
-        static constexpr bool __enable_default() {
-            return __all<is_default_constructible<_Args>::value...>::value;
+        template <int&...>
+        static constexpr bool __enable_implicit_default() {
+           return __all<__is_implicitly_default_constructible<_Tp>::value... >::value;
+        }
+
+        template <int&...>
+        static constexpr bool __enable_explicit_default() {
+            return
+                __all<is_default_constructible<_Tp>::value...>::value &&
+                !__enable_implicit_default< >();
         }
 
+
         template <class ..._Args>
         static constexpr bool __enable_explicit() {
             return
@@ -641,22 +649,26 @@ class _LIBCPP_TEMPLATE_VIS tuple
         const typename tuple_element<_Jp, tuple<_Up...> >::type&& get(const tuple<_Up...>&&) _NOEXCEPT;
 public:
 
-    template <bool _Dummy = true, class = typename enable_if<
-        _CheckArgsConstructor<_Dummy>::template __enable_default<_Tp...>()
-    >::type>
-    _LIBCPP_INLINE_VISIBILITY
-    _LIBCPP_CONSTEXPR tuple()
+    template <bool _Dummy = true, _EnableIf<
+        _CheckArgsConstructor<_Dummy>::__enable_implicit_default()
+    , void*> = nullptr>
+    _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+    tuple()
+        _NOEXCEPT_(__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
+
+    template <bool _Dummy = true, _EnableIf<
+        _CheckArgsConstructor<_Dummy>::__enable_explicit_default()
+    , void*> = nullptr>
+    explicit _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+    tuple()
         _NOEXCEPT_(__all<is_nothrow_default_constructible<_Tp>::value...>::value) {}
 
     tuple(tuple const&) = default;
     tuple(tuple&&) = default;
 
-    template <class _AllocArgT, class _Alloc, bool _Dummy = true, class = _EnableIf<
-        _And<
-            _IsSame<allocator_arg_t, _AllocArgT>,
-           __dependent_type<is_default_constructible<_Tp>, _Dummy>...
-       >::value
-      >
+    template <class _AllocArgT, class _Alloc, _EnableIf<
+             _CheckArgsConstructor<_IsSame<allocator_arg_t, _AllocArgT>::value >::__enable_implicit_default()
+      , void*> = nullptr
     >
     _LIBCPP_INLINE_VISIBILITY
     tuple(_AllocArgT, _Alloc const& __a)
@@ -665,6 +677,17 @@ public:
                     typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
                     __tuple_types<_Tp...>()) {}
 
+    template <class _AllocArgT, class _Alloc, _EnableIf<
+             _CheckArgsConstructor<_IsSame<allocator_arg_t, _AllocArgT>::value>::__enable_explicit_default()
+      , void*> = nullptr
+    >
+    explicit _LIBCPP_INLINE_VISIBILITY
+    tuple(_AllocArgT, _Alloc const& __a)
+      : __base_(allocator_arg_t(), __a,
+                    __tuple_indices<>(), __tuple_types<>(),
+                    typename __make_tuple_indices<sizeof...(_Tp), 0>::type(),
+                    __tuple_types<_Tp...>()) {}
+
     template <bool _Dummy = true,
               typename enable_if
                       <
diff --git a/libcxx/include/type_traits b/libcxx/include/type_traits
index 03e8bba5ec819..77dcc2321d991 100644
--- a/libcxx/include/type_traits
+++ b/libcxx/include/type_traits
@@ -509,8 +509,8 @@ struct _Lazy : _Func<_Args...> {};
 
 // Member detector base
 
-template <template <class...> class _Templ, class ..._Args>
-true_type __sfinae_test_impl(_FirstType<int, _Templ<_Args...> >);
+template <template <class...> class _Templ, class ..._Args, class = _Templ<_Args...> >
+true_type __sfinae_test_impl(int);
 template <template <class...> class, class ...>
 false_type __sfinae_test_impl(...);
 
@@ -2807,6 +2807,30 @@ _LIBCPP_INLINE_VAR _LIBCPP_CONSTEXPR bool is_default_constructible_v
     = is_default_constructible<_Tp>::value;
 #endif
 
+#ifndef _LIBCPP_CXX03_LANG
+// First of all, we can't implement this check in C++03 mode because the {}
+// default initialization syntax isn't valid.
+// Second, we implement the trait in a funny manner with two defaulted template
+// arguments to workaround Clang's PR43454.
+template <class _Tp>
+void __test_implicit_default_constructible(_Tp);
+
+template <class _Tp, class = void, bool = is_default_constructible<_Tp>::value>
+struct __is_implicitly_default_constructible
+    : false_type
+{ };
+
+template <class _Tp>
+struct __is_implicitly_default_constructible<_Tp, decltype(__test_implicit_default_constructible<_Tp const&>({})), true>
+    : true_type
+{ };
+
+template <class _Tp>
+struct __is_implicitly_default_constructible<_Tp, decltype(__test_implicit_default_constructible<_Tp const&>({})), false>
+    : false_type
+{ };
+#endif // !C++03
+
 // is_copy_constructible
 
 template <class _Tp>
diff --git a/libcxx/include/utility b/libcxx/include/utility
index 64599c8d55d0c..7ac322bfe7102 100644
--- a/libcxx/include/utility
+++ b/libcxx/include/utility
@@ -69,7 +69,7 @@ struct pair
 
     pair(const pair&) = default;
     pair(pair&&) = default;
-    constexpr pair();
+    explicit(see-below) constexpr pair();
     explicit(see-below) pair(const T1& x, const T2& y);                          // constexpr in C++14
     template <class U, class V> explicit(see-below) pair(U&& x, V&& y);          // constexpr in C++14
     template <class U, class V> explicit(see-below) pair(const pair<U, V>& p);   // constexpr in C++14
@@ -99,7 +99,7 @@ template <class T1, class T2>
 void
 swap(pair<T1, T2>& x, pair<T1, T2>& y) noexcept(noexcept(x.swap(y)));
 
-struct piecewise_construct_t { };
+struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
 inline constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
 
 template <class T> struct tuple_size;
@@ -276,7 +276,7 @@ template <class _Tp> constexpr add_const_t<_Tp>& as_const(_Tp& __t) noexcept { r
 template <class _Tp>                        void as_const(const _Tp&&) = delete;
 #endif
 
-struct _LIBCPP_TEMPLATE_VIS piecewise_construct_t { };
+struct _LIBCPP_TEMPLATE_VIS piecewise_construct_t { explicit piecewise_construct_t() = default; };
 #if defined(_LIBCPP_CXX03_LANG) || defined(_LIBCPP_BUILDING_LIBRARY)
 extern _LIBCPP_EXPORTED_FROM_ABI const piecewise_construct_t piecewise_construct;// = piecewise_construct_t();
 #else
@@ -334,10 +334,17 @@ struct _LIBCPP_TEMPLATE_VIS pair
     using _EnableB _LIBCPP_NODEBUG_TYPE = typename enable_if<_Val, bool>::type;
 
     struct _CheckArgs {
-      template <class _U1, class _U2>
-      static constexpr bool __enable_default() {
-          return is_default_constructible<_U1>::value
-              && is_default_constructible<_U2>::value;
+      template <int&...>
+      static constexpr bool __enable_explicit_default() {
+          return is_default_constructible<_T1>::value
+              && is_default_constructible<_T2>::value
+              && !__enable_implicit_default<>();
+      }
+
+      template <int&...>
+      static constexpr bool __enable_implicit_default() {
+          return __is_implicitly_default_constructible<_T1>::value
+              && __is_implicitly_default_constructible<_T2>::value;
       }
 
       template <class _U1, class _U2>
@@ -388,7 +395,15 @@ struct _LIBCPP_TEMPLATE_VIS pair
     >::type;
 
     template<bool _Dummy = true, _EnableB<
-            _CheckArgsDep<_Dummy>::template __enable_default<_T1, _T2>()
+            _CheckArgsDep<_Dummy>::__enable_explicit_default()
+    > = false>
+    explicit _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+    pair() _NOEXCEPT_(is_nothrow_default_constructible<first_type>::value &&
+                      is_nothrow_default_constructible<second_type>::value)
+        : first(), second() {}
+
+    template<bool _Dummy = true, _EnableB<
+            _CheckArgsDep<_Dummy>::__enable_implicit_default()
     > = false>
     _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
     pair() _NOEXCEPT_(is_nothrow_default_constructible<first_type>::value &&
diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 93be30b597e2d..c9c3f18087028 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -204,35 +204,6 @@ function(cxx_set_common_defines name)
   if(LIBCXX_CXX_ABI_HEADER_TARGET)
     add_dependencies(${name} ${LIBCXX_CXX_ABI_HEADER_TARGET})
   endif()
-  # Our visibility annotations are not quite right for non-Clang compilers,
-  # so we end up not exporting all the symbols we should. In the future, we
-  # can improve the situation by providing an explicit list of exported
-  # symbols on all compilers.
-  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
-    target_compile_options(${name} PRIVATE -fvisibility=hidden)
-  endif()
-  if(WIN32 AND NOT MINGW)
-    target_compile_definitions(${name}
-                               PRIVATE
-                                 # Ignore the -MSC_VER mismatch, as we may build
-                                 # with a different compatibility version.
-                                 _ALLOW_MSC_VER_MISMATCH
-                                 # Don't check the msvcprt iterator debug levels
-                                 # as we will define the iterator types; libc++
-                                 # uses a different macro to identify the debug
-                                 # level.
-                                 _ALLOW_ITERATOR_DEBUG_LEVEL_MISMATCH
-                                 # We are building the c++ runtime, don't pull in
-                                 # msvcprt.
-                                 _CRTBLD
-                                 # Don't warn on the use of "deprecated"
-                                 # "insecure" functions which are standards
-                                 # specified.
-                                 _CRT_SECURE_NO_WARNINGS
-                                 # Use the ISO conforming behaviour for conversion
-                                 # in printf, scanf.
-                                 _CRT_STDIO_ISO_WIDE_SPECIFIERS)
-  endif()
 
   if (LIBCXX_ENABLE_PARALLEL_ALGORITHMS)
     target_link_libraries(${name} PUBLIC pstl::ParallelSTL)
@@ -259,7 +230,10 @@ if (LIBCXX_ENABLE_SHARED)
       SOVERSION     "${LIBCXX_ABI_VERSION}"
       DEFINE_SYMBOL ""
   )
+  cxx_add_basic_build_flags(cxx_shared)
   cxx_set_common_defines(cxx_shared)
+  cxx_add_warning_flags(cxx_shared)
+  cxx_add_windows_flags(cxx_shared)
 
   # Link against LLVM libunwind
   if (LIBCXXABI_USE_LLVM_UNWINDER)
@@ -359,7 +333,10 @@ if (LIBCXX_ENABLE_STATIC)
       LINK_FLAGS    "${LIBCXX_LINK_FLAGS}"
       OUTPUT_NAME   "c++"
   )
+  cxx_add_basic_build_flags(cxx_static)
   cxx_set_common_defines(cxx_static)
+  cxx_add_warning_flags(cxx_static)
+  cxx_add_windows_flags(cxx_static)
 
   if (LIBCXX_HERMETIC_STATIC_LIBRARY)
     # If the hermetic library doesn't define the operator new/delete functions
diff --git a/libcxx/src/mutex.cpp b/libcxx/src/mutex.cpp
index 182785733efd5..7e979cd890413 100644
--- a/libcxx/src/mutex.cpp
+++ b/libcxx/src/mutex.cpp
@@ -21,9 +21,9 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 #ifndef _LIBCPP_HAS_NO_THREADS
 
-const defer_lock_t  defer_lock = {};
-const try_to_lock_t try_to_lock = {};
-const adopt_lock_t  adopt_lock = {};
+const defer_lock_t  defer_lock{};
+const try_to_lock_t try_to_lock{};
+const adopt_lock_t  adopt_lock{};
 
 // ~mutex is defined elsewhere
 
diff --git a/libcxx/src/utility.cpp b/libcxx/src/utility.cpp
index 016a5d91b85a4..6a690dc2879e1 100644
--- a/libcxx/src/utility.cpp
+++ b/libcxx/src/utility.cpp
@@ -10,6 +10,6 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-const piecewise_construct_t piecewise_construct = {};
+const piecewise_construct_t piecewise_construct{};
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/test/libcxx/debug/db_string_view.pass.cpp b/libcxx/test/libcxx/debug/db_string_view.pass.cpp
new file mode 100644
index 0000000000000..d4ae1cebdb431
--- /dev/null
+++ b/libcxx/test/libcxx/debug/db_string_view.pass.cpp
@@ -0,0 +1,41 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03, c++11, c++14
+// UNSUPPORTED: windows
+// UNSUPPORTED: libcpp-no-if-constexpr
+// MODULES_DEFINES: _LIBCPP_DEBUG=1
+
+// Can't test the system lib because this test enables debug mode
+// UNSUPPORTED: with_system_cxx_lib
+
+// test container debugging
+
+#define _LIBCPP_DEBUG 1
+#include <string_view>
+
+#include "test_macros.h"
+#include "debug_mode_helper.h"
+
+void test_null_argument() {
+  EXPECT_DEATH(std::string_view(nullptr));
+  EXPECT_DEATH(std::string_view(NULL));
+  EXPECT_DEATH(std::string_view(static_cast<const char*>(0)));
+  {
+    std::string_view v;
+    EXPECT_DEATH(((void)(v == nullptr)));
+    EXPECT_DEATH(((void)(nullptr == v)));
+  }
+}
+
+int main(int, char**)
+{
+  test_null_argument();
+
+  return 0;
+}
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
index 0cd74be2f500e..b42e4c4e962f0 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard.fail.cpp
@@ -14,7 +14,7 @@
 // UNSUPPORTED: c++98, c++03
 
 // GCC 7 is the first version to introduce [[nodiscard]]
-// UNSUPPORTED: gcc-4.9, gcc-5, gcc-6
+// UNSUPPORTED: gcc-5, gcc-6
 
 // MODULES_DEFINES: _LIBCPP_ENABLE_NODISCARD
 #define _LIBCPP_ENABLE_NODISCARD
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
index 530ea54ca6088..72fbc01198efa 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_after_cxx17.fail.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03
 
 // GCC 7 is the first version to introduce [[nodiscard]]
-// UNSUPPORTED: gcc-4.9, gcc-5, gcc-6
+// UNSUPPORTED: gcc-5, gcc-6
 
 // Test that _LIBCPP_DISABLE_NODISCARD_EXT only disables _LIBCPP_NODISCARD_EXT
 // and not _LIBCPP_NODISCARD_AFTER_CXX17.
diff --git a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
index 56df9248da1ba..65e5e1521331f 100644
--- a/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/enable_nodiscard_disable_nodiscard_ext.fail.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03
 
 // GCC 7 is the first version to introduce [[nodiscard]]
-// UNSUPPORTED: gcc-4.9, gcc-5, gcc-6
+// UNSUPPORTED: gcc-5, gcc-6
 
 
 // MODULES_DEFINES: _LIBCPP_ENABLE_NODISCARD
diff --git a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
index 43cc080e2e472..dd02a3dd25db6 100644
--- a/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
+++ b/libcxx/test/libcxx/diagnostics/nodiscard_extensions.fail.cpp
@@ -10,7 +10,7 @@
 // UNSUPPORTED: c++98, c++03
 
 // GCC versions prior to 7.0 don't provide the required [[nodiscard]] attribute.
-// UNSUPPORTED: gcc-4, gcc-5, gcc-6
+// UNSUPPORTED: gcc-5, gcc-6
 
 // AppleClang9 doesn't yet support C++17's implicitly synthesized deduction
 // guides from existing ctors, needed by default_searcher() below.
diff --git a/libcxx/test/libcxx/no_assert_include.sh.cpp b/libcxx/test/libcxx/no_assert_include.sh.cpp
new file mode 100644
index 0000000000000..25e3477b03de8
--- /dev/null
+++ b/libcxx/test/libcxx/no_assert_include.sh.cpp
@@ -0,0 +1,171 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Ensure that none of the standard C++ headers implicitly include cassert or
+// assert.h (because assert() is implemented as a macro).
+
+// RUN: %compile -fsyntax-only
+
+// Prevent <ext/hash_map> from generating deprecated warnings for this test.
+#if defined(__DEPRECATED)
+#undef __DEPRECATED
+#endif
+
+// Top level headers
+#include <algorithm>
+#include <any>
+#include <array>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <atomic>
+#endif
+#include <bit>
+#include <bitset>
+#include <ccomplex>
+#include <cctype>
+#include <cerrno>
+#include <cfenv>
+#include <cfloat>
+#include <charconv>
+#include <chrono>
+#include <cinttypes>
+#include <ciso646>
+#include <climits>
+#include <clocale>
+#include <cmath>
+#include <codecvt>
+#include <compare>
+#include <complex>
+#include <complex.h>
+#include <condition_variable>
+#include <csetjmp>
+#include <csignal>
+#include <cstdarg>
+#include <cstdbool>
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <ctgmath>
+#include <ctime>
+#include <ctype.h>
+#include <cwchar>
+#include <cwctype>
+#include <deque>
+#include <errno.h>
+#include <exception>
+#include <execution>
+#include <fenv.h>
+#include <filesystem>
+#include <float.h>
+#include <forward_list>
+#include <fstream>
+#include <functional>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <future>
+#endif
+#include <initializer_list>
+#include <inttypes.h>
+#include <iomanip>
+#include <ios>
+#include <iosfwd>
+#include <iostream>
+#include <istream>
+#include <iterator>
+#include <limits>
+#include <limits.h>
+#include <list>
+#include <locale>
+#include <locale.h>
+#include <map>
+#include <math.h>
+#include <memory>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <mutex>
+#endif
+#include <new>
+#include <numeric>
+#include <optional>
+#include <ostream>
+#include <queue>
+#include <random>
+#include <ratio>
+#include <regex>
+#include <scoped_allocator>
+#include <set>
+#include <setjmp.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <shared_mutex>
+#endif
+#include <span>
+#include <sstream>
+#include <stack>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdexcept>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <streambuf>
+#include <string>
+#include <string.h>
+#include <string_view>
+#include <strstream>
+#include <system_error>
+#include <tgmath.h>
+#ifndef _LIBCPP_HAS_NO_THREADS
+#include <thread>
+#endif
+#include <tuple>
+#include <typeindex>
+#include <typeinfo>
+#include <type_traits>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <valarray>
+#include <variant>
+#include <vector>
+#include <version>
+#include <wchar.h>
+#include <wctype.h>
+
+// experimental headers
+#if __cplusplus >= 201103L
+#include <experimental/algorithm>
+#if defined(__cpp_coroutines)
+#include <experimental/coroutine>
+#endif
+#include <experimental/deque>
+#include <experimental/filesystem>
+#include <experimental/forward_list>
+#include <experimental/functional>
+#include <experimental/iterator>
+#include <experimental/list>
+#include <experimental/map>
+#include <experimental/memory_resource>
+#include <experimental/propagate_const>
+#include <experimental/regex>
+#include <experimental/simd>
+#include <experimental/set>
+#include <experimental/string>
+#include <experimental/type_traits>
+#include <experimental/unordered_map>
+#include <experimental/unordered_set>
+#include <experimental/utility>
+#include <experimental/vector>
+#endif // __cplusplus >= 201103L
+
+// extended headers
+#include <ext/hash_map>
+#include <ext/hash_set>
+
+#ifdef assert
+#error "Do not include cassert or assert.h in standard header files"
+#endif
diff --git a/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp b/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
index ab7c7eb200c0e..802892a5eb39c 100644
--- a/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
+++ b/libcxx/test/libcxx/thread/thread.lock/thread.lock.guard/nodiscard.fail.cpp
@@ -11,6 +11,7 @@
 // [[nodiscard]] on constructors isn't supported by all compilers
 // UNSUPPORTED: clang-6, clang-7, clang-8, clang-9
 // UNSUPPORTED: apple-clang-9, apple-clang-10, apple-clang-11
+// UNSUPPORTED: gcc-5
 
 // [[nodiscard]] isn't supported in C++98 and C++03 (not even as an extension)
 // UNSUPPORTED: c++98, c++03
diff --git a/libcxx/test/libcxx/type_traits/is_implicitly_default_constructible.pass.cpp b/libcxx/test/libcxx/type_traits/is_implicitly_default_constructible.pass.cpp
new file mode 100644
index 0000000000000..eef2828a21747
--- /dev/null
+++ b/libcxx/test/libcxx/type_traits/is_implicitly_default_constructible.pass.cpp
@@ -0,0 +1,85 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, this trait fails. See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <type_traits>
+
+// __is_implicitly_default_constructible<Tp>
+
+#include <type_traits>
+
+
+struct ExplicitlyDefaultConstructible1 {
+    explicit ExplicitlyDefaultConstructible1() = default;
+};
+
+struct ExplicitlyDefaultConstructible2 {
+    explicit ExplicitlyDefaultConstructible2() { }
+};
+
+struct ImplicitlyDefaultConstructible1 {
+    ImplicitlyDefaultConstructible1() { }
+};
+
+struct ImplicitlyDefaultConstructible2 {
+    ImplicitlyDefaultConstructible2() = default;
+};
+
+struct NonDefaultConstructible1 {
+    NonDefaultConstructible1() = delete;
+};
+
+struct NonDefaultConstructible2 {
+    explicit NonDefaultConstructible2() = delete;
+};
+
+struct NonDefaultConstructible3 {
+    NonDefaultConstructible3(NonDefaultConstructible3&&) { }
+};
+
+struct ProtectedDefaultConstructible {
+protected:
+    ProtectedDefaultConstructible() = default;
+};
+
+struct PrivateDefaultConstructible {
+private:
+    PrivateDefaultConstructible() = default;
+};
+
+struct Base { };
+
+struct ProtectedDefaultConstructibleWithBase : Base {
+protected:
+    ProtectedDefaultConstructibleWithBase() = default;
+};
+
+struct PrivateDefaultConstructibleWithBase : Base {
+private:
+    PrivateDefaultConstructibleWithBase() = default;
+};
+
+static_assert(!std::__is_implicitly_default_constructible<ExplicitlyDefaultConstructible1>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<ExplicitlyDefaultConstructible2>::value, "");
+static_assert(std::__is_implicitly_default_constructible<ImplicitlyDefaultConstructible1>::value, "");
+static_assert(std::__is_implicitly_default_constructible<ImplicitlyDefaultConstructible2>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<NonDefaultConstructible1>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<NonDefaultConstructible2>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<NonDefaultConstructible3>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<ProtectedDefaultConstructible>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<PrivateDefaultConstructible>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<ProtectedDefaultConstructibleWithBase>::value, "");
+static_assert(!std::__is_implicitly_default_constructible<PrivateDefaultConstructibleWithBase>::value, "");
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/non_trivial_copy_move_ABI.pass.cpp b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/non_trivial_copy_move_ABI.pass.cpp
index 00c1910a3ed17..65fe65a1a03d8 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/non_trivial_copy_move_ABI.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/non_trivial_copy_move_ABI.pass.cpp
@@ -6,9 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// The test fails due to the missing is_trivially_constructible intrinsic.
-// XFAIL: gcc-4.9
-
 // The test suite needs to define the ABI macros on the command line when
 // modules are enabled.
 // UNSUPPORTED: -fmodules
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/trivial_copy_move_ABI.pass.cpp b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/trivial_copy_move_ABI.pass.cpp
index 1086011c37d82..135715751fa33 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/trivial_copy_move_ABI.pass.cpp
+++ b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/trivial_copy_move_ABI.pass.cpp
@@ -6,9 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// The test fails due to the missing is_trivially_constructible intrinsic.
-// XFAIL: gcc-4.9
-
 // <utility>
 
 // template <class T1, class T2> struct pair
diff --git a/libcxx/test/pretty_printers/gdb_pretty_printer_test.py b/libcxx/test/pretty_printers/gdb_pretty_printer_test.py
index 5e42568873430..b738c72b64c26 100644
--- a/libcxx/test/pretty_printers/gdb_pretty_printer_test.py
+++ b/libcxx/test/pretty_printers/gdb_pretty_printer_test.py
@@ -51,7 +51,7 @@ def invoke(self, arg, from_tty):
                 test_fails = not re.match(check_literal, value)
             else:
                 check_literal_string = expectation_val.string(encoding="utf-8")
-                check_literal = check_literal_string.encode("utf-8")
+                check_literal = str(check_literal_string.encode("utf-8"))
                 test_fails = value != check_literal
 
             if test_fails:
diff --git a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
index 0b32ef8437ac3..138af7e74bf1c 100644
--- a/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
+++ b/libcxx/test/pretty_printers/gdb_pretty_printer_test.sh.cpp
@@ -27,6 +27,8 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "test_macros.h"
+
 // To write a pretty-printer test:
 //
 // 1. Declare a variable of the type you want to test
@@ -54,11 +56,20 @@
 // ComparePrettyPrintTo*.  Also, make sure neither it, nor the
 // variables we need present in the Compare functions are optimized
 // away.
-void StopForDebugger(void *value, void *check) __attribute__((optnone)) { }
+#ifdef TEST_COMPILER_GCC
+#define OPT_NONE __attribute__((noinline))
+#else
+#define OPT_NONE __attribute__((optnone))
+#endif
+void StopForDebugger(void *value, void *check) OPT_NONE;
+void StopForDebugger(void *value, void *check)  {}
+
 
 // Prevents the compiler optimizing away the parameter in the caller function.
 template <typename Type>
-void MarkAsLive(Type &&t) __attribute__((optnone)) { }
+void MarkAsLive(Type &&t) OPT_NONE;
+template <typename Type>
+void MarkAsLive(Type &&t) {}
 
 // In all of the Compare(Expression)PrettyPrintTo(Regex/Chars) functions below,
 // the python script sets a breakpoint just before the call to StopForDebugger,
@@ -425,7 +436,7 @@ void set_iterator_test() {
   MarkAsLive(not_found);
   // Because the end_node is not easily detected, just be sure it doesn't crash.
   CompareExpressionPrettyPrintToRegex("not_found",
-      R"(std::__tree_const_iterator  = {\[0x[a-f0-9]+\] = .*})");
+      R"(std::__tree_const_iterator ( = {\[0x[a-f0-9]+\] = .*}|<error reading variable:.*>))");
 }
 
 void map_iterator_test() {
diff --git a/libcxx/test/std/containers/associative/map/map.cons/deduct.fail.cpp b/libcxx/test/std/containers/associative/map/map.cons/deduct.fail.cpp
index 2e719da6442bb..bb9be47129f7a 100644
--- a/libcxx/test/std/containers/associative/map/map.cons/deduct.fail.cpp
+++ b/libcxx/test/std/containers/associative/map/map.cons/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <map>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 //  clang-6 gives different error messages.
 
 // template<class InputIterator,
diff --git a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.fail.cpp b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.fail.cpp
index 70fa2522cc1e1..fe50a3de07de8 100644
--- a/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.fail.cpp
+++ b/libcxx/test/std/containers/associative/multimap/multimap.cons/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <map>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 //  clang-6 gives different error messages.
 
 // template<class InputIterator,
diff --git a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.fail.cpp b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.fail.cpp
index 34ad2efe1a1ad..bd58d9109a279 100644
--- a/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.fail.cpp
+++ b/libcxx/test/std/containers/associative/multiset/multiset.cons/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <set>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 //  clang-6 gives different error messages.
 
 // template<class InputIterator,
diff --git a/libcxx/test/std/containers/associative/set/set.cons/deduct.fail.cpp b/libcxx/test/std/containers/associative/set/set.cons/deduct.fail.cpp
index a1278073ec874..435eb97f4261e 100644
--- a/libcxx/test/std/containers/associative/set/set.cons/deduct.fail.cpp
+++ b/libcxx/test/std/containers/associative/set/set.cons/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <set>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 //  clang-6 gives different error messages.
 
 // template<class InputIterator,
diff --git a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.fail.cpp b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.fail.cpp
index 642abce8a27bb..e7ba02450919b 100644
--- a/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.map/unord.map.cnstr/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <unordered_map>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 
 // template<class InputIterator,
 //          class Hash = hash<iter-key-type<InputIterator>>,
diff --git a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.fail.cpp b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.fail.cpp
index 7f170472d7a52..eff8bed212980 100644
--- a/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multimap/unord.multimap.cnstr/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <unordered_map>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 
 // template<class InputIterator,
 //          class Hash = hash<iter-key-type<InputIterator>>,
diff --git a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.fail.cpp b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.fail.cpp
index ea359255f615f..ecd09daa6b5af 100644
--- a/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.multiset/unord.multiset.cnstr/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <unordered_set>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 
 // template<class InputIterator,
 //        class Hash = hash<iter-value-type<InputIterator>>,
diff --git a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.fail.cpp b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.fail.cpp
index 76fbd20d0c46e..fc381faf8a134 100644
--- a/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.fail.cpp
+++ b/libcxx/test/std/containers/unord/unord.set/unord.set.cnstr/deduct.fail.cpp
@@ -9,7 +9,7 @@
 // <unordered_set>
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 // UNSUPPORTED: libcpp-no-deduction-guides
-// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0
+// XFAIL: clang-6, apple-clang-9.0, apple-clang-9.1, apple-clang-10.0.0
 
 // template<class InputIterator,
 //        class Hash = hash<iter-value-type<InputIterator>>,
diff --git a/libcxx/test/std/experimental/simd/simd.cons/broadcast.pass.cpp b/libcxx/test/std/experimental/simd/simd.cons/broadcast.pass.cpp
index cf038e3842786..b55208a4eae94 100644
--- a/libcxx/test/std/experimental/simd/simd.cons/broadcast.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.cons/broadcast.pass.cpp
@@ -8,16 +8,14 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// See GCC PR63723.
-// UNSUPPORTED: gcc-4.9
-
 // <experimental/simd>
 //
 // [simd.class]
 // template <class U> simd(U&& value);
 
-#include <cstdint>
 #include <experimental/simd>
+#include <cstdint>
+#include <cassert>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/experimental/simd/simd.cons/generator.pass.cpp b/libcxx/test/std/experimental/simd/simd.cons/generator.pass.cpp
index 500ccce817d8c..59d0a9c98b9d5 100644
--- a/libcxx/test/std/experimental/simd/simd.cons/generator.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.cons/generator.pass.cpp
@@ -8,9 +8,6 @@
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14
 
-// See GCC PR63723.
-// UNSUPPORTED: gcc-4.9
-
 // <experimental/simd>
 //
 // [simd.class]
@@ -18,6 +15,7 @@
 
 #include <experimental/simd>
 #include <cstdint>
+#include <cassert>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/experimental/simd/simd.cons/load.pass.cpp b/libcxx/test/std/experimental/simd/simd.cons/load.pass.cpp
index 9440d5aefa358..c678bd5dbb684 100644
--- a/libcxx/test/std/experimental/simd/simd.cons/load.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.cons/load.pass.cpp
@@ -13,8 +13,9 @@
 // [simd.class]
 // template <class U, class Flags> simd(const U* mem, Flags f);
 
-#include <cstdint>
 #include <experimental/simd>
+#include <cstdint>
+#include <cassert>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/experimental/simd/simd.mem/load.pass.cpp b/libcxx/test/std/experimental/simd/simd.mem/load.pass.cpp
index 1a56161c5fb0a..bd9d563190a67 100644
--- a/libcxx/test/std/experimental/simd/simd.mem/load.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.mem/load.pass.cpp
@@ -15,6 +15,7 @@
 
 #include <experimental/simd>
 #include <cstdint>
+#include <cassert>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/experimental/simd/simd.mem/store.pass.cpp b/libcxx/test/std/experimental/simd/simd.mem/store.pass.cpp
index 3cc3d1fa14dc7..ad07239342437 100644
--- a/libcxx/test/std/experimental/simd/simd.mem/store.pass.cpp
+++ b/libcxx/test/std/experimental/simd/simd.mem/store.pass.cpp
@@ -15,6 +15,7 @@
 
 #include <experimental/simd>
 #include <cstdint>
+#include <cassert>
 
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/ios_Init/ios_Init.multiple.pass.cpp b/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/ios_Init/ios_Init.multiple.pass.cpp
index 37b4de9def52a..f17cb8e9e01c1 100644
--- a/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/ios_Init/ios_Init.multiple.pass.cpp
+++ b/libcxx/test/std/input.output/iostreams.base/ios.base/ios.types/ios_Init/ios_Init.multiple.pass.cpp
@@ -14,6 +14,18 @@
 // Test to make sure that the streams only get initialized once
 // Taken from https://bugs.llvm.org/show_bug.cgi?id=43300
 
+// The dylibs shipped on macOS so far do not contain the fix for PR43300, so
+// this test fails.
+// XFAIL: with_system_cxx_lib=macosx10.15
+// XFAIL: with_system_cxx_lib=macosx10.14
+// XFAIL: with_system_cxx_lib=macosx10.13
+// XFAIL: with_system_cxx_lib=macosx10.12
+// XFAIL: with_system_cxx_lib=macosx10.11
+// XFAIL: with_system_cxx_lib=macosx10.10
+// XFAIL: with_system_cxx_lib=macosx10.9
+// XFAIL: with_system_cxx_lib=macosx10.8
+// XFAIL: with_system_cxx_lib=macosx10.7
+
 int main(int, char**)
 {
 
diff --git a/libcxx/test/std/iterators/stream.iterators/istream.iterator/types.pass.cpp b/libcxx/test/std/iterators/stream.iterators/istream.iterator/types.pass.cpp
index 5170b1e536581..aa00a0d53b100 100644
--- a/libcxx/test/std/iterators/stream.iterators/istream.iterator/types.pass.cpp
+++ b/libcxx/test/std/iterators/stream.iterators/istream.iterator/types.pass.cpp
@@ -8,9 +8,6 @@
 
 // <iterator>
 
-// Test fails due to use of is_trivially_* trait.
-// XFAIL: gcc-4.9
-
 // template <class T, class charT = char, class traits = char_traits<charT>,
 //           class Distance = ptrdiff_t>
 // class istream_iterator
diff --git a/libcxx/test/std/iterators/stream.iterators/istreambuf.iterator/types.pass.cpp b/libcxx/test/std/iterators/stream.iterators/istreambuf.iterator/types.pass.cpp
index a6c6435f10a49..c2f323b03a6eb 100644
--- a/libcxx/test/std/iterators/stream.iterators/istreambuf.iterator/types.pass.cpp
+++ b/libcxx/test/std/iterators/stream.iterators/istreambuf.iterator/types.pass.cpp
@@ -8,9 +8,6 @@
 
 // <iterator>
 
-// Test fails due to use of is_trivially_* trait.
-// XFAIL: gcc-4.9
-
 // template<class charT, class traits = char_traits<charT> >
 // class istreambuf_iterator
 //     : public iterator<input_iterator_tag, charT,
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp
index d69c28be72b51..c4e10715f2793 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/sized_delete_array14.pass.cpp
@@ -15,10 +15,6 @@
 // (except clang-3.6 which temporarily enabled sized-deallocation)
 // UNSUPPORTED: clang, apple-clang
 
-// NOTE: GCC 4.9.1 does not support sized-deallocation in C++14. However
-// GCC 5.1 does.
-// XFAIL: gcc-4.7, gcc-4.8, gcc-4.9
-
 #include <new>
 #include <cstddef>
 #include <cstdlib>
diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp
index deb17d1ae7412..819d62ae2fbf7 100644
--- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/sized_delete14.pass.cpp
@@ -15,10 +15,6 @@
 // (except clang-3.6 which temporarily enabled sized-deallocation)
 // UNSUPPORTED: clang, apple-clang
 
-// NOTE: GCC 4.9.1 does not support sized-deallocation in C++14. However
-// GCC 5.1 does.
-// XFAIL: gcc-4.7, gcc-4.8, gcc-4.9
-
 #include <new>
 #include <cstddef>
 #include <cstdlib>
diff --git a/libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp b/libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp
new file mode 100644
index 0000000000000..78ab087c3ea1b
--- /dev/null
+++ b/libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// struct nothrow_t {
+//   explicit nothrow_t() = default;
+// };
+// extern const nothrow_t nothrow;
+
+// This test checks for LWG 2510.
+
+#include <new>
+
+
+std::nothrow_t f() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/language.support/support.dynamic/nothrow_t.pass.cpp b/libcxx/test/std/language.support/support.dynamic/nothrow_t.pass.cpp
new file mode 100644
index 0000000000000..3a6231329f096
--- /dev/null
+++ b/libcxx/test/std/language.support/support.dynamic/nothrow_t.pass.cpp
@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// struct nothrow_t {
+//   explicit nothrow_t() = default;
+// };
+// extern const nothrow_t nothrow;
+
+#include <new>
+
+
+int main(int, char**) {
+  std::nothrow_t x = std::nothrow;
+  (void)x;
+
+  return 0;
+}
diff --git a/libcxx/test/std/re/re.regex/re.regex.assign/assign_ptr_size_flag.pass.cpp b/libcxx/test/std/re/re.regex/re.regex.assign/assign_ptr_size_flag.pass.cpp
index cdbdae430543b..e74a42b693409 100644
--- a/libcxx/test/std/re/re.regex/re.regex.assign/assign_ptr_size_flag.pass.cpp
+++ b/libcxx/test/std/re/re.regex/re.regex.assign/assign_ptr_size_flag.pass.cpp
@@ -18,6 +18,16 @@
 
 int main(int, char**)
 {
+    std::regex r0;
+    r0.assign("(a([bc]))", 9);
+    assert(r0.flags() == std::regex::ECMAScript);
+    assert(r0.mark_count() == 2);
+
+    std::regex r1;
+    r1.assign("(a([bc]))", 9, std::regex::ECMAScript);
+    assert(r1.flags() == std::regex::ECMAScript);
+    assert(r1.mark_count() == 2);
+
     std::regex r2;
     r2.assign("(a([bc]))", 9, std::regex::extended);
     assert(r2.flags() == std::regex::extended);
diff --git a/libcxx/test/std/strings/string.view/string.view.io/stream_insert_decl_present.pass.cpp b/libcxx/test/std/strings/string.view/string.view.io/stream_insert_decl_present.pass.cpp
new file mode 100644
index 0000000000000..be0567a737231
--- /dev/null
+++ b/libcxx/test/std/strings/string.view/string.view.io/stream_insert_decl_present.pass.cpp
@@ -0,0 +1,26 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <string>
+
+// template<class charT, class traits, class Allocator>
+//   basic_ostream<charT, traits>&
+//   operator<<(basic_ostream<charT, traits>& os,
+//              const basic_string_view<charT,traits> str);
+
+#include <string_view>
+#include <iosfwd>
+
+template <class SV, class = void>
+struct HasDecl : std::false_type {};
+template <class SV>
+struct HasDecl<SV, decltype(static_cast<void>(std::declval<std::ostream&>() << std::declval<SV&>()))> : std::true_type {};
+
+int main() {
+  static_assert(HasDecl<std::string_view>::value, "streaming operator declaration not present");
+}
diff --git a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until.pass.cpp b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until.pass.cpp
index 12ccf3f1c06e0..e5c77f28eb84e 100644
--- a/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until.pass.cpp
+++ b/libcxx/test/std/thread/thread.condition/thread.condition.condvar/wait_until.pass.cpp
@@ -25,12 +25,12 @@
 
 #include "test_macros.h"
 
-struct Clock
+struct TestClock
 {
     typedef std::chrono::milliseconds duration;
     typedef duration::rep             rep;
     typedef duration::period          period;
-    typedef std::chrono::time_point<Clock> time_point;
+    typedef std::chrono::time_point<TestClock> time_point;
     static const bool is_steady =  true;
 
     static time_point now()
@@ -50,35 +50,40 @@ int test2 = 0;
 
 int runs = 0;
 
+template <typename Clock>
 void f()
 {
     std::unique_lock<std::mutex> lk(mut);
     assert(test2 == 0);
     test1 = 1;
     cv.notify_one();
-    Clock::time_point t0 = Clock::now();
-    Clock::time_point t = t0 + Clock::duration(250);
+    typename Clock::time_point t0 = Clock::now();
+    typename Clock::time_point t = t0 + std::chrono::milliseconds(250);
     while (test2 == 0 && cv.wait_until(lk, t) == std::cv_status::no_timeout)
         ;
-    Clock::time_point t1 = Clock::now();
+    typename Clock::time_point t1 = Clock::now();
     if (runs == 0)
     {
-        assert(t1 - t0 < Clock::duration(250));
+        assert(t1 - t0 < std::chrono::milliseconds(250));
         assert(test2 != 0);
     }
     else
     {
-        assert(t1 - t0 - Clock::duration(250) < Clock::duration(50));
+        assert(t1 - t0 - std::chrono::milliseconds(250) < std::chrono::milliseconds(50));
         assert(test2 == 0);
     }
     ++runs;
 }
 
-int main(int, char**)
+template <typename Clock>
+void run_test()
 {
+    runs = 0;
+    test1 = 0;
+    test2 = 0;
     {
         std::unique_lock<std::mutex>lk(mut);
-        std::thread t(f);
+        std::thread t(f<Clock>);
         assert(test1 == 0);
         while (test1 == 0)
             cv.wait(lk);
@@ -92,7 +97,7 @@ int main(int, char**)
     test2 = 0;
     {
         std::unique_lock<std::mutex>lk(mut);
-        std::thread t(f);
+        std::thread t(f<Clock>);
         assert(test1 == 0);
         while (test1 == 0)
             cv.wait(lk);
@@ -100,6 +105,12 @@ int main(int, char**)
         lk.unlock();
         t.join();
     }
+}
 
-  return 0;
+int main(int, char**)
+{
+    run_test<TestClock>();
+    run_test<std::chrono::steady_clock>();
+    run_test<std::chrono::system_clock>();
+    return 0;
 }
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp
new file mode 100644
index 0000000000000..4542c04c297f9
--- /dev/null
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp
@@ -0,0 +1,34 @@
+//===----------------------------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+// UNSUPPORTED: libcpp-has-no-threads
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <mutex>
+
+// struct defer_lock_t { explicit defer_lock_t() = default; };
+// struct try_to_lock_t { explicit try_to_lock_t() = default; };
+// struct adopt_lock_t { explicit adopt_lock_t() = default; };
+
+// This test checks for LWG 2510.
+
+#include <mutex>
+
+
+std::defer_lock_t f1() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::try_to_lock_t f2() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::adopt_lock_t f3() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/types.pass.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/types.pass.cpp
index 84c39baafa6aa..c03ca4622aabf 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/types.pass.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/types.pass.cpp
@@ -10,16 +10,15 @@
 
 // <mutex>
 
-// struct defer_lock_t {};
-// struct try_to_lock_t {};
-// struct adopt_lock_t {};
+// struct defer_lock_t { explicit defer_lock_t() = default; };
+// struct try_to_lock_t { explicit try_to_lock_t() = default; };
+// struct adopt_lock_t { explicit adopt_lock_t() = default; };
 //
 // constexpr defer_lock_t  defer_lock{};
 // constexpr try_to_lock_t try_to_lock{};
 // constexpr adopt_lock_t  adopt_lock{};
 
 #include <mutex>
-#include <type_traits>
 
 #include "test_macros.h"
 
@@ -33,5 +32,5 @@ int main(int, char**)
     T2 t2 = std::try_to_lock; ((void)t2);
     T3 t3 = std::adopt_lock; ((void)t3);
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/function.objects/refwrap/type_properties.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/type_properties.pass.cpp
index 17eef26f07a6d..d8d75443bf5d1 100644
--- a/libcxx/test/std/utilities/function.objects/refwrap/type_properties.pass.cpp
+++ b/libcxx/test/std/utilities/function.objects/refwrap/type_properties.pass.cpp
@@ -13,9 +13,6 @@
 // Test that reference wrapper meets the requirements of CopyConstructible and
 // CopyAssignable, and TriviallyCopyable (starting in C++14).
 
-// Test fails due to use of is_trivially_* trait.
-// XFAIL: gcc-4.9 && c++14
-
 #include <functional>
 #include <type_traits>
 #include <string>
diff --git a/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp
new file mode 100644
index 0000000000000..fb991d425f12f
--- /dev/null
+++ b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <memory>
+
+// struct allocator_arg_t { explicit allocator_arg_t() = default; };
+// const allocator_arg_t allocator_arg = allocator_arg_t();
+
+// This test checks for LWG 2510.
+
+#include <memory>
+
+
+std::allocator_arg_t f() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.pass.cpp b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.pass.cpp
index 0253243319af4..e22bec1e5144c 100644
--- a/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.pass.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.pass.cpp
@@ -8,7 +8,7 @@
 
 // <memory>
 
-// struct allocator_arg_t { };
+// struct allocator_arg_t { explicit allocator_arg_t() = default; };
 // const allocator_arg_t allocator_arg = allocator_arg_t();
 
 #include <memory>
@@ -21,5 +21,5 @@ int main(int, char**)
 {
     test(std::allocator_arg);
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.addressof/constexpr_addressof.pass.cpp b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.addressof/constexpr_addressof.pass.cpp
index 2aabf812a75f6..ae8a162ef2faa 100644
--- a/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.addressof/constexpr_addressof.pass.cpp
+++ b/libcxx/test/std/utilities/memory/specialized.algorithms/specialized.addressof/constexpr_addressof.pass.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14
-// XFAIL: gcc-4, gcc-5, gcc-6
+// XFAIL: gcc-5, gcc-6
 
 // <memory>
 
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp
index 40d4639d88102..72069456fa71a 100644
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp
+++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared.pass.cpp
@@ -6,8 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-// UNSUPPORTED: c++98, c++03
-
 // <memory>
 
 // shared_ptr
@@ -52,8 +50,87 @@ struct A
 
 int A::count = 0;
 
+struct Zero
+{
+    static int count;
+    Zero() {++count;}
+    Zero(Zero const &) {++count;}
+    ~Zero() {--count;}
+};
+
+int Zero::count = 0;
+
+struct One
+{
+    static int count;
+    int value;
+    explicit One(int v) : value(v) {++count;}
+    One(One const & o) : value(o.value) {++count;}
+    ~One() {--count;}
+};
+
+int One::count = 0;
+
+
+struct Two
+{
+    static int count;
+    int value;
+    Two(int v, int) : value(v) {++count;}
+    Two(Two const & o) : value(o.value) {++count;}
+    ~Two() {--count;}
+};
+
+int Two::count = 0;
+
+struct Three
+{
+    static int count;
+    int value;
+    Three(int v, int, int) : value(v) {++count;}
+    Three(Three const & o) : value(o.value) {++count;}
+    ~Three() {--count;}
+};
+
+int Three::count = 0;
+
+template <class Alloc>
+void test()
+{
+    int const bad = -1;
+    {
+    std::shared_ptr<Zero> p = std::allocate_shared<Zero>(Alloc());
+    assert(Zero::count == 1);
+    }
+    assert(Zero::count == 0);
+    {
+    int const i = 42;
+    std::shared_ptr<One> p = std::allocate_shared<One>(Alloc(), i);
+    assert(One::count == 1);
+    assert(p->value == i);
+    }
+    assert(One::count == 0);
+    {
+    int const i = 42;
+    std::shared_ptr<Two> p = std::allocate_shared<Two>(Alloc(), i, bad);
+    assert(Two::count == 1);
+    assert(p->value == i);
+    }
+    assert(Two::count == 0);
+    {
+    int const i = 42;
+    std::shared_ptr<Three> p = std::allocate_shared<Three>(Alloc(), i, bad, bad);
+    assert(Three::count == 1);
+    assert(p->value == i);
+    }
+    assert(Three::count == 0);
+}
+
 int main(int, char**)
 {
+    test<bare_allocator<void> >();
+    test<test_allocator<void> >();
+
     {
     int i = 67;
     char c = 'e';
diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_cxx03.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_cxx03.pass.cpp
deleted file mode 100644
index 13c929dd4f724..0000000000000
--- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_cxx03.pass.cpp
+++ /dev/null
@@ -1,120 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <memory>
-
-// shared_ptr
-
-// template<class T, class A, class... Args>
-//    shared_ptr<T> allocate_shared(const A& a, Args&&... args);
-
-
-#include <memory>
-#include <new>
-#include <cstdlib>
-#include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
-#include "min_allocator.h"
-
-struct Zero
-{
-    static int count;
-    Zero() {++count;}
-    Zero(Zero const &) {++count;}
-    ~Zero() {--count;}
-};
-
-int Zero::count = 0;
-
-struct One
-{
-    static int count;
-    int value;
-    explicit One(int v) : value(v) {++count;}
-    One(One const & o) : value(o.value) {++count;}
-    ~One() {--count;}
-};
-
-int One::count = 0;
-
-
-struct Two
-{
-    static int count;
-    int value;
-    Two(int v, int) : value(v) {++count;}
-    Two(Two const & o) : value(o.value) {++count;}
-    ~Two() {--count;}
-};
-
-int Two::count = 0;
-
-struct Three
-{
-    static int count;
-    int value;
-    Three(int v, int, int) : value(v) {++count;}
-    Three(Three const & o) : value(o.value) {++count;}
-    ~Three() {--count;}
-};
-
-int Three::count = 0;
-
-template <class Alloc>
-void test()
-{
-    int const bad = -1;
-    {
-    std::shared_ptr<Zero> p = std::allocate_shared<Zero>(Alloc());
-    assert(Zero::count == 1);
-    }
-    assert(Zero::count == 0);
-    {
-    int const i = 42;
-    std::shared_ptr<One> p = std::allocate_shared<One>(Alloc(), i);
-    assert(One::count == 1);
-    assert(p->value == i);
-    }
-    assert(One::count == 0);
-    {
-    int const i = 42;
-    std::shared_ptr<Two> p = std::allocate_shared<Two>(Alloc(), i, bad);
-    assert(Two::count == 1);
-    assert(p->value == i);
-    }
-    assert(Two::count == 0);
-    {
-    int const i = 42;
-    std::shared_ptr<Three> p = std::allocate_shared<Three>(Alloc(), i, bad, bad);
-    assert(Three::count == 1);
-    assert(p->value == i);
-    }
-    assert(Three::count == 0);
-}
-
-int main(int, char**)
-{
-    {
-    int i = 67;
-    int const bad = -1;
-    std::shared_ptr<Two> p = std::allocate_shared<Two>(test_allocator<Two>(54), i, bad);
-    assert(test_allocator<Two>::alloc_count == 1);
-    assert(Two::count == 1);
-    assert(p->value == 67);
-    }
-    assert(Two::count == 0);
-    assert(test_allocator<Two>::alloc_count == 0);
-
-    test<bare_allocator<void> >();
-#if TEST_STD_VER >= 11
-    test<min_allocator<void> >();
-#endif
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/has_unique_object_representations.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/has_unique_object_representations.pass.cpp
index e6f70123b2c97..8202fd73e992c 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/has_unique_object_representations.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/has_unique_object_representations.pass.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++98, c++03, c++11, c++14
-// UNSUPPORTED: clang-3, clang-4, clang-5, apple-clang, gcc-4, gcc-5, gcc-6
+// UNSUPPORTED: clang-3, clang-4, clang-5, apple-clang, gcc-5, gcc-6
 
 // type_traits
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_constructible.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_constructible.pass.cpp
index a5b723c0e5970..b51972432d036 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_constructible.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_constructible.pass.cpp
@@ -244,13 +244,17 @@ int main(int, char**)
 
     test_is_constructible<const int&, ExplicitTo<int&>&>();
     test_is_constructible<const int&, ExplicitTo<int&>>();
-    test_is_constructible<int&, ExplicitTo<int&>>();
-    test_is_constructible<const int&, ExplicitTo<int&&>>();
+
 
     // Binding through reference-compatible type is required to perform
     // direct-initialization as described in [over.match.ref] p. 1 b. 1:
+    //
+    // But the rvalue to lvalue reference binding isn't allowed according to
+    // [over.match.ref] despite Clang accepting it.
     test_is_constructible<int&, ExplicitTo<int&>>();
+#ifndef TEST_COMPILER_GCC
     test_is_constructible<const int&, ExplicitTo<int&&>>();
+#endif
 
     static_assert(std::is_constructible<int&&, ExplicitTo<int&&>>::value, "");
 #ifdef __clang__
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_assignable.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_assignable.pass.cpp
index 124480c075551..5113651dd39f4 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_assignable.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_assignable.pass.cpp
@@ -10,8 +10,6 @@
 
 // is_trivially_assignable
 
-// XFAIL: gcc-4.9
-
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_assignable.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_assignable.pass.cpp
index 91fbf69f4f29f..17104f4af1d6c 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_assignable.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_assignable.pass.cpp
@@ -10,8 +10,6 @@
 
 // is_trivially_copy_assignable
 
-// XFAIL: gcc-4.9
-
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_constructible.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_constructible.pass.cpp
index 5744d2ca0e8ff..09647c7107e18 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_constructible.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_copy_constructible.pass.cpp
@@ -10,8 +10,6 @@
 
 // is_trivially_copy_constructible
 
-// XFAIL: gcc-4.9
-
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_assignable.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_assignable.pass.cpp
index 0a91efdf089e1..b599ae56f0766 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_assignable.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_assignable.pass.cpp
@@ -10,8 +10,6 @@
 
 // is_trivially_move_assignable
 
-// XFAIL: gcc-4.9
-
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_constructible.pass.cpp b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_constructible.pass.cpp
index 78c10a1320a32..3503648ddea6e 100644
--- a/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_constructible.pass.cpp
+++ b/libcxx/test/std/utilities/meta/meta.unary/meta.unary.prop/is_trivially_move_constructible.pass.cpp
@@ -10,8 +10,6 @@
 
 // is_trivially_move_constructible
 
-// XFAIL: gcc-4.9
-
 #include <type_traits>
 #include "test_macros.h"
 
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp
new file mode 100644
index 0000000000000..abb8073a6a508
--- /dev/null
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp
@@ -0,0 +1,33 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// <tuple>
+
+// template <class... Types> class tuple;
+
+// template <class Alloc>
+//   explicit(see-below) tuple(allocator_arg_t, const Alloc& a);
+
+// Make sure we get the explicit-ness of the constructor right.
+// This is LWG 3158.
+
+#include <tuple>
+#include <memory>
+
+
+struct ExplicitDefault { explicit ExplicitDefault() { } };
+
+std::tuple<ExplicitDefault> explicit_default_test() {
+    return {std::allocator_arg, std::allocator<int>()}; // expected-error {{chosen constructor is explicit in copy-initialization}}
+}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.pass.cpp
index b0f957e3a374b..13511fa51483e 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.pass.cpp
@@ -13,7 +13,7 @@
 // template <class... Types> class tuple;
 
 // template <class Alloc>
-//   tuple(allocator_arg_t, const Alloc& a);
+//   explicit(see-below) tuple(allocator_arg_t, const Alloc& a);
 
 // NOTE: this constructor does not currently support tags derived from
 // allocator_arg_t because libc++ has to deduce the parameter as a template
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/deduct.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/deduct.pass.cpp
index 4944f64ddf053..b14447a63f916 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/deduct.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/deduct.pass.cpp
@@ -13,7 +13,7 @@
 // GCC's implementation of class template deduction is still immature and runs
 // into issues with libc++. However GCC accepts this code when compiling
 // against libstdc++.
-// XFAIL: gcc
+// XFAIL: gcc-5, gcc-6, gcc-7
 
 // <tuple>
 
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp
new file mode 100644
index 0000000000000..616550f60e57f
--- /dev/null
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp
@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <tuple>
+
+// template <class... Types> class tuple;
+
+// explicit(see-below) constexpr tuple();
+
+#include <tuple>
+
+
+struct Implicit {
+    Implicit() = default;
+};
+
+struct Explicit {
+    explicit Explicit() = default;
+};
+
+std::tuple<> test1() { return {}; }
+
+std::tuple<Implicit> test2() { return {}; }
+std::tuple<Explicit> test3() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+std::tuple<Implicit, Implicit> test4() { return {}; }
+std::tuple<Explicit, Implicit> test5() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Implicit, Explicit> test6() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Explicit, Explicit> test7() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+std::tuple<Implicit, Implicit, Implicit> test8() { return {}; }
+std::tuple<Implicit, Implicit, Explicit> test9() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Implicit, Explicit, Implicit> test10() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Implicit, Explicit, Explicit> test11() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Explicit, Implicit, Implicit> test12() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Explicit, Implicit, Explicit> test13() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Explicit, Explicit, Implicit> test14() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::tuple<Explicit, Explicit, Explicit> test15() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.pass.cpp
index 46fd35ad4ddd7..c5b650d79a1d8 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.pass.cpp
@@ -10,7 +10,7 @@
 
 // template <class... Types> class tuple;
 
-// constexpr tuple();
+// explicit(see-below) constexpr tuple();
 
 // UNSUPPORTED: c++98, c++03
 
@@ -107,6 +107,11 @@ int main(int, char**)
         IllFormedDefault v(0);
         std::tuple<IllFormedDefault> t(v);
     }
+    {
+        struct Base { };
+        struct Derived : Base { protected: Derived() = default; };
+        static_assert(!std::is_default_constructible<std::tuple<Derived, int> >::value, "");
+    }
 
-  return 0;
+    return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.pass.cpp
index f83ce7f13e7ab..3062bc7d1078d 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.pass.cpp
@@ -14,7 +14,6 @@
 //   struct tuple_size<tuple<Types...>>
 //     : public integral_constant<size_t, sizeof...(Types)> { };
 
-// XFAIL: gcc-4.9
 // UNSUPPORTED: c++98, c++03
 
 #include <tuple>
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_value_sfinae.pass.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_value_sfinae.pass.cpp
index aad003039ecda..62e584d3f0abd 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_value_sfinae.pass.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_value_sfinae.pass.cpp
@@ -14,7 +14,6 @@
 //   class tuple_size<tuple<Types...>>
 //     : public integral_constant<size_t, sizeof...(Types)> { };
 
-// XFAIL: gcc-4.9
 // UNSUPPORTED: c++98, c++03
 
 #include <tuple>
diff --git a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
index d70d060deb056..903aa9355832a 100644
--- a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct.pass.cpp
@@ -12,7 +12,7 @@
 
 // template <class T1, class T2> struct pair
 
-// struct piecewise_construct_t { };
+// struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
 // constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
 
 #include <utility>
diff --git a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.fail.cpp b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.fail.cpp
new file mode 100644
index 0000000000000..b8dd28a576458
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.fail.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <utility>
+
+// struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
+// constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
+
+// This test checks for LWG 2510.
+
+#include <utility>
+
+
+std::piecewise_construct_t f() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.pass.cpp
new file mode 100644
index 0000000000000..0fdba0674874c
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.pass.cpp
@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// <utility>
+
+// struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
+// constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
+
+#include <utility>
+
+
+int main(int, char**) {
+    std::piecewise_construct_t x = std::piecewise_construct;
+    (void)x;
+
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.explicit.fail.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.explicit.fail.cpp
new file mode 100644
index 0000000000000..511f21ec301c6
--- /dev/null
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.explicit.fail.cpp
@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++98, c++03
+
+// Before GCC 6, aggregate initialization kicks in.
+// See https://stackoverflow.com/q/41799015/627587.
+// UNSUPPORTED: gcc-5
+
+// <utility>
+
+// template <class T1, class T2> struct pair
+
+// explicit(see-below) constexpr pair();
+
+// This test checks the conditional explicitness of std::pair's default
+// constructor as introduced by the resolution of LWG 2510.
+
+#include <utility>
+
+
+struct ImplicitlyDefaultConstructible {
+    ImplicitlyDefaultConstructible() = default;
+};
+
+struct ExplicitlyDefaultConstructible {
+    explicit ExplicitlyDefaultConstructible() = default;
+};
+
+std::pair<ImplicitlyDefaultConstructible, ExplicitlyDefaultConstructible> test1() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::pair<ExplicitlyDefaultConstructible, ImplicitlyDefaultConstructible> test2() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::pair<ExplicitlyDefaultConstructible, ExplicitlyDefaultConstructible> test3() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
+std::pair<ImplicitlyDefaultConstructible, ImplicitlyDefaultConstructible> test4() { return {}; }
+
+int main(int, char**) {
+    return 0;
+}
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.pass.cpp
index de3730ef3cb28..dc05a2b488594 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/default.pass.cpp
@@ -10,12 +10,7 @@
 
 // template <class T1, class T2> struct pair
 
-// constexpr pair();
-
-// This test doesn't pass due to a constexpr bug in GCC 4.9 that fails
-// to initialize any type without a user provided constructor in a constant
-// expression (e.g. float).
-// XFAIL: gcc-4.9
+// explicit(see-below) constexpr pair();
 
 // NOTE: The SFINAE on the default constructor is tested in
 //       default-sfinae.pass.cpp
@@ -50,6 +45,11 @@ int main(int, char**)
         using P2 = std::pair<NoDefault, int>;
         static_assert(!std::is_default_constructible<P2>::value, "");
     }
+    {
+        struct Base { };
+        struct Derived : Base { protected: Derived() = default; };
+        static_assert(!std::is_default_constructible<std::pair<Derived, int> >::value, "");
+    }
 #endif
 
   return 0;
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/trivial_copy_move.pass.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/trivial_copy_move.pass.cpp
index 6841f28655388..be7c0a8a8be17 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/trivial_copy_move.pass.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/trivial_copy_move.pass.cpp
@@ -13,9 +13,6 @@
 // pair(pair const&) = default;
 // pair(pair&&) = default;
 
-// Doesn't pass due to use of is_trivially_* trait.
-// XFAIL: gcc-4.9
-
 #include <utility>
 #include <cassert>
 
diff --git a/libcxx/test/support/min_allocator.h b/libcxx/test/support/min_allocator.h
index 5228fab2e7034..fd23fc4383f34 100644
--- a/libcxx/test/support/min_allocator.h
+++ b/libcxx/test/support/min_allocator.h
@@ -14,6 +14,7 @@
 #include <cstddef>
 #include <cassert>
 #include <climits>
+#include <memory>
 
 #include "test_macros.h"
 
@@ -190,11 +191,6 @@ struct cpp03_overload_allocator : bare_allocator<T>
 };
 template <class T> bool cpp03_overload_allocator<T>::construct_called = false;
 
-
-#if TEST_STD_VER >= 11
-
-#include <memory>
-
 template <class T, class = std::integral_constant<size_t, 0> > class min_pointer;
 template <class T, class ID> class min_pointer<const T, ID>;
 template <class ID> class min_pointer<void, ID>;
@@ -462,6 +458,4 @@ class explicit_allocator
     friend bool operator!=(explicit_allocator x, explicit_allocator y) {return !(x == y);}
 };
 
-#endif  // TEST_STD_VER >= 11
-
 #endif  // MIN_ALLOCATOR_H
diff --git a/libcxx/test/support/test.workarounds/c1xx_broken_is_trivially_copyable.pass.cpp b/libcxx/test/support/test.workarounds/c1xx_broken_is_trivially_copyable.pass.cpp
index ef6a82c80f0c6..24d6869679f59 100644
--- a/libcxx/test/support/test.workarounds/c1xx_broken_is_trivially_copyable.pass.cpp
+++ b/libcxx/test/support/test.workarounds/c1xx_broken_is_trivially_copyable.pass.cpp
@@ -8,10 +8,6 @@
 
 // UNSUPPORTED: c++98, c++03
 
-// This workaround option is specific to MSVC's C1XX, so we don't care that
-// it isn't set for older GCC versions.
-// XFAIL: gcc-4.9
-
 // Verify TEST_WORKAROUND_C1XX_BROKEN_IS_TRIVIALLY_COPYABLE.
 
 #include <type_traits>
diff --git a/libcxx/utils/gdb/libcxx/printers.py b/libcxx/utils/gdb/libcxx/printers.py
index dc2f2e97aec85..ff45bde6172c1 100644
--- a/libcxx/utils/gdb/libcxx/printers.py
+++ b/libcxx/utils/gdb/libcxx/printers.py
@@ -27,6 +27,7 @@
 
 _long_int_type = gdb.lookup_type("unsigned long long")
 
+_libcpp_big_endian = False
 
 def addr_as_long(addr):
     return int(addr.cast(_long_int_type))
@@ -195,19 +196,14 @@ def _get_short_size(self, short_field, short_size):
         field = short_field.type.fields()[1].type.fields()[0]
         libcpp_abi_alternate_string_layout = field.name and "__padding" in field.name
 
-        # Strictly, this only tells us the current mode, not how libcxx was
-        # compiled.
-        libcpp_big_endian = "big endian" in gdb.execute("show endian",
-                                                        to_string=True)
-
         # This logical structure closely follows the original code (which is clearer
         # in C++).  Keep them parallel to make them easier to compare.
         if libcpp_abi_alternate_string_layout:
-            if libcpp_big_endian:
+            if _libcpp_big_endian:
                 return short_size >> 1
             else:
                 return short_size
-        elif libcpp_big_endian:
+        elif _libcpp_big_endian:
             return short_size
         else:
             return short_size >> 1
@@ -969,6 +965,14 @@ def __call__(self, val):
 # certain pathological cases. Limit our pretty printers to the progspace.
 def _register_libcxx_printers(event):
     progspace = event.new_objfile.progspace
+    # It would be ideal to get the endianness at print time, but
+    # gdb.execute clears gdb's internal wrap buffer, removing any values
+    # already generated as part of a larger data structure, and there is
+    # no python api to get the endianness. Mixed-endianness debugging
+    # rare enough that this workaround should be adequate.
+    _libcpp_big_endian = "big endian" in gdb.execute("show endian",
+                                                     to_string=True)
+
     if not getattr(progspace, _libcxx_printer_name, False):
         print("Loading libc++ pretty-printers.")
         gdb.printing.register_pretty_printer(
diff --git a/libcxx/www/cxx1z_status.html b/libcxx/www/cxx1z_status.html
index 227c05fa26a8d..627f9a085cb2c 100644
--- a/libcxx/www/cxx1z_status.html
+++ b/libcxx/www/cxx1z_status.html
@@ -372,7 +372,7 @@ <h3>Library Working group Issues Status</h3>
 	<tr><td><a href="https://wg21.link/LWG2468">2468</a></td><td>Self-move-assignment of library types</td><td>Issaquah</td><td></td></tr>
 	<tr><td><a href="https://wg21.link/LWG2475">2475</a></td><td>Allow overwriting of std::basic_string terminator with charT() to allow cleaner interoperation with legacy APIs</td><td>Issaquah</td><td>Complete</td></tr>
 	<tr><td><a href="https://wg21.link/LWG2503">2503</a></td><td>multiline option should be added to syntax_option_type</td><td>Issaquah</td><td></td></tr>
-	<tr><td><a href="https://wg21.link/LWG2510">2510</a></td><td>Tag types should not be DefaultConstructible</td><td>Issaquah</td><td></td></tr>
+	<tr><td><a href="https://wg21.link/LWG2510">2510</a></td><td>Tag types should not be DefaultConstructible</td><td>Issaquah</td><td>Complete</td></tr>
 	<tr><td><a href="https://wg21.link/LWG2514">2514</a></td><td>Type traits must not be final</td><td>Issaquah</td><td>Complete</td></tr>
 	<tr><td><a href="https://wg21.link/LWG2518">2518</a></td><td>[fund.ts.v2] Non-member swap for propagate_const should call member swap</td><td>Issaquah</td><td>Complete</td></tr>
 	<tr><td><a href="https://wg21.link/LWG2519">2519</a></td><td>Iterator operator-= has gratuitous undefined behaviour</td><td>Issaquah</td><td>Complete</td></tr>
diff --git a/libcxx/www/cxx2a_status.html b/libcxx/www/cxx2a_status.html
index cc84bbe7328de..6008c22068cf2 100644
--- a/libcxx/www/cxx2a_status.html
+++ b/libcxx/www/cxx2a_status.html
@@ -358,7 +358,7 @@ <h3>Library Working group Issues Status</h3>
 	<tr><td><a href="https://wg21.link/LWG3182">3182</a></td><td>Specification of <tt>Same</tt> could be clearer</td><td>Kona</td><td></td></tr>
 
 	<tr><td></td><td></td><td></td><td></td></tr>
-	<tr><td><a href="https://wg21.link/LWG2899">2899</a></td><td><tt>is_(nothrow_)move_constructible</tt> and <tt>tuple</tt>, <tt>optional</tt> and <tt>unique_ptr</tt></td><td>Cologne</td><td></td></tr>
+	<tr><td><a href="https://wg21.link/LWG2899">2899</a></td><td><tt>is_(nothrow_)move_constructible</tt> and <tt>tuple</tt>, <tt>optional</tt> and <tt>unique_ptr</tt></td><td>Cologne</td><td>Complete</td></tr>
 	<tr><td><a href="https://wg21.link/LWG3055">3055</a></td><td><tt>path::operator+=(<i>single-character</i>)</tt> misspecified</td><td>Cologne</td><td></td></tr>
 	<tr><td><a href="https://wg21.link/LWG3158">3158</a></td><td><tt>tuple(allocator_arg_t, const Alloc&amp;)</tt> should be conditionally explicit</td><td>Cologne</td><td></td></tr>
 	<tr><td><a href="https://wg21.link/LWG3169">3169</a></td><td><tt>ranges</tt> permutation generators discard useful information</td><td>Cologne</td><td></td></tr>
diff --git a/libcxx/www/upcoming_meeting.html b/libcxx/www/upcoming_meeting.html
index 29b5174ae307e..7c7d2a28af245 100644
--- a/libcxx/www/upcoming_meeting.html
+++ b/libcxx/www/upcoming_meeting.html
@@ -61,7 +61,7 @@ <h3>Library Working group Issues Status</h3>
 <tr><th>Issue #</th><th>Has P/R</th><th>Issue Name</th><th>Meeting</th><th>Status</th></tr>
 <tr><td><a href="https://wg21.link/LWG2899">2899</a></td><td>Yes</td><td><tt>is_(nothrow_)move_constructible</tt> and <tt>tuple</tt>, <tt>optional</tt> and <tt>unique_ptr</tt></td><td>Cologne</td><td></td></tr>
 <tr><td><a href="https://wg21.link/LWG3055">3055</a></td><td>Yes</td><td><tt>path::operator+=(<i>single-character</i>)</tt> misspecified</td><td>Cologne</td><td></td></tr>
-<tr><td><a href="https://wg21.link/LWG3158">3158</a></td><td>Yes</td><td><tt>tuple(allocator_arg_t, const Alloc&amp;)</tt> should be conditionally explicit</td><td>Cologne</td><td></td></tr>
+<tr><td><a href="https://wg21.link/LWG3158">3158</a></td><td>Yes</td><td><tt>tuple(allocator_arg_t, const Alloc&amp;)</tt> should be conditionally explicit</td><td>Cologne</td><td>Complete</td></tr>
 <tr><td><a href="https://wg21.link/LWG3169">3169</a></td><td>Yes</td><td><tt>ranges</tt> permutation generators discard useful information</td><td>Cologne</td><td></td></tr>
 <tr><td><a href="https://wg21.link/LWG3183">3183</a></td><td>Yes</td><td>Normative permission to specialize Ranges variable templates</td><td>Cologne</td><td></td></tr>
 <tr><td><a href="https://wg21.link/LWG3184">3184</a></td><td>Yes</td><td>Inconsistencies in <tt>bind_front</tt> wording</td><td>Cologne</td><td></td></tr>
@@ -95,7 +95,6 @@ <h3>Comments about the issues</h3>
 <ul>
 <li>2899 - Louis</li>
 <li>3055 - Eric?</li>
-<li>3158 - Louis</li>
 <li>3169 - We don't do ranges yet</li>
 <li>3183 - We don't do ranges yet</li>
 <li>3184 - We don't do bind_front yet</li>
diff --git a/libcxxabi/cmake/config-ix.cmake b/libcxxabi/cmake/config-ix.cmake
index 9195dfc74e19b..94f89bedd976c 100644
--- a/libcxxabi/cmake/config-ix.cmake
+++ b/libcxxabi/cmake/config-ix.cmake
@@ -7,6 +7,7 @@ include(CheckCSourceCompiles)
 check_library_exists(c fopen "" LIBCXXABI_HAS_C_LIB)
 if (NOT LIBCXXABI_USE_COMPILER_RT)
   check_library_exists(gcc_s __gcc_personality_v0 "" LIBCXXABI_HAS_GCC_S_LIB)
+  check_library_exists(gcc __aeabi_uldivmod "" LIBCXXABI_HAS_GCC_LIB)
 endif ()
 
 # libc++abi is built with -nodefaultlibs, so we want all our checks to also
@@ -26,8 +27,13 @@ if (LIBCXXABI_HAS_NODEFAULTLIBS_FLAG)
     list(APPEND CMAKE_REQUIRED_FLAGS -rtlib=compiler-rt)
     find_compiler_rt_library(builtins LIBCXXABI_BUILTINS_LIBRARY)
     list(APPEND CMAKE_REQUIRED_LIBRARIES "${LIBCXXABI_BUILTINS_LIBRARY}")
-  elseif (LIBCXXABI_HAS_GCC_S_LIB)
-    list(APPEND CMAKE_REQUIRED_LIBRARIES gcc_s)
+  else ()
+    if (LIBCXXABI_HAS_GCC_S_LIB)
+      list(APPEND CMAKE_REQUIRED_LIBRARIES gcc_s)
+    endif ()
+    if (LIBCXXABI_HAS_GCC_LIB)
+      list(APPEND CMAKE_REQUIRED_LIBRARIES gcc)
+    endif ()
   endif ()
   if (MINGW)
     # Mingw64 requires quite a few "C" runtime libraries in order for basic
diff --git a/libcxxabi/include/cxxabi.h b/libcxxabi/include/cxxabi.h
index c1271fd0f23f8..d21d3e1e2331d 100644
--- a/libcxxabi/include/cxxabi.h
+++ b/libcxxabi/include/cxxabi.h
@@ -11,7 +11,7 @@
 
 /*
  * This header provides the interface to the C++ ABI as defined at:
- *       http://www.codesourcery.com/cxx-abi/
+ *       https://itanium-cxx-abi.github.io/cxx-abi/
  */
 
 #include <stddef.h>
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 6b8f76b1a39f0..6cfcb9cb3df10 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -91,6 +91,10 @@ if (MINGW)
   list(APPEND LIBCXXABI_LIBRARIES ${MINGW_LIBRARIES})
 endif()
 
+if (NOT LIBCXXABI_USE_COMPILER_RT)
+  add_library_flags_if(LIBCXXABI_HAS_GCC_LIB gcc)
+endif ()
+
 # Setup flags.
 add_link_flags_if_supported(-nodefaultlibs)
 
diff --git a/libcxxabi/src/cxa_default_handlers.cpp b/libcxxabi/src/cxa_default_handlers.cpp
index 91a63f537662e..573694d4ebef3 100644
--- a/libcxxabi/src/cxa_default_handlers.cpp
+++ b/libcxxabi/src/cxa_default_handlers.cpp
@@ -8,10 +8,8 @@
 // This file implements the default terminate_handler and unexpected_handler.
 //===----------------------------------------------------------------------===//
 
-#include <stdexcept>
-#include <new>
 #include <exception>
-#include <cstdlib>
+#include <stdlib.h>
 #include "abort_message.h"
 #include "cxxabi.h"
 #include "cxa_handlers.h"
@@ -54,7 +52,7 @@ static void demangling_terminate_handler()
                     name = thrown_type->name();
                 // If the uncaught exception can be caught with std::exception&
                 const __shim_type_info* catch_type =
-				 static_cast<const __shim_type_info*>(&typeid(std::exception));
+                    static_cast<const __shim_type_info*>(&typeid(std::exception));
                 if (catch_type->can_catch(thrown_type, thrown_object))
                 {
                     // Include the what() message from the exception
@@ -87,7 +85,7 @@ static void demangling_unexpected_handler()
 static std::terminate_handler default_terminate_handler = demangling_terminate_handler;
 static std::terminate_handler default_unexpected_handler = demangling_unexpected_handler;
 #else
-static std::terminate_handler default_terminate_handler = std::abort;
+static std::terminate_handler default_terminate_handler = ::abort;
 static std::terminate_handler default_unexpected_handler = std::terminate;
 #endif
 
diff --git a/libcxxabi/src/cxa_exception.cpp b/libcxxabi/src/cxa_exception.cpp
index 688ba1ef74d57..ebb05ce54213d 100644
--- a/libcxxabi/src/cxa_exception.cpp
+++ b/libcxxabi/src/cxa_exception.cpp
@@ -4,16 +4,16 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the "Exception Handling APIs"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
-//  
+//
 //===----------------------------------------------------------------------===//
 
 #include "cxxabi.h"
 
 #include <exception>        // for std::terminate
-#include <cstring>          // for memset
+#include <string.h>         // for memset
 #include "cxa_exception.h"
 #include "cxa_handlers.h"
 #include "fallback_malloc.h"
@@ -79,9 +79,9 @@ size_t cxa_exception_size_from_exception_thrown_size(size_t size) {
 }
 
 void __setExceptionClass(_Unwind_Exception* unwind_exception, uint64_t newValue) {
-	::memcpy(&unwind_exception->exception_class, &newValue, sizeof(newValue));
-	}
-	
+    ::memcpy(&unwind_exception->exception_class, &newValue, sizeof(newValue));
+}
+
 
 static void setOurExceptionClass(_Unwind_Exception* unwind_exception) {
     __setExceptionClass(unwind_exception, kOurExceptionClass);
@@ -93,17 +93,17 @@ static void setDependentExceptionClass(_Unwind_Exception* unwind_exception) {
 
 //  Is it one of ours?
 uint64_t __getExceptionClass(const _Unwind_Exception* unwind_exception) {
-//	On x86 and some ARM unwinders, unwind_exception->exception_class is
-//		a uint64_t. On other ARM unwinders, it is a char[8]
-//	See: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf
-//	So we just copy it into a uint64_t to be sure.
-	uint64_t exClass;
-	::memcpy(&exClass, &unwind_exception->exception_class, sizeof(exClass));
-	return exClass;
+    // On x86 and some ARM unwinders, unwind_exception->exception_class is
+    // a uint64_t. On other ARM unwinders, it is a char[8].
+    // See: http://infocenter.arm.com/help/topic/com.arm.doc.ihi0038b/IHI0038B_ehabi.pdf
+    // So we just copy it into a uint64_t to be sure.
+    uint64_t exClass;
+    ::memcpy(&exClass, &unwind_exception->exception_class, sizeof(exClass));
+    return exClass;
 }
 
 bool __isOurExceptionClass(const _Unwind_Exception* unwind_exception) {
-    return (__getExceptionClass(unwind_exception) & get_vendor_and_language) == 
+    return (__getExceptionClass(unwind_exception) & get_vendor_and_language) ==
            (kOurExceptionClass                    & get_vendor_and_language);
 }
 
@@ -123,7 +123,7 @@ static inline  int decrementHandlerCount(__cxa_exception *exception) {
 
 /*
     If reason isn't _URC_FOREIGN_EXCEPTION_CAUGHT, then the terminateHandler
-    stored in exc is called.  Otherwise the exceptionDestructor stored in 
+    stored in exc is called.  Otherwise the exceptionDestructor stored in
     exc is called, and then the memory for the exception is deallocated.
 
     This is never called for a __cxa_dependent_exception.
@@ -145,7 +145,7 @@ static _LIBCXXABI_NORETURN void failed_throw(__cxa_exception* exception_header)
 //      * For purposes of this ABI, several things are considered exception handlers:
 //      ** A terminate() call due to a throw.
 //  and
-//      * Upon entry, Following initialization of the catch parameter, 
+//      * Upon entry, Following initialization of the catch parameter,
 //          a handler must call:
 //      * void *__cxa_begin_catch(void *exceptionObject );
     (void) __cxa_begin_catch(&exception_header->unwindHeader);
@@ -163,13 +163,12 @@ static size_t get_cxa_exception_offset() {
   } __attribute__((aligned));
 
   // Compute the maximum alignment for the target machine.
-  constexpr size_t alignment = std::alignment_of<S>::value;
+  constexpr size_t alignment = alignof(S);
   constexpr size_t excp_size = sizeof(__cxa_exception);
   constexpr size_t aligned_size =
       (excp_size + alignment - 1) / alignment * alignment;
   constexpr size_t offset = aligned_size - excp_size;
-  static_assert((offset == 0 ||
-                 std::alignment_of<_Unwind_Exception>::value < alignment),
+  static_assert((offset == 0 || alignof(_Unwind_Exception) < alignment),
                 "offset is non-zero only if _Unwind_Exception isn't aligned");
   return offset;
 }
@@ -193,7 +192,7 @@ void *__cxa_allocate_exception(size_t thrown_size) throw() {
         std::terminate();
     __cxa_exception *exception_header =
         static_cast<__cxa_exception *>((void *)(raw_buffer + header_offset));
-    std::memset(exception_header, 0, actual_size);
+    ::memset(exception_header, 0, actual_size);
     return thrown_object_from_cxa_exception(exception_header);
 }
 
@@ -216,7 +215,7 @@ void * __cxa_allocate_dependent_exception () {
     void *ptr = __aligned_malloc_with_fallback(actual_size);
     if (NULL == ptr)
         std::terminate();
-    std::memset(ptr, 0, actual_size);
+    ::memset(ptr, 0, actual_size);
     return ptr;
 }
 
@@ -237,15 +236,15 @@ routine never returns.
 The __cxa_throw routine will do the following:
 
 * Obtain the __cxa_exception header from the thrown exception object address,
-which can be computed as follows: 
- __cxa_exception *header = ((__cxa_exception *) thrown_exception - 1); 
+which can be computed as follows:
+ __cxa_exception *header = ((__cxa_exception *) thrown_exception - 1);
 * Save the current unexpected_handler and terminate_handler in the __cxa_exception header.
-* Save the tinfo and dest arguments in the __cxa_exception header. 
+* Save the tinfo and dest arguments in the __cxa_exception header.
 * Set the exception_class field in the unwind header. This is a 64-bit value
 representing the ASCII string "XXXXC++\0", where "XXXX" is a
 vendor-dependent string. That is, for implementations conforming to this
 ABI, the low-order 4 bytes of this 64-bit value will be "C++\0".
-* Increment the uncaught_exception flag. 
+* Increment the uncaught_exception flag.
 * Call _Unwind_RaiseException in the system unwind library, Its argument is the
 pointer to the thrown exception, which __cxa_throw itself received as an argument.
 __Unwind_RaiseException begins the process of stack unwinding, described
@@ -386,7 +385,7 @@ asm (
     "	.popsection"
 );
 #endif  // defined(_LIBCXXABI_ARM_EHABI)
-    
+
 /*
 This routine can catch foreign or native exceptions.  If native, the exception
 can be a primary or dependent variety.  This routine may remain blissfully
@@ -394,7 +393,7 @@ ignorant of whether the native exception is primary or dependent.
 
 If the exception is native:
 * Increment's the exception's handler count.
-* Push the exception on the stack of currently-caught exceptions if it is not 
+* Push the exception on the stack of currently-caught exceptions if it is not
   already there (from a rethrow).
 * Decrements the uncaught_exception count.
 * Returns the adjusted pointer to the exception object, which is stored in
@@ -415,7 +414,7 @@ to terminate or unexpected during unwinding.
   doesn't.  It has a stack of __cxa_exception (which has a next* in it).
 * Do not decrement the uncaught_exception count because we didn't increment it
   in __cxa_throw (or one of our rethrow functions).
-* If we haven't terminated, assume the exception object is just past the 
+* If we haven't terminated, assume the exception object is just past the
   _Unwind_Exception and return a pointer to that.
 */
 void*
@@ -539,7 +538,7 @@ void __cxa_end_catch() {
                     __cxa_decrement_exception_refcount(thrown_object_from_cxa_exception(exception_header));
                 }
             }
-        }       
+        }
         else
         {
             // The foreign exception has not been rethrown.  Pop the stack
@@ -572,10 +571,10 @@ std::type_info *__cxa_current_exception_type() {
 // 2.5.4 Rethrowing Exceptions
 /*  This routine can rethrow native or foreign exceptions.
 If the exception is native:
-* marks the exception object on top of the caughtExceptions stack 
-  (in an implementation-defined way) as being rethrown. 
-* If the caughtExceptions stack is empty, it calls terminate() 
-  (see [C++FDIS] [except.throw], 15.1.8). 
+* marks the exception object on top of the caughtExceptions stack
+  (in an implementation-defined way) as being rethrown.
+* If the caughtExceptions stack is empty, it calls terminate()
+  (see [C++FDIS] [except.throw], 15.1.8).
 * It then calls _Unwind_RaiseException which should not return
    (terminate if it does).
   Note:  exception_header may be masquerading as a __cxa_dependent_exception
@@ -609,7 +608,7 @@ void __cxa_rethrow() {
 #endif
 
     //  If we get here, some kind of unwinding error has occurred.
-    //  There is some weird code generation bug happening with 
+    //  There is some weird code generation bug happening with
     //     Apple clang version 4.0 (tags/Apple/clang-418.0.2) (based on LLVM 3.1svn)
     //     If we call failed_throw here.  Turns up with -O2 or higher, and -Os.
     __cxa_begin_catch(&exception_header->unwindHeader);
@@ -664,7 +663,7 @@ void __cxa_decrement_exception_refcount(void *thrown_object) throw() {
     returns null.
 
     We can use __cxa_get_globals_fast here to get the globals because if there have
-    been no exceptions thrown, ever, on this thread, we can return NULL without 
+    been no exceptions thrown, ever, on this thread, we can return NULL without
     the need to allocate the exception-handling globals.
 */
 void *__cxa_current_primary_exception() throw() {
@@ -697,7 +696,7 @@ static
 void
 dependent_exception_cleanup(_Unwind_Reason_Code reason, _Unwind_Exception* unwind_exception)
 {
-    __cxa_dependent_exception* dep_exception_header = 
+    __cxa_dependent_exception* dep_exception_header =
                       reinterpret_cast<__cxa_dependent_exception*>(unwind_exception + 1) - 1;
     if (_URC_FOREIGN_EXCEPTION_CAUGHT != reason)
         std::__terminate(dep_exception_header->terminateHandler);
diff --git a/libcxxabi/src/cxa_exception.h b/libcxxabi/src/cxa_exception.h
index d24b80a38c44a..d7999e0a17540 100644
--- a/libcxxabi/src/cxa_exception.h
+++ b/libcxxabi/src/cxa_exception.h
@@ -4,10 +4,10 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the "Exception Handling APIs"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
-//  
+//
 //===----------------------------------------------------------------------===//
 
 #ifndef _CXA_EXCEPTION_H
diff --git a/libcxxabi/src/cxa_exception_storage.cpp b/libcxxabi/src/cxa_exception_storage.cpp
index dea670203c1df..28c0122ff0746 100644
--- a/libcxxabi/src/cxa_exception_storage.cpp
+++ b/libcxxabi/src/cxa_exception_storage.cpp
@@ -4,10 +4,10 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the storage for the "Caught Exception Stack"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html#cxx-exc-stack
-//  
+//
 //===----------------------------------------------------------------------===//
 
 #include "cxa_exception.h"
@@ -69,13 +69,13 @@ namespace {
         if ( 0 != std::__libcpp_tls_create ( &key_, destruct_ ) )
             abort_message("cannot create thread specific key for __cxa_get_globals()");
         }
-}   
+}
 
 extern "C" {
     __cxa_eh_globals * __cxa_get_globals () {
     //  Try to get the globals for this thread
         __cxa_eh_globals* retVal = __cxa_get_globals_fast ();
-    
+
     //  If this is the first time we've been asked for these globals, create them
         if ( NULL == retVal ) {
             retVal = static_cast<__cxa_eh_globals*>
@@ -99,7 +99,7 @@ extern "C" {
 //        static int init = construct_();
         return static_cast<__cxa_eh_globals*>(std::__libcpp_tls_get(key_));
         }
-    
+
 }
 }
 #endif
diff --git a/libcxxabi/src/cxa_noexception.cpp b/libcxxabi/src/cxa_noexception.cpp
index 2182c70d0bcd6..73e0b2ae03f2d 100644
--- a/libcxxabi/src/cxa_noexception.cpp
+++ b/libcxxabi/src/cxa_noexception.cpp
@@ -4,10 +4,10 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the "Exception Handling APIs"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
-//  
+//
 //===----------------------------------------------------------------------===//
 
 // Support functions for the no-exceptions libc++ library
diff --git a/libcxxabi/src/cxa_personality.cpp b/libcxxabi/src/cxa_personality.cpp
index f140b7443fb83..f27625776111b 100644
--- a/libcxxabi/src/cxa_personality.cpp
+++ b/libcxxabi/src/cxa_personality.cpp
@@ -4,11 +4,11 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the "Exception Handling APIs"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi-eh.html
 //  http://www.intel.com/design/itanium/downloads/245358.htm
-//  
+//
 //===----------------------------------------------------------------------===//
 
 #include <assert.h>
@@ -188,8 +188,8 @@ enum
     DW_EH_PE_omit     = 0xFF
 };
 
-/// Read a uleb128 encoded value and advance pointer 
-/// See Variable Length Data Appendix C in: 
+/// Read a uleb128 encoded value and advance pointer
+/// See Variable Length Data Appendix C in:
 /// @link http://dwarfstd.org/Dwarf4.pdf @unlink
 /// @param data reference variable holding memory pointer to decode from
 /// @returns decoded value
@@ -211,8 +211,8 @@ readULEB128(const uint8_t** data)
     return result;
 }
 
-/// Read a sleb128 encoded value and advance pointer 
-/// See Variable Length Data Appendix C in: 
+/// Read a sleb128 encoded value and advance pointer
+/// See Variable Length Data Appendix C in:
 /// @link http://dwarfstd.org/Dwarf4.pdf @unlink
 /// @param data reference variable holding memory pointer to decode from
 /// @returns decoded value
@@ -236,8 +236,8 @@ readSLEB128(const uint8_t** data)
     return static_cast<intptr_t>(result);
 }
 
-/// Read a pointer encoded value and advance pointer 
-/// See Variable Length Data in: 
+/// Read a pointer encoded value and advance pointer
+/// See Variable Length Data in:
 /// @link http://dwarfstd.org/Dwarf3.pdf @unlink
 /// @param data reference variable holding memory pointer to decode from
 /// @param encoding dwarf encoding type
@@ -247,10 +247,10 @@ uintptr_t
 readEncodedPointer(const uint8_t** data, uint8_t encoding)
 {
     uintptr_t result = 0;
-    if (encoding == DW_EH_PE_omit) 
+    if (encoding == DW_EH_PE_omit)
         return result;
     const uint8_t* p = *data;
-    // first get value 
+    // first get value
     switch (encoding & 0x0F)
     {
     case DW_EH_PE_absptr:
@@ -281,15 +281,15 @@ readEncodedPointer(const uint8_t** data, uint8_t encoding)
         result = readPointerHelper<int64_t>(p);
         break;
     default:
-        // not supported 
+        // not supported
         abort();
         break;
     }
-    // then add relative offset 
+    // then add relative offset
     switch (encoding & 0x70)
     {
     case DW_EH_PE_absptr:
-        // do nothing 
+        // do nothing
         break;
     case DW_EH_PE_pcrel:
         if (result)
@@ -300,11 +300,11 @@ readEncodedPointer(const uint8_t** data, uint8_t encoding)
     case DW_EH_PE_funcrel:
     case DW_EH_PE_aligned:
     default:
-        // not supported 
+        // not supported
         abort();
         break;
     }
-    // then apply indirection 
+    // then apply indirection
     if (result && (encoding & DW_EH_PE_indirect))
         result = *((uintptr_t*)result);
     *data = p;
@@ -613,7 +613,7 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions,
     // Get the current instruction pointer and offset it before next
     // instruction in the current frame which threw the exception.
     uintptr_t ip = _Unwind_GetIP(context) - 1;
-    // Get beginning current frame's code (as defined by the 
+    // Get beginning current frame's code (as defined by the
     // emitted dwarf code)
     uintptr_t funcStart = _Unwind_GetRegionStart(context);
 #ifdef __USING_SJLJ_EXCEPTIONS__
@@ -646,8 +646,8 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions,
         uintptr_t classInfoOffset = readULEB128(&lsda);
         classInfo = lsda + classInfoOffset;
     }
-    // Walk call-site table looking for range that 
-    // includes current PC. 
+    // Walk call-site table looking for range that
+    // includes current PC.
     uint8_t callSiteEncoding = *lsda++;
 #ifdef __USING_SJLJ_EXCEPTIONS__
     (void)callSiteEncoding;  // When using SjLj exceptions, callSiteEncoding is never used
@@ -928,13 +928,13 @@ _UA_CLEANUP_PHASE
         else
             Recover state from header
         Transfer control to landing pad.  return _URC_INSTALL_CONTEXT
-    
+
     Else
 
         This branch handles both normal C++ non-catching handlers (cleanups)
-          and forced unwinding.    
+          and forced unwinding.
         Scan for anything that can not stop unwinding:
-    
+
             1.  A cleanup.
 
         If a cleanup is found
diff --git a/libcxxabi/src/cxa_thread_atexit.cpp b/libcxxabi/src/cxa_thread_atexit.cpp
index 392403bd047ce..923b265b27c2c 100644
--- a/libcxxabi/src/cxa_thread_atexit.cpp
+++ b/libcxxabi/src/cxa_thread_atexit.cpp
@@ -15,7 +15,7 @@
 #endif
 #endif
 
-#include <cstdlib>
+#include <stdlib.h>
 
 namespace __cxxabiv1 {
 
@@ -77,7 +77,7 @@ namespace {
     while (auto head = dtors) {
       dtors = head->next;
       head->dtor(head->obj);
-      std::free(head);
+      ::free(head);
     }
 
     dtors_alive = false;
@@ -126,7 +126,7 @@ extern "C" {
         dtors_alive = true;
       }
 
-      auto head = static_cast<DtorList*>(std::malloc(sizeof(DtorList)));
+      auto head = static_cast<DtorList*>(::malloc(sizeof(DtorList)));
       if (!head) {
         return -1;
       }
diff --git a/libcxxabi/src/cxa_vector.cpp b/libcxxabi/src/cxa_vector.cpp
index 3237b1df523d9..f20e978d36ef3 100644
--- a/libcxxabi/src/cxa_vector.cpp
+++ b/libcxxabi/src/cxa_vector.cpp
@@ -4,17 +4,17 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
-//  
+//
 //  This file implements the "Array Construction and Destruction APIs"
 //  https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-ctor
-//  
+//
 //===----------------------------------------------------------------------===//
 
 #include "cxxabi.h"
 #include "__cxxabi_config.h"
 
 #include <exception>        // for std::terminate
-#include <new>              // for std::bad_alloc
+#include <new>              // for std::bad_array_new_length
 
 #include "abort_message.h"
 
@@ -51,27 +51,27 @@ namespace {
     class st_heap_block2 {
     public:
         typedef void (*dealloc_f)(void *);
-        
-        st_heap_block2 ( dealloc_f dealloc, void *ptr ) 
+
+        st_heap_block2 ( dealloc_f dealloc, void *ptr )
             : dealloc_ ( dealloc ), ptr_ ( ptr ), enabled_ ( true ) {}
         ~st_heap_block2 () { if ( enabled_ ) dealloc_ ( ptr_ ) ; }
         void release () { enabled_ = false; }
-    
+
     private:
         dealloc_f dealloc_;
         void *ptr_;
         bool enabled_;
     };
-    
+
     class st_heap_block3 {
     public:
         typedef void (*dealloc_f)(void *, size_t);
-        
-        st_heap_block3 ( dealloc_f dealloc, void *ptr, size_t size ) 
+
+        st_heap_block3 ( dealloc_f dealloc, void *ptr, size_t size )
             : dealloc_ ( dealloc ), ptr_ ( ptr ), size_ ( size ), enabled_ ( true ) {}
         ~st_heap_block3 () { if ( enabled_ ) dealloc_ ( ptr_, size_ ) ; }
         void release () { enabled_ = false; }
-    
+
     private:
         dealloc_f dealloc_;
         void *ptr_;
@@ -82,17 +82,17 @@ namespace {
     class st_cxa_cleanup {
     public:
         typedef void (*destruct_f)(void *);
-        
+
         st_cxa_cleanup ( void *ptr, size_t &idx, size_t element_size, destruct_f destructor )
-            : ptr_ ( ptr ), idx_ ( idx ), element_size_ ( element_size ), 
+            : ptr_ ( ptr ), idx_ ( idx ), element_size_ ( element_size ),
                 destructor_ ( destructor ), enabled_ ( true ) {}
         ~st_cxa_cleanup () {
-            if ( enabled_ ) 
+            if ( enabled_ )
                 __cxa_vec_cleanup ( ptr_, idx_, element_size_, destructor_ );
             }
-        
+
         void release () { enabled_ = false; }
-    
+
     private:
         void *ptr_;
         size_t &idx_;
@@ -100,7 +100,7 @@ namespace {
         destruct_f destructor_;
         bool enabled_;
     };
-    
+
     class st_terminate {
     public:
         st_terminate ( bool enabled = true ) : enabled_ ( enabled ) {}
@@ -164,13 +164,13 @@ size_t calculate_allocation_size_or_throw(size_t element_count,
 extern "C" {
 
 // Equivalent to
-// 
+//
 //   __cxa_vec_new2(element_count, element_size, padding_size, constructor,
 //                  destructor, &::operator new[], &::operator delete[])
 _LIBCXXABI_FUNC_VIS void *
 __cxa_vec_new(size_t element_count, size_t element_size, size_t padding_size,
               void (*constructor)(void *), void (*destructor)(void *)) {
-    return __cxa_vec_new2 ( element_count, element_size, padding_size, 
+    return __cxa_vec_new2 ( element_count, element_size, padding_size,
         constructor, destructor, &::operator new [], &::operator delete [] );
 }
 
@@ -180,16 +180,16 @@ __cxa_vec_new(size_t element_count, size_t element_size, size_t padding_size,
 // the array preceded by the specified padding, initialize the cookie if
 // the padding is non-zero, and call the given constructor on each element.
 // Return the address of the array proper, after the padding.
-// 
+//
 // If alloc throws an exception, rethrow the exception. If alloc returns
 // NULL, return NULL. If the constructor throws an exception, call
 // destructor for any already constructed elements, and rethrow the
 // exception. If the destructor throws an exception, call std::terminate.
-// 
+//
 // The constructor may be NULL, in which case it must not be called. If the
 // padding_size is zero, the destructor may be NULL; in that case it must
 // not be called.
-// 
+//
 // Neither alloc nor dealloc may be NULL.
 _LIBCXXABI_FUNC_VIS void *
 __cxa_vec_new2(size_t element_count, size_t element_size, size_t padding_size,
@@ -208,12 +208,12 @@ __cxa_vec_new2(size_t element_count, size_t element_size, size_t padding_size,
             vec_base += padding_size;
             __set_element_count ( vec_base, element_count );
         }
-            
+
     //  Construct the elements
         __cxa_vec_ctor ( vec_base, element_count, element_size, constructor, destructor );
         heap.release ();    // We're good!
     }
-    
+
     return vec_base;
 }
 
@@ -237,16 +237,16 @@ __cxa_vec_new3(size_t element_count, size_t element_size, size_t padding_size,
             vec_base += padding_size;
             __set_element_count ( vec_base, element_count );
         }
-            
+
     //  Construct the elements
         __cxa_vec_ctor ( vec_base, element_count, element_size, constructor, destructor );
         heap.release ();    // We're good!
     }
-    
+
     return vec_base;
 }
- 
- 
+
+
 // Given the (data) addresses of a destination and a source array, an
 // element count and an element size, call the given copy constructor to
 // copy each element from the source array to the destination array. The
@@ -266,9 +266,9 @@ _LIBCXXABI_FUNC_VIS void __cxa_vec_cctor(void *dest_array, void *src_array,
         size_t idx = 0;
         char *src_ptr  = static_cast<char *>(src_array);
         char *dest_ptr = static_cast<char *>(dest_array);
-        st_cxa_cleanup cleanup ( dest_array, idx, element_size, destructor );        
+        st_cxa_cleanup cleanup ( dest_array, idx, element_size, destructor );
 
-        for ( idx = 0; idx < element_count; 
+        for ( idx = 0; idx < element_count;
                     ++idx, src_ptr += element_size, dest_ptr += element_size )
             constructor ( dest_ptr, src_ptr );
         cleanup.release ();     // We're good!
@@ -289,8 +289,8 @@ __cxa_vec_ctor(void *array_address, size_t element_count, size_t element_size,
     if ( NULL != constructor ) {
         size_t idx;
         char *ptr = static_cast <char *> ( array_address );
-        st_cxa_cleanup cleanup ( array_address, idx, element_size, destructor );        
-        
+        st_cxa_cleanup cleanup ( array_address, idx, element_size, destructor );
+
     //  Construct the elements
         for ( idx = 0; idx < element_count; ++idx, ptr += element_size )
             constructor ( ptr );
@@ -311,7 +311,7 @@ _LIBCXXABI_FUNC_VIS void __cxa_vec_dtor(void *array_address,
     if ( NULL != destructor ) {
         char *ptr = static_cast <char *> (array_address);
         size_t idx = element_count;
-        st_cxa_cleanup cleanup ( array_address, idx, element_size, destructor );        
+        st_cxa_cleanup cleanup ( array_address, idx, element_size, destructor );
         {
             st_terminate exception_guard (__cxa_uncaught_exception ());
             ptr +=  element_count * element_size;   // one past the last element
@@ -338,7 +338,7 @@ _LIBCXXABI_FUNC_VIS void __cxa_vec_cleanup(void *array_address,
         char *ptr = static_cast <char *> (array_address);
         size_t idx = element_count;
         st_terminate exception_guard;
-        
+
         ptr += element_count * element_size;    // one past the last element
         while ( idx-- > 0 ) {
             ptr -= element_size;
@@ -359,7 +359,7 @@ _LIBCXXABI_FUNC_VIS void __cxa_vec_cleanup(void *array_address,
 // throws a second exception, call terminate(). If padding_size is 0, the
 // destructor pointer must be NULL. If the destructor pointer is NULL, no
 // destructor call is to be made.
-// 
+//
 // The intent of this function is to permit an implementation to call this
 // function when confronted with an expression of the form delete[] p in
 // the source code, provided that the default deallocation function can be
@@ -387,9 +387,9 @@ __cxa_vec_delete2(void *array_address, size_t element_size, size_t padding_size,
         char *vec_base   = static_cast <char *> (array_address);
         char *heap_block = vec_base - padding_size;
         st_heap_block2 heap ( dealloc, heap_block );
-        
+
         if ( 0 != padding_size && NULL != destructor ) // call the destructors
-            __cxa_vec_dtor ( array_address, __get_element_count ( vec_base ), 
+            __cxa_vec_dtor ( array_address, __get_element_count ( vec_base ),
                                     element_size, destructor );
     }
 }
diff --git a/libcxxabi/src/fallback_malloc.cpp b/libcxxabi/src/fallback_malloc.cpp
index 9d6240752a1ab..8f301bcacd14c 100644
--- a/libcxxabi/src/fallback_malloc.cpp
+++ b/libcxxabi/src/fallback_malloc.cpp
@@ -18,8 +18,8 @@
 #endif
 #endif
 
-#include <cstdlib> // for malloc, calloc, free
-#include <cstring> // for memset
+#include <stdlib.h> // for malloc, calloc, free
+#include <string.h> // for memset
 
 //  A small, simple heap manager based (loosely) on
 //  the startup heap manager from FreeBSD, optimized for space.
@@ -214,7 +214,7 @@ void* __aligned_malloc_with_fallback(size_t size) {
   if (void* dest = _aligned_malloc(size, alignof(__aligned_type)))
     return dest;
 #elif defined(_LIBCPP_HAS_NO_LIBRARY_ALIGNED_ALLOCATION)
-  if (void* dest = std::malloc(size))
+  if (void* dest = ::malloc(size))
     return dest;
 #else
   if (size == 0)
@@ -227,13 +227,13 @@ void* __aligned_malloc_with_fallback(size_t size) {
 }
 
 void* __calloc_with_fallback(size_t count, size_t size) {
-  void* ptr = std::calloc(count, size);
+  void* ptr = ::calloc(count, size);
   if (NULL != ptr)
     return ptr;
   // if calloc fails, fall back to emergency stash
   ptr = fallback_malloc(size * count);
   if (NULL != ptr)
-    std::memset(ptr, 0, size * count);
+    ::memset(ptr, 0, size * count);
   return ptr;
 }
 
@@ -244,7 +244,7 @@ void __aligned_free_with_fallback(void* ptr) {
 #if defined(_WIN32)
     ::_aligned_free(ptr);
 #else
-    std::free(ptr);
+    ::free(ptr);
 #endif
   }
 }
@@ -253,7 +253,7 @@ void __free_with_fallback(void* ptr) {
   if (is_fallback_ptr(ptr))
     fallback_free(ptr);
   else
-    std::free(ptr);
+    ::free(ptr);
 }
 
 } // namespace __cxxabiv1
diff --git a/libcxxabi/src/fallback_malloc.h b/libcxxabi/src/fallback_malloc.h
index 033ecd63d1e1e..57808545197bc 100644
--- a/libcxxabi/src/fallback_malloc.h
+++ b/libcxxabi/src/fallback_malloc.h
@@ -10,7 +10,7 @@
 #define _FALLBACK_MALLOC_H
 
 #include "__cxxabi_config.h"
-#include <cstddef> // for size_t
+#include <stddef.h> // for size_t
 
 namespace __cxxabiv1 {
 
diff --git a/libcxxabi/src/private_typeinfo.cpp b/libcxxabi/src/private_typeinfo.cpp
index 43c91d123c9bd..5ce762a22bc4d 100644
--- a/libcxxabi/src/private_typeinfo.cpp
+++ b/libcxxabi/src/private_typeinfo.cpp
@@ -11,47 +11,41 @@
 // The flag _LIBCXX_DYNAMIC_FALLBACK is used to make dynamic_cast more
 // forgiving when type_info's mistakenly have hidden visibility and thus
 // multiple type_infos can exist for a single type.
-// 
+//
 // When _LIBCXX_DYNAMIC_FALLBACK is defined, and only in the case where
 // there is a detected inconsistency in the type_info hierarchy during a
 // dynamic_cast, then the equality operation will fall back to using strcmp
 // on type_info names to determine type_info equality.
-// 
+//
 // This change happens *only* under dynamic_cast, and only when
 // dynamic_cast is faced with the choice:  abort, or possibly give back the
 // wrong answer.  If when the dynamic_cast is done with this fallback
 // algorithm and an inconsistency is still detected, dynamic_cast will call
 // abort with an appropriate message.
-// 
+//
 // The current implementation of _LIBCXX_DYNAMIC_FALLBACK requires a
 // printf-like function called syslog:
-// 
+//
 //     void syslog(int facility_priority, const char* format, ...);
-// 
+//
 // If you want this functionality but your platform doesn't have syslog,
 // just implement it in terms of fprintf(stderr, ...).
-// 
+//
 // _LIBCXX_DYNAMIC_FALLBACK is currently off by default.
 
-
-#include <string.h>
-
-
-#ifdef _LIBCXX_DYNAMIC_FALLBACK
-#include "abort_message.h"
-#include <sys/syslog.h>
-#endif
-
 // On Windows, typeids are different between DLLs and EXEs, so comparing
 // type_info* will work for typeids from the same compiled file but fail
 // for typeids from a DLL and an executable. Among other things, exceptions
 // are not caught by handlers since can_catch() returns false.
 //
-// Defining _LIBCXX_DYNAMIC_FALLBACK does not help since can_catch() calls 
+// Defining _LIBCXX_DYNAMIC_FALLBACK does not help since can_catch() calls
 // is_equal() with use_strcmp=false so the string names are not compared.
 
-#ifdef _WIN32
 #include <string.h>
+
+#ifdef _LIBCXX_DYNAMIC_FALLBACK
+#include "abort_message.h"
+#include <sys/syslog.h>
 #endif
 
 static inline
@@ -158,11 +152,11 @@ __pointer_to_member_type_info::~__pointer_to_member_type_info()
 //      std::nullptr_t.
 
 // adjustedPtr:
-// 
+//
 // catch (A& a) : adjustedPtr == &a
 // catch (A* a) : adjustedPtr == a
 // catch (A** a) : adjustedPtr == a
-// 
+//
 // catch (D2& d2) : adjustedPtr == &d2  (d2 is base class of thrown object)
 // catch (D2* d2) : adjustedPtr == d2
 // catch (D2*& d2) : adjustedPtr == d2
@@ -1112,7 +1106,7 @@ __class_type_info::search_below_dst(__dynamic_cast_info* info,
             info->dst_ptr_not_leading_to_static_ptr = current_ptr;
             info->number_to_dst_ptr += 1;
             // If there exists another dst with a private path to
-            //    (static_ptr, static_type), then the cast from 
+            //    (static_ptr, static_type), then the cast from
             //     (dynamic_ptr, dynamic_type) to dst_type is now ambiguous.
             if (info->number_to_static_ptr == 1 &&
                     info->path_dst_ptr_to_static_ptr == not_public_path)
diff --git a/libcxxabi/src/private_typeinfo.h b/libcxxabi/src/private_typeinfo.h
index d7f3f830ba2c5..cf680d6b1f2f6 100644
--- a/libcxxabi/src/private_typeinfo.h
+++ b/libcxxabi/src/private_typeinfo.h
@@ -12,7 +12,7 @@
 #include "__cxxabi_config.h"
 
 #include <typeinfo>
-#include <cstddef>
+#include <stddef.h>
 
 namespace __cxxabiv1 {
 
@@ -72,7 +72,7 @@ struct _LIBCXXABI_HIDDEN __dynamic_cast_info
     const __class_type_info* dst_type;
     const void* static_ptr;
     const __class_type_info* static_type;
-    std::ptrdiff_t src2dst_offset;
+    ptrdiff_t src2dst_offset;
 
 // Data that represents the answer:
 
diff --git a/libcxxabi/test/catch_member_function_pointer_02.pass.cpp b/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
index 09770cbf4e687..41356e5daa7cc 100644
--- a/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
+++ b/libcxxabi/test/catch_member_function_pointer_02.pass.cpp
@@ -12,7 +12,7 @@
 
 // GCC 7 and 8 support noexcept function types but this test still fails.
 // This is likely a bug in their implementation. Investigation needed.
-// XFAIL: gcc-7, gcc-8, gcc-9
+// XFAIL: gcc-7, gcc-8, gcc-9, gcc-10
 
 #include <cassert>
 
diff --git a/libcxxabi/test/guard_test_basic.pass.cpp b/libcxxabi/test/guard_test_basic.pass.cpp
index a325dd0b732a4..7e5b7acc38872 100644
--- a/libcxxabi/test/guard_test_basic.pass.cpp
+++ b/libcxxabi/test/guard_test_basic.pass.cpp
@@ -10,6 +10,7 @@
 
 #define TESTING_CXA_GUARD
 #include "../src/cxa_guard_impl.h"
+#include <cassert>
 
 using namespace __cxxabiv1;
 
diff --git a/libunwind/src/AddressSpace.hpp b/libunwind/src/AddressSpace.hpp
index 2c9a083be2ace..bf57fdacbe6a7 100644
--- a/libunwind/src/AddressSpace.hpp
+++ b/libunwind/src/AddressSpace.hpp
@@ -497,9 +497,6 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
 #if !defined(Elf_Phdr)
         typedef ElfW(Phdr) Elf_Phdr;
 #endif
-#if !defined(Elf_Addr) && defined(__ANDROID__)
-        typedef ElfW(Addr) Elf_Addr;
-#endif
 
  #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
   #if !defined(_LIBUNWIND_SUPPORT_DWARF_INDEX)
@@ -507,6 +504,9 @@ inline bool LocalAddressSpace::findUnwindSections(pint_t targetAddr,
   #endif
         size_t object_length;
 #if defined(__ANDROID__)
+#if !defined(Elf_Addr)
+        typedef ElfW(Addr) Elf_Addr;
+#endif
         Elf_Addr image_base =
             pinfo->dlpi_phnum
                 ? reinterpret_cast<Elf_Addr>(pinfo->dlpi_phdr) -
diff --git a/libunwind/src/UnwindCursor.hpp b/libunwind/src/UnwindCursor.hpp
index a96c9f39958d6..488c3173dd7e3 100644
--- a/libunwind/src/UnwindCursor.hpp
+++ b/libunwind/src/UnwindCursor.hpp
@@ -1991,7 +1991,10 @@ int UnwindCursor<A, R>::step() {
 
 template <typename A, typename R>
 void UnwindCursor<A, R>::getInfo(unw_proc_info_t *info) {
-  *info = _info;
+  if (_unwindInfoMissing)
+    memset(info, 0, sizeof(*info));
+  else
+    *info = _info;
 }
 
 template <typename A, typename R>
diff --git a/libunwind/src/libunwind.cpp b/libunwind/src/libunwind.cpp
index c90032bd66c95..31f30f5cd709c 100644
--- a/libunwind/src/libunwind.cpp
+++ b/libunwind/src/libunwind.cpp
@@ -171,8 +171,7 @@ _LIBUNWIND_HIDDEN int __unw_get_proc_info(unw_cursor_t *cursor,
   co->getInfo(info);
   if (info->end_ip == 0)
     return UNW_ENOINFO;
-  else
-    return UNW_ESUCCESS;
+  return UNW_ESUCCESS;
 }
 _LIBUNWIND_WEAK_ALIAS(__unw_get_proc_info, unw_get_proc_info)
 
@@ -194,8 +193,7 @@ _LIBUNWIND_HIDDEN int __unw_get_proc_name(unw_cursor_t *cursor, char *buf,
   AbstractUnwindCursor *co = (AbstractUnwindCursor *)cursor;
   if (co->getFunctionName(buf, bufLen, offset))
     return UNW_ESUCCESS;
-  else
-    return UNW_EUNSPEC;
+  return UNW_EUNSPEC;
 }
 _LIBUNWIND_WEAK_ALIAS(__unw_get_proc_name, unw_get_proc_name)
 
diff --git a/libunwind/test/libunwind_01.pass.cpp b/libunwind/test/libunwind_01.pass.cpp
index 6957d98f956d7..830dcdd44f21a 100644
--- a/libunwind/test/libunwind_01.pass.cpp
+++ b/libunwind/test/libunwind_01.pass.cpp
@@ -35,8 +35,29 @@ void test3(int i, int j, int k) {
   test2(j, k);
 }
 
+void test_no_info() {
+  unw_context_t context;
+  unw_getcontext(&context);
+
+  unw_cursor_t cursor;
+  unw_init_local(&cursor, &context);
+
+  unw_proc_info_t info;
+  int ret = unw_get_proc_info(&cursor, &info);
+  if (ret != UNW_ESUCCESS)
+    abort();
+
+  // Set the IP to an address clearly outside any function.
+  unw_set_reg(&cursor, UNW_REG_IP, (unw_word_t)&context);
+
+  ret = unw_get_proc_info(&cursor, &info);
+  if (ret != UNW_ENOINFO)
+    abort();
+}
+
 int main() {
   test1(1);
   test2(1, 2);
   test3(1, 2, 3);
+  test_no_info();
 }
diff --git a/lld/COFF/CMakeLists.txt b/lld/COFF/CMakeLists.txt
index c7ef7c47fea17..da2916f0cfc08 100644
--- a/lld/COFF/CMakeLists.txt
+++ b/lld/COFF/CMakeLists.txt
@@ -30,12 +30,14 @@ add_lld_library(lldCOFF
   DebugInfoCodeView
   DebugInfoMSF
   DebugInfoPDB
+  Demangle
   LibDriver
   LTO
   MC
   Object
   Option
   Support
+  Symbolize
   WindowsManifest
 
   LINK_LIBS
diff --git a/lld/COFF/Config.h b/lld/COFF/Config.h
index 309e1fbf99e3b..cd597ddca4fa4 100644
--- a/lld/COFF/Config.h
+++ b/lld/COFF/Config.h
@@ -18,6 +18,12 @@
 #include <set>
 #include <string>
 
+namespace llvm {
+namespace symbolize {
+class LLVMSymbolizer;
+}
+} // namespace llvm
+
 namespace lld {
 namespace coff {
 
@@ -226,6 +232,8 @@ struct Configuration {
   bool swaprunNet = false;
   bool thinLTOEmitImportsFiles;
   bool thinLTOIndexOnly;
+
+  llvm::symbolize::LLVMSymbolizer *symbolizer = nullptr;
 };
 
 extern Configuration *config;
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index b0214ce4c7c6a..7688ff3d68d2a 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -27,6 +27,7 @@
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Magic.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/COFFModuleDefinition.h"
@@ -1070,12 +1071,6 @@ void LinkerDriver::maybeExportMinGWSymbols(const opt::InputArgList &args) {
   });
 }
 
-static const char *libcallRoutineNames[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
 void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   // Needed for LTO.
   InitializeAllTargetInfos();
@@ -1804,7 +1799,7 @@ void LinkerDriver::link(ArrayRef<const char *> argsArr) {
     // bitcode file in an archive member, we need to arrange to use LTO to
     // compile those archive members by adding them to the link beforehand.
     if (!BitcodeFile::instances.empty())
-      for (const char *s : libcallRoutineNames)
+      for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
         symtab->addLibcall(s);
 
     // Windows specific -- if __load_config_used can be resolved, resolve it.
diff --git a/lld/COFF/ICF.cpp b/lld/COFF/ICF.cpp
index 2b2818de3889c..abd71931cb463 100644
--- a/lld/COFF/ICF.cpp
+++ b/lld/COFF/ICF.cpp
@@ -13,7 +13,7 @@
 //
 // On Windows, ICF is enabled by default.
 //
-// See ELF/ICF.cpp for the details about the algortihm.
+// See ELF/ICF.cpp for the details about the algorithm.
 //
 //===----------------------------------------------------------------------===//
 
diff --git a/lld/COFF/Options.td b/lld/COFF/Options.td
index 6b6555bc030b8..d6d2e91f583b3 100644
--- a/lld/COFF/Options.td
+++ b/lld/COFF/Options.td
@@ -21,9 +21,9 @@ def aligncomm : P<"aligncomm", "Set common symbol alignment">;
 def alternatename : P<"alternatename", "Define weak alias">;
 def base    : P<"base", "Base address of the program">;
 def color_diagnostics: Flag<["--"], "color-diagnostics">,
-  HelpText<"Use colors in diagnostics">;
+    HelpText<"Use colors in diagnostics">;
 def color_diagnostics_eq: Joined<["--"], "color-diagnostics=">,
-  HelpText<"Use colors in diagnostics; one of 'always', 'never', 'auto'">;
+    HelpText<"Use colors in diagnostics; one of 'always', 'never', 'auto'">;
 def defaultlib : P<"defaultlib", "Add the library to the list of input files">;
 def delayload : P<"delayload", "Delay loaded DLL name">;
 def entry   : P<"entry", "Name of entry point symbol">;
@@ -34,7 +34,8 @@ def export  : P<"export", "Export a function">;
 def failifmismatch : P<"failifmismatch", "">;
 def filealign : P<"filealign", "Section alignment in the output file">;
 def functionpadmin : F<"functionpadmin">;
-def functionpadmin_opt : P<"functionpadmin", "Prepares an image for hotpatching">;
+def functionpadmin_opt : P<"functionpadmin",
+    "Prepares an image for hotpatching">;
 def guard   : P<"guard", "Control flow guard">;
 def heap    : P<"heap", "Size of the heap">;
 def ignore : P<"ignore", "Specify warning codes to ignore">;
@@ -42,11 +43,14 @@ def implib  : P<"implib", "Import library name">;
 def lib : F<"lib">,
     HelpText<"Act like lib.exe; must be first argument if present">;
 def libpath : P<"libpath", "Additional library search path">;
-def linkrepro : P<"linkrepro", "Dump linker invocation and input files for debugging">;
+def linkrepro : P<"linkrepro",
+    "Dump linker invocation and input files for debugging">;
 def lldignoreenv : F<"lldignoreenv">,
     HelpText<"Ignore environment variables like %LIB%">;
-def lldltocache : P<"lldltocache", "Path to ThinLTO cached object file directory">;
-def lldltocachepolicy : P<"lldltocachepolicy", "Pruning policy for the ThinLTO cache">;
+def lldltocache : P<"lldltocache",
+    "Path to ThinLTO cached object file directory">;
+def lldltocachepolicy : P<"lldltocachepolicy",
+    "Pruning policy for the ThinLTO cache">;
 def lldsavetemps : F<"lldsavetemps">,
     HelpText<"Save temporary files instead of deleting them">;
 def machine : P<"machine", "Specify target platform">;
@@ -58,7 +62,7 @@ def order   : P<"order", "Put functions in order">;
 def out     : P<"out", "Path to file to write output">;
 def natvis : P<"natvis", "Path to natvis file to embed in the PDB">;
 def no_color_diagnostics: F<"no-color-diagnostics">,
-  HelpText<"Do not use colors in diagnostics">;
+    HelpText<"Do not use colors in diagnostics">;
 def pdb : P<"pdb", "PDB file path">;
 def pdbaltpath : P<"pdbaltpath", "PDB file path to embed in the image">;
 def section : P<"section", "Specify section attributes">;
@@ -67,7 +71,8 @@ def stub    : P<"stub", "Specify DOS stub file">;
 def subsystem : P<"subsystem", "Specify subsystem">;
 def timestamp : P<"timestamp", "Specify the PE header timestamp">;
 def version : P<"version", "Specify a version number in the PE header">;
-def wholearchive_file : P<"wholearchive", "Include all object files from this archive">;
+def wholearchive_file : P<"wholearchive",
+    "Include all object files from this library">;
 
 def disallowlib : Joined<["/", "-", "/?", "-?"], "disallowlib:">,
     Alias<nodefaultlib>;
@@ -114,10 +119,11 @@ def swaprun_cd : F<"swaprun:cd">, Alias<swaprun>, AliasArgs<["cd"]>,
 def swaprun_net : F<"swaprun:net">, Alias<swaprun>, AliasArgs<["net"]>,
   HelpText<"Make loader run output binary from swap instead of from network">;
 def verbose : F<"verbose">;
-def wholearchive_flag : F<"wholearchive">;
+def wholearchive_flag : F<"wholearchive">,
+    HelpText<"Include all object files from all libraries">;
 
 def force : F<"force">,
-    HelpText<"Allow undefined and multiply defined symbols when creating executables">;
+    HelpText<"Allow undefined and multiply defined symbols">;
 def force_unresolved : F<"force:unresolved">,
     HelpText<"Allow undefined symbols when creating executables">;
 def force_multiple : F<"force:multiple">,
@@ -165,7 +171,7 @@ def help_q : Flag<["/??", "-??", "/?", "-?"], "">, Alias<help>;
 
 // LLD extensions
 def end_lib : F<"end-lib">,
-  HelpText<"End a grouping of objects that should be treated as if they were together in an archive">;
+  HelpText<"Ends group of objects treated as if they were in a library">;
 def exclude_all_symbols : F<"exclude-all-symbols">;
 def export_all_symbols : F<"export-all-symbols">;
 defm demangle : B<"demangle",
@@ -177,11 +183,11 @@ def kill_at : F<"kill-at">;
 def lldmingw : F<"lldmingw">;
 def output_def : Joined<["/", "-", "/?", "-?"], "output-def:">;
 def pdb_source_path : P<"pdbsourcepath",
-                        "Base path used to make relative source file path absolute in PDB">;
+    "Base path used to make relative source file path absolute in PDB">;
 def rsp_quoting : Joined<["--"], "rsp-quoting=">,
   HelpText<"Quoting style for response files, 'windows' (default) or 'posix'">;
 def start_lib : F<"start-lib">,
-  HelpText<"Start a grouping of objects that should be treated as if they were together in an archive">;
+  HelpText<"Starts group of objects treated as if they were in a library">;
 def thinlto_emit_imports_files :
     F<"thinlto-emit-imports-files">,
     HelpText<"Emit .imports files with -thinlto-index-only">;
diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp
index 25952b59234ab..41d2c5a80015b 100644
--- a/lld/COFF/PDB.cpp
+++ b/lld/COFF/PDB.cpp
@@ -1800,8 +1800,8 @@ static bool findLineTable(const SectionChunk *c, uint32_t addr,
 // Use CodeView line tables to resolve a file and line number for the given
 // offset into the given chunk and return them, or {"", 0} if a line table was
 // not found.
-std::pair<StringRef, uint32_t> coff::getFileLine(const SectionChunk *c,
-                                                 uint32_t addr) {
+std::pair<StringRef, uint32_t> coff::getFileLineCodeView(const SectionChunk *c,
+                                                         uint32_t addr) {
   ExitOnError exitOnErr;
 
   DebugStringTableSubsectionRef cVStrTab;
diff --git a/lld/COFF/PDB.h b/lld/COFF/PDB.h
index 3ac1adc85c5d0..4d8e3e2e5ea95 100644
--- a/lld/COFF/PDB.h
+++ b/lld/COFF/PDB.h
@@ -29,8 +29,8 @@ void createPDB(SymbolTable *symtab,
                llvm::ArrayRef<uint8_t> sectionTable,
                llvm::codeview::DebugInfo *buildId);
 
-std::pair<llvm::StringRef, uint32_t> getFileLine(const SectionChunk *c,
-                                                 uint32_t addr);
+std::pair<llvm::StringRef, uint32_t> getFileLineCodeView(const SectionChunk *c,
+                                                         uint32_t addr);
 }
 }
 
diff --git a/lld/COFF/SymbolTable.cpp b/lld/COFF/SymbolTable.cpp
index 4b75d667bd4c4..a4aa5d3982013 100644
--- a/lld/COFF/SymbolTable.cpp
+++ b/lld/COFF/SymbolTable.cpp
@@ -15,6 +15,7 @@
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
 #include "lld/Common/Timer.h"
+#include "llvm/DebugInfo/Symbolize/Symbolize.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Object/WindowsMachineFlag.h"
 #include "llvm/Support/Debug.h"
@@ -107,6 +108,30 @@ static std::vector<std::string> getSymbolLocations(BitcodeFile *file) {
   return {res};
 }
 
+static std::pair<StringRef, uint32_t> getFileLineDwarf(const SectionChunk *c,
+                                                       uint32_t addr) {
+  if (!config->symbolizer)
+    config->symbolizer = make<symbolize::LLVMSymbolizer>();
+  Expected<DILineInfo> expectedLineInfo = config->symbolizer->symbolizeCode(
+      *c->file->getCOFFObj(), {addr, c->getSectionNumber() - 1});
+  if (!expectedLineInfo)
+    return {"", 0};
+  const DILineInfo &lineInfo = *expectedLineInfo;
+  if (lineInfo.FileName == DILineInfo::BadString)
+    return {"", 0};
+  return {saver.save(lineInfo.FileName), lineInfo.Line};
+}
+
+static std::pair<StringRef, uint32_t> getFileLine(const SectionChunk *c,
+                                                  uint32_t addr) {
+  // MinGW can optionally use codeview, even if the default is dwarf.
+  std::pair<StringRef, uint32_t> fileLine = getFileLineCodeView(c, addr);
+  // If codeview didn't yield any result, check dwarf in MinGW mode.
+  if (fileLine.first.empty() && config->mingw)
+    fileLine = getFileLineDwarf(c, addr);
+  return fileLine;
+}
+
 // Given a file and the index of a symbol in that file, returns a description
 // of all references to that symbol from that file. If no debug information is
 // available, returns just the name of the file, else one string per actual
diff --git a/lld/COFF/Symbols.cpp b/lld/COFF/Symbols.cpp
index 0b89fc65f789f..817ae8b7488ce 100644
--- a/lld/COFF/Symbols.cpp
+++ b/lld/COFF/Symbols.cpp
@@ -12,6 +12,7 @@
 #include "lld/Common/Memory.h"
 #include "lld/Common/Strings.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/Demangle/Demangle.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -26,26 +27,25 @@ static_assert(sizeof(SymbolUnion) <= 48,
               "symbols should be optimized for memory usage");
 
 // Returns a symbol name for an error message.
-static std::string demangle(StringRef symName) {
+static std::string maybeDemangleSymbol(StringRef symName) {
   if (config->demangle) {
-    if (Optional<std::string> s = demangleMSVC(symName))
-      return *s;
-    if (config->mingw) {
-      StringRef demangleInput = symName;
-      std::string prefix;
-      if (demangleInput.consume_front("__imp_"))
-        prefix = "__declspec(dllimport) ";
-      if (config->machine == I386)
-        demangleInput.consume_front("_");
-      if (Optional<std::string> s = demangleItanium(demangleInput))
-        return prefix + *s;
-    }
+    std::string prefix;
+    StringRef demangleInput = symName;
+    if (demangleInput.consume_front("__imp_"))
+      prefix = "__declspec(dllimport) ";
+    if (config->machine == I386)
+      demangleInput.consume_front("_");
+    std::string demangled = demangle(demangleInput);
+    if (demangled != demangleInput)
+      return prefix + demangled;
   }
   return symName;
 }
-std::string toString(coff::Symbol &b) { return demangle(b.getName()); }
+std::string toString(coff::Symbol &b) {
+  return maybeDemangleSymbol(b.getName());
+}
 std::string toCOFFString(const Archive::Symbol &b) {
-  return demangle(b.getName());
+  return maybeDemangleSymbol(b.getName());
 }
 
 namespace coff {
diff --git a/lld/Common/Strings.cpp b/lld/Common/Strings.cpp
index 2240f9644e121..627435f141dad 100644
--- a/lld/Common/Strings.cpp
+++ b/lld/Common/Strings.cpp
@@ -18,39 +18,17 @@
 using namespace llvm;
 using namespace lld;
 
-// Returns the demangled C++ symbol name for Name.
-Optional<std::string> lld::demangleItanium(StringRef name) {
+// Returns the demangled C++ symbol name for name.
+std::string lld::demangleItanium(StringRef name) {
   // itaniumDemangle can be used to demangle strings other than symbol
   // names which do not necessarily start with "_Z". Name can be
-  // either a C or C++ symbol. Don't call itaniumDemangle if the name
+  // either a C or C++ symbol. Don't call demangle if the name
   // does not look like a C++ symbol name to avoid getting unexpected
   // result for a C symbol that happens to match a mangled type name.
   if (!name.startswith("_Z"))
-    return None;
+    return name;
 
-  char *buf = itaniumDemangle(name.str().c_str(), nullptr, nullptr, nullptr);
-  if (!buf)
-    return None;
-  std::string s(buf);
-  free(buf);
-  return s;
-}
-
-Optional<std::string> lld::demangleMSVC(StringRef name) {
-  std::string prefix;
-  if (name.consume_front("__imp_"))
-    prefix = "__declspec(dllimport) ";
-
-  // Demangle only C++ names.
-  if (!name.startswith("?"))
-    return None;
-
-  char *buf = microsoftDemangle(name.str().c_str(), nullptr, nullptr, nullptr);
-  if (!buf)
-    return None;
-  std::string s(buf);
-  free(buf);
-  return prefix + s;
+  return demangle(name);
 }
 
 StringMatcher::StringMatcher(ArrayRef<StringRef> pat) {
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp
index c073368aabe97..7fb3e02e7ee4f 100644
--- a/lld/ELF/AArch64ErrataFix.cpp
+++ b/lld/ELF/AArch64ErrataFix.cpp
@@ -6,7 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 // This file implements Section Patching for the purpose of working around
-// errata in CPUs. The general principle is that an erratum sequence of one or
+// the AArch64 Cortex-53 errata 843419 that affects r0p0, r0p1, r0p2 and r0p4
+// versions of the core.
+//
+// The general principle is that an erratum sequence of one or
 // more instructions is detected in the instruction stream, one of the
 // instructions in the sequence is replaced with a branch to a patch sequence
 // of replacement instructions. At the end of the replacement sequence the
@@ -20,12 +23,6 @@
 // - We can overwrite an instruction in the erratum sequence with a branch to
 // the replacement sequence.
 // - We can place the replacement sequence within range of the branch.
-
-// FIXME:
-// - The implementation here only supports one patch, the AArch64 Cortex-53
-// errata 843419 that affects r0p0, r0p1, r0p2 and r0p4 versions of the core.
-// To keep the initial version simple there is no support for multiple
-// architectures or selection of different patches.
 //===----------------------------------------------------------------------===//
 
 #include "AArch64ErrataFix.h"
@@ -333,16 +330,16 @@ static bool is843419ErratumSequence(uint32_t instr1, uint32_t instr2,
 }
 
 // Scan the instruction sequence starting at Offset Off from the base of
-// InputSection IS. We update Off in this function rather than in the caller as
-// we can skip ahead much further into the section when we know how many
+// InputSection isec. We update Off in this function rather than in the caller
+// as we can skip ahead much further into the section when we know how many
 // instructions we've scanned.
-// Return the offset of the load or store instruction in IS that we want to
+// Return the offset of the load or store instruction in isec that we want to
 // patch or 0 if no patch required.
 static uint64_t scanCortexA53Errata843419(InputSection *isec, uint64_t &off,
                                           uint64_t limit) {
   uint64_t isecAddr = isec->getVA(0);
 
-  // Advance Off so that (ISAddr + Off) modulo 0x1000 is at least 0xff8.
+  // Advance Off so that (isecAddr + Off) modulo 0x1000 is at least 0xff8.
   uint64_t initialPageOff = (isecAddr + off) & 0xfff;
   if (initialPageOff < 0xff8)
     off += 0xff8 - initialPageOff;
@@ -386,7 +383,7 @@ class Patch843419Section : public SyntheticSection {
 
   // The Section we are patching.
   const InputSection *patchee;
-  // The offset of the instruction in the Patchee section we are patching.
+  // The offset of the instruction in the patchee section we are patching.
   uint64_t patcheeOffset;
   // A label for the start of the Patch that we can use as a relocation target.
   Symbol *patchSym;
@@ -409,10 +406,10 @@ uint64_t Patch843419Section::getLDSTAddr() const {
 
 void Patch843419Section::writeTo(uint8_t *buf) {
   // Copy the instruction that we will be replacing with a branch in the
-  // Patchee Section.
+  // patchee Section.
   write32le(buf, read32le(patchee->data().begin() + patcheeOffset));
 
-  // Apply any relocation transferred from the original PatcheeSection.
+  // Apply any relocation transferred from the original patchee section.
   // For a SyntheticSection Buf already has outSecOff added, but relocateAlloc
   // also adds outSecOff so we need to subtract to avoid double counting.
   this->relocateAlloc(buf - outSecOff, buf - outSecOff + getSize());
@@ -467,10 +464,12 @@ void AArch64Err843419Patcher::init() {
     mapSyms.erase(
         std::unique(mapSyms.begin(), mapSyms.end(),
                     [=](const Defined *a, const Defined *b) {
-                      return (isCodeMapSymbol(a) && isCodeMapSymbol(b)) ||
-                             (isDataMapSymbol(a) && isDataMapSymbol(b));
+                      return isCodeMapSymbol(a) == isCodeMapSymbol(b);
                     }),
         mapSyms.end());
+    // Always start with a Code Mapping Symbol.
+    if (!mapSyms.empty() && !isCodeMapSymbol(mapSyms.front()))
+      mapSyms.erase(mapSyms.begin());
   }
   initialized = true;
 }
@@ -516,12 +515,9 @@ void AArch64Err843419Patcher::insertPatches(
   std::vector<InputSection *> tmp;
   tmp.reserve(isd.sections.size() + patches.size());
   auto mergeCmp = [](const InputSection *a, const InputSection *b) {
-    if (a->outSecOff < b->outSecOff)
-      return true;
-    if (a->outSecOff == b->outSecOff && isa<Patch843419Section>(a) &&
-        !isa<Patch843419Section>(b))
-      return true;
-    return false;
+    if (a->outSecOff != b->outSecOff)
+      return a->outSecOff < b->outSecOff;
+    return isa<Patch843419Section>(a) && !isa<Patch843419Section>(b);
   };
   std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(),
              patches.end(), std::back_inserter(tmp), mergeCmp);
@@ -530,7 +526,7 @@ void AArch64Err843419Patcher::insertPatches(
 
 // Given an erratum sequence that starts at address adrpAddr, with an
 // instruction that we need to patch at patcheeOffset from the start of
-// InputSection IS, create a Patch843419 Section and add it to the
+// InputSection isec, create a Patch843419 Section and add it to the
 // Patches that we need to insert.
 static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset,
                            InputSection *isec,
@@ -576,7 +572,7 @@ static void implementPatch(uint64_t adrpAddr, uint64_t patcheeOffset,
 
 // Scan all the instructions in InputSectionDescription, for each instance of
 // the erratum sequence create a Patch843419Section. We return the list of
-// Patch843419Sections that need to be applied to ISD.
+// Patch843419Sections that need to be applied to the InputSectionDescription.
 std::vector<Patch843419Section *>
 AArch64Err843419Patcher::patchInputSectionDescription(
     InputSectionDescription &isd) {
@@ -592,10 +588,7 @@ AArch64Err843419Patcher::patchInputSectionDescription(
     // section size).
     std::vector<const Defined *> &mapSyms = sectionMap[isec];
 
-    auto codeSym = llvm::find_if(mapSyms, [&](const Defined *ms) {
-      return ms->getName().startswith("$x");
-    });
-
+    auto codeSym = mapSyms.begin();
     while (codeSym != mapSyms.end()) {
       auto dataSym = std::next(codeSym);
       uint64_t off = (*codeSym)->value;
@@ -604,7 +597,8 @@ AArch64Err843419Patcher::patchInputSectionDescription(
 
       while (off < limit) {
         uint64_t startAddr = isec->getVA(off);
-        if (uint64_t patcheeOffset = scanCortexA53Errata843419(isec, off, limit))
+        if (uint64_t patcheeOffset =
+                scanCortexA53Errata843419(isec, off, limit))
           implementPatch(startAddr, patcheeOffset, isec, patches);
       }
       if (dataSym == mapSyms.end())
diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp
new file mode 100644
index 0000000000000..493fafc6a0b24
--- /dev/null
+++ b/lld/ELF/ARMErrataFix.cpp
@@ -0,0 +1,528 @@
+//===- ARMErrataFix.cpp ---------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file implements Section Patching for the purpose of working around the
+// Cortex-a8 erratum 657417 "A 32bit branch instruction that spans 2 4K regions
+// can result in an incorrect instruction fetch or processor deadlock." The
+// erratum affects all but r1p7, r2p5, r2p6, r3p1 and r3p2 revisions of the
+// Cortex-A8. A high level description of the patching technique is given in
+// the opening comment of AArch64ErrataFix.cpp.
+//===----------------------------------------------------------------------===//
+
+#include "ARMErrataFix.h"
+
+#include "Config.h"
+#include "LinkerScript.h"
+#include "OutputSections.h"
+#include "Relocations.h"
+#include "Symbols.h"
+#include "SyntheticSections.h"
+#include "Target.h"
+#include "lld/Common/Memory.h"
+#include "lld/Common/Strings.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/raw_ostream.h"
+#include <algorithm>
+
+using namespace llvm;
+using namespace llvm::ELF;
+using namespace llvm::object;
+using namespace llvm::support;
+using namespace llvm::support::endian;
+
+namespace lld {
+namespace elf {
+
+// The documented title for Erratum 657417 is:
+// "A 32bit branch instruction that spans two 4K regions can result in an
+// incorrect instruction fetch or processor deadlock". Graphically using a
+// 32-bit B.w instruction encoded as a pair of halfwords 0xf7fe 0xbfff
+// xxxxxx000 // Memory region 1 start
+// target:
+// ...
+// xxxxxxffe f7fe // First halfword of branch to target:
+// xxxxxx000 // Memory region 2 start
+// xxxxxx002 bfff // Second halfword of branch to target:
+//
+// The specific trigger conditions that can be detected at link time are:
+// - There is a 32-bit Thumb-2 branch instruction with an address of the form
+//   xxxxxxFFE. The first 2 bytes of the instruction are in 4KiB region 1, the
+//   second 2 bytes are in region 2.
+// - The branch instruction is one of BLX, BL, B.w BCC.w
+// - The instruction preceding the branch is a 32-bit non-branch instruction.
+// - The target of the branch is in region 1.
+//
+// The linker mitigation for the fix is to redirect any branch that meets the
+// erratum conditions to a patch section containing a branch to the target.
+//
+// As adding patch sections may move branches onto region boundaries the patch
+// must iterate until no more patches are added.
+//
+// Example, before:
+// 00000FFA func: NOP.w      // 32-bit Thumb function
+// 00000FFE       B.W func   // 32-bit branch spanning 2 regions, dest in 1st.
+// Example, after:
+// 00000FFA func: NOP.w      // 32-bit Thumb function
+// 00000FFE       B.w __CortexA8657417_00000FFE
+// 00001002       2 - bytes padding
+// 00001004 __CortexA8657417_00000FFE: B.w func
+
+class Patch657417Section : public SyntheticSection {
+public:
+  Patch657417Section(InputSection *p, uint64_t off, uint32_t instr, bool isARM);
+
+  void writeTo(uint8_t *buf) override;
+
+  size_t getSize() const override { return 4; }
+
+  // Get the virtual address of the branch instruction at patcheeOffset.
+  uint64_t getBranchAddr() const;
+
+  // The Section we are patching.
+  const InputSection *patchee;
+  // The offset of the instruction in the Patchee section we are patching.
+  uint64_t patcheeOffset;
+  // A label for the start of the Patch that we can use as a relocation target.
+  Symbol *patchSym;
+  // A decoding of the branch instruction at patcheeOffset.
+  uint32_t instr;
+  // True If the patch is to be written in ARM state, otherwise the patch will
+  // be written in Thumb state.
+  bool isARM;
+};
+
+// Return true if the half-word, when taken as the first of a pair of halfwords
+// is the first half of a 32-bit instruction.
+// Reference from ARM Architecure Reference Manual ARMv7-A and ARMv7-R edition
+// section A6.3: 32-bit Thumb instruction encoding
+// |             HW1                   |               HW2                |
+// | 1 1 1 | op1 (2) | op2 (7) | x (4) |op|           x (15)              |
+// With op1 == 0b00, a 16-bit instruction is encoded.
+//
+// We test only the first halfword, looking for op != 0b00.
+static bool is32bitInstruction(uint16_t hw) {
+  return (hw & 0xe000) == 0xe000 && (hw & 0x1800) != 0x0000;
+}
+
+// Reference from ARM Architecure Reference Manual ARMv7-A and ARMv7-R edition
+// section A6.3.4 Branches and miscellaneous control.
+// |             HW1              |               HW2                |
+// | 1 1 1 | 1 0 | op (7) | x (4) | 1 | op1 (3) | op2 (4) | imm8 (8) |
+// op1 == 0x0 op != x111xxx | Conditional branch (Bcc.W)
+// op1 == 0x1               | Branch (B.W)
+// op1 == 1x0               | Branch with Link and Exchange (BLX.w)
+// op1 == 1x1               | Branch with Link (BL.W)
+
+static bool isBcc(uint32_t instr) {
+  return (instr & 0xf800d000) == 0xf0008000 &&
+         (instr & 0x03800000) != 0x03800000;
+}
+
+static bool isB(uint32_t instr) { return (instr & 0xf800d000) == 0xf0009000; }
+
+static bool isBLX(uint32_t instr) { return (instr & 0xf800d000) == 0xf000c000; }
+
+static bool isBL(uint32_t instr) { return (instr & 0xf800d000) == 0xf000d000; }
+
+static bool is32bitBranch(uint32_t instr) {
+  return isBcc(instr) || isB(instr) || isBL(instr) || isBLX(instr);
+}
+
+Patch657417Section::Patch657417Section(InputSection *p, uint64_t off,
+                                       uint32_t instr, bool isARM)
+    : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4,
+                       ".text.patch"),
+      patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) {
+  parent = p->getParent();
+  patchSym = addSyntheticLocal(
+      saver.save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC,
+      isARM ? 0 : 1, getSize(), *this);
+  addSyntheticLocal(saver.save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this);
+}
+
+uint64_t Patch657417Section::getBranchAddr() const {
+  return patchee->getVA(patcheeOffset);
+}
+
+// Given a branch instruction instr at sourceAddr work out its destination
+// address. This is only used when the branch instruction has no relocation.
+static uint64_t getThumbDestAddr(uint64_t sourceAddr, uint32_t instr) {
+  uint8_t buf[4];
+  write16le(buf, instr >> 16);
+  write16le(buf + 2, instr & 0x0000ffff);
+  int64_t offset;
+  if (isBcc(instr))
+    offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP19);
+  else if (isB(instr))
+    offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP24);
+  else
+    offset = target->getImplicitAddend(buf, R_ARM_THM_CALL);
+  return sourceAddr + offset + 4;
+}
+
+void Patch657417Section::writeTo(uint8_t *buf) {
+  // The base instruction of the patch is always a 32-bit unconditional branch.
+  if (isARM)
+    write32le(buf, 0xea000000);
+  else
+    write32le(buf, 0x9000f000);
+  // If we have a relocation then apply it. For a SyntheticSection buf already
+  // has outSecOff added, but relocateAlloc also adds outSecOff so we need to
+  // subtract to avoid double counting.
+  if (!relocations.empty()) {
+    relocateAlloc(buf - outSecOff, buf - outSecOff + getSize());
+    return;
+  }
+
+  // If we don't have a relocation then we must calculate and write the offset
+  // ourselves.
+  // Get the destination offset from the addend in the branch instruction.
+  // We cannot use the instruction in the patchee section as this will have
+  // been altered to point to us!
+  uint64_t s = getThumbDestAddr(getBranchAddr(), instr);
+  uint64_t p = getVA(4);
+  target->relocateOne(buf, isARM ? R_ARM_JUMP24 : R_ARM_THM_JUMP24, s - p);
+}
+
+// Given a branch instruction spanning two 4KiB regions, at offset off from the
+// start of isec, return true if the destination of the branch is within the
+// first of the two 4Kib regions.
+static bool branchDestInFirstRegion(const InputSection *isec, uint64_t off,
+                                    uint32_t instr, const Relocation *r) {
+  uint64_t sourceAddr = isec->getVA(0) + off;
+  assert((sourceAddr & 0xfff) == 0xffe);
+  uint64_t destAddr = sourceAddr;
+  // If there is a branch relocation at the same offset we must use this to
+  // find the destination address as the branch could be indirected via a thunk
+  // or the PLT.
+  if (r) {
+    uint64_t dst = (r->expr == R_PLT_PC) ? r->sym->getPltVA() : r->sym->getVA();
+    // Account for Thumb PC bias, usually cancelled to 0 by addend of -4.
+    destAddr = dst + r->addend + 4;
+  } else {
+    // If there is no relocation, we must have an intra-section branch
+    // We must extract the offset from the addend manually.
+    destAddr = getThumbDestAddr(sourceAddr, instr);
+  }
+
+  return (destAddr & 0xfffff000) == (sourceAddr & 0xfffff000);
+}
+
+// Return true if a branch can reach a patch section placed after isec.
+// The Bcc.w instruction has a range of 1 MiB, all others have 16 MiB.
+static bool patchInRange(const InputSection *isec, uint64_t off,
+                         uint32_t instr) {
+
+  // We need the branch at source to reach a patch section placed immediately
+  // after isec. As there can be more than one patch in the patch section we
+  // add 0x100 as contingency to account for worst case of 1 branch every 4KiB
+  // for a 1 MiB range.
+  return target->inBranchRange(
+      isBcc(instr) ? R_ARM_THM_JUMP19 : R_ARM_THM_JUMP24, isec->getVA(off),
+      isec->getVA() + isec->getSize() + 0x100);
+}
+
+struct ScanResult {
+  // Offset of branch within its InputSection.
+  uint64_t off;
+  // Cached decoding of the branch instruction.
+  uint32_t instr;
+  // Branch relocation at off. Will be nullptr if no relocation exists.
+  Relocation *rel;
+};
+
+// Detect the erratum sequence, returning the offset of the branch instruction
+// and a decoding of the branch. If the erratum sequence is not found then
+// return an offset of 0 for the branch. 0 is a safe value to use for no patch
+// as there must be at least one 32-bit non-branch instruction before the
+// branch so the minimum offset for a patch is 4.
+static ScanResult scanCortexA8Errata657417(InputSection *isec, uint64_t &off,
+                                           uint64_t limit) {
+  uint64_t isecAddr = isec->getVA(0);
+  // Advance Off so that (isecAddr + off) modulo 0x1000 is at least 0xffa. We
+  // need to check for a 32-bit instruction immediately before a 32-bit branch
+  // at 0xffe modulo 0x1000.
+  off = alignTo(isecAddr + off, 0x1000, 0xffa) - isecAddr;
+  if (off >= limit || limit - off < 8) {
+    // Need at least 2 4-byte sized instructions to trigger erratum.
+    off = limit;
+    return {0, 0, nullptr};
+  }
+
+  ScanResult scanRes = {0, 0, nullptr};
+  const uint8_t *buf = isec->data().begin();
+  // ARMv7-A Thumb 32-bit instructions are encoded 2 consecutive
+  // little-endian halfwords.
+  const ulittle16_t *instBuf = reinterpret_cast<const ulittle16_t *>(buf + off);
+  uint16_t hw11 = *instBuf++;
+  uint16_t hw12 = *instBuf++;
+  uint16_t hw21 = *instBuf++;
+  uint16_t hw22 = *instBuf++;
+  if (is32bitInstruction(hw11) && is32bitInstruction(hw21)) {
+    uint32_t instr1 = (hw11 << 16) | hw12;
+    uint32_t instr2 = (hw21 << 16) | hw22;
+    if (!is32bitBranch(instr1) && is32bitBranch(instr2)) {
+      // Find a relocation for the branch if it exists. This will be used
+      // to determine the target.
+      uint64_t branchOff = off + 4;
+      auto relIt = llvm::find_if(isec->relocations, [=](const Relocation &r) {
+        return r.offset == branchOff &&
+               (r.type == R_ARM_THM_JUMP19 || r.type == R_ARM_THM_JUMP24 ||
+                r.type == R_ARM_THM_CALL);
+      });
+      if (relIt != isec->relocations.end())
+        scanRes.rel = &(*relIt);
+      if (branchDestInFirstRegion(isec, branchOff, instr2, scanRes.rel)) {
+        if (patchInRange(isec, branchOff, instr2)) {
+          scanRes.off = branchOff;
+          scanRes.instr = instr2;
+        } else {
+          warn(toString(isec->file) +
+               ": skipping cortex-a8 657417 erratum sequence, section " +
+               isec->name + " is too large to patch");
+        }
+      }
+    }
+  }
+  off += 0x1000;
+  return scanRes;
+}
+
+void ARMErr657417Patcher::init() {
+  // The Arm ABI permits a mix of ARM, Thumb and Data in the same
+  // InputSection. We must only scan Thumb instructions to avoid false
+  // matches. We use the mapping symbols in the InputObjects to identify this
+  // data, caching the results in sectionMap so we don't have to recalculate
+  // it each pass.
+
+  // The ABI Section 4.5.5 Mapping symbols; defines local symbols that describe
+  // half open intervals [Symbol Value, Next Symbol Value) of code and data
+  // within sections. If there is no next symbol then the half open interval is
+  // [Symbol Value, End of section). The type, code or data, is determined by
+  // the mapping symbol name, $a for Arm code, $t for Thumb code, $d for data.
+  auto isArmMapSymbol = [](const Symbol *s) {
+    return s->getName() == "$a" || s->getName().startswith("$a.");
+  };
+  auto isThumbMapSymbol = [](const Symbol *s) {
+    return s->getName() == "$t" || s->getName().startswith("$t.");
+  };
+  auto isDataMapSymbol = [](const Symbol *s) {
+    return s->getName() == "$d" || s->getName().startswith("$d.");
+  };
+
+  // Collect mapping symbols for every executable InputSection.
+  for (InputFile *file : objectFiles) {
+    auto *f = cast<ObjFile<ELF32LE>>(file);
+    for (Symbol *s : f->getLocalSymbols()) {
+      auto *def = dyn_cast<Defined>(s);
+      if (!def)
+        continue;
+      if (!isArmMapSymbol(def) && !isThumbMapSymbol(def) &&
+          !isDataMapSymbol(def))
+        continue;
+      if (auto *sec = dyn_cast_or_null<InputSection>(def->section))
+        if (sec->flags & SHF_EXECINSTR)
+          sectionMap[sec].push_back(def);
+    }
+  }
+  // For each InputSection make sure the mapping symbols are in sorted in
+  // ascending order and are in alternating Thumb, non-Thumb order.
+  for (auto &kv : sectionMap) {
+    std::vector<const Defined *> &mapSyms = kv.second;
+    llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {
+      return a->value < b->value;
+    });
+    mapSyms.erase(std::unique(mapSyms.begin(), mapSyms.end(),
+                              [=](const Defined *a, const Defined *b) {
+                                return (isThumbMapSymbol(a) ==
+                                        isThumbMapSymbol(b));
+                              }),
+                  mapSyms.end());
+    // Always start with a Thumb Mapping Symbol
+    if (!mapSyms.empty() && !isThumbMapSymbol(mapSyms.front()))
+      mapSyms.erase(mapSyms.begin());
+  }
+  initialized = true;
+}
+
+void ARMErr657417Patcher::insertPatches(
+    InputSectionDescription &isd, std::vector<Patch657417Section *> &patches) {
+  uint64_t spacing = 0x100000 - 0x7500;
+  uint64_t isecLimit;
+  uint64_t prevIsecLimit = isd.sections.front()->outSecOff;
+  uint64_t patchUpperBound = prevIsecLimit + spacing;
+  uint64_t outSecAddr = isd.sections.front()->getParent()->addr;
+
+  // Set the outSecOff of patches to the place where we want to insert them.
+  // We use a similar strategy to initial thunk placement, using 1 MiB as the
+  // range of the Thumb-2 conditional branch with a contingency accounting for
+  // thunk generation.
+  auto patchIt = patches.begin();
+  auto patchEnd = patches.end();
+  for (const InputSection *isec : isd.sections) {
+    isecLimit = isec->outSecOff + isec->getSize();
+    if (isecLimit > patchUpperBound) {
+      for (; patchIt != patchEnd; ++patchIt) {
+        if ((*patchIt)->getBranchAddr() - outSecAddr >= prevIsecLimit)
+          break;
+        (*patchIt)->outSecOff = prevIsecLimit;
+      }
+      patchUpperBound = prevIsecLimit + spacing;
+    }
+    prevIsecLimit = isecLimit;
+  }
+  for (; patchIt != patchEnd; ++patchIt)
+    (*patchIt)->outSecOff = isecLimit;
+
+  // Merge all patch sections. We use the outSecOff assigned above to
+  // determine the insertion point. This is ok as we only merge into an
+  // InputSectionDescription once per pass, and at the end of the pass
+  // assignAddresses() will recalculate all the outSecOff values.
+  std::vector<InputSection *> tmp;
+  tmp.reserve(isd.sections.size() + patches.size());
+  auto mergeCmp = [](const InputSection *a, const InputSection *b) {
+    if (a->outSecOff != b->outSecOff)
+      return a->outSecOff < b->outSecOff;
+    return isa<Patch657417Section>(a) && !isa<Patch657417Section>(b);
+  };
+  std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(),
+             patches.end(), std::back_inserter(tmp), mergeCmp);
+  isd.sections = std::move(tmp);
+}
+
+// Given a branch instruction described by ScanRes redirect it to a patch
+// section containing an unconditional branch instruction to the target.
+// Ensure that this patch section is 4-byte aligned so that the branch cannot
+// span two 4 KiB regions. Place the patch section so that it is always after
+// isec so the branch we are patching always goes forwards.
+static void implementPatch(ScanResult sr, InputSection *isec,
+                           std::vector<Patch657417Section *> &patches) {
+
+  log("detected cortex-a8-657419 erratum sequence starting at " +
+      utohexstr(isec->getVA(sr.off)) + " in unpatched output.");
+  Patch657417Section *psec;
+  // We have two cases to deal with.
+  // Case 1. There is a relocation at patcheeOffset to a symbol. The
+  // unconditional branch in the patch must have a relocation so that any
+  // further redirection via the PLT or a Thunk happens as normal. At
+  // patcheeOffset we redirect the existing relocation to a Symbol defined at
+  // the start of the patch section.
+  //
+  // Case 2. There is no relocation at patcheeOffset. We are unlikely to have
+  // a symbol that we can use as a target for a relocation in the patch section.
+  // Luckily we know that the destination cannot be indirected via the PLT or
+  // a Thunk so we can just write the destination directly.
+  if (sr.rel) {
+    // Case 1. We have an existing relocation to redirect to patch and a
+    // Symbol target.
+
+    // Create a branch relocation for the unconditional branch in the patch.
+    // This can be redirected via the PLT or Thunks.
+    RelType patchRelType = R_ARM_THM_JUMP24;
+    int64_t patchRelAddend = sr.rel->addend;
+    bool destIsARM = false;
+    if (isBL(sr.instr) || isBLX(sr.instr)) {
+      // The final target of the branch may be ARM or Thumb, if the target
+      // is ARM then we write the patch in ARM state to avoid a state change
+      // Thunk from the patch to the target.
+      uint64_t dstSymAddr = (sr.rel->expr == R_PLT_PC) ? sr.rel->sym->getPltVA()
+                                                       : sr.rel->sym->getVA();
+      destIsARM = (dstSymAddr & 1) == 0;
+    }
+    psec = make<Patch657417Section>(isec, sr.off, sr.instr, destIsARM);
+    if (destIsARM) {
+      // The patch will be in ARM state. Use an ARM relocation and account for
+      // the larger ARM PC-bias of 8 rather than Thumb's 4.
+      patchRelType = R_ARM_JUMP24;
+      patchRelAddend -= 4;
+    }
+    psec->relocations.push_back(
+        Relocation{sr.rel->expr, patchRelType, 0, patchRelAddend, sr.rel->sym});
+    // Redirect the existing branch relocation to the patch.
+    sr.rel->expr = R_PC;
+    sr.rel->addend = -4;
+    sr.rel->sym = psec->patchSym;
+  } else {
+    // Case 2. We do not have a relocation to the patch. Add a relocation of the
+    // appropriate type to the patch at patcheeOffset.
+
+    // The destination is ARM if we have a BLX.
+    psec = make<Patch657417Section>(isec, sr.off, sr.instr, isBLX(sr.instr));
+    RelType type;
+    if (isBcc(sr.instr))
+      type = R_ARM_THM_JUMP19;
+    else if (isB(sr.instr))
+      type = R_ARM_THM_JUMP24;
+    else
+      type = R_ARM_THM_CALL;
+    isec->relocations.push_back(
+        Relocation{R_PC, type, sr.off, -4, psec->patchSym});
+  }
+  patches.push_back(psec);
+}
+
+// Scan all the instructions in InputSectionDescription, for each instance of
+// the erratum sequence create a Patch657417Section. We return the list of
+// Patch657417Sections that need to be applied to the InputSectionDescription.
+std::vector<Patch657417Section *>
+ARMErr657417Patcher::patchInputSectionDescription(
+    InputSectionDescription &isd) {
+  std::vector<Patch657417Section *> patches;
+  for (InputSection *isec : isd.sections) {
+    // LLD doesn't use the erratum sequence in SyntheticSections.
+    if (isa<SyntheticSection>(isec))
+      continue;
+    // Use sectionMap to make sure we only scan Thumb code and not Arm or inline
+    // data. We have already sorted mapSyms in ascending order and removed
+    // consecutive mapping symbols of the same type. Our range of executable
+    // instructions to scan is therefore [thumbSym->value, nonThumbSym->value)
+    // or [thumbSym->value, section size).
+    std::vector<const Defined *> &mapSyms = sectionMap[isec];
+
+    auto thumbSym = mapSyms.begin();
+    while (thumbSym != mapSyms.end()) {
+      auto nonThumbSym = std::next(thumbSym);
+      uint64_t off = (*thumbSym)->value;
+      uint64_t limit = (nonThumbSym == mapSyms.end()) ? isec->data().size()
+                                                      : (*nonThumbSym)->value;
+
+      while (off < limit) {
+        ScanResult sr = scanCortexA8Errata657417(isec, off, limit);
+        if (sr.off)
+          implementPatch(sr, isec, patches);
+      }
+      if (nonThumbSym == mapSyms.end())
+        break;
+      thumbSym = std::next(nonThumbSym);
+    }
+  }
+  return patches;
+}
+
+bool ARMErr657417Patcher::createFixes() {
+  if (!initialized)
+    init();
+
+  bool addressesChanged = false;
+  for (OutputSection *os : outputSections) {
+    if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR))
+      continue;
+    for (BaseCommand *bc : os->sectionCommands)
+      if (auto *isd = dyn_cast<InputSectionDescription>(bc)) {
+        std::vector<Patch657417Section *> patches =
+            patchInputSectionDescription(*isd);
+        if (!patches.empty()) {
+          insertPatches(*isd, patches);
+          addressesChanged = true;
+        }
+      }
+  }
+  return addressesChanged;
+}
+
+} // namespace elf
+} // namespace lld
diff --git a/lld/ELF/ARMErrataFix.h b/lld/ELF/ARMErrataFix.h
new file mode 100644
index 0000000000000..5a39bcc75cd3b
--- /dev/null
+++ b/lld/ELF/ARMErrataFix.h
@@ -0,0 +1,51 @@
+//===- ARMErrataFix.h -------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_ELF_ARMA8ERRATAFIX_H
+#define LLD_ELF_ARMA8ERRATAFIX_H
+
+#include "lld/Common/LLVM.h"
+#include "llvm/ADT/DenseMap.h"
+#include <map>
+#include <vector>
+
+namespace lld {
+namespace elf {
+
+class Defined;
+class InputSection;
+struct InputSectionDescription;
+class OutputSection;
+class Patch657417Section;
+
+class ARMErr657417Patcher {
+public:
+  // Return true if Patches have been added to the OutputSections.
+  bool createFixes();
+
+private:
+  std::vector<Patch657417Section *>
+  patchInputSectionDescription(InputSectionDescription &isd);
+
+  void insertPatches(InputSectionDescription &isd,
+                     std::vector<Patch657417Section *> &patches);
+
+  void init();
+
+  // A cache of the mapping symbols defined by the InputSection sorted in order
+  // of ascending value with redundant symbols removed. These describe
+  // the ranges of code and data in an executable InputSection.
+  llvm::DenseMap<InputSection *, std::vector<const Defined *>> sectionMap;
+
+  bool initialized = false;
+};
+
+} // namespace elf
+} // namespace lld
+
+#endif
diff --git a/lld/ELF/Arch/Mips.cpp b/lld/ELF/Arch/Mips.cpp
index d218cf224d4db..3674e1cfad644 100644
--- a/lld/ELF/Arch/Mips.cpp
+++ b/lld/ELF/Arch/Mips.cpp
@@ -85,8 +85,14 @@ RelExpr MIPS<ELFT>::getRelExpr(RelType type, const Symbol &s,
 
   switch (type) {
   case R_MIPS_JALR:
+    // If the target symbol is not preemptible and is not microMIPS,
+    // it might be possible to replace jalr/jr instruction by bal/b.
+    // It depends on the target symbol's offset.
+    if (!s.isPreemptible && !(s.getVA() & 0x1))
+      return R_PC;
+    return R_NONE;
   case R_MICROMIPS_JALR:
-    return R_HINT;
+    return R_NONE;
   case R_MIPS_GPREL16:
   case R_MIPS_GPREL32:
   case R_MICROMIPS_GPREL16:
@@ -633,6 +639,20 @@ void MIPS<ELFT>::relocateOne(uint8_t *loc, RelType type, uint64_t val) const {
     writeValue<e>(loc, val + 0x800080008000, 16, 48);
     break;
   case R_MIPS_JALR:
+    val -= 4;
+    // Replace jalr/jr instructions by bal/b if the target
+    // offset fits into the 18-bit range.
+    if (isInt<18>(val)) {
+      switch (read32<e>(loc)) {
+      case 0x0320f809:  // jalr $25 => bal sym
+        write32<e>(loc, 0x04110000 | ((val >> 2) & 0xffff));
+        break;
+      case 0x03200008:  // jr $25 => b sym
+        write32<e>(loc, 0x10000000 | ((val >> 2) & 0xffff));
+        break;
+      }
+    }
+    break;
   case R_MICROMIPS_JALR:
     // Ignore this optimization relocation for now
     break;
diff --git a/lld/ELF/Arch/MipsArchTree.cpp b/lld/ELF/Arch/MipsArchTree.cpp
index f745a87a26377..85329c3bef536 100644
--- a/lld/ELF/Arch/MipsArchTree.cpp
+++ b/lld/ELF/Arch/MipsArchTree.cpp
@@ -298,8 +298,14 @@ template <class ELFT> uint32_t elf::calcMipsEFlags() {
   std::vector<FileFlags> v;
   for (InputFile *f : objectFiles)
     v.push_back({f, cast<ObjFile<ELFT>>(f)->getObj().getHeader()->e_flags});
-  if (v.empty())
-    return 0;
+  if (v.empty()) {
+    // If we don't have any input files, we'll have to rely on the information
+    // we can derive from emulation information, since this at least gets us
+    // ABI.
+    if (config->emulation.empty() || config->is64)
+      return 0;
+    return config->mipsN32Abi ? EF_MIPS_ABI2 : EF_MIPS_ABI_O32;
+  }
   checkFlags(v);
   return getMiscFlags(v) | getPicFlags(v) | getArchFlags(v);
 }
diff --git a/lld/ELF/CMakeLists.txt b/lld/ELF/CMakeLists.txt
index 70578746483df..1ba79bec73df5 100644
--- a/lld/ELF/CMakeLists.txt
+++ b/lld/ELF/CMakeLists.txt
@@ -22,6 +22,7 @@ add_lld_library(lldELF
   Arch/SPARCV9.cpp
   Arch/X86.cpp
   Arch/X86_64.cpp
+  ARMErrataFix.cpp
   CallGraphSort.cpp
   DWARF.cpp
   Driver.cpp
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index c038b50da81d4..0c68a8485fa2e 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -61,6 +61,9 @@ enum class Target2Policy { Abs, Rel, GotRel };
 // For tracking ARM Float Argument PCS
 enum class ARMVFPArgKind { Default, Base, VFP, ToolChain };
 
+// For -z noseparate-code, -z separate-code and -z separate-loadable-segments.
+enum class SeparateSegmentKind { None, Code, Loadable };
+
 struct SymbolVersion {
   llvm::StringRef name;
   bool isExternCpp;
@@ -145,6 +148,7 @@ struct Configuration {
   bool executeOnly;
   bool exportDynamic;
   bool fixCortexA53Errata843419;
+  bool fixCortexA8;
   bool forceBTI;
   bool formatBinary = false;
   bool requireCET;
@@ -208,7 +212,6 @@ struct Configuration {
   bool zOrigin;
   bool zRelro;
   bool zRodynamic;
-  bool zSeparateCode;
   bool zText;
   bool zRetpolineplt;
   bool zWxneeded;
@@ -221,6 +224,7 @@ struct Configuration {
   Target2Policy target2;
   ARMVFPArgKind armVFPArgs = ARMVFPArgKind::Default;
   BuildIdKind buildId = BuildIdKind::None;
+  SeparateSegmentKind zSeparate;
   ELFKind ekind = ELFNoneKind;
   uint16_t emachine = llvm::ELF::EM_NONE;
   llvm::Optional<uint64_t> imageBase;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 0bc40f0909f90..b2c728880b4a1 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -48,6 +48,7 @@
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/GlobPattern.h"
@@ -299,6 +300,9 @@ static void checkOptions() {
   if (config->fixCortexA53Errata843419 && config->emachine != EM_AARCH64)
     error("--fix-cortex-a53-843419 is only supported on AArch64 targets");
 
+  if (config->fixCortexA8 && config->emachine != EM_ARM)
+    error("--fix-cortex-a8 is only supported on ARM targets");
+
   if (config->tocOptimize && config->emachine != EM_PPC64)
     error("--toc-optimize is only supported on the PowerPC64 target");
 
@@ -376,16 +380,30 @@ static bool getZFlag(opt::InputArgList &args, StringRef k1, StringRef k2,
   return Default;
 }
 
+static SeparateSegmentKind getZSeparate(opt::InputArgList &args) {
+  for (auto *arg : args.filtered_reverse(OPT_z)) {
+    StringRef v = arg->getValue();
+    if (v == "noseparate-code")
+      return SeparateSegmentKind::None;
+    if (v == "separate-code")
+      return SeparateSegmentKind::Code;
+    if (v == "separate-loadable-segments")
+      return SeparateSegmentKind::Loadable;
+  }
+  return SeparateSegmentKind::None;
+}
+
 static bool isKnownZFlag(StringRef s) {
   return s == "combreloc" || s == "copyreloc" || s == "defs" ||
          s == "execstack" || s == "global" || s == "hazardplt" ||
          s == "ifunc-noplt" || s == "initfirst" || s == "interpose" ||
          s == "keep-text-section-prefix" || s == "lazy" || s == "muldefs" ||
-         s == "separate-code" || s == "nocombreloc" || s == "nocopyreloc" ||
-         s == "nodefaultlib" || s == "nodelete" || s == "nodlopen" ||
-         s == "noexecstack" || s == "nokeep-text-section-prefix" ||
-         s == "norelro" || s == "noseparate-code" || s == "notext" ||
-         s == "now" || s == "origin" || s == "relro" || s == "retpolineplt" ||
+         s == "separate-code" || s == "separate-loadable-segments" ||
+         s == "nocombreloc" || s == "nocopyreloc" || s == "nodefaultlib" ||
+         s == "nodelete" || s == "nodlopen" || s == "noexecstack" ||
+         s == "nokeep-text-section-prefix" || s == "norelro" ||
+         s == "noseparate-code" || s == "notext" || s == "now" ||
+         s == "origin" || s == "relro" || s == "retpolineplt" ||
          s == "rodynamic" || s == "text" || s == "undefs" || s == "wxneeded" ||
          s.startswith("common-page-size=") || s.startswith("max-page-size=") ||
          s.startswith("stack-size=");
@@ -753,6 +771,12 @@ static bool getCompressDebugSections(opt::InputArgList &args) {
   return true;
 }
 
+static StringRef getAliasSpelling(opt::Arg *arg) {
+  if (const opt::Arg *alias = arg->getAlias())
+    return alias->getSpelling();
+  return arg->getSpelling();
+}
+
 static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
                                                         unsigned id) {
   auto *arg = args.getLastArg(id);
@@ -762,7 +786,7 @@ static std::pair<StringRef, StringRef> getOldNewOptions(opt::InputArgList &args,
   StringRef s = arg->getValue();
   std::pair<StringRef, StringRef> ret = s.split(';');
   if (ret.second.empty())
-    error(arg->getSpelling() + " expects 'old;new' format, but got " + s);
+    error(getAliasSpelling(arg) + " expects 'old;new' format, but got " + s);
   return ret;
 }
 
@@ -835,6 +859,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->filterList = args::getStrings(args, OPT_filter);
   config->fini = args.getLastArgValue(OPT_fini, "_fini");
   config->fixCortexA53Errata843419 = args.hasArg(OPT_fix_cortex_a53_843419);
+  config->fixCortexA8 = args.hasArg(OPT_fix_cortex_a8);
   config->forceBTI = args.hasArg(OPT_force_bti);
   config->requireCET = args.hasArg(OPT_require_cet);
   config->gcSections = args.hasFlag(OPT_gc_sections, OPT_no_gc_sections, false);
@@ -853,7 +878,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->ltoNewPassManager = args.hasArg(OPT_lto_new_pass_manager);
   config->ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes);
   config->ltoo = args::getInteger(args, OPT_lto_O, 2);
-  config->ltoObjPath = args.getLastArgValue(OPT_plugin_opt_obj_path_eq);
+  config->ltoObjPath = args.getLastArgValue(OPT_lto_obj_path_eq);
   config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
   config->ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
   config->mapFile = args.getLastArgValue(OPT_Map);
@@ -898,17 +923,15 @@ static void readConfigs(opt::InputArgList &args) {
   config->thinLTOCachePolicy = CHECK(
       parseCachePruningPolicy(args.getLastArgValue(OPT_thinlto_cache_policy)),
       "--thinlto-cache-policy: invalid cache policy");
-  config->thinLTOEmitImportsFiles =
-      args.hasArg(OPT_plugin_opt_thinlto_emit_imports_files);
-  config->thinLTOIndexOnly = args.hasArg(OPT_plugin_opt_thinlto_index_only) ||
-                             args.hasArg(OPT_plugin_opt_thinlto_index_only_eq);
-  config->thinLTOIndexOnlyArg =
-      args.getLastArgValue(OPT_plugin_opt_thinlto_index_only_eq);
+  config->thinLTOEmitImportsFiles = args.hasArg(OPT_thinlto_emit_imports_files);
+  config->thinLTOIndexOnly = args.hasArg(OPT_thinlto_index_only) ||
+                             args.hasArg(OPT_thinlto_index_only_eq);
+  config->thinLTOIndexOnlyArg = args.getLastArgValue(OPT_thinlto_index_only_eq);
   config->thinLTOJobs = args::getInteger(args, OPT_thinlto_jobs, -1u);
   config->thinLTOObjectSuffixReplace =
-      getOldNewOptions(args, OPT_plugin_opt_thinlto_object_suffix_replace_eq);
+      getOldNewOptions(args, OPT_thinlto_object_suffix_replace_eq);
   config->thinLTOPrefixReplace =
-      getOldNewOptions(args, OPT_plugin_opt_thinlto_prefix_replace_eq);
+      getOldNewOptions(args, OPT_thinlto_prefix_replace_eq);
   config->trace = args.hasArg(OPT_trace);
   config->undefined = args::getStrings(args, OPT_undefined);
   config->undefinedVersion =
@@ -941,7 +964,7 @@ static void readConfigs(opt::InputArgList &args) {
   config->zRelro = getZFlag(args, "relro", "norelro", true);
   config->zRetpolineplt = hasZOption(args, "retpolineplt");
   config->zRodynamic = hasZOption(args, "rodynamic");
-  config->zSeparateCode = getZFlag(args, "separate-code", "noseparate-code", false);
+  config->zSeparate = getZSeparate(args);
   config->zStackSize = args::getZOptionValue(args, OPT_z, "stack-size", 0);
   config->zText = getZFlag(args, "text", "notext", true);
   config->zWxneeded = hasZOption(args, "wxneeded");
@@ -973,7 +996,8 @@ static void readConfigs(opt::InputArgList &args) {
     StringRef s = arg->getValue();
     std::tie(config->ekind, config->emachine, config->osabi) =
         parseEmulation(s);
-    config->mipsN32Abi = (s == "elf32btsmipn32" || s == "elf32ltsmipn32");
+    config->mipsN32Abi =
+        (s.startswith("elf32btsmipn32") || s.startswith("elf32ltsmipn32"));
     config->emulation = s;
   }
 
@@ -1668,12 +1692,6 @@ template <class ELFT> static uint32_t getAndFeatures() {
   return ret;
 }
 
-static const char *libcallRoutineNames[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
 // Do actual linking. Note that when this function is called,
 // all linker scripts have already been parsed.
 template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
@@ -1764,7 +1782,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // libcall symbols will be added to the link after LTO when we add the LTO
   // object file to the link.
   if (!bitcodeFiles.empty())
-    for (const char *s : libcallRoutineNames)
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
       handleLibcall(s);
 
   // Return if there were name resolution errors.
@@ -1889,8 +1907,7 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
            "feature detected");
   }
 
-  // This adds a .comment section containing a version string. We have to add it
-  // before mergeSections because the .comment section is a mergeable section.
+  // This adds a .comment section containing a version string.
   if (!config->relocatable)
     inputSections.push_back(createCommentSection());
 
@@ -1902,7 +1919,6 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   splitSections<ELFT>();
   markLive<ELFT>();
   demoteSharedSymbols();
-  mergeSections();
 
   // Make copies of any input sections that need to be copied into each
   // partition.
@@ -1926,6 +1942,16 @@ template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // they are assigned to output sections by the default rule. Process that.
   script->addOrphanSections();
 
+  // Migrate InputSectionDescription::sectionBases to sections. This includes
+  // merging MergeInputSections into a single MergeSyntheticSection. From this
+  // point onwards InputSectionDescription::sections should be used instead of
+  // sectionBases.
+  for (BaseCommand *base : script->sectionCommands)
+    if (auto *sec = dyn_cast<OutputSection>(base))
+      sec->finalizeInputSections();
+  llvm::erase_if(inputSections,
+                 [](InputSectionBase *s) { return isa<MergeInputSection>(s); });
+
   // Two input sections with different output sections should not be folded.
   // ICF runs after processSectionCommands() so that we know the output sections.
   if (config->icf != ICFLevel::None) {
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index 3a974074e0e51..d7c953262a41b 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -54,22 +54,9 @@ class SectionBase {
 
   unsigned sectionKind : 3;
 
-  // The next three bit fields are only used by InputSectionBase, but we
+  // The next two bit fields are only used by InputSectionBase, but we
   // put them here so the struct packs better.
 
-  // True if this section has already been placed to a linker script
-  // output section. This is needed because, in a linker script, you
-  // can refer to the same section more than once. For example, in
-  // the following linker script,
-  //
-  //   .foo : { *(.text) }
-  //   .bar : { *(.text) }
-  //
-  // .foo takes all .text sections, and .bar becomes empty. To achieve
-  // this, we need to memorize whether a section has been placed or
-  // not for each input section.
-  unsigned assigned : 1;
-
   unsigned bss : 1;
 
   // Set for sections that should not be folded by ICF.
@@ -108,9 +95,9 @@ class SectionBase {
   SectionBase(Kind sectionKind, StringRef name, uint64_t flags,
               uint64_t entsize, uint64_t alignment, uint32_t type,
               uint32_t info, uint32_t link)
-      : name(name), repl(this), sectionKind(sectionKind), assigned(false),
-        bss(false), keepUnique(false), partition(0), alignment(alignment),
-        flags(flags), entsize(entsize), type(type), link(link), info(info) {}
+      : name(name), repl(this), sectionKind(sectionKind), bss(false),
+        keepUnique(false), partition(0), alignment(alignment), flags(flags),
+        entsize(entsize), type(type), link(link), info(info) {}
 };
 
 // This corresponds to a section of an input file.
diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp
index 8d8f3b596fa94..70efc22829c64 100644
--- a/lld/ELF/LinkerScript.cpp
+++ b/lld/ELF/LinkerScript.cpp
@@ -341,19 +341,19 @@ bool LinkerScript::shouldKeep(InputSectionBase *s) {
 }
 
 // A helper function for the SORT() command.
-static bool matchConstraints(ArrayRef<InputSection *> sections,
+static bool matchConstraints(ArrayRef<InputSectionBase *> sections,
                              ConstraintKind kind) {
   if (kind == ConstraintKind::NoConstraint)
     return true;
 
   bool isRW = llvm::any_of(
-      sections, [](InputSection *sec) { return sec->flags & SHF_WRITE; });
+      sections, [](InputSectionBase *sec) { return sec->flags & SHF_WRITE; });
 
   return (isRW && kind == ConstraintKind::ReadWrite) ||
          (!isRW && kind == ConstraintKind::ReadOnly);
 }
 
-static void sortSections(MutableArrayRef<InputSection *> vec,
+static void sortSections(MutableArrayRef<InputSectionBase *> vec,
                          SortSectionPolicy k) {
   auto alignmentComparator = [](InputSectionBase *a, InputSectionBase *b) {
     // ">" is not a mistake. Sections with larger alignments are placed
@@ -392,7 +392,7 @@ static void sortSections(MutableArrayRef<InputSection *> vec,
 //    --sort-section is handled as an inner SORT command.
 // 3. If one SORT command is given, and if it is SORT_NONE, don't sort.
 // 4. If no SORT command is given, sort according to --sort-section.
-static void sortInputSections(MutableArrayRef<InputSection *> vec,
+static void sortInputSections(MutableArrayRef<InputSectionBase *> vec,
                               const SectionPattern &pat) {
   if (pat.sortOuter == SortSectionPolicy::None)
     return;
@@ -405,16 +405,16 @@ static void sortInputSections(MutableArrayRef<InputSection *> vec,
 }
 
 // Compute and remember which sections the InputSectionDescription matches.
-std::vector<InputSection *>
+std::vector<InputSectionBase *>
 LinkerScript::computeInputSections(const InputSectionDescription *cmd) {
-  std::vector<InputSection *> ret;
+  std::vector<InputSectionBase *> ret;
 
   // Collects all sections that satisfy constraints of Cmd.
   for (const SectionPattern &pat : cmd->sectionPatterns) {
     size_t sizeBefore = ret.size();
 
     for (InputSectionBase *sec : inputSections) {
-      if (!sec->isLive() || sec->assigned)
+      if (!sec->isLive() || sec->parent)
         continue;
 
       // For -emit-relocs we have to ignore entries like
@@ -422,10 +422,8 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd) {
       // which are common because they are in the default bfd script.
       // We do not ignore SHT_REL[A] linker-synthesized sections here because
       // want to support scripts that do custom layout for them.
-      //
-      // It is safe to assume that Sec is an InputSection because mergeable or
-      // EH input sections have already been handled and eliminated.
-      if (cast<InputSection>(sec)->getRelocatedSection())
+      if (isa<InputSection>(sec) &&
+          cast<InputSection>(sec)->getRelocatedSection())
         continue;
 
       std::string filename = getFilename(sec->file);
@@ -434,42 +432,43 @@ LinkerScript::computeInputSections(const InputSectionDescription *cmd) {
           !pat.sectionPat.match(sec->name))
         continue;
 
-      ret.push_back(cast<InputSection>(sec));
-      sec->assigned = true;
+      ret.push_back(sec);
     }
 
-    sortInputSections(MutableArrayRef<InputSection *>(ret).slice(sizeBefore),
-                      pat);
+    sortInputSections(
+        MutableArrayRef<InputSectionBase *>(ret).slice(sizeBefore), pat);
   }
   return ret;
 }
 
-void LinkerScript::discard(ArrayRef<InputSection *> v) {
-  for (InputSection *s : v) {
-    if (s == in.shStrTab || s == mainPart->relaDyn || s == mainPart->relrDyn)
-      error("discarding " + s->name + " section is not allowed");
-
-    // You can discard .hash and .gnu.hash sections by linker scripts. Since
-    // they are synthesized sections, we need to handle them differently than
-    // other regular sections.
-    if (s == mainPart->gnuHashTab)
-      mainPart->gnuHashTab = nullptr;
-    if (s == mainPart->hashTab)
-      mainPart->hashTab = nullptr;
-
-    s->markDead();
-    discard(s->dependentSections);
-  }
+void LinkerScript::discard(InputSectionBase *s) {
+  if (s == in.shStrTab || s == mainPart->relaDyn || s == mainPart->relrDyn)
+    error("discarding " + s->name + " section is not allowed");
+
+  // You can discard .hash and .gnu.hash sections by linker scripts. Since
+  // they are synthesized sections, we need to handle them differently than
+  // other regular sections.
+  if (s == mainPart->gnuHashTab)
+    mainPart->gnuHashTab = nullptr;
+  if (s == mainPart->hashTab)
+    mainPart->hashTab = nullptr;
+
+  s->markDead();
+  s->parent = nullptr;
+  for (InputSection *ds : s->dependentSections)
+    discard(ds);
 }
 
-std::vector<InputSection *>
+std::vector<InputSectionBase *>
 LinkerScript::createInputSectionList(OutputSection &outCmd) {
-  std::vector<InputSection *> ret;
+  std::vector<InputSectionBase *> ret;
 
   for (BaseCommand *base : outCmd.sectionCommands) {
     if (auto *cmd = dyn_cast<InputSectionDescription>(base)) {
-      cmd->sections = computeInputSections(cmd);
-      ret.insert(ret.end(), cmd->sections.begin(), cmd->sections.end());
+      cmd->sectionBases = computeInputSections(cmd);
+      for (InputSectionBase *s : cmd->sectionBases)
+        s->parent = &outCmd;
+      ret.insert(ret.end(), cmd->sectionBases.begin(), cmd->sectionBases.end());
     }
   }
   return ret;
@@ -480,12 +479,13 @@ void LinkerScript::processSectionCommands() {
   size_t i = 0;
   for (BaseCommand *base : sectionCommands) {
     if (auto *sec = dyn_cast<OutputSection>(base)) {
-      std::vector<InputSection *> v = createInputSectionList(*sec);
+      std::vector<InputSectionBase *> v = createInputSectionList(*sec);
 
       // The output section name `/DISCARD/' is special.
       // Any input section assigned to it is discarded.
       if (sec->name == "/DISCARD/") {
-        discard(v);
+        for (InputSectionBase *s : v)
+          discard(s);
         sec->sectionCommands.clear();
         continue;
       }
@@ -499,7 +499,7 @@ void LinkerScript::processSectionCommands() {
       // way to "make it as if it wasn't present" is to make it empty.
       if (!matchConstraints(v, sec->constraint)) {
         for (InputSectionBase *s : v)
-          s->assigned = false;
+          s->parent = nullptr;
         sec->sectionCommands.clear();
         continue;
       }
@@ -513,15 +513,12 @@ void LinkerScript::processSectionCommands() {
           s->alignment = subalign;
       }
 
-      // Some input sections may be removed from the list after ICF.
-      for (InputSection *s : v)
-        sec->addSection(s);
+      // Set the partition field the same way OutputSection::recordSection()
+      // does. Partitions cannot be used with the SECTIONS command, so this is
+      // always 1.
+      sec->partition = 1;
 
       sec->sectionIndex = i++;
-      if (sec->noload)
-        sec->type = SHT_NOBITS;
-      if (sec->nonAlloc)
-        sec->flags &= ~(uint64_t)SHF_ALLOC;
     }
   }
 }
@@ -565,7 +562,7 @@ static OutputSection *findByName(ArrayRef<BaseCommand *> vec,
 static OutputSection *createSection(InputSectionBase *isec,
                                     StringRef outsecName) {
   OutputSection *sec = script->createOutputSection(outsecName, "<internal>");
-  sec->addSection(cast<InputSection>(isec));
+  sec->recordSection(isec);
   return sec;
 }
 
@@ -594,7 +591,7 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map,
     OutputSection *out = sec->getRelocatedSection()->getOutputSection();
 
     if (out->relocationSection) {
-      out->relocationSection->addSection(sec);
+      out->relocationSection->recordSection(sec);
       return nullptr;
     }
 
@@ -602,12 +599,6 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map,
     return out->relocationSection;
   }
 
-  // When control reaches here, mergeable sections have already been merged into
-  // synthetic sections. For relocatable case we want to create one output
-  // section per syntetic section so that they have a valid sh_entsize.
-  if (config->relocatable && (isec->flags & SHF_MERGE))
-    return createSection(isec, outsecName);
-
   //  The ELF spec just says
   // ----------------------------------------------------------------
   // In the first phase, input sections that match in name, type and
@@ -654,7 +645,21 @@ addInputSec(StringMap<TinyPtrVector<OutputSection *>> &map,
   for (OutputSection *sec : v) {
     if (sec->partition != isec->partition)
       continue;
-    sec->addSection(cast<InputSection>(isec));
+
+    if (config->relocatable && (isec->flags & SHF_LINK_ORDER)) {
+      // Merging two SHF_LINK_ORDER sections with different sh_link fields will
+      // change their semantics, so we only merge them in -r links if they will
+      // end up being linked to the same output section. The casts are fine
+      // because everything in the map was created by the orphan placement code.
+      auto *firstIsec = cast<InputSectionBase>(
+          cast<InputSectionDescription>(sec->sectionCommands[0])
+              ->sectionBases[0]);
+      if (firstIsec->getLinkOrderDep()->getOutputSection() !=
+          isec->getLinkOrderDep()->getOutputSection())
+        continue;
+    }
+
+    sec->recordSection(isec);
     return nullptr;
   }
 
@@ -668,25 +673,30 @@ void LinkerScript::addOrphanSections() {
   StringMap<TinyPtrVector<OutputSection *>> map;
   std::vector<OutputSection *> v;
 
-  auto add = [&](InputSectionBase *s) {
-    if (!s->isLive() || s->parent)
-      return;
-
-    StringRef name = getOutputSectionName(s);
-
-    if (config->orphanHandling == OrphanHandlingPolicy::Error)
-      error(toString(s) + " is being placed in '" + name + "'");
-    else if (config->orphanHandling == OrphanHandlingPolicy::Warn)
-      warn(toString(s) + " is being placed in '" + name + "'");
-
-    if (OutputSection *sec = findByName(sectionCommands, name)) {
-      sec->addSection(cast<InputSection>(s));
-      return;
+  std::function<void(InputSectionBase *)> add;
+  add = [&](InputSectionBase *s) {
+    if (s->isLive() && !s->parent) {
+      StringRef name = getOutputSectionName(s);
+
+      if (config->orphanHandling == OrphanHandlingPolicy::Error)
+        error(toString(s) + " is being placed in '" + name + "'");
+      else if (config->orphanHandling == OrphanHandlingPolicy::Warn)
+        warn(toString(s) + " is being placed in '" + name + "'");
+
+      if (OutputSection *sec = findByName(sectionCommands, name)) {
+        sec->recordSection(s);
+      } else {
+        if (OutputSection *os = addInputSec(map, s, name))
+          v.push_back(os);
+        assert(isa<MergeInputSection>(s) ||
+               s->getOutputSection()->sectionIndex == UINT32_MAX);
+      }
     }
 
-    if (OutputSection *os = addInputSec(map, s, name))
-      v.push_back(os);
-    assert(s->getOutputSection()->sectionIndex == UINT32_MAX);
+    if (config->relocatable)
+      for (InputSectionBase *depSec : s->dependentSections)
+        if (depSec->flags & SHF_LINK_ORDER)
+          add(depSec);
   };
 
   // For futher --emit-reloc handling code we need target output section
@@ -694,6 +704,12 @@ void LinkerScript::addOrphanSections() {
   // to create target sections first. We do not want priority handling
   // for synthetic sections because them are special.
   for (InputSectionBase *isec : inputSections) {
+    // In -r links, SHF_LINK_ORDER sections are added while adding their parent
+    // sections because we need to know the parent's output section before we
+    // can select an output section for the SHF_LINK_ORDER section.
+    if (config->relocatable && (isec->flags & SHF_LINK_ORDER))
+      continue;
+
     if (auto *sec = dyn_cast<InputSection>(isec))
       if (InputSectionBase *rel = sec->getRelocatedSection())
         if (auto *relIS = dyn_cast_or_null<InputSectionBase>(rel->parent))
@@ -1006,17 +1022,13 @@ static uint64_t computeBase(uint64_t min, bool allocateHeaders) {
   return alignDown(min, config->maxPageSize);
 }
 
-// Try to find an address for the file and program headers output sections,
-// which were unconditionally added to the first PT_LOAD segment earlier.
+// When the SECTIONS command is used, try to find an address for the file and
+// program headers output sections, which can be added to the first PT_LOAD
+// segment when program headers are created.
 //
-// When using the default layout, we check if the headers fit below the first
-// allocated section. When using a linker script, we also check if the headers
-// are covered by the output section. This allows omitting the headers by not
-// leaving enough space for them in the linker script; this pattern is common
-// in embedded systems.
-//
-// If there isn't enough space for these sections, we'll remove them from the
-// PT_LOAD segment, and we'll also remove the PT_PHDR segment.
+// We check if the headers fit below the first allocated section. If there isn't
+// enough space for these sections, we'll remove them from the PT_LOAD segment,
+// and we'll also remove the PT_PHDR segment.
 void LinkerScript::allocateHeaders(std::vector<PhdrEntry *> &phdrs) {
   uint64_t min = std::numeric_limits<uint64_t>::max();
   for (OutputSection *sec : outputSections)
@@ -1062,28 +1074,23 @@ LinkerScript::AddressState::AddressState() {
   }
 }
 
-static uint64_t getInitialDot() {
-  // By default linker scripts use an initial value of 0 for '.',
-  // but prefer -image-base if set.
-  if (script->hasSectionsCommand)
-    return config->imageBase ? *config->imageBase : 0;
-
-  uint64_t startAddr = UINT64_MAX;
-  // The sections with -T<section> have been sorted in order of ascending
-  // address. We must lower startAddr if the lowest -T<section address> as
-  // calls to setDot() must be monotonically increasing.
-  for (auto &kv : config->sectionStartMap)
-    startAddr = std::min(startAddr, kv.second);
-  return std::min(startAddr, target->getImageBase() + elf::getHeaderSize());
-}
-
 // Here we assign addresses as instructed by linker script SECTIONS
 // sub-commands. Doing that allows us to use final VA values, so here
 // we also handle rest commands like symbol assignments and ASSERTs.
 // Returns a symbol that has changed its section or value, or nullptr if no
 // symbol has changed.
 const Defined *LinkerScript::assignAddresses() {
-  dot = getInitialDot();
+  if (script->hasSectionsCommand) {
+    // With a linker script, assignment of addresses to headers is covered by
+    // allocateHeaders().
+    dot = config->imageBase.getValueOr(0);
+  } else {
+    // Assign addresses to headers right now.
+    dot = target->getImageBase();
+    Out::elfHeader->addr = dot;
+    Out::programHeaders->addr = dot + Out::elfHeader->size;
+    dot += getHeaderSize();
+  }
 
   auto deleter = std::make_unique<AddressState>();
   ctx = deleter.get();
diff --git a/lld/ELF/LinkerScript.h b/lld/ELF/LinkerScript.h
index 4ac9e2a390930..621b8baeaae63 100644
--- a/lld/ELF/LinkerScript.h
+++ b/lld/ELF/LinkerScript.h
@@ -168,6 +168,12 @@ struct InputSectionDescription : BaseCommand {
   // will be associated with this InputSectionDescription.
   std::vector<SectionPattern> sectionPatterns;
 
+  // Includes InputSections and MergeInputSections. Used temporarily during
+  // assignment of input sections to output sections.
+  std::vector<InputSectionBase *> sectionBases;
+
+  // Used after the finalizeInputSections() pass. MergeInputSections have been
+  // merged into MergeSyntheticSections.
   std::vector<InputSection *> sections;
 
   // Temporary record of synthetic ThunkSection instances and the pass that
@@ -226,10 +232,10 @@ class LinkerScript final {
   void expandOutputSection(uint64_t size);
   void expandMemoryRegions(uint64_t size);
 
-  std::vector<InputSection *>
+  std::vector<InputSectionBase *>
   computeInputSections(const InputSectionDescription *);
 
-  std::vector<InputSection *> createInputSectionList(OutputSection &cmd);
+  std::vector<InputSectionBase *> createInputSectionList(OutputSection &cmd);
 
   std::vector<size_t> getPhdrIndices(OutputSection *sec);
 
@@ -259,7 +265,7 @@ class LinkerScript final {
 
   bool hasPhdrsCommands() { return !phdrsCommands.empty(); }
   uint64_t getDot() { return dot; }
-  void discard(ArrayRef<InputSection *> v);
+  void discard(InputSectionBase *s);
 
   ExprValue getSymbolValue(StringRef name, const Twine &loc);
 
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 6246d84946b6f..c540efb25e90b 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -171,6 +171,9 @@ defm fini: Eq<"fini", "Specify a finalizer function">, MetaVarName<"<symbol>">;
 def fix_cortex_a53_843419: F<"fix-cortex-a53-843419">,
   HelpText<"Apply fixes for AArch64 Cortex-A53 erratum 843419">;
 
+def fix_cortex_a8: F<"fix-cortex-a8">,
+  HelpText<"Apply fixes for ARM Cortex-A8 erratum 657417">;
+
 // This option is intentionally hidden from the user as the implementation
 // is not complete.
 def require_cet: F<"require-cet">;
@@ -478,6 +481,7 @@ def lto_cs_profile_generate: F<"lto-cs-profile-generate">,
   HelpText<"Perform context senstive PGO instrumentation">;
 def lto_cs_profile_file: J<"lto-cs-profile-file=">,
   HelpText<"Context sensitive profile file path">;
+def lto_obj_path_eq: J<"lto-obj-path=">;
 def lto_sample_profile: J<"lto-sample-profile=">,
   HelpText<"Sample profile file path">;
 def disable_verify: F<"disable-verify">;
@@ -495,7 +499,12 @@ def save_temps: F<"save-temps">;
 def thinlto_cache_dir: J<"thinlto-cache-dir=">,
   HelpText<"Path to ThinLTO cached object file directory">;
 defm thinlto_cache_policy: Eq<"thinlto-cache-policy", "Pruning policy for the ThinLTO cache">;
+def thinlto_emit_imports_files: F<"thinlto-emit-imports-files">;
+def thinlto_index_only: F<"thinlto-index-only">;
+def thinlto_index_only_eq: J<"thinlto-index-only=">;
 def thinlto_jobs: J<"thinlto-jobs=">, HelpText<"Number of ThinLTO jobs">;
+def thinlto_object_suffix_replace_eq: J<"thinlto-object-suffix-replace=">;
+def thinlto_prefix_replace_eq: J<"thinlto-prefix-replace=">;
 
 def: J<"plugin-opt=O">, Alias<lto_O>, HelpText<"Alias for -lto-O">;
 def: F<"plugin-opt=debug-pass-manager">,
@@ -509,19 +518,31 @@ def: J<"plugin-opt=lto-partitions=">, Alias<lto_partitions>, HelpText<"Alias for
 def plugin_opt_mcpu_eq: J<"plugin-opt=mcpu=">;
 def: F<"plugin-opt=new-pass-manager">,
   Alias<lto_new_pass_manager>, HelpText<"Alias for -lto-new-pass-manager">;
-def plugin_opt_obj_path_eq: J<"plugin-opt=obj-path=">;
 def: F<"plugin-opt=cs-profile-generate">,
   Alias<lto_cs_profile_generate>, HelpText<"Alias for -lto-cs-profile-generate">;
 def: J<"plugin-opt=cs-profile-path=">,
   Alias<lto_cs_profile_file>, HelpText<"Alias for -lto-cs-profile-file">;
+def: J<"plugin-opt=obj-path=">,
+  Alias<lto_obj_path_eq>,
+  HelpText<"Alias for -lto-obj-path=">;
 def: J<"plugin-opt=sample-profile=">,
   Alias<lto_sample_profile>, HelpText<"Alias for -lto-sample-profile">;
 def: F<"plugin-opt=save-temps">, Alias<save_temps>, HelpText<"Alias for -save-temps">;
-def plugin_opt_thinlto_emit_imports_files: F<"plugin-opt=thinlto-emit-imports-files">;
-def plugin_opt_thinlto_index_only: F<"plugin-opt=thinlto-index-only">;
-def plugin_opt_thinlto_index_only_eq: J<"plugin-opt=thinlto-index-only=">;
-def plugin_opt_thinlto_object_suffix_replace_eq: J<"plugin-opt=thinlto-object-suffix-replace=">;
-def plugin_opt_thinlto_prefix_replace_eq: J<"plugin-opt=thinlto-prefix-replace=">;
+def: F<"plugin-opt=thinlto-emit-imports-files">,
+  Alias<thinlto_emit_imports_files>,
+  HelpText<"Alias for -thinlto-emit-imports-files">;
+def: F<"plugin-opt=thinlto-index-only">,
+  Alias<thinlto_index_only>,
+  HelpText<"Alias for -thinlto-index-only">;
+def: J<"plugin-opt=thinlto-index-only=">,
+  Alias<thinlto_index_only_eq>,
+  HelpText<"Alias for -thinlto-index-only=">;
+def: J<"plugin-opt=thinlto-object-suffix-replace=">,
+ Alias<thinlto_object_suffix_replace_eq>,
+ HelpText<"Alias for -thinlto-object-suffix-replace=">;
+def: J<"plugin-opt=thinlto-prefix-replace=">,
+  Alias<thinlto_prefix_replace_eq>,
+  HelpText<"Alias for -thinlto-prefix-replace=">;
 
 // Ignore LTO plugin-related options.
 // clang -flto passes -plugin and -plugin-opt to the linker. This is required
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index a89bd509bc96d..d6164103867a9 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -83,12 +83,32 @@ static bool canMergeToProgbits(unsigned type) {
          type == SHT_NOTE;
 }
 
-void OutputSection::addSection(InputSection *isec) {
+// Record that isec will be placed in the OutputSection. isec does not become
+// permanent until finalizeInputSections() is called. The function should not be
+// used after finalizeInputSections() is called. If you need to add an
+// InputSection post finalizeInputSections(), then you must do the following:
+//
+// 1. Find or create an InputSectionDescription to hold InputSection.
+// 2. Add the InputSection to the InputSectionDesciption::sections.
+// 3. Call commitSection(isec).
+void OutputSection::recordSection(InputSectionBase *isec) {
+  partition = isec->partition;
+  isec->parent = this;
+  if (sectionCommands.empty() ||
+      !isa<InputSectionDescription>(sectionCommands.back()))
+    sectionCommands.push_back(make<InputSectionDescription>(""));
+  auto *isd = cast<InputSectionDescription>(sectionCommands.back());
+  isd->sectionBases.push_back(isec);
+}
+
+// Update fields (type, flags, alignment, etc) according to the InputSection
+// isec. Also check whether the InputSection flags and type are consistent with
+// other InputSections.
+void OutputSection::commitSection(InputSection *isec) {
   if (!hasInputSections) {
     // If IS is the first section to be added to this section,
-    // initialize Partition, Type, Entsize and flags from IS.
+    // initialize type, entsize and flags from isec.
     hasInputSections = true;
-    partition = isec->partition;
     type = isec->type;
     entsize = isec->entsize;
     flags = isec->flags;
@@ -110,6 +130,8 @@ void OutputSection::addSection(InputSection *isec) {
       type = SHT_PROGBITS;
     }
   }
+  if (noload)
+    type = SHT_NOBITS;
 
   isec->parent = this;
   uint64_t andMask =
@@ -118,6 +140,8 @@ void OutputSection::addSection(InputSection *isec) {
   uint64_t andFlags = (flags & isec->flags) & andMask;
   uint64_t orFlags = (flags | isec->flags) & orMask;
   flags = andFlags | orFlags;
+  if (nonAlloc)
+    flags &= ~(uint64_t)SHF_ALLOC;
 
   alignment = std::max(alignment, isec->alignment);
 
@@ -126,15 +150,69 @@ void OutputSection::addSection(InputSection *isec) {
   // set sh_entsize to 0.
   if (entsize != isec->entsize)
     entsize = 0;
+}
+
+// This function scans over the InputSectionBase list sectionBases to create
+// InputSectionDescription::sections.
+//
+// It removes MergeInputSections from the input section array and adds
+// new synthetic sections at the location of the first input section
+// that it replaces. It then finalizes each synthetic section in order
+// to compute an output offset for each piece of each input section.
+void OutputSection::finalizeInputSections() {
+  std::vector<MergeSyntheticSection *> mergeSections;
+  for (BaseCommand *base : sectionCommands) {
+    auto *cmd = dyn_cast<InputSectionDescription>(base);
+    if (!cmd)
+      continue;
+    cmd->sections.reserve(cmd->sectionBases.size());
+    for (InputSectionBase *s : cmd->sectionBases) {
+      MergeInputSection *ms = dyn_cast<MergeInputSection>(s);
+      if (!ms) {
+        cmd->sections.push_back(cast<InputSection>(s));
+        continue;
+      }
+
+      // We do not want to handle sections that are not alive, so just remove
+      // them instead of trying to merge.
+      if (!ms->isLive())
+        continue;
+
+      auto i = llvm::find_if(mergeSections, [=](MergeSyntheticSection *sec) {
+        // While we could create a single synthetic section for two different
+        // values of Entsize, it is better to take Entsize into consideration.
+        //
+        // With a single synthetic section no two pieces with different Entsize
+        // could be equal, so we may as well have two sections.
+        //
+        // Using Entsize in here also allows us to propagate it to the synthetic
+        // section.
+        //
+        // SHF_STRINGS section with different alignments should not be merged.
+        return sec->flags == ms->flags && sec->entsize == ms->entsize &&
+               (sec->alignment == ms->alignment || !(sec->flags & SHF_STRINGS));
+      });
+      if (i == mergeSections.end()) {
+        MergeSyntheticSection *syn =
+            createMergeSynthetic(name, ms->type, ms->flags, ms->alignment);
+        mergeSections.push_back(syn);
+        i = std::prev(mergeSections.end());
+        syn->entsize = ms->entsize;
+        cmd->sections.push_back(syn);
+      }
+      (*i)->addSection(ms);
+    }
+
+    // sectionBases should not be used from this point onwards. Clear it to
+    // catch misuses.
+    cmd->sectionBases.clear();
 
-  if (!isec->assigned) {
-    isec->assigned = true;
-    if (sectionCommands.empty() ||
-        !isa<InputSectionDescription>(sectionCommands.back()))
-      sectionCommands.push_back(make<InputSectionDescription>(""));
-    auto *isd = cast<InputSectionDescription>(sectionCommands.back());
-    isd->sections.push_back(isec);
+    // Some input sections may be removed from the list after ICF.
+    for (InputSection *s : cmd->sections)
+      commitSection(s);
   }
+  for (auto *ms : mergeSections)
+    ms->finalizeContents();
 }
 
 static void sortByOrder(MutableArrayRef<InputSection *> in,
diff --git a/lld/ELF/OutputSections.h b/lld/ELF/OutputSections.h
index fff8327ea376c..a24294eedf351 100644
--- a/lld/ELF/OutputSections.h
+++ b/lld/ELF/OutputSections.h
@@ -71,7 +71,9 @@ class OutputSection final : public BaseCommand, public SectionBase {
   uint64_t addr = 0;
   uint32_t shName = 0;
 
-  void addSection(InputSection *isec);
+  void recordSection(InputSectionBase *isec);
+  void commitSection(InputSection *isec);
+  void finalizeInputSections();
 
   // The following members are normally only used in linker scripts.
   MemoryRegion *memRegion = nullptr;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 42f0a873909bc..442ffdbb2f847 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -567,10 +567,16 @@ template <class ELFT> static void addCopyRelSymbol(SharedSymbol &ss) {
   bool isRO = isReadOnly<ELFT>(ss);
   BssSection *sec =
       make<BssSection>(isRO ? ".bss.rel.ro" : ".bss", symSize, ss.alignment);
-  if (isRO)
-    in.bssRelRo->getParent()->addSection(sec);
-  else
-    in.bss->getParent()->addSection(sec);
+  OutputSection *osec = (isRO ? in.bssRelRo : in.bss)->getParent();
+
+  // At this point, sectionBases has been migrated to sections. Append sec to
+  // sections.
+  if (osec->sectionCommands.empty() ||
+      !isa<InputSectionDescription>(osec->sectionCommands.back()))
+    osec->sectionCommands.push_back(make<InputSectionDescription>(""));
+  auto *isd = cast<InputSectionDescription>(osec->sectionCommands.back());
+  isd->sections.push_back(sec);
+  osec->commitSection(sec);
 
   // Look through the DSO's dynamic symbol table for aliases and create a
   // dynamic symbol for each one. This causes the copy relocation to correctly
diff --git a/lld/ELF/SymbolTable.cpp b/lld/ELF/SymbolTable.cpp
index 64dd1621ceaa1..8470a077e7f00 100644
--- a/lld/ELF/SymbolTable.cpp
+++ b/lld/ELF/SymbolTable.cpp
@@ -123,10 +123,7 @@ StringMap<std::vector<Symbol *>> &SymbolTable::getDemangledSyms() {
     for (Symbol *sym : symVector) {
       if (!sym->isDefined() && !sym->isCommon())
         continue;
-      if (Optional<std::string> s = demangleItanium(sym->getName()))
-        (*demangledSyms)[*s].push_back(sym);
-      else
-        (*demangledSyms)[sym->getName()].push_back(sym);
+      (*demangledSyms)[demangleItanium(sym->getName())].push_back(sym);
     }
   }
   return *demangledSyms;
diff --git a/lld/ELF/Symbols.cpp b/lld/ELF/Symbols.cpp
index 3845da7993463..e34f47d6830e9 100644
--- a/lld/ELF/Symbols.cpp
+++ b/lld/ELF/Symbols.cpp
@@ -45,8 +45,7 @@ Defined *ElfSym::tlsModuleBase;
 // Returns a symbol for an error message.
 static std::string demangle(StringRef symName) {
   if (config->demangle)
-    if (Optional<std::string> s = demangleItanium(symName))
-      return *s;
+    return demangleItanium(symName);
   return symName;
 }
 namespace lld {
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index e323ecaa9d7e0..5d9ebfb532530 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -257,10 +257,8 @@ InputSection *elf::createInterpSection() {
   StringRef s = saver.save(config->dynamicLinker);
   ArrayRef<uint8_t> contents = {(const uint8_t *)s.data(), s.size() + 1};
 
-  auto *sec = make<InputSection>(nullptr, SHF_ALLOC, SHT_PROGBITS, 1, contents,
-                                 ".interp");
-  sec->markLive();
-  return sec;
+  return make<InputSection>(nullptr, SHF_ALLOC, SHT_PROGBITS, 1, contents,
+                            ".interp");
 }
 
 Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
@@ -3125,10 +3123,9 @@ void MergeNoTailSection::finalizeContents() {
   });
 }
 
-static MergeSyntheticSection *createMergeSynthetic(StringRef name,
-                                                   uint32_t type,
-                                                   uint64_t flags,
-                                                   uint32_t alignment) {
+MergeSyntheticSection *elf::createMergeSynthetic(StringRef name, uint32_t type,
+                                                 uint64_t flags,
+                                                 uint32_t alignment) {
   bool shouldTailMerge = (flags & SHF_STRINGS) && config->optimize >= 2;
   if (shouldTailMerge)
     return make<MergeTailSection>(name, type, flags, alignment);
@@ -3146,63 +3143,6 @@ template <class ELFT> void elf::splitSections() {
   });
 }
 
-// This function scans over the inputsections to create mergeable
-// synthetic sections.
-//
-// It removes MergeInputSections from the input section array and adds
-// new synthetic sections at the location of the first input section
-// that it replaces. It then finalizes each synthetic section in order
-// to compute an output offset for each piece of each input section.
-void elf::mergeSections() {
-  std::vector<MergeSyntheticSection *> mergeSections;
-  for (InputSectionBase *&s : inputSections) {
-    MergeInputSection *ms = dyn_cast<MergeInputSection>(s);
-    if (!ms)
-      continue;
-
-    // We do not want to handle sections that are not alive, so just remove
-    // them instead of trying to merge.
-    if (!ms->isLive()) {
-      s = nullptr;
-      continue;
-    }
-
-    StringRef outsecName = getOutputSectionName(ms);
-
-    auto i = llvm::find_if(mergeSections, [=](MergeSyntheticSection *sec) {
-      // While we could create a single synthetic section for two different
-      // values of Entsize, it is better to take Entsize into consideration.
-      //
-      // With a single synthetic section no two pieces with different Entsize
-      // could be equal, so we may as well have two sections.
-      //
-      // Using Entsize in here also allows us to propagate it to the synthetic
-      // section.
-      //
-      // SHF_STRINGS section with different alignments should not be merged.
-      return sec->name == outsecName && sec->flags == ms->flags &&
-             sec->entsize == ms->entsize &&
-             (sec->alignment == ms->alignment || !(sec->flags & SHF_STRINGS));
-    });
-    if (i == mergeSections.end()) {
-      MergeSyntheticSection *syn =
-          createMergeSynthetic(outsecName, ms->type, ms->flags, ms->alignment);
-      mergeSections.push_back(syn);
-      i = std::prev(mergeSections.end());
-      s = syn;
-      syn->entsize = ms->entsize;
-    } else {
-      s = nullptr;
-    }
-    (*i)->addSection(ms);
-  }
-  for (auto *ms : mergeSections)
-    ms->finalizeContents();
-
-  std::vector<InputSectionBase *> &v = inputSections;
-  v.erase(std::remove(v.begin(), v.end(), nullptr), v.end());
-}
-
 MipsRldMapSection::MipsRldMapSection()
     : SyntheticSection(SHF_ALLOC | SHF_WRITE, SHT_PROGBITS, config->wordsize,
                        ".rld_map") {}
@@ -3218,17 +3158,23 @@ static InputSection *findExidxSection(InputSection *isec) {
   return nullptr;
 }
 
+static bool isValidExidxSectionDep(InputSection *isec) {
+  return (isec->flags & SHF_ALLOC) && (isec->flags & SHF_EXECINSTR) &&
+         isec->getSize() > 0;
+}
+
 bool ARMExidxSyntheticSection::addSection(InputSection *isec) {
   if (isec->type == SHT_ARM_EXIDX) {
-    exidxSections.push_back(isec);
-    return true;
+    if (InputSection* dep = isec->getLinkOrderDep())
+      if (isValidExidxSectionDep(dep)) {
+        exidxSections.push_back(isec);
+        return true;
+      }
+    return false;
   }
 
-  if ((isec->flags & SHF_ALLOC) && (isec->flags & SHF_EXECINSTR) &&
-      isec->getSize() > 0) {
+  if (isValidExidxSectionDep(isec)) {
     executableSections.push_back(isec);
-    if (empty && findExidxSection(isec))
-      empty = false;
     return false;
   }
 
@@ -3395,6 +3341,12 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) {
   assert(size == offset + 8);
 }
 
+bool ARMExidxSyntheticSection::isNeeded() const {
+  return llvm::find_if(exidxSections, [](InputSection *isec) {
+           return isec->isLive();
+         }) != exidxSections.end();
+}
+
 bool ARMExidxSyntheticSection::classof(const SectionBase *d) {
   return d->kind() == InputSectionBase::Synthetic && d->type == SHT_ARM_EXIDX;
 }
diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h
index 6846397895066..d592dcb84e126 100644
--- a/lld/ELF/SyntheticSections.h
+++ b/lld/ELF/SyntheticSections.h
@@ -994,7 +994,7 @@ class ARMExidxSyntheticSection : public SyntheticSection {
 
   size_t getSize() const override { return size; }
   void writeTo(uint8_t *buf) override;
-  bool isNeeded() const override { return !empty; }
+  bool isNeeded() const override;
   // Sort and remove duplicate entries.
   void finalizeContents() override;
   InputSection *getLinkOrderDep() const;
@@ -1008,9 +1008,6 @@ class ARMExidxSyntheticSection : public SyntheticSection {
 private:
   size_t size;
 
-  // Empty if ExecutableSections contains no dependent .ARM.exidx sections.
-  bool empty = true;
-
   // Instead of storing pointers to the .ARM.exidx InputSections from
   // InputObjects, we store pointers to the executable sections that need
   // .ARM.exidx sections. We can then use the dependentSections of these to
@@ -1102,8 +1099,9 @@ class PartitionIndexSection : public SyntheticSection {
 
 InputSection *createInterpSection();
 MergeInputSection *createCommentSection();
+MergeSyntheticSection *createMergeSynthetic(StringRef name, uint32_t type,
+                                            uint64_t flags, uint32_t alignment);
 template <class ELFT> void splitSections();
-void mergeSections();
 
 template <typename ELFT> void writeEhdr(uint8_t *buf, Partition &part);
 template <typename ELFT> void writePhdrs(uint8_t *buf, Partition &part);
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 68a16b2a79bef..2ce36079db30f 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -8,6 +8,7 @@
 
 #include "Writer.h"
 #include "AArch64ErrataFix.h"
+#include "ARMErrataFix.h"
 #include "CallGraphSort.h"
 #include "Config.h"
 #include "LinkerScript.h"
@@ -313,7 +314,18 @@ template <class ELFT> void elf::createSyntheticSections() {
   // you can call lld::elf::main more than once as a library.
   memset(&Out::first, 0, sizeof(Out));
 
-  auto add = [](InputSectionBase *sec) { inputSections.push_back(sec); };
+  // Add the .interp section first because it is not a SyntheticSection.
+  // The removeUnusedSyntheticSections() function relies on the
+  // SyntheticSections coming last.
+  if (needsInterpSection()) {
+    for (size_t i = 1; i <= partitions.size(); ++i) {
+      InputSection *sec = createInterpSection();
+      sec->partition = i;
+      inputSections.push_back(sec);
+    }
+  }
+
+  auto add = [](SyntheticSection *sec) { inputSections.push_back(sec); };
 
   in.shStrTab = make<StringTableSection>(".shstrtab", false);
 
@@ -355,7 +367,7 @@ template <class ELFT> void elf::createSyntheticSections() {
   StringRef relaDynName = config->isRela ? ".rela.dyn" : ".rel.dyn";
 
   for (Partition &part : partitions) {
-    auto add = [&](InputSectionBase *sec) {
+    auto add = [&](SyntheticSection *sec) {
       sec->partition = part.getNumber();
       inputSections.push_back(sec);
     };
@@ -383,9 +395,6 @@ template <class ELFT> void elf::createSyntheticSections() {
       part.relaDyn =
           make<RelocationSection<ELFT>>(relaDynName, config->zCombreloc);
 
-    if (needsInterpSection())
-      add(createInterpSection());
-
     if (config->hasDynSymTab) {
       part.dynSymTab = make<SymbolTableSection<ELFT>>(*part.dynStrTab);
       add(part.dynSymTab);
@@ -555,7 +564,8 @@ template <class ELFT> void Writer<ELFT>::run() {
   for (OutputSection *sec : outputSections)
     sec->maybeCompress<ELFT>();
 
-  script->allocateHeaders(mainPart->phdrs);
+  if (script->hasSectionsCommand)
+    script->allocateHeaders(mainPart->phdrs);
 
   // Remove empty PT_LOAD to avoid causing the dynamic linker to try to mmap a
   // 0 sized region. This has to be done late since only after assignAddresses
@@ -587,7 +597,8 @@ template <class ELFT> void Writer<ELFT>::run() {
     return;
 
   if (!config->oFormatBinary) {
-    writeTrapInstr();
+    if (config->zSeparate != SeparateSegmentKind::None)
+      writeTrapInstr();
     writeHeader();
     writeSections();
   } else {
@@ -703,7 +714,7 @@ template <class ELFT> void Writer<ELFT>::addSectionSymbols() {
     });
     if (i == sec->sectionCommands.end())
       continue;
-    InputSection *isec = cast<InputSectionDescription>(*i)->sections[0];
+    InputSectionBase *isec = cast<InputSectionDescription>(*i)->sections[0];
 
     // Relocations are not using REL[A] section symbols.
     if (isec->type == SHT_REL || isec->type == SHT_RELA)
@@ -1498,6 +1509,12 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
     if (!(sec->flags & SHF_LINK_ORDER))
       continue;
 
+    // The ARM.exidx section use SHF_LINK_ORDER, but we have consolidated
+    // this processing inside the ARMExidxsyntheticsection::finalizeContents().
+    if (!config->relocatable && config->emachine == EM_ARM &&
+        sec->type == SHT_ARM_EXIDX)
+      continue;
+
     // Link order may be distributed across several InputSectionDescriptions
     // but sort must consider them all at once.
     std::vector<InputSection **> scriptSections;
@@ -1507,14 +1524,16 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
         for (InputSection *&isec : isd->sections) {
           scriptSections.push_back(&isec);
           sections.push_back(isec);
+
+          InputSection *link = isec->getLinkOrderDep();
+          if (!link->getParent())
+            error(toString(isec) + ": sh_link points to discarded section " +
+                  toString(link));
         }
       }
     }
 
-    // The ARM.exidx section use SHF_LINK_ORDER, but we have consolidated
-    // this processing inside the ARMExidxsyntheticsection::finalizeContents().
-    if (!config->relocatable && config->emachine == EM_ARM &&
-        sec->type == SHT_ARM_EXIDX)
+    if (errorCount())
       continue;
 
     llvm::stable_sort(sections, compareByFilePosition);
@@ -1531,6 +1550,7 @@ template <class ELFT> void Writer<ELFT>::resolveShfLinkOrder() {
 template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
   ThunkCreator tc;
   AArch64Err843419Patcher a64p;
+  ARMErr657417Patcher a32p;
   script->assignAddresses();
 
   int assignPasses = 0;
@@ -1549,6 +1569,11 @@ template <class ELFT> void Writer<ELFT>::finalizeAddressDependentContent() {
         script->assignAddresses();
       changed |= a64p.createFixes();
     }
+    if (config->fixCortexA8) {
+      if (changed)
+        script->assignAddresses();
+      changed |= a32p.createFixes();
+    }
 
     if (in.mipsGot)
       in.mipsGot->updateAllocSize();
@@ -1886,6 +1911,8 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
   // SHFLinkOrder processing must be processed after relative section placements are
   // known but before addresses are allocated.
   resolveShfLinkOrder();
+  if (errorCount())
+    return;
 
   // This is used to:
   // 1) Create "thunks":
@@ -2215,13 +2242,8 @@ template <class ELFT> void Writer<ELFT>::fixSectionAlignments() {
       // maximum page size boundary so that we can find the ELF header at the
       // start. We cannot benefit from overlapping p_offset ranges with the
       // previous segment anyway.
-      //
-      // TODO Enable this technique on all targets.
-      bool enable = config->emachine != EM_HEXAGON &&
-                    config->emachine != EM_X86_64;
-
-      if (!enable ||
-          (config->zSeparateCode && prev &&
+      if (config->zSeparate == SeparateSegmentKind::Loadable ||
+          (config->zSeparate == SeparateSegmentKind::Code && prev &&
            (prev->p_flags & PF_X) != (p->p_flags & PF_X)) ||
           cmd->type == SHT_LLVM_PART_EHDR)
         cmd->addrExpr = [] {
@@ -2330,7 +2352,8 @@ template <class ELFT> void Writer<ELFT>::assignFileOffsets() {
     // If this is a last section of the last executable segment and that
     // segment is the last loadable segment, align the offset of the
     // following section to avoid loading non-segments parts of the file.
-    if (config->zSeparateCode && lastRX && lastRX->lastSec == sec)
+    if (config->zSeparate != SeparateSegmentKind::None && lastRX &&
+        lastRX->lastSec == sec)
       off = alignTo(off, config->commonPageSize);
   }
 
@@ -2602,9 +2625,6 @@ static void fillTrap(uint8_t *i, uint8_t *end) {
 // We'll leave other pages in segments as-is because the rest will be
 // overwritten by output sections.
 template <class ELFT> void Writer<ELFT>::writeTrapInstr() {
-  if (!config->zSeparateCode)
-    return;
-
   for (Partition &part : partitions) {
     // Fill the last page.
     for (PhdrEntry *p : part.phdrs)
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index 74a5140237a96..1746ae9f5fd42 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -591,18 +591,21 @@ Force load of all members in a static library.
 Use wrapper functions for symbol.
 .It Fl z Ar option
 Linker option extensions.
-.Bl -tag -width indent
+.Bl -tag -width indent -compact
+.Pp
 .It Cm execstack
 Make the main stack executable.
 Stack permissions are recorded in the
 .Dv PT_GNU_STACK
 segment.
+.Pp
 .It Cm global
 Sets the
 .Dv DF_1_GLOBAL flag in the
 .Dv DYNAMIC
 section.
 Different loaders can decide how to handle this flag on their own.
+.Pp
 .It Cm ifunc-noplt
 Do not emit PLT entries for ifunc symbols.
 Instead, emit text relocations referencing the resolver.
@@ -611,64 +614,78 @@ environments where text relocations do not have the usual drawbacks.
 This option must be combined with the
 .Fl z Li notext
 option.
+.Pp
 .It Cm initfirst
 Sets the
 .Dv DF_1_INITFIRST
 flag to indicate the module should be initialized first.
+.Pp
 .It Cm interpose
 Set the
 .Dv DF_1_INTERPOSE
 flag to indicate to the runtime linker that the object is an interposer.
 During symbol resolution interposers are searched after the application
 but before other dependencies.
+.Pp
 .It Cm muldefs
 Do not error if a symbol is defined multiple times.
 The first definition will be used.
 This is a synonym for
 .Fl -allow-multiple-definition.
+.Pp
 .It Cm nocombreloc
 Disable combining and sorting multiple relocation sections.
+.Pp
 .It Cm nocopyreloc
 Disable the creation of copy relocations.
+.Pp
 .It Cm nodefaultlib
 Set the
 .Dv DF_1_NODEFLIB
 flag to indicate that default library search paths should be ignored.
+.Pp
 .It Cm nodelete
 Set the
 .Dv DF_1_NODELETE
 flag to indicate that the object cannot be unloaded from a process.
+.Pp
 .It Cm nodlopen
 Set the
 .Dv DF_1_NOOPEN
 flag to indicate that the object may not be opened by
 .Xr dlopen 3 .
+.Pp
 .It Cm norelro
 Do not indicate that portions of the object shold be mapped read-only
 after initial relocation processing.
 The object will omit the
 .Dv PT_GNU_RELRO
 segment.
+.Pp
 .It Cm notext
 Allow relocations against read-only segments.
 Sets the
 .Dv DT_TEXTREL flag in the
 .Dv DYNAMIC
 section.
+.Pp
 .It Cm now
 Set the
 .Dv DF_BIND_NOW
 flag to indicate that the run-time loader should perform all relocation
 processing as part of object initialization.
 By default relocations may be performed on demand.
+.Pp
 .It Cm origin
 Set the
 .Dv DF_ORIGIN
 flag to indicate that the object requires
 $ORIGIN
 processing.
+.Pp
 .It Cm retpolineplt
 Emit retpoline format PLT entries as a mitigation for CVE-2017-5715.
+.Pp
 .It Cm rodynamic
 Make the
 .Li .dynamic
@@ -676,6 +693,18 @@ section read-only.
 The
 .Dv DT_DEBUG
 tag will not be emitted.
+.Pp
+.It Cm separate-loadable-segments
+.It Cm separate-code
+.It Cm noseparate-code
+Specify whether two adjacent PT_LOAD segments are allowed to overlap in pages.
+.Cm noseparate-code
+(default) allows overlap.
+.Cm separate-code
+allows overlap between two executable segments, or two non-executable segments.
+.Cm separate-loadable-segments
+disallows overlap.
+.Pp
 .It Cm stack-size Ns = Ns Ar size
 Set the main thread's stack size to
 .Ar size .
@@ -683,9 +712,11 @@ The stack size is recorded as the size of the
 .Ar size .
 .Dv PT_GNU_STACK
 program segment.
+.Pp
 .It Cm text
 Do not allow relocations against read-only segments.
 This is the default.
+.Pp
 .It Cm wxneeded
 Create a
 .Dv PT_OPENBSD_WXNEEDED
diff --git a/lld/include/lld/Common/Strings.h b/lld/include/lld/Common/Strings.h
index ded22dd769bed..9d002bf336deb 100644
--- a/lld/include/lld/Common/Strings.h
+++ b/lld/include/lld/Common/Strings.h
@@ -18,9 +18,8 @@
 
 namespace lld {
 // Returns a demangled C++ symbol name. If Name is not a mangled
-// name, it returns Optional::None.
-llvm::Optional<std::string> demangleItanium(llvm::StringRef name);
-llvm::Optional<std::string> demangleMSVC(llvm::StringRef s);
+// name, it returns name.
+std::string demangleItanium(llvm::StringRef name);
 
 std::vector<uint8_t> parseHex(llvm::StringRef s);
 bool isValidCIdentifier(llvm::StringRef s);
diff --git a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp
index da27c7cadf96c..ab7ea7e07f23e 100644
--- a/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp
+++ b/lld/lib/ReaderWriter/MachO/MachONormalizedFileBinaryWriter.cpp
@@ -626,17 +626,19 @@ llvm::Error MachOFileLayout::writeSingleSegmentLoadCommand(uint8_t *&lc) {
                           + _file.sections.size() * sizeof(typename T::section);
   uint8_t *next = lc + seg->cmdsize;
   memset(seg->segname, 0, 16);
+  seg->flags = 0;
   seg->vmaddr = 0;
-  seg->vmsize = _file.sections.back().address
-              + _file.sections.back().content.size();
   seg->fileoff = _endOfLoadCommands;
-  seg->filesize = _sectInfo[&_file.sections.back()].fileOffset +
-                  _file.sections.back().content.size() -
-                  _sectInfo[&_file.sections.front()].fileOffset;
   seg->maxprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE;
   seg->initprot = VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE;
   seg->nsects = _file.sections.size();
-  seg->flags = 0;
+  if (seg->nsects) {
+    seg->vmsize = _file.sections.back().address
+                + _file.sections.back().content.size();
+    seg->filesize = _sectInfo[&_file.sections.back()].fileOffset +
+                    _file.sections.back().content.size() -
+                    _sectInfo[&_file.sections.front()].fileOffset;
+  }
   if (_swap)
     swapStruct(*seg);
   typename T::section *sout = reinterpret_cast<typename T::section*>
diff --git a/lld/test/COFF/undefined-symbol-dwarf.s b/lld/test/COFF/undefined-symbol-dwarf.s
new file mode 100644
index 0000000000000..7e677f88b7e00
--- /dev/null
+++ b/lld/test/COFF/undefined-symbol-dwarf.s
@@ -0,0 +1,100 @@
+# REQUIRES: x86
+# RUN: llvm-mc -triple=x86_64-windows-gnu -filetype=obj -o %t.o %s
+# RUN: not lld-link /lldmingw /out:%t.exe %t.o /entry:entry 2>&1 | FileCheck %s
+
+# CHECK: error: undefined symbol: bar()
+# CHECK-NEXT: >>> referenced by /path/to/src{{[/\\]}}undef.cpp:17
+# CHECK-NEXT: >>>               {{.*}}.o:(entry)
+# CHECK-EMPTY:
+# CHECK-NEXT: error: undefined symbol: foo()
+# CHECK-NEXT: >>> referenced by /path/to/src{{[/\\]}}undef.cpp:7
+# CHECK-NEXT: >>>               {{.*}}.o:(A::afunc())
+
+        .text
+        .file   "undef.cpp"
+        .file   1 "/path/to/src" "undef.cpp"
+        .globl  entry                   # -- Begin function entry
+entry:                                  # @entry
+.Lfunc_begin0:
+        .loc    1 14 0                  # undef.cpp:14:0
+        subq    $40, %rsp
+.Ltmp0:
+        leaq    32(%rsp), %rcx
+.Ltmp1:
+        .loc    1 16 4 prologue_end     # undef.cpp:16:4
+        callq   _ZN1A5afuncEv
+        .loc    1 17 2                  # undef.cpp:17:2
+        callq   _Z3barv
+        .loc    1 18 1                  # undef.cpp:18:1
+        addq    $40, %rsp
+        retq
+.Ltmp2:
+.Lfunc_end0:
+
+        .def     _ZN1A5afuncEv;
+        .scl    2;
+        .type   32;
+        .endef
+        .section        .text$_ZN1A5afuncEv,"xr",discard,_ZN1A5afuncEv
+        .globl  _ZN1A5afuncEv           # -- Begin function _ZN1A5afuncEv
+        .p2align        1, 0x90
+_ZN1A5afuncEv:                          # @_ZN1A5afuncEv
+.Lfunc_begin1:
+        .loc    1 6 0                   # undef.cpp:6:0
+        .loc    1 7 3 prologue_end      # undef.cpp:7:3
+        jmp     _Z3foov                 # TAILCALL
+.Ltmp3:
+.Lfunc_end1:
+
+        .section        .debug_abbrev,"dr"
+.Lsection_abbrev:
+        .byte   1                       # Abbreviation Code
+        .byte   17                      # DW_TAG_compile_unit
+        .byte   1                       # DW_CHILDREN_yes
+        .byte   37                      # DW_AT_producer
+        .byte   37                      # DW_FORM_strx1
+        .byte   19                      # DW_AT_language
+        .byte   5                       # DW_FORM_data2
+        .byte   3                       # DW_AT_name
+        .byte   37                      # DW_FORM_strx1
+        .byte   16                      # DW_AT_stmt_list
+        .byte   23                      # DW_FORM_sec_offset
+        .byte   27                      # DW_AT_comp_dir
+        .byte   37                      # DW_FORM_strx1
+        .byte   17                      # DW_AT_low_pc
+        .byte   1                       # DW_FORM_addr
+        .byte   85                      # DW_AT_ranges
+        .byte   23                      # DW_FORM_sec_offset
+        .byte   0                       # EOM(1)
+        .byte   0                       # EOM(2)
+        .byte   0                       # EOM(3)
+        .section        .debug_info,"dr"
+.Lsection_info:
+.Lcu_begin0:
+        .long   .Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+        .short  4                       # DWARF version number
+        .secrel32       .Lsection_abbrev # Offset Into Abbrev. Section
+        .byte   8                       # Address Size (in bytes)
+        .byte   1                       # Abbrev [1] 0xb:0xb0 DW_TAG_compile_unit
+        .byte   0                       # DW_AT_producer
+        .short  4                       # DW_AT_language
+        .byte   0                       # DW_AT_name
+        .secrel32       .Lline_table_start0 # DW_AT_stmt_list
+        .byte   0                       # DW_AT_comp_dir
+        .quad   0                       # DW_AT_low_pc
+        .secrel32       .Ldebug_ranges0 # DW_AT_ranges
+        .byte   0                       # End Of Children Mark
+.Ldebug_info_end0:
+        .section        .debug_ranges,"dr"
+.Ldebug_range:
+.Ldebug_ranges0:
+        .quad   .Lfunc_begin0
+        .quad   .Lfunc_end0
+        .quad   .Lfunc_begin1
+        .quad   .Lfunc_end1
+        .quad   0
+        .quad   0
+
+        .section        .debug_line,"dr"
+.Lline_table_start0:
diff --git a/lld/test/COFF/undefined-symbol-itanium-i386.s b/lld/test/COFF/undefined-symbol-itanium-i386.s
index 228f2d81c1ae5..478e943bd88f5 100644
--- a/lld/test/COFF/undefined-symbol-itanium-i386.s
+++ b/lld/test/COFF/undefined-symbol-itanium-i386.s
@@ -7,6 +7,7 @@
 # NODEMANGLE: error: undefined symbol: __Z3fooi
 # NODEMANGLE: error: undefined symbol: __Z3barPKc
 # NODEMANGLE: error: undefined symbol: __imp___Z3bazv
+# NODEMANGLE: error: undefined symbol: _Z3fooi
 
 # CHECK: error: undefined symbol: foo(int)
 # CHECK-NEXT: >>> referenced by {{.*}}.o:(_main)
@@ -17,6 +18,9 @@
 # CHECK-NEXT: >>> referenced by {{.*}}.o:(_f1)
 # CHECK-EMPTY:
 # CHECK-NEXT: error: undefined symbol: __declspec(dllimport) baz()
+# CHECK-NEXT: >>> referenced by {{.*}}.o:(_f2)
+# CHECK-EMPTY:
+# CHECK-NEXT: error: undefined symbol: _Z3fooi
 # CHECK-NEXT: >>> referenced by {{.*}}.o:(_f2)
 
         .section        .text,"xr",one_only,_main
@@ -34,3 +38,4 @@ _f1:
 .globl _f2
 _f2:
 	call	*__imp___Z3bazv
+	call	_Z3fooi
diff --git a/lld/test/COFF/undefined-symbol.s b/lld/test/COFF/undefined-symbol.s
index 5fb2511ae25d0..e3f07ab07390f 100644
--- a/lld/test/COFF/undefined-symbol.s
+++ b/lld/test/COFF/undefined-symbol.s
@@ -7,6 +7,7 @@
 # NODEMANGLE: error: undefined symbol: ?foo@@YAHXZ
 # NODEMANGLE: error: undefined symbol: ?bar@@YAHXZ
 # NODEMANGLE: error: undefined symbol: __imp_?baz@@YAHXZ
+# NODEMANGLE: error: undefined symbol: __imp_undecorated
 
 # CHECK: error: undefined symbol: int __cdecl foo(void)
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(main)
@@ -17,6 +18,9 @@
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(f1)
 # CHECK-EMPTY:
 # CHECK-NEXT: error: undefined symbol: __declspec(dllimport) int __cdecl baz(void)
+# CHECK-NEXT: >>> referenced by {{.*}}.obj:(f2)
+# CHECK-EMPTY:
+# CHECK-NEXT: error: undefined symbol: __imp_undecorated
 # CHECK-NEXT: >>> referenced by {{.*}}.obj:(f2)
 
         .section        .text,"xr",one_only,main
@@ -34,3 +38,4 @@ f1:
 .globl f2
 f2:
 	callq	*"__imp_?baz@@YAHXZ"(%rip)
+	callq	*__imp_undecorated(%rip)
diff --git a/lld/test/ELF/Inputs/copy-rel-abs.s b/lld/test/ELF/Inputs/copy-rel-abs.s
index 66d08807b9c90..30cdd6c12872f 100644
--- a/lld/test/ELF/Inputs/copy-rel-abs.s
+++ b/lld/test/ELF/Inputs/copy-rel-abs.s
@@ -10,4 +10,4 @@ bar:
 
         .weak zed
         .type zed, @object
-        zed = 0x1000
+        zed = 0x1288
diff --git a/lld/test/ELF/Inputs/copy-rel-tls.s b/lld/test/ELF/Inputs/copy-rel-tls.s
index c34b6925e809d..d9bdf8bf9b0e8 100644
--- a/lld/test/ELF/Inputs/copy-rel-tls.s
+++ b/lld/test/ELF/Inputs/copy-rel-tls.s
@@ -6,7 +6,7 @@ foo:
 
 .section .tbss,"awT",@nobits
 .global tfoo
-.skip 0x2000
+.skip 0x3358
 .type tfoo,@object
 .size tfoo, 4
 tfoo:
diff --git a/lld/test/ELF/Inputs/shared.s b/lld/test/ELF/Inputs/shared.s
index c3c22fe4b4fbc..4a386be23e227 100644
--- a/lld/test/ELF/Inputs/shared.s
+++ b/lld/test/ELF/Inputs/shared.s
@@ -1,9 +1,9 @@
 .global bar
-.type bar, @function
+.type bar, %function
 bar:
 
 .global bar2
-.type bar2, @function
+.type bar2, %function
 bar2:
 
 .global zed
diff --git a/lld/test/ELF/abs-hidden.s b/lld/test/ELF/abs-hidden.s
index 5bdd1e4fd831a..ba97e0c12a31d 100644
--- a/lld/test/ELF/abs-hidden.s
+++ b/lld/test/ELF/abs-hidden.s
@@ -8,11 +8,11 @@
 
 # CHECK: There are no relocations in this file.
 # CHECK: section '.got':
-# CHECK: 0x00001070 2a000000 00000000
+# CHECK: 0x000022b8 2a000000 00000000
 
-## .got - (.data+8) = 0xfffff068
+## .got - (.data+8) = 0xfffff0ef
 # CHECK: section '.data':
-# CHECK: 0x00002000 2a000000 00000000 68f0ffff
+# CHECK: 0x000032c0 2a000000 00000000 f0efffff
 
 .data
 .quad foo
diff --git a/lld/test/ELF/apply-dynamic-relocs.s b/lld/test/ELF/apply-dynamic-relocs.s
index 70f83324d1bee..232c0edbb9115 100644
--- a/lld/test/ELF/apply-dynamic-relocs.s
+++ b/lld/test/ELF/apply-dynamic-relocs.s
@@ -14,7 +14,7 @@
 # CHECK:        Name: .got
 # CHECK:        Address: 0x[[GOT:.*]]
 # CHECK:        SectionData (
-# APPLY-NEXT:     0000: 00200000 00000000                |
+# APPLY-NEXT:     0000: 30220000 00000000                |
 # NOAPPLY-NEXT:   0000: 00000000 00000000                |
 # CHECK-NEXT:   )
 
@@ -25,7 +25,7 @@
 # CHECK-NEXT: ]
 
 # CHECK:      Type: PT_DYNAMIC
-# CHECK-NEXT: Offset: 0x[[ADDEND]]
+# CHECK-NEXT: Offset: 0x230
 # CHECK-NEXT: VirtualAddress: 0x[[ADDEND]]
 # CHECK-NEXT: PhysicalAddress: 0x[[ADDEND]]
 
diff --git a/lld/test/ELF/arm-attributes.s b/lld/test/ELF/arm-attributes.s
index de7cdfc550b31..4249dac26f465 100644
--- a/lld/test/ELF/arm-attributes.s
+++ b/lld/test/ELF/arm-attributes.s
@@ -3,11 +3,11 @@
 // RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t2.o
 
 // RUN: ld.lld %t1.o %t2.o -o %t
-// RUN: llvm-readobj --arm-attributes %t | FileCheck %s
+// RUN: llvm-readobj --arch-specific %t | FileCheck %s
 // RUN: ld.lld %t1.o %t2.o -shared -o %t2
-// RUN: llvm-readobj --arm-attributes %t2 | FileCheck %s
+// RUN: llvm-readobj --arch-specific %t2 | FileCheck %s
 // RUN: ld.lld %t1.o %t2.o -r -o %t3
-// RUN: llvm-readobj --arm-attributes %t3 | FileCheck %s
+// RUN: llvm-readobj --arch-specific %t3 | FileCheck %s
 
 // Check that we retain only 1 SHT_ARM_ATTRIBUTES section. At present we do not
 // try and merge or use the contents of SHT_ARM_ATTRIBUTES sections. We just
diff --git a/lld/test/ELF/arm-fix-cortex-a8-blx.s b/lld/test/ELF/arm-fix-cortex-a8-blx.s
new file mode 100644
index 0000000000000..bb3417bf0c947
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-blx.s
@@ -0,0 +1,33 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: ld.lld --fix-cortex-a8 -verbose %t.o -o %t2 2>&1 | FileCheck %s
+// RUN: llvm-objdump -d --no-show-raw-insn --start-address=0x12ffa --stop-address=0x13008 %t2 | FileCheck --check-prefix=CHECK-PATCH %s
+
+/// Test that the patch can work on an unrelocated BLX. Neither clang or GCC
+/// will emit these without a relocation, but they could be produced by ELF
+/// processing tools.
+
+// CHECK: ld.lld: detected cortex-a8-657419 erratum sequence starting at 12FFE in unpatched output.
+
+ .syntax unified
+ .text
+
+ .type _start, %function
+ .balign 4096
+ .global _start
+ .arm
+_start:
+ bx lr
+ .space 4086
+ .thumb
+/// 32-bit Branch link and exchange spans 2 4KiB regions, preceded by a
+/// 32-bit non branch instruction. Expect a patch.
+ nop.w
+/// Encoding for blx _start. Use .inst.n directives to avoid a relocation.
+ .inst.n 0xf7ff
+ .inst.n 0xe800
+
+// CHECK-PATCH:         12ffa:          nop.w
+// CHECK-PATCH-NEXT:    12ffe:          blx     #4
+// CHECK-PATCH:      00013004 __CortexA8657417_12FFE:
+// CHECK-PATCH-NEXT:    13004:          b       #-4104
diff --git a/lld/test/ELF/arm-fix-cortex-a8-nopatch.s b/lld/test/ELF/arm-fix-cortex-a8-nopatch.s
new file mode 100644
index 0000000000000..97ef29f55fe65
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-nopatch.s
@@ -0,0 +1,123 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: ld.lld --fix-cortex-a8 -verbose %t.o -o %t2
+// RUN: llvm-objdump -d %t2 --start-address=0x12ffa --stop-address=0x13002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE1 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x13ffa --stop-address=0x14002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE2 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x14ffa --stop-address=0x15002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE3 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x15ffa --stop-address=0x16006 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE4 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x16ffe --stop-address=0x17002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE5 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x18000 --stop-address=0x18004 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE6 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x19002 --stop-address=0x19006 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE7 %s
+
+/// Test boundary conditions of the cortex-a8 erratum. The following cases
+/// should not trigger the Erratum
+ .syntax unified
+ .thumb
+ .text
+ .global _start
+ .balign 4096
+ .thumb_func
+_start:
+ nop.w
+ .space 4086
+ .thumb_func
+target:
+/// 32-bit branch spans 2 4KiB regions, preceded by a 32-bit branch so no patch
+/// expected.
+ b.w target
+ b.w target
+
+// CALLSITE1:      00012ffa target:
+// CALLSITE1-NEXT:    12ffa:            b.w     #-4
+// CALLSITE1-NEXT:    12ffe:            b.w     #-8
+
+ .space 4088
+ .type target2, %function
+target2:
+/// 32-bit Branch and link spans 2 4KiB regions, preceded by a 16-bit
+/// instruction so no patch expected.
+ nop
+ nop
+ bl target2
+
+// CALLSITE2:      00013ffa target2:
+// CALLSITE2-NEXT:    13ffa:            nop
+// CALLSITE2-NEXT:    13ffc:            nop
+// CALLSITE2-NEXT:    13ffe:            bl      #-8
+
+ .space 4088
+ .type target3, %function
+target3:
+/// 32-bit conditional branch spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, branch is backwards but outside 4KiB region. So
+/// expect no patch.
+ nop.w
+ beq.w target2
+
+// CALLSITE3:      00014ffa target3:
+// CALLSITE3-NEXT:    14ffa:            nop.w
+// CALLSITE3-NEXT:    14ffe:            beq.w   #-4104
+
+ .space 4088
+ .type source4, %function
+source4:
+/// 32-bit conditional branch spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, branch is forwards to 2nd region so expect no patch.
+ nop.w
+ beq.w target4
+ .thumb_func
+target4:
+ nop.w
+
+// CALLSITE4:      00015ffa source4:
+// CALLSITE4-NEXT:    15ffa:            nop.w
+// CALLSITE4-NEXT:    15ffe:            beq.w   #0
+// CALLSITE4:      00016002 target4:
+// CALLSITE4-NEXT:    16002:            nop.w
+
+ .space 4084
+ .type target5, %function
+
+target5:
+/// 32-bit conditional branch spans 2 4KiB regions, preceded by the encoding of
+/// a 32-bit thumb instruction, but in ARM state (illegal instruction), we
+/// should not decode and match it as Thumb, expect no patch.
+ .arm
+ .inst 0x800f3af /// nop.w encoding in Thumb
+ .thumb
+ .thumb_func
+source5:
+ beq.w target5
+
+// CALLSITE5:      00016ffe source5:
+// CALLSITE5-NEXT:    16ffe:            beq.w   #-8
+
+/// Edge case where two word sequence starts at offset 0xffc, check that
+/// we don't match. In this case the branch will be completely in the 2nd
+/// region and the branch will target the second region. This will pass a
+/// branch destination in the same region test, but not the branch must have
+/// and address of the form xxxxxffe.
+ .space 4090
+ .type target6, %function
+ nop.w
+/// Make sure target of branch is in the same 4KiB region as the branch.
+target6:
+ bl target6
+
+// CALLSITE6:      00018000 target6:
+// CALLSITE6-NEXT:    18000:            bl      #-4
+
+/// Edge case where two word sequence starts at offset 0xffe, check that
+/// we don't match. In this case the branch will be completely in the 2nd
+/// region and the branch will target the second region. This will pass a
+/// branch destination in the same region test, but not the branch must have
+/// and address of the form xxxxxffe.
+ .space 4090
+ .type target7, %function
+ nop.w
+/// Make sure target of branch is in the same 4KiB region as the branch.
+target7:
+ bl target7
+
+// CALLSITE7:      00019002 target7:
+// CALLSITE7:         19002:            bl      #-4
diff --git a/lld/test/ELF/arm-fix-cortex-a8-plt.s b/lld/test/ELF/arm-fix-cortex-a8-plt.s
new file mode 100644
index 0000000000000..ba7c3f94b1596
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-plt.s
@@ -0,0 +1,39 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: echo "SECTIONS { \
+// RUN:       .plt  0x2000 : { *(.plt) *(.plt.*) } \
+// RUN:       .text : { *(.text) } \
+// RUN:       }" > %t.script
+
+// RUN: ld.lld --script %t.script --fix-cortex-a8 --shared -verbose %t.o -o %t2
+// RUN: llvm-objdump -d --start-address=0x2020 --stop-address=0x202c --no-show-raw-insn %t2 | FileCheck --check-prefix=CHECK-PLT %s
+// RUN: llvm-objdump -d --start-address=0x2ffa --stop-address=0x3008 --no-show-raw-insn %t2 | FileCheck %s
+
+/// If we patch a branch instruction that is indirected via the PLT then we
+/// must make sure the patch goes via the PLT
+
+// CHECK-PLT:          2020:            add     r12, pc, #0, #12
+// CHECK-PLT-NEXT:     2024:            add     r12, r12, #4096
+// CHECK-PLT-NEXT:     2028:            ldr     pc, [r12, #68]!
+
+ .syntax unified
+ .thumb
+
+ .global external
+ .type external, %function
+
+ .text
+ .balign 2048
+
+ .space 2042
+ .global source
+ .thumb_func
+source:
+ nop.w
+ bl external
+
+// CHECK:      00002ffa source:
+// CHECK-NEXT:     2ffa:        nop.w
+// CHECK-NEXT:     2ffe:        blx     #4
+// CHECK:      00003004 __CortexA8657417_2FFE:
+// CHECK-NEXT:     3004:        b       #-4076
diff --git a/lld/test/ELF/arm-fix-cortex-a8-recognize.s b/lld/test/ELF/arm-fix-cortex-a8-recognize.s
new file mode 100644
index 0000000000000..9971f15d884a7
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-recognize.s
@@ -0,0 +1,201 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: ld.lld --fix-cortex-a8 -verbose %t.o -o %t2 2>&1 | FileCheck %s
+// RUN: llvm-objdump -d %t2 --start-address=0x1a004 --stop-address=0x1a024 --no-show-raw-insn | FileCheck --check-prefix=CHECK-PATCHES %s
+// RUN: llvm-objdump -d %t2 --start-address=0x12ffa --stop-address=0x13002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE1 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x13ffa --stop-address=0x14002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE2 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x14ffa --stop-address=0x15002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE3 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x15ff4 --stop-address=0x16002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE4 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x16ffa --stop-address=0x17002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE5 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x17ffa --stop-address=0x18002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE6 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x18ffa --stop-address=0x19002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE7 %s
+// RUN: llvm-objdump -d %t2 --start-address=0x19ff4 --stop-address=0x1a002 --no-show-raw-insn | FileCheck --check-prefix=CALLSITE8 %s
+
+// CHECK:      ld.lld: detected cortex-a8-657419 erratum sequence starting at 12FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 13FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 14FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 15FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 16FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 17FFE in unpatched output.
+// CHECK-NEXT: ld.lld: detected cortex-a8-657419 erratum sequence starting at 18FFE in unpatched output.
+
+/// Basic tests for the -fix-cortex-a8 erratum fix. The full details of the
+/// erratum and the patch are in ARMA8ErrataFix.cpp . The test creates an
+/// instance of the erratum every 4KiB (32-bit non-branch, followed by 32-bit
+/// branch instruction, where the branch instruction spans two 4 KiB regions,
+/// and the branch destination is in the first 4KiB region.
+///
+/// Test each 32-bit branch b.w, bcc.w, bl, blx. For b.w, bcc.w, and bl we
+/// check the relocated and non-relocated forms. The blx instruction
+/// always has a relocation in assembler.
+ .syntax unified
+ .thumb
+ .text
+ .global _start
+ .type _start, %function
+ .balign 4096
+ .thumb_func
+_start:
+ nop.w
+ .space 4086
+ .thumb_func
+ .global target
+ .type target, %function
+target:
+/// 32-bit Branch spans 2 4KiB regions, preceded by a 32-bit non branch
+/// instruction, expect a patch.
+ nop.w
+ b.w target
+
+// CALLSITE1:      00012ffa target:
+// CALLSITE1-NEXT:    12ffa:            nop.w
+// CALLSITE1-NEXT:    12ffe:            b.w     #28674
+
+ .space 4088
+ .type target2, %function
+ .local target2
+target2:
+/// 32-bit Branch and link spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, expect a patch.
+ nop.w
+ bl target2
+
+// CALLSITE2:      00013ffa target2:
+// CALLSITE2-NEXT:    13ffa:            nop.w
+// CALLSITE2-NEXT:    13ffe:            bl      #24582
+
+ .space 4088
+ .type target3, %function
+ .local target3
+target3:
+/// 32-bit conditional branch spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, expect a patch.
+ nop.w
+ beq.w target3
+
+// CALLSITE3:      00014ffa target3:
+// CALLSITE3-NEXT:    14ffa:            nop.w
+// CALLSITE3-NEXT:    14ffe:            beq.w   #20490
+
+ .space 4082
+ .type target4, %function
+ .local target4
+ .arm
+target4:
+ bx lr
+ .space 2
+ .thumb
+/// 32-bit Branch link and exchange spans 2 4KiB regions, preceded by a
+/// 32-bit non branch instruction, blx always goes via relocation. Expect
+/// a patch.
+ nop.w
+ blx target4
+
+/// Target = 0x19010 __CortexA8657417_15FFE
+// CALLSITE4:      00015ff4 target4:
+// CALLSITE4-NEXT:    15ff4:            bx      lr
+// CALLSITE4:         15ff8:    00 00           .short  0x0000
+// CALLSITE4:         15ffa:            nop.w
+// CALLSITE4-NEXT:    15ffe:            blx     #16400
+
+/// Separate sections for source and destination of branches to force
+/// a relocation.
+ .section .text.0, "ax", %progbits
+ .balign 2
+ .global target5
+ .type target5, %function
+target5:
+ nop.w
+ .section .text.1, "ax", %progbits
+ .space 4084
+/// 32-bit branch spans 2 4KiB regions, preceded by a 32-bit non branch
+/// instruction, expect a patch. Branch to global symbol so goes via a
+/// relocation.
+ nop.w
+ b.w target5
+
+/// Target = 0x19014 __CortexA8657417_16FFE
+// CALLSITE5:         16ffa:            nop.w
+// CALLSITE5-NEXT:    16ffe:            b.w     #12306
+
+ .section .text.2, "ax", %progbits
+ .balign 2
+ .global target6
+ .type target6, %function
+target6:
+ nop.w
+ .section .text.3, "ax", %progbits
+ .space 4084
+/// 32-bit branch and link spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, expect a patch. Branch to global symbol so
+/// goes via a relocation.
+ nop.w
+ bl target6
+
+/// Target = 0x19018 __CortexA8657417_17FFE
+// CALLSITE6:         17ffa:            nop.w
+// CALLSITE6-NEXT:    17ffe:            bl      #8214
+
+ .section .text.4, "ax", %progbits
+ .global target7
+ .type target7, %function
+target7:
+ nop.w
+ .section .text.5, "ax", %progbits
+ .space 4084
+/// 32-bit conditional branch spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, expect a patch. Branch to global symbol so
+/// goes via a relocation.
+ nop.w
+ bne.w target7
+
+// CALLSITE7:         18ffa:            nop.w
+// CALLSITE7-NEXT:    18ffe:            bne.w   #4122
+
+ .section .text.6, "ax", %progbits
+ .space 4082
+ .arm
+ .global target8
+ .type target8, %function
+target8:
+ bx lr
+
+ .section .text.7, "ax", %progbits
+ .space 2
+ .thumb
+/// 32-bit Branch link spans 2 4KiB regions, preceded by a 32-bit non branch
+/// instruction, expect a patch. The target of the BL is in ARM state so we
+/// expect it to be turned into a BLX. The patch must be in ARM state to
+/// avoid a state change thunk.
+ nop.w
+ bl target8
+
+// CALLSITE8:      00019ff4 target8:
+// CALLSITE8-NEXT:    19ff4:            bx      lr
+// CALLSITE8:         19ff8:    00 00           .short  0x0000
+// CALLSITE8:         19ffa:            nop.w
+// CALLSITE8-NEXT:    19ffe:            blx     #32
+
+// CHECK-PATCHES: 0001a004 __CortexA8657417_12FFE:
+// CHECK-PATCHES-NEXT:    1a004:        b.w     #-28686
+
+// CHECK-PATCHES:      0001a008 __CortexA8657417_13FFE:
+// CHECK-PATCHES-NEXT:    1a008:        b.w     #-24594
+
+// CHECK-PATCHES:      0001a00c __CortexA8657417_14FFE:
+// CHECK-PATCHES-NEXT:    1a00c:        b.w     #-20502
+
+// CHECK-PATCHES:      0001a010 __CortexA8657417_15FFE:
+// CHECK-PATCHES-NEXT:    1a010:        b       #-16420
+
+// CHECK-PATCHES:      0001a014 __CortexA8657417_16FFE:
+// CHECK-PATCHES-NEXT:    1a014:        b.w     #-16406
+
+// CHECK-PATCHES:      0001a018 __CortexA8657417_17FFE:
+// CHECK-PATCHES-NEXT:    1a018:        b.w     #-12314
+
+// CHECK-PATCHES:      0001a01c __CortexA8657417_18FFE:
+// CHECK-PATCHES-NEXT:    1a01c:        b.w     #-8222
+
+// CHECK-PATCHES:      0001a020 __CortexA8657417_19FFE:
+// CHECK-PATCHES-NEXT:    1a020:        b       #-52
diff --git a/lld/test/ELF/arm-fix-cortex-a8-thunk.s b/lld/test/ELF/arm-fix-cortex-a8-thunk.s
new file mode 100644
index 0000000000000..e7e07c33c8eea
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-thunk.s
@@ -0,0 +1,69 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: echo "SECTIONS { \
+// RUN:          .text0 0x011006 : { *(.text.00) } \
+// RUN:          .text1 0x110000 : { *(.text.01) *(.text.02) *(.text.03) \
+// RUN:                             *(.text.04) } \
+// RUN:          .text2 0x210000 : { *(.text.05) } } " > %t.script
+// RUN: ld.lld --script %t.script --fix-cortex-a8 --shared -verbose %t.o -o %t2 2>&1
+// RUN: llvm-objdump -d --no-show-raw-insn --start-address=0x110000 --stop-address=0x110010 %t2 | FileCheck --check-prefix=THUNK %s
+// RUN: llvm-objdump -d --no-show-raw-insn --start-address=0x110ffa --stop-address=0x111008 %t2 | FileCheck --check-prefix=PATCH %s
+// RUN: llvm-objdump -d --no-show-raw-insn --start-address=0x111008 --stop-address=0x111010 %t2 | FileCheck --check-prefix=THUNK2 %s
+
+/// Test cases for Cortex-a8 Erratum 657417 that involve interactions with
+/// range extension thunks. Both erratum fixes and range extension thunks need
+/// precise information and after creation alter address information.
+ .thumb
+
+ .section .text.00, "ax", %progbits
+ .thumb_func
+early:
+ bx lr
+
+ .section .text.01, "ax", %progbits
+ .balign 4096
+ .globl _start
+ .type _start, %function
+_start:
+  beq.w far_away
+/// Thunk to far_away and state change needed, size 12-bytes goes here.
+// THUNK:      00110000 _start:
+// THUNK-NEXT:   110000:        beq.w   #0 <__ThumbV7PILongThunk_far_away+0x4>
+// THUNK:      00110004 __ThumbV7PILongThunk_far_away:
+// THUNK-NEXT:   110004:        movw    r12, #65524
+// THUNK-NEXT:   110008:        movt    r12, #15
+// THUNK-NEXT:   11000c:        add     r12, pc
+// THUNK-NEXT:   11000e:        bx      r12
+
+ .section .text.02, "ax", %progbits
+ .space 4096 - 22
+
+ .section .text.03, "ax", %progbits
+ .thumb_func
+target:
+/// After thunk is added this branch will line up across 2 4 KiB regions
+/// and will trigger a patch.
+ nop.w
+ bl target
+
+/// Expect erratum patch inserted here
+// PATCH:      00110ffa target:
+// PATCH-NEXT:   110ffa:        nop.w
+// PATCH-NEXT:   110ffe:        bl      #2
+// PATCH:      00111004 __CortexA8657417_110FFE:
+// PATCH-NEXT:   111004:        b.w     #-14
+
+// THUNK2: 00111008 __ThumbV7PILongThunk_early:
+// THUNK2-NEXT:   111008:       b.w     #-1048582
+ .section .text.04, "ax", %progbits
+/// The erratum patch will push this branch out of range, so another
+/// range extension thunk will be needed.
+ beq.w early
+// THUNK2-NEXT  11100c:         beq.w   #-8
+/// Expect range extension thunk here.
+ .section .text.05, "ax", %progbits
+ .arm
+ nop
+ .type far_away, %function
+far_away:
+  bx lr
diff --git a/lld/test/ELF/arm-fix-cortex-a8-toolarge.s b/lld/test/ELF/arm-fix-cortex-a8-toolarge.s
new file mode 100644
index 0000000000000..3937fc3d0eb85
--- /dev/null
+++ b/lld/test/ELF/arm-fix-cortex-a8-toolarge.s
@@ -0,0 +1,45 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple=armv7a-linux-gnueabihf --arm-add-build-attributes %s -o %t.o
+// RUN: ld.lld --fix-cortex-a8 -verbose %t.o -o /dev/null 2>&1 | FileCheck %s
+/// Test that we warn, but don't attempt to patch when it is impossible to
+/// redirect the branch as the Section is too large.
+
+// CHECK: skipping cortex-a8 657417 erratum sequence, section .text is too large to patch
+// CHECK: skipping cortex-a8 657417 erratum sequence, section .text.02 is too large to patch
+
+ .syntax unified
+ .thumb
+/// Case 1: 1 MiB conditional branch range without relocation.
+ .text
+ .global _start
+ .type _start, %function
+ .balign 4096
+ .thumb_func
+_start:
+ nop.w
+ .space 4086
+ .thumb_func
+ .global target
+ .type target, %function
+target:
+/// 32-bit Branch spans 2 4KiB regions, preceded by a 32-bit non branch
+/// instruction, a patch will be attempted. Unfortunately the branch
+/// cannot reach outside the section so we have to abort the patch.
+ nop.w
+ beq.w target
+ .space 1024 * 1024
+
+/// Case 2: 16 MiB
+ .section .text.01, "ax", %progbits
+ .balign 4096
+  .space 4090
+ .global target2
+ .thumb_func
+target2:
+ .section .text.02, "ax", %progbits
+/// 32-bit Branch and link spans 2 4KiB regions, preceded by a 32-bit
+/// non branch instruction, a patch will be be attempted. Unfortunately the
+/// the BL cannot reach outside the section so we have to abort the patch.
+ nop.w
+ bl target2
+ .space 16 * 1024 * 1024
diff --git a/lld/test/ELF/avoid-empty-program-headers.s b/lld/test/ELF/avoid-empty-program-headers.s
index 08aecb53b1d8f..4714ea9edcaa9 100644
--- a/lld/test/ELF/avoid-empty-program-headers.s
+++ b/lld/test/ELF/avoid-empty-program-headers.s
@@ -39,9 +39,9 @@ _start:
 // CHECK-NEXT:   }
 // CHECK-NEXT:   ProgramHeader {
 // CHECK-NEXT:     Type: PT_LOAD (0x1)
-// CHECK-NEXT:     Offset: 0x1000
-// CHECK-NEXT:     VirtualAddress: 0x201000
-// CHECK-NEXT:     PhysicalAddress: 0x201000
+// CHECK-NEXT:     Offset: 0x158
+// CHECK-NEXT:     VirtualAddress: 0x201158
+// CHECK-NEXT:     PhysicalAddress: 0x201158
 // CHECK-NEXT:     FileSize: 1
 // CHECK-NEXT:     MemSize: 1
 // CHECK-NEXT:     Flags [ (0x5)
@@ -52,9 +52,9 @@ _start:
 // CHECK-NEXT:   }
 // CHECK-NEXT:   ProgramHeader {
 // CHECK-NEXT:     Type: PT_TLS (0x7)
-// CHECK-NEXT:     Offset: 0x1001
-// CHECK-NEXT:     VirtualAddress: 0x201001
-// CHECK-NEXT:     PhysicalAddress: 0x201001
+// CHECK-NEXT:     Offset: 0x159
+// CHECK-NEXT:     VirtualAddress: 0x201159
+// CHECK-NEXT:     PhysicalAddress: 0x201159
 // CHECK-NEXT:     FileSize: 0
 // CHECK-NEXT:     MemSize: 4
 // CHECK-NEXT:     Flags [ (0x4)
diff --git a/lld/test/ELF/basic-aarch64.s b/lld/test/ELF/basic-aarch64.s
index 9882f9d0e65ac..2e3e5da45f17a 100644
--- a/lld/test/ELF/basic-aarch64.s
+++ b/lld/test/ELF/basic-aarch64.s
@@ -159,8 +159,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_PHDR (0x6)
 # CHECK-NEXT:     Offset: 0x40
-# CHECK-NEXT:     VirtualAddress: 0x210040
-# CHECK-NEXT:     PhysicalAddress: 0x210040
+# CHECK-NEXT:     VirtualAddress: 0x200040
+# CHECK-NEXT:     PhysicalAddress: 0x200040
 # CHECK-NEXT:     FileSize: 224
 # CHECK-NEXT:     MemSize: 224
 # CHECK-NEXT:     Flags [ (0x4)
@@ -171,8 +171,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD (0x1)
 # CHECK-NEXT:     Offset: 0x0
-# CHECK-NEXT:     VirtualAddress: 0x210000
-# CHECK-NEXT:     PhysicalAddress: 0x210000
+# CHECK-NEXT:     VirtualAddress: 0x200000
+# CHECK-NEXT:     PhysicalAddress: 0x200000
 # CHECK-NEXT:     FileSize: 288
 # CHECK-NEXT:     MemSize: 288
 # CHECK-NEXT:     Flags [
diff --git a/lld/test/ELF/basic-i386.s b/lld/test/ELF/basic-i386.s
index b0f2cda9279f8..24d4ff451826f 100644
--- a/lld/test/ELF/basic-i386.s
+++ b/lld/test/ELF/basic-i386.s
@@ -129,8 +129,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_PHDR (0x6)
 # CHECK-NEXT:     Offset: 0x34
-# CHECK-NEXT:     VirtualAddress: 0x401034
-# CHECK-NEXT:     PhysicalAddress: 0x401034
+# CHECK-NEXT:     VirtualAddress: 0x400034
+# CHECK-NEXT:     PhysicalAddress: 0x400034
 # CHECK-NEXT:     FileSize: 128
 # CHECK-NEXT:     MemSize: 128
 # CHECK-NEXT:     Flags [ (0x4)
@@ -141,8 +141,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD (0x1)
 # CHECK-NEXT:     Offset: 0x0
-# CHECK-NEXT:     VirtualAddress: 0x401000
-# CHECK-NEXT:     PhysicalAddress: 0x401000
+# CHECK-NEXT:     VirtualAddress: 0x400000
+# CHECK-NEXT:     PhysicalAddress: 0x400000
 # CHECK-NEXT:     FileSize: 180
 # CHECK-NEXT:     MemSize: 180
 # CHECK-NEXT:     Flags [
diff --git a/lld/test/ELF/basic-ppc.s b/lld/test/ELF/basic-ppc.s
index 567b55c13a106..ff753c8cebadd 100644
--- a/lld/test/ELF/basic-ppc.s
+++ b/lld/test/ELF/basic-ppc.s
@@ -143,8 +143,8 @@
 // CHECK-NEXT:   ProgramHeader {
 // CHECK-NEXT:     Type: PT_PHDR (0x6)
 // CHECK-NEXT:     Offset: 0x34
-// CHECK-NEXT:     VirtualAddress: 0x10010034
-// CHECK-NEXT:     PhysicalAddress: 0x10010034
+// CHECK-NEXT:     VirtualAddress: 0x10000034
+// CHECK-NEXT:     PhysicalAddress: 0x10000034
 // CHECK-NEXT:     FileSize: 128
 // CHECK-NEXT:     MemSize: 128
 // CHECK-NEXT:     Flags [ (0x4)
@@ -155,8 +155,8 @@
 // CHECK-NEXT:   ProgramHeader {
 // CHECK-NEXT:     Type: PT_LOAD (0x1)
 // CHECK-NEXT:     Offset: 0x0
-// CHECK-NEXT:     VirtualAddress: 0x10010000
-// CHECK-NEXT:     PhysicalAddress: 0x10010000
+// CHECK-NEXT:     VirtualAddress: 0x10000000
+// CHECK-NEXT:     PhysicalAddress: 0x10000000
 // CHECK-NEXT:     FileSize: 180
 // CHECK-NEXT:     MemSize: 180
 // CHECK-NEXT:     Flags [ (0x4)
diff --git a/lld/test/ELF/basic-sparcv9.s b/lld/test/ELF/basic-sparcv9.s
index 820dba556f5a1..43f3670ea6cef 100644
--- a/lld/test/ELF/basic-sparcv9.s
+++ b/lld/test/ELF/basic-sparcv9.s
@@ -150,8 +150,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_PHDR (0x6)
 # CHECK-NEXT:     Offset: 0x40
-# CHECK-NEXT:     VirtualAddress: 0x200040
-# CHECK-NEXT:     PhysicalAddress: 0x200040
+# CHECK-NEXT:     VirtualAddress: 0x100040
+# CHECK-NEXT:     PhysicalAddress: 0x100040
 # CHECK-NEXT:     FileSize: 224
 # CHECK-NEXT:     MemSize: 224
 # CHECK-NEXT:     Flags [ (0x4)
@@ -162,8 +162,8 @@ _start:
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD (0x1)
 # CHECK-NEXT:     Offset: 0x0
-# CHECK-NEXT:     VirtualAddress: 0x200000
-# CHECK-NEXT:     PhysicalAddress: 0x200000
+# CHECK-NEXT:     VirtualAddress: 0x100000
+# CHECK-NEXT:     PhysicalAddress: 0x100000
 # CHECK-NEXT:     FileSize: 288
 # CHECK-NEXT:     MemSize: 288
 # CHECK-NEXT:     Flags [
diff --git a/lld/test/ELF/basic.s b/lld/test/ELF/basic.s
index caab27d39951c..1343ab0febd96 100644
--- a/lld/test/ELF/basic.s
+++ b/lld/test/ELF/basic.s
@@ -28,7 +28,7 @@ _start:
 # CHECK-NEXT:   Version: 1
 # CHECK-NEXT:   Entry: [[ENTRY:0x[0-9A-F]+]]
 # CHECK-NEXT:   ProgramHeaderOffset: 0x40
-# CHECK-NEXT:   SectionHeaderOffset: 0x1080
+# CHECK-NEXT:   SectionHeaderOffset: 0x1A0
 # CHECK-NEXT:   Flags [ (0x0)
 # CHECK-NEXT:   ]
 # CHECK-NEXT:   HeaderSize: 64
@@ -61,8 +61,8 @@ _start:
 # CHECK-NEXT:       SHF_ALLOC (0x2)
 # CHECK-NEXT:       SHF_EXECINSTR (0x4)
 # CHECK-NEXT:     ]
-# CHECK-NEXT:     Address: 0x201000
-# CHECK-NEXT:     Offset: 0x1000
+# CHECK-NEXT:     Address: 0x201120
+# CHECK-NEXT:     Offset: 0x120
 # CHECK-NEXT:     Size: 16
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
@@ -78,7 +78,7 @@ _start:
 # CHECK-NEXT:       SHF_STRINGS (0x20)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x1010
+# CHECK-NEXT:     Offset: 0x130
 # CHECK-NEXT:     Size: 8
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
@@ -92,7 +92,7 @@ _start:
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x1018
+# CHECK-NEXT:     Offset: 0x138
 # CHECK-NEXT:     Size: 48
 # CHECK-NEXT:     Link: 5
 # CHECK-NEXT:     Info: 1
@@ -106,7 +106,7 @@ _start:
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x1048
+# CHECK-NEXT:     Offset: 0x168
 # CHECK-NEXT:     Size: 42
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
@@ -120,7 +120,7 @@ _start:
 # CHECK-NEXT:     Flags [ (0x0)
 # CHECK-NEXT:     ]
 # CHECK-NEXT:     Address: 0x0
-# CHECK-NEXT:     Offset: 0x1072
+# CHECK-NEXT:     Offset: 0x192
 # CHECK-NEXT:     Size: 8
 # CHECK-NEXT:     Link: 0
 # CHECK-NEXT:     Info: 0
@@ -175,9 +175,9 @@ _start:
 # CHECK-NEXT:   }
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD (0x1)
-# CHECK-NEXT:     Offset: 0x1000
-# CHECK-NEXT:     VirtualAddress: 0x201000
-# CHECK-NEXT:     PhysicalAddress: 0x201000
+# CHECK-NEXT:     Offset: 0x120
+# CHECK-NEXT:     VirtualAddress: 0x201120
+# CHECK-NEXT:     PhysicalAddress: 0x201120
 # CHECK-NEXT:     FileSize: 16
 # CHECK-NEXT:     MemSize: 16
 # CHECK-NEXT:     Flags [ (0x5)
diff --git a/lld/test/ELF/build-id.s b/lld/test/ELF/build-id.s
index 5e33f6fe4645b..2e555805fa3e0 100644
--- a/lld/test/ELF/build-id.s
+++ b/lld/test/ELF/build-id.s
@@ -65,15 +65,15 @@ _start:
 # DEFAULT:      Contents of section .note.test:
 # DEFAULT:      Contents of section .note.gnu.build-id:
 # DEFAULT-NEXT: 04000000 08000000 03000000 474e5500  ............GNU.
-# DEFAULT-NEXT: 605e19a6 30469e00
+# DEFAULT-NEXT: 7e8ddeff 3ed41fa3
 
 # MD5:      Contents of section .note.gnu.build-id:
 # MD5-NEXT: 04000000 10000000 03000000 474e5500  ............GNU.
-# MD5-NEXT: adbf65c5 42b4a428 184fd7c9 099cdc29
+# MD5-NEXT: 7b00fd9e 054ceb4b 06f64d0e 482cb476
 
 # SHA1:      Contents of section .note.gnu.build-id:
 # SHA1-NEXT: 04000000 14000000 03000000 474e5500  ............GNU.
-# SHA1-NEXT: fe148fd4 1add2878 6b298b61 5880148b
+# SHA1-NEXT: 221a99da dd1d2bf3 05e48a91 dde8a0cb
 
 # UUID:      Contents of section .note.gnu.build-id:
 # UUID-NEXT: 04000000 10000000 03000000 474e5500  ............GNU.
diff --git a/lld/test/ELF/cgprofile-bad-clusters.s b/lld/test/ELF/cgprofile-bad-clusters.s
index a981e37209ae5..da159d2e166ed 100644
--- a/lld/test/ELF/cgprofile-bad-clusters.s
+++ b/lld/test/ELF/cgprofile-bad-clusters.s
@@ -51,20 +51,20 @@ I:
     .fill 13, 1, 0
 
 # CHECK:          Name: B
-# CHECK-NEXT:     Value: 0x201011
+# CHECK-NEXT:     Value: 0x201131
 # CHECK:          Name: C
-# CHECK-NEXT:     Value: 0x20100F
+# CHECK-NEXT:     Value: 0x20112F
 # CHECK:          Name: D
-# CHECK-NEXT:     Value: 0x2013F9
+# CHECK-NEXT:     Value: 0x201519
 # CHECK:          Name: E
-# CHECK-NEXT:     Value: 0x201010
+# CHECK-NEXT:     Value: 0x201130
 # CHECK:          Name: F
-# CHECK-NEXT:     Value: 0x2017E1
+# CHECK-NEXT:     Value: 0x201901
 # CHECK:          Name: G
-# CHECK-NEXT:     Value: 0x3017E0
+# CHECK-NEXT:     Value: 0x301900
 # CHECK:          Name: H
-# CHECK-NEXT:     Value: 0x201000
+# CHECK-NEXT:     Value: 0x201120
 # CHECK:          Name: I
-# CHECK-NEXT:     Value: 0x201001
+# CHECK-NEXT:     Value: 0x201121
 # CHECK:          Name: A
-# CHECK-NEXT:     Value: 0x20100E
+# CHECK-NEXT:     Value: 0x20112E
diff --git a/lld/test/ELF/cgprofile-icf.s b/lld/test/ELF/cgprofile-icf.s
index ce6cfcf6546c4..c27a7b2566db0 100644
--- a/lld/test/ELF/cgprofile-icf.s
+++ b/lld/test/ELF/cgprofile-icf.s
@@ -35,19 +35,19 @@ A:
     retq
 
 # CHECK:          Name: A
-# CHECK-NEXT:     Value: 0x201000
+# CHECK-NEXT:     Value: 0x201120
 # CHECK:          Name: B
-# CHECK-NEXT:     Value: 0x201010
+# CHECK-NEXT:     Value: 0x201130
 # CHECK:          Name: C
-# CHECK-NEXT:     Value: 0x201008
+# CHECK-NEXT:     Value: 0x201128
 # CHECK:          Name: D
-# CHECK-NEXT:     Value: 0x201008
+# CHECK-NEXT:     Value: 0x201128
 
 # NOICF:          Name: A
-# NOICF-NEXT:     Value: 0x201000
+# NOICF-NEXT:     Value: 0x201120
 # NOICF:          Name: B
-# NOICF-NEXT:     Value: 0x201008
+# NOICF-NEXT:     Value: 0x201128
 # NOICF:          Name: C
-# NOICF-NEXT:     Value: 0x201010
+# NOICF-NEXT:     Value: 0x201130
 # NOICF:          Name: D
-# NOICF-NEXT:     Value: 0x201018
+# NOICF-NEXT:     Value: 0x201138
diff --git a/lld/test/ELF/cgprofile-obj.s b/lld/test/ELF/cgprofile-obj.s
index 4b7f465328cc0..57a4967ed7d54 100644
--- a/lld/test/ELF/cgprofile-obj.s
+++ b/lld/test/ELF/cgprofile-obj.s
@@ -33,12 +33,12 @@ Aa:
     .cg_profile B, C, 30
     .cg_profile C, D, 90
 
-# CHECK: 0000000000201003 t D
-# CHECK: 0000000000201000 T A
-# CHECK: 0000000000201001 T B
-# CHECK: 0000000000201002 T C
-
-# NO-CG: 0000000000201000 t D
-# NO-CG: 0000000000201003 T A
-# NO-CG: 0000000000201002 T B
-# NO-CG: 0000000000201001 T C
+# CHECK: 0000000000201123 t D
+# CHECK: 0000000000201120 T A
+# CHECK: 0000000000201121 T B
+# CHECK: 0000000000201122 T C
+
+# NO-CG: 0000000000201120 t D
+# NO-CG: 0000000000201123 T A
+# NO-CG: 0000000000201122 T B
+# NO-CG: 0000000000201121 T C
diff --git a/lld/test/ELF/cgprofile-reproduce.s b/lld/test/ELF/cgprofile-reproduce.s
index 778981496b489..b9cb269e4580d 100644
--- a/lld/test/ELF/cgprofile-reproduce.s
+++ b/lld/test/ELF/cgprofile-reproduce.s
@@ -10,13 +10,13 @@
 # RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 # CHECK:      Name: A
-# CHECK-NEXT: Value: 0x201003
+# CHECK-NEXT: Value: 0x201123
 # CHECK:      Name: B
-# CHECK-NEXT: Value: 0x201000
+# CHECK-NEXT: Value: 0x201120
 # CHECK:      Name: C
-# CHECK-NEXT: Value: 0x201001
+# CHECK-NEXT: Value: 0x201121
 # CHECK:      Name: D
-# CHECK-NEXT: Value: 0x201002
+# CHECK-NEXT: Value: 0x201122
 
 .section    .text.A,"ax",@progbits
 .globl  A
diff --git a/lld/test/ELF/cgprofile-txt.s b/lld/test/ELF/cgprofile-txt.s
index 650a4ef421542..2bc18376caebe 100644
--- a/lld/test/ELF/cgprofile-txt.s
+++ b/lld/test/ELF/cgprofile-txt.s
@@ -135,51 +135,51 @@ TooManyPreds10:
     retq
 
 # CHECK:          Name: D
-# CHECK-NEXT:     Value: 0x201003
+# CHECK-NEXT:     Value: 0x201123
 # CHECK:          Name: TooManyPreds
-# CHECK-NEXT:     Value: 0x201004
+# CHECK-NEXT:     Value: 0x201124
 # CHECK:          Name: TooManyPreds10
-# CHECK-NEXT:     Value: 0x201018
+# CHECK-NEXT:     Value: 0x201138
 # CHECK:          Name: A
-# CHECK-NEXT:     Value: 0x201000
+# CHECK-NEXT:     Value: 0x201120
 # CHECK:          Name: B
-# CHECK-NEXT:     Value: 0x201001
+# CHECK-NEXT:     Value: 0x201121
 # CHECK:          Name: C
-# CHECK-NEXT:     Value: 0x201002
+# CHECK-NEXT:     Value: 0x201122
 # CHECK:          Name: GB
-# CHECK-NEXT:     Value: 0x20101F
+# CHECK-NEXT:     Value: 0x20113F
 # CHECK:          Name: PP
-# CHECK-NEXT:     Value: 0x20101C
+# CHECK-NEXT:     Value: 0x20113C
 # CHECK:          Name: QC
-# CHECK-NEXT:     Value: 0x20101E
+# CHECK-NEXT:     Value: 0x20113E
 # CHECK:          Name: TS
-# CHECK-NEXT:     Value: 0x20101D
+# CHECK-NEXT:     Value: 0x20113D
 # CHECK:          Name: _init
-# CHECK-NEXT:     Value: 0x201020
+# CHECK-NEXT:     Value: 0x201140
 # CHECK:          Name: _init2
-# CHECK-NEXT:     Value: 0x201021
+# CHECK-NEXT:     Value: 0x201141
 
 # NOSORT:          Name: D
-# NOSORT-NEXT:     Value: 0x201000
+# NOSORT-NEXT:     Value: 0x201120
 # NOSORT:          Name: TooManyPreds
-# NOSORT-NEXT:     Value: 0x201004
+# NOSORT-NEXT:     Value: 0x201124
 # NOSORT:          Name: TooManyPreds10
-# NOSORT-NEXT:     Value: 0x201018
+# NOSORT-NEXT:     Value: 0x201138
 # NOSORT:          Name: A
-# NOSORT-NEXT:     Value: 0x201003
+# NOSORT-NEXT:     Value: 0x201123
 # NOSORT:          Name: B
-# NOSORT-NEXT:     Value: 0x201002
+# NOSORT-NEXT:     Value: 0x201122
 # NOSORT:          Name: C
-# NOSORT-NEXT:     Value: 0x201001
+# NOSORT-NEXT:     Value: 0x201121
 # NOSORT:          Name: GB
-# NOSORT-NEXT:     Value: 0x20101C
+# NOSORT-NEXT:     Value: 0x20113C
 # NOSORT:          Name: PP
-# NOSORT-NEXT:     Value: 0x20101A
+# NOSORT-NEXT:     Value: 0x20113A
 # NOSORT:          Name: QC
-# NOSORT-NEXT:     Value: 0x20101B
+# NOSORT-NEXT:     Value: 0x20113B
 # NOSORT:          Name: TS
-# NOSORT-NEXT:     Value: 0x201019
+# NOSORT-NEXT:     Value: 0x201139
 # NOSORT:          Name: _init
-# NOSORT-NEXT:     Value: 0x20101D
+# NOSORT-NEXT:     Value: 0x20113D
 # NOSORT:          Name: _init2
-# NOSORT-NEXT:     Value: 0x20101E
+# NOSORT-NEXT:     Value: 0x20113E
diff --git a/lld/test/ELF/cgprofile-txt2.s b/lld/test/ELF/cgprofile-txt2.s
index 7d59440f7cbdf..91961db39c3a8 100644
--- a/lld/test/ELF/cgprofile-txt2.s
+++ b/lld/test/ELF/cgprofile-txt2.s
@@ -9,13 +9,13 @@
 # RUN: llvm-readobj --symbols %t2 | FileCheck %s
 
 # CHECK:      Name: A
-# CHECK-NEXT: Value: 0x201003
+# CHECK-NEXT: Value: 0x201123
 # CHECK:      Name: B
-# CHECK-NEXT: Value: 0x201000
+# CHECK-NEXT: Value: 0x201120
 # CHECK:      Name: C
-# CHECK-NEXT: Value: 0x201001
+# CHECK-NEXT: Value: 0x201121
 # CHECK:      Name: D
-# CHECK-NEXT: Value: 0x201002
+# CHECK-NEXT: Value: 0x201122
 
 .section    .text.A,"ax",@progbits
 .globl  A
diff --git a/lld/test/ELF/combreloc.s b/lld/test/ELF/combreloc.s
index b8c7517928628..8ecd1af1ce957 100644
--- a/lld/test/ELF/combreloc.s
+++ b/lld/test/ELF/combreloc.s
@@ -14,12 +14,12 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-# CHECK-NEXT:     0x3020 R_X86_64_RELATIVE - 0x3028
-# CHECK-NEXT:     0x20B0 R_X86_64_GLOB_DAT aaa 0x0
-# CHECK-NEXT:     0x3000 R_X86_64_64 aaa 0x0
-# CHECK-NEXT:     0x3018 R_X86_64_64 aaa 0x0
-# CHECK-NEXT:     0x3010 R_X86_64_64 bbb 0x0
-# CHECK-NEXT:     0x3008 R_X86_64_64 ccc 0x0
+# CHECK-NEXT:     0x3428 R_X86_64_RELATIVE - 0x3430
+# CHECK-NEXT:     0x2400 R_X86_64_GLOB_DAT aaa 0x0
+# CHECK-NEXT:     0x3408 R_X86_64_64 aaa 0x0
+# CHECK-NEXT:     0x3420 R_X86_64_64 aaa 0x0
+# CHECK-NEXT:     0x3418 R_X86_64_64 bbb 0x0
+# CHECK-NEXT:     0x3410 R_X86_64_64 ccc 0x0
 # CHECK-NEXT:   }
 # CHECK:      DynamicSection [
 # CHECK:        RELACOUNT 1
@@ -29,12 +29,12 @@
 
 # NOCOMB:      Relocations [
 # NOCOMB-NEXT:   Section ({{.*}}) .rela.dyn {
-# NOCOMB-NEXT:     0x3000 R_X86_64_64 aaa 0x0
-# NOCOMB-NEXT:     0x3008 R_X86_64_64 ccc 0x0
-# NOCOMB-NEXT:     0x3010 R_X86_64_64 bbb 0x0
-# NOCOMB-NEXT:     0x3018 R_X86_64_64 aaa 0x0
-# NOCOMB-NEXT:     0x3020 R_X86_64_RELATIVE - 0x3028
-# NOCOMB-NEXT:     0x20A0 R_X86_64_GLOB_DAT aaa 0x0
+# NOCOMB-NEXT:     0x33F8 R_X86_64_64 aaa 0x0
+# NOCOMB-NEXT:     0x3400 R_X86_64_64 ccc 0x0
+# NOCOMB-NEXT:     0x3408 R_X86_64_64 bbb 0x0
+# NOCOMB-NEXT:     0x3410 R_X86_64_64 aaa 0x0
+# NOCOMB-NEXT:     0x3418 R_X86_64_RELATIVE - 0x3420
+# NOCOMB-NEXT:     0x23F0 R_X86_64_GLOB_DAT aaa 0x0
 # NOCOMB-NEXT:   }
 # NOCOMB:      DynamicSection [
 # NOCOMB-NOT:    RELACOUNT
diff --git a/lld/test/ELF/comdat.s b/lld/test/ELF/comdat.s
index 9e3f5a81d300e..a5ce6942a520c 100644
--- a/lld/test/ELF/comdat.s
+++ b/lld/test/ELF/comdat.s
@@ -21,7 +21,7 @@ foo:
 // CHECK: Disassembly of section .text2:
 // CHECK-EMPTY:
 // CHECK-NEXT: foo:
-// CHECK-NEXT:   1000: {{.*}}  nop
+// CHECK-NEXT:   1234: {{.*}}  nop
 // CHECK-NOT: nop
 
         .section bar, "ax"
@@ -30,8 +30,8 @@ foo:
 // CHECK: Disassembly of section bar:
 // CHECK-EMPTY:
 // CHECK-NEXT: bar:
-// 0x1000 - 0x1001 - 5 = -6
-// CHECK-NEXT:   1001:	{{.*}}  callq  -6
+// 0x1234 - 0x1235 - 5 = -6
+// CHECK-NEXT:   1235:	{{.*}}  callq  -6
 
         .section .text3,"axG",@progbits,zed,comdat,unique,0
 
@@ -71,7 +71,7 @@ foo:
 // READ-NEXT:   }
 // READ-NEXT:   Symbol {
 // READ-NEXT:     Name: _DYNAMIC
-// READ-NEXT:     Value: 0x2000
+// READ-NEXT:     Value: 0x2240
 // READ-NEXT:     Size: 0
 // READ-NEXT:     Binding: Local
 // READ-NEXT:     Type: None
diff --git a/lld/test/ELF/common-page.s b/lld/test/ELF/common-page.s
index eb0dfcd737714..bdde23feecfba 100644
--- a/lld/test/ELF/common-page.s
+++ b/lld/test/ELF/common-page.s
@@ -37,8 +37,8 @@ _start:
 # CHECK-MAX-NEXT:       SHF_ALLOC (0x2)
 # CHECK-MAX-NEXT:       SHF_EXECINSTR (0x4)
 # CHECK-MAX-NEXT:     ]
-# CHECK-MAX-NEXT:     Address: 0x210000
-# CHECK-MAX-NEXT:     Offset: 0x10000
+# CHECK-MAX-NEXT:     Address: 0x210120
+# CHECK-MAX-NEXT:     Offset: 0x120
 # CHECK-MAX-NEXT:     Size: 1
 # CHECK-MAX-NEXT:     Link: 0
 # CHECK-MAX-NEXT:     Info: 0
@@ -54,7 +54,7 @@ _start:
 # CHECK-MAX-NEXT:       SHF_STRINGS (0x20)
 # CHECK-MAX-NEXT:     ]
 # CHECK-MAX-NEXT:     Address: 0x0
-# CHECK-MAX-NEXT:     Offset: 0x10001
+# CHECK-MAX-NEXT:     Offset: 0x121
 # CHECK-MAX-NEXT:     Size: 8
 # CHECK-MAX-NEXT:     Link: 0
 # CHECK-MAX-NEXT:     Info: 0
@@ -88,9 +88,9 @@ _start:
 # CHECK-MAX-NEXT:   }
 # CHECK-MAX-NEXT:   ProgramHeader {
 # CHECK-MAX-NEXT:     Type: PT_LOAD (0x1)
-# CHECK-MAX-NEXT:     Offset: 0x10000
-# CHECK-MAX-NEXT:     VirtualAddress: 0x210000
-# CHECK-MAX-NEXT:     PhysicalAddress: 0x210000
+# CHECK-MAX-NEXT:     Offset: 0x120
+# CHECK-MAX-NEXT:     VirtualAddress: 0x210120
+# CHECK-MAX-NEXT:     PhysicalAddress: 0x210120
 # CHECK-MAX-NEXT:     FileSize: 1
 # CHECK-MAX-NEXT:     MemSize: 1
 # CHECK-MAX-NEXT:     Flags [ (0x5)
@@ -147,8 +147,8 @@ _start:
 # CHECK-COMMON-NEXT:       SHF_ALLOC (0x2)
 # CHECK-COMMON-NEXT:       SHF_EXECINSTR (0x4)
 # CHECK-COMMON-NEXT:     ]
-# CHECK-COMMON-NEXT:     Address: 0x210000
-# CHECK-COMMON-NEXT:     Offset: 0x10000
+# CHECK-COMMON-NEXT:     Address: 0x210120
+# CHECK-COMMON-NEXT:     Offset: 0x120
 # CHECK-COMMON-NEXT:     Size: 1
 # CHECK-COMMON-NEXT:     Link: 0
 # CHECK-COMMON-NEXT:     Info: 0
@@ -164,7 +164,7 @@ _start:
 # CHECK-COMMON-NEXT:       SHF_STRINGS (0x20)
 # CHECK-COMMON-NEXT:     ]
 # CHECK-COMMON-NEXT:     Address: 0x0
-# CHECK-COMMON-NEXT:     Offset: 0x10001
+# CHECK-COMMON-NEXT:     Offset: 0x121
 # CHECK-COMMON-NEXT:     Size: 8
 # CHECK-COMMON-NEXT:     Link: 0
 # CHECK-COMMON-NEXT:     Info: 0
@@ -198,9 +198,9 @@ _start:
 # CHECK-COMMON-NEXT:   }
 # CHECK-COMMON-NEXT:   ProgramHeader {
 # CHECK-COMMON-NEXT:     Type: PT_LOAD (0x1)
-# CHECK-COMMON-NEXT:     Offset: 0x10000
-# CHECK-COMMON-NEXT:     VirtualAddress: 0x210000
-# CHECK-COMMON-NEXT:     PhysicalAddress: 0x210000
+# CHECK-COMMON-NEXT:     Offset: 0x120
+# CHECK-COMMON-NEXT:     VirtualAddress: 0x210120
+# CHECK-COMMON-NEXT:     PhysicalAddress: 0x210120
 # CHECK-COMMON-NEXT:     FileSize: 1
 # CHECK-COMMON-NEXT:     MemSize: 1
 # CHECK-COMMON-NEXT:     Flags [ (0x5)
diff --git a/lld/test/ELF/common.s b/lld/test/ELF/common.s
index b9c7d95b44f11..1cdc91c2bf41e 100644
--- a/lld/test/ELF/common.s
+++ b/lld/test/ELF/common.s
@@ -10,7 +10,7 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x201000
+// CHECK-NEXT: Address: 0x202160
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 36
 // CHECK-NEXT: Link: 0
@@ -18,7 +18,7 @@
 // CHECK-NEXT: AddressAlignment: 16
 
 // CHECK:      Name: sym1
-// CHECK-NEXT: Value: 0x201000
+// CHECK-NEXT: Value: 0x202160
 // CHECK-NEXT: Size: 8
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: Object
@@ -26,7 +26,7 @@
 // CHECK-NEXT: Section: .bss
 
 // CHECK:      Name: sym2
-// CHECK-NEXT: Value: 0x201008
+// CHECK-NEXT: Value: 0x202168
 // CHECK-NEXT: Size: 8
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: Object
@@ -34,7 +34,7 @@
 // CHECK-NEXT: Section: .bss
 
 // CHECK:      Name: sym3
-// CHECK-NEXT: Value: 0x201010
+// CHECK-NEXT: Value: 0x202170
 // CHECK-NEXT: Size: 2
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: Object
@@ -42,7 +42,7 @@
 // CHECK-NEXT: Section: .bss
 
 // CHECK:      Name: sym4
-// CHECK-NEXT: Value: 0x201020
+// CHECK-NEXT: Value: 0x202180
 // CHECK-NEXT: Size: 4
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: Object
diff --git a/lld/test/ELF/copy-rel-abs.s b/lld/test/ELF/copy-rel-abs.s
index 725f4cd591392..de6fbc48a8976 100644
--- a/lld/test/ELF/copy-rel-abs.s
+++ b/lld/test/ELF/copy-rel-abs.s
@@ -12,11 +12,14 @@
 
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t2.o
 # RUN: ld.lld %t2.o %t1.so -o %t2
-# RUN: llvm-objdump -t %t2 | FileCheck %s --implicit-check-not=zed
+# RUN: llvm-objdump -t %t2 | FileCheck %s
 
 # CHECK: SYMBOL TABLE:
+# CHECK-NOT: zed
 # CHECK: .bss.rel.ro {{.*}} foo
+# CHECK-NOT: zed
 # CHECK: .bss.rel.ro {{.*}} bar
+# CHECK-NOT: zed
 
 .global _start
 _start:
diff --git a/lld/test/ELF/copy-rel-large.s b/lld/test/ELF/copy-rel-large.s
index f763a0e937f43..d383c0e2b74b6 100644
--- a/lld/test/ELF/copy-rel-large.s
+++ b/lld/test/ELF/copy-rel-large.s
@@ -2,7 +2,7 @@
 
 ## Test symbols larger than 2**32 can be copy relocated.
 
-# RUN: echo '.globl foo; .type foo,@object; foo: .size foo, 0x100000001' | \
+# RUN: echo '.globl foo; .type foo,@object; foo: .byte 0; .size foo, 0x100000001' | \
 # RUN:   llvm-mc -filetype=obj -triple=x86_64 - -o %t1.o
 # RUN: ld.lld -shared %t1.o -o %t1.so
 # RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
diff --git a/lld/test/ELF/copy-rel-tls.s b/lld/test/ELF/copy-rel-tls.s
index 265cce1bc7a07..df193a9be295a 100644
--- a/lld/test/ELF/copy-rel-tls.s
+++ b/lld/test/ELF/copy-rel-tls.s
@@ -6,8 +6,8 @@
 // RUN: llvm-nm %t1.so | FileCheck %s
 // RUN: llvm-nm %t | FileCheck --check-prefix=TLS %s
 // foo and tfoo have the same st_value but we should not copy tfoo.
-// CHECK: 2000 B foo
-// CHECK: 2000 B tfoo
+// CHECK: 0000000000003358 B foo
+// CHECK: 0000000000003358 B tfoo
 // TLS-NOT: tfoo
 
 .global _start
diff --git a/lld/test/ELF/defsym-reserved-syms.s b/lld/test/ELF/defsym-reserved-syms.s
index b71ca28fdbb5b..a9c2c054e7b56 100644
--- a/lld/test/ELF/defsym-reserved-syms.s
+++ b/lld/test/ELF/defsym-reserved-syms.s
@@ -17,10 +17,10 @@
 # RUN: llvm-readobj --symbols -S %t | FileCheck %s --check-prefix=EXPR
 # EXPR:     Symbol {
 # EXPR:      Name: foo2
-# EXPR-NEXT:  Value: 0x201007
+# EXPR-NEXT:  Value: 0x201127
 # EXPR:     Symbol {
 # EXPR:      Name: etext
-# EXPR-NEXT:  Value: 0x201005
+# EXPR-NEXT:  Value: 0x201125
 
 .globl foo1
  foo1 = 0x123
diff --git a/lld/test/ELF/dynamic-linker.s b/lld/test/ELF/dynamic-linker.s
index 3faf8e8a169e5..d0da77fab2953 100644
--- a/lld/test/ELF/dynamic-linker.s
+++ b/lld/test/ELF/dynamic-linker.s
@@ -4,10 +4,10 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
 
 # RUN: ld.lld --dynamic-linker foo %t.o %t.so -o %t
-# RUN: llvm-readelf -program-headers %t | FileCheck %s
+# RUN: llvm-readelf --program-headers --section-headers %t | FileCheck --implicit-check-not=.bss %s
 
 # RUN: ld.lld --dynamic-linker=foo %t.o %t.so -o %t
-# RUN: llvm-readelf -program-headers %t | FileCheck %s
+# RUN: llvm-readelf --program-headers --section-headers %t | FileCheck --implicit-check-not=.bss %s
 
 # CHECK: [Requesting program interpreter: foo]
 
diff --git a/lld/test/ELF/edata-etext.s b/lld/test/ELF/edata-etext.s
index 5c83256bfca1d..c559b06398a74 100644
--- a/lld/test/ELF/edata-etext.s
+++ b/lld/test/ELF/edata-etext.s
@@ -14,17 +14,17 @@
 # CHECK:      Sections:
 # CHECK-NEXT:  Idx Name          Size     VMA              Type
 # CHECK-NEXT:    0               00000000 0000000000000000
-# CHECK-NEXT:    1 .text         00000001 0000000000201000 TEXT
-# CHECK-NEXT:    2 .data         00000002 0000000000202000 DATA
-# CHECK-NEXT:    3 .bss          00000006 0000000000202004 BSS
+# CHECK-NEXT:    1 .text         00000001 0000000000201158 TEXT
+# CHECK-NEXT:    2 .data         00000002 0000000000202159 DATA
+# CHECK-NEXT:    3 .bss          00000006 000000000020215c BSS
 # CHECK:      SYMBOL TABLE:
-# CHECK-NEXT:  0000000000202002         .data 00000000 _edata
-# CHECK-NEXT:  000000000020200a         .bss  00000000 _end
-# CHECK-NEXT:  0000000000201001         .text 00000000 _etext
-# CHECK-NEXT:  0000000000201000         .text 00000000 _start
-# CHECK-NEXT:  0000000000202002         .data 00000000 edata
-# CHECK-NEXT:  000000000020200a         .bss  00000000 end
-# CHECK-NEXT:  0000000000201001         .text 00000000 etext
+# CHECK-NEXT:  000000000020215b         .data 00000000 _edata
+# CHECK-NEXT:  0000000000202162         .bss  00000000 _end
+# CHECK-NEXT:  0000000000201159         .text 00000000 _etext
+# CHECK-NEXT:  0000000000201158         .text 00000000 _start
+# CHECK-NEXT:  000000000020215b         .data 00000000 edata
+# CHECK-NEXT:  0000000000202162         .bss  00000000 end
+# CHECK-NEXT:  0000000000201159         .text 00000000 etext
 
 # RUN: ld.lld -r %t.o -o %t2
 # RUN: llvm-objdump -t %t2 | FileCheck %s --check-prefix=RELOCATABLE
diff --git a/lld/test/ELF/edata-no-bss.s b/lld/test/ELF/edata-no-bss.s
index b8660976cf60f..089ba42865ede 100644
--- a/lld/test/ELF/edata-no-bss.s
+++ b/lld/test/ELF/edata-no-bss.s
@@ -6,9 +6,9 @@
 # RUN: ld.lld %t.o -o %t --gc-sections
 # RUN: llvm-objdump -t -section-headers %t | FileCheck %s
 
-# CHECK: .data         00000008 0000000000202000 DATA
+# CHECK: .data         00000008 000000000020215c DATA
 
-# CHECK: 0000000000202008         .data                 00000000 _edata
+# CHECK: 0000000000202164         .data                 00000000 _edata
 
 .text
 .globl _start
diff --git a/lld/test/ELF/eh-align-cie.s b/lld/test/ELF/eh-align-cie.s
index 16b404e8fcd12..f621442bcd0a5 100644
--- a/lld/test/ELF/eh-align-cie.s
+++ b/lld/test/ELF/eh-align-cie.s
@@ -51,7 +51,7 @@ bar:
 // CHECK-NEXT: EntrySize:
 // CHECK-NEXT: SectionData (
 // CHECK-NEXT:   0000: 1C000000 00000000 017A5052 00017810
-// CHECK-NEXT:   0010: 061BF60D 00001B0C 07089001 00000000
-// CHECK-NEXT:   0020: 14000000 24000000 E00D0000 00000000
+// CHECK-NEXT:   0010: 061B2A10 00001B0C 07089001 00000000
+// CHECK-NEXT:   0020: 14000000 24000000 14100000 00000000
 // CHECK-NEXT:   0030: 00000000 00000000
 // CHECK-NEXT: )
diff --git a/lld/test/ELF/eh-frame-hdr-abs-fde.s b/lld/test/ELF/eh-frame-hdr-abs-fde.s
index d597ee94b463d..797f63992771c 100644
--- a/lld/test/ELF/eh-frame-hdr-abs-fde.s
+++ b/lld/test/ELF/eh-frame-hdr-abs-fde.s
@@ -8,12 +8,12 @@
 # RUN: llvm-objdump -s -dwarf=frames %t | FileCheck %s
 
 # CHECK:      Contents of section .eh_frame_hdr:
-# CHECK-NEXT:  200190 011b033b 14000000 01000000 700e0000
+# CHECK-NEXT:  200190 011b033b 14000000 01000000 54100000
 #                                                ^-- 0x201000 - 0x200190
 #                                                    .text    - .eh_frame_hdr
 # CHECK-NEXT:  2001a0 30000000
 # CHECK:      Contents of section .text:
-# CHECK-NEXT:  201000 90
+# CHECK-NEXT:  2011e4 90
 
 # CHECK: Augmentation:          "zLR"
 # CHECK: Augmentation data:     00 1B
diff --git a/lld/test/ELF/eh-frame-hdr-augmentation.s b/lld/test/ELF/eh-frame-hdr-augmentation.s
index c51ea1c4976b9..a87a6e2a3f687 100644
--- a/lld/test/ELF/eh-frame-hdr-augmentation.s
+++ b/lld/test/ELF/eh-frame-hdr-augmentation.s
@@ -11,7 +11,7 @@
 // CHECK-NEXT:   Code alignment factor: 1
 // CHECK-NEXT:   Data alignment factor: -8
 // CHECK-NEXT:   Return address column: 16
-// CHECK-NEXT:   Personality Address: 0000000000000dad
+// CHECK-NEXT:   Personality Address: 0000000000001029
 // CHECK-NEXT:   Augmentation data:
 
 // CHECK:      DW_CFA_def_cfa:  reg7 +8
@@ -19,8 +19,8 @@
 // CHECK-NEXT: DW_CFA_nop:
 // CHECK-NEXT: DW_CFA_nop:
 
-// CHECK:      00000020 00000014 00000024 FDE cie=00000024 pc=00000d98...00000d98
-// CHECK-NEXT:   LSDA Address: 0000000000000d8f
+// CHECK:      00000020 00000014 00000024 FDE cie=00000024 pc=00001014...00001014
+// CHECK-NEXT:   LSDA Address: 000000000000100b
 // CHECK-NEXT:   DW_CFA_nop:
 // CHECK-NEXT:   DW_CFA_nop:
 // CHECK-NEXT:   DW_CFA_nop:
diff --git a/lld/test/ELF/eh-frame-hdr-icf-fde.s b/lld/test/ELF/eh-frame-hdr-icf-fde.s
index ca7066d950233..0ab467e8020ae 100644
--- a/lld/test/ELF/eh-frame-hdr-icf-fde.s
+++ b/lld/test/ELF/eh-frame-hdr-icf-fde.s
@@ -32,10 +32,10 @@
 # CHECK-NEXT:   AddressAlignment:
 # CHECK-NEXT:   EntrySize: 0
 # CHECK-NEXT:   SectionData (
-# CHECK-NEXT:     0000: 011B033B 1C000000 02000000 A80E0000
+# CHECK-NEXT:     0000: 011B033B 1C000000 02000000 6C100000
 ##                               ^        ^-- FDE(1) PC
 ##                               ^-- Number of FDEs
-# CHECK-NEXT:     0010: 38000000 AA0E0000 50000000
+# CHECK-NEXT:     0010: 38000000 6E100000 50000000
 ##                               ^-- FDE(2) PC
 # CHECK-NEXT:   )
 # CHECK-NEXT: }
@@ -60,8 +60,8 @@
 # CHECK-NEXT:   SectionData (
 # CHECK-NEXT:     0000: 14000000 00000000 017A5200 01781001
 # CHECK-NEXT:     0010: 1B0C0708 90010000 14000000 1C000000
-# CHECK-NEXT:     0020: 680E0000 01000000 00000000 00000000
-# CHECK-NEXT:     0030: 14000000 34000000 520E0000 01000000
+# CHECK-NEXT:     0020: 2C100000 01000000 00000000 00000000
+# CHECK-NEXT:     0030: 14000000 34000000 16100000 01000000
 # CHECK-NEXT:     0040: 00000000 00000000 00000000
 # CHECK-NEXT:   )
 # CHECK-NEXT: }
@@ -74,7 +74,7 @@
 # CHECK-NEXT:    SHF_ALLOC
 # CHECK-NEXT:    SHF_EXECINSTR
 # CHECK-NEXT:  ]
-# CHECK-NEXT:  Address: 0x201000
+# CHECK-NEXT:  Address: 0x2011C4
 
 .section .text.f1, "ax"
 .cfi_startproc
diff --git a/lld/test/ELF/eh-frame-hdr.s b/lld/test/ELF/eh-frame-hdr.s
index d8507c1f1ea20..dc21fec6a0208 100644
--- a/lld/test/ELF/eh-frame-hdr.s
+++ b/lld/test/ELF/eh-frame-hdr.s
@@ -41,17 +41,17 @@ _start:
 // HDRDISASM:      Disassembly of section foo:
 // HDRDISASM-EMPTY:
 // HDRDISASM-NEXT: foo:
-// HDRDISASM-NEXT:    201000: 90 nop
+// HDRDISASM-NEXT:    2011e4: 90 nop
 // HDRDISASM-EMPTY:
 // HDRDISASM-NEXT: Disassembly of section bar:
 // HDRDISASM-EMPTY:
 // HDRDISASM-NEXT: bar:
-// HDRDISASM-NEXT:    201001: 90 nop
+// HDRDISASM-NEXT:    2011e5: 90 nop
 // HDRDISASM-EMPTY:
 // HDRDISASM-NEXT: Disassembly of section dah:
 // HDRDISASM-EMPTY:
 // HDRDISASM-NEXT: dah:
-// HDRDISASM-NEXT:    201002: 90 nop
+// HDRDISASM-NEXT:    2011e6: 90 nop
 
 // HDR:       Section {
 // HDR:         Index:
@@ -68,23 +68,23 @@ _start:
 // HDR-NEXT:    AddressAlignment: 4
 // HDR-NEXT:    EntrySize: 0
 // HDR-NEXT:    SectionData (
-// HDR-NEXT:      0000: 011B033B 24000000 03000000 A80E0000
-// HDR-NEXT:      0010: 40000000 A90E0000 58000000 AA0E0000
+// HDR-NEXT:      0000: 011B033B 24000000 03000000 8C100000
+// HDR-NEXT:      0010: 40000000 8D100000 58000000 8E100000
 // HDR-NEXT:      0020: 70000000
 // HDR-NEXT:    )
 //              Header (always 4 bytes): 0x011B033B
 //                 24000000 = .eh_frame(0x200180) - .eh_frame_hdr(0x200158) - 4
 //                 03000000 = 3 = the number of FDE pointers in the table.
-//              Entry(1): A80E0000 40000000
-//                 480E0000 = 0x201000 - .eh_frame_hdr(0x200158) = 0xEA8
+//              Entry(1): 8C100000 40000000
+//                 480E0000 = 0x2011e4 - .eh_frame_hdr(0x200158) = 0x108C
 //                 40000000 = address of FDE(1) - .eh_frame_hdr(0x200158) =
 //                    = .eh_frame(0x200180) + 24 - 0x200158 = 0x40
-//              Entry(2): A90E0000 58000000
-//                 A90E0000 = 0x201001 - .eh_frame_hdr(0x200158) = 0xEA9
+//              Entry(2): 8D100000 58000000
+//                 A90E0000 = 0x2011e5 - .eh_frame_hdr(0x200158) = 0x108D
 //                 58000000 = address of FDE(2) - .eh_frame_hdr(0x200158) =
 //                    = .eh_frame(0x200180) + 24 + 24 - 0x200158 = 0x58
-//              Entry(3): AA0E0000 70000000
-//                 AA0E0000 = 0x201002 - .eh_frame_hdr(0x200158) = 0xEAA
+//              Entry(3): 8E100000 70000000
+//                 AA0E0000 = 0x2011e6 - .eh_frame_hdr(0x200158) = 0x108E
 //                 70000000 = address of FDE(3) - .eh_frame_hdr(0x200158) =
 //                    = .eh_frame(0x200180) + 24 + 24 + 24 - 0x200158 = 0x70
 // HDR-NEXT:  }
@@ -105,10 +105,10 @@ _start:
 // HDR-NEXT:    SectionData (
 // HDR-NEXT:      0000: 14000000 00000000 017A5200 01781001
 // HDR-NEXT:      0010: 1B0C0708 90010000 14000000 1C000000
-// HDR-NEXT:      0020: 600E0000 01000000 00000000 00000000
-// HDR-NEXT:      0030: 14000000 34000000 490E0000 01000000
+// HDR-NEXT:      0020: 44100000 01000000 00000000 00000000
+// HDR-NEXT:      0030: 14000000 34000000 2D100000 01000000
 // HDR-NEXT:      0040: 00000000 00000000 14000000 4C000000
-// HDR-NEXT:      0050: 320E0000 01000000 00000000 00000000
+// HDR-NEXT:      0050: 16100000 01000000 00000000 00000000
 // HDR-NEXT:      0060: 00000000
 // HDR-NEXT:    )
 //            CIE: 14000000 00000000 017A5200 01781001 1B0C0708 90010000
diff --git a/lld/test/ELF/eh-frame-merge.s b/lld/test/ELF/eh-frame-merge.s
index d921798073610..ddfb8ec072607 100644
--- a/lld/test/ELF/eh-frame-merge.s
+++ b/lld/test/ELF/eh-frame-merge.s
@@ -35,10 +35,10 @@
 // CHECK-NEXT: SectionData (
 // CHECK-NEXT: 0000: 14000000 00000000 017A5200 01781001  |
 // CHECK-NEXT: 0010: 1B0C0708 90010000 14000000 1C000000  |
-// CHECK-NEXT: 0020: E80D0000 01000000 00000000 00000000  |
-// CHECK-NEXT: 0030: 14000000 34000000 D20D0000 02000000  |
+// CHECK-NEXT: 0020: 44100000 01000000 00000000 00000000  |
+// CHECK-NEXT: 0030: 14000000 34000000 2E100000 02000000  |
 // CHECK-NEXT: 0040: 00000000 00000000 14000000 4C000000  |
-// CHECK-NEXT: 0050: B90D0000 01000000 00000000 00000000  |
+// CHECK-NEXT: 0050: 15100000 01000000 00000000 00000000  |
 // CHECK-NEXT: 0060: 00000000
 // CHECK-NEXT: )
 
@@ -48,7 +48,7 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_EXECINSTR
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x1000
+// CHECK-NEXT: Address: 0x125C
 
 // CHECK:      Name: bar
 // CHECK-NEXT: Type: SHT_PROGBITS
@@ -56,4 +56,4 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_EXECINSTR
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x1002
+// CHECK-NEXT: Address: 0x125E
diff --git a/lld/test/ELF/eh-frame-pcrel-overflow.s b/lld/test/ELF/eh-frame-pcrel-overflow.s
index 9b88edab2a93c..676a3dea17a96 100644
--- a/lld/test/ELF/eh-frame-pcrel-overflow.s
+++ b/lld/test/ELF/eh-frame-pcrel-overflow.s
@@ -4,7 +4,7 @@
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/eh-frame-pcrel-overflow.s -o %t1.o
 # RUN: ld.lld --eh-frame-hdr -Ttext=0x90000000 %t.o -o /dev/null
 # RUN: not ld.lld --eh-frame-hdr %t.o %t1.o -o /dev/null 2>&1 | FileCheck %s
-# CHECK: error: {{.*}}.o:(.eh_frame): PC offset is too large: 0x90000eac
+# CHECK: error: {{.*}}.o:(.eh_frame): PC offset is too large: 0x90001054
 
 .text
 .global _start
diff --git a/lld/test/ELF/eh-frame-value-format7.s b/lld/test/ELF/eh-frame-value-format7.s
index ffe578188f3cd..038ed12a05b1e 100644
--- a/lld/test/ELF/eh-frame-value-format7.s
+++ b/lld/test/ELF/eh-frame-value-format7.s
@@ -13,20 +13,20 @@
 # CHECK-NEXT:   Flags [
 # CHECK-NEXT:     SHF_ALLOC
 # CHECK-NEXT:   ]
-# CHECK-NEXT:   Address: 0x2000
-# CHECK-NEXT:   Offset: 0x2000
+# CHECK-NEXT:   Address: 0x2004
+# CHECK-NEXT:   Offset: 0x1004
 # CHECK-NEXT:   Size: 20
 # CHECK-NEXT:   Link: 0
 # CHECK-NEXT:   Info: 0
 # CHECK-NEXT:   AddressAlignment: 4
 # CHECK-NEXT:   EntrySize: 0
 # CHECK-NEXT:   SectionData (
-# CHECK-NEXT:     0000: 011B033B 10000000 01000000 34F2FFFF
+# CHECK-NEXT:     0000: 011B033B 10000000 01000000 30F2FFFF
 # CHECK-NEXT:     0010: 24000000
 # Header (always 4 bytes): 011B033B
-#    10000000 = .eh_frame(0x2014) - .eh_frame_hdr(0x2000) - 4
+#    10000000 = .eh_frame(0x2018) - .eh_frame_hdr(0x2004) - 4
 #    01000000 = 1 = the number of FDE pointers in the table.
-# 34F2FFFF = foo(0x1000) - 0x234(addend) - .eh_frame_hdr(0x2000)
+# 30F2FFFF = foo(0x1000) - 0x234(addend) - .eh_frame_hdr(0x2004)
   
 # CHECK:      Section {
 # CHECK:        Index:
@@ -35,8 +35,8 @@
 # CHECK-NEXT:   Flags [
 # CHECK-NEXT:     SHF_ALLOC
 # CHECK-NEXT:   ]
-# CHECK-NEXT:   Address: 0x2014
-# CHECK-NEXT:   Offset: 0x2014
+# CHECK-NEXT:   Address: 0x2018
+# CHECK-NEXT:   Offset: 0x1018
 # CHECK-NEXT:   Size:
 # CHECK-NEXT:   Link:
 # CHECK-NEXT:   Info:
diff --git a/lld/test/ELF/eh-frame-value-format8.s b/lld/test/ELF/eh-frame-value-format8.s
index cb1f2410938ca..155e81a2c5f68 100644
--- a/lld/test/ELF/eh-frame-value-format8.s
+++ b/lld/test/ELF/eh-frame-value-format8.s
@@ -13,20 +13,20 @@
 # CHECK-NEXT:   Flags [
 # CHECK-NEXT:     SHF_ALLOC
 # CHECK-NEXT:   ]
-# CHECK-NEXT:   Address: 0x2000
-# CHECK-NEXT:   Offset: 0x2000
+# CHECK-NEXT:   Address: 0x2004
+# CHECK-NEXT:   Offset: 0x1004
 # CHECK-NEXT:   Size: 20
 # CHECK-NEXT:   Link: 0
 # CHECK-NEXT:   Info: 0
 # CHECK-NEXT:   AddressAlignment: 4
 # CHECK-NEXT:   EntrySize: 0
 # CHECK-NEXT:   SectionData (
-# CHECK-NEXT:     0000: 011B033B 10000000 01000000 34F2FFFF
+# CHECK-NEXT:     0000: 011B033B 10000000 01000000 30F2FFFF
 # CHECK-NEXT:     0010: 24000000
 # Header (always 4 bytes): 011B033B
-#    10000000 = .eh_frame(0x2014) - .eh_frame_hdr(0x2000) - 4
+#    10000000 = .eh_frame(0x2018) - .eh_frame_hdr(0x2004) - 4
 #    01000000 = 1 = the number of FDE pointers in the table.
-# 34F2FFFF = foo(0x1000) - 0x234(addend) - .eh_frame_hdr(0x2000)
+# 30F2FFFF = foo(0x1000) - 0x234(addend) - .eh_frame_hdr(0x2004)
   
 # CHECK:      Section {
 # CHECK:        Index:
@@ -35,8 +35,8 @@
 # CHECK-NEXT:   Flags [
 # CHECK-NEXT:     SHF_ALLOC
 # CHECK-NEXT:   ]
-# CHECK-NEXT:   Address: 0x2014
-# CHECK-NEXT:   Offset: 0x2014
+# CHECK-NEXT:   Address: 0x2018
+# CHECK-NEXT:   Offset: 0x1018
 # CHECK-NEXT:   Size:
 # CHECK-NEXT:   Link:
 # CHECK-NEXT:   Info:
diff --git a/lld/test/ELF/ehframe-relocation.s b/lld/test/ELF/ehframe-relocation.s
index 562fcfbeeae2f..953d53eca5a38 100644
--- a/lld/test/ELF/ehframe-relocation.s
+++ b/lld/test/ELF/ehframe-relocation.s
@@ -20,8 +20,8 @@
 // DISASM:      Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: _start:
-// DISASM-NEXT:   201000: {{.*}} movq 2097440, %rax
-// DISASM-NEXT:   201008: {{.*}} movq 2097445, %rax
+// DISASM-NEXT:   201154: {{.*}} movq 2097440, %rax
+// DISASM-NEXT:   20115c: {{.*}} movq 2097445, %rax
 
 .section .eh_frame,"ax",@unwind
 
diff --git a/lld/test/ELF/emit-relocs-icf1.s b/lld/test/ELF/emit-relocs-icf1.s
index 2f7e0cab27f61..28fd29cfc1dad 100644
--- a/lld/test/ELF/emit-relocs-icf1.s
+++ b/lld/test/ELF/emit-relocs-icf1.s
@@ -5,9 +5,9 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section (3) .rela.text {
-# CHECK-NEXT:     0x201000 R_X86_64_64 .text 0x11
-# CHECK-NEXT:     0x201008 R_X86_64_64 .text 0x11
-# CHECK-NEXT:     0x201011 R_X86_64_64 .rodata 0x0
+# CHECK-NEXT:     0x201128 R_X86_64_64 .text 0x11
+# CHECK-NEXT:     0x201130 R_X86_64_64 .text 0x11
+# CHECK-NEXT:     0x201139 R_X86_64_64 .rodata 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/emit-relocs-mergeable.s b/lld/test/ELF/emit-relocs-mergeable.s
index 3390d970bc8c3..d1ba49c62f809 100644
--- a/lld/test/ELF/emit-relocs-mergeable.s
+++ b/lld/test/ELF/emit-relocs-mergeable.s
@@ -27,10 +27,10 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section {{.*}} .rela.foo {
-# CHECK-NEXT:     0x201000 R_X86_64_64 .strings 0x0
-# CHECK-NEXT:     0x201008 R_X86_64_64 .strings 0x8
-# CHECK-NEXT:     0x201010 R_X86_64_64 .strings 0x8
-# CHECK-NEXT:     0x201018 R_X86_64_64 .strings 0x4
+# CHECK-NEXT:     0x201120 R_X86_64_64 .strings 0x0
+# CHECK-NEXT:     0x201128 R_X86_64_64 .strings 0x8
+# CHECK-NEXT:     0x201130 R_X86_64_64 .strings 0x8
+# CHECK-NEXT:     0x201138 R_X86_64_64 .strings 0x4
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/emit-relocs.s b/lld/test/ELF/emit-relocs.s
index 5d46535b749ef..d7cfcc165299e 100644
--- a/lld/test/ELF/emit-relocs.s
+++ b/lld/test/ELF/emit-relocs.s
@@ -20,10 +20,10 @@
 # CHECK-NEXT:   ]
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section ({{.*}}) .rela.text {
-# CHECK-NEXT:     0x201002 R_X86_64_32 .text 0x1
-# CHECK-NEXT:     0x201007 R_X86_64_PLT32 fn 0xFFFFFFFFFFFFFFFC
-# CHECK-NEXT:     0x20100E R_X86_64_32 .text 0xD
-# CHECK-NEXT:     0x201013 R_X86_64_PLT32 fn2 0xFFFFFFFFFFFFFFFC
+# CHECK-NEXT:     0x201122 R_X86_64_32 .text 0x1
+# CHECK-NEXT:     0x201127 R_X86_64_PLT32 fn 0xFFFFFFFFFFFFFFFC
+# CHECK-NEXT:     0x20112E R_X86_64_32 .text 0xD
+# CHECK-NEXT:     0x201133 R_X86_64_PLT32 fn2 0xFFFFFFFFFFFFFFFC
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 # CHECK-NEXT: Symbols [
@@ -38,7 +38,7 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: bar
-# CHECK-NEXT:     Value: 0x201001
+# CHECK-NEXT:     Value: 0x201121
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Local
 # CHECK-NEXT:     Type: None
@@ -47,7 +47,7 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: foo
-# CHECK-NEXT:     Value: 0x20100D
+# CHECK-NEXT:     Value: 0x20112D
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Local
 # CHECK-NEXT:     Type: None
@@ -56,7 +56,7 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name:
-# CHECK-NEXT:     Value: 0x201000
+# CHECK-NEXT:     Value: 0x201120
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Local
 # CHECK-NEXT:     Type: Section
@@ -74,7 +74,7 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: fn
-# CHECK-NEXT:     Value: 0x201000
+# CHECK-NEXT:     Value: 0x201120
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Global
 # CHECK-NEXT:     Type: Function
@@ -83,7 +83,7 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   Symbol {
 # CHECK-NEXT:     Name: fn2
-# CHECK-NEXT:     Value: 0x20100C
+# CHECK-NEXT:     Value: 0x20112C
 # CHECK-NEXT:     Size: 0
 # CHECK-NEXT:     Binding: Global
 # CHECK-NEXT:     Type: Function
diff --git a/lld/test/ELF/emulation-mips.s b/lld/test/ELF/emulation-mips.s
index 5b9f27290fd15..f6b04ca90abdc 100644
--- a/lld/test/ELF/emulation-mips.s
+++ b/lld/test/ELF/emulation-mips.s
@@ -1,22 +1,33 @@
 # REQUIRES: mips
 # RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o %tmips
 # RUN: ld.lld -m elf32btsmip -e _start %tmips -o %t2mips
-# RUN: llvm-readobj --file-headers %t2mips | FileCheck --check-prefix=MIPS %s
+# RUN: llvm-readobj --file-headers %t2mips \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS %s
+# RUN: ld.lld -m elf32btsmip_fbsd -e _start %tmips -o %t2mips
+# RUN: llvm-readobj --file-headers %t2mips \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS %s
 # RUN: ld.lld %tmips -e _start -o %t3mips
-# RUN: llvm-readobj --file-headers %t3mips | FileCheck --check-prefix=MIPS %s
+# RUN: llvm-readobj --file-headers %t3mips \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS %s
 # RUN: echo 'OUTPUT_FORMAT(elf32-tradbigmips)' > %tmips.script
 # RUN: ld.lld %tmips.script -e _start %tmips -o %t4mips
-# RUN: llvm-readobj --file-headers %t4mips | FileCheck --check-prefix=MIPS %s
+# RUN: llvm-readobj --file-headers %t4mips \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS %s
+# RUN: echo 'OUTPUT_FORMAT(elf32-tradbigmips-freebsd)' > %tmips.script
+# RUN: ld.lld %tmips.script -e _start %tmips -o %t4mips
+# RUN: llvm-readobj --file-headers %t4mips \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS %s
 # RUN: echo 'OUTPUT_FORMAT(elf32-bigmips)' > %tmips2.script
 # RUN: ld.lld %tmips2.script -e _start %tmips -o %t5mips
-# RUN: llvm-readobj --file-headers %t5mips | FileCheck --check-prefix=MIPS %s
+# RUN: llvm-readobj --file-headers %t5mips \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS %s
 # MIPS:      ElfHeader {
 # MIPS-NEXT:   Ident {
 # MIPS-NEXT:     Magic: (7F 45 4C 46)
 # MIPS-NEXT:     Class: 32-bit (0x1)
 # MIPS-NEXT:     DataEncoding: BigEndian (0x2)
 # MIPS-NEXT:     FileVersion: 1
-# MIPS-NEXT:     OS/ABI: SystemV (0x0)
+# MIPS-NEXT:     OS/ABI: [[OSABI]]
 # MIPS-NEXT:     ABIVersion: 1
 # MIPS-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPS-NEXT:   }
@@ -34,21 +45,32 @@
 
 # RUN: llvm-mc -filetype=obj -triple=mipsel-unknown-linux %s -o %tmipsel
 # RUN: ld.lld -m elf32ltsmip -e _start %tmipsel -o %t2mipsel
-# RUN: llvm-readobj --file-headers %t2mipsel | FileCheck --check-prefix=MIPSEL %s
+# RUN: llvm-readobj --file-headers %t2mipsel \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSEL %s
+# RUN: ld.lld -m elf32ltsmip_fbsd -e _start %tmipsel -o %t2mipsel
+# RUN: llvm-readobj --file-headers %t2mipsel \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSEL %s
 # RUN: ld.lld -melf32ltsmip -e _start %tmipsel -o %t2mipsel
-# RUN: llvm-readobj --file-headers %t2mipsel | FileCheck --check-prefix=MIPSEL %s
+# RUN: llvm-readobj --file-headers %t2mipsel \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSEL %s
 # RUN: ld.lld %tmipsel -e _start -o %t3mipsel
-# RUN: llvm-readobj --file-headers %t3mipsel | FileCheck --check-prefix=MIPSEL %s
+# RUN: llvm-readobj --file-headers %t3mipsel \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSEL %s
 # RUN: echo 'OUTPUT_FORMAT(elf32-tradlittlemips)' > %tmipsel.script
 # RUN: ld.lld %tmipsel.script -e _start %tmipsel -o %t4mipsel
-# RUN: llvm-readobj --file-headers %t4mipsel | FileCheck --check-prefix=MIPSEL %s
+# RUN: llvm-readobj --file-headers %t4mipsel \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSEL %s
+# RUN: echo 'OUTPUT_FORMAT(elf32-tradlittlemips-freebsd)' > %tmipsel.script
+# RUN: ld.lld %tmipsel.script -e _start %tmipsel -o %t4mipsel
+# RUN: llvm-readobj --file-headers %t4mipsel \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSEL %s
 # MIPSEL:      ElfHeader {
 # MIPSEL-NEXT:   Ident {
 # MIPSEL-NEXT:     Magic: (7F 45 4C 46)
 # MIPSEL-NEXT:     Class: 32-bit (0x1)
 # MIPSEL-NEXT:     DataEncoding: LittleEndian (0x1)
 # MIPSEL-NEXT:     FileVersion: 1
-# MIPSEL-NEXT:     OS/ABI: SystemV (0x0)
+# MIPSEL-NEXT:     OS/ABI: [[OSABI]]
 # MIPSEL-NEXT:     ABIVersion: 1
 # MIPSEL-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPSEL-NEXT:   }
@@ -66,19 +88,29 @@
 
 # RUN: llvm-mc -filetype=obj -triple=mips64-unknown-linux-gnuabin32 %s -o %tmipsn32
 # RUN: ld.lld -m elf32btsmipn32 -e _start %tmipsn32 -o %t2mipsn32
-# RUN: llvm-readobj --file-headers %t2mipsn32 | FileCheck --check-prefix=MIPSN32 %s
+# RUN: llvm-readobj --file-headers %t2mipsn32 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32 %s
+# RUN: ld.lld -m elf32btsmipn32_fbsd -e _start %tmipsn32 -o %t2mipsn32
+# RUN: llvm-readobj --file-headers %t2mipsn32 \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSN32 %s
 # RUN: ld.lld %tmipsn32 -e _start -o %t3mipsn32
-# RUN: llvm-readobj --file-headers %t3mipsn32 | FileCheck --check-prefix=MIPSN32 %s
+# RUN: llvm-readobj --file-headers %t3mipsn32 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32 %s
 # RUN: echo 'OUTPUT_FORMAT(elf32-ntradbigmips)' > %tmipsn32.script
 # RUN: ld.lld %tmipsn32.script -e _start %tmipsn32 -o %t4mipsn32
-# RUN: llvm-readobj --file-headers %t4mipsn32 | FileCheck --check-prefix=MIPSN32 %s
+# RUN: llvm-readobj --file-headers %t4mipsn32 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32 %s
+# RUN: echo 'OUTPUT_FORMAT(elf32-ntradbigmips-freebsd)' > %tmipsn32.script
+# RUN: ld.lld %tmipsn32.script -e _start %tmipsn32 -o %t4mipsn32
+# RUN: llvm-readobj --file-headers %t4mipsn32 \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSN32 %s
 # MIPSN32:      ElfHeader {
 # MIPSN32-NEXT:   Ident {
 # MIPSN32-NEXT:     Magic: (7F 45 4C 46)
 # MIPSN32-NEXT:     Class: 32-bit (0x1)
 # MIPSN32-NEXT:     DataEncoding: BigEndian (0x2)
 # MIPSN32-NEXT:     FileVersion: 1
-# MIPSN32-NEXT:     OS/ABI: SystemV (0x0)
+# MIPSN32-NEXT:     OS/ABI: [[OSABI]]
 # MIPSN32-NEXT:     ABIVersion: 1
 # MIPSN32-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPSN32-NEXT:   }
@@ -96,21 +128,32 @@
 
 # RUN: llvm-mc -filetype=obj -triple=mips64el-unknown-linux-gnuabin32 %s -o %tmipsn32el
 # RUN: ld.lld -m elf32ltsmipn32 -e _start %tmipsn32el -o %t2mipsn32el
-# RUN: llvm-readobj --file-headers %t2mipsn32el | FileCheck --check-prefix=MIPSN32EL %s
+# RUN: llvm-readobj --file-headers %t2mipsn32el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32EL %s
+# RUN: ld.lld -m elf32ltsmipn32_fbsd -e _start %tmipsn32el -o %t2mipsn32el
+# RUN: llvm-readobj --file-headers %t2mipsn32el \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSN32EL %s
 # RUN: ld.lld -melf32ltsmipn32 -e _start %tmipsn32el -o %t2mipsn32el
-# RUN: llvm-readobj --file-headers %t2mipsn32el | FileCheck --check-prefix=MIPSN32EL %s
+# RUN: llvm-readobj --file-headers %t2mipsn32el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32EL %s
 # RUN: ld.lld %tmipsn32el -e _start -o %t3mipsn32el
-# RUN: llvm-readobj --file-headers %t3mipsn32el | FileCheck --check-prefix=MIPSN32EL %s
+# RUN: llvm-readobj --file-headers %t3mipsn32el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32EL %s
 # RUN: echo 'OUTPUT_FORMAT(elf32-ntradlittlemips)' > %tmipsn32el.script
 # RUN: ld.lld %tmipsn32el.script -e _start %tmipsn32el -o %t4mipsn32el
-# RUN: llvm-readobj --file-headers %t4mipsn32el | FileCheck --check-prefix=MIPSN32EL %s
+# RUN: llvm-readobj --file-headers %t4mipsn32el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPSN32EL %s
+# RUN: echo 'OUTPUT_FORMAT(elf32-ntradlittlemips-freebsd)' > %tmipsn32el.script
+# RUN: ld.lld %tmipsn32el.script -e _start %tmipsn32el -o %t4mipsn32el
+# RUN: llvm-readobj --file-headers %t4mipsn32el \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPSN32EL %s
 # MIPSN32EL:      ElfHeader {
 # MIPSN32EL-NEXT:   Ident {
 # MIPSN32EL-NEXT:     Magic: (7F 45 4C 46)
 # MIPSN32EL-NEXT:     Class: 32-bit (0x1)
 # MIPSN32EL-NEXT:     DataEncoding: LittleEndian (0x1)
 # MIPSN32EL-NEXT:     FileVersion: 1
-# MIPSN32EL-NEXT:     OS/ABI: SystemV (0x0)
+# MIPSN32EL-NEXT:     OS/ABI: [[OSABI]]
 # MIPSN32EL-NEXT:     ABIVersion: 1
 # MIPSN32EL-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPSN32EL-NEXT:   }
@@ -129,19 +172,29 @@
 # RUN: llvm-mc -filetype=obj -triple=mips64-unknown-linux -position-independent \
 # RUN:         %s -o %tmips64
 # RUN: ld.lld -m elf64btsmip -e _start %tmips64 -o %t2mips64
-# RUN: llvm-readobj --file-headers %t2mips64 | FileCheck --check-prefix=MIPS64 %s
+# RUN: llvm-readobj --file-headers %t2mips64 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64 %s
+# RUN: ld.lld -m elf64btsmip_fbsd -e _start %tmips64 -o %t2mips64
+# RUN: llvm-readobj --file-headers %t2mips64 \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS64 %s
 # RUN: ld.lld %tmips64 -e _start -o %t3mips64
-# RUN: llvm-readobj --file-headers %t3mips64 | FileCheck --check-prefix=MIPS64 %s
+# RUN: llvm-readobj --file-headers %t3mips64 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64 %s
 # RUN: echo 'OUTPUT_FORMAT(elf64-tradbigmips)' > %tmips64.script
 # RUN: ld.lld %tmips64.script -e _start %tmips64 -o %t4mips64
-# RUN: llvm-readobj --file-headers %t4mips64 | FileCheck --check-prefix=MIPS64 %s
+# RUN: llvm-readobj --file-headers %t4mips64 \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64 %s
+# RUN: echo 'OUTPUT_FORMAT(elf64-tradbigmips-freebsd)' > %tmips64.script
+# RUN: ld.lld %tmips64.script -e _start %tmips64 -o %t4mips64
+# RUN: llvm-readobj --file-headers %t4mips64 \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS64 %s
 # MIPS64:      ElfHeader {
 # MIPS64-NEXT:   Ident {
 # MIPS64-NEXT:     Magic: (7F 45 4C 46)
 # MIPS64-NEXT:     Class: 64-bit (0x2)
 # MIPS64-NEXT:     DataEncoding: BigEndian (0x2)
 # MIPS64-NEXT:     FileVersion: 1
-# MIPS64-NEXT:     OS/ABI: SystemV (0x0)
+# MIPS64-NEXT:     OS/ABI: [[OSABI]]
 # MIPS64-NEXT:     ABIVersion: 0
 # MIPS64-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPS64-NEXT:   }
@@ -160,19 +213,29 @@
 # RUN: llvm-mc -filetype=obj -triple=mips64el-unknown-linux \
 # RUN:         -position-independent %s -o %tmips64el
 # RUN: ld.lld -m elf64ltsmip -e _start %tmips64el -o %t2mips64el
-# RUN: llvm-readobj --file-headers %t2mips64el | FileCheck --check-prefix=MIPS64EL %s
+# RUN: llvm-readobj --file-headers %t2mips64el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64EL %s
+# RUN: ld.lld -m elf64ltsmip_fbsd -e _start %tmips64el -o %t2mips64el
+# RUN: llvm-readobj --file-headers %t2mips64el \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS64EL %s
 # RUN: ld.lld %tmips64el -e _start -o %t3mips64el
-# RUN: llvm-readobj --file-headers %t3mips64el | FileCheck --check-prefix=MIPS64EL %s
+# RUN: llvm-readobj --file-headers %t3mips64el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64EL %s
 # RUN: echo 'OUTPUT_FORMAT(elf64-tradlittlemips)' > %tmips64el.script
 # RUN: ld.lld %tmips64el.script -e _start %tmips64el -o %t4mips64el
-# RUN: llvm-readobj --file-headers %t4mips64el | FileCheck --check-prefix=MIPS64EL %s
+# RUN: llvm-readobj --file-headers %t4mips64el \
+# RUN:   | FileCheck -DOSABI=SystemV --check-prefix=MIPS64EL %s
+# RUN: echo 'OUTPUT_FORMAT(elf64-tradlittlemips-freebsd)' > %tmips64el.script
+# RUN: ld.lld %tmips64el.script -e _start %tmips64el -o %t4mips64el
+# RUN: llvm-readobj --file-headers %t4mips64el \
+# RUN:   | FileCheck -DOSABI=FreeBSD --check-prefix=MIPS64EL %s
 # MIPS64EL:      ElfHeader {
 # MIPS64EL-NEXT:   Ident {
 # MIPS64EL-NEXT:     Magic: (7F 45 4C 46)
 # MIPS64EL-NEXT:     Class: 64-bit (0x2)
 # MIPS64EL-NEXT:     DataEncoding: LittleEndian (0x1)
 # MIPS64EL-NEXT:     FileVersion: 1
-# MIPS64EL-NEXT:     OS/ABI: SystemV (0x0)
+# MIPS64EL-NEXT:     OS/ABI: [[OSABI]]
 # MIPS64EL-NEXT:     ABIVersion: 0
 # MIPS64EL-NEXT:     Unused: (00 00 00 00 00 00 00)
 # MIPS64EL-NEXT:   }
diff --git a/lld/test/ELF/end-dso-defined.s b/lld/test/ELF/end-dso-defined.s
index 79078fce934b5..0b5e977296d10 100644
--- a/lld/test/ELF/end-dso-defined.s
+++ b/lld/test/ELF/end-dso-defined.s
@@ -1,8 +1,8 @@
 # REQUIRES: x86
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t1.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/resolution-end.s -o %t2.o
-# RUN: ld.lld -shared -o %t2.so %t2.o
-# RUN: ld.lld --hash-style=sysv %t1.o %t2.so -o %t
+# RUN: ld.lld -shared -soname=so -o %t2.so %t2.o
+# RUN: ld.lld %t1.o %t2.so -o %t
 # RUN: llvm-readobj --symbols -S --section-data  %t | FileCheck %s
 
 # Test that we resolve _end to the this executable.
@@ -21,16 +21,16 @@
 # CHECK-NEXT: AddressAlignment:
 # CHECK-NEXT: EntrySize:
 # CHECK-NEXT: SectionData (
-# CHECK-NEXT:   0000: 80202000 00000000 80202000 00000000
+# CHECK-NEXT:   0000: 08232000 00000000 08232000 00000000
 # CHECK-NEXT: )
 
 # CHECK:      Symbol {
 # CHECK:        Name: _end
-# CHECK-NEXT:   Value: 0x202080
+# CHECK-NEXT:   Value: 0x202308
 
 # CHECK:      Symbol {
 # CHECK:        Name: end
-# CHECK-NEXT:   Value: 0x202080
+# CHECK-NEXT:   Value: 0x202308
 
 .global _start
 _start:
diff --git a/lld/test/ELF/end-update.s b/lld/test/ELF/end-update.s
index c485d8a949e27..e8a2500674fd3 100644
--- a/lld/test/ELF/end-update.s
+++ b/lld/test/ELF/end-update.s
@@ -12,13 +12,13 @@
 // CHECK-NEXT:       SHF_ALLOC
 // CHECK-NEXT:       SHF_WRITE
 // CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x202000
+// CHECK-NEXT:     Address: 0x202159
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 6
 // CHECK: ]
 // CHECK: Symbols [
 // CHECK:     Name: end
-// CHECK-NEXT:     Value: 0x202006
+// CHECK-NEXT:     Value: 0x20215F
 // CHECK: ]
 
 .global _start,end
diff --git a/lld/test/ELF/end.s b/lld/test/ELF/end.s
index 9ddf208348ea1..c10943d70488f 100644
--- a/lld/test/ELF/end.s
+++ b/lld/test/ELF/end.s
@@ -14,13 +14,13 @@
 // DEFAULT-NEXT:       SHF_ALLOC
 // DEFAULT-NEXT:       SHF_WRITE
 // DEFAULT-NEXT:     ]
-// DEFAULT-NEXT:     Address: 0x202002
+// DEFAULT-NEXT:     Address: 0x20215B
 // DEFAULT-NEXT:     Offset:
 // DEFAULT-NEXT:     Size: 6
 // DEFAULT: ]
 // DEFAULT: Symbols [
 // DEFAULT:     Name: _end
-// DEFAULT-NEXT:     Value: 0x202008
+// DEFAULT-NEXT:     Value: 0x202161
 // DEFAULT: ]
 
 // RUN: ld.lld -r %t.o -o %t2
diff --git a/lld/test/ELF/entry.s b/lld/test/ELF/entry.s
index b469138ecc260..4f6ad1546ef9a 100644
--- a/lld/test/ELF/entry.s
+++ b/lld/test/ELF/entry.s
@@ -4,14 +4,14 @@
 # RUN: ld.lld -e foobar %t1 -o %t2 2>&1 | FileCheck -check-prefix=WARN1 %s
 # RUN: llvm-readobj --file-headers %t2 | FileCheck -check-prefix=TEXT %s
 
-# WARN1: warning: cannot find entry symbol foobar; defaulting to 0x201000
-# TEXT: Entry: 0x201000
+# WARN1: warning: cannot find entry symbol foobar; defaulting to 0x201120
+# TEXT: Entry: 0x201120
 
 # RUN: ld.lld %t1 -o %t2 2>&1 | FileCheck -check-prefix=WARN2 %s
-# WARN2: warning: cannot find entry symbol _start; defaulting to 0x201000
+# WARN2: warning: cannot find entry symbol _start; defaulting to 0x201120
 
 # RUN: ld.lld -shared -e foobar %t1 -o %t2 2>&1 | FileCheck -check-prefix=WARN3 %s
-# WARN3: warning: cannot find entry symbol foobar; defaulting to 0x1000
+# WARN3: warning: cannot find entry symbol foobar; defaulting to 0x1238
 
 # RUN: ld.lld -shared --fatal-warnings -e entry %t1 -o %t2
 # RUN: ld.lld -shared --fatal-warnings %t1 -o %t2
@@ -29,11 +29,11 @@
 
 # RUN: ld.lld %t1 -o %t2 -e entry
 # RUN: llvm-readobj --file-headers %t2 | FileCheck -check-prefix=SYM %s
-# SYM: Entry: 0x201008
+# SYM: Entry: 0x201128
 
 # RUN: ld.lld %t1 --fatal-warnings -shared -o %t2 -e entry
 # RUN: llvm-readobj --file-headers %t2 | FileCheck -check-prefix=DSO %s
-# DSO: Entry: 0x1008
+# DSO: Entry: 0x1240
 
 # RUN: ld.lld %t1 -o %t2 --entry=4096
 # RUN: llvm-readobj --file-headers %t2 | FileCheck -check-prefix=DEC %s
diff --git a/lld/test/ELF/fill-trap.s b/lld/test/ELF/fill-trap.s
index 9fc600045872f..bd26c3f8f8d7d 100644
--- a/lld/test/ELF/fill-trap.s
+++ b/lld/test/ELF/fill-trap.s
@@ -12,12 +12,18 @@
 # RUN: llvm-readobj -l %t | FileCheck %s --check-prefixes=CHECK,PAD
 # RUN: od -Ax -x -N16 -j0x1ff0 %t | FileCheck %s --check-prefix=FILL
 
+## -z separate-loadable-segments pads all segments, including the text segment.
+# RUN: ld.lld %t.o -z separate-loadable-segments -o %t
+# RUN: llvm-readobj -l %t | FileCheck %s --check-prefixes=CHECK,PAD
+# RUN: od -Ax -x -N16 -j0x1ff0 %t | FileCheck %s --check-prefix=FILL
+
 # RUN: ld.lld %t.o -z separate-code -z noseparate-code -o %t
 # RUN: llvm-readobj -l %t | FileCheck %s --check-prefixes=CHECK,NOPAD
 
 # CHECK: ProgramHeader {
 # CHECK:   Type: PT_LOAD
-# CHECK:   Offset: 0x1000
+# PAD:     Offset: 0x1000
+# NOPAD:   Offset: 0x120
 # CHECK-NEXT:   VirtualAddress:
 # CHECK-NEXT:   PhysicalAddress:
 # PAD-NEXT:     FileSize: 4096
diff --git a/lld/test/ELF/gc-sections-metadata-err.s b/lld/test/ELF/gc-sections-metadata-err.s
new file mode 100644
index 0000000000000..0f96e4157dfc2
--- /dev/null
+++ b/lld/test/ELF/gc-sections-metadata-err.s
@@ -0,0 +1,37 @@
+# REQUIRES: x86
+
+## Error if the linked-to section of an input section is discarded.
+
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+# RUN: not ld.lld --gc-sections --print-gc-sections %t.o -o /dev/null 2>&1 | FileCheck %s
+
+# CHECK:      removing unused section {{.*}}.o:(.foo0)
+# CHECK-NEXT: error: {{.*}}.o:(.bar): sh_link points to discarded section {{.*}}.o:(.foo0)
+# CHECK-NEXT: error: {{.*}}.o:(.baz): sh_link points to discarded section {{.*}}.o:(.foo0)
+
+.globl _start
+_start:
+  call .foo1
+  call bar0
+  call bar1
+  call baz0
+  call baz1
+
+.section .foo0,"a"
+.section .foo1,"a"
+
+## The linked-to section of the first input section is discarded.
+.section .bar,"ao",@progbits,.foo0,unique,0
+bar0:
+.byte 0
+.section .bar,"ao",@progbits,.foo1,unique,1
+bar1:
+.byte 1
+
+## Another case: the linked-to section of the second input section is discarded.
+.section .baz,"ao",@progbits,.foo1,unique,0
+baz0:
+.byte 0
+.section .baz,"ao",@progbits,.foo0,unique,1
+baz1:
+.byte 1
diff --git a/lld/test/ELF/gdb-index-base-addr.s b/lld/test/ELF/gdb-index-base-addr.s
index a16ea7b2c9dc0..92787688eb30f 100644
--- a/lld/test/ELF/gdb-index-base-addr.s
+++ b/lld/test/ELF/gdb-index-base-addr.s
@@ -5,8 +5,8 @@
 
 # CHECK:      .gdb_index contents:
 # CHECK:       Address area offset = 0x28, has 2 entries:
-# CHECK-NEXT:  Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
-# CHECK-NEXT:  Low/High address = [0x201003, 0x201006) (Size: 0x3), CU id = 0
+# CHECK-NEXT:  Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
+# CHECK-NEXT:  Low/High address = [0x201123, 0x201126) (Size: 0x3), CU id = 0
 
 .text
 .globl foo
diff --git a/lld/test/ELF/gdb-index-dwarf5-low-high.s b/lld/test/ELF/gdb-index-dwarf5-low-high.s
index 94b713c6d216b..a354016a212d4 100644
--- a/lld/test/ELF/gdb-index-dwarf5-low-high.s
+++ b/lld/test/ELF/gdb-index-dwarf5-low-high.s
@@ -4,7 +4,7 @@
 # RUN: llvm-dwarfdump -gdb-index %t | FileCheck %s
 
 # CHECK:       Address area offset = 0x28, has 1 entries:
-# CHECK-NEXT:    Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
+# CHECK-NEXT:    Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
 
   .text
   .globl main
diff --git a/lld/test/ELF/gdb-index-gc-sections.s b/lld/test/ELF/gdb-index-gc-sections.s
index 6016e9ccdd07f..cd7100ed67c50 100644
--- a/lld/test/ELF/gdb-index-gc-sections.s
+++ b/lld/test/ELF/gdb-index-gc-sections.s
@@ -4,7 +4,7 @@
 # RUN: llvm-dwarfdump -gdb-index %t2 | FileCheck %s
 
 # CHECK: Address area offset = 0x28, has 1 entries:
-# CHECK-NEXT:    Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
+# CHECK-NEXT:    Low/High address = [0x201154, 0x201155) (Size: 0x1), CU id = 0
 
 # Generated with: (clang r302976)
 # echo "void _start() {} void dead() {}" | \
diff --git a/lld/test/ELF/gdb-index-ranges-discarded.s b/lld/test/ELF/gdb-index-ranges-discarded.s
index 2d5ce15736fb9..5aa77fe3ea714 100644
--- a/lld/test/ELF/gdb-index-ranges-discarded.s
+++ b/lld/test/ELF/gdb-index-ranges-discarded.s
@@ -5,7 +5,7 @@
 
 # CHECK:      .gdb_index contents:
 # CHECK:       Address area offset = 0x28, has 1 entries:
-# CHECK-NEXT:    Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
+# CHECK-NEXT:    Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
 
 ## .debug_ranges contains 2 entries. .Lfunc_end0 is defined in the discarded
 ## .text.foo. Test we resolve it to a non-zero value, otherwise the address area
diff --git a/lld/test/ELF/gdb-index-ranges.s b/lld/test/ELF/gdb-index-ranges.s
index 2190a2442f525..3459ac4e6b044 100644
--- a/lld/test/ELF/gdb-index-ranges.s
+++ b/lld/test/ELF/gdb-index-ranges.s
@@ -5,8 +5,8 @@
 
 # CHECK:       .gdb_index contents:
 # CHECK:        Address area offset = 0x28, has 2 entries:
-# CHECK-NEXT:     Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
-# CHECK-NEXT:     Low/High address = [0x201001, 0x201003) (Size: 0x2), CU id = 0
+# CHECK-NEXT:     Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
+# CHECK-NEXT:     Low/High address = [0x201121, 0x201123) (Size: 0x2), CU id = 0
 
 .section .text.foo1,"ax",@progbits
 .Lfunc_begin0:
diff --git a/lld/test/ELF/gdb-index-rng-lists.s b/lld/test/ELF/gdb-index-rng-lists.s
index e2a63d77fc2af..e6c49ef675edd 100644
--- a/lld/test/ELF/gdb-index-rng-lists.s
+++ b/lld/test/ELF/gdb-index-rng-lists.s
@@ -14,8 +14,8 @@
 
 # CHECK:      .gdb_index contents:
 # CHECK:      Address area offset = 0x28, has 2 entries:
-# CHECK-NEXT:  Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
-# CHECK-NEXT:  Low/High address = [0x201001, 0x201003) (Size: 0x2), CU id = 0
+# CHECK-NEXT:  Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
+# CHECK-NEXT:  Low/High address = [0x201121, 0x201123) (Size: 0x2), CU id = 0
 
 .text
 .section .text._Z3zedv,"ax",@progbits
diff --git a/lld/test/ELF/gdb-index.s b/lld/test/ELF/gdb-index.s
index 77c16c2afbd22..97cee6b2d5925 100644
--- a/lld/test/ELF/gdb-index.s
+++ b/lld/test/ELF/gdb-index.s
@@ -32,8 +32,8 @@
 # DWARF-NEXT:    0: Offset = 0x0, Length = 0x34
 # DWARF-NEXT:    1: Offset = 0x34, Length = 0x34
 # DWARF:       Address area offset = 0x38, has 2 entries:
-# DWARF-NEXT:    Low/High address = [0x201000, 0x201001) (Size: 0x1), CU id = 0
-# DWARF-NEXT:    Low/High address = [0x201004, 0x201006) (Size: 0x2), CU id = 1
+# DWARF-NEXT:    Low/High address = [0x201120, 0x201121) (Size: 0x1), CU id = 0
+# DWARF-NEXT:    Low/High address = [0x201124, 0x201126) (Size: 0x2), CU id = 1
 # DWARF:       Symbol table offset = 0x60, size = 1024, filled slots:
 # DWARF-NEXT:    512: Name offset = 0x1c, CU vector offset = 0x0
 # DWARF-NEXT:      String name: aaaaaaaaaaaaaaaa, CU vector index: 0
diff --git a/lld/test/ELF/global_offset_table_shared.s b/lld/test/ELF/global_offset_table_shared.s
index 233d0858cda9f..f9e9c9837c61b 100644
--- a/lld/test/ELF/global_offset_table_shared.s
+++ b/lld/test/ELF/global_offset_table_shared.s
@@ -5,7 +5,7 @@
 .long _GLOBAL_OFFSET_TABLE_ - .
 
 // CHECK:      Name: _GLOBAL_OFFSET_TABLE_
-// CHECK-NEXT: Value: 0x3000
+// CHECK-NEXT: Value: 0x3290
 // CHECK-NEXT: Size: 0
 // CHECK-NEXT: Binding: Local
 // CHECK-NEXT: Type: None
diff --git a/lld/test/ELF/gnu-ifunc-canon.s b/lld/test/ELF/gnu-ifunc-canon.s
index a7ba3aff7edbf..05773f6c0f199 100644
--- a/lld/test/ELF/gnu-ifunc-canon.s
+++ b/lld/test/ELF/gnu-ifunc-canon.s
@@ -59,18 +59,18 @@
 
 // Make sure the static relocations look right, both with and without headers.
 // DUMP: Contents of section .plt:
-// DUMP-NEXT: 201010
+// DUMP-NEXT: 2011f0
 // DUMP: Contents of section .got:
-// DUMP-NEXT: 202000 10102000 00000000
+// DUMP-NEXT: 202200 f0112000 00000000
 // DUMP: Contents of section .data:
-// DUMP-NEXT: 203000 10102000 00000000 11102000 00000000
+// DUMP-NEXT: 203208 f0112000 00000000 f1112000 00000000
 
 // DUMP2: Contents of section .plt:
-// DUMP2-NEXT: 201010
+// DUMP2-NEXT: 2011f0
 // DUMP2: Contents of section .got:
-// DUMP2-NEXT: 202000 40102000 00000000
+// DUMP2-NEXT: 202240 20122000 00000000
 // DUMP2: Contents of section .data:
-// DUMP2-NEXT: 203000 40102000 00000000 41102000 00000000
+// DUMP2-NEXT: 203248 20122000 00000000 21122000 00000000
 
 lea ifunc@gotpcrel(%rip), %rbx
 
diff --git a/lld/test/ELF/gnu-ifunc-dyntags.s b/lld/test/ELF/gnu-ifunc-dyntags.s
index 7bd3567cbaa58..186a7a78ed64f 100644
--- a/lld/test/ELF/gnu-ifunc-dyntags.s
+++ b/lld/test/ELF/gnu-ifunc-dyntags.s
@@ -9,7 +9,7 @@
 
 # CHECK:  Name          Size   VMA
 # CHECK:  .rela.dyn   00000030 0000000000000248
-# CHECK:  .got.plt    00000010 0000000000003000
+# CHECK:  .got.plt    00000010 00000000000033a0
 
 # TAGS:      Relocations [
 # TAGS-NEXT:   Section {{.*}} .rela.dyn {
@@ -23,7 +23,7 @@
 # TAGS:   0x0000000000000008 RELASZ               48 (bytes)
 # TAGS:   0x0000000000000017 JMPREL               0x0
 # TAGS:   0x0000000000000002 PLTRELSZ             0 (bytes)
-# TAGS:   0x0000000000000003 PLTGOT               0x3000
+# TAGS:   0x0000000000000003 PLTGOT               0x33A0
 # TAGS:   0x0000000000000014 PLTREL               RELA
 
 .text
diff --git a/lld/test/ELF/gnu-ifunc-noplt.s b/lld/test/ELF/gnu-ifunc-noplt.s
index f3e50b1e41678..e697e4736d7b8 100644
--- a/lld/test/ELF/gnu-ifunc-noplt.s
+++ b/lld/test/ELF/gnu-ifunc-noplt.s
@@ -12,45 +12,45 @@
 // Check that we emitted relocations for the ifunc calls
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (4) .rela.dyn {
-// CHECK-NEXT:     0x201008 R_X86_64_PLT32 bar 0xFFFFFFFFFFFFFFFC
-// CHECK-NEXT:     0x201003 R_X86_64_PLT32 foo 0xFFFFFFFFFFFFFFFC
+// CHECK-NEXT:     0x201328 R_X86_64_PLT32 bar 0xFFFFFFFFFFFFFFFC
+// CHECK-NEXT:     0x201323 R_X86_64_PLT32 foo 0xFFFFFFFFFFFFFFFC
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section (5) .rela.plt {
-// CHECK-NEXT:     0x203018 R_X86_64_JUMP_SLOT bar2 0x0
-// CHECK-NEXT:     0x203020 R_X86_64_JUMP_SLOT zed2 0x0
+// CHECK-NEXT:     0x203498 R_X86_64_JUMP_SLOT bar2 0x0
+// CHECK-NEXT:     0x2034A0 R_X86_64_JUMP_SLOT zed2 0x0
 // CHECK-NEXT:   }
 
 // Check that ifunc call sites still require relocation
 // DISASM: Disassembly of section .text:
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201000 foo:
-// DISASM-NEXT:   201000:      	retq
+// DISASM-NEXT: 0000000000201320 foo:
+// DISASM-NEXT:   201320:      	retq
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201001 bar:
-// DISASM-NEXT:   201001:      	retq
+// DISASM-NEXT: 0000000000201321 bar:
+// DISASM-NEXT:   201321:      	retq
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201002 _start:
-// DISASM-NEXT:   201002:      	callq	0 <_start+0x5>
-// DISASM-NEXT:   201007:      	callq	0 <_start+0xa>
-// DISASM-NEXT:   20100c:      	callq	31 <bar2@plt>
-// DISASM-NEXT:   201011:      	callq	42 <zed2@plt>
+// DISASM-NEXT: 0000000000201322 _start:
+// DISASM-NEXT:   201322:      	callq	0 <_start+0x5>
+// DISASM-NEXT:   201327:      	callq	0 <_start+0xa>
+// DISASM-NEXT:   20132c:      	callq	31 <bar2@plt>
+// DISASM-NEXT:   201331:      	callq	42 <zed2@plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201020 .plt:
-// DISASM-NEXT:   201020:      	pushq	8162(%rip)
-// DISASM-NEXT:   201026:      	jmpq	*8164(%rip)
-// DISASM-NEXT:   20102c:      	nopl	(%rax)
+// DISASM-NEXT: 0000000000201340 .plt:
+// DISASM-NEXT:   201340:      	pushq	8514(%rip)
+// DISASM-NEXT:   201346:      	jmpq	*8516(%rip)
+// DISASM-NEXT:   20134c:      	nopl	(%rax)
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201030 bar2@plt:
-// DISASM-NEXT:   201030:      	jmpq	*8162(%rip)
-// DISASM-NEXT:   201036:      	pushq	$0
-// DISASM-NEXT:   20103b:      	jmp	-32 <.plt>
+// DISASM-NEXT: 0000000000201350 bar2@plt:
+// DISASM-NEXT:   201350:      	jmpq	*8514(%rip)
+// DISASM-NEXT:   201356:      	pushq	$0
+// DISASM-NEXT:   20135b:      	jmp	-32 <.plt>
 // DISASM-EMPTY:
-// DISASM-NEXT: 0000000000201040 zed2@plt:
-// DISASM-NEXT:   201040:      	jmpq	*8154(%rip)
-// DISASM-NEXT:   201046:      	pushq	$1
-// DISASM-NEXT:   20104b:      	jmp	-48 <.plt>
+// DISASM-NEXT: 0000000000201360 zed2@plt:
+// DISASM-NEXT:   201360:      	jmpq	*8506(%rip)
+// DISASM-NEXT:   201366:      	pushq	$1
+// DISASM-NEXT:   20136b:      	jmp	-48 <.plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-plt.s b/lld/test/ELF/gnu-ifunc-plt.s
index 25cefb8b3313b..0610d850759f5 100644
--- a/lld/test/ELF/gnu-ifunc-plt.s
+++ b/lld/test/ELF/gnu-ifunc-plt.s
@@ -13,20 +13,20 @@
 // Check that the IRELATIVE relocations are after the JUMP_SLOT in the plt
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (4) .rela.dyn {
-// CHECK-NEXT:     0x203028 R_X86_64_IRELATIVE - 0x201000
-// CHECK-NEXT:     0x203030 R_X86_64_IRELATIVE - 0x201001
+// CHECK-NEXT:     0x203458 R_X86_64_IRELATIVE - 0x2012D8
+// CHECK-NEXT:     0x203460 R_X86_64_IRELATIVE - 0x2012D9
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section (5) .rela.plt {
-// CHECK-NEXT:     0x203018 R_X86_64_JUMP_SLOT bar2 0x0
-// CHECK-NEXT:     0x203020 R_X86_64_JUMP_SLOT zed2 0x0
+// CHECK-NEXT:     0x203448 R_X86_64_JUMP_SLOT bar2 0x0
+// CHECK-NEXT:     0x203450 R_X86_64_JUMP_SLOT zed2 0x0
 // CHECK-NEXT:   }
 
 // Check that .got.plt entries point back to PLT header
 // GOTPLT: Contents of section .got.plt:
-// GOTPLT-NEXT:  203000 00202000 00000000 00000000 00000000
-// GOTPLT-NEXT:  203010 00000000 00000000 36102000 00000000
-// GOTPLT-NEXT:  203020 46102000 00000000 56102000 00000000
-// GOTPLT-NEXT:  203030 66102000 00000000
+// GOTPLT-NEXT:  203430 40232000 00000000 00000000 00000000
+// GOTPLT-NEXT:  203440 00000000 00000000 06132000 00000000
+// GOTPLT-NEXT:  203450 16132000 00000000 26132000 00000000
+// GOTPLT-NEXT:  203460 36132000 00000000
 
 // Check that the PLTRELSZ tag does not include the IRELATIVE relocations
 // CHECK: DynamicSection [
@@ -37,37 +37,37 @@
 // DISASM: Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: foo:
-// DISASM-NEXT:   201000:       retq
+// DISASM-NEXT:   2012d8:       retq
 // DISASM:      bar:
-// DISASM-NEXT:   201001:       retq
+// DISASM-NEXT:   2012d9:       retq
 // DISASM:      _start:
-// DISASM-NEXT:   201002:       callq   73
-// DISASM-NEXT:   201007:       callq   84
+// DISASM-NEXT:   2012da:       callq   65
+// DISASM-NEXT:   2012df:       callq   76
 // DISASM-NEXT:                 callq   {{.*}} <bar2@plt>
 // DISASM-NEXT:                 callq   {{.*}} <zed2@plt>
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
 // DISASM-NEXT: .plt:
-// DISASM-NEXT:   201020:       pushq   8162(%rip)
-// DISASM-NEXT:   201026:       jmpq    *8164(%rip)
-// DISASM-NEXT:   20102c:       nopl    (%rax)
+// DISASM-NEXT:   2012f0:       pushq   8514(%rip)
+// DISASM-NEXT:   2012f6:       jmpq    *8516(%rip)
+// DISASM-NEXT:   2012fc:       nopl    (%rax)
 // DISASM-EMPTY:
 // DISASM-NEXT:   bar2@plt:
-// DISASM-NEXT:   201030:       jmpq    *8162(%rip)
-// DISASM-NEXT:   201036:       pushq   $0
-// DISASM-NEXT:   20103b:       jmp     -32 <.plt>
+// DISASM-NEXT:   201300:       jmpq    *8514(%rip)
+// DISASM-NEXT:   201306:       pushq   $0
+// DISASM-NEXT:   20130b:       jmp     -32 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   zed2@plt:
-// DISASM-NEXT:   201040:       jmpq    *8154(%rip)
-// DISASM-NEXT:   201046:       pushq   $1
-// DISASM-NEXT:   20104b:       jmp     -48 <.plt>
-// DISASM-NEXT:   201050:       jmpq    *8146(%rip)
-// DISASM-NEXT:   201056:       pushq   $0
-// DISASM-NEXT:   20105b:       jmp     -32 <zed2@plt>
-// DISASM-NEXT:   201060:       jmpq    *8138(%rip)
-// DISASM-NEXT:   201066:       pushq   $1
-// DISASM-NEXT:   20106b:       jmp     -48 <zed2@plt>
+// DISASM-NEXT:   201310:       jmpq    *8506(%rip)
+// DISASM-NEXT:   201316:       pushq   $1
+// DISASM-NEXT:   20131b:       jmp     -48 <.plt>
+// DISASM-NEXT:   201320:       jmpq    *8498(%rip)
+// DISASM-NEXT:   201326:       pushq   $0
+// DISASM-NEXT:   20132b:       jmp     -32 <zed2@plt>
+// DISASM-NEXT:   201330:       jmpq    *8490(%rip)
+// DISASM-NEXT:   201336:       pushq   $1
+// DISASM-NEXT:   20133b:       jmp     -48 <zed2@plt>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/gnu-ifunc-shared.s b/lld/test/ELF/gnu-ifunc-shared.s
index 06789df18b467..d0e70100b6fdb 100644
--- a/lld/test/ELF/gnu-ifunc-shared.s
+++ b/lld/test/ELF/gnu-ifunc-shared.s
@@ -12,44 +12,44 @@
 // DISASM: Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: fct:
-// DISASM-NEXT:     1000:       retq
+// DISASM-NEXT:     1308:       retq
 // DISASM:     fct2:
-// DISASM-NEXT:     1001:       retq
+// DISASM-NEXT:     1309:       retq
 // DISASM:     f1:
-// DISASM-NEXT:     1002:       callq   73
-// DISASM-NEXT:     1007:       callq   36
-// DISASM-NEXT:     100c:       callq   47
-// DISASM-NEXT:     1011:       retq
+// DISASM-NEXT:     130a:       callq   65
+// DISASM-NEXT:     130f:       callq   28
+// DISASM-NEXT:     1314:       callq   39
+// DISASM-NEXT:     1319:       retq
 // DISASM:     f2:
-// DISASM-NEXT:     1012:       retq
+// DISASM-NEXT:     131a:       retq
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
 // DISASM-NEXT: .plt:
-// DISASM-NEXT:     1020:       pushq   8162(%rip)
-// DISASM-NEXT:     1026:       jmpq    *8164(%rip)
-// DISASM-NEXT:     102c:       nopl    (%rax)
+// DISASM-NEXT:     1320:       pushq   8482(%rip)
+// DISASM-NEXT:     1326:       jmpq    *8484(%rip)
+// DISASM-NEXT:     132c:       nopl    (%rax)
 // DISASM-EMPTY:
 // DISASM-NEXT:   fct2@plt:
-// DISASM-NEXT:     1030:       jmpq    *8162(%rip)
-// DISASM-NEXT:     1036:       pushq   $0
-// DISASM-NEXT:     103b:       jmp     -32 <.plt>
+// DISASM-NEXT:     1330:       jmpq    *8482(%rip)
+// DISASM-NEXT:     1336:       pushq   $0
+// DISASM-NEXT:     133b:       jmp     -32 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   f2@plt:
-// DISASM-NEXT:     1040:       jmpq    *8154(%rip)
-// DISASM-NEXT:     1046:       pushq   $1
-// DISASM-NEXT:     104b:       jmp     -48 <.plt>
-// DISASM-NEXT:     1050:       jmpq    *8146(%rip)
-// DISASM-NEXT:     1056:       pushq   $0
-// DISASM-NEXT:     105b:       jmp     -32 <f2@plt>
+// DISASM-NEXT:     1340:       jmpq    *8474(%rip)
+// DISASM-NEXT:     1346:       pushq   $1
+// DISASM-NEXT:     134b:       jmp     -48 <.plt>
+// DISASM-NEXT:     1350:       jmpq    *8466(%rip)
+// DISASM-NEXT:     1356:       pushq   $0
+// DISASM-NEXT:     135b:       jmp     -32 <f2@plt>
 
 // CHECK: Relocations [
 // CHECK-NEXT:   Section (5) .rela.dyn {
-// CHECK-NEXT:     0x3028 R_X86_64_IRELATIVE - 0x1000
+// CHECK-NEXT:     0x3468 R_X86_64_IRELATIVE - 0x1308
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Section (6) .rela.plt {
-// CHECK-NEXT:     0x3018 R_X86_64_JUMP_SLOT fct2 0x0
-// CHECK-NEXT:     0x3020 R_X86_64_JUMP_SLOT f2 0x0
+// CHECK-NEXT:     0x3458 R_X86_64_JUMP_SLOT fct2 0x0
+// CHECK-NEXT:     0x3460 R_X86_64_JUMP_SLOT f2 0x0
 // CHECK-NEXT:   }
 
  // Hidden expect IRELATIVE
diff --git a/lld/test/ELF/gnu-ifunc.s b/lld/test/ELF/gnu-ifunc.s
index 1efd4ab37c34a..97e8e9c5386dd 100644
--- a/lld/test/ELF/gnu-ifunc.s
+++ b/lld/test/ELF/gnu-ifunc.s
@@ -24,8 +24,8 @@
 // CHECK-NEXT: Name: .got.plt
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:     0x202000 R_X86_64_IRELATIVE
-// CHECK-NEXT:     0x202008 R_X86_64_IRELATIVE
+// CHECK-NEXT:     0x2021D0 R_X86_64_IRELATIVE
+// CHECK-NEXT:     0x2021D8 R_X86_64_IRELATIVE
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 // CHECK:      Symbols [
@@ -62,7 +62,7 @@
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: _start
-// CHECK-NEXT:    Value: 0x201002
+// CHECK-NEXT:    Value: 0x20118A
 // CHECK-NEXT:    Size: 0
 // CHECK-NEXT:    Binding: Global
 // CHECK-NEXT:    Type: None
@@ -71,7 +71,7 @@
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: bar
-// CHECK-NEXT:    Value: 0x201001
+// CHECK-NEXT:    Value: 0x201189
 // CHECK-NEXT:    Size: 0
 // CHECK-NEXT:    Binding: Global
 // CHECK-NEXT:    Type: GNU_IFunc
@@ -80,7 +80,7 @@
 // CHECK-NEXT:  }
 // CHECK-NEXT:  Symbol {
 // CHECK-NEXT:    Name: foo
-// CHECK-NEXT:    Value: 0x201000
+// CHECK-NEXT:    Value: 0x201188
 // CHECK-NEXT:    Size: 0
 // CHECK-NEXT:    Binding: Global
 // CHECK-NEXT:    Type: GNU_IFunc
@@ -92,25 +92,25 @@
 // DISASM: Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: foo:
-// DISASM-NEXT:  201000: {{.*}} retq
+// DISASM-NEXT:  201188: {{.*}} retq
 // DISASM:      bar:
-// DISASM-NEXT:  201001: {{.*}} retq
+// DISASM-NEXT:  201189: {{.*}} retq
 // DISASM:      _start:
-// DISASM-NEXT:  201002: {{.*}} callq 25
-// DISASM-NEXT:  201007: {{.*}} callq 36
-// DISASM-NEXT:  20100c: {{.*}} movl $2097496, %edx
-// DISASM-NEXT:  201011: {{.*}} movl $2097544, %edx
-// DISASM-NEXT:  201016: {{.*}} movl $2097545, %edx
+// DISASM-NEXT:  20118a: {{.*}} callq 33
+// DISASM-NEXT:  20118f: {{.*}} callq 44
+// DISASM-NEXT:  201194: {{.*}} movl $2097496, %edx
+// DISASM-NEXT:  201199: {{.*}} movl $2097544, %edx
+// DISASM-NEXT:  20119e: {{.*}} movl $2097545, %edx
 // DISASM-EMPTY:
 // DISASM-NEXT: Disassembly of section .plt:
 // DISASM-EMPTY:
 // DISASM-NEXT: .plt:
-// DISASM-NEXT:  201020: {{.*}} jmpq *4058(%rip)
-// DISASM-NEXT:  201026: {{.*}} pushq $0
-// DISASM-NEXT:  20102b: {{.*}} jmp -32 <_start+0xe>
-// DISASM-NEXT:  201030: {{.*}} jmpq *4050(%rip)
-// DISASM-NEXT:  201036: {{.*}} pushq $1
-// DISASM-NEXT:  20103b: {{.*}} jmp -48 <_start+0xe>
+// DISASM-NEXT:  2011b0: {{.*}} jmpq *4122(%rip)
+// DISASM-NEXT:  2011b6: {{.*}} pushq $0
+// DISASM-NEXT:  2011bb: {{.*}} jmp -32 <_start+0x16>
+// DISASM-NEXT:  2011c0: {{.*}} jmpq *4114(%rip)
+// DISASM-NEXT:  2011c6: {{.*}} pushq $1
+// DISASM-NEXT:  2011cb: {{.*}} jmp -48 <_start+0x16>
 
 .text
 .type foo STT_GNU_IFUNC
diff --git a/lld/test/ELF/got.s b/lld/test/ELF/got.s
index 3f53029ab19b6..1db9039d192c9 100644
--- a/lld/test/ELF/got.s
+++ b/lld/test/ELF/got.s
@@ -1,7 +1,7 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/shared.s -o %t2.o
-// RUN: ld.lld -shared %t2.o -o %t2.so
+// RUN: ld.lld -shared %t2.o -soname=t2.so -o %t2.so
 // RUN: ld.lld --hash-style=sysv %t.o %t2.so -o %t
 // RUN: llvm-readobj -S -r %t | FileCheck %s
 // RUN: llvm-objdump -d %t | FileCheck --check-prefix=DISASM %s
@@ -12,7 +12,7 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x2020B0
+// CHECK-NEXT: Address: 0x202338
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 16
 // CHECK-NEXT: Link: 0
@@ -21,22 +21,22 @@
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:     0x2020B0 R_X86_64_GLOB_DAT bar 0x0
-// CHECK-NEXT:     0x2020B8 R_X86_64_GLOB_DAT zed 0x0
+// CHECK-NEXT:     0x202338 R_X86_64_GLOB_DAT bar 0x0
+// CHECK-NEXT:     0x202340 R_X86_64_GLOB_DAT zed 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
 
 // Unfortunately FileCheck can't do math, so we have to check for explicit
 // values:
-//  0x2020B0 - (0x201000 + 2) - 4 = 4266
-//  0x2020B0 - (0x201006 + 2) - 4 = 4260
-//  0x2020A8 - (0x20100c + 2) - 4 = 4262
+//  0x202338 - (0x201270 + 2) - 4 = 4290
+//  0x202338 - (0x201276 + 2) - 4 = 4284
+//  0x202340 - (0x20127c + 2) - 4 = 4286
 
 // DISASM:      _start:
-// DISASM-NEXT:  201000:  {{.*}}  jmpq  *4266(%rip)
-// DISASM-NEXT:  201006:  {{.*}}  jmpq  *4260(%rip)
-// DISASM-NEXT:  20100c:  {{.*}}  jmpq  *4262(%rip)
+// DISASM-NEXT:  201270:  {{.*}}  jmpq  *4290(%rip)
+// DISASM-NEXT:  201276:  {{.*}}  jmpq  *4284(%rip)
+// DISASM-NEXT:  20127c:  {{.*}}  jmpq  *4286(%rip)
 
 .global _start
 _start:
diff --git a/lld/test/ELF/hexagon-gotrel.s b/lld/test/ELF/hexagon-gotrel.s
index 52edc0dc59ec1..74660bed2f448 100644
--- a/lld/test/ELF/hexagon-gotrel.s
+++ b/lld/test/ELF/hexagon-gotrel.s
@@ -1,7 +1,7 @@
 # REQUIRES: hexagon
 # RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %s -o %t.o
 # RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %S/Inputs/hexagon-shared.s -o %t2.o
-# RUN: ld.lld -shared %t2.o -o %t2.so
+# RUN: ld.lld -shared %t2.o -soname=so -o %t2.so
 # RUN: ld.lld -shared %t.o %t2.so -o %t3.so
 # RUN: llvm-objdump --print-imm-hex -d -j .text %t3.so | FileCheck --check-prefix=TEXT %s
 
@@ -19,9 +19,9 @@ foo:
 # R_HEX_GOTREL_32_6_X and R_HEX_GOTREL_16_X
   r0 = ##(.Lpc@GOTREL)
 
-# TEXT: r0.l = #0x0 }
-# TEXT: r0.h = #0xfffe }
-# TEXT: immext(#0xfffe0000)
-# TEXT: r0 = memw(r1+##-0x20000) }
-# TEXT: immext(#0xfffe0000)
-# TEXT: r0 = ##-0x20000 }
+# TEXT: r0.l = #0xffa8 }
+# TEXT: r0.h = #0xfffd }
+# TEXT: immext(#0xfffdff80)
+# TEXT: r0 = memw(r1+##-0x20058) }
+# TEXT: immext(#0xfffdff80)
+# TEXT: r0 = ##-0x20058 }
diff --git a/lld/test/ELF/hexagon-shared.s b/lld/test/ELF/hexagon-shared.s
index 98a5f2ddbdee6..4ec38d5c369c8 100644
--- a/lld/test/ELF/hexagon-shared.s
+++ b/lld/test/ELF/hexagon-shared.s
@@ -1,8 +1,8 @@
 # REQUIRES: hexagon
 # RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %s -o %t
 # RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %S/Inputs/hexagon-shared.s -o %t2.o
-# RUN: ld.lld -shared %t2.o -soname %t3.so -o %t3.so
-# RUN: ld.lld -shared %t %t3.so -soname %t4.so -o %t4.so
+# RUN: ld.lld -shared %t2.o -soname so -o %t3.so
+# RUN: ld.lld -shared %t %t3.so -o %t4.so
 # RUN: llvm-objdump -d -j .plt %t4.so | FileCheck --check-prefix=PLT %s
 # RUN: llvm-objdump -d -j .text %t4.so | FileCheck --check-prefix=TEXT %s
 # RUN: llvm-objdump -D -j .got %t4.so | FileCheck --check-prefix=GOT %s
@@ -40,26 +40,26 @@ pvar:
    .word var
    .size pvar, 4
 
-# PLT: { immext(#131008
-# PLT: r28 = add(pc,##131032) }
+# PLT: { immext(#131200
+# PLT: r28 = add(pc,##131252) }
 # PLT: { r14 -= add(r28,#16)
 # PLT: r15 = memw(r28+#8)
 # PLT: r28 = memw(r28+#4) }
 # PLT: { r14 = asr(r14,#2)
 # PLT: jumpr r28 }
 # PLT: { trap0(#219) }
-# PLT: immext(#131008)
-# PLT: r14 = add(pc,##131016) }
+# PLT: immext(#131200)
+# PLT: r14 = add(pc,##131236) }
 # PLT: r28 = memw(r14+#0) }
 # PLT: jumpr r28 }
 
-# TEXT:  10000: 00 00 01 00 00010000
-# TEXT: { 	call 0x10050 }
-# TEXT: r0 = add(r1,##-65416) }
+# TEXT:  10218: 68 00 01 00 00010068
+# TEXT: { 	call 0x10270 }
+# TEXT: r0 = add(r1,##-65548) }
 
 # GOT: .got:
-# GOT: 20080:	00 00 00 00 00000000 <unknown>
+# GOT: 202f8:	00 00 00 00 00000000 <unknown>
 
-# RELO: 00020080  00000121 R_HEX_GLOB_DAT
-# RELO: 00030004  00000406 R_HEX_32
-# RELO: 00030018  00000122 R_HEX_JMP_SLOT
+# RELO: 000202f8  00000121 R_HEX_GLOB_DAT
+# RELO: 00030300  00000406 R_HEX_32
+# RELO: 00030314  00000122 R_HEX_JMP_SLOT
diff --git a/lld/test/ELF/hexagon.s b/lld/test/ELF/hexagon.s
index c8dd271815e8d..394e6f00600bf 100644
--- a/lld/test/ELF/hexagon.s
+++ b/lld/test/ELF/hexagon.s
@@ -1,156 +1,156 @@
 # REQUIRES: hexagon
-# RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %s -o %t
-# RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %S/Inputs/hexagon.s -o %t2
-# RUN: ld.lld %t2 %t  -o %t3
-# RUN: llvm-objdump -d  %t3 | FileCheck %s
+# RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %s -o %t.o
+# RUN: llvm-mc -filetype=obj -triple=hexagon-unknown-elf %S/Inputs/hexagon.s -o %t1.o
+# RUN: ld.lld %t.o %t1.o -o %t
+# RUN: llvm-objdump -d %t | FileCheck %s
 
-# Note: 131072 == 0x20000
+# Note: 131648 == 0x20240
 # R_HEX_32_6_X
 # R_HEX_12_X
 if (p0) r0 = ##_start
-# CHECK: immext(#131072)
-# CHECK: if (p0) r0 = ##131072
+# CHECK: immext(#131648)
+# CHECK: if (p0) r0 = ##131648
 
 # R_HEX_B15_PCREL
 if (p0) jump:nt #_start
-# CHECK: if (p0) jump:nt 0x20000
+# CHECK: if (p0) jump:nt 0x20240
 
 # R_HEX_B32_PCREL_X
 # R_HEX_B15_PCREL_X
 if (p0) jump:nt ##_start
-# CHECK: if (p0) jump:nt 0x20000
+# CHECK: if (p0) jump:nt 0x20240
 
 # R_HEX_B22_PCREL
 call #_start
-# CHECK: call 0x20000
+# CHECK: call 0x20240
 
 # R_HEX_B32_PCREL_X
 # R_HEX_B22_PCREL_X
 call ##_start
-# CHECK: immext(#4294967232)
-# CHECK: call 0x20000
+# CHECK: immext(#320)
+# CHECK: call 0x20240
 
 # R_HEX_6_X tests:
 # One test for each mask in the lookup table.
 
 #0x38000000
 if (!P0) memw(r0+#8)=##_start
-# CHECK: 38c0c100   	if (!p0) memw(r0+#8) = ##131072 }
+# CHECK: 38c0c100   	if (!p0) memw(r0+#8) = ##131648 }
 
 #0x39000000
 { p0 = p1
   if (!P0.new) memw(r0+#0)=##_start }
-# CHECK: 39c0c000   	if (!p0.new) memw(r0+#0) = ##131072 }
+# CHECK: 39c0c000   	if (!p0.new) memw(r0+#0) = ##131648 }
 
 #0x3e000000
 memw(r0+##_start)+=r1
-# CHECK: 3e40c001   	memw(r0+##131072) += r1 }
+# CHECK: 3e40c001   	memw(r0+##131648) += r1 }
 
 #0x3f000000
 memw(r0+##_start)+=#4
-# CHECK: 3f40c004   	memw(r0+##131072) += #4 }
+# CHECK: 3f40c004   	memw(r0+##131648) += #4 }
 
 #0x40000000
 { r0 = r1
   if (p0) memb(r0+##_start)=r0.new }
-# CHECK: 40a0c200   	if (p0) memb(r0+##131072) = r0.new }
+# CHECK: 40a0c200   	if (p0) memb(r0+##131648) = r0.new }
 
 #0x41000000
 if (p0) r0=memb(r1+##_start)
-# CHECK: 4101c000   	if (p0) r0 = memb(r1+##131072) }
+# CHECK: 4101c000   	if (p0) r0 = memb(r1+##131648) }
 
 #0x42000000
 { r0 = r1
   p0 = p1
   if (p0.new) memb(r0+##_start)=r0.new }
-# CHECK: 42a0c200   	if (p0.new) memb(r0+##131072) = r0.new }
+# CHECK: 42a0c200   	if (p0.new) memb(r0+##131648) = r0.new }
 
 #0x43000000
 { p0 = p1
  if (P0.new) r0=memb(r0+##_start) }
-# CHECK: 4300c000   	if (p0.new) r0 = memb(r0+##131072) }
+# CHECK: 4300c000   	if (p0.new) r0 = memb(r0+##131648) }
 
 #0x44000000
 if (!p0) memb(r0+##_start)=r1
-# CHECK: 4400c100   	if (!p0) memb(r0+##131072) = r1 }
+# CHECK: 4400c100   	if (!p0) memb(r0+##131648) = r1 }
 
 #0x45000000
 if (!p0) r0=memb(r1+##_start)
-# CHECK: 4501c000   	if (!p0) r0 = memb(r1+##131072) }
+# CHECK: 4501c000   	if (!p0) r0 = memb(r1+##131648) }
 
 #0x46000000
 { p0 = p1
   if (!p0.new) memb(r0+##_start)=r1 }
-# CHECK: 4600c100   	if (!p0.new) memb(r0+##131072) = r1 }
+# CHECK: 4600c100   	if (!p0.new) memb(r0+##131648) = r1 }
 
 #0x47000000
 { p0 = p1
   if (!p0.new) r0=memb(r1+##_start) }
-# CHECK: 4701c000   	if (!p0.new) r0 = memb(r1+##131072) }
+# CHECK: 4701c000   	if (!p0.new) r0 = memb(r1+##131648) }
 
 #0x6a000000 -- Note 4294967132 == -0xa4 the distance between
 #              here and _start, so this will change if
 #              tests are added between here and _start
 r0=add(pc,##_start@pcrel)
-# CHECK: 6a49ce00  	r0 = add(pc,##4294967132) }
+# CHECK: 6a49d800  	r0 = add(pc,##240) }
 
 #0x7c000000
 r1:0=combine(#8,##_start)
-# CHECK: 7c80c100   	r1:0 = combine(#8,##131072) }
+# CHECK: 7c80c100   	r1:0 = combine(#8,##131648) }
 
 #0x9a000000
 r1:0=memb_fifo(r2=##_start)
-# CHECK: 9a82d000   	r1:0 = memb_fifo(r2=##131072) }
+# CHECK: 9a82d000   	r1:0 = memb_fifo(r2=##131648) }
 
 #0x9b000000
 r0=memb(r1=##_start)
-# CHECK: 9b01d000   	r0 = memb(r1=##131072) }
+# CHECK: 9b01d000   	r0 = memb(r1=##131648) }
 
 #0x9c000000
 r1:0=memb_fifo(r2<<#2+##_start)
-# CHECK: 9c82f000   	r1:0 = memb_fifo(r2<<#2+##131072) }
+# CHECK: 9c82f000   	r1:0 = memb_fifo(r2<<#2+##131648) }
 
 #0x9d000000
 r0=memb(r1<<#2+##_start)
-# CHECK: 9d01f000   	r0 = memb(r1<<#2+##131072) }
+# CHECK: 9d01f000   	r0 = memb(r1<<#2+##131648) }
 
 #0x9f000000
 if (!p0) r0=memb(##_start)
-# CHECK: 9f00e880   	if (!p0) r0 = memb(##131072) }
+# CHECK: 9f00e880   	if (!p0) r0 = memb(##131648) }
 
 #0xab000000
 memb(r0=##_start)=r1
-# CHECK: ab00c180   	memb(r0=##131072) = r1 }
+# CHECK: ab00c180   	memb(r0=##131648) = r1 }
 
 #0xad000000
 memb(r0<<#2+##_start)=r1
-# CHECK: ad00e180   	memb(r0<<#2+##131072) = r1 }
+# CHECK: ad00e180   	memb(r0<<#2+##131648) = r1 }
 
 #0xaf000000
 if (!p0) memb(##_start)=r1
-# CHECK: af00c184   	if (!p0) memb(##131072) = r1 }
+# CHECK: af00c184   	if (!p0) memb(##131648) = r1 }
 
 #0xd7000000
 r0=add(##_start,mpyi(r1,r2))
-# CHECK: d701c200   	r0 = add(##131072,mpyi(r1,r2)) }
+# CHECK: d701c200   	r0 = add(##131648,mpyi(r1,r2)) }
 
 #0xd8000000
 R0=add(##_start,mpyi(r0,#2))
-# CHECK: d800c002   	r0 = add(##131072,mpyi(r0,#2)) }
+# CHECK: d800c002   	r0 = add(##131648,mpyi(r0,#2)) }
 
 #0xdb000000
 r0=add(r1,add(r2,##_start))
-# CHECK: db01c002   	r0 = add(r1,add(r2,##131072)) }
+# CHECK: db01c002   	r0 = add(r1,add(r2,##131648)) }
 
 #0xdf000000
 r0=add(r1,mpyi(r2,##_start))
-# CHECK: df82c001   	r0 = add(r1,mpyi(r2,##131072)) }
+# CHECK: df82c001   	r0 = add(r1,mpyi(r2,##131648)) }
 
 # Duplex form of R_HEX_6_X
 # R_HEX_32_6_X
 # R_HEX_6_X
 { r0 = ##_start; r2 = r16 }
-# CHECK: 28003082   	r0 = ##131072; 	r2 = r16 }
+# CHECK: 28003082   	r0 = ##131648; 	r2 = r16 }
 
 # R_HEX_HI16
 r0.h = #HI(_start)
@@ -158,66 +158,66 @@ r0.h = #HI(_start)
 
 # R_HEX_LO16
 r0.l = #LO(_start)
-# CHECK: r0.l = #0
+# CHECK: r0.l = #576
 
 # R_HEX_8_X has 3 relocation mask variations
 #0xde000000
 r0=sub(##_start, asl(r0, #1))
-# CHECK: de00c106      r0 = sub(##131072,asl(r0,#1)) }
+# CHECK: de40c106      r0 = sub(##131648,asl(r0,#1)) }
 
 #0x3c000000
 memw(r0+#0) = ##_start
-# CHECK: 3c40c000   	memw(r0+#0) = ##131072 }
+# CHECK: 3c40c040   	memw(r0+#0) = ##131648 }
 
 # The rest:
 r1:0=combine(r2,##_start);
-# CHECK: 7302e000   	r1:0 = combine(r2,##131072) }
+# CHECK: 7302e800   	r1:0 = combine(r2,##131648) }
 
 # R_HEX_32:
 r_hex_32:
 .word _start
-# CHECK: 00020000
+# CHECK: 00020240
 
 # R_HEX_16_X has 4 relocation mask variations
 # 0x48000000
 memw(##_start) = r0
-# CHECK: 4880c000   memw(##131072) = r0 }
+# CHECK: 4880c000   memw(##131648) = r0 }
 
 # 0x49000000
 r0 = memw(##_start)
-# CHECK: 4980c000   r0 = memw(##131072)
+# CHECK: 4980c000   r0 = memw(##131648)
 
 # 0x78000000
 r0 = ##_start
-# CHECK: 7800c000   r0 = ##131072 }
+# CHECK: 7800c000   r0 = ##131648 }
 
 # 0xb0000000
 r0 = add(r1, ##_start)
-# CHECK: b001c000   r0 = add(r1,##131072) }
+# CHECK: b001c000   r0 = add(r1,##131648) }
 
 # R_HEX_B9_PCREL:
 {r0=#1 ; jump #_start}
-# CHECK: jump 0x20000
+# CHECK: jump 0x20240
 
 # R_HEX_B9_PCREL_X:
 {r0=#1 ; jump ##_start}
-# CHECK: jump 0x20000
+# CHECK: jump 0x20240
 
 # R_HEX_B13_PCREL
 if (r0 == #0) jump:t #_start
-# CHECK: if (r0==#0) jump:t 0x20000
+# CHECK: if (r0==#0) jump:t 0x20240
 
 # R_HEX_9_X
 p0 = !cmp.gtu(r0, ##_start)
-# CHECK: p0 = !cmp.gtu(r0,##131072)
+# CHECK: p0 = !cmp.gtu(r0,##131648)
 
 # R_HEX_10_X
 p0 = !cmp.gt(r0, ##_start)
-# CHECK: p0 = !cmp.gt(r0,##131072)
+# CHECK: p0 = !cmp.gt(r0,##131648)
 
 # R_HEX_11_X
 r0 = memw(r1+##_start)
-# CHECK: r0 = memw(r1+##131072)
+# CHECK: r0 = memw(r1+##131648)
 
 memw(r0+##_start) = r1
-# CHECK: memw(r0+##131072) = r1
+# CHECK: memw(r0+##131648) = r1
diff --git a/lld/test/ELF/image-base.s b/lld/test/ELF/image-base.s
index d8c6ba70061af..f19f900e79b8d 100644
--- a/lld/test/ELF/image-base.s
+++ b/lld/test/ELF/image-base.s
@@ -44,9 +44,9 @@ _start:
 # CHECK-NEXT:   }
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD (0x1)
-# CHECK-NEXT:     Offset: 0x1000
-# CHECK-NEXT:     VirtualAddress: 0x1001000
-# CHECK-NEXT:     PhysicalAddress: 0x1001000
+# CHECK-NEXT:     Offset: 0x120
+# CHECK-NEXT:     VirtualAddress: 0x1001120
+# CHECK-NEXT:     PhysicalAddress: 0x1001120
 # CHECK-NEXT:     FileSize: 1
 # CHECK-NEXT:     MemSize: 1
 # CHECK-NEXT:     Flags [ (0x5)
diff --git a/lld/test/ELF/just-symbols.s b/lld/test/ELF/just-symbols.s
index 856cf8c581ec1..cbceb2a1b2fd5 100644
--- a/lld/test/ELF/just-symbols.s
+++ b/lld/test/ELF/just-symbols.s
@@ -6,7 +6,7 @@
 # RUN: ld.lld -just-symbols=%t1.exe -o %t2.exe
 # RUN: llvm-readelf -symbols %t2.exe | FileCheck %s
 
-# CHECK: 0000000000011000    40 OBJECT  GLOBAL DEFAULT  ABS bar
+# CHECK: 0000000000011001    40 OBJECT  GLOBAL DEFAULT  ABS bar
 # CHECK: 0000000000010000     0 NOTYPE  GLOBAL DEFAULT  ABS foo
 
 .globl foo, bar
diff --git a/lld/test/ELF/linkerscript/absolute.s b/lld/test/ELF/linkerscript/absolute.s
index 54f1c7037d23f..c0db81b36848e 100644
--- a/lld/test/ELF/linkerscript/absolute.s
+++ b/lld/test/ELF/linkerscript/absolute.s
@@ -19,7 +19,7 @@
 # CHECK-NEXT: }
 
 # CHECK-RHS:        Name: foo
-# CHECK-RHS-NEXT:   Value: 0x201001
+# CHECK-RHS-NEXT:   Value: 0x201121
 # CHECK-RHS-NEXT:   Size:
 # CHECK-RHS-NEXT:   Binding:
 # CHECK-RHS-NEXT:   Type:
diff --git a/lld/test/ELF/linkerscript/arm-exidx-discard-all.s b/lld/test/ELF/linkerscript/arm-exidx-discard-all.s
new file mode 100644
index 0000000000000..ce74b279e583e
--- /dev/null
+++ b/lld/test/ELF/linkerscript/arm-exidx-discard-all.s
@@ -0,0 +1,19 @@
+// REQUIRES: arm
+// RUN: llvm-mc -filetype=obj -triple arm-gnu-linux-eabi -mcpu cortex-a7 -arm-add-build-attributes %s -o %t.o
+// RUN: echo "ENTRY(__entrypoint) SECTIONS { /DISCARD/ : { *(.text.1) } }" > %t.script
+// RUN: ld.lld -T %t.script %t.o -o %t.elf
+// RUN: llvm-readobj --sections %t.elf | FileCheck %s
+
+/// Test that when we /DISCARD/ all the input sections with associated
+/// .ARM.exidx sections then we also discard all the .ARM.exidx sections.
+
+ .section .text.1, "ax", %progbits
+ .global foo
+ .type foo, %function
+ .fnstart
+foo:
+  bx lr
+  .cantunwind
+  .fnend
+
+// CHECK-NOT: .ARM.exidx
diff --git a/lld/test/ELF/arm-exidx-discard.s b/lld/test/ELF/linkerscript/arm-exidx-discard.s
similarity index 100%
rename from lld/test/ELF/arm-exidx-discard.s
rename to lld/test/ELF/linkerscript/arm-exidx-discard.s
diff --git a/lld/test/ELF/linkerscript/dynamic.s b/lld/test/ELF/linkerscript/dynamic.s
index d236fba77b084..c69e752f9ad67 100644
--- a/lld/test/ELF/linkerscript/dynamic.s
+++ b/lld/test/ELF/linkerscript/dynamic.s
@@ -1,10 +1,14 @@
 # REQUIRES: x86
+
+## Test that DT_INIT_ARRAYSZ/DT_FINI_ARRAYSZ/DT_PREINIT_ARRAYSZ are computed
+## correctly, no matter their associated sections are orphans or not.
+
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t1.o
 # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/shared.s -o %t2.o
-# RUN: ld.lld -shared %t2.o -o %t2.so
+# RUN: ld.lld -shared %t2.o -soname=so -o %t2.so
 
-# RUN: echo "SECTIONS { }" > %t.script
-# RUN: ld.lld %t1.o %t2.so -o %t
+# RUN: echo "SECTIONS { .init_array : { *(.init_array) } }" > %t.script
+# RUN: ld.lld -T %t.script %t1.o %t2.so -o %t
 # RUN: llvm-readobj --dynamic-table %t | FileCheck %s
 
 # CHECK:      DynamicSection [
diff --git a/lld/test/ELF/linkerscript/merge-output-sections.s b/lld/test/ELF/linkerscript/merge-output-sections.s
new file mode 100644
index 0000000000000..b15596f84cf66
--- /dev/null
+++ b/lld/test/ELF/linkerscript/merge-output-sections.s
@@ -0,0 +1,35 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+## SHF_MERGE sections within the same output section can be freely merged.
+# RUN: echo 'SECTIONS { .rodata : { *(.rodata.*) }}' > %t.script
+# RUN: ld.lld %t.o -T %t.script -o %t
+# RUN: llvm-readelf -x .rodata %t | FileCheck --check-prefix=SAME %s --implicit-check-not=section
+
+# SAME:      section '.rodata':
+# SAME-NEXT: 0x00000000 01000200 0300
+
+## SHF_MERGE sections with different output sections cannot be merged.
+# RUN: echo 'SECTIONS { \
+# RUN:   .rodata.foo : { *(.rodata.foo) } \
+# RUN:   .rodata.bar : { *(.rodata.bar) } \
+# RUN: }' > %t2.script
+# RUN: ld.lld %t.o -T %t2.script -o %t2
+# RUN: llvm-readelf -x .rodata.foo -x .rodata.bar %t2 | FileCheck --check-prefix=DIFF %s --implicit-check-not=section
+
+# DIFF:      section '.rodata.foo':
+# DIFF-NEXT: 0x00000000 01000200 0300
+# DIFF:      section '.rodata.bar':
+# DIFF-NEXT: 0x00000006 0100
+
+.section .rodata.foo,"aM",@progbits,2,unique,0
+.short 1
+.short 2
+.section .rodata.foo,"aM",@progbits,2,unique,1
+.short 1
+.short 3
+
+.section .rodata.bar,"aM",@progbits,2,unique,0
+.short 1
+.section .rodata.bar,"aM",@progbits,2,unique,1
+.short 1
diff --git a/lld/test/ELF/linkerscript/merge-sections.s b/lld/test/ELF/linkerscript/merge-sections.s
index b9ebfe79714ab..ea53ba3e4201a 100644
--- a/lld/test/ELF/linkerscript/merge-sections.s
+++ b/lld/test/ELF/linkerscript/merge-sections.s
@@ -17,7 +17,7 @@
 # CHECK-NEXT:   ]
 # CHECK-NEXT:   Address: 0x[[ADDR1:.*]]
 # CHECK-NEXT:   Offset: 0x[[ADDR1]]
-# CHECK-NEXT:   Size: 14
+# CHECK-NEXT:   Size: 8
 # CHECK-NEXT:   Link: 0
 # CHECK-NEXT:   Info: 0
 # CHECK-NEXT:   AddressAlignment: 2
@@ -28,7 +28,7 @@
 # CHECK-NEXT: Value: 0x[[ADDR1]]
 
 # CHECK:      Name: end
-# CHECK-NEXT: Value: 0x236
+# CHECK-NEXT: Value: 0x230
 
 # Check that we don't crash with --gc-sections
 # RUN: ld.lld --gc-sections -o %t2 --script %t.script %t -shared
diff --git a/lld/test/ELF/linkerscript/page-size.s b/lld/test/ELF/linkerscript/page-size.s
index bf06611d9b321..eaa0acb7a0ad9 100644
--- a/lld/test/ELF/linkerscript/page-size.s
+++ b/lld/test/ELF/linkerscript/page-size.s
@@ -19,9 +19,9 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD
-# CHECK-NEXT:     Offset: 0x4000
-# CHECK-NEXT:     VirtualAddress: 0x204000
-# CHECK-NEXT:     PhysicalAddress: 0x204000
+# CHECK-NEXT:     Offset: 0x158
+# CHECK-NEXT:     VirtualAddress: 0x204158
+# CHECK-NEXT:     PhysicalAddress: 0x204158
 # CHECK-NEXT:     FileSize: 1
 # CHECK-NEXT:     MemSize: 1
 # CHECK-NEXT:     Flags [
@@ -32,9 +32,9 @@
 # CHECK-NEXT:   }
 # CHECK-NEXT:   ProgramHeader {
 # CHECK-NEXT:     Type: PT_LOAD
-# CHECK-NEXT:     Offset: 0x8000
-# CHECK-NEXT:     VirtualAddress: 0x208000
-# CHECK-NEXT:     PhysicalAddress: 0x208000
+# CHECK-NEXT:     Offset: 0x159
+# CHECK-NEXT:     VirtualAddress: 0x208159
+# CHECK-NEXT:     PhysicalAddress: 0x208159
 # CHECK-NEXT:     FileSize: 8
 # CHECK-NEXT:     MemSize: 8
 # CHECK-NEXT:     Flags [
diff --git a/lld/test/ELF/linkerscript/symbol-reserved.s b/lld/test/ELF/linkerscript/symbol-reserved.s
index a4c88daeb8000..0c686d2cf2562 100644
--- a/lld/test/ELF/linkerscript/symbol-reserved.s
+++ b/lld/test/ELF/linkerscript/symbol-reserved.s
@@ -34,7 +34,7 @@
 # RUN: ld.lld -o %t1 %t %t.script
 # RUN: llvm-objdump -t %t1 | FileCheck --check-prefix=RELATIVE %s
 # RELATIVE: 0000000000202005 .text 00000000 .hidden newsym
-# RELATIVE: 0000000000201007 .text 00000000 _end
+# RELATIVE: 0000000000201127 .text 00000000 _end
 
 # RUN: echo "PROVIDE_HIDDEN(newsym = ALIGN(_end, CONSTANT(MAXPAGESIZE)) + 5);" > %t.script
 # RUN: ld.lld -o %t1 --script %p/Inputs/symbol-reserved.script %t %t.script
diff --git a/lld/test/ELF/linkerscript/synthetic-symbols3.test b/lld/test/ELF/linkerscript/synthetic-symbols3.test
index a24ecccedf0b9..39e20e1468b7f 100644
--- a/lld/test/ELF/linkerscript/synthetic-symbols3.test
+++ b/lld/test/ELF/linkerscript/synthetic-symbols3.test
@@ -7,5 +7,5 @@
 PROVIDE_HIDDEN(_begin_sec = _start);
 PROVIDE_HIDDEN(_end_sec = ADDR(.text) + SIZEOF(.text));
 
-# CHECK:       0000000000201000         .text     00000000 .hidden _begin_sec
-# CHECK-NEXT:  0000000000201001         .text     00000000 .hidden _end_sec
+# CHECK:       00000000002011ac         .text     00000000 .hidden _begin_sec
+# CHECK-NEXT:  00000000002011ad         .text     00000000 .hidden _end_sec
diff --git a/lld/test/ELF/local-got-pie.s b/lld/test/ELF/local-got-pie.s
index 3a985458de5c7..b2a81a64f5032 100644
--- a/lld/test/ELF/local-got-pie.s
+++ b/lld/test/ELF/local-got-pie.s
@@ -13,13 +13,13 @@ _start:
 foo:
  nop
 
-// 0x20B0 - 1001 - 5 = 4266
+// 0x22C8 - 0x1210 - 5 = 4275
 // DISASM:      Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: _start:
-// DISASM-NEXT:   1000: {{.*}} callq 4267
+// DISASM-NEXT:   1210: {{.*}} callq 4275
 // DISASM:      foo:
-// DISASM-NEXT:   1005: {{.*}} nop
+// DISASM-NEXT:   1215: {{.*}} nop
 
 // CHECK:      Name: .got
 // CHECK-NEXT: Type: SHT_PROGBITS
@@ -27,13 +27,13 @@ foo:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x20B0
+// CHECK-NEXT: Address: 0x22C8
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 8
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:     0x20B0 R_X86_64_RELATIVE - 0x1005
+// CHECK-NEXT:     0x22C8 R_X86_64_RELATIVE - 0x1215
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 // CHECK:      0x000000006FFFFFF9 RELACOUNT            1
diff --git a/lld/test/ELF/local-got-shared.s b/lld/test/ELF/local-got-shared.s
index 34da3261e275a..979b282972e5a 100644
--- a/lld/test/ELF/local-got-shared.s
+++ b/lld/test/ELF/local-got-shared.s
@@ -12,12 +12,12 @@ bar:
 foo:
         nop
 
-// 0x20B0 - 0x1000 - 5 = 4267
+// 0x22E0 - 0x1228 - 5 = 4275
 // DISASM:      bar:
-// DISASM-NEXT:   1000:       callq 4267
+// DISASM-NEXT:   1228:       callq 4275
 
 // DISASM:      foo:
-// DISASM-NEXT:   1005:       nop
+// DISASM-NEXT:   122d:       nop
 
 // CHECK:      Name: .got
 // CHECK-NEXT: Type: SHT_PROGBITS
@@ -25,13 +25,13 @@ foo:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x20B0
+// CHECK-NEXT: Address: 0x22E0
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 8
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:     0x20B0 R_X86_64_RELATIVE - 0x1005
+// CHECK-NEXT:     0x22E0 R_X86_64_RELATIVE - 0x122D
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 // CHECK:      0x000000006FFFFFF9 RELACOUNT            1
diff --git a/lld/test/ELF/local-got.s b/lld/test/ELF/local-got.s
index 0ea1d664717b8..c9f1d9ee9ab44 100644
--- a/lld/test/ELF/local-got.s
+++ b/lld/test/ELF/local-got.s
@@ -15,14 +15,14 @@ _start:
 foo:
         nop
 
-// 0x2020C0 - 0x201000 - 5 =  4283
-// 0x2020C8 - 0x201005 - 5 =  4286
+// 0x202320 - 0x201250 - 5 =  4299
+// 0x202328 - 0x201255 - 5 =  4302
 // DISASM:      _start:
-// DISASM-NEXT:   201000:       callq 4283
-// DISASM-NEXT:   201005:       callq 4286
-                            
-// DISASM:      foo:        
-// DISASM-NEXT:   20100a:       nop
+// DISASM-NEXT:   201250:       callq 4299
+// DISASM-NEXT:   201255:       callq 4302
+
+// DISASM:      foo:
+// DISASM-NEXT:   20125a:       nop
 
 // CHECK:      Name: .got
 // CHECK-NEXT: Type: SHT_PROGBITS
@@ -30,7 +30,7 @@ foo:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x2020C0
+// CHECK-NEXT: Address: 0x202320
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 16
 // CHECK-NEXT: Link: 0
@@ -38,11 +38,11 @@ foo:
 // CHECK-NEXT: AddressAlignment: 8
 // CHECK-NEXT: EntrySize: 0
 // CHECK-NEXT: SectionData (
-// CHECK-NEXT:   0000:  00000000 00000000 0A102000 00000000
+// CHECK-NEXT:   0000:  00000000 00000000 5A122000 00000000
 // CHECK-NEXT: )
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:     0x2020C0 R_X86_64_GLOB_DAT bar 0x0
+// CHECK-NEXT:     0x202320 R_X86_64_GLOB_DAT bar 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/lld/test/ELF/lto/thinlto-emit-imports.ll b/lld/test/ELF/lto/thinlto-emit-imports.ll
index f85f409a26a1c..c2d047757001f 100644
--- a/lld/test/ELF/lto/thinlto-emit-imports.ll
+++ b/lld/test/ELF/lto/thinlto-emit-imports.ll
@@ -14,15 +14,15 @@
 
 ; The imports file for this module contains the bitcode file for
 ; Inputs/thinlto.ll
-; RUN: cat %t1.o.imports | count 1
-; RUN: cat %t1.o.imports | FileCheck %s --check-prefix=IMPORTS1
+; RUN: count 1 < %t1.o.imports
+; RUN: FileCheck %s --check-prefix=IMPORTS1 < %t1.o.imports
 ; IMPORTS1: thinlto-emit-imports.ll.tmp2.o
 
 ; The imports file for Input/thinlto.ll is empty as it does not import anything.
-; RUN: cat %t2.o.imports | count 0
+; RUN: count 0 < %t2.o.imports
 
 ; The imports file for Input/thinlto_empty.ll is empty but should exist.
-; RUN: cat %t3.o.imports | count 0
+; RUN: count 0 < %t3.o.imports
 
 ; The index file should be created even for the input with an empty summary.
 ; RUN: ls %t3.o.thinlto.bc
@@ -43,6 +43,13 @@
 ; RUN: not ls %t2.o.imports
 ; RUN: not ls %t3.o.imports
 
+; Check that imports files are generated also when -thinlto-index-only
+; is specified without --plugin-opt=.
+; RUN: rm -f %t1.o.imports
+; RUN: ld.lld -thinlto-index-only -thinlto-emit-imports-files -shared %t1.o %t2.o %t3.o -o %t4
+; RUN: count 1 < %t1.o.imports
+; RUN: FileCheck %s --check-prefix=IMPORTS1 < %t1.o.imports
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/lld/test/ELF/lto/thinlto-index-file.ll b/lld/test/ELF/lto/thinlto-index-file.ll
index 18f4350b3a4b8..5792f7b6a9bf5 100644
--- a/lld/test/ELF/lto/thinlto-index-file.ll
+++ b/lld/test/ELF/lto/thinlto-index-file.ll
@@ -6,12 +6,16 @@
 ; RUN: opt -module-summary %p/Inputs/thinlto_empty.ll -o %t3.o
 
 ; Ensure lld writes linked files to linked objects file
-; RUN: ld.lld --plugin-opt=thinlto-index-only=%t.idx -shared %t1.o %t2.o %t3.o -o %t4
+; RUN: ld.lld --plugin-opt=thinlto-index-only=%t.idx -shared %t1.o %t2.o %t3.o -o /dev/null
 ; RUN: FileCheck %s < %t.idx
 ; CHECK: {{.*}}thinlto-index-file.ll.tmp1.o
 ; CHECK: {{.*}}thinlto-index-file.ll.tmp2.o
 ; CHECK: {{.*}}thinlto-index-file.ll.tmp3.o
 
+; Check that this also works without the --plugin-opt= prefix.
+; RUN: ld.lld -thinlto-index-only=%t.idx -shared %t1.o %t2.o %t3.o -o /dev/null
+; RUN: FileCheck %s < %t.idx
+
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
diff --git a/lld/test/ELF/lto/thinlto-obj-path.ll b/lld/test/ELF/lto/thinlto-obj-path.ll
index 8878c8984d099..224b5b651861e 100644
--- a/lld/test/ELF/lto/thinlto-obj-path.ll
+++ b/lld/test/ELF/lto/thinlto-obj-path.ll
@@ -5,11 +5,16 @@
 
 ; Test to ensure that thinlto-index-only with obj-path creates the file.
 ; RUN: rm -f %t4.o
-; RUN: ld.lld --plugin-opt=thinlto-index-only --plugin-opt=obj-path=%t4.o -shared %t1.o %t2.o -o %t3
+; RUN: ld.lld --plugin-opt=thinlto-index-only --plugin-opt=obj-path=%t4.o -shared %t1.o %t2.o -o /dev/null
 ; RUN: llvm-readobj -h %t4.o | FileCheck %s
 ; RUN: llvm-nm %t4.o 2>&1 | FileCheck %s -check-prefix=NO-SYMBOLS
 ; NO-SYMBOLS: no symbols
 
+; Check that this also works without the --plugin-opt= prefix.
+; RUN: rm -f %t4.o
+; RUN: ld.lld -thinlto-index-only -lto-obj-path=%t4.o -shared %t1.o %t2.o -o /dev/null
+; RUN: llvm-readobj -h %t4.o | FileCheck %s
+
 ; CHECK: Format: ELF64-x86-64
 
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/lld/test/ELF/lto/thinlto-object-suffix-replace.ll b/lld/test/ELF/lto/thinlto-object-suffix-replace.ll
index 1749f816cd15a..0593e574b7246 100644
--- a/lld/test/ELF/lto/thinlto-object-suffix-replace.ll
+++ b/lld/test/ELF/lto/thinlto-object-suffix-replace.ll
@@ -21,6 +21,11 @@
 ; RUN: --plugin-opt=thinlto-object-suffix-replace=".thinlink.bc;.o" \
 ; RUN: -shared %t1.thinlink.bc -o %t3
 ; RUN: diff %t1.o.thinlto.bc.orig %t1.o.thinlto.bc
+; Also check that this works without the --plugin-opt= prefix.
+; RUN: ld.lld -thinlto-index-only \
+; RUN: -thinlto-object-suffix-replace=".thinlink.bc;.o" \
+; RUN: -shared %t1.thinlink.bc -o %t3
+; RUN: diff %t1.o.thinlto.bc.orig %t1.o.thinlto.bc
 
 ; Ensure lld generates error if object suffix replace option does not have 'old;new' format
 ; RUN: rm -f %t1.o.thinlto.bc
diff --git a/lld/test/ELF/lto/thinlto-prefix-replace.ll b/lld/test/ELF/lto/thinlto-prefix-replace.ll
index 67b11c74abf6e..2aa8e105ad3b9 100644
--- a/lld/test/ELF/lto/thinlto-prefix-replace.ll
+++ b/lld/test/ELF/lto/thinlto-prefix-replace.ll
@@ -9,6 +9,11 @@
 ; RUN: ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
 ; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
 
+; Check that this also works without the --plugin-opt= prefix.
+; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+; RUN: ld.lld -thinlto-index-only -thinlto-prefix-replace="%t/oldpath/;%t/newpath/" -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace
+; RUN: ls %t/newpath/thinlto_prefix_replace.o.thinlto.bc
+
 ; Ensure that lld generates error if prefix replace option does not have 'old;new' format
 ; RUN: rm -f %t/newpath/thinlto_prefix_replace.o.thinlto.bc
 ; RUN: not ld.lld --plugin-opt=thinlto-index-only --plugin-opt=thinlto-prefix-replace=abc:def -shared %t/oldpath/thinlto_prefix_replace.o -o %t/thinlto_prefix_replace 2>&1 | FileCheck %s --check-prefix=ERR
diff --git a/lld/test/ELF/map-file.s b/lld/test/ELF/map-file.s
index 2a23eb1925b4d..1071d85b6654d 100644
--- a/lld/test/ELF/map-file.s
+++ b/lld/test/ELF/map-file.s
@@ -57,44 +57,43 @@ labs = 0x1AB5
 // CHECK-NEXT: 200368           200368       2c     1         {{.*}}{{/|\\}}map-file.s.tmp1.o:(.eh_frame+0x0)
 // CHECK-NEXT: 200398           200398       14     1         {{.*}}{{/|\\}}map-file.s.tmp1.o:(.eh_frame+0x2c)
 // CHECK-NEXT: 2003b0           2003b0       18     1         {{.*}}{{/|\\}}map-file.s.tmp2.o:(.eh_frame+0x18)
-// CHECK-NEXT: 201000           201000       2d     4 .text
-// CHECK-NEXT: 201000           201000       28     4         {{.*}}{{/|\\}}map-file.s.tmp1.o:(.text)
-// CHECK-NEXT: 201000           201000        0     1                 _start
-// CHECK-NEXT: 20101f           20101f        0     1                 f(int)
-// CHECK-NEXT: 201028           201028        0     1                 local
-// CHECK-NEXT: 201028           201028        2     4         {{.*}}{{/|\\}}map-file.s.tmp2.o:(.text)
-// CHECK-NEXT: 201028           201028        0     1                 foo
-// CHECK-NEXT: 201029           201029        0     1                 bar
-// CHECK-NEXT: 20102a           20102a        0     1         {{.*}}{{/|\\}}map-file.s.tmp2.o:(.text.zed)
-// CHECK-NEXT: 20102a           20102a        0     1                 zed
-// CHECK-NEXT: 20102c           20102c        0     4         {{.*}}{{/|\\}}map-file.s.tmp3.o:(.text)
-// CHECK-NEXT: 20102c           20102c        0     1                 bah
-// CHECK-NEXT: 20102c           20102c        1     4         {{.*}}{{/|\\}}map-file.s.tmp4.a(map-file.s.tmp4.o):(.text)
-// CHECK-NEXT: 20102c           20102c        0     1                 baz
-// CHECK-NEXT: 201030           201030       30    16 .plt
-// CHECK-NEXT: 201030           201030       30    16         <internal>:(.plt)
-// CHECK-NEXT: 201040           201040        0     1                 sharedFunc1
-// CHECK-NEXT: 201050           201050        0     1                 sharedFunc2
-// CHECK-NEXT: 202000           202000      100     8 .dynamic
-// CHECK-NEXT: 202000           202000      100     8         <internal>:(.dynamic)
-// CHECK-NEXT: 203000           203000       28     8 .got.plt
-// CHECK-NEXT: 203000           203000       28     8         <internal>:(.got.plt)
-// CHECK-NEXT: 203030           203030       10    16 .bss
-// CHECK-NEXT: 203030           203030        4    16         {{.*}}{{/|\\}}map-file.s.tmp1.o:(COMMON)
-// CHECK-NEXT: 203030           203030        4     1                 common
-// CHECK-NEXT: 203034           203034        4     1         <internal>:(.bss)
-// CHECK-NEXT: 203034           203034        4     1                 sharedFoo
-// CHECK-NEXT: 203038           203038        8     1         <internal>:(.bss)
-// CHECK-NEXT: 203038           203038        8     1                 sharedBar
-// CHECK-NEXT:      0                0        8     1 .comment
-// CHECK-NEXT:      0                0        8     1         <internal>:(.comment)
-// CHECK-NEXT:      0                0      198     8 .symtab
-// CHECK-NEXT:      0                0      198     8         <internal>:(.symtab)
-// CHECK-NEXT:      0                0       84     1 .shstrtab
-// CHECK-NEXT:      0                0       84     1         <internal>:(.shstrtab)
-// CHECK-NEXT:      0                0       6d     1 .strtab
-// CHECK-NEXT:      0                0       6d     1         <internal>:(.strtab)
-
+// CHECK-NEXT: 2013cc           2013cc       2d     4 .text
+// CHECK-NEXT: 2013cc           2013cc       28     4         {{.*}}{{/|\\}}map-file.s.tmp1.o:(.text)
+// CHECK-NEXT: 2013cc           2013cc        0     1                 _start
+// CHECK-NEXT: 2013eb           2013eb        0     1                 f(int)
+// CHECK-NEXT: 2013f4           2013f4        0     1                 local
+// CHECK-NEXT: 2013f4           2013f4        2     4         {{.*}}{{/|\\}}map-file.s.tmp2.o:(.text)
+// CHECK-NEXT: 2013f4           2013f4        0     1                 foo
+// CHECK-NEXT: 2013f5           2013f5        0     1                 bar
+// CHECK-NEXT: 2013f6           2013f6        0     1         {{.*}}{{/|\\}}map-file.s.tmp2.o:(.text.zed)
+// CHECK-NEXT: 2013f6           2013f6        0     1                 zed
+// CHECK-NEXT: 2013f8           2013f8        0     4         {{.*}}{{/|\\}}map-file.s.tmp3.o:(.text)
+// CHECK-NEXT: 2013f8           2013f8        0     1                 bah
+// CHECK-NEXT: 2013f8           2013f8        1     4         {{.*}}{{/|\\}}map-file.s.tmp4.a(map-file.s.tmp4.o):(.text)
+// CHECK-NEXT: 2013f8           2013f8        0     1                 baz
+// CHECK-NEXT: 201400           201400       30    16 .plt
+// CHECK-NEXT: 201400           201400       30    16         <internal>:(.plt)
+// CHECK-NEXT: 201410           201410        0     1                 sharedFunc1
+// CHECK-NEXT: 201420           201420        0     1                 sharedFunc2
+// CHECK-NEXT: 202430           202430      100     8 .dynamic
+// CHECK-NEXT: 202430           202430      100     8         <internal>:(.dynamic)
+// CHECK-NEXT: 203530           203530       28     8 .got.plt
+// CHECk-NEXT: 203530           203530       28     8         <internal>:(.got.plt)
+// CHECk-NEXT: 203560           203560       10    16 .bss
+// CHECk-NEXT: 203560           203560        4    16         {{.*}}{{/|\\}}map-file.s.tmp1.o:(COMMON)
+// CHECk-NEXT: 203560           203560        4     1                 common
+// CHECk-NEXT: 203564           203564        4     1         <internal>:(.bss)
+// CHECk-NEXT: 203564           203564        4     1                 sharedFoo
+// CHECk-NEXT: 203568           203568        8     1         <internal>:(.bss)
+// CHECk-NEXT: 203568           203568        8     1                 sharedBar
+// CHECk-NEXT:      0                0       48     1 .comment
+// CHECk-NEXT:      0                0       48     1         <internal>:(.comment)
+// CHECk-NEXT:      0                0      198     8 .symtab
+// CHECk-NEXT:      0                0      198     8         <internal>:(.symtab)
+// CHECk-NEXT:      0                0       84     1 .shstrtab
+// CHECk-NEXT:      0                0       84     1         <internal>:(.shstrtab)
+// CHECk-NEXT:      0                0       6d     1 .strtab
+// CHECk-NEXT:      0                0       6d     1         <internal>:(.strtab)
 
 // RUN: not ld.lld %t1.o %t2.o %t3.o %t4.a -o %t -Map=/ 2>&1 \
 // RUN:  | FileCheck -check-prefix=FAIL %s
diff --git a/lld/test/ELF/merge-entsize2.s b/lld/test/ELF/merge-entsize2.s
index 26e40d3a55e37..25036beeea8e8 100644
--- a/lld/test/ELF/merge-entsize2.s
+++ b/lld/test/ELF/merge-entsize2.s
@@ -6,8 +6,8 @@
 # RUN: llvm-readelf -x .cst %t | FileCheck --check-prefix=HEX %s
 
 # RUN: ld.lld -O0 -r %t.o -o %t1.o
-# RUN: llvm-readelf -S %t1.o | FileCheck --check-prefix=SEC-R %s
-# RUN: llvm-readelf -x .cst %t1.o | FileCheck --check-prefix=HEX-R %s
+# RUN: llvm-readelf -S %t1.o | FileCheck --check-prefix=SEC %s
+# RUN: llvm-readelf -x .cst %t1.o | FileCheck --check-prefix=HEX %s
 
 ## Check that SHF_MERGE sections with the same name, sh_flags and sh_entsize
 ## are grouped together and can be merged within the group.
@@ -17,20 +17,10 @@
 # SEC:   Name  Type     {{.*}} Size   ES Flg Lk Inf Al
 # SEC:   .cst  PROGBITS {{.*}} 000020 00  AM  0   0  8
 
-## .cst 0 and .cst 1 are merged, but emitted as a separate output section.
-# SEC-R: .cst  PROGBITS {{.*}} 00000c 04  AM  0   0  4
-# SEC-R: .cst  PROGBITS {{.*}} 000010 08  AM  0   0  8
-
 # HEX:      Hex dump of section '.cst':
 # HEX-NEXT: 0x{{[0-9a-f]+}} 01000000 00000000 02000000 00000000
 # HEX-NEXT: 0x{{[0-9a-f]+}} 01000000 00000000 03000000 00000000
 
-# HEX-R:      Hex dump of section '.cst':
-# HEX-R-NEXT: 0x00000000 01000000 00000000 02000000
-# HEX-R-EMPTY:
-# HEX-R-NEXT: Hex dump of section '.cst':
-# HEX-R-NEXT: 0x00000000 01000000 00000000 03000000 00000000
-
 .section .cst,"aM",@progbits,4,unique,0
 .align 2
 .long 1
diff --git a/lld/test/ELF/mips-elf-flags-binary.s b/lld/test/ELF/mips-elf-flags-binary.s
new file mode 100644
index 0000000000000..3278fa0c7e23f
--- /dev/null
+++ b/lld/test/ELF/mips-elf-flags-binary.s
@@ -0,0 +1,25 @@
+# REQUIRES: mips
+# Check deducing MIPS specific ELF header flags from `emulation`.
+
+# RUN: echo -n "BLOB" > %t.binary
+# RUN: ld.lld -m elf32btsmip -r -b binary %t.binary -o %t.out
+# RUN: llvm-readobj -h %t.out | FileCheck -check-prefix=O32 %s
+
+# RUN: echo -n "BLOB" > %t.binary
+# RUN: ld.lld -m elf32btsmipn32 -r -b binary %t.binary -o %t.out
+# RUN: llvm-readobj -h %t.out | FileCheck -check-prefix=N32 %s
+
+# RUN: echo -n "BLOB" > %t.binary
+# RUN: ld.lld -m elf64btsmip -r -b binary %t.binary -o %t.out
+# RUN: llvm-readobj -h %t.out | FileCheck -check-prefix=N64 %s
+
+# O32:      Flags [
+# O32-NEXT:   EF_MIPS_ABI_O32
+# O32-NEXT: ]
+
+# N32:      Flags [
+# N32-NEXT:   EF_MIPS_ABI2
+# N32-NEXT: ]
+
+# N64:      Flags [
+# N64-NEXT: ]
diff --git a/lld/test/ELF/mips-jalr.s b/lld/test/ELF/mips-jalr.s
index a4a51ad5ca8de..5b3d3967aedee 100644
--- a/lld/test/ELF/mips-jalr.s
+++ b/lld/test/ELF/mips-jalr.s
@@ -1,20 +1,56 @@
 # REQUIRES: mips
-# Check that lld ignores R_MIPS_JALR relocation for now.
 
-# RUN: llvm-mc -filetype=obj -triple=mips-unknown-linux %s -o %t.o
-# RUN: ld.lld %t.o -o %t.exe 
+## Check handling of the R_MIPS_JALR relocation.
+
+# RUN: llvm-mc -filetype=obj -triple=mipsel-unknown-linux %s -o %t.o
 # RUN: llvm-readelf -r %t.o | FileCheck -check-prefix=REL %s
-# RUN: llvm-objdump -d --no-show-raw-insn %t.exe | FileCheck %s
 
-# REL: R_MIPS_CALL16 {{.*}} foo
+# RUN: ld.lld %t.o -shared -o %t.so
+# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck -check-prefix=SO %s
+
+# RUN: ld.lld %t.o --defsym=bar=__start -o %t.so
+# RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck -check-prefix=EXE %s
+
+# REL: R_MIPS_JALR   {{.*}} bar
 # REL: R_MIPS_JALR   {{.*}} foo
+# REL: R_MIPS_JALR   {{.*}} far
+
+# SO: jalr  $25
+# SO: bal   -24 <foo>
+# SO: jalr  $25
+
+# SO: jr    $25
+# SO: b     -64 <foo>
+# SO: jr    $25
 
-# CHECK: jalr  $25
+# EXE: bal   -4 <foo>
+# EXE: bal   -24 <foo>
+# EXE: jalr  $25
+
+# EXE: b     -56 <foo>
+# EXE: b     -64 <foo>
+# EXE: jr    $25
 
   .text
-  .global  __start
+  .global bar
+  .global __start
   .option pic2
+far:
+  .space 0x4fff0
 __start:
-  jal foo
 foo:
+  jal bar
+  nop
+  jal foo
+  nop
+  jal far
   nop
+l1:
+  jr $25
+  .reloc l1, R_MIPS_JALR, bar
+l2:
+  jr $25
+  .reloc l2, R_MIPS_JALR, foo
+l3:
+  jr $25
+  .reloc l3, R_MIPS_JALR, far
diff --git a/lld/test/ELF/no-inhibit-exec.s b/lld/test/ELF/no-inhibit-exec.s
index 203d37da44cc4..aadfd157f50ce 100644
--- a/lld/test/ELF/no-inhibit-exec.s
+++ b/lld/test/ELF/no-inhibit-exec.s
@@ -8,7 +8,7 @@
 # CHECK: Disassembly of section .text:
 # CHECK-EMPTY:
 # CHECK-NEXT: _start
-# CHECK-NEXT: 201000: {{.*}} callq -2101253
+# CHECK-NEXT: 201120: {{.*}} callq -2101541
 
 # RELOC:      Relocations [
 # RELOC-NEXT: ]
diff --git a/lld/test/ELF/partition-dynamic-linker.s b/lld/test/ELF/partition-dynamic-linker.s
new file mode 100644
index 0000000000000..7aafedd4137f4
--- /dev/null
+++ b/lld/test/ELF/partition-dynamic-linker.s
@@ -0,0 +1,25 @@
+## Test that we don't create a .ARM.exidx for the main partition.
+## Previously we were doing so, which is unnecessary and led to a crash.
+
+# REQUIRES: arm
+# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %p/Inputs/shared.s -o %t1.o
+# RUN: ld.lld -shared %t1.o -o %t.so
+# RUN: llvm-mc -filetype=obj -triple=armv7a-none-linux-gnueabi %s -o %t.o
+
+# RUN: ld.lld -shared --gc-sections --dynamic-linker foo %t.o %t.so -o %t
+# RUN: llvm-readelf --section-headers %t | FileCheck %s
+
+# CHECK: .ARM.exidx
+# CHECK-NOT: .ARM.exidx
+
+.section .llvm_sympart,"",%llvm_sympart
+.asciz "part1"
+.4byte p1
+
+.section .text.p1,"ax",%progbits
+.globl p1
+p1:
+.fnstart
+bx lr
+.cantunwind
+.fnend
diff --git a/lld/test/ELF/partition-notes.s b/lld/test/ELF/partition-notes.s
index 3c16f2c71e504..fd9803bbe595a 100644
--- a/lld/test/ELF/partition-notes.s
+++ b/lld/test/ELF/partition-notes.s
@@ -22,7 +22,7 @@
 // CHECK-NEXT: Displaying notes
 // CHECK-NEXT: Owner
 // CHECK-NEXT: GNU                   0x00000014	NT_GNU_BUILD_ID (unique build ID bitstring)
-// CHECK-NEXT: Build ID: 0f4d5297cbbe52e4bea558eeb792944670de22e1
+// CHECK-NEXT: Build ID: 08b93eab87177a2356d1b0d1148339463f98dac2
 
 .section .llvm_sympart,"",@llvm_sympart
 .asciz "part1"
diff --git a/lld/test/ELF/partition-pack-dyn-relocs.s b/lld/test/ELF/partition-pack-dyn-relocs.s
index d317759fc9c90..8b12ac1d02d91 100644
--- a/lld/test/ELF/partition-pack-dyn-relocs.s
+++ b/lld/test/ELF/partition-pack-dyn-relocs.s
@@ -15,17 +15,19 @@
 // CHECK: Section Headers:
 // CHECK: .rela.dyn      ANDROID_RELA {{0*}}[[ANDROID_RELA_ADDR:[^ ]*]]
 // CHECK: .relr.dyn      RELR         {{0*}}[[RELR_ADDR:[^ ]*]]
-// CHECK: .data          PROGBITS     000000000000[[DATA_SEGMENT:.]]000
+// PART0: .data          PROGBITS     000000000000[[DATA_SEGMENT:.]]378
+// PART1: .data          PROGBITS     000000000000[[DATA_SEGMENT:.]]340
 
 // CHECK: Relocation section '.rela.dyn'
 // CHECK-NEXT: Offset
-// PART0-NEXT: 000000000000[[DATA_SEGMENT]]008 {{.*}} R_X86_64_64 000000000000[[DATA_SEGMENT]]000 p0 + 0
-// PART1-NEXT: 000000000000[[DATA_SEGMENT]]008 {{.*}} R_X86_64_64 000000000000[[DATA_SEGMENT]]000 p1 + 0
+// PART0-NEXT: 000000000000[[DATA_SEGMENT]]380 {{.*}} R_X86_64_64 000000000000[[DATA_SEGMENT]]378 p0 + 0
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]348 {{.*}} R_X86_64_64 000000000000[[DATA_SEGMENT]]340 p1 + 0
 // CHECK-EMPTY:
 
 // CHECK: Relocation section '.relr.dyn'
 // CHECK-NEXT: Offset
-// CHECK-NEXT: 000000000000[[DATA_SEGMENT]]000 {{.*}} R_X86_64_RELATIVE
+// PART0-NEXT: 000000000000[[DATA_SEGMENT]]378 {{.*}} R_X86_64_RELATIVE
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]340 {{.*}} R_X86_64_RELATIVE
 // CHECK-EMPTY:
 
 // CHECK: Dynamic section
diff --git a/lld/test/ELF/partition-synthetic-sections.s b/lld/test/ELF/partition-synthetic-sections.s
index 4f51b3e92cf59..6edfd6e9da3e9 100644
--- a/lld/test/ELF/partition-synthetic-sections.s
+++ b/lld/test/ELF/partition-synthetic-sections.s
@@ -41,7 +41,8 @@
 // PART0-NEXT: .plt              PROGBITS
 // PART0-NEXT: .init_array       INIT_ARRAY      {{0*}}[[INIT_ARRAY_ADDR:[^ ]*]]
 // CHECK-NEXT: .dynamic          DYNAMIC         {{0*}}[[DYNAMIC_ADDR:[^ ]*]]
-// CHECK-NEXT: .data             PROGBITS        000000000000[[DATA_SEGMENT:.]]000
+// PART0-NEXT: .data             PROGBITS        000000000000[[DATA_SEGMENT:.]]178
+// PART1-NEXT: .data             PROGBITS        000000000000[[DATA_SEGMENT:.]]130
 // PART0-NEXT: .got.plt          PROGBITS        {{0*}}[[GOT_PLT_ADDR:[^ ]*]]
 // PART0-NEXT: .part.end         NOBITS          {{0*}}[[PART_END_ADDR:[^ ]*]]
 // CHECK-NEXT: .comment          PROGBITS
@@ -52,18 +53,18 @@
 
 // CHECK: Relocation section '.rela.dyn'
 // CHECK-NEXT: Offset
-// PART0-NEXT: 000000000000[[DATA_SEGMENT]]000 {{.*}} R_X86_64_64 {{.*}} f1@v3 + 0
+// PART0-NEXT: 000000000000[[DATA_SEGMENT]]178 {{.*}} R_X86_64_64 {{.*}} f1@v3 + 0
 // PART0-NEXT: {{0*}}[[INIT_ARRAY_ADDR]]       {{.*}} R_X86_64_64 {{.*}} p0@@x1 + 0
-// PART1-NEXT: 000000000000[[DATA_SEGMENT]]018 {{.*}} R_X86_64_RELATIVE 3000
-// PART1-NEXT: 000000000000[[DATA_SEGMENT]]000 {{.*}} R_X86_64_64 {{.*}} f2@v2 + 0
-// PART1-NEXT: 000000000000[[DATA_SEGMENT]]008 {{.*}} R_X86_64_64 {{.*}} p0@@x1 + 0
-// PART1-NEXT: 000000000000[[DATA_SEGMENT]]010 {{.*}} R_X86_64_64 {{.*}} p0@@x1 + 0
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]148 {{.*}} R_X86_64_RELATIVE 3178
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]130 {{.*}} R_X86_64_64 {{.*}} f2@v2 + 0
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]138 {{.*}} R_X86_64_64 {{.*}} p0@@x1 + 0
+// PART1-NEXT: 000000000000[[DATA_SEGMENT]]140 {{.*}} R_X86_64_64 {{.*}} p0@@x1 + 0
 // CHECK-EMPTY:
 
 // PART0: Relocation section '.rela.plt'
 // PART0-NEXT: Offset
-// PART0-NEXT: 000000000000[[DATA_SEGMENT]]020 {{.*}} R_X86_64_JUMP_SLOT {{.*}} f1@v3 + 0
-// PART0-NEXT: 000000000000[[DATA_SEGMENT]]028 {{.*}} R_X86_64_JUMP_SLOT {{.*}} f2@v2 + 0
+// PART0-NEXT: 000000000000[[DATA_SEGMENT]]198 {{.*}} R_X86_64_JUMP_SLOT {{.*}} f1@v3 + 0
+// PART0-NEXT: 000000000000[[DATA_SEGMENT]]1a0 {{.*}} R_X86_64_JUMP_SLOT {{.*}} f2@v2 + 0
 // PART0-EMPTY:
 
 // CHECK: Symbol table '.dynsym'
@@ -85,7 +86,7 @@
 // PART-INDEX: Contents of section .rodata:
 //                       0x48c + 0xffffff26 = 0x3b2
 //                                0x490 + 0x3b70 = 0x4000
-// PART-INDEX-NEXT: 048c 26ffffff 703b0000 00400000
+// PART-INDEX-NEXT: 048c 26ffffff 703b0000 50410000
 
 // CHECK: {{.*}}EH_FRAME Header
 // CHECK: Address: 0x[[EH_FRAME_HDR_ADDR]]
@@ -137,12 +138,12 @@
 // PART0-NEXT: LOAD           0x000000 0x0000000000000000 0x0000000000000000 {{.*}} R   0x1000
 // PART0-NEXT: LOAD           0x001000 0x0000000000001000 0x0000000000001000 {{.*}} R E 0x1000
 // PART0-NEXT: LOAD           0x002000 0x0000000000002000 0x0000000000002000 {{.*}} RW  0x1000
-// PART0-NEXT: LOAD           0x003000 0x0000000000003000 0x0000000000003000 {{.*}} RW  0x1000
-// PART0-NEXT: LOAD           0x004000 0x0000000000008000 0x0000000000008000 0x000000 0x001000 RW  0x1000
+// PART0-NEXT: LOAD           0x002178 0x0000000000003178 0x0000000000003178 {{.*}} RW  0x1000
+// PART0-NEXT: LOAD           0x003150 0x0000000000008150 0x0000000000008150 0x000000 0x001000 RW  0x1000
 // PART1-NEXT: LOAD           0x000000 0x0000000000004000 0x0000000000004000 {{.*}} R   0x1000
 // PART1-NEXT: LOAD           0x001000 0x0000000000005000 0x0000000000005000 {{.*}} R E 0x1000
 // PART1-NEXT: LOAD           0x002000 0x0000000000006000 0x0000000000006000 {{.*}} RW  0x1000
-// PART1-NEXT: LOAD           0x003000 0x0000000000007000 0x0000000000007000 {{.*}} RW  0x1000
+// PART1-NEXT: LOAD           0x002130 0x0000000000007130 0x0000000000007130 {{.*}} RW  0x1000
 // CHECK-NEXT: DYNAMIC        {{.*}} 0x{{0*}}[[DYNAMIC_ADDR]] 0x{{0*}}[[DYNAMIC_ADDR]] {{.*}} RW  0x8
 // PART0-NEXT: GNU_RELRO      0x002000 0x0000000000002000 0x0000000000002000 {{.*}} R   0x1
 // PART1-NEXT: GNU_RELRO      0x002000 0x0000000000006000 0x0000000000006000 {{.*}} R   0x1
@@ -191,9 +192,9 @@
 // FILL-NEXT: *
 // FILL-NEXT: 002000
 
-// FILL: 005010 cccc cccc cccc cccc cccc cccc cccc cccc
+// FILL: 004010 cccc cccc cccc cccc cccc cccc cccc cccc
 // FILL-NEXT: *
-// FILL-NEXT: 006000
+// FILL-NEXT: 005000
 
 .section .llvm_sympart,"",@llvm_sympart
 .asciz "part1"
diff --git a/lld/test/ELF/pre_init_fini_array_missing.s b/lld/test/ELF/pre_init_fini_array_missing.s
index 91ddbabff3df1..0b0477ef2608a 100644
--- a/lld/test/ELF/pre_init_fini_array_missing.s
+++ b/lld/test/ELF/pre_init_fini_array_missing.s
@@ -15,14 +15,14 @@ _start:
   call __fini_array_end
 
 // With no .init_array section the symbols resolve to .text.
-// 0x201000 - (0x201000 + 5) = -5
-// 0x201000 - (0x201005 + 5) = -10
+// 0x201120 - (0x201120 + 5) = -5
+// 0x201120 - (0x201125 + 5) = -10
 // ...
 
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT:  _start:
-// CHECK-NEXT:   201000:       callq    -5
+// CHECK-NEXT:   201120:       callq    -5
 // CHECK-NEXT:                 callq    -10
 // CHECK-NEXT:                 callq    -15
 // CHECK-NEXT:                 callq    -20
@@ -34,7 +34,7 @@ _start:
 // PIE:      Disassembly of section .text:
 // PIE-EMPTY:
 // PIE-NEXT: _start:
-// PIE-NEXT:     1000:       callq   -5
+// PIE-NEXT:     1210:       callq   -5
 // PIE-NEXT:                 callq   -10
 // PIE-NEXT:                 callq   -15
 // PIE-NEXT:                 callq   -20
diff --git a/lld/test/ELF/rel-offset.s b/lld/test/ELF/rel-offset.s
index 6f35bcebc4b02..69a96c8032171 100644
--- a/lld/test/ELF/rel-offset.s
+++ b/lld/test/ELF/rel-offset.s
@@ -10,6 +10,6 @@
         .quad   foo
 
 // CHECK:      Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:   0x2000 R_X86_64_64 foo 0x0
-// CHECK-NEXT:   0x2008 R_X86_64_64 foo 0x0
+// CHECK-NEXT:   0x3340 R_X86_64_64 foo 0x0
+// CHECK-NEXT:   0x3348 R_X86_64_64 foo 0x0
 // CHECK-NEXT: }
diff --git a/lld/test/ELF/relative-dynamic-reloc-pie.s b/lld/test/ELF/relative-dynamic-reloc-pie.s
index 790eed1199bda..22a1c01b35c52 100644
--- a/lld/test/ELF/relative-dynamic-reloc-pie.s
+++ b/lld/test/ELF/relative-dynamic-reloc-pie.s
@@ -6,9 +6,9 @@
 ## Test that we create R_X86_64_RELATIVE relocations with -pie.
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section ({{.*}}) .rela.dyn {
-# CHECK-NEXT:     0x3000 R_X86_64_RELATIVE - 0x3000
-# CHECK-NEXT:     0x3008 R_X86_64_RELATIVE - 0x3008
-# CHECK-NEXT:     0x3010 R_X86_64_RELATIVE - 0x3009
+# CHECK-NEXT:     0x3358 R_X86_64_RELATIVE - 0x3358
+# CHECK-NEXT:     0x3360 R_X86_64_RELATIVE - 0x3360
+# CHECK-NEXT:     0x3368 R_X86_64_RELATIVE - 0x3361
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/relocatable-linkorder.s b/lld/test/ELF/relocatable-linkorder.s
new file mode 100644
index 0000000000000..0a7dfc6c12a2c
--- /dev/null
+++ b/lld/test/ELF/relocatable-linkorder.s
@@ -0,0 +1,36 @@
+// REQUIRES: x86
+// RUN: llvm-mc %s -o %t.o -filetype=obj --triple=x86_64-unknown-linux
+// RUN: ld.lld %t.o -o %t -r
+// RUN: llvm-readelf -S %t | FileCheck --check-prefix=DIFFERENT %s
+// RUN: echo 'SECTIONS { .text.f1 : { *(.text.f1) } .text.f2 : { *(.text.f2) } }' > %t.lds
+// RUN: ld.lld %t.o -o %t -r %t.lds
+// RUN: llvm-readelf -S %t | FileCheck --check-prefix=DIFFERENT %s
+// RUN: echo 'SECTIONS { .text : { *(.text.f1) *(.text.f2) } }' > %t.lds
+// RUN: ld.lld %t.o -o %t -r %t.lds
+// RUN: llvm-readelf -S -x foo %t | FileCheck --check-prefix=SAME %s
+
+/// Test that SHF_LINK_ORDER sections with different linked sections
+/// aren't merged.
+
+.section .text.f1,"ax",@progbits
+.globl f1
+f1:
+ret
+
+.section .text.f2,"ax",@progbits
+.globl f2
+f2:
+ret
+
+// SAME: foo
+// DIFFERENT: foo
+.section foo,"ao",@progbits,.text.f2,unique,2
+.quad 2
+
+// SAME-NOT: foo
+// DIFFERENT: foo
+.section foo,"ao",@progbits,.text.f1,unique,1
+.quad 1
+
+// SAME: Hex dump of section 'foo':
+// SAME: 01000000 00000000 02000000 00000000
diff --git a/lld/test/ELF/relocatable.s b/lld/test/ELF/relocatable.s
index 1907ae46e1b8c..90ec024395c33 100644
--- a/lld/test/ELF/relocatable.s
+++ b/lld/test/ELF/relocatable.s
@@ -92,9 +92,9 @@
 # CHECKEXE-NEXT:    Type: Executable
 # CHECKEXE-NEXT:    Machine: EM_X86_64
 # CHECKEXE-NEXT:    Version: 1
-# CHECKEXE-NEXT:    Entry: 0x201000
+# CHECKEXE-NEXT:    Entry: 0x201160
 # CHECKEXE-NEXT:    ProgramHeaderOffset: 0x40
-# CHECKEXE-NEXT:    SectionHeaderOffset: 0x21A0
+# CHECKEXE-NEXT:    SectionHeaderOffset: 0x358
 # CHECKEXE-NEXT:    Flags [
 # CHECKEXE-NEXT:    ]
 # CHECKEXE-NEXT:    HeaderSize: 64
diff --git a/lld/test/ELF/relocation-common.s b/lld/test/ELF/relocation-common.s
index 392c495b1f17c..49c362764c37a 100644
--- a/lld/test/ELF/relocation-common.s
+++ b/lld/test/ELF/relocation-common.s
@@ -10,5 +10,5 @@ _start:
 .global sym1
 .comm sym1,4,4
 
-# CHECK: 201000: {{.*}} movl    $1, 4086(%rip)
-# CHECK: 0000000000202000 g     O .bss            00000004 sym1
+# CHECK: 201158: {{.*}} movl    $1, 4098(%rip)
+# CHECK: 0000000000202164 g     O .bss            00000004 sym1
diff --git a/lld/test/ELF/relocation-copy-align-common.s b/lld/test/ELF/relocation-copy-align-common.s
index 5928dfb904069..cef83d955bf41 100644
--- a/lld/test/ELF/relocation-copy-align-common.s
+++ b/lld/test/ELF/relocation-copy-align-common.s
@@ -18,8 +18,8 @@
 # CHECK-NEXT:     SHF_ALLOC
 # CHECK-NEXT:     SHF_WRITE
 # CHECK-NEXT:   ]
-# CHECK-NEXT:   Address: 0x203000
-# CHECK-NEXT:   Offset: 0x3000
+# CHECK-NEXT:   Address: 0x203360
+# CHECK-NEXT:   Offset: 0x360
 # CHECK-NEXT:   Size: 64
 # CHECK-NEXT:   Link: 0
 # CHECK-NEXT:   Info: 0
@@ -29,7 +29,7 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section {{.*}} .rela.dyn {
-# CHECK-NEXT:     0x203020 R_X86_64_COPY foo 0x0
+# CHECK-NEXT:     0x203380 R_X86_64_COPY foo 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/relocation-copy-flags.s b/lld/test/ELF/relocation-copy-flags.s
index cfbce61b664ac..6574ae5ae86e4 100644
--- a/lld/test/ELF/relocation-copy-flags.s
+++ b/lld/test/ELF/relocation-copy-flags.s
@@ -21,7 +21,7 @@ _start:
 
 // CHECK:      Name: .text
 // CHECK:      SectionData (
-// CHECK-NEXT:   0000: 10302000
+// CHECK-NEXT:   0000: 90332000
 // CHECK-NEXT: )
 
 // CHECK:      Name: bar
@@ -30,8 +30,8 @@ _start:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x203000
-// CHECK-NEXT: Offset: 0x3000
+// CHECK-NEXT: Address: 0x203380
+// CHECK-NEXT: Offset: 0x380
 // CHECK-NEXT: Size: 8
 // CHECK-NEXT: Link: 0
 // CHECK-NEXT: Info: 0
@@ -46,7 +46,7 @@ _start:
 // CHECK-NEXT: Flags [
 // CHECK-NEXT: ]
 // CHECK-NEXT: Address: 0x0
-// CHECK-NEXT: Offset: 0x3008
+// CHECK-NEXT: Offset: 0x388
 // CHECK-NEXT: Size: 8
 // CHECK-NEXT: Link: 0
 // CHECK-NEXT: Info: 0
@@ -58,7 +58,7 @@ _start:
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section (4) .rela.dyn {
-// CHECK-NEXT:     0x203010 R_X86_64_COPY x 0x0
-// CHECK-NEXT:     0x203000 R_X86_64_64 z 0x0
+// CHECK-NEXT:     0x203390 R_X86_64_COPY x 0x0
+// CHECK-NEXT:     0x203380 R_X86_64_64 z 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
diff --git a/lld/test/ELF/relocation-copy-relro.s b/lld/test/ELF/relocation-copy-relro.s
index a00dfe9b99e63..5b2460e399703 100644
--- a/lld/test/ELF/relocation-copy-relro.s
+++ b/lld/test/ELF/relocation-copy-relro.s
@@ -14,19 +14,19 @@
 // CHECK-NEXT:     SHF_ALLOC (0x2)
 // CHECK-NEXT:     SHF_WRITE (0x1)
 // CHECK-NEXT:   ]
-// CHECK-NEXT:   Address: 0x2020C0
-// CHECK-NEXT:   Offset: 0x20C0
+// CHECK-NEXT:   Address: 0x202368
+// CHECK-NEXT:   Offset: 0x368
 // CHECK-NEXT:   Size: 8
 
-// CHECK: 0x2020C0 R_X86_64_COPY a 0x0
-// CHECK: 0x2020C4 R_X86_64_COPY b 0x0
+// CHECK: 0x202368 R_X86_64_COPY a 0x0
+// CHECK: 0x20236C R_X86_64_COPY b 0x0
 
 // CHECK:      Type: PT_GNU_RELRO (0x6474E552)
-// CHECK-NEXT: Offset: 0x2000
-// CHECK-NEXT: VirtualAddress: 0x202000
-// CHECK-NEXT: PhysicalAddress: 0x202000
+// CHECK-NEXT: Offset: 0x2A8
+// CHECK-NEXT: VirtualAddress: 0x2022A8
+// CHECK-NEXT: PhysicalAddress: 0x2022A8
 // CHECK-NEXT: FileSize: 192
-// CHECK-NEXT: MemSize: 4096
+// CHECK-NEXT: MemSize: 3416
 
 .text
 .global _start
diff --git a/lld/test/ELF/relocation-copy.s b/lld/test/ELF/relocation-copy.s
index 08d648183fc59..e427cc91721bf 100644
--- a/lld/test/ELF/relocation-copy.s
+++ b/lld/test/ELF/relocation-copy.s
@@ -1,10 +1,10 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
 // RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/relocation-copy.s -o %t2.o
-// RUN: ld.lld -shared %t2.o -o %t.so
+// RUN: ld.lld -shared %t2.o -soname=so -o %t.so
 // RUN: ld.lld %t.o %t.so -o %t3
 // RUN: llvm-readobj -S -r --expand-relocs %t3 | FileCheck %s
-// RUN: llvm-objdump -d %t3 | FileCheck -check-prefix=CODE %s
+// RUN: llvm-objdump -d --no-show-raw-insn --print-imm-hex %t3 | FileCheck -check-prefix=CODE %s
 
 .text
 .global _start
@@ -22,7 +22,7 @@ movl $z, %edx
 // CHECK-NEXT:   SHF_ALLOC (0x2)
 // CHECK-NEXT:   SHF_WRITE (0x1)
 // CHECK-NEXT:  ]
-// CHECK-NEXT:  Address: 0x203000
+// CHECK-NEXT:  Address: 0x203400
 // CHECK-NEXT:  Offset:
 // CHECK-NEXT:  Size: 24
 // CHECK-NEXT:  Link: 0
@@ -53,16 +53,13 @@ movl $z, %edx
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
-// 2109440 = 0x203000
 // 16 is alignment here
-// 2109456 = 0x203000 + 16
-// 2109460 = 0x203000 + 16 + 4
 // CODE: Disassembly of section .text:
 // CODE-EMPTY:
 // CODE-NEXT: _start:
-// CODE-NEXT: 201000: {{.*}} movl $5, 2109440
-// CODE-NEXT: 20100b: {{.*}} movl $7, 2109456
-// CODE-NEXT: 201016: {{.*}} movl $9, 2109460
-// CODE-NEXT: 201021: {{.*}} movl $2109440, %edx
-// CODE-NEXT: 201026: {{.*}} movl $2109456, %edx
-// CODE-NEXT: 20102b: {{.*}} movl $2109460, %edx
+// CODE-NEXT:   movl $0x5, 0x203400
+// CODE-NEXT:   movl $0x7, 0x203410
+// CODE-NEXT:   movl $0x9, 0x203414
+// CODE-NEXT:   movl $0x203400, %edx
+// CODE-NEXT:   movl $0x203410, %edx
+// CODE-NEXT:   movl $0x203414, %edx
diff --git a/lld/test/ELF/relocation-non-alloc.s b/lld/test/ELF/relocation-non-alloc.s
index 04f3e39642061..db5b602802519 100644
--- a/lld/test/ELF/relocation-non-alloc.s
+++ b/lld/test/ELF/relocation-non-alloc.s
@@ -15,15 +15,15 @@
 // APPLYDYNREL-NEXT:   SHF_ALLOC
 // APPLYDYNREL-NEXT:   SHF_WRITE
 // APPLYDYNREL-NEXT: ]
-// APPLYDYNREL-NEXT: Address: 0x2000
-// APPLYDYNREL-NEXT: Offset: 0x2000
+// APPLYDYNREL-NEXT: Address: 0x3350
+// APPLYDYNREL-NEXT: Offset: 0x350
 // APPLYDYNREL-NEXT: Size: 16
 // APPLYDYNREL-NEXT: Link: 0
 // APPLYDYNREL-NEXT: Info: 0
 // APPLYDYNREL-NEXT: AddressAlignment: 1
 // APPLYDYNREL-NEXT: EntrySize: 0
 // APPLYDYNREL-NEXT: SectionData (
-// APPLYDYNREL-NEXT:   0000: 00200000 00000000 00000000 00000000
+// APPLYDYNREL-NEXT:   0000: 50330000 00000000 00000000 00000000
 // APPLYDYNREL-NEXT: )
 
 // NOAPPLYDYNREL:      Name: .data
@@ -32,8 +32,8 @@
 // NOAPPLYDYNREL-NEXT:   SHF_ALLOC
 // NOAPPLYDYNREL-NEXT:   SHF_WRITE
 // NOAPPLYDYNREL-NEXT: ]
-// NOAPPLYDYNREL-NEXT: Address: 0x2000
-// NOAPPLYDYNREL-NEXT: Offset: 0x2000
+// NOAPPLYDYNREL-NEXT: Address: 0x3350
+// NOAPPLYDYNREL-NEXT: Offset: 0x350
 // NOAPPLYDYNREL-NEXT: Size: 16
 // NOAPPLYDYNREL-NEXT: Link: 0
 // NOAPPLYDYNREL-NEXT: Info: 0
@@ -55,14 +55,14 @@
 // CHECK-NEXT: AddressAlignment: 1
 // CHECK-NEXT: EntrySize: 0
 // CHECK-NEXT: SectionData (
-// CHECK-NEXT:   0000: 00200000 00000000 00200000 00000000
-// CHECK-NEXT:   0010: 00200000 00000000 00200000 00000000
+// CHECK-NEXT:   0000: 50330000 00000000 50330000 00000000
+// CHECK-NEXT:   0010: 50330000 00000000 50330000 00000000
 // CHECK-NEXT: )
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.}}) .rela.dyn {
-// CHECK-NEXT:     0x2000 R_X86_64_RELATIVE - 0x2000
-// CHECK-NEXT:     0x2008 R_X86_64_64 zed 0x0
+// CHECK-NEXT:     0x3350 R_X86_64_RELATIVE - 0x3350
+// CHECK-NEXT:     0x3358 R_X86_64_64 zed 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
diff --git a/lld/test/ELF/relocation-undefined-weak.s b/lld/test/ELF/relocation-undefined-weak.s
index ce658f67ee0a8..18c1094f74119 100644
--- a/lld/test/ELF/relocation-undefined-weak.s
+++ b/lld/test/ELF/relocation-undefined-weak.s
@@ -14,4 +14,4 @@ _start:
 
 .weak sym1
 
-# CHECK: 201000: movl $0x1, -0x20100a(%rip)
+# CHECK: 201120: movl $0x1, -0x20112a(%rip)
diff --git a/lld/test/ELF/relocation.s b/lld/test/ELF/relocation.s
index 3905f8808fa73..e77e22ee0efc0 100644
--- a/lld/test/ELF/relocation.s
+++ b/lld/test/ELF/relocation.s
@@ -12,8 +12,8 @@
 // SEC-NEXT:   SHF_ALLOC
 // SEC-NEXT:   SHF_EXECINSTR
 // SEC-NEXT: ]
-// SEC-NEXT: Address: 0x201030
-// SEC-NEXT: Offset: 0x1030
+// SEC-NEXT: Address: 0x201340
+// SEC-NEXT: Offset: 0x340
 // SEC-NEXT: Size: 48
 
 // SEC:         Name: .got
@@ -22,7 +22,7 @@
 // SEC-NEXT:     SHF_ALLOC
 // SEC-NEXT:     SHF_WRITE
 // SEC-NEXT:   ]
-// SEC-NEXT:   Address: 0x2020F0
+// SEC-NEXT:   Address: 0x202460
 // SEC-NEXT:   Offset:
 // SEC-NEXT:   Size: 8
 // SEC-NEXT:   Link: 0
@@ -37,8 +37,8 @@
 // SEC-NEXT:     SHF_ALLOC
 // SEC-NEXT:     SHF_WRITE
 // SEC-NEXT:   ]
-// SEC-NEXT:   Address: 0x203000
-// SEC-NEXT:   Offset: 0x3000
+// SEC-NEXT:   Address: 0x203468
+// SEC-NEXT:   Offset: 0x468
 // SEC-NEXT:   Size: 40
 // SEC-NEXT:   Link: 0
 // SEC-NEXT:   Info: 0
@@ -60,11 +60,11 @@ lulz:
 // CHECK: Disassembly of section .text:
 // CHECK-EMPTY:
 // CHECK-NEXT: _start:
-// CHECK-NEXT:   201000:  e8 04 00 00 00   callq 4
-// CHECK-NEXT:   201005:
+// CHECK-NEXT:   201310:  e8 04 00 00 00   callq 4
+// CHECK-NEXT:   201315:
 
 // CHECK:      lulz:
-// CHECK-NEXT:   201009:  90  nop
+// CHECK-NEXT:   201319:  90  nop
 
 
 .section       .text2,"ax",@progbits
@@ -77,7 +77,7 @@ R_X86_64_32:
 // CHECK: Disassembly of section .text2:
 // CHECK-EMPTY:
 // CHECK-NEXT: R_X86_64_32:
-// CHECK-NEXT:  20100a: {{.*}} movl $2101258, %edx
+// CHECK-NEXT:  20131a: {{.*}} movl $2102042, %edx
 
 .section .R_X86_64_32S,"ax",@progbits
 .global R_X86_64_32S
@@ -87,7 +87,7 @@ R_X86_64_32S:
 // CHECK: Disassembly of section .R_X86_64_32S:
 // CHECK-EMPTY:
 // CHECK-NEXT: R_X86_64_32S:
-// CHECK-NEXT:  {{.*}}: {{.*}} movq 1052681, %rdx
+// CHECK-NEXT:  {{.*}}: {{.*}} movq 1053465, %rdx
 
 .section .R_X86_64_PC32,"ax",@progbits
 .global R_X86_64_PC32
@@ -95,22 +95,22 @@ R_X86_64_PC32:
  call bar
  movl $bar, %eax
 //16 is a size of PLT[0]
-// 0x201030 + 16 - (0x201017 + 5) = 20
+// 0x201340 + 16 - (0x201327 + 5) = 36
 // CHECK:      Disassembly of section .R_X86_64_PC32:
 // CHECK-EMPTY:
 // CHECK-NEXT: R_X86_64_PC32:
-// CHECK-NEXT:  201017:   {{.*}}  callq  36
-// CHECK-NEXT:  20101c:   {{.*}}  movl $2101312, %eax
+// CHECK-NEXT:  201327:   {{.*}}  callq  36
+// CHECK-NEXT:  20132c:   {{.*}}  movl $2102096, %eax
 
 .section .R_X86_64_32S_2,"ax",@progbits
 .global R_X86_64_32S_2
 R_X86_64_32S_2:
   mov bar2, %eax
-// plt is  at 0x201030. The second plt entry is at 0x201050 == 69712
+// plt is  at 0x201340. The second plt entry is at 0x201360 == 2102112
 // CHECK:      Disassembly of section .R_X86_64_32S_2:
 // CHECK-EMPTY:
 // CHECK-NEXT: R_X86_64_32S_2:
-// CHECK-NEXT: 201021: {{.*}}  movl    2101328, %eax
+// CHECK-NEXT: 201331: {{.*}}  movl    2102112, %eax
 
 .section .R_X86_64_64,"a",@progbits
 .global R_X86_64_64
@@ -125,9 +125,9 @@ R_X86_64_64:
 R_X86_64_GOTPCREL:
  .long zed@gotpcrel
 
-// 0x2020F0(.got) - 0x2002c8(.R_X86_64_GOTPCREL) = 0x1e28
+// 0x202460(.got) - 0x200300(.R_X86_64_GOTPCREL) = 0x2160
 // CHECK:      Contents of section .R_X86_64_GOTPCREL
-// CHECK-NEXT:   200300 f01d0000
+// CHECK-NEXT:   200300 60210000
 
 .section .R_X86_64_GOT32,"a",@progbits
 .global R_X86_64_GOT32
@@ -135,11 +135,11 @@ R_X86_64_GOT32:
         .long zed@got
 
 // CHECK: Contents of section .R_X86_64_GOT32:
-// CHECK-NEXT: f0f0ffff
+// CHECK-NEXT: f8efffff
 
 
 // CHECK: Contents of section .R_X86_64_GOT64:
-// CHECK-NEXT: f0f0ffff ffffffff
+// CHECK-NEXT: f8efffff ffffffff
 .section .R_X86_64_GOT64,"a",@progbits
 .global R_X86_64_GOT64
 R_X86_64_GOT64:
diff --git a/lld/test/ELF/relro-bss.s b/lld/test/ELF/relro-bss.s
index 20a3333f3c95e..8fc8167bb51b7 100644
--- a/lld/test/ELF/relro-bss.s
+++ b/lld/test/ELF/relro-bss.s
@@ -9,10 +9,10 @@
 # Because .bss.rel.ro (nobits) doesn't take space, p_filesz < p_memsz.
 
 #           Type           Offset   VirtAddr           PhysAddr           FileSiz  MemSiz   Flg Align
-# SEG:      LOAD           0x001000 0x0000000000201000 0x0000000000201000 0x000001 0x000001 R E 0x1000
-# SEG-NEXT: LOAD           0x002000 0x0000000000202000 0x0000000000202000 0x000001 0x002001 RW  0x1000
-# SEG-NEXT: LOAD           0x003000 0x0000000000205000 0x0000000000205000 0x000001 0x000002 RW  0x1000
-# SEG-NEXT: GNU_RELRO      0x002000 0x0000000000202000 0x0000000000202000 0x000001 0x003000 R   0x1
+# SEG:      LOAD           0x0001c8 0x00000000002011c8 0x00000000002011c8 0x000001 0x000001 R E 0x1000
+# SEG-NEXT: LOAD           0x0001c9 0x00000000002021c9 0x00000000002021c9 0x000001 0x002001 RW  0x1000
+# SEG-NEXT: LOAD           0x0001ca 0x00000000002051ca 0x00000000002051ca 0x000001 0x000002 RW  0x1000
+# SEG-NEXT: GNU_RELRO      0x0001c9 0x00000000002021c9 0x00000000002021c9 0x000001 0x002e37 R   0x1
 # SEG-NEXT: GNU_STACK      0x000000 0x0000000000000000 0x0000000000000000 0x000000 0x000000 RW  0x0
 
 # SEG:      .text
@@ -22,10 +22,10 @@
 # And .data is placed in the next page (sh_offset = alignTo(0x2001, 4096) = 0x3000).
 
 #        [Nr] Name              Type            Address          Off    Size
-# CHECK:      .data.rel.ro      PROGBITS        0000000000202000 002000 000001
-# CHECK-NEXT: .bss.rel.ro       NOBITS          0000000000202001 002001 002000
-# CHECK-NEXT: .data             PROGBITS        0000000000205000 003000 000001
-# CHECK-NEXT: .bss              NOBITS          0000000000205001 003001 000001
+# CHECK:      .data.rel.ro      PROGBITS        00000000002021c9 0001c9 000001
+# CHECK-NEXT: .bss.rel.ro       NOBITS          00000000002021ca 0001ca 002000
+# CHECK-NEXT: .data             PROGBITS        00000000002051ca 0001ca 000001
+# CHECK-NEXT: .bss              NOBITS          00000000002051cb 0001cb 000001
 
 .globl _start
 _start:
diff --git a/lld/test/ELF/relro-tls.s b/lld/test/ELF/relro-tls.s
index 9c391e6aa11b0..9d2ef6b8827fd 100644
--- a/lld/test/ELF/relro-tls.s
+++ b/lld/test/ELF/relro-tls.s
@@ -9,9 +9,9 @@
 ## Currently p_memsz of PT_GNU_RELRO is rounded up to protect the last page.
 
 # CHECK:      Type: PT_GNU_RELRO
-# CHECK:      VirtualAddress: 0x201000
+# CHECK:      VirtualAddress: 0x2021C8
 # CHECK:      FileSize: 4
-# CHECK-NEXT: MemSize: 4096
+# CHECK-NEXT: MemSize: 3640
 # CHECK:      Alignment: 1
 
 .section .foo,"awT",@progbits
diff --git a/lld/test/ELF/segments.s b/lld/test/ELF/segments.s
index 1199452a7af6a..5165fc3c48589 100644
--- a/lld/test/ELF/segments.s
+++ b/lld/test/ELF/segments.s
@@ -20,7 +20,7 @@
 # ROSEGMENT-NEXT:  }
 # ROSEGMENT-NEXT:  ProgramHeader {
 # ROSEGMENT-NEXT:    Type: PT_LOAD
-# ROSEGMENT-NEXT:    Offset: 0x1000
+# ROSEGMENT-NEXT:    Offset: 0x15C
 # ROSEGMENT-NEXT:    VirtualAddress:
 # ROSEGMENT-NEXT:    PhysicalAddress:
 # ROSEGMENT-NEXT:    FileSize:
@@ -33,7 +33,7 @@
 # ROSEGMENT-NEXT:  }
 # ROSEGMENT-NEXT:  ProgramHeader {
 # ROSEGMENT-NEXT:    Type: PT_LOAD
-# ROSEGMENT-NEXT:    Offset: 0x2000
+# ROSEGMENT-NEXT:    Offset: 0x15E
 # ROSEGMENT-NEXT:    VirtualAddress:
 # ROSEGMENT-NEXT:    PhysicalAddress:
 # ROSEGMENT-NEXT:    FileSize: 1
@@ -63,7 +63,7 @@
 # NOROSEGMENT-NEXT: }
 # NOROSEGMENT-NEXT: ProgramHeader {
 # NOROSEGMENT-NEXT:   Type: PT_LOAD
-# NOROSEGMENT-NEXT:   Offset: 0x1000
+# NOROSEGMENT-NEXT:   Offset: 0x126
 # NOROSEGMENT-NEXT:   VirtualAddress:
 # NOROSEGMENT-NEXT:   PhysicalAddress:
 # NOROSEGMENT-NEXT:   FileSize:
diff --git a/lld/test/ELF/separate-segments.s b/lld/test/ELF/separate-segments.s
new file mode 100644
index 0000000000000..a0c910f31338b
--- /dev/null
+++ b/lld/test/ELF/separate-segments.s
@@ -0,0 +1,33 @@
+# REQUIRES: x86
+# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o
+
+## -z noseparate-code is the default. All PT_LOAD can have overlapping p_offset
+## ranges at runtime.
+# RUN: ld.lld -pie %t.o -o %t
+# RUN: llvm-readelf -l %t | FileCheck --check-prefix=NONE %s
+# NONE:      LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x000245 0x000245 R   0x1000
+# NONE-NEXT: LOAD 0x000248 0x0000000000001248 0x0000000000001248 0x000001 0x000001 R E 0x1000
+# NONE-NEXT: LOAD 0x000250 0x0000000000002250 0x0000000000002250 0x000080 0x000080 RW  0x1000
+# NONE-NEXT: LOAD 0x0002d0 0x00000000000032d0 0x00000000000032d0 0x000001 0x000001 RW  0x1000
+
+## -z separate-code makes text segment (RX) separate.
+## The two RW can have overlapping p_offset ranges at runtime.
+# RUN: ld.lld -pie %t.o -z separate-code -o %t
+# RUN: llvm-readelf -l %t | FileCheck --check-prefix=CODE %s
+# CODE:      LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x000245 0x000245 R   0x1000
+# CODE-NEXT: LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000001 0x000001 R E 0x1000
+# CODE-NEXT: LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x000080 0x000080 RW  0x1000
+# CODE-NEXT: LOAD 0x002080 0x0000000000003080 0x0000000000003080 0x000001 0x000001 RW  0x1000
+
+## -z separate-loadable-segments makes all segments separate.
+# RUN: ld.lld -pie %t.o -z separate-loadable-segments -o %t
+# RUN: llvm-readelf -l %t | FileCheck --check-prefix=ALL %s
+# ALL:       LOAD 0x000000 0x0000000000000000 0x0000000000000000 0x000245 0x000245 R   0x1000
+# ALL-NEXT:  LOAD 0x001000 0x0000000000001000 0x0000000000001000 0x000001 0x000001 R E 0x1000
+# ALL-NEXT:  LOAD 0x002000 0x0000000000002000 0x0000000000002000 0x000080 0x000080 RW  0x1000
+# ALL-NEXT:  LOAD 0x003000 0x0000000000003000 0x0000000000003000 0x000001 0x000001 RW  0x1000
+
+nop
+
+.data
+.byte 0
diff --git a/lld/test/ELF/startstop-shared2.s b/lld/test/ELF/startstop-shared2.s
index 4b887bea56a36..1a2408c578cc5 100644
--- a/lld/test/ELF/startstop-shared2.s
+++ b/lld/test/ELF/startstop-shared2.s
@@ -8,10 +8,10 @@
 // RUN: ld.lld -o %t %t2.o %t.so
 // RUN: llvm-objdump -s -h %t | FileCheck %s
 
-// CHECK: foo           00000000 0000000000201008
+// CHECK: foo           00000000 0000000000201248
 
 // CHECK: Contents of section .text:
-// CHECK-NEXT: 201000 08102000 00000000
+// CHECK-NEXT: 201240 48122000 00000000
 
 .quad __start_foo
 .section foo,"ax"
diff --git a/lld/test/ELF/startstop.s b/lld/test/ELF/startstop.s
index 0ec5893e05913..a311fc7484a43 100644
--- a/lld/test/ELF/startstop.s
+++ b/lld/test/ELF/startstop.s
@@ -5,19 +5,19 @@
 // RUN: llvm-readobj --symbols -r %t.so | FileCheck -check-prefix=SYMBOL %s
 
 // DISASM: _start:
-// DISASM:    1000:       callq   10 <__start_foo>
-// DISASM:    1005:       callq   8 <__start_bar>
-// DISASM:    100a:       callq   3 <__start_bar>
+// DISASM:    1330:       callq   10 <__start_foo>
+// DISASM:    1335:       callq   8 <__start_bar>
+// DISASM:    133a:       callq   3 <__start_bar>
 // DISASM: Disassembly of section foo:
 // DISASM-EMPTY:
 // DISASM: __start_foo:
-// DISASM:    100f:       nop
+// DISASM:    133f:       nop
 // DISASM:                nop
 // DISASM:                nop
 // DISASM: Disassembly of section bar:
 // DISASM-EMPTY:
 // DISASM: __start_bar:
-// DISASM:    1012:       nop
+// DISASM:    1342:       nop
 // DISASM:                nop
 // DISASM:                nop
 
@@ -32,33 +32,33 @@
 
 // SYMBOL: Symbol {
 // SYMBOL:   Name: __start_bar
-// SYMBOL:   Value: 0x1012
+// SYMBOL:   Value: 0x1342
 // SYMBOL:   STV_HIDDEN
 // SYMBOL:   Section: bar
 // SYMBOL: }
 // SYMBOL-NOT:   Section: __stop_bar
 // SYMBOL: Symbol {
 // SYMBOL:   Name: __start_foo
-// SYMBOL:   Value: 0x100F
+// SYMBOL:   Value: 0x133F
 // SYMBOL:   STV_HIDDEN
 // SYMBOL:   Section: foo
 // SYMBOL: }
 // SYMBOL: Symbol {
 // SYMBOL:   Name: __stop_foo
-// SYMBOL:   Value: 0x1012
+// SYMBOL:   Value: 0x1342
 // SYMBOL:   STV_HIDDEN
 // SYMBOL:   Section: foo
 // SYMBOL: }
 
 // SYMBOL: Symbol {
 // SYMBOL:   Name: __stop_zed1
-// SYMBOL:   Value: 0x3010
+// SYMBOL:   Value: 0x3408
 // SYMBOL:   STV_PROTECTED
 // SYMBOL:   Section: zed1
 // SYMBOL: }
 // SYMBOL: Symbol {
 // SYMBOL:   Name: __stop_zed2
-// SYMBOL:   Value: 0x3020
+// SYMBOL:   Value: 0x3418
 // SYMBOL:   STV_PROTECTED
 // SYMBOL:   Section: zed2
 // SYMBOL: }
diff --git a/lld/test/ELF/symbol-ordering-file-cgprofile-conflicts.s b/lld/test/ELF/symbol-ordering-file-cgprofile-conflicts.s
index 5a60aff18e907..593f3c99891bf 100644
--- a/lld/test/ELF/symbol-ordering-file-cgprofile-conflicts.s
+++ b/lld/test/ELF/symbol-ordering-file-cgprofile-conflicts.s
@@ -2,18 +2,18 @@
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t.o
 // RUN: ld.lld -e A %t.o --no-call-graph-profile-sort -o %t
 // RUN: llvm-nm --numeric-sort %t | FileCheck %s --check-prefix=NO_ORDERING
-// NO_ORDERING:         0000000000201000 t D
-// NO_ORDERING-NEXT:    0000000000201001 T C
-// NO_ORDERING-NEXT:    0000000000201002 T B
-// NO_ORDERING-NEXT:    0000000000201003 T A
+// NO_ORDERING:         0000000000201120 t D
+// NO_ORDERING-NEXT:    0000000000201121 T C
+// NO_ORDERING-NEXT:    0000000000201122 T B
+// NO_ORDERING-NEXT:    0000000000201123 T A
 
 // RUN: ld.lld -e A %t.o -o %t
 // RUN: llvm-nm --numeric-sort %t | FileCheck %s --check-prefix=CALL_GRAPH
-// CALL_GRAPH:		0000000000201000 T A
-// CALL_GRAPH-NEXT:	0000000000201000 t Aa
-// CALL_GRAPH-NEXT:	0000000000201001 T B
-// CALL_GRAPH-NEXT:	0000000000201002 T C
-// CALL_GRAPH-NEXT:	0000000000201003 t D
+// CALL_GRAPH:		0000000000201120 T A
+// CALL_GRAPH-NEXT:	0000000000201120 t Aa
+// CALL_GRAPH-NEXT:	0000000000201121 T B
+// CALL_GRAPH-NEXT:	0000000000201122 T C
+// CALL_GRAPH-NEXT:	0000000000201123 t D
 
 // RUN: rm -f %t.symbol_order
 // RUN: echo "C" >> %t.symbol_order
@@ -23,10 +23,10 @@
 
 // RUN: ld.lld -e A %t.o --symbol-ordering-file %t.symbol_order -o %t
 // RUN: llvm-nm --numeric-sort %t | FileCheck %s --check-prefix=SYMBOL_ORDER
-// SYMBOL_ORDER:	0000000000201000 T C
-// SYMBOL_ORDER-NEXT:	0000000000201001 T B
-// SYMBOL_ORDER-NEXT:	0000000000201002 t D
-// SYMBOL_ORDER-NEXT:	0000000000201003 T A
+// SYMBOL_ORDER:	0000000000201120 T C
+// SYMBOL_ORDER-NEXT:	0000000000201121 T B
+// SYMBOL_ORDER-NEXT:	0000000000201122 t D
+// SYMBOL_ORDER-NEXT:	0000000000201123 T A
 
 // RUN: rm -f %t.call_graph
 // RUN: echo "A B 5" > %t.call_graph
diff --git a/lld/test/ELF/symbol-ordering-file-icf.s b/lld/test/ELF/symbol-ordering-file-icf.s
index 93e377cb53b08..82cf8211b7337 100644
--- a/lld/test/ELF/symbol-ordering-file-icf.s
+++ b/lld/test/ELF/symbol-ordering-file-icf.s
@@ -8,9 +8,9 @@
 
 ## Check that after ICF merges 'foo' and 'zed' we still
 ## place them before 'bar', in according to ordering file.
-# CHECK-DAG: 0000000000001000 T foo
-# CHECK-DAG: 0000000000001000 T zed
-# CHECK-DAG: 0000000000001004 T bar
+# CHECK-DAG: 0000000000001288 T foo
+# CHECK-DAG: 0000000000001288 T zed
+# CHECK-DAG: 000000000000128c T bar
 
 .section .text.foo,"ax",@progbits
 .align 4
diff --git a/lld/test/ELF/symbols.s b/lld/test/ELF/symbols.s
index eb73da83f37f2..56eec23f80e9f 100644
--- a/lld/test/ELF/symbols.s
+++ b/lld/test/ELF/symbols.s
@@ -58,7 +58,7 @@ internal:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_EXECINSTR
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x201000
+// CHECK-NEXT: Address: 0x201160
 
 // CHECK:      Name: .bss
 // CHECK-NEXT: Type: SHT_NOBITS
@@ -66,7 +66,7 @@ internal:
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x202000
+// CHECK-NEXT: Address: 0x202164
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 4
 
@@ -104,7 +104,7 @@ internal:
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: _start
-// CHECK-NEXT:     Value: 0x201000
+// CHECK-NEXT:     Value: 0x201160
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Global (0x1)
 // CHECK-NEXT:     Type: Function
@@ -131,7 +131,7 @@ internal:
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: common
-// CHECK-NEXT:     Value: 0x202000
+// CHECK-NEXT:     Value: 0x202164
 // CHECK-NEXT:     Size: 4
 // CHECK-NEXT:     Binding: Global
 // CHECK-NEXT:     Type: Object
@@ -140,7 +140,7 @@ internal:
 // CHECK-NEXT:   }
 // CHECK-NEXT:   Symbol {
 // CHECK-NEXT:     Name: foo
-// CHECK-NEXT:     Value: 0x201000
+// CHECK-NEXT:     Value: 0x201160
 // CHECK-NEXT:     Size: 0
 // CHECK-NEXT:     Binding: Weak (0x2)
 // CHECK-NEXT:     Type: Object
diff --git a/lld/test/ELF/tls-offset.s b/lld/test/ELF/tls-offset.s
index 2eeebdbf24da4..d52e2be657b2c 100644
--- a/lld/test/ELF/tls-offset.s
+++ b/lld/test/ELF/tls-offset.s
@@ -1,7 +1,7 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
 // RUN: ld.lld %t -o %tout
-// RUN: llvm-readobj -S %tout | FileCheck %s
+// RUN: llvm-readobj -S %tout | FileCheck --check-prefixes=CHECK,CHECK1 %s
 // RUN: echo "SECTIONS { \
 // RUN:   . = 0x201000; \
 // RUN:   .text : { *(.text) } \
@@ -11,7 +11,7 @@
 // RUN:   .data.rel.ro : { *(.data.rel.ro) } \
 // RUN: }" > %t.script
 // RUN: ld.lld -T %t.script %t -o %tout2
-// RUN: llvm-readobj -S %tout2 | FileCheck %s
+// RUN: llvm-readobj -S %tout2 | FileCheck --check-prefixes=CHECK,CHECK2 %s
         .global _start
 _start:
         retq
@@ -32,34 +32,40 @@ _start:
 // gets out of sync what we get a runtime is different from what the section
 // table says.
 
-// CHECK:      Name: .tdata
-// CHECK-NEXT: Type: SHT_PROGBITS
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_TLS
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x202000
-// CHECK-NEXT: Offset: 0x2000
-// CHECK-NEXT: Size: 4
+// CHECK:       Name: .tdata
+// CHECK-NEXT:  Type: SHT_PROGBITS
+// CHECK-NEXT:  Flags [
+// CHECK-NEXT:    SHF_ALLOC
+// CHECK-NEXT:    SHF_TLS
+// CHECK-NEXT:    SHF_WRITE
+// CHECK-NEXT:  ]
+// CHECK1-NEXT: Address: 0x2021D0
+// CHECK1-NEXT: Offset: 0x1D0
+// CHECK2-NEXT: Address: 0x202000
+// CHECK2-NEXT: Offset: 0x2000
+// CHECK-NEXT:  Size: 4
 
-// CHECK:      Name: .tbss
-// CHECK-NEXT: Type: SHT_NOBITS
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_TLS
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x202010
-// CHECK-NEXT: Offset: 0x2004
-// CHECK-NEXT: Size: 16
+// CHECK:       Name: .tbss
+// CHECK-NEXT:  Type: SHT_NOBITS
+// CHECK-NEXT:  Flags [
+// CHECK-NEXT:    SHF_ALLOC
+// CHECK-NEXT:    SHF_TLS
+// CHECK-NEXT:    SHF_WRITE
+// CHECK-NEXT:  ]
+// CHECK1-NEXT: Address: 0x2021E0
+// CHECK1-NEXT: Offset: 0x1D4
+// CHECK2-NEXT: Address: 0x202010
+// CHECK2-NEXT: Offset: 0x2004
+// CHECK-NEXT:  Size: 16
 
-// CHECK:      Name: .data.rel.ro
-// CHECK-NEXT: Type: SHT_PROGBITS
-// CHECK-NEXT: Flags [
-// CHECK-NEXT:   SHF_ALLOC
-// CHECK-NEXT:   SHF_WRITE
-// CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x202004
-// CHECK-NEXT: Offset: 0x2004
-// CHECK-NEXT: Size: 4
+// CHECK:       Name: .data.rel.ro
+// CHECK-NEXT:  Type: SHT_PROGBITS
+// CHECK-NEXT:  Flags [
+// CHECK-NEXT:    SHF_ALLOC
+// CHECK-NEXT:    SHF_WRITE
+// CHECK-NEXT:  ]
+// CHECK1-NEXT: Address: 0x2021D4
+// CHECK1-NEXT: Offset: 0x1D4
+// CHECK2-NEXT: Address: 0x202004
+// CHECK2-NEXT: Offset: 0x2004
+// CHECK-NEXT:  Size: 4
diff --git a/lld/test/ELF/tls.s b/lld/test/ELF/tls.s
index 156b8ae09b21f..80f83d66185d8 100644
--- a/lld/test/ELF/tls.s
+++ b/lld/test/ELF/tls.s
@@ -90,9 +90,9 @@ d:
 // CHECK-NEXT:       SHF_WRITE
 // CHECK-NEXT:     ]
 
-// 0x20200C = TBSS_ADDR + 4
+// 0x2021F4 = TBSS_ADDR + 4
 
-// CHECK-NEXT:     Address: 0x20200C
+// CHECK-NEXT:     Address: 0x2021F4
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 4
 // CHECK-NEXT:     Link:
diff --git a/lld/test/ELF/ttext-tdata-tbss.s b/lld/test/ELF/ttext-tdata-tbss.s
index 9637bdd55d044..fb9c4d5131749 100644
--- a/lld/test/ELF/ttext-tdata-tbss.s
+++ b/lld/test/ELF/ttext-tdata-tbss.s
@@ -5,36 +5,41 @@
 # RUN: ld.lld %t.o -o %t1
 # RUN: llvm-readelf -S -l  %t1 | FileCheck %s
 # CHECK:      .rodata   PROGBITS 0000000000200158 000158 000008
-# CHECK-NEXT: .text     PROGBITS 0000000000201000 001000 000001
-# CHECK-NEXT: .aw       PROGBITS 0000000000202000 002000 000008
-# CHECK-NEXT: .data     PROGBITS 0000000000202008 002008 000008
-# CHECK-NEXT: .bss      NOBITS   0000000000202010 002010 000008
+# CHECK-NEXT: .text     PROGBITS 0000000000201160 000160 000001
+# CHECK-NEXT: .aw       PROGBITS 0000000000202161 000161 000008
+# CHECK-NEXT: .data     PROGBITS 0000000000202169 000169 000008
+# CHECK-NEXT: .bss      NOBITS   0000000000202171 000171 000008
 # CHECK:      Type
 # CHECK-NEXT: PHDR
 # CHECK-NEXT: LOAD 0x000000 0x0000000000200000
 
-## With .text at 0 there is no space to allocate the headers.
+## If -Ttext is smaller than the image base (which defaults to 0x200000 for -no-pie),
+## the headers will still be allocated, but mapped at a higher address,
+## which may look strange.
 # RUN: ld.lld -Ttext 0x0 -Tdata 0x4000 -Tbss 0x8000 %t.o -o %t2
 # RUN: llvm-readelf -S -l %t2 | FileCheck %s --check-prefix=USER1
 # USER1:      .text   PROGBITS 0000000000000000 001000 000001
 # USER1-NEXT: .data   PROGBITS 0000000000004000 002000 000008
 # USER1-NEXT: .bss    NOBITS   0000000000008000 002008 000008
-# USER1-NEXT: .rodata PROGBITS 0000000000009000 003000 000008
-# USER1-NEXT: .aw     PROGBITS 000000000000a000 004000 000008
+# USER1-NEXT: .rodata PROGBITS 0000000000009008 002008 000008
+# USER1-NEXT: .aw     PROGBITS 000000000000a010 002010 000008
 # USER1:      Type
+# USER1-NEXT: PHDR 0x000040 0x0000000000200040
+# USER1-NEXT: LOAD 0x000000 0x0000000000200000
 # USER1-NEXT: LOAD 0x001000 0x0000000000000000
 
-## With .text at 0x1000 there is space to allocate the headers.
-# RUN: ld.lld -Ttext 0x1000 -Tdata 0x4000 -Tbss 0x8000 %t.o -o %t3
+## Specify --image-base to make program headers look normal.
+# RUN: ld.lld --image-base=0 -Ttext 0x1000 -Tdata 0x4000 -Tbss 0x8000 %t.o -o %t3
 # RUN: llvm-readelf -S -l  %t3 | FileCheck %s --check-prefix=USER2
 # USER2:      .text   PROGBITS 0000000000001000 001000 000001
 # USER2-NEXT: .data   PROGBITS 0000000000004000 002000 000008
 # USER2-NEXT: .bss    NOBITS   0000000000008000 002008 000008
-# USER2-NEXT: .rodata PROGBITS 0000000000009000 003000 000008
-# USER2-NEXT: .aw     PROGBITS 000000000000a000 004000 000008
+# USER2-NEXT: .rodata PROGBITS 0000000000009008 002008 000008
+# USER2-NEXT: .aw     PROGBITS 000000000000a010 002010 000008
 # USER2:      Type
-# USER2-NEXT: PHDR
+# USER2-NEXT: PHDR 0x000040 0x0000000000000040
 # USER2-NEXT: LOAD 0x000000 0x0000000000000000
+# USER2-NEXT: LOAD 0x001000 0x0000000000001000
 
 ## With .text well above 200000 we don't need to change the image base
 # RUN: ld.lld -Ttext 0x201000 %t.o -o %t4
@@ -45,8 +50,9 @@
 # USER3-NEX: .data   PROGBITS 0000000000203008 003008 000008
 # USER3-NEX: .bss    NOBITS   0000000000203010 003010 000008
 # USER3:      Type
-# USER3-NEXT: PHDR
+# USER3-NEXT: PHDR 0x000040 0x0000000000200040
 # USER3-NEXT: LOAD 0x000000 0x0000000000200000
+# USER3-NEXT: LOAD 0x001000 0x0000000000201000
 
 .text
 .globl _start
diff --git a/lld/test/ELF/undef-with-plt-addr.s b/lld/test/ELF/undef-with-plt-addr.s
index fe05dcfc9a84d..447c620401ef4 100644
--- a/lld/test/ELF/undef-with-plt-addr.s
+++ b/lld/test/ELF/undef-with-plt-addr.s
@@ -22,13 +22,13 @@ movabsq	$set_data, %rax
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_EXECINSTR
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x201010
+// CHECK-NEXT: Address: 0x2012E0
 
 // CHECK:      Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:   0x203000 R_X86_64_64 foo 0x0
+// CHECK-NEXT:   0x203400 R_X86_64_64 foo 0x0
 // CHECK-NEXT: }
 // CHECK-NEXT: Section ({{.*}}) .rela.plt {
-// CHECK-NEXT:   0x203020 R_X86_64_JUMP_SLOT set_data 0x0
+// CHECK-NEXT:   0x203420 R_X86_64_JUMP_SLOT set_data 0x0
 // CHECK-NEXT: }
 
 // CHECK:      Name: foo
@@ -40,7 +40,7 @@ movabsq	$set_data, %rax
 // CHECK-NEXT: Section: Undefined
 
 // CHECK:      Name:    set_data
-// CHECK-NEXT: Value:   0x201020
+// CHECK-NEXT: Value:   0x2012F0
 // CHECK-NEXT: Size: 0
 // CHECK-NEXT: Binding: Global
 // CHECK-NEXT: Type: Function
diff --git a/lld/test/ELF/undef.s b/lld/test/ELF/undef.s
index e6277e36667db..f16bd4769b1db 100644
--- a/lld/test/ELF/undef.s
+++ b/lld/test/ELF/undef.s
@@ -23,6 +23,12 @@
 # CHECK: error: undefined symbol: vtable for Foo
 # CHECK: the vtable symbol may be undefined because the class is missing its key function (see https://lld.llvm.org/missingkeyfunction)
 
+# Check that this symbol isn't demangled
+
+# CHECK: error: undefined symbol: __Z3fooi
+# CHECK: >>> referenced by undef.s
+# CHECK: >>>               {{.*}}:(.text+0x1A)
+
 # CHECK: error: undefined symbol: zed2
 # CHECK: >>> referenced by {{.*}}.o:(.text+0x0) in archive {{.*}}2.a
 
@@ -64,3 +70,4 @@ _start:
   call zed1
   call _Z3fooi
   call _ZTV3Foo
+  call __Z3fooi
diff --git a/lld/test/ELF/verdef-defaultver.s b/lld/test/ELF/verdef-defaultver.s
index 467f73a3bd4e4..2c1b183bdd82b 100644
--- a/lld/test/ELF/verdef-defaultver.s
+++ b/lld/test/ELF/verdef-defaultver.s
@@ -18,7 +18,7 @@
 # DSO-NEXT:    }
 # DSO-NEXT:    Symbol {
 # DSO-NEXT:      Name: a@@V1
-# DSO-NEXT:      Value: 0x1000
+# DSO-NEXT:      Value: 0x12E8
 # DSO-NEXT:      Size: 0
 # DSO-NEXT:      Binding: Global
 # DSO-NEXT:      Type: Function
@@ -27,7 +27,7 @@
 # DSO-NEXT:    }
 # DSO-NEXT:    Symbol {
 # DSO-NEXT:      Name: b@@V2
-# DSO-NEXT:      Value: 0x1002
+# DSO-NEXT:      Value: 0x12EA
 # DSO-NEXT:      Size: 0
 # DSO-NEXT:      Binding: Global
 # DSO-NEXT:      Type: Function
@@ -36,7 +36,7 @@
 # DSO-NEXT:    }
 # DSO-NEXT:    Symbol {
 # DSO-NEXT:      Name: b@V1
-# DSO-NEXT:      Value: 0x1001
+# DSO-NEXT:      Value: 0x12E9
 # DSO-NEXT:      Size: 0
 # DSO-NEXT:      Binding: Global
 # DSO-NEXT:      Type: Function
@@ -45,7 +45,7 @@
 # DSO-NEXT:    }
 # DSO-NEXT:    Symbol {
 # DSO-NEXT:      Name: c@@V2
-# DSO-NEXT:      Value: 0x1003
+# DSO-NEXT:      Value: 0x12EB
 # DSO-NEXT:      Size: 0
 # DSO-NEXT:      Binding: Global
 # DSO-NEXT:      Type: Function
@@ -122,7 +122,7 @@
 # EXE-NEXT:    }
 # EXE-NEXT:    Symbol {
 # EXE-NEXT:      Name: a@V1
-# EXE-NEXT:      Value: 0x201020
+# EXE-NEXT:      Value: 0x201340
 # EXE-NEXT:      Size: 0
 # EXE-NEXT:      Binding: Global
 # EXE-NEXT:      Type: Function
@@ -131,7 +131,7 @@
 # EXE-NEXT:    }
 # EXE-NEXT:    Symbol {
 # EXE-NEXT:      Name: b@V2
-# EXE-NEXT:      Value: 0x201030
+# EXE-NEXT:      Value: 0x201350
 # EXE-NEXT:      Size: 0
 # EXE-NEXT:      Binding: Global
 # EXE-NEXT:      Type: Function
@@ -140,7 +140,7 @@
 # EXE-NEXT:    }
 # EXE-NEXT:    Symbol {
 # EXE-NEXT:      Name: c@V2
-# EXE-NEXT:      Value: 0x201040
+# EXE-NEXT:      Value: 0x201360
 # EXE-NEXT:      Size: 0
 # EXE-NEXT:      Binding: Global
 # EXE-NEXT:      Type: Function
diff --git a/lld/test/ELF/x86-64-got-plt-header.s b/lld/test/ELF/x86-64-got-plt-header.s
index a65e49c512c3e..801e106886d9c 100644
--- a/lld/test/ELF/x86-64-got-plt-header.s
+++ b/lld/test/ELF/x86-64-got-plt-header.s
@@ -12,7 +12,7 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x2000
+// CHECK-NEXT: Address: 0x22B0
 
 // CHECK:      Name: .got.plt
 // CHECK-NEXT: Type: SHT_PROGBITS
@@ -20,12 +20,12 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x3000
-// CHECK-NEXT: Offset: 0x3000
+// CHECK-NEXT: Address: 0x3360
+// CHECK-NEXT: Offset: 0x360
 // CHECK-NEXT: Size: 32
 // CHECK-NEXT: Link: 0
 // CHECK-NEXT: Info: 0
 // CHECK-NEXT: AddressAlignment: 8
 // CHECK-NEXT: EntrySize: 0
 // CHECK-NEXT: SectionData (
-// CHECK-NEXT:   0000: 00200000 00000000 00000000 00000000
+// CHECK-NEXT:   0000: B0220000 00000000 00000000 00000000
diff --git a/lld/test/ELF/x86-64-gotpc-relax-nopic.s b/lld/test/ELF/x86-64-gotpc-relax-nopic.s
index 2d77e88bb4664..2e76e505870ba 100644
--- a/lld/test/ELF/x86-64-gotpc-relax-nopic.s
+++ b/lld/test/ELF/x86-64-gotpc-relax-nopic.s
@@ -10,21 +10,21 @@
 # SYMRELOC:      Symbols [
 # SYMRELOC:       Symbol {
 # SYMRELOC:        Name: bar
-# SYMRELOC-NEXT:   Value: 0x202000
+# SYMRELOC-NEXT:   Value: 0x202197
 
-## 2105344 = 0x202000 (bar)
+## 2105751 = 0x202197 (bar)
 # DISASM:      Disassembly of section .text:
 # DISASM-EMPTY:
 # DISASM-NEXT: _start:
-# DISASM-NEXT:   201000:       adcq  $2105344, %rax
-# DISASM-NEXT:                 addq  $2105344, %rbx
-# DISASM-NEXT:                 andq  $2105344, %rcx
-# DISASM-NEXT:                 cmpq  $2105344, %rdx
-# DISASM-NEXT:                 orq   $2105344, %rdi
-# DISASM-NEXT:                 sbbq  $2105344, %rsi
-# DISASM-NEXT:                 subq  $2105344, %rbp
-# DISASM-NEXT:                 xorq  $2105344, %r8
-# DISASM-NEXT:                 testq $2105344, %r15
+# DISASM-NEXT:   201158:       adcq  $2105751, %rax
+# DISASM-NEXT:                 addq  $2105751, %rbx
+# DISASM-NEXT:                 andq  $2105751, %rcx
+# DISASM-NEXT:                 cmpq  $2105751, %rdx
+# DISASM-NEXT:                 orq   $2105751, %rdi
+# DISASM-NEXT:                 sbbq  $2105751, %rsi
+# DISASM-NEXT:                 subq  $2105751, %rbp
+# DISASM-NEXT:                 xorq  $2105751, %r8
+# DISASM-NEXT:                 testq $2105751, %r15
 
 # RUN: ld.lld --hash-style=sysv -shared %t.o -o %t2
 # RUN: llvm-readobj -S -r -d %t2 | FileCheck --check-prefix=SEC-PIC    %s
@@ -37,8 +37,8 @@
 # SEC-PIC-NEXT:     SHF_ALLOC
 # SEC-PIC-NEXT:     SHF_WRITE
 # SEC-PIC-NEXT:   ]
-# SEC-PIC-NEXT:   Address: 0x20A0
-# SEC-PIC-NEXT:   Offset: 0x20A0
+# SEC-PIC-NEXT:   Address: 0x2348
+# SEC-PIC-NEXT:   Offset: 0x348
 # SEC-PIC-NEXT:   Size: 8
 # SEC-PIC-NEXT:   Link:
 # SEC-PIC-NEXT:   Info:
@@ -47,7 +47,7 @@
 # SEC-PIC-NEXT: }
 # SEC-PIC:      Relocations [
 # SEC-PIC-NEXT:   Section ({{.*}}) .rela.dyn {
-# SEC-PIC-NEXT:     0x20A0 R_X86_64_RELATIVE - 0x3000
+# SEC-PIC-NEXT:     0x2348 R_X86_64_RELATIVE - 0x3350
 # SEC-PIC-NEXT:   }
 # SEC-PIC-NEXT: ]
 # SEC-PIC:      0x000000006FFFFFF9 RELACOUNT            1
@@ -58,15 +58,15 @@
 # DISASM-PIC:      Disassembly of section .text:
 # DISASM-PIC-EMPTY:
 # DISASM-PIC-NEXT: _start:
-# DISASM-PIC-NEXT: 1000:       adcq  4249(%rip), %rax
-# DISASM-PIC-NEXT:             addq  4242(%rip), %rbx
-# DISASM-PIC-NEXT:             andq  4235(%rip), %rcx
-# DISASM-PIC-NEXT:             cmpq  4228(%rip), %rdx
-# DISASM-PIC-NEXT:             orq   4221(%rip), %rdi
-# DISASM-PIC-NEXT:             sbbq  4214(%rip), %rsi
-# DISASM-PIC-NEXT:             subq  4207(%rip), %rbp
-# DISASM-PIC-NEXT:             xorq  4200(%rip), %r8
-# DISASM-PIC-NEXT:             testq %r15, 4193(%rip)
+# DISASM-PIC-NEXT: 1268:       adcq  4313(%rip), %rax
+# DISASM-PIC-NEXT:             addq  4306(%rip), %rbx
+# DISASM-PIC-NEXT:             andq  4299(%rip), %rcx
+# DISASM-PIC-NEXT:             cmpq  4292(%rip), %rdx
+# DISASM-PIC-NEXT:             orq   4285(%rip), %rdi
+# DISASM-PIC-NEXT:             sbbq  4278(%rip), %rsi
+# DISASM-PIC-NEXT:             subq  4271(%rip), %rbp
+# DISASM-PIC-NEXT:             xorq  4264(%rip), %r8
+# DISASM-PIC-NEXT:             testq %r15, 4257(%rip)
 
 .data
 .type   bar, @object
diff --git a/lld/test/ELF/x86-64-gotpc-relax-und-dso.s b/lld/test/ELF/x86-64-gotpc-relax-und-dso.s
index 5e0fca49c4e50..5aded8f52b4d0 100644
--- a/lld/test/ELF/x86-64-gotpc-relax-und-dso.s
+++ b/lld/test/ELF/x86-64-gotpc-relax-und-dso.s
@@ -23,22 +23,22 @@
 # DISASM:      hid:
 # DISASM-NEXT:     nop
 # DISASM:      _start:
-# DISASM-NEXT:    movq    4247(%rip), %rax
-# DISASM-NEXT:    movq    4240(%rip), %rax
-# DISASM-NEXT:    movq    4241(%rip), %rax
-# DISASM-NEXT:    movq    4234(%rip), %rax
+# DISASM-NEXT:    movq    4367(%rip), %rax
+# DISASM-NEXT:    movq    4360(%rip), %rax
+# DISASM-NEXT:    movq    4361(%rip), %rax
+# DISASM-NEXT:    movq    4354(%rip), %rax
 # DISASM-NEXT:    leaq    -36(%rip), %rax
 # DISASM-NEXT:    leaq    -43(%rip), %rax
-# DISASM-NEXT:    movq    4221(%rip), %rax
-# DISASM-NEXT:    movq    4214(%rip), %rax
-# DISASM-NEXT:    movq    4191(%rip), %rax
-# DISASM-NEXT:    movq    4184(%rip), %rax
-# DISASM-NEXT:    movq    4185(%rip), %rax
-# DISASM-NEXT:    movq    4178(%rip), %rax
+# DISASM-NEXT:    movq    4341(%rip), %rax
+# DISASM-NEXT:    movq    4334(%rip), %rax
+# DISASM-NEXT:    movq    4311(%rip), %rax
+# DISASM-NEXT:    movq    4304(%rip), %rax
+# DISASM-NEXT:    movq    4305(%rip), %rax
+# DISASM-NEXT:    movq    4298(%rip), %rax
 # DISASM-NEXT:    leaq    -92(%rip), %rax
 # DISASM-NEXT:    leaq    -99(%rip), %rax
-# DISASM-NEXT:    movq    4165(%rip), %rax
-# DISASM-NEXT:    movq    4158(%rip), %rax
+# DISASM-NEXT:    movq    4285(%rip), %rax
+# DISASM-NEXT:    movq    4278(%rip), %rax
 
 .text
 .globl foo
diff --git a/lld/test/ELF/x86-64-gotpc-relax.s b/lld/test/ELF/x86-64-gotpc-relax.s
index 21ec4e10960e7..bf935f693e04e 100644
--- a/lld/test/ELF/x86-64-gotpc-relax.s
+++ b/lld/test/ELF/x86-64-gotpc-relax.s
@@ -8,37 +8,37 @@
 # RELOC:    Relocations [
 # RELOC:    ]
 
-# 0x201003 + 7 - 10 = 0x201000
-# 0x20100a + 7 - 17 = 0x201000
-# 0x201011 + 7 - 23 = 0x201001
-# 0x201018 + 7 - 30 = 0x201001
+# 0x201173 + 7 - 10 = 0x201170
+# 0x20117a + 7 - 17 = 0x201170
+# 0x201181 + 7 - 23 = 0x201171
+# 0x201188 + 7 - 30 = 0x201171
 # DISASM:      Disassembly of section .text:
 # DISASM-EMPTY:
 # DISASM-NEXT: foo:
-# DISASM-NEXT:   201000: 90 nop
+# DISASM-NEXT:   201170: 90 nop
 # DISASM:      hid:
-# DISASM-NEXT:   201001: 90 nop
+# DISASM-NEXT:   201171: 90 nop
 # DISASM:      ifunc:
-# DISASM-NEXT:   201002: c3 retq
+# DISASM-NEXT:   201172: c3 retq
 # DISASM:      _start:
 # DISASM-NEXT: leaq -10(%rip), %rax
 # DISASM-NEXT: leaq -17(%rip), %rax
 # DISASM-NEXT: leaq -23(%rip), %rax
 # DISASM-NEXT: leaq -30(%rip), %rax
-# DISASM-NEXT: movq 4058(%rip), %rax
-# DISASM-NEXT: movq 4051(%rip), %rax
+# DISASM-NEXT: movq 4234(%rip), %rax
+# DISASM-NEXT: movq 4227(%rip), %rax
 # DISASM-NEXT: leaq -52(%rip), %rax
 # DISASM-NEXT: leaq -59(%rip), %rax
 # DISASM-NEXT: leaq -65(%rip), %rax
 # DISASM-NEXT: leaq -72(%rip), %rax
-# DISASM-NEXT: movq 4016(%rip), %rax
-# DISASM-NEXT: movq 4009(%rip), %rax
+# DISASM-NEXT: movq 4192(%rip), %rax
+# DISASM-NEXT: movq 4185(%rip), %rax
 # DISASM-NEXT: callq -93 <foo>
 # DISASM-NEXT: callq -99 <foo>
 # DISASM-NEXT: callq -104 <hid>
 # DISASM-NEXT: callq -110 <hid>
-# DISASM-NEXT: callq *3979(%rip)
-# DISASM-NEXT: callq *3973(%rip)
+# DISASM-NEXT: callq *4155(%rip)
+# DISASM-NEXT: callq *4149(%rip)
 # DISASM-NEXT: jmp   -128 <foo>
 # DISASM-NEXT: nop
 # DISASM-NEXT: jmp   -134 <foo>
@@ -47,8 +47,8 @@
 # DISASM-NEXT: nop
 # DISASM-NEXT: jmp   -145 <hid>
 # DISASM-NEXT: nop
-# DISASM-NEXT: jmpq  *3943(%rip)
-# DISASM-NEXT: jmpq  *3937(%rip)
+# DISASM-NEXT: jmpq  *4119(%rip)
+# DISASM-NEXT: jmpq  *4113(%rip)
 
 .text
 .globl foo
diff --git a/lld/test/ELF/x86-64-plt-high-addr.s b/lld/test/ELF/x86-64-plt-high-addr.s
index ce2a876d1258d..381db32db5f9b 100644
--- a/lld/test/ELF/x86-64-plt-high-addr.s
+++ b/lld/test/ELF/x86-64-plt-high-addr.s
@@ -8,16 +8,16 @@
 // RUN: llvm-objdump -s -j .got.plt %t1.exe | FileCheck %s
 
 // CHECK:      Contents of section .got.plt:
-// CHECK-NEXT: cafe00003000 00200000 feca0000 00000000 00000000
-// CHECK-NEXT: cafe00003010 00000000 00000000 26100000 feca0000
+// CHECK-NEXT: cafe00003380 b0220000 feca0000 00000000 00000000
+// CHECK-NEXT: cafe00003390 00000000 00000000 a6120000 feca0000
 
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t2.o
 // RUN: ld.lld -o %t2.exe %t2.o %t.so -image-base=0xcafe00000000 -z retpolineplt
 // RUN: llvm-objdump -s -j .got.plt %t2.exe | FileCheck -check-prefix=RETPOLINE %s
 
 // RETPOLINE:      Contents of section .got.plt:
-// RETPOLINE-NEXT: cafe00003000 00200000 feca0000 00000000 00000000
-// RETPOLINE-NEXT: cafe00003010 00000000 00000000 51100000 feca0000
+// RETPOLINE-NEXT: cafe000033b0 e0220000 feca0000 00000000 00000000
+// RETPOLINE-NEXT: cafe000033c0 00000000 00000000 d1120000 feca0000
 
 .global _start
 _start:
diff --git a/lld/test/ELF/x86-64-plt.s b/lld/test/ELF/x86-64-plt.s
index 7fb37adea949d..33caf98c82d63 100644
--- a/lld/test/ELF/x86-64-plt.s
+++ b/lld/test/ELF/x86-64-plt.s
@@ -15,7 +15,7 @@
 // CHECK-NEXT:   SHF_ALLOC
 // CHECK-NEXT:   SHF_EXECINSTR
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x1020
+// CHECK-NEXT: Address: 0x1320
 // CHECK-NEXT: Offset:
 // CHECK-NEXT: Size: 64
 // CHECK-NEXT: Link: 0
@@ -24,9 +24,9 @@
 
 // CHECK:      Relocations [
 // CHECK-NEXT:   Section ({{.*}}) .rela.plt {
-// CHECK-NEXT:     0x3018 R_X86_64_JUMP_SLOT bar 0x0
-// CHECK-NEXT:     0x3020 R_X86_64_JUMP_SLOT zed 0x0
-// CHECK-NEXT:     0x3028 R_X86_64_JUMP_SLOT _start 0x0
+// CHECK-NEXT:     0x3438 R_X86_64_JUMP_SLOT bar 0x0
+// CHECK-NEXT:     0x3440 R_X86_64_JUMP_SLOT zed 0x0
+// CHECK-NEXT:     0x3448 R_X86_64_JUMP_SLOT _start 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
@@ -36,7 +36,7 @@
 // CHECK2-NEXT:   SHF_ALLOC
 // CHECK2-NEXT:   SHF_EXECINSTR
 // CHECK2-NEXT: ]
-// CHECK2-NEXT: Address: 0x201020
+// CHECK2-NEXT: Address: 0x2012E0
 // CHECK2-NEXT: Offset:
 // CHECK2-NEXT: Size: 48
 // CHECK2-NEXT: Link: 0
@@ -45,8 +45,8 @@
 
 // CHECK2:      Relocations [
 // CHECK2-NEXT:   Section ({{.*}}) .rela.plt {
-// CHECK2-NEXT:     0x203018 R_X86_64_JUMP_SLOT bar 0x0
-// CHECK2-NEXT:     0x203020 R_X86_64_JUMP_SLOT zed 0x0
+// CHECK2-NEXT:     0x2033F8 R_X86_64_JUMP_SLOT bar 0x0
+// CHECK2-NEXT:     0x203400 R_X86_64_JUMP_SLOT zed 0x0
 // CHECK2-NEXT:   }
 // CHECK2-NEXT: ]
 
@@ -63,22 +63,22 @@
 // DISASM:      Disassembly of section .plt:
 // DISASM-EMPTY:
 // DISASM-NEXT: .plt:
-// DISASM-NEXT:   1020:       pushq 8162(%rip)
-// DISASM-NEXT:               jmpq *8164(%rip)
+// DISASM-NEXT:   1320:       pushq 8450(%rip)
+// DISASM-NEXT:               jmpq *8452(%rip)
 // DISASM-NEXT:               nopl (%rax)
 // DISASM-EMPTY:
 // DISASM-NEXT:   bar@plt:
-// DISASM-NEXT:   1030:       jmpq *8162(%rip)
+// DISASM-NEXT:   1330:       jmpq *8450(%rip)
 // DISASM-NEXT:               pushq $0
 // DISASM-NEXT:               jmp -32 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   zed@plt:
-// DISASM-NEXT:   1040:       jmpq *8154(%rip)
+// DISASM-NEXT:   1340:       jmpq *8442(%rip)
 // DISASM-NEXT:               pushq $1
 // DISASM-NEXT:               jmp -48 <.plt>
 // DISASM-EMPTY:
 // DISASM-NEXT:   _start@plt:
-// DISASM-NEXT:   1050:       jmpq *8146(%rip)
+// DISASM-NEXT:   1350:       jmpq *8434(%rip)
 // DISASM-NEXT:               pushq $2
 // DISASM-NEXT:               jmp -64 <.plt>
 
@@ -99,17 +99,17 @@
 // DISASM2:      Disassembly of section .plt:
 // DISASM2-EMPTY:
 // DISASM2-NEXT: .plt:
-// DISASM2-NEXT:  201020:       pushq 8162(%rip)
-// DISASM2-NEXT:                jmpq *8164(%rip)
+// DISASM2-NEXT:  2012e0:       pushq 8450(%rip)
+// DISASM2-NEXT:                jmpq *8452(%rip)
 // DISASM2-NEXT:                nopl  (%rax)
 // DISASM2-EMPTY:
 // DISASM2-NEXT: bar@plt:
-// DISASM2-NEXT:  201030:       jmpq *8162(%rip)
+// DISASM2-NEXT:  2012f0:       jmpq *8450(%rip)
 // DISASM2-NEXT:                pushq $0
 // DISASM2-NEXT:                jmp -32 <.plt>
 // DISASM2-EMPTY:
 // DISASM2-NEXT: zed@plt:
-// DISASM2-NEXT:  201040:       jmpq *8154(%rip)
+// DISASM2-NEXT:  201300:       jmpq *8442(%rip)
 // DISASM2-NEXT:                pushq $1
 // DISASM2-NEXT:                jmp -48 <.plt>
 // DISASM2-NOT:   {{.}}
diff --git a/lld/test/ELF/x86-64-relax-got-abs.s b/lld/test/ELF/x86-64-relax-got-abs.s
index eaae8c528c2b3..07c9579b04f4f 100644
--- a/lld/test/ELF/x86-64-relax-got-abs.s
+++ b/lld/test/ELF/x86-64-relax-got-abs.s
@@ -7,7 +7,7 @@
 # RUN: ld.lld %t.o -o %t.so -shared
 # RUN: llvm-objdump -d %t.so | FileCheck %s
 
-# CHECK: movq  4201(%rip), %rax
+# CHECK: movq  4209(%rip), %rax
 
 	movq    bar@GOTPCREL(%rip), %rax
         .data
diff --git a/lld/test/ELF/x86-64-reloc-error2.s b/lld/test/ELF/x86-64-reloc-error2.s
index 230241b2be5f1..de5e2848df42b 100644
--- a/lld/test/ELF/x86-64-reloc-error2.s
+++ b/lld/test/ELF/x86-64-reloc-error2.s
@@ -4,7 +4,7 @@
 
 ## Check we are able to find a function symbol that encloses
 ## a given location when reporting error messages.
-# CHECK: {{.*}}.o:(function func: .text.func+0x3): relocation R_X86_64_32S out of range: -281474974609408 is not in [-2147483648, 2147483647]
+# CHECK: {{.*}}.o:(function func: .text.func+0x3): relocation R_X86_64_32S out of range: -281474974609120 is not in [-2147483648, 2147483647]
 
 # This mergeable section will be garbage collected. We had a crash issue in that case. Test it.
 .section .rodata.str1,"aMS",@progbits,1
diff --git a/lld/test/ELF/x86-64-reloc-gotoff64.s b/lld/test/ELF/x86-64-reloc-gotoff64.s
index 9cc9dd7d94854..39854354c452d 100644
--- a/lld/test/ELF/x86-64-reloc-gotoff64.s
+++ b/lld/test/ELF/x86-64-reloc-gotoff64.s
@@ -4,8 +4,8 @@
 // RUN: llvm-readelf -S %t.so | FileCheck %s -check-prefix=SECTION
 // RUN: llvm-objdump -d %t.so | FileCheck %s
 
-// SECTION: .dynamic DYNAMIC 0000000000002000
-// SECTION: .got.plt PROGBITS 0000000000003000 003000 000018
+// SECTION: .dynamic DYNAMIC 0000000000002290
+// SECTION: .got.plt PROGBITS 0000000000003300 000300 000018
 
 // The assembly is generated by
 // gcc -O2 -S -mcmodel=medium -fPIC a.c
@@ -14,10 +14,10 @@
 // extern long _DYNAMIC[] __attribute__((visibility("hidden")));
 // long* dynamic() { return _DYNAMIC; }
 
-// 0x3000 (.got.plt) - 0x1007 = 8185
-// 0x2000 (_DYNAMIC) - 0x3000 (.got.plt) = -4096
-// CHECK:      1000: {{.*}} leaq 8185(%rip), %rdx
-// CHECK-NEXT: 1007: {{.*}} movabsq $-4096, %rax
+// 0x3300 (.got.plt) - 0x127b = 8325
+// 0x2290 (_DYNAMIC) - 0x3300 (.got.plt) = -4208
+// CHECK:      1274: {{.*}} leaq 8325(%rip), %rdx
+// CHECK-NEXT: 127b: {{.*}} movabsq $-4208, %rax
 .global dynamic
 dynamic:
   leaq _GLOBAL_OFFSET_TABLE_(%rip), %rdx
diff --git a/lld/test/ELF/x86-64-reloc-gotpc64.s b/lld/test/ELF/x86-64-reloc-gotpc64.s
index cbd8a3acfe585..cb59baee9e158 100644
--- a/lld/test/ELF/x86-64-reloc-gotpc64.s
+++ b/lld/test/ELF/x86-64-reloc-gotpc64.s
@@ -4,11 +4,11 @@
 // RUN: llvm-readelf -S %t.so | FileCheck %s -check-prefix=SECTION
 // RUN: llvm-objdump -d %t.so | FileCheck %s
 
-// SECTION: .got.plt PROGBITS 0000000000003000 003000 000018
+// SECTION: .got.plt PROGBITS 00000000000032f0 0002f0 000018
 
-// 0x3000 (.got.plt) - 0x1000 = 8192
+// 0x3300 (.got.plt) - 0x1274 = 8316
 // CHECK: gotpc64:
-// CHECK-NEXT: 1000: {{.*}} movabsq $8192, %r11
+// CHECK-NEXT: 1274: {{.*}} movabsq $8316, %r11
 .global gotpc64
 gotpc64:
   movabsq $_GLOBAL_OFFSET_TABLE_-., %r11
diff --git a/lld/test/ELF/x86-64-reloc-range.s b/lld/test/ELF/x86-64-reloc-range.s
index 87e3379da859c..3f50076d85a0f 100644
--- a/lld/test/ELF/x86-64-reloc-range.s
+++ b/lld/test/ELF/x86-64-reloc-range.s
@@ -12,5 +12,5 @@
 
         .hidden foo
         .bss
-        .zero 0x7fffe007
+        .zero 0x7fffdf87
 foo:
diff --git a/lld/test/ELF/x86-64-reloc-size.s b/lld/test/ELF/x86-64-reloc-size.s
index 4edc4b75fad48..dbf2b2ca221e6 100644
--- a/lld/test/ELF/x86-64-reloc-size.s
+++ b/lld/test/ELF/x86-64-reloc-size.s
@@ -14,10 +14,10 @@
 # NORELOC-NEXT: ]
 
 # DATA:      section '.data':
-# DATA-NEXT: 0x00202000 00000000 00000000 00000000 00000000
-# DATA-NEXT: 0x00202010 00000000 00000000 00001900 00000000
-# DATA-NEXT: 0x00202020 00001b00 00000000 00001900 00000000
-# DATA-NEXT: 0x00202030 00001b00 00000000 0000
+# DATA-NEXT: 0x002031ac 00000000 00000000 00000000 00000000
+# DATA-NEXT: 0x002031bc 00000000 00000000 00001900 00000000
+# DATA-NEXT: 0x002031cc 00001b00 00000000 00001900 00000000
+# DATA-NEXT: 0x002031dc 00001b00 00000000 0000
 
 # DISASM:      _start:
 # DISASM-NEXT:   movl 25, %eax
@@ -27,10 +27,10 @@
 
 # RELOC2:      Relocations [
 # RELOC2-NEXT: Section ({{.*}}) .rela.dyn {
-# RELOC2-NEXT:    0x1003 R_X86_64_SIZE32 foo 0xFFFFFFFFFFFFFFFF
-# RELOC2-NEXT:    0x100A R_X86_64_SIZE32 foo 0x1
-# RELOC2-NEXT:    0x301A R_X86_64_SIZE64 foo 0xFFFFFFFFFFFFFFFF
-# RELOC2-NEXT:    0x3022 R_X86_64_SIZE64 foo 0x1
+# RELOC2-NEXT:    0x2333 R_X86_64_SIZE32 foo 0xFFFFFFFFFFFFFFFF
+# RELOC2-NEXT:    0x233A R_X86_64_SIZE32 foo 0x1
+# RELOC2-NEXT:    0x440A R_X86_64_SIZE64 foo 0xFFFFFFFFFFFFFFFF
+# RELOC2-NEXT:    0x4412 R_X86_64_SIZE64 foo 0x1
 # RELOC2-NEXT:  }
 # RELOC2-NEXT: ]
 
diff --git a/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s b/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
index 70355c05e2cd6..2ffc03df77a6f 100644
--- a/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
+++ b/lld/test/ELF/x86-64-retpoline-znow-static-iplt.s
@@ -3,18 +3,18 @@
 # RUN: ld.lld -z retpolineplt -z now %t.o -o %t
 # RUN: llvm-objdump -d -no-show-raw-insn %t | FileCheck %s
 
-#0x201001+5 + 42 = 0x201030 (foo@plt)
+#0x2011a9+5 + 34 = 0x2011d0 (foo@plt)
 # CHECK:      _start:
-# CHECK-NEXT:  201001:       callq   42
+# CHECK-NEXT:  2011a9:       callq   34
 
 #Static IPLT header due to -z retpolineplt
-# CHECK:       0000000000201010 .plt:
-# CHECK-NEXT:  201010:       callq   11 <.plt+0x10>
-# CHECK-NEXT:  201015:       pause
-# CHECK-NEXT:  201017:       lfence
+# CHECK:       00000000002011b0 .plt:
+# CHECK-NEXT:  2011b0:       callq   11 <.plt+0x10>
+# CHECK-NEXT:  2011b5:       pause
+# CHECK-NEXT:  2011b7:       lfence
 #foo@plt
-# CHECK:       201030:       movq    4041(%rip), %r11
-# CHECK-NEXT:  201037:       jmp     -44 <.plt>
+# CHECK:       2011d0:       movq    4105(%rip), %r11
+# CHECK-NEXT:  2011d7:       jmp     -44 <.plt>
 
 .type foo STT_GNU_IFUNC
 .globl foo
diff --git a/lld/test/ELF/x86-64-retpoline-znow.s b/lld/test/ELF/x86-64-retpoline-znow.s
index 061f4c384bd62..1b9da3ae0d771 100644
--- a/lld/test/ELF/x86-64-retpoline-znow.s
+++ b/lld/test/ELF/x86-64-retpoline-znow.s
@@ -9,7 +9,7 @@
 // CHECK:      Disassembly of section .plt:
 // CHECK-EMPTY:
 // CHECK-NEXT: .plt:
-// CHECK-NEXT: 1010:       callq   11 <.plt+0x10>
+// CHECK-NEXT: 12d0:       callq   11 <.plt+0x10>
 // CHECK-NEXT:             pause
 // CHECK-NEXT:             lfence
 // CHECK-NEXT:             jmp     -7 <.plt+0x5>
@@ -17,7 +17,7 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1020:       movq    %r11, (%rsp)
+// CHECK-NEXT: 12e0:       movq    %r11, (%rsp)
 // CHECK-NEXT:             retq
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
@@ -30,13 +30,13 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1030:       movq    4289(%rip), %r11
+// CHECK-NEXT: 12f0:       movq    4369(%rip), %r11
 // CHECK-NEXT:             jmp     -44 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1040:       movq    4281(%rip), %r11
+// CHECK-NEXT: 1300:       movq    4361(%rip), %r11
 // CHECK-NEXT:             jmp     -60 <.plt>
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
@@ -44,9 +44,9 @@
 // CHECK-NEXT:             int3
 
 // CHECK:      Contents of section .got.plt:
-// CHECK-NEXT: 20e0 00200000 00000000 00000000 00000000
-// CHECK-NEXT: 20f0 00000000 00000000 00000000 00000000
-// CHECK-NEXT: 2100 00000000 00000000
+// CHECK-NEXT: 23f0 10230000 00000000 00000000 00000000
+// CHECK-NEXT: 2400 00000000 00000000 00000000 00000000
+// CHECK-NEXT: 2410 00000000 00000000
 
 .global _start
 _start:
diff --git a/lld/test/ELF/x86-64-retpoline.s b/lld/test/ELF/x86-64-retpoline.s
index 8691f756b40b6..9cd2a3293e925 100644
--- a/lld/test/ELF/x86-64-retpoline.s
+++ b/lld/test/ELF/x86-64-retpoline.s
@@ -9,8 +9,8 @@
 // CHECK:      Disassembly of section .plt:
 // CHECK-EMPTY:
 // CHECK-NEXT: .plt:
-// CHECK-NEXT: 1010:       pushq   8178(%rip)
-// CHECK-NEXT:             movq    8179(%rip), %r11
+// CHECK-NEXT: 1300:       pushq   8498(%rip)
+// CHECK-NEXT:             movq    8499(%rip), %r11
 // CHECK-NEXT:             callq   14 <.plt+0x20>
 // CHECK-NEXT:             pause
 // CHECK-NEXT:             lfence
@@ -22,7 +22,7 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1030:       movq    %r11, (%rsp)
+// CHECK-NEXT: 1320:       movq    %r11, (%rsp)
 // CHECK-NEXT:             retq
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
@@ -35,7 +35,7 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1040:       movq    8145(%rip), %r11
+// CHECK-NEXT: 1330:       movq    8465(%rip), %r11
 // CHECK-NEXT:             callq   -28 <.plt+0x20>
 // CHECK-NEXT:             jmp     -47 <.plt+0x12>
 // CHECK-NEXT:             pushq   $0
@@ -45,7 +45,7 @@
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
 // CHECK-NEXT:             int3
-// CHECK-NEXT: 1060:       movq    8121(%rip), %r11
+// CHECK-NEXT: 1350:       movq    8441(%rip), %r11
 // CHECK-NEXT:             callq   -60 <.plt+0x20>
 // CHECK-NEXT:             jmp     -79 <.plt+0x12>
 // CHECK-NEXT:             pushq   $1
@@ -57,9 +57,9 @@
 // CHECK-NEXT:             int3
 
 // CHECK:      Contents of section .got.plt:
-// CHECK-NEXT: 3000 00200000 00000000 00000000 00000000
-// CHECK-NEXT: 3010 00000000 00000000 51100000 00000000
-// CHECK-NEXT: 3020 71100000 00000000
+// CHECK-NEXT: 3430 70230000 00000000 00000000 00000000
+// CHECK-NEXT: 3440 00000000 00000000 41130000 00000000
+// CHECK-NEXT: 3450 61130000 00000000
 
 .global _start
 _start:
diff --git a/lld/test/ELF/x86-64-tls-dynamic.s b/lld/test/ELF/x86-64-tls-dynamic.s
index 76396f6a16918..d78ae84e44aab 100644
--- a/lld/test/ELF/x86-64-tls-dynamic.s
+++ b/lld/test/ELF/x86-64-tls-dynamic.s
@@ -48,29 +48,29 @@ c:
 // CHECK-NEXT:       SHF_ALLOC
 // CHECK-NEXT:       SHF_WRITE
 // CHECK-NEXT:     ]
-// CHECK-NEXT:     Address: 0x20E0
+// CHECK-NEXT:     Address: 0x24A0
 // CHECK-NEXT:     Offset:
 // CHECK-NEXT:     Size: 40
 
 // CHECK:      Relocations [
 // CHECK:        Section ({{.+}}) .rela.dyn {
-// CHECK-NEXT:     0x20E0 R_X86_64_DTPMOD64 - 0x0
-// CHECK-NEXT:     0x20F0 R_X86_64_DTPMOD64 c 0x0
-// CHECK-NEXT:     0x20F8 R_X86_64_DTPOFF64 c 0x0
-// CHECK-NEXT:     0x2100 R_X86_64_TPOFF64 c 0x0
+// CHECK-NEXT:     0x24A0 R_X86_64_DTPMOD64 - 0x0
+// CHECK-NEXT:     0x24B0 R_X86_64_DTPMOD64 c 0x0
+// CHECK-NEXT:     0x24B8 R_X86_64_DTPOFF64 c 0x0
+// CHECK-NEXT:     0x24C0 R_X86_64_TPOFF64 c 0x0
 // CHECK-NEXT:   }
 
-// 4313 = (0x20E0 + -4) - (0x1000 + 3) // PC relative offset to got entry.
-// 4301 = (0x20F0 + -4) - (0x100c + 3) // PC relative offset to got entry.
-// 4283 = (0x20F8 + -4) - (0x102e + 3) // PC relative offset to got entry.
-// 4279 = (0x2100 + -4) - (0x1042 + 3) // PC relative offset to got entry.
+// 4457 = (0x24A0 + -4) - (0x1330 + 3) // PC relative offset to got entry.
+// 4445 = (0x24B0 + -4) - (0x133c + 3) // PC relative offset to got entry.
+// 4427 = (0x24B8 + -4) - (0x135e + 3) // PC relative offset to got entry.
+// 4423 = (0x24C0 + -4) - (0x1372 + 3) // PC relative offset to got entry.
 
 // DIS:      Disassembly of section .text:
 // DIS-EMPTY:
 // DIS-NEXT: .text:
-// DIS-NEXT:     1000: {{.+}} leaq    4313(%rip), %rdi
+// DIS-NEXT:     1330: {{.+}} leaq    4457(%rip), %rdi
 // DIS-NEXT:           {{.+}} callq
-// DIS-NEXT:     100c: {{.+}} leaq    4301(%rip), %rdi
+// DIS-NEXT:     133c: {{.+}} leaq    4445(%rip), %rdi
 // DIS-NEXT:           {{.+}} callq
 // DIS-NEXT:           {{.+}} leaq    (%rax), %rcx
 // DIS-NEXT:           {{.+}} leaq    4(%rax), %rcx
@@ -78,10 +78,10 @@ c:
 // DIS-NEXT:           00 00
 // DIS-NEXT:           00 00
 // DIS-NEXT:           00 00
-// DIS-NEXT:     102e: {{.+}} leaq    4283(%rip), %rdi
+// DIS-NEXT:     135e: {{.+}} leaq    4427(%rip), %rdi
 // DIS-NEXT:           {{.+}} callq
 // DIS-NEXT:           {{.+}} leaq    (%rax), %rcx
-// DIS-NEXT:     1042: {{.+}} movq    4279(%rip), %rax
+// DIS-NEXT:     1372: {{.+}} movq    4423(%rip), %rax
 // DIS-NEXT:           {{.+}} movq    %fs:(%rax), %rax
 // DIS-NEXT:           {{.+}} movabsq $0, %rax
 // DIS-NEXT:           {{.+}} movabsq $4, %rax
diff --git a/lld/test/ELF/x86-64-tls-gd-local.s b/lld/test/ELF/x86-64-tls-gd-local.s
index e6b61190213cf..51bd7ed45f58f 100644
--- a/lld/test/ELF/x86-64-tls-gd-local.s
+++ b/lld/test/ELF/x86-64-tls-gd-local.s
@@ -34,8 +34,8 @@ bar:
 // CHECK-NEXT:   SHF_ALLOC (0x2)
 // CHECK-NEXT:   SHF_WRITE (0x1)
 // CHECK-NEXT: ]
-// CHECK-NEXT: Address: 0x20D0
-// CHECK-NEXT: Offset: 0x20D0
+// CHECK-NEXT: Address: 0x23F0
+// CHECK-NEXT: Offset: 0x3F0
 // CHECK-NEXT: Size: 32
 // CHECK-NEXT: Link: 0
 // CHECK-NEXT: Info: 0
@@ -47,6 +47,6 @@ bar:
 // CHECK-NEXT: )
 
 // CHECK:      Section ({{.*}}) .rela.dyn {
-// CHECK-NEXT:   0x20D0 R_X86_64_DTPMOD64 - 0x0
-// CHECK-NEXT:   0x20E0 R_X86_64_DTPMOD64 - 0x0
+// CHECK-NEXT:   0x23F0 R_X86_64_DTPMOD64 - 0x0
+// CHECK-NEXT:   0x2400 R_X86_64_DTPMOD64 - 0x0
 // CHECK-NEXT: }
diff --git a/lld/test/ELF/x86-64-tls-gdie.s b/lld/test/ELF/x86-64-tls-gdie.s
index 541add2469cf0..366f562752b82 100644
--- a/lld/test/ELF/x86-64-tls-gdie.s
+++ b/lld/test/ELF/x86-64-tls-gdie.s
@@ -11,20 +11,20 @@
 
 //RELOC:      Relocations [
 //RELOC-NEXT:   Section (5) .rela.dyn {
-//RELOC-NEXT:     0x2020C0 R_X86_64_TPOFF64 tlsshared0 0x0
-//RELOC-NEXT:     0x2020C8 R_X86_64_TPOFF64 tlsshared1 0x0
+//RELOC-NEXT:     0x2023A8 R_X86_64_TPOFF64 tlsshared0 0x0
+//RELOC-NEXT:     0x2023B0 R_X86_64_TPOFF64 tlsshared1 0x0
 //RELOC-NEXT:   }
 //RELOC-NEXT: ]
 
-// 0x2020C0 - (201009+7) = 4272
-// 0x2020C8 - (201019+7) = 4264
+// 0x2023A8 - (2012d1+7) = 4304
+// 0x2023B0 - (2012e1+7) = 4296
 // DISASM:      Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: _start:
 // DISASM-NEXT:               movq %fs:0, %rax
-// DISASM-NEXT: 201009:       addq 4272(%rip), %rax
+// DISASM-NEXT: 2012d1:       addq 4304(%rip), %rax
 // DISASM-NEXT:               movq %fs:0, %rax
-// DISASM-NEXT: 201019:       addq 4264(%rip), %rax
+// DISASM-NEXT: 2012e1:       addq 4296(%rip), %rax
 
 .section .text
 .globl _start
diff --git a/lld/test/ELF/x86-64-tls-ie-local.s b/lld/test/ELF/x86-64-tls-ie-local.s
index 19e0740e679a4..ee21189245d50 100644
--- a/lld/test/ELF/x86-64-tls-ie-local.s
+++ b/lld/test/ELF/x86-64-tls-ie-local.s
@@ -5,18 +5,18 @@
 # RUN: llvm-readobj -r %t.so | FileCheck --check-prefix=REL %s
 # RUN: llvm-objdump -d --no-show-raw-insn %t.so | FileCheck %s
 
-# SEC: .got PROGBITS 00000000000020b0 0020b0 000010 00 WA 0 0 8
+# SEC: .got PROGBITS 0000000000002338 000338 000010 00 WA 0 0 8
 
 ## Dynamic relocations for non-preemptable symbols in a shared object have section index 0.
 # REL:      .rela.dyn {
-# REL-NEXT:   0x20B0 R_X86_64_TPOFF64 - 0x0
-# REL-NEXT:   0x20B8 R_X86_64_TPOFF64 - 0x4
+# REL-NEXT:   0x2338 R_X86_64_TPOFF64 - 0x0
+# REL-NEXT:   0x2340 R_X86_64_TPOFF64 - 0x4
 # REL-NEXT: }
 
-## &.got[0] - 0x1007 = 0x20B0 - 0x1007 = 4265
-## &.got[1] - 0x100e = 0x20B8 - 0x100e = 4266
-# CHECK:      1000:       addq 4265(%rip), %rax
-# CHECK-NEXT: 1007:       addq 4266(%rip), %rax
+## &.got[0] - 0x127f = 0x2338 - 0x127f = 4281
+## &.got[1] - 0x1286 = 0x2340 - 0x1286 = 4282
+# CHECK:      1278:       addq 4281(%rip), %rax
+# CHECK-NEXT: 127f:       addq 4282(%rip), %rax
 
 addq foo@GOTTPOFF(%rip), %rax
 addq bar@GOTTPOFF(%rip), %rax
diff --git a/lld/test/ELF/x86-64-tls-ie.s b/lld/test/ELF/x86-64-tls-ie.s
index d05208bae3c46..ab351191120a1 100644
--- a/lld/test/ELF/x86-64-tls-ie.s
+++ b/lld/test/ELF/x86-64-tls-ie.s
@@ -1,13 +1,13 @@
 // REQUIRES: x86
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t1.o
 // RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %p/Inputs/tls-got.s -o %t2.o
-// RUN: ld.lld -shared %t2.o -o %t2.so
-// RUN: ld.lld --hash-style=sysv -e main %t1.o %t2.so -o %t3
+// RUN: ld.lld -shared %t2.o -soname=so -o %t2.so
+// RUN: ld.lld -e main %t1.o %t2.so -o %t3
 // RUN: llvm-readobj -S -r %t3 | FileCheck %s
 // RUN: llvm-objdump -d %t3 | FileCheck --check-prefix=DISASM %s
 
 // CHECK:      Section {
-// CHECK:      Index: 8
+// CHECK:      Index: 9
 // CHECK-NEXT: Name: .got
 // CHECK-NEXT: Type: SHT_PROGBITS
 // CHECK-NEXT: Flags [
@@ -15,7 +15,7 @@
 // CHECK-NEXT:   SHF_WRITE
 // CHECK-NEXT: ]
 // CHECK-NEXT: Address: [[ADDR:.*]]
-// CHECK-NEXT: Offset: 0x20C0
+// CHECK-NEXT: Offset: 0x3C0
 // CHECK-NEXT: Size: 16
 // CHECK-NEXT: Link: 0
 // CHECK-NEXT: Info: 0
@@ -24,25 +24,25 @@
 // CHECK-NEXT: }
 
 // CHECK:      Relocations [
-// CHECK-NEXT:   Section (4) .rela.dyn {
-// CHECK-NEXT:     0x2020C8 R_X86_64_TPOFF64 tls0 0x0
+// CHECK-NEXT:   Section (5) .rela.dyn {
+// CHECK-NEXT:     0x2023C8 R_X86_64_TPOFF64 tls0 0x0
 // CHECK-NEXT:     [[ADDR]] R_X86_64_TPOFF64 tls1 0x0
 // CHECK-NEXT:   }
 // CHECK-NEXT: ]
 
-// 0x201000 + 4281 + 7 = 0x2020C0
-// 0x20100A + 4279 + 7 = 0x2020C8
-// 0x201014 + 4269 + 7 = 0x2020C8
+// 0x2012d0 + 4329 + 7 = 0x2023C0
+// 0x2012dA + 4327 + 7 = 0x2023C8
+// 0x2012e4 + 4317 + 7 = 0x2023C8
 // DISASM:      Disassembly of section .text:
 // DISASM-EMPTY:
 // DISASM-NEXT: main:
-// DISASM-NEXT: 201000: {{.*}} movq 4281(%rip), %rax
-// DISASM-NEXT: 201007: {{.*}} movl %fs:(%rax), %eax
-// DISASM-NEXT: 20100a: {{.*}} movq 4279(%rip), %rax
-// DISASM-NEXT: 201011: {{.*}} movl %fs:(%rax), %eax
-// DISASM-NEXT: 201014: {{.*}} movq 4269(%rip), %rax
-// DISASM-NEXT: 20101b: {{.*}} movl %fs:(%rax), %eax
-// DISASM-NEXT: 20101e: {{.*}} retq
+// DISASM-NEXT: 2012d0: {{.*}} movq 4329(%rip), %rax
+// DISASM-NEXT: 2012d7: {{.*}} movl %fs:(%rax), %eax
+// DISASM-NEXT: 2012da: {{.*}} movq 4327(%rip), %rax
+// DISASM-NEXT: 2012e1: {{.*}} movl %fs:(%rax), %eax
+// DISASM-NEXT: 2012e4: {{.*}} movq 4317(%rip), %rax
+// DISASM-NEXT: 2012eb: {{.*}} movl %fs:(%rax), %eax
+// DISASM-NEXT: 2012ee: {{.*}} retq
 
 .section .tdata,"awT",@progbits
 
diff --git a/lld/test/ELF/x86-64-tls-opt-noplt.s b/lld/test/ELF/x86-64-tls-opt-noplt.s
index 3f2f5ce04d4ee..931e9616493bc 100644
--- a/lld/test/ELF/x86-64-tls-opt-noplt.s
+++ b/lld/test/ELF/x86-64-tls-opt-noplt.s
@@ -12,8 +12,8 @@
 
 // RELOC:      Relocations [
 // RELOC-NEXT:  Section {{.*}} .rela.dyn {
-// RELOC-NEXT:    0x2020C0 R_X86_64_TPOFF64 tlsshared0 0x0
-// RELOC-NEXT:    0x2020C8 R_X86_64_TPOFF64 tlsshared1 0x0
+// RELOC-NEXT:    0x202420 R_X86_64_TPOFF64 tlsshared0 0x0
+// RELOC-NEXT:    0x202428 R_X86_64_TPOFF64 tlsshared1 0x0
 // RELOC-NEXT:  }
 // RELOC-NEXT: ]
 
@@ -21,9 +21,9 @@
 
 // Table 11.5: GD -> IE Code Transition (LP64)
 // DISASM-NEXT:               movq %fs:0, %rax
-// DISASM-NEXT: 201009:       addq 4272(%rip), %rax
+// DISASM-NEXT: 201309:       addq 4368(%rip), %rax
 // DISASM-NEXT:               movq %fs:0, %rax
-// DISASM-NEXT: 201019:       addq 4264(%rip), %rax
+// DISASM-NEXT: 201319:       addq 4360(%rip), %rax
 
 // Table 11.7: GD -> LE Code Transition (LP64)
 // DISASM-NEXT:               movq %fs:0, %rax
diff --git a/lld/test/ELF/x86-64-tlsdesc-gd.s b/lld/test/ELF/x86-64-tlsdesc-gd.s
index 1c72fbe40d04b..350b254fad078 100644
--- a/lld/test/ELF/x86-64-tlsdesc-gd.s
+++ b/lld/test/ELF/x86-64-tlsdesc-gd.s
@@ -16,18 +16,18 @@
 # RUN: llvm-objdump -d --no-show-raw-insn %t | FileCheck --check-prefix=IE %s
 
 # GD-REL:      .rela.dyn {
-# GD-REL-NEXT:   0x20A0 R_X86_64_TLSDESC a 0x0
-# GD-REL-NEXT:   0x20B0 R_X86_64_TLSDESC b 0x0
+# GD-REL-NEXT:   0x2380 R_X86_64_TLSDESC a 0x0
+# GD-REL-NEXT:   0x2390 R_X86_64_TLSDESC b 0x0
 # GD-REL-NEXT: }
 
-# 0x20a0-0x1007 = 4249
-# GD:            leaq 4249(%rip), %rax
-# GD-NEXT: 1007: callq *(%rax)
+# 0x2380-0x12cf = 4273
+# GD:            leaq 4273(%rip), %rax
+# GD-NEXT: 12cf: callq *(%rax)
 # GD-NEXT:       movl %fs:(%rax), %eax
 
-# 0x20b0-0x1013 = 4253
-# GD-NEXT:       leaq 4253(%rip), %rax
-# GD-NEXT: 1013: callq *(%rax)
+# 0x2390-0x12db = 4277
+# GD-NEXT:       leaq 4277(%rip), %rax
+# GD-NEXT: 12db: callq *(%rax)
 # GD-NEXT:       movl %fs:(%rax), %eax
 
 # NOREL: no relocations
@@ -42,16 +42,16 @@
 # LE-NEXT: movl %fs:(%rax), %eax
 
 # IE-REL:      .rela.dyn {
-# IE-REL-NEXT:   0x2020C0 R_X86_64_TPOFF64 b 0x0
+# IE-REL-NEXT:   0x202360 R_X86_64_TPOFF64 b 0x0
 # IE-REL-NEXT: }
 
 ## a is relaxed to use LE.
 # IE:              movq $-4, %rax
 # IE-NEXT:         nop
 # IE-NEXT:         movl %fs:(%rax), %eax
-## 0x2020C0 - 0x201013 = 4269
-# IE-NEXT:         movq 4269(%rip), %rax
-# IE-NEXT: 201013: nop
+## 0x202360 - 0x20129b = 4293
+# IE-NEXT:         movq 4293(%rip), %rax
+# IE-NEXT: 20129b: nop
 # IE-NEXT:         movl %fs:(%rax), %eax
 
 leaq a@tlsdesc(%rip), %rax
diff --git a/lld/test/ELF/x86-64-tlsdesc-ld.s b/lld/test/ELF/x86-64-tlsdesc-ld.s
index c478f0baa7ff7..c7e68d1a8c804 100644
--- a/lld/test/ELF/x86-64-tlsdesc-ld.s
+++ b/lld/test/ELF/x86-64-tlsdesc-ld.s
@@ -11,13 +11,13 @@
 
 ## Check _TLS_MODULE_BASE_ used by LD produces a dynamic relocation with a value of 0.
 # LD-REL:      .rela.dyn {
-# LD-REL-NEXT:   0x20A0 R_X86_64_TLSDESC - 0x0
+# LD-REL-NEXT:   0x2318 R_X86_64_TLSDESC - 0x0
 # LD-REL-NEXT: }
 
-## 0x20a0-0x1007 = 4249
+## 0x2318-0x1267 = 4273
 ## dtpoff(a) = 8, dtpoff(b) = 12
-# LD:            leaq 4249(%rip), %rax
-# LD-NEXT: 1007: callq *(%rax)
+# LD:            leaq 4273(%rip), %rax
+# LD-NEXT: 1267: callq *(%rax)
 # LD-NEXT:       movl %fs:8(%rax), %edx
 # LD-NEXT:       addl %fs:12(%rax), %edx
 
diff --git a/lld/test/ELF/ztext.s b/lld/test/ELF/ztext.s
index 814aa5f2f8897..43fab2f67e7f7 100644
--- a/lld/test/ELF/ztext.s
+++ b/lld/test/ELF/ztext.s
@@ -18,9 +18,9 @@
 
 # CHECK:      Relocations [
 # CHECK-NEXT:   Section {{.*}} .rela.dyn {
-# CHECK-NEXT:     0x1000 R_X86_64_RELATIVE - 0x1000
-# CHECK-NEXT:     0x1008 R_X86_64_64 bar 0x0
-# CHECK-NEXT:     0x1010 R_X86_64_PC64 zed 0x0
+# CHECK-NEXT:     0x12A0 R_X86_64_RELATIVE - 0x12A0
+# CHECK-NEXT:     0x12A8 R_X86_64_64 bar 0x0
+# CHECK-NEXT:     0x12B0 R_X86_64_PC64 zed 0x0
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
 
@@ -30,8 +30,8 @@
 
 # STATIC:      Relocations [
 # STATIC-NEXT:   Section {{.*}} .rela.dyn {
-# STATIC-NEXT:     0x201008 R_X86_64_64 bar 0x0
-# STATIC-NEXT:     0x201010 R_X86_64_PC64 zed 0x0
+# STATIC-NEXT:     0x201290 R_X86_64_64 bar 0x0
+# STATIC-NEXT:     0x201298 R_X86_64_PC64 zed 0x0
 # STATIC-NEXT:   }
 # STATIC-NEXT: ]
 
diff --git a/lld/test/mach-o/empty-sections.yaml b/lld/test/mach-o/empty-sections.yaml
new file mode 100644
index 0000000000000..ca6b150b1a467
--- /dev/null
+++ b/lld/test/mach-o/empty-sections.yaml
@@ -0,0 +1,9 @@
+# RUN: ld64.lld -arch x86_64 -r %s -o %t
+#
+# Test that writing empty mach-o sections does not segfault the linker.
+#
+
+--- !mach-o
+arch:            x86_64
+file-type:       MH_OBJECT
+...
diff --git a/lld/test/wasm/custom-section-name.ll b/lld/test/wasm/custom-section-name.ll
new file mode 100644
index 0000000000000..2d46810b13789
--- /dev/null
+++ b/lld/test/wasm/custom-section-name.ll
@@ -0,0 +1,58 @@
+; RUN: llc -filetype=obj %s -o %t.o
+; RUN: wasm-ld -no-gc-sections --no-entry -o %t.wasm %t.o
+; RUN: obj2yaml %t.wasm | FileCheck %s
+; RUN: wasm-ld -no-gc-sections --no-entry -o %t_reloc.o %t.o --relocatable
+; RUN: obj2yaml %t_reloc.o | FileCheck -check-prefix RELOC %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+@bss = hidden global i32 zeroinitializer, align 4
+@foo = hidden global i32 zeroinitializer, section "WowZero!", align 4
+@bar = hidden constant i32 42, section "MyAwesomeSection", align 4
+@baz = hidden global i32 7, section "AnotherGreatSection", align 4
+
+; CHECK-LABEL: - Type:            DATA
+; CHECK-NEXT:    Segments:
+; CHECK-NEXT:      - SectionOffset:   7
+; CHECK-NEXT:        InitFlags:       0
+; CHECK-NEXT:        Offset:
+; CHECK-NEXT:          Opcode:          I32_CONST
+; CHECK-NEXT:          Value:           1024
+; CHECK-NEXT:        Content:         '00000000'
+; CHECK-NEXT:      - SectionOffset:   17
+; CHECK-NEXT:        InitFlags:       0
+; CHECK-NEXT:        Offset:
+; CHECK-NEXT:          Opcode:          I32_CONST
+; CHECK-NEXT:          Value:           1028
+; CHECK-NEXT:        Content:         2A000000
+; CHECK-NEXT:      - SectionOffset:   27
+; CHECK-NEXT:        InitFlags:       0
+; CHECK-NEXT:        Offset:
+; CHECK-NEXT:          Opcode:          I32_CONST
+; CHECK-NEXT:          Value:           1032
+; CHECK-NEXT:        Content:         '07000000'
+; CHECK-NEXT:      - SectionOffset:   37
+; CHECK-NEXT:        InitFlags:       0
+; CHECK-NEXT:        Offset:
+; CHECK-NEXT:          Opcode:          I32_CONST
+; CHECK-NEXT:          Value:           1036
+; CHECK-NEXT:        Content:         '00000000'
+
+; RELOC-LABEL: SegmentInfo:
+; RELOC-NEXT:    - Index:           0
+; RELOC-NEXT:      Name:            'WowZero!'
+; RELOC-NEXT:      Alignment:       2
+; RELOC-NEXT:      Flags:           [  ]
+; RELOC-NEXT:    - Index:           1
+; RELOC-NEXT:      Name:            MyAwesomeSection
+; RELOC-NEXT:      Alignment:       2
+; RELOC-NEXT:      Flags:           [  ]
+; RELOC-NEXT:    - Index:           2
+; RELOC-NEXT:      Name:            AnotherGreatSection
+; RELOC-NEXT:      Alignment:       2
+; RELOC-NEXT:      Flags:           [  ]
+; RELOC-NEXT:    - Index:           3
+; RELOC-NEXT:      Name:            .bss.bss
+; RELOC-NEXT:      Alignment:       2
+; RELOC-NEXT:      Flags:           [  ]
diff --git a/lld/test/wasm/data-layout.ll b/lld/test/wasm/data-layout.ll
index 888512376e75d..2db83e9935525 100644
--- a/lld/test/wasm/data-layout.ll
+++ b/lld/test/wasm/data-layout.ll
@@ -26,19 +26,19 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:         Mutable:         true
 ; CHECK-NEXT:         InitExpr:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           66608
+; CHECK-NEXT:           Value:           66624
 ; CHECK-NEXT:       - Index:           1
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           1071
+; CHECK-NEXT:           Value:           1080
 ; CHECK-NEXT:       - Index:           2
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           66608
+; CHECK-NEXT:           Value:           66624
 
 ; CHECK:        - Type:            DATA
 ; CHECK-NEXT:     Segments:
@@ -47,19 +47,19 @@ target triple = "wasm32-unknown-unknown"
 ; CHECK-NEXT:         Offset:
 ; CHECK-NEXT:           Opcode:          I32_CONST
 ; CHECK-NEXT:           Value:           1024
-; CHECK-NEXT:         Content:         '0100000000000000000000000000000003000000000000002804000024040000'
-; CHECK-NEXT:       - SectionOffset:   45
+; CHECK-NEXT:         Content:         68656C6C6F0A00
+; CHECK-NEXT:       - SectionOffset:   20
 ; CHECK-NEXT:         InitFlags:       0
 ; CHECK-NEXT:         Offset:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           1056
-; CHECK-NEXT:         Content:         '0000000000000000'
-; CHECK-NEXT:       - SectionOffset:   59
+; CHECK-NEXT:           Value:           1040
+; CHECK-NEXT:         Content:         '0100000000000000000000000000000003000000000000000004000034040000'
+; CHECK-NEXT:       - SectionOffset:   58
 ; CHECK-NEXT:         InitFlags:       0
 ; CHECK-NEXT:         Offset:
 ; CHECK-NEXT:           Opcode:          I32_CONST
-; CHECK-NEXT:           Value:           1064
-; CHECK-NEXT:         Content:         68656C6C6F0A00
+; CHECK-NEXT:           Value:           1072
+; CHECK-NEXT:         Content:         '0000000000000000'
 ; CHECK-NEXT:    - Type:            CUSTOM
 
 
@@ -92,10 +92,10 @@ target triple = "wasm32-unknown-unknown"
 ; RELOC-NEXT:     Relocations:
 ; RELOC-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; RELOC-NEXT:         Index:           3
-; RELOC-NEXT:         Offset:          0x00000018
+; RELOC-NEXT:         Offset:          0x00000024
 ; RELOC-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; RELOC-NEXT:         Index:           4
-; RELOC-NEXT:         Offset:          0x0000002E
+; RELOC-NEXT:         Offset:          0x0000002D
 ; RELOC-NEXT:         Addend:          4
 ; RELOC-NEXT:     Segments:
 ; RELOC-NEXT:       - SectionOffset:   6
@@ -103,60 +103,60 @@ target triple = "wasm32-unknown-unknown"
 ; RELOC-NEXT:         Offset:
 ; RELOC-NEXT:           Opcode:          I32_CONST
 ; RELOC-NEXT:           Value:           0
+; RELOC-NEXT:         Content:         68656C6C6F0A00
+; RELOC-NEXT:       - SectionOffset:   18
+; RELOC-NEXT:         InitFlags:       0
+; RELOC-NEXT:         Offset:
+; RELOC-NEXT:           Opcode:          I32_CONST
+; RELOC-NEXT:           Value:           8
 ; RELOC-NEXT:         Content:         '01000000'
-; RELOC-NEXT:       - SectionOffset:   15
+; RELOC-NEXT:       - SectionOffset:   27
 ; RELOC-NEXT:         InitFlags:       0
 ; RELOC-NEXT:         Offset:
 ; RELOC-NEXT:           Opcode:          I32_CONST
 ; RELOC-NEXT:           Value:           16
 ; RELOC-NEXT:         Content:         '03000000'
-; RELOC-NEXT:       - SectionOffset:   24
+; RELOC-NEXT:       - SectionOffset:   36
 ; RELOC-NEXT:         InitFlags:       0
 ; RELOC-NEXT:         Offset:
 ; RELOC-NEXT:           Opcode:          I32_CONST
 ; RELOC-NEXT:           Value:           24
-; RELOC-NEXT:         Content:         '28000000'
-; RELOC-NEXT:       - SectionOffset:   33
+; RELOC-NEXT:         Content:         '00000000'
+; RELOC-NEXT:       - SectionOffset:   45
 ; RELOC-NEXT:         InitFlags:       0
 ; RELOC-NEXT:         Offset:
 ; RELOC-NEXT:           Opcode:          I32_CONST
 ; RELOC-NEXT:           Value:           28
-; RELOC-NEXT:         Content:         '0000000000000000'
-; RELOC-NEXT:       - SectionOffset:   46
-; RELOC-NEXT:         InitFlags:       0
-; RELOC-NEXT:         Offset:
-; RELOC-NEXT:           Opcode:          I32_CONST
-; RELOC-NEXT:           Value:           36
-; RELOC-NEXT:         Content:         '20000000'
-; RELOC-NEXT:       - SectionOffset:   55
+; RELOC-NEXT:         Content:         '24000000'
+; RELOC-NEXT:       - SectionOffset:   54
 ; RELOC-NEXT:         InitFlags:       0
 ; RELOC-NEXT:         Offset:
 ; RELOC-NEXT:           Opcode:          I32_CONST
-; RELOC-NEXT:           Value:           40
-; RELOC-NEXT:         Content:         68656C6C6F0A00
+; RELOC-NEXT:           Value:           32
+; RELOC-NEXT:         Content:         '0000000000000000'
 
 ; RELOC:          SymbolTable:
 ; RELOC-NEXT:       - Index:           0
 ; RELOC-NEXT:         Kind:            DATA
 ; RELOC-NEXT:         Name:            foo
 ; RELOC-NEXT:         Flags:           [ VISIBILITY_HIDDEN ]
-; RELOC-NEXT:         Segment:         0
+; RELOC-NEXT:         Segment:         1
 ; RELOC-NEXT:         Size:            4
 ; RELOC-NEXT:       - Index:           1
 ; RELOC-NEXT:         Kind:            DATA
 ; RELOC-NEXT:         Name:            aligned_bar
 ; RELOC-NEXT:         Flags:           [ VISIBILITY_HIDDEN ]
-; RELOC-NEXT:         Segment:         1
+; RELOC-NEXT:         Segment:         2
 ; RELOC-NEXT:         Size:            4
 ; RELOC-NEXT:       - Index:           2
 ; RELOC-NEXT:         Kind:            DATA
 ; RELOC-NEXT:         Name:            external_ref
 ; RELOC-NEXT:         Flags:           [  ]
-; RELOC-NEXT:         Segment:         2
+; RELOC-NEXT:         Segment:         3
 ; RELOC-NEXT:         Size:            4
 ; RELOC-NEXT:       - Index:           3
 ; RELOC-NEXT:         Kind:            DATA
 ; RELOC-NEXT:         Name:            hello_str
 ; RELOC-NEXT:         Flags:           [  ]
-; RELOC-NEXT:         Segment:         5
+; RELOC-NEXT:         Segment:         0
 ; RELOC-NEXT:         Size:            7
diff --git a/lld/test/wasm/data-segment-merging.ll b/lld/test/wasm/data-segment-merging.ll
index 4ab323b902795..453382337038b 100644
--- a/lld/test/wasm/data-segment-merging.ll
+++ b/lld/test/wasm/data-segment-merging.ll
@@ -19,11 +19,11 @@ target triple = "wasm32-unknown-unknown"
 ; MERGE-NEXT:      - SectionOffset:   7
 ; MERGE-NEXT:        InitFlags:       0
 ; MERGE-NEXT:        Offset:
-; MERGE:             Content:         68656C6C6F00676F6F6462796500776861746576657200002A000000
-; MERGE-NEXT:      - SectionOffset:   41
+; MERGE:             Content:         636F6E7374616E74000000002B
+; MERGE-NEXT:      - SectionOffset:   26
 ; MERGE-NEXT:        InitFlags:       0
 ; MERGE-NEXT:        Offset:
-; MERGE:             Content:         636F6E7374616E74000000002B
+; MERGE:             Content:         68656C6C6F00676F6F6462796500776861746576657200002A000000
 ; MERGE-NEXT:  - Type:            CUSTOM
 ; MERGE-NEXT:    Name:            name
 ; MERGE-NEXT:    FunctionNames:
@@ -40,27 +40,27 @@ target triple = "wasm32-unknown-unknown"
 ; SEPARATE-NEXT:      - SectionOffset:   7
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         68656C6C6F00
-; SEPARATE-NEXT:      - SectionOffset:   19
+; SEPARATE:             Content:         636F6E7374616E7400
+; SEPARATE-NEXT:      - SectionOffset:   22
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         676F6F6462796500
-; SEPARATE-NEXT:      - SectionOffset:   33
+; SEPARATE:             Content:         2B
+; SEPARATE-NEXT:      - SectionOffset:   29
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         '776861746576657200'
-; SEPARATE-NEXT:      - SectionOffset:   48
+; SEPARATE:             Content:         68656C6C6F00
+; SEPARATE-NEXT:      - SectionOffset:   41
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         2A000000
-; SEPARATE-NEXT:      - SectionOffset:   58
+; SEPARATE:             Content:         676F6F6462796500
+; SEPARATE-NEXT:      - SectionOffset:   55
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         636F6E7374616E7400
-; SEPARATE-NEXT:      - SectionOffset:   73
+; SEPARATE:             Content:         '776861746576657200'
+; SEPARATE-NEXT:      - SectionOffset:   70
 ; SEPARATE-NEXT:        InitFlags:       0
 ; SEPARATE-NEXT:        Offset:
-; SEPARATE:             Content:         2B
+; SEPARATE:             Content:         2A000000
 ; SEPARATE-NEXT:  - Type:            CUSTOM
 ; SEPARATE-NEXT:    Name:            name
 ; SEPARATE-NEXT:    FunctionNames:
@@ -77,10 +77,10 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-MERGE-NEXT:    Segments:
 ; PASSIVE-MERGE-NEXT:      - SectionOffset:   3
 ; PASSIVE-MERGE-NEXT:        InitFlags:       1
-; PASSIVE-MERGE-NEXT:        Content:         68656C6C6F00676F6F6462796500776861746576657200002A000000
-; PASSIVE-MERGE-NEXT:      - SectionOffset:   33
-; PASSIVE-MERGE-NEXT:        InitFlags:       1
 ; PASSIVE-MERGE-NEXT:        Content:         636F6E7374616E74000000002B
+; PASSIVE-MERGE-NEXT:      - SectionOffset:   18
+; PASSIVE-MERGE-NEXT:        InitFlags:       1
+; PASSIVE-MERGE-NEXT:        Content:         68656C6C6F00676F6F6462796500776861746576657200002A000000
 ; PASSIVE-MERGE-NEXT:  - Type:            CUSTOM
 ; PASSIVE-MERGE-NEXT:    Name:            name
 ; PASSIVE-MERGE-NEXT:    FunctionNames:
@@ -100,22 +100,22 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-SEPARATE-NEXT:    Segments:
 ; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   3
 ; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
+; PASSIVE-SEPARATE-NEXT:        Content:         636F6E7374616E7400
+; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   14
+; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
+; PASSIVE-SEPARATE-NEXT:        Content:         2B
+; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   17
+; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
 ; PASSIVE-SEPARATE-NEXT:        Content:         68656C6C6F00
-; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   11
+; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   25
 ; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
 ; PASSIVE-SEPARATE-NEXT:        Content:         676F6F6462796500
-; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   21
+; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   35
 ; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
 ; PASSIVE-SEPARATE-NEXT:        Content:         '776861746576657200'
-; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   32
+; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   46
 ; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
 ; PASSIVE-SEPARATE-NEXT:        Content:         2A000000
-; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   38
-; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
-; PASSIVE-SEPARATE-NEXT:        Content:         636F6E7374616E7400
-; PASSIVE-SEPARATE-NEXT:      - SectionOffset:   49
-; PASSIVE-SEPARATE-NEXT:        InitFlags:       1
-; PASSIVE-SEPARATE-NEXT:        Content:         2B
 ; PASSIVE-SEPARATE-NEXT:    - Type:            CUSTOM
 ; PASSIVE-SEPARATE-NEXT:      Name:            name
 ; PASSIVE-SEPARATE-NEXT:      FunctionNames:
diff --git a/lld/test/wasm/data-segments.ll b/lld/test/wasm/data-segments.ll
index b3530ed0b8d6c..0a15b241cb388 100644
--- a/lld/test/wasm/data-segments.ll
+++ b/lld/test/wasm/data-segments.ll
@@ -37,20 +37,20 @@ target triple = "wasm32-unknown-unknown"
 ; ACTIVE-NEXT:        Offset:
 ; ACTIVE-NEXT:          Opcode:          I32_CONST
 ; ACTIVE-NEXT:          Value:           1024
-; ACTIVE-NEXT:        Content:         68656C6C6F00676F6F646279650000002A000000
-; ACTIVE-NEXT:      - SectionOffset:   34
+; ACTIVE-NEXT:        Content:         636F6E7374616E74000000002B
+; ACTIVE-NEXT:      - SectionOffset:   26
 ; ACTIVE-NEXT:        InitFlags:       0
 ; ACTIVE-NEXT:        Offset:
 ; ACTIVE-NEXT:          Opcode:          I32_CONST
-; ACTIVE-NEXT:          Value:           1044
-; ACTIVE-NEXT:        Content:         '0000000000
-; ACTIVE-SAME:                           0000000000'
-; ACTIVE-NEXT:      - SectionOffset:   10041
+; ACTIVE-NEXT:          Value:           1040
+; ACTIVE-NEXT:        Content:         68656C6C6F00676F6F646279650000002A000000
+; ACTIVE-NEXT:      - SectionOffset:   53
 ; ACTIVE-NEXT:        InitFlags:       0
 ; ACTIVE-NEXT:        Offset:
 ; ACTIVE-NEXT:          Opcode:          I32_CONST
-; ACTIVE-NEXT:          Value:           11044
-; ACTIVE-NEXT:        Content:         636F6E7374616E74000000002B
+; ACTIVE-NEXT:          Value:           1060
+; ACTIVE-NEXT:        Content:         '0000000000
+; ACTIVE-SAME:                           0000000000'
 ; ACTIVE-NEXT:  - Type:            CUSTOM
 ; ACTIVE-NEXT:    Name:            name
 ; ACTIVE-NEXT:    FunctionNames:
@@ -66,7 +66,7 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-NEXT:        Body:            0B
 ; PASSIVE-NEXT:      - Index:           1
 ; PASSIVE-NEXT:        Locals:          []
-; PASSIVE-NEXT:        Body:            41B4D60041004101FE480200044041B4D6004101427FFE0102001A0541800841004114FC08000041940841004190CE00FC08010041A4D6004100410DFC08020041B4D6004102FE17020041B4D600417FFE0002001A0BFC0900FC0901FC09020B
+; PASSIVE-NEXT:        Body:            41B4D60041004101FE480200044041B4D6004101427FFE0102001A054180084100410DFC08000041900841004114FC08010041A40841004190CE00FC08020041B4D6004102FE17020041B4D600417FFE0002001A0BFC0900FC0901FC09020B
 ; PASSIVE-NEXT:  - Index:           2
 ; PASSIVE-NEXT:    Locals:          []
 ; PASSIVE-NEXT:    Body:            0B
@@ -74,14 +74,14 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-NEXT:    Segments:
 ; PASSIVE-NEXT:      - SectionOffset:   3
 ; PASSIVE-NEXT:        InitFlags:       1
+; PASSIVE-NEXT:        Content:         636F6E7374616E74000000002B
+; PASSIVE-NEXT:      - SectionOffset:   18
+; PASSIVE-NEXT:        InitFlags:       1
 ; PASSIVE-NEXT:        Content:         68656C6C6F00676F6F646279650000002A000000
-; PASSIVE-NEXT:      - SectionOffset:   26
+; PASSIVE-NEXT:      - SectionOffset:   41
 ; PASSIVE-NEXT:        InitFlags:       1
 ; PASSIVE-NEXT:        Content:         '0000000000
 ; PASSIVE-SAME:                           0000000000'
-; PASSIVE-NEXT:      - SectionOffset:   10028
-; PASSIVE-NEXT:        InitFlags:       1
-; PASSIVE-NEXT:        Content:         636F6E7374616E74000000002B
 ; PASSIVE-NEXT:  - Type:            CUSTOM
 ; PASSIVE-NEXT:    Name:            name
 ; PASSIVE-NEXT:    FunctionNames:
@@ -89,5 +89,5 @@ target triple = "wasm32-unknown-unknown"
 ; PASSIVE-NEXT:        Name:            __wasm_call_ctors
 ; PASSIVE-NEXT:      - Index:           1
 ; PASSIVE-NEXT:        Name:            __wasm_init_memory
-; PASSIVE-NEXT:  - Index:           2
-; PASSIVE-NEXT:    Name:            __wasm_init_tls
+; PASSIVE-NEXT:      - Index:           2
+; PASSIVE-NEXT:        Name:            __wasm_init_tls
diff --git a/lld/test/wasm/pic-static.ll b/lld/test/wasm/pic-static.ll
index f0187ec32ec06..d13d74d8a2143 100644
--- a/lld/test/wasm/pic-static.ll
+++ b/lld/test/wasm/pic-static.ll
@@ -3,12 +3,13 @@
 ; fixed values.
 ; RUN: llc -relocation-model=pic -filetype=obj %p/Inputs/ret32.ll -o %t.ret32.o
 ; RUN: llc -relocation-model=pic -filetype=obj %s -o %t.o
-; RUN: wasm-ld -o %t.wasm %t.o %t.ret32.o
+; RUN: wasm-ld --allow-undefined --export-all -o %t.wasm %t.o %t.ret32.o
 ; RUN: obj2yaml %t.wasm | FileCheck %s
 
 target triple = "wasm32-unknown-emscripten"
 
 declare i32 @ret32(float)
+declare i32 @missing_function(float)
 @global_float = global float 1.0
 @hidden_float = hidden global float 2.0
 
@@ -18,6 +19,10 @@ define i32 (float)* @getaddr_external() {
   ret i32 (float)* @ret32;
 }
 
+define i32 (float)* @getaddr_missing_function() {
+  ret i32 (float)* @missing_function;
+}
+
 define i32 ()* @getaddr_hidden() {
   ret i32 ()* @hidden_func;
 }
@@ -60,10 +65,18 @@ entry:
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
 ; CHECK-NEXT:           Opcode:          I32_CONST
+; CHECK-NEXT:           Value:           1
+
+; GOT.func.missing_function
+; CHECK-NEXT:       - Index:           2
+; CHECK-NEXT:         Type:            I32
+; CHECK-NEXT:         Mutable:         false
+; CHECK-NEXT:         InitExpr:
+; CHECK-NEXT:           Opcode:          I32_CONST
 ; CHECK-NEXT:           Value:           2
 
 ; __table_base
-; CHECK-NEXT:       - Index:           2
+; CHECK-NEXT:       - Index:           3
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
@@ -71,7 +84,7 @@ entry:
 ; CHECK-NEXT:           Value:           1
 
 ; GOT.mem.global_float
-; CHECK-NEXT:       - Index:           3
+; CHECK-NEXT:       - Index:           4
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
@@ -79,7 +92,7 @@ entry:
 ; CHECK-NEXT:           Value:           1024
 
 ; GOT.mem.ret32_ptr
-; CHECK-NEXT:       - Index:           4
+; CHECK-NEXT:       - Index:           5
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
@@ -87,7 +100,7 @@ entry:
 ; CHECK-NEXT:           Value:           1032
 
 ; __memory_base
-; CHECK-NEXT:       - Index:           5
+; CHECK-NEXT:       - Index:           6
 ; CHECK-NEXT:         Type:            I32
 ; CHECK-NEXT:         Mutable:         false
 ; CHECK-NEXT:         InitExpr:
diff --git a/lld/test/wasm/reloc-addend.ll b/lld/test/wasm/reloc-addend.ll
index fc00b8ebc31cf..f232c26d48a1f 100644
--- a/lld/test/wasm/reloc-addend.ll
+++ b/lld/test/wasm/reloc-addend.ll
@@ -12,8 +12,8 @@ target triple = "wasm32-unknown-unknown"
 @bar = hidden local_unnamed_addr global i32* getelementptr inbounds ([76 x i32], [76 x i32]* @foo, i32 0, i32 16), align 4
 
 ; CHECK:        - Type:            DATA
-; CHECK-NEXT:     Relocations:     
+; CHECK-NEXT:     Relocations:
 ; CHECK-NEXT:       - Type:            R_WASM_MEMORY_ADDR_I32
 ; CHECK-NEXT:         Index:           0
-; CHECK-NEXT:         Offset:          0x0000013D
+; CHECK-NEXT:         Offset:          0x00000006
 ; CHECK-NEXT:         Addend:          64
diff --git a/lld/test/wasm/relocatable.ll b/lld/test/wasm/relocatable.ll
index 64dc59cbaa35e..5ccaadf124650 100644
--- a/lld/test/wasm/relocatable.ll
+++ b/lld/test/wasm/relocatable.ll
@@ -34,6 +34,9 @@ entry:
   ret i32 ptrtoint ([3 x i8]* @data_comdat to i32)
 }
 
+; Test that __attribute__(used) (i.e NO_STRIP) is preserved in the relocated symbol table
+@llvm.used = appending global [1 x i8*] [i8* bitcast (i32 ()* @my_func to i8*)], section "llvm.metadata"
+
 ; CHECK:      --- !WASM
 ; CHECK-NEXT: FileHeader:
 ; CHECK-NEXT:   Version:         0x00000001
@@ -113,21 +116,21 @@ entry:
 ; CHECK-NEXT:         Body:          1081808080001A1082808080001A41010B
 ; CHECK-NEXT:       - Index:         5
 ; CHECK-NEXT:         Locals:
-; CHECK-NEXT:         Body:          419C808080000B
+; CHECK-NEXT:         Body:          4187808080000B
 ; NORMAL-NEXT:  - Type:            DATA
 ; NORMAL-NEXT:    Relocations:
 ; NORMAL-NEXT:      - Type:            R_WASM_TABLE_INDEX_I32
 ; NORMAL-NEXT:        Index:           3
-; NORMAL-NEXT:        Offset:          0x00000012
+; NORMAL-NEXT:        Offset:          0x0000001A
 ; NORMAL-NEXT:      - Type:            R_WASM_TABLE_INDEX_I32
 ; NORMAL-NEXT:        Index:           4
-; NORMAL-NEXT:        Offset:          0x0000001B
+; NORMAL-NEXT:        Offset:          0x00000023
 ; NORMAL-NEXT:      - Type:            R_WASM_TABLE_INDEX_I32
 ; NORMAL-NEXT:        Index:           5
-; NORMAL-NEXT:        Offset:          0x00000024
+; NORMAL-NEXT:        Offset:          0x0000002C
 ; NORMAL-NEXT:      - Type:            R_WASM_MEMORY_ADDR_I32
 ; NORMAL-NEXT:        Index:           12
-; NORMAL-NEXT:        Offset:          0x0000002D
+; NORMAL-NEXT:        Offset:          0x00000035
 ; NORMAL-NEXT:    Segments:
 ; NORMAL-NEXT:      - SectionOffset:   6
 ; NORMAL-NEXT:        InitFlags:       0
@@ -139,32 +142,32 @@ entry:
 ; NORMAL-NEXT:        InitFlags:       0
 ; NORMAL-NEXT:        Offset:
 ; NORMAL-NEXT:          Opcode:          I32_CONST
-; NORMAL-NEXT:          Value:           8
-; NORMAL-NEXT:        Content:         '01000000'
-; NORMAL-NEXT:      - SectionOffset:   27
+; NORMAL-NEXT:          Value:           7
+; NORMAL-NEXT:        Content:         '616263'
+; NORMAL-NEXT:      - SectionOffset:   26
 ; NORMAL-NEXT:        InitFlags:       0
 ; NORMAL-NEXT:        Offset:
 ; NORMAL-NEXT:          Opcode:          I32_CONST
 ; NORMAL-NEXT:          Value:           12
-; NORMAL-NEXT:        Content:         '02000000'
-; NORMAL-NEXT:      - SectionOffset:   36
+; NORMAL-NEXT:        Content:         '01000000'
+; NORMAL-NEXT:      - SectionOffset:   35
 ; NORMAL-NEXT:        InitFlags:       0
 ; NORMAL-NEXT:        Offset:
 ; NORMAL-NEXT:          Opcode:          I32_CONST
 ; NORMAL-NEXT:          Value:           16
-; NORMAL-NEXT:        Content:         '03000000'
-; NORMAL-NEXT:      - SectionOffset:   45
+; NORMAL-NEXT:        Content:         '02000000'
+; NORMAL-NEXT:      - SectionOffset:   44
 ; NORMAL-NEXT:        InitFlags:       0
 ; NORMAL-NEXT:        Offset:
 ; NORMAL-NEXT:          Opcode:          I32_CONST
-; NORMAL-NEXT:          Value:           24
-; NORMAL-NEXT:        Content:         '00000000'
-; NORMAL-NEXT:      - SectionOffset:   54
+; NORMAL-NEXT:          Value:           20
+; NORMAL-NEXT:        Content:         '03000000'
+; NORMAL-NEXT:      - SectionOffset:   53
 ; NORMAL-NEXT:        InitFlags:       0
 ; NORMAL-NEXT:        Offset:
 ; NORMAL-NEXT:          Opcode:          I32_CONST
-; NORMAL-NEXT:          Value:           28
-; NORMAL-NEXT:        Content:         '616263'
+; NORMAL-NEXT:          Value:           24
+; NORMAL-NEXT:        Content:         '00000000'
 ; NORMAL-NEXT:  - Type:            CUSTOM
 ; NORMAL-NEXT:    Name:            linking
 ; NORMAL-NEXT:    Version:         2
@@ -188,7 +191,7 @@ entry:
 ; NORMAL-NEXT:      - Index:           3
 ; NORMAL-NEXT:        Kind:            FUNCTION
 ; NORMAL-NEXT:        Name:            my_func
-; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN ]
+; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN, NO_STRIP ]
 ; NORMAL-NEXT:        Function:        4
 ; NORMAL-NEXT:      - Index:           4
 ; NORMAL-NEXT:        Kind:            FUNCTION
@@ -209,31 +212,31 @@ entry:
 ; NORMAL-NEXT:        Kind:            DATA
 ; NORMAL-NEXT:        Name:            data_comdat
 ; NORMAL-NEXT:        Flags:           [ BINDING_WEAK ]
-; NORMAL-NEXT:        Segment:         5
+; NORMAL-NEXT:        Segment:         1
 ; NORMAL-NEXT:        Size:            3
 ; NORMAL-NEXT:      - Index:           8
 ; NORMAL-NEXT:        Kind:            DATA
 ; NORMAL-NEXT:        Name:            func_addr1
 ; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN ]
-; NORMAL-NEXT:        Segment:         1
+; NORMAL-NEXT:        Segment:         2
 ; NORMAL-NEXT:        Size:            4
 ; NORMAL-NEXT:      - Index:           9
 ; NORMAL-NEXT:        Kind:            DATA
 ; NORMAL-NEXT:        Name:            func_addr2
 ; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN ]
-; NORMAL-NEXT:        Segment:         2
+; NORMAL-NEXT:        Segment:         3
 ; NORMAL-NEXT:        Size:            4
 ; NORMAL-NEXT:      - Index:           10
 ; NORMAL-NEXT:        Kind:            DATA
 ; NORMAL-NEXT:        Name:            func_addr3
 ; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN ]
-; NORMAL-NEXT:        Segment:         3
+; NORMAL-NEXT:        Segment:         4
 ; NORMAL-NEXT:        Size:            4
 ; NORMAL-NEXT:      - Index:           11
 ; NORMAL-NEXT:        Kind:            DATA
 ; NORMAL-NEXT:        Name:            data_addr1
 ; NORMAL-NEXT:        Flags:           [ VISIBILITY_HIDDEN ]
-; NORMAL-NEXT:        Segment:         4
+; NORMAL-NEXT:        Segment:         5
 ; NORMAL-NEXT:        Size:            4
 ; NORMAL-NEXT:      - Index:           12
 ; NORMAL-NEXT:        Kind:            DATA
@@ -245,32 +248,32 @@ entry:
 ; NORMAL-NEXT:        Alignment:       0
 ; NORMAL-NEXT:        Flags:           [  ]
 ; NORMAL-NEXT:      - Index:           1
+; NORMAL-NEXT:        Name:            .rodata.data_comdat
+; NORMAL-NEXT:        Alignment:       0
+; NORMAL-NEXT:        Flags:           [  ]
+; NORMAL-NEXT:      - Index:           2
 ; NORMAL-NEXT:        Name:            .data.func_addr1
 ; NORMAL-NEXT:        Alignment:       2
 ; NORMAL-NEXT:        Flags:           [  ]
-; NORMAL-NEXT:      - Index:           2
+; NORMAL-NEXT:      - Index:           3
 ; NORMAL-NEXT:        Name:            .data.func_addr2
 ; NORMAL-NEXT:        Alignment:       2
 ; NORMAL-NEXT:        Flags:           [  ]
-; NORMAL-NEXT:      - Index:           3
+; NORMAL-NEXT:      - Index:           4
 ; NORMAL-NEXT:        Name:            .data.func_addr3
 ; NORMAL-NEXT:        Alignment:       2
 ; NORMAL-NEXT:        Flags:           [  ]
-; NORMAL-NEXT:      - Index:           4
+; NORMAL-NEXT:      - Index:           5
 ; NORMAL-NEXT:        Name:            .data.data_addr1
 ; NORMAL-NEXT:        Alignment:       3
 ; NORMAL-NEXT:        Flags:           [  ]
-; NORMAL-NEXT:      - Index:           5
-; NORMAL-NEXT:        Name:            .rodata.data_comdat
-; NORMAL-NEXT:        Alignment:       0
-; NORMAL-NEXT:        Flags:           [  ]
 ; NORMAL-NEXT:    Comdats:
 ; NORMAL-NEXT:      - Name:            func_comdat
 ; NORMAL-NEXT:        Entries:
 ; NORMAL-NEXT:          - Kind:            FUNCTION
 ; NORMAL-NEXT:            Index:           5
 ; NORMAL-NEXT:          - Kind:            DATA
-; NORMAL-NEXT:            Index:           5
+; NORMAL-NEXT:            Index:           1
 ; NORMAL-NEXT:  - Type:            CUSTOM
 ; NORMAL-NEXT:    Name:            name
 ; NORMAL-NEXT:    FunctionNames:
diff --git a/lld/test/wasm/tls-align.ll b/lld/test/wasm/tls-align.ll
index 62e7c492e8f60..8363c82bb29f9 100644
--- a/lld/test/wasm/tls-align.ll
+++ b/lld/test/wasm/tls-align.ll
@@ -24,7 +24,7 @@ define i32* @tls2_addr() {
 ; CHECK-NEXT:       Mutable:         true
 ; CHECK-NEXT:       InitExpr:
 ; CHECK-NEXT:         Opcode:          I32_CONST
-; CHECK-NEXT:         Value:           66608
+; CHECK-NEXT:         Value:           66592
 
 ; __tls_base
 ; CHECK-NEXT:     - Index:           1
diff --git a/lld/test/wasm/tls.ll b/lld/test/wasm/tls.ll
index 304c00f7cae5e..126c2237c1be8 100644
--- a/lld/test/wasm/tls.ll
+++ b/lld/test/wasm/tls.ll
@@ -66,7 +66,7 @@ declare i32 @llvm.wasm.tls.align.i32()
 ; Skip __wasm_call_ctors and __wasm_init_memory
 ; CHECK:          - Index:           2
 ; CHECK-NEXT:       Locals:          []
-; CHECK-NEXT:       Body:            20002401200041004108FC0801000B
+; CHECK-NEXT:       Body:            20002401200041004108FC0800000B
 
 ; Expected body of __wasm_init_tls:
 ;   local.get 0
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index 9a42fa1d2c0d1..34801c1b50e12 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -441,10 +441,8 @@ static Symbol *handleUndefined(StringRef name) {
 
 static UndefinedGlobal *
 createUndefinedGlobal(StringRef name, llvm::wasm::WasmGlobalType *type) {
-  auto *sym =
-      cast<UndefinedGlobal>(symtab->addUndefinedGlobal(name, name,
-                                                       defaultModule, 0,
-                                                       nullptr, type));
+  auto *sym = cast<UndefinedGlobal>(symtab->addUndefinedGlobal(
+      name, name, defaultModule, WASM_SYMBOL_UNDEFINED, nullptr, type));
   config->allowUndefinedSymbols.insert(sym->getName());
   sym->isUsedInRegularObj = true;
   return sym;
@@ -582,7 +580,8 @@ struct WrappedSymbol {
 };
 
 static Symbol *addUndefined(StringRef name) {
-  return symtab->addUndefinedFunction(name, "", "", 0, nullptr, nullptr, false);
+  return symtab->addUndefinedFunction(name, "", "", WASM_SYMBOL_UNDEFINED,
+                                      nullptr, nullptr, false);
 }
 
 // Handles -wrap option.
diff --git a/lld/wasm/InputFiles.cpp b/lld/wasm/InputFiles.cpp
index 6ec4b9dcd01ca..198ac1c41a77b 100644
--- a/lld/wasm/InputFiles.cpp
+++ b/lld/wasm/InputFiles.cpp
@@ -165,10 +165,15 @@ uint32_t ObjFile::calcNewValue(const WasmRelocation &reloc) const {
   switch (reloc.Type) {
   case R_WASM_TABLE_INDEX_I32:
   case R_WASM_TABLE_INDEX_SLEB:
-  case R_WASM_TABLE_INDEX_REL_SLEB:
+  case R_WASM_TABLE_INDEX_REL_SLEB: {
     if (!getFunctionSymbol(reloc.Index)->hasTableIndex())
       return 0;
-    return getFunctionSymbol(reloc.Index)->getTableIndex();
+    uint32_t index = getFunctionSymbol(reloc.Index)->getTableIndex();
+    if (reloc.Type == R_WASM_TABLE_INDEX_REL_SLEB)
+      index -= config->tableBase;
+    return index;
+
+  }
   case R_WASM_MEMORY_ADDR_SLEB:
   case R_WASM_MEMORY_ADDR_I32:
   case R_WASM_MEMORY_ADDR_LEB:
@@ -421,7 +426,7 @@ Symbol *ObjFile::createDefined(const WasmSymbol &sym) {
 
 Symbol *ObjFile::createUndefined(const WasmSymbol &sym, bool isCalledDirectly) {
   StringRef name = sym.Info.Name;
-  uint32_t flags = sym.Info.Flags;
+  uint32_t flags = sym.Info.Flags | WASM_SYMBOL_UNDEFINED;
 
   switch (sym.Info.Kind) {
   case WASM_SYMBOL_TYPE_FUNCTION:
@@ -509,6 +514,7 @@ static Symbol *createBitcodeSymbol(const std::vector<bool> &keptComdats,
   bool excludedByComdat = c != -1 && !keptComdats[c];
 
   if (objSym.isUndefined() || excludedByComdat) {
+    flags |= WASM_SYMBOL_UNDEFINED;
     if (objSym.isExecutable())
       return symtab->addUndefinedFunction(name, name, defaultModule, flags, &f,
                                           nullptr, true);
diff --git a/lld/wasm/InputFiles.h b/lld/wasm/InputFiles.h
index 2b25906f44350..4a445a718352f 100644
--- a/lld/wasm/InputFiles.h
+++ b/lld/wasm/InputFiles.h
@@ -54,7 +54,7 @@ class InputFile {
   Kind kind() const { return fileKind; }
 
   // An archive file name if this file is created from an archive.
-  StringRef archiveName;
+  std::string archiveName;
 
   ArrayRef<Symbol *> getSymbols() const { return symbols; }
 
diff --git a/lld/wasm/OutputSegment.h b/lld/wasm/OutputSegment.h
index 14142fbd44fd1..663a9b8228229 100644
--- a/lld/wasm/OutputSegment.h
+++ b/lld/wasm/OutputSegment.h
@@ -20,7 +20,7 @@ class InputSegment;
 
 class OutputSegment {
 public:
-  OutputSegment(StringRef n, uint32_t index) : name(n), index(index) {}
+  OutputSegment(StringRef n) : name(n) {}
 
   void addInputSegment(InputSegment *inSeg) {
     alignment = std::max(alignment, inSeg->getAlignment());
@@ -32,7 +32,7 @@ class OutputSegment {
   }
 
   StringRef name;
-  const uint32_t index;
+  uint32_t index = 0;
   uint32_t initFlags = 0;
   uint32_t sectionOffset = 0;
   uint32_t alignment = 0;
diff --git a/lld/wasm/Relocations.cpp b/lld/wasm/Relocations.cpp
index ed160a57ea138..018fea9a9c898 100644
--- a/lld/wasm/Relocations.cpp
+++ b/lld/wasm/Relocations.cpp
@@ -51,7 +51,7 @@ static void addGOTEntry(Symbol *sym) {
   if (config->isPic)
     out.importSec->addGOTEntry(sym);
   else
-    out.globalSec->addDummyGOTEntry(sym);
+    out.globalSec->addStaticGOTEntry(sym);
 }
 
 void lld::wasm::scanRelocations(InputChunk *chunk) {
diff --git a/lld/wasm/SymbolTable.cpp b/lld/wasm/SymbolTable.cpp
index 44417b076c7fc..b53ddaf575013 100644
--- a/lld/wasm/SymbolTable.cpp
+++ b/lld/wasm/SymbolTable.cpp
@@ -401,6 +401,7 @@ Symbol *SymbolTable::addUndefinedFunction(StringRef name, StringRef importName,
   LLVM_DEBUG(dbgs() << "addUndefinedFunction: " << name << " ["
                     << (sig ? toString(*sig) : "none")
                     << "] IsCalledDirectly:" << isCalledDirectly << "\n");
+  assert(flags & WASM_SYMBOL_UNDEFINED);
 
   Symbol *s;
   bool wasInserted;
@@ -443,6 +444,7 @@ Symbol *SymbolTable::addUndefinedFunction(StringRef name, StringRef importName,
 Symbol *SymbolTable::addUndefinedData(StringRef name, uint32_t flags,
                                       InputFile *file) {
   LLVM_DEBUG(dbgs() << "addUndefinedData: " << name << "\n");
+  assert(flags & WASM_SYMBOL_UNDEFINED);
 
   Symbol *s;
   bool wasInserted;
@@ -464,6 +466,7 @@ Symbol *SymbolTable::addUndefinedGlobal(StringRef name, StringRef importName,
                                         InputFile *file,
                                         const WasmGlobalType *type) {
   LLVM_DEBUG(dbgs() << "addUndefinedGlobal: " << name << "\n");
+  assert(flags & WASM_SYMBOL_UNDEFINED);
 
   Symbol *s;
   bool wasInserted;
diff --git a/lld/wasm/Symbols.cpp b/lld/wasm/Symbols.cpp
index c5d9aff3130a4..d36e33adf2d07 100644
--- a/lld/wasm/Symbols.cpp
+++ b/lld/wasm/Symbols.cpp
@@ -304,8 +304,7 @@ std::string lld::toString(const wasm::Symbol &sym) {
 
 std::string lld::maybeDemangleSymbol(StringRef name) {
   if (config->demangle)
-    if (Optional<std::string> s = demangleItanium(name))
-      return *s;
+    return demangleItanium(name);
   return name;
 }
 
diff --git a/lld/wasm/Symbols.h b/lld/wasm/Symbols.h
index 7ee76d7fe5e13..787ce7deae630 100644
--- a/lld/wasm/Symbols.h
+++ b/lld/wasm/Symbols.h
@@ -113,8 +113,6 @@ class Symbol {
 
   const WasmSignature* getSignature() const;
 
-  bool isInGOT() const { return gotIndex != INVALID_INDEX; }
-
   uint32_t getGOTIndex() const {
     assert(gotIndex != INVALID_INDEX);
     return gotIndex;
@@ -126,8 +124,9 @@ class Symbol {
 protected:
   Symbol(StringRef name, Kind k, uint32_t flags, InputFile *f)
       : name(name), file(f), flags(flags), symbolKind(k),
-        referenced(!config->gcSections), isUsedInRegularObj(false),
-        forceExport(false), canInline(false), traced(false) {}
+        referenced(!config->gcSections), requiresGOT(false),
+        isUsedInRegularObj(false), forceExport(false), canInline(false),
+        traced(false) {}
 
   StringRef name;
   InputFile *file;
@@ -139,6 +138,10 @@ class Symbol {
 public:
   bool referenced : 1;
 
+  // True for data symbols that needs a dummy GOT entry.  Used for static
+  // linking of GOT accesses.
+  bool requiresGOT : 1;
+
   // True if the symbol was used for linking and thus need to be added to the
   // output file's symbol table. This is true for all symbols except for
   // unreferenced DSO symbols, lazy (archive) symbols, and bitcode symbols that
diff --git a/lld/wasm/SyntheticSections.cpp b/lld/wasm/SyntheticSections.cpp
index f2d4c2e07e1f1..7c3ea321232fc 100644
--- a/lld/wasm/SyntheticSections.cpp
+++ b/lld/wasm/SyntheticSections.cpp
@@ -103,9 +103,9 @@ uint32_t ImportSection::getNumImports() const {
 
 void ImportSection::addGOTEntry(Symbol *sym) {
   assert(!isSealed);
-  LLVM_DEBUG(dbgs() << "addGOTEntry: " << toString(*sym) << "\n");
   if (sym->hasGOTIndex())
     return;
+  LLVM_DEBUG(dbgs() << "addGOTEntry: " << toString(*sym) << "\n");
   sym->setGOTIndex(numImportedGlobals++);
   gotSymbols.push_back(sym);
 }
@@ -244,15 +244,21 @@ void GlobalSection::assignIndexes() {
   uint32_t globalIndex = out.importSec->getNumImportedGlobals();
   for (InputGlobal *g : inputGlobals)
     g->setGlobalIndex(globalIndex++);
-  for (Symbol *sym : gotSymbols)
+  for (Symbol *sym : staticGotSymbols)
     sym->setGOTIndex(globalIndex++);
+  isSealed = true;
 }
 
-void GlobalSection::addDummyGOTEntry(Symbol *sym) {
-  LLVM_DEBUG(dbgs() << "addDummyGOTEntry: " << toString(*sym) << "\n");
-  if (sym->hasGOTIndex())
+void GlobalSection::addStaticGOTEntry(Symbol *sym) {
+  assert(!isSealed);
+  if (sym->requiresGOT)
     return;
-  gotSymbols.push_back(sym);
+  LLVM_DEBUG(dbgs() << "addStaticGOTEntry: " << sym->getName() << " "
+                    << toString(sym->kind()) << "\n");
+  sym->requiresGOT = true;
+  if (auto *F = dyn_cast<FunctionSymbol>(sym))
+    out.elemSec->addEntry(F);
+  staticGotSymbols.push_back(sym);
 }
 
 void GlobalSection::writeBody() {
@@ -261,26 +267,27 @@ void GlobalSection::writeBody() {
   writeUleb128(os, numGlobals(), "global count");
   for (InputGlobal *g : inputGlobals)
     writeGlobal(os, g->global);
-  for (const DefinedData *sym : definedFakeGlobals) {
+  for (const Symbol *sym : staticGotSymbols) {
     WasmGlobal global;
     global.Type = {WASM_TYPE_I32, false};
     global.InitExpr.Opcode = WASM_OPCODE_I32_CONST;
-    global.InitExpr.Value.Int32 = sym->getVirtualAddress();
+    if (auto *d = dyn_cast<DefinedData>(sym))
+      global.InitExpr.Value.Int32 = d->getVirtualAddress();
+    else if (auto *f = cast<FunctionSymbol>(sym))
+      global.InitExpr.Value.Int32 = f->getTableIndex();
     writeGlobal(os, global);
   }
-  for (const Symbol *sym : gotSymbols) {
+  for (const DefinedData *sym : dataAddressGlobals) {
     WasmGlobal global;
     global.Type = {WASM_TYPE_I32, false};
     global.InitExpr.Opcode = WASM_OPCODE_I32_CONST;
-    if (auto *d = dyn_cast<DefinedData>(sym))
-      global.InitExpr.Value.Int32 = d->getVirtualAddress();
-    else if (auto *f = cast<DefinedFunction>(sym))
-      global.InitExpr.Value.Int32 = f->getTableIndex();
+    global.InitExpr.Value.Int32 = sym->getVirtualAddress();
     writeGlobal(os, global);
   }
 }
 
 void GlobalSection::addGlobal(InputGlobal *global) {
+  assert(!isSealed);
   if (!global->live)
     return;
   inputGlobals.push_back(global);
@@ -362,26 +369,6 @@ bool DataCountSection::isNeeded() const {
   return numSegments && config->sharedMemory;
 }
 
-static uint32_t getWasmFlags(const Symbol *sym) {
-  uint32_t flags = 0;
-  if (sym->isLocal())
-    flags |= WASM_SYMBOL_BINDING_LOCAL;
-  if (sym->isWeak())
-    flags |= WASM_SYMBOL_BINDING_WEAK;
-  if (sym->isHidden())
-    flags |= WASM_SYMBOL_VISIBILITY_HIDDEN;
-  if (sym->isUndefined())
-    flags |= WASM_SYMBOL_UNDEFINED;
-  if (auto *f = dyn_cast<UndefinedFunction>(sym)) {
-    if (f->getName() != f->importName)
-      flags |= WASM_SYMBOL_EXPLICIT_NAME;
-  } else if (auto *g = dyn_cast<UndefinedGlobal>(sym)) {
-    if (g->getName() != g->importName)
-      flags |= WASM_SYMBOL_EXPLICIT_NAME;
-  }
-  return flags;
-}
-
 void LinkingSection::writeBody() {
   raw_ostream &os = bodyOutputStream;
 
@@ -394,7 +381,7 @@ void LinkingSection::writeBody() {
     for (const Symbol *sym : symtabEntries) {
       assert(sym->isDefined() || sym->isUndefined());
       WasmSymbolType kind = sym->getWasmType();
-      uint32_t flags = getWasmFlags(sym);
+      uint32_t flags = sym->getFlags();
 
       writeU8(sub.os, kind, "sym kind");
       writeUleb128(sub.os, flags, "sym flags");
diff --git a/lld/wasm/SyntheticSections.h b/lld/wasm/SyntheticSections.h
index 88dc1fed87f0c..1557ce79d2d3d 100644
--- a/lld/wasm/SyntheticSections.h
+++ b/lld/wasm/SyntheticSections.h
@@ -175,17 +175,23 @@ class GlobalSection : public SyntheticSection {
 public:
   GlobalSection() : SyntheticSection(llvm::wasm::WASM_SEC_GLOBAL) {}
   uint32_t numGlobals() const {
-    return inputGlobals.size() + definedFakeGlobals.size() + gotSymbols.size();
+    assert(isSealed);
+    return inputGlobals.size() + dataAddressGlobals.size() +
+           staticGotSymbols.size();
   }
   bool isNeeded() const override { return numGlobals() > 0; }
   void assignIndexes() override;
   void writeBody() override;
   void addGlobal(InputGlobal *global);
-  void addDummyGOTEntry(Symbol *sym);
+  void addDataAddressGlobal(DefinedData *global);
+  void addStaticGOTEntry(Symbol *sym);
+
+  std::vector<const DefinedData *> dataAddressGlobals;
 
-  std::vector<const DefinedData *> definedFakeGlobals;
+protected:
+  bool isSealed = false;
   std::vector<InputGlobal *> inputGlobals;
-  std::vector<Symbol *> gotSymbols;
+  std::vector<Symbol *> staticGotSymbols;
 };
 
 // The event section contains a list of declared wasm events associated with the
diff --git a/lld/wasm/Writer.cpp b/lld/wasm/Writer.cpp
index 58d585bf0cb0f..d841d21b79825 100644
--- a/lld/wasm/Writer.cpp
+++ b/lld/wasm/Writer.cpp
@@ -263,7 +263,6 @@ void Writer::layoutMemory() {
     memoryPtr += 4;
   }
 
-  // TODO: Add .bss space here.
   if (WasmSym::dataEnd)
     WasmSym::dataEnd->setVirtualAddress(memoryPtr);
 
@@ -510,8 +509,8 @@ void Writer::calculateExports() {
     out.exportSec->exports.push_back(
         WasmExport{functionTableName, WASM_EXTERNAL_TABLE, 0});
 
-  unsigned fakeGlobalIndex = out.importSec->getNumImportedGlobals() +
-                             out.globalSec->inputGlobals.size();
+  unsigned globalIndex =
+      out.importSec->getNumImportedGlobals() + out.globalSec->numGlobals();
 
   for (Symbol *sym : symtab->getSymbols()) {
     if (!sym->isExported())
@@ -537,8 +536,8 @@ void Writer::calculateExports() {
       export_ = {name, WASM_EXTERNAL_EVENT, e->getEventIndex()};
     } else {
       auto *d = cast<DefinedData>(sym);
-      out.globalSec->definedFakeGlobals.emplace_back(d);
-      export_ = {name, WASM_EXTERNAL_GLOBAL, fakeGlobalIndex++};
+      out.globalSec->dataAddressGlobals.push_back(d);
+      export_ = {name, WASM_EXTERNAL_GLOBAL, globalIndex++};
     }
 
     LLVM_DEBUG(dbgs() << "Export: " << name << "\n");
@@ -667,7 +666,7 @@ void Writer::createOutputSegments() {
       OutputSegment *&s = segmentMap[name];
       if (s == nullptr) {
         LLVM_DEBUG(dbgs() << "new segment: " << name << "\n");
-        s = make<OutputSegment>(name, segments.size());
+        s = make<OutputSegment>(name);
         if (config->sharedMemory || name == ".tdata")
           s->initFlags = WASM_SEGMENT_IS_PASSIVE;
         segments.push_back(s);
@@ -676,6 +675,23 @@ void Writer::createOutputSegments() {
       LLVM_DEBUG(dbgs() << "added data: " << name << ": " << s->size << "\n");
     }
   }
+
+  // Sort segments by type, placing .bss last
+  std::stable_sort(segments.begin(), segments.end(),
+                   [](const OutputSegment *a, const OutputSegment *b) {
+                     auto order = [](StringRef name) {
+                       return StringSwitch<int>(name)
+                           .StartsWith(".rodata", 0)
+                           .StartsWith(".data", 1)
+                           .StartsWith(".tdata", 2)
+                           .StartsWith(".bss", 4)
+                           .Default(3);
+                     };
+                     return order(a->name) < order(b->name);
+                   });
+
+  for (size_t i = 0; i < segments.size(); ++i)
+    segments[i]->index = i;
 }
 
 static void createFunction(DefinedFunction *func, StringRef bodyContent) {
@@ -1019,7 +1035,7 @@ void Writer::run() {
 
   if (errorHandler().verbose) {
     log("Defined Functions: " + Twine(out.functionSec->inputFunctions.size()));
-    log("Defined Globals  : " + Twine(out.globalSec->inputGlobals.size()));
+    log("Defined Globals  : " + Twine(out.globalSec->numGlobals()));
     log("Defined Events   : " + Twine(out.eventSec->inputEvents.size()));
     log("Function Imports : " +
         Twine(out.importSec->getNumImportedFunctions()));
diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index 792fd5207415d..e6e78a831825d 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -250,10 +250,10 @@ function(find_python_libs_windows)
   endif()
 
   # Set the same variables as FindPythonInterp and FindPythonLibs.
-  set(PYTHON_EXECUTABLE         "${PYTHON_EXECUTABLE}"          PARENT_SCOPE)
-  set(PYTHON_LIBRARY            "${PYTHON_LIBRARY}"             PARENT_SCOPE)
-  set(PYTHON_DLL                "${PYTHON_DLL}"                 PARENT_SCOPE)
-  set(PYTHON_INCLUDE_DIR        "${PYTHON_INCLUDE_DIR}"         PARENT_SCOPE)
+  set(PYTHON_EXECUTABLE         "${PYTHON_EXECUTABLE}"          CACHE PATH "")
+  set(PYTHON_LIBRARY            "${PYTHON_LIBRARY}"             CACHE PATH "")
+  set(PYTHON_DLL                "${PYTHON_DLL}"                 CACHE PATH "")
+  set(PYTHON_INCLUDE_DIR        "${PYTHON_INCLUDE_DIR}"         CACHE PATH "")
   set(PYTHONLIBS_VERSION_STRING "${PYTHONLIBS_VERSION_STRING}"  PARENT_SCOPE)
   set(PYTHON_VERSION_STRING     "${PYTHON_VERSION_STRING}"      PARENT_SCOPE)
   set(PYTHON_VERSION_MAJOR      "${PYTHON_VERSION_MAJOR}"       PARENT_SCOPE)
diff --git a/lldb/cmake/modules/LLDBFramework.cmake b/lldb/cmake/modules/LLDBFramework.cmake
index 7a00ec49fdede..638515659f356 100644
--- a/lldb/cmake/modules/LLDBFramework.cmake
+++ b/lldb/cmake/modules/LLDBFramework.cmake
@@ -44,6 +44,9 @@ else()
     XCODE_ATTRIBUTE_MACOSX_DEPLOYMENT_TARGET "${MACOSX_DEPLOYMENT_TARGET}")
 endif()
 
+# Add -Wdocumentation parameter
+set(CMAKE_XCODE_ATTRIBUTE_CLANG_WARN_DOCUMENTATION_COMMENTS "YES")
+
 # Apart from this one, CMake creates all required symlinks in the framework bundle.
 add_custom_command(TARGET liblldb POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E create_symlink
diff --git a/lldb/cmake/modules/LLDBStandalone.cmake b/lldb/cmake/modules/LLDBStandalone.cmake
index 4b67d164d1c24..752113bcc6c41 100644
--- a/lldb/cmake/modules/LLDBStandalone.cmake
+++ b/lldb/cmake/modules/LLDBStandalone.cmake
@@ -1,7 +1,7 @@
 option(LLVM_INSTALL_TOOLCHAIN_ONLY "Only include toolchain files in the 'install' target." OFF)
 
-find_package(LLVM REQUIRED CONFIG HINTS "${LLVM_DIR}" NO_CMAKE_FIND_ROOT_PATH)
-find_package(Clang REQUIRED CONFIG HINTS "${Clang_DIR}" NO_CMAKE_FIND_ROOT_PATH)
+find_package(LLVM REQUIRED CONFIG HINTS ${LLVM_DIR} NO_CMAKE_FIND_ROOT_PATH)
+find_package(Clang REQUIRED CONFIG HINTS ${Clang_DIR} ${LLVM_DIR}/../clang NO_CMAKE_FIND_ROOT_PATH)
 
 # We set LLVM_CMAKE_PATH so that GetSVN.cmake is found correctly when building SVNVersion.inc
 set(LLVM_CMAKE_PATH ${LLVM_CMAKE_DIR} CACHE PATH "Path to LLVM CMake modules")
diff --git a/lldb/docs/resources/bots.rst b/lldb/docs/resources/bots.rst
index d40c8fe47e2a4..e054575432da8 100644
--- a/lldb/docs/resources/bots.rst
+++ b/lldb/docs/resources/bots.rst
@@ -10,8 +10,8 @@ build slaves. Everyone can `add a buildbot for LLDB
 
 
 * `lldb-x64-windows-ninja <http://lab.llvm.org:8011/builders/lldb-x64-windows-ninja>`_
-* `lldb-x86_64-debian <http://lab.llvm.org:8011/builders/lldb-x86_64-debian>`_
-* `lldb-x86_64-fedora <http://lab.llvm.org:8011/builders/lldb-x86_64-fedora>`_
+* `lldb-x86_64-debian <http://lab.llvm.org:8014/builders/lldb-x86_64-debian>`_
+* `lldb-x86_64-fedora <http://lab.llvm.org:8014/builders/lldb-x86_64-fedora>`_
 
 Documentation
 -------------
diff --git a/lldb/docs/resources/build.rst b/lldb/docs/resources/build.rst
index 62981e0f23344..f0a5273cb87f8 100644
--- a/lldb/docs/resources/build.rst
+++ b/lldb/docs/resources/build.rst
@@ -120,10 +120,11 @@ Standalone builds
 *****************
 
 This is another way to build LLDB. We can use the same source-tree as we
-checked out above, but now we will have two build-trees:
+checked out above, but now we will have multiple build-trees:
 
 * the main build-tree for LLDB in ``/path/to/lldb-build``
-* a provided build-tree for LLVM and Clang in ``/path/to/llvm-build``
+* one or more provided build-trees for LLVM and Clang; for simplicity we use a
+  single one in ``/path/to/llvm-build``
 
 Run CMake with ``-B`` pointing to a new directory for the provided
 build-tree\ :sup:`1` and the positional argument pointing to the ``llvm``
@@ -139,15 +140,15 @@ Clang. Then we build the ``ALL`` target with ninja:
 
 Now run CMake a second time with ``-B`` pointing to a new directory for the
 main build-tree and the positional argument pointing to the ``lldb`` directory
-in the source-tree. In order to find the provided build-tree, the build-system
-needs the options ``LLVM_DIR`` and ``Clang_DIR`` (CMake variables are
-case-sensitive!):
+in the source-tree. In order to find the provided build-tree, the build system
+looks for the path to its CMake modules in ``LLVM_DIR``. If you use a separate
+build directory for Clang, remember to pass its module path via ``Clang_DIR``
+(CMake variables are case-sensitive!):
 
 ::
 
   > cmake -B /path/to/lldb-build -G Ninja \
           -DLLVM_DIR=/path/to/llvm-build/lib/cmake/llvm \
-          -DClang_DIR=/path/to/llvm-build/lib/cmake/clang \
           [<more cmake options>] /path/to/llvm-project/lldb
   > ninja lldb
 
@@ -320,7 +321,6 @@ Build LLDB standalone for development with Xcode:
   > cmake -B /path/to/lldb-build \
           -C /path/to/llvm-project/lldb/cmake/caches/Apple-lldb-Xcode.cmake \
           -DLLVM_DIR=/path/to/llvm-build/lib/cmake/llvm \
-          -DClang_DIR=/path/to/llvm-build/lib/cmake/clang \
           llvm-project/lldb
   > open lldb.xcodeproj
   > cmake --build /path/to/lldb-build --target check-lldb
diff --git a/lldb/docs/resources/sbapi.rst b/lldb/docs/resources/sbapi.rst
index 20c332990c1f9..048f6c12d9720 100644
--- a/lldb/docs/resources/sbapi.rst
+++ b/lldb/docs/resources/sbapi.rst
@@ -53,3 +53,43 @@ file, and adding documentation and the Python decorations, etc. We do this in a
 decidedly low-tech way, by maintaining the two files in parallel. That
 simplifies the build process, but it does mean that if you add a method to the
 C++ API's for an SB class, you have to copy the interface to the .i file.
+
+API Instrumentation
+-------------------
+
+The reproducer infrastructure requires API methods to be instrumented so that
+they can be captured and replayed. Instrumentation consists of two macros,
+``LLDB_REGISTER`` and ``LLDB_RECORD``. Both can be automatically generated with
+the ``lldb-instr`` utility.
+
+To add instrumentation for a given file, pass it to the ``lldb-instr`` tool.
+Like other clang-based tools it requires a compilation database
+(``compile_commands.json``) to be present in the current working directory.
+
+::
+
+    ./bin/lldb-instr /path/to/lldb/source/API/SBDebugger.cpp
+
+
+The tool will automatically insert ``LLDB_RECORD`` macros inline, however you
+will need to run ``clang-format`` over the processed file, as the tool
+(intentionally) makes no attempt to get that right.
+
+The ``LLDB_REGISTER`` macros are printed to standard out between curly braces.
+You'll have to copy-paste those into the corresponding `RegsiterMethods`
+function in the implementation file. This function is fully specialized in the
+corresponding type.
+
+::
+
+  template <> void RegisterMethods<SBDebugger>(Registry &R) {
+    ...
+  }
+
+
+When adding a new class, you'll also have to add a call to ``RegisterMethods``
+in the ``SBRegistry`` constructor.
+
+The tool can be used incrementally. However, it will ignore existing macros
+even if their signature is wrong. It will only generate a ``LLDB_REGISTER`` if
+it emitted a corresponding ``LLDB_RECORD`` macro.
diff --git a/lldb/docs/resources/test.rst b/lldb/docs/resources/test.rst
index d77941a8b2b0c..2305aefaa8445 100644
--- a/lldb/docs/resources/test.rst
+++ b/lldb/docs/resources/test.rst
@@ -1,11 +1,20 @@
 Testing
 =======
 
-The LLDB test suite consists of Python scripts located under the test
-directory. Each script contains a number of test cases and is usually
-accompanied by a C (C++, ObjC, etc.) source file. Each test first compiles the
-source file and then uses LLDB to debug the resulting executable. The tests
-verify both the LLDB command line interface and the scripting API.
+The LLDB test suite consists of three different kinds of test:
+
+* Python scripts located under ``lldb/packages/Python/lldbsuite``.
+  These are written using python's unittest2 testing framework.
+
+* Unit tests, located under ``lldb/unittests``.   These are written in C++,
+  using googletest.
+
+* LIT tests, located under ``lldb/lit``.  These use the LLVM Integrated Tester.
+
+Many of the tests are accompanied by a C (C++, ObjC, etc.) source file. Each test
+first compiles the source file and then uses LLDB to debug the resulting executable.
+
+The tests verify both the LLDB command line interface and the scripting API.
 
 .. contents::
    :local:
@@ -46,8 +55,8 @@ Note that multiple ``-A`` and ``-C`` flags can be specified to
 ``LLDB_TEST_USER_ARGS``.
 
 
-Running a Specific Test or Set of Tests
----------------------------------------
+Running a Specific Test or Set of Tests: Python
+-----------------------------------------------
 
 In addition to running all the LLDB test suites with the ``check-lldb`` CMake
 target above, it is possible to run individual LLDB tests. If you have a CMake
@@ -86,6 +95,52 @@ Many more options that are available. To see a list of all of them, run:
 
    > python dotest.py -h
 
+
+Running a Specific Test or Set of Tests: Unit Tests
+---------------------------------------------------
+
+The unit tests are simple executables, located in the build directory under ``tools/lldb/unittests``.
+
+To run them, just run the test binary, for example, to run all the Host tests:
+
+::
+
+   > ./tools/lldb/unittests/Host/HostTests
+
+
+To run a specific test, pass a filter, for example:
+
+::
+
+   > ./tools/lldb/unittests/Host/HostTests --gtest_filter=SocketTest.DomainListenConnectAccept
+
+
+Running a Specific Test or Set of Tests: LIT
+--------------------------------------------
+
+LIT automatically scans a directory for tests.   To run a subset of the LIT tests, pass it a
+subdirectory, for example:
+
+::
+
+   > ./bin/llvm-lit -sv tools/lldb/lit/Commands/CommandScriptImmediateOutput
+
+
+LIT can also filter based on test name.
+
+::
+
+   > ./bin/llvm-lit -sv tools/lldb/lit --filter CommandScriptImmediateOutput
+
+
+It is also possible to forward arguments to dotest.py by passing ``--param`` to
+lit and setting a value for ``dotest-args``.
+
+::
+
+   > ./bin/llvm-lit -sv tools/lldb/lit --param dotest-args='-C gcc'
+
+
 Running the Test Suite Remotely
 -------------------------------
 
diff --git a/lldb/include/lldb/API/SBCommandReturnObject.h b/lldb/include/lldb/API/SBCommandReturnObject.h
index 2877afb9b8b05..95eed2e3f6201 100644
--- a/lldb/include/lldb/API/SBCommandReturnObject.h
+++ b/lldb/include/lldb/API/SBCommandReturnObject.h
@@ -98,8 +98,6 @@ class LLDB_API SBCommandReturnObject {
 
   lldb_private::CommandReturnObject &ref() const;
 
-  void SetLLDBObjectPtr(lldb_private::CommandReturnObject *ptr);
-
 private:
   std::unique_ptr<lldb_private::CommandReturnObject> m_opaque_up;
 };
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 2b5bf5136510d..90eefe115244a 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -17,6 +17,7 @@
 #include "lldb/Core/FormatEntity.h"
 #include "lldb/Core/IOHandler.h"
 #include "lldb/Core/SourceManager.h"
+#include "lldb/Core/StreamFile.h"
 #include "lldb/Core/UserSettingsController.h"
 #include "lldb/Host/HostThread.h"
 #include "lldb/Host/Terminal.h"
@@ -113,11 +114,21 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   void SetAsyncExecution(bool async);
 
-  lldb::StreamFileSP GetInputFile() { return m_input_file_sp; }
+  lldb::FileSP GetInputFileSP() { return m_input_file_sp; }
 
-  lldb::StreamFileSP GetOutputFile() { return m_output_file_sp; }
+  lldb::StreamFileSP GetOutputStreamSP() { return m_output_stream_sp; }
 
-  lldb::StreamFileSP GetErrorFile() { return m_error_file_sp; }
+  lldb::StreamFileSP GetErrorStreamSP() { return m_error_stream_sp; }
+
+  File &GetInputFile() { return *m_input_file_sp; }
+
+  File &GetOutputFile() { return m_output_stream_sp->GetFile(); }
+
+  File &GetErrorFile() { return m_error_stream_sp->GetFile(); }
+
+  StreamFile &GetOutputStream() { return *m_output_stream_sp; }
+
+  StreamFile &GetErrorStream() { return *m_error_stream_sp; }
 
   repro::DataRecorder *GetInputRecorder();
 
@@ -174,7 +185,7 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   // If any of the streams are not set, set them to the in/out/err stream of
   // the top most input reader to ensure they at least have something
-  void AdoptTopIOHandlerFilesIfInvalid(lldb::StreamFileSP &in,
+  void AdoptTopIOHandlerFilesIfInvalid(lldb::FileSP &in,
                                        lldb::StreamFileSP &out,
                                        lldb::StreamFileSP &err);
 
@@ -356,9 +367,10 @@ class Debugger : public std::enable_shared_from_this<Debugger>,
 
   void InstanceInitialize();
 
-  lldb::StreamFileSP m_input_file_sp;
-  lldb::StreamFileSP m_output_file_sp;
-  lldb::StreamFileSP m_error_file_sp;
+  // these should never be NULL
+  lldb::FileSP m_input_file_sp;
+  lldb::StreamFileSP m_output_stream_sp;
+  lldb::StreamFileSP m_error_stream_sp;
 
   /// Used for shadowing the input file when capturing a reproducer.
   repro::DataRecorder *m_input_recorder;
diff --git a/lldb/include/lldb/Core/IOHandler.h b/lldb/include/lldb/Core/IOHandler.h
index 3a77615833d93..0c50a2d80a973 100644
--- a/lldb/include/lldb/Core/IOHandler.h
+++ b/lldb/include/lldb/Core/IOHandler.h
@@ -58,8 +58,7 @@ class IOHandler {
   IOHandler(Debugger &debugger, IOHandler::Type type);
 
   IOHandler(Debugger &debugger, IOHandler::Type type,
-            const lldb::StreamFileSP &input_sp,
-            const lldb::StreamFileSP &output_sp,
+            const lldb::FileSP &input_sp, const lldb::StreamFileSP &output_sp,
             const lldb::StreamFileSP &error_sp, uint32_t flags,
             repro::DataRecorder *data_recorder);
 
@@ -123,11 +122,11 @@ class IOHandler {
 
   FILE *GetErrorFILE();
 
-  lldb::StreamFileSP &GetInputStreamFile();
+  lldb::FileSP &GetInputFileSP();
 
-  lldb::StreamFileSP &GetOutputStreamFile();
+  lldb::StreamFileSP &GetOutputStreamFileSP();
 
-  lldb::StreamFileSP &GetErrorStreamFile();
+  lldb::StreamFileSP &GetErrorStreamFileSP();
 
   Debugger &GetDebugger() { return m_debugger; }
 
@@ -165,7 +164,7 @@ class IOHandler {
 
 protected:
   Debugger &m_debugger;
-  lldb::StreamFileSP m_input_sp;
+  lldb::FileSP m_input_sp;
   lldb::StreamFileSP m_output_sp;
   lldb::StreamFileSP m_error_sp;
   repro::DataRecorder *m_data_recorder;
@@ -333,7 +332,7 @@ class IOHandlerEditline : public IOHandler {
                     repro::DataRecorder *data_recorder);
 
   IOHandlerEditline(Debugger &debugger, IOHandler::Type type,
-                    const lldb::StreamFileSP &input_sp,
+                    const lldb::FileSP &input_sp,
                     const lldb::StreamFileSP &output_sp,
                     const lldb::StreamFileSP &error_sp, uint32_t flags,
                     const char *editline_name, // Used for saving history files
@@ -349,7 +348,7 @@ class IOHandlerEditline : public IOHandler {
                     const char *, bool, bool, uint32_t,
                     IOHandlerDelegate &) = delete;
 
-  IOHandlerEditline(Debugger &, IOHandler::Type, const lldb::StreamFileSP &,
+  IOHandlerEditline(Debugger &, IOHandler::Type, const lldb::FileSP &,
                     const lldb::StreamFileSP &, const lldb::StreamFileSP &,
                     uint32_t, const char *, const char *, const char *, bool,
                     bool, uint32_t, IOHandlerDelegate &) = delete;
diff --git a/lldb/include/lldb/Core/LoadedModuleInfoList.h b/lldb/include/lldb/Core/LoadedModuleInfoList.h
index f7f83604fc1a8..04e58fcdf3130 100644
--- a/lldb/include/lldb/Core/LoadedModuleInfoList.h
+++ b/lldb/include/lldb/Core/LoadedModuleInfoList.h
@@ -84,9 +84,6 @@ class LoadedModuleInfoList {
     }
 
     bool operator==(LoadedModuleInfo const &rhs) const {
-      if (e_num != rhs.e_num)
-        return false;
-
       for (size_t i = 0; i < e_num; ++i) {
         if (m_has[i] != rhs.m_has[i])
           return false;
diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h
index 146bbe8ca1b31..f0b8a552a29f1 100644
--- a/lldb/include/lldb/Core/Module.h
+++ b/lldb/include/lldb/Core/Module.h
@@ -447,9 +447,7 @@ class Module : public std::enable_shared_from_this<Module>,
   /// \param[out] type_list
   ///     A type list gets populated with any matches.
   ///
-  /// \return
-  ///     The number of matches added to \a type_list.
-  size_t
+  void
   FindTypes(ConstString type_name, bool exact_match, size_t max_matches,
             llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
             TypeList &types);
@@ -459,8 +457,8 @@ class Module : public std::enable_shared_from_this<Module>,
   /// This behaves like the other FindTypes method but allows to
   /// specify a DeclContext and a language for the type being searched
   /// for.
-  size_t FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                   LanguageSet languages, bool append, TypeMap &types);
+  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
+                 TypeMap &types);
 
   lldb::TypeSP FindFirstType(const SymbolContext &sc,
                              ConstString type_name, bool exact_match);
@@ -479,11 +477,9 @@ class Module : public std::enable_shared_from_this<Module>,
   /// \param[out] type_list
   ///     A type list gets populated with any matches.
   ///
-  /// \return
-  ///     The number of matches added to \a type_list.
-  size_t FindTypesInNamespace(ConstString type_name,
-                              const CompilerDeclContext *parent_decl_ctx,
-                              size_t max_matches, TypeList &type_list);
+  void FindTypesInNamespace(ConstString type_name,
+                            const CompilerDeclContext *parent_decl_ctx,
+                            size_t max_matches, TypeList &type_list);
 
   /// Get const accessor for the module architecture.
   ///
@@ -1074,9 +1070,9 @@ class Module : public std::enable_shared_from_this<Module>,
 private:
   Module(); // Only used internally by CreateJITModule ()
 
-  size_t FindTypes_Impl(
+  void FindTypes_Impl(
       ConstString name, const CompilerDeclContext *parent_decl_ctx,
-      bool append, size_t max_matches,
+      size_t max_matches,
       llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
       TypeMap &types);
 
diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h
index 1e26c35e4c157..898f4b14219fa 100644
--- a/lldb/include/lldb/Core/ModuleList.h
+++ b/lldb/include/lldb/Core/ModuleList.h
@@ -393,12 +393,10 @@ class ModuleList {
   /// \param[out] type_list
   ///     A type list gets populated with any matches.
   ///
-  /// \return
-  ///     The number of matches added to \a type_list.
-  size_t FindTypes(Module *search_first, ConstString name,
-                   bool name_is_fully_qualified, size_t max_matches,
-                   llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                   TypeList &types) const;
+  void FindTypes(Module *search_first, ConstString name,
+                 bool name_is_fully_qualified, size_t max_matches,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeList &types) const;
 
   bool FindSourceFile(const FileSpec &orig_spec, FileSpec &new_spec) const;
 
diff --git a/lldb/include/lldb/Core/StreamFile.h b/lldb/include/lldb/Core/StreamFile.h
index 54aeab16b3f1a..77e1557cb91df 100644
--- a/lldb/include/lldb/Core/StreamFile.h
+++ b/lldb/include/lldb/Core/StreamFile.h
@@ -35,18 +35,22 @@ class StreamFile : public Stream {
 
   StreamFile(FILE *fh, bool transfer_ownership);
 
+  StreamFile(std::shared_ptr<File> file) : m_file_sp(file) { assert(file); };
+
   ~StreamFile() override;
 
-  File &GetFile() { return m_file; }
+  File &GetFile() { return *m_file_sp; }
+
+  const File &GetFile() const { return *m_file_sp; }
 
-  const File &GetFile() const { return m_file; }
+  std::shared_ptr<File> GetFileSP() { return m_file_sp; }
 
   void Flush() override;
 
 
 protected:
   // Classes that inherit from StreamFile can see and modify these
-  File m_file;
+  std::shared_ptr<File> m_file_sp; // never NULL
   size_t WriteImpl(const void *s, size_t length) override;
 
 private:
diff --git a/lldb/include/lldb/Core/StructuredDataImpl.h b/lldb/include/lldb/Core/StructuredDataImpl.h
index 79a709af7ca31..c66e4736dc26c 100644
--- a/lldb/include/lldb/Core/StructuredDataImpl.h
+++ b/lldb/include/lldb/Core/StructuredDataImpl.h
@@ -54,7 +54,8 @@ class StructuredDataImpl {
       return error;
     }
 
-    m_data_sp->Dump(stream);
+    llvm::json::OStream s(stream.AsRawOstream());
+    m_data_sp->Serialize(s);
     return error;
   }
 
diff --git a/lldb/include/lldb/Core/dwarf.h b/lldb/include/lldb/Core/dwarf.h
index afe62e09fe168..832109e55c70b 100644
--- a/lldb/include/lldb/Core/dwarf.h
+++ b/lldb/include/lldb/Core/dwarf.h
@@ -22,7 +22,7 @@ typedef uint32_t dw_uleb128_t;
 typedef int32_t dw_sleb128_t;
 typedef uint16_t dw_attr_t;
 typedef uint16_t dw_form_t;
-typedef uint16_t dw_tag_t;
+typedef llvm::dwarf::Tag dw_tag_t;
 typedef uint64_t dw_addr_t; // Dwarf address define that must be big enough for
                             // any addresses in the compile units that get
                             // parsed
diff --git a/lldb/include/lldb/DataFormatters/FormattersContainer.h b/lldb/include/lldb/DataFormatters/FormattersContainer.h
index 0c9ddec684024..de2edb103151a 100644
--- a/lldb/include/lldb/DataFormatters/FormattersContainer.h
+++ b/lldb/include/lldb/DataFormatters/FormattersContainer.h
@@ -65,7 +65,7 @@ template <typename KeyType, typename ValueType> class FormattersContainer;
 template <typename KeyType, typename ValueType> class FormatMap {
 public:
   typedef typename ValueType::SharedPointer ValueSP;
-  typedef std::map<KeyType, ValueSP> MapType;
+  typedef std::vector<std::pair<KeyType, ValueSP>> MapType;
   typedef typename MapType::iterator MapIterator;
   typedef std::function<bool(const KeyType &, const ValueSP &)> ForEachCallback;
 
@@ -79,20 +79,22 @@ template <typename KeyType, typename ValueType> class FormatMap {
       entry->GetRevision() = 0;
 
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    m_map[std::move(name)] = entry;
+    Delete(name);
+    m_map.emplace_back(std::move(name), std::move(entry));
     if (listener)
       listener->Changed();
   }
 
   bool Delete(const KeyType &name) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    MapIterator iter = m_map.find(name);
-    if (iter == m_map.end())
-      return false;
-    m_map.erase(iter);
-    if (listener)
-      listener->Changed();
-    return true;
+    for (MapIterator iter = m_map.begin(); iter != m_map.end(); ++iter)
+      if (iter->first == name) {
+        m_map.erase(iter);
+        if (listener)
+          listener->Changed();
+        return true;
+      }
+    return false;
   }
 
   void Clear() {
@@ -104,20 +106,20 @@ template <typename KeyType, typename ValueType> class FormatMap {
 
   bool Get(const KeyType &name, ValueSP &entry) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    MapIterator iter = m_map.find(name);
-    if (iter == m_map.end())
-      return false;
-    entry = iter->second;
-    return true;
+    for (const auto &pos : m_map)
+      if (pos.first == name) {
+        entry = pos.second;
+        return true;
+      }
+    return false;
   }
 
   void ForEach(ForEachCallback callback) {
     if (callback) {
       std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-      MapIterator pos, end = m_map.end();
-      for (pos = m_map.begin(); pos != end; pos++) {
-        const KeyType &type = pos->first;
-        if (!callback(type, pos->second))
+      for (const auto &pos : m_map) {
+        const KeyType &type = pos.first;
+        if (!callback(type, pos.second))
           break;
       }
     }
@@ -127,29 +129,17 @@ template <typename KeyType, typename ValueType> class FormatMap {
 
   ValueSP GetValueAtIndex(size_t index) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    MapIterator iter = m_map.begin();
-    MapIterator end = m_map.end();
-    while (index > 0) {
-      iter++;
-      index--;
-      if (end == iter)
-        return ValueSP();
-    }
-    return iter->second;
+    if (index >= m_map.size())
+      return ValueSP();
+    return m_map[index].second;
   }
 
   // If caller holds the mutex we could return a reference without copy ctor.
   KeyType GetKeyAtIndex(size_t index) {
     std::lock_guard<std::recursive_mutex> guard(m_map_mutex);
-    MapIterator iter = m_map.begin();
-    MapIterator end = m_map.end();
-    while (index > 0) {
-      iter++;
-      index--;
-      if (end == iter)
-        return KeyType();
-    }
-    return iter->first;
+    if (index >= m_map.size())
+      return {};
+    return m_map[index].first;
   }
 
 protected:
@@ -172,8 +162,8 @@ template <typename KeyType, typename ValueType> class FormattersContainer {
 public:
   typedef typename BackEndType::MapType MapType;
   typedef typename MapType::iterator MapIterator;
-  typedef typename MapType::key_type MapKeyType;
-  typedef typename MapType::mapped_type MapValueType;
+  typedef KeyType MapKeyType;
+  typedef std::shared_ptr<ValueType> MapValueType;
   typedef typename BackEndType::ForEachCallback ForEachCallback;
   typedef typename std::shared_ptr<FormattersContainer<KeyType, ValueType>>
       SharedPointer;
@@ -295,11 +285,11 @@ template <typename KeyType, typename ValueType> class FormattersContainer {
                 RegularExpression *dummy) {
     llvm::StringRef key_str = key.GetStringRef();
     std::lock_guard<std::recursive_mutex> guard(m_format_map.mutex());
-    MapIterator pos, end = m_format_map.map().end();
-    for (pos = m_format_map.map().begin(); pos != end; pos++) {
-      const RegularExpression &regex = pos->first;
+    // Patterns are matched in reverse-chronological order.
+    for (const auto &pos : llvm::reverse(m_format_map.map())) {
+      const RegularExpression &regex = pos.first;
       if (regex.Execute(key_str)) {
-        value = pos->second;
+        value = pos.second;
         return true;
       }
     }
@@ -309,11 +299,10 @@ template <typename KeyType, typename ValueType> class FormattersContainer {
   bool GetExact_Impl(ConstString key, MapValueType &value,
                      RegularExpression *dummy) {
     std::lock_guard<std::recursive_mutex> guard(m_format_map.mutex());
-    MapIterator pos, end = m_format_map.map().end();
-    for (pos = m_format_map.map().begin(); pos != end; pos++) {
-      const RegularExpression &regex = pos->first;
+    for (const auto &pos : m_format_map.map()) {
+      const RegularExpression &regex = pos.first;
       if (regex.GetText() == key.GetStringRef()) {
-        value = pos->second;
+        value = pos.second;
         return true;
       }
     }
diff --git a/lldb/include/lldb/DataFormatters/StringPrinter.h b/lldb/include/lldb/DataFormatters/StringPrinter.h
index 41d5edd880216..43b92019e6fd0 100644
--- a/lldb/include/lldb/DataFormatters/StringPrinter.h
+++ b/lldb/include/lldb/DataFormatters/StringPrinter.h
@@ -24,238 +24,121 @@ class StringPrinter {
 
   enum class GetPrintableElementType { ASCII, UTF8 };
 
-  class ReadStringAndDumpToStreamOptions {
+  class DumpToStreamOptions {
   public:
-    ReadStringAndDumpToStreamOptions()
-        : m_location(0), m_process_sp(), m_stream(nullptr), m_prefix_token(),
-          m_suffix_token(), m_quote('"'), m_source_size(0),
-          m_needs_zero_termination(true), m_escape_non_printables(true),
-          m_ignore_max_length(false), m_zero_is_terminator(true),
-          m_language_type(lldb::eLanguageTypeUnknown) {}
+    DumpToStreamOptions() = default;
 
-    ReadStringAndDumpToStreamOptions(ValueObject &valobj);
-
-    ReadStringAndDumpToStreamOptions &SetLocation(uint64_t l) {
-      m_location = l;
-      return *this;
-    }
-
-    uint64_t GetLocation() const { return m_location; }
-
-    ReadStringAndDumpToStreamOptions &SetProcessSP(lldb::ProcessSP p) {
-      m_process_sp = p;
-      return *this;
-    }
-
-    lldb::ProcessSP GetProcessSP() const { return m_process_sp; }
-
-    ReadStringAndDumpToStreamOptions &SetStream(Stream *s) {
-      m_stream = s;
-      return *this;
-    }
+    void SetStream(Stream *s) { m_stream = s; }
 
     Stream *GetStream() const { return m_stream; }
 
-    ReadStringAndDumpToStreamOptions &SetPrefixToken(const std::string &p) {
-      m_prefix_token = p;
-      return *this;
-    }
+    void SetPrefixToken(const std::string &p) { m_prefix_token = p; }
 
-    ReadStringAndDumpToStreamOptions &SetPrefixToken(std::nullptr_t) {
-      m_prefix_token.clear();
-      return *this;
-    }
+    void SetPrefixToken(std::nullptr_t) { m_prefix_token.clear(); }
 
     const char *GetPrefixToken() const { return m_prefix_token.c_str(); }
 
-    ReadStringAndDumpToStreamOptions &SetSuffixToken(const std::string &p) {
-      m_suffix_token = p;
-      return *this;
-    }
+    void SetSuffixToken(const std::string &p) { m_suffix_token = p; }
 
-    ReadStringAndDumpToStreamOptions &SetSuffixToken(std::nullptr_t) {
-      m_suffix_token.clear();
-      return *this;
-    }
+    void SetSuffixToken(std::nullptr_t) { m_suffix_token.clear(); }
 
     const char *GetSuffixToken() const { return m_suffix_token.c_str(); }
 
-    ReadStringAndDumpToStreamOptions &SetQuote(char q) {
-      m_quote = q;
-      return *this;
-    }
+    void SetQuote(char q) { m_quote = q; }
 
     char GetQuote() const { return m_quote; }
 
-    ReadStringAndDumpToStreamOptions &SetSourceSize(uint32_t s) {
-      m_source_size = s;
-      return *this;
-    }
+    void SetSourceSize(uint32_t s) { m_source_size = s; }
 
     uint32_t GetSourceSize() const { return m_source_size; }
 
-    ReadStringAndDumpToStreamOptions &SetNeedsZeroTermination(bool z) {
-      m_needs_zero_termination = z;
-      return *this;
-    }
+    void SetNeedsZeroTermination(bool z) { m_needs_zero_termination = z; }
 
     bool GetNeedsZeroTermination() const { return m_needs_zero_termination; }
 
-    ReadStringAndDumpToStreamOptions &SetBinaryZeroIsTerminator(bool e) {
-      m_zero_is_terminator = e;
-      return *this;
-    }
+    void SetBinaryZeroIsTerminator(bool e) { m_zero_is_terminator = e; }
 
     bool GetBinaryZeroIsTerminator() const { return m_zero_is_terminator; }
 
-    ReadStringAndDumpToStreamOptions &SetEscapeNonPrintables(bool e) {
-      m_escape_non_printables = e;
-      return *this;
-    }
+    void SetEscapeNonPrintables(bool e) { m_escape_non_printables = e; }
 
     bool GetEscapeNonPrintables() const { return m_escape_non_printables; }
 
-    ReadStringAndDumpToStreamOptions &SetIgnoreMaxLength(bool e) {
-      m_ignore_max_length = e;
-      return *this;
-    }
+    void SetIgnoreMaxLength(bool e) { m_ignore_max_length = e; }
 
     bool GetIgnoreMaxLength() const { return m_ignore_max_length; }
 
-    ReadStringAndDumpToStreamOptions &SetLanguage(lldb::LanguageType l) {
-      m_language_type = l;
-      return *this;
-    }
-
-    lldb::LanguageType GetLanguage() const
+    void SetLanguage(lldb::LanguageType l) { m_language_type = l; }
 
-    {
-      return m_language_type;
-    }
+    lldb::LanguageType GetLanguage() const { return m_language_type; }
 
   private:
-    uint64_t m_location;
-    lldb::ProcessSP m_process_sp;
-    Stream *m_stream;
+    /// The used output stream.
+    Stream *m_stream = nullptr;
+    /// String that should be printed before the heading quote character.
     std::string m_prefix_token;
+    /// String that should be printed after the trailing quote character.
     std::string m_suffix_token;
-    char m_quote;
-    uint32_t m_source_size;
-    bool m_needs_zero_termination;
-    bool m_escape_non_printables;
-    bool m_ignore_max_length;
-    bool m_zero_is_terminator;
-    lldb::LanguageType m_language_type;
+    /// The quote character that should surround the string.
+    char m_quote = '"';
+    /// The length of the memory region that should be dumped in bytes.
+    uint32_t m_source_size = 0;
+    bool m_needs_zero_termination = true;
+    /// True iff non-printable characters should be escaped when dumping
+    /// them to the stream.
+    bool m_escape_non_printables = true;
+    /// True iff the max-string-summary-length setting of the target should
+    /// be ignored.
+    bool m_ignore_max_length = false;
+    /// True iff a zero bytes ('\0') should terminate the memory region that
+    /// is being dumped.
+    bool m_zero_is_terminator = true;
+    /// The language that the generated string literal is supposed to be valid
+    /// for. This changes for example what and how certain characters are
+    /// escaped.
+    /// For example, printing the a string containing only a quote (") char
+    /// with eLanguageTypeC would escape the quote character.
+    lldb::LanguageType m_language_type = lldb::eLanguageTypeUnknown;
   };
 
-  class ReadBufferAndDumpToStreamOptions {
+  class ReadStringAndDumpToStreamOptions : public DumpToStreamOptions {
   public:
-    ReadBufferAndDumpToStreamOptions()
-        : m_data(), m_stream(nullptr), m_prefix_token(), m_suffix_token(),
-          m_quote('"'), m_source_size(0), m_escape_non_printables(true),
-          m_zero_is_terminator(true), m_is_truncated(false),
-          m_language_type(lldb::eLanguageTypeUnknown) {}
+    ReadStringAndDumpToStreamOptions() = default;
 
-    ReadBufferAndDumpToStreamOptions(ValueObject &valobj);
-
-    ReadBufferAndDumpToStreamOptions(
-        const ReadStringAndDumpToStreamOptions &options);
-
-    ReadBufferAndDumpToStreamOptions &SetData(DataExtractor d) {
-      m_data = d;
-      return *this;
-    }
-
-    lldb_private::DataExtractor GetData() const { return m_data; }
-
-    ReadBufferAndDumpToStreamOptions &SetStream(Stream *s) {
-      m_stream = s;
-      return *this;
-    }
-
-    Stream *GetStream() const { return m_stream; }
-
-    ReadBufferAndDumpToStreamOptions &SetPrefixToken(const std::string &p) {
-      m_prefix_token = p;
-      return *this;
-    }
-
-    ReadBufferAndDumpToStreamOptions &SetPrefixToken(std::nullptr_t) {
-      m_prefix_token.clear();
-      return *this;
-    }
-
-    const char *GetPrefixToken() const { return m_prefix_token.c_str(); }
-
-    ReadBufferAndDumpToStreamOptions &SetSuffixToken(const std::string &p) {
-      m_suffix_token = p;
-      return *this;
-    }
+    ReadStringAndDumpToStreamOptions(ValueObject &valobj);
 
-    ReadBufferAndDumpToStreamOptions &SetSuffixToken(std::nullptr_t) {
-      m_suffix_token.clear();
-      return *this;
-    }
+    void SetLocation(uint64_t l) { m_location = l; }
 
-    const char *GetSuffixToken() const { return m_suffix_token.c_str(); }
+    uint64_t GetLocation() const { return m_location; }
 
-    ReadBufferAndDumpToStreamOptions &SetQuote(char q) {
-      m_quote = q;
-      return *this;
-    }
+    void SetProcessSP(lldb::ProcessSP p) { m_process_sp = p; }
 
-    char GetQuote() const { return m_quote; }
+    lldb::ProcessSP GetProcessSP() const { return m_process_sp; }
 
-    ReadBufferAndDumpToStreamOptions &SetSourceSize(uint32_t s) {
-      m_source_size = s;
-      return *this;
-    }
+  private:
+    uint64_t m_location = 0;
+    lldb::ProcessSP m_process_sp;
+  };
 
-    uint32_t GetSourceSize() const { return m_source_size; }
+  class ReadBufferAndDumpToStreamOptions : public DumpToStreamOptions {
+  public:
+    ReadBufferAndDumpToStreamOptions() = default;
 
-    ReadBufferAndDumpToStreamOptions &SetEscapeNonPrintables(bool e) {
-      m_escape_non_printables = e;
-      return *this;
-    }
+    ReadBufferAndDumpToStreamOptions(ValueObject &valobj);
 
-    bool GetEscapeNonPrintables() const { return m_escape_non_printables; }
+    ReadBufferAndDumpToStreamOptions(
+        const ReadStringAndDumpToStreamOptions &options);
 
-    ReadBufferAndDumpToStreamOptions &SetBinaryZeroIsTerminator(bool e) {
-      m_zero_is_terminator = e;
-      return *this;
-    }
+    void SetData(DataExtractor d) { m_data = d; }
 
-    bool GetBinaryZeroIsTerminator() const { return m_zero_is_terminator; }
+    lldb_private::DataExtractor GetData() const { return m_data; }
 
-    ReadBufferAndDumpToStreamOptions &SetIsTruncated(bool t) {
-      m_is_truncated = t;
-      return *this;
-    }
+    void SetIsTruncated(bool t) { m_is_truncated = t; }
 
     bool GetIsTruncated() const { return m_is_truncated; }
-
-    ReadBufferAndDumpToStreamOptions &SetLanguage(lldb::LanguageType l) {
-      m_language_type = l;
-      return *this;
-    }
-
-    lldb::LanguageType GetLanguage() const
-
-    {
-      return m_language_type;
-    }
-
   private:
     DataExtractor m_data;
-    Stream *m_stream;
-    std::string m_prefix_token;
-    std::string m_suffix_token;
-    char m_quote;
-    uint32_t m_source_size;
-    bool m_escape_non_printables;
-    bool m_zero_is_terminator;
-    bool m_is_truncated;
-    lldb::LanguageType m_language_type;
+    bool m_is_truncated = false;
   };
 
   // I can't use a std::unique_ptr for this because the Deleter is a template
diff --git a/lldb/include/lldb/Host/File.h b/lldb/include/lldb/Host/File.h
index 688db17e07ea9..9ad5f4715da44 100644
--- a/lldb/include/lldb/Host/File.h
+++ b/lldb/include/lldb/Host/File.h
@@ -51,22 +51,23 @@ class File : public IOObject {
   static mode_t ConvertOpenOptionsForPOSIXOpen(uint32_t open_options);
 
   File()
-      : IOObject(eFDTypeFile, false), m_descriptor(kInvalidDescriptor),
-        m_stream(kInvalidStream), m_options(0), m_own_stream(false),
-        m_is_interactive(eLazyBoolCalculate),
+      : IOObject(eFDTypeFile), m_descriptor(kInvalidDescriptor),
+        m_own_descriptor(false), m_stream(kInvalidStream), m_options(0),
+        m_own_stream(false), m_is_interactive(eLazyBoolCalculate),
         m_is_real_terminal(eLazyBoolCalculate),
         m_supports_colors(eLazyBoolCalculate) {}
 
   File(FILE *fh, bool transfer_ownership)
-      : IOObject(eFDTypeFile, false), m_descriptor(kInvalidDescriptor),
-        m_stream(fh), m_options(0), m_own_stream(transfer_ownership),
-        m_is_interactive(eLazyBoolCalculate),
+      : IOObject(eFDTypeFile), m_descriptor(kInvalidDescriptor),
+        m_own_descriptor(false), m_stream(fh), m_options(0),
+        m_own_stream(transfer_ownership), m_is_interactive(eLazyBoolCalculate),
         m_is_real_terminal(eLazyBoolCalculate),
         m_supports_colors(eLazyBoolCalculate) {}
 
-  File(int fd, bool transfer_ownership)
-      : IOObject(eFDTypeFile, transfer_ownership), m_descriptor(fd),
-        m_stream(kInvalidStream), m_options(0), m_own_stream(false),
+  File(int fd, uint32_t options, bool transfer_ownership)
+      : IOObject(eFDTypeFile), m_descriptor(fd),
+        m_own_descriptor(transfer_ownership), m_stream(kInvalidStream),
+        m_options(options), m_own_stream(false),
         m_is_interactive(eLazyBoolCalculate),
         m_is_real_terminal(eLazyBoolCalculate) {}
 
@@ -119,18 +120,26 @@ class File : public IOObject {
 
   Status Close() override;
 
-  void Clear();
+  /// DEPRECATED! Extract the underlying FILE* and reset this File without closing it.
+  ///
+  /// This is only here to support legacy SB interfaces that need to convert scripting
+  /// language objects into FILE* streams.   That conversion is inherently sketchy and
+  /// doing so may cause the stream to be leaked.
+  ///
+  /// After calling this the File will be reset to its original state.  It will be
+  /// invalid and it will not hold on to any resources.
+  ///
+  /// \return
+  ///     The underlying FILE* stream from this File, if one exists and can be extracted,
+  ///     nullptr otherwise.
+  FILE *TakeStreamAndClear();
 
   int GetDescriptor() const;
 
   WaitableHandle GetWaitableHandle() override;
 
-  void SetDescriptor(int fd, bool transfer_ownership);
-
   FILE *GetStream();
 
-  void SetStream(FILE *fh, bool transfer_ownership);
-
   /// Read bytes from a file from the current file position.
   ///
   /// NOTE: This function is NOT thread safe. Use the read function
@@ -332,8 +341,6 @@ class File : public IOObject {
 
   size_t PrintfVarArg(const char *format, va_list args);
 
-  void SetOptions(uint32_t options) { m_options = options; }
-
   static bool DescriptorIsValid(int descriptor) { return descriptor >= 0; };
 
 protected:
@@ -345,6 +352,7 @@ class File : public IOObject {
 
   // Member variables
   int m_descriptor;
+  bool m_own_descriptor;
   FILE *m_stream;
   uint32_t m_options;
   bool m_own_stream;
diff --git a/lldb/include/lldb/Host/FileCache.h b/lldb/include/lldb/Host/FileCache.h
index 0c1ef1961645a..49f966058f82f 100644
--- a/lldb/include/lldb/Host/FileCache.h
+++ b/lldb/include/lldb/Host/FileCache.h
@@ -22,7 +22,7 @@ class FileCache {
 private:
   FileCache() {}
 
-  typedef std::map<lldb::user_id_t, lldb::FileSP> FDToFileMap;
+  typedef std::map<lldb::user_id_t, lldb::FileUP> FDToFileMap;
 
 public:
   static FileCache &GetInstance();
diff --git a/lldb/include/lldb/Host/FileSystem.h b/lldb/include/lldb/Host/FileSystem.h
index 733a4a4fe395c..6900f3f73595c 100644
--- a/lldb/include/lldb/Host/FileSystem.h
+++ b/lldb/include/lldb/Host/FileSystem.h
@@ -63,9 +63,10 @@ class FileSystem {
   /// Wraps ::open in a platform-independent way.
   int Open(const char *path, int flags, int mode);
 
-  Status Open(File &File, const FileSpec &file_spec, uint32_t options,
-              uint32_t permissions = lldb::eFilePermissionsFileDefault,
-              bool should_close_fd = true);
+  llvm::Expected<std::unique_ptr<File>>
+  Open(const FileSpec &file_spec, uint32_t options,
+       uint32_t permissions = lldb::eFilePermissionsFileDefault,
+       bool should_close_fd = true);
 
   /// Get a directory iterator.
   /// \{
diff --git a/lldb/include/lldb/Host/Socket.h b/lldb/include/lldb/Host/Socket.h
index 6f96bd73e753e..c6df5634e24e4 100644
--- a/lldb/include/lldb/Host/Socket.h
+++ b/lldb/include/lldb/Host/Socket.h
@@ -31,7 +31,7 @@ class StringRef;
 
 namespace lldb_private {
 
-#if defined(_MSC_VER)
+#if defined(_WIN32)
 typedef SOCKET NativeSocket;
 #else
 typedef int NativeSocket;
@@ -122,6 +122,7 @@ class Socket : public IOObject {
   SocketProtocol m_protocol;
   NativeSocket m_socket;
   bool m_child_processes_inherit;
+  bool m_should_close_fd;
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Host/windows/PosixApi.h b/lldb/include/lldb/Host/windows/PosixApi.h
index 6a6ed3ebd61e9..a4c3dcdbc5214 100644
--- a/lldb/include/lldb/Host/windows/PosixApi.h
+++ b/lldb/include/lldb/Host/windows/PosixApi.h
@@ -31,20 +31,30 @@
 #define SIGKILL 9
 #define SIGSTOP 20
 
-#if defined(_MSC_VER)
+#ifndef S_IRUSR
 #define S_IRUSR S_IREAD  /* read, user */
 #define S_IWUSR S_IWRITE /* write, user */
 #define S_IXUSR 0        /* execute, user */
 #endif
+#ifndef S_IRGRP
 #define S_IRGRP 0 /* read, group */
 #define S_IWGRP 0 /* write, group */
 #define S_IXGRP 0 /* execute, group */
+#endif
+#ifndef S_IROTH
 #define S_IROTH 0 /* read, others */
 #define S_IWOTH 0 /* write, others */
 #define S_IXOTH 0 /* execute, others */
+#endif
+#ifndef S_IRWXU
 #define S_IRWXU 0
+#endif
+#ifndef S_IRWXG
 #define S_IRWXG 0
+#endif
+#ifndef S_IRWXO
 #define S_IRWXO 0
+#endif
 
 #if HAVE_SYS_TYPES_H
 // pyconfig.h typedefs this.  We require python headers to be included before
diff --git a/lldb/include/lldb/Interpreter/CommandInterpreter.h b/lldb/include/lldb/Interpreter/CommandInterpreter.h
index edf92d7ca9a9e..3b98b2a521dd9 100644
--- a/lldb/include/lldb/Interpreter/CommandInterpreter.h
+++ b/lldb/include/lldb/Interpreter/CommandInterpreter.h
@@ -502,6 +502,8 @@ class CommandInterpreter : public Broadcaster,
 
   void GetProcessOutput();
 
+  bool DidProcessStopAbnormally() const;
+
   void SetSynchronous(bool value);
 
   lldb::CommandObjectSP GetCommandSP(llvm::StringRef cmd,
diff --git a/lldb/include/lldb/Interpreter/CommandReturnObject.h b/lldb/include/lldb/Interpreter/CommandReturnObject.h
index a5f61261455ee..670002dada837 100644
--- a/lldb/include/lldb/Interpreter/CommandReturnObject.h
+++ b/lldb/include/lldb/Interpreter/CommandReturnObject.h
@@ -144,14 +144,6 @@ class CommandReturnObject {
 
   void SetInteractive(bool b);
 
-  bool GetAbnormalStopWasExpected() const {
-    return m_abnormal_stop_was_expected;
-  }
-
-  void SetAbnormalStopWasExpected(bool signal_was_expected) {
-    m_abnormal_stop_was_expected = signal_was_expected;
-  }
-
 private:
   enum { eStreamStringIndex = 0, eImmediateStreamIndex = 1 };
 
@@ -162,14 +154,6 @@ class CommandReturnObject {
   bool m_did_change_process_state;
   bool m_interactive; // If true, then the input handle from the debugger will
                       // be hooked up
-  bool m_abnormal_stop_was_expected; // This is to support
-                                     // eHandleCommandFlagStopOnCrash vrs.
-                                     // attach.
-  // The attach command often ends up with the process stopped due to a signal.
-  // Normally that would mean stop on crash should halt batch execution, but we
-  // obviously don't want that for attach.  Using this flag, the attach command
-  // (and anything else for which this is relevant) can say that the signal is
-  // expected, and batch command execution can continue.
 };
 
 } // namespace lldb_private
diff --git a/lldb/include/lldb/Interpreter/ScriptInterpreter.h b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
index c8fa3901350d7..9c68ffb07172b 100644
--- a/lldb/include/lldb/Interpreter/ScriptInterpreter.h
+++ b/lldb/include/lldb/Interpreter/ScriptInterpreter.h
@@ -208,6 +208,7 @@ class ScriptInterpreter : public PluginInterface {
 
   virtual StructuredData::ObjectSP
   CreateScriptedThreadPlan(const char *class_name,
+                           std::string &error_str,
                            lldb::ThreadPlanSP thread_plan_sp) {
     return StructuredData::ObjectSP();
   }
@@ -463,8 +464,6 @@ class ScriptInterpreter : public PluginInterface {
 
   static lldb::ScriptLanguage StringToLanguage(const llvm::StringRef &string);
 
-  virtual void ResetOutputFileHandle(FILE *new_fh) {} // By default, do nothing.
-
   lldb::ScriptLanguage GetLanguage() { return m_script_lang; }
 
 protected:
diff --git a/lldb/include/lldb/Symbol/ClangASTContext.h b/lldb/include/lldb/Symbol/ClangASTContext.h
index 877763407ed39..188635a13b86f 100644
--- a/lldb/include/lldb/Symbol/ClangASTContext.h
+++ b/lldb/include/lldb/Symbol/ClangASTContext.h
@@ -52,7 +52,14 @@ class ClangASTContext : public TypeSystem {
   }
 
   // Constructors and Destructors
-  ClangASTContext(const char *triple = nullptr);
+  explicit ClangASTContext(llvm::StringRef triple = "");
+  explicit ClangASTContext(ArchSpec arch);
+
+  /// Constructs a ClangASTContext that uses an existing ASTContext internally.
+  /// Useful when having an existing ASTContext created by Clang.
+  ///
+  /// \param existing_ctxt An existing ASTContext.
+  explicit ClangASTContext(clang::ASTContext &existing_ctxt);
 
   ~ClangASTContext() override;
 
@@ -79,8 +86,6 @@ class ClangASTContext : public TypeSystem {
 
   clang::ASTContext *getASTContext();
 
-  void setASTContext(clang::ASTContext *ast_ctx);
-
   clang::Builtin::Context *getBuiltinContext();
 
   clang::IdentifierTable *getIdentifierTable();
@@ -106,21 +111,11 @@ class ClangASTContext : public TypeSystem {
   void setSema(clang::Sema *s);
   clang::Sema *getSema() { return m_sema; }
 
-  void Clear();
-
   const char *GetTargetTriple();
 
-  void SetTargetTriple(const char *target_triple);
-
-  void SetArchitecture(const ArchSpec &arch);
-
-  bool HasExternalSource();
-
   void SetExternalSource(
       llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> &ast_source_up);
 
-  void RemoveExternalSource();
-
   bool GetCompleteDecl(clang::Decl *decl) {
     return ClangASTContext::GetCompleteDecl(getASTContext(), decl);
   }
@@ -862,7 +857,6 @@ class ClangASTContext : public TypeSystem {
   static bool SetHasExternalStorage(lldb::opaque_compiler_type_t type,
                                     bool has_extern);
 
-  static bool GetHasExternalStorage(const CompilerType &type);
   // Tag Declarations
   static bool StartTagDeclarationDefinition(const CompilerType &type);
 
@@ -978,43 +972,45 @@ class ClangASTContext : public TypeSystem {
   GetAsTemplateSpecialization(lldb::opaque_compiler_type_t type);
 
   // Classes that inherit from ClangASTContext can see and modify these
-  // clang-format off
-    std::string                                     m_target_triple;
-    std::unique_ptr<clang::ASTContext>              m_ast_up;
-    std::unique_ptr<clang::LangOptions>             m_language_options_up;
-    std::unique_ptr<clang::FileManager>             m_file_manager_up;
-    std::unique_ptr<clang::SourceManager>           m_source_manager_up;
-    std::unique_ptr<clang::DiagnosticsEngine>       m_diagnostics_engine_up;
-    std::unique_ptr<clang::DiagnosticConsumer>      m_diagnostic_consumer_up;
-    std::shared_ptr<clang::TargetOptions>           m_target_options_rp;
-    std::unique_ptr<clang::TargetInfo>              m_target_info_up;
-    std::unique_ptr<clang::IdentifierTable>         m_identifier_table_up;
-    std::unique_ptr<clang::SelectorTable>           m_selector_table_up;
-    std::unique_ptr<clang::Builtin::Context>        m_builtins_up;
-    std::unique_ptr<DWARFASTParserClang>            m_dwarf_ast_parser_up;
-    std::unique_ptr<PDBASTParser>                   m_pdb_ast_parser_up;
-    std::unique_ptr<ClangASTSource>                 m_scratch_ast_source_up;
-    std::unique_ptr<clang::MangleContext>           m_mangle_ctx_up;
-    CompleteTagDeclCallback                         m_callback_tag_decl;
-    CompleteObjCInterfaceDeclCallback               m_callback_objc_decl;
-    void *                                          m_callback_baton;
-    clang::ExternalASTMerger::OriginMap             m_origins;
-    uint32_t                                        m_pointer_byte_size;
-    bool                                            m_ast_owned;
-    /// The sema associated that is currently used to build this ASTContext.
-    /// May be null if we are already done parsing this ASTContext or the
-    /// ASTContext wasn't created by parsing source code.
-    clang::Sema *                                   m_sema = nullptr;
-  // clang-format on
+  std::string m_target_triple;
+  std::unique_ptr<clang::ASTContext> m_ast_up;
+  std::unique_ptr<clang::LangOptions> m_language_options_up;
+  std::unique_ptr<clang::FileManager> m_file_manager_up;
+  std::unique_ptr<clang::SourceManager> m_source_manager_up;
+  std::unique_ptr<clang::DiagnosticsEngine> m_diagnostics_engine_up;
+  std::unique_ptr<clang::DiagnosticConsumer> m_diagnostic_consumer_up;
+  std::shared_ptr<clang::TargetOptions> m_target_options_rp;
+  std::unique_ptr<clang::TargetInfo> m_target_info_up;
+  std::unique_ptr<clang::IdentifierTable> m_identifier_table_up;
+  std::unique_ptr<clang::SelectorTable> m_selector_table_up;
+  std::unique_ptr<clang::Builtin::Context> m_builtins_up;
+  std::unique_ptr<DWARFASTParserClang> m_dwarf_ast_parser_up;
+  std::unique_ptr<PDBASTParser> m_pdb_ast_parser_up;
+  std::unique_ptr<ClangASTSource> m_scratch_ast_source_up;
+  std::unique_ptr<clang::MangleContext> m_mangle_ctx_up;
+  CompleteTagDeclCallback m_callback_tag_decl = nullptr;
+  CompleteObjCInterfaceDeclCallback m_callback_objc_decl = nullptr;
+  void *m_callback_baton = nullptr;
+  clang::ExternalASTMerger::OriginMap m_origins;
+  uint32_t m_pointer_byte_size = 0;
+  bool m_ast_owned = false;
+  /// The sema associated that is currently used to build this ASTContext.
+  /// May be null if we are already done parsing this ASTContext or the
+  /// ASTContext wasn't created by parsing source code.
+  clang::Sema *m_sema = nullptr;
+
 private:
   // For ClangASTContext only
   ClangASTContext(const ClangASTContext &);
   const ClangASTContext &operator=(const ClangASTContext &);
+  /// Creates the internal ASTContext.
+  void CreateASTContext();
+  void SetTargetTriple(llvm::StringRef target_triple);
 };
 
 class ClangASTContextForExpressions : public ClangASTContext {
 public:
-  ClangASTContextForExpressions(Target &target);
+  ClangASTContextForExpressions(Target &target, ArchSpec arch);
 
   ~ClangASTContextForExpressions() override = default;
 
diff --git a/lldb/include/lldb/Symbol/ClangASTImporter.h b/lldb/include/lldb/Symbol/ClangASTImporter.h
index eb3689485cfb8..f963f9b2b1dc6 100644
--- a/lldb/include/lldb/Symbol/ClangASTImporter.h
+++ b/lldb/include/lldb/Symbol/ClangASTImporter.h
@@ -210,7 +210,7 @@ class ClangASTImporter {
   void ForgetDestination(clang::ASTContext *dst_ctx);
   void ForgetSource(clang::ASTContext *dst_ctx, clang::ASTContext *src_ctx);
 
-private:
+public:
   struct DeclOrigin {
     DeclOrigin() : ctx(nullptr), decl(nullptr) {}
 
@@ -235,23 +235,29 @@ class ClangASTImporter {
 
   typedef std::map<const clang::Decl *, DeclOrigin> OriginMap;
 
+  /// Listener interface used by the ASTImporterDelegate to inform other code
+  /// about decls that have been imported the first time.
+  struct NewDeclListener {
+    virtual ~NewDeclListener() = default;
+    /// A decl has been imported for the first time.
+    virtual void NewDeclImported(clang::Decl *from, clang::Decl *to) = 0;
+  };
+
   /// ASTImporter that intercepts and records the import process of the
   /// underlying ASTImporter.
   ///
   /// This class updates the map from declarations to their original
-  /// declarations and can record and complete declarations that have been
-  /// imported in a certain interval.
+  /// declarations and can record declarations that have been imported in a
+  /// certain interval.
   ///
   /// When intercepting a declaration import, the ASTImporterDelegate uses the
   /// CxxModuleHandler to replace any missing or malformed declarations with
   /// their counterpart from a C++ module.
-  class ASTImporterDelegate : public clang::ASTImporter {
-  public:
+  struct ASTImporterDelegate : public clang::ASTImporter {
     ASTImporterDelegate(ClangASTImporter &master, clang::ASTContext *target_ctx,
                         clang::ASTContext *source_ctx)
         : clang::ASTImporter(*target_ctx, master.m_file_manager, *source_ctx,
                              master.m_file_manager, true /*minimal*/),
-          m_decls_to_deport(nullptr), m_decls_already_deported(nullptr),
           m_master(master), m_source_ctx(source_ctx) {
       setODRHandling(clang::ASTImporter::ODRHandlingType::Liberal);
     }
@@ -287,43 +293,32 @@ class ClangASTImporter {
       }
     };
 
-  protected:
-    llvm::Expected<clang::Decl *> ImportImpl(clang::Decl *From) override;
-
-  public:
-    // A call to "InitDeportWorkQueues" puts the delegate into deport mode.
-    // In deport mode, every copied Decl that could require completion is
-    // recorded and placed into the decls_to_deport set.
-    //
-    // A call to "ExecuteDeportWorkQueues" completes all the Decls that
-    // are in decls_to_deport, adding any Decls it sees along the way that it
-    // hasn't already deported.  It proceeds until decls_to_deport is empty.
-    //
-    // These calls must be paired.  Leaving a delegate in deport mode or trying
-    // to start deport delegate with a new pair of queues will result in an
-    // assertion failure.
-
-    void
-    InitDeportWorkQueues(std::set<clang::NamedDecl *> *decls_to_deport,
-                         std::set<clang::NamedDecl *> *decls_already_deported);
-    void ExecuteDeportWorkQueues();
-
     void ImportDefinitionTo(clang::Decl *to, clang::Decl *from);
 
     void Imported(clang::Decl *from, clang::Decl *to) override;
 
     clang::Decl *GetOriginalDecl(clang::Decl *To) override;
 
+    void SetImportListener(NewDeclListener *listener) {
+      assert(m_new_decl_listener == nullptr && "Already attached a listener?");
+      m_new_decl_listener = listener;
+    }
+    void RemoveImportListener() { m_new_decl_listener = nullptr; }
+
+  protected:
+    llvm::Expected<clang::Decl *> ImportImpl(clang::Decl *From) override;
+
+  private:
     /// Decls we should ignore when mapping decls back to their original
     /// ASTContext. Used by the CxxModuleHandler to mark declarations that
     /// were created from the 'std' C++ module to prevent that the Importer
     /// tries to sync them with the broken equivalent in the debug info AST.
     std::set<clang::Decl *> m_decls_to_ignore;
-    std::set<clang::NamedDecl *> *m_decls_to_deport;
-    std::set<clang::NamedDecl *> *m_decls_already_deported;
     ClangASTImporter &m_master;
     clang::ASTContext *m_source_ctx;
     CxxModuleHandler *m_std_handler = nullptr;
+    /// The currently attached listener.
+    NewDeclListener *m_new_decl_listener = nullptr;
   };
 
   typedef std::shared_ptr<ASTImporterDelegate> ImporterDelegateSP;
@@ -389,6 +384,7 @@ class ClangASTImporter {
     }
   }
 
+public:
   DeclOrigin GetDeclOrigin(const clang::Decl *decl);
 
   clang::FileManager m_file_manager;
diff --git a/lldb/include/lldb/Symbol/CompileUnit.h b/lldb/include/lldb/Symbol/CompileUnit.h
index dc4896c92383f..d132c367b9990 100644
--- a/lldb/include/lldb/Symbol/CompileUnit.h
+++ b/lldb/include/lldb/Symbol/CompileUnit.h
@@ -225,6 +225,14 @@ class CompileUnit : public std::enable_shared_from_this<CompileUnit>,
 
   DebugMacros *GetDebugMacros();
 
+  /// Apply a lambda to each external lldb::Module referenced by this
+  /// compilation unit. Recursively also descends into the referenced external
+  /// modules of any encountered compilation unit.
+  ///
+  /// \param[in] lambda
+  ///     The lambda that should be applied to every module.
+  void ForEachExternalModule(llvm::function_ref<void(lldb::ModuleSP)> f);
+
   /// Get the compile unit's support file list.
   ///
   /// The support file list is used by the line table, and any objects that
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index ce8da8014c5c8..d246c6d809447 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -19,8 +19,8 @@
 #include "lldb/Symbol/TypeList.h"
 #include "lldb/Symbol/TypeSystem.h"
 #include "lldb/lldb-private.h"
-
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/Support/Errc.h"
 
 #include <mutex>
 
@@ -122,6 +122,9 @@ class SymbolFile : public PluginInterface {
   virtual size_t ParseFunctions(CompileUnit &comp_unit) = 0;
   virtual bool ParseLineTable(CompileUnit &comp_unit) = 0;
   virtual bool ParseDebugMacros(CompileUnit &comp_unit) = 0;
+  virtual void
+  ForEachExternalModule(CompileUnit &comp_unit,
+                        llvm::function_ref<void(lldb::ModuleSP)> f) {}
   virtual bool ParseSupportFiles(CompileUnit &comp_unit,
                                  FileSpecList &support_files) = 0;
   virtual size_t ParseTypes(CompileUnit &comp_unit) = 0;
@@ -185,25 +188,24 @@ class SymbolFile : public PluginInterface {
   virtual uint32_t FindFunctions(const RegularExpression &regex,
                                  bool include_inlines, bool append,
                                  SymbolContextList &sc_list);
-  virtual uint32_t
+  virtual void
   FindTypes(ConstString name, const CompilerDeclContext *parent_decl_ctx,
-            bool append, uint32_t max_matches,
+            uint32_t max_matches,
             llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
             TypeMap &types);
 
   /// Find types specified by a CompilerContextPattern.
   /// \param languages    Only return results in these languages.
-  virtual size_t FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                           LanguageSet languages, bool append,
-                           TypeMap &types);
+  virtual void FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                           LanguageSet languages, TypeMap &types);
 
   virtual void
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
                              std::vector<ConstString> &mangled_names);
 
-  virtual size_t GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                          lldb::TypeClass type_mask,
-                          lldb_private::TypeList &type_list) = 0;
+  virtual void GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                        lldb::TypeClass type_mask,
+                        lldb_private::TypeList &type_list) = 0;
 
   virtual void PreloadSymbols();
 
@@ -242,6 +244,13 @@ class SymbolFile : public PluginInterface {
     return nullptr;
   }
 
+  /// Return the number of stack bytes taken up by the parameters to this
+  /// function.
+  virtual llvm::Expected<lldb::addr_t> GetParameterStackSize(Symbol &symbol) {
+    return llvm::createStringError(make_error_code(llvm::errc::not_supported),
+                                   "Operation not supported.");
+  }
+
   virtual void Dump(Stream &s);
 
 protected:
diff --git a/lldb/include/lldb/Symbol/TypeList.h b/lldb/include/lldb/Symbol/TypeList.h
index 982f673fff6e6..38342b6d6f4fb 100644
--- a/lldb/include/lldb/Symbol/TypeList.h
+++ b/lldb/include/lldb/Symbol/TypeList.h
@@ -28,15 +28,14 @@ class TypeList {
 
   void Dump(Stream *s, bool show_context);
 
-  //    lldb::TypeSP
-  //    FindType(lldb::user_id_t uid);
-
   TypeList FindTypes(ConstString name);
 
   void Insert(const lldb::TypeSP &type);
 
   uint32_t GetSize() const;
 
+  bool Empty() const { return !GetSize(); }
+
   lldb::TypeSP GetTypeAtIndex(uint32_t idx);
 
   typedef std::vector<lldb::TypeSP> collection;
diff --git a/lldb/include/lldb/Symbol/UnwindPlan.h b/lldb/include/lldb/Symbol/UnwindPlan.h
index fc3e3ccf4ff1b..fa41ba5d056ad 100644
--- a/lldb/include/lldb/Symbol/UnwindPlan.h
+++ b/lldb/include/lldb/Symbol/UnwindPlan.h
@@ -201,7 +201,8 @@ class UnwindPlan {
         unspecified,            // not specified
         isRegisterPlusOffset,   // FA = register + offset
         isRegisterDereferenced, // FA = [reg]
-        isDWARFExpression       // FA = eval(dwarf_expr)
+        isDWARFExpression,      // FA = eval(dwarf_expr)
+        isRaSearch,             // FA = SP + offset + ???
       };
 
       FAValue() : m_type(unspecified), m_value() {}
@@ -214,6 +215,11 @@ class UnwindPlan {
 
       bool IsUnspecified() const { return m_type == unspecified; }
 
+      void SetRaSearch(int32_t offset) {
+        m_type = isRaSearch;
+        m_value.ra_search_offset = offset;
+      }
+
       bool IsRegisterPlusOffset() const {
         return m_type == isRegisterPlusOffset;
       }
@@ -250,9 +256,14 @@ class UnwindPlan {
       ValueType GetValueType() const { return m_type; }
 
       int32_t GetOffset() const {
-        if (m_type == isRegisterPlusOffset)
-          return m_value.reg.offset;
-        return 0;
+        switch (m_type) {
+          case isRegisterPlusOffset:
+            return m_value.reg.offset;
+          case isRaSearch:
+            return m_value.ra_search_offset;
+          default:
+            return 0;
+        }
       }
 
       void IncOffset(int32_t delta) {
@@ -304,6 +315,8 @@ class UnwindPlan {
           const uint8_t *opcodes;
           uint16_t length;
         } expr;
+        // For m_type == isRaSearch
+        int32_t ra_search_offset;
       } m_value;
     }; // class FAValue
 
diff --git a/lldb/include/lldb/Symbol/Variable.h b/lldb/include/lldb/Symbol/Variable.h
index 03a20985462bf..30f57f1f91104 100644
--- a/lldb/include/lldb/Symbol/Variable.h
+++ b/lldb/include/lldb/Symbol/Variable.h
@@ -26,15 +26,14 @@ class Variable : public UserID, public std::enable_shared_from_this<Variable> {
 public:
   typedef RangeVector<lldb::addr_t, lldb::addr_t> RangeList;
 
-  // Constructors and Destructors
-  Variable(lldb::user_id_t uid, const char *name,
-           const char
-               *mangled, // The mangled or fully qualified name of the variable.
-           const lldb::SymbolFileTypeSP &symfile_type_sp,
-           lldb::ValueType scope, SymbolContextScope *owner_scope,
-           const RangeList &scope_range, Declaration *decl,
-           const DWARFExpression &location, bool external, bool artificial,
-           bool static_member = false);
+  /// Constructors and Destructors.
+  ///
+  /// \param mangled The mangled or fully qualified name of the variable.
+  Variable(lldb::user_id_t uid, const char *name, const char *mangled,
+           const lldb::SymbolFileTypeSP &symfile_type_sp, lldb::ValueType scope,
+           SymbolContextScope *owner_scope, const RangeList &scope_range,
+           Declaration *decl, const DWARFExpression &location, bool external,
+           bool artificial, bool static_member = false);
 
   virtual ~Variable();
 
@@ -50,11 +49,11 @@ class Variable : public UserID, public std::enable_shared_from_this<Variable> {
 
   SymbolContextScope *GetSymbolContextScope() const { return m_owner_scope; }
 
-  // Since a variable can have a basename "i" and also a mangled named
-  // "_ZN12_GLOBAL__N_11iE" and a demangled mangled name "(anonymous
-  // namespace)::i", this function will allow a generic match function that can
-  // be called by commands and expression parsers to make sure we match
-  // anything we come across.
+  /// Since a variable can have a basename "i" and also a mangled named
+  /// "_ZN12_GLOBAL__N_11iE" and a demangled mangled name "(anonymous
+  /// namespace)::i", this function will allow a generic match function that can
+  /// be called by commands and expression parsers to make sure we match
+  /// anything we come across.
   bool NameMatches(ConstString name) const;
 
   bool NameMatches(const RegularExpression &regex) const;
@@ -107,26 +106,34 @@ class Variable : public UserID, public std::enable_shared_from_this<Variable> {
   CompilerDecl GetDecl();
 
 protected:
-  ConstString m_name; // The basename of the variable (no namespaces)
-  Mangled m_mangled;  // The mangled name of the variable
-  lldb::SymbolFileTypeSP m_symfile_type_sp; // The type pointer of the variable
-                                            // (int, struct, class, etc)
-  lldb::ValueType m_scope;                  // global, parameter, local
-  SymbolContextScope
-      *m_owner_scope; // The symbol file scope that this variable was defined in
-  RangeList m_scope_range; // The list of ranges inside the owner's scope where
-                           // this variable is valid
-  Declaration m_declaration;  // Declaration location for this item.
-  DWARFExpression m_location; // The location of this variable that can be fed
-                              // to DWARFExpression::Evaluate()
-  uint8_t m_external : 1,     // Visible outside the containing compile unit?
-      m_artificial : 1, // Non-zero if the variable is not explicitly declared
-                        // in source
-      m_loc_is_const_data : 1, // The m_location expression contains the
-                               // constant variable value data, not a DWARF
-                               // location
-      m_static_member : 1; // Non-zero if variable is static member of a class
-                           // or struct.
+  /// The basename of the variable (no namespaces).
+  ConstString m_name;
+  /// The mangled name of the variable.
+  Mangled m_mangled;
+  /// The type pointer of the variable (int, struct, class, etc)
+  /// global, parameter, local.
+  lldb::SymbolFileTypeSP m_symfile_type_sp;
+  lldb::ValueType m_scope;
+  /// The symbol file scope that this variable was defined in
+  SymbolContextScope *m_owner_scope;
+  /// The list of ranges inside the owner's scope where this variable
+  /// is valid.
+  RangeList m_scope_range;
+  /// Declaration location for this item.
+  Declaration m_declaration;
+  /// The location of this variable that can be fed to
+  /// DWARFExpression::Evaluate().
+  DWARFExpression m_location;
+  /// Visible outside the containing compile unit?
+  unsigned m_external : 1;
+  /// Non-zero if the variable is not explicitly declared in source.
+  unsigned m_artificial : 1;
+  /// The m_location expression contains the constant variable value
+  /// data, not a DWARF location.
+  unsigned m_loc_is_const_data : 1;
+  /// Non-zero if variable is static member of a class or struct.
+  unsigned m_static_member : 1;
+
 private:
   Variable(const Variable &rhs) = delete;
   Variable &operator=(const Variable &rhs) = delete;
diff --git a/lldb/include/lldb/Target/ABI.h b/lldb/include/lldb/Target/ABI.h
index f254839fc975a..93378abc2ac2c 100644
--- a/lldb/include/lldb/Target/ABI.h
+++ b/lldb/include/lldb/Target/ABI.h
@@ -15,8 +15,8 @@
 #include "lldb/lldb-private.h"
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/MC/MCRegisterInfo.h"
 
-// forward define the llvm::Type class
 namespace llvm {
 class Type;
 }
@@ -124,6 +124,8 @@ class ABI : public PluginInterface {
     return pc;
   }
 
+  llvm::MCRegisterInfo &GetMCRegisterInfo() { return *m_mc_register_info_up; }
+
   virtual const RegisterInfo *GetRegisterInfoArray(uint32_t &count) = 0;
 
   bool GetRegisterInfoByName(ConstString name, RegisterInfo &info);
@@ -136,13 +138,19 @@ class ABI : public PluginInterface {
   static lldb::ABISP FindPlugin(lldb::ProcessSP process_sp, const ArchSpec &arch);
 
 protected:
-  // Classes that inherit from ABI can see and modify these
-  ABI(lldb::ProcessSP process_sp) {
-    if (process_sp.get())
-        m_process_wp = process_sp;
+  ABI(lldb::ProcessSP process_sp, std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : m_process_wp(process_sp), m_mc_register_info_up(std::move(info_up)) {
+    assert(m_mc_register_info_up && "ABI must have MCRegisterInfo");
   }
 
+  /// Utility function to construct a MCRegisterInfo using the ArchSpec triple.
+  /// Plugins wishing to customize the construction can construct the
+  /// MCRegisterInfo themselves.
+  static std::unique_ptr<llvm::MCRegisterInfo>
+  MakeMCRegisterInfo(const ArchSpec &arch);
+
   lldb::ProcessWP m_process_wp;
+  std::unique_ptr<llvm::MCRegisterInfo> m_mc_register_info_up;
 
 private:
   DISALLOW_COPY_AND_ASSIGN(ABI);
diff --git a/lldb/include/lldb/Target/Platform.h b/lldb/include/lldb/Target/Platform.h
index 3ba58c0ec741e..ba9e97be1b68e 100644
--- a/lldb/include/lldb/Target/Platform.h
+++ b/lldb/include/lldb/Target/Platform.h
@@ -257,19 +257,6 @@ class Platform : public PluginInterface {
 
   virtual bool SetRemoteWorkingDirectory(const FileSpec &working_dir);
 
-  /// Retrieve the system include directories on this platform for the
-  /// given language.
-  ///
-  /// \param[in] lang
-  ///     The language for which the include directories should be queried.
-  ///
-  /// \param[out] directories
-  ///     The include directories for this system.
-  virtual std::vector<std::string>
-  GetSystemIncludeDirectories(lldb::LanguageType lang) {
-    return {};
-  }
-
   virtual UserIDResolver &GetUserIDResolver() = 0;
 
   /// Locate a file for a platform.
diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h
index 65c68566f075b..a4ef1cc40414a 100644
--- a/lldb/include/lldb/Target/Process.h
+++ b/lldb/include/lldb/Target/Process.h
@@ -2272,6 +2272,8 @@ class Process : public std::enable_shared_from_this<Process>,
   void ClearPreResumeAction(PreResumeActionCallback callback, void *baton);
 
   ProcessRunLock &GetRunLock();
+  
+  bool CurrentThreadIsPrivateStateThread();
 
   virtual Status SendEventData(const char *data) {
     Status return_error("Sending an event is not supported for this process.");
diff --git a/lldb/include/lldb/Target/Target.h b/lldb/include/lldb/Target/Target.h
index dbec198c15eb7..e465046959f23 100644
--- a/lldb/include/lldb/Target/Target.h
+++ b/lldb/include/lldb/Target/Target.h
@@ -535,7 +535,7 @@ class Target : public std::enable_shared_from_this<Target>,
 
   static const lldb::TargetPropertiesSP &GetGlobalProperties();
 
-  std::recursive_mutex &GetAPIMutex() { return m_mutex; }
+  std::recursive_mutex &GetAPIMutex();
 
   void DeleteCurrentProcess();
 
@@ -1288,6 +1288,12 @@ class Target : public std::enable_shared_from_this<Target>,
   lldb::PlatformSP m_platform_sp; ///< The platform for this target.
   std::recursive_mutex m_mutex; ///< An API mutex that is used by the lldb::SB*
                                 /// classes make the SB interface thread safe
+  /// When the private state thread calls SB API's - usually because it is 
+  /// running OS plugin or Python ThreadPlan code - it should not block on the
+  /// API mutex that is held by the code that kicked off the sequence of events
+  /// that led us to run the code.  We hand out this mutex instead when we 
+  /// detect that code is running on the private state thread.
+  std::recursive_mutex m_private_mutex; 
   Arch m_arch;
   ModuleList m_images; ///< The list of images for this process (shared
                        /// libraries and anything dynamically loaded).
diff --git a/lldb/include/lldb/Target/ThreadPlanPython.h b/lldb/include/lldb/Target/ThreadPlanPython.h
index 3825bf6ee4f31..58eb33e2c5b79 100644
--- a/lldb/include/lldb/Target/ThreadPlanPython.h
+++ b/lldb/include/lldb/Target/ThreadPlanPython.h
@@ -55,6 +55,7 @@ class ThreadPlanPython : public ThreadPlan {
 
 private:
   std::string m_class_name;
+  std::string m_error_str;
   StructuredData::ObjectSP m_implementation_sp;
   bool m_did_push;
 
diff --git a/lldb/include/lldb/Utility/CompletionRequest.h b/lldb/include/lldb/Utility/CompletionRequest.h
index 28ac78a875659..570f626ac54e8 100644
--- a/lldb/include/lldb/Utility/CompletionRequest.h
+++ b/lldb/include/lldb/Utility/CompletionRequest.h
@@ -103,9 +103,6 @@ class CompletionRequest {
                     CompletionResult &result);
 
   llvm::StringRef GetRawLine() const { return m_command; }
-  llvm::StringRef GetRawLineUntilCursor() const {
-    return m_command.substr(0, m_cursor_index);
-  }
 
   unsigned GetRawCursorPos() const { return m_raw_cursor_pos; }
 
@@ -113,17 +110,25 @@ class CompletionRequest {
 
   Args &GetParsedLine() { return m_parsed_line; }
 
-  const Args &GetPartialParsedLine() const { return m_partial_parsed_line; }
-
   const Args::ArgEntry &GetParsedArg() {
     return GetParsedLine()[GetCursorIndex()];
   }
 
-  void SetCursorIndex(int i) { m_cursor_index = i; }
-  int GetCursorIndex() const { return m_cursor_index; }
+  /// Drops the first argument from the argument list.
+  void ShiftArguments() {
+    m_cursor_index--;
+    m_parsed_line.Shift();
+  }
 
-  void SetCursorCharPosition(int pos) { m_cursor_char_position = pos; }
-  int GetCursorCharPosition() const { return m_cursor_char_position; }
+  /// Adds an empty argument at the end of the argument list and moves
+  /// the cursor to this new argument.
+  void AppendEmptyArgument() {
+    m_parsed_line.AppendArgument(llvm::StringRef());
+    m_cursor_index++;
+    m_cursor_char_position = 0;
+  }
+
+  size_t GetCursorIndex() const { return m_cursor_index; }
 
   /// Adds a possible completion string. If the completion was already
   /// suggested before, it will not be added to the list of results. A copy of
@@ -131,14 +136,33 @@ class CompletionRequest {
   /// afterwards.
   ///
   /// \param match The suggested completion.
-  /// \param match An optional description of the completion string. The
+  /// \param completion An optional description of the completion string. The
   ///     description will be displayed to the user alongside the completion.
+  /// \param mode The CompletionMode for this completion.
   void AddCompletion(llvm::StringRef completion,
                      llvm::StringRef description = "",
                      CompletionMode mode = CompletionMode::Normal) {
     m_result.AddResult(completion, description, mode);
   }
 
+  /// Adds a possible completion string if the completion would complete the
+  /// current argument.
+  ///
+  /// \param match The suggested completion.
+  /// \param description An optional description of the completion string. The
+  ///     description will be displayed to the user alongside the completion.
+  template <CompletionMode M = CompletionMode::Normal>
+  void TryCompleteCurrentArg(llvm::StringRef completion,
+                             llvm::StringRef description = "") {
+    // Trying to rewrite the whole line while checking for the current
+    // argument never makes sense. Completion modes are always hardcoded, so
+    // this can be a static_assert.
+    static_assert(M != CompletionMode::RewriteLine,
+                  "Shouldn't rewrite line with this function");
+    if (completion.startswith(GetCursorArgumentPrefix()))
+      AddCompletion(completion, description, M);
+  }
+
   /// Adds multiple possible completion strings.
   ///
   /// \param completions The list of completions.
@@ -165,12 +189,8 @@ class CompletionRequest {
                     descriptions.GetStringAtIndex(i));
   }
 
-  llvm::StringRef GetCursorArgument() const {
-    return GetParsedLine().GetArgumentAtIndex(GetCursorIndex());
-  }
-
   llvm::StringRef GetCursorArgumentPrefix() const {
-    return GetCursorArgument().substr(0, GetCursorCharPosition());
+    return GetParsedLine().GetArgumentAtIndex(GetCursorIndex());
   }
 
 private:
@@ -180,12 +200,10 @@ class CompletionRequest {
   unsigned m_raw_cursor_pos;
   /// The command line parsed as arguments.
   Args m_parsed_line;
-  /// The command line until the cursor position parsed as arguments.
-  Args m_partial_parsed_line;
   /// The index of the argument in which the completion cursor is.
-  int m_cursor_index;
+  size_t m_cursor_index;
   /// The cursor position in the argument indexed by m_cursor_index.
-  int m_cursor_char_position;
+  size_t m_cursor_char_position;
 
   /// The result this request is supposed to fill out.
   /// We keep this object private to ensure that no backend can in any way
diff --git a/lldb/include/lldb/Utility/DataExtractor.h b/lldb/include/lldb/Utility/DataExtractor.h
index 2898c61df333a..333baf9fd3494 100644
--- a/lldb/include/lldb/Utility/DataExtractor.h
+++ b/lldb/include/lldb/Utility/DataExtractor.h
@@ -14,6 +14,7 @@
 #include "lldb/lldb-forward.h"
 #include "lldb/lldb-types.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/DataExtractor.h"
 
 #include <cassert>
 #include <stdint.h>
@@ -995,6 +996,11 @@ class DataExtractor {
     return {GetDataStart(), size_t(GetByteSize())};
   }
 
+  llvm::DataExtractor GetAsLLVM() const {
+    return {GetData(), GetByteOrder() == lldb::eByteOrderLittle,
+            uint8_t(GetAddressByteSize())};
+  }
+
 protected:
   // Member variables
   const uint8_t *m_start; ///< A pointer to the first byte of data.
diff --git a/lldb/include/lldb/Utility/Flags.h b/lldb/include/lldb/Utility/Flags.h
index 48b14e7d2a2ca..aa869eca0c6e0 100644
--- a/lldb/include/lldb/Utility/Flags.h
+++ b/lldb/include/lldb/Utility/Flags.h
@@ -121,32 +121,6 @@ class Flags {
   ///     \b true if \a bit is 0, \b false otherwise.
   bool IsClear(ValueType bit) const { return (m_flags & bit) == 0; }
 
-  /// Get the number of zero bits in \a m_flags.
-  ///
-  /// \return
-  ///     The number of bits that are set to 0 in the current flags.
-  size_t ClearCount() const {
-    size_t count = 0;
-    for (ValueType shift = 0; shift < sizeof(ValueType) * 8; ++shift) {
-      if ((m_flags & (1u << shift)) == 0)
-        ++count;
-    }
-    return count;
-  }
-
-  /// Get the number of one bits in \a m_flags.
-  ///
-  /// \return
-  ///     The number of bits that are set to 1 in the current flags.
-  size_t SetCount() const {
-    size_t count = 0;
-    for (ValueType mask = m_flags; mask; mask >>= 1) {
-      if (mask & 1u)
-        ++count;
-    }
-    return count;
-  }
-
 protected:
   ValueType m_flags; ///< The flags.
 };
diff --git a/lldb/include/lldb/Utility/GDBRemote.h b/lldb/include/lldb/Utility/GDBRemote.h
new file mode 100644
index 0000000000000..b4adeb3685248
--- /dev/null
+++ b/lldb/include/lldb/Utility/GDBRemote.h
@@ -0,0 +1,117 @@
+//===-- GDBRemote.h ----------------------------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef liblldb_GDBRemote_h_
+#define liblldb_GDBRemote_h_
+
+#include "lldb/Utility/StreamString.h"
+#include "lldb/lldb-enumerations.h"
+#include "lldb/lldb-public.h"
+#include "llvm/Support/YAMLTraits.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+namespace lldb_private {
+
+class StreamGDBRemote : public StreamString {
+public:
+  StreamGDBRemote();
+
+  StreamGDBRemote(uint32_t flags, uint32_t addr_size,
+                  lldb::ByteOrder byte_order);
+
+  ~StreamGDBRemote() override;
+
+  /// Output a block of data to the stream performing GDB-remote escaping.
+  ///
+  /// \param[in] s
+  ///     A block of data.
+  ///
+  /// \param[in] src_len
+  ///     The amount of data to write.
+  ///
+  /// \return
+  ///     Number of bytes written.
+  // TODO: Convert this function to take ArrayRef<uint8_t>
+  int PutEscapedBytes(const void *s, size_t src_len);
+};
+
+/// GDB remote packet as used by the reproducer and the GDB remote
+/// communication history. Packets can be serialized to file.
+struct GDBRemotePacket {
+
+  friend llvm::yaml::MappingTraits<GDBRemotePacket>;
+
+  enum Type { ePacketTypeInvalid = 0, ePacketTypeSend, ePacketTypeRecv };
+
+  GDBRemotePacket()
+      : packet(), type(ePacketTypeInvalid), bytes_transmitted(0), packet_idx(0),
+        tid(LLDB_INVALID_THREAD_ID) {}
+
+  void Clear() {
+    packet.data.clear();
+    type = ePacketTypeInvalid;
+    bytes_transmitted = 0;
+    packet_idx = 0;
+    tid = LLDB_INVALID_THREAD_ID;
+  }
+
+  struct BinaryData {
+    std::string data;
+  };
+
+  void Serialize(llvm::raw_ostream &strm) const;
+  void Dump(Stream &strm) const;
+
+  BinaryData packet;
+  Type type;
+  uint32_t bytes_transmitted;
+  uint32_t packet_idx;
+  lldb::tid_t tid;
+
+private:
+  llvm::StringRef GetTypeStr() const;
+};
+
+} // namespace lldb_private
+
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(lldb_private::GDBRemotePacket)
+LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(std::vector<lldb_private::GDBRemotePacket>)
+
+namespace llvm {
+namespace yaml {
+
+template <>
+struct ScalarEnumerationTraits<lldb_private::GDBRemotePacket::Type> {
+  static void enumeration(IO &io, lldb_private::GDBRemotePacket::Type &value);
+};
+
+template <> struct ScalarTraits<lldb_private::GDBRemotePacket::BinaryData> {
+  static void output(const lldb_private::GDBRemotePacket::BinaryData &, void *,
+                     raw_ostream &);
+
+  static StringRef input(StringRef, void *,
+                         lldb_private::GDBRemotePacket::BinaryData &);
+
+  static QuotingType mustQuote(StringRef S) { return QuotingType::None; }
+};
+
+template <> struct MappingTraits<lldb_private::GDBRemotePacket> {
+  static void mapping(IO &io, lldb_private::GDBRemotePacket &Packet);
+
+  static StringRef validate(IO &io, lldb_private::GDBRemotePacket &);
+};
+
+} // namespace yaml
+} // namespace llvm
+
+#endif // liblldb_GDBRemote_h_
diff --git a/lldb/include/lldb/Utility/IOObject.h b/lldb/include/lldb/Utility/IOObject.h
index 1640200a01d98..16ed580abf714 100644
--- a/lldb/include/lldb/Utility/IOObject.h
+++ b/lldb/include/lldb/Utility/IOObject.h
@@ -29,8 +29,7 @@ class IOObject {
   typedef int WaitableHandle;
   static const WaitableHandle kInvalidHandleValue;
 
-  IOObject(FDType type, bool should_close)
-      : m_fd_type(type), m_should_close_fd(should_close) {}
+  IOObject(FDType type) : m_fd_type(type) {}
   virtual ~IOObject();
 
   virtual Status Read(void *buf, size_t &num_bytes) = 0;
@@ -44,8 +43,6 @@ class IOObject {
 
 protected:
   FDType m_fd_type;
-  bool m_should_close_fd; // True if this class should close the file descriptor
-                          // when it goes away.
 
 private:
   DISALLOW_COPY_AND_ASSIGN(IOObject);
diff --git a/lldb/include/lldb/Utility/JSON.h b/lldb/include/lldb/Utility/JSON.h
deleted file mode 100644
index 172f77afb01fe..0000000000000
--- a/lldb/include/lldb/Utility/JSON.h
+++ /dev/null
@@ -1,283 +0,0 @@
-//===---------------------JSON.h --------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef utility_JSON_h_
-#define utility_JSON_h_
-
-#include "lldb/Utility/StringExtractor.h"
-
-#include <map>
-#include <memory>
-#include <string>
-#include <type_traits>
-#include <vector>
-
-#include <stdint.h>
-
-namespace lldb_private {
-class Stream;
-
-class JSONValue {
-public:
-  virtual void Write(Stream &s) = 0;
-
-  typedef std::shared_ptr<JSONValue> SP;
-
-  enum class Kind { String, Number, True, False, Null, Object, Array };
-
-  JSONValue(Kind k) : m_kind(k) {}
-
-  Kind GetKind() const { return m_kind; }
-
-  virtual ~JSONValue() = default;
-
-private:
-  const Kind m_kind;
-};
-
-class JSONString : public JSONValue {
-public:
-  JSONString();
-  JSONString(const char *s);
-  JSONString(const std::string &s);
-
-  JSONString(const JSONString &s) = delete;
-  JSONString &operator=(const JSONString &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONString> SP;
-
-  std::string GetData() { return m_data; }
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::String;
-  }
-
-  ~JSONString() override = default;
-
-private:
-  static std::string json_string_quote_metachars(const std::string &);
-
-  std::string m_data;
-};
-
-class JSONNumber : public JSONValue {
-public:
-  typedef std::shared_ptr<JSONNumber> SP;
-
-  // We cretae a constructor for all integer and floating point type with using
-  // templates and
-  // SFINAE to avoid having ambiguous overloads because of the implicit type
-  // promotion. If we
-  // would have constructors only with int64_t, uint64_t and double types then
-  // constructing a JSONNumber from an int32_t (or any other similar type)
-  // would fail to compile.
-
-  template <typename T, typename std::enable_if<
-                            std::is_integral<T>::value &&
-                            std::is_unsigned<T>::value>::type * = nullptr>
-  explicit JSONNumber(T u)
-      : JSONValue(JSONValue::Kind::Number), m_data_type(DataType::Unsigned) {
-    m_data.m_unsigned = u;
-  }
-
-  template <typename T,
-            typename std::enable_if<std::is_integral<T>::value &&
-                                    std::is_signed<T>::value>::type * = nullptr>
-  explicit JSONNumber(T s)
-      : JSONValue(JSONValue::Kind::Number), m_data_type(DataType::Signed) {
-    m_data.m_signed = s;
-  }
-
-  template <typename T, typename std::enable_if<
-                            std::is_floating_point<T>::value>::type * = nullptr>
-  explicit JSONNumber(T d)
-      : JSONValue(JSONValue::Kind::Number), m_data_type(DataType::Double) {
-    m_data.m_double = d;
-  }
-
-  ~JSONNumber() override = default;
-
-  JSONNumber(const JSONNumber &s) = delete;
-  JSONNumber &operator=(const JSONNumber &s) = delete;
-
-  void Write(Stream &s) override;
-
-  uint64_t GetAsUnsigned() const;
-
-  int64_t GetAsSigned() const;
-
-  double GetAsDouble() const;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::Number;
-  }
-
-private:
-  enum class DataType : uint8_t { Unsigned, Signed, Double } m_data_type;
-
-  union {
-    uint64_t m_unsigned;
-    int64_t m_signed;
-    double m_double;
-  } m_data;
-};
-
-class JSONTrue : public JSONValue {
-public:
-  JSONTrue();
-
-  JSONTrue(const JSONTrue &s) = delete;
-  JSONTrue &operator=(const JSONTrue &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONTrue> SP;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::True;
-  }
-
-  ~JSONTrue() override = default;
-};
-
-class JSONFalse : public JSONValue {
-public:
-  JSONFalse();
-
-  JSONFalse(const JSONFalse &s) = delete;
-  JSONFalse &operator=(const JSONFalse &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONFalse> SP;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::False;
-  }
-
-  ~JSONFalse() override = default;
-};
-
-class JSONNull : public JSONValue {
-public:
-  JSONNull();
-
-  JSONNull(const JSONNull &s) = delete;
-  JSONNull &operator=(const JSONNull &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONNull> SP;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::Null;
-  }
-
-  ~JSONNull() override = default;
-};
-
-class JSONObject : public JSONValue {
-public:
-  JSONObject();
-
-  JSONObject(const JSONObject &s) = delete;
-  JSONObject &operator=(const JSONObject &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONObject> SP;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::Object;
-  }
-
-  bool SetObject(const std::string &key, JSONValue::SP value);
-
-  JSONValue::SP GetObject(const std::string &key);
-
-  ~JSONObject() override = default;
-
-private:
-  typedef std::map<std::string, JSONValue::SP> Map;
-  typedef Map::iterator Iterator;
-  Map m_elements;
-};
-
-class JSONArray : public JSONValue {
-public:
-  JSONArray();
-
-  JSONArray(const JSONArray &s) = delete;
-  JSONArray &operator=(const JSONArray &s) = delete;
-
-  void Write(Stream &s) override;
-
-  typedef std::shared_ptr<JSONArray> SP;
-
-  static bool classof(const JSONValue *V) {
-    return V->GetKind() == JSONValue::Kind::Array;
-  }
-
-private:
-  typedef std::vector<JSONValue::SP> Vector;
-  typedef Vector::iterator Iterator;
-  typedef Vector::size_type Index;
-  typedef Vector::size_type Size;
-
-public:
-  bool SetObject(Index i, JSONValue::SP value);
-
-  bool AppendObject(JSONValue::SP value);
-
-  JSONValue::SP GetObject(Index i);
-
-  Size GetNumElements();
-
-  ~JSONArray() override = default;
-
-  Vector m_elements;
-};
-
-class JSONParser : public StringExtractor {
-public:
-  enum Token {
-    Invalid,
-    Status,
-    ObjectStart,
-    ObjectEnd,
-    ArrayStart,
-    ArrayEnd,
-    Comma,
-    Colon,
-    String,
-    Integer,
-    Float,
-    True,
-    False,
-    Null,
-    EndOfFile
-  };
-
-  JSONParser(llvm::StringRef data);
-
-  int GetEscapedChar(bool &was_escaped);
-
-  Token GetToken(std::string &value);
-
-  JSONValue::SP ParseJSONValue();
-
-protected:
-  JSONValue::SP ParseJSONObject();
-
-  JSONValue::SP ParseJSONArray();
-};
-} // namespace lldb_private
-
-#endif // utility_JSON_h_
diff --git a/lldb/include/lldb/Utility/Log.h b/lldb/include/lldb/Utility/Log.h
index 90bf38347bb3d..09c0f69544784 100644
--- a/lldb/include/lldb/Utility/Log.h
+++ b/lldb/include/lldb/Utility/Log.h
@@ -14,6 +14,7 @@
 #include "lldb/lldb-defines.h"
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
@@ -112,6 +113,14 @@ class Log final {
   static bool ListChannelCategories(llvm::StringRef channel,
                                     llvm::raw_ostream &stream);
 
+  /// Returns the list of log channels.
+  static std::vector<llvm::StringRef> ListChannels();
+  /// Calls the given lambda for every category in the given channel.
+  /// If no channel with the given name exists, lambda is never called.
+  static void ForEachChannelCategory(
+      llvm::StringRef channel,
+      llvm::function_ref<void(llvm::StringRef, llvm::StringRef)> lambda);
+
   static void DisableAllLogChannels();
 
   static void ListAllLogChannels(llvm::raw_ostream &stream);
@@ -193,6 +202,10 @@ class Log final {
   typedef llvm::StringMap<Log> ChannelMap;
   static llvm::ManagedStatic<ChannelMap> g_channel_map;
 
+  static void ForEachCategory(
+      const Log::ChannelMap::value_type &entry,
+      llvm::function_ref<void(llvm::StringRef, llvm::StringRef)> lambda);
+
   static void ListCategories(llvm::raw_ostream &stream,
                              const ChannelMap::value_type &entry);
   static uint32_t GetFlags(llvm::raw_ostream &stream, const ChannelMap::value_type &entry,
diff --git a/lldb/include/lldb/Utility/RegularExpression.h b/lldb/include/lldb/Utility/RegularExpression.h
index ee2e2f5dbc398..6acc203d8e7c0 100644
--- a/lldb/include/lldb/Utility/RegularExpression.h
+++ b/lldb/include/lldb/Utility/RegularExpression.h
@@ -78,10 +78,6 @@ class RegularExpression {
   ///     otherwise.
   llvm::Error GetError() const;
 
-  bool operator<(const RegularExpression &rhs) const {
-    return GetText() < rhs.GetText();
-  }
-
   bool operator==(const RegularExpression &rhs) const {
     return GetText() == rhs.GetText();
   }
diff --git a/lldb/include/lldb/Utility/Reproducer.h b/lldb/include/lldb/Utility/Reproducer.h
index 76bdfa83037a9..1ad7638eec433 100644
--- a/lldb/include/lldb/Utility/Reproducer.h
+++ b/lldb/include/lldb/Utility/Reproducer.h
@@ -10,7 +10,6 @@
 #define LLDB_UTILITY_REPRODUCER_H
 
 #include "lldb/Utility/FileSpec.h"
-
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileCollector.h"
@@ -216,7 +215,7 @@ class ProcessGDBRemoteProvider
 class Generator final {
 
 public:
-  Generator(const FileSpec &root);
+  Generator(FileSpec root);
   ~Generator();
 
   /// Method to indicate we want to keep the reproducer. If reproducer
@@ -273,7 +272,7 @@ class Generator final {
 
 class Loader final {
 public:
-  Loader(const FileSpec &root);
+  Loader(FileSpec root);
 
   template <typename T> FileSpec GetFile() {
     if (!HasFile(T::file))
diff --git a/lldb/include/lldb/Utility/ReproducerInstrumentation.h b/lldb/include/lldb/Utility/ReproducerInstrumentation.h
index 9be632ec6a950..2151708ff4fd5 100644
--- a/lldb/include/lldb/Utility/ReproducerInstrumentation.h
+++ b/lldb/include/lldb/Utility/ReproducerInstrumentation.h
@@ -40,7 +40,7 @@ inline void stringify_append(llvm::raw_string_ostream &ss, const T *t) {
 template <>
 inline void stringify_append<char>(llvm::raw_string_ostream &ss,
                                    const char *t) {
-  ss << t;
+  ss << '\"' << t << '\"';
 }
 
 template <typename Head>
@@ -105,8 +105,8 @@ template <typename... Ts> inline std::string stringify_args(const Ts &... ts) {
   }
 
 #define LLDB_RECORD_METHOD(Result, Class, Method, Signature, ...)              \
-  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION,              \
-                                            stringify_args(__VA_ARGS__));      \
+  lldb_private::repro::Recorder sb_recorder(                                   \
+      LLVM_PRETTY_FUNCTION, stringify_args(*this, __VA_ARGS__));               \
   if (lldb_private::repro::InstrumentationData data =                          \
           LLDB_GET_INSTRUMENTATION_DATA()) {                                   \
     sb_recorder.Record(                                                        \
@@ -117,8 +117,8 @@ template <typename... Ts> inline std::string stringify_args(const Ts &... ts) {
   }
 
 #define LLDB_RECORD_METHOD_CONST(Result, Class, Method, Signature, ...)        \
-  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION,              \
-                                            stringify_args(__VA_ARGS__));      \
+  lldb_private::repro::Recorder sb_recorder(                                   \
+      LLVM_PRETTY_FUNCTION, stringify_args(*this, __VA_ARGS__));               \
   if (lldb_private::repro::InstrumentationData data =                          \
           LLDB_GET_INSTRUMENTATION_DATA()) {                                   \
     sb_recorder.Record(                                                        \
@@ -129,7 +129,8 @@ template <typename... Ts> inline std::string stringify_args(const Ts &... ts) {
   }
 
 #define LLDB_RECORD_METHOD_NO_ARGS(Result, Class, Method)                      \
-  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION);             \
+  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION,              \
+                                            stringify_args(*this));            \
   if (lldb_private::repro::InstrumentationData data =                          \
           LLDB_GET_INSTRUMENTATION_DATA()) {                                   \
     sb_recorder.Record(data.GetSerializer(), data.GetRegistry(),               \
@@ -139,7 +140,8 @@ template <typename... Ts> inline std::string stringify_args(const Ts &... ts) {
   }
 
 #define LLDB_RECORD_METHOD_CONST_NO_ARGS(Result, Class, Method)                \
-  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION);             \
+  lldb_private::repro::Recorder sb_recorder(LLVM_PRETTY_FUNCTION,              \
+                                            stringify_args(*this));            \
   if (lldb_private::repro::InstrumentationData data =                          \
           LLDB_GET_INSTRUMENTATION_DATA()) {                                   \
     sb_recorder.Record(                                                        \
@@ -542,9 +544,7 @@ class Serializer {
     SerializeAll(tail...);
   }
 
-  void SerializeAll() {
-    m_stream.flush();
-  }
+  void SerializeAll() { m_stream.flush(); }
 
 private:
   /// Serialize pointers. We need to differentiate between pointers to
diff --git a/lldb/include/lldb/Utility/StreamGDBRemote.h b/lldb/include/lldb/Utility/StreamGDBRemote.h
deleted file mode 100644
index dd0ea31126d93..0000000000000
--- a/lldb/include/lldb/Utility/StreamGDBRemote.h
+++ /dev/null
@@ -1,45 +0,0 @@
-//===-- StreamGDBRemote.h ----------------------------------------*- C++-*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef liblldb_StreamGDBRemote_h_
-#define liblldb_StreamGDBRemote_h_
-
-#include "lldb/Utility/StreamString.h"
-#include "lldb/lldb-enumerations.h"
-
-#include <stddef.h>
-#include <stdint.h>
-
-namespace lldb_private {
-
-class StreamGDBRemote : public StreamString {
-public:
-  StreamGDBRemote();
-
-  StreamGDBRemote(uint32_t flags, uint32_t addr_size,
-                  lldb::ByteOrder byte_order);
-
-  ~StreamGDBRemote() override;
-
-  /// Output a block of data to the stream performing GDB-remote escaping.
-  ///
-  /// \param[in] s
-  ///     A block of data.
-  ///
-  /// \param[in] src_len
-  ///     The amount of data to write.
-  ///
-  /// \return
-  ///     Number of bytes written.
-  // TODO: Convert this function to take ArrayRef<uint8_t>
-  int PutEscapedBytes(const void *s, size_t src_len);
-};
-
-} // namespace lldb_private
-
-#endif // liblldb_StreamGDBRemote_h_
diff --git a/lldb/include/lldb/Utility/StructuredData.h b/lldb/include/lldb/Utility/StructuredData.h
index 75eb2f7b7291e..01b14fc3d4d39 100644
--- a/lldb/include/lldb/Utility/StructuredData.h
+++ b/lldb/include/lldb/Utility/StructuredData.h
@@ -10,9 +10,11 @@
 #define liblldb_StructuredData_h_
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
 
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/FileSpec.h"
+#include "lldb/Utility/Stream.h"
 #include "lldb/lldb-enumerations.h"
 
 #include <cassert>
@@ -28,7 +30,6 @@
 
 namespace lldb_private {
 class Status;
-class Stream;
 }
 
 namespace lldb_private {
@@ -150,7 +151,12 @@ class StructuredData {
 
     void DumpToStdout(bool pretty_print = true) const;
 
-    virtual void Dump(Stream &s, bool pretty_print = true) const = 0;
+    virtual void Serialize(llvm::json::OStream &s) const = 0;
+
+    void Dump(lldb_private::Stream &s, bool pretty_print = true) const {
+      llvm::json::OStream jso(s.AsRawOstream(), pretty_print ? 2 : 0);
+      Serialize(jso);
+    }
 
   private:
     lldb::StructuredDataType m_type;
@@ -269,7 +275,7 @@ class StructuredData {
 
     void AddItem(ObjectSP item) { m_items.push_back(item); }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     typedef std::vector<ObjectSP> collection;
@@ -287,7 +293,7 @@ class StructuredData {
 
     uint64_t GetValue() { return m_value; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     uint64_t m_value;
@@ -304,7 +310,7 @@ class StructuredData {
 
     double GetValue() { return m_value; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     double m_value;
@@ -321,7 +327,7 @@ class StructuredData {
 
     bool GetValue() { return m_value; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     bool m_value;
@@ -337,7 +343,7 @@ class StructuredData {
 
     llvm::StringRef GetValue() { return m_value; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     std::string m_value;
@@ -506,7 +512,7 @@ class StructuredData {
       AddItem(key, std::make_shared<Boolean>(value));
     }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   protected:
     typedef std::map<ConstString, ObjectSP> collection;
@@ -521,7 +527,7 @@ class StructuredData {
 
     bool IsValid() const override { return false; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
   };
 
   class Generic : public Object {
@@ -535,14 +541,13 @@ class StructuredData {
 
     bool IsValid() const override { return m_object != nullptr; }
 
-    void Dump(Stream &s, bool pretty_print = true) const override;
+    void Serialize(llvm::json::OStream &s) const override;
 
   private:
     void *m_object;
   };
 
   static ObjectSP ParseJSON(std::string json_text);
-
   static ObjectSP ParseJSONFromFile(const FileSpec &file, Status &error);
 };
 
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 0cdaf1cf9bf81..35f8928debeaf 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -331,6 +331,7 @@ typedef std::shared_ptr<lldb_private::EventDataStructuredData>
 typedef std::shared_ptr<lldb_private::ExecutionContextRef>
     ExecutionContextRefSP;
 typedef std::shared_ptr<lldb_private::ExpressionVariable> ExpressionVariableSP;
+typedef std::unique_ptr<lldb_private::File> FileUP;
 typedef std::shared_ptr<lldb_private::File> FileSP;
 typedef std::shared_ptr<lldb_private::Function> FunctionSP;
 typedef std::shared_ptr<lldb_private::FunctionCaller> FunctionCallerSP;
diff --git a/lldb/lit/Commands/Inputs/frame.py b/lldb/lit/Commands/Inputs/frame.py
new file mode 100644
index 0000000000000..405917428393e
--- /dev/null
+++ b/lldb/lit/Commands/Inputs/frame.py
@@ -0,0 +1,2 @@
+import lldb
+print("frame.py: {}".format(lldb.frame))
diff --git a/lldb/lit/Commands/Inputs/main.c b/lldb/lit/Commands/Inputs/main.c
new file mode 100644
index 0000000000000..c029ddd96cd52
--- /dev/null
+++ b/lldb/lit/Commands/Inputs/main.c
@@ -0,0 +1,2 @@
+int foo() { return 0; }
+int main() { return foo(); }
diff --git a/lldb/lit/Commands/command-script-import.test b/lldb/lit/Commands/command-script-import.test
new file mode 100644
index 0000000000000..18a043b494076
--- /dev/null
+++ b/lldb/lit/Commands/command-script-import.test
@@ -0,0 +1,13 @@
+# REQUIRES: python
+# RUN: echo 'b main' > %t.in
+# RUN: echo 'run' >> %t.in
+# RUN: echo 'command script import %S/Inputs/frame.py' >> %t.in
+
+# RUN: %clang -g -O0 %S/Inputs/main.c -o %t.out
+# RUN: %lldb -b -s %t.in -o 'script print("script: {}".format(lldb.frame))' %t.out | FileCheck %s
+
+# Make sure that we don't have access to lldb.frame from the Python script.
+# CHECK: frame.py: None
+
+# Make sure that we do have access to lldb.frame from the script command.
+# CHECK: script: frame #0
diff --git a/lldb/lit/Modules/PECOFF/basic-info-arm.yaml b/lldb/lit/Modules/PECOFF/basic-info-arm.yaml
new file mode 100644
index 0000000000000..39ef947685768
--- /dev/null
+++ b/lldb/lit/Modules/PECOFF/basic-info-arm.yaml
@@ -0,0 +1,86 @@
+# RUN: yaml2obj %s > %t
+# RUN: lldb-test object-file %t | FileCheck %s
+
+# CHECK: Plugin name: pe-coff
+# CHECK: Architecture: armv7-pc-windows-msvc
+# CHECK: UUID: 
+# CHECK: Executable: true
+# CHECK: Stripped: false
+# CHECK: Type: executable
+# CHECK: Strata: user
+# CHECK: Base VM address: 0x40000
+
+--- !COFF
+OptionalHeader:
+  AddressOfEntryPoint: 4097
+  ImageBase:       4194304
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [ IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE, IMAGE_DLL_CHARACTERISTICS_NX_COMPAT, IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+  ExportTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ImportTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ResourceTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ExceptionTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  CertificateTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  BaseRelocationTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  Debug:
+    RelativeVirtualAddress: 0
+    Size:            0
+  Architecture:
+    RelativeVirtualAddress: 0
+    Size:            0
+  GlobalPtr:
+    RelativeVirtualAddress: 0
+    Size:            0
+  TlsTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  LoadConfigTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  BoundImport:
+    RelativeVirtualAddress: 0
+    Size:            0
+  IAT:
+    RelativeVirtualAddress: 0
+    Size:            0
+  DelayImportDescriptor:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ClrRuntimeHeader:
+    RelativeVirtualAddress: 0
+    Size:            0
+header:
+  Machine:         IMAGE_FILE_MACHINE_ARMNT
+  Characteristics: [ IMAGE_FILE_EXECUTABLE_IMAGE, IMAGE_FILE_32BIT_MACHINE ]
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  4096
+    VirtualSize:     2
+    SectionData:     '7047'
+symbols:         []
+...
diff --git a/lldb/lit/Modules/PECOFF/basic-info-arm64.yaml b/lldb/lit/Modules/PECOFF/basic-info-arm64.yaml
new file mode 100644
index 0000000000000..1fcf3b35a8def
--- /dev/null
+++ b/lldb/lit/Modules/PECOFF/basic-info-arm64.yaml
@@ -0,0 +1,86 @@
+# RUN: yaml2obj %s > %t
+# RUN: lldb-test object-file %t | FileCheck %s
+
+# CHECK: Plugin name: pe-coff
+# CHECK: Architecture: aarch64-unknown-windows-msvc
+# CHECK: UUID: 
+# CHECK: Executable: true
+# CHECK: Stripped: false
+# CHECK: Type: executable
+# CHECK: Strata: user
+# CHECK: Base VM address: 0x40000000
+
+--- !COFF
+OptionalHeader:
+  AddressOfEntryPoint: 4096
+  ImageBase:       1073741824
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [ IMAGE_DLL_CHARACTERISTICS_HIGH_ENTROPY_VA, IMAGE_DLL_CHARACTERISTICS_DYNAMIC_BASE, IMAGE_DLL_CHARACTERISTICS_NX_COMPAT, IMAGE_DLL_CHARACTERISTICS_TERMINAL_SERVER_AWARE ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+  ExportTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ImportTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ResourceTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ExceptionTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  CertificateTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  BaseRelocationTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  Debug:
+    RelativeVirtualAddress: 0
+    Size:            0
+  Architecture:
+    RelativeVirtualAddress: 0
+    Size:            0
+  GlobalPtr:
+    RelativeVirtualAddress: 0
+    Size:            0
+  TlsTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  LoadConfigTable:
+    RelativeVirtualAddress: 0
+    Size:            0
+  BoundImport:
+    RelativeVirtualAddress: 0
+    Size:            0
+  IAT:
+    RelativeVirtualAddress: 0
+    Size:            0
+  DelayImportDescriptor:
+    RelativeVirtualAddress: 0
+    Size:            0
+  ClrRuntimeHeader:
+    RelativeVirtualAddress: 0
+    Size:            0
+header:
+  Machine:         IMAGE_FILE_MACHINE_ARM64
+  Characteristics: [ IMAGE_FILE_EXECUTABLE_IMAGE, IMAGE_FILE_LARGE_ADDRESS_AWARE ]
+sections:
+  - Name:            .text
+    Characteristics: [ IMAGE_SCN_CNT_CODE, IMAGE_SCN_MEM_EXECUTE, IMAGE_SCN_MEM_READ ]
+    VirtualAddress:  4096
+    VirtualSize:     4
+    SectionData:     C0035FD6
+symbols:         []
+...
diff --git a/lldb/lit/Reproducer/Inputs/FileCapture.in b/lldb/lit/Reproducer/Inputs/FileCapture.in
index bf6f852dbedb7..2c69487575e2d 100644
--- a/lldb/lit/Reproducer/Inputs/FileCapture.in
+++ b/lldb/lit/Reproducer/Inputs/FileCapture.in
@@ -1,3 +1,4 @@
 run
 reproducer status
+reproducer dump -p files
 reproducer generate
diff --git a/lldb/lit/Reproducer/TestDump.test b/lldb/lit/Reproducer/TestDump.test
new file mode 100644
index 0000000000000..2e717152057ca
--- /dev/null
+++ b/lldb/lit/Reproducer/TestDump.test
@@ -0,0 +1,26 @@
+# UNSUPPORTED: system-windows
+# This tests the reproducer dump functionality.
+
+# Generate a reproducer.
+# RUN: mkdir -p %t
+# RUN: rm -rf %t.repro
+# RUN: %clang %S/Inputs/simple.c -g -o %t/reproducer.out
+# RUN: %lldb -x -b -s %S/Inputs/FileCapture.in -o 'reproducer dump -p files' --capture --capture-path %t.repro %t/reproducer.out
+
+# RUN: %lldb -b -o 'reproducer dump -p files -f %t.repro' | FileCheck %s --check-prefix FILES
+# FILES-DAG: 'reproducer.out'
+# FILES-DAG: 'FileCapture.in'
+
+# RUN: %lldb -b -o 'reproducer dump -p version -f %t.repro' | FileCheck %s --check-prefix VERSION
+# VERSION: lldb version
+
+# RUN: %lldb -b -o 'reproducer dump -p commands -f %t.repro' | FileCheck %s --check-prefix COMMANDS
+# COMMANDS: command source
+# COMMANDS: target create
+# COMMANDS: command source
+
+# RUN: %lldb -b -o 'reproducer dump -p gdb -f %t.repro' | FileCheck %s --check-prefix GDB
+# GDB: send packet: $QStartNoAckMode#b0
+# GDB: read packet: $OK#9a
+
+# RUN: %lldb --replay %t.repro | FileCheck %s --check-prefix FILES
diff --git a/lldb/lit/Reproducer/TestRelativePath.test b/lldb/lit/Reproducer/TestRelativePath.test
new file mode 100644
index 0000000000000..1c871ee81e8be
--- /dev/null
+++ b/lldb/lit/Reproducer/TestRelativePath.test
@@ -0,0 +1,8 @@
+# This tests relative capture paths.
+
+# RUN: mkdir -p %t
+# RUN: cd %t
+# RUN: rm -rf ./foo
+# RUN: %clang %S/Inputs/simple.c -g -o %t/reproducer.out
+# RUN: %lldb -x -b -s %S/Inputs/FileCapture.in -o 'reproducer dump -p files' --capture --capture-path ./foo %t/reproducer.out
+# RUN: %lldb --replay ./foo
diff --git a/lldb/lit/Reproducer/TestSynchronous.test b/lldb/lit/Reproducer/TestSynchronous.test
index a2ff189bfc73f..f32ce8c57224f 100644
--- a/lldb/lit/Reproducer/TestSynchronous.test
+++ b/lldb/lit/Reproducer/TestSynchronous.test
@@ -1,3 +1,4 @@
+# REQUIRES: python
 # Ensure that replay happens in synchronous mode.
 
 # RUN: rm -rf %t.repro
diff --git a/lldb/lit/Suite/lldbtest.py b/lldb/lit/Suite/lldbtest.py
index d6ed8b61d73ce..99eb1457259e8 100644
--- a/lldb/lit/Suite/lldbtest.py
+++ b/lldb/lit/Suite/lldbtest.py
@@ -102,6 +102,11 @@ def execute(self, test, litConfig):
             if 'XPASS:' in out or 'XPASS:' in err:
                 return lit.Test.XPASS, out + err
 
+        has_unsupported_tests = 'UNSUPPORTED:' in out or 'UNSUPPORTED:' in err
+        has_passing_tests = 'PASS:' in out or 'PASS:' in err
+        if has_unsupported_tests and not has_passing_tests:
+            return lit.Test.UNSUPPORTED, out + err
+
         passing_test_line = 'RESULT: PASSED'
         if passing_test_line not in out and passing_test_line not in err:
             msg = ('Unable to find %r in dotest output (exit code %d):\n\n%s%s'
diff --git a/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms b/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms
new file mode 100644
index 0000000000000..329a280bde858
--- /dev/null
+++ b/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-raSearch.syms
@@ -0,0 +1,15 @@
+MODULE windows x86 897DD83EA8C8411897F3A925EE4BF7411 unwind-via-stack-win.pdb
+INFO CODE_ID 5D499B5C5000 unwind-via-stack-win.exe
+PUBLIC  0 0 dummy
+PUBLIC 10 0 call_many
+PUBLIC 80 0 main
+PUBLIC 90 0 many_pointer_args
+PUBLIC 100 0 complex_rasearch
+PUBLIC 110 0 esp_rasearch
+PUBLIC 120 0 nonzero_frame_size
+STACK WIN 4 10 6d 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 80 8 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 90 5 0 0 50 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 100 4 0 0 0 0 0 0 1 $T0 .raSearch 80 + = $eip $T0 ^ = $esp $T0 4 + =
+STACK WIN 4 110 4 0 0 0 0 0 0 1 $T0 .raSearch = $eip $T0 ^ = $esp .raSearch 4 + =
+STACK WIN 4 120 4 0 0 0 4 8 0 1 $T0 .raSearch = $eip $T0 ^ = $esp $T0 4 + =
diff --git a/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml b/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
index e788575aad6d6..399cf2bbaa351 100644
--- a/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
+++ b/lldb/lit/SymbolFile/Breakpad/Inputs/unwind-via-stack-win.yaml
@@ -23,6 +23,8 @@ Streams:
     Memory Ranges:
       - Start of Memory Range: 0x0000000000CFFE78
         Content:         0000000079100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0000100B0085100B0094842777
+  - Type:            MemoryInfoList
+    Content:         1000000030000000020000000000000000100B00000000000000000000000000000000000000000000400000000000000010000010000000000000010000000000002677000000000000000000000000000000000000000000000E000000000000100000100000000000000100000000
   - Type:            SystemInfo
     Processor Arch:  X86
     Platform ID:     Win32NT
diff --git a/lldb/lit/SymbolFile/Breakpad/unwind-via-raSearch.test b/lldb/lit/SymbolFile/Breakpad/unwind-via-raSearch.test
new file mode 100644
index 0000000000000..ea259a9de7e47
--- /dev/null
+++ b/lldb/lit/SymbolFile/Breakpad/unwind-via-raSearch.test
@@ -0,0 +1,43 @@
+# RUN: yaml2obj %S/Inputs/unwind-via-stack-win.yaml > %t
+# RUN: %lldb -c %t \
+# RUN:   -o "target symbols add %S/Inputs/unwind-via-raSearch.syms" \
+# RUN:   -s %s -b | FileCheck %s
+
+# First check that unwind plan generation works correctly.
+# This function has a "typical" unwind rule.
+image show-unwind -n call_many
+# CHECK-LABEL: image show-unwind -n call_many
+# CHECK: UNWIND PLANS for unwind-via-stack-win.exe`call_many
+# CHECK: Symbol file UnwindPlan:
+# CHECK: This UnwindPlan originally sourced from breakpad STACK WIN
+# CHECK: This UnwindPlan is sourced from the compiler: yes.
+# CHECK: This UnwindPlan is valid at all instruction locations: no.
+# CHECK: Address range of this UnwindPlan: [unwind-via-stack-win.exe..module_image + 16-0x0000007d)
+# CHECK: row[0]:    0: CFA=RaSearch@SP+0 => esp=DW_OP_pick 0x00, DW_OP_consts +4, DW_OP_plus  eip=DW_OP_pick 0x00, DW_OP_deref
+
+image show-unwind -n nonzero_frame_size
+# CHECK-LABEL: image show-unwind -n nonzero_frame_size
+# CHECK: UNWIND PLANS for unwind-via-stack-win.exe`nonzero_frame_size
+# CHECK: Symbol file UnwindPlan:
+# CHECK: row[0]:    0: CFA=RaSearch@SP+12 => esp=DW_OP_pick 0x00, DW_OP_consts +4, DW_OP_plus  eip=DW_OP_pick 0x00, DW_OP_deref
+
+# Then, some invalid rules.
+image show-unwind -n complex_rasearch
+# CHECK-LABEL: image show-unwind -n complex_rasearch
+# CHECK: UNWIND PLANS for unwind-via-stack-win.exe`complex_rasearch
+# CHECK-NOT: Symbol file
+
+image show-unwind -n esp_rasearch
+# CHECK-LABEL: image show-unwind -n esp_rasearch
+# CHECK: UNWIND PLANS for unwind-via-stack-win.exe`esp_rasearch
+# CHECK-NOT: Symbol file
+
+# And finally, check that backtracing works as a whole by unwinding a simple
+# stack.
+thread backtrace
+# CHECK-LABEL: thread backtrace
+# CHECK: frame #0: 0x000b1092 unwind-via-stack-win.exe`many_pointer_args
+# CHECK: frame #1: 0x000b1079 unwind-via-stack-win.exe`call_many + 105
+# CHECK: frame #2: 0x000b1085 unwind-via-stack-win.exe`main + 5
+# CHECK: frame #3: 0x77278494 kernel32.dll
+# CHECK-NOT: frame
diff --git a/lldb/lit/SymbolFile/DWARF/debug-types-address-ranges.s b/lldb/lit/SymbolFile/DWARF/debug-types-address-ranges.s
index b12d1468e8aec..e87f8c8569db4 100644
--- a/lldb/lit/SymbolFile/DWARF/debug-types-address-ranges.s
+++ b/lldb/lit/SymbolFile/DWARF/debug-types-address-ranges.s
@@ -7,7 +7,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -dwarf-version=5 -triple x86_64-pc-linux %s -filetype=obj >%t.o
-# RUN: ld.lld %t.o -o %t -image-base=0x47000
+# RUN: ld.lld %t.o -o %t -image-base=0x47000 -z separate-code
 # RUN: %lldb %t -o "image lookup -a 0x48000 -v" -o exit | FileCheck %s
 
 # CHECK:   CompileUnit: id = {0x00000001}, file = "/tmp/a.cc", language = "c++"
diff --git a/lldb/lit/SymbolFile/DWARF/debug_loc.s b/lldb/lit/SymbolFile/DWARF/debug_loc.s
index 1fc862f8a1fc5..8dec979f694fc 100644
--- a/lldb/lit/SymbolFile/DWARF/debug_loc.s
+++ b/lldb/lit/SymbolFile/DWARF/debug_loc.s
@@ -9,12 +9,12 @@
 # RUN:   | FileCheck %s
 
 # CHECK-LABEL: image lookup -v -a 0
-# CHECK: Variable: {{.*}}, name = "x0", type = "int", location = rdi,
+# CHECK: Variable: {{.*}}, name = "x0", type = "int", location = DW_OP_reg5 RDI,
 # CHECK: Variable: {{.*}}, name = "x1", type = "int", location = ,
 # CHECK: Variable: {{.*}}, name = "x2", type = "int", location = ,
 
 # CHECK-LABEL: image lookup -v -a 2
-# CHECK: Variable: {{.*}}, name = "x0", type = "int", location = rax,
+# CHECK: Variable: {{.*}}, name = "x0", type = "int", location = DW_OP_reg0 RAX,
 # CHECK: Variable: {{.*}}, name = "x1", type = "int", location = ,
 # CHECK: Variable: {{.*}}, name = "x2", type = "int", location = ,
 
diff --git a/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir-relative-name.s b/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir-relative-name.s
index 2a798f1796294..7fee15d65ca21 100644
--- a/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir-relative-name.s
+++ b/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir-relative-name.s
@@ -5,7 +5,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -triple x86_64-pc-linux %s -filetype=obj > %t.o
-# RUN: ld.lld %t.o -o %t
+# RUN: ld.lld %t.o -o %t -z separate-code
 # RUN: %lldb %t -s %S/Inputs/dir-separator-no-comp-dir-relative-name.lldbinit -o exit | FileCheck %s
 
 # CHECK-LABEL: image dump line-table a.c
diff --git a/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir.s b/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir.s
index a1d0ea5e71ba4..de6bde9b5c627 100644
--- a/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir.s
+++ b/lldb/lit/SymbolFile/DWARF/dir-separator-no-comp-dir.s
@@ -4,7 +4,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -triple x86_64-pc-linux %s -filetype=obj > %t.o
-# RUN: ld.lld %t.o -o %t
+# RUN: ld.lld %t.o -o %t -z separate-code
 # RUN: %lldb %t -s %S/Inputs/dir-separator-windows.lldbinit -o exit | FileCheck %s
 
 # CHECK-LABEL: image dump line-table a.c
diff --git a/lldb/lit/SymbolFile/DWARF/dir-separator-posix.s b/lldb/lit/SymbolFile/DWARF/dir-separator-posix.s
index 14d110016e664..d646670891925 100644
--- a/lldb/lit/SymbolFile/DWARF/dir-separator-posix.s
+++ b/lldb/lit/SymbolFile/DWARF/dir-separator-posix.s
@@ -4,7 +4,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -triple x86_64-pc-linux %s -filetype=obj > %t.o
-# RUN: ld.lld %t.o -o %t
+# RUN: ld.lld %t.o -o %t -z separate-code
 # RUN: %lldb %t -s %S/Inputs/dir-separator-posix.lldbinit -o exit | FileCheck %s
 
 # CHECK-LABEL: image dump line-table a.c
diff --git a/lldb/lit/SymbolFile/DWARF/dir-separator-windows.s b/lldb/lit/SymbolFile/DWARF/dir-separator-windows.s
index 1ba03cc6ab5fc..d64033546f7fa 100644
--- a/lldb/lit/SymbolFile/DWARF/dir-separator-windows.s
+++ b/lldb/lit/SymbolFile/DWARF/dir-separator-windows.s
@@ -4,7 +4,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -triple x86_64-pc-linux %s -filetype=obj > %t.o
-# RUN: ld.lld %t.o -o %t
+# RUN: ld.lld %t.o -o %t -z separate-code
 # RUN: %lldb %t -s %S/Inputs/dir-separator-windows.lldbinit -o exit | FileCheck %s
 
 # CHECK-LABEL: image dump line-table a.c
diff --git a/lldb/lit/SymbolFile/DWARF/dwarf5_locations.s b/lldb/lit/SymbolFile/DWARF/dwarf5_locations.s
index 2cf16ca78103c..4093ea059808d 100644
--- a/lldb/lit/SymbolFile/DWARF/dwarf5_locations.s
+++ b/lldb/lit/SymbolFile/DWARF/dwarf5_locations.s
@@ -7,7 +7,7 @@
 # RUN: lldb-test symbols %t | FileCheck %s
 
 # CHECK: Variable{0x7fffffff00000011}, name = "color"
-# CHECK-SAME: location = DW_OP_addrx(0x0)
+# CHECK-SAME: location = DW_OP_addrx 0x0
 
         .text
         .section        .debug_str,"MS",@progbits,1
diff --git a/lldb/lit/SymbolFile/DWARF/find-inline-method.s b/lldb/lit/SymbolFile/DWARF/find-inline-method.s
index c88c9a21460e2..cfc5ef01dd359 100644
--- a/lldb/lit/SymbolFile/DWARF/find-inline-method.s
+++ b/lldb/lit/SymbolFile/DWARF/find-inline-method.s
@@ -1,7 +1,7 @@
 # REQUIRES: lld, x86
 
 # RUN: llvm-mc -triple x86_64-pc-linux %s -filetype=obj > %t.o
-# RUN: ld.lld %t.o -o %t
+# RUN: ld.lld %t.o -o %t -z separate-code
 # RUN: lldb-test symbols --find=function --name=inl --function-flags=method %t \
 # RUN:   | FileCheck %s
 
diff --git a/lldb/packages/Python/lldbsuite/test/api/log/TestAPILog.py b/lldb/packages/Python/lldbsuite/test/api/log/TestAPILog.py
new file mode 100644
index 0000000000000..c0ffa2c6f5093
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/api/log/TestAPILog.py
@@ -0,0 +1,51 @@
+"""
+Test API logging.
+"""
+
+import re
+
+import lldb
+import lldbsuite.test.lldbutil as lldbutil
+from lldbsuite.test.lldbtest import *
+
+
+class APILogTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_api_log(self):
+        """Test API logging"""
+        logfile = os.path.join(self.getBuildDir(), "api-log.txt")
+
+        def cleanup():
+            if os.path.exists(logfile):
+                os.unlink(logfile)
+
+        self.addTearDownHook(cleanup)
+        self.expect("log enable lldb api -f {}".format(logfile))
+
+        self.dbg.SetDefaultArchitecture(None)
+        self.dbg.GetScriptingLanguage(None)
+        target = self.dbg.CreateTarget(None)
+
+        print(logfile)
+        with open(logfile, 'r') as f:
+            log = f.read()
+
+        # Find the SBDebugger's address.
+        debugger_addr = re.findall(
+            r"lldb::SBDebugger::GetScriptingLanguage\([^)]*\) \(0x([0-9a-fA-F]+),",
+            log)
+
+        # Make sure we've found a match.
+        self.assertTrue(debugger_addr, log)
+
+        # Make sure the GetScriptingLanguage matches.
+        self.assertTrue(re.search(r'lldb::SBDebugger::GetScriptingLanguage\([^)]*\) \(0x{}, ""\)'.format(
+            debugger_addr[0]), log), log)
+
+        # Make sure the address matches.
+        self.assertTrue(re.search(r'lldb::SBDebugger::CreateTarget\([^)]*\) \(0x{}, ""\)'.format(
+            debugger_addr[0]), log), log)
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile b/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile
index dac447e5c70b3..f8f686a345520 100644
--- a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/arm/breakpoint-it/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS = -mthumb
+CFLAGS_EXTRAS := -mthumb
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile b/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile
index 0d6bbe2df0108..3f8e5597f7dde 100644
--- a/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/arm/breakpoint-thumb-codesection/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS = -mthumb
+CFLAGS_EXTRAS := -mthumb
 
 include Makefile.rules
\ No newline at end of file
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py b/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py
index 87787f3479a21..0065d98f4738a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/call-function/TestCallBuiltinFunction.py
@@ -40,14 +40,7 @@ def test(self):
 
         # Test different builtin functions.
 
-        interp.HandleCommand("expr __builtin_isinf(0.0f)", result)
-        self.assertEqual(result.GetOutput(), "(int) $0 = 0\n")
-
-        interp.HandleCommand("expr __builtin_isnormal(0.0f)", result)
-        self.assertEqual(result.GetOutput(), "(int) $1 = 0\n")
-
-        interp.HandleCommand("expr __builtin_constant_p(1)", result)
-        self.assertEqual(result.GetOutput(), "(int) $2 = 1\n")
-
-        interp.HandleCommand("expr __builtin_abs(-14)", result)
-        self.assertEqual(result.GetOutput(), "(int) $3 = 14\n")
+        self.expect("expr __builtin_isinf(0.0f)", substrs=["(int) $", " = 0\n"])
+        self.expect("expr __builtin_isnormal(0.0f)", substrs=["(int) $", " = 0\n"])
+        self.expect("expr __builtin_constant_p(1)", substrs=["(int) $", " = 1\n"])
+        self.expect("expr __builtin_abs(-14)", substrs=["(int) $", " = 14\n"])
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/TestCallOverriddenMethod.py b/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/TestCallOverriddenMethod.py
index 1e61aa9f46e01..c22fffb4f23e4 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/TestCallOverriddenMethod.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/TestCallOverriddenMethod.py
@@ -26,7 +26,6 @@ def setUp(self):
         # Find the line number to break for main.c.
         self.line = line_number('main.cpp', '// Set breakpoint here')
 
-    @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr21765")
     def test(self):
         """Test calls to overridden methods in derived classes."""
         self.build()
@@ -41,9 +40,12 @@ def test(self):
 
         # Test call to method in base class (this should always work as the base
         # class method is never an override).
-        self.expect("expr b->foo()")
+        self.expect("expr b->foo()", substrs=["2"])
 
         # Test call to overridden method in derived class (this will fail if the
         # overrides table is not correctly set up, as Derived::foo will be assigned
         # a vtable entry that does not exist in the compiled program).
-        self.expect("expr d.foo()")
+        self.expect("expr d.foo()", substrs=["2"])
+
+        # Test calling the base class.
+        self.expect("expr realbase.foo()", substrs=["1"])
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/main.cpp b/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/main.cpp
index 54ae705d29785..712c374eefd9f 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/main.cpp
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/call-overridden-method/main.cpp
@@ -1,15 +1,16 @@
 class Base {
 public:
   virtual ~Base() {}
-  virtual void foo() {}
+  virtual int foo() { return 1; }
 };
 
 class Derived : public Base {
 public:
-  virtual void foo() {}
+  virtual int foo() { return 2; }
 };
 
 int main() {
+  Base realbase;
   Derived d;
   Base *b = &d;
   return 0; // Set breakpoint here
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile
index 33746217ecbae..becb2f0965868 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/call-throws/Makefile
@@ -1,4 +1,3 @@
 OBJC_SOURCES := call-throws.m
-
+LD_EXTRAS := -framework Foundation
 include Makefile.rules
-LDFLAGS += -framework Foundation
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile
index 2f36b673031d6..a3198db9e8e88 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/context-object-objc/Makefile
@@ -1,4 +1,3 @@
 OBJC_SOURCES := main.m
-
+LD_EXTRAS := -framework Foundation
 include Makefile.rules
-LDFLAGS += -framework Foundation
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/Makefile
new file mode 100644
index 0000000000000..99998b20bcb05
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/TestExprDiagnostics.py b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/TestExprDiagnostics.py
new file mode 100644
index 0000000000000..dcd01a7f3b588
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/TestExprDiagnostics.py
@@ -0,0 +1,113 @@
+"""
+Test the diagnostics emitted by our embeded Clang instance that parses expressions.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test.decorators import *
+
+class ExprDiagnosticsTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def setUp(self):
+        # Call super's setUp().
+        TestBase.setUp(self)
+
+        self.main_source = "main.cpp"
+        self.main_source_spec = lldb.SBFileSpec(self.main_source)
+
+    def test_source_and_caret_printing(self):
+        """Test that the source and caret positions LLDB prints are correct"""
+        self.build()
+
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+                                          '// Break here', self.main_source_spec)
+        frame = thread.GetFrameAtIndex(0)
+
+        # Test that source/caret are at the right position.
+        value = frame.EvaluateExpression("unknown_identifier")
+        self.assertFalse(value.GetError().Success())
+        # We should get a nice diagnostic with a caret pointing at the start of
+        # the identifier.
+        self.assertIn("\nunknown_identifier\n^\n", value.GetError().GetCString())
+        self.assertIn("<user expression 0>:1:1", value.GetError().GetCString())
+
+        # Same as above but with the identifier in the middle.
+        value = frame.EvaluateExpression("1 + unknown_identifier  ")
+        self.assertFalse(value.GetError().Success())
+        self.assertIn("\n1 + unknown_identifier", value.GetError().GetCString())
+        self.assertIn("\n    ^\n", value.GetError().GetCString())
+
+        # Multiline expressions.
+        value = frame.EvaluateExpression("int a = 0;\nfoobar +=1;\na")
+        self.assertFalse(value.GetError().Success())
+        # We should still get the right line information and caret position.
+        self.assertIn("\nfoobar +=1;\n^\n", value.GetError().GetCString())
+        # It's the second line of the user expression.
+        self.assertIn("<user expression 2>:2:1", value.GetError().GetCString())
+
+        # Top-level expressions.
+        top_level_opts = lldb.SBExpressionOptions();
+        top_level_opts.SetTopLevel(True)
+
+        value = frame.EvaluateExpression("void foo(unknown_type x) {}", top_level_opts)
+        self.assertFalse(value.GetError().Success())
+        self.assertIn("\nvoid foo(unknown_type x) {}\n         ^\n", value.GetError().GetCString())
+        # Top-level expressions might use a different wrapper code, but the file name should still
+        # be the same.
+        self.assertIn("<user expression 3>:1:10", value.GetError().GetCString())
+
+        # Multiline top-level expressions.
+        value = frame.EvaluateExpression("void x() {}\nvoid foo(unknown_type x) {}", top_level_opts)
+        self.assertFalse(value.GetError().Success())
+        self.assertIn("\nvoid foo(unknown_type x) {}\n         ^\n", value.GetError().GetCString())
+        self.assertIn("<user expression 4>:2:10", value.GetError().GetCString())
+
+        # Test that we render Clang's 'notes' correctly.
+        value = frame.EvaluateExpression("struct SFoo{}; struct SFoo { int x; };", top_level_opts)
+        self.assertFalse(value.GetError().Success())
+        self.assertIn("<user expression 5>:1:8: previous definition is here\nstruct SFoo{}; struct SFoo { int x; };\n       ^\n", value.GetError().GetCString())
+
+        # Declarations from the debug information currently have no debug information. It's not clear what
+        # we should do in this case, but we should at least not print anything that's wrong.
+        # In the future our declarations should have valid source locations.
+        value = frame.EvaluateExpression("struct FooBar { double x };", top_level_opts)
+        self.assertFalse(value.GetError().Success())
+        self.assertEqual("error: <user expression 6>:1:8: redefinition of 'FooBar'\nstruct FooBar { double x };\n       ^\n", value.GetError().GetCString())
+
+        value = frame.EvaluateExpression("foo(1, 2)")
+        self.assertFalse(value.GetError().Success())
+        self.assertEqual("error: <user expression 7>:1:1: no matching function for call to 'foo'\nfoo(1, 2)\n^~~\nnote: candidate function not viable: requires single argument 'x', but 2 arguments were provided\n\n", value.GetError().GetCString())
+
+        # Redefine something that we defined in a user-expression. We should use the previous expression file name
+        # for the original decl.
+        value = frame.EvaluateExpression("struct Redef { double x; };", top_level_opts)
+        value = frame.EvaluateExpression("struct Redef { float y; };", top_level_opts)
+        self.assertFalse(value.GetError().Success())
+        self.assertIn("error: <user expression 9>:1:8: redefinition of 'Redef'\nstruct Redef { float y; };\n       ^\n<user expression 8>:1:8: previous definition is here\nstruct Redef { double x; };\n       ^", value.GetError().GetCString())
+
+    @skipUnlessDarwin
+    def test_source_locations_from_objc_modules(self):
+        self.build()
+
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+                                          '// Break here', self.main_source_spec)
+        frame = thread.GetFrameAtIndex(0)
+
+        # Import foundation so that the Obj-C module is loaded (which contains source locations
+        # that can be used by LLDB).
+        self.runCmd("expr @import Foundation")
+        value = frame.EvaluateExpression("NSLog(1);")
+        self.assertFalse(value.GetError().Success())
+        print(value.GetError().GetCString())
+        # LLDB should print the source line that defines NSLog. To not rely on any
+        # header paths/line numbers or the actual formatting of the Foundation headers, only look
+        # for a few tokens in the output.
+        # File path should come from Foundation framework.
+        self.assertIn("/Foundation.framework/", value.GetError().GetCString())
+        # The NSLog definition source line should be printed. Return value and
+        # the first argument are probably stable enough that this test can check for them.
+        self.assertIn("void NSLog(NSString *format", value.GetError().GetCString())
+
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/main.cpp b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/main.cpp
new file mode 100644
index 0000000000000..f4ad1ad220cf8
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/diagnostics/main.cpp
@@ -0,0 +1,11 @@
+void foo(int x) {}
+
+struct FooBar {
+  int i;
+};
+
+int main() {
+  FooBar f;
+  foo(1);
+  return 0; // Break here
+}
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/dont_allow_jit/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py
index 19e02ca84c992..de0d796505e77 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/basic/TestImportStdModule.py
@@ -11,12 +11,8 @@ class ImportStdModule(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
@@ -34,12 +30,8 @@ def test(self):
         self.expect("expr char char_a = 'b'; char char_b = 'a'; std::swap(char_a, char_b); char_a",
                     substrs=["(char) $3 = 'a'"])
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test_non_cpp_language(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
index 2abaece27fed7..ad5322baab007 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/conflicts/TestStdModuleWithConflicts.py
@@ -16,12 +16,8 @@ class TestImportStdModuleConflicts(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestBasicDeque.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestBasicDeque.py
index 0ede19e835088..4615d249bb1b7 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestBasicDeque.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-basic/TestBasicDeque.py
@@ -10,12 +10,8 @@ class TestBasicDeque(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDeque.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDeque.py
index eeea34819dd8c..cde74af8e1f43 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDeque.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/deque-dbg-info-content/TestDbgInfoContentDeque.py
@@ -10,12 +10,8 @@ class TestDbgInfoContentDeque(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/TestBasicForwardList.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/TestBasicForwardList.py
index 528d555e0af7d..9e45bf23c120c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/TestBasicForwardList.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-basic/TestBasicForwardList.py
@@ -10,12 +10,8 @@ class TestBasicForwardList(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardList.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardList.py
index a2fc559893068..290a0450fc8cd 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardList.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/forward_list-dbg-info-content/TestDbgInfoContentForwardList.py
@@ -10,12 +10,8 @@ class TestDbgInfoContentForwardList(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/TestBasicList.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/TestBasicList.py
index 154c8ed7bd0a3..e8dfc88473949 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/TestBasicList.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-basic/TestBasicList.py
@@ -10,12 +10,8 @@ class TestBasicList(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentList.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentList.py
index 713ec1abb85e1..fa8be7cabee69 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentList.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/list-dbg-info-content/TestDbgInfoContentList.py
@@ -11,12 +11,8 @@ class TestDbgInfoContentList(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
index f4d3d1fd49267..83d672fb1fdfb 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/no-std-module/TestMissingStdModule.py
@@ -17,12 +17,8 @@ class STLTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueue.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueue.py
index aad1b11719bda..42e5f8d8c4307 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueue.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/queue/TestQueue.py
@@ -10,12 +10,8 @@ class TestQueue(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContent.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContent.py
index 028798f72f3c6..ec39651b81509 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContent.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr-dbg-info-content/TestSharedPtrDbgInfoContent.py
@@ -10,12 +10,8 @@ class TestSharedPtrDbgInfoContent(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtr.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtr.py
index 6f1e7b12d1f86..7e05bcca4770f 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtr.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/shared_ptr/TestSharedPtr.py
@@ -10,12 +10,8 @@ class TestSharedPtr(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStack.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStack.py
index 1c2e5da54ed92..fe19a4e5f1b3a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStack.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/stack/TestStack.py
@@ -10,12 +10,8 @@ class TestStack(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile
index 459a66c5c8b3f..c1790dfbf767c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/Makefile
@@ -3,7 +3,7 @@
 # system headers.
 NO_TEST_COMMON_H := 1
 
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
-CXXFLAGS += -I $(SRCDIR)/root/usr/include/c++/include/ -I $(SRCDIR)/root/usr/include/ -nostdinc -nostdinc++ -nostdlib++
+CXXFLAGS_EXTRAS = -I $(SRCDIR)/root/usr/include/c++/include/ -I $(SRCDIR)/root/usr/include/ -nostdinc -nostdinc++
 CXX_SOURCES := main.cpp
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
index dfd90e80e4528..fbf00cddcfa92 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/sysroot/TestStdModuleSysroot.py
@@ -11,11 +11,7 @@ class ImportStdModule(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
index c321d8022e19f..9389de9712869 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr-dbg-info-content/TestUniquePtrDbgInfoContent.py
@@ -10,12 +10,8 @@ class TestUniquePtrDbgInfoContent(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtr.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtr.py
index 295ec51232762..a0e05b2c42027 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtr.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/unique_ptr/TestUniquePtr.py
@@ -10,12 +10,8 @@ class TestUniquePtr(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/TestBasicVector.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/TestBasicVector.py
index deaf8f0cb7538..840bacb86b30a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/TestBasicVector.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-basic/TestBasicVector.py
@@ -10,12 +10,8 @@ class TestBasicVector(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestBoolVector.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestBoolVector.py
index 0ab52695fafa4..45a2507f77651 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestBoolVector.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-bool/TestBoolVector.py
@@ -10,12 +10,8 @@ class TestBoolVector(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVector.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVector.py
index c89265f2cd337..4d51f1ba3b602 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVector.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-dbg-info-content/TestDbgInfoContentVector.py
@@ -11,12 +11,8 @@ class TestDbgInfoContentVector(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectors.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectors.py
index 2b82e1a039a35..feaeac9be5e07 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectors.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/vector-of-vectors/TestVectorOfVectors.py
@@ -10,12 +10,8 @@ class TestVectorOfVectors(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtr.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtr.py
index 0acd54e803cf8..5ed30312ea76c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtr.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr-dbg-info-content/TestDbgInfoContentWeakPtr.py
@@ -10,12 +10,8 @@ class TestDbgInfoContentWeakPtr(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile
index 12e489d83b2bc..f938f7428468a 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/Makefile
@@ -1,4 +1,3 @@
 USE_LIBCPP := 1
-CXXFLAGS += $(MANDATORY_CXXMODULE_BUILD_CFLAGS)
 CXX_SOURCES := main.cpp
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtr.py b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtr.py
index 6b8b9ceb7aab4..76241740b682c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtr.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import-std-module/weak_ptr/TestWeakPtr.py
@@ -10,12 +10,8 @@ class TestSharedPtr(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
-    # FIXME: This should work on more setups, so remove these
-    # skipIf's in the future.
     @add_test_categories(["libc++"])
     @skipIf(compiler=no_match("clang"))
-    @skipIf(oslist=no_match(["linux"]))
-    @skipIf(debug_info=no_match(["dwarf"]))
     def test(self):
         self.build()
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile
index 780b2660cd076..b46bd7f39a92b 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/import_builtin_fileid/Makefile
@@ -1,4 +1,3 @@
 OBJC_SOURCES := main.m
-
+LD_EXTRAS := -framework Cocoa
 include Makefile.rules
-LDFLAGS += -framework Cocoa
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile
index 9267f734396cb..4d12ff1b43e26 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/namespace_local_var_same_name_obj_c/Makefile
@@ -1,4 +1,5 @@
 OBJCXX_SOURCES := main.mm util.mm
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile
index add737b475224..69a2ad6b8d8b7 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/persist_objc_pointeetype/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
-LDFLAGS += -framework Foundation -framework CloudKit
 
+LD_EXTRAS := -framework Foundation -framework CloudKit
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile
index 2f36b673031d6..9e812e1cf7854 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/po_verbosity/Makefile
@@ -1,4 +1,5 @@
 OBJC_SOURCES := main.m
 
+
+LD_EXTRAS := -framework Foundation
 include Makefile.rules
-LDFLAGS += -framework Foundation
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile
index 4b46fe22a8b04..4eb417eeb57ac 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/scoped_enums/Makefile
@@ -1,4 +1,5 @@
 CXX_SOURCES := main.cpp
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
+
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile
index b8b1d12fe2ba8..c82383d9400cc 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/two-files/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m foo.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile b/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile
index 9786ff38a8461..c8b38907ac922 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/expression/weak_symbols/Makefile
@@ -1,4 +1,4 @@
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 LD_FLAGS := -dynamiclib
 include Makefile.rules
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py b/lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py
index 19fa3e9f0d882..ed3e9efb0a576 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/frame/language/TestGuessLanguage.py
@@ -20,6 +20,7 @@ class TestFrameGuessLanguage(TestBase):
     # each debug info format.
     NO_DEBUG_INFO_TESTCASE = True
 
+    @skipIf(compiler="clang", compiler_version=['<', '10.0'])
     @expectedFailureAll(oslist=["windows"], bugnumber="llvm.org/pr37658")
     def test_guess_language(self):
         """Test GuessLanguage for C and C++."""
@@ -74,12 +75,14 @@ def do_test(self):
         thread = threads[0]
 
         c_frame_language = lldb.eLanguageTypeC99
+        cxx_frame_language = lldb.eLanguageTypeC_plus_plus_11
         # gcc emits DW_LANG_C89 even if -std=c99 was specified
         if "gcc" in self.getCompiler():
             c_frame_language = lldb.eLanguageTypeC89
+            cxx_frame_language = lldb.eLanguageTypeC_plus_plus
 
         self.check_language(thread, 0, c_frame_language)
-        self.check_language(thread, 1, lldb.eLanguageTypeC_plus_plus)
+        self.check_language(thread, 1, cxx_frame_language)
         self.check_language(thread, 2, lldb.eLanguageTypeC_plus_plus)
 
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile b/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile
index b016328910eaf..8248c01fe6a8c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/Makefile
@@ -1,8 +1,9 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -g0 # No debug info.
+CFLAGS_EXTRAS := -g0 # No debug info.
 MAKE_DSYM := NO
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py b/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py
index a8a506745118e..548676c0a4cee 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/frame/recognizer/recognizer.py
@@ -7,12 +7,12 @@ def get_recognized_arguments(self, frame):
         if frame.name == "foo":
             arg1 = frame.EvaluateExpression("$arg1").signed
             arg2 = frame.EvaluateExpression("$arg2").signed
-            val1 = lldb.target.CreateValueFromExpression("a", "%d" % arg1)
-            val2 = lldb.target.CreateValueFromExpression("b", "%d" % arg2)
+            val1 = frame.GetThread().GetProcess().GetTarget().CreateValueFromExpression("a", "%d" % arg1)
+            val2 = frame.GetThread().GetProcess().GetTarget().CreateValueFromExpression("b", "%d" % arg2)
             return [val1, val2]
         elif frame.name == "bar":
             arg1 = frame.EvaluateExpression("$arg1").signed
-            val1 = lldb.target.CreateValueFromExpression("a", "(int *)%d" % arg1)
+            val1 = frame.GetThread().GetProcess().GetTarget().CreateValueFromExpression("a", "(int *)%d" % arg1)
             return [val1]
         return []
 
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py b/lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py
index c6b665ee94153..2f2dd5aa49309 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py
+++ b/lldb/packages/Python/lldbsuite/test/commands/frame/select/TestFrameSelect.py
@@ -36,3 +36,42 @@ def test_relative(self):
 
         self.expect("frame select -r 100")
         self.expect("frame select -r 1", error=True, substrs=["Already at the top of the stack."])
+
+    @no_debug_info_test
+    @skipIfWindows
+    def test_mixing_relative_and_abs(self):
+        self.build()
+
+        lldbutil.run_to_source_breakpoint(self,
+            "// Set break point at this line.", lldb.SBFileSpec("main.cpp"))
+
+        # The function associated with each frame index can change depending
+        # on the function calling main (e.g. `start`), so this only tests that
+        # the frame index number is correct. We test the actual functions
+        # in the relative test.
+
+        # Jump to the top of the stack.
+        self.expect("frame select 0", substrs=["frame #0"])
+
+        # Run some relative commands.
+        self.expect("up", substrs=["frame #1"])
+        self.expect("frame select -r 1", substrs=["frame #2"])
+        self.expect("frame select -r -1", substrs=["frame #1"])
+
+        # Test that absolute indices still work.
+        self.expect("frame select 2", substrs=["frame #2"])
+        self.expect("frame select 1", substrs=["frame #1"])
+        self.expect("frame select 3", substrs=["frame #3"])
+        self.expect("frame select 0", substrs=["frame #0"])
+        self.expect("frame select 1", substrs=["frame #1"])
+
+        # Run some other relative frame select commands.
+        self.expect("down", substrs=["frame #0"])
+        self.expect("frame select -r 1", substrs=["frame #1"])
+        self.expect("frame select -r -1", substrs=["frame #0"])
+
+        # Test that absolute indices still work.
+        self.expect("frame select 2", substrs=["frame #2"])
+        self.expect("frame select 1", substrs=["frame #1"])
+        self.expect("frame select 3", substrs=["frame #3"])
+        self.expect("frame select 0", substrs=["frame #0"])
diff --git a/lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile b/lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/frame/var/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile b/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile
index 202f88e410e36..5cc7382f1d92f 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -mmpx -fcheck-pointer-bounds -fuse-ld=bfd
+CFLAGS_EXTRAS := -mmpx -fcheck-pointer-bounds -fuse-ld=bfd
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile b/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
index 202f88e410e36..5cc7382f1d92f 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/register/register/intel_xtended_registers/mpx_bound_violation/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -mmpx -fcheck-pointer-bounds -fuse-ld=bfd
+CFLAGS_EXTRAS := -mmpx -fcheck-pointer-bounds -fuse-ld=bfd
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile b/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/commands/target/stop-hooks/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py b/lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py
index 01e11fc096ce0..41c6019adfbcb 100644
--- a/lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py
+++ b/lldb/packages/Python/lldbsuite/test/driver/batch_mode/TestBatchMode.py
@@ -78,6 +78,35 @@ def test_batch_mode_run_exit(self):
         import pexpect
         child.expect(pexpect.EOF)
 
+    @expectedFlakeyFreeBSD("llvm.org/pr25172 fails rarely on the buildbot")
+    def test_batch_mode_launch_stop_at_entry(self):
+        """Test that the lldb driver's batch mode works correctly for process launch."""
+        self.build()
+
+        exe = self.getBuildArtifact("a.out")
+
+        # Launch with the option '--stop-at-entry' stops with a signal (usually SIGSTOP)
+        # that should be suppressed since it doesn't imply a crash and
+        # this is not a reason to exit batch mode.
+        extra_args = ['-b',
+            '-o', 'process launch --stop-at-entry',
+            '-o', 'continue',
+        ]
+        self.launch(executable=exe, extra_args=extra_args)
+        child = self.child
+
+        # Check that the process has been launched:
+        child.expect("Process ([0-9]+) launched:")
+        # We should have continued:
+        child.expect_exact("continue")
+        # The App should have not have crashed:
+        child.expect_exact("Got there on time and it did not crash.")
+        
+        # Then lldb should exit.
+        child.expect_exact("exited")
+        import pexpect
+        child.expect(pexpect.EOF)
+
     def closeVictim(self):
         if self.victim is not None:
             self.victim.close()
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/address_breakpoints/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/auto_continue/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
index b6309174c4c71..ad42b20df4ed5 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_by_line_and_column/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99 -gcolumn-info
+CFLAGS_EXTRAS := -std=c99 -gcolumn-info
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_conditions/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile
index 56bf159ec0729..2c00681fa2280 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_ids/Makefile
@@ -1,7 +1,8 @@
 CXX_SOURCES := main.cpp
 
 ifneq (,$(findstring icc,$(CC)))
-    CXXFLAGS += -debug inline-debug-info
+    CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile
index f25c0ea8055c0..9645fd9cc8dfb 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/breakpoint_locations/Makefile
@@ -1,7 +1,7 @@
 C_SOURCES := main.c
 
 ifneq (,$(findstring icc,$(CC)))
-    CFLAGS += -debug inline-debug-info
+    CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile
index 56bf159ec0729..2c00681fa2280 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/consecutive_breakpoints/Makefile
@@ -1,7 +1,8 @@
 CXX_SOURCES := main.cpp
 
 ifneq (,$(findstring icc,$(CC)))
-    CXXFLAGS += -debug inline-debug-info
+    CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile
index 56bf159ec0729..2c00681fa2280 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/cpp/Makefile
@@ -1,7 +1,8 @@
 CXX_SOURCES := main.cpp
 
 ifneq (,$(findstring icc,$(CC)))
-    CXXFLAGS += -debug inline-debug-info
+    CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile
index f25c0ea8055c0..9645fd9cc8dfb 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/dummy_target_breakpoints/Makefile
@@ -1,7 +1,7 @@
 C_SOURCES := main.c
 
 ifneq (,$(findstring icc,$(CC)))
-    CFLAGS += -debug inline-debug-info
+    CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/objc/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile
index f25c0ea8055c0..9645fd9cc8dfb 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/require_hw_breakpoints/Makefile
@@ -1,7 +1,7 @@
 C_SOURCES := main.c
 
 ifneq (,$(findstring icc,$(CC)))
-    CFLAGS += -debug inline-debug-info
+    CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/scripted_bkpt/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile
index 56bf159ec0729..2c00681fa2280 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/breakpoint/step_over_breakpoint/Makefile
@@ -1,7 +1,8 @@
 CXX_SOURCES := main.cpp
 
 ifneq (,$(findstring icc,$(CC)))
-    CXXFLAGS += -debug inline-debug-info
+    CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py b/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py
index df975345c1cbf..efd80f93754c8 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/completion/TestCompletion.py
@@ -101,6 +101,27 @@ def test_ambiguous_long_opt(self):
     def test_plugin_load(self):
         self.complete_from_to('plugin load ', [])
 
+    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
+    def test_log_enable(self):
+        self.complete_from_to('log enable ll', ['lldb'])
+        self.complete_from_to('log enable dw', ['dwarf'])
+        self.complete_from_to('log enable lldb al', ['all'])
+        self.complete_from_to('log enable lldb sym', ['symbol'])
+
+    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
+    def test_log_enable(self):
+        self.complete_from_to('log disable ll', ['lldb'])
+        self.complete_from_to('log disable dw', ['dwarf'])
+        self.complete_from_to('log disable lldb al', ['all'])
+        self.complete_from_to('log disable lldb sym', ['symbol'])
+
+    @skipIfFreeBSD  # timing out on the FreeBSD buildbot
+    def test_log_list(self):
+        self.complete_from_to('log list ll', ['lldb'])
+        self.complete_from_to('log list dw', ['dwarf'])
+        self.complete_from_to('log list ll', ['lldb'])
+        self.complete_from_to('log list lldb dwa', ['dwarf'])
+
     @skipIfFreeBSD  # timing out on the FreeBSD buildbot
     def test_quoted_command(self):
         self.complete_from_to('"set',
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile
index 377a1a0373a0d..876340159d9c8 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/boolreference/Makefile
@@ -1,7 +1,8 @@
 OBJCXX_SOURCES := main.mm
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile
index 86bbe461026ec..a537b994079ee 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/compactvectors/Makefile
@@ -1,5 +1,4 @@
 CXX_SOURCES := main.cpp
-
+LD_EXTRAS := -framework Accelerate
 include Makefile.rules
 
-LDFLAGS += -framework Accelerate
\ No newline at end of file
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
index 8615fe65dd577..3390da4bf3fd9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-advanced/TestDataFormatterAdv.py
@@ -136,6 +136,13 @@ def cleanup():
         self.expect("frame variable int_array",
                     substrs=['1,2'])
 
+        # Test the patterns are matched in reverse-chronological order.
+        self.runCmd(
+            'type summary add --summary-string \"${var[2-3]}\" "int []"')
+
+        self.expect("frame variable int_array",
+                    substrs=['3,4'])
+
         self.runCmd("type summary clear")
 
         self.runCmd("type summary add -c -x \"i_am_cool \[[0-9]\]\"")
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsindexpath/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-objc/nsstring/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-proper-plurals/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile
index 720d1b1ca5afb..6bfb97cd2c2bc 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-skip-summary/Makefile
@@ -1,6 +1,7 @@
 CXX_SOURCES := main.cpp
 USE_LIBSTDCPP := 0
 
-include Makefile.rules
 
-CXXFLAGS += -O0
+
+CXXFLAGS_EXTRAS := -O0
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
index 22307c41c1c32..b016f006747da 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/atomic/Makefile
@@ -1,4 +1,5 @@
 CXX_SOURCES := main.cpp
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 USE_LIBCPP := 1
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
index 22307c41c1c32..b016f006747da 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/function/Makefile
@@ -1,4 +1,5 @@
 CXX_SOURCES := main.cpp
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 USE_LIBCPP := 1
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
index 29c68ae574016..e78030cbf7528 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/initializerlist/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/iterator/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/list/loop/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
index 3e127306e94c0..64c1f301865da 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/map/TestDataFormatterLibccMap.py
@@ -52,13 +52,26 @@ def cleanup():
         self.addTearDownHook(cleanup)
 
         ns = self.namespace
-        self.expect('frame variable ii',
+        self.expect('p ii',
+                    substrs=['%s::map' % ns,
+                             'size=0',
+                             '{}'])
+        self.expect('frame var ii',
                     substrs=['%s::map' % ns,
                              'size=0',
                              '{}'])
 
         lldbutil.continue_to_breakpoint(self.process(), bkpt)
 
+        self.expect('p ii',
+                    substrs=['%s::map' % ns, 'size=2',
+                             '[0] = ',
+                             'first = 0',
+                             'second = 0',
+                             '[1] = ',
+                             'first = 1',
+                             'second = 1'])
+
         self.expect('frame variable ii',
                     substrs=['%s::map' % ns, 'size=2',
                              '[0] = ',
@@ -81,7 +94,7 @@ def cleanup():
 
         lldbutil.continue_to_breakpoint(self.process(), bkpt)
 
-        self.expect("frame variable ii",
+        self.expect("p ii",
                     substrs=['%s::map' % ns, 'size=8',
                              '[5] = ',
                              'first = 5',
@@ -90,7 +103,7 @@ def cleanup():
                              'first = 7',
                              'second = 1'])
 
-        self.expect("p ii",
+        self.expect("frame var ii",
                     substrs=['%s::map' % ns, 'size=8',
                              '[5] = ',
                              'first = 5',
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multimap/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/multiset/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
index 5ad5241858d80..23496eb206572 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/optional/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -std=c++17 -fno-exceptions
 include Makefile.rules
-CXXFLAGS += -std=c++17 -fno-exceptions
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/set/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
index 8b7b61c59c094..c7c91da728d1e 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -std=c++11 -O0
 include Makefile.rules
-CXXFLAGS += -std=c++11 -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
index e2048e9e99c64..f36876ad51af9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/TestDataFormatterLibcxxString.py
@@ -59,16 +59,23 @@ def cleanup():
         self.expect(
             "frame variable",
             substrs=[
+                '(%s::wstring) wempty = L""'%ns,
                 '(%s::wstring) s = L"hello world! מזל טוב!"'%ns,
                 '(%s::wstring) S = L"!!!!"'%ns,
                 '(const wchar_t *) mazeltov = 0x',
                 'L"מזל טוב"',
+                '(%s::string) empty = ""'%ns,
                 '(%s::string) q = "hello world"'%ns,
                 '(%s::string) Q = "quite a long std::strin with lots of info inside it"'%ns,
                 '(%s::string) IHaveEmbeddedZeros = "a\\0b\\0c\\0d"'%ns,
                 '(%s::wstring) IHaveEmbeddedZerosToo = L"hello world!\\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監"'%ns,
                 '(%s::u16string) u16_string = u"ß水氶"'%ns,
-                '(%s::u32string) u32_string = U"🍄🍅🍆🍌"'%ns])
+                # FIXME: This should have a 'u' prefix.
+                '(%s::u16string) u16_empty = ""'%ns,
+                '(%s::u32string) u32_string = U"🍄🍅🍆🍌"'%ns,
+                # FIXME: This should have a 'U' prefix.
+                '(%s::u32string) u32_empty = ""'%ns
+])
 
         self.runCmd("n")
 
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
index 838a4c71be1ab..ec9b8170b2f00 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/string/main.cpp
@@ -2,16 +2,20 @@
 
 int main()
 {
+    std::wstring wempty(L"");
     std::wstring s(L"hello world! מזל טוב!");
     std::wstring S(L"!!!!");
     const wchar_t *mazeltov = L"מזל טוב";
+    std::string empty("");
     std::string q("hello world");
     std::string Q("quite a long std::strin with lots of info inside it");
     std::string TheVeryLongOne("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890someText1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890");
     std::string IHaveEmbeddedZeros("a\0b\0c\0d",7);
     std::wstring IHaveEmbeddedZerosToo(L"hello world!\0てざ ル゜䋨ミ㠧槊 きゅへ狦穤襩 じゃ馩リョ 䤦監", 38);
     std::u16string u16_string(u"ß水氶");
+    std::u16string u16_empty(u"");
     std::u32string u32_string(U"🍄🍅🍆🍌");
+    std::u32string u32_empty(U"");
     S.assign(L"!!!!!"); // Set break point at this line.
     return 0;
 }
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
index 442886ce40f9f..913a52fb191c1 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/unordered/Makefile
@@ -3,8 +3,7 @@ CXX_SOURCES := main.cpp
 # Work around "exception specification in declaration does not match previous
 # declaration" errors present in older libc++ releases. This error was fixed in
 # the 3.8 release.
-CFLAGS_EXTRAS += -fno-exceptions
+CFLAGS_EXTRAS := -fno-exceptions
 
 USE_LIBCPP := 1
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
index 3fc4d80271867..7eeff7407804d 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/variant/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -std=c++17
 include Makefile.rules
-CXXFLAGS += -std=c++17
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
index c7df6a4cbf9b8..d87cf7d402787 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vbool/Makefile
@@ -1,5 +1,4 @@
 CXX_SOURCES := main.cpp
 USE_LIBCPP := 1
 include Makefile.rules
-CXXFLAGS += -O0
 
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
index 4a33784693a21..564cbada74e08 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libcxx/vector/Makefile
@@ -1,5 +1,6 @@
 CXX_SOURCES := main.cpp
 
 USE_LIBCPP := 1
+
+CXXFLAGS_EXTRAS := -O0
 include Makefile.rules
-CXXFLAGS += -O0
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
index 7d43aa33e9ae7..c825977b1a5dc 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/iterator/Makefile
@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -O0
+CFLAGS_EXTRAS := -O0
 USE_LIBSTDCPP := 1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
index 7d43aa33e9ae7..c825977b1a5dc 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/list/Makefile
@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -O0
+CFLAGS_EXTRAS := -O0
 USE_LIBSTDCPP := 1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
index 7d43aa33e9ae7..c825977b1a5dc 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/Makefile
@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -O0
+CFLAGS_EXTRAS := -O0
 USE_LIBSTDCPP := 1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
index 183aabe649352..0adbaa91d6f96 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/TestDataFormatterStdString.py
@@ -52,12 +52,16 @@ def cleanup():
         # Execute the cleanup function during test case tear down.
         self.addTearDownHook(cleanup)
 
+        var_wempty = self.frame().FindVariable('wempty')
         var_s = self.frame().FindVariable('s')
         var_S = self.frame().FindVariable('S')
         var_mazeltov = self.frame().FindVariable('mazeltov')
+        var_empty = self.frame().FindVariable('empty')
         var_q = self.frame().FindVariable('q')
         var_Q = self.frame().FindVariable('Q')
 
+        # TODO: This is currently broken
+        # self.assertTrue(var_wempty.GetSummary() == 'L""', "wempty summary wrong")
         self.assertTrue(
             var_s.GetSummary() == 'L"hello world! מזל טוב!"',
             "s summary wrong")
@@ -65,6 +69,7 @@ def cleanup():
         self.assertTrue(
             var_mazeltov.GetSummary() == 'L"מזל טוב"',
             "mazeltov summary wrong")
+        self.assertTrue(var_empty.GetSummary() == '""', "empty summary wrong")
         self.assertTrue(
             var_q.GetSummary() == '"hello world"',
             "q summary wrong")
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
index f6e56cf12456f..6227a4d493a53 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/string/main.cpp
@@ -2,9 +2,11 @@
 
 int main()
 {
+    std::wstring wempty(L"");
     std::wstring s(L"hello world! מזל טוב!");
     std::wstring S(L"!!!!");
     const wchar_t *mazeltov = L"מזל טוב";
+    std::string empty("");
     std::string q("hello world");
     std::string Q("quite a long std::strin with lots of info inside it");
     S.assign(L"!!!!!"); // Set break point at this line.
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
index 7d43aa33e9ae7..c825977b1a5dc 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/data-formatter-stl/libstdcpp/vbool/Makefile
@@ -1,6 +1,6 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -O0
+CFLAGS_EXTRAS := -O0
 USE_LIBSTDCPP := 1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsarraysynth/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nsdictionarysynth/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/nssetsynth/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile
index 377a1a0373a0d..876340159d9c8 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ostypeformatting/Makefile
@@ -1,7 +1,8 @@
 OBJCXX_SOURCES := main.mm
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile
index 377a1a0373a0d..876340159d9c8 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/poarray/Makefile
@@ -1,7 +1,8 @@
 OBJCXX_SOURCES := main.mm
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile
index 348edcd88b443..83eb77ec8a94b 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/ptr_ref_typedef/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
 
-CFLAGS_EXTRAS += -std=c++11
+CFLAGS_EXTRAS := -std=c++11
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile
index 9048c7205237e..3767ff17e35bb 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/data-formatter/synthupdate/Makefile
@@ -1,10 +1,8 @@
 OBJC_SOURCES := main.m
-
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+LD_EXTRAS := -framework Foundation
 
 include Makefile.rules
 
-LDFLAGS += -framework Foundation
-
 clean::
 	rm -rf $(wildcard *.o *.d *.dSYM *.log)
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile
index 62c0a39c94a44..eb07d2fbdf3bd 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/inferior-crashing/recursive-inferior/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := main.c
 
-CFLAGS_EXTRAS += -fomit-frame-pointer
+CFLAGS_EXTRAS := -fomit-frame-pointer
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile
index d94874a7a05ca..362b89d7f995b 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/inline-stepping/Makefile
@@ -1,7 +1,8 @@
 CXX_SOURCES := calling.cpp
 
 ifneq (,$(findstring icc,$(CC)))
-    CXXFLAGS += -debug inline-debug-info
+    CXXFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/Makefile
new file mode 100644
index 0000000000000..afecbf969483e
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/Makefile
@@ -0,0 +1,4 @@
+OBJC_SOURCES := main.m
+LD_EXTRAS := -lobjc -framework Foundation
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/TestBasicObjcModernTypeLookup.py b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/TestBasicObjcModernTypeLookup.py
new file mode 100644
index 0000000000000..700b3ecbe6bb4
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/TestBasicObjcModernTypeLookup.py
@@ -0,0 +1,18 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestBasicObjcModernTypeLookup(TestBase):
+  mydir = TestBase.compute_mydir(__file__)
+
+  @skipUnlessDarwin
+  def test(self):
+    self.build()
+    # Activate modern-type-lookup.
+    # FIXME: This has to happen before we create any target otherwise we crash...
+    self.runCmd("settings set target.experimental.use-modern-type-lookup true")
+    (target, process, thread, main_breakpoint) = lldbutil.run_to_source_breakpoint(self,
+          "break here", lldb.SBFileSpec("main.m"))
+    self.expect("expr 1", substrs=["(int) ", " = 1"])
+    self.expect("expr (int)[Foo bar:22]", substrs=["(int) ", " = 44"])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/main.m b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/main.m
new file mode 100644
index 0000000000000..b427971e7bea0
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic-objc/main.m
@@ -0,0 +1,17 @@
+#import <Foundation/Foundation.h>
+
+@interface Foo : NSObject
++(int) bar: (int) string;
+@end
+
+@implementation Foo
++(int) bar: (int) x
+{
+  return x + x;
+}
+@end
+
+int main() {
+  NSLog(@"Hello World");
+  return 0; // break here
+}
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/Makefile
new file mode 100644
index 0000000000000..3d0b98f13f3d7
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/Makefile
@@ -0,0 +1,2 @@
+CXX_SOURCES := main.cpp
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/TestBasicModernTypeLookup.py b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/TestBasicModernTypeLookup.py
new file mode 100644
index 0000000000000..613fb95658bff
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/TestBasicModernTypeLookup.py
@@ -0,0 +1,21 @@
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class BasicModernTypeLookup(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    def test(self):
+        self.build()
+
+        # Activate modern-type-lookup.
+        self.runCmd("settings set target.experimental.use-modern-type-lookup true")
+
+        lldbutil.run_to_source_breakpoint(self,
+            "// Set break point at this line.", lldb.SBFileSpec("main.cpp"))
+
+        # Test a few simple expressions that should still work with modern-type-lookup.
+        self.expect("expr 1", substrs=["(int) ", " = 1\n"])
+        self.expect("expr f.x", substrs=["(int) ", " = 44\n"])
+        self.expect("expr f", substrs=["(Foo) ", "x = 44"])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/main.cpp
new file mode 100644
index 0000000000000..e0c3ead7f757b
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/basic/main.cpp
@@ -0,0 +1,7 @@
+struct Foo { int x; };
+
+int main(int argc, char **argv) {
+  Foo f;
+  f.x = 44;
+  return f.x; // Set break point at this line.
+}
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/Makefile
new file mode 100644
index 0000000000000..574c53d7cb1c5
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp
+USE_LIBCPP := 1
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/TestLibCxxModernTypeLookup.py b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/TestLibCxxModernTypeLookup.py
new file mode 100644
index 0000000000000..a33c75494ece2
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/TestLibCxxModernTypeLookup.py
@@ -0,0 +1,20 @@
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class LibcxxModernTypeLookup(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @add_test_categories(["libc++"])
+    def test(self):
+        self.build()
+
+        # Activate modern-type-lookup.
+        self.runCmd("settings set target.experimental.use-modern-type-lookup true")
+
+        lldbutil.run_to_source_breakpoint(self,
+            "// Set break point at this line.", lldb.SBFileSpec("main.cpp"))
+
+        # Test a few simple expressions that should still work with modern-type-lookup.
+        self.expect("expr pair", substrs=["(std::", "pair<int, long", "= (first = 1, second = 2)"])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/main.cpp
new file mode 100644
index 0000000000000..6db6160ba6215
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/libcxx/main.cpp
@@ -0,0 +1,6 @@
+#include <utility>
+
+int main(int argc, char **argv) {
+  std::pair<int, long> pair = std::make_pair<int, float>(1, 2L);
+  return pair.first; // Set break point at this line.
+}
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/Makefile
new file mode 100644
index 0000000000000..afecbf969483e
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/Makefile
@@ -0,0 +1,4 @@
+OBJC_SOURCES := main.m
+LD_EXTRAS := -lobjc -framework Foundation
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/TestObjModulesModernTypeLookup.py b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/TestObjModulesModernTypeLookup.py
new file mode 100644
index 0000000000000..3983263bd65b2
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/TestObjModulesModernTypeLookup.py
@@ -0,0 +1,26 @@
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+class TestObjcModulesModernTypeLookup(TestBase):
+  mydir = TestBase.compute_mydir(__file__)
+
+  @skipUnlessDarwin
+  # gmodules causes this to crash as we seem to get a NSURL type from the debug information.
+  @skipIf(debug_info="gmodules")
+  def test(self):
+    self.build()
+    # Activate modern-type-lookup.
+    # FIXME: This has to happen before we create any target otherwise we crash...
+    self.runCmd("settings set target.experimental.use-modern-type-lookup true")
+    (target, process, thread, main_breakpoint) = lldbutil.run_to_source_breakpoint(self,
+          "break here", lldb.SBFileSpec("main.m"))
+    self.expect("expr @import Foundation")
+    self.expect(
+            "p *[NSURL URLWithString:@\"http://lldb.llvm.org\"]",
+            VARIABLES_DISPLAYED_CORRECTLY,
+            substrs=[
+                "NSURL",
+                "isa",
+                "_urlString"])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/main.m b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/main.m
new file mode 100644
index 0000000000000..9eb93f0e8c5fb
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/modern-type-lookup/objc-modules/main.m
@@ -0,0 +1,6 @@
+#import <Foundation/Foundation.h>
+
+int main() {
+  NSLog(@"Hello World");
+  return 0; // break here
+}
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile
index 5a7351146d241..44093101a5076 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/mtc/simple/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation -framework AppKit
+LD_EXTRAS := -lobjc -framework Foundation -framework AppKit
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/multiword-commands/TestMultiWordCommands.py b/lldb/packages/Python/lldbsuite/test/functionalities/multiword-commands/TestMultiWordCommands.py
new file mode 100644
index 0000000000000..80b144f58d573
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/multiword-commands/TestMultiWordCommands.py
@@ -0,0 +1,33 @@
+"""
+Test multiword commands ('platform' in this case).
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+
+class MultiwordCommandsTestCase(TestBase):
+
+    mydir = TestBase.compute_mydir(__file__)
+
+    @no_debug_info_test
+    def test_ambiguous_subcommand(self):
+        self.expect("platform s", error=True,
+                    substrs=["ambiguous command 'platform s'. Possible completions:",
+                             "\tselect\n",
+                             "\tshell\n",
+                             "\tsettings\n"])
+
+    @no_debug_info_test
+    def test_empty_subcommand(self):
+        self.expect("platform \"\"", error=True, substrs=["Need to specify a non-empty subcommand."])
+
+    @no_debug_info_test
+    def test_help(self):
+        # <multiword> help brings up help.
+        self.expect("platform help",
+                    substrs=["Commands to manage and create platforms.",
+                             "Syntax: platform [",
+                             "The following subcommands are supported:",
+                             "connect",
+                             "Select the current platform"])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
index 0c26fc7269707..1adf3fc44a694 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/Makefile
@@ -1,4 +1,5 @@
 LEVEL = ../../../make
 CXX_SOURCES := main.cpp
 include $(LEVEL)/Makefile.rules
-CXXFLAGS += -O1 -glldb -Xclang -femit-debug-entry-values
+CXXFLAGS_EXTRAS := -O1 -glldb -Xclang -femit-debug-entry-values
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
index 5b5099bff5d25..ad50f12c6b827 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/TestBasicEntryValuesX86_64.py
@@ -1,8 +1,12 @@
 from lldbsuite.test import lldbinline
 from lldbsuite.test import decorators
+from lldbsuite.test import lldbplatformutil
+
+supported_platforms = ["linux"]
+supported_platforms.extend(lldbplatformutil.getDarwinOSTriples())
 
 lldbinline.MakeInlineTest(__file__, globals(),
-        [decorators.skipUnlessDarwin,
+        [decorators.skipUnlessPlatform(supported_platforms),
          decorators.skipUnlessArch('x86_64'),
          decorators.skipUnlessHasCallSiteInfo,
          decorators.skipIf(dwarf_version=['<', '4'])])
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
index 6efd5708a73cf..5a38376b680e9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/param_entry_vals/basic_entry_values_x86_64/main.cpp
@@ -31,7 +31,7 @@ void func1(int &sink, int x) {
   DESTROY_RSI;
 
   //% self.filecheck("image lookup -va $pc", "main.cpp", "-check-prefix=FUNC1-DESC")
-  // FUNC1-DESC: name = "x", type = "int", location = DW_OP_entry_value( rsi)
+  // FUNC1-DESC: name = "x", type = "int", location = DW_OP_entry_value(DW_OP_reg4 RSI)
 
   ++sink;
 }
@@ -152,7 +152,10 @@ int main() {
   func2(sink, 123);
 
   // Test evaluation of "DW_OP_fbreg -24, DW_OP_deref" in the parent frame.
+  // Disabled for now, see: llvm.org/PR43343
+#if 0
   func3(sink, s1.field2);
+#endif
 
   // The sequences `main -> func4 -> func{5,6}_amb -> sink` are both plausible.
   // Test that lldb doesn't attempt to guess which one occurred: entry value
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/set-data/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py
index 1383a03f4647d..f93559af7365e 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/Steps.py
@@ -35,3 +35,38 @@ def __init__(self, thread_plan, dict):
 
     def queue_child_thread_plan(self):
         return self.thread_plan.QueueThreadPlanForStepScripted("Steps.StepOut")
+
+# This plan does a step-over until a variable changes value.
+class StepUntil(StepWithChild):
+    def __init__(self, thread_plan, dict):
+        self.frame = thread_plan.GetThread().frames[0]
+        self.target = thread_plan.GetThread().GetProcess().GetTarget()
+        self.value = self.frame.FindVariable("foo")
+        StepWithChild.__init__(self, thread_plan)
+
+    def queue_child_thread_plan(self):
+        le = self.frame.GetLineEntry()
+        start_addr = le.GetStartAddress() 
+        start = start_addr.GetLoadAddress(self.target)
+        end = le.GetEndAddress().GetLoadAddress(self.target)
+        print("Stepping from 0x%x to 0x%x (0x%x)"%(start, end, end - start))
+        return self.thread_plan.QueueThreadPlanForStepOverRange(start_addr,
+                                                                end - start)
+
+    def should_stop(self, event):
+        if not self.child_thread_plan.IsPlanComplete():
+            return False
+
+        # If we've stepped out of this frame, stop.
+        if not self.frame.IsValid():
+            return True
+
+        if not self.value.IsValid():
+            return True
+
+        print("Got next value: %d"%(self.value.GetValueAsUnsigned()))
+        if not self.value.GetValueDidChange():
+            self.child_thread_plan = self.queue_child_thread_plan()
+            return False
+        else:
+            return True
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py
index a111ede6739c9..cadadde96778d 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/TestStepScripted.py
@@ -12,24 +12,27 @@ class StepScriptedTestCase(TestBase):
 
     NO_DEBUG_INFO_TESTCASE = True
 
+    def setUp(self):
+        TestBase.setUp(self)
+        self.main_source_file = lldb.SBFileSpec("main.c")
+        self.runCmd("command script import Steps.py")
+
     def test_standard_step_out(self):
-        """Tests stepping with the scripted thread plan laying over a standard thread plan for stepping out."""
+        """Tests stepping with the scripted thread plan laying over a standard 
+        thread plan for stepping out."""
         self.build()
-        self.main_source_file = lldb.SBFileSpec("main.c")
         self.step_out_with_scripted_plan("Steps.StepOut")
 
     def test_scripted_step_out(self):
-        """Tests stepping with the scripted thread plan laying over an another scripted thread plan for stepping out."""
+        """Tests stepping with the scripted thread plan laying over an another 
+        scripted thread plan for stepping out."""
         self.build()
-        self.main_source_file = lldb.SBFileSpec("main.c")
         self.step_out_with_scripted_plan("Steps.StepScripted")
 
-    def setUp(self):
-        TestBase.setUp(self)
-        self.runCmd("command script import Steps.py")
-
     def step_out_with_scripted_plan(self, name):
-        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self, "Set a breakpoint here", self.main_source_file)
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+                                                                            "Set a breakpoint here",
+                                                                            self.main_source_file)
 
         frame = thread.GetFrameAtIndex(0)
         self.assertEqual("foo", frame.GetFunctionName())
@@ -39,3 +42,47 @@ def step_out_with_scripted_plan(self, name):
 
         frame = thread.GetFrameAtIndex(0)
         self.assertEqual("main", frame.GetFunctionName())
+
+
+    def test_misspelled_plan_name(self):
+        """Test that we get a useful error if we misspell the plan class name"""
+        self.build()
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+                                                                            "Set a breakpoint here",
+                                                                            self.main_source_file)
+        stop_id = process.GetStopID()
+        # Pass a non-existent class for the plan class:
+        err = thread.StepUsingScriptedThreadPlan("NoSuchModule.NoSuchPlan")
+        
+        # Make sure we got a good error:
+        self.assertTrue(err.Fail(), "We got a failure state")
+        msg = err.GetCString()
+        self.assertTrue("NoSuchModule.NoSuchPlan" in msg, "Mentioned missing class")
+        
+        # Make sure we didn't let the process run:
+        self.assertEqual(stop_id, process.GetStopID(), "Process didn't run")
+        
+    def test_checking_variable(self):
+        """Test that we can call SBValue API's from a scripted thread plan"""
+        self.build()
+        (target, process, thread, bkpt) = lldbutil.run_to_source_breakpoint(self,
+                                                                            "Set a breakpoint here",
+                                                                            self.main_source_file)
+
+        frame = thread.GetFrameAtIndex(0)
+        self.assertEqual("foo", frame.GetFunctionName())
+        foo_val = frame.FindVariable("foo")
+        self.assertTrue(foo_val.GetError().Success(), "Got the foo variable")
+        self.assertEqual(foo_val.GetValueAsUnsigned(), 10, "foo starts at 10")
+
+        err = thread.StepUsingScriptedThreadPlan("Steps.StepUntil")
+        self.assertTrue(err.Success(), err.GetCString())
+
+        # We should not have exited:
+        self.assertEqual(process.GetState(), lldb.eStateStopped, "We are stopped")
+        
+        # We should still be in foo:
+        self.assertEqual("foo", frame.GetFunctionName())
+
+        # And foo should have changed:
+        self.assertTrue(foo_val.GetValueDidChange(), "Foo changed")
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c
index 88b3c17125db0..bfd8a35d55626 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/step_scripted/main.c
@@ -1,7 +1,10 @@
 #include <stdio.h>
 
 void foo() {
-  printf("Set a breakpoint here.\n");
+  int foo = 10; 
+  printf("%d\n", foo); // Set a breakpoint here. 
+  foo = 20;
+  printf("%d\n", foo);
 }
 
 int main() {
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq1/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/ambiguous_tail_call_seq2/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_call_site/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_paths_to_common_sink/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/disambiguate_tail_call_seq/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/inlining_and_tail_calls/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/sbapi_support/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_message/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/thread_step_out_or_return/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile
index 41c46c6626b3e..48342e8e3afb9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/tail_call_frames/unambiguous_sequence/Makefile
@@ -1,3 +1,4 @@
 CXX_SOURCES := main.cpp
+
+CXXFLAGS_EXTRAS := -g -O1 -glldb
 include Makefile.rules
-CXXFLAGS += -g -O1 -glldb
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile
index 78f60047cd787..cd092b7725618 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_all/Makefile
@@ -1,4 +1,5 @@
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 CXX_SOURCES := ParallelTask.cpp
 ENABLE_THREADS := YES
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile
index 1e84d63f106af..5fd720c6d064c 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/backtrace_limit/Makefile
@@ -1,4 +1,5 @@
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 CXX_SOURCES := main.cpp
 ENABLE_THREADS := YES
+
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
index 9814c0ba9ad55..0ceaed937a937 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyBreakpoints.py
@@ -15,6 +15,7 @@ class ConcurrentManyBreakpoints(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
+    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 breakpoints from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
index c4e8f11b8233d..945b1b9a3405d 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyCrash.py
@@ -15,6 +15,7 @@ class ConcurrentManyCrash(ConcurrentEventsBase):
     # Atomic sequences are not supported yet for MIPS in LLDB.
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
+    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 threads that cause a segfault."""
         self.build(dictionary=self.getBuildFlags())
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
index ca179cf4ca980..35e4bb3e83f9e 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManySignals.py
@@ -17,6 +17,7 @@ class ConcurrentManySignals(ConcurrentEventsBase):
     # This test is flaky on Darwin.
     @skipIfDarwin
     @expectedFailureNetBSD
+    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 signals from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
index 25a40d2c768db..cce537ac77ae9 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/thread/concurrent_events/TestConcurrentManyWatchpoints.py
@@ -16,6 +16,7 @@ class ConcurrentManyWatchpoints(ConcurrentEventsBase):
     @skipIf(triple='^mips')
     @expectedFailureNetBSD
     @add_test_categories(["watchpoint"])
+    @skipIfOutOfTreeDebugserver
     def test(self):
         """Test 100 watchpoints from 100 threads."""
         self.build(dictionary=self.getBuildFlags())
diff --git a/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile b/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile
index 377a1a0373a0d..876340159d9c8 100644
--- a/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/functionalities/type_lookup/Makefile
@@ -1,7 +1,8 @@
 OBJCXX_SOURCES := main.mm
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile b/lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile
index a90dc493b8c38..57083c9527456 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/c/blocks/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -fblocks
+CFLAGS_EXTRAS := -fblocks
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile b/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile
index 47b3ca4be66a0..325be90d17ed0 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/c/const_variables/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := main.c functions.c
 
-CFLAGS_EXTRAS += -O3
+CFLAGS_EXTRAS := -O3
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile b/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile
index ae2e4c45e1162..d3998eeef9951 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/c/local_variables/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := main.c
 
-CFLAGS_EXTRAS += -O1
+CFLAGS_EXTRAS := -O1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile b/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile
index 569979a0653ca..e4d42f6525d14 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/c/register_variables/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := test.c
 
-CFLAGS_EXTRAS += -O1 -D_FORTIFY_SOURCE=0
+CFLAGS_EXTRAS := -O1 -D_FORTIFY_SOURCE=0
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile b/lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile
index db8ac60dd1931..a40a666476c43 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/c/unicode/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -finput-charset=UTF-8 -fextended-identifiers -std=c99
+CFLAGS_EXTRAS := -finput-charset=UTF-8 -fextended-identifiers -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile
index ac4b37c7c7bfa..a02c72adc2028 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/enum_types/Makefile
@@ -1,8 +1,9 @@
 CXX_SOURCES := main.cpp
 
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 
 clean: OBJECTS+=$(wildcard main.d.*)
 
-include Makefile.rules
 
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile
index 4a430eccc701d..a98dca64e8a9d 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/gmodules/Makefile
@@ -1,5 +1,5 @@
 PCH_CXX_SOURCE = pch.h
 CXX_SOURCES = main.cpp
-CFLAGS_EXTRAS += $(MODULE_DEBUG_INFO_FLAGS)
+CFLAGS_EXTRAS := $(MODULE_DEBUG_INFO_FLAGS)
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile
index 8e811fdeb67a5..ba7e01537522e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/limit-debug-info/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES = main.cpp derived.cpp base.cpp
 
-CFLAGS_EXTRAS += $(LIMIT_DEBUG_INFO_FLAGS)
+CFLAGS_EXTRAS := $(LIMIT_DEBUG_INFO_FLAGS)
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile
index 9e7837a769538..3dff43b34c774 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/modules-import/Makefile
@@ -1,4 +1,4 @@
 CXX_SOURCES := main.cpp
-CFLAGS_EXTRAS = $(MANDATORY_MODULE_BUILD_CFLAGS) -I$(BUILDDIR)/include
+CXXFLAGS_EXTRAS = $(MANDATORY_MODULE_BUILD_CFLAGS) -I$(BUILDDIR)/include
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/Makefile
new file mode 100644
index 0000000000000..020dce7c31d11
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/Makefile
@@ -0,0 +1,3 @@
+CXX_SOURCES := main.cpp other.cpp
+
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py
new file mode 100644
index 0000000000000..5b590d6363ab3
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/TestNestedClassWithParentInAnotherCU.py
@@ -0,0 +1,29 @@
+"""
+Test that the expression evaluator can access members of nested classes even if
+the parents of the nested classes were imported from another compilation unit.
+"""
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TestNestedClassWithParentInAnotherCU(TestBase):
+    mydir = TestBase.compute_mydir(__file__)
+
+    def test_nested_class_with_parent_in_another_cu(self):
+        self.main_source_file = lldb.SBFileSpec("main.cpp")
+        self.build()
+        (_, _, thread, _) = lldbutil.run_to_source_breakpoint(self, "// break here", self.main_source_file)
+        frame = thread.GetSelectedFrame()
+        # Parse the DIEs of the parent classes and the nested classes from
+        # other.cpp's CU.
+        warmup_result = frame.EvaluateExpression("b")
+        self.assertTrue(warmup_result.IsValid())
+        # Inspect fields of the nested classes. This will reuse the types that
+        # were parsed during the evaluation above. By accessing a chain of
+        # fields, we try to verify that all the DIEs, reused types and
+        # declaration contexts were wired properly into lldb's parser's state.
+        expr_result = frame.EvaluateExpression("a.y.oY_inner.oX_inner")
+        self.assertTrue(expr_result.IsValid())
+        self.assertEqual(expr_result.GetValue(), "42")
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/main.cpp b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/main.cpp
new file mode 100644
index 0000000000000..8a6e6ff81d23c
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/main.cpp
@@ -0,0 +1,22 @@
+#include "shared.h"
+
+struct WrapperA {
+  OuterY::Inner<unsigned int> y;
+};
+
+int main() {
+  // WrapperA refers to the Inner and Outer class DIEs from this CU.
+  WrapperA a;
+  // WrapperB refers to the Inner and Outer DIEs from the other.cpp CU.
+  // It is important that WrapperB is only forward-declared in shared.h.
+  WrapperB* b = foo();
+
+  // Evaluating 'b' here will parse other.cpp's DIEs for all
+  // the Inner and Outer classes from shared.h.
+  //
+  // Evaluating 'a' here will find and reuse the already-parsed
+  // versions of the Inner and Outer classes. In the associated test
+  // we make sure that we can still resolve all the types properly
+  // by evaluating 'a.y.oY_inner.oX_inner'.
+  return 0;  // break here
+}
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/other.cpp b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/other.cpp
new file mode 100644
index 0000000000000..71a9579c4e820
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/other.cpp
@@ -0,0 +1,10 @@
+#include "shared.h"
+
+struct WrapperB {
+  OuterY y;
+  OuterX x;
+};
+
+WrapperB* foo() {
+  return new WrapperB();
+}
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/shared.h b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/shared.h
new file mode 100644
index 0000000000000..627f49ab79c59
--- /dev/null
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/nested-class-other-compilation-unit/shared.h
@@ -0,0 +1,17 @@
+struct OuterX {
+  template<typename T>
+  struct Inner {
+    int oX_inner = 42;
+  };
+};
+
+struct OuterY {
+  template<typename T>
+  struct Inner {
+    typename OuterX::Inner<T> oY_inner;
+  };
+};
+
+struct WrapperB;
+
+WrapperB* foo();
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile
index 57ee6052ae9ce..e891bb25dd912 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/rvalue-references/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
 
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 
-include Makefile.rules
\ No newline at end of file
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile b/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile
index f96dd95399ee9..b016f006747da 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/std-function-step-into-callable/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
-CXXFLAGS += -std=c++11
+CXXFLAGS_EXTRAS := -std=c++11
 USE_LIBCPP := 1
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py b/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py
index 9f8ed89375eb5..5152c0010d102 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py
+++ b/lldb/packages/Python/lldbsuite/test/lang/cpp/thread_local/TestThreadLocal.py
@@ -2,4 +2,5 @@
 from lldbsuite.test import decorators
 
 lldbinline.MakeInlineTest(__file__, globals(),
-                          lldbinline.expectedFailureAll(oslist=["windows", "linux"]))
+                          lldbinline.expectedFailureAll(oslist=[
+                              "windows", "linux", "netbsd"]))
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile
index 48c06c712cdb2..df76ed3069fcf 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/blocks/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := ivars-in-blocks.m main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile
index 377a1a0373a0d..876340159d9c8 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/exceptions/Makefile
@@ -1,7 +1,8 @@
 OBJCXX_SOURCES := main.mm
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile
index b9d3a985fbe9b..cfae251ead4b0 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/forward-decl/Makefile
@@ -2,6 +2,7 @@ DYLIB_NAME := Container
 DYLIB_OBJC_SOURCES := Container.m
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile
index 38ef76b8d5852..e95ebd94a94aa 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/Makefile
@@ -1,6 +1,7 @@
 OBJC_SOURCES := main.m my-base.m
 #OBJC_SOURCES := const-strings.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py
index 329407d448a2d..b42aeb14226b3 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestFoundationDisassembly.py
@@ -15,6 +15,8 @@ class FoundationDisassembleTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
+    NO_DEBUG_INFO_TESTCASE = True
+
     def test_foundation_disasm(self):
         """Do 'disassemble -n func' on each and every 'Code' symbol entry from the Foundation.framework."""
         self.build()
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py
index 85d34c3d50062..7d4990c4f38fe 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods.py
@@ -191,7 +191,7 @@ def test_data_type_and_expr(self):
             "expression self->non_existent_member",
             COMMAND_FAILED_AS_EXPECTED,
             error=True,
-            startstr="error: 'MyString' does not have a member named 'non_existent_member'")
+            substrs=["error:", "'MyString' does not have a member named 'non_existent_member'"])
 
         # Use expression parser.
         self.runCmd("expression self->str")
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py
index 20137e3d118f8..28d3558126d0d 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/foundation/TestObjCMethods2.py
@@ -16,61 +16,26 @@ class FoundationTestCase2(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
 
+    NO_DEBUG_INFO_TESTCASE = True
+
     def test_expr_commands(self):
         """More expression commands for objective-c."""
         self.build()
-        exe = self.getBuildArtifact("a.out")
-        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)
-
-        lines = []
-        lines.append(
-            line_number(
-                'main.m',
-                '// Break here for selector: tests'))
-        lines.append(
-            line_number(
-                'main.m',
-                '// Break here for NSArray tests'))
-        lines.append(
-            line_number(
-                'main.m',
-                '// Break here for NSString tests'))
-        lines.append(
-            line_number(
-                'main.m',
-                '// Break here for description test'))
-        lines.append(
-            line_number(
-                'main.m',
-                '// Set break point at this line'))
-
-        # Create a bunch of breakpoints.
-        for line in lines:
-            lldbutil.run_break_set_by_file_and_line(
-                self, "main.m", line, num_expected_locations=1, loc_exact=True)
-
-        self.runCmd("run", RUN_SUCCEEDED)
+        main_spec = lldb.SBFileSpec("main.m")
 
+        (target, process, thread, bp) = lldbutil.run_to_source_breakpoint(
+            self, "Break here for selector: tests", main_spec)
+        
         # Test_Selector:
-        self.runCmd("thread backtrace")
         self.expect("expression (char *)sel_getName(sel)",
                     substrs=["(char *)",
                              "length"])
 
-        self.runCmd("process continue")
-
-        # Test_NSArray:
-        self.runCmd("thread backtrace")
-        self.runCmd("process continue")
-
-        # Test_NSString:
-        self.runCmd("thread backtrace")
-        self.runCmd("process continue")
-
-        # Test_MyString:
-        self.runCmd("thread backtrace")
+        desc_bkpt = target.BreakpointCreateBySourceRegex("Break here for description test",
+                                                          main_spec)
+        self.assertEqual(desc_bkpt.GetNumLocations(), 1, "description breakpoint has a location")
+        lldbutil.continue_to_breakpoint(process, desc_bkpt)
+        
         self.expect("expression (char *)sel_getName(_cmd)",
                     substrs=["(char *)",
                              "description"])
-
-        self.runCmd("process continue")
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/global_ptrs/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile
index 843400a08cfce..ba7e23acabab1 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/ivar-IMP/Makefile
@@ -1,18 +1,5 @@
-CC ?= clang
-ifeq "$(ARCH)" ""
-        ARCH = x86_64
-endif
-
-ifeq "$(OS)" ""
-        OS = $(shell uname -s)
-endif
-
-CFLAGS ?= -g -O0
+CFLAGS := -g -O0
 CFLAGS_NO_DEBUG = 
-ifeq "$(OS)" "Darwin"
-        CFLAGS += -arch $(ARCH)
-		CFLAGS_NO_DEBUG += -arch $(ARCH)
-endif
 
 all: aout
 
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile
index c6bd24d99ada4..3b2bd504c89d6 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-auto-import/Makefile
@@ -1,5 +1,5 @@
 OBJC_SOURCES := main.m
 
-CFLAGS += $(MANDATORY_MODULE_BUILD_CFLAGS)
+CFLAGS_EXTRAS = $(MANDATORY_MODULE_BUILD_CFLAGS)
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile
index c32d0fb655d06..abb36e281b2f9 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-incomplete/Makefile
@@ -1,6 +1,5 @@
 OBJC_SOURCES := main.m myModule.m
 
+CFLAGS_EXTRAS = $(MANDATORY_MODULE_BUILD_CFLAGS)
+LD_EXTRAS := -framework Foundation
 include Makefile.rules
-
-CFLAGS += $(MANDATORY_MODULE_BUILD_CFLAGS)
-LDFLAGS += -framework Foundation
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile
index 78d1c67dec167..b4afe2cb3e856 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/modules-inline-functions/Makefile
@@ -2,6 +2,6 @@ C_SOURCES := myModule.c
 
 OBJC_SOURCES := main.m
 
-include Makefile.rules
+CFLAGS_EXTRAS = $(MANDATORY_MODULE_BUILD_CFLAGS) -I$(BUILDDIR)
 
-CFLAGS += $(MANDATORY_MODULE_BUILD_CFLAGS) -I$(PWD)
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/modules/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-checker/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile
index e0c4678f5d902..e6db3dee37bfa 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-class-method/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := class.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile
index 8677e22655681..a34977712c366 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-dynamic-value/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := dynamic-value.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile
index 701fd53ab273d..5408f4199f2ad 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-offsets/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := objc-ivar-offsets.m main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile
index 32c0f4aef4086..a218ce37e61df 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-ivar-stripped/Makefile
@@ -1,5 +1,5 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 all:        a.out.stripped
 
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-new-syntax/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile
index 9be48503826ae..5fb128dcc7e91 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-optimized/Makefile
@@ -1,6 +1,6 @@
 OBJC_SOURCES := main.m
 
 CFLAGS ?= -arch $(ARCH) -g -O2
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-property/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-runtime-ivars/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile
index 5ba751c1567e7..0ad7e3563167f 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method-stripped/Makefile
@@ -1,5 +1,5 @@
 OBJC_SOURCES := static.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 default:        a.out.stripped
 
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile
index 1c9e629925f81..954dea47b32ec 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-static-method/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := static.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile
index 09e6a5ea3387d..9906470d5308d 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-stepping/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := stepping-tests.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile
index 1db77a61ecdd8..d059a5c1c292e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-argument/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := test.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile
index 1db77a61ecdd8..d059a5c1c292e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-struct-return/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := test.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile
index e0c4678f5d902..e6db3dee37bfa 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/objc-super/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := class.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/orderedset/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile
index 1d34d4ed2fff0..2eab56265f57b 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/print-obj/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := blocked.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/radar-9691614/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-10967107/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/rdar-11355592/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile
index 1b3732be2528e..f1498efac1d29 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/real-definition/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := Bar.m Foo.m main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile
index 32d01e345df68..afecbf969483e 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/sample/Makefile
@@ -1,4 +1,4 @@
 OBJC_SOURCES := main.m
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile
index 7202932571dc3..37dd8f40a9d4a 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objc/single-entry-dictionary/Makefile
@@ -1,5 +1,6 @@
 OBJC_SOURCES := main.m
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile
index 7423b54218a52..7cfe4a3f0fdc6 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objcxx/cxx-bridged-po/Makefile
@@ -1,4 +1,4 @@
 OBJCXX_SOURCES := main.mm
-LDFLAGS = $(CFLAGS) -lobjc -framework CoreFoundation
+LD_EXTRAS := -lobjc -framework CoreFoundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile
index c0b4b1a3470aa..3af75c304c35d 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objcxx/hide-runtime-values/Makefile
@@ -1,4 +1,4 @@
 OBJCXX_SOURCES := main.mm
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile b/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile
index c0b4b1a3470aa..3af75c304c35d 100644
--- a/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/lang/objcxx/sample/Makefile
@@ -1,4 +1,4 @@
 OBJCXX_SOURCES := main.mm
-LDFLAGS = $(CFLAGS) -lobjc -framework Foundation
+LD_EXTRAS := -lobjc -framework Foundation
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile b/lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile
index fa3eadef65f56..98042d61ea562 100644
--- a/lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/linux/add-symbols/Makefile
@@ -1,5 +1,5 @@
 CXX_SOURCES := main.cpp
-LD_EXTRAS += -Wl,--build-id=none
+LD_EXTRAS := -Wl,--build-id=none
 
 all: stripped.out
 
diff --git a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile b/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile
index 169460598a25c..4f3dd568d5f18 100644
--- a/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/linux/mix-dwo-and-regular-objects/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := a.c b.c
-a.o: CFLAGS_EXTRAS += -gsplit-dwarf
+a.o: CFLAGS_EXTRAS := -gsplit-dwarf
 
 include Makefile.rules
 
diff --git a/lldb/packages/Python/lldbsuite/test/macosx/order/Makefile b/lldb/packages/Python/lldbsuite/test/macosx/order/Makefile
index 357a6032bcd29..ee67988d0650f 100644
--- a/lldb/packages/Python/lldbsuite/test/macosx/order/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/macosx/order/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := main.c
-LDFLAGS = $(CFLAGS) -Xlinker -order_file -Xlinker $(SRCDIR)/order-file
+LD_EXTRAS = -Xlinker -order_file -Xlinker $(SRCDIR)/order-file
 MAKE_DSYM := NO
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
index 784307b98acef..5a056b1475a0c 100644
--- a/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
+++ b/lldb/packages/Python/lldbsuite/test/make/Makefile.rules
@@ -13,6 +13,12 @@
 # the building of the a.out executable program.  For example,
 # DYLIB_ONLY := YES
 #
+# Specifying FRAMEWORK and its variants has the effect of building a NeXT-style
+# framework.
+# FRAMEWORK := "Foo"
+# FRAMEWORK_HEADERS := "Foo.h"
+# FRAMEWORK_MODULES := "module.modulemap"
+#
 # Also might be of interest:
 # FRAMEWORK_INCLUDES (Darwin only) :=
 # CFLAGS_EXTRAS :=
@@ -329,7 +335,7 @@ ifeq "$(MAKE_GMODULES)" "YES"
 	CXXFLAGS += $(MANDATORY_MODULE_BUILD_CFLAGS)
 endif
 
-CXXFLAGS += -std=c++11 $(CFLAGS) $(ARCH_CXXFLAGS)
+CXXFLAGS += -std=c++11 $(CFLAGS) $(ARCH_CXXFLAGS) $(CXXFLAGS_EXTRAS)
 LD = $(CC)
 LDFLAGS ?= $(CFLAGS)
 LDFLAGS += $(LD_EXTRAS) $(ARCH_LDFLAGS)
@@ -346,10 +352,19 @@ endif
 OBJECTS =
 EXE ?= a.out
 
+ifneq "$(FRAMEWORK)" ""
+	DYLIB_NAME ?= $(FRAMEWORK).framework/Versions/A/$(FRAMEWORK)
+	DYLIB_FILENAME ?= $(FRAMEWORK).framework/Versions/A/$(FRAMEWORK)
+endif
+
 ifneq "$(DYLIB_NAME)" ""
 	ifeq "$(OS)" "Darwin"
-		DYLIB_FILENAME = lib$(DYLIB_NAME).dylib
-		DYLIB_EXECUTABLE_PATH ?= @executable_path
+		ifneq "$(FRAMEWORK)" ""
+			DYLIB_INSTALL_NAME ?= @executable_path/$(FRAMEWORK).framework/Versions/A/$(FRAMEWORK)
+		else
+			DYLIB_FILENAME = lib$(DYLIB_NAME).dylib
+			DYLIB_INSTALL_NAME ?= @executable_path/$(DYLIB_FILENAME)
+		endif
 	else ifeq "$(OS)" "Windows_NT"
 		DYLIB_FILENAME = $(DYLIB_NAME).dll
 	else
@@ -467,16 +482,11 @@ endif
 #----------------------------------------------------------------------
 # dylib settings
 #----------------------------------------------------------------------
-ifneq "$(strip $(DYLIB_C_SOURCES))" ""
-	DYLIB_OBJECTS +=$(strip $(DYLIB_C_SOURCES:.c=.o))
-endif
-
-ifneq "$(strip $(DYLIB_OBJC_SOURCES))" ""
-	DYLIB_OBJECTS +=$(strip $(DYLIB_OBJC_SOURCES:.m=.o))
-endif
 
+DYLIB_OBJECTS +=$(strip $(DYLIB_C_SOURCES:.c=.o))
+DYLIB_OBJECTS +=$(strip $(DYLIB_OBJC_SOURCES:.m=.o))
 ifneq "$(strip $(DYLIB_CXX_SOURCES))" ""
-	DYLIB_OBJECTS +=$(strip $(DYLIB_CXX_SOURCES:.cpp=.o))
+	DYLIB_OBJECTS +=$(strip $(patsubst %.mm, %.o, $(DYLIB_CXX_SOURCES:.cpp=.o)))
 	CXX = $(call cxx_compiler,$(CC))
 	LD = $(call cxx_linker,$(CC))
 endif
@@ -650,7 +660,23 @@ endif
 
 $(DYLIB_FILENAME) : $(DYLIB_OBJECTS)
 ifeq "$(OS)" "Darwin"
-	$(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -install_name "$(DYLIB_EXECUTABLE_PATH)/$(DYLIB_FILENAME)" -dynamiclib -o "$(DYLIB_FILENAME)"
+ifneq "$(FRAMEWORK)" ""
+	mkdir -p $(FRAMEWORK).framework/Versions/A/Headers
+	mkdir -p $(FRAMEWORK).framework/Versions/A/Modules
+	mkdir -p $(FRAMEWORK).framework/Versions/A/Resources
+ifneq "$(FRAMEWORK_MODULES)" ""
+	cp -r $(FRAMEWORK_MODULES) $(FRAMEWORK).framework/Versions/A/Modules
+endif
+ifneq "$(FRAMEWORK_HEADERS)" ""
+	cp -r $(FRAMEWORK_HEADERS) $(FRAMEWORK).framework/Versions/A/Headers
+endif
+	(cd $(FRAMEWORK).framework/Versions; ln -sf A Current)
+	(cd $(FRAMEWORK).framework/; ln -sf Versions/A/Headers Headers)
+	(cd $(FRAMEWORK).framework/; ln -sf Versions/A/Modules Modules)
+	(cd $(FRAMEWORK).framework/; ln -sf Versions/A/Resources Resources)
+	(cd $(FRAMEWORK).framework/; ln -sf Versions/A/$(FRAMEWORK) $(FRAMEWORK))
+endif
+	$(LD) $(DYLIB_OBJECTS) $(LDFLAGS) -install_name "$(DYLIB_INSTALL_NAME)" -dynamiclib -o "$(DYLIB_FILENAME)"
 ifneq "$(CODESIGN)" ""
 	$(CODESIGN) -s - "$(DYLIB_FILENAME)"
 endif
@@ -743,6 +769,9 @@ ifneq "$(DYLIB_NAME)" ""
 	$(RM) -r $(DYLIB_FILENAME).dSYM
 	$(RM) $(DYLIB_OBJECTS) $(DYLIB_PREREQS) $(DYLIB_PREREQS:.d=.d.tmp) $(DYLIB_DWOS) $(DYLIB_FILENAME) $(DYLIB_FILENAME).debug
 endif
+ifneq "$(FRAMEWORK)" ""
+	$(RM) -rf $(FRAMEWORK).framework
+endif
 ifneq "$(PCH_OUTPUT)" ""
 	$(RM) $(PCH_OUTPUT)
 endif
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile
index b5256564c2f16..33da5d0645ec4 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/findvalue_duplist/Makefile
@@ -1,6 +1,8 @@
 CXX_SOURCES := main.cpp
 
+include Makefile.rules
+
 # Clean renamed executable on 'make clean'
-clean: OBJECTS+=no_synth
+clean::
+	$(RM) -f no_synth
 
-include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile
index b5256564c2f16..16a823213db6d 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/formatters/Makefile
@@ -1,6 +1,7 @@
 CXX_SOURCES := main.cpp
 
-# Clean renamed executable on 'make clean'
-clean: OBJECTS+=no_synth
-
 include Makefile.rules
+
+# Clean renamed executable on 'make clean'
+clean::
+	$(RM) -f no_synth
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile
index 4cb3dc98c8c1e..e6d9d8310a0fa 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/frame/inlines/Makefile
@@ -1,7 +1,7 @@
 C_SOURCES := inlines.c
 
 ifneq (,$(findstring icc,$(CC)))
-    CFLAGS += -debug inline-debug-info
+    CFLAGS_EXTRAS := -debug inline-debug-info
 endif
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile
index 200bdafd69fe4..4d11bbc8b6a05 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/iter/Makefile
@@ -1,4 +1,4 @@
-CFLAGS_EXTRAS += -D__STDC_LIMIT_MACROS
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS
 ENABLE_THREADS := YES
 CXX_SOURCES := main.cpp
 MAKE_DSYM := NO
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile
index 938161724be55..6b33049a78bdd 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/lldbutil/process/Makefile
@@ -1,4 +1,4 @@
-CFLAGS_EXTRAS += -D__STDC_LIMIT_MACROS
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS
 ENABLE_THREADS := YES
 CXX_SOURCES := main.cpp
 MAKE_DSYM :=NO
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile
index 5eed4cb698c42..79209db9696aa 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/module_section/Makefile
@@ -1,4 +1,4 @@
-CFLAGS_EXTRAS += -D__STDC_LIMIT_MACROS
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS
 ENABLE_THREADS := YES
 CXX_SOURCES := main.cpp b.cpp c.cpp
 MAKE_DSYM :=NO
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile
index 03b578913862c..8b322ff320bb0 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/objc_type/Makefile
@@ -1,7 +1,8 @@
 OBJC_SOURCES := main.m
 
-CFLAGS_EXTRAS += -w
+CFLAGS_EXTRAS := -w
+
 
-include Makefile.rules
 
-LDFLAGS += -framework Foundation
+LD_EXTRAS := -framework Foundation
+include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp b/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp
index eb9c8e0c142a3..318a45bc21a85 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp
+++ b/lldb/packages/Python/lldbsuite/test/python_api/sbvalue_const_addrof/main.cpp
@@ -28,7 +28,7 @@ ThreadInfo *g_thread_list_ptr = &g_thread1;
 int main (int argc, char const *argv[], char const *envp[])
 {
     printf ("g_thread_list is %p\n", g_thread_list_ptr);
-    return 0; //% v = lldb.target.FindFirstGlobalVariable('g_thread_list_ptr')
+    return 0; //% v = self.dbg.GetSelectedTarget().FindFirstGlobalVariable('g_thread_list_ptr')
     //% v_gla = v.GetChildMemberWithName('regs').GetLoadAddress()
     //% v_aof = v.GetChildMemberWithName('regs').AddressOf().GetValueAsUnsigned(lldb.LLDB_INVALID_ADDRESS)
     //% expr = '(%s)0x%x' % (v.GetType().GetName(), v.GetValueAsUnsigned(0))
diff --git a/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile b/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile
index 95cdf295fbcf9..3716c6e29e990 100644
--- a/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/python_api/value_var_update/Makefile
@@ -1,5 +1,5 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 # See TestHelloWorld.py, which specifies the executable name with a dictionary.
 EXE := hello_world
 
diff --git a/lldb/packages/Python/lldbsuite/test/sample_test/Makefile b/lldb/packages/Python/lldbsuite/test/sample_test/Makefile
index aa5f222c08e31..695335e068c0c 100644
--- a/lldb/packages/Python/lldbsuite/test/sample_test/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/sample_test/Makefile
@@ -1,4 +1,4 @@
 C_SOURCES := main.c
-CFLAGS_EXTRAS += -std=c99
+CFLAGS_EXTRAS := -std=c99
 
 include Makefile.rules
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile
index 1e0987a3022b0..0c441eda98b2e 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/Makefile
@@ -1,4 +1,4 @@
-override CFLAGS_EXTRAS += -D__STDC_LIMIT_MACROS -D__STDC_FORMAT_MACROS
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS -D__STDC_FORMAT_MACROS
 ENABLE_THREADS := YES
 CXX_SOURCES := main.cpp
 MAKE_DSYM :=NO
diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile
index 7fe9a3e92a740..536d2e8db48c5 100644
--- a/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile
+++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-server/inferior-crash/Makefile
@@ -1,4 +1,4 @@
-CFLAGS_EXTRAS += -D__STDC_LIMIT_MACROS -D__STDC_FORMAT_MACROS -std=c++11
+CFLAGS_EXTRAS := -D__STDC_LIMIT_MACROS -D__STDC_FORMAT_MACROS -std=c++11
 # LD_EXTRAS := -lpthread
 CXX_SOURCES := main.cpp
 MAKE_DSYM :=NO
diff --git a/lldb/scripts/CMakeLists.txt b/lldb/scripts/CMakeLists.txt
index 1a0ea96ff9f65..ebe850c56c410 100644
--- a/lldb/scripts/CMakeLists.txt
+++ b/lldb/scripts/CMakeLists.txt
@@ -1,13 +1,15 @@
 file(GLOB SWIG_INTERFACES interface/*.i)
 file(GLOB_RECURSE SWIG_SOURCES *.swig)
-set(SWIG_HEADERS
-  ${LLDB_SOURCE_DIR}/include/lldb/API/SBDefines.h
-  ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h
-  ${LLDB_SOURCE_DIR}/include/lldb/lldb-enumerations.h
-  ${LLDB_SOURCE_DIR}/include/lldb/lldb-forward.h
-  ${LLDB_SOURCE_DIR}/include/lldb/lldb-types.h
-  ${LLDB_SOURCE_DIR}/include/lldb/lldb-versioning.h
+file(GLOB SWIG_HEADERS
+  ${LLDB_SOURCE_DIR}/include/lldb/API/*.h
+  ${LLDB_SOURCE_DIR}/include/lldb/*.h
 )
+file(GLOB SWIG_PRIVATE_HEADERS
+  ${LLDB_SOURCE_DIR}/include/lldb/lldb-private*.h
+)
+foreach(private_header ${SWIG_PRIVATE_HEADERS})
+  list(REMOVE_ITEM SWIG_HEADERS ${private_header})
+endforeach()
 
 if(LLDB_BUILD_FRAMEWORK)
   set(framework_arg --framework --target-platform Darwin)
@@ -19,22 +21,34 @@ if (${SWIG_VERSION} VERSION_LESS ${SWIG_MIN_VERSION})
   message(FATAL_ERROR "LLDB requires swig ${SWIG_MIN_VERSION}, your version is ${SWIG_VERSION}.")
 endif()
 
+if(APPLE)
+  set(DARWIN_EXTRAS "-D__APPLE__")
+else()
+  set(DARWIN_EXTRAS "")
+endif()
+
 add_custom_command(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/LLDBWrapPython.cpp
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lldb.py
   DEPENDS ${SWIG_SOURCES}
   DEPENDS ${SWIG_INTERFACES}
   DEPENDS ${SWIG_HEADERS}
-  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Python/prepare_binding_Python.py
-  COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/prepare_bindings.py
-      ${framework_arg}
-      --srcRoot=${LLDB_SOURCE_DIR}
-      --targetDir=${CMAKE_CURRENT_BINARY_DIR}
-      --cfgBldDir=${CMAKE_CURRENT_BINARY_DIR}
-      --prefix=${CMAKE_BINARY_DIR}
-      --swigExecutable=${SWIG_EXECUTABLE}
+  COMMAND ${SWIG_EXECUTABLE} 
+      -c++
+      -shadow
+      -python
+      -features autodoc
+      -threads
+      -I${LLDB_SOURCE_DIR}/include
+      -I${CMAKE_CURRENT_SOURCE_DIR}
+      -D__STDC_LIMIT_MACROS
+      -D__STDC_CONSTANT_MACROS
+      ${DARWIN_EXTRAS}
+      -outdir ${CMAKE_CURRENT_BINARY_DIR}
+      -o ${CMAKE_CURRENT_BINARY_DIR}/LLDBWrapPython.cpp
+      ${LLDB_SOURCE_DIR}/scripts/lldb.swig
   VERBATIM
-  COMMENT "Python script building LLDB Python wrapper")
+  COMMENT "Builds LLDB Python wrapper")
 
 add_custom_target(swig_wrapper ALL DEPENDS
   ${CMAKE_CURRENT_BINARY_DIR}/LLDBWrapPython.cpp
@@ -42,15 +56,18 @@ add_custom_target(swig_wrapper ALL DEPENDS
 )
 
 if(NOT LLDB_BUILD_FRAMEWORK)
-  if(CMAKE_SYSTEM_NAME MATCHES "Windows")
-    set(swig_python_subdir site-packages)
-  else()
-    set(swig_python_subdir python${PYTHON_VERSION_MAJOR}.${PYTHON_VERSION_MINOR})
-  endif()
-
-  set(SWIG_PYTHON_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/${swig_python_subdir})
-  set(SWIG_INSTALL_DIR lib${LLVM_LIBDIR_SUFFIX})
+  execute_process(
+    COMMAND ${PYTHON_EXECUTABLE}
+        -c "import distutils.sysconfig, sys; print(distutils.sysconfig.get_python_lib(True, False, sys.argv[1]))"
+        ${CMAKE_BINARY_DIR}
+    OUTPUT_VARIABLE SWIG_PYTHON_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+  execute_process(
+    COMMAND ${PYTHON_EXECUTABLE}
+        -c "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(True, False, ''))"
+    OUTPUT_VARIABLE SWIG_INSTALL_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
 
   # Install the LLDB python module
-  install(DIRECTORY ${SWIG_PYTHON_DIR} DESTINATION ${SWIG_INSTALL_DIR})
+  install(DIRECTORY ${SWIG_PYTHON_DIR}/ DESTINATION ${SWIG_INSTALL_DIR})
 endif()
diff --git a/lldb/scripts/Python/prepare_binding_Python.py b/lldb/scripts/Python/prepare_binding_Python.py
deleted file mode 100644
index 7d13d6ebb17f7..0000000000000
--- a/lldb/scripts/Python/prepare_binding_Python.py
+++ /dev/null
@@ -1,394 +0,0 @@
-"""
-Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-Python binding preparation script.
-"""
-
-# Python modules:
-from __future__ import print_function
-
-import logging
-import os
-import re
-import shutil
-import subprocess
-import sys
-import platform
-
-
-class SwigSettings(object):
-    """Provides a single object to represent swig files and settings."""
-
-    def __init__(self):
-        self.extensions_file = None
-        self.header_files = None
-        self.input_file = None
-        self.interface_files = None
-        self.output_file = None
-        self.safecast_file = None
-        self.typemaps_file = None
-        self.wrapper_file = None
-
-    @classmethod
-    def _any_files_newer(cls, files, check_mtime):
-        """Returns if any of the given files has a newer modified time.
-
-        @param cls the class
-        @param files a list of zero or more file paths to check
-        @param check_mtime the modification time to use as a reference.
-
-        @return True if any file's modified time is newer than check_mtime.
-        """
-        for path in files:
-            path_mtime = os.path.getmtime(path)
-            if path_mtime > check_mtime:
-                # This path was modified more recently than the
-                # check_mtime.
-                return True
-        # If we made it here, nothing was newer than the check_mtime
-        return False
-
-    @classmethod
-    def _file_newer(cls, path, check_mtime):
-        """Tests how recently a file has been modified.
-
-        @param cls the class
-        @param path a file path to check
-        @param check_mtime the modification time to use as a reference.
-
-        @return True if the file's modified time is newer than check_mtime.
-        """
-        path_mtime = os.path.getmtime(path)
-        return path_mtime > check_mtime
-
-    def output_out_of_date(self):
-        """Returns whether the output file is out of date.
-
-        Compares output file time to all the input files.
-
-        @return True if any of the input files are newer than
-        the output file, or if the output file doesn't exist;
-        False otherwise.
-        """
-        if not os.path.exists(self.output_file):
-            logging.info("will generate, missing binding output file")
-            return True
-        output_mtime = os.path.getmtime(self.output_file)
-        if self._any_files_newer(self.header_files, output_mtime):
-            logging.info("will generate, header files newer")
-            return True
-        if self._any_files_newer(self.interface_files, output_mtime):
-            logging.info("will generate, interface files newer")
-            return True
-        if self._file_newer(self.input_file, output_mtime):
-            logging.info("will generate, swig input file newer")
-            return True
-        if self._file_newer(self.extensions_file, output_mtime):
-            logging.info("will generate, swig extensions file newer")
-            return True
-        if self._file_newer(self.wrapper_file, output_mtime):
-            logging.info("will generate, swig wrapper file newer")
-            return True
-        if self._file_newer(self.typemaps_file, output_mtime):
-            logging.info("will generate, swig typemaps file newer")
-            return True
-        if self._file_newer(self.safecast_file, output_mtime):
-            logging.info("will generate, swig safecast file newer")
-            return True
-
-        # If we made it here, nothing is newer than the output file.
-        # Thus, the output file is not out of date.
-        return False
-
-
-def get_header_files(options):
-    """Returns a list of paths to C++ header files for the LLDB API.
-
-    These are the files that define the C++ API that will be wrapped by Python.
-
-    @param options the dictionary of options parsed from the command line.
-
-    @return a list of full paths to the include files used to define the public
-    LLDB C++ API.
-    """
-
-    header_file_paths = []
-    header_base_dir = os.path.join(options.src_root, "include", "lldb")
-
-    # Specify the include files in include/lldb that are not easy to
-    # grab programatically.
-    for header in [
-            "lldb-defines.h",
-            "lldb-enumerations.h",
-            "lldb-forward.h",
-            "lldb-types.h"]:
-        header_file_paths.append(os.path.normcase(
-            os.path.join(header_base_dir, header)))
-
-    # Include the main LLDB.h file.
-    api_dir = os.path.join(header_base_dir, "API")
-    header_file_paths.append(os.path.normcase(
-        os.path.join(api_dir, "LLDB.h")))
-
-    filename_regex = re.compile(r"^SB.+\.h$")
-
-    # Include all the SB*.h files in the API dir.
-    for filename in os.listdir(api_dir):
-        if filename_regex.match(filename):
-            header_file_paths.append(
-                os.path.normcase(os.path.join(api_dir, filename)))
-
-    logging.debug("found public API header file paths: %s", header_file_paths)
-    return header_file_paths
-
-
-def get_interface_files(options):
-    """Returns a list of interface files used as input to swig.
-
-    @param options the options dictionary parsed from the command line args.
-
-    @return a list of full paths to the interface (.i) files used to describe
-    the public API language binding.
-    """
-    interface_file_paths = []
-    interface_dir = os.path.join(options.src_root, "scripts", "interface")
-
-    for filepath in [f for f in os.listdir(interface_dir)
-                     if os.path.splitext(f)[1] == ".i"]:
-        interface_file_paths.append(
-            os.path.normcase(os.path.join(interface_dir, filepath)))
-
-    logging.debug("found swig interface files: %s", interface_file_paths)
-    return interface_file_paths
-
-
-def remove_ignore_enoent(filename):
-    """Removes given file, ignoring error if it doesn't exist.
-
-    @param filename the path of the file to remove.
-    """
-    try:
-        os.remove(filename)
-    except OSError as error:
-        import errno
-        if error.errno != errno.ENOENT:
-            raise
-
-
-def do_swig_rebuild(options, dependency_file, config_build_dir, settings):
-    """Generates Python bindings file from swig.
-
-    This method will do a sys.exit() if something fails.  If it returns to
-    the caller, it succeeded.
-
-    @param options the parsed command line options structure.
-    @param dependency_file path to the bindings dependency file
-    to be generated; otherwise, None if a dependency file is not
-    to be generated.
-    @param config_build_dir used as the output directory used by swig
-    @param settings the SwigSettings that specify a number of aspects used
-    to configure building the Python binding with swig (mostly paths)
-    """
-    if options.generate_dependency_file:
-        temp_dep_file_path = dependency_file + ".tmp"
-
-    # Build the SWIG args list
-    is_darwin = options.target_platform == "Darwin"
-    gen_deps = options.generate_dependency_file
-    darwin_extras = ["-D__APPLE__"] if is_darwin else []
-    deps_args = ["-MMD", "-MF", temp_dep_file_path] if gen_deps else []
-    command = ([
-            options.swig_executable,
-            "-c++",
-            "-shadow",
-            "-python",
-            "-features", "autodoc",
-            "-threads",
-            "-I" + os.path.normpath(os.path.join(options.src_root, "include")),
-            "-I" + os.path.curdir,
-            "-D__STDC_LIMIT_MACROS",
-            "-D__STDC_CONSTANT_MACROS"
-        ]
-        + darwin_extras
-        + deps_args
-        + [
-            "-outdir", config_build_dir,
-            "-o", settings.output_file,
-            settings.input_file
-        ]
-    )
-    logging.info("running swig with: %r", command)
-
-    # Execute swig
-    process = subprocess.Popen(
-        command,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
-    # Wait for SWIG process to terminate
-    swig_stdout, swig_stderr = process.communicate()
-    return_code = process.returncode
-    if return_code != 0:
-        logging.error(
-            "swig failed with error code %d: stdout=%s, stderr=%s",
-            return_code,
-            swig_stdout,
-            swig_stderr)
-        logging.error(
-            "command line:\n%s", ' '.join(command))
-        sys.exit(return_code)
-
-    logging.info("swig generation succeeded")
-    if swig_stdout is not None and len(swig_stdout) > 0:
-        logging.info("swig output: %s", swig_stdout)
-
-    # Move the depedency file we just generated to the proper location.
-    if options.generate_dependency_file:
-        if os.path.exists(temp_dep_file_path):
-            shutil.move(temp_dep_file_path, dependency_file)
-        else:
-            logging.error(
-                "failed to generate Python binding depedency file '%s'",
-                temp_dep_file_path)
-            if os.path.exists(dependency_file):
-                # Delete the old one.
-                os.remove(dependency_file)
-            sys.exit(-10)
-
-
-def get_python_module_path(options):
-    """Returns the location where the lldb Python module should be placed.
-
-    @param options dictionary of options parsed from the command line.
-
-    @return the directory where the lldb module should be placed.
-    """
-    if options.framework:
-        # Caller wants to use the OS X framework packaging.
-
-        # We are packaging in an OS X-style framework bundle. The
-        # module dir will be within the
-        # LLDB.framework/Resources/Python subdirectory.
-        return os.path.join(
-            options.target_dir,
-            "LLDB.framework",
-            "Resources",
-            "Python",
-            "lldb")
-    else:
-        from distutils.sysconfig import get_python_lib
-
-        if options.prefix is not None:
-            module_path = get_python_lib(True, False, options.prefix)
-        else:
-            module_path = get_python_lib(True, False)
-        return os.path.normcase(
-            os.path.join(module_path, "lldb"))
-
-
-def main(options):
-    """Pepares the Python language binding to LLDB.
-
-    @param options the parsed command line argument dictionary
-    """
-    # Setup generated dependency file options.
-    if options.generate_dependency_file:
-        dependency_file = os.path.normcase(os.path.join(
-            options.target_dir, "LLDBWrapPython.cpp.d"))
-    else:
-        dependency_file = None
-
-    # Keep track of all the swig-related settings.
-    settings = SwigSettings()
-
-    # Determine the final binding file path.
-    settings.output_file = os.path.normcase(
-        os.path.join(options.target_dir, "LLDBWrapPython.cpp"))
-
-    # Touch the output file (but don't really generate it) if python
-    # is disabled.
-    disable_python = os.getenv("LLDB_DISABLE_PYTHON", None)
-    if disable_python is not None and disable_python == "1":
-        remove_ignore_enoent(settings.output_file)
-        # Touch the file.
-        open(settings.output_file, 'w').close()
-        logging.info(
-            "Created empty python binding file due to LLDB_DISABLE_PYTHON "
-            "being set")
-        return
-
-    # We also check the GCC_PREPROCESSOR_DEFINITIONS to see if it
-    # contains LLDB_DISABLE_PYTHON.  If so, we skip generating
-    # the binding.
-    gcc_preprocessor_defs = os.getenv("GCC_PREPROCESSOR_DEFINITIONS", None)
-    if gcc_preprocessor_defs is not None:
-        if re.search(r"LLDB_DISABLE_PYTHON", gcc_preprocessor_defs):
-            remove_ignore_enoent(settings.output_file)
-            # Touch the file
-            open(settings.output_file, 'w').close()
-            logging.info(
-                "Created empty python binding file due to "
-                "finding LLDB_DISABLE_PYTHON in GCC_PREPROCESSOR_DEFINITIONS")
-            return
-
-    # Setup paths used during swig invocation.
-    settings.input_file = os.path.normcase(
-        os.path.join(options.src_root, "scripts", "lldb.swig"))
-    scripts_python_dir = os.path.dirname(os.path.realpath(__file__))
-    settings.extensions_file = os.path.normcase(
-        os.path.join(scripts_python_dir, "python-extensions.swig"))
-    settings.wrapper_file = os.path.normcase(
-        os.path.join(scripts_python_dir, "python-wrapper.swig"))
-    settings.typemaps_file = os.path.normcase(
-        os.path.join(scripts_python_dir, "python-typemaps.swig"))
-    settings.safecast_file = os.path.normcase(
-        os.path.join(scripts_python_dir, "python-swigsafecast.swig"))
-
-    settings.header_files = get_header_files(options)
-    settings.interface_files = get_interface_files(options)
-
-    generate_output = settings.output_out_of_date()
-
-    # Determine where to put the module.
-    python_module_path = get_python_module_path(options)
-    logging.info("python module path: %s", python_module_path)
-
-    # Handle the configuration build dir.
-    if options.config_build_dir is not None:
-        config_build_dir = options.config_build_dir
-    else:
-        config_build_dir = python_module_path
-
-    # Allow missing/non-link _lldb.so to force regeneration.
-    if not generate_output:
-        # Ensure the _lldb.so file exists.
-        so_path = os.path.join(python_module_path, "_lldb.so")
-        if not os.path.exists(so_path) or not os.path.islink(so_path):
-            logging.info("_lldb.so doesn't exist or isn't a symlink")
-            generate_output = True
-
-    # Allow missing __init__.py to force regeneration.
-    if not generate_output:
-        # Ensure the __init__.py for the lldb module can be found.
-        init_path = os.path.join(python_module_path, "__init__.py")
-        if not os.path.exists(init_path):
-            logging.info("__init__.py doesn't exist")
-            generate_output = True
-
-    if not generate_output:
-        logging.info(
-            "Skipping Python binding generation: everything is up to date")
-        return
-
-    # Generate the Python binding with swig.
-    logging.info("Python binding is out of date, regenerating")
-    do_swig_rebuild(options, dependency_file, config_build_dir, settings)
-
-
-# This script can be called by another Python script by calling the main()
-# function directly
-if __name__ == "__main__":
-    print("Script cannot be called directly.")
-    sys.exit(-1)
diff --git a/lldb/scripts/Python/python-typemaps.swig b/lldb/scripts/Python/python-typemaps.swig
index e3956faf20cc6..fe6a1798c53aa 100644
--- a/lldb/scripts/Python/python-typemaps.swig
+++ b/lldb/scripts/Python/python-typemaps.swig
@@ -372,6 +372,9 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
   $1 = $1 || PyCallable_Check(reinterpret_cast<PyObject*>($input));
 }
 
+// FIXME both of these paths wind up calling fdopen() with no provision for ever calling
+// fclose() on the result.  SB interfaces that use FILE* should be deprecated for scripting
+// use and this typemap should eventually be removed.
 %typemap(in) FILE * {
    using namespace lldb_private;
    if ($input == Py_None)
@@ -395,13 +398,10 @@ bool SetNumberFromPyObject<double>(double &number, PyObject *obj) {
    else
    {
       PythonFile py_file(PyRefType::Borrowed, $input);
-      File file;
-      if (!py_file.GetUnderlyingFile(file))
+      lldb::FileUP file = py_file.GetUnderlyingFile();
+      if (!file)
          return nullptr;
-
-      $1 = file.GetStream();
-      if ($1)
-         file.Clear();
+      $1 = file->TakeStreamAndClear();
     }
 }
 
diff --git a/lldb/scripts/Python/python-wrapper.swig b/lldb/scripts/Python/python-wrapper.swig
index 8509899df27b0..8a99daa90c833 100644
--- a/lldb/scripts/Python/python-wrapper.swig
+++ b/lldb/scripts/Python/python-wrapper.swig
@@ -250,6 +250,7 @@ LLDBSwigPythonCreateScriptedThreadPlan
 (
     const char *python_class_name,
     const char *session_dictionary_name,
+    std::string &error_string,
     const lldb::ThreadPlanSP& thread_plan_sp
 )
 {
@@ -267,8 +268,11 @@ LLDBSwigPythonCreateScriptedThreadPlan
     auto dict = PythonModule::MainModule().ResolveName<PythonDictionary>(session_dictionary_name);
     auto pfunc = PythonObject::ResolveNameWithDictionary<PythonCallable>(python_class_name, dict);
 
-    if (!pfunc.IsAllocated())
+    if (!pfunc.IsAllocated()) {
+        error_string.append("could not find script class: ");
+        error_string.append(python_class_name);
         return nullptr;
+    }
 
     PythonObject tp_arg(PyRefType::Owned, SBTypeToSWIGWrapper(tp_value));
 
diff --git a/lldb/scripts/get_relative_lib_dir.py b/lldb/scripts/get_relative_lib_dir.py
index f7020d653fddc..3afeeafd7b482 100644
--- a/lldb/scripts/get_relative_lib_dir.py
+++ b/lldb/scripts/get_relative_lib_dir.py
@@ -23,7 +23,7 @@ def get_python_relative_libdir():
     # right answer always.
     arch_specific_libdir = distutils.sysconfig.get_python_lib(True, False)
     split_libdir = arch_specific_libdir.split(os.sep)
-    lib_re = re.compile(r"^lib.+$")
+    lib_re = re.compile(r"^lib.*$")
 
     for i in range(len(split_libdir)):
         match = lib_re.match(split_libdir[i])
diff --git a/lldb/scripts/interface/SBBreakpoint.i b/lldb/scripts/interface/SBBreakpoint.i
index 8aabb2b3d280f..6df0b95808594 100644
--- a/lldb/scripts/interface/SBBreakpoint.i
+++ b/lldb/scripts/interface/SBBreakpoint.i
@@ -223,12 +223,6 @@ public:
     SBError
     AddLocation(SBAddress &address);
 
-    bool
-    operator == (const lldb::SBBreakpoint& rhs);
-
-    bool
-    operator != (const lldb::SBBreakpoint& rhs);
-
     static bool
     EventIsBreakpointEvent (const lldb::SBEvent &event);
 
diff --git a/lldb/scripts/interface/SBBreakpointName.i b/lldb/scripts/interface/SBBreakpointName.i
index f9feaa9a5e4ee..42dd423633860 100644
--- a/lldb/scripts/interface/SBBreakpointName.i
+++ b/lldb/scripts/interface/SBBreakpointName.i
@@ -34,8 +34,6 @@ public:
 
   ~SBBreakpointName();
 
-  const lldb::SBBreakpointName &operator=(const lldb::SBBreakpointName &rhs);
-
   // Tests to see if the opaque breakpoint object in this object matches the
   // opaque breakpoint object in "rhs".
   bool operator==(const lldb::SBBreakpointName &rhs);
diff --git a/lldb/scripts/interface/SBModule.i b/lldb/scripts/interface/SBModule.i
index ba8f5fba2a31c..06a4628d52b25 100644
--- a/lldb/scripts/interface/SBModule.i
+++ b/lldb/scripts/interface/SBModule.i
@@ -342,12 +342,6 @@ public:
     lldb::SBAddress
     GetObjectFileEntryPointAddress() const;
 
-    bool
-    operator == (const lldb::SBModule &rhs) const;
-
-    bool
-    operator != (const lldb::SBModule &rhs) const;
-
     %pythoncode %{
         def __len__(self):
             '''Return the number of symbols in a lldb.SBModule object.'''
diff --git a/lldb/scripts/interface/SBStructuredData.i b/lldb/scripts/interface/SBStructuredData.i
index 99b32474b171f..c7601bf6cf952 100644
--- a/lldb/scripts/interface/SBStructuredData.i
+++ b/lldb/scripts/interface/SBStructuredData.i
@@ -33,8 +33,6 @@ namespace lldb {
         void
         Clear();
 
-        lldb::SBStructuredData &operator=(const lldb::SBStructuredData &rhs);
-
         lldb::StructuredDataType GetType() const;
 
         size_t GetSize() const;
diff --git a/lldb/scripts/interface/SBThreadPlan.i b/lldb/scripts/interface/SBThreadPlan.i
index 05789f1fabfe8..edce0ee49c876 100644
--- a/lldb/scripts/interface/SBThreadPlan.i
+++ b/lldb/scripts/interface/SBThreadPlan.i
@@ -37,6 +37,9 @@ public:
 
    ~SBThreadPlan ();
 
+    bool
+    IsValid();
+
     bool
     IsValid() const;
 
@@ -89,11 +92,6 @@ public:
     bool
     IsPlanStale();
 
-    bool
-    IsValid();
-
-    explicit operator bool() const;
-
     SBThreadPlan
     QueueThreadPlanForStepOverRange (SBAddress &start_address,
                                      lldb::addr_t range_size);
diff --git a/lldb/scripts/lldb.swig b/lldb/scripts/lldb.swig
index 3cad53ad76298..ec39ad1b3358d 100644
--- a/lldb/scripts/lldb.swig
+++ b/lldb/scripts/lldb.swig
@@ -265,8 +265,8 @@ def lldb_iter(obj, getsize, getelem):
 debugger_unique_id = 0
 SBDebugger.Initialize()
 debugger = None
-target = SBTarget()
-process = SBProcess()
-thread = SBThread()
-frame = SBFrame()
+target = None
+process = None
+thread = None
+frame = None
 %}
diff --git a/lldb/scripts/prepare_bindings.py b/lldb/scripts/prepare_bindings.py
deleted file mode 100755
index 447fc5dc8f6e4..0000000000000
--- a/lldb/scripts/prepare_bindings.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/env python
-"""
-Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-See https://llvm.org/LICENSE.txt for license information.
-SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-Prepares language bindings for LLDB build process.  Run with --help
-to see a description of the supported command line arguments.
-"""
-
-# Python modules:
-import argparse
-import logging
-import os
-import platform
-import sys
-
-# LLDB modules:
-import use_lldb_suite
-from lldbsuite.support import fs
-
-
-def prepare_binding_for_language(scripts_dir, script_lang, options):
-    """Prepares the binding for a specific language.
-
-    @param scripts_dir the full path to the scripts source directory.
-    @param script_lang the name of the script language.  Should be a child
-    directory within the scripts dir, and should contain a
-    prepare_scripts_{script_lang}.py script file in it.
-    @param options the dictionary of parsed command line options.
-
-    There is no return value.  If it returns, the process succeeded; otherwise,
-    the process will exit where it fails.
-    """
-    # Ensure the language-specific prepare module exists.
-    script_name = "prepare_binding_{}.py".format(script_lang)
-    lang_path = os.path.join(scripts_dir, script_lang)
-    script_path = os.path.join(lang_path, script_name)
-    if not os.path.exists(script_path):
-        logging.error(
-            "failed to find prepare script for language '%s' at '%s'",
-            script_lang,
-            script_path)
-        sys.exit(-9)
-
-    # Include this language-specific directory in the Python search
-    # path.
-    sys.path.append(os.path.normcase(lang_path))
-
-    # Execute the specific language script
-    module_name = os.path.splitext(script_name)[0]
-    module = __import__(module_name)
-    module.main(options)
-
-    # Remove the language-specific directory from the Python search path.
-    sys.path.remove(os.path.normcase(lang_path))
-
-
-def prepare_all_bindings(options):
-    """Prepares bindings for each of the languages supported.
-
-    @param options the parsed arguments from the command line
-
-    @return the exit value for the program. 0 is success, all othes
-    indicate some kind of failure.
-    """
-    # Check for the existence of the SWIG scripts folder
-    scripts_dir = os.path.join(options.src_root, "scripts")
-    if not os.path.exists(scripts_dir):
-        logging.error("failed to find scripts dir: '%s'", scripts_dir)
-        sys.exit(-8)
-
-    child_dirs = ["Python"]
-
-    # Iterate script directory find any script language directories
-    for script_lang in child_dirs:
-        logging.info("executing language script for: '%s'", script_lang)
-        prepare_binding_for_language(scripts_dir, script_lang, options)
-
-
-def process_args(args):
-    """Returns options processed from the provided command line.
-
-    @param args the command line to process.
-    """
-
-    # Setup the parser arguments that are accepted.
-    parser = argparse.ArgumentParser(
-        description="Prepare language bindings for LLDB build.")
-
-    # Arguments to control logging verbosity.
-    parser.add_argument(
-        "--debug", "-d",
-        action="store_true",
-        help="Set program logging level to DEBUG.")
-    parser.add_argument(
-        "--verbose", "-v",
-        action="count",
-        default=0,
-        help=(
-            "Increase logging verbosity level.  Default: only error and "
-            "higher are displayed.  Each -v increases level of verbosity."))
-
-    # Arguments to control whether we're building an OS X-style
-    # framework.  This is the opposite of the older "-m" (makefile)
-    # option.
-    parser.add_argument(
-        "--config-build-dir",
-        "--cfgBldDir",
-        help=(
-            "Configuration build dir, will use python module path "
-            "if unspecified."))
-    parser.add_argument(
-        "--find-swig",
-        action="store_true",
-        help=(
-            "Indicates the swig executable should be searched for "
-            "if not eplicitly provided.  Either this or the explicit "
-            "swig executable option must be provided."))
-    parser.add_argument(
-        "--framework",
-        action="store_true",
-        help="Prepare as OS X-style framework.")
-    parser.add_argument(
-        "--generate-dependency-file",
-        "-M",
-        action="store_true",
-        help="Make the dependency (.d) file for the wrappers.")
-    parser.add_argument(
-        "--prefix",
-        help="Override path where the LLDB module is placed.")
-    parser.add_argument(
-        "--src-root",
-        "--srcRoot",
-        "-s",
-        # Default to the parent directory of this script's directory.
-        default=os.path.abspath(
-            os.path.join(
-                os.path.dirname(os.path.realpath(__file__)),
-                os.path.pardir)),
-        help="Specifies the LLDB source root directory.")
-    parser.add_argument(
-        "--swig-executable",
-        "--swigExecutable",
-        help="Path to the swig executable.")
-    parser.add_argument(
-        "--target-dir",
-        "--targetDir",
-        required=True,
-        help=(
-            "Specifies the build dir where the language binding "
-            "should be placed"))
-
-    parser.add_argument(
-        "--target-platform",
-        help=(
-            "Specifies the platform we are building for."
-            "Should be the same as what platform.system() returns."))
-    # Process args.
-    options = parser.parse_args(args)
-
-    # Set logging level based on verbosity count.
-    if options.debug:
-        log_level = logging.DEBUG
-    else:
-        # See logging documentation for error levels.  We'll default
-        # to showing ERROR or higher error messages.  For each -v
-        # specified, we'll shift to the next lower-priority log level.
-        log_level = logging.ERROR - 10 * options.verbose
-        if log_level < logging.NOTSET:
-            # Displays all logged messages.
-            log_level = logging.NOTSET
-    logging.basicConfig(level=log_level)
-    logging.info("logging is using level: %d", log_level)
-
-    return options
-
-
-def main(args):
-    """Drives the main script preparation steps.
-
-    @param args list of command line arguments.
-    """
-    # Process command line arguments.
-    options = process_args(args)
-    logging.debug("Processed args: options=%s", options)
-
-    # Ensure we have a swig executable.
-    if not options.swig_executable or len(options.swig_executable) == 0:
-        if options.find_swig:
-            try:
-                options.swig_executable = fs.find_executable("swig")
-            except Exception as e:
-                logging.error("Unable to find swig executable: %s" % e.message)
-                sys.exit(-6)
-        else:
-            logging.error(
-                "The --find-swig option must be specified "
-                "when the swig executable location is not "
-                "explicitly provided.")
-            sys.exit(-12)
-
-    # Check if the swig file exists.
-    swig_path = os.path.normcase(
-        os.path.join(options.src_root, "scripts", "lldb.swig"))
-    if not os.path.isfile(swig_path):
-        logging.error("swig file not found at '%s'", swig_path)
-        sys.exit(-3)
-
-    # Prepare bindings for each supported language binding.
-    # This will error out if it doesn't succeed.
-    prepare_all_bindings(options)
-    sys.exit(0)
-
-if __name__ == "__main__":
-    # Run the main driver loop.
-    main(sys.argv[1:])
diff --git a/lldb/source/API/SBCommandReturnObject.cpp b/lldb/source/API/SBCommandReturnObject.cpp
index 58740eeaade5d..6e01843944ca2 100644
--- a/lldb/source/API/SBCommandReturnObject.cpp
+++ b/lldb/source/API/SBCommandReturnObject.cpp
@@ -35,6 +35,7 @@ SBCommandReturnObject::SBCommandReturnObject(CommandReturnObject *ptr)
     : m_opaque_up(ptr) {
   LLDB_RECORD_CONSTRUCTOR(SBCommandReturnObject,
                           (lldb_private::CommandReturnObject *), ptr);
+  assert(ptr != nullptr);
 }
 
 SBCommandReturnObject::~SBCommandReturnObject() = default;
@@ -65,41 +66,34 @@ bool SBCommandReturnObject::IsValid() const {
 SBCommandReturnObject::operator bool() const {
   LLDB_RECORD_METHOD_CONST_NO_ARGS(bool, SBCommandReturnObject, operator bool);
 
-  return m_opaque_up != nullptr;
+  // This method is not useful but it needs to stay to keep SB API stable.
+  return true;
 }
 
 const char *SBCommandReturnObject::GetOutput() {
   LLDB_RECORD_METHOD_NO_ARGS(const char *, SBCommandReturnObject, GetOutput);
 
-  if (m_opaque_up) {
-    ConstString output(m_opaque_up->GetOutputData());
-    return output.AsCString(/*value_if_empty*/ "");
-  }
-
-  return nullptr;
+  ConstString output(m_opaque_up->GetOutputData());
+  return output.AsCString(/*value_if_empty*/ "");
 }
 
 const char *SBCommandReturnObject::GetError() {
   LLDB_RECORD_METHOD_NO_ARGS(const char *, SBCommandReturnObject, GetError);
 
-  if (m_opaque_up) {
-    ConstString output(m_opaque_up->GetErrorData());
-    return output.AsCString(/*value_if_empty*/ "");
-  }
-
-  return nullptr;
+  ConstString output(m_opaque_up->GetErrorData());
+  return output.AsCString(/*value_if_empty*/ "");
 }
 
 size_t SBCommandReturnObject::GetOutputSize() {
   LLDB_RECORD_METHOD_NO_ARGS(size_t, SBCommandReturnObject, GetOutputSize);
 
-  return (m_opaque_up ? m_opaque_up->GetOutputData().size() : 0);
+  return m_opaque_up->GetOutputData().size();
 }
 
 size_t SBCommandReturnObject::GetErrorSize() {
   LLDB_RECORD_METHOD_NO_ARGS(size_t, SBCommandReturnObject, GetErrorSize);
 
-  return (m_opaque_up ? m_opaque_up->GetErrorData().size() : 0);
+  return m_opaque_up->GetErrorData().size();
 }
 
 size_t SBCommandReturnObject::PutOutput(FILE *fh) {
@@ -127,51 +121,47 @@ size_t SBCommandReturnObject::PutError(FILE *fh) {
 void SBCommandReturnObject::Clear() {
   LLDB_RECORD_METHOD_NO_ARGS(void, SBCommandReturnObject, Clear);
 
-  if (m_opaque_up)
-    m_opaque_up->Clear();
+  m_opaque_up->Clear();
 }
 
 lldb::ReturnStatus SBCommandReturnObject::GetStatus() {
   LLDB_RECORD_METHOD_NO_ARGS(lldb::ReturnStatus, SBCommandReturnObject,
                              GetStatus);
 
-  return (m_opaque_up ? m_opaque_up->GetStatus() : lldb::eReturnStatusInvalid);
+  return m_opaque_up->GetStatus();
 }
 
 void SBCommandReturnObject::SetStatus(lldb::ReturnStatus status) {
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, SetStatus,
                      (lldb::ReturnStatus), status);
 
-  if (m_opaque_up)
-    m_opaque_up->SetStatus(status);
+  m_opaque_up->SetStatus(status);
 }
 
 bool SBCommandReturnObject::Succeeded() {
   LLDB_RECORD_METHOD_NO_ARGS(bool, SBCommandReturnObject, Succeeded);
 
-  return (m_opaque_up ? m_opaque_up->Succeeded() : false);
+  return m_opaque_up->Succeeded();
 }
 
 bool SBCommandReturnObject::HasResult() {
   LLDB_RECORD_METHOD_NO_ARGS(bool, SBCommandReturnObject, HasResult);
 
-  return (m_opaque_up ? m_opaque_up->HasResult() : false);
+  return m_opaque_up->HasResult();
 }
 
 void SBCommandReturnObject::AppendMessage(const char *message) {
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, AppendMessage, (const char *),
                      message);
 
-  if (m_opaque_up)
-    m_opaque_up->AppendMessage(message);
+  m_opaque_up->AppendMessage(message);
 }
 
 void SBCommandReturnObject::AppendWarning(const char *message) {
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, AppendWarning, (const char *),
                      message);
 
-  if (m_opaque_up)
-    m_opaque_up->AppendWarning(message);
+  m_opaque_up->AppendWarning(message);
 }
 
 CommandReturnObject *SBCommandReturnObject::operator->() const {
@@ -192,36 +182,28 @@ CommandReturnObject &SBCommandReturnObject::ref() const {
   return *(m_opaque_up.get());
 }
 
-void SBCommandReturnObject::SetLLDBObjectPtr(CommandReturnObject *ptr) {
-  if (m_opaque_up)
-    m_opaque_up.reset(ptr);
-}
-
 bool SBCommandReturnObject::GetDescription(SBStream &description) {
   LLDB_RECORD_METHOD(bool, SBCommandReturnObject, GetDescription,
                      (lldb::SBStream &), description);
 
   Stream &strm = description.ref();
 
-  if (m_opaque_up) {
-    description.Printf("Error:  ");
-    lldb::ReturnStatus status = m_opaque_up->GetStatus();
-    if (status == lldb::eReturnStatusStarted)
-      strm.PutCString("Started");
-    else if (status == lldb::eReturnStatusInvalid)
-      strm.PutCString("Invalid");
-    else if (m_opaque_up->Succeeded())
-      strm.PutCString("Success");
-    else
-      strm.PutCString("Fail");
-
-    if (GetOutputSize() > 0)
-      strm.Printf("\nOutput Message:\n%s", GetOutput());
-
-    if (GetErrorSize() > 0)
-      strm.Printf("\nError Message:\n%s", GetError());
-  } else
-    strm.PutCString("No value");
+  description.Printf("Error:  ");
+  lldb::ReturnStatus status = m_opaque_up->GetStatus();
+  if (status == lldb::eReturnStatusStarted)
+    strm.PutCString("Started");
+  else if (status == lldb::eReturnStatusInvalid)
+    strm.PutCString("Invalid");
+  else if (m_opaque_up->Succeeded())
+    strm.PutCString("Success");
+  else
+    strm.PutCString("Fail");
+
+  if (GetOutputSize() > 0)
+    strm.Printf("\nOutput Message:\n%s", GetOutput());
+
+  if (GetErrorSize() > 0)
+    strm.Printf("\nError Message:\n%s", GetError());
 
   return true;
 }
@@ -245,8 +227,7 @@ void SBCommandReturnObject::SetImmediateOutputFile(FILE *fh,
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, SetImmediateOutputFile,
                      (FILE *, bool), fh, transfer_ownership);
 
-  if (m_opaque_up)
-    m_opaque_up->SetImmediateOutputFile(fh, transfer_ownership);
+  m_opaque_up->SetImmediateOutputFile(fh, transfer_ownership);
 }
 
 void SBCommandReturnObject::SetImmediateErrorFile(FILE *fh,
@@ -254,31 +235,26 @@ void SBCommandReturnObject::SetImmediateErrorFile(FILE *fh,
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, SetImmediateErrorFile,
                      (FILE *, bool), fh, transfer_ownership);
 
-  if (m_opaque_up)
-    m_opaque_up->SetImmediateErrorFile(fh, transfer_ownership);
+  m_opaque_up->SetImmediateErrorFile(fh, transfer_ownership);
 }
 
 void SBCommandReturnObject::PutCString(const char *string, int len) {
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, PutCString,
                      (const char *, int), string, len);
 
-  if (m_opaque_up) {
-    if (len == 0 || string == nullptr || *string == 0) {
-      return;
-    } else if (len > 0) {
-      std::string buffer(string, len);
-      m_opaque_up->AppendMessage(buffer.c_str());
-    } else
-      m_opaque_up->AppendMessage(string);
-  }
+  if (len == 0 || string == nullptr || *string == 0) {
+    return;
+  } else if (len > 0) {
+    std::string buffer(string, len);
+    m_opaque_up->AppendMessage(buffer.c_str());
+  } else
+    m_opaque_up->AppendMessage(string);
 }
 
 const char *SBCommandReturnObject::GetOutput(bool only_if_no_immediate) {
   LLDB_RECORD_METHOD(const char *, SBCommandReturnObject, GetOutput, (bool),
                      only_if_no_immediate);
 
-  if (!m_opaque_up)
-    return nullptr;
   if (!only_if_no_immediate ||
       m_opaque_up->GetImmediateOutputStream().get() == nullptr)
     return GetOutput();
@@ -289,8 +265,6 @@ const char *SBCommandReturnObject::GetError(bool only_if_no_immediate) {
   LLDB_RECORD_METHOD(const char *, SBCommandReturnObject, GetError, (bool),
                      only_if_no_immediate);
 
-  if (!m_opaque_up)
-    return nullptr;
   if (!only_if_no_immediate ||
       m_opaque_up->GetImmediateErrorStream().get() == nullptr)
     return GetError();
@@ -298,14 +272,11 @@ const char *SBCommandReturnObject::GetError(bool only_if_no_immediate) {
 }
 
 size_t SBCommandReturnObject::Printf(const char *format, ...) {
-  if (m_opaque_up) {
-    va_list args;
-    va_start(args, format);
-    size_t result = m_opaque_up->GetOutputStream().PrintfVarArg(format, args);
-    va_end(args);
-    return result;
-  }
-  return 0;
+  va_list args;
+  va_start(args, format);
+  size_t result = m_opaque_up->GetOutputStream().PrintfVarArg(format, args);
+  va_end(args);
+  return result;
 }
 
 void SBCommandReturnObject::SetError(lldb::SBError &error,
@@ -314,19 +285,17 @@ void SBCommandReturnObject::SetError(lldb::SBError &error,
                      (lldb::SBError &, const char *), error,
                      fallback_error_cstr);
 
-  if (m_opaque_up) {
-    if (error.IsValid())
-      m_opaque_up->SetError(error.ref(), fallback_error_cstr);
-    else if (fallback_error_cstr)
-      m_opaque_up->SetError(Status(), fallback_error_cstr);
-  }
+  if (error.IsValid())
+    m_opaque_up->SetError(error.ref(), fallback_error_cstr);
+  else if (fallback_error_cstr)
+    m_opaque_up->SetError(Status(), fallback_error_cstr);
 }
 
 void SBCommandReturnObject::SetError(const char *error_cstr) {
   LLDB_RECORD_METHOD(void, SBCommandReturnObject, SetError, (const char *),
                      error_cstr);
 
-  if (m_opaque_up && error_cstr)
+  if (error_cstr)
     m_opaque_up->SetError(error_cstr);
 }
 
diff --git a/lldb/source/API/SBDebugger.cpp b/lldb/source/API/SBDebugger.cpp
index dcc3e83ccad4b..893ede5e41499 100644
--- a/lldb/source/API/SBDebugger.cpp
+++ b/lldb/source/API/SBDebugger.cpp
@@ -329,9 +329,8 @@ FILE *SBDebugger::GetInputFileHandle() {
   LLDB_RECORD_METHOD_NO_ARGS(FILE *, SBDebugger, GetInputFileHandle);
 
   if (m_opaque_sp) {
-    StreamFileSP stream_file_sp(m_opaque_sp->GetInputFile());
-    if (stream_file_sp)
-      return LLDB_RECORD_RESULT(stream_file_sp->GetFile().GetStream());
+    File &file_sp = m_opaque_sp->GetInputFile();
+    return LLDB_RECORD_RESULT(file_sp.GetStream());
   }
   return nullptr;
 }
@@ -340,9 +339,8 @@ FILE *SBDebugger::GetOutputFileHandle() {
   LLDB_RECORD_METHOD_NO_ARGS(FILE *, SBDebugger, GetOutputFileHandle);
 
   if (m_opaque_sp) {
-    StreamFileSP stream_file_sp(m_opaque_sp->GetOutputFile());
-    if (stream_file_sp)
-      return LLDB_RECORD_RESULT(stream_file_sp->GetFile().GetStream());
+    StreamFile &stream_file = m_opaque_sp->GetOutputStream();
+    return LLDB_RECORD_RESULT(stream_file.GetFile().GetStream());
   }
   return nullptr;
 }
@@ -351,9 +349,8 @@ FILE *SBDebugger::GetErrorFileHandle() {
   LLDB_RECORD_METHOD_NO_ARGS(FILE *, SBDebugger, GetErrorFileHandle);
 
   if (m_opaque_sp) {
-    StreamFileSP stream_file_sp(m_opaque_sp->GetErrorFile());
-    if (stream_file_sp)
-      return LLDB_RECORD_RESULT(stream_file_sp->GetFile().GetStream());
+    StreamFile &stream_file = m_opaque_sp->GetErrorStream();
+    return LLDB_RECORD_RESULT(stream_file.GetFile().GetStream());
   }
   return nullptr;
 }
diff --git a/lldb/source/API/SBModule.cpp b/lldb/source/API/SBModule.cpp
index 253a5393486ee..5eb0f54bed62e 100644
--- a/lldb/source/API/SBModule.cpp
+++ b/lldb/source/API/SBModule.cpp
@@ -503,16 +503,10 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) {
     const bool exact_match = false;
     ConstString name(type);
     llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    const uint32_t num_matches = module_sp->FindTypes(
-        name, exact_match, UINT32_MAX, searched_symbol_files, type_list);
+    module_sp->FindTypes(name, exact_match, UINT32_MAX, searched_symbol_files,
+                         type_list);
 
-    if (num_matches > 0) {
-      for (size_t idx = 0; idx < num_matches; idx++) {
-        TypeSP type_sp(type_list.GetTypeAtIndex(idx));
-        if (type_sp)
-          retval.Append(SBType(type_sp));
-      }
-    } else {
+    if (type_list.Empty()) {
       auto type_system_or_err =
           module_sp->GetTypeSystemForLanguage(eLanguageTypeC);
       if (auto err = type_system_or_err.takeError()) {
@@ -523,9 +517,14 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) {
         if (compiler_type)
           retval.Append(SBType(compiler_type));
       }
+    } else {
+      for (size_t idx = 0; idx < type_list.GetSize(); idx++) {
+        TypeSP type_sp(type_list.GetTypeAtIndex(idx));
+        if (type_sp)
+          retval.Append(SBType(type_sp));
+      }
     }
   }
-
   return LLDB_RECORD_RESULT(retval);
 }
 
diff --git a/lldb/source/API/SBStream.cpp b/lldb/source/API/SBStream.cpp
index ae652338e1ea5..910563824a88c 100644
--- a/lldb/source/API/SBStream.cpp
+++ b/lldb/source/API/SBStream.cpp
@@ -82,26 +82,27 @@ void SBStream::RedirectToFile(const char *path, bool append) {
     if (!m_is_file)
       local_data = static_cast<StreamString *>(m_opaque_up.get())->GetString();
   }
-  StreamFile *stream_file = new StreamFile;
   uint32_t open_options = File::eOpenOptionWrite | File::eOpenOptionCanCreate;
   if (append)
     open_options |= File::eOpenOptionAppend;
   else
     open_options |= File::eOpenOptionTruncate;
 
-  FileSystem::Instance().Open(stream_file->GetFile(), FileSpec(path),
-                              open_options);
-  m_opaque_up.reset(stream_file);
+  llvm::Expected<FileUP> file =
+      FileSystem::Instance().Open(FileSpec(path), open_options);
+  if (!file) {
+    LLDB_LOG_ERROR(GetLogIfAllCategoriesSet(LIBLLDB_LOG_API), file.takeError(),
+                   "Cannot open {1}: {0}", path);
+    return;
+  }
 
-  if (m_opaque_up) {
-    m_is_file = true;
-
-    // If we had any data locally in our StreamString, then pass that along to
-    // the to new file we are redirecting to.
-    if (!local_data.empty())
-      m_opaque_up->Write(&local_data[0], local_data.size());
-  } else
-    m_is_file = false;
+  m_opaque_up = std::make_unique<StreamFile>(std::move(file.get()));
+  m_is_file = true;
+
+  // If we had any data locally in our StreamString, then pass that along to
+  // the to new file we are redirecting to.
+  if (!local_data.empty())
+    m_opaque_up->Write(&local_data[0], local_data.size());
 }
 
 void SBStream::RedirectToFileHandle(FILE *fh, bool transfer_fh_ownership) {
@@ -118,17 +119,14 @@ void SBStream::RedirectToFileHandle(FILE *fh, bool transfer_fh_ownership) {
     if (!m_is_file)
       local_data = static_cast<StreamString *>(m_opaque_up.get())->GetString();
   }
-  m_opaque_up.reset(new StreamFile(fh, transfer_fh_ownership));
 
-  if (m_opaque_up) {
-    m_is_file = true;
-
-    // If we had any data locally in our StreamString, then pass that along to
-    // the to new file we are redirecting to.
-    if (!local_data.empty())
-      m_opaque_up->Write(&local_data[0], local_data.size());
-  } else
-    m_is_file = false;
+  m_opaque_up = std::make_unique<StreamFile>(fh, transfer_fh_ownership);
+  m_is_file = true;
+
+  // If we had any data locally in our StreamString, then pass that along to
+  // the to new file we are redirecting to.
+  if (!local_data.empty())
+    m_opaque_up->Write(&local_data[0], local_data.size());
 }
 
 void SBStream::RedirectToFileDescriptor(int fd, bool transfer_fh_ownership) {
@@ -143,16 +141,13 @@ void SBStream::RedirectToFileDescriptor(int fd, bool transfer_fh_ownership) {
       local_data = static_cast<StreamString *>(m_opaque_up.get())->GetString();
   }
 
-  m_opaque_up.reset(new StreamFile(::fdopen(fd, "w"), transfer_fh_ownership));
-  if (m_opaque_up) {
-    m_is_file = true;
-
-    // If we had any data locally in our StreamString, then pass that along to
-    // the to new file we are redirecting to.
-    if (!local_data.empty())
-      m_opaque_up->Write(&local_data[0], local_data.size());
-  } else
-    m_is_file = false;
+  m_opaque_up = std::make_unique<StreamFile>(fd, transfer_fh_ownership);
+  m_is_file = true;
+
+  // If we had any data locally in our StreamString, then pass that along to
+  // the to new file we are redirecting to.
+  if (!local_data.empty())
+    m_opaque_up->Write(&local_data[0], local_data.size());
 }
 
 lldb_private::Stream *SBStream::operator->() { return m_opaque_up.get(); }
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index 10cdf5b93ea7a..0d0a470e202f6 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -1891,16 +1891,13 @@ lldb::SBTypeList SBTarget::FindTypes(const char *typename_cstr) {
     bool exact_match = false;
     TypeList type_list;
     llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    uint32_t num_matches =
-        images.FindTypes(nullptr, const_typename, exact_match, UINT32_MAX,
-                         searched_symbol_files, type_list);
+    images.FindTypes(nullptr, const_typename, exact_match, UINT32_MAX,
+                     searched_symbol_files, type_list);
 
-    if (num_matches > 0) {
-      for (size_t idx = 0; idx < num_matches; idx++) {
-        TypeSP type_sp(type_list.GetTypeAtIndex(idx));
-        if (type_sp)
-          sb_type_list.Append(SBType(type_sp));
-      }
+    for (size_t idx = 0; idx < type_list.GetSize(); idx++) {
+      TypeSP type_sp(type_list.GetTypeAtIndex(idx));
+      if (type_sp)
+        sb_type_list.Append(SBType(type_sp));
     }
 
     // Try the loaded language runtimes
diff --git a/lldb/source/API/SystemInitializerFull.cpp b/lldb/source/API/SystemInitializerFull.cpp
index e7f2206b9a59d..ba6ad6f8c5efb 100644
--- a/lldb/source/API/SystemInitializerFull.cpp
+++ b/lldb/source/API/SystemInitializerFull.cpp
@@ -131,6 +131,38 @@ SystemInitializerFull::SystemInitializerFull() {}
 
 SystemInitializerFull::~SystemInitializerFull() {}
 
+#define LLDB_PROCESS_AArch64(op)                                               \
+  ABIMacOSX_arm64::op();                                                       \
+  ABISysV_arm64::op();
+#define LLDB_PROCESS_ARM(op)                                                   \
+  ABIMacOSX_arm::op();                                                         \
+  ABISysV_arm::op();
+#define LLDB_PROCESS_Hexagon(op) ABISysV_hexagon::op();
+#define LLDB_PROCESS_Mips(op)                                                  \
+  ABISysV_mips::op();                                                          \
+  ABISysV_mips64::op();
+#define LLDB_PROCESS_PowerPC(op)                                               \
+  ABISysV_ppc::op();                                                          \
+  ABISysV_ppc64::op();
+#define LLDB_PROCESS_SystemZ(op) ABISysV_s390x::op();
+#define LLDB_PROCESS_X86(op)                                                   \
+  ABIMacOSX_i386::op();                                                        \
+  ABISysV_i386::op();                                                          \
+  ABISysV_x86_64::op();                                                        \
+  ABIWindows_x86_64::op();
+
+#define LLDB_PROCESS_AMDGPU(op)
+#define LLDB_PROCESS_ARC(op)
+#define LLDB_PROCESS_AVR(op)
+#define LLDB_PROCESS_BPF(op)
+#define LLDB_PROCESS_Lanai(op)
+#define LLDB_PROCESS_MSP430(op)
+#define LLDB_PROCESS_NVPTX(op)
+#define LLDB_PROCESS_RISCV(op)
+#define LLDB_PROCESS_Sparc(op)
+#define LLDB_PROCESS_WebAssembly(op)
+#define LLDB_PROCESS_XCore(op)
+
 llvm::Error SystemInitializerFull::Initialize() {
   if (auto e = SystemInitializerCommon::Initialize())
     return e;
@@ -174,20 +206,8 @@ llvm::Error SystemInitializerFull::Initialize() {
 
   ClangASTContext::Initialize();
 
-  ABIMacOSX_i386::Initialize();
-  ABIMacOSX_arm::Initialize();
-  ABIMacOSX_arm64::Initialize();
-  ABISysV_arm::Initialize();
-  ABISysV_arm64::Initialize();
-  ABISysV_hexagon::Initialize();
-  ABISysV_i386::Initialize();
-  ABISysV_x86_64::Initialize();
-  ABISysV_ppc::Initialize();
-  ABISysV_ppc64::Initialize();
-  ABISysV_mips::Initialize();
-  ABISysV_mips64::Initialize();
-  ABISysV_s390x::Initialize();
-  ABIWindows_x86_64::Initialize();
+#define LLVM_TARGET(t) LLDB_PROCESS_ ## t(Initialize)
+#include "llvm/Config/Targets.def"
 
   ArchitectureArm::Initialize();
   ArchitectureMips::Initialize();
@@ -288,20 +308,9 @@ void SystemInitializerFull::Terminate() {
   ArchitectureMips::Terminate();
   ArchitecturePPC64::Terminate();
 
-  ABIMacOSX_i386::Terminate();
-  ABIMacOSX_arm::Terminate();
-  ABIMacOSX_arm64::Terminate();
-  ABISysV_arm::Terminate();
-  ABISysV_arm64::Terminate();
-  ABISysV_hexagon::Terminate();
-  ABISysV_i386::Terminate();
-  ABISysV_x86_64::Terminate();
-  ABISysV_ppc::Terminate();
-  ABISysV_ppc64::Terminate();
-  ABISysV_mips::Terminate();
-  ABISysV_mips64::Terminate();
-  ABISysV_s390x::Terminate();
-  ABIWindows_x86_64::Terminate();
+#define LLVM_TARGET(t) LLDB_PROCESS_ ## t(Terminate)
+#include "llvm/Config/Targets.def"
+
   DisassemblerLLVMC::Terminate();
 
   JITLoaderGDB::Terminate();
diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp
index f00a35d3d81b1..00ba108f3889a 100644
--- a/lldb/source/Commands/CommandCompletions.cpp
+++ b/lldb/source/Commands/CommandCompletions.cpp
@@ -308,10 +308,8 @@ void CommandCompletions::SettingsNames(CommandInterpreter &interpreter,
     }
   }
 
-  for (const std::string &s : g_property_names) {
-    if (llvm::StringRef(s).startswith(request.GetCursorArgumentPrefix()))
-      request.AddCompletion(s);
-  }
+  for (const std::string &s : g_property_names)
+    request.TryCompleteCurrentArg(s);
 }
 
 void CommandCompletions::PlatformPluginNames(CommandInterpreter &interpreter,
diff --git a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
index 11fdb85e84451..a6bcd1d8dc32e 100644
--- a/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectBreakpointCommand.cpp
@@ -221,7 +221,7 @@ are no syntax errors may indicate that a function was declared but never called.
   Options *GetOptions() override { return &m_options; }
 
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(g_reader_instructions);
       output_sp->Flush();
diff --git a/lldb/source/Commands/CommandObjectCommands.cpp b/lldb/source/Commands/CommandObjectCommands.cpp
index c20c624ab9044..259affbe6e0ac 100644
--- a/lldb/source/Commands/CommandObjectCommands.cpp
+++ b/lldb/source/Commands/CommandObjectCommands.cpp
@@ -945,7 +945,7 @@ a number follows 'f':"
 
 protected:
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString("Enter one or more sed substitution commands in "
                             "the form: 's/<regex>/<subst>/'.\nTerminate the "
@@ -1585,7 +1585,7 @@ class CommandObjectCommandsScriptAdd : public CommandObjectParsed,
   };
 
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(g_python_command_instructions);
       output_sp->Flush();
@@ -1594,7 +1594,7 @@ class CommandObjectCommandsScriptAdd : public CommandObjectParsed,
 
   void IOHandlerInputComplete(IOHandler &io_handler,
                               std::string &data) override {
-    StreamFileSP error_sp = io_handler.GetErrorStreamFile();
+    StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
 
     ScriptInterpreter *interpreter = GetDebugger().GetScriptInterpreter();
     if (interpreter) {
diff --git a/lldb/source/Commands/CommandObjectExpression.cpp b/lldb/source/Commands/CommandObjectExpression.cpp
index e8e86560971de..9bafdc1498041 100644
--- a/lldb/source/Commands/CommandObjectExpression.cpp
+++ b/lldb/source/Commands/CommandObjectExpression.cpp
@@ -498,8 +498,8 @@ void CommandObjectExpression::IOHandlerInputComplete(IOHandler &io_handler,
   //    StreamSP output_stream =
   //    io_handler.GetDebugger().GetAsyncOutputStream();
   //    StreamSP error_stream = io_handler.GetDebugger().GetAsyncErrorStream();
-  StreamFileSP output_sp(io_handler.GetOutputStreamFile());
-  StreamFileSP error_sp(io_handler.GetErrorStreamFile());
+  StreamFileSP output_sp = io_handler.GetOutputStreamFileSP();
+  StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
 
   EvaluateExpression(line.c_str(), output_sp.get(), error_sp.get());
   if (output_sp)
@@ -537,7 +537,7 @@ void CommandObjectExpression::GetMultilineExpression() {
                             1, // Show line numbers starting at 1
                             *this, nullptr));
 
-  StreamFileSP output_sp(io_handler_sp->GetOutputStreamFile());
+  StreamFileSP output_sp = io_handler_sp->GetOutputStreamFileSP();
   if (output_sp) {
     output_sp->PutCString(
         "Enter expressions, then terminate with an empty line to evaluate:\n");
diff --git a/lldb/source/Commands/CommandObjectFrame.cpp b/lldb/source/Commands/CommandObjectFrame.cpp
index 89efdab03c460..6a7facdaff35f 100644
--- a/lldb/source/Commands/CommandObjectFrame.cpp
+++ b/lldb/source/Commands/CommandObjectFrame.cpp
@@ -246,7 +246,7 @@ class CommandObjectFrameSelect : public CommandObjectParsed {
       Status error;
       const int short_option = m_getopt_table[option_idx].val;
       switch (short_option) {
-        case 'r': {
+      case 'r': {
         int32_t offset = 0;
         if (option_arg.getAsInteger(0, offset) || offset == INT32_MIN) {
           error.SetErrorStringWithFormat("invalid frame offset argument '%s'",
@@ -263,8 +263,10 @@ class CommandObjectFrameSelect : public CommandObjectParsed {
       return error;
     }
 
-    void OptionParsingStarting(ExecutionContext *execution_context) override {}
-    
+    void OptionParsingStarting(ExecutionContext *execution_context) override {
+      relative_frame_offset.reset();
+    }
+
     llvm::ArrayRef<OptionDefinition> GetDefinitions() override {
       return llvm::makeArrayRef(g_frame_select_options);
     }
@@ -314,7 +316,8 @@ class CommandObjectFrameSelect : public CommandObjectParsed {
         frame_idx = 0;
 
       if (*m_options.relative_frame_offset < 0) {
-        if (static_cast<int32_t>(frame_idx) >= -*m_options.relative_frame_offset)
+        if (static_cast<int32_t>(frame_idx) >=
+            -*m_options.relative_frame_offset)
           frame_idx += *m_options.relative_frame_offset;
         else {
           if (frame_idx == 0) {
diff --git a/lldb/source/Commands/CommandObjectGUI.cpp b/lldb/source/Commands/CommandObjectGUI.cpp
index 21ed510d1264c..09cb685a42f28 100644
--- a/lldb/source/Commands/CommandObjectGUI.cpp
+++ b/lldb/source/Commands/CommandObjectGUI.cpp
@@ -28,9 +28,8 @@ bool CommandObjectGUI::DoExecute(Args &args, CommandReturnObject &result) {
   if (args.GetArgumentCount() == 0) {
     Debugger &debugger = GetDebugger();
 
-    lldb::StreamFileSP input_sp = debugger.GetInputFile();
-    if (input_sp && input_sp->GetFile().GetIsRealTerminal() &&
-        input_sp->GetFile().GetIsInteractive()) {
+    File &input = debugger.GetInputFile();
+    if (input.GetIsRealTerminal() && input.GetIsInteractive()) {
       IOHandlerSP io_handler_sp(new IOHandlerCursesGUI(debugger));
       if (io_handler_sp)
         debugger.PushIOHandler(io_handler_sp);
diff --git a/lldb/source/Commands/CommandObjectHelp.cpp b/lldb/source/Commands/CommandObjectHelp.cpp
index 34f6e5d544dc9..c02a583bf9dfd 100644
--- a/lldb/source/Commands/CommandObjectHelp.cpp
+++ b/lldb/source/Commands/CommandObjectHelp.cpp
@@ -215,8 +215,7 @@ void CommandObjectHelp::HandleCompletion(CompletionRequest &request) {
   // user is getting help on...
 
   if (cmd_obj) {
-    request.GetParsedLine().Shift();
-    request.SetCursorIndex(request.GetCursorIndex() - 1);
+    request.ShiftArguments();
     cmd_obj->HandleCompletion(request);
     return;
   }
diff --git a/lldb/source/Commands/CommandObjectLog.cpp b/lldb/source/Commands/CommandObjectLog.cpp
index 3035a646f0609..31a876c3430ec 100644
--- a/lldb/source/Commands/CommandObjectLog.cpp
+++ b/lldb/source/Commands/CommandObjectLog.cpp
@@ -34,6 +34,21 @@ using namespace lldb_private;
 #define LLDB_OPTIONS_log
 #include "CommandOptions.inc"
 
+/// Common completion logic for log enable/disable.
+static void CompleteEnableDisable(CompletionRequest &request) {
+  size_t arg_index = request.GetCursorIndex();
+  if (arg_index == 0) { // We got: log enable/disable x[tab]
+    for (llvm::StringRef channel : Log::ListChannels())
+      request.TryCompleteCurrentArg(channel);
+  } else if (arg_index >= 1) { // We got: log enable/disable channel x[tab]
+    llvm::StringRef channel = request.GetParsedLine().GetArgumentAtIndex(0);
+    Log::ForEachChannelCategory(
+        channel, [&request](llvm::StringRef name, llvm::StringRef desc) {
+          request.TryCompleteCurrentArg(name, desc);
+        });
+  }
+}
+
 class CommandObjectLogEnable : public CommandObjectParsed {
 public:
   // Constructors and Destructors
@@ -134,6 +149,12 @@ class CommandObjectLogEnable : public CommandObjectParsed {
     uint32_t log_options;
   };
 
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    CompleteEnableDisable(request);
+  }
+
 protected:
   bool DoExecute(Args &args, CommandReturnObject &result) override {
     if (args.GetArgumentCount() < 2) {
@@ -202,6 +223,12 @@ class CommandObjectLogDisable : public CommandObjectParsed {
 
   ~CommandObjectLogDisable() override = default;
 
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    CompleteEnableDisable(request);
+  }
+
 protected:
   bool DoExecute(Args &args, CommandReturnObject &result) override {
     if (args.empty()) {
@@ -254,6 +281,13 @@ class CommandObjectLogList : public CommandObjectParsed {
 
   ~CommandObjectLogList() override = default;
 
+  void
+  HandleArgumentCompletion(CompletionRequest &request,
+                           OptionElementVector &opt_element_vector) override {
+    for (llvm::StringRef channel : Log::ListChannels())
+      request.TryCompleteCurrentArg(channel);
+  }
+
 protected:
   bool DoExecute(Args &args, CommandReturnObject &result) override {
     std::string output;
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index 1e9c9909f54fd..ca2b3f521416b 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -765,8 +765,8 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
     m_prev_varobj_options = m_varobj_options;
     m_prev_compiler_type = compiler_type;
 
-    StreamFile outfile_stream;
-    Stream *output_stream = nullptr;
+    std::unique_ptr<Stream> output_stream_storage;
+    Stream *output_stream_p = nullptr;
     const FileSpec &outfile_spec =
         m_outfile_options.GetFile().GetCurrentValue();
 
@@ -779,12 +779,14 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
       if (append)
         open_options |= File::eOpenOptionAppend;
 
-      Status error = FileSystem::Instance().Open(outfile_stream.GetFile(),
-                                                 outfile_spec, open_options);
-      if (error.Success()) {
+      auto outfile = FileSystem::Instance().Open(outfile_spec, open_options);
+
+      if (outfile) {
+        auto outfile_stream_up =
+            std::make_unique<StreamFile>(std::move(outfile.get()));
         if (m_memory_options.m_output_as_binary) {
           const size_t bytes_written =
-              outfile_stream.Write(data_sp->GetBytes(), bytes_read);
+              outfile_stream_up->Write(data_sp->GetBytes(), bytes_read);
           if (bytes_written > 0) {
             result.GetOutputStream().Printf(
                 "%zi bytes %s to '%s'\n", bytes_written,
@@ -800,16 +802,19 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
         } else {
           // We are going to write ASCII to the file just point the
           // output_stream to our outfile_stream...
-          output_stream = &outfile_stream;
+          output_stream_storage = std::move(outfile_stream_up);
+          output_stream_p = output_stream_storage.get();
         }
       } else {
-        result.AppendErrorWithFormat("Failed to open file '%s' for %s.\n",
+        result.AppendErrorWithFormat("Failed to open file '%s' for %s:\n",
                                      path.c_str(), append ? "append" : "write");
+
+        result.AppendError(llvm::toString(outfile.takeError()));
         result.SetStatus(eReturnStatusFailed);
         return false;
       }
     } else {
-      output_stream = &result.GetOutputStream();
+      output_stream_p = &result.GetOutputStream();
     }
 
     ExecutionContextScope *exe_scope = m_exe_ctx.GetBestExecutionContextScope();
@@ -829,7 +834,7 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
           DumpValueObjectOptions options(m_varobj_options.GetAsDumpOptions(
               eLanguageRuntimeDescriptionDisplayVerbosityFull, format));
 
-          valobj_sp->Dump(*output_stream, options);
+          valobj_sp->Dump(*output_stream_p, options);
         } else {
           result.AppendErrorWithFormat(
               "failed to create a value object for: (%s) %s\n",
@@ -869,13 +874,13 @@ class CommandObjectMemoryRead : public CommandObjectParsed {
       }
     }
 
-    assert(output_stream);
+    assert(output_stream_p);
     size_t bytes_dumped = DumpDataExtractor(
-        data, output_stream, 0, format, item_byte_size, item_count,
+        data, output_stream_p, 0, format, item_byte_size, item_count,
         num_per_line / target->GetArchitecture().GetDataByteSize(), addr, 0, 0,
         exe_scope);
     m_next_addr = addr + bytes_dumped;
-    output_stream->EOL();
+    output_stream_p->EOL();
     return true;
   }
 
diff --git a/lldb/source/Commands/CommandObjectMultiword.cpp b/lldb/source/Commands/CommandObjectMultiword.cpp
index e33a5e049693c..03a3770d8df70 100644
--- a/lldb/source/Commands/CommandObjectMultiword.cpp
+++ b/lldb/source/Commands/CommandObjectMultiword.cpp
@@ -94,8 +94,10 @@ bool CommandObjectMultiword::Execute(const char *args_string,
   }
 
   auto sub_command = args[0].ref();
-  if (sub_command.empty())
+  if (sub_command.empty()) {
+    result.AppendError("Need to specify a non-empty subcommand.");
     return result.Succeeded();
+  }
 
   if (sub_command.equals_lower("help")) {
     this->CommandObject::GenerateHelpText(result);
@@ -195,8 +197,7 @@ void CommandObjectMultiword::HandleCompletion(CompletionRequest &request) {
       if (cmd_obj != nullptr) {
         if (request.GetParsedLine().GetArgumentCount() != 1) {
           request.GetParsedLine().Shift();
-          request.SetCursorCharPosition(0);
-          request.GetParsedLine().AppendArgument(llvm::StringRef());
+          request.AppendEmptyArgument();
           cmd_obj->HandleCompletion(request);
         }
       }
@@ -214,8 +215,7 @@ void CommandObjectMultiword::HandleCompletion(CompletionRequest &request) {
   // Remove the one match that we got from calling GetSubcommandObject.
   new_matches.DeleteStringAtIndex(0);
   request.AddCompletions(new_matches);
-  request.GetParsedLine().Shift();
-  request.SetCursorIndex(request.GetCursorIndex() - 1);
+  request.ShiftArguments();
   sub_command_object->HandleCompletion(request);
 }
 
diff --git a/lldb/source/Commands/CommandObjectProcess.cpp b/lldb/source/Commands/CommandObjectProcess.cpp
index b494740c62a4f..e5aa78afabb31 100644
--- a/lldb/source/Commands/CommandObjectProcess.cpp
+++ b/lldb/source/Commands/CommandObjectProcess.cpp
@@ -429,7 +429,6 @@ class CommandObjectProcessAttach : public CommandObjectProcessLaunchOrAttach {
         result.AppendMessage(stream.GetString());
         result.SetStatus(eReturnStatusSuccessFinishNoResult);
         result.SetDidChangeProcessState(true);
-        result.SetAbnormalStopWasExpected(true);
       } else {
         result.AppendError(
             "no error returned from Target::Attach, and target has no process");
diff --git a/lldb/source/Commands/CommandObjectReproducer.cpp b/lldb/source/Commands/CommandObjectReproducer.cpp
index 895a9cc0b0d09..404702d3640e4 100644
--- a/lldb/source/Commands/CommandObjectReproducer.cpp
+++ b/lldb/source/Commands/CommandObjectReproducer.cpp
@@ -8,7 +8,9 @@
 
 #include "CommandObjectReproducer.h"
 
+#include "lldb/Host/OptionParser.h"
 #include "lldb/Utility/Reproducer.h"
+#include "lldb/Utility/GDBRemote.h"
 
 #include "lldb/Interpreter/CommandInterpreter.h"
 #include "lldb/Interpreter/CommandReturnObject.h"
@@ -16,7 +18,52 @@
 #include "lldb/Interpreter/OptionGroupBoolean.h"
 
 using namespace lldb;
+using namespace llvm;
 using namespace lldb_private;
+using namespace lldb_private::repro;
+
+enum ReproducerProvider {
+  eReproducerProviderCommands,
+  eReproducerProviderFiles,
+  eReproducerProviderGDB,
+  eReproducerProviderVersion,
+  eReproducerProviderNone
+};
+
+static constexpr OptionEnumValueElement g_reproducer_provider_type[] = {
+    {
+        eReproducerProviderCommands,
+        "commands",
+        "Command Interpreter Commands",
+    },
+    {
+        eReproducerProviderFiles,
+        "files",
+        "Files",
+    },
+    {
+        eReproducerProviderGDB,
+        "gdb",
+        "GDB Remote Packets",
+    },
+    {
+        eReproducerProviderVersion,
+        "version",
+        "Version",
+    },
+    {
+        eReproducerProviderNone,
+        "none",
+        "None",
+    },
+};
+
+static constexpr OptionEnumValues ReproducerProviderType() {
+  return OptionEnumValues(g_reproducer_provider_type);
+}
+
+#define LLDB_OPTIONS_reproducer
+#include "CommandOptions.inc"
 
 class CommandObjectReproducerGenerate : public CommandObjectParsed {
 public:
@@ -38,7 +85,7 @@ class CommandObjectReproducerGenerate : public CommandObjectParsed {
       return false;
     }
 
-    auto &r = repro::Reproducer::Instance();
+    auto &r = Reproducer::Instance();
     if (auto generator = r.GetGenerator()) {
       generator->Keep();
     } else if (r.GetLoader()) {
@@ -84,7 +131,7 @@ class CommandObjectReproducerStatus : public CommandObjectParsed {
       return false;
     }
 
-    auto &r = repro::Reproducer::Instance();
+    auto &r = Reproducer::Instance();
     if (r.GetGenerator()) {
       result.GetOutputStream() << "Reproducer is in capture mode.\n";
     } else if (r.GetLoader()) {
@@ -98,6 +145,208 @@ class CommandObjectReproducerStatus : public CommandObjectParsed {
   }
 };
 
+static void SetError(CommandReturnObject &result, Error err) {
+  result.GetErrorStream().Printf("error: %s\n",
+                                 toString(std::move(err)).c_str());
+  result.SetStatus(eReturnStatusFailed);
+}
+
+class CommandObjectReproducerDump : public CommandObjectParsed {
+public:
+  CommandObjectReproducerDump(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "reproducer dump",
+                            "Dump the information contained in a reproducer.",
+                            nullptr) {}
+
+  ~CommandObjectReproducerDump() override = default;
+
+  Options *GetOptions() override { return &m_options; }
+
+  class CommandOptions : public Options {
+  public:
+    CommandOptions() : Options(), file() {}
+
+    ~CommandOptions() override = default;
+
+    Status SetOptionValue(uint32_t option_idx, StringRef option_arg,
+                          ExecutionContext *execution_context) override {
+      Status error;
+      const int short_option = m_getopt_table[option_idx].val;
+
+      switch (short_option) {
+      case 'f':
+        file.SetFile(option_arg, FileSpec::Style::native);
+        FileSystem::Instance().Resolve(file);
+        break;
+      case 'p':
+        provider = (ReproducerProvider)OptionArgParser::ToOptionEnum(
+            option_arg, GetDefinitions()[option_idx].enum_values, 0, error);
+        if (!error.Success())
+          error.SetErrorStringWithFormat("unrecognized value for provider '%s'",
+                                         option_arg.str().c_str());
+        break;
+      default:
+        llvm_unreachable("Unimplemented option");
+      }
+
+      return error;
+    }
+
+    void OptionParsingStarting(ExecutionContext *execution_context) override {
+      file.Clear();
+      provider = eReproducerProviderNone;
+    }
+
+    ArrayRef<OptionDefinition> GetDefinitions() override {
+      return makeArrayRef(g_reproducer_options);
+    }
+
+    FileSpec file;
+    ReproducerProvider provider = eReproducerProviderNone;
+  };
+
+protected:
+  bool DoExecute(Args &command, CommandReturnObject &result) override {
+    if (!command.empty()) {
+      result.AppendErrorWithFormat("'%s' takes no arguments",
+                                   m_cmd_name.c_str());
+      return false;
+    }
+
+    // If no reproducer path is specified, use the loader currently used for
+    // replay. Otherwise create a new loader just for dumping.
+    llvm::Optional<Loader> loader_storage;
+    Loader *loader = nullptr;
+    if (!m_options.file) {
+      loader = Reproducer::Instance().GetLoader();
+      if (loader == nullptr) {
+        result.SetError(
+            "Not specifying a reproducer is only support during replay.");
+        result.SetStatus(eReturnStatusSuccessFinishNoResult);
+        return false;
+      }
+    } else {
+      loader_storage.emplace(m_options.file);
+      loader = &(*loader_storage);
+      if (Error err = loader->LoadIndex()) {
+        SetError(result, std::move(err));
+        return false;
+      }
+    }
+
+    // If we get here we should have a valid loader.
+    assert(loader);
+
+    switch (m_options.provider) {
+    case eReproducerProviderFiles: {
+      FileSpec vfs_mapping = loader->GetFile<FileProvider::Info>();
+
+      // Read the VFS mapping.
+      ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
+          vfs::getRealFileSystem()->getBufferForFile(vfs_mapping.GetPath());
+      if (!buffer) {
+        SetError(result, errorCodeToError(buffer.getError()));
+        return false;
+      }
+
+      // Initialize a VFS from the given mapping.
+      IntrusiveRefCntPtr<vfs::FileSystem> vfs = vfs::getVFSFromYAML(
+          std::move(buffer.get()), nullptr, vfs_mapping.GetPath());
+
+      // Dump the VFS to a buffer.
+      std::string str;
+      raw_string_ostream os(str);
+      static_cast<vfs::RedirectingFileSystem &>(*vfs).dump(os);
+      os.flush();
+
+      // Return the string.
+      result.AppendMessage(str);
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+      return true;
+    }
+    case eReproducerProviderVersion: {
+      FileSpec version_file = loader->GetFile<VersionProvider::Info>();
+
+      // Load the version info into a buffer.
+      ErrorOr<std::unique_ptr<MemoryBuffer>> buffer =
+          vfs::getRealFileSystem()->getBufferForFile(version_file.GetPath());
+      if (!buffer) {
+        SetError(result, errorCodeToError(buffer.getError()));
+        return false;
+      }
+
+      // Return the version string.
+      StringRef version = (*buffer)->getBuffer();
+      result.AppendMessage(version.str());
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+      return true;
+    }
+    case eReproducerProviderCommands: {
+      // Create a new command loader.
+      std::unique_ptr<repro::CommandLoader> command_loader =
+          repro::CommandLoader::Create(loader);
+      if (!command_loader) {
+        SetError(result,
+                 make_error<StringError>(llvm::inconvertibleErrorCode(),
+                                         "Unable to create command loader."));
+        return false;
+      }
+
+      // Iterate over the command files and dump them.
+      while (true) {
+        llvm::Optional<std::string> command_file =
+            command_loader->GetNextFile();
+        if (!command_file)
+          break;
+
+        auto command_buffer = llvm::MemoryBuffer::getFile(*command_file);
+        if (auto err = command_buffer.getError()) {
+          SetError(result, errorCodeToError(err));
+          return false;
+        }
+        result.AppendMessage((*command_buffer)->getBuffer());
+      }
+
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+      return true;
+    }
+    case eReproducerProviderGDB: {
+      FileSpec gdb_file = loader->GetFile<ProcessGDBRemoteProvider::Info>();
+      auto error_or_file = MemoryBuffer::getFile(gdb_file.GetPath());
+      if (auto err = error_or_file.getError()) {
+        SetError(result, errorCodeToError(err));
+        return false;
+      }
+
+      std::vector<GDBRemotePacket> packets;
+      yaml::Input yin((*error_or_file)->getBuffer());
+      yin >> packets;
+
+      if (auto err = yin.error()) {
+        SetError(result, errorCodeToError(err));
+        return false;
+      }
+
+      for (GDBRemotePacket& packet : packets) {
+        packet.Dump(result.GetOutputStream());
+      }
+
+      result.SetStatus(eReturnStatusSuccessFinishResult);
+      return true;
+    }
+    case eReproducerProviderNone:
+      result.SetError("No valid provider specified.");
+      return false;
+    }
+
+    result.SetStatus(eReturnStatusSuccessFinishNoResult);
+    return result.Succeeded();
+  }
+
+private:
+  CommandOptions m_options;
+};
+
 CommandObjectReproducer::CommandObjectReproducer(
     CommandInterpreter &interpreter)
     : CommandObjectMultiword(
@@ -109,6 +358,8 @@ CommandObjectReproducer::CommandObjectReproducer(
       CommandObjectSP(new CommandObjectReproducerGenerate(interpreter)));
   LoadSubCommand("status", CommandObjectSP(
                                new CommandObjectReproducerStatus(interpreter)));
+  LoadSubCommand("dump",
+                 CommandObjectSP(new CommandObjectReproducerDump(interpreter)));
 }
 
 CommandObjectReproducer::~CommandObjectReproducer() = default;
diff --git a/lldb/source/Commands/CommandObjectSettings.cpp b/lldb/source/Commands/CommandObjectSettings.cpp
index f1e3bf1277e06..6ed63c134f23f 100644
--- a/lldb/source/Commands/CommandObjectSettings.cpp
+++ b/lldb/source/Commands/CommandObjectSettings.cpp
@@ -130,9 +130,8 @@ insert-before or insert-after.");
 
     const size_t argc = request.GetParsedLine().GetArgumentCount();
     const char *arg = nullptr;
-    int setting_var_idx;
-    for (setting_var_idx = 0; setting_var_idx < static_cast<int>(argc);
-         ++setting_var_idx) {
+    size_t setting_var_idx;
+    for (setting_var_idx = 0; setting_var_idx < argc; ++setting_var_idx) {
       arg = request.GetParsedLine().GetArgumentAtIndex(setting_var_idx);
       if (arg && arg[0] != '-')
         break; // We found our setting variable name index
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 37e61ecf068c9..566de9ba071b0 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -1629,75 +1629,30 @@ static size_t LookupFunctionInModule(CommandInterpreter &interpreter,
 static size_t LookupTypeInModule(CommandInterpreter &interpreter, Stream &strm,
                                  Module *module, const char *name_cstr,
                                  bool name_is_regex) {
+  TypeList type_list;
   if (module && name_cstr && name_cstr[0]) {
-    TypeList type_list;
     const uint32_t max_num_matches = UINT32_MAX;
     size_t num_matches = 0;
     bool name_is_fully_qualified = false;
 
     ConstString name(name_cstr);
     llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-    num_matches =
-        module->FindTypes(name, name_is_fully_qualified, max_num_matches,
-                          searched_symbol_files, type_list);
-
-    if (num_matches) {
-      strm.Indent();
-      strm.Printf("%" PRIu64 " match%s found in ", (uint64_t)num_matches,
-                  num_matches > 1 ? "es" : "");
-      DumpFullpath(strm, &module->GetFileSpec(), 0);
-      strm.PutCString(":\n");
-      for (TypeSP type_sp : type_list.Types()) {
-        if (type_sp) {
-          // Resolve the clang type so that any forward references to types
-          // that haven't yet been parsed will get parsed.
-          type_sp->GetFullCompilerType();
-          type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
-          // Print all typedef chains
-          TypeSP typedef_type_sp(type_sp);
-          TypeSP typedefed_type_sp(typedef_type_sp->GetTypedefType());
-          while (typedefed_type_sp) {
-            strm.EOL();
-            strm.Printf("     typedef '%s': ",
-                        typedef_type_sp->GetName().GetCString());
-            typedefed_type_sp->GetFullCompilerType();
-            typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull,
-                                              true);
-            typedef_type_sp = typedefed_type_sp;
-            typedefed_type_sp = typedef_type_sp->GetTypedefType();
-          }
-        }
-        strm.EOL();
-      }
-    }
-    return num_matches;
-  }
-  return 0;
-}
+    module->FindTypes(name, name_is_fully_qualified, max_num_matches,
+                      searched_symbol_files, type_list);
 
-static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
-                             Module &module, const char *name_cstr,
-                             bool name_is_regex) {
-  TypeList type_list;
-  const uint32_t max_num_matches = UINT32_MAX;
-  size_t num_matches = 1;
-  bool name_is_fully_qualified = false;
+    if (type_list.Empty())
+      return 0;
 
-  ConstString name(name_cstr);
-  llvm::DenseSet<SymbolFile *> searched_symbol_files;
-  num_matches = module.FindTypes(name, name_is_fully_qualified, max_num_matches,
-                                 searched_symbol_files, type_list);
-
-  if (num_matches) {
     strm.Indent();
-    strm.PutCString("Best match found in ");
-    DumpFullpath(strm, &module.GetFileSpec(), 0);
+    strm.Printf("%" PRIu64 " match%s found in ", (uint64_t)num_matches,
+                num_matches > 1 ? "es" : "");
+    DumpFullpath(strm, &module->GetFileSpec(), 0);
     strm.PutCString(":\n");
-
-    TypeSP type_sp(type_list.GetTypeAtIndex(0));
-    if (type_sp) {
-      // Resolve the clang type so that any forward references to types that
-      // haven't yet been parsed will get parsed.
+    for (TypeSP type_sp : type_list.Types()) {
+      if (!type_sp)
+        continue;
+      // Resolve the clang type so that any forward references to types
+      // that haven't yet been parsed will get parsed.
       type_sp->GetFullCompilerType();
       type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
       // Print all typedef chains
@@ -1715,7 +1670,50 @@ static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
     }
     strm.EOL();
   }
-  return num_matches;
+  return type_list.GetSize();
+}
+
+static size_t LookupTypeHere(CommandInterpreter &interpreter, Stream &strm,
+                             Module &module, const char *name_cstr,
+                             bool name_is_regex) {
+  TypeList type_list;
+  const uint32_t max_num_matches = UINT32_MAX;
+  bool name_is_fully_qualified = false;
+
+  ConstString name(name_cstr);
+  llvm::DenseSet<SymbolFile *> searched_symbol_files;
+  module.FindTypes(name, name_is_fully_qualified, max_num_matches,
+                   searched_symbol_files, type_list);
+
+  if (type_list.Empty())
+    return 0;
+
+  strm.Indent();
+  strm.PutCString("Best match found in ");
+  DumpFullpath(strm, &module.GetFileSpec(), 0);
+  strm.PutCString(":\n");
+
+  TypeSP type_sp(type_list.GetTypeAtIndex(0));
+  if (type_sp) {
+    // Resolve the clang type so that any forward references to types that
+    // haven't yet been parsed will get parsed.
+    type_sp->GetFullCompilerType();
+    type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
+    // Print all typedef chains
+    TypeSP typedef_type_sp(type_sp);
+    TypeSP typedefed_type_sp(typedef_type_sp->GetTypedefType());
+    while (typedefed_type_sp) {
+      strm.EOL();
+      strm.Printf("     typedef '%s': ",
+                  typedef_type_sp->GetName().GetCString());
+      typedefed_type_sp->GetFullCompilerType();
+      typedefed_type_sp->GetDescription(&strm, eDescriptionLevelFull, true);
+      typedef_type_sp = typedefed_type_sp;
+      typedefed_type_sp = typedef_type_sp->GetTypedefType();
+    }
+  }
+  strm.EOL();
+  return type_list.GetSize();
 }
 
 static uint32_t LookupFileAndLineInModule(CommandInterpreter &interpreter,
@@ -4555,7 +4553,7 @@ class CommandObjectTargetStopHookAdd : public CommandObjectParsed,
 
 protected:
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(
           "Enter your stop hook command(s).  Type 'DONE' to end.\n");
@@ -4567,7 +4565,7 @@ class CommandObjectTargetStopHookAdd : public CommandObjectParsed,
                               std::string &line) override {
     if (m_stop_hook_sp) {
       if (line.empty()) {
-        StreamFileSP error_sp(io_handler.GetErrorStreamFile());
+        StreamFileSP error_sp(io_handler.GetErrorStreamFileSP());
         if (error_sp) {
           error_sp->Printf("error: stop hook #%" PRIu64
                            " aborted, no commands.\n",
@@ -4579,7 +4577,7 @@ class CommandObjectTargetStopHookAdd : public CommandObjectParsed,
           target->RemoveStopHookByID(m_stop_hook_sp->GetID());
       } else {
         m_stop_hook_sp->GetCommandPointer()->SplitIntoLines(line);
-        StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+        StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
         if (output_sp) {
           output_sp->Printf("Stop hook #%" PRIu64 " added.\n",
                             m_stop_hook_sp->GetID());
diff --git a/lldb/source/Commands/CommandObjectType.cpp b/lldb/source/Commands/CommandObjectType.cpp
index fe4cb57efaeff..5e31fd5e8bcee 100644
--- a/lldb/source/Commands/CommandObjectType.cpp
+++ b/lldb/source/Commands/CommandObjectType.cpp
@@ -151,7 +151,7 @@ class CommandObjectTypeSummaryAdd : public CommandObjectParsed,
         "for\n"
         "        internal_dict: an LLDB support object not to be used\"\"\"\n";
 
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(g_summary_addreader_instructions);
       output_sp->Flush();
@@ -160,7 +160,7 @@ class CommandObjectTypeSummaryAdd : public CommandObjectParsed,
 
   void IOHandlerInputComplete(IOHandler &io_handler,
                               std::string &data) override {
-    StreamFileSP error_sp = io_handler.GetErrorStreamFile();
+    StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
 
 #ifndef LLDB_DISABLE_PYTHON
     ScriptInterpreter *interpreter = GetDebugger().GetScriptInterpreter();
@@ -383,7 +383,7 @@ class CommandObjectTypeSynthAdd : public CommandObjectParsed,
   }
 
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(g_synth_addreader_instructions);
       output_sp->Flush();
@@ -392,7 +392,7 @@ class CommandObjectTypeSynthAdd : public CommandObjectParsed,
 
   void IOHandlerInputComplete(IOHandler &io_handler,
                               std::string &data) override {
-    StreamFileSP error_sp = io_handler.GetErrorStreamFile();
+    StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
 
 #ifndef LLDB_DISABLE_PYTHON
     ScriptInterpreter *interpreter = GetDebugger().GetScriptInterpreter();
diff --git a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
index 4e46abac95146..5683381efc858 100644
--- a/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
+++ b/lldb/source/Commands/CommandObjectWatchpointCommand.cpp
@@ -207,7 +207,7 @@ are no syntax errors may indicate that a function was declared but never called.
   Options *GetOptions() override { return &m_options; }
 
   void IOHandlerActivated(IOHandler &io_handler, bool interactive) override {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(
           "Enter your debugger command(s).  Type 'DONE' to end.\n");
diff --git a/lldb/source/Commands/Options.td b/lldb/source/Commands/Options.td
index d71029015eade..7791302c3da37 100644
--- a/lldb/source/Commands/Options.td
+++ b/lldb/source/Commands/Options.td
@@ -442,6 +442,15 @@ let Command = "log" in {
     Desc<"Prepend the names of files and function that generate the logs.">;
 }
 
+let Command = "reproducer" in {
+  def reproducer_provider : Option<"provider", "p">, Group<1>,
+    EnumArg<"None", "ReproducerProviderType()">,
+    Required, Desc<"The reproducer provider to dump.">;
+  def reproducer_file : Option<"file", "f">, Group<1>, Arg<"Filename">,
+    Desc<"The reproducer path. If a reproducer is replayed and no path is "
+    "provided, that reproducer is dumped.">;
+}
+
 let Command = "memory read" in {
   def memory_read_num_per_line : Option<"num-per-line", "l">, Group<1>,
     Arg<"NumberPerLine">, Desc<"The number of items per line to display.">;
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index 476e92878e059..e45b5e7c84ab7 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -65,8 +65,11 @@ add_lldb_library(lldbCore
   ValueObjectSyntheticFilter.cpp
   ValueObjectVariable.cpp
 
+  DEPENDS
+    clang-tablegen-targets
+
   LINK_LIBS
-    clangAST
+    clangDriver
     lldbBreakpoint
     lldbDataFormatters
     lldbExpression
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 2e293f57c81c7..fe2815029ca40 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -297,14 +297,12 @@ Status Debugger::SetPropertyValue(const ExecutionContext *exe_ctx,
         std::list<Status> errors;
         StreamString feedback_stream;
         if (!target_sp->LoadScriptingResources(errors, &feedback_stream)) {
-          StreamFileSP stream_sp(GetErrorFile());
-          if (stream_sp) {
-            for (auto error : errors) {
-              stream_sp->Printf("%s\n", error.AsCString());
-            }
-            if (feedback_stream.GetSize())
-              stream_sp->PutCString(feedback_stream.GetString());
+          Stream &s = GetErrorStream();
+          for (auto error : errors) {
+            s.Printf("%s\n", error.AsCString());
           }
+          if (feedback_stream.GetSize())
+            s.PutCString(feedback_stream.GetString());
         }
       }
     }
@@ -700,9 +698,9 @@ TargetSP Debugger::FindTargetWithProcess(Process *process) {
 Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
     : UserID(g_unique_id++),
       Properties(std::make_shared<OptionValueProperties>()),
-      m_input_file_sp(std::make_shared<StreamFile>(stdin, false)),
-      m_output_file_sp(std::make_shared<StreamFile>(stdout, false)),
-      m_error_file_sp(std::make_shared<StreamFile>(stderr, false)),
+      m_input_file_sp(std::make_shared<File>(stdin, false)),
+      m_output_stream_sp(std::make_shared<StreamFile>(stdout, false)),
+      m_error_stream_sp(std::make_shared<StreamFile>(stderr, false)),
       m_input_recorder(nullptr),
       m_broadcaster_manager_sp(BroadcasterManager::MakeBroadcasterManager()),
       m_terminal_state(), m_target_list(*this), m_platform_list(),
@@ -757,7 +755,7 @@ Debugger::Debugger(lldb::LogOutputCallback log_callback, void *baton)
   if (term && !strcmp(term, "dumb"))
     SetUseColor(false);
   // Turn off use-color if we don't write to a terminal with color support.
-  if (!m_output_file_sp->GetFile().GetIsTerminalWithColors())
+  if (!GetOutputFile().GetIsTerminalWithColors())
     SetUseColor(false);
 
 #if defined(_WIN32) && defined(ENABLE_VIRTUAL_TERMINAL_PROCESSING)
@@ -798,8 +796,7 @@ void Debugger::Clear() {
     // Close the input file _before_ we close the input read communications
     // class as it does NOT own the input file, our m_input_file does.
     m_terminal_state.Clear();
-    if (m_input_file_sp)
-      m_input_file_sp->GetFile().Close();
+    GetInputFile().Close();
 
     m_command_interpreter_up->Clear();
   });
@@ -827,14 +824,10 @@ repro::DataRecorder *Debugger::GetInputRecorder() { return m_input_recorder; }
 void Debugger::SetInputFileHandle(FILE *fh, bool tranfer_ownership,
                                   repro::DataRecorder *recorder) {
   m_input_recorder = recorder;
-  if (m_input_file_sp)
-    m_input_file_sp->GetFile().SetStream(fh, tranfer_ownership);
-  else
-    m_input_file_sp = std::make_shared<StreamFile>(fh, tranfer_ownership);
 
-  File &in_file = m_input_file_sp->GetFile();
-  if (!in_file.IsValid())
-    in_file.SetStream(stdin, true);
+  m_input_file_sp = std::make_shared<File>(fh, tranfer_ownership);
+  if (!m_input_file_sp->IsValid())
+    m_input_file_sp = std::make_shared<File>(stdin, false);
 
   // Save away the terminal state if that is relevant, so that we can restore
   // it in RestoreInputState.
@@ -842,39 +835,24 @@ void Debugger::SetInputFileHandle(FILE *fh, bool tranfer_ownership,
 }
 
 void Debugger::SetOutputFileHandle(FILE *fh, bool tranfer_ownership) {
-  if (m_output_file_sp)
-    m_output_file_sp->GetFile().SetStream(fh, tranfer_ownership);
-  else
-    m_output_file_sp = std::make_shared<StreamFile>(fh, tranfer_ownership);
-
-  File &out_file = m_output_file_sp->GetFile();
-  if (!out_file.IsValid())
-    out_file.SetStream(stdout, false);
+  FileSP file_sp = std::make_shared<File>(fh, tranfer_ownership);
+  if (!file_sp->IsValid())
+    file_sp = std::make_shared<File>(stdout, false);
+  m_output_stream_sp = std::make_shared<StreamFile>(file_sp);
 
-  // Do not create the ScriptInterpreter just for setting the output file
-  // handle as the constructor will know how to do the right thing on its own.
-  if (ScriptInterpreter *script_interpreter =
-          GetScriptInterpreter(/*can_create=*/false))
-    script_interpreter->ResetOutputFileHandle(fh);
 }
 
 void Debugger::SetErrorFileHandle(FILE *fh, bool tranfer_ownership) {
-  if (m_error_file_sp)
-    m_error_file_sp->GetFile().SetStream(fh, tranfer_ownership);
-  else
-    m_error_file_sp = std::make_shared<StreamFile>(fh, tranfer_ownership);
-
-  File &err_file = m_error_file_sp->GetFile();
-  if (!err_file.IsValid())
-    err_file.SetStream(stderr, false);
+  FileSP file_sp = std::make_shared<File>(fh, tranfer_ownership);
+  if (!file_sp->IsValid())
+    file_sp = std::make_shared<File>(stderr, false);
+  m_error_stream_sp = std::make_shared<StreamFile>(file_sp);
 }
 
 void Debugger::SaveInputTerminalState() {
-  if (m_input_file_sp) {
-    File &in_file = m_input_file_sp->GetFile();
-    if (in_file.GetDescriptor() != File::kInvalidDescriptor)
-      m_terminal_state.Save(in_file.GetDescriptor(), true);
-  }
+  int fd = GetInputFile().GetDescriptor();
+  if (fd != File::kInvalidDescriptor)
+    m_terminal_state.Save(fd, true);
 }
 
 void Debugger::RestoreInputTerminalState() { m_terminal_state.Restore(); }
@@ -955,8 +933,9 @@ bool Debugger::CheckTopIOHandlerTypes(IOHandler::Type top_type,
 }
 
 void Debugger::PrintAsync(const char *s, size_t len, bool is_stdout) {
-  lldb::StreamFileSP stream = is_stdout ? GetOutputFile() : GetErrorFile();
-  m_input_reader_stack.PrintAsync(stream.get(), s, len);
+  lldb_private::StreamFile &stream =
+      is_stdout ? GetOutputStream() : GetErrorStream();
+  m_input_reader_stack.PrintAsync(&stream, s, len);
 }
 
 ConstString Debugger::GetTopIOHandlerControlSequence(char ch) {
@@ -993,8 +972,7 @@ void Debugger::RunIOHandler(const IOHandlerSP &reader_sp) {
   }
 }
 
-void Debugger::AdoptTopIOHandlerFilesIfInvalid(StreamFileSP &in,
-                                               StreamFileSP &out,
+void Debugger::AdoptTopIOHandlerFilesIfInvalid(FileSP &in, StreamFileSP &out,
                                                StreamFileSP &err) {
   // Before an IOHandler runs, it must have in/out/err streams. This function
   // is called when one ore more of the streams are nullptr. We use the top
@@ -1006,20 +984,20 @@ void Debugger::AdoptTopIOHandlerFilesIfInvalid(StreamFileSP &in,
   // If no STDIN has been set, then set it appropriately
   if (!in) {
     if (top_reader_sp)
-      in = top_reader_sp->GetInputStreamFile();
+      in = top_reader_sp->GetInputFileSP();
     else
-      in = GetInputFile();
+      in = GetInputFileSP();
 
     // If there is nothing, use stdin
     if (!in)
-      in = std::make_shared<StreamFile>(stdin, false);
+      in = std::make_shared<File>(stdin, false);
   }
   // If no STDOUT has been set, then set it appropriately
   if (!out) {
     if (top_reader_sp)
-      out = top_reader_sp->GetOutputStreamFile();
+      out = top_reader_sp->GetOutputStreamFileSP();
     else
-      out = GetOutputFile();
+      out = GetOutputStreamSP();
 
     // If there is nothing, use stdout
     if (!out)
@@ -1028,13 +1006,13 @@ void Debugger::AdoptTopIOHandlerFilesIfInvalid(StreamFileSP &in,
   // If no STDERR has been set, then set it appropriately
   if (!err) {
     if (top_reader_sp)
-      err = top_reader_sp->GetErrorStreamFile();
+      err = top_reader_sp->GetErrorStreamFileSP();
     else
-      err = GetErrorFile();
+      err = GetErrorStreamSP();
 
     // If there is nothing, use stderr
     if (!err)
-      err = std::make_shared<StreamFile>(stdout, false);
+      err = std::make_shared<StreamFile>(stderr, false);
   }
 }
 
@@ -1202,7 +1180,7 @@ bool Debugger::EnableLog(llvm::StringRef channel,
         LLDB_LOG_OPTION_PREPEND_TIMESTAMP | LLDB_LOG_OPTION_PREPEND_THREAD_NAME;
   } else if (log_file.empty()) {
     log_stream_sp = std::make_shared<llvm::raw_fd_ostream>(
-        GetOutputFile()->GetFile().GetDescriptor(), !should_close, unbuffered);
+        GetOutputFile().GetDescriptor(), !should_close, unbuffered);
   } else {
     auto pos = m_log_streams.find(log_file);
     if (pos != m_log_streams.end())
@@ -1597,8 +1575,7 @@ bool Debugger::StartIOHandlerThread() {
 
 void Debugger::StopIOHandlerThread() {
   if (m_io_handler_thread.IsJoinable()) {
-    if (m_input_file_sp)
-      m_input_file_sp->GetFile().Close();
+    GetInputFile().Close();
     m_io_handler_thread.Join(nullptr);
   }
 }
diff --git a/lldb/source/Core/DumpDataExtractor.cpp b/lldb/source/Core/DumpDataExtractor.cpp
index 8ca31e0f2002c..12e98de2675c6 100644
--- a/lldb/source/Core/DumpDataExtractor.cpp
+++ b/lldb/source/Core/DumpDataExtractor.cpp
@@ -22,9 +22,6 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Stream.h"
 
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/CanonicalType.h"
-
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
diff --git a/lldb/source/Core/IOHandler.cpp b/lldb/source/Core/IOHandler.cpp
index 5394f16b7f981..6f4b1df6ad4c2 100644
--- a/lldb/source/Core/IOHandler.cpp
+++ b/lldb/source/Core/IOHandler.cpp
@@ -52,7 +52,7 @@
 
 #include "llvm/ADT/StringRef.h"
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 #include "lldb/Host/windows/windows.h"
 #endif
 
@@ -73,7 +73,7 @@ using namespace lldb_private;
 
 IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type)
     : IOHandler(debugger, type,
-                StreamFileSP(), // Adopt STDIN from top input reader
+                FileSP(),       // Adopt STDIN from top input reader
                 StreamFileSP(), // Adopt STDOUT from top input reader
                 StreamFileSP(), // Adopt STDERR from top input reader
                 0,              // Flags
@@ -81,7 +81,7 @@ IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type)
       ) {}
 
 IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type,
-                     const lldb::StreamFileSP &input_sp,
+                     const lldb::FileSP &input_sp,
                      const lldb::StreamFileSP &output_sp,
                      const lldb::StreamFileSP &error_sp, uint32_t flags,
                      repro::DataRecorder *data_recorder)
@@ -98,7 +98,7 @@ IOHandler::IOHandler(Debugger &debugger, IOHandler::Type type,
 IOHandler::~IOHandler() = default;
 
 int IOHandler::GetInputFD() {
-  return (m_input_sp ? m_input_sp->GetFile().GetDescriptor() : -1);
+  return (m_input_sp ? m_input_sp->GetDescriptor() : -1);
 }
 
 int IOHandler::GetOutputFD() {
@@ -110,7 +110,7 @@ int IOHandler::GetErrorFD() {
 }
 
 FILE *IOHandler::GetInputFILE() {
-  return (m_input_sp ? m_input_sp->GetFile().GetStream() : nullptr);
+  return (m_input_sp ? m_input_sp->GetStream() : nullptr);
 }
 
 FILE *IOHandler::GetOutputFILE() {
@@ -121,18 +121,18 @@ FILE *IOHandler::GetErrorFILE() {
   return (m_error_sp ? m_error_sp->GetFile().GetStream() : nullptr);
 }
 
-StreamFileSP &IOHandler::GetInputStreamFile() { return m_input_sp; }
+FileSP &IOHandler::GetInputFileSP() { return m_input_sp; }
 
-StreamFileSP &IOHandler::GetOutputStreamFile() { return m_output_sp; }
+StreamFileSP &IOHandler::GetOutputStreamFileSP() { return m_output_sp; }
 
-StreamFileSP &IOHandler::GetErrorStreamFile() { return m_error_sp; }
+StreamFileSP &IOHandler::GetErrorStreamFileSP() { return m_error_sp; }
 
 bool IOHandler::GetIsInteractive() {
-  return GetInputStreamFile()->GetFile().GetIsInteractive();
+  return GetInputFileSP() ? GetInputFileSP()->GetIsInteractive() : false;
 }
 
 bool IOHandler::GetIsRealTerminal() {
-  return GetInputStreamFile()->GetFile().GetIsRealTerminal();
+  return GetInputFileSP() ? GetInputFileSP()->GetIsRealTerminal() : false;
 }
 
 void IOHandler::SetPopped(bool b) { m_popped.SetValue(b, eBroadcastOnChange); }
@@ -235,7 +235,7 @@ IOHandlerEditline::IOHandlerEditline(
     bool multi_line, bool color_prompts, uint32_t line_number_start,
     IOHandlerDelegate &delegate, repro::DataRecorder *data_recorder)
     : IOHandlerEditline(debugger, type,
-                        StreamFileSP(), // Inherit input from top input reader
+                        FileSP(),       // Inherit input from top input reader
                         StreamFileSP(), // Inherit output from top input reader
                         StreamFileSP(), // Inherit error from top input reader
                         0,              // Flags
@@ -244,9 +244,9 @@ IOHandlerEditline::IOHandlerEditline(
                         line_number_start, delegate, data_recorder) {}
 
 IOHandlerEditline::IOHandlerEditline(
-    Debugger &debugger, IOHandler::Type type,
-    const lldb::StreamFileSP &input_sp, const lldb::StreamFileSP &output_sp,
-    const lldb::StreamFileSP &error_sp, uint32_t flags,
+    Debugger &debugger, IOHandler::Type type, const lldb::FileSP &input_sp,
+    const lldb::StreamFileSP &output_sp, const lldb::StreamFileSP &error_sp,
+    uint32_t flags,
     const char *editline_name, // Used for saving history files
     llvm::StringRef prompt, llvm::StringRef continuation_prompt,
     bool multi_line, bool color_prompts, uint32_t line_number_start,
@@ -266,7 +266,7 @@ IOHandlerEditline::IOHandlerEditline(
 #ifndef LLDB_DISABLE_LIBEDIT
   bool use_editline = false;
 
-  use_editline = m_input_sp->GetFile().GetIsRealTerminal();
+  use_editline = m_input_sp && m_input_sp->GetIsRealTerminal();
 
   if (use_editline) {
     m_editline_up.reset(new Editline(editline_name, GetInputFILE(),
@@ -579,7 +579,7 @@ void IOHandlerEditline::PrintAsync(Stream *stream, const char *s, size_t len) {
   else
 #endif
   {
-#ifdef _MSC_VER
+#ifdef _WIN32
     const char *prompt = GetPrompt();
     if (prompt) {
       // Back up over previous prompt using Windows API
@@ -594,9 +594,9 @@ void IOHandlerEditline::PrintAsync(Stream *stream, const char *s, size_t len) {
     }
 #endif
     IOHandler::PrintAsync(stream, s, len);
-#ifdef _MSC_VER
+#ifdef _WIN32
     if (prompt)
-      IOHandler::PrintAsync(GetOutputStreamFile().get(), prompt,
+      IOHandler::PrintAsync(GetOutputStreamFileSP().get(), prompt,
                             strlen(prompt));
 #endif
   }
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index 491ce93cdbf22..eb4119a385854 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -939,56 +939,49 @@ void Module::FindAddressesForLine(const lldb::TargetSP target_sp,
   }
 }
 
-size_t Module::FindTypes_Impl(
+void Module::FindTypes_Impl(
     ConstString name, const CompilerDeclContext *parent_decl_ctx,
-    bool append, size_t max_matches,
+    size_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     TypeMap &types) {
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(func_cat, LLVM_PRETTY_FUNCTION);
   if (SymbolFile *symbols = GetSymbolFile())
-    return symbols->FindTypes(name, parent_decl_ctx, append, max_matches,
-                              searched_symbol_files, types);
-  return 0;
+    symbols->FindTypes(name, parent_decl_ctx, max_matches,
+                       searched_symbol_files, types);
 }
 
-size_t Module::FindTypesInNamespace(ConstString type_name,
-                                    const CompilerDeclContext *parent_decl_ctx,
-                                    size_t max_matches, TypeList &type_list) {
-  const bool append = true;
+void Module::FindTypesInNamespace(ConstString type_name,
+                                  const CompilerDeclContext *parent_decl_ctx,
+                                  size_t max_matches, TypeList &type_list) {
   TypeMap types_map;
   llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  size_t num_types =
-      FindTypes_Impl(type_name, parent_decl_ctx, append, max_matches,
-                     searched_symbol_files, types_map);
-  if (num_types > 0) {
+  FindTypes_Impl(type_name, parent_decl_ctx, max_matches, searched_symbol_files,
+                 types_map);
+  if (types_map.GetSize()) {
     SymbolContext sc;
     sc.module_sp = shared_from_this();
     sc.SortTypeList(types_map, type_list);
   }
-  return num_types;
 }
 
 lldb::TypeSP Module::FindFirstType(const SymbolContext &sc,
                                    ConstString name, bool exact_match) {
   TypeList type_list;
   llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  const size_t num_matches =
-      FindTypes(name, exact_match, 1, searched_symbol_files, type_list);
-  if (num_matches)
+  FindTypes(name, exact_match, 1, searched_symbol_files, type_list);
+  if (type_list.GetSize())
     return type_list.GetTypeAtIndex(0);
   return TypeSP();
 }
 
-size_t Module::FindTypes(
+void Module::FindTypes(
     ConstString name, bool exact_match, size_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     TypeList &types) {
-  size_t num_matches = 0;
   const char *type_name_cstr = name.GetCString();
   llvm::StringRef type_scope;
   llvm::StringRef type_basename;
-  const bool append = true;
   TypeClass type_class = eTypeClassAny;
   TypeMap typesmap;
 
@@ -1001,49 +994,43 @@ size_t Module::FindTypes(
     exact_match = type_scope.consume_front("::");
 
     ConstString type_basename_const_str(type_basename);
-    if (FindTypes_Impl(type_basename_const_str, nullptr, append, max_matches,
-                       searched_symbol_files, typesmap)) {
+    FindTypes_Impl(type_basename_const_str, nullptr, max_matches,
+                   searched_symbol_files, typesmap);
+    if (typesmap.GetSize())
       typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class,
                                      exact_match);
-      num_matches = typesmap.GetSize();
-    }
   } else {
     // The type is not in a namespace/class scope, just search for it by
     // basename
     if (type_class != eTypeClassAny && !type_basename.empty()) {
       // The "type_name_cstr" will have been modified if we have a valid type
       // class prefix (like "struct", "class", "union", "typedef" etc).
-      FindTypes_Impl(ConstString(type_basename), nullptr, append, UINT_MAX,
+      FindTypes_Impl(ConstString(type_basename), nullptr, UINT_MAX,
                      searched_symbol_files, typesmap);
       typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class,
                                      exact_match);
-      num_matches = typesmap.GetSize();
     } else {
-      num_matches = FindTypes_Impl(name, nullptr, append, UINT_MAX,
-                                   searched_symbol_files, typesmap);
+      FindTypes_Impl(name, nullptr, UINT_MAX, searched_symbol_files, typesmap);
       if (exact_match) {
         std::string name_str(name.AsCString(""));
         typesmap.RemoveMismatchedTypes(type_scope, name_str, type_class,
                                        exact_match);
-        num_matches = typesmap.GetSize();
       }
     }
   }
-  if (num_matches > 0) {
+  if (typesmap.GetSize()) {
     SymbolContext sc;
     sc.module_sp = shared_from_this();
     sc.SortTypeList(typesmap, types);
   }
-  return num_matches;
 }
 
-size_t Module::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                         LanguageSet languages, bool append, TypeMap &types) {
+void Module::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                       LanguageSet languages, TypeMap &types) {
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(func_cat, LLVM_PRETTY_FUNCTION);
   if (SymbolFile *symbols = GetSymbolFile())
-    return symbols->FindTypes(pattern, languages, append, types);
-  return 0;
+    symbols->FindTypes(pattern, languages, types);
 }
 
 SymbolFile *Module::GetSymbolFile(bool can_create, Stream *feedback_strm) {
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 999ed6b74d7b0..317a2e3319664 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -17,6 +17,7 @@
 #include "lldb/Symbol/LocateSymbolFile.h"
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/SymbolContext.h"
+#include "lldb/Symbol/TypeList.h"
 #include "lldb/Symbol/VariableList.h"
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/ConstString.h"
@@ -56,9 +57,6 @@ class SymbolFile;
 namespace lldb_private {
 class Target;
 }
-namespace lldb_private {
-class TypeList;
-}
 
 using namespace lldb;
 using namespace lldb_private;
@@ -525,44 +523,36 @@ ModuleSP ModuleList::FindModule(const UUID &uuid) const {
   return module_sp;
 }
 
-size_t
-ModuleList::FindTypes(Module *search_first, ConstString name,
-                      bool name_is_fully_qualified, size_t max_matches,
-                      llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                      TypeList &types) const {
+void ModuleList::FindTypes(Module *search_first, ConstString name,
+                           bool name_is_fully_qualified, size_t max_matches,
+                           llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                           TypeList &types) const {
   std::lock_guard<std::recursive_mutex> guard(m_modules_mutex);
 
-  size_t total_matches = 0;
   collection::const_iterator pos, end = m_modules.end();
   if (search_first) {
     for (pos = m_modules.begin(); pos != end; ++pos) {
       if (search_first == pos->get()) {
-        total_matches +=
-            search_first->FindTypes(name, name_is_fully_qualified, max_matches,
-                                    searched_symbol_files, types);
+        search_first->FindTypes(name, name_is_fully_qualified, max_matches,
+                                searched_symbol_files, types);
 
-        if (total_matches >= max_matches)
-          break;
+        if (types.GetSize() >= max_matches)
+          return;
       }
     }
   }
 
-  if (total_matches < max_matches) {
-    for (pos = m_modules.begin(); pos != end; ++pos) {
-      // Search the module if the module is not equal to the one in the symbol
-      // context "sc". If "sc" contains a empty module shared pointer, then the
-      // comparison will always be true (valid_module_ptr != nullptr).
-      if (search_first != pos->get())
-        total_matches +=
-            (*pos)->FindTypes(name, name_is_fully_qualified, max_matches,
-                              searched_symbol_files, types);
-
-      if (total_matches >= max_matches)
-        break;
-    }
+  for (pos = m_modules.begin(); pos != end; ++pos) {
+    // Search the module if the module is not equal to the one in the symbol
+    // context "sc". If "sc" contains a empty module shared pointer, then the
+    // comparison will always be true (valid_module_ptr != nullptr).
+    if (search_first != pos->get())
+      (*pos)->FindTypes(name, name_is_fully_qualified, max_matches,
+                        searched_symbol_files, types);
+
+    if (types.GetSize() >= max_matches)
+      return;
   }
-
-  return total_matches;
 }
 
 bool ModuleList::FindSourceFile(const FileSpec &orig_spec,
diff --git a/lldb/source/Core/StreamFile.cpp b/lldb/source/Core/StreamFile.cpp
index ce6e1ea2c18f5..fe5d7413114ac 100644
--- a/lldb/source/Core/StreamFile.cpp
+++ b/lldb/source/Core/StreamFile.cpp
@@ -8,6 +8,7 @@
 
 #include "lldb/Core/StreamFile.h"
 #include "lldb/Host/FileSystem.h"
+#include "lldb/Utility/Log.h"
 
 #include <stdio.h>
 
@@ -15,35 +16,54 @@ using namespace lldb;
 using namespace lldb_private;
 
 // StreamFile constructor
-StreamFile::StreamFile() : Stream(), m_file() {}
+StreamFile::StreamFile() : Stream() { m_file_sp = std::make_shared<File>(); }
 
 StreamFile::StreamFile(uint32_t flags, uint32_t addr_size, ByteOrder byte_order)
-    : Stream(flags, addr_size, byte_order), m_file() {}
+    : Stream(flags, addr_size, byte_order) {
+  m_file_sp = std::make_shared<File>();
+}
 
-StreamFile::StreamFile(int fd, bool transfer_ownership)
-    : Stream(), m_file(fd, transfer_ownership) {}
+StreamFile::StreamFile(int fd, bool transfer_ownership) : Stream() {
+  m_file_sp =
+      std::make_shared<File>(fd, File::eOpenOptionWrite, transfer_ownership);
+}
 
-StreamFile::StreamFile(FILE *fh, bool transfer_ownership)
-    : Stream(), m_file(fh, transfer_ownership) {}
+StreamFile::StreamFile(FILE *fh, bool transfer_ownership) : Stream() {
+  m_file_sp = std::make_shared<File>(fh, transfer_ownership);
+}
 
-StreamFile::StreamFile(const char *path) : Stream(), m_file() {
-  FileSystem::Instance().Open(m_file, FileSpec(path),
-                              File::eOpenOptionWrite |
-                                  File::eOpenOptionCanCreate |
-                                  File::eOpenOptionCloseOnExec);
+StreamFile::StreamFile(const char *path) : Stream() {
+  auto file = FileSystem::Instance().Open(
+      FileSpec(path), File::eOpenOptionWrite | File::eOpenOptionCanCreate |
+                          File::eOpenOptionCloseOnExec);
+  if (file)
+    m_file_sp = std::move(file.get());
+  else {
+    // TODO refactor this so the error gets popagated up instead of logged here.
+    LLDB_LOG_ERROR(GetLogIfAllCategoriesSet(LIBLLDB_LOG_HOST), file.takeError(),
+                   "Cannot open {1}: {0}", path);
+    m_file_sp = std::make_shared<File>();
+  }
 }
 
 StreamFile::StreamFile(const char *path, uint32_t options, uint32_t permissions)
-    : Stream(), m_file() {
-
-  FileSystem::Instance().Open(m_file, FileSpec(path), options, permissions);
+    : Stream() {
+  auto file = FileSystem::Instance().Open(FileSpec(path), options, permissions);
+  if (file)
+    m_file_sp = std::move(file.get());
+  else {
+    // TODO refactor this so the error gets popagated up instead of logged here.
+    LLDB_LOG_ERROR(GetLogIfAllCategoriesSet(LIBLLDB_LOG_HOST), file.takeError(),
+                   "Cannot open {1}: {0}", path);
+    m_file_sp = std::make_shared<File>();
+  }
 }
 
 StreamFile::~StreamFile() {}
 
-void StreamFile::Flush() { m_file.Flush(); }
+void StreamFile::Flush() { m_file_sp->Flush(); }
 
 size_t StreamFile::WriteImpl(const void *s, size_t length) {
-  m_file.Write(s, length);
+  m_file_sp->Write(s, length);
   return length;
 }
diff --git a/lldb/source/DataFormatters/TypeFormat.cpp b/lldb/source/DataFormatters/TypeFormat.cpp
index b526e9a744bc4..b272c2e8dc3b3 100644
--- a/lldb/source/DataFormatters/TypeFormat.cpp
+++ b/lldb/source/DataFormatters/TypeFormat.cpp
@@ -164,7 +164,7 @@ bool TypeFormatImpl_EnumType::FormatObject(ValueObject *valobj,
     llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
     images.FindTypes(nullptr, m_enum_type, false, UINT32_MAX,
                      searched_symbol_files, types);
-    if (types.GetSize() == 0)
+    if (types.Empty())
       return false;
     for (lldb::TypeSP type_sp : types.Types()) {
       if (!type_sp)
diff --git a/lldb/source/Expression/DWARFExpression.cpp b/lldb/source/Expression/DWARFExpression.cpp
index 48bfa72a19b5d..d1ef95726855f 100644
--- a/lldb/source/Expression/DWARFExpression.cpp
+++ b/lldb/source/Expression/DWARFExpression.cpp
@@ -88,413 +88,10 @@ void DWARFExpression::DumpLocation(Stream *s, lldb::offset_t offset,
                                    lldb::offset_t length,
                                    lldb::DescriptionLevel level,
                                    ABI *abi) const {
-  if (!m_data.ValidOffsetForDataOfSize(offset, length))
-    return;
-  const lldb::offset_t start_offset = offset;
-  const lldb::offset_t end_offset = offset + length;
-
-  // An operation within a DWARF expression may contain a sub-expression. The
-  // motivating example for this is DW_OP_entry_value. Keep track of where each
-  // each sub-expression ends.
-  std::vector<lldb::offset_t> ends_of_subexprs;
-
-  // "Finish" (i.e. print the closing right-parens) for sub-expressions up to
-  // the specified \p op_offset.
-  auto finish_subexpressions_to = [&](const lldb::offset_t op_offset) {
-    while (!ends_of_subexprs.empty() && op_offset >= ends_of_subexprs.back()) {
-      ends_of_subexprs.pop_back();
-      s->Printf(")");
-      if (!ends_of_subexprs.empty())
-        s->Printf(" ");
-    }
-  };
-
-  while (m_data.ValidOffset(offset) && offset < end_offset) {
-    const lldb::offset_t op_offset = offset;
-    const uint8_t op = m_data.GetU8(&offset);
-    finish_subexpressions_to(op_offset);
-
-    switch (level) {
-    default:
-      break;
-
-    case lldb::eDescriptionLevelBrief:
-      if (op_offset > start_offset)
-        s->PutChar(' ');
-      break;
-
-    case lldb::eDescriptionLevelFull:
-    case lldb::eDescriptionLevelVerbose:
-      if (op_offset > start_offset)
-        s->EOL();
-      s->Indent();
-      if (level == lldb::eDescriptionLevelFull)
-        break;
-      // Fall through for verbose and print offset and DW_OP prefix..
-      s->Printf("0x%8.8" PRIx64 ": %s", op_offset,
-                op >= DW_OP_APPLE_uninit ? "DW_OP_APPLE_" : "DW_OP_");
-      break;
-    }
-
-    switch (op) {
-    case DW_OP_addr:
-      *s << "DW_OP_addr(" << m_data.GetAddress(&offset) << ") ";
-      break; // 0x03 1 address
-    case DW_OP_deref:
-      *s << "DW_OP_deref";
-      break; // 0x06
-    case DW_OP_const1u:
-      s->Printf("DW_OP_const1u(0x%2.2x)", m_data.GetU8(&offset));
-      break; // 0x08 1 1-byte constant
-    case DW_OP_const1s:
-      s->Printf("DW_OP_const1s(0x%2.2x)", m_data.GetU8(&offset));
-      break; // 0x09 1 1-byte constant
-    case DW_OP_const2u:
-      s->Printf("DW_OP_const2u(0x%4.4x)", m_data.GetU16(&offset));
-      break; // 0x0a 1 2-byte constant
-    case DW_OP_const2s:
-      s->Printf("DW_OP_const2s(0x%4.4x)", m_data.GetU16(&offset));
-      break; // 0x0b 1 2-byte constant
-    case DW_OP_const4u:
-      s->Printf("DW_OP_const4u(0x%8.8x)", m_data.GetU32(&offset));
-      break; // 0x0c 1 4-byte constant
-    case DW_OP_const4s:
-      s->Printf("DW_OP_const4s(0x%8.8x)", m_data.GetU32(&offset));
-      break; // 0x0d 1 4-byte constant
-    case DW_OP_const8u:
-      s->Printf("DW_OP_const8u(0x%16.16" PRIx64 ")", m_data.GetU64(&offset));
-      break; // 0x0e 1 8-byte constant
-    case DW_OP_const8s:
-      s->Printf("DW_OP_const8s(0x%16.16" PRIx64 ")", m_data.GetU64(&offset));
-      break; // 0x0f 1 8-byte constant
-    case DW_OP_constu:
-      s->Printf("DW_OP_constu(0x%" PRIx64 ")", m_data.GetULEB128(&offset));
-      break; // 0x10 1 ULEB128 constant
-    case DW_OP_consts:
-      s->Printf("DW_OP_consts(0x%" PRId64 ")", m_data.GetSLEB128(&offset));
-      break; // 0x11 1 SLEB128 constant
-    case DW_OP_dup:
-      s->PutCString("DW_OP_dup");
-      break; // 0x12
-    case DW_OP_drop:
-      s->PutCString("DW_OP_drop");
-      break; // 0x13
-    case DW_OP_over:
-      s->PutCString("DW_OP_over");
-      break; // 0x14
-    case DW_OP_pick:
-      s->Printf("DW_OP_pick(0x%2.2x)", m_data.GetU8(&offset));
-      break; // 0x15 1 1-byte stack index
-    case DW_OP_swap:
-      s->PutCString("DW_OP_swap");
-      break; // 0x16
-    case DW_OP_rot:
-      s->PutCString("DW_OP_rot");
-      break; // 0x17
-    case DW_OP_xderef:
-      s->PutCString("DW_OP_xderef");
-      break; // 0x18
-    case DW_OP_abs:
-      s->PutCString("DW_OP_abs");
-      break; // 0x19
-    case DW_OP_and:
-      s->PutCString("DW_OP_and");
-      break; // 0x1a
-    case DW_OP_div:
-      s->PutCString("DW_OP_div");
-      break; // 0x1b
-    case DW_OP_minus:
-      s->PutCString("DW_OP_minus");
-      break; // 0x1c
-    case DW_OP_mod:
-      s->PutCString("DW_OP_mod");
-      break; // 0x1d
-    case DW_OP_mul:
-      s->PutCString("DW_OP_mul");
-      break; // 0x1e
-    case DW_OP_neg:
-      s->PutCString("DW_OP_neg");
-      break; // 0x1f
-    case DW_OP_not:
-      s->PutCString("DW_OP_not");
-      break; // 0x20
-    case DW_OP_or:
-      s->PutCString("DW_OP_or");
-      break; // 0x21
-    case DW_OP_plus:
-      s->PutCString("DW_OP_plus");
-      break;                // 0x22
-    case DW_OP_plus_uconst: // 0x23 1 ULEB128 addend
-      s->Printf("DW_OP_plus_uconst(0x%" PRIx64 ")",
-                m_data.GetULEB128(&offset));
-      break;
-
-    case DW_OP_shl:
-      s->PutCString("DW_OP_shl");
-      break; // 0x24
-    case DW_OP_shr:
-      s->PutCString("DW_OP_shr");
-      break; // 0x25
-    case DW_OP_shra:
-      s->PutCString("DW_OP_shra");
-      break; // 0x26
-    case DW_OP_xor:
-      s->PutCString("DW_OP_xor");
-      break; // 0x27
-    case DW_OP_skip:
-      s->Printf("DW_OP_skip(0x%4.4x)", m_data.GetU16(&offset));
-      break; // 0x2f 1 signed 2-byte constant
-    case DW_OP_bra:
-      s->Printf("DW_OP_bra(0x%4.4x)", m_data.GetU16(&offset));
-      break; // 0x28 1 signed 2-byte constant
-    case DW_OP_eq:
-      s->PutCString("DW_OP_eq");
-      break; // 0x29
-    case DW_OP_ge:
-      s->PutCString("DW_OP_ge");
-      break; // 0x2a
-    case DW_OP_gt:
-      s->PutCString("DW_OP_gt");
-      break; // 0x2b
-    case DW_OP_le:
-      s->PutCString("DW_OP_le");
-      break; // 0x2c
-    case DW_OP_lt:
-      s->PutCString("DW_OP_lt");
-      break; // 0x2d
-    case DW_OP_ne:
-      s->PutCString("DW_OP_ne");
-      break; // 0x2e
-
-    case DW_OP_lit0:  // 0x30
-    case DW_OP_lit1:  // 0x31
-    case DW_OP_lit2:  // 0x32
-    case DW_OP_lit3:  // 0x33
-    case DW_OP_lit4:  // 0x34
-    case DW_OP_lit5:  // 0x35
-    case DW_OP_lit6:  // 0x36
-    case DW_OP_lit7:  // 0x37
-    case DW_OP_lit8:  // 0x38
-    case DW_OP_lit9:  // 0x39
-    case DW_OP_lit10: // 0x3A
-    case DW_OP_lit11: // 0x3B
-    case DW_OP_lit12: // 0x3C
-    case DW_OP_lit13: // 0x3D
-    case DW_OP_lit14: // 0x3E
-    case DW_OP_lit15: // 0x3F
-    case DW_OP_lit16: // 0x40
-    case DW_OP_lit17: // 0x41
-    case DW_OP_lit18: // 0x42
-    case DW_OP_lit19: // 0x43
-    case DW_OP_lit20: // 0x44
-    case DW_OP_lit21: // 0x45
-    case DW_OP_lit22: // 0x46
-    case DW_OP_lit23: // 0x47
-    case DW_OP_lit24: // 0x48
-    case DW_OP_lit25: // 0x49
-    case DW_OP_lit26: // 0x4A
-    case DW_OP_lit27: // 0x4B
-    case DW_OP_lit28: // 0x4C
-    case DW_OP_lit29: // 0x4D
-    case DW_OP_lit30: // 0x4E
-    case DW_OP_lit31:
-      s->Printf("DW_OP_lit%i", op - DW_OP_lit0);
-      break; // 0x4f
-
-    case DW_OP_reg0:  // 0x50
-    case DW_OP_reg1:  // 0x51
-    case DW_OP_reg2:  // 0x52
-    case DW_OP_reg3:  // 0x53
-    case DW_OP_reg4:  // 0x54
-    case DW_OP_reg5:  // 0x55
-    case DW_OP_reg6:  // 0x56
-    case DW_OP_reg7:  // 0x57
-    case DW_OP_reg8:  // 0x58
-    case DW_OP_reg9:  // 0x59
-    case DW_OP_reg10: // 0x5A
-    case DW_OP_reg11: // 0x5B
-    case DW_OP_reg12: // 0x5C
-    case DW_OP_reg13: // 0x5D
-    case DW_OP_reg14: // 0x5E
-    case DW_OP_reg15: // 0x5F
-    case DW_OP_reg16: // 0x60
-    case DW_OP_reg17: // 0x61
-    case DW_OP_reg18: // 0x62
-    case DW_OP_reg19: // 0x63
-    case DW_OP_reg20: // 0x64
-    case DW_OP_reg21: // 0x65
-    case DW_OP_reg22: // 0x66
-    case DW_OP_reg23: // 0x67
-    case DW_OP_reg24: // 0x68
-    case DW_OP_reg25: // 0x69
-    case DW_OP_reg26: // 0x6A
-    case DW_OP_reg27: // 0x6B
-    case DW_OP_reg28: // 0x6C
-    case DW_OP_reg29: // 0x6D
-    case DW_OP_reg30: // 0x6E
-    case DW_OP_reg31: // 0x6F
-    {
-      uint32_t reg_num = op - DW_OP_reg0;
-      if (abi) {
-        RegisterInfo reg_info;
-        if (abi->GetRegisterInfoByKind(m_reg_kind, reg_num, reg_info)) {
-          if (reg_info.name) {
-            s->PutCString(reg_info.name);
-            break;
-          } else if (reg_info.alt_name) {
-            s->PutCString(reg_info.alt_name);
-            break;
-          }
-        }
-      }
-      s->Printf("DW_OP_reg%u", reg_num);
-      break;
-    } break;
-
-    case DW_OP_breg0:
-    case DW_OP_breg1:
-    case DW_OP_breg2:
-    case DW_OP_breg3:
-    case DW_OP_breg4:
-    case DW_OP_breg5:
-    case DW_OP_breg6:
-    case DW_OP_breg7:
-    case DW_OP_breg8:
-    case DW_OP_breg9:
-    case DW_OP_breg10:
-    case DW_OP_breg11:
-    case DW_OP_breg12:
-    case DW_OP_breg13:
-    case DW_OP_breg14:
-    case DW_OP_breg15:
-    case DW_OP_breg16:
-    case DW_OP_breg17:
-    case DW_OP_breg18:
-    case DW_OP_breg19:
-    case DW_OP_breg20:
-    case DW_OP_breg21:
-    case DW_OP_breg22:
-    case DW_OP_breg23:
-    case DW_OP_breg24:
-    case DW_OP_breg25:
-    case DW_OP_breg26:
-    case DW_OP_breg27:
-    case DW_OP_breg28:
-    case DW_OP_breg29:
-    case DW_OP_breg30:
-    case DW_OP_breg31: {
-      uint32_t reg_num = op - DW_OP_breg0;
-      int64_t reg_offset = m_data.GetSLEB128(&offset);
-      if (abi) {
-        RegisterInfo reg_info;
-        if (abi->GetRegisterInfoByKind(m_reg_kind, reg_num, reg_info)) {
-          if (reg_info.name) {
-            s->Printf("[%s%+" PRIi64 "]", reg_info.name, reg_offset);
-            break;
-          } else if (reg_info.alt_name) {
-            s->Printf("[%s%+" PRIi64 "]", reg_info.alt_name, reg_offset);
-            break;
-          }
-        }
-      }
-      s->Printf("DW_OP_breg%i(0x%" PRIx64 ")", reg_num, reg_offset);
-    } break;
-
-    case DW_OP_regx: // 0x90 1 ULEB128 register
-    {
-      uint32_t reg_num = m_data.GetULEB128(&offset);
-      if (abi) {
-        RegisterInfo reg_info;
-        if (abi->GetRegisterInfoByKind(m_reg_kind, reg_num, reg_info)) {
-          if (reg_info.name) {
-            s->PutCString(reg_info.name);
-            break;
-          } else if (reg_info.alt_name) {
-            s->PutCString(reg_info.alt_name);
-            break;
-          }
-        }
-      }
-      s->Printf("DW_OP_regx(%" PRIu32 ")", reg_num);
-      break;
-    } break;
-    case DW_OP_fbreg: // 0x91 1 SLEB128 offset
-      s->Printf("DW_OP_fbreg(%" PRIi64 ")", m_data.GetSLEB128(&offset));
-      break;
-    case DW_OP_bregx: // 0x92 2 ULEB128 register followed by SLEB128 offset
-    {
-      uint32_t reg_num = m_data.GetULEB128(&offset);
-      int64_t reg_offset = m_data.GetSLEB128(&offset);
-      if (abi) {
-        RegisterInfo reg_info;
-        if (abi->GetRegisterInfoByKind(m_reg_kind, reg_num, reg_info)) {
-          if (reg_info.name) {
-            s->Printf("[%s%+" PRIi64 "]", reg_info.name, reg_offset);
-            break;
-          } else if (reg_info.alt_name) {
-            s->Printf("[%s%+" PRIi64 "]", reg_info.alt_name, reg_offset);
-            break;
-          }
-        }
-      }
-      s->Printf("DW_OP_bregx(reg=%" PRIu32 ",offset=%" PRIi64 ")", reg_num,
-                reg_offset);
-    } break;
-    case DW_OP_piece: // 0x93 1 ULEB128 size of piece addressed
-      s->Printf("DW_OP_piece(0x%" PRIx64 ")", m_data.GetULEB128(&offset));
-      break;
-    case DW_OP_deref_size: // 0x94 1 1-byte size of data retrieved
-      s->Printf("DW_OP_deref_size(0x%2.2x)", m_data.GetU8(&offset));
-      break;
-    case DW_OP_xderef_size: // 0x95 1 1-byte size of data retrieved
-      s->Printf("DW_OP_xderef_size(0x%2.2x)", m_data.GetU8(&offset));
-      break;
-    case DW_OP_nop:
-      s->PutCString("DW_OP_nop");
-      break; // 0x96
-    case DW_OP_push_object_address:
-      s->PutCString("DW_OP_push_object_address");
-      break;          // 0x97 DWARF3
-    case DW_OP_call2: // 0x98 DWARF3 1 2-byte offset of DIE
-      s->Printf("DW_OP_call2(0x%4.4x)", m_data.GetU16(&offset));
-      break;
-    case DW_OP_call4: // 0x99 DWARF3 1 4-byte offset of DIE
-      s->Printf("DW_OP_call4(0x%8.8x)", m_data.GetU32(&offset));
-      break;
-    case DW_OP_call_ref: // 0x9a DWARF3 1 4- or 8-byte offset of DIE
-      s->Printf("DW_OP_call_ref(0x%8.8" PRIx64 ")", m_data.GetAddress(&offset));
-      break;
-    case DW_OP_form_tls_address:
-      s->PutCString("DW_OP_form_tls_address"); // 0x9b
-      break;
-    case DW_OP_GNU_addr_index: // 0xfb
-      s->Printf("DW_OP_GNU_addr_index(0x%" PRIx64 ")",
-                m_data.GetULEB128(&offset));
-      break;
-    case DW_OP_addrx:
-      s->Printf("DW_OP_addrx(0x%" PRIx64 ")",
-                m_data.GetULEB128(&offset));
-      break;
-    case DW_OP_GNU_const_index: // 0xfc
-      s->Printf("DW_OP_GNU_const_index(0x%" PRIx64 ")",
-                m_data.GetULEB128(&offset));
-      break;
-    case DW_OP_GNU_push_tls_address:
-      s->PutCString("DW_OP_GNU_push_tls_address"); // 0xe0
-      break;
-    case DW_OP_APPLE_uninit:
-      s->PutCString("DW_OP_APPLE_uninit"); // 0xF0
-      break;
-    case DW_OP_entry_value: {
-      uint32_t subexpr_len = m_data.GetULEB128(&offset);
-      s->PutCString("DW_OP_entry_value(");
-      ends_of_subexprs.push_back(offset + subexpr_len);
-      break;
-    }
-    }
-  }
-
-  finish_subexpressions_to(end_offset);
+  llvm::DWARFExpression(DataExtractor(m_data, offset, length).GetAsLLVM(),
+                        llvm::dwarf::DWARF_VERSION, m_data.GetAddressByteSize())
+      .print(s->AsRawOstream(), abi ? &abi->GetMCRegisterInfo() : nullptr,
+             nullptr);
 }
 
 void DWARFExpression::SetLocationListSlide(addr_t slide) {
diff --git a/lldb/source/Expression/REPL.cpp b/lldb/source/Expression/REPL.cpp
index 7935251345891..78e4688102344 100644
--- a/lldb/source/Expression/REPL.cpp
+++ b/lldb/source/Expression/REPL.cpp
@@ -96,7 +96,7 @@ void REPL::IOHandlerActivated(IOHandler &io_handler, bool interactive) {
   lldb::ProcessSP process_sp = m_target.GetProcessSP();
   if (process_sp && process_sp->IsAlive())
     return;
-  lldb::StreamFileSP error_sp(io_handler.GetErrorStreamFile());
+  lldb::StreamFileSP error_sp(io_handler.GetErrorStreamFileSP());
   error_sp->Printf("REPL requires a running target process.\n");
   io_handler.SetIsDone(true);
 }
@@ -180,8 +180,8 @@ int REPL::IOHandlerFixIndentation(IOHandler &io_handler,
 }
 
 void REPL::IOHandlerInputComplete(IOHandler &io_handler, std::string &code) {
-  lldb::StreamFileSP output_sp(io_handler.GetOutputStreamFile());
-  lldb::StreamFileSP error_sp(io_handler.GetErrorStreamFile());
+  lldb::StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
+  lldb::StreamFileSP error_sp(io_handler.GetErrorStreamFileSP());
   bool extra_line = false;
   bool did_quit = false;
 
@@ -398,17 +398,22 @@ void REPL::IOHandlerInputComplete(IOHandler &io_handler, std::string &code) {
 
           // Update our code on disk
           if (!m_repl_source_path.empty()) {
-            lldb_private::File file;
-            FileSystem::Instance().Open(file, FileSpec(m_repl_source_path),
-                                        File::eOpenOptionWrite |
-                                            File::eOpenOptionTruncate |
-                                            File::eOpenOptionCanCreate,
-                                        lldb::eFilePermissionsFileDefault);
-            std::string code(m_code.CopyList());
-            code.append(1, '\n');
-            size_t bytes_written = code.size();
-            file.Write(code.c_str(), bytes_written);
-            file.Close();
+            auto file = FileSystem::Instance().Open(
+                FileSpec(m_repl_source_path),
+                File::eOpenOptionWrite | File::eOpenOptionTruncate |
+                    File::eOpenOptionCanCreate,
+                lldb::eFilePermissionsFileDefault);
+            if (file) {
+              std::string code(m_code.CopyList());
+              code.append(1, '\n');
+              size_t bytes_written = code.size();
+              file.get()->Write(code.c_str(), bytes_written);
+              file.get()->Close();
+            } else {
+              std::string message = llvm::toString(file.takeError());
+              error_sp->Printf("error: couldn't open %s: %s\n",
+                               m_repl_source_path.c_str(), message.c_str());
+            }
 
             // Now set the default file and line to the REPL source file
             m_target.GetSourceManager().SetDefaultFileAndLine(
@@ -453,7 +458,7 @@ void REPL::IOHandlerComplete(IOHandler &io_handler,
   }
 
   // Strip spaces from the line and see if we had only spaces
-  if (request.GetRawLineUntilCursor().trim().empty()) {
+  if (request.GetRawLine().trim().empty()) {
     // Only spaces on this line, so just indent
     request.AddCompletion(m_indent_str);
     return;
@@ -479,7 +484,7 @@ void REPL::IOHandlerComplete(IOHandler &io_handler,
   }
 
   current_code.append("\n");
-  current_code += request.GetRawLineUntilCursor();
+  current_code += request.GetRawLine();
 
   StringList matches;
   int result = CompleteCode(current_code, matches);
diff --git a/lldb/source/Host/common/File.cpp b/lldb/source/Host/common/File.cpp
index 269548974d1d5..b79c00378c9d8 100644
--- a/lldb/source/Host/common/File.cpp
+++ b/lldb/source/Host/common/File.cpp
@@ -91,21 +91,14 @@ int File::GetDescriptor() const {
   return kInvalidDescriptor;
 }
 
-IOObject::WaitableHandle File::GetWaitableHandle() { return m_descriptor; }
-
-void File::SetDescriptor(int fd, bool transfer_ownership) {
-  if (IsValid())
-    Close();
-  m_descriptor = fd;
-  m_should_close_fd = transfer_ownership;
-}
+IOObject::WaitableHandle File::GetWaitableHandle() { return GetDescriptor(); }
 
 FILE *File::GetStream() {
   if (!StreamIsValid()) {
     if (DescriptorIsValid()) {
       const char *mode = GetStreamOpenModeFromOptions(m_options);
       if (mode) {
-        if (!m_should_close_fd) {
+        if (!m_own_descriptor) {
 // We must duplicate the file descriptor if we don't own it because when you
 // call fdopen, the stream will own the fd
 #ifdef _WIN32
@@ -113,7 +106,7 @@ FILE *File::GetStream() {
 #else
           m_descriptor = dup(GetDescriptor());
 #endif
-          m_should_close_fd = true;
+          m_own_descriptor = true;
         }
 
         m_stream =
@@ -124,7 +117,7 @@ FILE *File::GetStream() {
 
         if (m_stream) {
           m_own_stream = true;
-          m_should_close_fd = false;
+          m_own_descriptor = false;
         }
       }
     }
@@ -132,13 +125,6 @@ FILE *File::GetStream() {
   return m_stream;
 }
 
-void File::SetStream(FILE *fh, bool transfer_ownership) {
-  if (IsValid())
-    Close();
-  m_stream = fh;
-  m_own_stream = transfer_ownership;
-}
-
 uint32_t File::GetPermissions(Status &error) const {
   int fd = GetDescriptor();
   if (fd != kInvalidDescriptor) {
@@ -162,7 +148,7 @@ Status File::Close() {
       error.SetErrorToErrno();
   }
 
-  if (DescriptorIsValid() && m_should_close_fd) {
+  if (DescriptorIsValid() && m_own_descriptor) {
     if (::close(m_descriptor) != 0)
       error.SetErrorToErrno();
   }
@@ -170,19 +156,22 @@ Status File::Close() {
   m_stream = kInvalidStream;
   m_options = 0;
   m_own_stream = false;
-  m_should_close_fd = false;
+  m_own_descriptor = false;
   m_is_interactive = eLazyBoolCalculate;
   m_is_real_terminal = eLazyBoolCalculate;
   return error;
 }
 
-void File::Clear() {
-  m_stream = nullptr;
+FILE *File::TakeStreamAndClear() {
+  FILE *stream = GetStream();
+  m_stream = NULL;
   m_descriptor = kInvalidDescriptor;
   m_options = 0;
   m_own_stream = false;
+  m_own_descriptor = false;
   m_is_interactive = m_supports_colors = m_is_real_terminal =
       eLazyBoolCalculate;
+  return stream;
 }
 
 Status File::GetFileSpec(FileSpec &file_spec) const {
diff --git a/lldb/source/Host/common/FileCache.cpp b/lldb/source/Host/common/FileCache.cpp
index 4bd3efda7fb0e..0a7197e55b0b5 100644
--- a/lldb/source/Host/common/FileCache.cpp
+++ b/lldb/source/Host/common/FileCache.cpp
@@ -29,12 +29,13 @@ lldb::user_id_t FileCache::OpenFile(const FileSpec &file_spec, uint32_t flags,
     error.SetErrorString("empty path");
     return UINT64_MAX;
   }
-  FileSP file_sp(new File());
-  error = FileSystem::Instance().Open(*file_sp, file_spec, flags, mode);
-  if (!file_sp->IsValid())
+  auto file = FileSystem::Instance().Open(file_spec, flags, mode);
+  if (!file) {
+    error = file.takeError();
     return UINT64_MAX;
-  lldb::user_id_t fd = file_sp->GetDescriptor();
-  m_cache[fd] = file_sp;
+  }
+  lldb::user_id_t fd = file.get()->GetDescriptor();
+  m_cache[fd] = std::move(file.get());
   return fd;
 }
 
@@ -48,12 +49,12 @@ bool FileCache::CloseFile(lldb::user_id_t fd, Status &error) {
     error.SetErrorStringWithFormat("invalid host file descriptor %" PRIu64, fd);
     return false;
   }
-  FileSP file_sp = pos->second;
-  if (!file_sp) {
+  FileUP &file_up = pos->second;
+  if (!file_up) {
     error.SetErrorString("invalid host backing file");
     return false;
   }
-  error = file_sp->Close();
+  error = file_up->Close();
   m_cache.erase(pos);
   return error.Success();
 }
@@ -70,16 +71,16 @@ uint64_t FileCache::WriteFile(lldb::user_id_t fd, uint64_t offset,
     error.SetErrorStringWithFormat("invalid host file descriptor %" PRIu64, fd);
     return false;
   }
-  FileSP file_sp = pos->second;
-  if (!file_sp) {
+  FileUP &file_up = pos->second;
+  if (!file_up) {
     error.SetErrorString("invalid host backing file");
     return UINT64_MAX;
   }
-  if (static_cast<uint64_t>(file_sp->SeekFromStart(offset, &error)) != offset ||
+  if (static_cast<uint64_t>(file_up->SeekFromStart(offset, &error)) != offset ||
       error.Fail())
     return UINT64_MAX;
   size_t bytes_written = src_len;
-  error = file_sp->Write(src, bytes_written);
+  error = file_up->Write(src, bytes_written);
   if (error.Fail())
     return UINT64_MAX;
   return bytes_written;
@@ -96,16 +97,16 @@ uint64_t FileCache::ReadFile(lldb::user_id_t fd, uint64_t offset, void *dst,
     error.SetErrorStringWithFormat("invalid host file descriptor %" PRIu64, fd);
     return false;
   }
-  FileSP file_sp = pos->second;
-  if (!file_sp) {
+  FileUP &file_up = pos->second;
+  if (!file_up) {
     error.SetErrorString("invalid host backing file");
     return UINT64_MAX;
   }
-  if (static_cast<uint64_t>(file_sp->SeekFromStart(offset, &error)) != offset ||
+  if (static_cast<uint64_t>(file_up->SeekFromStart(offset, &error)) != offset ||
       error.Fail())
     return UINT64_MAX;
   size_t bytes_read = dst_len;
-  error = file_sp->Read(dst, bytes_read);
+  error = file_up->Read(dst, bytes_read);
   if (error.Fail())
     return UINT64_MAX;
   return bytes_read;
diff --git a/lldb/source/Host/common/FileSystem.cpp b/lldb/source/Host/common/FileSystem.cpp
index 0d9417120fdd1..b3442770f010b 100644
--- a/lldb/source/Host/common/FileSystem.cpp
+++ b/lldb/source/Host/common/FileSystem.cpp
@@ -415,34 +415,29 @@ static mode_t GetOpenMode(uint32_t permissions) {
   return mode;
 }
 
-Status FileSystem::Open(File &File, const FileSpec &file_spec, uint32_t options,
-                        uint32_t permissions, bool should_close_fd) {
+Expected<FileUP> FileSystem::Open(const FileSpec &file_spec, uint32_t options,
+                                  uint32_t permissions, bool should_close_fd) {
   if (m_collector)
     m_collector->addFile(file_spec.GetPath());
 
-  if (File.IsValid())
-    File.Close();
-
   const int open_flags = GetOpenFlags(options);
   const mode_t open_mode =
       (open_flags & O_CREAT) ? GetOpenMode(permissions) : 0;
 
   auto path = GetExternalPath(file_spec);
   if (!path)
-    return Status(path.getError());
+    return errorCodeToError(path.getError());
 
   int descriptor = llvm::sys::RetryAfterSignal(
       -1, OpenWithFS, *this, path->c_str(), open_flags, open_mode);
 
-  Status error;
-  if (!File::DescriptorIsValid(descriptor)) {
-    File.SetDescriptor(descriptor, false);
-    error.SetErrorToErrno();
-  } else {
-    File.SetDescriptor(descriptor, should_close_fd);
-    File.SetOptions(options);
-  }
-  return error;
+  if (!File::DescriptorIsValid(descriptor))
+    return llvm::errorCodeToError(
+        std::error_code(errno, std::system_category()));
+
+  auto file = std::make_unique<File>(descriptor, options, should_close_fd);
+  assert(file->IsValid());
+  return std::move(file);
 }
 
 ErrorOr<std::string> FileSystem::GetExternalPath(const llvm::Twine &path) {
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index 525727ed94510..6358ab8a8e774 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -74,9 +74,10 @@ bool IsInterrupted() {
 
 Socket::Socket(SocketProtocol protocol, bool should_close,
                bool child_processes_inherit)
-    : IOObject(eFDTypeSocket, should_close), m_protocol(protocol),
+    : IOObject(eFDTypeSocket), m_protocol(protocol),
       m_socket(kInvalidSocketValue),
-      m_child_processes_inherit(child_processes_inherit) {}
+      m_child_processes_inherit(child_processes_inherit),
+      m_should_close_fd(should_close) {}
 
 Socket::~Socket() { Close(); }
 
@@ -476,11 +477,11 @@ NativeSocket Socket::AcceptSocket(NativeSocket sockfd, struct sockaddr *addr,
   if (!child_processes_inherit) {
     flags |= SOCK_CLOEXEC;
   }
-  NativeSocket fd = llvm::sys::RetryAfterSignal(-1, ::accept4,
-      sockfd, addr, addrlen, flags);
+  NativeSocket fd = llvm::sys::RetryAfterSignal(
+      static_cast<NativeSocket>(-1), ::accept4, sockfd, addr, addrlen, flags);
 #else
-  NativeSocket fd = llvm::sys::RetryAfterSignal(-1, ::accept,
-      sockfd, addr, addrlen);
+  NativeSocket fd = llvm::sys::RetryAfterSignal(
+      static_cast<NativeSocket>(-1), ::accept, sockfd, addr, addrlen);
 #endif
   if (fd == kInvalidSocketValue)
     SetLastError(error);
diff --git a/lldb/source/Host/common/UDPSocket.cpp b/lldb/source/Host/common/UDPSocket.cpp
index 03ad7f587974a..7accbb651ba9b 100644
--- a/lldb/source/Host/common/UDPSocket.cpp
+++ b/lldb/source/Host/common/UDPSocket.cpp
@@ -80,7 +80,7 @@ Status UDPSocket::Connect(llvm::StringRef name, bool child_processes_inherit,
                           &service_info_list);
   if (err != 0) {
     error.SetErrorStringWithFormat(
-#if defined(_MSC_VER) && defined(UNICODE)
+#if defined(_WIN32) && defined(UNICODE)
         "getaddrinfo(%s, %s, &hints, &info) returned error %i (%S)",
 #else
         "getaddrinfo(%s, %s, &hints, &info) returned error %i (%s)",
diff --git a/lldb/source/Host/macosx/objcxx/Host.mm b/lldb/source/Host/macosx/objcxx/Host.mm
index 75d88eb4d02f6..92aa4778fc653 100644
--- a/lldb/source/Host/macosx/objcxx/Host.mm
+++ b/lldb/source/Host/macosx/objcxx/Host.mm
@@ -10,8 +10,8 @@
 
 #include <AvailabilityMacros.h>
 
-#if !defined(MAC_OS_X_VERSION_10_7) ||                                         \
-    MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_7
+// On device doesn't have supporty for XPC.
+#if defined(__APPLE__) && (defined(__arm64__) || defined(__aarch64__))
 #define NO_XPC_SERVICES 1
 #endif
 
diff --git a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
index a8425638555aa..f9759ec3cafd7 100644
--- a/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
+++ b/lldb/source/Host/posix/ConnectionFileDescriptorPosix.cpp
@@ -86,8 +86,8 @@ ConnectionFileDescriptor::ConnectionFileDescriptor(bool child_processes_inherit)
 ConnectionFileDescriptor::ConnectionFileDescriptor(int fd, bool owns_fd)
     : Connection(), m_pipe(), m_mutex(), m_shutting_down(false),
       m_waiting_for_accept(false), m_child_processes_inherit(false) {
-  m_write_sp = std::make_shared<File>(fd, owns_fd);
-  m_read_sp = std::make_shared<File>(fd, false);
+  m_write_sp = std::make_shared<File>(fd, File::eOpenOptionWrite, owns_fd);
+  m_read_sp = std::make_shared<File>(fd, File::eOpenOptionRead, false);
 
   Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_CONNECTION |
                                                   LIBLLDB_LOG_OBJECT));
@@ -218,8 +218,10 @@ ConnectionStatus ConnectionFileDescriptor::Connect(llvm::StringRef path,
             m_read_sp = std::move(tcp_socket);
             m_write_sp = m_read_sp;
           } else {
-            m_read_sp = std::make_shared<File>(fd, false);
-            m_write_sp = std::make_shared<File>(fd, false);
+            m_read_sp =
+                std::make_shared<File>(fd, File::eOpenOptionRead, false);
+            m_write_sp =
+                std::make_shared<File>(fd, File::eOpenOptionWrite, false);
           }
           m_uri = *addr;
           return eConnectionStatusSuccess;
@@ -268,8 +270,8 @@ ConnectionStatus ConnectionFileDescriptor::Connect(llvm::StringRef path,
           ::fcntl(fd, F_SETFL, flags);
         }
       }
-      m_read_sp = std::make_shared<File>(fd, true);
-      m_write_sp = std::make_shared<File>(fd, false);
+      m_read_sp = std::make_shared<File>(fd, File::eOpenOptionRead, true);
+      m_write_sp = std::make_shared<File>(fd, File::eOpenOptionWrite, false);
       return eConnectionStatusSuccess;
     }
 #endif
@@ -564,7 +566,7 @@ ConnectionFileDescriptor::BytesAvailable(const Timeout<std::micro> &timeout,
       select_helper.SetTimeout(*timeout);
 
     select_helper.FDSetRead(handle);
-#if defined(_MSC_VER)
+#if defined(_WIN32)
     // select() won't accept pipes on Windows.  The entire Windows codepath
     // needs to be converted over to using WaitForMultipleObjects and event
     // HANDLEs, but for now at least this will allow ::select() to not return
diff --git a/lldb/source/Host/windows/ConnectionGenericFileWindows.cpp b/lldb/source/Host/windows/ConnectionGenericFileWindows.cpp
index f6445d753d7d5..174ed73e98159 100644
--- a/lldb/source/Host/windows/ConnectionGenericFileWindows.cpp
+++ b/lldb/source/Host/windows/ConnectionGenericFileWindows.cpp
@@ -245,13 +245,11 @@ size_t ConnectionGenericFile::Read(void *dst, size_t dst_len,
 
   IncrementFilePointer(return_info.GetBytes());
   Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_CONNECTION));
-  if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionGenericFile::Read()  handle = %p, dst = %p, "
-              "dst_len = %zu) => %zu, error = %s",
-              this, m_file, dst, dst_len, return_info.GetBytes(),
-              return_info.GetError().AsCString());
-  }
+  LLDB_LOGF(log,
+            "%p ConnectionGenericFile::Read()  handle = %p, dst = %p, "
+            "dst_len = %zu) => %zu, error = %s",
+            static_cast<void *>(this), m_file, dst, dst_len,
+            return_info.GetBytes(), return_info.GetError().AsCString());
 
   return return_info.GetBytes();
 }
@@ -296,13 +294,11 @@ size_t ConnectionGenericFile::Write(const void *src, size_t src_len,
 
   IncrementFilePointer(return_info.GetBytes());
   Log *log(lldb_private::GetLogIfAnyCategoriesSet(LIBLLDB_LOG_CONNECTION));
-  if (log) {
-    LLDB_LOGF(log,
-              "%p ConnectionGenericFile::Write()  handle = %p, src = %p, "
-              "src_len = %zu) => %zu, error = %s",
-              this, m_file, src, src_len, return_info.GetBytes(),
-              return_info.GetError().AsCString());
-  }
+  LLDB_LOGF(log,
+            "%p ConnectionGenericFile::Write()  handle = %p, src = %p, "
+            "src_len = %zu) => %zu, error = %s",
+            static_cast<void *>(this), m_file, src, src_len,
+            return_info.GetBytes(), return_info.GetError().AsCString());
   return return_info.GetBytes();
 }
 
diff --git a/lldb/source/Host/windows/Host.cpp b/lldb/source/Host/windows/Host.cpp
index 6464264873420..19be271e831bd 100644
--- a/lldb/source/Host/windows/Host.cpp
+++ b/lldb/source/Host/windows/Host.cpp
@@ -34,9 +34,11 @@ namespace {
 bool GetTripleForProcess(const FileSpec &executable, llvm::Triple &triple) {
   // Open the PE File as a binary file, and parse just enough information to
   // determine the machine type.
-  File imageBinary;
-  FileSystem::Instance().Open(imageBinary, executable, File::eOpenOptionRead,
-                              lldb::eFilePermissionsUserRead);
+  auto imageBinaryP = FileSystem::Instance().Open(
+      executable, File::eOpenOptionRead, lldb::eFilePermissionsUserRead);
+  if (!imageBinaryP)
+    return llvm::errorToBool(imageBinaryP.takeError());
+  File &imageBinary = *imageBinaryP.get();
   imageBinary.SeekFromStart(0x3c);
   int32_t peOffset = 0;
   uint32_t peHead = 0;
@@ -56,6 +58,10 @@ bool GetTripleForProcess(const FileSpec &executable, llvm::Triple &triple) {
     triple.setArch(llvm::Triple::x86_64);
   else if (machineType == 0x14c)
     triple.setArch(llvm::Triple::x86);
+  else if (machineType == 0x1c4)
+    triple.setArch(llvm::Triple::arm);
+  else if (machineType == 0xaa64)
+    triple.setArch(llvm::Triple::aarch64);
 
   return true;
 }
diff --git a/lldb/source/Host/windows/ProcessLauncherWindows.cpp b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
index 3f5c637a5a346..b74308264fe65 100644
--- a/lldb/source/Host/windows/ProcessLauncherWindows.cpp
+++ b/lldb/source/Host/windows/ProcessLauncherWindows.cpp
@@ -46,7 +46,7 @@ bool GetFlattenedWindowsCommandString(Args args, std::string &command) {
 
   std::vector<llvm::StringRef> args_ref;
   for (auto &entry : args.entries())
-    args_ref.push_back(entry.ref);
+    args_ref.push_back(entry.ref());
 
   command = llvm::sys::flattenWindowsCommandLine(args_ref);
   return true;
diff --git a/lldb/source/Host/windows/Windows.cpp b/lldb/source/Host/windows/Windows.cpp
index e76f6d2dba6a8..fa664f5398401 100644
--- a/lldb/source/Host/windows/Windows.cpp
+++ b/lldb/source/Host/windows/Windows.cpp
@@ -48,13 +48,8 @@ int vasprintf(char **ret, const char *fmt, va_list ap) {
   size_t buflen;
   va_list ap2;
 
-#if defined(_MSC_VER) || defined(__MINGW64)
-  ap2 = ap;
-  len = _vscprintf(fmt, ap2);
-#else
   va_copy(ap2, ap);
   len = vsnprintf(NULL, 0, fmt, ap2);
-#endif
 
   if (len >= 0 &&
       (buf = (char *)malloc((buflen = (size_t)(len + 1)))) != NULL) {
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index e9f22b5ebd2c1..0c059096c6cdb 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -63,11 +63,14 @@
 #include "lldb/Utility/Args.h"
 
 #include "lldb/Target/Process.h"
+#include "lldb/Target/StopInfo.h"
 #include "lldb/Target/TargetList.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Target/UnixSignals.h"
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FormatAdapters.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/PrettyStackTrace.h"
 
@@ -1756,7 +1759,7 @@ void CommandInterpreter::HandleCompletionMatches(CompletionRequest &request) {
 
   // For any of the command completions a unique match will be a complete word.
 
-  if (request.GetCursorIndex() == -1) {
+  if (request.GetParsedLine().GetArgumentCount() == 0) {
     // We got nothing on the command line, so return the list of commands
     bool include_aliases = true;
     StringList new_matches, descriptions;
@@ -1779,9 +1782,7 @@ void CommandInterpreter::HandleCompletionMatches(CompletionRequest &request) {
         look_for_subcommand = true;
         new_matches.DeleteStringAtIndex(0);
         new_descriptions.DeleteStringAtIndex(0);
-        request.GetParsedLine().AppendArgument(llvm::StringRef());
-        request.SetCursorIndex(request.GetCursorIndex() + 1);
-        request.SetCursorCharPosition(0);
+        request.AppendEmptyArgument();
       }
     }
     request.AddCompletions(new_matches, new_descriptions);
@@ -1794,8 +1795,7 @@ void CommandInterpreter::HandleCompletionMatches(CompletionRequest &request) {
     CommandObject *command_object =
         GetCommandObject(request.GetParsedLine().GetArgumentAtIndex(0));
     if (command_object) {
-      request.GetParsedLine().Shift();
-      request.SetCursorIndex(request.GetCursorIndex() - 1);
+      request.ShiftArguments();
       command_object->HandleCompletion(request);
     }
   }
@@ -2141,6 +2141,45 @@ PlatformSP CommandInterpreter::GetPlatform(bool prefer_target_platform) {
   return platform_sp;
 }
 
+bool CommandInterpreter::DidProcessStopAbnormally() const {
+  TargetSP target_sp = m_debugger.GetTargetList().GetSelectedTarget();
+  if (!target_sp)
+    return false;
+
+  ProcessSP process_sp(target_sp->GetProcessSP());
+  if (!process_sp)
+    return false;
+
+  if (eStateStopped != process_sp->GetState())
+    return false;
+
+  for (const auto &thread_sp : process_sp->GetThreadList().Threads()) {
+    StopInfoSP stop_info = thread_sp->GetStopInfo();
+    if (!stop_info)
+      return false;
+
+    const StopReason reason = stop_info->GetStopReason();
+    if (reason == eStopReasonException || reason == eStopReasonInstrumentation)
+      return true;
+
+    if (reason == eStopReasonSignal) {
+      const auto stop_signal = static_cast<int32_t>(stop_info->GetValue());
+      UnixSignalsSP signals_sp = process_sp->GetUnixSignals();
+      if (!signals_sp || !signals_sp->SignalIsValid(stop_signal))
+        // The signal is unknown, treat it as abnormal.
+        return true;
+
+      const auto sigint_num = signals_sp->GetSignalNumberFromName("SIGINT");
+      const auto sigstop_num = signals_sp->GetSignalNumberFromName("SIGSTOP");
+      if ((stop_signal != sigint_num) && (stop_signal != sigstop_num))
+        // The signal very likely implies a crash.
+        return true;
+    }
+  }
+
+  return false;
+}
+
 void CommandInterpreter::HandleCommands(const StringList &commands,
                                         ExecutionContext *override_context,
                                         CommandInterpreterRunOptions &options,
@@ -2251,38 +2290,22 @@ void CommandInterpreter::HandleCommands(const StringList &commands,
     }
 
     // Also check for "stop on crash here:
-    bool should_stop = false;
-    if (tmp_result.GetDidChangeProcessState() && options.GetStopOnCrash()) {
-      TargetSP target_sp(m_debugger.GetTargetList().GetSelectedTarget());
-      if (target_sp) {
-        ProcessSP process_sp(target_sp->GetProcessSP());
-        if (process_sp) {
-          for (ThreadSP thread_sp : process_sp->GetThreadList().Threads()) {
-            StopReason reason = thread_sp->GetStopReason();
-            if (reason == eStopReasonSignal || reason == eStopReasonException ||
-                reason == eStopReasonInstrumentation) {
-              should_stop = true;
-              break;
-            }
-          }
-        }
-      }
-      if (should_stop) {
-        if (idx != num_lines - 1)
-          result.AppendErrorWithFormat(
-              "Aborting reading of commands after command #%" PRIu64
-              ": '%s' stopped with a signal or exception.\n",
-              (uint64_t)idx + 1, cmd);
-        else
-          result.AppendMessageWithFormat(
-              "Command #%" PRIu64 " '%s' stopped with a signal or exception.\n",
-              (uint64_t)idx + 1, cmd);
+    if (tmp_result.GetDidChangeProcessState() && options.GetStopOnCrash() &&
+        DidProcessStopAbnormally()) {
+      if (idx != num_lines - 1)
+        result.AppendErrorWithFormat(
+            "Aborting reading of commands after command #%" PRIu64
+            ": '%s' stopped with a signal or exception.\n",
+            (uint64_t)idx + 1, cmd);
+      else
+        result.AppendMessageWithFormat(
+            "Command #%" PRIu64 " '%s' stopped with a signal or exception.\n",
+            (uint64_t)idx + 1, cmd);
 
-        result.SetStatus(tmp_result.GetStatus());
-        m_debugger.SetAsyncExecution(old_async_execution);
+      result.SetStatus(tmp_result.GetStatus());
+      m_debugger.SetAsyncExecution(old_async_execution);
 
-        return;
-      }
+      return;
     }
   }
 
@@ -2315,18 +2338,18 @@ void CommandInterpreter::HandleCommandsFromFile(
     return;
   }
 
-  StreamFileSP input_file_sp(new StreamFile());
   std::string cmd_file_path = cmd_file.GetPath();
-  Status error = FileSystem::Instance().Open(input_file_sp->GetFile(), cmd_file,
-                                             File::eOpenOptionRead);
-
-  if (error.Fail()) {
-    result.AppendErrorWithFormat(
-        "error: an error occurred read file '%s': %s\n", cmd_file_path.c_str(),
-        error.AsCString());
+  auto input_file_up =
+      FileSystem::Instance().Open(cmd_file, File::eOpenOptionRead);
+  if (!input_file_up) {
+    std::string error = llvm::toString(input_file_up.takeError());
+    result.AppendErrorWithFormatv(
+        "error: an error occurred read file '{0}': {1}\n", cmd_file_path,
+        llvm::fmt_consume(input_file_up.takeError()));
     result.SetStatus(eReturnStatusFailed);
     return;
   }
+  FileSP input_file_sp = FileSP(std::move(input_file_up.get()));
 
   Debugger &debugger = GetDebugger();
 
@@ -2412,8 +2435,8 @@ void CommandInterpreter::HandleCommandsFromFile(
   }
 
   if (flags & eHandleCommandFlagPrintResult) {
-    debugger.GetOutputFile()->Printf("Executing commands in '%s'.\n",
-                                     cmd_file_path.c_str());
+    debugger.GetOutputFile().Printf("Executing commands in '%s'.\n",
+                                    cmd_file_path.c_str());
   }
 
   // Used for inheriting the right settings when "command source" might
@@ -2713,8 +2736,8 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
     // from a file) we need to echo the command out so we don't just see the
     // command output and no command...
     if (EchoCommandNonInteractive(line, io_handler.GetFlags()))
-      io_handler.GetOutputStreamFile()->Printf("%s%s\n", io_handler.GetPrompt(),
-                                               line.c_str());
+      io_handler.GetOutputStreamFileSP()->Printf(
+          "%s%s\n", io_handler.GetPrompt(), line.c_str());
   }
 
   StartHandlingCommand();
@@ -2731,13 +2754,13 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
 
     if (!result.GetImmediateOutputStream()) {
       llvm::StringRef output = result.GetOutputData();
-      PrintCommandOutput(*io_handler.GetOutputStreamFile(), output);
+      PrintCommandOutput(*io_handler.GetOutputStreamFileSP(), output);
     }
 
     // Now emit the command error text from the command we just executed
     if (!result.GetImmediateErrorStream()) {
       llvm::StringRef error = result.GetErrorData();
-      PrintCommandOutput(*io_handler.GetErrorStreamFile(), error);
+      PrintCommandOutput(*io_handler.GetErrorStreamFileSP(), error);
     }
   }
 
@@ -2770,27 +2793,10 @@ void CommandInterpreter::IOHandlerInputComplete(IOHandler &io_handler,
 
   // Finally, if we're going to stop on crash, check that here:
   if (!m_quit_requested && result.GetDidChangeProcessState() &&
-      io_handler.GetFlags().Test(eHandleCommandFlagStopOnCrash)) {
-    bool should_stop = false;
-    TargetSP target_sp(m_debugger.GetTargetList().GetSelectedTarget());
-    if (target_sp) {
-      ProcessSP process_sp(target_sp->GetProcessSP());
-      if (process_sp) {
-        for (ThreadSP thread_sp : process_sp->GetThreadList().Threads()) {
-          StopReason reason = thread_sp->GetStopReason();
-          if ((reason == eStopReasonSignal || reason == eStopReasonException ||
-               reason == eStopReasonInstrumentation) &&
-              !result.GetAbnormalStopWasExpected()) {
-            should_stop = true;
-            break;
-          }
-        }
-      }
-    }
-    if (should_stop) {
-      io_handler.SetIsDone(true);
-      m_stopped_for_crash = true;
-    }
+      io_handler.GetFlags().Test(eHandleCommandFlagStopOnCrash) &&
+      DidProcessStopAbnormally()) {
+    io_handler.SetIsDone(true);
+    m_stopped_for_crash = true;
   }
 }
 
@@ -2904,8 +2910,8 @@ CommandInterpreter::GetIOHandler(bool force_create,
 
     m_command_io_handler_sp = std::make_shared<IOHandlerEditline>(
         m_debugger, IOHandler::Type::CommandInterpreter,
-        m_debugger.GetInputFile(), m_debugger.GetOutputFile(),
-        m_debugger.GetErrorFile(), flags, "lldb", m_debugger.GetPrompt(),
+        m_debugger.GetInputFileSP(), m_debugger.GetOutputStreamSP(),
+        m_debugger.GetErrorStreamSP(), flags, "lldb", m_debugger.GetPrompt(),
         llvm::StringRef(), // Continuation prompt
         false, // Don't enable multiple line input, just single line commands
         m_debugger.GetUseColor(),
diff --git a/lldb/source/Interpreter/CommandReturnObject.cpp b/lldb/source/Interpreter/CommandReturnObject.cpp
index 3a7a8755d9755..c17390be7311d 100644
--- a/lldb/source/Interpreter/CommandReturnObject.cpp
+++ b/lldb/source/Interpreter/CommandReturnObject.cpp
@@ -33,8 +33,7 @@ static void DumpStringToStreamWithNewline(Stream &strm, const std::string &s,
 
 CommandReturnObject::CommandReturnObject()
     : m_out_stream(), m_err_stream(), m_status(eReturnStatusStarted),
-      m_did_change_process_state(false), m_interactive(true),
-      m_abnormal_stop_was_expected(false) {}
+      m_did_change_process_state(false), m_interactive(true) {}
 
 CommandReturnObject::~CommandReturnObject() {}
 
diff --git a/lldb/source/Interpreter/OptionValueBoolean.cpp b/lldb/source/Interpreter/OptionValueBoolean.cpp
index bfc7760e207a2..4a084156a0a92 100644
--- a/lldb/source/Interpreter/OptionValueBoolean.cpp
+++ b/lldb/source/Interpreter/OptionValueBoolean.cpp
@@ -82,8 +82,6 @@ void OptionValueBoolean::AutoComplete(CommandInterpreter &interpreter,
   if (request.GetCursorArgumentPrefix().empty())
     entries = entries.take_front(2);
 
-  for (auto entry : entries) {
-    if (entry.startswith_lower(request.GetCursorArgumentPrefix()))
-      request.AddCompletion(entry);
-  }
+  for (auto entry : entries)
+    request.TryCompleteCurrentArg(entry);
 }
diff --git a/lldb/source/Interpreter/OptionValueEnumeration.cpp b/lldb/source/Interpreter/OptionValueEnumeration.cpp
index 126a4f53e10f1..26933aa782404 100644
--- a/lldb/source/Interpreter/OptionValueEnumeration.cpp
+++ b/lldb/source/Interpreter/OptionValueEnumeration.cpp
@@ -108,8 +108,7 @@ void OptionValueEnumeration::AutoComplete(CommandInterpreter &interpreter,
   if (!request.GetCursorArgumentPrefix().empty()) {
     for (size_t i = 0; i < num_enumerators; ++i) {
       llvm::StringRef name = m_enumerations.GetCStringAtIndex(i).GetStringRef();
-      if (name.startswith(request.GetCursorArgumentPrefix()))
-        request.AddCompletion(name);
+      request.TryCompleteCurrentArg(name);
     }
     return;
   }
diff --git a/lldb/source/Interpreter/OptionValueUUID.cpp b/lldb/source/Interpreter/OptionValueUUID.cpp
index 483c47807d2ad..7a6bc65b25a40 100644
--- a/lldb/source/Interpreter/OptionValueUUID.cpp
+++ b/lldb/source/Interpreter/OptionValueUUID.cpp
@@ -80,10 +80,6 @@ void OptionValueUUID::AutoComplete(CommandInterpreter &interpreter,
     const UUID &module_uuid = module_sp->GetUUID();
     if (!module_uuid.IsValid())
       continue;
-    llvm::ArrayRef<uint8_t> module_bytes = module_uuid.GetBytes();
-    if (module_bytes.size() >= uuid_bytes.size() &&
-        module_bytes.take_front(uuid_bytes.size()).equals(uuid_bytes)) {
-      request.AddCompletion(module_uuid.GetAsString());
-    }
+    request.TryCompleteCurrentArg(module_uuid.GetAsString());
   }
 }
diff --git a/lldb/source/Interpreter/Options.cpp b/lldb/source/Interpreter/Options.cpp
index 09ecfed206c3d..465271d2f9694 100644
--- a/lldb/source/Interpreter/Options.cpp
+++ b/lldb/source/Interpreter/Options.cpp
@@ -655,8 +655,8 @@ bool Options::HandleOptionCompletion(CompletionRequest &request,
   llvm::StringRef cur_opt_str = request.GetCursorArgumentPrefix();
 
   for (size_t i = 0; i < opt_element_vector.size(); i++) {
-    int opt_pos = opt_element_vector[i].opt_pos;
-    int opt_arg_pos = opt_element_vector[i].opt_arg_pos;
+    size_t opt_pos = static_cast<size_t>(opt_element_vector[i].opt_pos);
+    size_t opt_arg_pos = static_cast<size_t>(opt_element_vector[i].opt_arg_pos);
     int opt_defs_index = opt_element_vector[i].opt_defs_index;
     if (opt_pos == request.GetCursorIndex()) {
       // We're completing the option itself.
@@ -698,7 +698,7 @@ bool Options::HandleOptionCompletion(CompletionRequest &request,
           request.AddCompletion("--" + long_option.str(), opt.usage_text);
           return true;
         } else
-          request.AddCompletion(request.GetCursorArgument());
+          request.AddCompletion(request.GetCursorArgumentPrefix());
         return true;
       } else {
         // FIXME - not handling wrong options yet:
@@ -720,11 +720,8 @@ bool Options::HandleOptionCompletion(CompletionRequest &request,
     } else if (opt_arg_pos == request.GetCursorIndex()) {
       // Okay the cursor is on the completion of an argument. See if it has a
       // completion, otherwise return no matches.
-
-      CompletionRequest subrequest = request;
-      subrequest.SetCursorCharPosition(subrequest.GetCursorArgument().size());
       if (opt_defs_index != -1) {
-        HandleOptionArgumentCompletion(subrequest, opt_element_vector, i,
+        HandleOptionArgumentCompletion(request, opt_element_vector, i,
                                        interpreter);
         return true;
       } else {
@@ -746,25 +743,14 @@ void Options::HandleOptionArgumentCompletion(
   auto opt_defs = GetDefinitions();
   std::unique_ptr<SearchFilter> filter_up;
 
-  int opt_arg_pos = opt_element_vector[opt_element_index].opt_arg_pos;
   int opt_defs_index = opt_element_vector[opt_element_index].opt_defs_index;
 
   // See if this is an enumeration type option, and if so complete it here:
 
   const auto &enum_values = opt_defs[opt_defs_index].enum_values;
-  if (!enum_values.empty()) {
-    std::string match_string(
-        request.GetParsedLine().GetArgumentAtIndex(opt_arg_pos),
-        request.GetParsedLine().GetArgumentAtIndex(opt_arg_pos) +
-            request.GetCursorCharPosition());
-
-    for (const auto &enum_value : enum_values) {
-      if (strstr(enum_value.string_value, match_string.c_str()) ==
-          enum_value.string_value) {
-        request.AddCompletion(enum_value.string_value);
-      }
-    }
-  }
+  if (!enum_values.empty())
+    for (const auto &enum_value : enum_values)
+      request.TryCompleteCurrentArg(enum_value.string_value);
 
   // If this is a source file or symbol type completion, and  there is a -shlib
   // option somewhere in the supplied arguments, then make a search filter for
diff --git a/lldb/source/Plugins/ABI/CMakeLists.txt b/lldb/source/Plugins/ABI/CMakeLists.txt
index c88affd806377..3d10f96114beb 100644
--- a/lldb/source/Plugins/ABI/CMakeLists.txt
+++ b/lldb/source/Plugins/ABI/CMakeLists.txt
@@ -1,14 +1,28 @@
-add_subdirectory(SysV-arm)
-add_subdirectory(SysV-arm64)
-add_subdirectory(SysV-hexagon)
-add_subdirectory(SysV-ppc)
-add_subdirectory(SysV-ppc64)
-add_subdirectory(SysV-mips)
-add_subdirectory(SysV-mips64)
-add_subdirectory(SysV-s390x)
-add_subdirectory(SysV-i386)
-add_subdirectory(SysV-x86_64)
-add_subdirectory(MacOSX-i386)
-add_subdirectory(MacOSX-arm)
-add_subdirectory(MacOSX-arm64)
-add_subdirectory(Windows-x86_64)
+if ("AArch64" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(MacOSX-arm64)
+  add_subdirectory(SysV-arm64)
+endif()
+if ("ARM" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(MacOSX-arm)
+  add_subdirectory(SysV-arm)
+endif()
+if ("Hexagon" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(SysV-hexagon)
+endif()
+if ("Mips" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(SysV-mips)
+  add_subdirectory(SysV-mips64)
+endif()
+if ("PowerPC" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(SysV-ppc)
+  add_subdirectory(SysV-ppc64)
+endif()
+if ("SystemZ" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(SysV-s390x)
+endif()
+if ("X86" IN_LIST LLVM_TARGETS_TO_BUILD)
+  add_subdirectory(SysV-i386)
+  add_subdirectory(SysV-x86_64)
+  add_subdirectory(MacOSX-i386)
+  add_subdirectory(Windows-x86_64)
+endif()
diff --git a/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.cpp b/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.cpp
index f2d01dba69de1..9dff12bcc7482 100644
--- a/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.cpp
+++ b/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.cpp
@@ -1326,7 +1326,8 @@ ABIMacOSX_arm::CreateInstance(ProcessSP process_sp, const ArchSpec &arch) {
   if (vendor_type == llvm::Triple::Apple) {
     if ((arch_type == llvm::Triple::arm) ||
         (arch_type == llvm::Triple::thumb)) {
-      return ABISP(new ABIMacOSX_arm(process_sp));
+      return ABISP(
+          new ABIMacOSX_arm(std::move(process_sp), MakeMCRegisterInfo(arch)));
     }
   }
 
diff --git a/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.h b/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.h
index ac9ba00b9d918..e512651f86e51 100644
--- a/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.h
+++ b/lldb/source/Plugins/ABI/MacOSX-arm/ABIMacOSX_arm.h
@@ -85,7 +85,9 @@ class ABIMacOSX_arm : public lldb_private::ABI {
                            lldb_private::CompilerType &ast_type) const override;
 
 private:
-  ABIMacOSX_arm(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABIMacOSX_arm(lldb::ProcessSP process_sp,
+                std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.cpp b/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.cpp
index e5c72bd549b68..a9c57dfdb2473 100644
--- a/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.cpp
+++ b/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.cpp
@@ -1666,7 +1666,8 @@ ABIMacOSX_arm64::CreateInstance(ProcessSP process_sp, const ArchSpec &arch) {
 
   if (vendor_type == llvm::Triple::Apple) {
     if (arch_type == llvm::Triple::aarch64) {
-      return ABISP(new ABIMacOSX_arm64(process_sp));
+      return ABISP(
+          new ABIMacOSX_arm64(std::move(process_sp), MakeMCRegisterInfo(arch)));
     }
   }
 
diff --git a/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.h b/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.h
index bfacbcd54a944..c7a91ba9c4683 100644
--- a/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.h
+++ b/lldb/source/Plugins/ABI/MacOSX-arm64/ABIMacOSX_arm64.h
@@ -93,7 +93,9 @@ class ABIMacOSX_arm64 : public lldb_private::ABI {
                            lldb_private::CompilerType &ast_type) const override;
 
 private:
-  ABIMacOSX_arm64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABIMacOSX_arm64(lldb::ProcessSP process_sp,
+                  std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.cpp b/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.cpp
index adb40cf1e7ea3..76ebd6476ffda 100644
--- a/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.cpp
+++ b/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.cpp
@@ -710,7 +710,8 @@ ABIMacOSX_i386::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch)
   if ((arch.GetTriple().getArch() == llvm::Triple::x86) &&
       (arch.GetTriple().isMacOSX() || arch.GetTriple().isiOS() ||
        arch.GetTriple().isWatchOS())) {
-    return ABISP(new ABIMacOSX_i386(process_sp));
+    return ABISP(
+        new ABIMacOSX_i386(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.h b/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.h
index 57def683283fb..50062b84d878d 100644
--- a/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.h
+++ b/lldb/source/Plugins/ABI/MacOSX-i386/ABIMacOSX_i386.h
@@ -92,7 +92,9 @@ class ABIMacOSX_i386 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABIMacOSX_i386(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABIMacOSX_i386(lldb::ProcessSP process_sp,
+                 std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.cpp b/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.cpp
index b0747147bfad1..b6e8f8806829a 100644
--- a/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.cpp
+++ b/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.cpp
@@ -1327,7 +1327,8 @@ ABISysV_arm::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (vendor_type != llvm::Triple::Apple) {
     if ((arch_type == llvm::Triple::arm) ||
         (arch_type == llvm::Triple::thumb)) {
-      return ABISP(new ABISysV_arm(process_sp));
+      return ABISP(
+          new ABISysV_arm(std::move(process_sp), MakeMCRegisterInfo(arch)));
     }
   }
 
diff --git a/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.h b/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.h
index a0f00c8f227da..60fb14be5f7bd 100644
--- a/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.h
+++ b/lldb/source/Plugins/ABI/SysV-arm/ABISysV_arm.h
@@ -85,7 +85,9 @@ class ABISysV_arm : public lldb_private::ABI {
                            lldb_private::CompilerType &ast_type) const override;
 
 private:
-  ABISysV_arm(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_arm(lldb::ProcessSP process_sp,
+              std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.cpp b/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.cpp
index beb54031c64d9..781d71c6c05dc 100644
--- a/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.cpp
+++ b/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.cpp
@@ -1669,7 +1669,8 @@ ABISysV_arm64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch)
 
   if (vendor_type != llvm::Triple::Apple) {
     if (arch_type == llvm::Triple::aarch64) {
-      return ABISP(new ABISysV_arm64(process_sp));
+      return ABISP(
+          new ABISysV_arm64(std::move(process_sp), MakeMCRegisterInfo(arch)));
     }
   }
 
diff --git a/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.h b/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.h
index 1fbdc793ed6ec..1bf5773e2db3c 100644
--- a/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.h
+++ b/lldb/source/Plugins/ABI/SysV-arm64/ABISysV_arm64.h
@@ -92,7 +92,9 @@ class ABISysV_arm64 : public lldb_private::ABI {
                            lldb_private::CompilerType &ast_type) const override;
 
 private:
-  ABISysV_arm64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_arm64(lldb::ProcessSP process_sp,
+                std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.cpp b/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.cpp
index ae836851162b2..34d9258ccb924 100644
--- a/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.cpp
+++ b/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.cpp
@@ -1014,7 +1014,8 @@ size_t ABISysV_hexagon::GetRedZoneSize() const { return 0; }
 ABISP
 ABISysV_hexagon::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().getArch() == llvm::Triple::hexagon) {
-    return ABISP(new ABISysV_hexagon(process_sp));
+    return ABISP(
+        new ABISysV_hexagon(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.h b/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.h
index 459b6315dba27..bef64a22d95f6 100644
--- a/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.h
+++ b/lldb/source/Plugins/ABI/SysV-hexagon/ABISysV_hexagon.h
@@ -97,7 +97,9 @@ class ABISysV_hexagon : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_hexagon(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_hexagon(lldb::ProcessSP process_sp,
+                  std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.cpp b/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.cpp
index 41148b7f42619..69e4cff90ebf2 100644
--- a/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.cpp
+++ b/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.cpp
@@ -198,7 +198,8 @@ ABISP
 ABISysV_i386::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().getVendor() != llvm::Triple::Apple) {
     if (arch.GetTriple().getArch() == llvm::Triple::x86) {
-      return ABISP(new ABISysV_i386(process_sp));
+      return ABISP(
+          new ABISysV_i386(std::move(process_sp), MakeMCRegisterInfo(arch)));
     }
   }
   return ABISP();
diff --git a/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.h b/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.h
index 982bdd676b74e..2362e9adda980 100644
--- a/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.h
+++ b/lldb/source/Plugins/ABI/SysV-i386/ABISysV_i386.h
@@ -100,7 +100,9 @@ class ABISysV_i386 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_i386(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_i386(lldb::ProcessSP process_sp,
+               std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.cpp b/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.cpp
index df63a31d198df..416db9f5ae873 100644
--- a/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.cpp
+++ b/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.cpp
@@ -556,7 +556,8 @@ ABISysV_mips::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   const llvm::Triple::ArchType arch_type = arch.GetTriple().getArch();
   if ((arch_type == llvm::Triple::mips) ||
       (arch_type == llvm::Triple::mipsel)) {
-    return ABISP(new ABISysV_mips(process_sp));
+    return ABISP(
+        new ABISysV_mips(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.h b/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.h
index 6cd9c19c22acc..8143f552fc4d4 100644
--- a/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.h
+++ b/lldb/source/Plugins/ABI/SysV-mips/ABISysV_mips.h
@@ -87,7 +87,9 @@ class ABISysV_mips : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_mips(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_mips(lldb::ProcessSP process_sp,
+               std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.cpp b/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.cpp
index 7fb1ea243eacf..72ec0715b6cd5 100644
--- a/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.cpp
+++ b/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.cpp
@@ -554,7 +554,8 @@ size_t ABISysV_mips64::GetRedZoneSize() const { return 0; }
 ABISP
 ABISysV_mips64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().isMIPS64())
-    return ABISP(new ABISysV_mips64(process_sp));
+    return ABISP(
+        new ABISysV_mips64(std::move(process_sp), MakeMCRegisterInfo(arch)));
   return ABISP();
 }
 
diff --git a/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.h b/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.h
index 7da71b36b4b72..76c3c5413b920 100644
--- a/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.h
+++ b/lldb/source/Plugins/ABI/SysV-mips64/ABISysV_mips64.h
@@ -100,7 +100,9 @@ class ABISysV_mips64 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_mips64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_mips64(lldb::ProcessSP process_sp,
+                 std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.cpp b/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.cpp
index e37a6ae046a08..857b7fee10e3e 100644
--- a/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.cpp
+++ b/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.cpp
@@ -218,7 +218,8 @@ size_t ABISysV_ppc::GetRedZoneSize() const { return 224; }
 ABISP
 ABISysV_ppc::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().getArch() == llvm::Triple::ppc) {
-     return ABISP(new ABISysV_ppc(process_sp));
+    return ABISP(
+        new ABISysV_ppc(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.h b/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.h
index 3b199852c30d3..59907c4648ba9 100644
--- a/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.h
+++ b/lldb/source/Plugins/ABI/SysV-ppc/ABISysV_ppc.h
@@ -96,7 +96,9 @@ class ABISysV_ppc : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_ppc(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_ppc(lldb::ProcessSP process_sp,
+              std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.cpp b/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.cpp
index 92d5f9ce08d81..935353c38ca42 100644
--- a/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.cpp
+++ b/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.cpp
@@ -70,7 +70,8 @@ ABISP
 ABISysV_ppc64::CreateInstance(lldb::ProcessSP process_sp,
                               const ArchSpec &arch) {
   if (arch.GetTriple().isPPC64())
-    return ABISP(new ABISysV_ppc64(process_sp));
+    return ABISP(
+        new ABISysV_ppc64(std::move(process_sp), MakeMCRegisterInfo(arch)));
   return ABISP();
 }
 
diff --git a/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.h b/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.h
index d5fb09eec0d02..1b58975dd9d9d 100644
--- a/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.h
+++ b/lldb/source/Plugins/ABI/SysV-ppc64/ABISysV_ppc64.h
@@ -96,7 +96,9 @@ class ABISysV_ppc64 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_ppc64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_ppc64(lldb::ProcessSP process_sp,
+                std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 
diff --git a/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.cpp b/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.cpp
index 227925fd90315..f4f803a8277d2 100644
--- a/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.cpp
+++ b/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.cpp
@@ -200,7 +200,7 @@ size_t ABISysV_s390x::GetRedZoneSize() const { return 0; }
 ABISP
 ABISysV_s390x::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().getArch() == llvm::Triple::systemz) {
-    return ABISP(new ABISysV_s390x(process_sp));
+    return ABISP(new ABISysV_s390x(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.h b/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.h
index 13df477e84bc0..671d6a18260e3 100644
--- a/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.h
+++ b/lldb/source/Plugins/ABI/SysV-s390x/ABISysV_s390x.h
@@ -88,7 +88,9 @@ class ABISysV_s390x : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_s390x(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_s390x(lldb::ProcessSP process_sp,
+                std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.cpp b/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.cpp
index 80b99e6b42bc4..bf1c48f778e1b 100644
--- a/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.cpp
@@ -222,17 +222,35 @@ ABISP
 ABISysV_x86_64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   const llvm::Triple::ArchType arch_type = arch.GetTriple().getArch();
   const llvm::Triple::OSType os_type = arch.GetTriple().getOS();
+  const llvm::Triple::EnvironmentType os_env =
+      arch.GetTriple().getEnvironment();
   if (arch_type == llvm::Triple::x86_64) {
     switch(os_type) {
-      case llvm::Triple::OSType::MacOSX:
-      case llvm::Triple::OSType::Linux:
-      case llvm::Triple::OSType::FreeBSD:
-      case llvm::Triple::OSType::NetBSD:
-      case llvm::Triple::OSType::Solaris:
-      case llvm::Triple::OSType::UnknownOS:
-        return ABISP(new ABISysV_x86_64(process_sp));
-      default: 
+    case llvm::Triple::OSType::IOS:
+    case llvm::Triple::OSType::TvOS:
+    case llvm::Triple::OSType::WatchOS:
+      switch (os_env) {
+      case llvm::Triple::EnvironmentType::MacABI:
+      case llvm::Triple::EnvironmentType::Simulator:
+      case llvm::Triple::EnvironmentType::UnknownEnvironment:
+        // UnknownEnvironment is needed for older compilers that don't
+        // support the simulator environment.
+        return ABISP(new ABISysV_x86_64(std::move(process_sp),
+                                        MakeMCRegisterInfo(arch)));
+      default:
         return ABISP();
+      }
+    case llvm::Triple::OSType::Darwin:
+    case llvm::Triple::OSType::FreeBSD:
+    case llvm::Triple::OSType::Linux:
+    case llvm::Triple::OSType::MacOSX:
+    case llvm::Triple::OSType::NetBSD:
+    case llvm::Triple::OSType::Solaris:
+    case llvm::Triple::OSType::UnknownOS:
+      return ABISP(
+          new ABISysV_x86_64(std::move(process_sp), MakeMCRegisterInfo(arch)));
+    default:
+      return ABISP();
     }
   }
   return ABISP();
diff --git a/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.h b/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.h
index f6704aff348ce..d445d8f4142a5 100644
--- a/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.h
+++ b/lldb/source/Plugins/ABI/SysV-x86_64/ABISysV_x86_64.h
@@ -98,7 +98,9 @@ class ABISysV_x86_64 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABISysV_x86_64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABISysV_x86_64(lldb::ProcessSP process_sp,
+                 std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.cpp b/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.cpp
index fdd10a602099f..ac24426914e1f 100644
--- a/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.cpp
+++ b/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.cpp
@@ -1092,7 +1092,8 @@ ABISP
 ABIWindows_x86_64::CreateInstance(lldb::ProcessSP process_sp, const ArchSpec &arch) {
   if (arch.GetTriple().getArch() == llvm::Triple::x86_64 &&
       arch.GetTriple().isOSWindows()) {
-    return ABISP(new ABIWindows_x86_64(process_sp));
+    return ABISP(
+        new ABIWindows_x86_64(std::move(process_sp), MakeMCRegisterInfo(arch)));
   }
   return ABISP();
 }
diff --git a/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.h b/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.h
index 9f6b2ceef2997..2366566d7809d 100644
--- a/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.h
+++ b/lldb/source/Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.h
@@ -91,7 +91,9 @@ class ABIWindows_x86_64 : public lldb_private::ABI {
   bool RegisterIsCalleeSaved(const lldb_private::RegisterInfo *reg_info);
 
 private:
-  ABIWindows_x86_64(lldb::ProcessSP process_sp) : lldb_private::ABI(process_sp) {
+  ABIWindows_x86_64(lldb::ProcessSP process_sp,
+                    std::unique_ptr<llvm::MCRegisterInfo> info_up)
+      : lldb_private::ABI(std::move(process_sp), std::move(info_up)) {
     // Call CreateInstance instead.
   }
 };
diff --git a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
index 790e786651656..0c212735be7cb 100644
--- a/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
+++ b/lldb/source/Plugins/DynamicLoader/Darwin-Kernel/DynamicLoaderDarwinKernel.cpp
@@ -773,12 +773,9 @@ bool DynamicLoaderDarwinKernel::KextImageInfo::LoadImageUsingMemoryModule(
   }
 
   if (IsKernel() && m_uuid.IsValid()) {
-    Stream *s = target.GetDebugger().GetOutputFile().get();
-    if (s) {
-      s->Printf("Kernel UUID: %s\n",
-                m_uuid.GetAsString().c_str());
-      s->Printf("Load Address: 0x%" PRIx64 "\n", m_load_address);
-    }
+    Stream &s = target.GetDebugger().GetOutputStream();
+    s.Printf("Kernel UUID: %s\n", m_uuid.GetAsString().c_str());
+    s.Printf("Load Address: 0x%" PRIx64 "\n", m_load_address);
   }
 
   if (!m_module_sp) {
@@ -839,11 +836,9 @@ bool DynamicLoaderDarwinKernel::KextImageInfo::LoadImageUsingMemoryModule(
       }
 
       if (IsKernel() && !m_module_sp) {
-        Stream *s = target.GetDebugger().GetOutputFile().get();
-        if (s) {
-          s->Printf("WARNING: Unable to locate kernel binary on the debugger "
-                    "system.\n");
-        }
+        Stream &s = target.GetDebugger().GetOutputStream();
+        s.Printf("WARNING: Unable to locate kernel binary on the debugger "
+                 "system.\n");
       }
     }
 
@@ -932,25 +927,24 @@ bool DynamicLoaderDarwinKernel::KextImageInfo::LoadImageUsingMemoryModule(
   bool is_loaded = IsLoaded();
 
   if (is_loaded && m_module_sp && IsKernel()) {
-    Stream *s = target.GetDebugger().GetOutputFile().get();
-    if (s) {
-      ObjectFile *kernel_object_file = m_module_sp->GetObjectFile();
-      if (kernel_object_file) {
-        addr_t file_address =
-            kernel_object_file->GetBaseAddress().GetFileAddress();
-        if (m_load_address != LLDB_INVALID_ADDRESS &&
-            file_address != LLDB_INVALID_ADDRESS) {
-          s->Printf("Kernel slid 0x%" PRIx64 " in memory.\n",
-                    m_load_address - file_address);
-        }
+    Stream &s = target.GetDebugger().GetOutputStream();
+    ObjectFile *kernel_object_file = m_module_sp->GetObjectFile();
+    if (kernel_object_file) {
+      addr_t file_address =
+          kernel_object_file->GetBaseAddress().GetFileAddress();
+      if (m_load_address != LLDB_INVALID_ADDRESS &&
+          file_address != LLDB_INVALID_ADDRESS) {
+        s.Printf("Kernel slid 0x%" PRIx64 " in memory.\n",
+                 m_load_address - file_address);
       }
-      {
-        s->Printf("Loaded kernel file %s\n",
-                  m_module_sp->GetFileSpec().GetPath().c_str());
-      }
-      s->Flush();
     }
+    {
+      s.Printf("Loaded kernel file %s\n",
+               m_module_sp->GetFileSpec().GetPath().c_str());
+    }
+    s.Flush();
   }
+
   return is_loaded;
 }
 
@@ -1109,11 +1103,10 @@ bool DynamicLoaderDarwinKernel::ReadKextSummaryHeader() {
           lldb::offset_t offset = 0;
           m_kext_summary_header.version = data.GetU32(&offset);
           if (m_kext_summary_header.version > 128) {
-            Stream *s =
-                m_process->GetTarget().GetDebugger().GetOutputFile().get();
-            s->Printf("WARNING: Unable to read kext summary header, got "
-                      "improbable version number %u\n",
-                      m_kext_summary_header.version);
+            Stream &s = m_process->GetTarget().GetDebugger().GetOutputStream();
+            s.Printf("WARNING: Unable to read kext summary header, got "
+                     "improbable version number %u\n",
+                     m_kext_summary_header.version);
             // If we get an improbably large version number, we're probably
             // getting bad memory.
             m_kext_summary_header_addr.Clear();
@@ -1124,11 +1117,11 @@ bool DynamicLoaderDarwinKernel::ReadKextSummaryHeader() {
             if (m_kext_summary_header.entry_size > 4096) {
               // If we get an improbably large entry_size, we're probably
               // getting bad memory.
-              Stream *s =
-                  m_process->GetTarget().GetDebugger().GetOutputFile().get();
-              s->Printf("WARNING: Unable to read kext summary header, got "
-                        "improbable entry_size %u\n",
-                        m_kext_summary_header.entry_size);
+              Stream &s =
+                  m_process->GetTarget().GetDebugger().GetOutputStream();
+              s.Printf("WARNING: Unable to read kext summary header, got "
+                       "improbable entry_size %u\n",
+                       m_kext_summary_header.entry_size);
               m_kext_summary_header_addr.Clear();
               return false;
             }
@@ -1142,11 +1135,10 @@ bool DynamicLoaderDarwinKernel::ReadKextSummaryHeader() {
           if (m_kext_summary_header.entry_count > 10000) {
             // If we get an improbably large number of kexts, we're probably
             // getting bad memory.
-            Stream *s =
-                m_process->GetTarget().GetDebugger().GetOutputFile().get();
-            s->Printf("WARNING: Unable to read kext summary header, got "
-                      "improbable number of kexts %u\n",
-                      m_kext_summary_header.entry_count);
+            Stream &s = m_process->GetTarget().GetDebugger().GetOutputStream();
+            s.Printf("WARNING: Unable to read kext summary header, got "
+                     "improbable number of kexts %u\n",
+                     m_kext_summary_header.entry_count);
             m_kext_summary_header_addr.Clear();
             return false;
           }
@@ -1247,18 +1239,17 @@ bool DynamicLoaderDarwinKernel::ParseKextSummaries(
       number_of_old_kexts_being_removed == 0)
     return true;
 
-  Stream *s = m_process->GetTarget().GetDebugger().GetOutputFile().get();
-  if (s && load_kexts) {
+  Stream &s = m_process->GetTarget().GetDebugger().GetOutputStream();
+  if (load_kexts) {
     if (number_of_new_kexts_being_added > 0 &&
         number_of_old_kexts_being_removed > 0) {
-      s->Printf("Loading %d kext modules and unloading %d kext modules ",
-                number_of_new_kexts_being_added,
-                number_of_old_kexts_being_removed);
+      s.Printf("Loading %d kext modules and unloading %d kext modules ",
+               number_of_new_kexts_being_added,
+               number_of_old_kexts_being_removed);
     } else if (number_of_new_kexts_being_added > 0) {
-      s->Printf("Loading %d kext modules ", number_of_new_kexts_being_added);
+      s.Printf("Loading %d kext modules ", number_of_new_kexts_being_added);
     } else if (number_of_old_kexts_being_removed > 0) {
-      s->Printf("Unloading %d kext modules ",
-                number_of_old_kexts_being_removed);
+      s.Printf("Unloading %d kext modules ", number_of_old_kexts_being_removed);
     }
   }
 
@@ -1304,11 +1295,11 @@ bool DynamicLoaderDarwinKernel::ParseKextSummaries(
             m_process->GetStopID() == image_info.GetProcessStopId())
           loaded_module_list.AppendIfNeeded(image_info.GetModule());
 
-        if (s && load_kexts) {
+        if (load_kexts) {
           if (kext_successfully_added)
-            s->Printf(".");
+            s.Printf(".");
           else
-            s->Printf("-");
+            s.Printf("-");
         }
 
         if (log)
@@ -1330,8 +1321,7 @@ bool DynamicLoaderDarwinKernel::ParseKextSummaries(
           if (image_info.GetModule()) {
             unloaded_module_list.AppendIfNeeded(image_info.GetModule());
           }
-          if (s)
-            s->Printf(".");
+          s.Printf(".");
           image_info.Clear();
           // should pull it out of the KextImageInfos vector but that would
           // mutate the list and invalidate the to_be_removed bool vector;
@@ -1342,12 +1332,12 @@ bool DynamicLoaderDarwinKernel::ParseKextSummaries(
     }
   }
 
-  if (s && load_kexts) {
-    s->Printf(" done.\n");
+  if (load_kexts) {
+    s.Printf(" done.\n");
     if (kexts_failed_to_load.size() > 0 && number_of_new_kexts_being_added > 0) {
-      s->Printf("Failed to load %d of %d kexts:\n",
-                (int)kexts_failed_to_load.size(),
-                number_of_new_kexts_being_added);
+      s.Printf("Failed to load %d of %d kexts:\n",
+               (int)kexts_failed_to_load.size(),
+               number_of_new_kexts_being_added);
       // print a sorted list of <kext-name, uuid> kexts which failed to load
       unsigned longest_name = 0;
       std::sort(kexts_failed_to_load.begin(), kexts_failed_to_load.end());
@@ -1359,10 +1349,10 @@ bool DynamicLoaderDarwinKernel::ParseKextSummaries(
         std::string uuid;
         if (ku.second.IsValid())
           uuid = ku.second.GetAsString();
-        s->Printf (" %-*s %s\n", longest_name, ku.first.c_str(), uuid.c_str());
+        s.Printf(" %-*s %s\n", longest_name, ku.first.c_str(), uuid.c_str());
       }
     }
-    s->Flush();
+    s.Flush();
   }
 
   return true;
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt b/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
index 99d50c4007f92..b114ce7762dec 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CMakeLists.txt
@@ -19,6 +19,7 @@ add_lldb_library(lldbPluginExpressionParserClang PLUGIN
   ClangPersistentVariables.cpp
   ClangUserExpression.cpp
   ClangUtilityFunction.cpp
+  CppModuleConfiguration.cpp
   IRForTarget.cpp
   IRDynamicChecks.cpp
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index e33bfc774b800..b8cab4fdc378c 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -84,9 +84,9 @@ void ClangASTSource::InstallASTContext(clang::ASTContext &ast_context,
                      &type_system_or_err.get())) {
         lldbassert(module_ast_ctx->getASTContext());
         lldbassert(module_ast_ctx->getFileManager());
-        sources.push_back({*module_ast_ctx->getASTContext(),
-                           *module_ast_ctx->getFileManager(),
-                           module_ast_ctx->GetOriginMap()});
+        sources.emplace_back(*module_ast_ctx->getASTContext(),
+                             *module_ast_ctx->getFileManager(),
+                             module_ast_ctx->GetOriginMap());
       }
     }
 
@@ -108,8 +108,7 @@ void ClangASTSource::InstallASTContext(clang::ASTContext &ast_context,
     } while (false);
 
     do {
-      auto *modules_decl_vendor = llvm::cast<ClangModulesDeclVendor>(
-          m_target->GetClangModulesDeclVendor());
+      auto *modules_decl_vendor = m_target->GetClangModulesDeclVendor();
 
       if (!modules_decl_vendor)
         break;
@@ -130,8 +129,6 @@ void ClangASTSource::InstallASTContext(clang::ASTContext &ast_context,
                          *scratch_ast_context->getFileManager(),
                          scratch_ast_context->GetOriginMap()});
     }
-    while (false)
-      ;
 
     m_merger_up =
         std::make_unique<clang::ExternalASTMerger>(target, sources);
@@ -650,6 +647,20 @@ void ClangASTSource::FindExternalLexicalDecls(
 
         m_ast_importer_sp->RequireCompleteType(copied_field_type);
       }
+      auto decl_context_non_const = const_cast<DeclContext *>(decl_context);
+
+      // The decl ended up in the wrong DeclContext. Let's fix that so
+      // the decl we copied will actually be found.
+      // FIXME: This is a horrible hack that shouldn't be necessary. However
+      // it seems our current setup sometimes fails to copy decls to the right
+      // place. See rdar://55129537.
+      if (copied_decl->getDeclContext() != decl_context) {
+        assert(copied_decl->getDeclContext()->containsDecl(copied_decl));
+        copied_decl->getDeclContext()->removeDecl(copied_decl);
+        copied_decl->setDeclContext(decl_context_non_const);
+        assert(!decl_context_non_const->containsDecl(copied_decl));
+        decl_context_non_const->addDeclInternal(copied_decl);
+      }
     } else {
       SkippedDecls = true;
     }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangDiagnostic.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangDiagnostic.h
index 4db1ba096fa88..48cd1c4b99fad 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangDiagnostic.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangDiagnostic.h
@@ -29,7 +29,7 @@ class ClangDiagnostic : public Diagnostic {
     return diag->getKind() == eDiagnosticOriginClang;
   }
 
-  ClangDiagnostic(const char *message, DiagnosticSeverity severity,
+  ClangDiagnostic(llvm::StringRef message, DiagnosticSeverity severity,
                   uint32_t compiler_id)
       : Diagnostic(message, severity, eDiagnosticOriginClang, compiler_id) {}
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
index 3092994f2c0b2..7b5a349005df9 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionDeclMap.cpp
@@ -277,8 +277,8 @@ static clang::QualType ExportAllDeclaredTypes(
     clang::FileManager &source_file_manager,
     const clang::ExternalASTMerger::OriginMap &source_origin_map,
     clang::FileID file, clang::QualType root) {
-  clang::ExternalASTMerger::ImporterSource importer_source = {
-      source, source_file_manager, source_origin_map};
+  clang::ExternalASTMerger::ImporterSource importer_source(
+      source, source_file_manager, source_origin_map);
   merger.AddSources(importer_source);
   clang::ASTImporter &exporter = merger.ImporterForOrigin(source);
   CompleteAllDeclContexts(exporter, file, root);
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
index 9487237f8d136..880185053c343 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.cpp
@@ -105,16 +105,26 @@ using namespace lldb_private;
 class ClangExpressionParser::LLDBPreprocessorCallbacks : public PPCallbacks {
   ClangModulesDeclVendor &m_decl_vendor;
   ClangPersistentVariables &m_persistent_vars;
+  clang::SourceManager &m_source_mgr;
   StreamString m_error_stream;
   bool m_has_errors = false;
 
 public:
   LLDBPreprocessorCallbacks(ClangModulesDeclVendor &decl_vendor,
-                            ClangPersistentVariables &persistent_vars)
-      : m_decl_vendor(decl_vendor), m_persistent_vars(persistent_vars) {}
+                            ClangPersistentVariables &persistent_vars,
+                            clang::SourceManager &source_mgr)
+      : m_decl_vendor(decl_vendor), m_persistent_vars(persistent_vars),
+        m_source_mgr(source_mgr) {}
 
   void moduleImport(SourceLocation import_location, clang::ModuleIdPath path,
                     const clang::Module * /*null*/) override {
+    // Ignore modules that are imported in the wrapper code as these are not
+    // loaded by the user.
+    llvm::StringRef filename =
+        m_source_mgr.getPresumedLoc(import_location).getFilename();
+    if (filename == ClangExpressionSourceCode::g_prefix_file_name)
+      return;
+
     SourceModule module;
 
     for (const std::pair<IdentifierInfo *, SourceLocation> &component : path)
@@ -137,12 +147,14 @@ class ClangExpressionParser::LLDBPreprocessorCallbacks : public PPCallbacks {
 
 class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
 public:
-  ClangDiagnosticManagerAdapter()
-      : m_passthrough(new clang::TextDiagnosticBuffer) {}
-
-  ClangDiagnosticManagerAdapter(
-      const std::shared_ptr<clang::TextDiagnosticBuffer> &passthrough)
-      : m_passthrough(passthrough) {}
+  ClangDiagnosticManagerAdapter(DiagnosticOptions &opts) {
+    DiagnosticOptions *m_options = new DiagnosticOptions(opts);
+    m_options->ShowPresumedLoc = true;
+    m_options->ShowLevel = false;
+    m_os.reset(new llvm::raw_string_ostream(m_output));
+    m_passthrough.reset(
+        new clang::TextDiagnosticPrinter(*m_os, m_options, false));
+  }
 
   void ResetManager(DiagnosticManager *manager = nullptr) {
     m_manager = manager;
@@ -150,12 +162,12 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
 
   void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                         const clang::Diagnostic &Info) override {
-    if (m_manager) {
-      llvm::SmallVector<char, 32> diag_str;
-      Info.FormatDiagnostic(diag_str);
-      diag_str.push_back('\0');
-      const char *data = diag_str.data();
+    // Render diagnostic message to m_output.
+    m_output.clear();
+    m_passthrough->HandleDiagnostic(DiagLevel, Info);
+    m_os->flush();
 
+    if (m_manager) {
       lldb_private::DiagnosticSeverity severity;
       bool make_new_diagnostic = true;
 
@@ -172,12 +184,16 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
         severity = eDiagnosticSeverityRemark;
         break;
       case DiagnosticsEngine::Level::Note:
-        m_manager->AppendMessageToDiagnostic(data);
+        m_manager->AppendMessageToDiagnostic(m_output);
         make_new_diagnostic = false;
       }
       if (make_new_diagnostic) {
+        // ClangDiagnostic messages are expected to have no whitespace/newlines
+        // around them.
+        std::string stripped_output = llvm::StringRef(m_output).trim();
+
         ClangDiagnostic *new_diagnostic =
-            new ClangDiagnostic(data, severity, Info.getID());
+            new ClangDiagnostic(stripped_output, severity, Info.getID());
         m_manager->AddDiagnostic(new_diagnostic);
 
         // Don't store away warning fixits, since the compiler doesn't have
@@ -194,23 +210,17 @@ class ClangDiagnosticManagerAdapter : public clang::DiagnosticConsumer {
         }
       }
     }
-
-    m_passthrough->HandleDiagnostic(DiagLevel, Info);
-  }
-
-  void FlushDiagnostics(DiagnosticsEngine &Diags) {
-    m_passthrough->FlushDiagnostics(Diags);
-  }
-
-  DiagnosticConsumer *clone(DiagnosticsEngine &Diags) const {
-    return new ClangDiagnosticManagerAdapter(m_passthrough);
   }
 
-  clang::TextDiagnosticBuffer *GetPassthrough() { return m_passthrough.get(); }
+  clang::TextDiagnosticPrinter *GetPassthrough() { return m_passthrough.get(); }
 
 private:
   DiagnosticManager *m_manager = nullptr;
-  std::shared_ptr<clang::TextDiagnosticBuffer> m_passthrough;
+  std::shared_ptr<clang::TextDiagnosticPrinter> m_passthrough;
+  /// Output stream of m_passthrough.
+  std::shared_ptr<llvm::raw_string_ostream> m_os;
+  /// Output string filled by m_os.
+  std::string m_output;
 };
 
 static void SetupModuleHeaderPaths(CompilerInstance *compiler,
@@ -231,27 +241,9 @@ static void SetupModuleHeaderPaths(CompilerInstance *compiler,
   search_opts.ModuleCachePath = module_cache.str();
   LLDB_LOG(log, "Using module cache path: {0}", module_cache.c_str());
 
-  FileSpec clang_resource_dir = GetClangResourceDir();
-  std::string resource_dir = clang_resource_dir.GetPath();
-  if (FileSystem::Instance().IsDirectory(resource_dir)) {
-    search_opts.ResourceDir = resource_dir;
-    std::string resource_include = resource_dir + "/include";
-    search_opts.AddPath(resource_include, frontend::System, false, true);
-
-    LLDB_LOG(log, "Added resource include dir: {0}", resource_include);
-  }
+  search_opts.ResourceDir = GetClangResourceDir().GetPath();
 
   search_opts.ImplicitModuleMaps = true;
-
-  std::vector<std::string> system_include_directories =
-      target_sp->GetPlatform()->GetSystemIncludeDirectories(
-          lldb::eLanguageTypeC_plus_plus);
-
-  for (const std::string &include_dir : system_include_directories) {
-    search_opts.AddPath(include_dir, frontend::System, false, true);
-
-    LLDB_LOG(log, "Added system include dir: {0}", include_dir);
-  }
 }
 
 //===----------------------------------------------------------------------===//
@@ -260,10 +252,12 @@ static void SetupModuleHeaderPaths(CompilerInstance *compiler,
 
 ClangExpressionParser::ClangExpressionParser(
     ExecutionContextScope *exe_scope, Expression &expr,
-    bool generate_debug_info, std::vector<std::string> include_directories)
+    bool generate_debug_info, std::vector<std::string> include_directories,
+    std::string filename)
     : ExpressionParser(exe_scope, expr, generate_debug_info), m_compiler(),
       m_pp_callbacks(nullptr),
-      m_include_directories(std::move(include_directories)) {
+      m_include_directories(std::move(include_directories)),
+      m_filename(std::move(filename)) {
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
 
   // We can't compile expressions without a target.  So if the exe_scope is
@@ -557,7 +551,9 @@ ClangExpressionParser::ClangExpressionParser(
 
   // 6. Set up the diagnostic buffer for reporting errors
 
-  m_compiler->getDiagnostics().setClient(new ClangDiagnosticManagerAdapter);
+  auto diag_mgr = new ClangDiagnosticManagerAdapter(
+      m_compiler->getDiagnostics().getDiagnosticOptions());
+  m_compiler->getDiagnostics().setClient(diag_mgr);
 
   // 7. Set up the source management objects inside the compiler
   m_compiler->createFileManager();
@@ -571,8 +567,8 @@ ClangExpressionParser::ClangExpressionParser(
         llvm::cast<ClangPersistentVariables>(
             target_sp->GetPersistentExpressionStateForLanguage(
                 lldb::eLanguageTypeC));
-    std::unique_ptr<PPCallbacks> pp_callbacks(
-        new LLDBPreprocessorCallbacks(*decl_vendor, *clang_persistent_vars));
+    std::unique_ptr<PPCallbacks> pp_callbacks(new LLDBPreprocessorCallbacks(
+        *decl_vendor, *clang_persistent_vars, m_compiler->getSourceManager()));
     m_pp_callbacks =
         static_cast<LLDBPreprocessorCallbacks *>(pp_callbacks.get());
     m_compiler->getPreprocessor().addPPCallbacks(std::move(pp_callbacks));
@@ -589,9 +585,7 @@ ClangExpressionParser::ClangExpressionParser(
   m_compiler->createASTContext();
   clang::ASTContext &ast_context = m_compiler->getASTContext();
 
-  m_ast_context.reset(
-      new ClangASTContext(m_compiler->getTargetOpts().Triple.c_str()));
-  m_ast_context->setASTContext(&ast_context);
+  m_ast_context.reset(new ClangASTContext(ast_context));
 
   std::string module_name("$__lldb_module");
 
@@ -869,8 +863,7 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
   ClangDiagnosticManagerAdapter *adapter =
       static_cast<ClangDiagnosticManagerAdapter *>(
           m_compiler->getDiagnostics().getClient());
-  clang::TextDiagnosticBuffer *diag_buf = adapter->GetPassthrough();
-  diag_buf->FlushDiagnostics(m_compiler->getDiagnostics());
+  auto diag_buf = adapter->GetPassthrough();
 
   adapter->ResetManager(&diagnostic_manager);
 
@@ -901,7 +894,7 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
     }
 
     if (temp_fd != -1) {
-      lldb_private::File file(temp_fd, true);
+      lldb_private::File file(temp_fd, File::eOpenOptionWrite, true);
       const size_t expr_text_len = strlen(expr_text);
       size_t bytes_written = expr_text_len;
       if (file.Write(expr_text, bytes_written).Success()) {
@@ -921,7 +914,7 @@ ClangExpressionParser::ParseInternal(DiagnosticManager &diagnostic_manager,
 
   if (!created_main_file) {
     std::unique_ptr<MemoryBuffer> memory_buffer =
-        MemoryBuffer::getMemBufferCopy(expr_text, "<lldb-expr>");
+        MemoryBuffer::getMemBufferCopy(expr_text, m_filename);
     source_mgr.setMainFileID(source_mgr.createFileID(std::move(memory_buffer)));
   }
 
@@ -1228,12 +1221,10 @@ lldb_private::Status ClangExpressionParser::PrepareForExecution(
       type_system_helper->DeclMap(); // result can be NULL
 
   if (decl_map) {
-    Stream *error_stream = nullptr;
     Target *target = exe_ctx.GetTargetPtr();
-    error_stream = target->GetDebugger().GetErrorFile().get();
-
+    auto &error_stream = target->GetDebugger().GetErrorStream();
     IRForTarget ir_for_target(decl_map, m_expr.NeedsVariableResolution(),
-                              *execution_unit_sp, *error_stream,
+                              *execution_unit_sp, error_stream,
                               function_name.AsCString());
 
     bool ir_can_run =
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.h
index 80a92691eaacf..79ad5728bf741 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionParser.h
@@ -53,9 +53,14 @@ class ClangExpressionParser : public ExpressionParser {
   /// @param[in] include_directories
   ///     List of include directories that should be used when parsing the
   ///     expression.
+  ///
+  /// @param[in] filename
+  ///     Name of the source file that should be used when rendering
+  ///     diagnostics (i.e. errors, warnings or notes from Clang).
   ClangExpressionParser(ExecutionContextScope *exe_scope, Expression &expr,
                         bool generate_debug_info,
-                        std::vector<std::string> include_directories = {});
+                        std::vector<std::string> include_directories = {},
+                        std::string filename = "<clang expression>");
 
   /// Destructor
   ~ClangExpressionParser() override;
@@ -178,6 +183,8 @@ class ClangExpressionParser : public ExpressionParser {
   std::unique_ptr<ClangASTContext> m_ast_context;
 
   std::vector<std::string> m_include_directories;
+  /// File name used for the user expression.
+  std::string m_filename;
 };
 }
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
index 88cfe5f081919..ee890ac08e644 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.cpp
@@ -29,7 +29,12 @@
 
 using namespace lldb_private;
 
-const char *ClangExpressionSourceCode::g_expression_prefix = R"(
+#define PREFIX_NAME "<lldb wrapper prefix>"
+
+const llvm::StringRef ClangExpressionSourceCode::g_prefix_file_name = PREFIX_NAME;
+
+const char *ClangExpressionSourceCode::g_expression_prefix =
+"#line 1 \"" PREFIX_NAME R"("
 #ifndef offsetof
 #define offsetof(t, d) __builtin_offsetof(t, d)
 #endif
@@ -67,9 +72,6 @@ extern "C"
 }
 )";
 
-static const char *c_start_marker = "    /*LLDB_BODY_START*/\n    ";
-static const char *c_end_marker = ";\n    /*LLDB_BODY_END*/\n";
-
 namespace {
 
 class AddMacroState {
@@ -169,6 +171,17 @@ static void AddMacros(const DebugMacros *dm, CompileUnit *comp_unit,
   }
 }
 
+lldb_private::ClangExpressionSourceCode::ClangExpressionSourceCode(
+    llvm::StringRef filename, llvm::StringRef name, llvm::StringRef prefix,
+    llvm::StringRef body, Wrapping wrap)
+    : ExpressionSourceCode(name, prefix, body, wrap) {
+  // Use #line markers to pretend that we have a single-line source file
+  // containing only the user expression. This will hide our wrapper code
+  // from the user when we render diagnostics with Clang.
+  m_start_marker = "#line 1 \"" + filename.str() + "\"\n";
+  m_end_marker = "\n;\n#line 1 \"<lldb wrapper suffix>\"\n";
+}
+
 namespace {
 /// Allows checking if a token is contained in a given expression.
 class TokenVerifier {
@@ -401,9 +414,9 @@ bool ClangExpressionSourceCode::GetText(
     case lldb::eLanguageTypeC:
     case lldb::eLanguageTypeC_plus_plus:
     case lldb::eLanguageTypeObjC:
-      tagged_body.append(c_start_marker);
+      tagged_body.append(m_start_marker);
       tagged_body.append(m_body);
-      tagged_body.append(c_end_marker);
+      tagged_body.append(m_end_marker);
       break;
     }
     switch (wrapping_language) {
@@ -477,24 +490,19 @@ bool ClangExpressionSourceCode::GetText(
 bool ClangExpressionSourceCode::GetOriginalBodyBounds(
     std::string transformed_text, lldb::LanguageType wrapping_language,
     size_t &start_loc, size_t &end_loc) {
-  const char *start_marker;
-  const char *end_marker;
-
   switch (wrapping_language) {
   default:
     return false;
   case lldb::eLanguageTypeC:
   case lldb::eLanguageTypeC_plus_plus:
   case lldb::eLanguageTypeObjC:
-    start_marker = c_start_marker;
-    end_marker = c_end_marker;
     break;
   }
 
-  start_loc = transformed_text.find(start_marker);
+  start_loc = transformed_text.find(m_start_marker);
   if (start_loc == std::string::npos)
     return false;
-  start_loc += strlen(start_marker);
-  end_loc = transformed_text.find(end_marker);
+  start_loc += m_start_marker.size();
+  end_loc = transformed_text.find(m_end_marker);
   return end_loc != std::string::npos;
 }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.h
index 5b5f398dd93d6..1d159670b962a 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangExpressionSourceCode.h
@@ -23,11 +23,16 @@ class ExecutionContext;
 
 class ClangExpressionSourceCode : public ExpressionSourceCode {
 public:
+  /// The file name we use for the wrapper code that we inject before
+  /// the user expression.
+  static const llvm::StringRef g_prefix_file_name;
   static const char *g_expression_prefix;
 
-  static ClangExpressionSourceCode *CreateWrapped(llvm::StringRef prefix,
+  static ClangExpressionSourceCode *CreateWrapped(llvm::StringRef filename,
+                                                  llvm::StringRef prefix,
                                                   llvm::StringRef body) {
-    return new ClangExpressionSourceCode("$__lldb_expr", prefix, body, Wrap);
+    return new ClangExpressionSourceCode(filename, "$__lldb_expr", prefix, body,
+                                         Wrap);
   }
 
   /// Generates the source code that will evaluate the expression.
@@ -54,14 +59,20 @@ class ClangExpressionSourceCode : public ExpressionSourceCode {
   // Given a string returned by GetText, find the beginning and end of the body
   // passed to CreateWrapped. Return true if the bounds could be found.  This
   // will also work on text with FixItHints applied.
-  static bool GetOriginalBodyBounds(std::string transformed_text,
-                                    lldb::LanguageType wrapping_language,
-                                    size_t &start_loc, size_t &end_loc);
+  bool GetOriginalBodyBounds(std::string transformed_text,
+                             lldb::LanguageType wrapping_language,
+                             size_t &start_loc, size_t &end_loc);
 
 protected:
-  ClangExpressionSourceCode(llvm::StringRef name, llvm::StringRef prefix,
-                            llvm::StringRef body, Wrapping wrap)
-      : ExpressionSourceCode(name, prefix, body, wrap) {}
+  ClangExpressionSourceCode(llvm::StringRef filename, llvm::StringRef name,
+                            llvm::StringRef prefix, llvm::StringRef body,
+                            Wrapping wrap);
+
+private:
+  /// String marking the start of the user expression.
+  std::string m_start_marker;
+  /// String marking the end of the user expression.
+  std::string m_end_marker;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
index c569291700750..f3df589d73112 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp
@@ -163,9 +163,7 @@ ClangModulesDeclVendorImpl::ClangModulesDeclVendorImpl(
       m_parser(std::move(parser)), m_origin_map() {
 
   // Initialize our ClangASTContext.
-  auto target_opts = m_compiler_invocation->getTargetOpts();
-  m_ast_context.reset(new ClangASTContext(target_opts.Triple.c_str()));
-  m_ast_context->setASTContext(&m_compiler_instance->getASTContext());
+  m_ast_context.reset(new ClangASTContext(m_compiler_instance->getASTContext()));
 }
 
 void ClangModulesDeclVendorImpl::ReportModuleExportsHelper(
@@ -573,8 +571,9 @@ ClangModulesDeclVendorImpl::DoGetModule(clang::ModuleIdPath path,
 
 clang::ExternalASTMerger::ImporterSource
 ClangModulesDeclVendorImpl::GetImporterSource() {
-  return {m_compiler_instance->getASTContext(),
-          m_compiler_instance->getFileManager(), m_origin_map};
+  return clang::ExternalASTMerger::ImporterSource(
+      m_compiler_instance->getASTContext(),
+      m_compiler_instance->getFileManager(), m_origin_map);
 }
 
 static const char *ModuleImportBufferName = "LLDBModulesMemoryBuffer";
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.h
index a53d3b985ecbe..95e6c3ac963de 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangPersistentVariables.h
@@ -50,6 +50,15 @@ class ClangPersistentVariables : public PersistentExpressionState {
     return "$";
   }
 
+  /// Returns the next file name that should be used for user expressions.
+  std::string GetNextExprFileName() {
+    std::string name;
+    name.append("<user expression ");
+    name.append(std::to_string(m_next_user_file_id++));
+    name.append(">");
+    return name;
+  }
+
   llvm::Optional<CompilerType>
   GetCompilerTypeFromPersistentDecl(ConstString type_name) override;
 
@@ -66,6 +75,8 @@ class ClangPersistentVariables : public PersistentExpressionState {
   }
 
 private:
+  /// The counter used by GetNextExprFileName.
+  uint32_t m_next_user_file_id = 0;
   // The counter used by GetNextPersistentVariableName
   uint32_t m_next_persistent_variable_id = 0;
 
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
index 57bf654528add..31c85d2087459 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.cpp
@@ -23,9 +23,9 @@
 #include "ClangDiagnostic.h"
 #include "ClangExpressionDeclMap.h"
 #include "ClangExpressionParser.h"
-#include "ClangExpressionSourceCode.h"
 #include "ClangModulesDeclVendor.h"
 #include "ClangPersistentVariables.h"
+#include "CppModuleConfiguration.h"
 
 #include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
@@ -313,17 +313,13 @@ void ClangUserExpression::ScanContext(ExecutionContext &exe_ctx, Status &err) {
 // count is not available, [myArray count] returns id, which can't be directly
 // cast to int without causing a clang error.
 static void ApplyObjcCastHack(std::string &expr) {
-#define OBJC_CAST_HACK_FROM "(int)["
-#define OBJC_CAST_HACK_TO "(int)(long long)["
+  const std::string from = "(int)[";
+  const std::string to = "(int)(long long)[";
 
-  size_t from_offset;
+  size_t offset;
 
-  while ((from_offset = expr.find(OBJC_CAST_HACK_FROM)) != expr.npos)
-    expr.replace(from_offset, sizeof(OBJC_CAST_HACK_FROM) - 1,
-                 OBJC_CAST_HACK_TO);
-
-#undef OBJC_CAST_HACK_TO
-#undef OBJC_CAST_HACK_FROM
+  while ((offset = expr.find(from)) != expr.npos)
+    expr.replace(offset, from.size(), to);
 }
 
 bool ClangUserExpression::SetupPersistentState(DiagnosticManager &diagnostic_manager,
@@ -332,6 +328,7 @@ bool ClangUserExpression::SetupPersistentState(DiagnosticManager &diagnostic_man
     if (PersistentExpressionState *persistent_state =
             target->GetPersistentExpressionStateForLanguage(
                 lldb::eLanguageTypeC)) {
+      m_clang_state = llvm::cast<ClangPersistentVariables>(persistent_state);
       m_result_delegate.RegisterPersistentState(persistent_state);
     } else {
       diagnostic_manager.PutString(
@@ -396,18 +393,18 @@ void ClangUserExpression::CreateSourceCode(
     DiagnosticManager &diagnostic_manager, ExecutionContext &exe_ctx,
     std::vector<std::string> modules_to_import, bool for_completion) {
 
+  m_filename = m_clang_state->GetNextExprFileName();
   std::string prefix = m_expr_prefix;
 
   if (m_options.GetExecutionPolicy() == eExecutionPolicyTopLevel) {
     m_transformed_text = m_expr_text;
   } else {
-    std::unique_ptr<ClangExpressionSourceCode> source_code(
-        ClangExpressionSourceCode::CreateWrapped(prefix.c_str(),
-                                                 m_expr_text.c_str()));
+    m_source_code.reset(ClangExpressionSourceCode::CreateWrapped(
+        m_filename, prefix.c_str(), m_expr_text.c_str()));
 
-    if (!source_code->GetText(m_transformed_text, m_expr_lang,
-                              m_in_static_method, exe_ctx, !m_ctx_obj,
-                              for_completion, modules_to_import)) {
+    if (!m_source_code->GetText(m_transformed_text, m_expr_lang,
+                                m_in_static_method, exe_ctx, !m_ctx_obj,
+                                for_completion, modules_to_import)) {
       diagnostic_manager.PutString(eDiagnosticSeverityError,
                                    "couldn't construct expression body");
       return;
@@ -417,7 +414,7 @@ void ClangUserExpression::CreateSourceCode(
     // transformed code. We need this later for the code completion.
     std::size_t original_start;
     std::size_t original_end;
-    bool found_bounds = source_code->GetOriginalBodyBounds(
+    bool found_bounds = m_source_code->GetOriginalBodyBounds(
         m_transformed_text, m_expr_lang, original_start, original_end);
     if (found_bounds)
       m_user_expression_start_pos = original_start;
@@ -437,48 +434,59 @@ static bool SupportsCxxModuleImport(lldb::LanguageType language) {
   }
 }
 
-std::vector<std::string>
-ClangUserExpression::GetModulesToImport(ExecutionContext &exe_ctx) {
+/// Utility method that puts a message into the expression log and
+/// returns an invalid module configuration.
+static CppModuleConfiguration LogConfigError(const std::string &msg) {
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_EXPRESSIONS));
+  LLDB_LOG(log, "[C++ module config] {0}", msg);
+  return CppModuleConfiguration();
+}
 
-  if (!SupportsCxxModuleImport(Language()))
-    return {};
+CppModuleConfiguration GetModuleConfig(lldb::LanguageType language,
+                                       ExecutionContext &exe_ctx) {
+  // Don't do anything if this is not a C++ module configuration.
+  if (!SupportsCxxModuleImport(language))
+    return LogConfigError("Language doesn't support C++ modules");
 
   Target *target = exe_ctx.GetTargetPtr();
-  if (!target || !target->GetEnableImportStdModule())
-    return {};
+  if (!target)
+    return LogConfigError("No target");
+
+  if (!target->GetEnableImportStdModule())
+    return LogConfigError("Importing std module not enabled in settings");
 
   StackFrame *frame = exe_ctx.GetFramePtr();
   if (!frame)
-    return {};
+    return LogConfigError("No frame");
 
   Block *block = frame->GetFrameBlock();
   if (!block)
-    return {};
+    return LogConfigError("No block");
 
   SymbolContext sc;
   block->CalculateSymbolContext(&sc);
   if (!sc.comp_unit)
-    return {};
-
-  if (log) {
-    for (const SourceModule &m : sc.comp_unit->GetImportedModules()) {
-      LLDB_LOG(log, "Found module in compile unit: {0:$[.]} - include dir: {1}",
-                  llvm::make_range(m.path.begin(), m.path.end()), m.search_path);
+    return LogConfigError("Couldn't calculate symbol context");
+
+  // Build a list of files we need to analyze to build the configuration.
+  FileSpecList files;
+  for (const FileSpec &f : sc.comp_unit->GetSupportFiles())
+    files.AppendIfUnique(f);
+  // We also need to look at external modules in the case of -gmodules as they
+  // contain the support files for libc++ and the C library.
+  sc.comp_unit->ForEachExternalModule([&files](lldb::ModuleSP module) {
+    for (std::size_t i = 0; i < module->GetNumCompileUnits(); ++i) {
+      const FileSpecList &support_files =
+          module->GetCompileUnitAtIndex(i)->GetSupportFiles();
+      for (const FileSpec &f : support_files) {
+        files.AppendIfUnique(f);
+      }
     }
-  }
-
-  for (const SourceModule &m : sc.comp_unit->GetImportedModules())
-    m_include_directories.emplace_back(m.search_path.GetCString());
-
-  // Check if we imported 'std' or any of its submodules.
-  // We currently don't support importing any other modules in the expression
-  // parser.
-  for (const SourceModule &m : sc.comp_unit->GetImportedModules())
-    if (!m.path.empty() && m.path.front() == "std")
-      return {"std"};
-
-  return {};
+  });
+  // Try to create a configuration from the files. If there is no valid
+  // configuration possible with the files, this just returns an invalid
+  // configuration.
+  return CppModuleConfiguration(files);
 }
 
 bool ClangUserExpression::PrepareForParsing(
@@ -506,14 +514,21 @@ bool ClangUserExpression::PrepareForParsing(
 
   SetupDeclVendor(exe_ctx, m_target);
 
-  std::vector<std::string> used_modules = GetModulesToImport(exe_ctx);
-  m_imported_cpp_modules = !used_modules.empty();
+  CppModuleConfiguration module_config = GetModuleConfig(m_language, exe_ctx);
+  llvm::ArrayRef<std::string> imported_modules =
+      module_config.GetImportedModules();
+  m_imported_cpp_modules = !imported_modules.empty();
+  m_include_directories = module_config.GetIncludeDirs();
 
   LLDB_LOG(log, "List of imported modules in expression: {0}",
-           llvm::make_range(used_modules.begin(), used_modules.end()));
+           llvm::make_range(imported_modules.begin(), imported_modules.end()));
+  LLDB_LOG(log, "List of include directories gathered for modules: {0}",
+           llvm::make_range(m_include_directories.begin(),
+                            m_include_directories.end()));
 
   UpdateLanguageForExpr();
-  CreateSourceCode(diagnostic_manager, exe_ctx, used_modules, for_completion);
+  CreateSourceCode(diagnostic_manager, exe_ctx, imported_modules,
+                   for_completion);
   return true;
 }
 
@@ -572,7 +587,7 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
   // parser_sp will never be empty.
 
   ClangExpressionParser parser(exe_scope, *this, generate_debug_info,
-                               m_include_directories);
+                               m_include_directories, m_filename);
 
   unsigned num_errors = parser.Parse(diagnostic_manager);
 
@@ -585,8 +600,11 @@ bool ClangUserExpression::Parse(DiagnosticManager &diagnostic_manager,
         size_t fixed_end;
         const std::string &fixed_expression =
             diagnostic_manager.GetFixedExpression();
-        if (ClangExpressionSourceCode::GetOriginalBodyBounds(
-                fixed_expression, m_expr_lang, fixed_start, fixed_end))
+        // Retrieve the original expression in case we don't have a top level
+        // expression (which has no surrounding source code).
+        if (m_source_code &&
+            m_source_code->GetOriginalBodyBounds(fixed_expression, m_expr_lang,
+                                                 fixed_start, fixed_end))
           m_fixed_text =
               fixed_expression.substr(fixed_start, fixed_end - fixed_start);
       }
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
index 9af5ce6cf24b8..d94f9cc5e0660 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangUserExpression.h
@@ -15,6 +15,7 @@
 #include "ASTStructExtractor.h"
 #include "ClangExpressionDeclMap.h"
 #include "ClangExpressionHelper.h"
+#include "ClangExpressionSourceCode.h"
 #include "ClangExpressionVariable.h"
 #include "IRForTarget.h"
 
@@ -178,7 +179,6 @@ class ClangUserExpression : public LLVMUserExpression {
                     lldb::addr_t struct_address,
                     DiagnosticManager &diagnostic_manager) override;
 
-  std::vector<std::string> GetModulesToImport(ExecutionContext &exe_ctx);
   void CreateSourceCode(DiagnosticManager &diagnostic_manager,
                         ExecutionContext &exe_ctx,
                         std::vector<std::string> modules_to_import,
@@ -216,6 +216,10 @@ class ClangUserExpression : public LLVMUserExpression {
   /// were not able to calculate this position.
   llvm::Optional<size_t> m_user_expression_start_pos;
   ResultDelegate m_result_delegate;
+  ClangPersistentVariables *m_clang_state;
+  std::unique_ptr<ClangExpressionSourceCode> m_source_code;
+  /// File name used for the expression.
+  std::string m_filename;
 
   /// The object (if any) in which context the expression is evaluated.
   /// See the comment to `UserExpression::Evaluate` for details.
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp
new file mode 100644
index 0000000000000..1f9d33e9beb4d
--- /dev/null
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.cpp
@@ -0,0 +1,82 @@
+//===-- CppModuleConfiguration.cpp ----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CppModuleConfiguration.h"
+
+#include "ClangHost.h"
+#include "lldb/Host/FileSystem.h"
+
+using namespace lldb_private;
+
+bool CppModuleConfiguration::SetOncePath::TrySet(llvm::StringRef path) {
+  // Setting for the first time always works.
+  if (m_first) {
+    m_path = path.str();
+    m_valid = true;
+    m_first = false;
+    return true;
+  }
+  // Changing the path to the same value is fine.
+  if (m_path == path)
+    return true;
+
+  // Changing the path after it was already set is not allowed.
+  m_valid = false;
+  return false;
+}
+
+bool CppModuleConfiguration::analyzeFile(const FileSpec &f) {
+  using namespace llvm::sys::path;
+  // Convert to slashes to make following operations simpler.
+  std::string dir_buffer = convert_to_slash(f.GetDirectory().GetStringRef());
+  llvm::StringRef posix_dir(dir_buffer);
+
+  // Check for /c++/vX/ that is used by libc++.
+  static llvm::Regex libcpp_regex(R"regex(/c[+][+]/v[0-9]/)regex");
+  if (libcpp_regex.match(f.GetPath())) {
+    // Strip away libc++'s /experimental directory if there is one.
+    posix_dir.consume_back("/experimental");
+    return m_std_inc.TrySet(posix_dir);
+  }
+
+  // Check for /usr/include. On Linux this might be /usr/include/bits, so
+  // we should remove that '/bits' suffix to get the actual include directory.
+  if (posix_dir.endswith("/usr/include/bits"))
+    posix_dir.consume_back("/bits");
+  if (posix_dir.endswith("/usr/include"))
+    return m_c_inc.TrySet(posix_dir);
+
+  // File wasn't interesting, continue analyzing.
+  return true;
+}
+
+bool CppModuleConfiguration::hasValidConfig() {
+  // We all these include directories to have a valid usable configuration.
+  return m_c_inc.Valid() && m_std_inc.Valid();
+}
+
+CppModuleConfiguration::CppModuleConfiguration(
+    const FileSpecList &support_files) {
+  // Analyze all files we were given to build the configuration.
+  bool error = !std::all_of(support_files.begin(), support_files.end(),
+                            std::bind(&CppModuleConfiguration::analyzeFile,
+                                      this, std::placeholders::_1));
+  // If we have a valid configuration at this point, set the
+  // include directories and module list that should be used.
+  if (!error && hasValidConfig()) {
+    // Calculate the resource directory for LLDB.
+    llvm::SmallString<256> resource_dir;
+    llvm::sys::path::append(resource_dir, GetClangResourceDir().GetPath(),
+                            "include");
+    m_resource_inc = resource_dir.str();
+
+    // This order matches the way Clang orders these directories.
+    m_include_dirs = {m_std_inc.Get(), m_resource_inc, m_c_inc.Get()};
+    m_imported_modules = {"std"};
+  }
+}
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
new file mode 100644
index 0000000000000..8e892e37d0de9
--- /dev/null
+++ b/lldb/source/Plugins/ExpressionParser/Clang/CppModuleConfiguration.h
@@ -0,0 +1,84 @@
+//===-- CppModuleConfiguration.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef liblldb_CppModuleConfiguration_h_
+#define liblldb_CppModuleConfiguration_h_
+
+#include <lldb/Core/FileSpecList.h>
+#include <llvm/Support/Regex.h>
+
+namespace lldb_private {
+
+/// A Clang configuration when importing C++ modules.
+///
+/// Includes a list of include paths that should be used when importing
+/// and a list of modules that can be imported. Currently only used when
+/// importing the 'std' module and its dependencies.
+class CppModuleConfiguration {
+  /// Utility class for a path that can only be set once.
+  class SetOncePath {
+    std::string m_path;
+    bool m_valid = false;
+    /// True iff this path hasn't been set yet.
+    bool m_first = true;
+
+  public:
+    /// Try setting the path. Returns true if the path was set and false if
+    /// the path was already set.
+    LLVM_NODISCARD bool TrySet(llvm::StringRef path);
+    /// Return the path if there is one.
+    std::string Get() const {
+      assert(m_valid && "Called Get() on an invalid SetOncePath?");
+      return m_path;
+    }
+    /// Returns true iff this path was set exactly once so far.
+    bool Valid() const { return m_valid; }
+  };
+
+  /// If valid, the include path used for the std module.
+  SetOncePath m_std_inc;
+  /// If valid, the include path to the C library (e.g. /usr/include).
+  SetOncePath m_c_inc;
+  /// The Clang resource include path for this configuration.
+  std::string m_resource_inc;
+
+  std::vector<std::string> m_include_dirs;
+  std::vector<std::string> m_imported_modules;
+
+  /// Analyze a given source file to build the current configuration.
+  /// Returns false iff there was a fatal error that makes analyzing any
+  /// further files pointless as the configuration is now invalid.
+  bool analyzeFile(const FileSpec &f);
+
+public:
+  /// Creates a configuraiton by analyzing the given list of used source files.
+  ///
+  /// Currently only looks at the used paths and doesn't actually access the
+  /// files on the disk.
+  explicit CppModuleConfiguration(const FileSpecList &support_files);
+  /// Creates an empty and invalid configuration.
+  CppModuleConfiguration() {}
+
+  /// Returns true iff this is a valid configuration that can be used to
+  /// load and compile modules.
+  bool hasValidConfig();
+
+  /// Returns a list of include directories that should be used when using this
+  /// configuration (e.g. {"/usr/include", "/usr/include/c++/v1"}).
+  llvm::ArrayRef<std::string> GetIncludeDirs() const { return m_include_dirs; }
+
+  /// Returns a list of (top level) modules that should be imported when using
+  /// this configuration (e.g. {"std"}).
+  llvm::ArrayRef<std::string> GetImportedModules() const {
+    return m_imported_modules;
+  }
+};
+
+} // namespace lldb_private
+
+#endif
diff --git a/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp b/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp
index c8ac04641e685..2e5dd5989e774 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/ASan/ASanRuntime.cpp
@@ -264,7 +264,7 @@ bool AddressSanitizerRuntime::NotifyBreakpointHit(
                                      *thread_sp, description, report));
 
     StreamFileSP stream_sp(
-        process_sp->GetTarget().GetDebugger().GetOutputFile());
+        process_sp->GetTarget().GetDebugger().GetOutputStreamSP());
     if (stream_sp) {
       stream_sp->Printf("AddressSanitizer report breakpoint hit. Use 'thread "
                         "info -s' to get extended information about the "
diff --git a/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp b/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp
index 89f2139db71b1..45a3aeeb204ef 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/TSan/TSanRuntime.cpp
@@ -863,13 +863,11 @@ bool ThreadSanitizerRuntime::NotifyBreakpointHit(
               CreateStopReasonWithInstrumentationData(
                   *thread_sp, stop_reason_description, report));
 
-    StreamFileSP stream_sp(
-        process_sp->GetTarget().GetDebugger().GetOutputFile());
-    if (stream_sp) {
-      stream_sp->Printf("ThreadSanitizer report breakpoint hit. Use 'thread "
-                        "info -s' to get extended information about the "
-                        "report.\n");
-    }
+    StreamFile &s = process_sp->GetTarget().GetDebugger().GetOutputStream();
+    s.Printf("ThreadSanitizer report breakpoint hit. Use 'thread "
+             "info -s' to get extended information about the "
+             "report.\n");
+
     return true; // Return true to stop the target
   } else
     return false; // Let target run
diff --git a/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp b/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp
index 367098bb448e8..50f1d48d03e0b 100644
--- a/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp
+++ b/lldb/source/Plugins/InstrumentationRuntime/UBSan/UBSanRuntime.cpp
@@ -124,7 +124,7 @@ StructuredData::ObjectSP UndefinedBehaviorSanitizerRuntime::RetrieveReportData(
   if (!frame_sp)
     return StructuredData::ObjectSP();
 
-  StreamFileSP Stream(target.GetDebugger().GetOutputFile());
+  StreamFileSP Stream = target.GetDebugger().GetOutputStreamSP();
 
   EvaluateExpressionOptions options;
   options.SetUnwindOnError(true);
diff --git a/lldb/source/Plugins/Language/ObjC/NSString.cpp b/lldb/source/Plugins/Language/ObjC/NSString.cpp
index 4800c955e5f51..55e129b098dce 100644
--- a/lldb/source/Plugins/Language/ObjC/NSString.cpp
+++ b/lldb/source/Plugins/Language/ObjC/NSString.cpp
@@ -78,12 +78,12 @@ bool lldb_private::formatters::NSStringSummaryProvider(
     return false;
 
   ConstString class_name_cs = descriptor->GetClassName();
-  const char *class_name = class_name_cs.GetCString();
+  llvm::StringRef class_name = class_name_cs.GetStringRef();
 
-  if (!class_name || !*class_name)
+  if (class_name.empty())
     return false;
 
-  bool is_tagged_ptr = (0 == strcmp(class_name, "NSTaggedPointerString")) &&
+  bool is_tagged_ptr = class_name == "NSTaggedPointerString" &&
                        descriptor->GetTaggedPointerInfo();
   // for a tagged pointer, the descriptor has everything we need
   if (is_tagged_ptr)
@@ -111,7 +111,7 @@ bool lldb_private::formatters::NSStringSummaryProvider(
   bool is_inline = (info_bits & 0x60) == 0;
   bool has_explicit_length = (info_bits & (1 | 4)) != 4;
   bool is_unicode = (info_bits & 0x10) == 0x10;
-  bool is_path_store = strcmp(class_name, "NSPathStore2") == 0;
+  bool is_path_store = class_name == "NSPathStore2";
   bool has_null = (info_bits & 8) == 8;
 
   size_t explicit_length = 0;
@@ -135,14 +135,14 @@ bool lldb_private::formatters::NSStringSummaryProvider(
     }
   }
 
-  if (strcmp(class_name, "NSString") && strcmp(class_name, "CFStringRef") &&
-      strcmp(class_name, "CFMutableStringRef") &&
-      strcmp(class_name, "__NSCFConstantString") &&
-      strcmp(class_name, "__NSCFString") &&
-      strcmp(class_name, "NSCFConstantString") &&
-      strcmp(class_name, "NSCFString") && strcmp(class_name, "NSPathStore2")) {
+  const llvm::StringSet<> supported_string_classes = {
+      "NSString",     "CFMutableStringRef",
+      "CFStringRef",  "__NSCFConstantString",
+      "__NSCFString", "NSCFConstantString",
+      "NSCFString",   "NSPathStore2"};
+  if (supported_string_classes.count(class_name) == 0) {
     // not one of us - but tell me class name
-    stream.Printf("class name = %s", class_name);
+    stream.Printf("class name = %s", class_name_cs.GetCString());
     return true;
   }
 
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index 6713a0ad674d0..72aea1f2a5a4a 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -95,31 +95,27 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfoFromVTableAddress(
             const bool exact_match = true;
             TypeList class_types;
 
-            uint32_t num_matches = 0;
             // First look in the module that the vtable symbol came from and
             // look for a single exact match.
             llvm::DenseSet<SymbolFile *> searched_symbol_files;
-            if (sc.module_sp) {
-              num_matches = sc.module_sp->FindTypes(
-                  ConstString(lookup_name), exact_match, 1,
-                  searched_symbol_files, class_types);
-            }
+            if (sc.module_sp)
+              sc.module_sp->FindTypes(ConstString(lookup_name), exact_match, 1,
+                                      searched_symbol_files, class_types);
 
             // If we didn't find a symbol, then move on to the entire module
             // list in the target and get as many unique matches as possible
-            if (num_matches == 0) {
-              num_matches = target.GetImages().FindTypes(
-                  nullptr, ConstString(lookup_name), exact_match, UINT32_MAX,
-                  searched_symbol_files, class_types);
-            }
+            if (class_types.Empty())
+              target.GetImages().FindTypes(nullptr, ConstString(lookup_name),
+                                           exact_match, UINT32_MAX,
+                                           searched_symbol_files, class_types);
 
             lldb::TypeSP type_sp;
-            if (num_matches == 0) {
+            if (class_types.Empty()) {
               LLDB_LOGF(log, "0x%16.16" PRIx64 ": is not dynamic\n",
                         original_ptr);
               return TypeAndOrName();
             }
-            if (num_matches == 1) {
+            if (class_types.GetSize() == 1) {
               type_sp = class_types.GetTypeAtIndex(0);
               if (type_sp) {
                 if (ClangASTContext::IsCXXClassType(
@@ -134,10 +130,10 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfoFromVTableAddress(
                   type_info.SetTypeSP(type_sp);
                 }
               }
-            } else if (num_matches > 1) {
+            } else {
               size_t i;
               if (log) {
-                for (i = 0; i < num_matches; i++) {
+                for (i = 0; i < class_types.GetSize(); i++) {
                   type_sp = class_types.GetTypeAtIndex(i);
                   if (type_sp) {
                     LLDB_LOGF(
@@ -151,7 +147,7 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfoFromVTableAddress(
                 }
               }
 
-              for (i = 0; i < num_matches; i++) {
+              for (i = 0; i < class_types.GetSize(); i++) {
                 type_sp = class_types.GetTypeAtIndex(i);
                 if (type_sp) {
                   if (ClangASTContext::IsCXXClassType(
@@ -168,7 +164,7 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfoFromVTableAddress(
                 }
               }
 
-              if (log && i == num_matches) {
+              if (log) {
                 LLDB_LOGF(log,
                           "0x%16.16" PRIx64
                           ": static-type = '%s' has multiple matching dynamic "
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
index 3800b81d9768a..1f27a4f0b3ed0 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCDeclVendor.cpp
@@ -657,8 +657,7 @@ AppleObjCDeclVendor::FindDecls(ConstString name, bool append,
 
 clang::ExternalASTMerger::ImporterSource
 AppleObjCDeclVendor::GetImporterSource() {
-        return {*m_ast_ctx.getASTContext(),
-                *m_ast_ctx.getFileManager(),
-                m_ast_ctx.GetOriginMap()
-        };
+  return clang::ExternalASTMerger::ImporterSource(*m_ast_ctx.getASTContext(),
+                                                  *m_ast_ctx.getFileManager(),
+                                                  m_ast_ctx.GetOriginMap());
 }
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
index 3be07937eeaa0..379ef3dca86c9 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/AppleObjCRuntime/AppleObjCTrampolineHandler.cpp
@@ -704,7 +704,7 @@ AppleObjCTrampolineHandler::AppleObjCTrampolineHandler(
     // step through any method dispatches.  Warn to that effect and get out of
     // here.
     if (process_sp->CanJIT()) {
-      process_sp->GetTarget().GetDebugger().GetErrorFile()->Printf(
+      process_sp->GetTarget().GetDebugger().GetErrorStream().Printf(
           "Could not find implementation lookup function \"%s\""
           " step in through ObjC method dispatch will not work.\n",
           get_impl_name.AsCString());
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index 1e39ff14a894f..31aac7284602c 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -121,20 +121,17 @@ ObjCLanguageRuntime::LookupInCompleteClassCache(ConstString &name) {
     TypeList types;
 
     llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    const uint32_t num_types = module_sp->FindTypes(
-        name, exact_match, max_matches, searched_symbol_files, types);
-
-    if (num_types) {
-      uint32_t i;
-      for (i = 0; i < num_types; ++i) {
-        TypeSP type_sp(types.GetTypeAtIndex(i));
-
-        if (ClangASTContext::IsObjCObjectOrInterfaceType(
-                type_sp->GetForwardCompilerType())) {
-          if (type_sp->IsCompleteObjCClass()) {
-            m_complete_class_cache[name] = type_sp;
-            return type_sp;
-          }
+    module_sp->FindTypes(name, exact_match, max_matches, searched_symbol_files,
+                         types);
+
+    for (uint32_t i = 0; i < types.GetSize(); ++i) {
+      TypeSP type_sp(types.GetTypeAtIndex(i));
+
+      if (ClangASTContext::IsObjCObjectOrInterfaceType(
+              type_sp->GetForwardCompilerType())) {
+        if (type_sp->IsCompleteObjCClass()) {
+          m_complete_class_cache[name] = type_sp;
+          return type_sp;
         }
       }
     }
diff --git a/lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptRuntime.cpp
index 38bd79d68cf90..ad1b3411c67e9 100644
--- a/lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/RenderScript/RenderScriptRuntime/RenderScriptRuntime.cpp
@@ -2651,14 +2651,14 @@ bool RenderScriptRuntime::SaveAllocation(Stream &strm, const uint32_t alloc_id,
   // Check we can create writable file
   FileSpec file_spec(path);
   FileSystem::Instance().Resolve(file_spec);
-  File file;
-  FileSystem::Instance().Open(file, file_spec,
-                              File::eOpenOptionWrite |
-                                  File::eOpenOptionCanCreate |
-                                  File::eOpenOptionTruncate);
+  auto file = FileSystem::Instance().Open(
+      file_spec, File::eOpenOptionWrite | File::eOpenOptionCanCreate |
+                     File::eOpenOptionTruncate);
 
   if (!file) {
-    strm.Printf("Error: Failed to open '%s' for writing", path);
+    std::string error = llvm::toString(file.takeError());
+    strm.Printf("Error: Failed to open '%s' for writing: %s", path,
+                error.c_str());
     strm.EOL();
     return false;
   }
@@ -2690,7 +2690,7 @@ bool RenderScriptRuntime::SaveAllocation(Stream &strm, const uint32_t alloc_id,
   LLDB_LOGF(log, "%s - writing File Header, 0x%" PRIx64 " bytes", __FUNCTION__,
             (uint64_t)num_bytes);
 
-  Status err = file.Write(&head, num_bytes);
+  Status err = file.get()->Write(&head, num_bytes);
   if (!err.Success()) {
     strm.Printf("Error: '%s' when writing to file '%s'", err.AsCString(), path);
     strm.EOL();
@@ -2715,7 +2715,7 @@ bool RenderScriptRuntime::SaveAllocation(Stream &strm, const uint32_t alloc_id,
   LLDB_LOGF(log, "%s - writing element headers, 0x%" PRIx64 " bytes.",
             __FUNCTION__, (uint64_t)num_bytes);
 
-  err = file.Write(element_header_buffer.get(), num_bytes);
+  err = file.get()->Write(element_header_buffer.get(), num_bytes);
   if (!err.Success()) {
     strm.Printf("Error: '%s' when writing to file '%s'", err.AsCString(), path);
     strm.EOL();
@@ -2727,7 +2727,7 @@ bool RenderScriptRuntime::SaveAllocation(Stream &strm, const uint32_t alloc_id,
   LLDB_LOGF(log, "%s - writing 0x%" PRIx64 " bytes", __FUNCTION__,
             (uint64_t)num_bytes);
 
-  err = file.Write(buffer.get(), num_bytes);
+  err = file.get()->Write(buffer.get(), num_bytes);
   if (!err.Success()) {
     strm.Printf("Error: '%s' when writing to file '%s'", err.AsCString(), path);
     strm.EOL();
@@ -4578,32 +4578,36 @@ class CommandObjectRenderScriptRuntimeAllocationDump
       return false;
     }
 
-    Stream *output_strm = nullptr;
-    StreamFile outfile_stream;
+    Stream *output_stream_p = nullptr;
+    std::unique_ptr<Stream> output_stream_storage;
+
     const FileSpec &outfile_spec =
         m_options.m_outfile; // Dump allocation to file instead
     if (outfile_spec) {
       // Open output file
       std::string path = outfile_spec.GetPath();
-      auto error = FileSystem::Instance().Open(
-          outfile_stream.GetFile(), outfile_spec,
-          File::eOpenOptionWrite | File::eOpenOptionCanCreate);
-      if (error.Success()) {
-        output_strm = &outfile_stream;
+      auto file = FileSystem::Instance().Open(
+          outfile_spec, File::eOpenOptionWrite | File::eOpenOptionCanCreate);
+      if (file) {
+        output_stream_storage =
+            std::make_unique<StreamFile>(std::move(file.get()));
+        output_stream_p = output_stream_storage.get();
         result.GetOutputStream().Printf("Results written to '%s'",
                                         path.c_str());
         result.GetOutputStream().EOL();
       } else {
-        result.AppendErrorWithFormat("Couldn't open file '%s'", path.c_str());
+        std::string error = llvm::toString(file.takeError());
+        result.AppendErrorWithFormat("Couldn't open file '%s': %s",
+                                     path.c_str(), error.c_str());
         result.SetStatus(eReturnStatusFailed);
         return false;
       }
     } else
-      output_strm = &result.GetOutputStream();
+      output_stream_p = &result.GetOutputStream();
 
-    assert(output_strm != nullptr);
+    assert(output_stream_p != nullptr);
     bool dumped =
-        runtime->DumpAllocation(*output_strm, m_exe_ctx.GetFramePtr(), id);
+        runtime->DumpAllocation(*output_stream_p, m_exe_ctx.GetFramePtr(), id);
 
     if (dumped)
       result.SetStatus(eReturnStatusSuccessFinishResult);
diff --git a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
index 577d80f03cdcb..bc46b97eeccd2 100644
--- a/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
+++ b/lldb/source/Plugins/ObjectFile/Mach-O/ObjectFileMachO.cpp
@@ -2554,8 +2554,8 @@ size_t ObjectFileMachO::ParseSymtab() {
           "dyld_shared_cache_", /* DYLD_SHARED_CACHE_BASE_NAME */
           header_arch.GetArchitectureName(), ".development");
 
-      FileSpec dsc_nondevelopment_filespec(dsc_path, false);
-      FileSpec dsc_development_filespec(dsc_path_development, false);
+      FileSpec dsc_nondevelopment_filespec(dsc_path);
+      FileSpec dsc_development_filespec(dsc_path_development);
       FileSpec dsc_filespec;
 
       UUID dsc_uuid;
@@ -3001,11 +3001,11 @@ size_t ObjectFileMachO::ParseSymtab() {
                                   // second is the directory for the source
                                   // file so you end up with a path that looks
                                   // like "/tmp/src//tmp/src/"
-                                  FileSpec so_dir(so_path, false);
+                                  FileSpec so_dir(so_path);
                                   if (!FileSystem::Instance().Exists(so_dir)) {
                                     so_dir.SetFile(
                                         &full_so_path[double_slash_pos + 1],
-                                        false);
+                                        FileSpec::Style::native);
                                     if (FileSystem::Instance().Exists(so_dir)) {
                                       // Trim off the incorrect path
                                       full_so_path.erase(0,
@@ -6271,22 +6271,23 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
             buffer.PutHex32(segment.flags);
           }
 
-          File core_file;
           std::string core_file_path(outfile.GetPath());
-          error = FileSystem::Instance().Open(core_file, outfile,
-                                              File::eOpenOptionWrite |
-                                                  File::eOpenOptionTruncate |
-                                                  File::eOpenOptionCanCreate);
-          if (error.Success()) {
+          auto core_file = FileSystem::Instance().Open(
+              outfile, File::eOpenOptionWrite | File::eOpenOptionTruncate |
+                           File::eOpenOptionCanCreate);
+          if (!core_file) {
+            error = core_file.takeError();
+          } else {
             // Read 1 page at a time
             uint8_t bytes[0x1000];
             // Write the mach header and load commands out to the core file
             size_t bytes_written = buffer.GetString().size();
-            error = core_file.Write(buffer.GetString().data(), bytes_written);
+            error = core_file.get()->Write(buffer.GetString().data(),
+                                           bytes_written);
             if (error.Success()) {
               // Now write the file data for all memory segments in the process
               for (const auto &segment : segment_load_commands) {
-                if (core_file.SeekFromStart(segment.fileoff) == -1) {
+                if (core_file.get()->SeekFromStart(segment.fileoff) == -1) {
                   error.SetErrorStringWithFormat(
                       "unable to seek to offset 0x%" PRIx64 " in '%s'",
                       segment.fileoff, core_file_path.c_str());
@@ -6311,7 +6312,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
 
                   if (bytes_read == bytes_to_read) {
                     size_t bytes_written = bytes_read;
-                    error = core_file.Write(bytes, bytes_written);
+                    error = core_file.get()->Write(bytes, bytes_written);
                     bytes_left -= bytes_read;
                     addr += bytes_read;
                   } else {
@@ -6319,7 +6320,7 @@ bool ObjectFileMachO::SaveCore(const lldb::ProcessSP &process_sp,
                     // be zero filled
                     memset(bytes, 0, bytes_to_read);
                     size_t bytes_written = bytes_to_read;
-                    error = core_file.Write(bytes, bytes_written);
+                    error = core_file.get()->Write(bytes, bytes_written);
                     bytes_left -= bytes_to_read;
                     addr += bytes_to_read;
                   }
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt b/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
index 04321f2765510..6981f7ecb71a8 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/CMakeLists.txt
@@ -1,3 +1,10 @@
+# Dbghelp is used on windows for writing minidump files.
+if(WIN32)
+  set(DBGHELP_LINK_FILES dbghelp)
+else()
+  set(DBGHELP_LINK_FILES "")
+endif()
+
 add_lldb_library(lldbPluginObjectFilePECOFF PLUGIN
   ObjectFilePECOFF.cpp
   WindowsMiniDump.cpp
@@ -7,6 +14,7 @@ add_lldb_library(lldbPluginObjectFilePECOFF PLUGIN
     lldbHost
     lldbSymbol
     lldbTarget
+    ${DBGHELP_LINK_FILES}
   LINK_COMPONENTS
     BinaryFormat
     Support
diff --git a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
index 0328931959da0..16c131fa469a0 100644
--- a/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
+++ b/lldb/source/Plugins/ObjectFile/PECOFF/ObjectFilePECOFF.cpp
@@ -195,7 +195,11 @@ size_t ObjectFilePECOFF::GetModuleSpecifications(
     specs.Append(module_spec);
     break;
   case MachineArmNt:
-    spec.SetTriple("arm-pc-windows");
+    spec.SetTriple("armv7-pc-windows");
+    specs.Append(module_spec);
+    break;
+  case MachineArm64:
+    spec.SetTriple("aarch64-unknown-windows");
     specs.Append(module_spec);
     break;
   default:
@@ -1200,6 +1204,7 @@ ArchSpec ObjectFilePECOFF::GetArchitecture() {
   case llvm::COFF::IMAGE_FILE_MACHINE_ARM:
   case llvm::COFF::IMAGE_FILE_MACHINE_ARMNT:
   case llvm::COFF::IMAGE_FILE_MACHINE_THUMB:
+  case llvm::COFF::IMAGE_FILE_MACHINE_ARM64:
     ArchSpec arch;
     arch.SetArchitecture(eArchTypeCOFF, machine, LLDB_INVALID_CPUTYPE,
                          IsWindowsSubsystem() ? llvm::Triple::Win32
diff --git a/lldb/source/Plugins/Platform/Linux/PlatformLinux.cpp b/lldb/source/Plugins/Platform/Linux/PlatformLinux.cpp
index 52d72f5e6fdd7..f17fb8f048334 100644
--- a/lldb/source/Plugins/Platform/Linux/PlatformLinux.cpp
+++ b/lldb/source/Plugins/Platform/Linux/PlatformLinux.cpp
@@ -260,25 +260,6 @@ bool PlatformLinux::CanDebugProcess() {
   }
 }
 
-std::vector<std::string>
-PlatformLinux::GetSystemIncludeDirectories(lldb::LanguageType lang) {
-  std::string sys_root = GetSDKRootDirectory().AsCString("");
-  switch (lang) {
-  case lldb::eLanguageTypeC:
-  case lldb::eLanguageTypeC89:
-  case lldb::eLanguageTypeC99:
-  case lldb::eLanguageTypeC11:
-  case lldb::eLanguageTypeC_plus_plus:
-  case lldb::eLanguageTypeC_plus_plus_03:
-  case lldb::eLanguageTypeC_plus_plus_11:
-  case lldb::eLanguageTypeC_plus_plus_14:
-  case lldb::eLanguageTypeObjC_plus_plus:
-    return {sys_root + "/usr/include/"};
-  default:
-    return {};
-  }
-}
-
 // For local debugging, Linux will override the debug logic to use llgs-launch
 // rather than lldb-launch, llgs-attach.  This differs from current lldb-
 // launch, debugserver-attach approach on MacOSX.
diff --git a/lldb/source/Plugins/Platform/Linux/PlatformLinux.h b/lldb/source/Plugins/Platform/Linux/PlatformLinux.h
index a843558f1df81..67d211ac82433 100644
--- a/lldb/source/Plugins/Platform/Linux/PlatformLinux.h
+++ b/lldb/source/Plugins/Platform/Linux/PlatformLinux.h
@@ -48,9 +48,6 @@ class PlatformLinux : public PlatformPOSIX {
 
   bool CanDebugProcess() override;
 
-  std::vector<std::string>
-  GetSystemIncludeDirectories(lldb::LanguageType lang) override;
-
   lldb::ProcessSP DebugProcess(ProcessLaunchInfo &launch_info,
                                Debugger &debugger, Target *target,
                                Status &error) override;
diff --git a/lldb/source/Plugins/Platform/MacOSX/objcxx/PlatformiOSSimulatorCoreSimulatorSupport.mm b/lldb/source/Plugins/Platform/MacOSX/objcxx/PlatformiOSSimulatorCoreSimulatorSupport.mm
index ed18a1cb7569f..1d259554fbcde 100644
--- a/lldb/source/Plugins/Platform/MacOSX/objcxx/PlatformiOSSimulatorCoreSimulatorSupport.mm
+++ b/lldb/source/Plugins/Platform/MacOSX/objcxx/PlatformiOSSimulatorCoreSimulatorSupport.mm
@@ -382,7 +382,7 @@ should outlive the calling process (default false) */
 
 static Status HandleFileAction(ProcessLaunchInfo &launch_info,
                                NSMutableDictionary *options, NSString *key,
-                               const int fd, File &file) {
+                               const int fd, lldb::FileSP &file) {
   Status error;
   const FileAction *file_action = launch_info.GetFileActionForFD(fd);
   if (file_action) {
@@ -425,11 +425,16 @@ static Status HandleFileAction(ProcessLaunchInfo &launch_info,
           }
         }
         Status posix_error;
+        int oflag = file_action->GetActionArgument();
         int created_fd =
-            open(file_spec.GetPath().c_str(), file_action->GetActionArgument(),
-                 S_IRUSR | S_IWUSR);
+            open(file_spec.GetPath().c_str(), oflag, S_IRUSR | S_IWUSR);
         if (created_fd >= 0) {
-          file.SetDescriptor(created_fd, true);
+          uint32_t file_options = 0;
+          if ((oflag & O_RDWR) || (oflag & O_RDONLY))
+            file_options |= File::eOpenOptionRead;
+          if ((oflag & O_RDWR) || (oflag & O_RDONLY))
+            file_options |= File::eOpenOptionWrite;
+          file = std::make_shared<File>(created_fd, file_options, true);
           [options setValue:[NSNumber numberWithInteger:created_fd] forKey:key];
           return error; // Success
         } else {
@@ -489,9 +494,9 @@ static Status HandleFileAction(ProcessLaunchInfo &launch_info,
   [options setObject:env_dict forKey:kSimDeviceSpawnEnvironment];
 
   Status error;
-  File stdin_file;
-  File stdout_file;
-  File stderr_file;
+  lldb::FileSP stdin_file;
+  lldb::FileSP stdout_file;
+  lldb::FileSP stderr_file;
   error = HandleFileAction(launch_info, options, kSimDeviceSpawnStdin,
                            STDIN_FILENO, stdin_file);
 
diff --git a/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp b/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
index 6186e1fb83f2a..32e3320150f83 100644
--- a/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
+++ b/lldb/source/Plugins/Process/FreeBSD/ProcessFreeBSD.cpp
@@ -977,7 +977,7 @@ Status ProcessFreeBSD::SetSoftwareSingleStepBreakpoint(lldb::tid_t tid,
   Breakpoint *const sw_step_break =
       m_process->GetTarget().CreateBreakpoint(addr, true, false).get();
   sw_step_break->SetCallback(SingleStepBreakpointHit, this, true);
-  sw_step_break->SetBreakpointKind("software-signle-step");
+  sw_step_break->SetBreakpointKind("software-single-step");
 
   LLDB_LOGF(log, "ProcessFreeBSD::%s addr = 0x%" PRIx64 " -- SUCCESS",
             __FUNCTION__, addr);
diff --git a/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp b/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp
index 87feca3c4337a..8b6f9fbc33c34 100644
--- a/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp
+++ b/lldb/source/Plugins/Process/NetBSD/NativeProcessNetBSD.cpp
@@ -340,12 +340,14 @@ Status NativeProcessNetBSD::Resume(const ResumeActionList &resume_actions) {
   }
 
   Status error;
+  int signal =
+      action->signal != LLDB_INVALID_SIGNAL_NUMBER ? action->signal : 0;
 
   switch (action->state) {
   case eStateRunning: {
     // Run the thread, possibly feeding it the signal.
     error = NativeProcessNetBSD::PtraceWrapper(PT_CONTINUE, GetID(), (void *)1,
-                                               action->signal);
+                                               signal);
     if (!error.Success())
       return error;
     for (const auto &thread : m_threads)
@@ -356,7 +358,7 @@ Status NativeProcessNetBSD::Resume(const ResumeActionList &resume_actions) {
   case eStateStepping:
     // Run the thread, possibly feeding it the signal.
     error = NativeProcessNetBSD::PtraceWrapper(PT_STEP, GetID(), (void *)1,
-                                               action->signal);
+                                               signal);
     if (!error.Success())
       return error;
     for (const auto &thread : m_threads)
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm64.cpp
index 66e3b75630752..b3ec24d8905de 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextDarwin_arm64.cpp
@@ -25,6 +25,11 @@
 
 #include <memory>
 
+#if defined(__APPLE__) && (defined(__arm64__) || defined(__aarch64__))
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#endif
+
 // Support building against older versions of LLVM, this macro was added
 // recently.
 #ifndef LLVM_EXTENSION
@@ -503,7 +508,7 @@ bool RegisterContextDarwin_arm64::ReadRegister(const RegisterInfo *reg_info,
   case fpu_d31: {
     ProcessSP process_sp(m_thread.GetProcess());
     if (process_sp.get()) {
-      DataExtractor regdata(&fpu.v[reg - fpu_s0], 8, process_sp->GetByteOrder(),
+      DataExtractor regdata(&fpu.v[reg - fpu_d0], 8, process_sp->GetByteOrder(),
                             process_sp->GetAddressByteSize());
       offset_t offset = 0;
       value.SetDouble(regdata.GetDouble(&offset));
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.cpp b/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.cpp
index 0da687ad0b910..50795c09f2808 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.cpp
@@ -18,6 +18,7 @@
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Symbol/Symbol.h"
 #include "lldb/Symbol/SymbolContext.h"
+#include "lldb/Symbol/SymbolFile.h"
 #include "lldb/Target/ABI.h"
 #include "lldb/Target/DynamicLoader.h"
 #include "lldb/Target/ExecutionContext.h"
@@ -1852,12 +1853,66 @@ bool RegisterContextLLDB::ReadFrameAddress(
                  error.AsCString());
     break;
   }
+  case UnwindPlan::Row::FAValue::isRaSearch: {
+    Process &process = *m_thread.GetProcess();
+    lldb::addr_t return_address_hint = GetReturnAddressHint(fa.GetOffset());
+    if (return_address_hint == LLDB_INVALID_ADDRESS)
+      return false;
+    const unsigned max_iterations = 256;
+    for (unsigned i = 0; i < max_iterations; ++i) {
+      Status st;
+      lldb::addr_t candidate_addr =
+          return_address_hint + i * process.GetAddressByteSize();
+      lldb::addr_t candidate =
+          process.ReadPointerFromMemory(candidate_addr, st);
+      if (st.Fail()) {
+        UnwindLogMsg("Cannot read memory at 0x%" PRIx64 ": %s", candidate_addr,
+                     st.AsCString());
+        return false;
+      }
+      Address addr;
+      uint32_t permissions;
+      if (process.GetLoadAddressPermissions(candidate, permissions) &&
+          permissions & lldb::ePermissionsExecutable) {
+        address = candidate_addr;
+        UnwindLogMsg("Heuristically found CFA: 0x%" PRIx64, address);
+        return true;
+      }
+    }
+    UnwindLogMsg("No suitable CFA found");
+    break;
+  }
   default:
     return false;
   }
   return false;
 }
 
+lldb::addr_t RegisterContextLLDB::GetReturnAddressHint(int32_t plan_offset) {
+  addr_t hint;
+  if (!ReadGPRValue(eRegisterKindGeneric, LLDB_REGNUM_GENERIC_SP, hint))
+    return LLDB_INVALID_ADDRESS;
+  if (!m_sym_ctx.module_sp || !m_sym_ctx.symbol)
+    return LLDB_INVALID_ADDRESS;
+
+  hint += plan_offset;
+
+  if (auto next = GetNextFrame()) {
+    if (!next->m_sym_ctx.module_sp || !next->m_sym_ctx.symbol)
+      return LLDB_INVALID_ADDRESS;
+    if (auto expected_size =
+            next->m_sym_ctx.module_sp->GetSymbolFile()->GetParameterStackSize(
+                *next->m_sym_ctx.symbol))
+      hint += *expected_size;
+    else {
+      UnwindLogMsgVerbose("Could not retrieve parameter size: %s",
+                          llvm::toString(expected_size.takeError()).c_str());
+      return LLDB_INVALID_ADDRESS;
+    }
+  }
+  return hint;
+}
+
 // Retrieve a general purpose register value for THIS frame, as saved by the
 // NEXT frame, i.e. the frame that
 // this frame called.  e.g.
diff --git a/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.h b/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.h
index 1ca663906ed35..114ac35591e7e 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterContextLLDB.h
@@ -201,6 +201,8 @@ class RegisterContextLLDB : public lldb_private::RegisterContext {
   bool IsUnwindPlanValidForCurrentPC(lldb::UnwindPlanSP unwind_plan_sp,
                                      int &valid_pc_offset);
 
+  lldb::addr_t GetReturnAddressHint(int32_t plan_offset);
+
   lldb_private::Thread &m_thread;
 
   ///
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
index 71ad57c45c779..f888d152caa38 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeProcessWindows.cpp
@@ -177,9 +177,9 @@ Status NativeProcessWindows::Detach() {
     else
       LLDB_LOG(log, "Detaching process error: {0}", error);
   } else {
-    error.SetErrorStringWithFormat("error: process {0} in state = {1}, but "
-                                   "cannot detach it in this state.",
-                                   GetID(), state);
+    error.SetErrorStringWithFormatv("error: process {0} in state = {1}, but "
+                                    "cannot detach it in this state.",
+                                    GetID(), state);
     LLDB_LOG(log, "error: {0}", error);
   }
   return error;
@@ -431,13 +431,34 @@ NativeProcessWindows::OnDebugException(bool first_chance,
   ExceptionResult result = ExceptionResult::SendToApplication;
   switch (record.GetExceptionCode()) {
   case DWORD(STATUS_SINGLE_STEP):
-  case STATUS_WX86_SINGLE_STEP:
-    StopThread(record.GetThreadID(), StopReason::eStopReasonTrace);
+  case STATUS_WX86_SINGLE_STEP: {
+    uint32_t wp_id = LLDB_INVALID_INDEX32;
+    if (NativeThreadWindows *thread = GetThreadByID(record.GetThreadID())) {
+      NativeRegisterContextWindows &reg_ctx = thread->GetRegisterContext();
+      Status error =
+          reg_ctx.GetWatchpointHitIndex(wp_id, record.GetExceptionAddress());
+      if (error.Fail())
+        LLDB_LOG(log,
+                 "received error while checking for watchpoint hits, pid = "
+                 "{0}, error = {1}",
+                 thread->GetID(), error);
+      if (wp_id != LLDB_INVALID_INDEX32) {
+        addr_t wp_addr = reg_ctx.GetWatchpointAddress(wp_id);
+        addr_t wp_hit_addr = reg_ctx.GetWatchpointHitAddress(wp_id);
+        std::string desc =
+            formatv("{0} {1} {2}", wp_addr, wp_id, wp_hit_addr).str();
+        StopThread(record.GetThreadID(), StopReason::eStopReasonWatchpoint,
+                   desc);
+      }
+    }
+    if (wp_id == LLDB_INVALID_INDEX32)
+      StopThread(record.GetThreadID(), StopReason::eStopReasonTrace);
+
     SetState(eStateStopped, true);
 
     // Continue the debugger.
     return ExceptionResult::MaskException;
-
+  }
   case DWORD(STATUS_BREAKPOINT):
   case STATUS_WX86_BREAKPOINT:
     if (FindSoftwareBreakpoint(record.GetExceptionAddress())) {
@@ -479,7 +500,7 @@ NativeProcessWindows::OnDebugException(bool first_chance,
       return ExceptionResult::BreakInDebugger;
     }
 
-    // Fall through
+    LLVM_FALLTHROUGH;
   default:
     LLDB_LOG(log,
              "Debugger thread reported exception {0:x} at address {1:x} "
@@ -513,8 +534,16 @@ NativeProcessWindows::OnDebugException(bool first_chance,
 
 void NativeProcessWindows::OnCreateThread(const HostThread &new_thread) {
   llvm::sys::ScopedLock lock(m_mutex);
-  m_threads.push_back(
-      std::make_unique<NativeThreadWindows>(*this, new_thread));
+
+  auto thread = std::make_unique<NativeThreadWindows>(*this, new_thread);
+  thread->GetRegisterContext().ClearAllHardwareWatchpoints();
+  for (const auto &pair : GetWatchpointMap()) {
+    const NativeWatchpoint &wp = pair.second;
+    thread->SetWatchpoint(wp.m_addr, wp.m_size, wp.m_watch_flags,
+                          wp.m_hardware);
+  }
+
+  m_threads.push_back(std::move(thread));
 }
 
 void NativeProcessWindows::OnExitThread(lldb::tid_t thread_id,
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
index ca3badd5cc04c..841b8547f3e90 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows.h
@@ -19,7 +19,7 @@ class NativeThreadWindows;
 
 class NativeRegisterContextWindows : public NativeRegisterContextRegisterInfo {
 public:
-  NativeRegisterContextWindows::NativeRegisterContextWindows(
+  NativeRegisterContextWindows(
       NativeThreadProtocol &native_thread,
       RegisterInfoInterface *reg_info_interface_p);
 
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
index 8128d5550c4b5..a5a86f5e12ca4 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.cpp
@@ -56,12 +56,14 @@ CreateRegisterInfoInterface(const ArchSpec &target_arch) {
   return new RegisterContextWindows_i386(target_arch);
 }
 
-static Status GetWoW64ThreadContextHelper(lldb::thread_t thread_handle,
-                                          PWOW64_CONTEXT context_ptr) {
+static Status
+GetWoW64ThreadContextHelper(lldb::thread_t thread_handle,
+                            PWOW64_CONTEXT context_ptr,
+                            const DWORD control_flag = kWoW64ContextFlags) {
   Log *log = ProcessWindowsLog::GetLogIfAny(WINDOWS_LOG_REGISTERS);
   Status error;
   memset(context_ptr, 0, sizeof(::WOW64_CONTEXT));
-  context_ptr->ContextFlags = kWoW64ContextFlags;
+  context_ptr->ContextFlags = control_flag;
   if (!::Wow64GetThreadContext(thread_handle, context_ptr)) {
     error.SetError(GetLastError(), eErrorTypeWin32);
     LLDB_LOG(log, "{0} Wow64GetThreadContext failed with error {1}",
@@ -93,6 +95,10 @@ bool NativeRegisterContextWindows_WoW64::IsGPR(uint32_t reg_index) const {
   return (reg_index >= k_first_gpr_i386 && reg_index < k_first_alias_i386);
 }
 
+bool NativeRegisterContextWindows_WoW64::IsDR(uint32_t reg_index) const {
+  return (reg_index >= lldb_dr0_i386 && reg_index <= lldb_dr7_i386);
+}
+
 uint32_t NativeRegisterContextWindows_WoW64::GetRegisterSetCount() const {
   return k_num_register_sets;
 }
@@ -228,6 +234,82 @@ NativeRegisterContextWindows_WoW64::GPRWrite(const uint32_t reg,
   return SetWoW64ThreadContextHelper(thread_handle, &tls_context);
 }
 
+Status NativeRegisterContextWindows_WoW64::DRRead(const uint32_t reg,
+                                                  RegisterValue &reg_value) {
+  ::WOW64_CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  Status error = GetWoW64ThreadContextHelper(GetThreadHandle(), &tls_context,
+                                             context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_i386:
+    reg_value.SetUInt32(tls_context.Dr0);
+    break;
+  case lldb_dr1_i386:
+    reg_value.SetUInt32(tls_context.Dr1);
+    break;
+  case lldb_dr2_i386:
+    reg_value.SetUInt32(tls_context.Dr2);
+    break;
+  case lldb_dr3_i386:
+    reg_value.SetUInt32(tls_context.Dr3);
+    break;
+  case lldb_dr4_i386:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_i386:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_i386:
+    reg_value.SetUInt32(tls_context.Dr6);
+    break;
+  case lldb_dr7_i386:
+    reg_value.SetUInt32(tls_context.Dr7);
+    break;
+  }
+
+  return {};
+}
+
+Status
+NativeRegisterContextWindows_WoW64::DRWrite(const uint32_t reg,
+                                            const RegisterValue &reg_value) {
+  ::WOW64_CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  auto thread_handle = GetThreadHandle();
+  Status error =
+      GetWoW64ThreadContextHelper(thread_handle, &tls_context, context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_i386:
+    tls_context.Dr0 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr1_i386:
+    tls_context.Dr1 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr2_i386:
+    tls_context.Dr2 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr3_i386:
+    tls_context.Dr3 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr4_i386:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_i386:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_i386:
+    tls_context.Dr6 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr7_i386:
+    tls_context.Dr7 = reg_value.GetAsUInt32();
+    break;
+  }
+
+  return SetWoW64ThreadContextHelper(thread_handle, &tls_context);
+}
+
 Status
 NativeRegisterContextWindows_WoW64::ReadRegister(const RegisterInfo *reg_info,
                                                  RegisterValue &reg_value) {
@@ -250,6 +332,9 @@ NativeRegisterContextWindows_WoW64::ReadRegister(const RegisterInfo *reg_info,
   if (IsGPR(reg))
     return GPRRead(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRRead(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -275,6 +360,9 @@ Status NativeRegisterContextWindows_WoW64::WriteRegister(
   if (IsGPR(reg))
     return GPRWrite(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRWrite(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -317,46 +405,198 @@ Status NativeRegisterContextWindows_WoW64::WriteAllRegisterValues(
 
 Status NativeRegisterContextWindows_WoW64::IsWatchpointHit(uint32_t wp_index,
                                                            bool &is_hit) {
-  return Status("unimplemented");
+  is_hit = false;
+
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_hit = reg_value.GetAsUInt32() & (1 << wp_index);
+
+  return {};
 }
 
 Status NativeRegisterContextWindows_WoW64::GetWatchpointHitIndex(
     uint32_t &wp_index, lldb::addr_t trap_addr) {
-  return Status("unimplemented");
+  wp_index = LLDB_INVALID_INDEX32;
+
+  for (uint32_t i = 0; i < NumSupportedHardwareWatchpoints(); i++) {
+    bool is_hit;
+    Status error = IsWatchpointHit(i, is_hit);
+    if (error.Fail())
+      return error;
+
+    if (is_hit) {
+      wp_index = i;
+      return {};
+    }
+  }
+
+  return {};
 }
 
 Status NativeRegisterContextWindows_WoW64::IsWatchpointVacant(uint32_t wp_index,
                                                               bool &is_vacant) {
-  return Status("unimplemented");
-}
+  is_vacant = false;
 
-Status NativeRegisterContextWindows_WoW64::SetHardwareWatchpointWithIndex(
-    lldb::addr_t addr, size_t size, uint32_t watch_flags, uint32_t wp_index) {
-  return Status("unimplemented");
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("Watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_vacant = !(reg_value.GetAsUInt32() & (1 << (2 * wp_index)));
+
+  return error;
 }
 
 bool NativeRegisterContextWindows_WoW64::ClearHardwareWatchpoint(
     uint32_t wp_index) {
-  return false;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits 0, 1, 2, or 3 of
+  // the debug status register (DR6)
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return false;
+
+  uint32_t bit_mask = 1 << wp_index;
+  uint32_t status_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  error = DRWrite(lldb_dr6_i386, RegisterValue(status_bits));
+  if (error.Fail())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits {0-1,16-19},
+  // {2-3,20-23}, {4-5,24-27}, or {6-7,28-31} of the debug control register
+  // (DR7)
+
+  error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return false;
+
+  bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  return DRWrite(lldb_dr7_i386, RegisterValue(control_bits)).Success();
 }
 
 Status NativeRegisterContextWindows_WoW64::ClearAllHardwareWatchpoints() {
-  return Status("unimplemented");
+  RegisterValue reg_value;
+
+  // clear bits {0-4} of the debug status register (DR6)
+
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint32_t status_bits = reg_value.GetAsUInt32() & ~0xF;
+  error = DRWrite(lldb_dr6_i386, RegisterValue(status_bits));
+  if (error.Fail())
+    return error;
+
+  // clear bits {0-7,16-31} of the debug control register (DR7)
+
+  error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~0xFFFF00FF;
+  return DRWrite(lldb_dr7_i386, RegisterValue(control_bits));
 }
 
 uint32_t NativeRegisterContextWindows_WoW64::SetHardwareWatchpoint(
     lldb::addr_t addr, size_t size, uint32_t watch_flags) {
+  switch (size) {
+  case 1:
+  case 2:
+  case 4:
+    break;
+  default:
+    return LLDB_INVALID_INDEX32;
+  }
+
+  if (watch_flags == 0x2)
+    watch_flags = 0x3;
+
+  if (watch_flags != 0x1 && watch_flags != 0x3)
+    return LLDB_INVALID_INDEX32;
+
+  for (uint32_t wp_index = 0; wp_index < NumSupportedHardwareWatchpoints();
+       ++wp_index) {
+    bool is_vacant;
+    if (IsWatchpointVacant(wp_index, is_vacant).Fail())
+      return LLDB_INVALID_INDEX32;
+
+    if (is_vacant) {
+      if (!ClearHardwareWatchpoint(wp_index))
+        return LLDB_INVALID_INDEX32;
+
+      if (ApplyHardwareBreakpoint(wp_index, addr, size, watch_flags).Fail())
+        return LLDB_INVALID_INDEX32;
+
+      return wp_index;
+    }
+  }
   return LLDB_INVALID_INDEX32;
 }
 
+Status NativeRegisterContextWindows_WoW64::ApplyHardwareBreakpoint(
+    uint32_t wp_index, lldb::addr_t addr, size_t size, uint32_t flags) {
+  RegisterValue reg_value;
+  auto error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, set bits 1, 3, 5, or 7
+  uint32_t enable_bit = 1 << (2 * wp_index);
+
+  // set bits 16-17, 20-21, 24-25, or 28-29
+  // with 0b01 for write, and 0b11 for read/write
+  uint32_t rw_bits = flags << (16 + 4 * wp_index);
+
+  // set bits 18-19, 22-23, 26-27, or 30-31
+  // with 0b00, 0b01, 0b10, or 0b11
+  // for 1, 2, 8 (if supported), or 4 bytes, respectively
+  uint32_t size_bits = (size == 8 ? 0x2 : size - 1) << (18 + 4 * wp_index);
+
+  uint32_t bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  control_bits |= enable_bit | rw_bits | size_bits;
+
+  error = DRWrite(lldb_dr7_i386, RegisterValue(control_bits));
+  if (error.Fail())
+    return error;
+
+  error = DRWrite(lldb_dr0_i386 + wp_index, RegisterValue(addr));
+  if (error.Fail())
+    return error;
+
+  return {};
+}
+
 lldb::addr_t
 NativeRegisterContextWindows_WoW64::GetWatchpointAddress(uint32_t wp_index) {
-  return LLDB_INVALID_ADDRESS;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return LLDB_INVALID_ADDRESS;
+
+  RegisterValue reg_value;
+  if (DRRead(lldb_dr0_i386 + wp_index, reg_value).Fail())
+    return LLDB_INVALID_ADDRESS;
+
+  return reg_value.GetAsUInt32();
 }
 
 uint32_t NativeRegisterContextWindows_WoW64::NumSupportedHardwareWatchpoints() {
-  // Not implemented
-  return 0;
+  return 4;
 }
 
 #endif // defined(__x86_64__) || defined(_M_X64)
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.h
index 473a9dd5d6468..b1bf725e9639a 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_WoW64.h
@@ -49,10 +49,6 @@ class NativeRegisterContextWindows_WoW64 : public NativeRegisterContextWindows {
 
   Status ClearAllHardwareWatchpoints() override;
 
-  Status SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size,
-                                        uint32_t watch_flags,
-                                        uint32_t wp_index);
-
   uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size,
                                  uint32_t watch_flags) override;
 
@@ -64,8 +60,15 @@ class NativeRegisterContextWindows_WoW64 : public NativeRegisterContextWindows {
   Status GPRRead(const uint32_t reg, RegisterValue &reg_value);
   Status GPRWrite(const uint32_t reg, const RegisterValue &reg_value);
 
+  Status DRRead(const uint32_t reg, RegisterValue &reg_value);
+  Status DRWrite(const uint32_t reg, const RegisterValue &reg_value);
+
 private:
+  Status ApplyHardwareBreakpoint(uint32_t wp_index, lldb::addr_t addr,
+                                 size_t size, uint32_t flags);
+
   bool IsGPR(uint32_t reg_index) const;
+  bool IsDR(uint32_t reg_index) const;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
index eb7b4b86f05c9..3ad721b60a189 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.cpp
@@ -87,7 +87,7 @@ std::unique_ptr<NativeRegisterContextWindows>
 NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
     const ArchSpec &target_arch, NativeThreadProtocol &native_thread) {
   return std::make_unique<NativeRegisterContextWindows_i386>(target_arch,
-                                                              native_thread);
+                                                             native_thread);
 }
 
 NativeRegisterContextWindows_i386::NativeRegisterContextWindows_i386(
@@ -99,6 +99,10 @@ bool NativeRegisterContextWindows_i386::IsGPR(uint32_t reg_index) const {
   return (reg_index < k_first_alias_i386);
 }
 
+bool NativeRegisterContextWindows_i386::IsDR(uint32_t reg_index) const {
+  return (reg_index >= lldb_dr0_i386 && reg_index <= lldb_dr7_i386);
+}
+
 uint32_t NativeRegisterContextWindows_i386::GetRegisterSetCount() const {
   return k_num_register_sets;
 }
@@ -238,6 +242,82 @@ NativeRegisterContextWindows_i386::GPRWrite(const uint32_t reg,
   return SetThreadContextHelper(thread_handle, &tls_context);
 }
 
+Status NativeRegisterContextWindows_i386::DRRead(const uint32_t reg,
+                                                 RegisterValue &reg_value) {
+  ::CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  Status error =
+      GetThreadContextHelper(GetThreadHandle(), &tls_context, context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_i386:
+    reg_value.SetUInt32(tls_context.Dr0);
+    break;
+  case lldb_dr1_i386:
+    reg_value.SetUInt32(tls_context.Dr1);
+    break;
+  case lldb_dr2_i386:
+    reg_value.SetUInt32(tls_context.Dr2);
+    break;
+  case lldb_dr3_i386:
+    reg_value.SetUInt32(tls_context.Dr3);
+    break;
+  case lldb_dr4_i386:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_i386:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_i386:
+    reg_value.SetUInt32(tls_context.Dr6);
+    break;
+  case lldb_dr7_i386:
+    reg_value.SetUInt32(tls_context.Dr7);
+    break;
+  }
+
+  return {};
+}
+
+Status
+NativeRegisterContextWindows_i386::DRWrite(const uint32_t reg,
+                                           const RegisterValue &reg_value) {
+  ::CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  auto thread_handle = GetThreadHandle();
+  Status error =
+      GetThreadContextHelper(thread_handle, &tls_context, context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_i386:
+    tls_context.Dr0 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr1_i386:
+    tls_context.Dr1 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr2_i386:
+    tls_context.Dr2 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr3_i386:
+    tls_context.Dr3 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr4_i386:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_i386:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_i386:
+    tls_context.Dr6 = reg_value.GetAsUInt32();
+    break;
+  case lldb_dr7_i386:
+    tls_context.Dr7 = reg_value.GetAsUInt32();
+    break;
+  }
+
+  return SetThreadContextHelper(thread_handle, &tls_context);
+}
+
 Status
 NativeRegisterContextWindows_i386::ReadRegister(const RegisterInfo *reg_info,
                                                 RegisterValue &reg_value) {
@@ -261,6 +341,9 @@ NativeRegisterContextWindows_i386::ReadRegister(const RegisterInfo *reg_info,
   if (IsGPR(reg))
     return GPRRead(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRRead(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -286,6 +369,9 @@ Status NativeRegisterContextWindows_i386::WriteRegister(
   if (IsGPR(reg))
     return GPRWrite(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRWrite(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -329,46 +415,198 @@ Status NativeRegisterContextWindows_i386::WriteAllRegisterValues(
 
 Status NativeRegisterContextWindows_i386::IsWatchpointHit(uint32_t wp_index,
                                                           bool &is_hit) {
-  return Status("unimplemented");
+  is_hit = false;
+
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_hit = reg_value.GetAsUInt32() & (1 << wp_index);
+
+  return {};
 }
 
 Status NativeRegisterContextWindows_i386::GetWatchpointHitIndex(
     uint32_t &wp_index, lldb::addr_t trap_addr) {
-  return Status("unimplemented");
+  wp_index = LLDB_INVALID_INDEX32;
+
+  for (uint32_t i = 0; i < NumSupportedHardwareWatchpoints(); i++) {
+    bool is_hit;
+    Status error = IsWatchpointHit(i, is_hit);
+    if (error.Fail())
+      return error;
+
+    if (is_hit) {
+      wp_index = i;
+      return {};
+    }
+  }
+
+  return {};
 }
 
 Status NativeRegisterContextWindows_i386::IsWatchpointVacant(uint32_t wp_index,
                                                              bool &is_vacant) {
-  return Status("unimplemented");
-}
+  is_vacant = false;
 
-Status NativeRegisterContextWindows_i386::SetHardwareWatchpointWithIndex(
-    lldb::addr_t addr, size_t size, uint32_t watch_flags, uint32_t wp_index) {
-  return Status("unimplemented");
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("Watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_vacant = !(reg_value.GetAsUInt32() & (1 << (2 * wp_index)));
+
+  return error;
 }
 
 bool NativeRegisterContextWindows_i386::ClearHardwareWatchpoint(
     uint32_t wp_index) {
-  return false;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits 0, 1, 2, or 3 of
+  // the debug status register (DR6)
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return false;
+
+  uint32_t bit_mask = 1 << wp_index;
+  uint32_t status_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  error = DRWrite(lldb_dr6_i386, RegisterValue(status_bits));
+  if (error.Fail())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits {0-1,16-19},
+  // {2-3,20-23}, {4-5,24-27}, or {6-7,28-31} of the debug control register
+  // (DR7)
+
+  error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return false;
+
+  bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  return DRWrite(lldb_dr7_i386, RegisterValue(control_bits)).Success();
 }
 
 Status NativeRegisterContextWindows_i386::ClearAllHardwareWatchpoints() {
-  return Status("unimplemented");
+  RegisterValue reg_value;
+
+  // clear bits {0-4} of the debug status register (DR6)
+
+  Status error = DRRead(lldb_dr6_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint32_t status_bits = reg_value.GetAsUInt32() & ~0xF;
+  error = DRWrite(lldb_dr6_i386, RegisterValue(status_bits));
+  if (error.Fail())
+    return error;
+
+  // clear bits {0-7,16-31} of the debug control register (DR7)
+
+  error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~0xFFFF00FF;
+  return DRWrite(lldb_dr7_i386, RegisterValue(control_bits));
 }
 
 uint32_t NativeRegisterContextWindows_i386::SetHardwareWatchpoint(
     lldb::addr_t addr, size_t size, uint32_t watch_flags) {
+  switch (size) {
+  case 1:
+  case 2:
+  case 4:
+    break;
+  default:
+    return LLDB_INVALID_INDEX32;
+  }
+
+  if (watch_flags == 0x2)
+    watch_flags = 0x3;
+
+  if (watch_flags != 0x1 && watch_flags != 0x3)
+    return LLDB_INVALID_INDEX32;
+
+  for (uint32_t wp_index = 0; wp_index < NumSupportedHardwareWatchpoints();
+       ++wp_index) {
+    bool is_vacant;
+    if (IsWatchpointVacant(wp_index, is_vacant).Fail())
+      return LLDB_INVALID_INDEX32;
+
+    if (is_vacant) {
+      if (!ClearHardwareWatchpoint(wp_index))
+        return LLDB_INVALID_INDEX32;
+
+      if (ApplyHardwareBreakpoint(wp_index, addr, size, watch_flags).Fail())
+        return LLDB_INVALID_INDEX32;
+
+      return wp_index;
+    }
+  }
   return LLDB_INVALID_INDEX32;
 }
 
+Status NativeRegisterContextWindows_i386::ApplyHardwareBreakpoint(
+    uint32_t wp_index, lldb::addr_t addr, size_t size, uint32_t flags) {
+  RegisterValue reg_value;
+  auto error = DRRead(lldb_dr7_i386, reg_value);
+  if (error.Fail())
+    return error;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, set bits 1, 3, 5, or 7
+  uint32_t enable_bit = 1 << (2 * wp_index);
+
+  // set bits 16-17, 20-21, 24-25, or 28-29
+  // with 0b01 for write, and 0b11 for read/write
+  uint32_t rw_bits = flags << (16 + 4 * wp_index);
+
+  // set bits 18-19, 22-23, 26-27, or 30-31
+  // with 0b00, 0b01, 0b10, or 0b11
+  // for 1, 2, 8 (if supported), or 4 bytes, respectively
+  uint32_t size_bits = (size == 8 ? 0x2 : size - 1) << (18 + 4 * wp_index);
+
+  uint32_t bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+
+  uint32_t control_bits = reg_value.GetAsUInt32() & ~bit_mask;
+  control_bits |= enable_bit | rw_bits | size_bits;
+
+  error = DRWrite(lldb_dr7_i386, RegisterValue(control_bits));
+  if (error.Fail())
+    return error;
+
+  error = DRWrite(lldb_dr0_i386 + wp_index, RegisterValue(addr));
+  if (error.Fail())
+    return error;
+
+  return {};
+}
+
 lldb::addr_t
 NativeRegisterContextWindows_i386::GetWatchpointAddress(uint32_t wp_index) {
-  return LLDB_INVALID_ADDRESS;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return LLDB_INVALID_ADDRESS;
+
+  RegisterValue reg_value;
+  if (DRRead(lldb_dr0_i386 + wp_index, reg_value).Fail())
+    return LLDB_INVALID_ADDRESS;
+
+  return reg_value.GetAsUInt32();
 }
 
 uint32_t NativeRegisterContextWindows_i386::NumSupportedHardwareWatchpoints() {
-  // Not implemented
-  return 0;
+  return 4;
 }
 
 #endif
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.h
index 1e1689cc95708..d357e63423814 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_i386.h
@@ -49,10 +49,6 @@ class NativeRegisterContextWindows_i386 : public NativeRegisterContextWindows {
 
   Status ClearAllHardwareWatchpoints() override;
 
-  Status SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size,
-                                        uint32_t watch_flags,
-                                        uint32_t wp_index);
-
   uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size,
                                  uint32_t watch_flags) override;
 
@@ -64,8 +60,15 @@ class NativeRegisterContextWindows_i386 : public NativeRegisterContextWindows {
   Status GPRRead(const uint32_t reg, RegisterValue &reg_value);
   Status GPRWrite(const uint32_t reg, const RegisterValue &reg_value);
 
+  Status DRRead(const uint32_t reg, RegisterValue &reg_value);
+  Status DRWrite(const uint32_t reg, const RegisterValue &reg_value);
+
 private:
+  Status ApplyHardwareBreakpoint(uint32_t wp_index, lldb::addr_t addr,
+                                 size_t size, uint32_t flags);
+
   bool IsGPR(uint32_t reg_index) const;
+  bool IsDR(uint32_t reg_index) const;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
index f202bda3fbebe..fed644946643f 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.cpp
@@ -101,11 +101,11 @@ NativeRegisterContextWindows::CreateHostNativeRegisterContextWindows(
   // Register context for a WoW64 application.
   if (target_arch.GetAddressByteSize() == 4)
     return std::make_unique<NativeRegisterContextWindows_WoW64>(target_arch,
-                                                                 native_thread);
+                                                                native_thread);
 
   // Register context for a native 64-bit application.
   return std::make_unique<NativeRegisterContextWindows_x86_64>(target_arch,
-                                                                native_thread);
+                                                               native_thread);
 }
 
 NativeRegisterContextWindows_x86_64::NativeRegisterContextWindows_x86_64(
@@ -121,6 +121,10 @@ bool NativeRegisterContextWindows_x86_64::IsFPR(uint32_t reg_index) const {
   return (reg_index >= lldb_xmm0_x86_64 && reg_index <= k_last_fpr_x86_64);
 }
 
+bool NativeRegisterContextWindows_x86_64::IsDR(uint32_t reg_index) const {
+  return (reg_index >= lldb_dr0_x86_64 && reg_index <= lldb_dr7_x86_64);
+}
+
 uint32_t NativeRegisterContextWindows_x86_64::GetRegisterSetCount() const {
   return k_num_register_sets;
 }
@@ -436,6 +440,82 @@ NativeRegisterContextWindows_x86_64::FPRWrite(const uint32_t reg,
   return SetThreadContextHelper(thread_handle, &tls_context);
 }
 
+Status NativeRegisterContextWindows_x86_64::DRRead(const uint32_t reg,
+                                                   RegisterValue &reg_value) {
+  ::CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  Status error =
+      GetThreadContextHelper(GetThreadHandle(), &tls_context, context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_x86_64:
+    reg_value.SetUInt64(tls_context.Dr0);
+    break;
+  case lldb_dr1_x86_64:
+    reg_value.SetUInt64(tls_context.Dr1);
+    break;
+  case lldb_dr2_x86_64:
+    reg_value.SetUInt64(tls_context.Dr2);
+    break;
+  case lldb_dr3_x86_64:
+    reg_value.SetUInt64(tls_context.Dr3);
+    break;
+  case lldb_dr4_x86_64:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_x86_64:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_x86_64:
+    reg_value.SetUInt64(tls_context.Dr6);
+    break;
+  case lldb_dr7_x86_64:
+    reg_value.SetUInt64(tls_context.Dr7);
+    break;
+  }
+
+  return {};
+}
+
+Status
+NativeRegisterContextWindows_x86_64::DRWrite(const uint32_t reg,
+                                             const RegisterValue &reg_value) {
+  ::CONTEXT tls_context;
+  DWORD context_flag = CONTEXT_DEBUG_REGISTERS;
+  auto thread_handle = GetThreadHandle();
+  Status error =
+      GetThreadContextHelper(thread_handle, &tls_context, context_flag);
+  if (error.Fail())
+    return error;
+
+  switch (reg) {
+  case lldb_dr0_x86_64:
+    tls_context.Dr0 = reg_value.GetAsUInt64();
+    break;
+  case lldb_dr1_x86_64:
+    tls_context.Dr1 = reg_value.GetAsUInt64();
+    break;
+  case lldb_dr2_x86_64:
+    tls_context.Dr2 = reg_value.GetAsUInt64();
+    break;
+  case lldb_dr3_x86_64:
+    tls_context.Dr3 = reg_value.GetAsUInt64();
+    break;
+  case lldb_dr4_x86_64:
+    return Status("register DR4 is obsolete");
+  case lldb_dr5_x86_64:
+    return Status("register DR5 is obsolete");
+  case lldb_dr6_x86_64:
+    tls_context.Dr6 = reg_value.GetAsUInt64();
+    break;
+  case lldb_dr7_x86_64:
+    tls_context.Dr7 = reg_value.GetAsUInt64();
+    break;
+  }
+
+  return SetThreadContextHelper(thread_handle, &tls_context);
+}
+
 Status
 NativeRegisterContextWindows_x86_64::ReadRegister(const RegisterInfo *reg_info,
                                                   RegisterValue &reg_value) {
@@ -461,6 +541,9 @@ NativeRegisterContextWindows_x86_64::ReadRegister(const RegisterInfo *reg_info,
   if (IsFPR(reg))
     return FPRRead(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRRead(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -489,6 +572,9 @@ Status NativeRegisterContextWindows_x86_64::WriteRegister(
   if (IsFPR(reg))
     return FPRWrite(reg, reg_value);
 
+  if (IsDR(reg))
+    return DRWrite(reg, reg_value);
+
   return Status("unimplemented");
 }
 
@@ -532,48 +618,201 @@ Status NativeRegisterContextWindows_x86_64::WriteAllRegisterValues(
 
 Status NativeRegisterContextWindows_x86_64::IsWatchpointHit(uint32_t wp_index,
                                                             bool &is_hit) {
-  return Status("unimplemented");
+  is_hit = false;
+
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_x86_64, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_hit = reg_value.GetAsUInt64() & (1ULL << wp_index);
+
+  return {};
 }
 
 Status NativeRegisterContextWindows_x86_64::GetWatchpointHitIndex(
     uint32_t &wp_index, lldb::addr_t trap_addr) {
-  return Status("unimplemented");
+  wp_index = LLDB_INVALID_INDEX32;
+
+  for (uint32_t i = 0; i < NumSupportedHardwareWatchpoints(); i++) {
+    bool is_hit;
+    Status error = IsWatchpointHit(i, is_hit);
+    if (error.Fail())
+      return error;
+
+    if (is_hit) {
+      wp_index = i;
+      return {};
+    }
+  }
+
+  return {};
 }
 
 Status
 NativeRegisterContextWindows_x86_64::IsWatchpointVacant(uint32_t wp_index,
                                                         bool &is_vacant) {
-  return Status("unimplemented");
-}
+  is_vacant = false;
 
-Status NativeRegisterContextWindows_x86_64::SetHardwareWatchpointWithIndex(
-    lldb::addr_t addr, size_t size, uint32_t watch_flags, uint32_t wp_index) {
-  return Status("unimplemented");
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return Status("Watchpoint index out of range");
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr7_x86_64, reg_value);
+  if (error.Fail())
+    return error;
+
+  is_vacant = !(reg_value.GetAsUInt64() & (1ULL << (2 * wp_index)));
+
+  return error;
 }
 
 bool NativeRegisterContextWindows_x86_64::ClearHardwareWatchpoint(
     uint32_t wp_index) {
-  return false;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits 0, 1, 2, or 3 of
+  // the debug status register (DR6)
+
+  RegisterValue reg_value;
+  Status error = DRRead(lldb_dr6_x86_64, reg_value);
+  if (error.Fail())
+    return false;
+
+  uint64_t bit_mask = 1ULL << wp_index;
+  uint64_t status_bits = reg_value.GetAsUInt64() & ~bit_mask;
+  error = DRWrite(lldb_dr6_x86_64, RegisterValue(status_bits));
+  if (error.Fail())
+    return false;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, clear bits {0-1,16-19},
+  // {2-3,20-23}, {4-5,24-27}, or {6-7,28-31} of the debug control register
+  // (DR7)
+
+  error = DRRead(lldb_dr7_x86_64, reg_value);
+  if (error.Fail())
+    return false;
+
+  bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+  uint64_t control_bits = reg_value.GetAsUInt64() & ~bit_mask;
+  return DRWrite(lldb_dr7_x86_64, RegisterValue(control_bits)).Success();
 }
 
 Status NativeRegisterContextWindows_x86_64::ClearAllHardwareWatchpoints() {
-  return Status("unimplemented");
+  RegisterValue reg_value;
+
+  // clear bits {0-4} of the debug status register (DR6)
+
+  Status error = DRRead(lldb_dr6_x86_64, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint64_t status_bits = reg_value.GetAsUInt64() & ~0xFULL;
+  error = DRWrite(lldb_dr6_x86_64, RegisterValue(status_bits));
+  if (error.Fail())
+    return error;
+
+  // clear bits {0-7,16-31} of the debug control register (DR7)
+
+  error = DRRead(lldb_dr7_x86_64, reg_value);
+  if (error.Fail())
+    return error;
+
+  uint64_t control_bits = reg_value.GetAsUInt64() & ~0xFFFF00FFULL;
+  return DRWrite(lldb_dr7_x86_64, RegisterValue(control_bits));
 }
 
 uint32_t NativeRegisterContextWindows_x86_64::SetHardwareWatchpoint(
     lldb::addr_t addr, size_t size, uint32_t watch_flags) {
+  switch (size) {
+  case 1:
+  case 2:
+  case 4:
+  case 8:
+    break;
+  default:
+    return LLDB_INVALID_INDEX32;
+  }
+
+  if (watch_flags == 0x2)
+    watch_flags = 0x3;
+
+  if (watch_flags != 0x1 && watch_flags != 0x3)
+    return LLDB_INVALID_INDEX32;
+
+  for (uint32_t wp_index = 0; wp_index < NumSupportedHardwareWatchpoints();
+       ++wp_index) {
+    bool is_vacant;
+    if (IsWatchpointVacant(wp_index, is_vacant).Fail())
+      return LLDB_INVALID_INDEX32;
+
+    if (is_vacant) {
+      if (!ClearHardwareWatchpoint(wp_index))
+        return LLDB_INVALID_INDEX32;
+
+      if (ApplyHardwareBreakpoint(wp_index, addr, size, watch_flags).Fail())
+        return LLDB_INVALID_INDEX32;
+
+      return wp_index;
+    }
+  }
   return LLDB_INVALID_INDEX32;
 }
 
+Status NativeRegisterContextWindows_x86_64::ApplyHardwareBreakpoint(
+    uint32_t wp_index, lldb::addr_t addr, size_t size, uint32_t flags) {
+  RegisterValue reg_value;
+  auto error = DRRead(lldb_dr7_x86_64, reg_value);
+  if (error.Fail())
+    return error;
+
+  // for watchpoints 0, 1, 2, or 3, respectively, set bits 1, 3, 5, or 7
+  uint64_t enable_bit = 1ULL << (2 * wp_index);
+
+  // set bits 16-17, 20-21, 24-25, or 28-29
+  // with 0b01 for write, and 0b11 for read/write
+  uint64_t rw_bits = flags << (16 + 4 * wp_index);
+
+  // set bits 18-19, 22-23, 26-27, or 30-31
+  // with 0b00, 0b01, 0b10, or 0b11
+  // for 1, 2, 8 (if supported), or 4 bytes, respectively
+  uint64_t size_bits = (size == 8 ? 0x2 : size - 1) << (18 + 4 * wp_index);
+
+  uint64_t bit_mask = (0x3 << (2 * wp_index)) | (0xF << (16 + 4 * wp_index));
+
+  uint64_t control_bits = reg_value.GetAsUInt64() & ~bit_mask;
+  control_bits |= enable_bit | rw_bits | size_bits;
+
+  error = DRWrite(lldb_dr7_x86_64, RegisterValue(control_bits));
+  if (error.Fail())
+    return error;
+
+  error = DRWrite(lldb_dr0_x86_64 + wp_index, RegisterValue(addr));
+  if (error.Fail())
+    return error;
+
+  return {};
+}
+
 lldb::addr_t
 NativeRegisterContextWindows_x86_64::GetWatchpointAddress(uint32_t wp_index) {
-  return LLDB_INVALID_ADDRESS;
+  if (wp_index >= NumSupportedHardwareWatchpoints())
+    return LLDB_INVALID_ADDRESS;
+
+  RegisterValue reg_value;
+  if (DRRead(lldb_dr0_x86_64 + wp_index, reg_value).Fail())
+    return LLDB_INVALID_ADDRESS;
+
+  return reg_value.GetAsUInt64();
 }
 
 uint32_t
 NativeRegisterContextWindows_x86_64::NumSupportedHardwareWatchpoints() {
-  // Not implemented
-  return 0;
+  return 4;
 }
 
 #endif // defined(__x86_64__) || defined(_M_X64)
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.h b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.h
index 49f541efa640e..8699f740c8f68 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.h
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeRegisterContextWindows_x86_64.h
@@ -50,10 +50,6 @@ class NativeRegisterContextWindows_x86_64
 
   Status ClearAllHardwareWatchpoints() override;
 
-  Status SetHardwareWatchpointWithIndex(lldb::addr_t addr, size_t size,
-                                        uint32_t watch_flags,
-                                        uint32_t wp_index);
-
   uint32_t SetHardwareWatchpoint(lldb::addr_t addr, size_t size,
                                  uint32_t watch_flags) override;
 
@@ -63,17 +59,21 @@ class NativeRegisterContextWindows_x86_64
 
 protected:
   Status GPRRead(const uint32_t reg, RegisterValue &reg_value);
-
   Status GPRWrite(const uint32_t reg, const RegisterValue &reg_value);
 
   Status FPRRead(const uint32_t reg, RegisterValue &reg_value);
-
   Status FPRWrite(const uint32_t reg, const RegisterValue &reg_value);
 
+  Status DRRead(const uint32_t reg, RegisterValue &reg_value);
+  Status DRWrite(const uint32_t reg, const RegisterValue &reg_value);
+
 private:
-  bool IsGPR(uint32_t reg_index) const;
+  Status ApplyHardwareBreakpoint(uint32_t wp_index, lldb::addr_t addr,
+                                 size_t size, uint32_t flags);
 
+  bool IsGPR(uint32_t reg_index) const;
   bool IsFPR(uint32_t reg_index) const;
+  bool IsDR(uint32_t reg_index) const;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.cpp
index 91af4c8cd72fa..45bdec40b40b3 100644
--- a/lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/NativeThreadWindows.cpp
@@ -132,11 +132,30 @@ bool NativeThreadWindows::GetStopReason(ThreadStopInfo &stop_info,
 
 Status NativeThreadWindows::SetWatchpoint(lldb::addr_t addr, size_t size,
                                           uint32_t watch_flags, bool hardware) {
-  return Status("unimplemented.");
+  if (!hardware)
+    return Status("not implemented");
+  if (m_state == eStateLaunching)
+    return Status();
+  Status error = RemoveWatchpoint(addr);
+  if (error.Fail())
+    return error;
+  uint32_t wp_index =
+      m_reg_context_up->SetHardwareWatchpoint(addr, size, watch_flags);
+  if (wp_index == LLDB_INVALID_INDEX32)
+    return Status("Setting hardware watchpoint failed.");
+  m_watchpoint_index_map.insert({addr, wp_index});
+  return Status();
 }
 
 Status NativeThreadWindows::RemoveWatchpoint(lldb::addr_t addr) {
-  return Status("unimplemented");
+  auto wp = m_watchpoint_index_map.find(addr);
+  if (wp == m_watchpoint_index_map.end())
+    return Status();
+  uint32_t wp_index = wp->second;
+  m_watchpoint_index_map.erase(wp);
+  if (m_reg_context_up->ClearHardwareWatchpoint(wp_index))
+    return Status();
+  return Status("Clearing hardware watchpoint failed.");
 }
 
 Status NativeThreadWindows::SetHardwareBreakpoint(lldb::addr_t addr,
diff --git a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
index 461dbebb3d8a7..8a06fb7ed3793 100644
--- a/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/ProcessWindows.cpp
@@ -170,9 +170,9 @@ Status ProcessWindows::DoDetach(bool keep_stopped) {
     else
       LLDB_LOG(log, "Detaching process error: {0}", error);
   } else {
-    error.SetErrorStringWithFormat("error: process {0} in state = {1}, but "
-                                   "cannot detach it in this state.",
-                                   GetID(), private_state);
+    error.SetErrorStringWithFormatv("error: process {0} in state = {1}, but "
+                                    "cannot detach it in this state.",
+                                    GetID(), private_state);
     LLDB_LOG(log, "error: {0}", error);
   }
   return error;
diff --git a/lldb/source/Plugins/Process/Windows/Common/RegisterContextWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/RegisterContextWindows.cpp
index 597673b852255..28e7a590ff9f3 100644
--- a/lldb/source/Plugins/Process/Windows/Common/RegisterContextWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/RegisterContextWindows.cpp
@@ -84,7 +84,7 @@ bool RegisterContextWindows::AddHardwareBreakpoint(uint32_t slot,
   case 1:
   case 2:
   case 4:
-#if defined(_M_AMD64)
+#if defined(_WIN64)
   case 8:
 #endif
     break;
@@ -95,6 +95,7 @@ bool RegisterContextWindows::AddHardwareBreakpoint(uint32_t slot,
   if (!CacheAllRegisterValues())
     return false;
 
+#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_AMD64)
   unsigned shift = 2 * slot;
   m_context.Dr7 |= 1ULL << shift;
 
@@ -109,6 +110,12 @@ bool RegisterContextWindows::AddHardwareBreakpoint(uint32_t slot,
   m_context.Dr7 |= (read ? 3ULL : (write ? 1ULL : 0)) << shift;
 
   return ApplyAllRegisterValues();
+
+#else
+  Log *log = ProcessWindowsLog::GetLogIfAny(WINDOWS_LOG_REGISTERS);
+  LLDB_LOG(log, "hardware breakpoints not currently supported on this arch");
+  return false;
+#endif
 }
 
 bool RegisterContextWindows::RemoveHardwareBreakpoint(uint32_t slot) {
@@ -118,19 +125,25 @@ bool RegisterContextWindows::RemoveHardwareBreakpoint(uint32_t slot) {
   if (!CacheAllRegisterValues())
     return false;
 
+#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_AMD64)
   unsigned shift = 2 * slot;
   m_context.Dr7 &= ~(1ULL << shift);
 
   return ApplyAllRegisterValues();
+#else
+  return false;
+#endif
 }
 
 uint32_t RegisterContextWindows::GetTriggeredHardwareBreakpointSlotId() {
   if (!CacheAllRegisterValues())
     return LLDB_INVALID_INDEX32;
 
+#if defined(__i386__) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_AMD64)
   for (unsigned i = 0UL; i < NUM_HARDWARE_BREAKPOINT_SLOTS; i++)
     if (m_context.Dr6 & (1ULL << i))
       return i;
+#endif
 
   return LLDB_INVALID_INDEX32;
 }
diff --git a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp
index 0af0b56925bde..8fca982da14ea 100644
--- a/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp
+++ b/lldb/source/Plugins/Process/Windows/Common/TargetThreadWindows.cpp
@@ -21,9 +21,9 @@
 #include "TargetThreadWindows.h"
 
 // TODO support _M_ARM and _M_ARM64
-#if defined(_M_AMD64)
+#if defined(__x86_64__) || defined(_M_AMD64)
 #include "x64/RegisterContextWindows_x64.h"
-#elif defined(_M_IX86)
+#elif defined(__i386__) || defined(_M_IX86)
 #include "x86/RegisterContextWindows_x86.h"
 #endif
 
@@ -77,7 +77,7 @@ TargetThreadWindows::CreateRegisterContextForFrame(StackFrame *frame) {
         break;
 
       case llvm::Triple::x86:
-#if defined(_M_IX86)
+#if defined(__i386__) || defined(_M_IX86)
         m_thread_reg_ctx_sp.reset(
             new RegisterContextWindows_x86(*this, concrete_frame_idx));
 #else
@@ -86,7 +86,7 @@ TargetThreadWindows::CreateRegisterContextForFrame(StackFrame *frame) {
         break;
 
       case llvm::Triple::x86_64:
-#if defined(_M_AMD64)
+#if defined(__x86_64__) || defined(_M_AMD64)
         m_thread_reg_ctx_sp.reset(
             new RegisterContextWindows_x64(*this, concrete_frame_idx));
 #else
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
index bb467df8bb6e3..144ae103faa4b 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunication.cpp
@@ -103,8 +103,7 @@ size_t GDBRemoteCommunication::SendAck() {
   char ch = '+';
   const size_t bytes_written = Write(&ch, 1, status, nullptr);
   LLDB_LOGF(log, "<%4" PRIu64 "> send packet: %c", (uint64_t)bytes_written, ch);
-  m_history.AddPacket(ch, GDBRemoteCommunicationHistory::ePacketTypeSend,
-                      bytes_written);
+  m_history.AddPacket(ch, GDBRemotePacket::ePacketTypeSend, bytes_written);
   return bytes_written;
 }
 
@@ -114,8 +113,7 @@ size_t GDBRemoteCommunication::SendNack() {
   char ch = '-';
   const size_t bytes_written = Write(&ch, 1, status, nullptr);
   LLDB_LOGF(log, "<%4" PRIu64 "> send packet: %c", (uint64_t)bytes_written, ch);
-  m_history.AddPacket(ch, GDBRemoteCommunicationHistory::ePacketTypeSend,
-                      bytes_written);
+  m_history.AddPacket(ch, GDBRemotePacket::ePacketTypeSend, bytes_written);
   return bytes_written;
 }
 
@@ -178,8 +176,7 @@ GDBRemoteCommunication::SendRawPacketNoLock(llvm::StringRef packet,
     }
 
     m_history.AddPacket(packet.str(), packet_length,
-                        GDBRemoteCommunicationHistory::ePacketTypeSend,
-                        bytes_written);
+                        GDBRemotePacket::ePacketTypeSend, bytes_written);
 
     if (bytes_written == packet_length) {
       if (!skip_ack && GetSendAcks())
@@ -809,8 +806,7 @@ GDBRemoteCommunication::CheckForPacket(const uint8_t *src, size_t src_len,
       }
 
       m_history.AddPacket(m_bytes, total_length,
-                          GDBRemoteCommunicationHistory::ePacketTypeRecv,
-                          total_length);
+                          GDBRemotePacket::ePacketTypeRecv, total_length);
 
       // Copy the packet from m_bytes to packet_str expanding the run-length
       // encoding in the process. Reserve enough byte for the most common case
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
index 40a88fb7de715..63d0522da3b08 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.cpp
@@ -23,7 +23,6 @@
 #include "lldb/Target/UnixSignals.h"
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/DataBufferHeap.h"
-#include "lldb/Utility/JSON.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/State.h"
@@ -35,14 +34,15 @@
 #include "lldb/Utility/StringExtractorGDBRemote.h"
 
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/JSON.h"
 
 #if defined(HAVE_LIBCOMPRESSION)
 #include <compression.h>
 #endif
 
 using namespace lldb;
-using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
+using namespace lldb_private;
 using namespace std::chrono;
 
 // GDBRemoteCommunicationClient constructor
@@ -3609,21 +3609,21 @@ ParseModuleSpec(StructuredData::Dictionary *dict) {
 llvm::Optional<std::vector<ModuleSpec>>
 GDBRemoteCommunicationClient::GetModulesInfo(
     llvm::ArrayRef<FileSpec> module_file_specs, const llvm::Triple &triple) {
+  namespace json = llvm::json;
+
   if (!m_supports_jModulesInfo)
     return llvm::None;
 
-  JSONArray::SP module_array_sp = std::make_shared<JSONArray>();
+  json::Array module_array;
   for (const FileSpec &module_file_spec : module_file_specs) {
-    JSONObject::SP module_sp = std::make_shared<JSONObject>();
-    module_array_sp->AppendObject(module_sp);
-    module_sp->SetObject(
-        "file", std::make_shared<JSONString>(module_file_spec.GetPath(false)));
-    module_sp->SetObject("triple",
-                         std::make_shared<JSONString>(triple.getTriple()));
+    module_array.push_back(
+        json::Object{{"file", module_file_spec.GetPath(false)},
+                     {"triple", triple.getTriple()}});
   }
   StreamString unescaped_payload;
   unescaped_payload.PutCString("jModulesInfo:");
-  module_array_sp->Write(unescaped_payload);
+  unescaped_payload.AsRawOstream() << std::move(module_array);
+
   StreamGDBRemote payload;
   payload.PutEscapedBytes(unescaped_payload.GetString().data(),
                           unescaped_payload.GetSize());
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
index daee68059520b..898942bb54594 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationClient.h
@@ -18,7 +18,7 @@
 #include <vector>
 
 #include "lldb/Utility/ArchSpec.h"
-#include "lldb/Utility/StreamGDBRemote.h"
+#include "lldb/Utility/GDBRemote.h"
 #include "lldb/Utility/StructuredData.h"
 #if defined(_WIN32)
 #include "lldb/Host/windows/PosixApi.h"
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.cpp
index f9f67fcbcaa92..d2cc32f63f20a 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.cpp
@@ -18,12 +18,6 @@ using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
 
-void GDBRemoteCommunicationHistory::Entry::Serialize(raw_ostream &strm) const {
-  yaml::Output yout(strm);
-  yout << const_cast<GDBRemoteCommunicationHistory::Entry &>(*this);
-  strm.flush();
-}
-
 GDBRemoteCommunicationHistory::GDBRemoteCommunicationHistory(uint32_t size)
     : m_packets(), m_curr_idx(0), m_total_packet_count(0),
       m_dumped_to_log(false) {
@@ -33,7 +27,8 @@ GDBRemoteCommunicationHistory::GDBRemoteCommunicationHistory(uint32_t size)
 
 GDBRemoteCommunicationHistory::~GDBRemoteCommunicationHistory() {}
 
-void GDBRemoteCommunicationHistory::AddPacket(char packet_char, PacketType type,
+void GDBRemoteCommunicationHistory::AddPacket(char packet_char,
+                                              GDBRemotePacket::Type type,
                                               uint32_t bytes_transmitted) {
   const size_t size = m_packets.size();
   if (size == 0)
@@ -50,7 +45,8 @@ void GDBRemoteCommunicationHistory::AddPacket(char packet_char, PacketType type,
 }
 
 void GDBRemoteCommunicationHistory::AddPacket(const std::string &src,
-                                              uint32_t src_len, PacketType type,
+                                              uint32_t src_len,
+                                              GDBRemotePacket::Type type,
                                               uint32_t bytes_transmitted) {
   const size_t size = m_packets.size();
   if (size == 0)
@@ -72,13 +68,12 @@ void GDBRemoteCommunicationHistory::Dump(Stream &strm) const {
   const uint32_t stop_idx = m_curr_idx + size;
   for (uint32_t i = first_idx; i < stop_idx; ++i) {
     const uint32_t idx = NormalizeIndex(i);
-    const Entry &entry = m_packets[idx];
-    if (entry.type == ePacketTypeInvalid || entry.packet.data.empty())
+    const GDBRemotePacket &entry = m_packets[idx];
+    if (entry.type == GDBRemotePacket::ePacketTypeInvalid ||
+        entry.packet.data.empty())
       break;
-    strm.Printf("history[%u] tid=0x%4.4" PRIx64 " <%4u> %s packet: %s\n",
-                entry.packet_idx, entry.tid, entry.bytes_transmitted,
-                (entry.type == ePacketTypeSend) ? "send" : "read",
-                entry.packet.data.c_str());
+    strm.Printf("history[%u] ", entry.packet_idx);
+    entry.Dump(strm);
   }
 }
 
@@ -92,51 +87,15 @@ void GDBRemoteCommunicationHistory::Dump(Log *log) const {
   const uint32_t stop_idx = m_curr_idx + size;
   for (uint32_t i = first_idx; i < stop_idx; ++i) {
     const uint32_t idx = NormalizeIndex(i);
-    const Entry &entry = m_packets[idx];
-    if (entry.type == ePacketTypeInvalid || entry.packet.data.empty())
+    const GDBRemotePacket &entry = m_packets[idx];
+    if (entry.type == GDBRemotePacket::ePacketTypeInvalid ||
+        entry.packet.data.empty())
       break;
     LLDB_LOGF(log, "history[%u] tid=0x%4.4" PRIx64 " <%4u> %s packet: %s",
               entry.packet_idx, entry.tid, entry.bytes_transmitted,
-              (entry.type == ePacketTypeSend) ? "send" : "read",
+              (entry.type == GDBRemotePacket::ePacketTypeSend) ? "send"
+                                                               : "read",
               entry.packet.data.c_str());
   }
 }
 
-void yaml::ScalarEnumerationTraits<GDBRemoteCommunicationHistory::PacketType>::
-    enumeration(IO &io, GDBRemoteCommunicationHistory::PacketType &value) {
-  io.enumCase(value, "Invalid",
-              GDBRemoteCommunicationHistory::ePacketTypeInvalid);
-  io.enumCase(value, "Send", GDBRemoteCommunicationHistory::ePacketTypeSend);
-  io.enumCase(value, "Recv", GDBRemoteCommunicationHistory::ePacketTypeRecv);
-}
-
-void yaml::ScalarTraits<GDBRemoteCommunicationHistory::Entry::BinaryData>::
-    output(const GDBRemoteCommunicationHistory::Entry::BinaryData &Val, void *,
-           raw_ostream &Out) {
-  Out << toHex(Val.data);
-}
-
-StringRef
-yaml::ScalarTraits<GDBRemoteCommunicationHistory::Entry::BinaryData>::input(
-    StringRef Scalar, void *,
-    GDBRemoteCommunicationHistory::Entry::BinaryData &Val) {
-  Val.data = fromHex(Scalar);
-  return {};
-}
-
-void yaml::MappingTraits<GDBRemoteCommunicationHistory::Entry>::mapping(
-    IO &io, GDBRemoteCommunicationHistory::Entry &Entry) {
-  io.mapRequired("packet", Entry.packet);
-  io.mapRequired("type", Entry.type);
-  io.mapRequired("bytes", Entry.bytes_transmitted);
-  io.mapRequired("index", Entry.packet_idx);
-  io.mapRequired("tid", Entry.tid);
-}
-
-StringRef yaml::MappingTraits<GDBRemoteCommunicationHistory::Entry>::validate(
-    IO &io, GDBRemoteCommunicationHistory::Entry &Entry) {
-  if (Entry.bytes_transmitted != Entry.packet.data.size())
-    return "BinaryData size doesn't match bytes transmitted";
-
-  return {};
-}
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h
index 85f112b506236..c006fbd34a4b9 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationHistory.h
@@ -12,6 +12,7 @@
 #include <string>
 #include <vector>
 
+#include "lldb/Utility/GDBRemote.h"
 #include "lldb/lldb-public.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
@@ -25,46 +26,17 @@ class GDBRemoteCommunicationHistory {
 public:
   friend llvm::yaml::MappingTraits<GDBRemoteCommunicationHistory>;
 
-  enum PacketType { ePacketTypeInvalid = 0, ePacketTypeSend, ePacketTypeRecv };
-
-  /// Entry in the ring buffer containing the packet data, its type, size and
-  /// index. Entries can be serialized to file.
-  struct Entry {
-    Entry()
-        : packet(), type(ePacketTypeInvalid), bytes_transmitted(0),
-          packet_idx(0), tid(LLDB_INVALID_THREAD_ID) {}
-
-    void Clear() {
-      packet.data.clear();
-      type = ePacketTypeInvalid;
-      bytes_transmitted = 0;
-      packet_idx = 0;
-      tid = LLDB_INVALID_THREAD_ID;
-    }
-
-    struct BinaryData {
-      std::string data;
-    };
-
-    void Serialize(llvm::raw_ostream &strm) const;
-
-    BinaryData packet;
-    PacketType type;
-    uint32_t bytes_transmitted;
-    uint32_t packet_idx;
-    lldb::tid_t tid;
-  };
-
   GDBRemoteCommunicationHistory(uint32_t size = 0);
 
   ~GDBRemoteCommunicationHistory();
 
   // For single char packets for ack, nack and /x03
-  void AddPacket(char packet_char, PacketType type, uint32_t bytes_transmitted);
-
-  void AddPacket(const std::string &src, uint32_t src_len, PacketType type,
+  void AddPacket(char packet_char, GDBRemotePacket::Type type,
                  uint32_t bytes_transmitted);
 
+  void AddPacket(const std::string &src, uint32_t src_len,
+                 GDBRemotePacket::Type type, uint32_t bytes_transmitted);
+
   void Dump(Stream &strm) const;
   void Dump(Log *log) const;
   bool DidDumpToLog() const { return m_dumped_to_log; }
@@ -97,7 +69,7 @@ class GDBRemoteCommunicationHistory {
     return m_packets.empty() ? 0 : i % m_packets.size();
   }
 
-  std::vector<Entry> m_packets;
+  std::vector<GDBRemotePacket> m_packets;
   uint32_t m_curr_idx;
   uint32_t m_total_packet_count;
   mutable bool m_dumped_to_log;
@@ -107,49 +79,4 @@ class GDBRemoteCommunicationHistory {
 } // namespace process_gdb_remote
 } // namespace lldb_private
 
-LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(
-    lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry)
-
-namespace llvm {
-namespace yaml {
-
-template <>
-struct ScalarEnumerationTraits<lldb_private::process_gdb_remote::
-                                   GDBRemoteCommunicationHistory::PacketType> {
-  static void enumeration(IO &io,
-                          lldb_private::process_gdb_remote::
-                              GDBRemoteCommunicationHistory::PacketType &value);
-};
-
-template <>
-struct ScalarTraits<lldb_private::process_gdb_remote::
-                        GDBRemoteCommunicationHistory::Entry::BinaryData> {
-  static void output(const lldb_private::process_gdb_remote::
-                         GDBRemoteCommunicationHistory::Entry::BinaryData &,
-                     void *, raw_ostream &);
-
-  static StringRef
-  input(StringRef, void *,
-        lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry::
-            BinaryData &);
-
-  static QuotingType mustQuote(StringRef S) { return QuotingType::None; }
-};
-
-template <>
-struct MappingTraits<
-    lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry> {
-  static void
-  mapping(IO &io,
-          lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry
-              &Entry);
-
-  static StringRef validate(
-      IO &io,
-      lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry &);
-};
-
-} // namespace yaml
-} // namespace llvm
-
 #endif // liblldb_GDBRemoteCommunicationHistory_h_
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.cpp
index 39304a6ff94e8..2d26c550dc76a 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.cpp
@@ -128,7 +128,7 @@ GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
   Log *log(ProcessGDBRemoteLog::GetLogIfAllCategoriesSet(GDBR_LOG_PROCESS));
   while (!m_packet_history.empty()) {
     // Pop last packet from the history.
-    GDBRemoteCommunicationHistory::Entry entry = m_packet_history.back();
+    GDBRemotePacket entry = m_packet_history.back();
     m_packet_history.pop_back();
 
     // We've handled the handshake implicitly before. Skip the packet and move
@@ -136,7 +136,7 @@ GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
     if (entry.packet.data == "+")
       continue;
 
-    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeSend) {
+    if (entry.type == GDBRemotePacket::ePacketTypeSend) {
       if (unexpected(entry.packet.data, packet.GetStringRef())) {
         LLDB_LOG(log,
                  "GDBRemoteCommunicationReplayServer expected packet: '{0}'",
@@ -150,14 +150,14 @@ GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
       // Ignore QEnvironment packets as they're handled earlier.
       if (entry.packet.data.find("QEnvironment") == 1) {
         assert(m_packet_history.back().type ==
-               GDBRemoteCommunicationHistory::ePacketTypeRecv);
+               GDBRemotePacket::ePacketTypeRecv);
         m_packet_history.pop_back();
       }
 
       continue;
     }
 
-    if (entry.type == GDBRemoteCommunicationHistory::ePacketTypeInvalid) {
+    if (entry.type == GDBRemotePacket::ePacketTypeInvalid) {
       LLDB_LOG(
           log,
           "GDBRemoteCommunicationReplayServer skipped invalid packet: '{0}'",
@@ -176,10 +176,6 @@ GDBRemoteCommunicationReplayServer::GetPacketAndSendResponse(
   return packet_result;
 }
 
-LLVM_YAML_IS_DOCUMENT_LIST_VECTOR(
-    std::vector<
-        lldb_private::process_gdb_remote::GDBRemoteCommunicationHistory::Entry>)
-
 llvm::Error
 GDBRemoteCommunicationReplayServer::LoadReplayHistory(const FileSpec &path) {
   auto error_or_file = MemoryBuffer::getFile(path.GetPath());
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.h b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.h
index 26d65e265463b..0b5e910f7c6a6 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.h
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationReplayServer.h
@@ -62,7 +62,7 @@ class GDBRemoteCommunicationReplayServer : public GDBRemoteCommunication {
   static lldb::thread_result_t AsyncThread(void *arg);
 
   /// Replay history with the oldest packet at the end.
-  std::vector<GDBRemoteCommunicationHistory::Entry> m_packet_history;
+  std::vector<GDBRemotePacket> m_packet_history;
 
   /// Server thread.
   Broadcaster m_async_broadcaster;
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
index d0ae5191efd6f..c5a07a88d1e73 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerCommon.cpp
@@ -29,12 +29,13 @@
 #include "lldb/Symbol/ObjectFile.h"
 #include "lldb/Target/Platform.h"
 #include "lldb/Utility/Endian.h"
-#include "lldb/Utility/JSON.h"
+#include "lldb/Utility/GDBRemote.h"
 #include "lldb/Utility/Log.h"
-#include "lldb/Utility/StreamGDBRemote.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StructuredData.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Support/JSON.h"
 
 #include "ProcessGDBRemoteLog.h"
 #include "lldb/Utility/StringExtractorGDBRemote.h"
@@ -43,11 +44,10 @@
 #include "lldb/Host/android/HostInfoAndroid.h"
 #endif
 
-#include "llvm/ADT/StringSwitch.h"
 
 using namespace lldb;
-using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
+using namespace lldb_private;
 
 #ifdef __ANDROID__
 const static uint32_t g_default_packet_timeout_sec = 20; // seconds
@@ -512,14 +512,23 @@ GDBRemoteCommunicationServerCommon::Handle_vFile_Open(
         mode_t mode = packet.GetHexMaxU32(false, 0600);
         FileSpec path_spec(path);
         FileSystem::Instance().Resolve(path_spec);
-        File file;
         // Do not close fd.
-        Status error =
-            FileSystem::Instance().Open(file, path_spec, flags, mode, false);
-        const int save_errno = error.GetError();
+        auto file = FileSystem::Instance().Open(path_spec, flags, mode, false);
+
+        int save_errno = 0;
+        int descriptor = File::kInvalidDescriptor;
+        if (file) {
+          descriptor = file.get()->GetDescriptor();
+        } else {
+          std::error_code code = errorToErrorCode(file.takeError());
+          if (code.category() == std::system_category()) {
+            save_errno = code.value();
+          }
+        }
+
         StreamString response;
         response.PutChar('F');
-        response.Printf("%i", file.GetDescriptor());
+        response.Printf("%i", descriptor);
         if (save_errno)
           response.Printf(",%i", save_errno);
         return SendPacketNoLock(response.GetString());
@@ -537,7 +546,7 @@ GDBRemoteCommunicationServerCommon::Handle_vFile_Close(
   int err = -1;
   int save_errno = 0;
   if (fd >= 0) {
-    File file(fd, true);
+    File file(fd, 0, true);
     Status error = file.Close();
     err = 0;
     save_errno = error.GetError();
@@ -568,7 +577,7 @@ GDBRemoteCommunicationServerCommon::Handle_vFile_pRead(
       }
 
       std::string buffer(count, 0);
-      File file(fd, false);
+      File file(fd, File::eOpenOptionRead, false);
       Status error = file.Read(static_cast<void *>(&buffer[0]), count, offset);
       const ssize_t bytes_read = error.Success() ? count : -1;
       const int save_errno = error.GetError();
@@ -600,7 +609,7 @@ GDBRemoteCommunicationServerCommon::Handle_vFile_pWrite(
     if (packet.GetChar() == ',') {
       std::string buffer;
       if (packet.GetEscapedBinaryData(buffer)) {
-        File file(fd, false);
+        File file(fd, File::eOpenOptionWrite, false);
         size_t count = buffer.size();
         Status error =
             file.Write(static_cast<const void *>(&buffer[0]), count, offset);
@@ -1111,6 +1120,8 @@ GDBRemoteCommunicationServerCommon::Handle_qModuleInfo(
 GDBRemoteCommunication::PacketResult
 GDBRemoteCommunicationServerCommon::Handle_jModulesInfo(
     StringExtractorGDBRemote &packet) {
+  namespace json = llvm::json;
+
   packet.SetFilePos(::strlen("jModulesInfo:"));
 
   StructuredData::ObjectSP object_sp = StructuredData::ParseJSON(packet.Peek());
@@ -1121,7 +1132,7 @@ GDBRemoteCommunicationServerCommon::Handle_jModulesInfo(
   if (!packet_array)
     return SendErrorResponse(2);
 
-  JSONArray::SP response_array_sp = std::make_shared<JSONArray>();
+  json::Array response_array;
   for (size_t i = 0; i < packet_array->GetSize(); ++i) {
     StructuredData::Dictionary *query =
         packet_array->GetItemAtIndex(i)->GetAsDictionary();
@@ -1139,27 +1150,22 @@ GDBRemoteCommunicationServerCommon::Handle_jModulesInfo(
     const auto file_offset = matched_module_spec.GetObjectOffset();
     const auto file_size = matched_module_spec.GetObjectSize();
     const auto uuid_str = matched_module_spec.GetUUID().GetAsString("");
-
     if (uuid_str.empty())
       continue;
-
-    JSONObject::SP response = std::make_shared<JSONObject>();
-    response_array_sp->AppendObject(response);
-    response->SetObject("uuid", std::make_shared<JSONString>(uuid_str));
-    response->SetObject(
-        "triple",
-        std::make_shared<JSONString>(
-            matched_module_spec.GetArchitecture().GetTriple().getTriple()));
-    response->SetObject("file_path",
-                        std::make_shared<JSONString>(
-                            matched_module_spec.GetFileSpec().GetPath()));
-    response->SetObject("file_offset",
-                        std::make_shared<JSONNumber>(file_offset));
-    response->SetObject("file_size", std::make_shared<JSONNumber>(file_size));
+    const auto triple_str =
+        matched_module_spec.GetArchitecture().GetTriple().getTriple();
+    const auto file_path = matched_module_spec.GetFileSpec().GetPath();
+
+    json::Object response{{"uuid", uuid_str},
+                          {"triple", triple_str},
+                          {"file_path", file_path},
+                          {"file_offset", static_cast<int64_t>(file_offset)},
+                          {"file_size", static_cast<int64_t>(file_size)}};
+    response_array.push_back(std::move(response));
   }
 
   StreamString response;
-  response_array_sp->Write(response);
+  response.AsRawOstream() << std::move(response_array);
   StreamGDBRemote escaped_response;
   escaped_response.PutEscapedBytes(response.GetString().data(),
                                    response.GetSize());
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
index bf7860ad1a5db..4e719143da5d9 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerLLGS.cpp
@@ -11,7 +11,7 @@
 #include "lldb/Host/Config.h"
 
 #include "GDBRemoteCommunicationServerLLGS.h"
-#include "lldb/Utility/StreamGDBRemote.h"
+#include "lldb/Utility/GDBRemote.h"
 
 #include <chrono>
 #include <cstring>
@@ -32,7 +32,6 @@
 #include "lldb/Utility/Args.h"
 #include "lldb/Utility/DataBuffer.h"
 #include "lldb/Utility/Endian.h"
-#include "lldb/Utility/JSON.h"
 #include "lldb/Utility/LLDBAssert.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/RegisterValue.h"
@@ -40,6 +39,7 @@
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/UriParser.h"
 #include "llvm/ADT/Triple.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/ScopedPrinter.h"
 
 #include "ProcessGDBRemote.h"
@@ -402,19 +402,21 @@ static void WriteRegisterValueInHexFixedWidth(
   }
 }
 
-static JSONObject::SP GetRegistersAsJSON(NativeThreadProtocol &thread) {
+static llvm::Expected<json::Object>
+GetRegistersAsJSON(NativeThreadProtocol &thread) {
   Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_THREAD));
 
   NativeRegisterContext& reg_ctx = thread.GetRegisterContext();
 
-  JSONObject::SP register_object_sp = std::make_shared<JSONObject>();
+  json::Object register_object;
 
 #ifdef LLDB_JTHREADSINFO_FULL_REGISTER_SET
   // Expedite all registers in the first register set (i.e. should be GPRs)
   // that are not contained in other registers.
   const RegisterSet *reg_set_p = reg_ctx_sp->GetRegisterSet(0);
   if (!reg_set_p)
-    return nullptr;
+    return llvm::make_error<llvm::StringError>("failed to get registers",
+                                               llvm::inconvertibleErrorCode());
   for (const uint32_t *reg_num_p = reg_set_p->registers;
        *reg_num_p != LLDB_INVALID_REGNUM; ++reg_num_p) {
     uint32_t reg_num = *reg_num_p;
@@ -460,12 +462,10 @@ static JSONObject::SP GetRegistersAsJSON(NativeThreadProtocol &thread) {
     WriteRegisterValueInHexFixedWidth(stream, reg_ctx, *reg_info_p,
                                       &reg_value, lldb::eByteOrderBig);
 
-    register_object_sp->SetObject(
-        llvm::to_string(reg_num),
-        std::make_shared<JSONString>(stream.GetString()));
+    register_object.try_emplace(llvm::to_string(reg_num), stream.GetString());
   }
 
-  return register_object_sp;
+  return register_object;
 }
 
 static const char *GetStopReasonString(StopReason stop_reason) {
@@ -492,11 +492,11 @@ static const char *GetStopReasonString(StopReason stop_reason) {
   return nullptr;
 }
 
-static JSONArray::SP GetJSONThreadsInfo(NativeProcessProtocol &process,
-                                        bool abridged) {
+static llvm::Expected<json::Value>
+GetJSONThreadsInfo(NativeProcessProtocol &process, bool abridged) {
   Log *log(GetLogIfAnyCategoriesSet(LIBLLDB_LOG_PROCESS | LIBLLDB_LOG_THREAD));
 
-  JSONArray::SP threads_array_sp = std::make_shared<JSONArray>();
+  json::Array threads_array;
 
   // Ensure we can get info on the given thread.
   uint32_t thread_idx = 0;
@@ -510,7 +510,8 @@ static JSONArray::SP GetJSONThreadsInfo(NativeProcessProtocol &process,
     struct ThreadStopInfo tid_stop_info;
     std::string description;
     if (!thread->GetStopReason(tid_stop_info, description))
-      return nullptr;
+      return llvm::make_error<llvm::StringError>(
+          "failed to get stop reason", llvm::inconvertibleErrorCode());
 
     const int signum = tid_stop_info.details.signal.signo;
     if (log) {
@@ -522,50 +523,49 @@ static JSONArray::SP GetJSONThreadsInfo(NativeProcessProtocol &process,
                 tid_stop_info.reason, tid_stop_info.details.exception.type);
     }
 
-    JSONObject::SP thread_obj_sp = std::make_shared<JSONObject>();
-    threads_array_sp->AppendObject(thread_obj_sp);
+    json::Object thread_obj;
 
     if (!abridged) {
-      if (JSONObject::SP registers_sp = GetRegistersAsJSON(*thread))
-        thread_obj_sp->SetObject("registers", registers_sp);
+      if (llvm::Expected<json::Object> registers =
+              GetRegistersAsJSON(*thread)) {
+        thread_obj.try_emplace("registers", std::move(*registers));
+      } else {
+        return registers.takeError();
+      }
     }
 
-    thread_obj_sp->SetObject("tid", std::make_shared<JSONNumber>(tid));
+    thread_obj.try_emplace("tid", static_cast<int64_t>(tid));
+
     if (signum != 0)
-      thread_obj_sp->SetObject("signal", std::make_shared<JSONNumber>(signum));
+      thread_obj.try_emplace("signal", signum);
 
     const std::string thread_name = thread->GetName();
     if (!thread_name.empty())
-      thread_obj_sp->SetObject("name",
-                               std::make_shared<JSONString>(thread_name));
+      thread_obj.try_emplace("name", thread_name);
 
-    if (const char *stop_reason_str = GetStopReasonString(tid_stop_info.reason))
-      thread_obj_sp->SetObject("reason",
-                               std::make_shared<JSONString>(stop_reason_str));
+    const char *stop_reason = GetStopReasonString(tid_stop_info.reason);
+    if (stop_reason)
+      thread_obj.try_emplace("reason", stop_reason);
 
     if (!description.empty())
-      thread_obj_sp->SetObject("description",
-                               std::make_shared<JSONString>(description));
+      thread_obj.try_emplace("description", description);
 
     if ((tid_stop_info.reason == eStopReasonException) &&
         tid_stop_info.details.exception.type) {
-      thread_obj_sp->SetObject(
-          "metype",
-          std::make_shared<JSONNumber>(tid_stop_info.details.exception.type));
+      thread_obj.try_emplace(
+          "metype", static_cast<int64_t>(tid_stop_info.details.exception.type));
 
-      JSONArray::SP medata_array_sp = std::make_shared<JSONArray>();
+      json::Array medata_array;
       for (uint32_t i = 0; i < tid_stop_info.details.exception.data_count;
            ++i) {
-        medata_array_sp->AppendObject(std::make_shared<JSONNumber>(
-            tid_stop_info.details.exception.data[i]));
+        medata_array.push_back(
+            static_cast<int64_t>(tid_stop_info.details.exception.data[i]));
       }
-      thread_obj_sp->SetObject("medata", medata_array_sp);
+      thread_obj.try_emplace("medata", std::move(medata_array));
     }
-
-    // TODO: Expedite interesting regions of inferior memory
+    threads_array.push_back(std::move(thread_obj));
   }
-
-  return threads_array_sp;
+  return threads_array;
 }
 
 GDBRemoteCommunication::PacketResult
@@ -657,19 +657,21 @@ GDBRemoteCommunicationServerLLGS::SendStopReplyPacketForThread(
     // is hex ascii JSON that contains the thread IDs thread stop info only for
     // threads that have stop reasons. Only send this if we have more than one
     // thread otherwise this packet has all the info it needs.
-    if (thread_index > 0) {
+    if (thread_index > 1) {
       const bool threads_with_valid_stop_info_only = true;
-      JSONArray::SP threads_info_sp = GetJSONThreadsInfo(
+      llvm::Expected<json::Value> threads_info = GetJSONThreadsInfo(
           *m_debugged_process_up, threads_with_valid_stop_info_only);
-      if (threads_info_sp) {
+      if (threads_info) {
         response.PutCString("jstopinfo:");
         StreamString unescaped_response;
-        threads_info_sp->Write(unescaped_response);
+        unescaped_response.AsRawOstream() << *threads_info;
         response.PutStringAsRawHex8(unescaped_response.GetData());
         response.PutChar(';');
-      } else
-        LLDB_LOG(log, "failed to prepare a jstopinfo field for pid {0}",
-                 m_debugged_process_up->GetID());
+      } else {
+        LLDB_LOG(log, "failed to prepare a jstopinfo field for pid {0}:",
+                 m_debugged_process_up->GetID(),
+                 llvm::toString(threads_info.takeError()));
+      }
     }
 
     uint32_t i = 0;
@@ -1383,7 +1385,8 @@ GDBRemoteCommunicationServerLLGS::Handle_C(StringExtractorGDBRemote &packet) {
           packet, "unexpected content after $C{signal-number}");
   }
 
-  ResumeActionList resume_actions(StateType::eStateRunning, 0);
+  ResumeActionList resume_actions(StateType::eStateRunning,
+                                  LLDB_INVALID_SIGNAL_NUMBER);
   Status error;
 
   // We have two branches: what to do if a continue thread is specified (in
@@ -1453,7 +1456,8 @@ GDBRemoteCommunicationServerLLGS::Handle_c(StringExtractorGDBRemote &packet) {
   }
 
   // Build the ResumeActionList
-  ResumeActionList actions(StateType::eStateRunning, 0);
+  ResumeActionList actions(StateType::eStateRunning,
+                           LLDB_INVALID_SIGNAL_NUMBER);
 
   Status error = m_debugged_process_up->Resume(actions);
   if (error.Fail()) {
@@ -1520,7 +1524,7 @@ GDBRemoteCommunicationServerLLGS::Handle_vCont(
     ResumeAction thread_action;
     thread_action.tid = LLDB_INVALID_THREAD_ID;
     thread_action.state = eStateInvalid;
-    thread_action.signal = 0;
+    thread_action.signal = LLDB_INVALID_SIGNAL_NUMBER;
 
     const char action = packet.GetChar();
     switch (action) {
@@ -2723,7 +2727,7 @@ GDBRemoteCommunicationServerLLGS::Handle_s(StringExtractorGDBRemote &packet) {
     return SendErrorResponse(0x33);
 
   // Create the step action for the given thread.
-  ResumeAction action = {tid, eStateStepping, 0};
+  ResumeAction action = {tid, eStateStepping, LLDB_INVALID_SIGNAL_NUMBER};
 
   // Setup the actions list.
   ResumeActionList actions;
@@ -3079,15 +3083,16 @@ GDBRemoteCommunicationServerLLGS::Handle_jThreadsInfo(
 
   StreamString response;
   const bool threads_with_valid_stop_info_only = false;
-  JSONArray::SP threads_array_sp = GetJSONThreadsInfo(
+  llvm::Expected<json::Value> threads_info = GetJSONThreadsInfo(
       *m_debugged_process_up, threads_with_valid_stop_info_only);
-  if (!threads_array_sp) {
-    LLDB_LOG(log, "failed to prepare a packet for pid {0}",
-             m_debugged_process_up->GetID());
+  if (!threads_info) {
+    LLDB_LOG(log, "failed to prepare a packet for pid {0}: {1}",
+             m_debugged_process_up->GetID(),
+             llvm::toString(threads_info.takeError()));
     return SendErrorResponse(52);
   }
 
-  threads_array_sp->Write(response);
+  response.AsRawOstream() << *threads_info;
   StreamGDBRemote escaped_response;
   escaped_response.PutEscapedBytes(response.GetData(), response.GetSize());
   return SendPacketNoLock(escaped_response.GetString());
@@ -3322,4 +3327,4 @@ std::string GDBRemoteCommunicationServerLLGS::XMLEncodeAttributeValue(
     }
   }
   return result;
-}
\ No newline at end of file
+}
diff --git a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
index 677f348dec209..25cebbba8f7b3 100644
--- a/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/GDBRemoteCommunicationServerPlatform.cpp
@@ -18,6 +18,7 @@
 #include <thread>
 
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/Threading.h"
 
 #include "lldb/Host/Config.h"
@@ -27,9 +28,8 @@
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Target/Platform.h"
 #include "lldb/Target/UnixSignals.h"
-#include "lldb/Utility/JSON.h"
+#include "lldb/Utility/GDBRemote.h"
 #include "lldb/Utility/Log.h"
-#include "lldb/Utility/StreamGDBRemote.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/UriParser.h"
@@ -37,8 +37,8 @@
 #include "lldb/Utility/StringExtractorGDBRemote.h"
 
 using namespace lldb;
-using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
+using namespace lldb_private;
 
 // GDBRemoteCommunicationServerPlatform constructor
 GDBRemoteCommunicationServerPlatform::GDBRemoteCommunicationServerPlatform(
@@ -215,22 +215,21 @@ GDBRemoteCommunicationServerPlatform::Handle_qLaunchGDBServer(
 GDBRemoteCommunication::PacketResult
 GDBRemoteCommunicationServerPlatform::Handle_qQueryGDBServer(
     StringExtractorGDBRemote &packet) {
+  namespace json = llvm::json;
+
   if (m_pending_gdb_server.pid == LLDB_INVALID_PROCESS_ID)
     return SendErrorResponse(4);
 
-  JSONObject::SP server_sp = std::make_shared<JSONObject>();
-  server_sp->SetObject("port",
-                       std::make_shared<JSONNumber>(m_pending_gdb_server.port));
+  json::Object server{{"port", m_pending_gdb_server.port}};
+
   if (!m_pending_gdb_server.socket_name.empty())
-    server_sp->SetObject(
-        "socket_name",
-        std::make_shared<JSONString>(m_pending_gdb_server.socket_name.c_str()));
+    server.try_emplace("socket_name", m_pending_gdb_server.socket_name);
 
-  JSONArray server_list;
-  server_list.AppendObject(server_sp);
+  json::Array server_list;
+  server_list.push_back(std::move(server));
 
   StreamGDBRemote response;
-  server_list.Write(response);
+  response.AsRawOstream() << std::move(server_list);
 
   StreamGDBRemote escaped_response;
   escaped_response.PutEscapedBytes(response.GetString().data(),
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
index d54363cae264d..f1762abc55f82 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp
@@ -86,7 +86,6 @@
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUGSERVER_BASENAME "debugserver"
-using namespace llvm;
 using namespace lldb;
 using namespace lldb_private;
 using namespace lldb_private::process_gdb_remote;
@@ -99,12 +98,14 @@ namespace lldb {
 // namespace. This allows you to attach with a debugger and call this function
 // and get the packet history dumped to a file.
 void DumpProcessGDBRemotePacketHistory(void *p, const char *path) {
-  StreamFile strm;
-  Status error = FileSystem::Instance().Open(strm.GetFile(), FileSpec(path),
-                                             File::eOpenOptionWrite |
-                                                 File::eOpenOptionCanCreate);
-  if (error.Success())
-    ((ProcessGDBRemote *)p)->GetGDBRemote().DumpHistory(strm);
+  auto file = FileSystem::Instance().Open(
+      FileSpec(path), File::eOpenOptionWrite | File::eOpenOptionCanCreate);
+  if (!file) {
+    llvm::consumeError(file.takeError());
+    return;
+  }
+  StreamFile stream(std::move(file.get()));
+  ((ProcessGDBRemote *)p)->GetGDBRemote().DumpHistory(stream);
 }
 } // namespace lldb
 
@@ -5088,7 +5089,7 @@ ParseStructuredDataPacket(llvm::StringRef packet) {
   if (log) {
     if (json_sp) {
       StreamString json_str;
-      json_sp->Dump(json_str);
+      json_sp->Dump(json_str, true);
       json_str.Flush();
       LLDB_LOGF(log,
                 "ProcessGDBRemote::%s() "
diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
index 84004c195660f..0e3e3b39d9c8e 100644
--- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
+++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.h
@@ -24,8 +24,8 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/Broadcaster.h"
 #include "lldb/Utility/ConstString.h"
+#include "lldb/Utility/GDBRemote.h"
 #include "lldb/Utility/Status.h"
-#include "lldb/Utility/StreamGDBRemote.h"
 #include "lldb/Utility/StreamString.h"
 #include "lldb/Utility/StringExtractor.h"
 #include "lldb/Utility/StringList.h"
diff --git a/lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.cpp b/lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.cpp
index 7b95983795530..3517f831970dd 100644
--- a/lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/None/ScriptInterpreterNone.cpp
@@ -28,13 +28,13 @@ ScriptInterpreterNone::~ScriptInterpreterNone() {}
 bool ScriptInterpreterNone::ExecuteOneLine(llvm::StringRef command,
                                            CommandReturnObject *,
                                            const ExecuteScriptOptions &) {
-  m_debugger.GetErrorFile()->PutCString(
+  m_debugger.GetErrorStream().PutCString(
       "error: there is no embedded script interpreter in this mode.\n");
   return false;
 }
 
 void ScriptInterpreterNone::ExecuteInterpreterLoop() {
-  m_debugger.GetErrorFile()->PutCString(
+  m_debugger.GetErrorStream().PutCString(
       "error: there is no embedded script interpreter in this mode.\n");
 }
 
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
index 29dd037efd86b..6d9a74d39cc31 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp
@@ -29,8 +29,8 @@
 using namespace lldb_private;
 using namespace lldb;
 
-void StructuredPythonObject::Dump(Stream &s, bool pretty_print) const {
-  s << "Python Obj: 0x" << GetValue();
+void StructuredPythonObject::Serialize(llvm::json::OStream &s) const {
+  s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str());
 }
 
 // PythonObject
@@ -949,11 +949,6 @@ PythonFile::PythonFile() : PythonObject() {}
 
 PythonFile::PythonFile(File &file, const char *mode) { Reset(file, mode); }
 
-PythonFile::PythonFile(const char *path, const char *mode) {
-  lldb_private::File file;
-  FileSystem::Instance().Open(file, FileSpec(path), GetOptionsFromMode(mode));
-  Reset(file, mode);
-}
 
 PythonFile::PythonFile(PyRefType type, PyObject *o) { Reset(type, o); }
 
@@ -1036,17 +1031,19 @@ uint32_t PythonFile::GetOptionsFromMode(llvm::StringRef mode) {
       .Default(0);
 }
 
-bool PythonFile::GetUnderlyingFile(File &file) const {
+FileUP PythonFile::GetUnderlyingFile() const {
   if (!IsValid())
-    return false;
+    return nullptr;
 
-  file.Close();
   // We don't own the file descriptor returned by this function, make sure the
   // File object knows about that.
-  file.SetDescriptor(PyObject_AsFileDescriptor(m_py_obj), false);
   PythonString py_mode = GetAttributeValue("mode").AsType<PythonString>();
-  file.SetOptions(PythonFile::GetOptionsFromMode(py_mode.GetString()));
-  return file.IsValid();
+  auto options = PythonFile::GetOptionsFromMode(py_mode.GetString());
+  auto file = std::make_unique<File>(PyObject_AsFileDescriptor(m_py_obj),
+                                     options, false);
+  if (!file->IsValid())
+    return nullptr;
+  return file;
 }
 
 #endif
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
index 049ce90bb1b1c..14484d9314828 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.h
@@ -43,7 +43,7 @@ class StructuredPythonObject : public StructuredData::Generic {
 
   bool IsValid() const override { return GetValue() && GetValue() != Py_None; }
 
-  void Dump(Stream &s, bool pretty_print = true) const override;
+  void Serialize(llvm::json::OStream &s) const override;
 
 private:
   DISALLOW_COPY_AND_ASSIGN(StructuredPythonObject);
@@ -455,7 +455,6 @@ class PythonFile : public PythonObject {
 public:
   PythonFile();
   PythonFile(File &file, const char *mode);
-  PythonFile(const char *path, const char *mode);
   PythonFile(PyRefType type, PyObject *o);
 
   ~PythonFile() override;
@@ -469,7 +468,7 @@ class PythonFile : public PythonObject {
 
   static uint32_t GetOptionsFromMode(llvm::StringRef mode);
 
-  bool GetUnderlyingFile(File &file) const;
+  lldb::FileUP GetUnderlyingFile() const;
 };
 
 } // namespace lldb_private
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
index 7fe3157dfc200..4438a9b1dbef5 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPython.cpp
@@ -45,6 +45,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FormatAdapters.h"
 
 #include <memory>
 #include <mutex>
@@ -96,6 +97,7 @@ LLDBSwigPythonCreateCommandObject(const char *python_class_name,
 
 extern "C" void *LLDBSwigPythonCreateScriptedThreadPlan(
     const char *python_class_name, const char *session_dictionary_name,
+    std::string &error_string,
     const lldb::ThreadPlanSP &thread_plan_sp);
 
 extern "C" bool LLDBSWIGPythonCallThreadPlan(void *implementor,
@@ -535,7 +537,7 @@ def function (frame, bp_loc, internal_dict):
   }
 
   if (instructions) {
-    StreamFileSP output_sp(io_handler.GetOutputStreamFile());
+    StreamFileSP output_sp(io_handler.GetOutputStreamFileSP());
     if (output_sp && interactive) {
       output_sp->PutCString(instructions);
       output_sp->Flush();
@@ -571,7 +573,7 @@ void ScriptInterpreterPythonImpl::IOHandlerInputComplete(IOHandler &io_handler,
         bp_options->SetCallback(
             ScriptInterpreterPythonImpl::BreakpointCallbackFunction, baton_sp);
       } else if (!batch_mode) {
-        StreamFileSP error_sp = io_handler.GetErrorStreamFile();
+        StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
         if (error_sp) {
           error_sp->Printf("Warning: No command attached to breakpoint.\n");
           error_sp->Flush();
@@ -593,7 +595,7 @@ void ScriptInterpreterPythonImpl::IOHandlerInputComplete(IOHandler &io_handler,
       wp_options->SetCallback(
           ScriptInterpreterPythonImpl::WatchpointCallbackFunction, baton_sp);
     } else if (!batch_mode) {
-      StreamFileSP error_sp = io_handler.GetErrorStreamFile();
+      StreamFileSP error_sp = io_handler.GetErrorStreamFileSP();
       if (error_sp) {
         error_sp->Printf("Warning: No command attached to breakpoint.\n");
         error_sp->Flush();
@@ -609,13 +611,15 @@ ScriptInterpreterPythonImpl::CreateInstance(Debugger &debugger) {
   return std::make_shared<ScriptInterpreterPythonImpl>(debugger);
 }
 
-void ScriptInterpreterPythonImpl::ResetOutputFileHandle(FILE *fh) {}
-
 void ScriptInterpreterPythonImpl::LeaveSession() {
   Log *log(lldb_private::GetLogIfAllCategoriesSet(LIBLLDB_LOG_SCRIPT));
   if (log)
     log->PutCString("ScriptInterpreterPythonImpl::LeaveSession()");
 
+  // Unset the LLDB global variables.
+  PyRun_SimpleString("lldb.debugger = None; lldb.target = None; lldb.process "
+                     "= None; lldb.thread = None; lldb.frame = None");
+
   // checking that we have a valid thread state - since we use our own
   // threading and locking in some (rare) cases during cleanup Python may end
   // up believing we have no thread state and PyImport_AddModule will crash if
@@ -720,7 +724,7 @@ bool ScriptInterpreterPythonImpl::EnterSession(uint16_t on_entry_flags,
     File out_file(out, false);
     File err_file(err, false);
 
-    lldb::StreamFileSP in_sp;
+    lldb::FileSP in_sp;
     lldb::StreamFileSP out_sp;
     lldb::StreamFileSP err_sp;
     if (!in_file.IsValid() || !out_file.IsValid() || !err_file.IsValid())
@@ -731,7 +735,7 @@ bool ScriptInterpreterPythonImpl::EnterSession(uint16_t on_entry_flags,
     } else {
       if (!SetStdHandle(in_file, "stdin", m_saved_stdin, "r")) {
         if (in_sp)
-          SetStdHandle(in_sp->GetFile(), "stdin", m_saved_stdin, "r");
+          SetStdHandle(*in_sp, "stdin", m_saved_stdin, "r");
       }
     }
 
@@ -850,7 +854,7 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
     // directly down to Python.
     Debugger &debugger = m_debugger;
 
-    StreamFileSP input_file_sp;
+    FileSP input_file_sp;
     StreamFileSP output_file_sp;
     StreamFileSP error_file_sp;
     Communication output_comm(
@@ -858,7 +862,7 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
     bool join_read_thread = false;
     if (options.GetEnableIO()) {
       if (result) {
-        input_file_sp = debugger.GetInputFile();
+        input_file_sp = debugger.GetInputFileSP();
         // Set output to a temporary file so we can forward the results on to
         // the result object
 
@@ -889,9 +893,8 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
               ::setbuf(outfile_handle, nullptr);
 
             result->SetImmediateOutputFile(
-                debugger.GetOutputFile()->GetFile().GetStream());
-            result->SetImmediateErrorFile(
-                debugger.GetErrorFile()->GetFile().GetStream());
+                debugger.GetOutputFile().GetStream());
+            result->SetImmediateErrorFile(debugger.GetErrorFile().GetStream());
           }
         }
       }
@@ -899,20 +902,27 @@ bool ScriptInterpreterPythonImpl::ExecuteOneLine(
         debugger.AdoptTopIOHandlerFilesIfInvalid(input_file_sp, output_file_sp,
                                                  error_file_sp);
     } else {
-      input_file_sp = std::make_shared<StreamFile>();
-      FileSystem::Instance().Open(input_file_sp->GetFile(),
+      auto nullin = FileSystem::Instance().Open(
                                   FileSpec(FileSystem::DEV_NULL),
                                   File::eOpenOptionRead);
-
-      output_file_sp = std::make_shared<StreamFile>();
-      FileSystem::Instance().Open(output_file_sp->GetFile(),
+      auto nullout = FileSystem::Instance().Open(
                                   FileSpec(FileSystem::DEV_NULL),
                                   File::eOpenOptionWrite);
-
-      error_file_sp = output_file_sp;
+      if (!nullin) {
+        result->AppendErrorWithFormatv("failed to open /dev/null: {0}\n",
+                                       llvm::fmt_consume(nullin.takeError()));
+        return false;
+      }
+      if (!nullout) {
+        result->AppendErrorWithFormatv("failed to open /dev/null: {0}\n",
+                                       llvm::fmt_consume(nullout.takeError()));
+        return false;
+      }
+      input_file_sp = std::move(nullin.get());
+      error_file_sp = output_file_sp = std::make_shared<StreamFile>(std::move(nullout.get()));
     }
 
-    FILE *in_file = input_file_sp->GetFile().GetStream();
+    FILE *in_file = input_file_sp->GetStream();
     FILE *out_file = output_file_sp->GetFile().GetStream();
     FILE *err_file = error_file_sp->GetFile().GetStream();
     bool success = false;
@@ -1003,7 +1013,7 @@ void ScriptInterpreterPythonImpl::ExecuteInterpreterLoop() {
   // a running interpreter loop inside the already running Python interpreter
   // loop, so we won't do it.
 
-  if (!debugger.GetInputFile()->GetFile().IsValid())
+  if (!debugger.GetInputFile().IsValid())
     return;
 
   IOHandlerSP io_handler_sp(new IOHandlerPythonInterpreter(debugger, this));
@@ -1835,12 +1845,13 @@ StructuredData::DictionarySP ScriptInterpreterPythonImpl::OSPlugin_CreateThread(
 }
 
 StructuredData::ObjectSP ScriptInterpreterPythonImpl::CreateScriptedThreadPlan(
-    const char *class_name, lldb::ThreadPlanSP thread_plan_sp) {
+    const char *class_name, std::string &error_str, 
+    lldb::ThreadPlanSP thread_plan_sp) {
   if (class_name == nullptr || class_name[0] == '\0')
     return StructuredData::ObjectSP();
 
   if (!thread_plan_sp.get())
-    return StructuredData::ObjectSP();
+    return {};
 
   Debugger &debugger = thread_plan_sp->GetTarget().GetDebugger();
   ScriptInterpreter *script_interpreter = debugger.GetScriptInterpreter();
@@ -1848,17 +1859,18 @@ StructuredData::ObjectSP ScriptInterpreterPythonImpl::CreateScriptedThreadPlan(
       static_cast<ScriptInterpreterPythonImpl *>(script_interpreter);
 
   if (!script_interpreter)
-    return StructuredData::ObjectSP();
+    return {};
 
   void *ret_val;
 
   {
     Locker py_lock(this,
                    Locker::AcquireLock | Locker::InitSession | Locker::NoSTDIN);
-
     ret_val = LLDBSwigPythonCreateScriptedThreadPlan(
         class_name, python_interpreter->m_dictionary_name.c_str(),
-        thread_plan_sp);
+        error_str, thread_plan_sp);
+    if (!ret_val)
+      return {};
   }
 
   return StructuredData::ObjectSP(new StructuredPythonObject(ret_val));
@@ -2212,18 +2224,6 @@ bool ScriptInterpreterPythonImpl::GetScriptedSummary(
   return ret_val;
 }
 
-void ScriptInterpreterPythonImpl::Clear() {
-  // Release any global variables that might have strong references to
-  // LLDB objects when clearing the python script interpreter.
-  Locker locker(this, Locker::AcquireLock, Locker::FreeAcquiredLock);
-
-  // This may be called as part of Py_Finalize.  In that case the modules are
-  // destroyed in random order and we can't guarantee that we can access these.
-  if (Py_IsInitialized())
-    PyRun_SimpleString("lldb.debugger = None; lldb.target = None; lldb.process "
-                       "= None; lldb.thread = None; lldb.frame = None");
-}
-
 bool ScriptInterpreterPythonImpl::BreakpointCallbackFunction(
     void *baton, StoppointCallbackContext *context, user_id_t break_id,
     user_id_t break_loc_id) {
diff --git a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
index f68e2c6eb96b5..9cf32059a5031 100644
--- a/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
+++ b/lldb/source/Plugins/ScriptInterpreter/Python/ScriptInterpreterPythonImpl.h
@@ -78,6 +78,7 @@ class ScriptInterpreterPythonImpl : public ScriptInterpreterPython {
 
   StructuredData::ObjectSP
   CreateScriptedThreadPlan(const char *class_name,
+                           std::string &error_str,
                            lldb::ThreadPlanSP thread_plan) override;
 
   bool ScriptedThreadPlanExplainsStop(StructuredData::ObjectSP implementor_sp,
@@ -188,8 +189,6 @@ class ScriptInterpreterPythonImpl : public ScriptInterpreterPython {
                           const TypeSummaryOptions &options,
                           std::string &retval) override;
 
-  void Clear() override;
-
   bool GetDocumentationForItem(const char *item, std::string &dest) override;
 
   bool GetShortHelpForCommandObject(StructuredData::GenericSP cmd_obj_sp,
@@ -256,8 +255,6 @@ class ScriptInterpreterPythonImpl : public ScriptInterpreterPython {
   void SetWatchpointCommandCallback(WatchpointOptions *wp_options,
                                     const char *oneliner) override;
 
-  void ResetOutputFileHandle(FILE *new_fh) override;
-
   const char *GetDictionaryName() { return m_dictionary_name.c_str(); }
 
   PyThreadState *GetThreadState() { return m_command_thread_state; }
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
index d946ae84a0eb0..e665a3e9c0642 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
@@ -308,22 +308,13 @@ uint32_t SymbolFileBreakpad::FindFunctions(const RegularExpression &regex,
   return sc_list.GetSize();
 }
 
-uint32_t SymbolFileBreakpad::FindTypes(
+void SymbolFileBreakpad::FindTypes(
     ConstString name, const CompilerDeclContext *parent_decl_ctx,
-    bool append, uint32_t max_matches,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return types.GetSize();
-}
+    uint32_t max_matches, llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+    TypeMap &types) {}
 
-size_t SymbolFileBreakpad::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                                     LanguageSet languages, bool append,
-                                     TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return types.GetSize();
-}
+void SymbolFileBreakpad::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                                   LanguageSet languages, TypeMap &types) {}
 
 void SymbolFileBreakpad::AddSymbols(Symtab &symtab) {
   Log *log = GetLogIfAllCategoriesSet(LIBLLDB_LOG_SYMBOLS);
@@ -374,6 +365,20 @@ void SymbolFileBreakpad::AddSymbols(Symtab &symtab) {
   symtab.CalculateSymbolSizes();
 }
 
+llvm::Expected<lldb::addr_t>
+SymbolFileBreakpad::GetParameterStackSize(Symbol &symbol) {
+  ParseUnwindData();
+  if (auto *entry = m_unwind_data->win.FindEntryThatContains(
+          symbol.GetAddress().GetFileAddress())) {
+    auto record = StackWinRecord::parse(
+        *LineIterator(*m_objfile_sp, Record::StackWin, entry->data));
+    assert(record.hasValue());
+    return record->ParameterSize;
+  }
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "Parameter size unknown.");
+}
+
 static llvm::Optional<std::pair<llvm::StringRef, llvm::StringRef>>
 GetRule(llvm::StringRef &unwind_rules) {
   // Unwind rules are of the form
@@ -585,12 +590,19 @@ SymbolFileBreakpad::ParseWinUnwindPlan(const Bookmark &bookmark,
 
   // We assume the first value will be the CFA. It is usually called T0, but
   // clang will use T1, if it needs to realign the stack.
-  if (!postfix::ResolveSymbols(it->second, symbol_resolver)) {
-    LLDB_LOG(log, "Resolving symbols in `{0}` failed.", record->ProgramString);
-    return nullptr;
+  auto *symbol = llvm::dyn_cast<postfix::SymbolNode>(it->second);
+  if (symbol && symbol->GetName() == ".raSearch") {
+    row_sp->GetCFAValue().SetRaSearch(record->LocalSize +
+                                      record->SavedRegisterSize);
+  } else {
+    if (!postfix::ResolveSymbols(it->second, symbol_resolver)) {
+      LLDB_LOG(log, "Resolving symbols in `{0}` failed.",
+               record->ProgramString);
+      return nullptr;
+    }
+    llvm::ArrayRef<uint8_t> saved  = SaveAsDWARF(*it->second);
+    row_sp->GetCFAValue().SetIsDWARFExpression(saved.data(), saved.size());
   }
-  llvm::ArrayRef<uint8_t> saved = SaveAsDWARF(*it->second);
-  row_sp->GetCFAValue().SetIsDWARFExpression(saved.data(), saved.size());
 
   // Replace the node value with InitialValueNode, so that subsequent
   // expressions refer to the CFA value instead of recomputing the whole
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
index dc309094c17a3..8906e4bfa25ba 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
@@ -97,10 +97,8 @@ class SymbolFileBreakpad : public SymbolFile {
                                 lldb::SymbolContextItem resolve_scope,
                                 SymbolContextList &sc_list) override;
 
-  size_t GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
-                  TypeList &type_list) override {
-    return 0;
-  }
+  void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
+                TypeList &type_list) override {}
 
   uint32_t FindFunctions(ConstString name,
                          const CompilerDeclContext *parent_decl_ctx,
@@ -111,14 +109,13 @@ class SymbolFileBreakpad : public SymbolFile {
   uint32_t FindFunctions(const RegularExpression &regex, bool include_inlines,
                          bool append, SymbolContextList &sc_list) override;
 
-  uint32_t FindTypes(ConstString name,
-                     const CompilerDeclContext *parent_decl_ctx, bool append,
-                     uint32_t max_matches,
-                     llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                     TypeMap &types) override;
+  void FindTypes(ConstString name, const CompilerDeclContext *parent_decl_ctx,
+                 uint32_t max_matches,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeMap &types) override;
 
-  size_t FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                   LanguageSet languages, bool append, TypeMap &types) override;
+  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
+                 TypeMap &types) override;
 
   llvm::Expected<TypeSystem &>
   GetTypeSystemForLanguage(lldb::LanguageType language) override {
@@ -135,6 +132,8 @@ class SymbolFileBreakpad : public SymbolFile {
 
   void AddSymbols(Symtab &symtab) override;
 
+  llvm::Expected<lldb::addr_t> GetParameterStackSize(Symbol &symbol) override;
+
   lldb::UnwindPlanSP
   GetUnwindPlan(const Address &address,
                 const RegisterInfoResolver &resolver) override;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
index e7927b31b9c31..4f6885e611071 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParser.h
@@ -48,8 +48,8 @@ class DWARFASTParser {
   virtual lldb_private::CompilerDeclContext
   GetDeclContextContainingUIDFromDWARF(const DWARFDIE &die) = 0;
 
-  virtual std::vector<DWARFDIE>
-  GetDIEForDeclContext(lldb_private::CompilerDeclContext decl_context) = 0;
+  virtual void EnsureAllDIEsInDeclContextHaveBeenParsed(
+      lldb_private::CompilerDeclContext decl_context) = 0;
 
   static llvm::Optional<lldb_private::SymbolFile::ArrayInfo>
   ParseChildArrayInfo(const DWARFDIE &parent_die,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 3c799f920a272..e031618a11874 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -148,22 +148,23 @@ TypeSP DWARFASTParserClang::ParseTypeFromDWO(const DWARFDIE &die, Log *log) {
   die.GetDeclContext(decl_context);
   TypeMap dwo_types;
 
-  // The type in the Clang module must have the same langage as the current CU.
+  // The type in the Clang module must have the same language as the current CU.
   LanguageSet languages;
   languages.Insert(die.GetCU()->GetLanguageType());
-  if (!dwo_module_sp->GetSymbolFile()->FindTypes(decl_context, languages, true,
-                                                 dwo_types)) {
+  dwo_module_sp->GetSymbolFile()->FindTypes(decl_context, languages, dwo_types);
+  if (dwo_types.Empty()) {
     if (!IsClangModuleFwdDecl(die))
       return TypeSP();
 
-    // Since this this type is defined in one of the Clang modules imported by
-    // this symbol file, search all of them.
+    // Since this type is defined in one of the Clang modules imported
+    // by this symbol file, search all of them.
     auto &sym_file = die.GetCU()->GetSymbolFileDWARF();
     for (const auto &name_module : sym_file.getExternalTypeModules()) {
       if (!name_module.second)
         continue;
-      if (name_module.second->GetSymbolFile()->FindTypes(
-              decl_context, languages, true, dwo_types))
+      name_module.second->GetSymbolFile()->FindTypes(decl_context,
+                                                     languages, dwo_types);
+      if (dwo_types.GetSize())
         break;
     }
   }
@@ -690,6 +691,8 @@ TypeSP DWARFASTParserClang::ParseTypeFromDWARF(const SymbolContext &sc,
         type_sp = unique_ast_entry_up->m_type_sp;
         if (type_sp) {
           dwarf->GetDIEToType()[die.GetDIE()] = type_sp.get();
+          LinkDeclContextToDIE(
+              GetCachedClangDeclContextForDIE(unique_ast_entry_up->m_die), die);
           return type_sp;
         }
       }
@@ -2183,15 +2186,16 @@ bool DWARFASTParserClang::CompleteTypeFromDWARF(const DWARFDIE &die,
   return false;
 }
 
-std::vector<DWARFDIE> DWARFASTParserClang::GetDIEForDeclContext(
+void DWARFASTParserClang::EnsureAllDIEsInDeclContextHaveBeenParsed(
     lldb_private::CompilerDeclContext decl_context) {
-  std::vector<DWARFDIE> result;
   auto opaque_decl_ctx =
       (clang::DeclContext *)decl_context.GetOpaqueDeclContext();
   for (auto it = m_decl_ctx_to_die.find(opaque_decl_ctx);
-       it != m_decl_ctx_to_die.end() && it->first == opaque_decl_ctx; it++)
-    result.push_back(it->second);
-  return result;
+       it != m_decl_ctx_to_die.end() && it->first == opaque_decl_ctx;
+       it = m_decl_ctx_to_die.erase(it))
+    for (DWARFDIE decl = it->second.GetFirstChild(); decl;
+         decl = decl.GetSibling())
+      GetClangDeclForDIE(decl);
 }
 
 CompilerDecl DWARFASTParserClang::GetDeclForUIDFromDWARF(const DWARFDIE &die) {
@@ -3276,6 +3280,8 @@ DWARFASTParser::ParseChildArrayInfo(const DWARFDIE &parent_die,
         array_info.element_orders.push_back(num_elements);
       }
     } break;
+    default:
+      break;
     }
   }
   return array_info;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 5b5d83d659325..a8daa10596e57 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -51,8 +51,8 @@ class DWARFASTParserClang : public DWARFASTParser {
   lldb_private::CompilerDecl
   GetDeclForUIDFromDWARF(const DWARFDIE &die) override;
 
-  std::vector<DWARFDIE>
-  GetDIEForDeclContext(lldb_private::CompilerDeclContext decl_context) override;
+  void EnsureAllDIEsInDeclContextHaveBeenParsed(
+      lldb_private::CompilerDeclContext decl_context) override;
 
   lldb_private::CompilerDeclContext
   GetDeclContextForUIDFromDWARF(const DWARFDIE &die) override;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.cpp
index 6128163a29269..741669b05754b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFAbbreviationDeclaration.cpp
@@ -18,7 +18,8 @@
 using namespace lldb_private;
 
 DWARFAbbreviationDeclaration::DWARFAbbreviationDeclaration()
-    : m_code(InvalidCode), m_tag(0), m_has_children(0), m_attributes() {}
+    : m_code(InvalidCode), m_tag(llvm::dwarf::DW_TAG_null), m_has_children(0),
+      m_attributes() {}
 
 DWARFAbbreviationDeclaration::DWARFAbbreviationDeclaration(dw_tag_t tag,
                                                            uint8_t has_children)
@@ -33,7 +34,7 @@ DWARFAbbreviationDeclaration::extract(const DWARFDataExtractor &data,
     return DWARFEnumState::Complete;
 
   m_attributes.clear();
-  m_tag = data.GetULEB128(offset_ptr);
+  m_tag = static_cast<dw_tag_t>(data.GetULEB128(offset_ptr));
   if (m_tag == DW_TAG_null)
     return llvm::make_error<llvm::object::GenericBinaryError>(
         "abbrev decl requires non-null tag.");
@@ -68,7 +69,7 @@ DWARFAbbreviationDeclaration::extract(const DWARFDataExtractor &data,
 }
 
 bool DWARFAbbreviationDeclaration::IsValid() {
-  return m_code != 0 && m_tag != 0;
+  return m_code != 0 && m_tag != llvm::dwarf::DW_TAG_null;
 }
 
 uint32_t
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
index 8b2ae12b4bb2f..033105efdc533 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFBaseDIE.cpp
@@ -30,7 +30,7 @@ dw_tag_t DWARFBaseDIE::Tag() const {
   if (m_die)
     return m_die->Tag();
   else
-    return 0;
+    return llvm::dwarf::DW_TAG_null;
 }
 
 const char *DWARFBaseDIE::GetTagAsCString() const {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
index e5b301d24c0c5..8c0fbeb4b717b 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.cpp
@@ -192,7 +192,7 @@ bool DWARFDebugInfoEntry::Extract(const DWARFDataExtractor &data,
     *offset_ptr = offset;
     return true;
   } else {
-    m_tag = 0;
+    m_tag = llvm::dwarf::DW_TAG_null;
     m_has_children = false;
     return true; // NULL debug tag entry
   }
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
index 1e7b5f27642d3..25c885608d859 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDebugInfoEntry.h
@@ -31,7 +31,7 @@ class DWARFDebugInfoEntry {
 
   DWARFDebugInfoEntry()
       : m_offset(DW_INVALID_OFFSET), m_parent_idx(0), m_sibling_idx(0),
-        m_has_children(false), m_abbr_idx(0), m_tag(0) {}
+        m_has_children(false), m_abbr_idx(0), m_tag(llvm::dwarf::DW_TAG_null) {}
 
   explicit operator bool() const { return m_offset != DW_INVALID_OFFSET; }
   bool operator==(const DWARFDebugInfoEntry &rhs) const;
@@ -178,8 +178,9 @@ class DWARFDebugInfoEntry {
       // a single NULL terminating child.
       m_has_children : 1;
   uint16_t m_abbr_idx;
-  uint16_t m_tag; // A copy of the DW_TAG value so we don't have to go through
-                  // the compile unit abbrev table
+  /// A copy of the DW_TAG value so we don't have to go through the compile
+  /// unit abbrev table
+  dw_tag_t m_tag = llvm::dwarf::DW_TAG_null;
 };
 
 #endif // SymbolFileDWARF_DWARFDebugInfoEntry_h_
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
index d0d70dd5123e1..6501ac27f27d4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.h
@@ -23,7 +23,7 @@
 class DWARFDeclContext {
 public:
   struct Entry {
-    Entry() : tag(0), name(nullptr) {}
+    Entry() : tag(llvm::dwarf::DW_TAG_null), name(nullptr) {}
     Entry(dw_tag_t t, const char *n) : tag(t), name(n) {}
 
     bool NameMatches(const Entry &rhs) const {
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h
index 02dfd734ba131..56d9bc5488770 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/HashedNameToDIE.h
@@ -53,7 +53,7 @@ class DWARFMappedHash {
 
   struct DIEInfo {
     dw_offset_t die_offset = DW_INVALID_OFFSET;
-    dw_tag_t tag = 0;
+    dw_tag_t tag = llvm::dwarf::DW_TAG_null;
 
     /// Any flags for this DIEInfo.
     uint32_t type_flags = 0;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index 08b9434f68e47..1db516387c67d 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -314,6 +314,8 @@ void SymbolFileDWARF::GetTypes(const DWARFDIE &die, dw_offset_t min_die_offset,
       case DW_TAG_ptr_to_member_type:
         add_type = (type_mask & eTypeClassMemberPointer) != 0;
         break;
+      default:
+        break;
       }
 
       if (add_type) {
@@ -333,8 +335,8 @@ void SymbolFileDWARF::GetTypes(const DWARFDIE &die, dw_offset_t min_die_offset,
   }
 }
 
-size_t SymbolFileDWARF::GetTypes(SymbolContextScope *sc_scope,
-                                 TypeClass type_mask, TypeList &type_list)
+void SymbolFileDWARF::GetTypes(SymbolContextScope *sc_scope,
+                               TypeClass type_mask, TypeList &type_list)
 
 {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
@@ -347,8 +349,8 @@ size_t SymbolFileDWARF::GetTypes(SymbolContextScope *sc_scope,
 
   if (comp_unit) {
     dwarf_cu = GetDWARFCompileUnit(comp_unit);
-    if (dwarf_cu == nullptr)
-      return 0;
+    if (!dwarf_cu)
+      return;
     GetTypes(dwarf_cu->DIE(), dwarf_cu->GetOffset(),
              dwarf_cu->GetNextUnitOffset(), type_mask, type_set);
   } else {
@@ -365,16 +367,13 @@ size_t SymbolFileDWARF::GetTypes(SymbolContextScope *sc_scope,
   }
 
   std::set<CompilerType> compiler_type_set;
-  size_t num_types_added = 0;
   for (Type *type : type_set) {
     CompilerType compiler_type = type->GetForwardCompilerType();
     if (compiler_type_set.find(compiler_type) == compiler_type_set.end()) {
       compiler_type_set.insert(compiler_type);
       type_list.Insert(type->shared_from_this());
-      ++num_types_added;
     }
   }
-  return num_types_added;
 }
 
 // Gets the first parent that is a lexical block, function or inlined
@@ -392,6 +391,8 @@ SymbolFileDWARF::GetParentSymbolContextDIE(const DWARFDIE &child_die) {
     case DW_TAG_inlined_subroutine:
     case DW_TAG_lexical_block:
       return die;
+    default:
+      break;
     }
   }
   return DWARFDIE();
@@ -845,6 +846,18 @@ size_t SymbolFileDWARF::ParseFunctions(CompileUnit &comp_unit) {
   return functions_added;
 }
 
+void SymbolFileDWARF::ForEachExternalModule(
+    CompileUnit &comp_unit, llvm::function_ref<void(ModuleSP)> f) {
+  UpdateExternalModuleListIfNeeded();
+
+  for (auto &p : m_external_type_modules) {
+    ModuleSP module = p.second;
+    f(module);
+    for (std::size_t i = 0; i < module->GetNumCompileUnits(); ++i)
+      module->GetCompileUnitAtIndex(i)->ForEachExternalModule(f);
+  }
+}
+
 bool SymbolFileDWARF::ParseSupportFiles(CompileUnit &comp_unit,
                                         FileSpecList &support_files) {
   if (!comp_unit.GetLineTable())
@@ -1205,16 +1218,9 @@ bool SymbolFileDWARF::ClassOrStructIsVirtual(const DWARFDIE &parent_die) {
 
 void SymbolFileDWARF::ParseDeclsForContext(CompilerDeclContext decl_ctx) {
   auto *type_system = decl_ctx.GetTypeSystem();
-  if (!type_system)
-    return;
-  DWARFASTParser *ast_parser = type_system->GetDWARFParser();
-  std::vector<DWARFDIE> decl_ctx_die_list =
-      ast_parser->GetDIEForDeclContext(decl_ctx);
-
-  for (DWARFDIE decl_ctx_die : decl_ctx_die_list)
-    for (DWARFDIE decl = decl_ctx_die.GetFirstChild(); decl;
-         decl = decl.GetSibling())
-      ast_parser->GetDeclForUIDFromDWARF(decl);
+  if (type_system != nullptr)
+    type_system->GetDWARFParser()->EnsureAllDIEsInDeclContextHaveBeenParsed(
+        decl_ctx);
 }
 
 user_id_t SymbolFileDWARF::GetUID(DIERef ref) {
@@ -2374,25 +2380,21 @@ void SymbolFileDWARF::GetMangledNamesForFunction(
   }
 }
 
-uint32_t SymbolFileDWARF::FindTypes(
-    ConstString name, const CompilerDeclContext *parent_decl_ctx, bool append,
+void SymbolFileDWARF::FindTypes(
+    ConstString name, const CompilerDeclContext *parent_decl_ctx,
     uint32_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     TypeMap &types) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  // If we aren't appending the results to this list, then clear the list
-  if (!append)
-    types.Clear();
-
   // Make sure we haven't already searched this SymbolFile before...
   if (searched_symbol_files.count(this))
-    return 0;
-  else
-    searched_symbol_files.insert(this);
+    return;
+
+  searched_symbol_files.insert(this);
 
   DWARFDebugInfo *info = DebugInfo();
-  if (info == nullptr)
-    return 0;
+  if (!info)
+    return;
 
   Log *log(LogChannelDWARF::GetLogIfAll(DWARF_LOG_LOOKUPS));
 
@@ -2401,63 +2403,41 @@ uint32_t SymbolFileDWARF::FindTypes(
       GetObjectFile()->GetModule()->LogMessage(
           log,
           "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx = "
-          "%p (\"%s\"), append=%u, max_matches=%u, type_list)",
+          "%p (\"%s\"), max_matches=%u, type_list)",
           name.GetCString(), static_cast<const void *>(parent_decl_ctx),
-          parent_decl_ctx->GetName().AsCString("<NULL>"), append, max_matches);
+          parent_decl_ctx->GetName().AsCString("<NULL>"), max_matches);
     else
       GetObjectFile()->GetModule()->LogMessage(
           log,
           "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx = "
-          "NULL, append=%u, max_matches=%u, type_list)",
-          name.GetCString(), append, max_matches);
+          "NULL, max_matches=%u, type_list)",
+          name.GetCString(), max_matches);
   }
 
   if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx))
-    return 0;
+    return;
 
   DIEArray die_offsets;
   m_index->GetTypes(name, die_offsets);
   const size_t num_die_matches = die_offsets.size();
 
-  if (num_die_matches) {
-    const uint32_t initial_types_size = types.GetSize();
-    for (size_t i = 0; i < num_die_matches; ++i) {
-      const DIERef &die_ref = die_offsets[i];
-      DWARFDIE die = GetDIE(die_ref);
+  for (size_t i = 0; i < num_die_matches; ++i) {
+    const DIERef &die_ref = die_offsets[i];
+    DWARFDIE die = GetDIE(die_ref);
+    if (die) {
+      if (!DIEInDeclContext(parent_decl_ctx, die))
+        continue; // The containing decl contexts don't match
 
-      if (die) {
-        if (!DIEInDeclContext(parent_decl_ctx, die))
-          continue; // The containing decl contexts don't match
-
-        Type *matching_type = ResolveType(die, true, true);
-        if (matching_type) {
-          // We found a type pointer, now find the shared pointer form our type
-          // list
-          types.InsertUnique(matching_type->shared_from_this());
-          if (types.GetSize() >= max_matches)
-            break;
-        }
-      } else {
-        m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
-      }
-    }
-    const uint32_t num_matches = types.GetSize() - initial_types_size;
-    if (log && num_matches) {
-      if (parent_decl_ctx) {
-        GetObjectFile()->GetModule()->LogMessage(
-            log,
-            "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx "
-            "= %p (\"%s\"), append=%u, max_matches=%u, type_list) => %u",
-            name.GetCString(), static_cast<const void *>(parent_decl_ctx),
-            parent_decl_ctx->GetName().AsCString("<NULL>"), append, max_matches,
-            num_matches);
-      } else {
-        GetObjectFile()->GetModule()->LogMessage(
-            log,
-            "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx "
-            "= NULL, append=%u, max_matches=%u, type_list) => %u",
-            name.GetCString(), append, max_matches, num_matches);
+      Type *matching_type = ResolveType(die, true, true);
+      if (matching_type) {
+        // We found a type pointer, now find the shared pointer form our type
+        // list
+        types.InsertUnique(matching_type->shared_from_this());
+        if (types.GetSize() >= max_matches)
+          break;
       }
+    } else {
+      m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
     }
   }
 
@@ -2467,69 +2447,68 @@ uint32_t SymbolFileDWARF::FindTypes(
   if (num_die_matches < max_matches) {
     UpdateExternalModuleListIfNeeded();
 
-    for (const auto &pair : m_external_type_modules) {
-      ModuleSP external_module_sp = pair.second;
-      if (external_module_sp) {
-        SymbolFile *sym_file = external_module_sp->GetSymbolFile();
-        if (sym_file) {
-          const uint32_t num_external_matches =
-              sym_file->FindTypes(name, parent_decl_ctx, append, max_matches,
-                                  searched_symbol_files, types);
-          if (num_external_matches)
-            return num_external_matches;
-        }
-      }
-    }
+    for (const auto &pair : m_external_type_modules)
+      if (ModuleSP external_module_sp = pair.second) 
+        if (SymbolFile *sym_file = external_module_sp->GetSymbolFile())
+          sym_file->FindTypes(name, parent_decl_ctx, max_matches,
+                              searched_symbol_files, types);
   }
 
-  return num_die_matches;
+  if (log && types.GetSize()) {
+    if (parent_decl_ctx) {
+      GetObjectFile()->GetModule()->LogMessage(
+          log,
+          "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx "
+          "= %p (\"%s\"), max_matches=%u, type_list) => %u",
+          name.GetCString(), static_cast<const void *>(parent_decl_ctx),
+          parent_decl_ctx->GetName().AsCString("<NULL>"), max_matches,
+          types.GetSize());
+    } else {
+      GetObjectFile()->GetModule()->LogMessage(
+          log,
+          "SymbolFileDWARF::FindTypes (sc, name=\"%s\", parent_decl_ctx "
+          "= NULL, max_matches=%u, type_list) => %u",
+          name.GetCString(), max_matches, types.GetSize());
+    }
+  }
 }
 
-size_t SymbolFileDWARF::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                                  LanguageSet languages, bool append,
-                                  TypeMap &types) {
+void SymbolFileDWARF::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                                  LanguageSet languages, TypeMap &types) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!append)
-    types.Clear();
-
   if (pattern.empty())
-    return 0;
+    return;
 
   ConstString name = pattern.back().name;
 
   if (!name)
-    return 0;
+    return;
 
   DIEArray die_offsets;
   m_index->GetTypes(name, die_offsets);
   const size_t num_die_matches = die_offsets.size();
 
-  size_t num_matches = 0;
   for (size_t i = 0; i < num_die_matches; ++i) {
     const DIERef &die_ref = die_offsets[i];
     DWARFDIE die = GetDIE(die_ref);
 
-    if (die) {
-      if (!languages[die.GetCU()->GetLanguageType()])
-        continue;
-
-      llvm::SmallVector<CompilerContext, 4> die_context;
-      die.GetDeclContext(die_context);
-      if (!contextMatches(die_context, pattern))
-        continue;
-
-      Type *matching_type = ResolveType(die, true, true);
-      if (matching_type) {
-        // We found a type pointer, now find the shared pointer form our type
-        // list
-        types.InsertUnique(matching_type->shared_from_this());
-        ++num_matches;
-      }
-    } else {
+    if (!die) {
       m_index->ReportInvalidDIERef(die_ref, name.GetStringRef());
+      continue;
     }
+    if (!languages[die.GetCU()->GetLanguageType()])
+      continue;
+
+    llvm::SmallVector<CompilerContext, 4> die_context;
+    die.GetDeclContext(die_context);
+    if (!contextMatches(die_context, pattern))
+      continue;
+
+    if (Type *matching_type = ResolveType(die, true, true))
+      // We found a type pointer, now find the shared pointer form our type
+      // list.
+      types.InsertUnique(matching_type->shared_from_this());
   }
-  return num_matches;
 }
 
 CompilerDeclContext
@@ -3578,6 +3557,8 @@ SymbolFileDWARF::FindBlockContainingSpecification(
           spec_block_die_offset)
         return die;
     } break;
+    default:
+      break;
     }
 
     // Give the concrete function die specified by "func_die_offset", find the
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index 2a1eb8ab9abeb..4f04d416eacca 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -105,6 +105,10 @@ class SymbolFileDWARF : public lldb_private::SymbolFile,
 
   bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override;
 
+  void
+  ForEachExternalModule(lldb_private::CompileUnit &comp_unit,
+                        llvm::function_ref<void(lldb::ModuleSP)> f) override;
+
   bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit,
                          lldb_private::FileSpecList &support_files) override;
 
@@ -177,20 +181,20 @@ class SymbolFileDWARF : public lldb_private::SymbolFile,
       const std::string &scope_qualified_name,
       std::vector<lldb_private::ConstString> &mangled_names) override;
 
-  uint32_t
+  void
   FindTypes(lldb_private::ConstString name,
             const lldb_private::CompilerDeclContext *parent_decl_ctx,
-            bool append, uint32_t max_matches,
+            uint32_t max_matches,
             llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
             lldb_private::TypeMap &types) override;
 
-  size_t FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
-                   lldb_private::LanguageSet languages, bool append,
-                   lldb_private::TypeMap &types) override;
+  void FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
+                 lldb_private::LanguageSet languages,
+                 lldb_private::TypeMap &types) override;
 
-  size_t GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                  lldb::TypeClass type_mask,
-                  lldb_private::TypeList &type_list) override;
+  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                lldb::TypeClass type_mask,
+                lldb_private::TypeList &type_list) override;
 
   llvm::Expected<lldb_private::TypeSystem &>
   GetTypeSystemForLanguage(lldb::LanguageType language) override;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
index 77e2fcf887c46..c286dc90e5d6f 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
@@ -652,6 +652,14 @@ bool SymbolFileDWARFDebugMap::ParseDebugMacros(CompileUnit &comp_unit) {
   return false;
 }
 
+void SymbolFileDWARFDebugMap::ForEachExternalModule(
+    CompileUnit &comp_unit, llvm::function_ref<void(ModuleSP)> f) {
+  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
+  SymbolFileDWARF *oso_dwarf = GetSymbolFile(comp_unit);
+  if (oso_dwarf)
+    oso_dwarf->ForEachExternalModule(comp_unit, f);
+}
+
 bool SymbolFileDWARFDebugMap::ParseSupportFiles(CompileUnit &comp_unit,
                                                 FileSpecList &support_files) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
@@ -1062,16 +1070,15 @@ uint32_t SymbolFileDWARFDebugMap::FindFunctions(const RegularExpression &regex,
   return sc_list.GetSize() - initial_size;
 }
 
-size_t SymbolFileDWARFDebugMap::GetTypes(SymbolContextScope *sc_scope,
-                                         lldb::TypeClass type_mask,
-                                         TypeList &type_list) {
+void SymbolFileDWARFDebugMap::GetTypes(SymbolContextScope *sc_scope,
+                                       lldb::TypeClass type_mask,
+                                       TypeList &type_list) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   static Timer::Category func_cat(LLVM_PRETTY_FUNCTION);
   Timer scoped_timer(func_cat,
                      "SymbolFileDWARFDebugMap::GetTypes (type_mask = 0x%8.8x)",
                      type_mask);
 
-  uint32_t initial_size = type_list.GetSize();
   SymbolFileDWARF *oso_dwarf = nullptr;
   if (sc_scope) {
     SymbolContext sc;
@@ -1089,7 +1096,6 @@ size_t SymbolFileDWARFDebugMap::GetTypes(SymbolContextScope *sc_scope,
       return false;
     });
   }
-  return type_list.GetSize() - initial_size;
 }
 
 std::vector<lldb_private::CallEdge>
@@ -1191,24 +1197,17 @@ TypeSP SymbolFileDWARFDebugMap::FindCompleteObjCDefinitionTypeForDIE(
   return TypeSP();
 }
 
-uint32_t SymbolFileDWARFDebugMap::FindTypes(
+void SymbolFileDWARFDebugMap::FindTypes(
     ConstString name, const CompilerDeclContext *parent_decl_ctx,
-    bool append, uint32_t max_matches,
+    uint32_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     TypeMap &types) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!append)
-    types.Clear();
-
-  const uint32_t initial_types_size = types.GetSize();
-
   ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool {
-    oso_dwarf->FindTypes(name, parent_decl_ctx, append, max_matches,
+    oso_dwarf->FindTypes(name, parent_decl_ctx, max_matches,
                          searched_symbol_files, types);
     return types.GetSize() >= max_matches;
   });
-
-  return types.GetSize() - initial_types_size;
 }
 
 //
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index 0b4f96b203645..0643e6d24e6a7 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -53,6 +53,10 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFile {
 
   bool ParseDebugMacros(lldb_private::CompileUnit &comp_unit) override;
 
+  void
+  ForEachExternalModule(lldb_private::CompileUnit &comp_unit,
+                        llvm::function_ref<void(lldb::ModuleSP)> f) override;
+
   bool ParseSupportFiles(lldb_private::CompileUnit &comp_unit,
                          lldb_private::FileSpecList &support_files) override;
 
@@ -104,18 +108,18 @@ class SymbolFileDWARFDebugMap : public lldb_private::SymbolFile {
   uint32_t FindFunctions(const lldb_private::RegularExpression &regex,
                          bool include_inlines, bool append,
                          lldb_private::SymbolContextList &sc_list) override;
-  uint32_t
+  void
   FindTypes(lldb_private::ConstString name,
             const lldb_private::CompilerDeclContext *parent_decl_ctx,
-            bool append, uint32_t max_matches,
+            uint32_t max_matches,
             llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
             lldb_private::TypeMap &types) override;
   lldb_private::CompilerDeclContext FindNamespace(
       lldb_private::ConstString name,
       const lldb_private::CompilerDeclContext *parent_decl_ctx) override;
-  size_t GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                  lldb::TypeClass type_mask,
-                  lldb_private::TypeList &type_list) override;
+  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                lldb::TypeClass type_mask,
+                lldb_private::TypeList &type_list) override;
   std::vector<lldb_private::CallEdge>
   ParseCallEdgesInFunction(lldb_private::UserID func_id) override;
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
index 8da7e22262664..4862fea8d0798 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/UniqueDWARFASTType.cpp
@@ -55,6 +55,8 @@ bool UniqueDWARFASTTypeList::Find(const DWARFDIE &die,
               case DW_TAG_partial_unit:
                 done = true;
                 break;
+              default:
+                break;
               }
             }
             parent_arg_die = parent_arg_die.GetParent();
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index 05e48787cbebc..ceaa877459373 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -1250,38 +1250,28 @@ uint32_t SymbolFileNativePDB::FindFunctions(const RegularExpression &regex,
   return 0;
 }
 
-uint32_t SymbolFileNativePDB::FindTypes(
+void SymbolFileNativePDB::FindTypes(
     ConstString name, const CompilerDeclContext *parent_decl_ctx,
-    bool append, uint32_t max_matches,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {
+    uint32_t max_matches, llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+    TypeMap &types) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!append)
-    types.Clear();
   if (!name)
-    return 0;
+    return;
 
   searched_symbol_files.clear();
   searched_symbol_files.insert(this);
 
   // There is an assumption 'name' is not a regex
-  size_t match_count = FindTypesByName(name.GetStringRef(), max_matches, types);
-
-  return match_count;
+  FindTypesByName(name.GetStringRef(), max_matches, types);
 }
 
-size_t SymbolFileNativePDB::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                                      LanguageSet languages, bool append,
-                                      TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return 0;
-}
+void SymbolFileNativePDB::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                                    LanguageSet languages, TypeMap &types) {}
 
-size_t SymbolFileNativePDB::FindTypesByName(llvm::StringRef name,
-                                            uint32_t max_matches,
-                                            TypeMap &types) {
+void SymbolFileNativePDB::FindTypesByName(llvm::StringRef name,
+                                          uint32_t max_matches,
+                                          TypeMap &types) {
 
-  size_t match_count = 0;
   std::vector<TypeIndex> matches = m_index->tpi().findRecordsByName(name);
   if (max_matches > 0 && max_matches < matches.size())
     matches.resize(max_matches);
@@ -1292,9 +1282,7 @@ size_t SymbolFileNativePDB::FindTypesByName(llvm::StringRef name,
       continue;
 
     types.Insert(type);
-    ++match_count;
   }
-  return match_count;
 }
 
 size_t SymbolFileNativePDB::ParseTypes(CompileUnit &comp_unit) {
@@ -1583,11 +1571,9 @@ bool SymbolFileNativePDB::CompleteType(CompilerType &compiler_type) {
   return m_ast->CompleteType(qt);
 }
 
-size_t SymbolFileNativePDB::GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                                     TypeClass type_mask,
-                                     lldb_private::TypeList &type_list) {
-  return 0;
-}
+void SymbolFileNativePDB::GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                                   TypeClass type_mask,
+                                   lldb_private::TypeList &type_list) {}
 
 CompilerDeclContext
 SymbolFileNativePDB::FindNamespace(ConstString name,
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
index f2e840cdfc115..d8634b28ccfb9 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
@@ -116,8 +116,8 @@ class SymbolFileNativePDB : public SymbolFile {
                                 lldb::SymbolContextItem resolve_scope,
                                 SymbolContextList &sc_list) override;
 
-  size_t GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
-                  TypeList &type_list) override;
+  void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
+                TypeList &type_list) override;
 
   uint32_t FindFunctions(ConstString name,
                          const CompilerDeclContext *parent_decl_ctx,
@@ -128,14 +128,13 @@ class SymbolFileNativePDB : public SymbolFile {
   uint32_t FindFunctions(const RegularExpression &regex, bool include_inlines,
                          bool append, SymbolContextList &sc_list) override;
 
-  uint32_t FindTypes(ConstString name,
-                     const CompilerDeclContext *parent_decl_ctx, bool append,
-                     uint32_t max_matches,
-                     llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                     TypeMap &types) override;
+  void FindTypes(ConstString name, const CompilerDeclContext *parent_decl_ctx,
+                 uint32_t max_matches,
+                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
+                 TypeMap &types) override;
 
-  size_t FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                   LanguageSet languages, bool append, TypeMap &types) override;
+  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
+                 TypeMap &types) override;
 
   llvm::Expected<TypeSystem &>
   GetTypeSystemForLanguage(lldb::LanguageType language) override;
@@ -158,8 +157,8 @@ class SymbolFileNativePDB : public SymbolFile {
 
   lldb::CompUnitSP ParseCompileUnitAtIndex(uint32_t index) override;
 
-  size_t FindTypesByName(llvm::StringRef name, uint32_t max_matches,
-                         TypeMap &types);
+  void FindTypesByName(llvm::StringRef name, uint32_t max_matches,
+                       TypeMap &types);
 
   lldb::TypeSP CreateModifierType(PdbTypeSymId type_id,
                                   const llvm::codeview::ModifierRecord &mr,
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index efd3b0cc94018..80eb1c7603077 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -1441,27 +1441,23 @@ void SymbolFilePDB::AddSymbols(lldb_private::Symtab &symtab) {
   symtab.Finalize();
 }
 
-uint32_t SymbolFilePDB::FindTypes(
+void SymbolFilePDB::FindTypes(
     lldb_private::ConstString name,
-    const lldb_private::CompilerDeclContext *parent_decl_ctx, bool append,
+    const lldb_private::CompilerDeclContext *parent_decl_ctx,
     uint32_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
     lldb_private::TypeMap &types) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!append)
-    types.Clear();
   if (!name)
-    return 0;
+    return;
   if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx))
-    return 0;
+    return;
 
   searched_symbol_files.clear();
   searched_symbol_files.insert(this);
 
   // There is an assumption 'name' is not a regex
   FindTypesByName(name.GetStringRef(), parent_decl_ctx, max_matches, types);
-
-  return types.GetSize();
 }
 
 void SymbolFilePDB::DumpClangAST(Stream &s) {
@@ -1584,13 +1580,9 @@ void SymbolFilePDB::FindTypesByName(
   }
 }
 
-size_t SymbolFilePDB::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                                LanguageSet languages, bool append,
-                                lldb_private::TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return 0;
-}
+void SymbolFilePDB::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                              LanguageSet languages,
+                              lldb_private::TypeMap &types) {}
 
 void SymbolFilePDB::GetTypesForPDBSymbol(const llvm::pdb::PDBSymbol &pdb_symbol,
                                          uint32_t type_mask,
@@ -1642,18 +1634,17 @@ void SymbolFilePDB::GetTypesForPDBSymbol(const llvm::pdb::PDBSymbol &pdb_symbol,
     GetTypesForPDBSymbol(*symbol_up, type_mask, type_collection);
 }
 
-size_t SymbolFilePDB::GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                               TypeClass type_mask,
-                               lldb_private::TypeList &type_list) {
+void SymbolFilePDB::GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                             TypeClass type_mask,
+                             lldb_private::TypeList &type_list) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   TypeCollection type_collection;
-  uint32_t old_size = type_list.GetSize();
   CompileUnit *cu =
       sc_scope ? sc_scope->CalculateSymbolContextCompileUnit() : nullptr;
   if (cu) {
     auto compiland_up = GetPDBCompilandByUID(cu->GetID());
     if (!compiland_up)
-      return 0;
+      return;
     GetTypesForPDBSymbol(*compiland_up, type_mask, type_collection);
   } else {
     for (uint32_t cu_idx = 0; cu_idx < GetNumCompileUnits(); ++cu_idx) {
@@ -1669,7 +1660,6 @@ size_t SymbolFilePDB::GetTypes(lldb_private::SymbolContextScope *sc_scope,
     type->GetForwardCompilerType();
     type_list.Insert(type->shared_from_this());
   }
-  return type_list.GetSize() - old_size;
 }
 
 llvm::Expected<lldb_private::TypeSystem &>
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
index a360a18ae262f..1fdacf0c8257d 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
@@ -125,23 +125,23 @@ class SymbolFilePDB : public lldb_private::SymbolFile {
 
   void AddSymbols(lldb_private::Symtab &symtab) override;
 
-  uint32_t
+  void
   FindTypes(lldb_private::ConstString name,
             const lldb_private::CompilerDeclContext *parent_decl_ctx,
-            bool append, uint32_t max_matches,
+            uint32_t max_matches,
             llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
             lldb_private::TypeMap &types) override;
 
-  size_t FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
-                   lldb_private::LanguageSet languages, bool append,
-                   lldb_private::TypeMap &types) override;
+  void FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
+                 lldb_private::LanguageSet languages,
+                 lldb_private::TypeMap &types) override;
 
   void FindTypesByRegex(const lldb_private::RegularExpression &regex,
                         uint32_t max_matches, lldb_private::TypeMap &types);
 
-  size_t GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                  lldb::TypeClass type_mask,
-                  lldb_private::TypeList &type_list) override;
+  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                lldb::TypeClass type_mask,
+                lldb_private::TypeList &type_list) override;
 
   llvm::Expected<lldb_private::TypeSystem &>
   GetTypeSystemForLanguage(lldb::LanguageType language) override;
diff --git a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp
index 91f49e8babb6c..62da76581c3ec 100644
--- a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp
+++ b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.cpp
@@ -47,11 +47,9 @@ SymbolFile *SymbolFileSymtab::CreateInstance(ObjectFileSP objfile_sp) {
   return new SymbolFileSymtab(std::move(objfile_sp));
 }
 
-size_t SymbolFileSymtab::GetTypes(SymbolContextScope *sc_scope,
-                                  TypeClass type_mask,
-                                  lldb_private::TypeList &type_list) {
-  return 0;
-}
+void SymbolFileSymtab::GetTypes(SymbolContextScope *sc_scope,
+                                TypeClass type_mask,
+                                lldb_private::TypeList &type_list) {}
 
 SymbolFileSymtab::SymbolFileSymtab(ObjectFileSP objfile_sp)
     : SymbolFile(std::move(objfile_sp)), m_source_indexes(), m_func_indexes(),
diff --git a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h
index 147290031b2d9..2ac4660f01250 100644
--- a/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h
+++ b/lldb/source/Plugins/SymbolFile/Symtab/SymbolFileSymtab.h
@@ -71,9 +71,9 @@ class SymbolFileSymtab : public lldb_private::SymbolFile {
                                 lldb::SymbolContextItem resolve_scope,
                                 lldb_private::SymbolContext &sc) override;
 
-  size_t GetTypes(lldb_private::SymbolContextScope *sc_scope,
-                  lldb::TypeClass type_mask,
-                  lldb_private::TypeList &type_list) override;
+  void GetTypes(lldb_private::SymbolContextScope *sc_scope,
+                lldb::TypeClass type_mask,
+                lldb_private::TypeList &type_list) override;
 
   // PluginInterface protocol
   lldb_private::ConstString GetPluginName() override;
diff --git a/lldb/source/Symbol/ClangASTContext.cpp b/lldb/source/Symbol/ClangASTContext.cpp
index 7a2f81a07248d..30d8b1dd038c6 100644
--- a/lldb/source/Symbol/ClangASTContext.cpp
+++ b/lldb/source/Symbol/ClangASTContext.cpp
@@ -103,8 +103,8 @@
 
 using namespace lldb;
 using namespace lldb_private;
-using namespace llvm;
 using namespace clang;
+using llvm::StringSwitch;
 
 namespace {
 #ifdef LLDB_CONFIGURATION_DEBUG
@@ -522,15 +522,29 @@ static void ParseLangArgs(LangOptions &Opts, InputKind IK, const char *triple) {
   Opts.NoInlineDefine = !Opt;
 }
 
-ClangASTContext::ClangASTContext(const char *target_triple)
-    : TypeSystem(TypeSystem::eKindClang), m_target_triple(), m_ast_up(),
-      m_language_options_up(), m_source_manager_up(), m_diagnostics_engine_up(),
-      m_target_options_rp(), m_target_info_up(), m_identifier_table_up(),
-      m_selector_table_up(), m_builtins_up(), m_callback_tag_decl(nullptr),
-      m_callback_objc_decl(nullptr), m_callback_baton(nullptr),
-      m_pointer_byte_size(0), m_ast_owned(false) {
-  if (target_triple && target_triple[0])
+ClangASTContext::ClangASTContext(llvm::StringRef target_triple)
+    : TypeSystem(TypeSystem::eKindClang) {
+  if (!target_triple.empty())
     SetTargetTriple(target_triple);
+  // The caller didn't pass an ASTContext so create a new one for this
+  // ClangASTContext.
+  CreateASTContext();
+}
+
+ClangASTContext::ClangASTContext(ArchSpec arch)
+    : TypeSystem(TypeSystem::eKindClang) {
+  SetTargetTriple(arch.GetTriple().str());
+  // The caller didn't pass an ASTContext so create a new one for this
+  // ClangASTContext.
+  CreateASTContext();
+}
+
+ClangASTContext::ClangASTContext(ASTContext &existing_ctxt)
+  : TypeSystem(TypeSystem::eKindClang) {
+  SetTargetTriple(existing_ctxt.getTargetInfo().getTriple().str());
+
+  m_ast_up.reset(&existing_ctxt);
+  GetASTMap().Insert(&existing_ctxt, this);
 }
 
 // Destructor
@@ -572,26 +586,21 @@ lldb::TypeSystemSP ClangASTContext::CreateInstance(lldb::LanguageType language,
       }
 
       if (module) {
-        std::shared_ptr<ClangASTContext> ast_sp(new ClangASTContext);
-        if (ast_sp) {
-          ast_sp->SetArchitecture(fixed_arch);
-        }
+        std::shared_ptr<ClangASTContext> ast_sp(
+            new ClangASTContext(fixed_arch));
         return ast_sp;
       } else if (target && target->IsValid()) {
         std::shared_ptr<ClangASTContextForExpressions> ast_sp(
-            new ClangASTContextForExpressions(*target));
-        if (ast_sp) {
-          ast_sp->SetArchitecture(fixed_arch);
-          ast_sp->m_scratch_ast_source_up.reset(
-              new ClangASTSource(target->shared_from_this()));
-          lldbassert(ast_sp->getFileManager());
-          ast_sp->m_scratch_ast_source_up->InstallASTContext(
-              *ast_sp->getASTContext(), *ast_sp->getFileManager(), true);
-          llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> proxy_ast_source(
-              ast_sp->m_scratch_ast_source_up->CreateProxy());
-          ast_sp->SetExternalSource(proxy_ast_source);
-          return ast_sp;
-        }
+            new ClangASTContextForExpressions(*target, fixed_arch));
+        ast_sp->m_scratch_ast_source_up.reset(
+            new ClangASTSource(target->shared_from_this()));
+        lldbassert(ast_sp->getFileManager());
+        ast_sp->m_scratch_ast_source_up->InstallASTContext(
+            *ast_sp->getASTContext(), *ast_sp->getFileManager(), true);
+        llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> proxy_ast_source(
+            ast_sp->m_scratch_ast_source_up->CreateProxy());
+        ast_sp->SetExternalSource(proxy_ast_source);
+        return ast_sp;
       }
     }
   }
@@ -635,11 +644,10 @@ void ClangASTContext::Terminate() {
 }
 
 void ClangASTContext::Finalize() {
-  if (m_ast_up) {
-    GetASTMap().Erase(m_ast_up.get());
-    if (!m_ast_owned)
-      m_ast_up.release();
-  }
+  assert(m_ast_up);
+  GetASTMap().Erase(m_ast_up.get());
+  if (!m_ast_owned)
+    m_ast_up.release();
 
   m_builtins_up.reset();
   m_selector_table_up.reset();
@@ -649,23 +657,9 @@ void ClangASTContext::Finalize() {
   m_diagnostics_engine_up.reset();
   m_source_manager_up.reset();
   m_language_options_up.reset();
-  m_ast_up.reset();
   m_scratch_ast_source_up.reset();
 }
 
-void ClangASTContext::Clear() {
-  m_ast_up.reset();
-  m_language_options_up.reset();
-  m_source_manager_up.reset();
-  m_diagnostics_engine_up.reset();
-  m_target_options_rp.reset();
-  m_target_info_up.reset();
-  m_identifier_table_up.reset();
-  m_selector_table_up.reset();
-  m_builtins_up.reset();
-  m_pointer_byte_size = 0;
-}
-
 void ClangASTContext::setSema(Sema *s) {
   // Ensure that the new sema actually belongs to our ASTContext.
   assert(s == nullptr || &s->getASTContext() == m_ast_up.get());
@@ -676,20 +670,8 @@ const char *ClangASTContext::GetTargetTriple() {
   return m_target_triple.c_str();
 }
 
-void ClangASTContext::SetTargetTriple(const char *target_triple) {
-  Clear();
-  m_target_triple.assign(target_triple);
-}
-
-void ClangASTContext::SetArchitecture(const ArchSpec &arch) {
-  SetTargetTriple(arch.GetTriple().str().c_str());
-}
-
-bool ClangASTContext::HasExternalSource() {
-  ASTContext *ast = getASTContext();
-  if (ast)
-    return ast->getExternalSource() != nullptr;
-  return false;
+void ClangASTContext::SetTargetTriple(llvm::StringRef target_triple) {
+  m_target_triple = target_triple.str();
 }
 
 void ClangASTContext::SetExternalSource(
@@ -701,56 +683,40 @@ void ClangASTContext::SetExternalSource(
   }
 }
 
-void ClangASTContext::RemoveExternalSource() {
-  ASTContext *ast = getASTContext();
-
-  if (ast) {
-    llvm::IntrusiveRefCntPtr<ExternalASTSource> empty_ast_source_up;
-    ast->setExternalSource(empty_ast_source_up);
-    ast->getTranslationUnitDecl()->setHasExternalLexicalStorage(false);
-  }
-}
-
-void ClangASTContext::setASTContext(clang::ASTContext *ast_ctx) {
-  if (!m_ast_owned) {
-    m_ast_up.release();
-  }
-  m_ast_owned = false;
-  m_ast_up.reset(ast_ctx);
-  GetASTMap().Insert(ast_ctx, this);
+ASTContext *ClangASTContext::getASTContext() {
+  assert(m_ast_up);
+  return m_ast_up.get();
 }
 
-ASTContext *ClangASTContext::getASTContext() {
-  if (m_ast_up == nullptr) {
-    m_ast_owned = true;
-    m_ast_up.reset(new ASTContext(*getLanguageOptions(), *getSourceManager(),
-                                  *getIdentifierTable(), *getSelectorTable(),
-                                  *getBuiltinContext()));
+void ClangASTContext::CreateASTContext() {
+  assert(!m_ast_up);
+  m_ast_owned = true;
+  m_ast_up.reset(new ASTContext(*getLanguageOptions(), *getSourceManager(),
+                                *getIdentifierTable(), *getSelectorTable(),
+                                *getBuiltinContext()));
 
-    m_ast_up->getDiagnostics().setClient(getDiagnosticConsumer(), false);
+  m_ast_up->getDiagnostics().setClient(getDiagnosticConsumer(), false);
 
-    // This can be NULL if we don't know anything about the architecture or if
-    // the target for an architecture isn't enabled in the llvm/clang that we
-    // built
-    TargetInfo *target_info = getTargetInfo();
-    if (target_info)
-      m_ast_up->InitBuiltinTypes(*target_info);
+  // This can be NULL if we don't know anything about the architecture or if
+  // the target for an architecture isn't enabled in the llvm/clang that we
+  // built
+  TargetInfo *target_info = getTargetInfo();
+  if (target_info)
+    m_ast_up->InitBuiltinTypes(*target_info);
 
-    if ((m_callback_tag_decl || m_callback_objc_decl) && m_callback_baton) {
-      m_ast_up->getTranslationUnitDecl()->setHasExternalLexicalStorage();
-      // m_ast_up->getTranslationUnitDecl()->setHasExternalVisibleStorage();
-    }
+  if ((m_callback_tag_decl || m_callback_objc_decl) && m_callback_baton) {
+    m_ast_up->getTranslationUnitDecl()->setHasExternalLexicalStorage();
+    // m_ast_up->getTranslationUnitDecl()->setHasExternalVisibleStorage();
+  }
 
-    GetASTMap().Insert(m_ast_up.get(), this);
+  GetASTMap().Insert(m_ast_up.get(), this);
 
-    llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> ast_source_up(
-        new ClangExternalASTSourceCallbacks(
-            ClangASTContext::CompleteTagDecl,
-            ClangASTContext::CompleteObjCInterfaceDecl, nullptr,
-            ClangASTContext::LayoutRecordType, this));
-    SetExternalSource(ast_source_up);
-  }
-  return m_ast_up.get();
+  llvm::IntrusiveRefCntPtr<clang::ExternalASTSource> ast_source_up(
+      new ClangExternalASTSourceCallbacks(
+          ClangASTContext::CompleteTagDecl,
+          ClangASTContext::CompleteObjCInterfaceDecl, nullptr,
+          ClangASTContext::LayoutRecordType, this));
+  SetExternalSource(ast_source_up);
 }
 
 ClangASTContext *ClangASTContext::GetASTContext(clang::ASTContext *ast) {
@@ -8229,7 +8195,8 @@ clang::CXXMethodDecl *ClangASTContext::AddMethodToCXXRecordType(
             getASTContext()->DeclarationNames.getCXXDestructorName(
                 getASTContext()->getCanonicalType(record_qual_type)),
             clang::SourceLocation()),
-        method_qual_type, nullptr, is_inline, is_artificial);
+        method_qual_type, nullptr, is_inline, is_artificial,
+          ConstexprSpecKind::CSK_unspecified);
     cxx_method_decl = cxx_dtor_decl;
   } else if (decl_name == cxx_record_decl->getDeclName()) {
     cxx_ctor_decl = clang::CXXConstructorDecl::Create(
@@ -8300,8 +8267,8 @@ clang::CXXMethodDecl *ClangASTContext::AddMethodToCXXRecordType(
     cxx_method_decl->addAttr(clang::UsedAttr::CreateImplicit(*getASTContext()));
 
   if (mangled_name != nullptr) {
-    cxx_method_decl->addAttr(
-        clang::AsmLabelAttr::CreateImplicit(*getASTContext(), mangled_name));
+    cxx_method_decl->addAttr(clang::AsmLabelAttr::CreateImplicit(
+        *getASTContext(), mangled_name, /*literal=*/false));
   }
 
   // Populate the method decl with parameter decls
@@ -8743,74 +8710,6 @@ clang::ObjCMethodDecl *ClangASTContext::AddMethodToObjCObjectType(
   return objc_method_decl;
 }
 
-bool ClangASTContext::GetHasExternalStorage(const CompilerType &type) {
-  if (ClangUtil::IsClangType(type))
-    return false;
-
-  clang::QualType qual_type(ClangUtil::GetCanonicalQualType(type));
-
-  const clang::Type::TypeClass type_class = qual_type->getTypeClass();
-  switch (type_class) {
-  case clang::Type::Record: {
-    clang::CXXRecordDecl *cxx_record_decl = qual_type->getAsCXXRecordDecl();
-    if (cxx_record_decl)
-      return cxx_record_decl->hasExternalLexicalStorage() ||
-             cxx_record_decl->hasExternalVisibleStorage();
-  } break;
-
-  case clang::Type::Enum: {
-    clang::EnumDecl *enum_decl =
-        llvm::cast<clang::EnumType>(qual_type)->getDecl();
-    if (enum_decl)
-      return enum_decl->hasExternalLexicalStorage() ||
-             enum_decl->hasExternalVisibleStorage();
-  } break;
-
-  case clang::Type::ObjCObject:
-  case clang::Type::ObjCInterface: {
-    const clang::ObjCObjectType *objc_class_type =
-        llvm::dyn_cast<clang::ObjCObjectType>(qual_type.getTypePtr());
-    assert(objc_class_type);
-    if (objc_class_type) {
-      clang::ObjCInterfaceDecl *class_interface_decl =
-          objc_class_type->getInterface();
-
-      if (class_interface_decl)
-        return class_interface_decl->hasExternalLexicalStorage() ||
-               class_interface_decl->hasExternalVisibleStorage();
-    }
-  } break;
-
-  case clang::Type::Typedef:
-    return GetHasExternalStorage(CompilerType(
-        type.GetTypeSystem(), llvm::cast<clang::TypedefType>(qual_type)
-                                  ->getDecl()
-                                  ->getUnderlyingType()
-                                  .getAsOpaquePtr()));
-
-  case clang::Type::Auto:
-    return GetHasExternalStorage(CompilerType(
-        type.GetTypeSystem(), llvm::cast<clang::AutoType>(qual_type)
-                                  ->getDeducedType()
-                                  .getAsOpaquePtr()));
-
-  case clang::Type::Elaborated:
-    return GetHasExternalStorage(CompilerType(
-        type.GetTypeSystem(), llvm::cast<clang::ElaboratedType>(qual_type)
-                                  ->getNamedType()
-                                  .getAsOpaquePtr()));
-
-  case clang::Type::Paren:
-    return GetHasExternalStorage(CompilerType(
-        type.GetTypeSystem(),
-        llvm::cast<clang::ParenType>(qual_type)->desugar().getAsOpaquePtr()));
-
-  default:
-    break;
-  }
-  return false;
-}
-
 bool ClangASTContext::SetHasExternalStorage(lldb::opaque_compiler_type_t type,
                                             bool has_extern) {
   if (!type)
@@ -10332,9 +10231,9 @@ ClangASTContext::DeclContextGetClangASTContext(const CompilerDeclContext &dc) {
   return nullptr;
 }
 
-ClangASTContextForExpressions::ClangASTContextForExpressions(Target &target)
-    : ClangASTContext(target.GetArchitecture().GetTriple().getTriple().c_str()),
-      m_target_wp(target.shared_from_this()),
+ClangASTContextForExpressions::ClangASTContextForExpressions(Target &target,
+                                                             ArchSpec arch)
+    : ClangASTContext(arch), m_target_wp(target.shared_from_this()),
       m_persistent_variables(new ClangPersistentVariables) {}
 
 UserExpression *ClangASTContextForExpressions::GetUserExpression(
diff --git a/lldb/source/Symbol/ClangASTImporter.cpp b/lldb/source/Symbol/ClangASTImporter.cpp
index 5319f794ef931..92d51f79a007a 100644
--- a/lldb/source/Symbol/ClangASTImporter.cpp
+++ b/lldb/source/Symbol/ClangASTImporter.cpp
@@ -250,6 +250,94 @@ class DeclContextOverride {
   }
 };
 
+namespace {
+/// Completes all imported TagDecls at the end of the scope.
+///
+/// While in a CompleteTagDeclsScope, every decl that could be completed will
+/// be completed at the end of the scope (including all Decls that are
+/// imported while completing the original Decls).
+class CompleteTagDeclsScope : public ClangASTImporter::NewDeclListener {
+  ClangASTImporter::ImporterDelegateSP m_delegate;
+  // FIXME: Investigate how many decls we usually have in these sets and
+  // see if we can use SmallPtrSet instead here.
+  std::set<NamedDecl *> m_decls_to_complete;
+  std::set<NamedDecl *> m_decls_already_completed;
+  clang::ASTContext *m_dst_ctx;
+  clang::ASTContext *m_src_ctx;
+  ClangASTImporter &importer;
+
+public:
+  /// Constructs a CompleteTagDeclsScope.
+  /// \param importer The ClangASTImporter that we should observe.
+  /// \param dst_ctx The ASTContext to which Decls are imported.
+  /// \param src_ctx The ASTContext from which Decls are imported.
+  explicit CompleteTagDeclsScope(ClangASTImporter &importer,
+                            clang::ASTContext *dst_ctx,
+                            clang::ASTContext *src_ctx)
+      : m_delegate(importer.GetDelegate(dst_ctx, src_ctx)), m_dst_ctx(dst_ctx),
+        m_src_ctx(src_ctx), importer(importer) {
+    m_delegate->SetImportListener(this);
+  }
+
+  virtual ~CompleteTagDeclsScope() {
+    ClangASTImporter::ASTContextMetadataSP to_context_md =
+        importer.GetContextMetadata(m_dst_ctx);
+
+    // Complete all decls we collected until now.
+    while (!m_decls_to_complete.empty()) {
+      NamedDecl *decl = *m_decls_to_complete.begin();
+
+      m_decls_already_completed.insert(decl);
+      m_decls_to_complete.erase(decl);
+
+      // We should only complete decls coming from the source context.
+      assert(to_context_md->m_origins[decl].ctx == m_src_ctx);
+
+      Decl *original_decl = to_context_md->m_origins[decl].decl;
+
+      // Complete the decl now.
+      ClangASTContext::GetCompleteDecl(m_src_ctx, original_decl);
+      if (auto *tag_decl = dyn_cast<TagDecl>(decl)) {
+        if (auto *original_tag_decl = dyn_cast<TagDecl>(original_decl)) {
+          if (original_tag_decl->isCompleteDefinition()) {
+            m_delegate->ImportDefinitionTo(tag_decl, original_tag_decl);
+            tag_decl->setCompleteDefinition(true);
+          }
+        }
+
+        tag_decl->setHasExternalLexicalStorage(false);
+        tag_decl->setHasExternalVisibleStorage(false);
+      } else if (auto *container_decl = dyn_cast<ObjCContainerDecl>(decl)) {
+        container_decl->setHasExternalLexicalStorage(false);
+        container_decl->setHasExternalVisibleStorage(false);
+      }
+
+      to_context_md->m_origins.erase(decl);
+    }
+
+    // Stop listening to imported decls. We do this after clearing the
+    // Decls we needed to import to catch all Decls they might have pulled in.
+    m_delegate->RemoveImportListener();
+  }
+
+  void NewDeclImported(clang::Decl *from, clang::Decl *to) override {
+    // Filter out decls that we can't complete later.
+    if (!isa<TagDecl>(to) && !isa<ObjCInterfaceDecl>(to))
+      return;
+    RecordDecl *from_record_decl = dyn_cast<RecordDecl>(from);
+    // We don't need to complete injected class name decls.
+    if (from_record_decl && from_record_decl->isInjectedClassName())
+      return;
+
+    NamedDecl *to_named_decl = dyn_cast<NamedDecl>(to);
+    // Check if we already completed this type.
+    if (m_decls_already_completed.count(to_named_decl) != 0)
+      return;
+    m_decls_to_complete.insert(to_named_decl);
+  }
+};
+} // namespace
+
 lldb::opaque_compiler_type_t
 ClangASTImporter::DeportType(clang::ASTContext *dst_ctx,
                              clang::ASTContext *src_ctx,
@@ -263,27 +351,16 @@ ClangASTImporter::DeportType(clang::ASTContext *dst_ctx,
             (unsigned long long)type, static_cast<void *>(src_ctx),
             static_cast<void *>(dst_ctx));
 
-  ImporterDelegateSP delegate_sp(GetDelegate(dst_ctx, src_ctx));
-
-  if (!delegate_sp)
-    return nullptr;
-
-  std::set<NamedDecl *> decls_to_deport;
-  std::set<NamedDecl *> decls_already_deported;
-
   DeclContextOverride decl_context_override;
 
-  if (const clang::TagType *tag_type =
-          clang::QualType::getFromOpaquePtr(type)->getAs<TagType>()) {
-    decl_context_override.OverrideAllDeclsFromContainingFunction(
-        tag_type->getDecl());
-  }
-
-  delegate_sp->InitDeportWorkQueues(&decls_to_deport, &decls_already_deported);
-
-  lldb::opaque_compiler_type_t result = CopyType(dst_ctx, src_ctx, type);
+  if (auto *t = QualType::getFromOpaquePtr(type)->getAs<TagType>())
+    decl_context_override.OverrideAllDeclsFromContainingFunction(t->getDecl());
 
-  delegate_sp->ExecuteDeportWorkQueues();
+  lldb::opaque_compiler_type_t result;
+  {
+    CompleteTagDeclsScope complete_scope(*this, dst_ctx, src_ctx);
+    result = CopyType(dst_ctx, src_ctx, type);
+  }
 
   if (!result)
     return nullptr;
@@ -302,23 +379,15 @@ clang::Decl *ClangASTImporter::DeportDecl(clang::ASTContext *dst_ctx,
             decl->getDeclKindName(), static_cast<void *>(decl),
             static_cast<void *>(src_ctx), static_cast<void *>(dst_ctx));
 
-  ImporterDelegateSP delegate_sp(GetDelegate(dst_ctx, src_ctx));
-
-  if (!delegate_sp)
-    return nullptr;
-
-  std::set<NamedDecl *> decls_to_deport;
-  std::set<NamedDecl *> decls_already_deported;
-
   DeclContextOverride decl_context_override;
 
   decl_context_override.OverrideAllDeclsFromContainingFunction(decl);
 
-  delegate_sp->InitDeportWorkQueues(&decls_to_deport, &decls_already_deported);
-
-  clang::Decl *result = CopyDecl(dst_ctx, src_ctx, decl);
-
-  delegate_sp->ExecuteDeportWorkQueues();
+  clang::Decl *result;
+  {
+    CompleteTagDeclsScope complete_scope(*this, dst_ctx, src_ctx);
+    result = CopyDecl(dst_ctx, src_ctx, decl);
+  }
 
   if (!result)
     return nullptr;
@@ -870,63 +939,6 @@ ClangASTImporter::ASTImporterDelegate::ImportImpl(Decl *From) {
   return ASTImporter::ImportImpl(From);
 }
 
-void ClangASTImporter::ASTImporterDelegate::InitDeportWorkQueues(
-    std::set<clang::NamedDecl *> *decls_to_deport,
-    std::set<clang::NamedDecl *> *decls_already_deported) {
-  assert(!m_decls_to_deport);
-  assert(!m_decls_already_deported);
-
-  m_decls_to_deport = decls_to_deport;
-  m_decls_already_deported = decls_already_deported;
-}
-
-void ClangASTImporter::ASTImporterDelegate::ExecuteDeportWorkQueues() {
-  assert(m_decls_to_deport);
-  assert(m_decls_already_deported);
-
-  ASTContextMetadataSP to_context_md =
-      m_master.GetContextMetadata(&getToContext());
-
-  while (!m_decls_to_deport->empty()) {
-    NamedDecl *decl = *m_decls_to_deport->begin();
-
-    m_decls_already_deported->insert(decl);
-    m_decls_to_deport->erase(decl);
-
-    DeclOrigin &origin = to_context_md->m_origins[decl];
-    UNUSED_IF_ASSERT_DISABLED(origin);
-
-    assert(origin.ctx ==
-           m_source_ctx); // otherwise we should never have added this
-                          // because it doesn't need to be deported
-
-    Decl *original_decl = to_context_md->m_origins[decl].decl;
-
-    ClangASTContext::GetCompleteDecl(m_source_ctx, original_decl);
-
-    if (TagDecl *tag_decl = dyn_cast<TagDecl>(decl)) {
-      if (TagDecl *original_tag_decl = dyn_cast<TagDecl>(original_decl)) {
-        if (original_tag_decl->isCompleteDefinition()) {
-          ImportDefinitionTo(tag_decl, original_tag_decl);
-          tag_decl->setCompleteDefinition(true);
-        }
-      }
-
-      tag_decl->setHasExternalLexicalStorage(false);
-      tag_decl->setHasExternalVisibleStorage(false);
-    } else if (ObjCContainerDecl *container_decl =
-                   dyn_cast<ObjCContainerDecl>(decl)) {
-      container_decl->setHasExternalLexicalStorage(false);
-      container_decl->setHasExternalVisibleStorage(false);
-    }
-
-    to_context_md->m_origins.erase(decl);
-  }
-
-  m_decls_to_deport = nullptr;
-  m_decls_already_deported = nullptr;
-}
-
 void ClangASTImporter::ASTImporterDelegate::ImportDefinitionTo(
     clang::Decl *to, clang::Decl *from) {
   ASTImporter::Imported(from, to);
@@ -1092,18 +1104,8 @@ void ClangASTImporter::ASTImporterDelegate::Imported(clang::Decl *from,
                 static_cast<void *>(&from->getASTContext()),
                 static_cast<void *>(&to->getASTContext()));
     } else {
-      if (m_decls_to_deport && m_decls_already_deported) {
-        if (isa<TagDecl>(to) || isa<ObjCInterfaceDecl>(to)) {
-          RecordDecl *from_record_decl = dyn_cast<RecordDecl>(from);
-          if (from_record_decl == nullptr ||
-              !from_record_decl->isInjectedClassName()) {
-            NamedDecl *to_named_decl = dyn_cast<NamedDecl>(to);
-
-            if (!m_decls_already_deported->count(to_named_decl))
-              m_decls_to_deport->insert(to_named_decl);
-          }
-        }
-      }
+      if (m_new_decl_listener)
+        m_new_decl_listener->NewDeclImported(from, to);
 
       if (to_context_md->m_origins.find(to) == to_context_md->m_origins.end() ||
           user_id != LLDB_INVALID_UID) {
diff --git a/lldb/source/Symbol/CompileUnit.cpp b/lldb/source/Symbol/CompileUnit.cpp
index dc50e832aa62c..41086d2df3df1 100644
--- a/lldb/source/Symbol/CompileUnit.cpp
+++ b/lldb/source/Symbol/CompileUnit.cpp
@@ -353,6 +353,11 @@ const std::vector<SourceModule> &CompileUnit::GetImportedModules() {
   return m_imported_modules;
 }
 
+void CompileUnit::ForEachExternalModule(llvm::function_ref<void(ModuleSP)> f) {
+  if (SymbolFile *symfile = GetModule()->GetSymbolFile())
+    symfile->ForEachExternalModule(*this, f);
+}
+
 const FileSpecList &CompileUnit::GetSupportFiles() {
   if (m_support_files.GetSize() == 0) {
     if (m_flags.IsClear(flagsParsedSupportFiles)) {
diff --git a/lldb/source/Symbol/Function.cpp b/lldb/source/Symbol/Function.cpp
index 10a6032179fca..be5ea258de5be 100644
--- a/lldb/source/Symbol/Function.cpp
+++ b/lldb/source/Symbol/Function.cpp
@@ -173,6 +173,7 @@ void CallEdge::ParseSymbolFileAndResolve(ModuleList &images) {
 
 Function *CallEdge::GetCallee(ModuleList &images) {
   ParseSymbolFileAndResolve(images);
+  assert(resolved && "Did not resolve lazy callee");
   return lazy_callee.def;
 }
 
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index b1fc8520d7698..6dcdfbc089baa 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -139,23 +139,14 @@ void SymbolFile::GetMangledNamesForFunction(
   return;
 }
 
-uint32_t SymbolFile::FindTypes(
+void SymbolFile::FindTypes(
     ConstString name, const CompilerDeclContext *parent_decl_ctx,
-    bool append, uint32_t max_matches,
+    uint32_t max_matches,
     llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return 0;
-}
+    TypeMap &types) {}
 
-size_t SymbolFile::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                             LanguageSet languages, bool append,
-                             TypeMap &types) {
-  if (!append)
-    types.Clear();
-  return 0;
-}
+void SymbolFile::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
+                           LanguageSet languages, TypeMap &types) {}
 
 void SymbolFile::AssertModuleLock() {
   // The code below is too expensive to leave enabled in release builds. It's
diff --git a/lldb/source/Symbol/UnwindPlan.cpp b/lldb/source/Symbol/UnwindPlan.cpp
index 6af85d699d6bf..15443ce5d8ac0 100644
--- a/lldb/source/Symbol/UnwindPlan.cpp
+++ b/lldb/source/Symbol/UnwindPlan.cpp
@@ -170,7 +170,8 @@ operator==(const UnwindPlan::Row::FAValue &rhs) const {
   if (m_type == rhs.m_type) {
     switch (m_type) {
     case unspecified:
-      return true;
+    case isRaSearch:
+      return m_value.ra_search_offset == rhs.m_value.ra_search_offset;
 
     case isRegisterPlusOffset:
       return m_value.reg.offset == rhs.m_value.reg.offset;
@@ -205,9 +206,12 @@ void UnwindPlan::Row::FAValue::Dump(Stream &s, const UnwindPlan *unwind_plan,
                   llvm::makeArrayRef(m_value.expr.opcodes, m_value.expr.length),
                   thread);
     break;
-  default:
+  case unspecified:
     s.PutCString("unspecified");
     break;
+  case isRaSearch:
+    s.Printf("RaSearch@SP%+d", m_value.ra_search_offset);
+    break;
   }
 }
 
diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp
index 3ec97b1fce7f1..3f3d7c198f15b 100644
--- a/lldb/source/Symbol/Variable.cpp
+++ b/lldb/source/Symbol/Variable.cpp
@@ -35,14 +35,12 @@
 using namespace lldb;
 using namespace lldb_private;
 
-// Variable constructor
-Variable::Variable(
-    lldb::user_id_t uid, const char *name,
-    const char *mangled, // The mangled or fully qualified name of the variable.
-    const lldb::SymbolFileTypeSP &symfile_type_sp, ValueType scope,
-    SymbolContextScope *context, const RangeList &scope_range,
-    Declaration *decl_ptr, const DWARFExpression &location, bool external,
-    bool artificial, bool static_member)
+Variable::Variable(lldb::user_id_t uid, const char *name, const char *mangled,
+                   const lldb::SymbolFileTypeSP &symfile_type_sp,
+                   ValueType scope, SymbolContextScope *context,
+                   const RangeList &scope_range, Declaration *decl_ptr,
+                   const DWARFExpression &location, bool external,
+                   bool artificial, bool static_member)
     : UserID(uid), m_name(name), m_mangled(ConstString(mangled)),
       m_symfile_type_sp(symfile_type_sp), m_scope(scope),
       m_owner_scope(context), m_scope_range(scope_range),
@@ -50,7 +48,6 @@ Variable::Variable(
       m_artificial(artificial), m_loc_is_const_data(false),
       m_static_member(static_member) {}
 
-// Destructor
 Variable::~Variable() {}
 
 lldb::LanguageType Variable::GetLanguage() const {
diff --git a/lldb/source/Target/ABI.cpp b/lldb/source/Target/ABI.cpp
index 28cd9aec665cb..005261e0ddee0 100644
--- a/lldb/source/Target/ABI.cpp
+++ b/lldb/source/Target/ABI.cpp
@@ -15,6 +15,8 @@
 #include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/Thread.h"
+#include "lldb/Utility/Log.h"
+#include "llvm/Support/TargetRegistry.h"
 
 using namespace lldb;
 using namespace lldb_private;
@@ -210,3 +212,20 @@ bool ABI::GetFallbackRegisterLocation(
 
   return false;
 }
+
+std::unique_ptr<llvm::MCRegisterInfo> ABI::MakeMCRegisterInfo(const ArchSpec &arch) {
+  std::string triple = arch.GetTriple().getTriple();
+  std::string lookup_error;
+  const llvm::Target *target =
+      llvm::TargetRegistry::lookupTarget(triple, lookup_error);
+  if (!target) {
+    LLDB_LOG(GetLogIfAllCategoriesSet(LIBLLDB_LOG_PROCESS),
+             "Failed to create an llvm target for {0}: {1}", triple,
+             lookup_error);
+    return nullptr;
+  }
+  std::unique_ptr<llvm::MCRegisterInfo> info_up(
+      target->createMCRegInfo(triple));
+  assert(info_up);
+  return info_up;
+}
diff --git a/lldb/source/Target/ModuleCache.cpp b/lldb/source/Target/ModuleCache.cpp
index cd3f211b81cf1..124cdacfb4d2b 100644
--- a/lldb/source/Target/ModuleCache.cpp
+++ b/lldb/source/Target/ModuleCache.cpp
@@ -48,7 +48,7 @@ std::string GetEscapedHostname(const char *hostname) {
 
 class ModuleLock {
 private:
-  File m_file;
+  FileUP m_file_up;
   std::unique_ptr<lldb_private::LockFile> m_lock;
   FileSpec m_file_spec;
 
@@ -157,16 +157,19 @@ ModuleLock::ModuleLock(const FileSpec &root_dir_spec, const UUID &uuid,
     return;
 
   m_file_spec = JoinPath(lock_dir_spec, uuid.GetAsString().c_str());
-  FileSystem::Instance().Open(m_file, m_file_spec,
-                              File::eOpenOptionWrite |
-                                  File::eOpenOptionCanCreate |
-                                  File::eOpenOptionCloseOnExec);
-  if (!m_file) {
-    error.SetErrorToErrno();
+
+  auto file = FileSystem::Instance().Open(
+      m_file_spec, File::eOpenOptionWrite | File::eOpenOptionCanCreate |
+                       File::eOpenOptionCloseOnExec);
+  if (file)
+    m_file_up = std::move(file.get());
+  else {
+    m_file_up.reset();
+    error = Status(file.takeError());
     return;
   }
 
-  m_lock.reset(new lldb_private::LockFile(m_file.GetDescriptor()));
+  m_lock.reset(new lldb_private::LockFile(m_file_up->GetDescriptor()));
   error = m_lock->WriteLock(0, 1);
   if (error.Fail())
     error.SetErrorStringWithFormat("Failed to lock file: %s",
@@ -174,10 +177,11 @@ ModuleLock::ModuleLock(const FileSpec &root_dir_spec, const UUID &uuid,
 }
 
 void ModuleLock::Delete() {
-  if (!m_file)
+  if (!m_file_up)
     return;
 
-  m_file.Close();
+  m_file_up->Close();
+  m_file_up.reset();
   llvm::sys::fs::remove(m_file_spec.GetPath());
 }
 
diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index fac9b56026bbd..00af754382f39 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1226,15 +1226,15 @@ Status Platform::PutFile(const FileSpec &source, const FileSpec &destination,
   if (fs::is_symlink_file(source.GetPath()))
     source_open_options |= File::eOpenOptionDontFollowSymlinks;
 
-  File source_file;
-  Status error = FileSystem::Instance().Open(
-      source_file, source, source_open_options, lldb::eFilePermissionsUserRW);
-  uint32_t permissions = source_file.GetPermissions(error);
+  auto source_file = FileSystem::Instance().Open(source, source_open_options,
+                                                 lldb::eFilePermissionsUserRW);
+  if (!source_file)
+    return Status(source_file.takeError());
+  Status error;
+  uint32_t permissions = source_file.get()->GetPermissions(error);
   if (permissions == 0)
     permissions = lldb::eFilePermissionsFileDefault;
 
-  if (!source_file.IsValid())
-    return Status("PutFile: unable to open source file");
   lldb::user_id_t dest_file = OpenFile(
       destination, File::eOpenOptionCanCreate | File::eOpenOptionWrite |
                        File::eOpenOptionTruncate | File::eOpenOptionCloseOnExec,
@@ -1249,7 +1249,7 @@ Status Platform::PutFile(const FileSpec &source, const FileSpec &destination,
   uint64_t offset = 0;
   for (;;) {
     size_t bytes_read = buffer_sp->GetByteSize();
-    error = source_file.Read(buffer_sp->GetBytes(), bytes_read);
+    error = source_file.get()->Read(buffer_sp->GetBytes(), bytes_read);
     if (error.Fail() || bytes_read == 0)
       break;
 
@@ -1262,7 +1262,7 @@ Status Platform::PutFile(const FileSpec &source, const FileSpec &destination,
     if (bytes_written != bytes_read) {
       // We didn't write the correct number of bytes, so adjust the file
       // position in the source file we are reading from...
-      source_file.SeekFromStart(offset);
+      source_file.get()->SeekFromStart(offset);
     }
   }
   CloseFile(dest_file, error);
@@ -1796,8 +1796,7 @@ lldb::ProcessSP Platform::ConnectProcess(llvm::StringRef connect_url,
   if (!process_sp)
     return nullptr;
 
-  error =
-      process_sp->ConnectRemote(debugger.GetOutputFile().get(), connect_url);
+  error = process_sp->ConnectRemote(&debugger.GetOutputStream(), connect_url);
   if (error.Fail())
     return nullptr;
 
diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp
index 55570d02b9f8a..d0008023147ed 100644
--- a/lldb/source/Target/Process.cpp
+++ b/lldb/source/Target/Process.cpp
@@ -1649,7 +1649,7 @@ Process::CreateBreakpointSite(const BreakpointLocationSP &owner,
       Address symbol_address = symbol->GetAddress();
       load_addr = ResolveIndirectFunction(&symbol_address, error);
       if (!error.Success() && show_error) {
-        GetTarget().GetDebugger().GetErrorFile()->Printf(
+        GetTarget().GetDebugger().GetErrorStream().Printf(
             "warning: failed to resolve indirect function at 0x%" PRIx64
             " for breakpoint %i.%i: %s\n",
             symbol->GetLoadAddress(&GetTarget()),
@@ -1688,7 +1688,7 @@ Process::CreateBreakpointSite(const BreakpointLocationSP &owner,
         } else {
           if (show_error || use_hardware) {
             // Report error for setting breakpoint...
-            GetTarget().GetDebugger().GetErrorFile()->Printf(
+            GetTarget().GetDebugger().GetErrorStream().Printf(
                 "warning: failed to set breakpoint site at 0x%" PRIx64
                 " for breakpoint %i.%i: %s\n",
                 load_addr, owner->GetBreakpoint().GetID(), owner->GetID(),
@@ -4299,9 +4299,10 @@ class IOHandlerProcessSTDIO : public IOHandler {
   IOHandlerProcessSTDIO(Process *process, int write_fd)
       : IOHandler(process->GetTarget().GetDebugger(),
                   IOHandler::Type::ProcessIO),
-        m_process(process), m_write_file(write_fd, false) {
+        m_process(process),
+        m_read_file(GetInputFD(), File::eOpenOptionRead, false),
+        m_write_file(write_fd, File::eOpenOptionWrite, false) {
     m_pipe.CreateNew(false);
-    m_read_file.SetDescriptor(GetInputFD(), false);
   }
 
   ~IOHandlerProcessSTDIO() override = default;
@@ -5537,6 +5538,12 @@ ProcessRunLock &Process::GetRunLock() {
     return m_public_run_lock;
 }
 
+bool Process::CurrentThreadIsPrivateStateThread()
+{
+  return m_private_state_thread.EqualsThread(Host::GetCurrentThread());
+}
+
+
 void Process::Flush() {
   m_thread_list.Flush();
   m_extended_thread_list.Flush();
diff --git a/lldb/source/Target/StackFrameList.cpp b/lldb/source/Target/StackFrameList.cpp
index fbced5fd132d9..6d0c46259c20f 100644
--- a/lldb/source/Target/StackFrameList.cpp
+++ b/lldb/source/Target/StackFrameList.cpp
@@ -286,15 +286,15 @@ static void FindInterveningFrames(Function &begin, Function &end,
 
     DFS(Function *end, ModuleList &images) : end(end), images(images) {}
 
-    void search(Function *first_callee, std::vector<Function *> &path) {
+    void search(Function &first_callee, std::vector<Function *> &path) {
       dfs(first_callee);
       if (!ambiguous)
         path = std::move(solution_path);
     }
 
-    void dfs(Function *callee) {
+    void dfs(Function &callee) {
       // Found a path to the target function.
-      if (callee == end) {
+      if (&callee == end) {
         if (solution_path.empty())
           solution_path = active_path;
         else
@@ -306,19 +306,19 @@ static void FindInterveningFrames(Function &begin, Function &end,
       // there's more than one way to reach a target. This errs on the side of
       // caution: it conservatively stops searching when some solutions are
       // still possible to save time in the average case.
-      if (!visited_nodes.insert(callee).second) {
+      if (!visited_nodes.insert(&callee).second) {
         ambiguous = true;
         return;
       }
 
       // Search the calls made from this callee.
-      active_path.push_back(callee);
-      for (CallEdge &edge : callee->GetTailCallingEdges()) {
+      active_path.push_back(&callee);
+      for (CallEdge &edge : callee.GetTailCallingEdges()) {
         Function *next_callee = edge.GetCallee(images);
         if (!next_callee)
           continue;
 
-        dfs(next_callee);
+        dfs(*next_callee);
         if (ambiguous)
           return;
       }
@@ -326,7 +326,7 @@ static void FindInterveningFrames(Function &begin, Function &end,
     }
   };
 
-  DFS(&end, images).search(first_callee, path);
+  DFS(&end, images).search(*first_callee, path);
 }
 
 /// Given that \p next_frame will be appended to the frame list, synthesize
diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index c92d63882c0f8..38406214ba12f 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -1358,15 +1358,15 @@ static void LoadScriptingResourceForModule(const ModuleSP &module_sp,
   if (module_sp && !module_sp->LoadScriptingResourceInTarget(
                        target, error, &feedback_stream)) {
     if (error.AsCString())
-      target->GetDebugger().GetErrorFile()->Printf(
+      target->GetDebugger().GetErrorStream().Printf(
           "unable to load scripting data for module %s - error reported was "
           "%s\n",
           module_sp->GetFileSpec().GetFileNameStrippingExtension().GetCString(),
           error.AsCString());
   }
   if (feedback_stream.GetSize())
-    target->GetDebugger().GetErrorFile()->Printf("%s\n",
-                                                 feedback_stream.GetData());
+    target->GetDebugger().GetErrorStream().Printf("%s\n",
+                                                  feedback_stream.GetData());
 }
 
 void Target::ClearModules(bool delete_locations) {
@@ -4129,3 +4129,10 @@ Target::TargetEventData::GetModuleListFromEvent(const Event *event_ptr) {
     module_list = event_data->m_module_list;
   return module_list;
 }
+
+std::recursive_mutex &Target::GetAPIMutex() { 
+  if (GetProcessSP() && GetProcessSP()->CurrentThreadIsPrivateStateThread())
+    return m_private_mutex;
+  else
+    return m_mutex;
+}
diff --git a/lldb/source/Target/ThreadPlanPython.cpp b/lldb/source/Target/ThreadPlanPython.cpp
index cc0f55bf8d853..ff57e0a25ff20 100644
--- a/lldb/source/Target/ThreadPlanPython.cpp
+++ b/lldb/source/Target/ThreadPlanPython.cpp
@@ -45,7 +45,9 @@ bool ThreadPlanPython::ValidatePlan(Stream *error) {
 
   if (!m_implementation_sp) {
     if (error)
-      error->Printf("Python thread plan does not have an implementation");
+      error->Printf("Error constructing Python ThreadPlan: %s",
+          m_error_str.empty() ? "<unknown error>"
+                                : m_error_str.c_str());
     return false;
   }
 
@@ -63,7 +65,7 @@ void ThreadPlanPython::DidPush() {
                                            .GetScriptInterpreter();
     if (script_interp) {
       m_implementation_sp = script_interp->CreateScriptedThreadPlan(
-          m_class_name.c_str(), this->shared_from_this());
+          m_class_name.c_str(), m_error_str, this->shared_from_this());
     }
   }
 }
diff --git a/lldb/source/Target/ThreadPlanTracer.cpp b/lldb/source/Target/ThreadPlanTracer.cpp
index ed4e17840cd3f..5782fe8e6443d 100644
--- a/lldb/source/Target/ThreadPlanTracer.cpp
+++ b/lldb/source/Target/ThreadPlanTracer.cpp
@@ -46,7 +46,7 @@ Stream *ThreadPlanTracer::GetLogStream() {
   else {
     TargetSP target_sp(m_thread.CalculateTarget());
     if (target_sp)
-      return target_sp->GetDebugger().GetOutputFile().get();
+      return &(target_sp->GetDebugger().GetOutputStream());
   }
   return nullptr;
 }
diff --git a/lldb/source/Utility/ArchSpec.cpp b/lldb/source/Utility/ArchSpec.cpp
index fc2d43925a3c3..9907ca9b24095 100644
--- a/lldb/source/Utility/ArchSpec.cpp
+++ b/lldb/source/Utility/ArchSpec.cpp
@@ -244,17 +244,8 @@ void ArchSpec::ListSupportedArchNames(StringList &list) {
 }
 
 void ArchSpec::AutoComplete(CompletionRequest &request) {
-  if (!request.GetCursorArgumentPrefix().empty()) {
-    for (uint32_t i = 0; i < llvm::array_lengthof(g_core_definitions); ++i) {
-      if (NameMatches(g_core_definitions[i].name, NameMatch::StartsWith,
-                      request.GetCursorArgumentPrefix()))
-        request.AddCompletion(g_core_definitions[i].name);
-    }
-  } else {
-    StringList matches;
-    ListSupportedArchNames(matches);
-    request.AddCompletions(matches);
-  }
+  for (uint32_t i = 0; i < llvm::array_lengthof(g_core_definitions); ++i)
+    request.TryCompleteCurrentArg(g_core_definitions[i].name);
 }
 
 #define CPU_ANY (UINT32_MAX)
@@ -468,7 +459,9 @@ static const ArchDefinitionEntry g_coff_arch_entries[] = {
     {ArchSpec::eCore_thumb, llvm::COFF::IMAGE_FILE_MACHINE_THUMB,
      LLDB_INVALID_CPUTYPE, 0xFFFFFFFFu, 0xFFFFFFFFu}, // ARMv7
     {ArchSpec::eCore_x86_64_x86_64, llvm::COFF::IMAGE_FILE_MACHINE_AMD64,
-     LLDB_INVALID_CPUTYPE, 0xFFFFFFFFu, 0xFFFFFFFFu} // AMD64
+     LLDB_INVALID_CPUTYPE, 0xFFFFFFFFu, 0xFFFFFFFFu}, // AMD64
+    {ArchSpec::eCore_arm_arm64, llvm::COFF::IMAGE_FILE_MACHINE_ARM64,
+     LLDB_INVALID_CPUTYPE, 0xFFFFFFFFu, 0xFFFFFFFFu} // ARM64
 };
 
 static const ArchDefinition g_coff_arch_def = {
diff --git a/lldb/source/Utility/CMakeLists.txt b/lldb/source/Utility/CMakeLists.txt
index e5cee780d4f73..df486e2c0a4ca 100644
--- a/lldb/source/Utility/CMakeLists.txt
+++ b/lldb/source/Utility/CMakeLists.txt
@@ -25,8 +25,8 @@ add_lldb_library(lldbUtility
   Environment.cpp
   Event.cpp
   FileSpec.cpp
+  GDBRemote.cpp
   IOObject.cpp
-  JSON.cpp
   LLDBAssert.cpp
   Listener.cpp
   Log.cpp
@@ -44,7 +44,6 @@ add_lldb_library(lldbUtility
   Status.cpp
   Stream.cpp
   StreamCallback.cpp
-  StreamGDBRemote.cpp
   StreamString.cpp
   StringExtractor.cpp
   StringExtractorGDBRemote.cpp
diff --git a/lldb/source/Utility/CompletionRequest.cpp b/lldb/source/Utility/CompletionRequest.cpp
index 7fd5260f7d53a..3b5a4570e3242 100644
--- a/lldb/source/Utility/CompletionRequest.cpp
+++ b/lldb/source/Utility/CompletionRequest.cpp
@@ -21,36 +21,24 @@ CompletionRequest::CompletionRequest(llvm::StringRef command_line,
   // We parse the argument up to the cursor, so the last argument in
   // parsed_line is the one containing the cursor, and the cursor is after the
   // last character.
-  m_parsed_line = Args(command_line);
-  m_partial_parsed_line = Args(command_line.substr(0, raw_cursor_pos));
+  llvm::StringRef partial_command(command_line.substr(0, raw_cursor_pos));
+  m_parsed_line = Args(partial_command);
 
-  m_cursor_index = m_partial_parsed_line.GetArgumentCount() - 1;
-
-  if (m_cursor_index == -1)
+  if (GetParsedLine().GetArgumentCount() == 0) {
+    m_cursor_index = 0;
     m_cursor_char_position = 0;
-  else
+  } else {
+    m_cursor_index = GetParsedLine().GetArgumentCount() - 1U;
     m_cursor_char_position =
-        strlen(m_partial_parsed_line.GetArgumentAtIndex(m_cursor_index));
-
-  const char *cursor = command_line.data() + raw_cursor_pos;
-  if (raw_cursor_pos > 0 && cursor[-1] == ' ') {
-    // We are just after a space.  If we are in an argument, then we will
-    // continue parsing, but if we are between arguments, then we have to
-    // complete whatever the next element would be. We can distinguish the two
-    // cases because if we are in an argument (e.g. because the space is
-    // protected by a quote) then the space will also be in the parsed
-    // argument...
-
-    const char *current_elem =
-        m_partial_parsed_line.GetArgumentAtIndex(m_cursor_index);
-    if (m_cursor_char_position == 0 ||
-        current_elem[m_cursor_char_position - 1] != ' ') {
-      m_parsed_line.InsertArgumentAtIndex(m_cursor_index + 1, llvm::StringRef(),
-                                          '\0');
-      m_cursor_index++;
-      m_cursor_char_position = 0;
-    }
+        strlen(GetParsedLine().GetArgumentAtIndex(m_cursor_index));
   }
+
+  // The cursor is after a space but the space is not part of the argument.
+  // Let's add an empty fake argument to the end to make sure the completion
+  // code. Note: The space could be part of the last argument when it's quoted.
+  if (partial_command.endswith(" ") &&
+      !GetCursorArgumentPrefix().endswith(" "))
+    AppendEmptyArgument();
 }
 
 std::string CompletionResult::Completion::GetUniqueKey() const {
diff --git a/lldb/source/Utility/GDBRemote.cpp b/lldb/source/Utility/GDBRemote.cpp
new file mode 100644
index 0000000000000..85c4bc69a8d10
--- /dev/null
+++ b/lldb/source/Utility/GDBRemote.cpp
@@ -0,0 +1,105 @@
+//===-- GDBRemote.cpp -----------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Utility/GDBRemote.h"
+
+#include "lldb/Utility/Flags.h"
+#include "lldb/Utility/Stream.h"
+
+#include <stdio.h>
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace llvm;
+
+StreamGDBRemote::StreamGDBRemote() : StreamString() {}
+
+StreamGDBRemote::StreamGDBRemote(uint32_t flags, uint32_t addr_size,
+                                 ByteOrder byte_order)
+    : StreamString(flags, addr_size, byte_order) {}
+
+StreamGDBRemote::~StreamGDBRemote() {}
+
+int StreamGDBRemote::PutEscapedBytes(const void *s, size_t src_len) {
+  int bytes_written = 0;
+  const uint8_t *src = static_cast<const uint8_t *>(s);
+  bool binary_is_set = m_flags.Test(eBinary);
+  m_flags.Clear(eBinary);
+  while (src_len) {
+    uint8_t byte = *src;
+    src++;
+    src_len--;
+    if (byte == 0x23 || byte == 0x24 || byte == 0x7d || byte == 0x2a) {
+      bytes_written += PutChar(0x7d);
+      byte ^= 0x20;
+    }
+    bytes_written += PutChar(byte);
+  };
+  if (binary_is_set)
+    m_flags.Set(eBinary);
+  return bytes_written;
+}
+
+void GDBRemotePacket::Serialize(raw_ostream &strm) const {
+  yaml::Output yout(strm);
+  yout << const_cast<GDBRemotePacket &>(*this);
+  strm.flush();
+}
+
+llvm::StringRef GDBRemotePacket::GetTypeStr() const {
+  switch (type) {
+  case GDBRemotePacket::ePacketTypeSend:
+    return "send";
+  case GDBRemotePacket::ePacketTypeRecv:
+    return "read";
+  case GDBRemotePacket::ePacketTypeInvalid:
+    return "invalid";
+  }
+  llvm_unreachable("All enum cases should be handled");
+}
+
+void GDBRemotePacket::Dump(Stream &strm) const {
+  strm.Printf("tid=0x%4.4" PRIx64 " <%4u> %s packet: %s\n", tid,
+              bytes_transmitted, GetTypeStr().data(), packet.data.c_str());
+}
+
+void yaml::ScalarEnumerationTraits<GDBRemotePacket::Type>::enumeration(
+    IO &io, GDBRemotePacket::Type &value) {
+  io.enumCase(value, "Invalid", GDBRemotePacket::ePacketTypeInvalid);
+  io.enumCase(value, "Send", GDBRemotePacket::ePacketTypeSend);
+  io.enumCase(value, "Recv", GDBRemotePacket::ePacketTypeRecv);
+}
+
+void yaml::ScalarTraits<GDBRemotePacket::BinaryData>::output(
+    const GDBRemotePacket::BinaryData &Val, void *, raw_ostream &Out) {
+  Out << toHex(Val.data);
+}
+
+StringRef yaml::ScalarTraits<GDBRemotePacket::BinaryData>::input(
+    StringRef Scalar, void *, GDBRemotePacket::BinaryData &Val) {
+  Val.data = fromHex(Scalar);
+  return {};
+}
+
+void yaml::MappingTraits<GDBRemotePacket>::mapping(IO &io,
+                                                   GDBRemotePacket &Packet) {
+  io.mapRequired("packet", Packet.packet);
+  io.mapRequired("type", Packet.type);
+  io.mapRequired("bytes", Packet.bytes_transmitted);
+  io.mapRequired("index", Packet.packet_idx);
+  io.mapRequired("tid", Packet.tid);
+}
+
+StringRef
+yaml::MappingTraits<GDBRemotePacket>::validate(IO &io,
+                                               GDBRemotePacket &Packet) {
+  if (Packet.bytes_transmitted != Packet.packet.data.size())
+    return "BinaryData size doesn't match bytes transmitted";
+
+  return {};
+}
diff --git a/lldb/source/Utility/JSON.cpp b/lldb/source/Utility/JSON.cpp
deleted file mode 100644
index 2c3f6229eda11..0000000000000
--- a/lldb/source/Utility/JSON.cpp
+++ /dev/null
@@ -1,550 +0,0 @@
-//===--------------------- JSON.cpp -----------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Utility/JSON.h"
-
-#include "lldb/Utility/Stream.h"
-#include "lldb/Utility/StreamString.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/ErrorHandling.h"
-
-#include <inttypes.h>
-#include <limits.h>
-#include <stddef.h>
-#include <utility>
-
-using namespace lldb_private;
-
-std::string JSONString::json_string_quote_metachars(const std::string &s) {
-  if (s.find_first_of("\\\n\"") == std::string::npos)
-    return s;
-
-  std::string output;
-  const size_t s_size = s.size();
-  const char *s_chars = s.c_str();
-  for (size_t i = 0; i < s_size; i++) {
-    unsigned char ch = *(s_chars + i);
-    if (ch == '"' || ch == '\\' || ch == '\n') {
-      output.push_back('\\');
-      if (ch == '\n') ch = 'n';
-    }
-    output.push_back(ch);
-  }
-  return output;
-}
-
-JSONString::JSONString() : JSONValue(JSONValue::Kind::String), m_data() {}
-
-JSONString::JSONString(const char *s)
-    : JSONValue(JSONValue::Kind::String), m_data(s ? s : "") {}
-
-JSONString::JSONString(const std::string &s)
-    : JSONValue(JSONValue::Kind::String), m_data(s) {}
-
-void JSONString::Write(Stream &s) {
-  s.Printf("\"%s\"", json_string_quote_metachars(m_data).c_str());
-}
-
-uint64_t JSONNumber::GetAsUnsigned() const {
-  switch (m_data_type) {
-  case DataType::Unsigned:
-    return m_data.m_unsigned;
-  case DataType::Signed:
-    return static_cast<uint64_t>(m_data.m_signed);
-  case DataType::Double:
-    return static_cast<uint64_t>(m_data.m_double);
-  }
-  llvm_unreachable("Unhandled data type");
-}
-
-int64_t JSONNumber::GetAsSigned() const {
-  switch (m_data_type) {
-  case DataType::Unsigned:
-    return static_cast<int64_t>(m_data.m_unsigned);
-  case DataType::Signed:
-    return m_data.m_signed;
-  case DataType::Double:
-    return static_cast<int64_t>(m_data.m_double);
-  }
-  llvm_unreachable("Unhandled data type");
-}
-
-double JSONNumber::GetAsDouble() const {
-  switch (m_data_type) {
-  case DataType::Unsigned:
-    return static_cast<double>(m_data.m_unsigned);
-  case DataType::Signed:
-    return static_cast<double>(m_data.m_signed);
-  case DataType::Double:
-    return m_data.m_double;
-  }
-  llvm_unreachable("Unhandled data type");
-}
-
-void JSONNumber::Write(Stream &s) {
-  switch (m_data_type) {
-  case DataType::Unsigned:
-    s.Printf("%" PRIu64, m_data.m_unsigned);
-    break;
-  case DataType::Signed:
-    s.Printf("%" PRId64, m_data.m_signed);
-    break;
-  case DataType::Double:
-    s.Printf("%g", m_data.m_double);
-    break;
-  }
-}
-
-JSONTrue::JSONTrue() : JSONValue(JSONValue::Kind::True) {}
-
-void JSONTrue::Write(Stream &s) { s.Printf("true"); }
-
-JSONFalse::JSONFalse() : JSONValue(JSONValue::Kind::False) {}
-
-void JSONFalse::Write(Stream &s) { s.Printf("false"); }
-
-JSONNull::JSONNull() : JSONValue(JSONValue::Kind::Null) {}
-
-void JSONNull::Write(Stream &s) { s.Printf("null"); }
-
-JSONObject::JSONObject() : JSONValue(JSONValue::Kind::Object) {}
-
-void JSONObject::Write(Stream &s) {
-  bool first = true;
-  s.PutChar('{');
-  auto iter = m_elements.begin(), end = m_elements.end();
-  for (; iter != end; iter++) {
-    if (first)
-      first = false;
-    else
-      s.PutChar(',');
-    JSONString key(iter->first);
-    JSONValue::SP value(iter->second);
-    key.Write(s);
-    s.PutChar(':');
-    value->Write(s);
-  }
-  s.PutChar('}');
-}
-
-bool JSONObject::SetObject(const std::string &key, JSONValue::SP value) {
-  if (key.empty() || nullptr == value.get())
-    return false;
-  m_elements[key] = value;
-  return true;
-}
-
-JSONValue::SP JSONObject::GetObject(const std::string &key) {
-  auto iter = m_elements.find(key), end = m_elements.end();
-  if (iter == end)
-    return JSONValue::SP();
-  return iter->second;
-}
-
-JSONArray::JSONArray() : JSONValue(JSONValue::Kind::Array) {}
-
-void JSONArray::Write(Stream &s) {
-  bool first = true;
-  s.PutChar('[');
-  auto iter = m_elements.begin(), end = m_elements.end();
-  for (; iter != end; iter++) {
-    if (first)
-      first = false;
-    else
-      s.PutChar(',');
-    (*iter)->Write(s);
-  }
-  s.PutChar(']');
-}
-
-bool JSONArray::SetObject(Index i, JSONValue::SP value) {
-  if (value.get() == nullptr)
-    return false;
-  if (i < m_elements.size()) {
-    m_elements[i] = value;
-    return true;
-  }
-  if (i == m_elements.size()) {
-    m_elements.push_back(value);
-    return true;
-  }
-  return false;
-}
-
-bool JSONArray::AppendObject(JSONValue::SP value) {
-  if (value.get() == nullptr)
-    return false;
-  m_elements.push_back(value);
-  return true;
-}
-
-JSONValue::SP JSONArray::GetObject(Index i) {
-  if (i < m_elements.size())
-    return m_elements[i];
-  return JSONValue::SP();
-}
-
-JSONArray::Size JSONArray::GetNumElements() { return m_elements.size(); }
-
-JSONParser::JSONParser(llvm::StringRef data) : StringExtractor(data) {}
-
-JSONParser::Token JSONParser::GetToken(std::string &value) {
-  StreamString error;
-
-  value.clear();
-  SkipSpaces();
-  const uint64_t start_index = m_index;
-  const char ch = GetChar();
-  switch (ch) {
-  case '{':
-    return Token::ObjectStart;
-  case '}':
-    return Token::ObjectEnd;
-  case '[':
-    return Token::ArrayStart;
-  case ']':
-    return Token::ArrayEnd;
-  case ',':
-    return Token::Comma;
-  case ':':
-    return Token::Colon;
-  case '\0':
-    return Token::EndOfFile;
-  case 't':
-    if (GetChar() == 'r')
-      if (GetChar() == 'u')
-        if (GetChar() == 'e')
-          return Token::True;
-    break;
-
-  case 'f':
-    if (GetChar() == 'a')
-      if (GetChar() == 'l')
-        if (GetChar() == 's')
-          if (GetChar() == 'e')
-            return Token::False;
-    break;
-
-  case 'n':
-    if (GetChar() == 'u')
-      if (GetChar() == 'l')
-        if (GetChar() == 'l')
-          return Token::Null;
-    break;
-
-  case '"': {
-    while (true) {
-      bool was_escaped = false;
-      int escaped_ch = GetEscapedChar(was_escaped);
-      if (escaped_ch == -1) {
-        error.Printf(
-            "error: an error occurred getting a character from offset %" PRIu64,
-            start_index);
-        value = std::move(error.GetString());
-        return Token::Status;
-
-      } else {
-        const bool is_end_quote = escaped_ch == '"';
-        const bool is_null = escaped_ch == 0;
-        if (was_escaped || (!is_end_quote && !is_null)) {
-          if (CHAR_MIN <= escaped_ch && escaped_ch <= CHAR_MAX) {
-            value.append(1, static_cast<char>(escaped_ch));
-          } else {
-            error.Printf("error: wide character support is needed for unicode "
-                         "character 0x%4.4x at offset %" PRIu64,
-                         escaped_ch, start_index);
-            value = std::move(error.GetString());
-            return Token::Status;
-          }
-        } else if (is_end_quote) {
-          return Token::String;
-        } else if (is_null) {
-          value = "error: missing end quote for string";
-          return Token::Status;
-        }
-      }
-    }
-  } break;
-
-  case '-':
-  case '0':
-  case '1':
-  case '2':
-  case '3':
-  case '4':
-  case '5':
-  case '6':
-  case '7':
-  case '8':
-  case '9': {
-    bool done = false;
-    bool got_decimal_point = false;
-    uint64_t exp_index = 0;
-    bool got_int_digits = (ch >= '0') && (ch <= '9');
-    bool got_frac_digits = false;
-    bool got_exp_digits = false;
-    while (!done) {
-      const char next_ch = PeekChar();
-      switch (next_ch) {
-      case '0':
-      case '1':
-      case '2':
-      case '3':
-      case '4':
-      case '5':
-      case '6':
-      case '7':
-      case '8':
-      case '9':
-        if (exp_index != 0) {
-          got_exp_digits = true;
-        } else if (got_decimal_point) {
-          got_frac_digits = true;
-        } else {
-          got_int_digits = true;
-        }
-        ++m_index; // Skip this character
-        break;
-
-      case '.':
-        if (got_decimal_point) {
-          error.Printf("error: extra decimal point found at offset %" PRIu64,
-                       start_index);
-          value = std::move(error.GetString());
-          return Token::Status;
-        } else {
-          got_decimal_point = true;
-          ++m_index; // Skip this character
-        }
-        break;
-
-      case 'e':
-      case 'E':
-        if (exp_index != 0) {
-          error.Printf(
-              "error: extra exponent character found at offset %" PRIu64,
-              start_index);
-          value = std::move(error.GetString());
-          return Token::Status;
-        } else {
-          exp_index = m_index;
-          ++m_index; // Skip this character
-        }
-        break;
-
-      case '+':
-      case '-':
-        // The '+' and '-' can only come after an exponent character...
-        if (exp_index == m_index - 1) {
-          ++m_index; // Skip the exponent sign character
-        } else {
-          error.Printf("error: unexpected %c character at offset %" PRIu64,
-                       next_ch, start_index);
-          value = std::move(error.GetString());
-          return Token::Status;
-        }
-        break;
-
-      default:
-        done = true;
-        break;
-      }
-    }
-
-    if (m_index > start_index) {
-      value = m_packet.substr(start_index, m_index - start_index);
-      if (got_decimal_point) {
-        if (exp_index != 0) {
-          // We have an exponent, make sure we got exponent digits
-          if (got_exp_digits) {
-            return Token::Float;
-          } else {
-            error.Printf("error: got exponent character but no exponent digits "
-                         "at offset in float value \"%s\"",
-                         value.c_str());
-            value = std::move(error.GetString());
-            return Token::Status;
-          }
-        } else {
-          // No exponent, but we need at least one decimal after the decimal
-          // point
-          if (got_frac_digits) {
-            return Token::Float;
-          } else {
-            error.Printf("error: no digits after decimal point \"%s\"",
-                         value.c_str());
-            value = std::move(error.GetString());
-            return Token::Status;
-          }
-        }
-      } else {
-        // No decimal point
-        if (got_int_digits) {
-          // We need at least some integer digits to make an integer
-          return Token::Integer;
-        } else {
-          error.Printf("error: no digits negate sign \"%s\"", value.c_str());
-          value = std::move(error.GetString());
-          return Token::Status;
-        }
-      }
-    } else {
-      error.Printf("error: invalid number found at offset %" PRIu64,
-                   start_index);
-      value = std::move(error.GetString());
-      return Token::Status;
-    }
-  } break;
-  default:
-    break;
-  }
-  error.Printf("error: failed to parse token at offset %" PRIu64
-               " (around character '%c')",
-               start_index, ch);
-  value = std::move(error.GetString());
-  return Token::Status;
-}
-
-int JSONParser::GetEscapedChar(bool &was_escaped) {
-  was_escaped = false;
-  const char ch = GetChar();
-  if (ch == '\\') {
-    was_escaped = true;
-    const char ch2 = GetChar();
-    switch (ch2) {
-    case '"':
-    case '\\':
-    case '/':
-    default:
-      break;
-
-    case 'b':
-      return '\b';
-    case 'f':
-      return '\f';
-    case 'n':
-      return '\n';
-    case 'r':
-      return '\r';
-    case 't':
-      return '\t';
-    case 'u': {
-      const int hi_byte = DecodeHexU8();
-      const int lo_byte = DecodeHexU8();
-      if (hi_byte >= 0 && lo_byte >= 0)
-        return hi_byte << 8 | lo_byte;
-      return -1;
-    } break;
-    }
-    return ch2;
-  }
-  return ch;
-}
-
-JSONValue::SP JSONParser::ParseJSONObject() {
-  // The "JSONParser::Token::ObjectStart" token should have already been
-  // consumed by the time this function is called
-  std::unique_ptr<JSONObject> dict_up(new JSONObject());
-
-  std::string value;
-  std::string key;
-  while (true) {
-    JSONParser::Token token = GetToken(value);
-
-    if (token == JSONParser::Token::String) {
-      key.swap(value);
-      token = GetToken(value);
-      if (token == JSONParser::Token::Colon) {
-        JSONValue::SP value_sp = ParseJSONValue();
-        if (value_sp)
-          dict_up->SetObject(key, value_sp);
-        else
-          break;
-      }
-    } else if (token == JSONParser::Token::ObjectEnd) {
-      return JSONValue::SP(dict_up.release());
-    } else if (token == JSONParser::Token::Comma) {
-      continue;
-    } else {
-      break;
-    }
-  }
-  return JSONValue::SP();
-}
-
-JSONValue::SP JSONParser::ParseJSONArray() {
-  // The "JSONParser::Token::ObjectStart" token should have already been
-  // consumed by the time this function is called
-  std::unique_ptr<JSONArray> array_up(new JSONArray());
-
-  std::string value;
-  std::string key;
-  while (true) {
-    JSONValue::SP value_sp = ParseJSONValue();
-    if (value_sp)
-      array_up->AppendObject(value_sp);
-    else
-      break;
-
-    JSONParser::Token token = GetToken(value);
-    if (token == JSONParser::Token::Comma) {
-      continue;
-    } else if (token == JSONParser::Token::ArrayEnd) {
-      return JSONValue::SP(array_up.release());
-    } else {
-      break;
-    }
-  }
-  return JSONValue::SP();
-}
-
-JSONValue::SP JSONParser::ParseJSONValue() {
-  std::string value;
-  const JSONParser::Token token = GetToken(value);
-  switch (token) {
-  case JSONParser::Token::ObjectStart:
-    return ParseJSONObject();
-
-  case JSONParser::Token::ArrayStart:
-    return ParseJSONArray();
-
-  case JSONParser::Token::Integer: {
-    if (value.front() == '-') {
-      int64_t sval = 0;
-      if (!llvm::StringRef(value).getAsInteger(0, sval))
-        return JSONValue::SP(new JSONNumber(sval));
-    } else {
-      uint64_t uval = 0;
-      if (!llvm::StringRef(value).getAsInteger(0, uval))
-        return JSONValue::SP(new JSONNumber(uval));
-    }
-  } break;
-
-  case JSONParser::Token::Float: {
-    double D;
-    if (!llvm::StringRef(value).getAsDouble(D))
-      return JSONValue::SP(new JSONNumber(D));
-  } break;
-
-  case JSONParser::Token::String:
-    return JSONValue::SP(new JSONString(value));
-
-  case JSONParser::Token::True:
-    return JSONValue::SP(new JSONTrue());
-
-  case JSONParser::Token::False:
-    return JSONValue::SP(new JSONFalse());
-
-  case JSONParser::Token::Null:
-    return JSONValue::SP(new JSONNull());
-
-  default:
-    break;
-  }
-  return JSONValue::SP();
-}
diff --git a/lldb/source/Utility/Log.cpp b/lldb/source/Utility/Log.cpp
index 217b0d2ba97b1..ab5e630114a6b 100644
--- a/lldb/source/Utility/Log.cpp
+++ b/lldb/source/Utility/Log.cpp
@@ -9,7 +9,6 @@
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/VASPrintf.h"
 
-#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/ADT/iterator.h"
@@ -38,13 +37,22 @@ using namespace lldb_private;
 
 llvm::ManagedStatic<Log::ChannelMap> Log::g_channel_map;
 
-void Log::ListCategories(llvm::raw_ostream &stream, const ChannelMap::value_type &entry) {
-  stream << llvm::formatv("Logging categories for '{0}':\n", entry.first());
-  stream << "  all - all available logging categories\n";
-  stream << "  default - default set of logging categories\n";
+void Log::ForEachCategory(
+    const Log::ChannelMap::value_type &entry,
+    llvm::function_ref<void(llvm::StringRef, llvm::StringRef)> lambda) {
+  lambda("all", "all available logging categories");
+  lambda("default", "default set of logging categories");
   for (const auto &category : entry.second.m_channel.categories)
-    stream << llvm::formatv("  {0} - {1}\n", category.name,
-                            category.description);
+    lambda(category.name, category.description);
+}
+
+void Log::ListCategories(llvm::raw_ostream &stream,
+                         const ChannelMap::value_type &entry) {
+  stream << llvm::formatv("Logging categories for '{0}':\n", entry.first());
+  ForEachCategory(entry,
+                  [&stream](llvm::StringRef name, llvm::StringRef description) {
+                    stream << llvm::formatv("  {0} - {1}\n", name, description);
+                  });
 }
 
 uint32_t Log::GetFlags(llvm::raw_ostream &stream, const ChannelMap::value_type &entry,
@@ -237,6 +245,23 @@ void Log::DisableAllLogChannels() {
     entry.second.Disable(UINT32_MAX);
 }
 
+void Log::ForEachChannelCategory(
+    llvm::StringRef channel,
+    llvm::function_ref<void(llvm::StringRef, llvm::StringRef)> lambda) {
+  auto ch = g_channel_map->find(channel);
+  if (ch == g_channel_map->end())
+    return;
+
+  ForEachCategory(*ch, lambda);
+}
+
+std::vector<llvm::StringRef> Log::ListChannels() {
+  std::vector<llvm::StringRef> result;
+  for (const auto &channel : *g_channel_map)
+    result.push_back(channel.first());
+  return result;
+}
+
 void Log::ListAllLogChannels(llvm::raw_ostream &stream) {
   if (g_channel_map->empty()) {
     stream << "No logging channels are currently registered.\n";
diff --git a/lldb/source/Utility/Reproducer.cpp b/lldb/source/Utility/Reproducer.cpp
index 8e44f08017725..a8a581f0761a0 100644
--- a/lldb/source/Utility/Reproducer.cpp
+++ b/lldb/source/Utility/Reproducer.cpp
@@ -136,7 +136,15 @@ FileSpec Reproducer::GetReproducerPath() const {
   return {};
 }
 
-Generator::Generator(const FileSpec &root) : m_root(root), m_done(false) {}
+static FileSpec MakeAbsolute(FileSpec file_spec) {
+  SmallString<128> path;
+  file_spec.GetPath(path, false);
+  llvm::sys::fs::make_absolute(path);
+  return FileSpec(path, file_spec.GetPathStyle());
+}
+
+Generator::Generator(FileSpec root)
+    : m_root(MakeAbsolute(std::move(root))), m_done(false) {}
 
 Generator::~Generator() {}
 
@@ -188,7 +196,8 @@ void Generator::AddProvidersToIndex() {
   yout << files;
 }
 
-Loader::Loader(const FileSpec &root) : m_root(root), m_loaded(false) {}
+Loader::Loader(FileSpec root)
+    : m_root(MakeAbsolute(std::move(root))), m_loaded(false) {}
 
 llvm::Error Loader::LoadIndex() {
   if (m_loaded)
diff --git a/lldb/source/Utility/Scalar.cpp b/lldb/source/Utility/Scalar.cpp
index 3393af921ddac..e9aec17b6650d 100644
--- a/lldb/source/Utility/Scalar.cpp
+++ b/lldb/source/Utility/Scalar.cpp
@@ -434,7 +434,7 @@ Scalar::Type Scalar::GetBestTypeForBitSize(size_t bit_size, bool sign) {
     if (bit_size <= 512) return Scalar::e_uint512;
   }
   return Scalar::e_void;
-};
+}
 
 void Scalar::TruncOrExtendTo(Scalar::Type type, uint16_t bits) {
   switch (type) {
diff --git a/lldb/source/Utility/SelectHelper.cpp b/lldb/source/Utility/SelectHelper.cpp
index ff21d99e400ab..9f5ca586e1ef9 100644
--- a/lldb/source/Utility/SelectHelper.cpp
+++ b/lldb/source/Utility/SelectHelper.cpp
@@ -92,7 +92,7 @@ static void updateMaxFd(llvm::Optional<lldb::socket_t> &vold,
 
 lldb_private::Status SelectHelper::Select() {
   lldb_private::Status error;
-#ifdef _MSC_VER
+#ifdef _WIN32
   // On windows FD_SETSIZE limits the number of file descriptors, not their
   // numeric value.
   lldbassert(m_fd_map.size() <= FD_SETSIZE);
@@ -107,7 +107,7 @@ lldb_private::Status SelectHelper::Select() {
   for (auto &pair : m_fd_map) {
     pair.second.PrepareForSelect();
     const lldb::socket_t fd = pair.first;
-#if !defined(__APPLE__) && !defined(_MSC_VER)
+#if !defined(__APPLE__) && !defined(_WIN32)
     lldbassert(fd < static_cast<int>(FD_SETSIZE));
     if (fd >= static_cast<int>(FD_SETSIZE)) {
       error.SetErrorStringWithFormat("%i is too large for select()", fd);
diff --git a/lldb/source/Utility/StreamGDBRemote.cpp b/lldb/source/Utility/StreamGDBRemote.cpp
deleted file mode 100644
index c710bbe3eecb3..0000000000000
--- a/lldb/source/Utility/StreamGDBRemote.cpp
+++ /dev/null
@@ -1,45 +0,0 @@
-//===-- StreamGDBRemote.cpp -------------------------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "lldb/Utility/StreamGDBRemote.h"
-
-#include "lldb/Utility/Flags.h"
-#include "lldb/Utility/Stream.h"
-
-#include <stdio.h>
-
-using namespace lldb;
-using namespace lldb_private;
-
-StreamGDBRemote::StreamGDBRemote() : StreamString() {}
-
-StreamGDBRemote::StreamGDBRemote(uint32_t flags, uint32_t addr_size,
-                                 ByteOrder byte_order)
-    : StreamString(flags, addr_size, byte_order) {}
-
-StreamGDBRemote::~StreamGDBRemote() {}
-
-int StreamGDBRemote::PutEscapedBytes(const void *s, size_t src_len) {
-  int bytes_written = 0;
-  const uint8_t *src = static_cast<const uint8_t *>(s);
-  bool binary_is_set = m_flags.Test(eBinary);
-  m_flags.Clear(eBinary);
-  while (src_len) {
-    uint8_t byte = *src;
-    src++;
-    src_len--;
-    if (byte == 0x23 || byte == 0x24 || byte == 0x7d || byte == 0x2a) {
-      bytes_written += PutChar(0x7d);
-      byte ^= 0x20;
-    }
-    bytes_written += PutChar(byte);
-  };
-  if (binary_is_set)
-    m_flags.Set(eBinary);
-  return bytes_written;
-}
diff --git a/lldb/source/Utility/StructuredData.cpp b/lldb/source/Utility/StructuredData.cpp
index fde5093428f6a..783a080821744 100644
--- a/lldb/source/Utility/StructuredData.cpp
+++ b/lldb/source/Utility/StructuredData.cpp
@@ -9,9 +9,7 @@
 #include "lldb/Utility/StructuredData.h"
 #include "lldb/Utility/DataBuffer.h"
 #include "lldb/Utility/FileSpec.h"
-#include "lldb/Utility/JSON.h"
 #include "lldb/Utility/Status.h"
-#include "lldb/Utility/Stream.h"
 #include "lldb/Utility/StreamString.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -21,11 +19,20 @@
 #include <limits>
 
 using namespace lldb_private;
+using namespace llvm;
 
-// Functions that use a JSONParser to parse JSON into StructuredData
-static StructuredData::ObjectSP ParseJSONValue(JSONParser &json_parser);
-static StructuredData::ObjectSP ParseJSONObject(JSONParser &json_parser);
-static StructuredData::ObjectSP ParseJSONArray(JSONParser &json_parser);
+static StructuredData::ObjectSP ParseJSONValue(json::Value &value);
+static StructuredData::ObjectSP ParseJSONObject(json::Object *object);
+static StructuredData::ObjectSP ParseJSONArray(json::Array *array);
+
+StructuredData::ObjectSP StructuredData::ParseJSON(std::string json_text) {
+  llvm::Expected<json::Value> value = json::parse(json_text);
+  if (!value) {
+    llvm::consumeError(value.takeError());
+    return nullptr;
+  }
+  return ParseJSONValue(*value);
+}
 
 StructuredData::ObjectSP
 StructuredData::ParseJSONFromFile(const FileSpec &input_spec, Status &error) {
@@ -38,112 +45,53 @@ StructuredData::ParseJSONFromFile(const FileSpec &input_spec, Status &error) {
                                     buffer_or_error.getError().message());
     return return_sp;
   }
-
-  JSONParser json_parser(buffer_or_error.get()->getBuffer());
-  return_sp = ParseJSONValue(json_parser);
-  return return_sp;
+  return ParseJSON(buffer_or_error.get()->getBuffer().str());
 }
 
-static StructuredData::ObjectSP ParseJSONObject(JSONParser &json_parser) {
-  // The "JSONParser::Token::ObjectStart" token should have already been
-  // consumed by the time this function is called
-  auto dict_up = std::make_unique<StructuredData::Dictionary>();
+static StructuredData::ObjectSP ParseJSONValue(json::Value &value) {
+  if (json::Object *O = value.getAsObject())
+    return ParseJSONObject(O);
 
-  std::string value;
-  std::string key;
-  while (true) {
-    JSONParser::Token token = json_parser.GetToken(value);
+  if (json::Array *A = value.getAsArray())
+    return ParseJSONArray(A);
 
-    if (token == JSONParser::Token::String) {
-      key.swap(value);
-      token = json_parser.GetToken(value);
-      if (token == JSONParser::Token::Colon) {
-        StructuredData::ObjectSP value_sp = ParseJSONValue(json_parser);
-        if (value_sp)
-          dict_up->AddItem(key, value_sp);
-        else
-          break;
-      }
-    } else if (token == JSONParser::Token::ObjectEnd) {
-      return StructuredData::ObjectSP(dict_up.release());
-    } else if (token == JSONParser::Token::Comma) {
-      continue;
-    } else {
-      break;
-    }
-  }
-  return StructuredData::ObjectSP();
-}
+  std::string s;
+  if (json::fromJSON(value, s))
+    return std::make_shared<StructuredData::String>(s);
 
-static StructuredData::ObjectSP ParseJSONArray(JSONParser &json_parser) {
-  // The "JSONParser::Token::ObjectStart" token should have already been
-  // consumed by the time this function is called
-  auto array_up = std::make_unique<StructuredData::Array>();
+  bool b;
+  if (json::fromJSON(value, b))
+    return std::make_shared<StructuredData::Boolean>(b);
 
-  std::string value;
-  std::string key;
-  while (true) {
-    StructuredData::ObjectSP value_sp = ParseJSONValue(json_parser);
-    if (value_sp)
-      array_up->AddItem(value_sp);
-    else
-      break;
+  int64_t i;
+  if (json::fromJSON(value, i))
+    return std::make_shared<StructuredData::Integer>(i);
+
+  double d;
+  if (json::fromJSON(value, d))
+    return std::make_shared<StructuredData::Float>(d);
 
-    JSONParser::Token token = json_parser.GetToken(value);
-    if (token == JSONParser::Token::Comma) {
-      continue;
-    } else if (token == JSONParser::Token::ArrayEnd) {
-      return StructuredData::ObjectSP(array_up.release());
-    } else {
-      break;
-    }
-  }
   return StructuredData::ObjectSP();
 }
 
-static StructuredData::ObjectSP ParseJSONValue(JSONParser &json_parser) {
-  std::string value;
-  const JSONParser::Token token = json_parser.GetToken(value);
-  switch (token) {
-  case JSONParser::Token::ObjectStart:
-    return ParseJSONObject(json_parser);
-
-  case JSONParser::Token::ArrayStart:
-    return ParseJSONArray(json_parser);
-
-  case JSONParser::Token::Integer: {
-    uint64_t uval;
-    if (llvm::to_integer(value, uval, 0))
-      return std::make_shared<StructuredData::Integer>(uval);
-  } break;
-
-  case JSONParser::Token::Float: {
-    double val;
-    if (llvm::to_float(value, val))
-      return std::make_shared<StructuredData::Float>(val);
-  } break;
-
-  case JSONParser::Token::String:
-    return std::make_shared<StructuredData::String>(value);
-
-  case JSONParser::Token::True:
-  case JSONParser::Token::False:
-    return std::make_shared<StructuredData::Boolean>(token ==
-                                                     JSONParser::Token::True);
-
-  case JSONParser::Token::Null:
-    return std::make_shared<StructuredData::Null>();
-
-  default:
-    break;
+static StructuredData::ObjectSP ParseJSONObject(json::Object *object) {
+  auto dict_up = std::make_unique<StructuredData::Dictionary>();
+  for (auto &KV : *object) {
+    StringRef key = KV.first;
+    json::Value value = KV.second;
+    if (StructuredData::ObjectSP value_sp = ParseJSONValue(value))
+      dict_up->AddItem(key, value_sp);
   }
-  return StructuredData::ObjectSP();
+  return dict_up;
 }
 
-StructuredData::ObjectSP StructuredData::ParseJSON(std::string json_text) {
-  JSONParser json_parser(json_text);
-  StructuredData::ObjectSP object_sp = ParseJSONValue(json_parser);
-  return object_sp;
+static StructuredData::ObjectSP ParseJSONArray(json::Array *array) {
+  auto array_up = std::make_unique<StructuredData::Array>();
+  for (json::Value &value : *array) {
+    if (StructuredData::ObjectSP value_sp = ParseJSONValue(value))
+      array_up->AddItem(value_sp);
+  }
+  return array_up;
 }
 
 StructuredData::ObjectSP
@@ -181,98 +129,48 @@ StructuredData::Object::GetObjectForDotSeparatedPath(llvm::StringRef path) {
 }
 
 void StructuredData::Object::DumpToStdout(bool pretty_print) const {
-  StreamString stream;
-  Dump(stream, pretty_print);
-  llvm::outs() << stream.GetString();
+  json::OStream stream(llvm::outs(), pretty_print ? 2 : 0);
+  Serialize(stream);
 }
 
-void StructuredData::Array::Dump(Stream &s, bool pretty_print) const {
-  bool first = true;
-  s << "[";
-  if (pretty_print) {
-    s << "\n";
-    s.IndentMore();
-  }
+void StructuredData::Array::Serialize(json::OStream &s) const {
+  s.arrayBegin();
   for (const auto &item_sp : m_items) {
-    if (first) {
-      first = false;
-    } else {
-      s << ",";
-      if (pretty_print)
-        s << "\n";
-    }
-
-    if (pretty_print)
-      s.Indent();
-    item_sp->Dump(s, pretty_print);
-  }
-  if (pretty_print) {
-    s.IndentLess();
-    s.EOL();
-    s.Indent();
+    item_sp->Serialize(s);
   }
-  s << "]";
+  s.arrayEnd();
 }
 
-void StructuredData::Integer::Dump(Stream &s, bool pretty_print) const {
-  s.Printf("%" PRIu64, m_value);
+void StructuredData::Integer::Serialize(json::OStream &s) const {
+  s.value(static_cast<int64_t>(m_value));
 }
 
-void StructuredData::Float::Dump(Stream &s, bool pretty_print) const {
-  s.Printf("%lg", m_value);
+void StructuredData::Float::Serialize(json::OStream &s) const {
+  s.value(m_value);
 }
 
-void StructuredData::Boolean::Dump(Stream &s, bool pretty_print) const {
-  if (m_value)
-    s.PutCString("true");
-  else
-    s.PutCString("false");
+void StructuredData::Boolean::Serialize(json::OStream &s) const {
+  s.value(m_value);
 }
 
-void StructuredData::String::Dump(Stream &s, bool pretty_print) const {
-  std::string quoted;
-  const size_t strsize = m_value.size();
-  for (size_t i = 0; i < strsize; ++i) {
-    char ch = m_value[i];
-    if (ch == '"' || ch == '\\')
-      quoted.push_back('\\');
-    quoted.push_back(ch);
-  }
-  s.Printf("\"%s\"", quoted.c_str());
+void StructuredData::String::Serialize(json::OStream &s) const {
+  s.value(m_value);
 }
 
-void StructuredData::Dictionary::Dump(Stream &s, bool pretty_print) const {
-  bool first = true;
-  s << "{";
-  if (pretty_print) {
-    s << "\n";
-    s.IndentMore();
-  }
+void StructuredData::Dictionary::Serialize(json::OStream &s) const {
+  s.objectBegin();
   for (const auto &pair : m_dict) {
-    if (first)
-      first = false;
-    else {
-      s << ",";
-      if (pretty_print)
-        s << "\n";
-    }
-    if (pretty_print)
-      s.Indent();
-    s << "\"" << pair.first.AsCString() << "\" : ";
-    pair.second->Dump(s, pretty_print);
-  }
-  if (pretty_print) {
-    s.IndentLess();
-    s.EOL();
-    s.Indent();
+    s.attributeBegin(pair.first.AsCString());
+    pair.second->Serialize(s);
+    s.attributeEnd();
   }
-  s << "}";
+  s.objectEnd();
 }
 
-void StructuredData::Null::Dump(Stream &s, bool pretty_print) const {
-  s << "null";
+void StructuredData::Null::Serialize(json::OStream &s) const {
+  s.value(nullptr);
 }
 
-void StructuredData::Generic::Dump(Stream &s, bool pretty_print) const {
-  s << "0x" << m_object;
+void StructuredData::Generic::Serialize(json::OStream &s) const {
+  s.value(llvm::formatv("{0:X}", m_object));
 }
diff --git a/lldb/test/CMakeLists.txt b/lldb/test/CMakeLists.txt
index 3bfb2332862c1..41dd30ec77c08 100644
--- a/lldb/test/CMakeLists.txt
+++ b/lldb/test/CMakeLists.txt
@@ -97,8 +97,19 @@ if(CMAKE_HOST_APPLE)
 
   if(LLDB_USE_SYSTEM_DEBUGSERVER)
     lldb_find_system_debugserver(system_debugserver_path)
+    add_custom_target(debugserver
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      ${system_debugserver_path} ${LLVM_RUNTIME_OUTPUT_INTDIR}
+      COMMENT "Copying the system debugserver to LLDB's binaries directory for testing.")
+    # The custom target for the system debugserver has no install target, so we
+    # need to remove it from the LLVM_DISTRIBUTION_COMPONENTS list.
+    if (LLVM_DISTRIBUTION_COMPONENTS)
+      list(REMOVE_ITEM LLVM_DISTRIBUTION_COMPONENTS debugserver)
+      set(LLVM_DISTRIBUTION_COMPONENTS ${LLVM_DISTRIBUTION_COMPONENTS} PARENT_SCOPE)
+    endif()
     message(STATUS "LLDB tests use out-of-tree debugserver: ${system_debugserver_path}")
     list(APPEND LLDB_TEST_COMMON_ARGS --out-of-tree-debugserver)
+    add_dependencies(lldb-test-deps debugserver)
   elseif(TARGET debugserver)
     set(debugserver_path ${LLVM_RUNTIME_OUTPUT_INTDIR}/debugserver)
     message(STATUS "LLDB Tests use just-built debugserver: ${debugserver_path}")
diff --git a/lldb/third_party/Python/module/unittest2/unittest2/loader.py b/lldb/third_party/Python/module/unittest2/unittest2/loader.py
index 996dd76647579..87edddca40281 100644
--- a/lldb/third_party/Python/module/unittest2/unittest2/loader.py
+++ b/lldb/third_party/Python/module/unittest2/unittest2/loader.py
@@ -117,7 +117,7 @@ def loadTestsFromName(self, name, module=None):
             return self.loadTestsFromModule(obj)
         elif isinstance(obj, type) and issubclass(obj, unittest.TestCase):
             return self.loadTestsFromTestCase(obj)
-        elif (isinstance(obj, types.UnboundMethodType) and
+        elif (isinstance(obj, (types.MethodType, types.FunctionType)) and
               isinstance(parent, type) and
               issubclass(parent, case.TestCase)):
             return self.suiteClass([parent(obj.__name__)])
diff --git a/lldb/tools/CMakeLists.txt b/lldb/tools/CMakeLists.txt
index 3556f9eceae1a..1585fd4dc4b9e 100644
--- a/lldb/tools/CMakeLists.txt
+++ b/lldb/tools/CMakeLists.txt
@@ -12,7 +12,9 @@ add_lldb_tool_subdirectory(lldb-vscode)
 
 if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_lldb_tool_subdirectory(darwin-debug)
-  add_lldb_tool_subdirectory(debugserver)
+  if(NOT LLDB_USE_SYSTEM_DEBUGSERVER)
+    add_lldb_tool_subdirectory(debugserver)
+  endif()
 endif()
 
 if (LLDB_CAN_USE_LLDB_SERVER)
diff --git a/lldb/tools/argdumper/CMakeLists.txt b/lldb/tools/argdumper/CMakeLists.txt
index d0767483781fe..924946325194a 100644
--- a/lldb/tools/argdumper/CMakeLists.txt
+++ b/lldb/tools/argdumper/CMakeLists.txt
@@ -1,6 +1,6 @@
 add_lldb_tool(lldb-argdumper ADD_TO_FRAMEWORK
   argdumper.cpp
 
-  LINK_LIBS
-    lldbUtility
+  LINK_COMPONENTS
+    Support
 )
diff --git a/lldb/tools/argdumper/argdumper.cpp b/lldb/tools/argdumper/argdumper.cpp
index 7fd8999c5dced..1cf0d6dae7a46 100644
--- a/lldb/tools/argdumper/argdumper.cpp
+++ b/lldb/tools/argdumper/argdumper.cpp
@@ -6,27 +6,15 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lldb/Utility/JSON.h"
-#include "lldb/Utility/StreamString.h"
+#include "llvm/Support/JSON.h"
 
-#include <iostream>
-
-using namespace lldb_private;
+using namespace llvm;
 
 int main(int argc, char *argv[]) {
-  JSONArray::SP arguments(new JSONArray());
+  json::Array Arguments;
   for (int i = 1; i < argc; i++) {
-    arguments->AppendObject(JSONString::SP(new JSONString(argv[i])));
+    Arguments.push_back(argv[i]);
   }
-
-  JSONObject::SP object(new JSONObject());
-  object->SetObject("arguments", arguments);
-
-  StreamString ss;
-
-  object->Write(ss);
-
-  std::cout << ss.GetData() << std::endl;
-
+  llvm::outs() << json::Object({{"arguments", std::move(Arguments)}});
   return 0;
 }
diff --git a/lldb/tools/debugserver/source/CMakeLists.txt b/lldb/tools/debugserver/source/CMakeLists.txt
index aea8bb5d8e67b..cfb46817e6dfb 100644
--- a/lldb/tools/debugserver/source/CMakeLists.txt
+++ b/lldb/tools/debugserver/source/CMakeLists.txt
@@ -28,7 +28,7 @@ function(get_debugserver_codesign_identity result)
   if(LLDB_CODESIGN_IDENTITY)
     set(${result} ${LLDB_CODESIGN_IDENTITY} PARENT_SCOPE)
     check_certificate(${LLDB_CODESIGN_IDENTITY} available)
-    if(NOT available AND NOT LLDB_USE_SYSTEM_DEBUGSERVER)
+    if(NOT available)
       message(WARNING "LLDB_CODESIGN_IDENTITY not found: '${LLDB_CODESIGN_IDENTITY}' ${not_found_help}")
     endif()
     return()
@@ -41,9 +41,7 @@ function(get_debugserver_codesign_identity result)
     return()
   endif()
 
-  if(NOT LLDB_USE_SYSTEM_DEBUGSERVER)
-    message(WARNING "Development code sign identiy not found: 'lldb_codesign' ${not_found_help}")
-  endif()
+  message(WARNING "Development code sign identiy not found: 'lldb_codesign' ${not_found_help}")
 
   # LLVM pendant: fallback if available
   if(LLVM_CODESIGNING_IDENTITY)
diff --git a/lldb/tools/debugserver/source/debugserver.cpp b/lldb/tools/debugserver/source/debugserver.cpp
index fa19bba58fa79..42205dedf4bb8 100644
--- a/lldb/tools/debugserver/source/debugserver.cpp
+++ b/lldb/tools/debugserver/source/debugserver.cpp
@@ -789,6 +789,12 @@ void FileLogCallback(void *baton, uint32_t flags, const char *format,
   ::fflush((FILE *)baton);
 }
 
+void show_version_and_exit(int exit_code) {
+  printf("%s-%s for %s.\n", DEBUGSERVER_PROGRAM_NAME, DEBUGSERVER_VERSION_STR,
+         RNB_ARCH);
+  exit(exit_code);
+}
+
 void show_usage_and_exit(int exit_code) {
   RNBLogSTDERR(
       "Usage:\n  %s host:port [program-name program-arg1 program-arg2 ...]\n",
@@ -811,6 +817,7 @@ static struct option g_long_options[] = {
     {"debug", no_argument, NULL, 'g'},
     {"kill-on-error", no_argument, NULL, 'K'},
     {"verbose", no_argument, NULL, 'v'},
+    {"version", no_argument, NULL, 'V'},
     {"lockdown", no_argument, &g_lockdown_opt, 1}, // short option "-k"
     {"applist", no_argument, &g_applist_opt, 1},   // short option "-t"
     {"log-file", required_argument, NULL, 'l'},
@@ -1173,6 +1180,10 @@ int main(int argc, char *argv[]) {
       DNBLogSetVerbose(1);
       break;
 
+    case 'V':
+      show_version_and_exit(0);
+      break;
+
     case 's':
       ctx.GetSTDIN().assign(optarg);
       ctx.GetSTDOUT().assign(optarg);
diff --git a/lldb/tools/lldb-server/lldb-gdbserver.cpp b/lldb/tools/lldb-server/lldb-gdbserver.cpp
index 493eaf5674154..204c67bc470ff 100644
--- a/lldb/tools/lldb-server/lldb-gdbserver.cpp
+++ b/lldb/tools/lldb-server/lldb-gdbserver.cpp
@@ -108,10 +108,10 @@ static struct option g_long_options[] = {
     {"fd", required_argument, nullptr, 'F'},
     {nullptr, 0, nullptr, 0}};
 
+#ifndef _WIN32
 // Watch for signals
 static int g_sighup_received_count = 0;
 
-#ifndef _WIN32
 static void sighup_handler(MainLoopBase &mainloop) {
   ++g_sighup_received_count;
 
diff --git a/lldb/tools/lldb-server/lldb-platform.cpp b/lldb/tools/lldb-server/lldb-platform.cpp
index ee438da07fe76..a6fb5639d6422 100644
--- a/lldb/tools/lldb-server/lldb-platform.cpp
+++ b/lldb/tools/lldb-server/lldb-platform.cpp
@@ -22,6 +22,7 @@
 
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/raw_ostream.h"
 
 #include "Acceptor.h"
 #include "LLDBServerUtilities.h"
@@ -103,29 +104,36 @@ static Status save_socket_id_to_file(const std::string &socket_id,
 
   llvm::SmallString<64> temp_file_path;
   temp_file_spec.AppendPathComponent("port-file.%%%%%%");
-  int FD;
-  auto err_code = llvm::sys::fs::createUniqueFile(temp_file_spec.GetPath(), FD,
-                                                  temp_file_path);
-  if (err_code)
-    return Status("Failed to create temp file: %s", err_code.message().c_str());
-
-  llvm::FileRemover tmp_file_remover(temp_file_path);
-
-  {
-    llvm::raw_fd_ostream temp_file(FD, true);
-    temp_file << socket_id;
-    temp_file.close();
-    if (temp_file.has_error())
-      return Status("Failed to write to port file.");
-  }
-
-  err_code = llvm::sys::fs::rename(temp_file_path, file_spec.GetPath());
-  if (err_code)
-    return Status("Failed to rename file %s to %s: %s", temp_file_path.c_str(),
-                  file_spec.GetPath().c_str(), err_code.message().c_str());
 
-  tmp_file_remover.releaseFile();
-  return Status();
+  Status status;
+  if (auto Err =
+          handleErrors(llvm::writeFileAtomically(
+                           temp_file_path, temp_file_spec.GetPath(), socket_id),
+                       [&status, &file_spec](const AtomicFileWriteError &E) {
+                         std::string ErrorMsgBuffer;
+                         llvm::raw_string_ostream S(ErrorMsgBuffer);
+                         E.log(S);
+
+                         switch (E.Error) {
+                         case atomic_write_error::failed_to_create_uniq_file:
+                           status = Status("Failed to create temp file: %s",
+                                           ErrorMsgBuffer.c_str());
+                           break;
+                         case atomic_write_error::output_stream_error:
+                           status = Status("Failed to write to port file.");
+                           break;
+                         case atomic_write_error::failed_to_rename_temp_file:
+                           status = Status("Failed to rename file %s to %s: %s",
+                                           ErrorMsgBuffer.c_str(),
+                                           file_spec.GetPath().c_str(),
+                                           ErrorMsgBuffer.c_str());
+                           break;
+                         }
+                       })) {
+    return Status("Failed to atomically write file %s",
+                  file_spec.GetPath().c_str());
+  }
+  return status;
 }
 
 // main
diff --git a/lldb/tools/lldb-test/SystemInitializerTest.cpp b/lldb/tools/lldb-test/SystemInitializerTest.cpp
index 44b960cca101e..8841e4d6583af 100644
--- a/lldb/tools/lldb-test/SystemInitializerTest.cpp
+++ b/lldb/tools/lldb-test/SystemInitializerTest.cpp
@@ -28,6 +28,7 @@
 #include "Plugins/ABI/SysV-ppc64/ABISysV_ppc64.h"
 #include "Plugins/ABI/SysV-s390x/ABISysV_s390x.h"
 #include "Plugins/ABI/SysV-x86_64/ABISysV_x86_64.h"
+#include "Plugins/ABI/Windows-x86_64/ABIWindows_x86_64.h"
 #include "Plugins/Architecture/Arm/ArchitectureArm.h"
 #include "Plugins/Architecture/PPC64/ArchitecturePPC64.h"
 #include "Plugins/Disassembler/llvm/DisassemblerLLVMC.h"
@@ -111,6 +112,38 @@ SystemInitializerTest::SystemInitializerTest() {}
 
 SystemInitializerTest::~SystemInitializerTest() {}
 
+#define LLDB_PROCESS_AArch64(op)                                               \
+  ABIMacOSX_arm64::op();                                                       \
+  ABISysV_arm64::op();
+#define LLDB_PROCESS_ARM(op)                                                   \
+  ABIMacOSX_arm::op();                                                         \
+  ABISysV_arm::op();
+#define LLDB_PROCESS_Hexagon(op) ABISysV_hexagon::op();
+#define LLDB_PROCESS_Mips(op)                                                  \
+  ABISysV_mips::op();                                                          \
+  ABISysV_mips64::op();
+#define LLDB_PROCESS_PowerPC(op)                                               \
+  ABISysV_ppc::op();                                                          \
+  ABISysV_ppc64::op();
+#define LLDB_PROCESS_SystemZ(op) ABISysV_s390x::op();
+#define LLDB_PROCESS_X86(op)                                                   \
+  ABIMacOSX_i386::op();                                                        \
+  ABISysV_i386::op();                                                          \
+  ABISysV_x86_64::op();                                                        \
+  ABIWindows_x86_64::op();
+
+#define LLDB_PROCESS_AMDGPU(op)
+#define LLDB_PROCESS_ARC(op)
+#define LLDB_PROCESS_AVR(op)
+#define LLDB_PROCESS_BPF(op)
+#define LLDB_PROCESS_Lanai(op)
+#define LLDB_PROCESS_MSP430(op)
+#define LLDB_PROCESS_NVPTX(op)
+#define LLDB_PROCESS_RISCV(op)
+#define LLDB_PROCESS_Sparc(op)
+#define LLDB_PROCESS_WebAssembly(op)
+#define LLDB_PROCESS_XCore(op)
+
 llvm::Error SystemInitializerTest::Initialize() {
   if (auto e = SystemInitializerCommon::Initialize())
     return e;
@@ -144,19 +177,8 @@ llvm::Error SystemInitializerTest::Initialize() {
 
   ClangASTContext::Initialize();
 
-  ABIMacOSX_i386::Initialize();
-  ABIMacOSX_arm::Initialize();
-  ABIMacOSX_arm64::Initialize();
-  ABISysV_arm::Initialize();
-  ABISysV_arm64::Initialize();
-  ABISysV_hexagon::Initialize();
-  ABISysV_i386::Initialize();
-  ABISysV_x86_64::Initialize();
-  ABISysV_ppc::Initialize();
-  ABISysV_ppc64::Initialize();
-  ABISysV_mips::Initialize();
-  ABISysV_mips64::Initialize();
-  ABISysV_s390x::Initialize();
+#define LLVM_TARGET(t) LLDB_PROCESS_ ## t(Initialize)
+#include "llvm/Config/Targets.def"
 
   ArchitectureArm::Initialize();
   ArchitecturePPC64::Initialize();
@@ -246,19 +268,9 @@ void SystemInitializerTest::Terminate() {
 
   ClangASTContext::Terminate();
 
-  ABIMacOSX_i386::Terminate();
-  ABIMacOSX_arm::Terminate();
-  ABIMacOSX_arm64::Terminate();
-  ABISysV_arm::Terminate();
-  ABISysV_arm64::Terminate();
-  ABISysV_hexagon::Terminate();
-  ABISysV_i386::Terminate();
-  ABISysV_x86_64::Terminate();
-  ABISysV_ppc::Terminate();
-  ABISysV_ppc64::Terminate();
-  ABISysV_mips::Terminate();
-  ABISysV_mips64::Terminate();
-  ABISysV_s390x::Terminate();
+#define LLVM_TARGET(t) LLDB_PROCESS_ ## t(Terminate)
+#include "llvm/Config/Targets.def"
+
   DisassemblerLLVMC::Terminate();
 
   JITLoaderGDB::Terminate();
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 001d52d94bbda..112c2fa876404 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -519,10 +519,10 @@ Error opts::symbols::findTypes(lldb_private::Module &Module) {
   DenseSet<SymbolFile *> SearchedFiles;
   TypeMap Map;
   if (!Name.empty())
-    Symfile.FindTypes(ConstString(Name), ContextPtr, true, UINT32_MAX,
-                      SearchedFiles, Map);
+    Symfile.FindTypes(ConstString(Name), ContextPtr, UINT32_MAX, SearchedFiles,
+                      Map);
   else
-    Module.FindTypes(parseCompilerContext(), languages, true, Map);
+    Module.FindTypes(parseCompilerContext(), languages, Map);
 
   outs() << formatv("Found {0} types:\n", Map.GetSize());
   StreamString Stream;
diff --git a/lldb/tools/lldb-vscode/VSCode.cpp b/lldb/tools/lldb-vscode/VSCode.cpp
index bba39d70e5faa..283798eb7abaf 100644
--- a/lldb/tools/lldb-vscode/VSCode.cpp
+++ b/lldb/tools/lldb-vscode/VSCode.cpp
@@ -15,10 +15,8 @@
 #include "llvm/Support/FormatVariadic.h"
 
 #if defined(_WIN32)
-#ifndef __MINGW32__
 #define NOMINMAX
-#include <Windows.h>
-#endif // __MINGW32__
+#include <windows.h>
 #include <fcntl.h>
 #include <io.h>
 #endif
diff --git a/lldb/tools/lldb-vscode/lldb-vscode.cpp b/lldb/tools/lldb-vscode/lldb-vscode.cpp
index 2810da62f6073..f39dbbf65e1fe 100644
--- a/lldb/tools/lldb-vscode/lldb-vscode.cpp
+++ b/lldb/tools/lldb-vscode/lldb-vscode.cpp
@@ -93,8 +93,9 @@ SOCKET AcceptConnection(int portno) {
     } else {
       listen(sockfd, 5);
       socklen_t clilen = sizeof(cli_addr);
-      newsockfd = llvm::sys::RetryAfterSignal(-1, accept,
-          sockfd, (struct sockaddr *)&cli_addr, &clilen);
+      newsockfd =
+          llvm::sys::RetryAfterSignal(static_cast<SOCKET>(-1), accept, sockfd,
+                                      (struct sockaddr *)&cli_addr, &clilen);
       if (newsockfd < 0)
         if (g_vsc.log)
           *g_vsc.log << "error: accept (" << strerror(errno) << ")"
@@ -435,7 +436,7 @@ void SetSourceMapFromArguments(const llvm::json::Object &arguments) {
       }
       auto mapFrom = GetAsString((*mapping)[0]);
       auto mapTo = GetAsString((*mapping)[1]);
-      strm << "\"" << mapFrom << "\" \"" << mapTo << "\"";
+      strm << "\"" << mapFrom << "\" \"" << mapTo << "\" ";
     }
   } else {
     if (ObjectContainsKey(arguments, "sourceMap")) {
@@ -2581,7 +2582,7 @@ const std::map<std::string, RequestCallback> &GetRequestHandlers() {
 #undef REQUEST_CALLBACK
   return g_request_handlers;
 }
-  
+
 } // anonymous namespace
 
 int main(int argc, char *argv[]) {
diff --git a/lldb/unittests/Expression/CMakeLists.txt b/lldb/unittests/Expression/CMakeLists.txt
index 2dcc8d3d65587..3408b278b0bfa 100644
--- a/lldb/unittests/Expression/CMakeLists.txt
+++ b/lldb/unittests/Expression/CMakeLists.txt
@@ -2,6 +2,7 @@ add_lldb_unittest(ExpressionTests
   ClangParserTest.cpp
   DiagnosticManagerTest.cpp
   DWARFExpressionTest.cpp
+  CppModuleConfigurationTest.cpp
 
   LINK_LIBS
     lldbCore
diff --git a/lldb/unittests/Expression/CppModuleConfigurationTest.cpp b/lldb/unittests/Expression/CppModuleConfigurationTest.cpp
new file mode 100644
index 0000000000000..0babf766a3a7a
--- /dev/null
+++ b/lldb/unittests/Expression/CppModuleConfigurationTest.cpp
@@ -0,0 +1,168 @@
+//===-- CppModuleConfigurationTest.cpp ---------------------------*- C++-*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/ExpressionParser/Clang/CppModuleConfiguration.h"
+#include "Plugins/ExpressionParser/Clang/ClangHost.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace lldb_private;
+
+namespace {
+struct CppModuleConfigurationTest : public testing::Test {
+  static void SetUpTestCase() {
+    // Getting the resource directory uses those subsystems, so we should
+    // initialize them.
+    FileSystem::Initialize();
+    HostInfo::Initialize();
+  }
+  static void TearDownTestCase() {
+    HostInfo::Terminate();
+    FileSystem::Terminate();
+  }
+};
+} // namespace
+
+/// Returns the Clang resource include directory.
+static std::string ResourceInc() {
+  llvm::SmallString<256> resource_dir;
+  llvm::sys::path::append(resource_dir, GetClangResourceDir().GetPath(),
+                          "include");
+  return resource_dir.str().str();
+}
+
+/// Utility function turningn a list of paths into a FileSpecList.
+static FileSpecList makeFiles(llvm::ArrayRef<std::string> paths) {
+  FileSpecList result;
+  for (const std::string &path : paths)
+    result.Append(FileSpec(path, FileSpec::Style::posix));
+  return result;
+}
+
+TEST_F(CppModuleConfigurationTest, Linux) {
+  // Test the average Linux configuration.
+  std::string libcpp = "/usr/include/c++/v1";
+  std::string usr = "/usr/include";
+  CppModuleConfiguration config(
+      makeFiles({usr + "/bits/types.h", libcpp + "/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre(libcpp, ResourceInc(), usr));
+}
+
+TEST_F(CppModuleConfigurationTest, Sysroot) {
+  // Test that having a sysroot for the whole system works fine.
+  std::string libcpp = "/home/user/sysroot/usr/include/c++/v1";
+  std::string usr = "/home/user/sysroot/usr/include";
+  CppModuleConfiguration config(
+      makeFiles({usr + "/bits/types.h", libcpp + "/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre(libcpp, ResourceInc(), usr));
+}
+
+TEST_F(CppModuleConfigurationTest, LinuxLocalLibCpp) {
+  // Test that a locally build libc++ is detected.
+  std::string libcpp = "/home/user/llvm-build/include/c++/v1";
+  std::string usr = "/usr/include";
+  CppModuleConfiguration config(
+      makeFiles({usr + "/bits/types.h", libcpp + "/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre(libcpp, ResourceInc(), usr));
+}
+
+TEST_F(CppModuleConfigurationTest, UnrelatedLibrary) {
+  // Test that having an unrelated library in /usr/include doesn't break.
+  std::string libcpp = "/home/user/llvm-build/include/c++/v1";
+  std::string usr = "/usr/include";
+  CppModuleConfiguration config(makeFiles(
+      {usr + "/bits/types.h", libcpp + "/vector", usr + "/boost/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre(libcpp, ResourceInc(), usr));
+}
+
+TEST_F(CppModuleConfigurationTest, Xcode) {
+  // Test detection of libc++ coming from Xcode with generic platform names.
+  std::string p = "/Applications/Xcode.app/Contents/Developer/";
+  std::string libcpp = p + "Toolchains/B.xctoolchain/usr/include/c++/v1";
+  std::string usr =
+      p + "Platforms/A.platform/Developer/SDKs/OSVers.sdk/usr/include";
+  CppModuleConfiguration config(
+      makeFiles({libcpp + "/unordered_map", usr + "/stdio.h"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre(libcpp, ResourceInc(), usr));
+}
+
+TEST_F(CppModuleConfigurationTest, LibCppV2) {
+  // Test that a "v2" of libc++ is still correctly detected.
+  CppModuleConfiguration config(
+      makeFiles({"/usr/include/bits/types.h", "/usr/include/c++/v2/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre("/usr/include/c++/v2", ResourceInc(),
+                                   "/usr/include"));
+}
+
+TEST_F(CppModuleConfigurationTest, UnknownLibCppFile) {
+  // Test that having some unknown file in the libc++ path doesn't break
+  // anything.
+  CppModuleConfiguration config(makeFiles(
+      {"/usr/include/bits/types.h", "/usr/include/c++/v1/non_existing_file"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre("std"));
+  EXPECT_THAT(config.GetIncludeDirs(),
+              testing::ElementsAre("/usr/include/c++/v1", ResourceInc(),
+                                   "/usr/include"));
+}
+
+TEST_F(CppModuleConfigurationTest, MissingUsrInclude) {
+  // Test that we don't load 'std' if we can't find the C standard library.
+  CppModuleConfiguration config(makeFiles({"/usr/include/c++/v1/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre());
+  EXPECT_THAT(config.GetIncludeDirs(), testing::ElementsAre());
+}
+
+TEST_F(CppModuleConfigurationTest, MissingLibCpp) {
+  // Test that we don't load 'std' if we don't have a libc++.
+  CppModuleConfiguration config(makeFiles({"/usr/include/bits/types.h"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre());
+  EXPECT_THAT(config.GetIncludeDirs(), testing::ElementsAre());
+}
+
+TEST_F(CppModuleConfigurationTest, IgnoreLibStdCpp) {
+  // Test that we don't do anything bad when we encounter libstdc++ paths.
+  CppModuleConfiguration config(makeFiles(
+      {"/usr/include/bits/types.h", "/usr/include/c++/8.0.1/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre());
+  EXPECT_THAT(config.GetIncludeDirs(), testing::ElementsAre());
+}
+
+TEST_F(CppModuleConfigurationTest, AmbiguousCLib) {
+  // Test that we don't do anything when we are not sure where the
+  // right C standard library is.
+  CppModuleConfiguration config(
+      makeFiles({"/usr/include/bits/types.h", "/usr/include/c++/v1/vector",
+                 "/sysroot/usr/include/bits/types.h"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre());
+  EXPECT_THAT(config.GetIncludeDirs(), testing::ElementsAre());
+}
+
+TEST_F(CppModuleConfigurationTest, AmbiguousLibCpp) {
+  // Test that we don't do anything when we are not sure where the
+  // right libc++ is.
+  CppModuleConfiguration config(
+      makeFiles({"/usr/include/bits/types.h", "/usr/include/c++/v1/vector",
+                 "/usr/include/c++/v2/vector"}));
+  EXPECT_THAT(config.GetImportedModules(), testing::ElementsAre());
+  EXPECT_THAT(config.GetIncludeDirs(), testing::ElementsAre());
+}
diff --git a/lldb/unittests/Host/CMakeLists.txt b/lldb/unittests/Host/CMakeLists.txt
index cf7c7cacfe2bb..b88aa97a90058 100644
--- a/lldb/unittests/Host/CMakeLists.txt
+++ b/lldb/unittests/Host/CMakeLists.txt
@@ -2,6 +2,7 @@ set (FILES
   ConnectionFileDescriptorTest.cpp
   FileActionTest.cpp
   FileSystemTest.cpp
+  FileTest.cpp
   HostInfoTest.cpp
   HostTest.cpp
   MainLoopTest.cpp
diff --git a/lldb/unittests/Host/FileSystemTest.cpp b/lldb/unittests/Host/FileSystemTest.cpp
index 7fea3b3a52908..31184d9ad539f 100644
--- a/lldb/unittests/Host/FileSystemTest.cpp
+++ b/lldb/unittests/Host/FileSystemTest.cpp
@@ -288,3 +288,18 @@ TEST(FileSystemTest, EnumerateDirectory) {
   EXPECT_THAT(visited,
               testing::UnorderedElementsAre("/foo", "/bar", "/baz", "/qux"));
 }
+
+TEST(FileSystemTest, OpenErrno) {
+#ifdef _WIN32
+  FileSpec spec("C:\\FILE\\THAT\\DOES\\NOT\\EXIST.TXT");
+#else
+  FileSpec spec("/file/that/does/not/exist.txt");
+#endif
+  FileSystem fs;
+  auto file = fs.Open(spec, File::eOpenOptionRead, 0, true);
+  ASSERT_FALSE(file);
+  std::error_code code = errorToErrorCode(file.takeError());
+  EXPECT_EQ(code.category(), std::system_category());
+  EXPECT_EQ(code.value(), ENOENT);
+}
+
diff --git a/lldb/unittests/Host/FileTest.cpp b/lldb/unittests/Host/FileTest.cpp
new file mode 100644
index 0000000000000..8cf1ebd2d99e6
--- /dev/null
+++ b/lldb/unittests/Host/FileTest.cpp
@@ -0,0 +1,57 @@
+//===-- FileTest.cpp --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Host/File.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "gtest/gtest.h"
+
+using namespace lldb;
+using namespace lldb_private;
+
+TEST(File, GetWaitableHandleFileno) {
+  const auto *Info = testing::UnitTest::GetInstance()->current_test_info();
+
+  llvm::SmallString<128> name;
+  int fd;
+  llvm::sys::fs::createTemporaryFile(llvm::Twine(Info->test_case_name()) + "-" +
+                                         Info->name(),
+                                     "test", fd, name);
+  llvm::FileRemover remover(name);
+  ASSERT_GE(fd, 0);
+
+  FILE *stream = fdopen(fd, "r");
+  ASSERT_TRUE(stream);
+
+  File file(stream, true);
+  EXPECT_EQ(file.GetWaitableHandle(), fd);
+}
+
+TEST(File, GetStreamFromDescriptor) {
+  const auto *Info = testing::UnitTest::GetInstance()->current_test_info();
+  llvm::SmallString<128> name;
+  int fd;
+  llvm::sys::fs::createTemporaryFile(llvm::Twine(Info->test_case_name()) + "-" +
+                                         Info->name(),
+                                     "test", fd, name);
+
+  llvm::FileRemover remover(name);
+  ASSERT_GE(fd, 0);
+
+  File file(fd, File::eOpenOptionWrite, true);
+  ASSERT_TRUE(file.IsValid());
+
+  FILE *stream = file.GetStream();
+  ASSERT_TRUE(stream != NULL);
+
+  EXPECT_EQ(file.GetDescriptor(), fd);
+  EXPECT_EQ(file.GetWaitableHandle(), fd);
+}
diff --git a/lldb/unittests/Host/SocketTest.cpp b/lldb/unittests/Host/SocketTest.cpp
index 4e897975e5d61..9408ecdaabe83 100644
--- a/lldb/unittests/Host/SocketTest.cpp
+++ b/lldb/unittests/Host/SocketTest.cpp
@@ -93,8 +93,10 @@ TEST_F(SocketTest, DomainListenConnectAccept) {
   std::error_code EC = llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", Path);
   ASSERT_FALSE(EC);
   llvm::sys::path::append(Path, "test");
-  // If this fails, $TMPDIR is too long to hold a domain socket.
-  EXPECT_LE(Path.size(), 107u);
+
+  // Skip the test if the $TMPDIR is too long to hold a domain socket.
+  if (Path.size() > 107u)
+    return;
 
   std::unique_ptr<DomainSocket> socket_a_up;
   std::unique_ptr<DomainSocket> socket_b_up;
@@ -196,8 +198,10 @@ TEST_F(SocketTest, DomainGetConnectURI) {
       llvm::sys::fs::createUniqueDirectory("DomainListenConnectAccept", domain_path);
   ASSERT_FALSE(EC);
   llvm::sys::path::append(domain_path, "test");
-  // If this fails, $TMPDIR is too long to hold a domain socket.
-  EXPECT_LE(domain_path.size(), 107u);
+
+  // Skip the test if the $TMPDIR is too long to hold a domain socket.
+  if (domain_path.size() > 107u)
+    return;
 
   std::unique_ptr<DomainSocket> socket_a_up;
   std::unique_ptr<DomainSocket> socket_b_up;
diff --git a/lldb/unittests/Process/gdb-remote/GDBRemoteClientBaseTest.cpp b/lldb/unittests/Process/gdb-remote/GDBRemoteClientBaseTest.cpp
index afba5c64266d5..d56e9cf81bc75 100644
--- a/lldb/unittests/Process/gdb-remote/GDBRemoteClientBaseTest.cpp
+++ b/lldb/unittests/Process/gdb-remote/GDBRemoteClientBaseTest.cpp
@@ -12,7 +12,7 @@
 #include "Plugins/Process/Utility/LinuxSignals.h"
 #include "Plugins/Process/gdb-remote/GDBRemoteClientBase.h"
 #include "Plugins/Process/gdb-remote/GDBRemoteCommunicationServer.h"
-#include "lldb/Utility/StreamGDBRemote.h"
+#include "lldb/Utility/GDBRemote.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Testing/Support/Error.h"
 
diff --git a/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp b/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
index 4a5741c809b07..e52f3782437d4 100644
--- a/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
+++ b/lldb/unittests/Process/gdb-remote/GDBRemoteCommunicationClientTest.cpp
@@ -384,9 +384,9 @@ TEST_F(GDBRemoteCommunicationClientTest, SendStartTracePacket) {
 
   // Since the line is exceeding 80 characters.
   std::string expected_packet1 =
-      R"(jTraceStart:{"buffersize" : 8192,"metabuffersize" : 8192,"params" :)";
+      R"(jTraceStart:{"buffersize":8192,"metabuffersize":8192,"params":)";
   std::string expected_packet2 =
-      R"( {"psb" : 1,"tracetech" : "intel-pt"},"threadid" : 35,"type" : 1})";
+      R"({"psb":1,"tracetech":"intel-pt"},"threadid":35,"type":1})";
   HandlePacket(server, (expected_packet1 + expected_packet2), "1");
   ASSERT_TRUE(error.Success());
   ASSERT_EQ(result.get(), 1u);
@@ -409,8 +409,7 @@ TEST_F(GDBRemoteCommunicationClientTest, SendStopTracePacket) {
     return client.SendStopTracePacket(trace_id, thread_id);
   });
 
-  const char *expected_packet =
-      R"(jTraceStop:{"threadid" : 35,"traceid" : 3})";
+  const char *expected_packet = R"(jTraceStop:{"threadid":35,"traceid":3})";
   HandlePacket(server, expected_packet, "OK");
   ASSERT_TRUE(result.get().Success());
 
@@ -435,8 +434,8 @@ TEST_F(GDBRemoteCommunicationClientTest, SendGetDataPacket) {
   });
 
   std::string expected_packet1 =
-      R"(jTraceBufferRead:{"buffersize" : 32,"offset" : 0,"threadid" : 35,)";
-  std::string expected_packet2 = R"("traceid" : 3})";
+      R"(jTraceBufferRead:{"buffersize":32,"offset":0,"threadid":35,)";
+  std::string expected_packet2 = R"("traceid":3})";
   HandlePacket(server, expected_packet1+expected_packet2, "123456");
   ASSERT_TRUE(result.get().Success());
   ASSERT_EQ(buffer.size(), 3u);
@@ -467,8 +466,8 @@ TEST_F(GDBRemoteCommunicationClientTest, SendGetMetaDataPacket) {
   });
 
   std::string expected_packet1 =
-      R"(jTraceMetaRead:{"buffersize" : 32,"offset" : 0,"threadid" : 35,)";
-  std::string expected_packet2 = R"("traceid" : 3})";
+      R"(jTraceMetaRead:{"buffersize":32,"offset":0,"threadid":35,)";
+  std::string expected_packet2 = R"("traceid":3})";
   HandlePacket(server, expected_packet1+expected_packet2, "123456");
   ASSERT_TRUE(result.get().Success());
   ASSERT_EQ(buffer.size(), 3u);
@@ -497,11 +496,10 @@ TEST_F(GDBRemoteCommunicationClientTest, SendGetTraceConfigPacket) {
   });
 
   const char *expected_packet =
-      R"(jTraceConfigRead:{"threadid" : 35,"traceid" : 3})";
+      R"(jTraceConfigRead:{"threadid":35,"traceid":3})";
   std::string response1 =
-      R"({"buffersize" : 8192,"params" : {"psb" : 1,"tracetech" : "intel-pt"})";
-  std::string response2 =
-      R"(],"metabuffersize" : 8192,"threadid" : 35,"type" : 1}])";
+      R"({"buffersize":8192,"params":{"psb":1,"tracetech":"intel-pt"})";
+  std::string response2 = R"(],"metabuffersize":8192,"threadid":35,"type":1}])";
   HandlePacket(server, expected_packet, response1+response2);
   ASSERT_TRUE(result.get().Success());
   ASSERT_EQ(options.getTraceBufferSize(), 8192u);
diff --git a/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt b/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
index fdb86db7cf3ed..618767de91ed4 100644
--- a/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
+++ b/lldb/unittests/ScriptInterpreter/Python/CMakeLists.txt
@@ -6,6 +6,7 @@ add_lldb_unittest(ScriptInterpreterPythonTests
   LINK_LIBS
     lldbHost
     lldbPluginScriptInterpreterPython
+    LLVMTestingSupport
   LINK_COMPONENTS
     Support
   )
\ No newline at end of file
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
index d43da010b1dfd..e6d1c4d14b0f9 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonDataObjectsTests.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Host/FileSystem.h"
 #include "lldb/Host/HostInfo.h"
 #include "lldb/lldb-enumerations.h"
+#include "llvm/Testing/Support/Error.h"
 
 #include "PythonTestSuite.h"
 
@@ -581,10 +582,10 @@ TEST_F(PythonDataObjectsTest, TestPythonCallableInvoke) {
 }
 
 TEST_F(PythonDataObjectsTest, TestPythonFile) {
-  File file;
-  FileSystem::Instance().Open(file, FileSpec(FileSystem::DEV_NULL),
-                              File::eOpenOptionRead);
-  PythonFile py_file(file, "r");
+  auto file = FileSystem::Instance().Open(FileSpec(FileSystem::DEV_NULL),
+                                          File::eOpenOptionRead);
+  ASSERT_THAT_EXPECTED(file, llvm::Succeeded());
+  PythonFile py_file(*file.get(), "r");
   EXPECT_TRUE(PythonFile::Check(py_file.get()));
 }
 
diff --git a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
index f6b617d26ae50..3ccda8c6fed53 100644
--- a/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
+++ b/lldb/unittests/ScriptInterpreter/Python/PythonTestSuite.cpp
@@ -95,6 +95,7 @@ LLDBSwigPythonCreateCommandObject(const char *python_class_name,
 
 extern "C" void *LLDBSwigPythonCreateScriptedThreadPlan(
     const char *python_class_name, const char *session_dictionary_name,
+    std::string &error_string,
     const lldb::ThreadPlanSP &thread_plan_sp) {
   return nullptr;
 }
diff --git a/lldb/unittests/Symbol/TestLineEntry.cpp b/lldb/unittests/Symbol/TestLineEntry.cpp
index 0d7a0567c6c88..b6325fa115b78 100644
--- a/lldb/unittests/Symbol/TestLineEntry.cpp
+++ b/lldb/unittests/Symbol/TestLineEntry.cpp
@@ -22,7 +22,6 @@
 #include "lldb/Host/HostInfo.h"
 #include "lldb/Symbol/CompileUnit.h"
 #include "lldb/Symbol/SymbolContext.h"
-#include "lldb/Utility/StreamString.h"
 
 #include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Program.h"
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 8ffd330985226..c68e35125b4c1 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 #include "Plugins/SymbolFile/DWARF/DWARFASTParserClang.h"
@@ -14,34 +15,54 @@
 using namespace lldb;
 using namespace lldb_private;
 
+class DWARFASTParserClangTests : public testing::Test {
+public:
+  void SetUp() override {
+    FileSystem::Initialize();
+    ClangASTContext::Initialize();
+  }
+
+  void TearDown() override {
+    ClangASTContext::Terminate();
+    FileSystem::Terminate();
+  }
+};
+
 namespace {
 class DWARFASTParserClangStub : public DWARFASTParserClang {
 public:
   using DWARFASTParserClang::DWARFASTParserClang;
   using DWARFASTParserClang::LinkDeclContextToDIE;
+
+  std::vector<const clang::DeclContext *> GetDeclContextToDIEMapKeys() {
+    std::vector<const clang::DeclContext *> keys;
+    for (const auto &it : m_decl_ctx_to_die)
+      keys.push_back(it.first);
+    return keys;
+  }
 };
 } // namespace
 
 // If your implementation needs to dereference the dummy pointers we are
 // defining here, causing this test to fail, feel free to delete it.
-TEST(DWARFASTParserClangTests,
-     TestGetDIEForDeclContextReturnsOnlyMatchingEntries) {
+TEST_F(DWARFASTParserClangTests,
+       EnsureAllDIEsInDeclContextHaveBeenParsedParsesOnlyMatchingEntries) {
   ClangASTContext ast_ctx;
   DWARFASTParserClangStub ast_parser(ast_ctx);
 
   DWARFUnit *unit = nullptr;
-  DWARFDIE die1(unit, (DWARFDebugInfoEntry *)1LL);
-  DWARFDIE die2(unit, (DWARFDebugInfoEntry *)2LL);
-  DWARFDIE die3(unit, (DWARFDebugInfoEntry *)3LL);
-  DWARFDIE die4(unit, (DWARFDebugInfoEntry *)4LL);
-  ast_parser.LinkDeclContextToDIE((clang::DeclContext *)1LL, die1);
-  ast_parser.LinkDeclContextToDIE((clang::DeclContext *)2LL, die2);
-  ast_parser.LinkDeclContextToDIE((clang::DeclContext *)2LL, die3);
-  ast_parser.LinkDeclContextToDIE((clang::DeclContext *)3LL, die4);
-
-  auto die_list = ast_parser.GetDIEForDeclContext(
-      CompilerDeclContext(nullptr, (clang::DeclContext *)2LL));
-  ASSERT_EQ(2u, die_list.size());
-  ASSERT_EQ(die2, die_list[0]);
-  ASSERT_EQ(die3, die_list[1]);
+  std::vector<DWARFDIE> dies = {DWARFDIE(unit, (DWARFDebugInfoEntry *)1LL),
+                                DWARFDIE(unit, (DWARFDebugInfoEntry *)2LL),
+                                DWARFDIE(unit, (DWARFDebugInfoEntry *)3LL),
+                                DWARFDIE(unit, (DWARFDebugInfoEntry *)4LL)};
+  std::vector<clang::DeclContext *> decl_ctxs = {
+      (clang::DeclContext *)1LL, (clang::DeclContext *)2LL,
+      (clang::DeclContext *)2LL, (clang::DeclContext *)3LL};
+  for (int i = 0; i < 4; ++i)
+    ast_parser.LinkDeclContextToDIE(decl_ctxs[i], dies[i]);
+  ast_parser.EnsureAllDIEsInDeclContextHaveBeenParsed(
+      CompilerDeclContext(nullptr, decl_ctxs[1]));
+
+  EXPECT_THAT(ast_parser.GetDeclContextToDIEMapKeys(),
+              testing::UnorderedElementsAre(decl_ctxs[0], decl_ctxs[3]));
 }
diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
index 64d4338deac91..909ef601e448b 100644
--- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
+++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
@@ -31,7 +31,7 @@
 #include "lldb/Utility/ArchSpec.h"
 #include "lldb/Utility/FileSpec.h"
 
-#if defined(_MSC_VER)
+#if defined(_WIN32)
 #include "lldb/Host/windows/windows.h"
 #include <objbase.h>
 #endif
@@ -46,7 +46,7 @@ class SymbolFilePDBTests : public testing::Test {
 // Initialize and TearDown the plugin every time, so we get a brand new
 // AST every time so that modifications to the AST from each test don't
 // leak into the next test.
-#if defined(_MSC_VER)
+#if defined(_WIN32)
     ::CoInitializeEx(nullptr, COINIT_MULTITHREADED);
 #endif
 
@@ -69,7 +69,7 @@ class SymbolFilePDBTests : public testing::Test {
     HostInfo::Terminate();
     FileSystem::Terminate();
 
-#if defined(_MSC_VER)
+#if defined(_WIN32)
     ::CoUninitialize();
 #endif
   }
@@ -355,8 +355,7 @@ TEST_F(SymbolFilePDBTests, TestSimpleClassTypes) {
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
   llvm::DenseSet<SymbolFile *> searched_files;
   TypeMap results;
-  EXPECT_EQ(1u, symfile->FindTypes(ConstString("Class"), nullptr, false, 0,
-                                   searched_files, results));
+  symfile->FindTypes(ConstString("Class"), nullptr, 0, searched_files, results);
   EXPECT_EQ(1u, results.GetSize());
   lldb::TypeSP udt_type = results.GetTypeAtIndex(0);
   EXPECT_EQ(ConstString("Class"), udt_type->GetName());
@@ -385,8 +384,7 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
       llvm::dyn_cast_or_null<ClangASTContext>(&clang_ast_ctx_or_err.get());
   EXPECT_NE(nullptr, clang_ast_ctx);
 
-  EXPECT_EQ(1u, symfile->FindTypes(ConstString("Class"), nullptr, false, 0,
-                                   searched_files, results));
+  symfile->FindTypes(ConstString("Class"), nullptr, 0, searched_files, results);
   EXPECT_EQ(1u, results.GetSize());
 
   auto Class = results.GetTypeAtIndex(0);
@@ -403,13 +401,14 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
   // and one is global. We process correctly this case and create the same
   // compiler type for both, but `FindTypes` may return more than one type
   // (with the same compiler type) because the symbols have different IDs.
+
+  TypeMap more_results;
   auto ClassCompilerDeclCtx = CompilerDeclContext(clang_ast_ctx, ClassDeclCtx);
-  EXPECT_LE(1u, symfile->FindTypes(ConstString("NestedClass"),
-                                   &ClassCompilerDeclCtx, false, 0,
-                                   searched_files, results));
-  EXPECT_LE(1u, results.GetSize());
+  symfile->FindTypes(ConstString("NestedClass"), &ClassCompilerDeclCtx, 0,
+                     searched_files, more_results);
+  EXPECT_LE(1u, more_results.GetSize());
 
-  lldb::TypeSP udt_type = results.GetTypeAtIndex(0);
+  lldb::TypeSP udt_type = more_results.GetTypeAtIndex(0);
   EXPECT_EQ(ConstString("NestedClass"), udt_type->GetName());
 
   CompilerType compiler_type = udt_type->GetForwardCompilerType();
@@ -450,8 +449,8 @@ TEST_F(SymbolFilePDBTests, TestClassInNamespace) {
   auto ns_namespace = symfile->FindNamespace(ConstString("NS"), nullptr);
   EXPECT_TRUE(ns_namespace.IsValid());
 
-  EXPECT_EQ(1u, symfile->FindTypes(ConstString("NSClass"), &ns_namespace, false,
-                                   0, searched_files, results));
+  symfile->FindTypes(ConstString("NSClass"), &ns_namespace, 0, searched_files,
+                     results);
   EXPECT_EQ(1u, results.GetSize());
 
   lldb::TypeSP udt_type = results.GetTypeAtIndex(0);
@@ -476,8 +475,7 @@ TEST_F(SymbolFilePDBTests, TestEnumTypes) {
   const char *EnumsToCheck[] = {"Enum", "ShortEnum"};
   for (auto Enum : EnumsToCheck) {
     TypeMap results;
-    EXPECT_EQ(1u, symfile->FindTypes(ConstString(Enum), nullptr, false, 0,
-                                     searched_files, results));
+    symfile->FindTypes(ConstString(Enum), nullptr, 0, searched_files, results);
     EXPECT_EQ(1u, results.GetSize());
     lldb::TypeSP enum_type = results.GetTypeAtIndex(0);
     EXPECT_EQ(ConstString(Enum), enum_type->GetName());
@@ -525,8 +523,8 @@ TEST_F(SymbolFilePDBTests, TestTypedefs) {
                                    "VariadicFuncPointerTypedef"};
   for (auto Typedef : TypedefsToCheck) {
     TypeMap results;
-    EXPECT_EQ(1u, symfile->FindTypes(ConstString(Typedef), nullptr, false, 0,
-                                     searched_files, results));
+    symfile->FindTypes(ConstString(Typedef), nullptr, 0, searched_files,
+                       results);
     EXPECT_EQ(1u, results.GetSize());
     lldb::TypeSP typedef_type = results.GetTypeAtIndex(0);
     EXPECT_EQ(ConstString(Typedef), typedef_type->GetName());
@@ -571,19 +569,18 @@ TEST_F(SymbolFilePDBTests, TestMaxMatches) {
   llvm::DenseSet<SymbolFile *> searched_files;
   TypeMap results;
   const ConstString name("ClassTypedef");
-  uint32_t num_results =
-      symfile->FindTypes(name, nullptr, false, 0, searched_files, results);
-  // Try to limit ourselves from 1 to 10 results, otherwise we could be doing
-  // this thousands of times.
-  // The idea is just to make sure that for a variety of values, the number of
-  // limited results always
-  // comes out to the number we are expecting.
+  symfile->FindTypes(name, nullptr, 0, searched_files, results);
+  // Try to limit ourselves from 1 to 10 results, otherwise we could
+  // be doing this thousands of times.  The idea is just to make sure
+  // that for a variety of values, the number of limited results
+  // always comes out to the number we are expecting.
+  uint32_t num_results = results.GetSize();
   uint32_t iterations = std::min(num_results, 10u);
   for (uint32_t i = 1; i <= iterations; ++i) {
-    uint32_t num_limited_results =
-        symfile->FindTypes(name, nullptr, false, i, searched_files, results);
+    TypeMap more_results;
+    symfile->FindTypes(name, nullptr, i, searched_files, more_results);
+    uint32_t num_limited_results = more_results.GetSize();
     EXPECT_EQ(i, num_limited_results);
-    EXPECT_EQ(num_limited_results, results.GetSize());
   }
 }
 
@@ -596,9 +593,7 @@ TEST_F(SymbolFilePDBTests, TestNullName) {
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::DenseSet<SymbolFile *> searched_files;
   TypeMap results;
-  uint32_t num_results = symfile->FindTypes(ConstString(), nullptr, false, 0,
-                                            searched_files, results);
-  EXPECT_EQ(0u, num_results);
+  symfile->FindTypes(ConstString(), nullptr, 0, searched_files, results);
   EXPECT_EQ(0u, results.GetSize());
 }
 
diff --git a/lldb/unittests/Utility/CMakeLists.txt b/lldb/unittests/Utility/CMakeLists.txt
index adc161e892022..1021d928b00d1 100644
--- a/lldb/unittests/Utility/CMakeLists.txt
+++ b/lldb/unittests/Utility/CMakeLists.txt
@@ -11,7 +11,6 @@ add_lldb_unittest(UtilityTests
   EventTest.cpp
   FileSpecTest.cpp
   FlagsTest.cpp
-  JSONTest.cpp
   ListenerTest.cpp
   LogTest.cpp
   NameMatchesTest.cpp
diff --git a/lldb/unittests/Utility/CompletionRequestTest.cpp b/lldb/unittests/Utility/CompletionRequestTest.cpp
index 54bd342992194..f1971711a50f0 100644
--- a/lldb/unittests/Utility/CompletionRequestTest.cpp
+++ b/lldb/unittests/Utility/CompletionRequestTest.cpp
@@ -14,8 +14,7 @@ using namespace lldb_private;
 TEST(CompletionRequest, Constructor) {
   std::string command = "a bad c";
   const unsigned cursor_pos = 3;
-  const int arg_index = 1;
-  const int arg_cursor_pos = 1;
+  const size_t arg_index = 1;
   StringList matches;
   CompletionResult result;
 
@@ -25,10 +24,93 @@ TEST(CompletionRequest, Constructor) {
   EXPECT_STREQ(request.GetRawLine().str().c_str(), command.c_str());
   EXPECT_EQ(request.GetRawCursorPos(), cursor_pos);
   EXPECT_EQ(request.GetCursorIndex(), arg_index);
-  EXPECT_EQ(request.GetCursorCharPosition(), arg_cursor_pos);
 
-  EXPECT_EQ(request.GetPartialParsedLine().GetArgumentCount(), 2u);
-  EXPECT_STREQ(request.GetPartialParsedLine().GetArgumentAtIndex(1), "b");
+  EXPECT_EQ(request.GetParsedLine().GetArgumentCount(), 2u);
+  EXPECT_EQ(request.GetCursorArgumentPrefix().str(), "b");
+}
+
+TEST(CompletionRequest, FakeLastArg) {
+  // We insert an empty fake argument into the argument list when the
+  // cursor is after a space.
+  std::string command = "a bad c ";
+  const unsigned cursor_pos = command.size();
+  CompletionResult result;
+
+  CompletionRequest request(command, cursor_pos, result);
+
+  EXPECT_STREQ(request.GetRawLine().str().c_str(), command.c_str());
+  EXPECT_EQ(request.GetRawCursorPos(), cursor_pos);
+  EXPECT_EQ(request.GetCursorIndex(), 3U);
+
+  EXPECT_EQ(request.GetParsedLine().GetArgumentCount(), 4U);
+  EXPECT_EQ(request.GetCursorArgumentPrefix().str(), "");
+}
+
+TEST(CompletionRequest, TryCompleteCurrentArgGood) {
+  std::string command = "a bad c";
+  StringList matches, descriptions;
+  CompletionResult result;
+
+  CompletionRequest request(command, 3, result);
+  request.TryCompleteCurrentArg("boo", "car");
+  result.GetMatches(matches);
+  result.GetDescriptions(descriptions);
+
+  EXPECT_EQ(1U, result.GetResults().size());
+  EXPECT_STREQ("boo", matches.GetStringAtIndex(0U));
+  EXPECT_EQ(1U, descriptions.GetSize());
+  EXPECT_STREQ("car", descriptions.GetStringAtIndex(0U));
+}
+
+TEST(CompletionRequest, TryCompleteCurrentArgBad) {
+  std::string command = "a bad c";
+  CompletionResult result;
+
+  CompletionRequest request(command, 3, result);
+  request.TryCompleteCurrentArg("car", "card");
+
+  EXPECT_EQ(0U, result.GetResults().size());
+}
+
+TEST(CompletionRequest, TryCompleteCurrentArgMode) {
+  std::string command = "a bad c";
+  CompletionResult result;
+
+  CompletionRequest request(command, 3, result);
+  request.TryCompleteCurrentArg<CompletionMode::Partial>("bar", "bard");
+
+  EXPECT_EQ(1U, result.GetResults().size());
+  EXPECT_EQ(CompletionMode::Partial, result.GetResults()[0].GetMode());
+}
+
+TEST(CompletionRequest, ShiftArguments) {
+  std::string command = "a bad c";
+  const unsigned cursor_pos = 3;
+  const size_t arg_index = 1;
+  StringList matches;
+  CompletionResult result;
+
+  CompletionRequest request(command, cursor_pos, result);
+  result.GetMatches(matches);
+
+  EXPECT_STREQ(request.GetRawLine().str().c_str(), command.c_str());
+  EXPECT_EQ(request.GetRawCursorPos(), cursor_pos);
+  EXPECT_EQ(request.GetCursorIndex(), arg_index);
+
+  EXPECT_EQ(request.GetParsedLine().GetArgumentCount(), 2u);
+  EXPECT_STREQ(request.GetParsedLine().GetArgumentAtIndex(1), "b");
+
+  // Shift away the 'a' argument.
+  request.ShiftArguments();
+
+  // The raw line/cursor stays identical.
+  EXPECT_STREQ(request.GetRawLine().str().c_str(), command.c_str());
+  EXPECT_EQ(request.GetRawCursorPos(), cursor_pos);
+
+  // Partially parsed line and cursor should be updated.
+  EXPECT_EQ(request.GetCursorIndex(), arg_index - 1U);
+  EXPECT_EQ(request.GetParsedLine().GetArgumentCount(), 1u);
+  EXPECT_EQ(request.GetCursorArgumentPrefix().str(), "b");
 }
 
 TEST(CompletionRequest, DuplicateFiltering) {
diff --git a/lldb/unittests/Utility/FlagsTest.cpp b/lldb/unittests/Utility/FlagsTest.cpp
index e26b620f9e2e9..6a46bcc2d6f09 100644
--- a/lldb/unittests/Utility/FlagsTest.cpp
+++ b/lldb/unittests/Utility/FlagsTest.cpp
@@ -30,19 +30,18 @@ TEST(Flags, Reset) {
   Flags f;
   f.Reset(0x3);
   EXPECT_EQ(0x3U, f.Get());
-  EXPECT_EQ(2U, f.SetCount());
 }
 
 TEST(Flags, Clear) {
   Flags f;
   f.Reset(0x3);
-  EXPECT_EQ(2U, f.SetCount());
+  EXPECT_EQ(0x3U, f.Get());
 
   f.Clear(0x5);
-  EXPECT_EQ(1U, f.SetCount());
+  EXPECT_EQ(0x2U, f.Get());
 
   f.Clear();
-  EXPECT_EQ(0U, f.SetCount());
+  EXPECT_EQ(0x0U, f.Get());
 }
 
 TEST(Flags, AllSet) {
@@ -162,37 +161,3 @@ TEST(Flags, IsClear) {
   EXPECT_TRUE(f.IsClear(eFlag0));
   EXPECT_TRUE(f.IsClear(eFlag1));
 }
-
-TEST(Flags, ClearCount) {
-  Flags f;
-  EXPECT_EQ(32U, f.ClearCount());
-
-  f.Set(eFlag0);
-  EXPECT_EQ(31U, f.ClearCount());
-
-  f.Set(eFlag0);
-  EXPECT_EQ(31U, f.ClearCount());
-
-  f.Set(eFlag1);
-  EXPECT_EQ(30U, f.ClearCount());
-
-  f.Set(eAllFlags);
-  EXPECT_EQ(29U, f.ClearCount());
-}
-
-TEST(Flags, SetCount) {
-  Flags f;
-  EXPECT_EQ(0U, f.SetCount());
-
-  f.Set(eFlag0);
-  EXPECT_EQ(1U, f.SetCount());
-
-  f.Set(eFlag0);
-  EXPECT_EQ(1U, f.SetCount());
-
-  f.Set(eFlag1);
-  EXPECT_EQ(2U, f.SetCount());
-
-  f.Set(eAllFlags);
-  EXPECT_EQ(3U, f.SetCount());
-}
diff --git a/lldb/unittests/Utility/JSONTest.cpp b/lldb/unittests/Utility/JSONTest.cpp
deleted file mode 100644
index 0c23350517ce6..0000000000000
--- a/lldb/unittests/Utility/JSONTest.cpp
+++ /dev/null
@@ -1,26 +0,0 @@
-#include "gtest/gtest.h"
-
-#include "lldb/Utility/JSON.h"
-#include "lldb/Utility/StreamString.h"
-
-using namespace lldb_private;
-
-TEST(JSONTest, Dictionary) {
-  JSONObject o;
-  o.SetObject("key", std::make_shared<JSONString>("value"));
-
-  StreamString stream;
-  o.Write(stream);
-
-  ASSERT_EQ(stream.GetString(), R"({"key":"value"})");
-}
-
-TEST(JSONTest, Newlines) {
-  JSONObject o;
-  o.SetObject("key", std::make_shared<JSONString>("hello\nworld"));
-
-  StreamString stream;
-  o.Write(stream);
-
-  ASSERT_EQ(stream.GetString(), R"({"key":"hello\nworld"})");
-}
diff --git a/lldb/unittests/Utility/ReproducerTest.cpp b/lldb/unittests/Utility/ReproducerTest.cpp
index a89812a03cb67..e4ac030350c95 100644
--- a/lldb/unittests/Utility/ReproducerTest.cpp
+++ b/lldb/unittests/Utility/ReproducerTest.cpp
@@ -52,14 +52,19 @@ TEST(ReproducerTest, SetCapture) {
   EXPECT_EQ(nullptr, reproducer.GetLoader());
 
   // Enable capture and check that means we have a generator.
-  EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec("/bogus/path")),
-                    Succeeded());
+  EXPECT_THAT_ERROR(
+      reproducer.SetCapture(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Succeeded());
   EXPECT_NE(nullptr, reproducer.GetGenerator());
-  EXPECT_EQ(FileSpec("/bogus/path"), reproducer.GetGenerator()->GetRoot());
-  EXPECT_EQ(FileSpec("/bogus/path"), reproducer.GetReproducerPath());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            reproducer.GetGenerator()->GetRoot());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            reproducer.GetReproducerPath());
 
   // Ensure that we cannot enable replay.
-  EXPECT_THAT_ERROR(reproducer.SetReplay(FileSpec("/bogus/path")), Failed());
+  EXPECT_THAT_ERROR(
+      reproducer.SetReplay(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Failed());
   EXPECT_EQ(nullptr, reproducer.GetLoader());
 
   // Ensure we can disable the generator again.
@@ -76,36 +81,45 @@ TEST(ReproducerTest, SetReplay) {
   EXPECT_EQ(nullptr, reproducer.GetLoader());
 
   // Expected to fail because we can't load the index.
-  EXPECT_THAT_ERROR(reproducer.SetReplay(FileSpec("/bogus/path")), Failed());
+  EXPECT_THAT_ERROR(
+      reproducer.SetReplay(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Failed());
   // However the loader should still be set, which we check here.
   EXPECT_NE(nullptr, reproducer.GetLoader());
 
   // Make sure the bogus path is correctly set.
-  EXPECT_EQ(FileSpec("/bogus/path"), reproducer.GetLoader()->GetRoot());
-  EXPECT_EQ(FileSpec("/bogus/path"), reproducer.GetReproducerPath());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            reproducer.GetLoader()->GetRoot());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            reproducer.GetReproducerPath());
 
   // Ensure that we cannot enable replay.
-  EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec("/bogus/path")), Failed());
+  EXPECT_THAT_ERROR(
+      reproducer.SetCapture(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Failed());
   EXPECT_EQ(nullptr, reproducer.GetGenerator());
 }
 
 TEST(GeneratorTest, Create) {
   DummyReproducer reproducer;
 
-  EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec("/bogus/path")),
-                    Succeeded());
+  EXPECT_THAT_ERROR(
+      reproducer.SetCapture(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Succeeded());
   auto &generator = *reproducer.GetGenerator();
 
   auto *provider = generator.Create<DummyProvider>();
   EXPECT_NE(nullptr, provider);
-  EXPECT_EQ(FileSpec("/bogus/path"), provider->GetRoot());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            provider->GetRoot());
 }
 
 TEST(GeneratorTest, Get) {
   DummyReproducer reproducer;
 
-  EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec("/bogus/path")),
-                    Succeeded());
+  EXPECT_THAT_ERROR(
+      reproducer.SetCapture(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Succeeded());
   auto &generator = *reproducer.GetGenerator();
 
   auto *provider = generator.Create<DummyProvider>();
@@ -118,12 +132,14 @@ TEST(GeneratorTest, Get) {
 TEST(GeneratorTest, GetOrCreate) {
   DummyReproducer reproducer;
 
-  EXPECT_THAT_ERROR(reproducer.SetCapture(FileSpec("/bogus/path")),
-                    Succeeded());
+  EXPECT_THAT_ERROR(
+      reproducer.SetCapture(FileSpec("//bogus/path", FileSpec::Style::posix)),
+      Succeeded());
   auto &generator = *reproducer.GetGenerator();
 
   auto &provider = generator.GetOrCreate<DummyProvider>();
-  EXPECT_EQ(FileSpec("/bogus/path"), provider.GetRoot());
+  EXPECT_EQ(FileSpec("//bogus/path", FileSpec::Style::posix),
+            provider.GetRoot());
 
   auto &provider_alt = generator.GetOrCreate<DummyProvider>();
   EXPECT_EQ(&provider, &provider_alt);
diff --git a/lldb/unittests/Utility/ScalarTest.cpp b/lldb/unittests/Utility/ScalarTest.cpp
index 54012d8d0593c..3a955e9a66877 100644
--- a/lldb/unittests/Utility/ScalarTest.cpp
+++ b/lldb/unittests/Utility/ScalarTest.cpp
@@ -16,7 +16,9 @@
 #include "llvm/Testing/Support/Error.h"
 
 using namespace lldb_private;
-using namespace llvm;
+using llvm::APInt;
+using llvm::Failed;
+using llvm::Succeeded;
 
 template <typename T>
 bool checkInequality(T c1, T c2) {
diff --git a/lldb/unittests/tools/lldb-server/CMakeLists.txt b/lldb/unittests/tools/lldb-server/CMakeLists.txt
index 1d8d312c9cef6..1a7603077b8d8 100644
--- a/lldb/unittests/tools/lldb-server/CMakeLists.txt
+++ b/lldb/unittests/tools/lldb-server/CMakeLists.txt
@@ -13,7 +13,7 @@ endfunction()
 add_lldb_test_executable(thread_inferior inferior/thread_inferior.cpp)
 add_lldb_test_executable(environment_check inferior/environment_check.cpp)
 
-if(LLDB_CAN_USE_DEBUGSERVER AND LLDB_TOOL_DEBUGSERVER_BUILD)
+if(LLDB_CAN_USE_DEBUGSERVER AND (LLDB_TOOL_DEBUGSERVER_BUILD OR LLDB_USE_SYSTEM_DEBUGSERVER))
   if(LLDB_USE_SYSTEM_DEBUGSERVER)
     lldb_find_system_debugserver(debugserver_path)
   else()
diff --git a/llvm-spirv/lib/SPIRV/SPIRVLowerMemmove.cpp b/llvm-spirv/lib/SPIRV/SPIRVLowerMemmove.cpp
index 84a939cb51e8c..9e2f92ab82952 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVLowerMemmove.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVLowerMemmove.cpp
@@ -95,7 +95,7 @@ class SPIRVLowerMemmove : public ModulePass,
 
     auto *Alloca =
         Builder.CreateAlloca(SrcTy->getPointerElementType(), NumElements);
-    Alloca->setAlignment(Align);
+    Alloca->setAlignment(MaybeAlign(Align));
     Builder.CreateLifetimeStart(Alloca);
     Builder.CreateMemCpy(Alloca, Align, Src, Align, Length, Volatile);
     auto *SecondCpy = Builder.CreateMemCpy(Dest, I.getDestAlignment(), Alloca,
diff --git a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
index c36492ac69795..6315f5bac6acc 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVReader.cpp
@@ -2774,13 +2774,13 @@ bool SPIRVToLLVM::transAlign(SPIRVValue *BV, Value *V) {
   if (auto AL = dyn_cast<AllocaInst>(V)) {
     SPIRVWord Align = 0;
     if (BV->hasAlignment(&Align))
-      AL->setAlignment(Align);
+      AL->setAlignment(MaybeAlign(Align));
     return true;
   }
   if (auto GV = dyn_cast<GlobalVariable>(V)) {
     SPIRVWord Align = 0;
     if (BV->hasAlignment(&Align))
-      GV->setAlignment(Align);
+      GV->setAlignment(MaybeAlign(Align));
     return true;
   }
   return true;
diff --git a/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp b/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
index 765591a52506a..93f4319f70f90 100644
--- a/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
+++ b/llvm-spirv/lib/SPIRV/SPIRVToOCL20.cpp
@@ -283,7 +283,7 @@ Instruction *SPIRVToOCL20::visitCallSPIRVAtomicCmpExchg(CallInst *CI, Op OC) {
                                                       ->getParent()
                                                       ->getEntryBlock()
                                                       .getFirstInsertionPt()));
-        PExpected->setAlignment(CI->getType()->getScalarSizeInBits() / 8);
+        PExpected->setAlignment(MaybeAlign(CI->getType()->getScalarSizeInBits() / 8));
         new StoreInst(Args[1], PExpected, PInsertBefore);
         unsigned AddrSpc = SPIRAS_Generic;
         Type *PtrTyAS =
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 678f1a74bdd14..4ebc726c1a93b 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -618,6 +618,9 @@ mark_as_advanced(LLVM_TARGET_TRIPLE_ENV)
 set(LLVM_ENABLE_PER_TARGET_RUNTIME_DIR OFF CACHE BOOL
   "Enable per-target runtimes directory")
 
+set(LLVM_PROFDATA_FILE "" CACHE FILEPATH
+  "Profiling data file to use when compiling in order to improve runtime performance.")
+
 # All options referred to from HandleLLVMOptions have to be specified
 # BEFORE this include, otherwise options will not be correctly set on
 # first cmake run
@@ -880,17 +883,6 @@ endif( ${CMAKE_SYSTEM_NAME} MATCHES SunOS )
 # use export_executable_symbols(target).
 set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
 
-set(LLVM_PROFDATA_FILE "" CACHE FILEPATH
-  "Profiling data file to use when compiling in order to improve runtime performance.")
-
-if(LLVM_PROFDATA_FILE AND EXISTS ${LLVM_PROFDATA_FILE})
-  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
-    add_definitions("-fprofile-instr-use=${LLVM_PROFDATA_FILE}")
-  else()
-    message(FATAL_ERROR "LLVM_PROFDATA_FILE can only be specified when compiling with clang")
-  endif()
-endif()
-
 include(AddLLVM)
 include(TableGen)
 
@@ -1105,3 +1097,7 @@ if (LLVM_INCLUDE_BENCHMARKS)
   add_subdirectory(utils/benchmark)
   add_subdirectory(benchmarks)
 endif()
+
+if (LLVM_INCLUDE_UTILS AND LLVM_INCLUDE_TOOLS)
+  add_subdirectory(utils/llvm-locstats)
+endif()
diff --git a/llvm/bindings/go/llvm/dibuilder.go b/llvm/bindings/go/llvm/dibuilder.go
index 98c11c151153b..2c6bee411c0ec 100644
--- a/llvm/bindings/go/llvm/dibuilder.go
+++ b/llvm/bindings/go/llvm/dibuilder.go
@@ -44,7 +44,7 @@ const (
 	FlagProtected
 	FlagFwdDecl
 	FlagAppleBlock
-	FlagBlockByrefStruct
+	FlagReserved
 	FlagVirtual
 	FlagArtificial
 	FlagExplicit
diff --git a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
index e4226d8b370c3..9fcaa10534f6b 100644
--- a/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
+++ b/llvm/bindings/ocaml/transforms/ipo/ipo_ocaml.c
@@ -31,6 +31,12 @@ CAMLprim value llvm_add_constant_merge(LLVMPassManagerRef PM) {
   return Val_unit;
 }
 
+/* [`Module] Llvm.PassManager.t -> unit */
+CAMLprim value llvm_add_merge_functions(LLVMPassManagerRef PM) {
+  LLVMAddMergeFunctionsPass(PM);
+  return Val_unit;
+}
+
 /* [`Module] Llvm.PassManager.t -> unit */
 CAMLprim value llvm_add_dead_arg_elimination(LLVMPassManagerRef PM) {
   LLVMAddDeadArgEliminationPass(PM);
diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
index cc6b0507968f8..1fb5594fcc7d6 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.ml
@@ -12,6 +12,9 @@ external add_argument_promotion
 external add_constant_merge
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_constant_merge"
+external add_merge_functions
+  : [ `Module ] Llvm.PassManager.t -> unit
+  = "llvm_add_merge_functions"
 external add_dead_arg_elimination
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_dead_arg_elimination"
diff --git a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
index 4ae988d8e579b..6507c5d92c2b1 100644
--- a/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
+++ b/llvm/bindings/ocaml/transforms/ipo/llvm_ipo.mli
@@ -21,6 +21,11 @@ external add_constant_merge
   : [ `Module ] Llvm.PassManager.t -> unit
   = "llvm_add_constant_merge"
 
+(** See the [llvm::createMergeFunctionsPass] function. *)
+external add_merge_functions
+  : [ `Module ] Llvm.PassManager.t -> unit
+  = "llvm_add_merge_functions"
+
 (** See the [llvm::createDeadArgEliminationPass] function. *)
 external add_dead_arg_elimination
   : [ `Module ] Llvm.PassManager.t -> unit
diff --git a/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml b/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
index 0c99b0f34f768..6864076053b67 100644
--- a/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
+++ b/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.ml
@@ -9,6 +9,9 @@
 external add_aggressive_dce
   : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
   = "llvm_add_aggressive_dce"
+external add_dce
+  : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
+  = "llvm_add_dce"
 external add_alignment_from_assumptions
   : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
   = "llvm_add_alignment_from_assumptions"
diff --git a/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli b/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
index 76d86ddf00147..8970431aeab27 100644
--- a/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
+++ b/llvm/bindings/ocaml/transforms/scalar_opts/llvm_scalar_opts.mli
@@ -16,6 +16,11 @@ external add_aggressive_dce
   : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
   = "llvm_add_aggressive_dce"
 
+(** See the [llvm::createDCEPass] function. *)
+external add_dce
+  : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
+  = "llvm_add_dce"
+
 (** See the [llvm::createAlignmentFromAssumptionsPass] function. *)
 external add_alignment_from_assumptions
   : [< Llvm.PassManager.any ] Llvm.PassManager.t -> unit
diff --git a/llvm/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c b/llvm/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
index cb3d50d31fb7c..56ebbf3c43c76 100644
--- a/llvm/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
+++ b/llvm/bindings/ocaml/transforms/scalar_opts/scalar_opts_ocaml.c
@@ -26,6 +26,11 @@ CAMLprim value llvm_add_aggressive_dce(LLVMPassManagerRef PM) {
   return Val_unit;
 }
 
+CAMLprim value llvm_add_dce(LLVMPassManagerRef PM) {
+  LLVMAddDCEPass(PM);
+  return Val_unit;
+}
+
 /* [<Llvm.PassManager.any] Llvm.PassManager.t -> unit */
 CAMLprim value llvm_add_alignment_from_assumptions(LLVMPassManagerRef PM) {
   LLVMAddAlignmentFromAssumptionsPass(PM);
diff --git a/llvm/cmake/modules/AddSphinxTarget.cmake b/llvm/cmake/modules/AddSphinxTarget.cmake
index 22e3dcb776aa2..2bf654b60c444 100644
--- a/llvm/cmake/modules/AddSphinxTarget.cmake
+++ b/llvm/cmake/modules/AddSphinxTarget.cmake
@@ -71,6 +71,11 @@ function (add_sphinx_target builder project)
                 COMPONENT "${project}-sphinx-man"
                 DESTINATION ${INSTALL_MANDIR}man1)
 
+        if(NOT LLVM_ENABLE_IDE)
+          add_llvm_install_targets("install-${SPHINX_TARGET_NAME}"
+                                   DEPENDS ${SPHINX_TARGET_NAME}
+                                   COMPONENT "${project}-sphinx-man")
+        endif()
       elseif (builder STREQUAL html)
         string(TOUPPER "${project}" project_upper)
         set(${project_upper}_INSTALL_SPHINX_HTML_DIR "share/doc/${project}/html"
@@ -82,6 +87,12 @@ function (add_sphinx_target builder project)
         install(DIRECTORY "${SPHINX_BUILD_DIR}/."
                 COMPONENT "${project}-sphinx-html"
                 DESTINATION "${${project_upper}_INSTALL_SPHINX_HTML_DIR}")
+
+        if(NOT LLVM_ENABLE_IDE)
+          add_llvm_install_targets("install-${SPHINX_TARGET_NAME}"
+                                   DEPENDS ${SPHINX_TARGET_NAME}
+                                   COMPONENT "${project}-sphinx-html")
+        endif()
       else()
         message(WARNING Installation of ${builder} not supported)
       endif()
diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 120438e5faca8..a84aff104a449 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -821,37 +821,53 @@ string(TOUPPER "${LLVM_BUILD_INSTRUMENTED}" uppercase_LLVM_BUILD_INSTRUMENTED)
 
 if (LLVM_BUILD_INSTRUMENTED)
   if (LLVM_ENABLE_IR_PGO OR uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "IR")
-    append("-fprofile-generate='${LLVM_PROFILE_DATA_DIR}'"
+    append("-fprofile-generate=\"${LLVM_PROFILE_DATA_DIR}\""
       CMAKE_CXX_FLAGS
-      CMAKE_C_FLAGS
-      CMAKE_EXE_LINKER_FLAGS
-      CMAKE_SHARED_LINKER_FLAGS)
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+        append("-fprofile-generate=\"${LLVM_PROFILE_DATA_DIR}\""
+          CMAKE_EXE_LINKER_FLAGS
+          CMAKE_SHARED_LINKER_FLAGS)
+    endif()
   elseif(uppercase_LLVM_BUILD_INSTRUMENTED STREQUAL "CSIR")
-    append("-fcs-profile-generate='${LLVM_CSPROFILE_DATA_DIR}'"
+    append("-fcs-profile-generate=\"${LLVM_CSPROFILE_DATA_DIR}\""
       CMAKE_CXX_FLAGS
-      CMAKE_C_FLAGS
-      CMAKE_EXE_LINKER_FLAGS
-      CMAKE_SHARED_LINKER_FLAGS)
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+      append("-fcs-profile-generate=\"${LLVM_CSPROFILE_DATA_DIR}\""
+        CMAKE_EXE_LINKER_FLAGS
+        CMAKE_SHARED_LINKER_FLAGS)
+    endif()
   else()
-    append("-fprofile-instr-generate='${LLVM_PROFILE_FILE_PATTERN}'"
+    append("-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\""
       CMAKE_CXX_FLAGS
-      CMAKE_C_FLAGS
-      CMAKE_EXE_LINKER_FLAGS
-      CMAKE_SHARED_LINKER_FLAGS)
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+      append("-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\""
+        CMAKE_EXE_LINKER_FLAGS
+        CMAKE_SHARED_LINKER_FLAGS)
+    endif()
   endif()
 endif()
 
-# Need to pass -fprofile-instr-use to linker for context-sensitive PGO
-# compilation.
 if(LLVM_PROFDATA_FILE AND EXISTS ${LLVM_PROFDATA_FILE})
-    append("-fprofile-instr-use='${LLVM_PROFDATA_FILE}'"
-      CMAKE_EXE_LINKER_FLAGS
-      CMAKE_SHARED_LINKER_FLAGS)
+  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" )
+    append("-fprofile-instr-use=\"${LLVM_PROFDATA_FILE}\""
+      CMAKE_CXX_FLAGS
+      CMAKE_C_FLAGS)
+    if(NOT LINKER_IS_LLD_LINK)
+      append("-fprofile-instr-use=\"${LLVM_PROFDATA_FILE}\""
+        CMAKE_EXE_LINKER_FLAGS
+        CMAKE_SHARED_LINKER_FLAGS)
+    endif()
+  else()
+    message(FATAL_ERROR "LLVM_PROFDATA_FILE can only be specified when compiling with clang")
+  endif()
 endif()
 
 option(LLVM_BUILD_INSTRUMENTED_COVERAGE "Build LLVM and tools with Code Coverage instrumentation" Off)
 mark_as_advanced(LLVM_BUILD_INSTRUMENTED_COVERAGE)
-append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate='${LLVM_PROFILE_FILE_PATTERN}' -fcoverage-mapping"
+append_if(LLVM_BUILD_INSTRUMENTED_COVERAGE "-fprofile-instr-generate=\"${LLVM_PROFILE_FILE_PATTERN}\" -fcoverage-mapping"
   CMAKE_CXX_FLAGS
   CMAKE_C_FLAGS
   CMAKE_EXE_LINKER_FLAGS
diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in
index 6dcbaac446987..7fdca536c1fdb 100644
--- a/llvm/cmake/modules/LLVMConfig.cmake.in
+++ b/llvm/cmake/modules/LLVMConfig.cmake.in
@@ -10,6 +10,11 @@ set(LLVM_PACKAGE_VERSION @PACKAGE_VERSION@)
 
 set(LLVM_BUILD_TYPE @CMAKE_BUILD_TYPE@)
 
+set(LLVM_USE_CRT_DEBUG @LLVM_USE_CRT_DEBUG@)
+set(LLVM_USE_CRT_MINSIZEREL @LLVM_USE_CRT_MINSIZEREL@)
+set(LLVM_USE_CRT_RELEASE @LLVM_USE_CRT_RELEASE@)
+set(LLVM_USE_CRT_RELWITHDEBINFO @LLVM_USE_CRT_RELWITHDEBINFO@)
+
 set(LLVM_COMMON_DEPENDS @LLVM_COMMON_DEPENDS@)
 
 set(LLVM_AVAILABLE_LIBS @LLVM_AVAILABLE_LIBS@)
@@ -40,6 +45,8 @@ set(LLVM_ENABLE_TERMINFO @LLVM_ENABLE_TERMINFO@)
 
 set(LLVM_ENABLE_THREADS @LLVM_ENABLE_THREADS@)
 
+set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@)
+
 set(LLVM_ENABLE_ZLIB @LLVM_ENABLE_ZLIB@)
 
 set(LLVM_LIBXML2_ENABLED @LLVM_LIBXML2_ENABLED@)
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index ca9357e295ebd..38e755d6d15a2 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -23,7 +23,7 @@ function(tablegen project ofn)
     file(RELATIVE_PATH ofn_rel
       ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR}/${ofn})
     set(additional_cmdline
-      -o ${ofn_rel}
+      -o ${ofn_rel}.tmp
       -d ${ofn_rel}.d
       WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
       DEPFILE ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.d
@@ -34,7 +34,7 @@ function(tablegen project ofn)
     file(GLOB local_tds "*.td")
     file(GLOB_RECURSE global_tds "${LLVM_MAIN_INCLUDE_DIR}/llvm/*.td")
     set(additional_cmdline
-      -o ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
+      -o ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
       )
   endif()
 
@@ -67,7 +67,8 @@ function(tablegen project ofn)
   # dependency twice in the result file when
   # ("${${project}_TABLEGEN_TARGET}" STREQUAL "${${project}_TABLEGEN_EXE}")
   # but lets us having smaller and cleaner code here.
-  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
+  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
+    # Generate tablegen output in a temporary file.
     COMMAND ${${project}_TABLEGEN_EXE} ${ARGN} -I ${CMAKE_CURRENT_SOURCE_DIR}
     ${LLVM_TABLEGEN_FLAGS}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
@@ -80,9 +81,20 @@ function(tablegen project ofn)
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
     COMMENT "Building ${ofn}..."
     )
+  add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
+    # Only update the real output file if there are any differences.
+    # This prevents recompilation of all the files depending on it if there
+    # aren't any.
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+        ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
+        ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
+    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${ofn}.tmp
+    COMMENT "Updating ${ofn}..."
+    )
 
   # `make clean' must remove all those generated files:
-  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${ofn})
+  set_property(DIRECTORY APPEND
+    PROPERTY ADDITIONAL_MAKE_CLEAN_FILES ${ofn}.tmp ${ofn})
 
   set(TABLEGEN_OUTPUT ${TABLEGEN_OUTPUT} ${CMAKE_CURRENT_BINARY_DIR}/${ofn} PARENT_SCOPE)
   set_source_files_properties(${CMAKE_CURRENT_BINARY_DIR}/${ofn} PROPERTIES
@@ -159,7 +171,13 @@ macro(add_tablegen target project)
 
     install(TARGETS ${target}
             ${export_to_llvmexports}
+            COMPONENT ${target}
             RUNTIME DESTINATION ${LLVM_TOOLS_INSTALL_DIR})
+    if(NOT LLVM_ENABLE_IDE)
+      add_llvm_install_targets("install-${target}"
+                               DEPENDS ${target}
+                               COMPONENT ${target})
+    endif()
   endif()
   set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS ${target})
 endmacro()
diff --git a/llvm/docs/AMDGPU/AMDGPUAsmGFX8.rst b/llvm/docs/AMDGPU/AMDGPUAsmGFX8.rst
index 9c762aa0f838b..a0e514e29b2c1 100644
--- a/llvm/docs/AMDGPU/AMDGPUAsmGFX8.rst
+++ b/llvm/docs/AMDGPU/AMDGPUAsmGFX8.rst
@@ -566,7 +566,7 @@ SOPC
     s_cmp_lg_u64                   :ref:`ssrc0<amdgpu_synid8_ssrc64_0>`,    :ref:`ssrc1<amdgpu_synid8_ssrc64_0>`
     s_cmp_lt_i32                   :ref:`ssrc0<amdgpu_synid8_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid8_ssrc32_0>`
     s_cmp_lt_u32                   :ref:`ssrc0<amdgpu_synid8_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid8_ssrc32_0>`
-    s_set_gpr_idx_on               :ref:`ssrc<amdgpu_synid8_ssrc32_0>`,     :ref:`imm4<amdgpu_synid8_imm4>`
+    s_set_gpr_idx_on               :ref:`ssrc<amdgpu_synid8_ssrc32_0>`,     :ref:`imask<amdgpu_synid8_imask>`
     s_setvskip                     :ref:`ssrc0<amdgpu_synid8_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid8_ssrc32_0>`
 
 SOPK
@@ -624,7 +624,7 @@ SOPP
     s_nop                          :ref:`imm16<amdgpu_synid8_bimm16>`
     s_sendmsg                      :ref:`msg<amdgpu_synid8_msg>`
     s_sendmsghalt                  :ref:`msg<amdgpu_synid8_msg>`
-    s_set_gpr_idx_mode             :ref:`imm4<amdgpu_synid8_imm4>`
+    s_set_gpr_idx_mode             :ref:`imask<amdgpu_synid8_imask>`
     s_set_gpr_idx_off
     s_sethalt                      :ref:`imm16<amdgpu_synid8_bimm16>`
     s_setkill                      :ref:`imm16<amdgpu_synid8_bimm16>`
@@ -1756,7 +1756,7 @@ VOPC
     gfx8_fimm16
     gfx8_fimm32
     gfx8_hwreg
-    gfx8_imm4
+    gfx8_imask
     gfx8_label
     gfx8_msg
     gfx8_param
diff --git a/llvm/docs/AMDGPU/AMDGPUAsmGFX9.rst b/llvm/docs/AMDGPU/AMDGPUAsmGFX9.rst
index 3ffb9fd5c8dbc..8ce056c8caf8e 100644
--- a/llvm/docs/AMDGPU/AMDGPUAsmGFX9.rst
+++ b/llvm/docs/AMDGPU/AMDGPUAsmGFX9.rst
@@ -736,7 +736,7 @@ SOPC
     s_cmp_lg_u64                   :ref:`ssrc0<amdgpu_synid9_ssrc64_0>`,    :ref:`ssrc1<amdgpu_synid9_ssrc64_0>`
     s_cmp_lt_i32                   :ref:`ssrc0<amdgpu_synid9_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid9_ssrc32_0>`
     s_cmp_lt_u32                   :ref:`ssrc0<amdgpu_synid9_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid9_ssrc32_0>`
-    s_set_gpr_idx_on               :ref:`ssrc<amdgpu_synid9_ssrc32_0>`,     :ref:`imm4<amdgpu_synid9_imm4>`
+    s_set_gpr_idx_on               :ref:`ssrc<amdgpu_synid9_ssrc32_0>`,     :ref:`imask<amdgpu_synid9_imask>`
     s_setvskip                     :ref:`ssrc0<amdgpu_synid9_ssrc32_0>`,    :ref:`ssrc1<amdgpu_synid9_ssrc32_0>`
 
 SOPK
@@ -796,7 +796,7 @@ SOPP
     s_nop                          :ref:`imm16<amdgpu_synid9_bimm16>`
     s_sendmsg                      :ref:`msg<amdgpu_synid9_msg>`
     s_sendmsghalt                  :ref:`msg<amdgpu_synid9_msg>`
-    s_set_gpr_idx_mode             :ref:`imm4<amdgpu_synid9_imm4>`
+    s_set_gpr_idx_mode             :ref:`imask<amdgpu_synid9_imask>`
     s_set_gpr_idx_off
     s_sethalt                      :ref:`imm16<amdgpu_synid9_bimm16>`
     s_setkill                      :ref:`imm16<amdgpu_synid9_bimm16>`
@@ -2010,7 +2010,7 @@ VOPC
     gfx9_fimm16
     gfx9_fimm32
     gfx9_hwreg
-    gfx9_imm4
+    gfx9_imask
     gfx9_label
     gfx9_msg
     gfx9_param
diff --git a/llvm/docs/AMDGPU/gfx10_bimm16.rst b/llvm/docs/AMDGPU/gfx10_bimm16.rst
index 00e9b71b92e7e..689ac46d94bae 100644
--- a/llvm/docs/AMDGPU/gfx10_bimm16.rst
+++ b/llvm/docs/AMDGPU/gfx10_bimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits.
+A 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx10_bimm32.rst b/llvm/docs/AMDGPU/gfx10_bimm32.rst
index c4b87df907f70..7e1bd334ebfcd 100644
--- a/llvm/docs/AMDGPU/gfx10_bimm32.rst
+++ b/llvm/docs/AMDGPU/gfx10_bimm32.rst
@@ -10,5 +10,5 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value is truncated to 32 bits.
 
diff --git a/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_cmp.rst b/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_cmp.rst
index bd257cf570b8c..6b5cd68143509 100644
--- a/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_cmp.rst
+++ b/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_cmp.rst
@@ -21,7 +21,7 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 2 data elements for 32-bit-per-pixel surfaces or 4 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_reg.rst b/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_reg.rst
index 930dab31321dd..7b74290a5652a 100644
--- a/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_reg.rst
+++ b/llvm/docs/AMDGPU/gfx10_data_mimg_atomic_reg.rst
@@ -21,6 +21,6 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 1 data element for 32-bit-per-pixel surfaces or 2 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx10_fimm16.rst b/llvm/docs/AMDGPU/gfx10_fimm16.rst
index c4d85be860d64..5c1dccc468928 100644
--- a/llvm/docs/AMDGPU/gfx10_fimm16.rst
+++ b/llvm/docs/AMDGPU/gfx10_fimm16.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The number is converted to *f16* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f16* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx10_fimm32.rst b/llvm/docs/AMDGPU/gfx10_fimm32.rst
index 7ff84cc051b6b..258762c4ed7e0 100644
--- a/llvm/docs/AMDGPU/gfx10_fimm32.rst
+++ b/llvm/docs/AMDGPU/gfx10_fimm32.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The value is converted to *f32* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f32* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx10_hwreg.rst b/llvm/docs/AMDGPU/gfx10_hwreg.rst
index 64d441bc72f59..56e2c66cbcf2f 100644
--- a/llvm/docs/AMDGPU/gfx10_hwreg.rst
+++ b/llvm/docs/AMDGPU/gfx10_hwreg.rst
@@ -14,18 +14,21 @@ Bits of a hardware register being accessed.
 
 The bits of this operand have the following meaning:
 
-    ============ ===================================
-    Bits         Description
-    ============ ===================================
-    5:0          Register *id*.
-    10:6         First bit *offset* (0..31).
-    15:11        *Size* in bits (1..32).
-    ============ ===================================
+    ======= ===================== ============
+    Bits    Description           Value Range
+    ======= ===================== ============
+    5:0     Register *id*.        0..63
+    10:6    First bit *offset*.   0..31
+    15:11   *Size* in bits.       1..32
+    ======= ===================== ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below.
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* An *hwreg* value described below.
 
     ==================================== ============================================================================
-    Syntax                               Description
+    Hwreg Value Syntax                   Description
     ==================================== ============================================================================
     hwreg({0..63})                       All bits of a register indicated by its *id*.
     hwreg(<*name*>)                      All bits of a register indicated by its *name*.
@@ -33,7 +36,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
     hwreg(<*name*>, {0..31}, {1..32})    Register bits indicated by register *name*, first bit *offset* and *size*.
     ==================================== ============================================================================
 
-Register *id*, *offset* and *size* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Defined register *names* include:
 
@@ -62,7 +66,16 @@ Examples:
 
 .. parsed-literal::
 
-    s_getreg_b32 s2, 0x6
+    reg = 1
+    offset = 2
+    size = 4
+    hwreg_enc = reg | (offset << 6) | ((size - 1) << 11)
+
+    s_getreg_b32 s2, 0x1881
+    s_getreg_b32 s2, hwreg_enc                     // the same as above
+    s_getreg_b32 s2, hwreg(1, 2, 4)                // the same as above
+    s_getreg_b32 s2, hwreg(reg, offset, size)      // the same as above
+
     s_getreg_b32 s2, hwreg(15)
     s_getreg_b32 s2, hwreg(51, 1, 31)
     s_getreg_b32 s2, hwreg(HW_REG_LDS_ALLOC, 0, 1)
diff --git a/llvm/docs/AMDGPU/gfx10_label.rst b/llvm/docs/AMDGPU/gfx10_label.rst
index 288c6c023a5c3..40c973eaf675c 100644
--- a/llvm/docs/AMDGPU/gfx10_label.rst
+++ b/llvm/docs/AMDGPU/gfx10_label.rst
@@ -12,19 +12,26 @@ label
 
 A branch target which is a 16-bit signed integer treated as a PC-relative dword offset.
 
-This operand may be specified as:
+This operand may be specified as one of the following:
 
-* An :ref:`integer_number<amdgpu_synid_integer_number>`. The number is truncated to 16 bits.
-* An :ref:`absolute_expression<amdgpu_synid_absolute_expression>` which must start with an :ref:`integer_number<amdgpu_synid_integer_number>`. The value of the expression is truncated to 16 bits.
-* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label). The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
+* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label) representing a relocatable address in the same compilation unit where it is referred from. The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
 
 Examples:
 
 .. parsed-literal::
 
   offset = 30
-  s_branch loop_end
-  s_branch 2 + offset
+  label_1:
+  label_2 = . + 4
+
   s_branch 32
-  loop_end:
+  s_branch offset + 2
+  s_branch label_1
+  s_branch label_2
+  s_branch label_3
+  s_branch label_4
+
+  label_3 = label_2 + 4
+  label_4:
 
diff --git a/llvm/docs/AMDGPU/gfx10_msg.rst b/llvm/docs/AMDGPU/gfx10_msg.rst
index ef531a14db19f..3e6c532dd85a8 100644
--- a/llvm/docs/AMDGPU/gfx10_msg.rst
+++ b/llvm/docs/AMDGPU/gfx10_msg.rst
@@ -12,24 +12,29 @@ msg
 
 A 16-bit message code. The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          Message *type*.
-    6:4          Optional *operation*.
-    9:7          Optional *parameters*.
-    15:10        Unused.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below:
-
-    ======================================== ========================================================================
-    Syntax                                   Description
-    ======================================== ========================================================================
-    sendmsg(<*type*>)                        A message identified by its *type*.
-    sendmsg(<*type*>, <*op*>)                A message identified by its *type* and *operation*.
-    sendmsg(<*type*>, <*op*>, <*stream*>)    A message identified by its *type* and *operation* with a stream *id*.
-    ======================================== ========================================================================
+    ============ =============================== ===============
+    Bits         Description                     Value Range
+    ============ =============================== ===============
+    3:0          Message *type*.                 0..15
+    6:4          Optional *operation*.           0..7
+    7:7          Unused.                         \-
+    9:8          Optional *stream*.              0..3
+    15:10        Unused.                         \-
+    ============ =============================== ===============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A *sendmsg* value described below.
+
+    ==================================== ====================================================
+    Sendmsg Value Syntax                 Description
+    ==================================== ====================================================
+    sendmsg(<*type*>)                    A message identified by its *type*.
+    sendmsg(<*type*>,<*op*>)             A message identified by its *type* and *operation*.
+    sendmsg(<*type*>,<*op*>,<*stream*>)  A message identified by its *type* and *operation*
+                                         with a stream *id*.
+    ==================================== ====================================================
 
 *Type* may be specified using message *name* or message *id*.
 
@@ -37,7 +42,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
 
 Stream *id* is an integer in the range 0..3.
 
-Message *id*, operation *id* and stream *id* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Each message type supports specific operations:
 
@@ -60,16 +66,32 @@ Each message type supports specific operations:
     \                            SYSMSG_OP_TTRACE_PC            4            \-
     ================= ========== ============================== ============ ==========
 
+*Sendmsg* arguments are validated depending on how *type* value is specified:
+
+* If message *type* is specified by name, arguments values must satisfy limitations detailed in the table above.
+* If message *type* is specified as a number, each argument must not exceed corresponding value range (see the first table).
+
 Examples:
 
 .. parsed-literal::
 
+    // numeric message code
+    msg = 0x10
     s_sendmsg 0x12
+    s_sendmsg msg + 2
+
+    // sendmsg with strict arguments validation
     s_sendmsg sendmsg(MSG_INTERRUPT)
-    s_sendmsg sendmsg(MSG_GET_DOORBELL)
-    s_sendmsg sendmsg(2, GS_OP_CUT)
     s_sendmsg sendmsg(MSG_GS, GS_OP_EMIT)
     s_sendmsg sendmsg(MSG_GS, 2)
     s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_EMIT_CUT, 1)
     s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_TTRACE_PC)
+    s_sendmsg sendmsg(MSG_GET_DOORBELL)
+
+    // sendmsg with validation of value range only
+    msg = 2
+    op = 3
+    stream = 1
+    s_sendmsg sendmsg(msg, op, stream)
+    s_sendmsg sendmsg(2, GS_OP_CUT)
 
diff --git a/llvm/docs/AMDGPU/gfx10_perm_smem.rst b/llvm/docs/AMDGPU/gfx10_perm_smem.rst
index 879b33de1dff8..bc12d1566517c 100644
--- a/llvm/docs/AMDGPU/gfx10_perm_smem.rst
+++ b/llvm/docs/AMDGPU/gfx10_perm_smem.rst
@@ -12,7 +12,8 @@ imm3
 
 A bit mask which indicates request permissions.
 
-This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 7 bits, but only 3 low bits are significant.
+This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is truncated to 7 bits, but only 3 low bits are significant.
 
     ============ ==============================
     Bit Number   Description
diff --git a/llvm/docs/AMDGPU/gfx10_simm16.rst b/llvm/docs/AMDGPU/gfx10_simm16.rst
index eb1d171e52903..365600a5b2e38 100644
--- a/llvm/docs/AMDGPU/gfx10_simm16.rst
+++ b/llvm/docs/AMDGPU/gfx10_simm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then sign-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx10_uimm16.rst b/llvm/docs/AMDGPU/gfx10_uimm16.rst
index f4bfe8c924ba7..8ade8dd60daec 100644
--- a/llvm/docs/AMDGPU/gfx10_uimm16.rst
+++ b/llvm/docs/AMDGPU/gfx10_uimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then zero-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx10_waitcnt.rst b/llvm/docs/AMDGPU/gfx10_waitcnt.rst
index e4c4bcdc169f5..c861fb0113c91 100644
--- a/llvm/docs/AMDGPU/gfx10_waitcnt.rst
+++ b/llvm/docs/AMDGPU/gfx10_waitcnt.rst
@@ -14,30 +14,31 @@ Counts of outstanding instructions to wait for.
 
 The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          VM_CNT: vector memory operations count, lower bits.
-    6:4          EXP_CNT: export count.
-    11:8         LGKM_CNT: LDS, GDS, Constant and Message count.
-    15:14        VM_CNT: vector memory operations count, upper bits.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>`
-or as a combination of the following symbolic helpers:
+    ========== ========= ================================================ ============
+    High Bits  Low Bits  Description                                      Value Range
+    ========== ========= ================================================ ============
+    15:14      3:0       VM_CNT: vector memory operations count.          0..63
+    \-         6:4       EXP_CNT: export count.                           0..7
+    \-         11:8      LGKM_CNT: LDS, GDS, Constant and Message count.  0..15
+    ========== ========= ================================================ ============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A combination of *vmcnt*, *expcnt*, *lgkmcnt* and other values described below.
 
     ====================== ======================================================================
     Syntax                 Description
     ====================== ======================================================================
-    vmcnt(<*N*>)           VM_CNT value. *N* must not exceed the largest VM_CNT value.
-    expcnt(<*N*>)          EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
-    lgkmcnt(<*N*>)         LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
-    vmcnt_sat(<*N*>)       VM_CNT value computed as min(*N*, the largest VM_CNT value).
-    expcnt_sat(<*N*>)      EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
-    lgkmcnt_sat(<*N*>)     LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
+    vmcnt(<*N*>)           A VM_CNT value. *N* must not exceed the largest VM_CNT value.
+    expcnt(<*N*>)          An EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
+    lgkmcnt(<*N*>)         An LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
+    vmcnt_sat(<*N*>)       A VM_CNT value computed as min(*N*, the largest VM_CNT value).
+    expcnt_sat(<*N*>)      An EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
+    lgkmcnt_sat(<*N*>)     An LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
     ====================== ======================================================================
 
-These helpers may be specified in any order. Ampersands and commas may be used as optional separators.
+These values may be specified in any order. Spaces, ampersands and commas may be used as optional separators.
 
 *N* is either an
 :ref:`integer number<amdgpu_synid_integer_number>` or an
@@ -47,10 +48,18 @@ Examples:
 
 .. parsed-literal::
 
-    s_waitcnt 0
+    vm_cnt = 1
+    exp_cnt = 2
+    lgkm_cnt = 3
+    cnt = vm_cnt | (exp_cnt << 4) | (lgkm_cnt << 8)
+
+    s_waitcnt cnt
+    s_waitcnt 1 | (2 << 4) | (3 << 8)                          // the same as above
+    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)                    // the same as above
+    s_waitcnt vmcnt(vm_cnt) expcnt(exp_cnt) lgkmcnt(lgkm_cnt)  // the same as above
+
     s_waitcnt vmcnt(1)
     s_waitcnt expcnt(2) lgkmcnt(3)
-    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)
     s_waitcnt vmcnt(1), expcnt(2), lgkmcnt(3)
     s_waitcnt vmcnt(1) & lgkmcnt_sat(100) & expcnt(2)
 
diff --git a/llvm/docs/AMDGPU/gfx7_bimm16.rst b/llvm/docs/AMDGPU/gfx7_bimm16.rst
index eb43f9b36a9d5..5f1fbc1dcd6c9 100644
--- a/llvm/docs/AMDGPU/gfx7_bimm16.rst
+++ b/llvm/docs/AMDGPU/gfx7_bimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits.
+A 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx7_bimm32.rst b/llvm/docs/AMDGPU/gfx7_bimm32.rst
index 4d8f89d7ae39a..45d35ff7adebc 100644
--- a/llvm/docs/AMDGPU/gfx7_bimm32.rst
+++ b/llvm/docs/AMDGPU/gfx7_bimm32.rst
@@ -10,5 +10,5 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value is truncated to 32 bits.
 
diff --git a/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_cmp.rst b/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_cmp.rst
index 82c3337aeb2bd..0328519d17b61 100644
--- a/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_cmp.rst
+++ b/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_cmp.rst
@@ -21,7 +21,7 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 2 data elements for 32-bit-per-pixel surfaces or 4 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_reg.rst b/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_reg.rst
index 729548dcb87b5..30785e47ff985 100644
--- a/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_reg.rst
+++ b/llvm/docs/AMDGPU/gfx7_data_mimg_atomic_reg.rst
@@ -21,6 +21,6 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 1 data element for 32-bit-per-pixel surfaces or 2 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx7_fimm32.rst b/llvm/docs/AMDGPU/gfx7_fimm32.rst
index 70c81891e9784..de261aea8dbfb 100644
--- a/llvm/docs/AMDGPU/gfx7_fimm32.rst
+++ b/llvm/docs/AMDGPU/gfx7_fimm32.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The value is converted to *f32* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f32* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx7_hwreg.rst b/llvm/docs/AMDGPU/gfx7_hwreg.rst
index 1e2d96417c339..b303b6ce8f97f 100644
--- a/llvm/docs/AMDGPU/gfx7_hwreg.rst
+++ b/llvm/docs/AMDGPU/gfx7_hwreg.rst
@@ -14,18 +14,21 @@ Bits of a hardware register being accessed.
 
 The bits of this operand have the following meaning:
 
-    ============ ===================================
-    Bits         Description
-    ============ ===================================
-    5:0          Register *id*.
-    10:6         First bit *offset* (0..31).
-    15:11        *Size* in bits (1..32).
-    ============ ===================================
+    ======= ===================== ============
+    Bits    Description           Value Range
+    ======= ===================== ============
+    5:0     Register *id*.        0..63
+    10:6    First bit *offset*.   0..31
+    15:11   *Size* in bits.       1..32
+    ======= ===================== ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below.
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* An *hwreg* value described below.
 
     ==================================== ============================================================================
-    Syntax                               Description
+    Hwreg Value Syntax                   Description
     ==================================== ============================================================================
     hwreg({0..63})                       All bits of a register indicated by its *id*.
     hwreg(<*name*>)                      All bits of a register indicated by its *name*.
@@ -33,7 +36,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
     hwreg(<*name*>, {0..31}, {1..32})    Register bits indicated by register *name*, first bit *offset* and *size*.
     ==================================== ============================================================================
 
-Register *id*, *offset* and *size* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Defined register *names* include:
 
@@ -53,7 +57,16 @@ Examples:
 
 .. parsed-literal::
 
-    s_getreg_b32 s2, 0x6
+    reg = 1
+    offset = 2
+    size = 4
+    hwreg_enc = reg | (offset << 6) | ((size - 1) << 11)
+
+    s_getreg_b32 s2, 0x1881
+    s_getreg_b32 s2, hwreg_enc                     // the same as above
+    s_getreg_b32 s2, hwreg(1, 2, 4)                // the same as above
+    s_getreg_b32 s2, hwreg(reg, offset, size)      // the same as above
+
     s_getreg_b32 s2, hwreg(15)
     s_getreg_b32 s2, hwreg(51, 1, 31)
     s_getreg_b32 s2, hwreg(HW_REG_LDS_ALLOC, 0, 1)
diff --git a/llvm/docs/AMDGPU/gfx7_label.rst b/llvm/docs/AMDGPU/gfx7_label.rst
index ed2f3a4166677..2f6bd68ddc330 100644
--- a/llvm/docs/AMDGPU/gfx7_label.rst
+++ b/llvm/docs/AMDGPU/gfx7_label.rst
@@ -12,19 +12,26 @@ label
 
 A branch target which is a 16-bit signed integer treated as a PC-relative dword offset.
 
-This operand may be specified as:
+This operand may be specified as one of the following:
 
-* An :ref:`integer_number<amdgpu_synid_integer_number>`. The number is truncated to 16 bits.
-* An :ref:`absolute_expression<amdgpu_synid_absolute_expression>` which must start with an :ref:`integer_number<amdgpu_synid_integer_number>`. The value of the expression is truncated to 16 bits.
-* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label). The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
+* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label) representing a relocatable address in the same compilation unit where it is referred from. The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
 
 Examples:
 
 .. parsed-literal::
 
   offset = 30
-  s_branch loop_end
-  s_branch 2 + offset
+  label_1:
+  label_2 = . + 4
+
   s_branch 32
-  loop_end:
+  s_branch offset + 2
+  s_branch label_1
+  s_branch label_2
+  s_branch label_3
+  s_branch label_4
+
+  label_3 = label_2 + 4
+  label_4:
 
diff --git a/llvm/docs/AMDGPU/gfx7_msg.rst b/llvm/docs/AMDGPU/gfx7_msg.rst
index 5476053ccc1cd..72a895ef5b36e 100644
--- a/llvm/docs/AMDGPU/gfx7_msg.rst
+++ b/llvm/docs/AMDGPU/gfx7_msg.rst
@@ -12,24 +12,29 @@ msg
 
 A 16-bit message code. The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          Message *type*.
-    6:4          Optional *operation*.
-    9:7          Optional *parameters*.
-    15:10        Unused.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below:
-
-    ======================================== ========================================================================
-    Syntax                                   Description
-    ======================================== ========================================================================
-    sendmsg(<*type*>)                        A message identified by its *type*.
-    sendmsg(<*type*>, <*op*>)                A message identified by its *type* and *operation*.
-    sendmsg(<*type*>, <*op*>, <*stream*>)    A message identified by its *type* and *operation* with a stream *id*.
-    ======================================== ========================================================================
+    ============ =============================== ===============
+    Bits         Description                     Value Range
+    ============ =============================== ===============
+    3:0          Message *type*.                 0..15
+    6:4          Optional *operation*.           0..7
+    7:7          Unused.                         \-
+    9:8          Optional *stream*.              0..3
+    15:10        Unused.                         \-
+    ============ =============================== ===============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A *sendmsg* value described below.
+
+    ==================================== ====================================================
+    Sendmsg Value Syntax                 Description
+    ==================================== ====================================================
+    sendmsg(<*type*>)                    A message identified by its *type*.
+    sendmsg(<*type*>,<*op*>)             A message identified by its *type* and *operation*.
+    sendmsg(<*type*>,<*op*>,<*stream*>)  A message identified by its *type* and *operation*
+                                         with a stream *id*.
+    ==================================== ====================================================
 
 *Type* may be specified using message *name* or message *id*.
 
@@ -37,7 +42,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
 
 Stream *id* is an integer in the range 0..3.
 
-Message *id*, operation *id* and stream *id* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Each message type supports specific operations:
 
@@ -58,15 +64,31 @@ Each message type supports specific operations:
     \                            SYSMSG_OP_TTRACE_PC            4            \-
     ================= ========== ============================== ============ ==========
 
+*Sendmsg* arguments are validated depending on how *type* value is specified:
+
+* If message *type* is specified by name, arguments values must satisfy limitations detailed in the table above.
+* If message *type* is specified as a number, each argument must not exceed corresponding value range (see the first table).
+
 Examples:
 
 .. parsed-literal::
 
+    // numeric message code
+    msg = 0x10
     s_sendmsg 0x12
+    s_sendmsg msg + 2
+
+    // sendmsg with strict arguments validation
     s_sendmsg sendmsg(MSG_INTERRUPT)
-    s_sendmsg sendmsg(2, GS_OP_CUT)
     s_sendmsg sendmsg(MSG_GS, GS_OP_EMIT)
     s_sendmsg sendmsg(MSG_GS, 2)
     s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_EMIT_CUT, 1)
     s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_TTRACE_PC)
 
+    // sendmsg with validation of value range only
+    msg = 2
+    op = 3
+    stream = 1
+    s_sendmsg sendmsg(msg, op, stream)
+    s_sendmsg sendmsg(2, GS_OP_CUT)
+
diff --git a/llvm/docs/AMDGPU/gfx7_simm16.rst b/llvm/docs/AMDGPU/gfx7_simm16.rst
index 66e560ecec8c3..3e5d3700863f2 100644
--- a/llvm/docs/AMDGPU/gfx7_simm16.rst
+++ b/llvm/docs/AMDGPU/gfx7_simm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then sign-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx7_uimm16.rst b/llvm/docs/AMDGPU/gfx7_uimm16.rst
index bd0d4c2fb1442..d8a1a20528ae4 100644
--- a/llvm/docs/AMDGPU/gfx7_uimm16.rst
+++ b/llvm/docs/AMDGPU/gfx7_uimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then zero-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx7_waitcnt.rst b/llvm/docs/AMDGPU/gfx7_waitcnt.rst
index 3f5e07d164934..9566e6c67327a 100644
--- a/llvm/docs/AMDGPU/gfx7_waitcnt.rst
+++ b/llvm/docs/AMDGPU/gfx7_waitcnt.rst
@@ -14,29 +14,31 @@ Counts of outstanding instructions to wait for.
 
 The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          VM_CNT: vector memory operations count.
-    6:4          EXP_CNT: export count.
-    12:8         LGKM_CNT: LDS, GDS, Constant and Message count.
-    ============ ======================================================
+   ===== ================================================ ============
+   Bits  Description                                      Value Range
+   ===== ================================================ ============
+   3:0   VM_CNT: vector memory operations count.          0..15
+   6:4   EXP_CNT: export count.                           0..7
+   12:8  LGKM_CNT: LDS, GDS, Constant and Message count.  0..31
+   ===== ================================================ ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>`
-or as a combination of the following symbolic helpers:
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A combination of *vmcnt*, *expcnt*, *lgkmcnt* and other values described below.
 
     ====================== ======================================================================
     Syntax                 Description
     ====================== ======================================================================
-    vmcnt(<*N*>)           VM_CNT value. *N* must not exceed the largest VM_CNT value.
-    expcnt(<*N*>)          EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
-    lgkmcnt(<*N*>)         LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
-    vmcnt_sat(<*N*>)       VM_CNT value computed as min(*N*, the largest VM_CNT value).
-    expcnt_sat(<*N*>)      EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
-    lgkmcnt_sat(<*N*>)     LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
+    vmcnt(<*N*>)           A VM_CNT value. *N* must not exceed the largest VM_CNT value.
+    expcnt(<*N*>)          An EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
+    lgkmcnt(<*N*>)         An LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
+    vmcnt_sat(<*N*>)       A VM_CNT value computed as min(*N*, the largest VM_CNT value).
+    expcnt_sat(<*N*>)      An EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
+    lgkmcnt_sat(<*N*>)     An LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
     ====================== ======================================================================
 
-These helpers may be specified in any order. Ampersands and commas may be used as optional separators.
+These values may be specified in any order. Spaces, ampersands and commas may be used as optional separators.
 
 *N* is either an
 :ref:`integer number<amdgpu_synid_integer_number>` or an
@@ -46,10 +48,18 @@ Examples:
 
 .. parsed-literal::
 
-    s_waitcnt 0
+    vm_cnt = 1
+    exp_cnt = 2
+    lgkm_cnt = 3
+    cnt = vm_cnt | (exp_cnt << 4) | (lgkm_cnt << 8)
+
+    s_waitcnt cnt
+    s_waitcnt 1 | (2 << 4) | (3 << 8)                          // the same as above
+    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)                    // the same as above
+    s_waitcnt vmcnt(vm_cnt) expcnt(exp_cnt) lgkmcnt(lgkm_cnt)  // the same as above
+
     s_waitcnt vmcnt(1)
     s_waitcnt expcnt(2) lgkmcnt(3)
-    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)
     s_waitcnt vmcnt(1), expcnt(2), lgkmcnt(3)
     s_waitcnt vmcnt(1) & lgkmcnt_sat(100) & expcnt(2)
 
diff --git a/llvm/docs/AMDGPU/gfx8_bimm16.rst b/llvm/docs/AMDGPU/gfx8_bimm16.rst
index ed50e5582329a..66875c6024f48 100644
--- a/llvm/docs/AMDGPU/gfx8_bimm16.rst
+++ b/llvm/docs/AMDGPU/gfx8_bimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits.
+A 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx8_bimm32.rst b/llvm/docs/AMDGPU/gfx8_bimm32.rst
index d03c27b1732b8..e46bc3c7ac81f 100644
--- a/llvm/docs/AMDGPU/gfx8_bimm32.rst
+++ b/llvm/docs/AMDGPU/gfx8_bimm32.rst
@@ -10,5 +10,5 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value is truncated to 32 bits.
 
diff --git a/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_cmp.rst b/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_cmp.rst
index 80222ead81a83..237b91cadfee0 100644
--- a/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_cmp.rst
+++ b/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_cmp.rst
@@ -21,7 +21,7 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 2 data elements for 32-bit-per-pixel surfaces or 4 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_reg.rst b/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_reg.rst
index 8baf9269b887b..35bcdc379a10b 100644
--- a/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_reg.rst
+++ b/llvm/docs/AMDGPU/gfx8_data_mimg_atomic_reg.rst
@@ -21,6 +21,6 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 1 data element for 32-bit-per-pixel surfaces or 2 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx8_fimm16.rst b/llvm/docs/AMDGPU/gfx8_fimm16.rst
index 5e387f5cb7746..7abd5987bcdda 100644
--- a/llvm/docs/AMDGPU/gfx8_fimm16.rst
+++ b/llvm/docs/AMDGPU/gfx8_fimm16.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The number is converted to *f16* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f16* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx8_fimm32.rst b/llvm/docs/AMDGPU/gfx8_fimm32.rst
index e29e7704b8aa5..a1556557f1da9 100644
--- a/llvm/docs/AMDGPU/gfx8_fimm32.rst
+++ b/llvm/docs/AMDGPU/gfx8_fimm32.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The value is converted to *f32* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f32* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx8_hwreg.rst b/llvm/docs/AMDGPU/gfx8_hwreg.rst
index ffa1ea5afde3d..87d7888491105 100644
--- a/llvm/docs/AMDGPU/gfx8_hwreg.rst
+++ b/llvm/docs/AMDGPU/gfx8_hwreg.rst
@@ -14,18 +14,21 @@ Bits of a hardware register being accessed.
 
 The bits of this operand have the following meaning:
 
-    ============ ===================================
-    Bits         Description
-    ============ ===================================
-    5:0          Register *id*.
-    10:6         First bit *offset* (0..31).
-    15:11        *Size* in bits (1..32).
-    ============ ===================================
+    ======= ===================== ============
+    Bits    Description           Value Range
+    ======= ===================== ============
+    5:0     Register *id*.        0..63
+    10:6    First bit *offset*.   0..31
+    15:11   *Size* in bits.       1..32
+    ======= ===================== ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below.
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* An *hwreg* value described below.
 
     ==================================== ============================================================================
-    Syntax                               Description
+    Hwreg Value Syntax                   Description
     ==================================== ============================================================================
     hwreg({0..63})                       All bits of a register indicated by its *id*.
     hwreg(<*name*>)                      All bits of a register indicated by its *name*.
@@ -33,7 +36,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
     hwreg(<*name*>, {0..31}, {1..32})    Register bits indicated by register *name*, first bit *offset* and *size*.
     ==================================== ============================================================================
 
-Register *id*, *offset* and *size* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Defined register *names* include:
 
@@ -53,7 +57,16 @@ Examples:
 
 .. parsed-literal::
 
-    s_getreg_b32 s2, 0x6
+    reg = 1
+    offset = 2
+    size = 4
+    hwreg_enc = reg | (offset << 6) | ((size - 1) << 11)
+
+    s_getreg_b32 s2, 0x1881
+    s_getreg_b32 s2, hwreg_enc                     // the same as above
+    s_getreg_b32 s2, hwreg(1, 2, 4)                // the same as above
+    s_getreg_b32 s2, hwreg(reg, offset, size)      // the same as above
+
     s_getreg_b32 s2, hwreg(15)
     s_getreg_b32 s2, hwreg(51, 1, 31)
     s_getreg_b32 s2, hwreg(HW_REG_LDS_ALLOC, 0, 1)
diff --git a/llvm/docs/AMDGPU/gfx8_imask.rst b/llvm/docs/AMDGPU/gfx8_imask.rst
new file mode 100644
index 0000000000000..55e9a244bdcbf
--- /dev/null
+++ b/llvm/docs/AMDGPU/gfx8_imask.rst
@@ -0,0 +1,66 @@
+..
+    **************************************************
+    *                                                *
+    *   Automatically generated file, do not edit!   *
+    *                                                *
+    **************************************************
+
+.. _amdgpu_synid8_imask:
+
+imask
+===========================
+
+This operand is a mask which controls indexing mode for operands of subsequent instructions.
+Bits 0, 1 and 2 control indexing of *src0*, *src1* and *src2*, while bit 3 controls indexing of *dst*.
+Value 1 enables indexing and value 0 disables it.
+
+    ===== ========================================
+    Bit   Meaning
+    ===== ========================================
+    0     Enables or disables *src0* indexing.
+    1     Enables or disables *src1* indexing.
+    2     Enables or disables *src2* indexing.
+    3     Enables or disables *dst* indexing.
+    ===== ========================================
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..15.
+* A *gpr_idx* value described below.
+
+    ==================================== ===========================================
+    Gpr_idx Value Syntax                 Description
+    ==================================== ===========================================
+    gpr_idx(*<operands>*)                Enable indexing for specified *operands*
+                                         and disable it for the rest.
+                                         *Operands* is a comma-separated list of
+                                         values which may include:
+
+                                         * "SRC0" - enable *src0* indexing.
+
+                                         * "SRC1" - enable *src1* indexing.
+
+                                         * "SRC2" - enable *src2* indexing.
+
+                                         * "DST"  - enable *dst* indexing.
+
+                                         Each of these values may be specified only
+                                         once.
+
+                                         *Operands* list may be empty; this syntax
+                                         disables indexing for all operands.
+    ==================================== ===========================================
+
+Examples:
+
+.. parsed-literal::
+
+    s_set_gpr_idx_mode 0
+    s_set_gpr_idx_mode gpr_idx()                        // the same as above
+
+    s_set_gpr_idx_mode 15
+    s_set_gpr_idx_mode gpr_idx(DST,SRC0,SRC1,SRC2)      // the same as above
+    s_set_gpr_idx_mode gpr_idx(SRC0,SRC1,SRC2,DST)      // the same as above
+
+    s_set_gpr_idx_mode gpr_idx(DST,SRC1)
+
diff --git a/llvm/docs/AMDGPU/gfx8_imm4.rst b/llvm/docs/AMDGPU/gfx8_imm4.rst
deleted file mode 100644
index a03de76e86efb..0000000000000
--- a/llvm/docs/AMDGPU/gfx8_imm4.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-..
-    **************************************************
-    *                                                *
-    *   Automatically generated file, do not edit!   *
-    *                                                *
-    **************************************************
-
-.. _amdgpu_synid8_imm4:
-
-imm4
-===========================
-
-A positive :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 4 bits.
-
-This operand is a mask which controls indexing mode for operands of subsequent instructions. Value 1 enables indexing and value 0 disables it.
-
-    ============ ========================================
-    Bit          Meaning
-    ============ ========================================
-    0            Enables or disables *src0* indexing.
-    1            Enables or disables *src1* indexing.
-    2            Enables or disables *src2* indexing.
-    3            Enables or disables *dst* indexing.
-    ============ ========================================
-
diff --git a/llvm/docs/AMDGPU/gfx8_label.rst b/llvm/docs/AMDGPU/gfx8_label.rst
index 99e384ee392e8..4f10c76f26875 100644
--- a/llvm/docs/AMDGPU/gfx8_label.rst
+++ b/llvm/docs/AMDGPU/gfx8_label.rst
@@ -12,19 +12,26 @@ label
 
 A branch target which is a 16-bit signed integer treated as a PC-relative dword offset.
 
-This operand may be specified as:
+This operand may be specified as one of the following:
 
-* An :ref:`integer_number<amdgpu_synid_integer_number>`. The number is truncated to 16 bits.
-* An :ref:`absolute_expression<amdgpu_synid_absolute_expression>` which must start with an :ref:`integer_number<amdgpu_synid_integer_number>`. The value of the expression is truncated to 16 bits.
-* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label). The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
+* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label) representing a relocatable address in the same compilation unit where it is referred from. The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
 
 Examples:
 
 .. parsed-literal::
 
   offset = 30
-  s_branch loop_end
-  s_branch 2 + offset
+  label_1:
+  label_2 = . + 4
+
   s_branch 32
-  loop_end:
+  s_branch offset + 2
+  s_branch label_1
+  s_branch label_2
+  s_branch label_3
+  s_branch label_4
+
+  label_3 = label_2 + 4
+  label_4:
 
diff --git a/llvm/docs/AMDGPU/gfx8_msg.rst b/llvm/docs/AMDGPU/gfx8_msg.rst
index 313d8e68b4b80..0b0b2f3074820 100644
--- a/llvm/docs/AMDGPU/gfx8_msg.rst
+++ b/llvm/docs/AMDGPU/gfx8_msg.rst
@@ -12,24 +12,29 @@ msg
 
 A 16-bit message code. The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          Message *type*.
-    6:4          Optional *operation*.
-    9:7          Optional *parameters*.
-    15:10        Unused.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below:
-
-    ======================================== ========================================================================
-    Syntax                                   Description
-    ======================================== ========================================================================
-    sendmsg(<*type*>)                        A message identified by its *type*.
-    sendmsg(<*type*>, <*op*>)                A message identified by its *type* and *operation*.
-    sendmsg(<*type*>, <*op*>, <*stream*>)    A message identified by its *type* and *operation* with a stream *id*.
-    ======================================== ========================================================================
+    ============ =============================== ===============
+    Bits         Description                     Value Range
+    ============ =============================== ===============
+    3:0          Message *type*.                 0..15
+    6:4          Optional *operation*.           0..7
+    7:7          Unused.                         \-
+    9:8          Optional *stream*.              0..3
+    15:10        Unused.                         \-
+    ============ =============================== ===============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A *sendmsg* value described below.
+
+    ==================================== ====================================================
+    Sendmsg Value Syntax                 Description
+    ==================================== ====================================================
+    sendmsg(<*type*>)                    A message identified by its *type*.
+    sendmsg(<*type*>,<*op*>)             A message identified by its *type* and *operation*.
+    sendmsg(<*type*>,<*op*>,<*stream*>)  A message identified by its *type* and *operation*
+                                         with a stream *id*.
+    ==================================== ====================================================
 
 *Type* may be specified using message *name* or message *id*.
 
@@ -37,7 +42,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
 
 Stream *id* is an integer in the range 0..3.
 
-Message *id*, operation *id* and stream *id* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Each message type supports specific operations:
 
@@ -58,15 +64,31 @@ Each message type supports specific operations:
     \                            SYSMSG_OP_TTRACE_PC            4            \-
     ================= ========== ============================== ============ ==========
 
+*Sendmsg* arguments are validated depending on how *type* value is specified:
+
+* If message *type* is specified by name, arguments values must satisfy limitations detailed in the table above.
+* If message *type* is specified as a number, each argument must not exceed corresponding value range (see the first table).
+
 Examples:
 
 .. parsed-literal::
 
+    // numeric message code
+    msg = 0x10
     s_sendmsg 0x12
+    s_sendmsg msg + 2
+
+    // sendmsg with strict arguments validation
     s_sendmsg sendmsg(MSG_INTERRUPT)
-    s_sendmsg sendmsg(2, GS_OP_CUT)
     s_sendmsg sendmsg(MSG_GS, GS_OP_EMIT)
     s_sendmsg sendmsg(MSG_GS, 2)
     s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_EMIT_CUT, 1)
     s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_TTRACE_PC)
 
+    // sendmsg with validation of value range only
+    msg = 2
+    op = 3
+    stream = 1
+    s_sendmsg sendmsg(msg, op, stream)
+    s_sendmsg sendmsg(2, GS_OP_CUT)
+
diff --git a/llvm/docs/AMDGPU/gfx8_perm_smem.rst b/llvm/docs/AMDGPU/gfx8_perm_smem.rst
index 0035ac821a724..75d02ddac796a 100644
--- a/llvm/docs/AMDGPU/gfx8_perm_smem.rst
+++ b/llvm/docs/AMDGPU/gfx8_perm_smem.rst
@@ -12,7 +12,8 @@ imm3
 
 A bit mask which indicates request permissions.
 
-This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 7 bits, but only 3 low bits are significant.
+This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is truncated to 7 bits, but only 3 low bits are significant.
 
     ============ ==============================
     Bit Number   Description
diff --git a/llvm/docs/AMDGPU/gfx8_simm16.rst b/llvm/docs/AMDGPU/gfx8_simm16.rst
index 730f239b6bea7..86161c5400ef9 100644
--- a/llvm/docs/AMDGPU/gfx8_simm16.rst
+++ b/llvm/docs/AMDGPU/gfx8_simm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then sign-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx8_uimm16.rst b/llvm/docs/AMDGPU/gfx8_uimm16.rst
index a20abcc134486..9da1c60a8a73b 100644
--- a/llvm/docs/AMDGPU/gfx8_uimm16.rst
+++ b/llvm/docs/AMDGPU/gfx8_uimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then zero-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx8_waitcnt.rst b/llvm/docs/AMDGPU/gfx8_waitcnt.rst
index 4bad594171164..29b430e074199 100644
--- a/llvm/docs/AMDGPU/gfx8_waitcnt.rst
+++ b/llvm/docs/AMDGPU/gfx8_waitcnt.rst
@@ -14,29 +14,31 @@ Counts of outstanding instructions to wait for.
 
 The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          VM_CNT: vector memory operations count.
-    6:4          EXP_CNT: export count.
-    11:8         LGKM_CNT: LDS, GDS, Constant and Message count.
-    ============ ======================================================
+   ===== ================================================ ============
+   Bits  Description                                      Value Range
+   ===== ================================================ ============
+   3:0   VM_CNT: vector memory operations count.          0..15
+   6:4   EXP_CNT: export count.                           0..7
+   11:8  LGKM_CNT: LDS, GDS, Constant and Message count.  0..15
+   ===== ================================================ ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>`
-or as a combination of the following symbolic helpers:
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A combination of *vmcnt*, *expcnt*, *lgkmcnt* and other values described below.
 
     ====================== ======================================================================
     Syntax                 Description
     ====================== ======================================================================
-    vmcnt(<*N*>)           VM_CNT value. *N* must not exceed the largest VM_CNT value.
-    expcnt(<*N*>)          EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
-    lgkmcnt(<*N*>)         LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
-    vmcnt_sat(<*N*>)       VM_CNT value computed as min(*N*, the largest VM_CNT value).
-    expcnt_sat(<*N*>)      EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
-    lgkmcnt_sat(<*N*>)     LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
+    vmcnt(<*N*>)           A VM_CNT value. *N* must not exceed the largest VM_CNT value.
+    expcnt(<*N*>)          An EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
+    lgkmcnt(<*N*>)         An LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
+    vmcnt_sat(<*N*>)       A VM_CNT value computed as min(*N*, the largest VM_CNT value).
+    expcnt_sat(<*N*>)      An EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
+    lgkmcnt_sat(<*N*>)     An LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
     ====================== ======================================================================
 
-These helpers may be specified in any order. Ampersands and commas may be used as optional separators.
+These values may be specified in any order. Spaces, ampersands and commas may be used as optional separators.
 
 *N* is either an
 :ref:`integer number<amdgpu_synid_integer_number>` or an
@@ -46,10 +48,18 @@ Examples:
 
 .. parsed-literal::
 
-    s_waitcnt 0
+    vm_cnt = 1
+    exp_cnt = 2
+    lgkm_cnt = 3
+    cnt = vm_cnt | (exp_cnt << 4) | (lgkm_cnt << 8)
+
+    s_waitcnt cnt
+    s_waitcnt 1 | (2 << 4) | (3 << 8)                          // the same as above
+    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)                    // the same as above
+    s_waitcnt vmcnt(vm_cnt) expcnt(exp_cnt) lgkmcnt(lgkm_cnt)  // the same as above
+
     s_waitcnt vmcnt(1)
     s_waitcnt expcnt(2) lgkmcnt(3)
-    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)
     s_waitcnt vmcnt(1), expcnt(2), lgkmcnt(3)
     s_waitcnt vmcnt(1) & lgkmcnt_sat(100) & expcnt(2)
 
diff --git a/llvm/docs/AMDGPU/gfx9_bimm16.rst b/llvm/docs/AMDGPU/gfx9_bimm16.rst
index 2c9dc5c521561..6e961167f47ab 100644
--- a/llvm/docs/AMDGPU/gfx9_bimm16.rst
+++ b/llvm/docs/AMDGPU/gfx9_bimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits.
+A 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx9_bimm32.rst b/llvm/docs/AMDGPU/gfx9_bimm32.rst
index e9b89674a2ef8..22286f1d27200 100644
--- a/llvm/docs/AMDGPU/gfx9_bimm32.rst
+++ b/llvm/docs/AMDGPU/gfx9_bimm32.rst
@@ -10,5 +10,5 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value is truncated to 32 bits.
 
diff --git a/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_cmp.rst b/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_cmp.rst
index 08fe297b4f39f..79d10fdb4e969 100644
--- a/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_cmp.rst
+++ b/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_cmp.rst
@@ -21,7 +21,7 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 2 data elements for 32-bit-per-pixel surfaces or 4 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_reg.rst b/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_reg.rst
index 2037dfd5356b4..6889c468d26e4 100644
--- a/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_reg.rst
+++ b/llvm/docs/AMDGPU/gfx9_data_mimg_atomic_reg.rst
@@ -21,6 +21,6 @@ Optionally may serve as an output data:
 * :ref:`dmask<amdgpu_synid_dmask>` may specify 1 data element for 32-bit-per-pixel surfaces or 2 data elements for 64-bit-per-pixel surfaces. Each data element occupies 1 dword.
 * :ref:`tfe<amdgpu_synid_tfe>` adds 1 dword if specified.
 
-  Note. The surface data format is indicated in the image resource constant but not in the instruction.
+  Note: the surface data format is indicated in the image resource constant but not in the instruction.
 
 *Operands:* :ref:`v<amdgpu_synid_v>`
diff --git a/llvm/docs/AMDGPU/gfx9_fimm16.rst b/llvm/docs/AMDGPU/gfx9_fimm16.rst
index a438b452eecfa..53432d2594a21 100644
--- a/llvm/docs/AMDGPU/gfx9_fimm16.rst
+++ b/llvm/docs/AMDGPU/gfx9_fimm16.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The number is converted to *f16* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f16* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx9_fimm32.rst b/llvm/docs/AMDGPU/gfx9_fimm32.rst
index 11103e70529e3..e3198735ae7ab 100644
--- a/llvm/docs/AMDGPU/gfx9_fimm32.rst
+++ b/llvm/docs/AMDGPU/gfx9_fimm32.rst
@@ -10,5 +10,6 @@
 imm32
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>` or a :ref:`floating-point_number<amdgpu_synid_floating-point_number>`. The value is converted to *f32* as described :ref:`here<amdgpu_synid_lit_conv>`.
+A :ref:`floating-point_number<amdgpu_synid_floating-point_number>`, an :ref:`integer_number<amdgpu_synid_integer_number>`, or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is converted to *f32* as described :ref:`here<amdgpu_synid_fp_conv>`.
 
diff --git a/llvm/docs/AMDGPU/gfx9_hwreg.rst b/llvm/docs/AMDGPU/gfx9_hwreg.rst
index 7ebb38b42fe01..b0c70cb6ccf80 100644
--- a/llvm/docs/AMDGPU/gfx9_hwreg.rst
+++ b/llvm/docs/AMDGPU/gfx9_hwreg.rst
@@ -14,18 +14,21 @@ Bits of a hardware register being accessed.
 
 The bits of this operand have the following meaning:
 
-    ============ ===================================
-    Bits         Description
-    ============ ===================================
-    5:0          Register *id*.
-    10:6         First bit *offset* (0..31).
-    15:11        *Size* in bits (1..32).
-    ============ ===================================
+    ======= ===================== ============
+    Bits    Description           Value Range
+    ======= ===================== ============
+    5:0     Register *id*.        0..63
+    10:6    First bit *offset*.   0..31
+    15:11   *Size* in bits.       1..32
+    ======= ===================== ============
 
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below.
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* An *hwreg* value described below.
 
     ==================================== ============================================================================
-    Syntax                               Description
+    Hwreg Value Syntax                   Description
     ==================================== ============================================================================
     hwreg({0..63})                       All bits of a register indicated by its *id*.
     hwreg(<*name*>)                      All bits of a register indicated by its *name*.
@@ -33,7 +36,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
     hwreg(<*name*>, {0..31}, {1..32})    Register bits indicated by register *name*, first bit *offset* and *size*.
     ==================================== ============================================================================
 
-Register *id*, *offset* and *size* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Defined register *names* include:
 
@@ -54,7 +58,16 @@ Examples:
 
 .. parsed-literal::
 
-    s_getreg_b32 s2, 0x6
+    reg = 1
+    offset = 2
+    size = 4
+    hwreg_enc = reg | (offset << 6) | ((size - 1) << 11)
+
+    s_getreg_b32 s2, 0x1881
+    s_getreg_b32 s2, hwreg_enc                     // the same as above
+    s_getreg_b32 s2, hwreg(1, 2, 4)                // the same as above
+    s_getreg_b32 s2, hwreg(reg, offset, size)      // the same as above
+
     s_getreg_b32 s2, hwreg(15)
     s_getreg_b32 s2, hwreg(51, 1, 31)
     s_getreg_b32 s2, hwreg(HW_REG_LDS_ALLOC, 0, 1)
diff --git a/llvm/docs/AMDGPU/gfx9_imask.rst b/llvm/docs/AMDGPU/gfx9_imask.rst
new file mode 100644
index 0000000000000..22d73cb4ec042
--- /dev/null
+++ b/llvm/docs/AMDGPU/gfx9_imask.rst
@@ -0,0 +1,66 @@
+..
+    **************************************************
+    *                                                *
+    *   Automatically generated file, do not edit!   *
+    *                                                *
+    **************************************************
+
+.. _amdgpu_synid9_imask:
+
+imask
+===========================
+
+This operand is a mask which controls indexing mode for operands of subsequent instructions.
+Bits 0, 1 and 2 control indexing of *src0*, *src1* and *src2*, while bit 3 controls indexing of *dst*.
+Value 1 enables indexing and value 0 disables it.
+
+    ===== ========================================
+    Bit   Meaning
+    ===== ========================================
+    0     Enables or disables *src0* indexing.
+    1     Enables or disables *src1* indexing.
+    2     Enables or disables *src2* indexing.
+    3     Enables or disables *dst* indexing.
+    ===== ========================================
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..15.
+* A *gpr_idx* value described below.
+
+    ==================================== ===========================================
+    Gpr_idx Value Syntax                 Description
+    ==================================== ===========================================
+    gpr_idx(*<operands>*)                Enable indexing for specified *operands*
+                                         and disable it for the rest.
+                                         *Operands* is a comma-separated list of
+                                         values which may include:
+
+                                         * "SRC0" - enable *src0* indexing.
+
+                                         * "SRC1" - enable *src1* indexing.
+
+                                         * "SRC2" - enable *src2* indexing.
+
+                                         * "DST"  - enable *dst* indexing.
+
+                                         Each of these values may be specified only
+                                         once.
+
+                                         *Operands* list may be empty; this syntax
+                                         disables indexing for all operands.
+    ==================================== ===========================================
+
+Examples:
+
+.. parsed-literal::
+
+    s_set_gpr_idx_mode 0
+    s_set_gpr_idx_mode gpr_idx()                        // the same as above
+
+    s_set_gpr_idx_mode 15
+    s_set_gpr_idx_mode gpr_idx(DST,SRC0,SRC1,SRC2)      // the same as above
+    s_set_gpr_idx_mode gpr_idx(SRC0,SRC1,SRC2,DST)      // the same as above
+
+    s_set_gpr_idx_mode gpr_idx(DST,SRC1)
+
diff --git a/llvm/docs/AMDGPU/gfx9_imm4.rst b/llvm/docs/AMDGPU/gfx9_imm4.rst
deleted file mode 100644
index b1c97fb0b29d8..0000000000000
--- a/llvm/docs/AMDGPU/gfx9_imm4.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-..
-    **************************************************
-    *                                                *
-    *   Automatically generated file, do not edit!   *
-    *                                                *
-    **************************************************
-
-.. _amdgpu_synid9_imm4:
-
-imm4
-===========================
-
-A positive :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 4 bits.
-
-This operand is a mask which controls indexing mode for operands of subsequent instructions. Value 1 enables indexing and value 0 disables it.
-
-    ============ ========================================
-    Bit          Meaning
-    ============ ========================================
-    0            Enables or disables *src0* indexing.
-    1            Enables or disables *src1* indexing.
-    2            Enables or disables *src2* indexing.
-    3            Enables or disables *dst* indexing.
-    ============ ========================================
-
diff --git a/llvm/docs/AMDGPU/gfx9_label.rst b/llvm/docs/AMDGPU/gfx9_label.rst
index 32771722f71de..7348fc9148876 100644
--- a/llvm/docs/AMDGPU/gfx9_label.rst
+++ b/llvm/docs/AMDGPU/gfx9_label.rst
@@ -12,19 +12,26 @@ label
 
 A branch target which is a 16-bit signed integer treated as a PC-relative dword offset.
 
-This operand may be specified as:
+This operand may be specified as one of the following:
 
-* An :ref:`integer_number<amdgpu_synid_integer_number>`. The number is truncated to 16 bits.
-* An :ref:`absolute_expression<amdgpu_synid_absolute_expression>` which must start with an :ref:`integer_number<amdgpu_synid_integer_number>`. The value of the expression is truncated to 16 bits.
-* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label). The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
+* A :ref:`symbol<amdgpu_synid_symbol>` (for example, a label) representing a relocatable address in the same compilation unit where it is referred from. The value is handled as a 16-bit PC-relative dword offset to be resolved by a linker.
 
 Examples:
 
 .. parsed-literal::
 
   offset = 30
-  s_branch loop_end
-  s_branch 2 + offset
+  label_1:
+  label_2 = . + 4
+
   s_branch 32
-  loop_end:
+  s_branch offset + 2
+  s_branch label_1
+  s_branch label_2
+  s_branch label_3
+  s_branch label_4
+
+  label_3 = label_2 + 4
+  label_4:
 
diff --git a/llvm/docs/AMDGPU/gfx9_msg.rst b/llvm/docs/AMDGPU/gfx9_msg.rst
index 14dff9050e658..34ede7100381d 100644
--- a/llvm/docs/AMDGPU/gfx9_msg.rst
+++ b/llvm/docs/AMDGPU/gfx9_msg.rst
@@ -12,24 +12,29 @@ msg
 
 A 16-bit message code. The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          Message *type*.
-    6:4          Optional *operation*.
-    9:7          Optional *parameters*.
-    15:10        Unused.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>` or using the syntax described below:
-
-    ======================================== ========================================================================
-    Syntax                                   Description
-    ======================================== ========================================================================
-    sendmsg(<*type*>)                        A message identified by its *type*.
-    sendmsg(<*type*>, <*op*>)                A message identified by its *type* and *operation*.
-    sendmsg(<*type*>, <*op*>, <*stream*>)    A message identified by its *type* and *operation* with a stream *id*.
-    ======================================== ========================================================================
+    ============ =============================== ===============
+    Bits         Description                     Value Range
+    ============ =============================== ===============
+    3:0          Message *type*.                 0..15
+    6:4          Optional *operation*.           0..7
+    7:7          Unused.                         \-
+    9:8          Optional *stream*.              0..3
+    15:10        Unused.                         \-
+    ============ =============================== ===============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A *sendmsg* value described below.
+
+    ==================================== ====================================================
+    Sendmsg Value Syntax                 Description
+    ==================================== ====================================================
+    sendmsg(<*type*>)                    A message identified by its *type*.
+    sendmsg(<*type*>,<*op*>)             A message identified by its *type* and *operation*.
+    sendmsg(<*type*>,<*op*>,<*stream*>)  A message identified by its *type* and *operation*
+                                         with a stream *id*.
+    ==================================== ====================================================
 
 *Type* may be specified using message *name* or message *id*.
 
@@ -37,7 +42,8 @@ This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_s
 
 Stream *id* is an integer in the range 0..3.
 
-Message *id*, operation *id* and stream *id* must be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`.
+Numeric values may be specified as positive :ref:`integer numbers<amdgpu_synid_integer_number>`
+or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Each message type supports specific operations:
 
@@ -60,16 +66,32 @@ Each message type supports specific operations:
     \                            SYSMSG_OP_TTRACE_PC            4            \-
     ================= ========== ============================== ============ ==========
 
+*Sendmsg* arguments are validated depending on how *type* value is specified:
+
+* If message *type* is specified by name, arguments values must satisfy limitations detailed in the table above.
+* If message *type* is specified as a number, each argument must not exceed corresponding value range (see the first table).
+
 Examples:
 
 .. parsed-literal::
 
+    // numeric message code
+    msg = 0x10
     s_sendmsg 0x12
+    s_sendmsg msg + 2
+
+    // sendmsg with strict arguments validation
     s_sendmsg sendmsg(MSG_INTERRUPT)
-    s_sendmsg sendmsg(MSG_GET_DOORBELL)
-    s_sendmsg sendmsg(2, GS_OP_CUT)
     s_sendmsg sendmsg(MSG_GS, GS_OP_EMIT)
     s_sendmsg sendmsg(MSG_GS, 2)
     s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_EMIT_CUT, 1)
     s_sendmsg sendmsg(MSG_SYSMSG, SYSMSG_OP_TTRACE_PC)
+    s_sendmsg sendmsg(MSG_GET_DOORBELL)
+
+    // sendmsg with validation of value range only
+    msg = 2
+    op = 3
+    stream = 1
+    s_sendmsg sendmsg(msg, op, stream)
+    s_sendmsg sendmsg(2, GS_OP_CUT)
 
diff --git a/llvm/docs/AMDGPU/gfx9_perm_smem.rst b/llvm/docs/AMDGPU/gfx9_perm_smem.rst
index 370fb0d67b33f..d13c566789cf1 100644
--- a/llvm/docs/AMDGPU/gfx9_perm_smem.rst
+++ b/llvm/docs/AMDGPU/gfx9_perm_smem.rst
@@ -12,7 +12,8 @@ imm3
 
 A bit mask which indicates request permissions.
 
-This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 7 bits, but only 3 low bits are significant.
+This operand must be specified as an :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`.
+The value is truncated to 7 bits, but only 3 low bits are significant.
 
     ============ ==============================
     Bit Number   Description
diff --git a/llvm/docs/AMDGPU/gfx9_simm16.rst b/llvm/docs/AMDGPU/gfx9_simm16.rst
index 47b200a72070c..4f734a04239a6 100644
--- a/llvm/docs/AMDGPU/gfx9_simm16.rst
+++ b/llvm/docs/AMDGPU/gfx9_simm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then sign-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range -32768..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx9_uimm16.rst b/llvm/docs/AMDGPU/gfx9_uimm16.rst
index 4d1fe1de3f20b..dd3c9b4a2995d 100644
--- a/llvm/docs/AMDGPU/gfx9_uimm16.rst
+++ b/llvm/docs/AMDGPU/gfx9_uimm16.rst
@@ -10,5 +10,5 @@
 imm16
 ===========================
 
-An :ref:`integer_number<amdgpu_synid_integer_number>`. The value is truncated to 16 bits and then zero-extended to 32 bits.
+An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..65535.
 
diff --git a/llvm/docs/AMDGPU/gfx9_waitcnt.rst b/llvm/docs/AMDGPU/gfx9_waitcnt.rst
index 015a51ae8c329..342f8c7fdb9e5 100644
--- a/llvm/docs/AMDGPU/gfx9_waitcnt.rst
+++ b/llvm/docs/AMDGPU/gfx9_waitcnt.rst
@@ -14,30 +14,31 @@ Counts of outstanding instructions to wait for.
 
 The bits of this operand have the following meaning:
 
-    ============ ======================================================
-    Bits         Description
-    ============ ======================================================
-    3:0          VM_CNT: vector memory operations count, lower bits.
-    6:4          EXP_CNT: export count.
-    11:8         LGKM_CNT: LDS, GDS, Constant and Message count.
-    15:14        VM_CNT: vector memory operations count, upper bits.
-    ============ ======================================================
-
-This operand may be specified as a positive 16-bit :ref:`integer_number<amdgpu_synid_integer_number>`
-or as a combination of the following symbolic helpers:
+    ========== ========= ================================================ ============
+    High Bits  Low Bits  Description                                      Value Range
+    ========== ========= ================================================ ============
+    15:14      3:0       VM_CNT: vector memory operations count.          0..63
+    \-         6:4       EXP_CNT: export count.                           0..7
+    \-         11:8      LGKM_CNT: LDS, GDS, Constant and Message count.  0..15
+    ========== ========= ================================================ ============
+
+This operand may be specified as one of the following:
+
+* An :ref:`integer_number<amdgpu_synid_integer_number>` or an :ref:`absolute_expression<amdgpu_synid_absolute_expression>`. The value must be in the range 0..0xFFFF.
+* A combination of *vmcnt*, *expcnt*, *lgkmcnt* and other values described below.
 
     ====================== ======================================================================
     Syntax                 Description
     ====================== ======================================================================
-    vmcnt(<*N*>)           VM_CNT value. *N* must not exceed the largest VM_CNT value.
-    expcnt(<*N*>)          EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
-    lgkmcnt(<*N*>)         LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
-    vmcnt_sat(<*N*>)       VM_CNT value computed as min(*N*, the largest VM_CNT value).
-    expcnt_sat(<*N*>)      EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
-    lgkmcnt_sat(<*N*>)     LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
+    vmcnt(<*N*>)           A VM_CNT value. *N* must not exceed the largest VM_CNT value.
+    expcnt(<*N*>)          An EXP_CNT value. *N* must not exceed the largest EXP_CNT value.
+    lgkmcnt(<*N*>)         An LGKM_CNT value. *N* must not exceed the largest LGKM_CNT value.
+    vmcnt_sat(<*N*>)       A VM_CNT value computed as min(*N*, the largest VM_CNT value).
+    expcnt_sat(<*N*>)      An EXP_CNT value computed as min(*N*, the largest EXP_CNT value).
+    lgkmcnt_sat(<*N*>)     An LGKM_CNT value computed as min(*N*, the largest LGKM_CNT value).
     ====================== ======================================================================
 
-These helpers may be specified in any order. Ampersands and commas may be used as optional separators.
+These values may be specified in any order. Spaces, ampersands and commas may be used as optional separators.
 
 *N* is either an
 :ref:`integer number<amdgpu_synid_integer_number>` or an
@@ -47,10 +48,18 @@ Examples:
 
 .. parsed-literal::
 
-    s_waitcnt 0
+    vm_cnt = 1
+    exp_cnt = 2
+    lgkm_cnt = 3
+    cnt = vm_cnt | (exp_cnt << 4) | (lgkm_cnt << 8)
+
+    s_waitcnt cnt
+    s_waitcnt 1 | (2 << 4) | (3 << 8)                          // the same as above
+    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)                    // the same as above
+    s_waitcnt vmcnt(vm_cnt) expcnt(exp_cnt) lgkmcnt(lgkm_cnt)  // the same as above
+
     s_waitcnt vmcnt(1)
     s_waitcnt expcnt(2) lgkmcnt(3)
-    s_waitcnt vmcnt(1) expcnt(2) lgkmcnt(3)
     s_waitcnt vmcnt(1), expcnt(2), lgkmcnt(3)
     s_waitcnt vmcnt(1) & lgkmcnt_sat(100) & expcnt(2)
 
diff --git a/llvm/docs/AMDGPUModifierSyntax.rst b/llvm/docs/AMDGPUModifierSyntax.rst
index d66e94dcb91a0..526016d4f4f81 100644
--- a/llvm/docs/AMDGPUModifierSyntax.rst
+++ b/llvm/docs/AMDGPUModifierSyntax.rst
@@ -34,19 +34,21 @@ Specifies an immediate unsigned 8-bit offset, in bytes. The default value is 0.
 
 Used with DS instructions which have 2 addresses.
 
-    =================== =====================================================
+    =================== ====================================================================
     Syntax              Description
-    =================== =====================================================
+    =================== ====================================================================
     offset:{0..0xFF}    Specifies an unsigned 8-bit offset as a positive
-                        :ref:`integer number <amdgpu_synid_integer_number>`.
-    =================== =====================================================
+                        :ref:`integer number <amdgpu_synid_integer_number>`
+                        or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    =================== ====================================================================
 
 Examples:
 
 .. parsed-literal::
 
-  offset:255
   offset:0xff
+  offset:2-x
+  offset:-x-y
 
 .. _amdgpu_synid_ds_offset16:
 
@@ -57,12 +59,13 @@ Specifies an immediate unsigned 16-bit offset, in bytes. The default value is 0.
 
 Used with DS instructions which have 1 address.
 
-    ==================== ======================================================
+    ==================== ====================================================================
     Syntax               Description
-    ==================== ======================================================
+    ==================== ====================================================================
     offset:{0..0xFFFF}   Specifies an unsigned 16-bit offset as a positive
-                         :ref:`integer number <amdgpu_synid_integer_number>`.
-    ==================== ======================================================
+                         :ref:`integer number <amdgpu_synid_integer_number>`
+                         or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ==================== ====================================================================
 
 Examples:
 
@@ -70,6 +73,7 @@ Examples:
 
   offset:65535
   offset:0xffff
+  offset:-x-y
 
 .. _amdgpu_synid_sw_offset16:
 
@@ -95,7 +99,7 @@ See AMD documentation for more information.
 
                                                             *mask* is a 5 character sequence which
                                                             specifies how to transform the bits of the
-                                                            lane *id*. 
+                                                            lane *id*.
 
                                                             The following characters are allowed:
 
@@ -116,7 +120,7 @@ See AMD documentation for more information.
                                                             size and must be equal to 2, 4, 8, 16 or 32.
 
                                                             The second numeric parameter is an index of the
-                                                            lane being broadcasted. 
+                                                            lane being broadcasted.
 
                                                             The index must not exceed group size.
     offset:swizzle(SWAP,{1..16})                            Specifies a swap mode.
@@ -128,7 +132,7 @@ See AMD documentation for more information.
                                                             Reverses the lanes for groups of 2, 4, 8, 16 or 32 lanes.
     ======================================================= ===========================================================
 
-Numeric parameters may be specified as either :ref:`integer numbers<amdgpu_synid_integer_number>` or
+Note: numeric values may be specified as either :ref:`integer numbers<amdgpu_synid_integer_number>` or
 :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
 Examples:
@@ -137,7 +141,7 @@ Examples:
 
   offset:255
   offset:0xffff
-  offset:swizzle(QUAD_PERM, 0, 1, 2 ,3)
+  offset:swizzle(QUAD_PERM, 0, 1, 2, 3)
   offset:swizzle(BITMASK_PERM, "01pi0")
   offset:swizzle(BROADCAST, 2, 0)
   offset:swizzle(SWAP, 8)
@@ -212,19 +216,20 @@ Specifies an immediate unsigned 12-bit offset, in bytes. The default value is 0.
 
 Cannot be used with *global/scratch* opcodes. GFX9 only.
 
-    ================= ======================================================
+    ================= ====================================================================
     Syntax            Description
-    ================= ======================================================
+    ================= ====================================================================
     offset:{0..4095}  Specifies a 12-bit unsigned offset as a positive
-                      :ref:`integer number <amdgpu_synid_integer_number>`.
-    ================= ======================================================
+                      :ref:`integer number <amdgpu_synid_integer_number>`
+                      or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ================= ====================================================================
 
 Examples:
 
 .. parsed-literal::
 
   offset:4095
-  offset:0xff
+  offset:x-0xff
 
 .. _amdgpu_synid_flat_offset13s:
 
@@ -235,12 +240,13 @@ Specifies an immediate signed 13-bit offset, in bytes. The default value is 0.
 
 Can be used with *global/scratch* opcodes only. GFX9 only.
 
-    ============================ =======================================================
-    Syntax                       Description
-    ============================ =======================================================
-    offset:{-4096..4095}         Specifies a 13-bit signed offset as an
-                                 :ref:`integer number <amdgpu_synid_integer_number>`.
-    ============================ =======================================================
+    ===================== ====================================================================
+    Syntax                Description
+    ===================== ====================================================================
+    offset:{-4096..4095}  Specifies a 13-bit signed offset as an
+                          :ref:`integer number <amdgpu_synid_integer_number>`
+                          or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ===================== ====================================================================
 
 Examples:
 
@@ -248,6 +254,7 @@ Examples:
 
   offset:-4000
   offset:0x10
+  offset:-x
 
 .. _amdgpu_synid_flat_offset12s:
 
@@ -260,12 +267,13 @@ Can be used with *global/scratch* opcodes only.
 
 GFX10 only.
 
-    ============================ =======================================================
-    Syntax                       Description
-    ============================ =======================================================
-    offset:{-2048..2047}         Specifies a 12-bit signed offset as an
-                                 :ref:`integer number <amdgpu_synid_integer_number>`.
-    ============================ =======================================================
+    ===================== ====================================================================
+    Syntax                Description
+    ===================== ====================================================================
+    offset:{-2048..2047}  Specifies a 12-bit signed offset as an
+                          :ref:`integer number <amdgpu_synid_integer_number>`
+                          or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ===================== ====================================================================
 
 Examples:
 
@@ -273,6 +281,7 @@ Examples:
 
   offset:-2000
   offset:0x10
+  offset:-x+y
 
 .. _amdgpu_synid_flat_offset11:
 
@@ -285,19 +294,20 @@ Cannot be used with *global/scratch* opcodes.
 
 GFX10 only.
 
-    ================= ======================================================
+    ================= ====================================================================
     Syntax            Description
-    ================= ======================================================
+    ================= ====================================================================
     offset:{0..2047}  Specifies an 11-bit unsigned offset as a positive
-                      :ref:`integer number <amdgpu_synid_integer_number>`.
-    ================= ======================================================
+                      :ref:`integer number <amdgpu_synid_integer_number>`
+                      or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ================= ====================================================================
 
 Examples:
 
 .. parsed-literal::
 
   offset:2047
-  offset:0xff
+  offset:x+0xff
 
 dlc
 ~~~
@@ -340,19 +350,18 @@ dmask
 Specifies which channels (image components) are used by the operation. By default, no channels
 are used.
 
-    =============== =====================================================
+    =============== ====================================================================
     Syntax          Description
-    =============== =====================================================
+    =============== ====================================================================
     dmask:{0..15}   Specifies image channels as a positive
-                    :ref:`integer number <amdgpu_synid_integer_number>`.
+                    :ref:`integer number <amdgpu_synid_integer_number>`
+                    or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
 
-                    Each bit corresponds to one of 4 image
-                    components (RGBA).
+                    Each bit corresponds to one of 4 image components (RGBA).
 
-                    If the specified bit value
-                    is 0, the component is not used, value 1 means
-                    that the component is used.
-    =============== =====================================================
+                    If the specified bit value is 0, the component is not used,
+                    value 1 means that the component is used.
+    =============== ====================================================================
 
 This modifier has some limitations depending on instruction kind:
 
@@ -373,7 +382,7 @@ Examples:
 
   dmask:0xf
   dmask:0b1111
-  dmask:3
+  dmask:x|y|z
 
 .. _amdgpu_synid_unorm:
 
@@ -468,7 +477,7 @@ Specifies data size: 16 or 32 bits (32 bits by default). Not supported by GFX7.
                                              Each 16-bit data element occupies 1 VGPR.
 
                                              GFX8.1, GFX9 and GFX10 support data packing.
-                                             Each pair of 16-bit data elements 
+                                             Each pair of 16-bit data elements
                                              occupies 1 VGPR.
     ======================================== ================================================
 
@@ -684,18 +693,19 @@ offset12
 
 Specifies an immediate unsigned 12-bit offset, in bytes. The default value is 0.
 
-    =============================== ======================================================
-    Syntax                          Description
-    =============================== ======================================================
-    offset:{0..0xFFF}               Specifies a 12-bit unsigned offset as a positive
-                                    :ref:`integer number <amdgpu_synid_integer_number>`.
-    =============================== ======================================================
+    ================== ====================================================================
+    Syntax             Description
+    ================== ====================================================================
+    offset:{0..0xFFF}  Specifies a 12-bit unsigned offset as a positive
+                       :ref:`integer number <amdgpu_synid_integer_number>`
+                       or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+    ================== ====================================================================
 
 Examples:
 
 .. parsed-literal::
 
-  offset:0
+  offset:x+y
   offset:0x10
 
 glc
@@ -782,14 +792,18 @@ GFX10 only.
 dpp8_sel
 ~~~~~~~~
 
-Selects which lane to pull data from, within a group of 8 lanes. This is a mandatory modifier.
+Selects which lanes to pull data from, within a group of 8 lanes. This is a mandatory modifier.
 There is no default value.
 
 GFX10 only.
 
-The *dpp8_sel* modifier must specify exactly 8 values, each ranging from 0 to 7.
+The *dpp8_sel* modifier must specify exactly 8 values.
 First value selects which lane to read from to supply data into lane 0.
-Second value controls value for lane 1 and so on.
+Second value controls lane 1 and so on.
+
+Each value may be specified as either
+an :ref:`integer number<amdgpu_synid_integer_number>` or
+an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
 
     =============================================================== ===========================
     Syntax                                                          Description
@@ -811,7 +825,7 @@ fi
 
 Controls interaction with inactive lanes for *dpp8* instructions. The default value is zero.
 
-Note. *Inactive* lanes are those whose :ref:`exec<amdgpu_synid_exec>` mask bit is zero.
+Note: *inactive* lanes are those whose :ref:`exec<amdgpu_synid_exec>` mask bit is zero.
 
 GFX10 only.
 
@@ -822,6 +836,9 @@ GFX10 only.
     fi:1                                 Fetch pre-exist values from inactive lanes.
     ==================================== =====================================================
 
+Note: numeric values may be specified as either :ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 DPP/DPP16 Modifiers
 -------------------
 
@@ -837,7 +854,7 @@ There is no default value.
 
 GFX8 and GFX9 only. Use :ref:`dpp16_ctrl<amdgpu_synid_dpp16_ctrl>` for GFX10.
 
-Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
+Note: the lanes of a wavefront are organized in four *rows* and four *banks*.
 
     ======================================== ================================================
     Syntax                                   Description
@@ -856,7 +873,7 @@ Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
     row_ror:{1..15}                          Row rotate right by 1-15 threads.
     ======================================== ================================================
 
-Note: Numeric parameters may be specified as either
+Note: numeric values may be specified as either
 :ref:`integer numbers<amdgpu_synid_integer_number>` or
 :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
@@ -877,7 +894,7 @@ There is no default value.
 
 GFX10 only. Use :ref:`dpp_ctrl<amdgpu_synid_dpp_ctrl>` for GFX8 and GFX9.
 
-Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
+Note: the lanes of a wavefront are organized in four *rows* and four *banks*.
 (There are only two rows in *wave32* mode.)
 
     ======================================== ====================================================
@@ -894,7 +911,7 @@ Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
     row_ror:{1..15}                          Row rotate right by 1-15 threads.
     ======================================== ====================================================
 
-Note: Numeric parameters may be specified as either
+Note: numeric values may be specified as either
 :ref:`integer numbers<amdgpu_synid_integer_number>` or
 :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
 
@@ -912,21 +929,21 @@ row_mask
 
 Controls which rows are enabled for data sharing. By default, all rows are enabled.
 
-Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
+Note: the lanes of a wavefront are organized in four *rows* and four *banks*.
 (There are only two rows in *wave32* mode.)
 
-    ======================================== =====================================================
-    Syntax                                   Description
-    ======================================== =====================================================
-    row_mask:{0..15}                         Specifies a *row mask* as a positive
-                                             :ref:`integer number <amdgpu_synid_integer_number>`.
+    ================= ====================================================================
+    Syntax            Description
+    ================= ====================================================================
+    row_mask:{0..15}  Specifies a *row mask* as a positive
+                      :ref:`integer number <amdgpu_synid_integer_number>`
+                      or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
 
-                                             Each of 4 bits in the mask controls one
-                                             row (0 - disabled, 1 - enabled).
+                      Each of 4 bits in the mask controls one row
+                      (0 - disabled, 1 - enabled).
 
-                                             In *wave32* mode the values should be limited to
-                                             {0..7}.
-    ======================================== =====================================================
+                      In *wave32* mode the values should be limited to 0..7.
+    ================= ====================================================================
 
 Examples:
 
@@ -934,7 +951,7 @@ Examples:
 
   row_mask:0xf
   row_mask:0b1010
-  row_mask:0b1111
+  row_mask:x|y
 
 .. _amdgpu_synid_bank_mask:
 
@@ -943,18 +960,19 @@ bank_mask
 
 Controls which banks are enabled for data sharing. By default, all banks are enabled.
 
-Note. The lanes of a wavefront are organized in four *rows* and four *banks*.
+Note: the lanes of a wavefront are organized in four *rows* and four *banks*.
 (There are only two rows in *wave32* mode.)
 
-    ======================================== =======================================================
-    Syntax                                   Description
-    ======================================== =======================================================
-    bank_mask:{0..15}                        Specifies a *bank mask* as a positive
-                                             :ref:`integer number <amdgpu_synid_integer_number>`.
+    ================== ====================================================================
+    Syntax             Description
+    ================== ====================================================================
+    bank_mask:{0..15}  Specifies a *bank mask* as a positive
+                       :ref:`integer number <amdgpu_synid_integer_number>`
+                       or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
 
-                                             Each of 4 bits in the mask controls one
-                                             bank (0 - disabled, 1 - enabled).
-    ======================================== =======================================================
+                       Each of 4 bits in the mask controls one bank
+                       (0 - disabled, 1 - enabled).
+    ================== ====================================================================
 
 Examples:
 
@@ -962,7 +980,7 @@ Examples:
 
   bank_mask:0x3
   bank_mask:0b0011
-  bank_mask:0b1111
+  bank_mask:x&y
 
 .. _amdgpu_synid_bound_ctrl:
 
@@ -988,7 +1006,7 @@ fi
 
 Controls interaction with *inactive* lanes for *dpp16* instructions. The default value is zero.
 
-Note. *Inactive* lanes are those whose :ref:`exec<amdgpu_synid_exec>` mask bit is zero.
+Note: *inactive* lanes are those whose :ref:`exec<amdgpu_synid_exec>` mask bit is zero.
 
 GFX10 only.
 
@@ -1001,6 +1019,9 @@ GFX10 only.
     fi:1                                     Fetch pre-exist values from inactive lanes.
     ======================================== ==================================================
 
+Note: numeric values may be specified as either :ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 SDWA Modifiers
 --------------
 
@@ -1037,7 +1058,6 @@ Selects which bits in the destination are affected. By default, all bits are aff
     dst_sel:WORD_1                           Use bits 31:16.
     ======================================== ================================================
 
-
 .. _amdgpu_synid_dst_unused:
 
 dst_unused
@@ -1151,7 +1171,7 @@ operands (both source and destination). First value controls src0, second value
 and so on, except that the last value controls destination.
 The value 0 selects the low bits, while 1 selects the high bits.
 
-Note. op_sel modifier affects 16-bit operands only. For 32-bit operands the value specified
+Note: op_sel modifier affects 16-bit operands only. For 32-bit operands the value specified
 by op_sel must be 0.
 
 GFX9 and GFX10 only.
@@ -1164,6 +1184,10 @@ GFX9 and GFX10 only.
     op_sel:[{0..1},{0..1},{0..1},{0..1}]     Select operand bits for instructions with 3 source operands.
     ======================================== ============================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1189,7 +1213,7 @@ Integer clamping is not supported by GFX7.
 For floating point operations, clamp modifier indicates that the result must be clamped
 to the range [0.0, 1.0]. By default, there is no clamping.
 
-Note. Clamp modifier is applied after :ref:`output modifiers<amdgpu_synid_omod>` (if any).
+Note: clamp modifier is applied after :ref:`output modifiers<amdgpu_synid_omod>` (if any).
 
     ======================================== ================================================
     Syntax                                   Description
@@ -1205,12 +1229,12 @@ omod
 Specifies if an output modifier must be applied to the result.
 By default, no output modifiers are applied.
 
-Note. Output modifiers are applied before :ref:`clamping<amdgpu_synid_clamp>` (if any).
+Note: output modifiers are applied before :ref:`clamping<amdgpu_synid_clamp>` (if any).
 
 Output modifiers are valid for f32 and f64 floating point results only.
 They must not be used with f16.
 
-Note. *v_cvt_f16_f32* is an exception. This instruction produces f16 result
+Note: *v_cvt_f16_f32* is an exception. This instruction produces f16 result
 but accepts output modifiers.
 
     ======================================== ================================================
@@ -1221,6 +1245,16 @@ but accepts output modifiers.
     div:2                                    Multiply the result by 0.5.
     ======================================== ================================================
 
+Note: numeric values may be specified as either :ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
+Examples:
+
+.. parsed-literal::
+
+  mul:2
+  mul:x      // x must be equal to 2 or 4
+
 .. _amdgpu_synid_vop3_operand_modifiers:
 
 VOP3 Operand Modifiers
@@ -1233,15 +1267,19 @@ Operand modifiers are not used separately. They are applied to source operands.
 abs
 ~~~
 
-Computes absolute value of its operand. Applied before :ref:`neg<amdgpu_synid_neg>` (if any).
-Valid for floating point operands only.
+Computes the absolute value of its operand. Must be applied before :ref:`neg<amdgpu_synid_neg>`
+(if any). Valid for floating point operands only.
 
-    ======================================== ================================================
+    ======================================== ====================================================
     Syntax                                   Description
-    ======================================== ================================================
-    abs(<operand>)                           Get absolute value of operand.
-    \|<operand>|                             The same as above.
-    ======================================== ================================================
+    ======================================== ====================================================
+    abs(<operand>)                           Get the absolute value of a floating-point operand.
+    \|<operand>|                             The same as above (an SP3 syntax).
+    ======================================== ====================================================
+
+Note: avoid using SP3 syntax with operands specified as expressions because the trailing '|'
+may be misinterpreted. Such operands should be enclosed into additional parentheses as shown
+in examples below.
 
 Examples:
 
@@ -1249,28 +1287,50 @@ Examples:
 
   abs(v36)
   \|v36|
+  abs(x|y)     // ok
+  \|(x|y)|      // additional parentheses are required
 
 .. _amdgpu_synid_neg:
 
 neg
 ~~~
 
-Computes negative value of its operand. Applied after :ref:`abs<amdgpu_synid_abs>` (if any).
-Valid for floating point operands only.
+Computes the negative value of its operand. Must be applied after :ref:`abs<amdgpu_synid_abs>`
+(if any). Valid for floating point operands only.
 
-    ======================================== ================================================
-    Syntax                                   Description
-    ======================================== ================================================
-    neg(<operand>)                           Get negative value of operand.
-    -<operand>                               The same as above.
-    ======================================== ================================================
+    ================== ====================================================
+    Syntax             Description
+    ================== ====================================================
+    neg(<operand>)     Get the negative value of a floating-point operand.
+                       The operand may include an optional
+                       :ref:`abs<amdgpu_synid_abs>` modifier.
+    -<operand>         The same as above (an SP3 syntax).
+    ================== ====================================================
+
+Note: SP3 syntax is supported with limitations because of a potential ambiguity.
+Currently it is allowed in the following cases:
+
+* Before a register.
+* Before an :ref:`abs<amdgpu_synid_abs>` modifier.
+* Before an SP3 :ref:`abs<amdgpu_synid_abs>` modifier.
+
+In all other cases "-" is handled as a part of an expression that follows the sign.
 
 Examples:
 
 .. parsed-literal::
 
+  // Operands with negate modifiers
   neg(v[0])
-  -v4
+  neg(1.0)
+  neg(abs(v0))
+  -v5
+  -abs(v5)
+  -\|v5|
+
+  // Operands without negate modifiers
+  -1
+  -x+y
 
 VOP3P Modifiers
 ---------------
@@ -1304,6 +1364,10 @@ The value 0 selects the low bits, while 1 selects the high bits.
     op_sel:[{0..1},{0..1},{0..1}]     Select operand bits for instructions with 3 source operands.
     ================================= =============================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1333,6 +1397,10 @@ The value 0 selects the low bits, while 1 selects the high bits.
     op_sel_hi:[{0..1},{0..1},{0..1}]    Select operand bits for instructions with 3 source operands.
     =================================== =============================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1367,6 +1435,10 @@ This modifier is valid for floating point operands only.
     neg_lo:[{0..1},{0..1},{0..1}]    Select affected operands for instructions with 3 source operands.
     ================================ ==================================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1401,6 +1473,10 @@ This modifier is valid for floating point operands only.
     neg_hi:[{0..1},{0..1},{0..1}]   Select affected operands for instructions with 3 source operands.
     =============================== ==================================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1419,7 +1495,7 @@ VOP3P V_MAD_MIX Modifiers
 -------------------------
 
 *v_mad_mix_f32*, *v_mad_mixhi_f16* and *v_mad_mixlo_f16* instructions
-use *op_sel* and *op_sel_hi* modifiers 
+use *op_sel* and *op_sel_hi* modifiers
 in a manner different from *regular* VOP3P instructions.
 
 See a description below.
@@ -1449,6 +1525,10 @@ By default, low bits are used for all operands.
     op_sel:[{0..1},{0..1},{0..1}]   Select location of each 16-bit source operand.
     =============================== ================================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
@@ -1477,6 +1557,10 @@ The location of 16 bits in the operand may be specified by
     op_sel_hi:[{0..1},{0..1},{0..1}]         Select size of each source operand.
     ======================================== ====================================
 
+Note: numeric values may be specified as either
+:ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
 Examples:
 
 .. parsed-literal::
diff --git a/llvm/docs/AMDGPUOperandSyntax.rst b/llvm/docs/AMDGPUOperandSyntax.rst
index 523c5ac7179cc..c20da0047296b 100644
--- a/llvm/docs/AMDGPUOperandSyntax.rst
+++ b/llvm/docs/AMDGPUOperandSyntax.rst
@@ -38,7 +38,8 @@ Assembler currently supports sequences of 1, 2, 3, 4, 8 and 16 *vector* register
     =================================================== ====================================================================
     **v**\<N>                                           A single 32-bit *vector* register.
 
-                                                        *N* must be a decimal integer number.
+                                                        *N* must be a decimal
+                                                        :ref:`integer number<amdgpu_synid_integer_number>`.
     **v[**\ <N>\ **]**                                  A single 32-bit *vector* register.
 
                                                         *N* may be specified as an
@@ -51,10 +52,11 @@ Assembler currently supports sequences of 1, 2, 3, 4, 8 and 16 *vector* register
                                                         or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
     **[v**\ <N>, \ **v**\ <N+1>, ... **v**\ <K>\ **]**  A sequence of (\ *K-N+1*\ ) *vector* registers.
 
-                                                        Register indices must be specified as decimal integer numbers.
+                                                        Register indices must be specified as decimal
+                                                        :ref:`integer numbers<amdgpu_synid_integer_number>`.
     =================================================== ====================================================================
 
-Note. *N* and *K* must satisfy the following conditions:
+Note: *N* and *K* must satisfy the following conditions:
 
 * *N* <= *K*.
 * 0 <= *N* <= 255.
@@ -77,26 +79,27 @@ Examples:
 
 .. _amdgpu_synid_nsa:
 
-*Image* instructions may use special *NSA* (Non-Sequential Address) syntax for *image addresses*:
+GFX10 *Image* instructions may use special *NSA* (Non-Sequential Address) syntax for *image addresses*:
 
-    =================================================== ====================================================================
-    Syntax                                              Description
-    =================================================== ====================================================================
-    **[v**\ <A>, \ **v**\ <B>, ... **v**\ <X>\ **]**    A sequence of *vector* registers. At least one register
-                                                        must be specified.
+    ===================================== =================================================
+    Syntax                                Description
+    ===================================== =================================================
+    **[Vm**, \ **Vn**, ... **Vk**\ **]**  A sequence of 32-bit *vector* registers.
+                                          Each register may be specified using a syntax
+                                          defined :ref:`above<amdgpu_synid_v>`.
 
-                                                        In contrast with standard syntax described above, registers in
-                                                        this sequence are not required to have consecutive indices.
-                                                        Moreover, the same register may appear in the list more than once.
-    =================================================== ====================================================================
-
-Note. Reqister indices must be in the range 0..255. They must be specified as decimal integer numbers.
+                                          In contrast with standard syntax, registers
+                                          in *NSA* sequence are not required to have
+                                          consecutive indices. Moreover, the same register
+                                          may appear in the list more than once.
+    ===================================== =================================================
 
 Examples:
 
 .. parsed-literal::
 
-  [v32,v1,v2]
+  [v32,v1,v[2]]
+  [v[32],v[1:1],[v2]]
   [v4,v4,v4,v4]
 
 .. _amdgpu_synid_s:
@@ -126,7 +129,9 @@ Sequences of 4 and more *scalar* registers must be quad-aligned.
     ======================================================== ====================================================================
     **s**\ <N>                                               A single 32-bit *scalar* register.
 
-                                                             *N* must be a decimal integer number.
+                                                             *N* must be a decimal
+                                                             :ref:`integer number<amdgpu_synid_integer_number>`.
+
     **s[**\ <N>\ **]**                                       A single 32-bit *scalar* register.
 
                                                              *N* may be specified as an
@@ -137,12 +142,14 @@ Sequences of 4 and more *scalar* registers must be quad-aligned.
                                                              *N* and *K* may be specified as
                                                              :ref:`integer numbers<amdgpu_synid_integer_number>`
                                                              or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
+
     **[s**\ <N>, \ **s**\ <N+1>, ... **s**\ <K>\ **]**       A sequence of (\ *K-N+1*\ ) *scalar* registers.
 
-                                                             Register indices must be specified as decimal integer numbers.
+                                                             Register indices must be specified as decimal
+                                                             :ref:`integer numbers<amdgpu_synid_integer_number>`.
     ======================================================== ====================================================================
 
-Note. *N* and *K* must satisfy the following conditions:
+Note: *N* and *K* must satisfy the following conditions:
 
 * *N* must be properly aligned based on sequence size.
 * *N* <= *K*.
@@ -210,7 +217,8 @@ Sequences of 4 and more *ttmp* registers must be quad-aligned.
     ============================================================= ====================================================================
     **ttmp**\ <N>                                                 A single 32-bit *ttmp* register.
 
-                                                                  *N* must be a decimal integer number.
+                                                                  *N* must be a decimal
+                                                                  :ref:`integer number<amdgpu_synid_integer_number>`.
     **ttmp[**\ <N>\ **]**                                         A single 32-bit *ttmp* register.
 
                                                                   *N* may be specified as an
@@ -223,10 +231,11 @@ Sequences of 4 and more *ttmp* registers must be quad-aligned.
                                                                   or :ref:`absolute expressions<amdgpu_synid_absolute_expression>`.
     **[ttmp**\ <N>, \ **ttmp**\ <N+1>, ... **ttmp**\ <K>\ **]**   A sequence of (\ *K-N+1*\ ) *ttmp* registers.
 
-                                                                  Register indices must be specified as decimal integer numbers.
+                                                                  Register indices must be specified as decimal
+                                                                  :ref:`integer numbers<amdgpu_synid_integer_number>`.
     ============================================================= ====================================================================
 
-Note. *N* and *K* must satisfy the following conditions:
+Note: *N* and *K* must satisfy the following conditions:
 
 * *N* must be properly aligned based on sequence size.
 * *N* <= *K*.
@@ -266,8 +275,8 @@ Trap base address, 64-bits wide. Holds the pointer to the current trap handler p
     Syntax             Description                                                             Availability
     ================== ======================================================================= =============
     tba                64-bit *trap base address* register.                                    GFX7, GFX8
-    [tba]              64-bit *trap base address* register (an alternative syntax).            GFX7, GFX8
-    [tba_lo,tba_hi]    64-bit *trap base address* register (an alternative syntax).            GFX7, GFX8
+    [tba]              64-bit *trap base address* register (an SP3 syntax).                    GFX7, GFX8
+    [tba_lo,tba_hi]    64-bit *trap base address* register (an SP3 syntax).                    GFX7, GFX8
     ================== ======================================================================= =============
 
 High and low 32 bits of *trap base address* may be accessed as separate registers:
@@ -277,8 +286,8 @@ High and low 32 bits of *trap base address* may be accessed as separate register
     ================== ======================================================================= =============
     tba_lo             Low 32 bits of *trap base address* register.                            GFX7, GFX8
     tba_hi             High 32 bits of *trap base address* register.                           GFX7, GFX8
-    [tba_lo]           Low 32 bits of *trap base address* register (an alternative syntax).    GFX7, GFX8
-    [tba_hi]           High 32 bits of *trap base address* register (an alternative syntax).   GFX7, GFX8
+    [tba_lo]           Low 32 bits of *trap base address* register (an SP3 syntax).            GFX7, GFX8
+    [tba_hi]           High 32 bits of *trap base address* register (an SP3 syntax).           GFX7, GFX8
     ================== ======================================================================= =============
 
 Note that *tba*, *tba_lo* and *tba_hi* are not accessible as assembler registers in GFX9 and GFX10,
@@ -295,8 +304,8 @@ Trap memory address, 64-bits wide.
     Syntax            Description                                                             Availability
     ================= ======================================================================= ==================
     tma               64-bit *trap memory address* register.                                  GFX7, GFX8
-    [tma]             64-bit *trap memory address* register (an alternative syntax).          GFX7, GFX8
-    [tma_lo,tma_hi]   64-bit *trap memory address* register (an alternative syntax).          GFX7, GFX8
+    [tma]             64-bit *trap memory address* register (an SP3 syntax).                  GFX7, GFX8
+    [tma_lo,tma_hi]   64-bit *trap memory address* register (an SP3 syntax).                  GFX7, GFX8
     ================= ======================================================================= ==================
 
 High and low 32 bits of *trap memory address* may be accessed as separate registers:
@@ -306,8 +315,8 @@ High and low 32 bits of *trap memory address* may be accessed as separate regist
     ================= ======================================================================= ==================
     tma_lo            Low 32 bits of *trap memory address* register.                          GFX7, GFX8
     tma_hi            High 32 bits of *trap memory address* register.                         GFX7, GFX8
-    [tma_lo]          Low 32 bits of *trap memory address* register (an alternative syntax).  GFX7, GFX8
-    [tma_hi]          High 32 bits of *trap memory address* register (an alternative syntax). GFX7, GFX8
+    [tma_lo]          Low 32 bits of *trap memory address* register (an SP3 syntax).          GFX7, GFX8
+    [tma_hi]          High 32 bits of *trap memory address* register (an SP3 syntax).         GFX7, GFX8
     ================= ======================================================================= ==================
 
 Note that *tma*, *tma_lo* and *tma_hi* are not accessible as assembler registers in GFX9 and GFX10,
@@ -324,8 +333,8 @@ Flat scratch address, 64-bits wide. Holds the base address of scratch memory.
     Syntax                             Description
     ================================== ================================================================
     flat_scratch                       64-bit *flat scratch* address register.
-    [flat_scratch]                     64-bit *flat scratch* address register (an alternative syntax).
-    [flat_scratch_lo,flat_scratch_hi]  64-bit *flat scratch* address register (an alternative syntax).
+    [flat_scratch]                     64-bit *flat scratch* address register (an SP3 syntax).
+    [flat_scratch_lo,flat_scratch_hi]  64-bit *flat scratch* address register (an SP3 syntax).
     ================================== ================================================================
 
 High and low 32 bits of *flat scratch* address may be accessed as separate registers:
@@ -335,8 +344,8 @@ High and low 32 bits of *flat scratch* address may be accessed as separate regis
     ========================= =========================================================================
     flat_scratch_lo           Low 32 bits of *flat scratch* address register.
     flat_scratch_hi           High 32 bits of *flat scratch* address register.
-    [flat_scratch_lo]         Low 32 bits of *flat scratch* address register (an alternative syntax).
-    [flat_scratch_hi]         High 32 bits of *flat scratch* address register (an alternative syntax).
+    [flat_scratch_lo]         Low 32 bits of *flat scratch* address register (an SP3 syntax).
+    [flat_scratch_hi]         High 32 bits of *flat scratch* address register (an SP3 syntax).
     ========================= =========================================================================
 
 .. _amdgpu_synid_xnack:
@@ -355,8 +364,8 @@ received an *XNACK* due to a vector memory operation.
     Syntax                         Description
     ============================== =====================================================
     xnack_mask                     64-bit *xnack mask* register.
-    [xnack_mask]                   64-bit *xnack mask* register (an alternative syntax).
-    [xnack_mask_lo,xnack_mask_hi]  64-bit *xnack mask* register (an alternative syntax).
+    [xnack_mask]                   64-bit *xnack mask* register (an SP3 syntax).
+    [xnack_mask_lo,xnack_mask_hi]  64-bit *xnack mask* register (an SP3 syntax).
     ============================== =====================================================
 
 High and low 32 bits of *xnack mask* may be accessed as separate registers:
@@ -366,8 +375,8 @@ High and low 32 bits of *xnack mask* may be accessed as separate registers:
     ===================== ==============================================================
     xnack_mask_lo         Low 32 bits of *xnack mask* register.
     xnack_mask_hi         High 32 bits of *xnack mask* register.
-    [xnack_mask_lo]       Low 32 bits of *xnack mask* register (an alternative syntax).
-    [xnack_mask_hi]       High 32 bits of *xnack mask* register (an alternative syntax).
+    [xnack_mask_lo]       Low 32 bits of *xnack mask* register (an SP3 syntax).
+    [xnack_mask_hi]       High 32 bits of *xnack mask* register (an SP3 syntax).
     ===================== ==============================================================
 
 .. _amdgpu_synid_vcc:
@@ -385,8 +394,8 @@ Note that GFX10 H/W does not use high 32 bits of *vcc* in *wave32* mode.
     Syntax           Description
     ================ =========================================================================
     vcc              64-bit *vector condition code* register.
-    [vcc]            64-bit *vector condition code* register (an alternative syntax).
-    [vcc_lo,vcc_hi]  64-bit *vector condition code* register (an alternative syntax).
+    [vcc]            64-bit *vector condition code* register (an SP3 syntax).
+    [vcc_lo,vcc_hi]  64-bit *vector condition code* register (an SP3 syntax).
     ================ =========================================================================
 
 High and low 32 bits of *vector condition code* may be accessed as separate registers:
@@ -396,8 +405,8 @@ High and low 32 bits of *vector condition code* may be accessed as separate regi
     ================ =========================================================================
     vcc_lo           Low 32 bits of *vector condition code* register.
     vcc_hi           High 32 bits of *vector condition code* register.
-    [vcc_lo]         Low 32 bits of *vector condition code* register (an alternative syntax).
-    [vcc_hi]         High 32 bits of *vector condition code* register (an alternative syntax).
+    [vcc_lo]         Low 32 bits of *vector condition code* register (an SP3 syntax).
+    [vcc_hi]         High 32 bits of *vector condition code* register (an SP3 syntax).
     ================ =========================================================================
 
 .. _amdgpu_synid_m0:
@@ -412,7 +421,7 @@ including register indexing and bounds checking.
     Syntax      Description
     =========== ===================================================
     m0          A 32-bit *memory* register.
-    [m0]        A 32-bit *memory* register (an alternative syntax).
+    [m0]        A 32-bit *memory* register (an SP3 syntax).
     =========== ===================================================
 
 .. _amdgpu_synid_exec:
@@ -430,8 +439,8 @@ Note that GFX10 H/W does not use high 32 bits of *exec* in *wave32* mode.
     Syntax                Description
     ===================== =================================================================
     exec                  64-bit *execute mask* register.
-    [exec]                64-bit *execute mask* register (an alternative syntax).
-    [exec_lo,exec_hi]     64-bit *execute mask* register (an alternative syntax).
+    [exec]                64-bit *execute mask* register (an SP3 syntax).
+    [exec_lo,exec_hi]     64-bit *execute mask* register (an SP3 syntax).
     ===================== =================================================================
 
 High and low 32 bits of *execute mask* may be accessed as separate registers:
@@ -441,8 +450,8 @@ High and low 32 bits of *execute mask* may be accessed as separate registers:
     ===================== =================================================================
     exec_lo               Low 32 bits of *execute mask* register.
     exec_hi               High 32 bits of *execute mask* register.
-    [exec_lo]             Low 32 bits of *execute mask* register (an alternative syntax).
-    [exec_hi]             High 32 bits of *execute mask* register (an alternative syntax).
+    [exec_lo]             Low 32 bits of *execute mask* register (an SP3 syntax).
+    [exec_hi]             High 32 bits of *execute mask* register (an SP3 syntax).
     ===================== =================================================================
 
 .. _amdgpu_synid_vccz:
@@ -452,7 +461,7 @@ vccz
 
 A single bit flag indicating that the :ref:`vcc<amdgpu_synid_vcc>` is all zeros.
 
-Note. When GFX10 operates in *wave32* mode, this register reflects state of :ref:`vcc_lo<amdgpu_synid_vcc_lo>`.
+Note: when GFX10 operates in *wave32* mode, this register reflects state of :ref:`vcc_lo<amdgpu_synid_vcc_lo>`.
 
 .. _amdgpu_synid_execz:
 
@@ -461,7 +470,7 @@ execz
 
 A single bit flag indicating that the :ref:`exec<amdgpu_synid_exec>` is all zeros.
 
-Note. When GFX10 operates in *wave32* mode, this register reflects state of :ref:`exec_lo<amdgpu_synid_exec>`.
+Note: when GFX10 operates in *wave32* mode, this register reflects state of :ref:`exec_lo<amdgpu_synid_exec>`.
 
 .. _amdgpu_synid_scc:
 
@@ -495,19 +504,20 @@ GFX10 only.
 
 .. _amdgpu_synid_constant:
 
-constant
---------
+inline constant
+---------------
+
+An *inline constant* is an integer or a floating-point value encoded as a part of an instruction.
+Compare *inline constants* with :ref:`literals<amdgpu_synid_literal>`.
 
-A set of integer and floating-point *inline* constants and values:
+Inline constants include:
 
 * :ref:`iconst<amdgpu_synid_iconst>`
 * :ref:`fconst<amdgpu_synid_fconst>`
 * :ref:`ival<amdgpu_synid_ival>`
 
-In contrast with :ref:`literals<amdgpu_synid_literal>`, these operands are encoded as a part of instruction.
-
 If a number may be encoded as either
-a :ref:`literal<amdgpu_synid_literal>` or 
+a :ref:`literal<amdgpu_synid_literal>` or
 a :ref:`constant<amdgpu_synid_constant>`,
 assembler selects the latter encoding as more efficient.
 
@@ -516,17 +526,14 @@ assembler selects the latter encoding as more efficient.
 iconst
 ~~~~~~
 
-An :ref:`integer number<amdgpu_synid_integer_number>`
+An :ref:`integer number<amdgpu_synid_integer_number>` or
+an :ref:`absolute expression<amdgpu_synid_absolute_expression>`
 encoded as an *inline constant*.
 
 Only a small fraction of integer numbers may be encoded as *inline constants*.
 They are enumerated in the table below.
 Other integer numbers have to be encoded as :ref:`literals<amdgpu_synid_literal>`.
 
-Integer *inline constants* are converted to
-:ref:`expected operand type<amdgpu_syn_instruction_type>`
-as described :ref:`here<amdgpu_synid_int_const_conv>`.
-
     ================================== ====================================
     Value                              Note
     ================================== ====================================
@@ -548,10 +555,6 @@ Only a small fraction of floating-point numbers may be encoded as *inline consta
 They are enumerated in the table below.
 Other floating-point numbers have to be encoded as :ref:`literals<amdgpu_synid_literal>`.
 
-Floating-point *inline constants* are converted to
-:ref:`expected operand type<amdgpu_syn_instruction_type>`
-as described :ref:`here<amdgpu_synid_fp_const_conv>`.
-
     ===================== ===================================================== ==================
     Value                 Note                                                  Availability
     ===================== ===================================================== ==================
@@ -594,21 +597,18 @@ These operands provide read-only access to H/W registers.
 literal
 -------
 
-A literal is a 64-bit value which is encoded as a separate 32-bit dword in the instruction stream.
+A *literal* is a 64-bit value encoded as a separate 32-bit dword in the instruction stream.
+Compare *literals* with :ref:`inline constants<amdgpu_synid_constant>`.
 
 If a number may be encoded as either
-a :ref:`literal<amdgpu_synid_literal>` or 
+a :ref:`literal<amdgpu_synid_literal>` or
 an :ref:`inline constant<amdgpu_synid_constant>`,
 assembler selects the latter encoding as more efficient.
 
 Literals may be specified as :ref:`integer numbers<amdgpu_synid_integer_number>`,
-:ref:`floating-point numbers<amdgpu_synid_floating-point_number>` or
-:ref:`expressions<amdgpu_synid_expression>`
-(expressions are currently supported for 32-bit operands only).
-
-A 64-bit literal value is converted by assembler
-to an :ref:`expected operand type<amdgpu_syn_instruction_type>`
-as described :ref:`here<amdgpu_synid_lit_conv>`.
+:ref:`floating-point numbers<amdgpu_synid_floating-point_number>`,
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>` or
+:ref:`relocatable expressions<amdgpu_synid_relocatable_expression>`.
 
 An instruction may use only one literal but several operands may refer the same literal.
 
@@ -617,30 +617,38 @@ An instruction may use only one literal but several operands may refer the same
 uimm8
 -----
 
-A 8-bit positive :ref:`integer number<amdgpu_synid_integer_number>`.
-The value is encoded as part of the opcode so it is free to use.
+A 8-bit :ref:`integer number<amdgpu_synid_integer_number>`
+or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+The value must be in the range 0..0xFF.
 
 .. _amdgpu_synid_uimm32:
 
 uimm32
 ------
 
-A 32-bit positive :ref:`integer number<amdgpu_synid_integer_number>`.
-The value is stored as a separate 32-bit dword in the instruction stream.
+A 32-bit :ref:`integer number<amdgpu_synid_integer_number>`
+or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+The value must be in the range 0..0xFFFFFFFF.
 
 .. _amdgpu_synid_uimm20:
 
 uimm20
 ------
 
-A 20-bit positive :ref:`integer number<amdgpu_synid_integer_number>`.
+A 20-bit :ref:`integer number<amdgpu_synid_integer_number>`
+or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+
+The value must be in the range 0..0xFFFFF.
 
 .. _amdgpu_synid_uimm21:
 
 uimm21
 ------
 
-A 21-bit positive :ref:`integer number<amdgpu_synid_integer_number>`.
+A 21-bit :ref:`integer number<amdgpu_synid_integer_number>`
+or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+
+The value must be in the range 0..0x1FFFFF.
 
 .. WARNING:: Assembler currently supports 20-bit offsets only. Use :ref:`uimm20<amdgpu_synid_uimm20>` as a replacement.
 
@@ -649,7 +657,10 @@ A 21-bit positive :ref:`integer number<amdgpu_synid_integer_number>`.
 simm21
 ------
 
-A 21-bit :ref:`integer number<amdgpu_synid_integer_number>`.
+A 21-bit :ref:`integer number<amdgpu_synid_integer_number>`
+or an :ref:`absolute expression<amdgpu_synid_absolute_expression>`.
+
+The value must be in the range -0x100000..0x0FFFFF.
 
 .. WARNING:: Assembler currently supports 20-bit unsigned offsets only. Use :ref:`uimm20<amdgpu_synid_uimm20>` as a replacement.
 
@@ -678,27 +689,20 @@ Integer Numbers
 ---------------
 
 Integer numbers are 64 bits wide.
-They may be specified in binary, octal, hexadecimal and decimal formats:
-
-    ============== ====================================
-    Format         Syntax
-    ============== ====================================
-    Decimal        [-]?[1-9][0-9]*
-    Binary         [-]?0b[01]+
-    Octal          [-]?0[0-7]+
-    Hexadecimal    [-]?0x[0-9a-fA-F]+
-    \              [-]?[0x]?[0-9][0-9a-fA-F]*[hH]
-    ============== ====================================
+They are converted to :ref:`expected operand type<amdgpu_syn_instruction_type>`
+as described :ref:`here<amdgpu_synid_int_conv>`.
 
-Examples:
+Integer numbers may be specified in binary, octal, hexadecimal and decimal formats:
 
-.. parsed-literal::
-
-  -1234
-  0b1010
-  010
-  0xff
-  0ffh
+    ============ =============================== ========
+    Format       Syntax                          Example
+    ============ =============================== ========
+    Decimal      [-]?[1-9][0-9]*                 -1234
+    Binary       [-]?0b[01]+                     0b1010
+    Octal        [-]?0[0-7]+                     010
+    Hexadecimal  [-]?0x[0-9a-fA-F]+              0xff
+    \            [-]?[0x]?[0-9][0-9a-fA-F]*[hH]  0ffh
+    ============ =============================== ========
 
 .. _amdgpu_synid_floating-point_number:
 
@@ -706,31 +710,29 @@ Floating-Point Numbers
 ----------------------
 
 All floating-point numbers are handled as double (64 bits wide).
+They are converted to
+:ref:`expected operand type<amdgpu_syn_instruction_type>`
+as described :ref:`here<amdgpu_synid_fp_conv>`.
 
 Floating-point numbers may be specified in hexadecimal and decimal formats:
 
-    ============== ======================================================== ========================================================
-    Format         Syntax                                                   Note
-    ============== ======================================================== ========================================================
-    Decimal        [-]?[0-9]*[.][0-9]*([eE][+-]?[0-9]*)?                    Must include either a decimal separator or an exponent.
-    Hexadecimal    [-]0x[0-9a-fA-F]*(.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+
-    ============== ======================================================== ========================================================
-
-Examples:
-
-.. parsed-literal::
-
- -1.234
- 234e2
- -0x1afp-10
- 0x.1afp10
+    ============ ======================================================== ====================== ====================
+    Format       Syntax                                                   Examples               Note
+    ============ ======================================================== ====================== ====================
+    Decimal      [-]?[0-9]*[.][0-9]*([eE][+-]?[0-9]*)?                    -1.234, 234e2          Must include either
+                                                                                                 a decimal separator
+                                                                                                 or an exponent.
+    Hexadecimal  [-]0x[0-9a-fA-F]*(.[0-9a-fA-F]*)?[pP][+-]?[0-9a-fA-F]+   -0x1afp-10, 0x.1afp10
+    ============ ======================================================== ====================== ====================
 
 .. _amdgpu_synid_expression:
 
 Expressions
 ===========
 
-An expression specifies an address or a numeric value.
+An expression is evaluated to a 64-bit integer.
+Note that floating-point expressions are not supported.
+
 There are two kinds of expressions:
 
 * :ref:`Absolute<amdgpu_synid_absolute_expression>`.
@@ -741,10 +743,14 @@ There are two kinds of expressions:
 Absolute Expressions
 --------------------
 
-The value of an absolute expression remains the same after program relocation.
+The value of an absolute expression does not change after program relocation.
 Absolute expressions must not include unassigned and relocatable values
 such as labels.
 
+Absolute expressions are evaluated to 64-bit integer values and converted to
+:ref:`expected operand type<amdgpu_syn_instruction_type>`
+as described :ref:`here<amdgpu_synid_int_conv>`.
+
 Examples:
 
 .. parsed-literal::
@@ -760,45 +766,38 @@ Relocatable Expressions
 The value of a relocatable expression depends on program relocation.
 
 Note that use of relocatable expressions is limited with branch targets
-and 32-bit :ref:`literals<amdgpu_synid_literal>`.
+and 32-bit integer operands.
 
-Addition information about relocation may be found :ref:`here<amdgpu-relocation-records>`.
-
-Examples:
+A relocatable expression is evaluated to a 64-bit integer value
+which depends on operand kind and :ref:`relocation type<amdgpu-relocation-records>`
+of symbol(s) used in the expression. For example, if an instruction refers a label,
+this reference is evaluated to an offset from the address after the instruction
+to the label address:
 
 .. parsed-literal::
 
-    y = x + 10 // x is not yet defined. Undefined symbols are assumed to be PC-relative.
-    z = .
-
-Expression Data Type
---------------------
-
-Expressions and operands of expressions are interpreted as 64-bit integers.
+    label:
+    v_add_co_u32_e32 v0, vcc, label, v1  // 'label' operand is evaluated to -4
 
-Expressions may include 64-bit :ref:`floating-point numbers<amdgpu_synid_floating-point_number>` (double).
-However these operands are also handled as 64-bit integers
-using binary representation of specified floating-point numbers.
-No conversion from floating-point to integer is performed.
-
-Examples:
+Note that values of relocatable expressions are usually unknown at assembly time;
+they are resolved later by a linker and converted to
+:ref:`expected operand type<amdgpu_syn_instruction_type>`
+as described :ref:`here<amdgpu_synid_rl_conv>`.
 
-.. parsed-literal::
+Operands and Operations
+-----------------------
 
-    x = 0.1    // x is assigned an integer 4591870180066957722 which is a binary representation of 0.1.
-    y = x + x  // y is a sum of two integer values; it is not equal to 0.2!
+Expressions are composed of 64-bit integer operands and operations.
+Operands include :ref:`integer numbers<amdgpu_synid_integer_number>`
+and :ref:`symbols<amdgpu_synid_symbol>`.
 
-Syntax
-------
+Expressions may also use "." which is a reference to the current PC (program counter).
 
-Expressions are composed of
-:ref:`symbols<amdgpu_synid_symbol>`,
-:ref:`integer numbers<amdgpu_synid_integer_number>`,
-:ref:`floating-point numbers<amdgpu_synid_floating-point_number>`,
-:ref:`binary operators<amdgpu_synid_expression_bin_op>`,
-:ref:`unary operators<amdgpu_synid_expression_un_op>` and subexpressions.
+:ref:`Unary<amdgpu_synid_expression_un_op>` and :ref:`binary<amdgpu_synid_expression_bin_op>`
+operations produce 64-bit integer results.
 
-Expressions may also use "." which is a reference to the current PC (program counter).
+Syntax of Expressions
+---------------------
 
 The syntax of expressions is shown below::
 
@@ -887,7 +886,7 @@ They operate on and produce 64-bit integers.
 Symbols
 -------
 
-A symbol is a named 64-bit value, representing a relocatable
+A symbol is a named 64-bit integer value, representing a relocatable
 address or an absolute (non-relocatable) number.
 
 Symbol names have the following syntax:
@@ -907,128 +906,78 @@ The table below provides several examples of syntax used for symbol definition.
 A symbol may be used before it is declared or assigned;
 unassigned symbols are assumed to be PC-relative.
 
-Addition information about symbols may be found :ref:`here<amdgpu-symbols>`.
+Additional information about symbols may be found :ref:`here<amdgpu-symbols>`.
 
 .. _amdgpu_synid_conv:
 
-Conversions
-===========
+Type and Size Conversion
+========================
 
 This section describes what happens when a 64-bit
 :ref:`integer number<amdgpu_synid_integer_number>`, a
-:ref:`floating-point numbers<amdgpu_synid_floating-point_number>` or a
-:ref:`symbol<amdgpu_synid_symbol>`
+:ref:`floating-point number<amdgpu_synid_floating-point_number>` or an
+:ref:`expression<amdgpu_synid_expression>`
 is used for an operand which has a different type or size.
 
-Depending on operand kind, this conversion is performed by either assembler or AMDGPU H/W:
-
-* Values encoded as :ref:`inline constants<amdgpu_synid_constant>` are handled by H/W.
-* Values encoded as :ref:`literals<amdgpu_synid_literal>` are converted by assembler.
-
-.. _amdgpu_synid_const_conv:
-
-Inline Constants
-----------------
-
-.. _amdgpu_synid_int_const_conv:
-
-Integer Inline Constants
-~~~~~~~~~~~~~~~~~~~~~~~~
-
-Integer :ref:`inline constants<amdgpu_synid_constant>`
-may be thought of as 64-bit
-:ref:`integer numbers<amdgpu_synid_integer_number>`;
-when used as operands they are truncated to the size of
-:ref:`expected operand type<amdgpu_syn_instruction_type>`.
-No data type conversions are performed.
-
-Examples:
-
-.. parsed-literal::
-
-    // GFX9
-
-    v_add_u16 v0, -1, 0    // v0 = 0xFFFF
-    v_add_f16 v0, -1, 0    // v0 = 0xFFFF (NaN)
-
-    v_add_u32 v0, -1, 0    // v0 = 0xFFFFFFFF
-    v_add_f32 v0, -1, 0    // v0 = 0xFFFFFFFF (NaN)
+.. _amdgpu_synid_int_conv:
 
-.. _amdgpu_synid_fp_const_conv:
+Conversion of Integer Values
+----------------------------
 
-Floating-Point Inline Constants
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Instruction operands may be specified as 64-bit :ref:`integer numbers<amdgpu_synid_integer_number>` or
+:ref:`absolute expressions<amdgpu_synid_absolute_expression>`. These values are converted to
+the :ref:`expected operand type<amdgpu_syn_instruction_type>` using the following steps:
 
-Floating-point :ref:`inline constants<amdgpu_synid_constant>`
-may be thought of as 64-bit
-:ref:`floating-point numbers<amdgpu_synid_floating-point_number>`;
-when used as operands they are converted to a floating-point number of
-:ref:`expected operand size<amdgpu_syn_instruction_type>`.
+1. *Validation*. Assembler checks if the input value may be truncated without loss to the required *truncation width*
+(see the table below). There are two cases when this operation is enabled:
 
-Examples:
-
-.. parsed-literal::
-
-    // GFX9
-
-    v_add_f16 v0, 1.0, 0    // v0 = 0x3C00 (1.0)
-    v_add_u16 v0, 1.0, 0    // v0 = 0x3C00
-
-    v_add_f32 v0, 1.0, 0    // v0 = 0x3F800000 (1.0)
-    v_add_u32 v0, 1.0, 0    // v0 = 0x3F800000
-
-
-.. _amdgpu_synid_lit_conv:
-
-Literals
---------
+    * The truncated bits are all 0.
+    * The truncated bits are all 1 and the value after truncation has its MSB bit set.
 
-.. _amdgpu_synid_int_lit_conv:
+In all other cases assembler triggers an error.
 
-Integer Literals
-~~~~~~~~~~~~~~~~
+2. *Conversion*. The input value is converted to the expected type as described in the table below.
+Depending on operand kind, this conversion is performed by either assembler or AMDGPU H/W (or both).
 
-Integer :ref:`literals<amdgpu_synid_literal>`
-are specified as 64-bit :ref:`integer numbers<amdgpu_synid_integer_number>`.
+    ============== ================= =============== ====================================================================
+    Expected type  Truncation Width  Conversion      Description
+    ============== ================= =============== ====================================================================
+    i16, u16, b16  16                num.u16         Truncate to 16 bits.
+    i32, u32, b32  32                num.u32         Truncate to 32 bits.
+    i64            32                {-1,num.i32}    Truncate to 32 bits and then sign-extend the result to 64 bits.
+    u64, b64       32                {0,num.u32}     Truncate to 32 bits and then zero-extend the result to 64 bits.
+    f16            16                num.u16         Use low 16 bits as an f16 value.
+    f32            32                num.u32         Use low 32 bits as an f32 value.
+    f64            32                {num.u32,0}     Use low 32 bits of the number as high 32 bits
+                                                     of the result; low 32 bits of the result are zeroed.
+    ============== ================= =============== ====================================================================
 
-When used as operands they are converted to
-:ref:`expected operand type<amdgpu_syn_instruction_type>` as described below.
-
-    ============== ============== =============== ====================================================================
-    Expected type  Condition      Result          Note
-    ============== ============== =============== ====================================================================
-    i16, u16, b16  cond(num,16)   num.u16         Truncate to 16 bits.
-    i32, u32, b32  cond(num,32)   num.u32         Truncate to 32 bits.
-    i64            cond(num,32)   {-1,num.i32}    Truncate to 32 bits and then sign-extend the result to 64 bits.
-    u64, b64       cond(num,32)   { 0,num.u32}    Truncate to 32 bits and then zero-extend the result to 64 bits.
-    f16            cond(num,16)   num.u16         Use low 16 bits as an f16 value.
-    f32            cond(num,32)   num.u32         Use low 32 bits as an f32 value.
-    f64            cond(num,32)   {num.u32,0}     Use low 32 bits of the number as high 32 bits
-                                                  of the result; low 32 bits of the result are zeroed.
-    ============== ============== =============== ====================================================================
-
-The condition *cond(X,S)* indicates if a 64-bit number *X*
-can be converted to a smaller size *S* by truncation of upper bits.
-There are two cases when the conversion is possible:
-
-* The truncated bits are all 0.
-* The truncated bits are all 1 and the value after truncation has its MSB bit set.
-
-Examples of valid literals:
+Examples of enabled conversions:
 
 .. parsed-literal::
 
     // GFX9
-                                             // Literal value after conversion:
-    v_add_u16 v0, 0xff00, v0                 //   0xff00
-    v_add_u16 v0, 0xffffffffffffff00, v0     //   0xff00
-    v_add_u16 v0, -256, v0                   //   0xff00
-                                             // Literal value after conversion:
-    s_bfe_i64 s[0:1], 0xffefffff, s3         //   0xffffffffffefffff
-    s_bfe_u64 s[0:1], 0xffefffff, s3         //   0x00000000ffefffff
-    v_ceil_f64_e32 v[0:1], 0xffefffff        //   0xffefffff00000000 (-1.7976922776554302e308)
 
-Examples of invalid literals:
+    v_add_u16 v0, -1, 0                   // src0 = 0xFFFF
+    v_add_f16 v0, -1, 0                   // src0 = 0xFFFF (NaN)
+                                          //
+    v_add_u32 v0, -1, 0                   // src0 = 0xFFFFFFFF
+    v_add_f32 v0, -1, 0                   // src0 = 0xFFFFFFFF (NaN)
+                                          //
+    v_add_u16 v0, 0xff00, v0              // src0 = 0xff00
+    v_add_u16 v0, 0xffffffffffffff00, v0  // src0 = 0xff00
+    v_add_u16 v0, -256, v0                // src0 = 0xff00
+                                          //
+    s_bfe_i64 s[0:1], 0xffefffff, s3      // src0 = 0xffffffffffefffff
+    s_bfe_u64 s[0:1], 0xffefffff, s3      // src0 = 0x00000000ffefffff
+    v_ceil_f64_e32 v[0:1], 0xffefffff     // src0 = 0xffefffff00000000 (-1.7976922776554302e308)
+                                          //
+    x = 0xffefffff                        //
+    s_bfe_i64 s[0:1], x, s3               // src0 = 0xffffffffffefffff
+    s_bfe_u64 s[0:1], x, s3               // src0 = 0x00000000ffefffff
+    v_ceil_f64_e32 v[0:1], x              // src0 = 0xffefffff00000000 (-1.7976922776554302e308)
+
+Examples of disabled conversions:
 
 .. parsed-literal::
 
@@ -1037,49 +986,57 @@ Examples of invalid literals:
     v_add_u16 v0, 0x1ff00, v0               // truncated bits are not all 0 or 1
     v_add_u16 v0, 0xffffffffffff00ff, v0    // truncated bits do not match MSB of the result
 
-.. _amdgpu_synid_fp_lit_conv:
+.. _amdgpu_synid_fp_conv:
 
-Floating-Point Literals
-~~~~~~~~~~~~~~~~~~~~~~~
+Conversion of Floating-Point Values
+-----------------------------------
 
-Floating-point :ref:`literals<amdgpu_synid_literal>` are specified as 64-bit
-:ref:`floating-point numbers<amdgpu_synid_floating-point_number>`.
+Instruction operands may be specified as 64-bit :ref:`floating-point numbers<amdgpu_synid_floating-point_number>`.
+These values are converted to the :ref:`expected operand type<amdgpu_syn_instruction_type>` using the following steps:
 
-When used as operands they are converted to
-:ref:`expected operand type<amdgpu_syn_instruction_type>` as described below.
+1. *Validation*. Assembler checks if the input f64 number can be converted
+to the *required floating-point type* (see the table below) without overflow or underflow.
+Precision lost is allowed. If this conversion is not possible, assembler triggers an error.
 
-    ============== ============== ================= =================================================================
-    Expected type  Condition      Result            Note
-    ============== ============== ================= =================================================================
-    i16, u16, b16  cond(num,16)   f16(num)          Convert to f16 and use bits of the result as an integer value.
-    i32, u32, b32  cond(num,32)   f32(num)          Convert to f32 and use bits of the result as an integer value.
-    i64, u64, b64  false          \-                Conversion disabled because of an unclear semantics.
-    f16            cond(num,16)   f16(num)          Convert to f16.
-    f32            cond(num,32)   f32(num)          Convert to f32.
-    f64            true           {num.u32.hi,0}    Use high 32 bits of the number as high 32 bits of the result;
-                                                    zero-fill low 32 bits of the result.
+2. *Conversion*. The input value is converted to the expected type as described in the table below.
+Depending on operand kind, this is performed by either assembler or AMDGPU H/W (or both).
 
-                                                    Note that the result may differ from the original number.
-    ============== ============== ================= =================================================================
+    ============== ================ ================= =================================================================
+    Expected type  Required FP Type Conversion        Description
+    ============== ================ ================= =================================================================
+    i16, u16, b16  f16              f16(num)          Convert to f16 and use bits of the result as an integer value.
+    i32, u32, b32  f32              f32(num)          Convert to f32 and use bits of the result as an integer value.
+    i64, u64, b64  \-               \-                Conversion disabled.
+    f16            f16              f16(num)          Convert to f16.
+    f32            f32              f32(num)          Convert to f32.
+    f64            f64              {num.u32.hi,0}    Use high 32 bits of the number as high 32 bits of the result;
+                                                      zero-fill low 32 bits of the result.
 
-The condition *cond(X,S)* indicates if an f64 number *X* can be converted
-to a smaller *S*-bit floating-point type without overflow or underflow.
-Precision lost is allowed.
+                                                      Note that the result may differ from the original number.
+    ============== ================ ================= =================================================================
 
-Examples of valid literals:
+Examples of enabled conversions:
 
 .. parsed-literal::
 
     // GFX9
 
-    v_add_f16 v1, 65500.0, v2
-    v_add_f32 v1, 65600.0, v2
+    v_add_f16 v0, 1.0, 0        // src0 = 0x3C00 (1.0)
+    v_add_u16 v0, 1.0, 0        // src0 = 0x3C00
+                                //
+    v_add_f32 v0, 1.0, 0        // src0 = 0x3F800000 (1.0)
+    v_add_u32 v0, 1.0, 0        // src0 = 0x3F800000
 
-    // Literal value before conversion: 1.7976931348623157e308 (0x7fefffffffffffff)
-    // Literal value after conversion:  1.7976922776554302e308 (0x7fefffff00000000)
+                                // src0 before conversion:
+                                //   1.7976931348623157e308 = 0x7fefffffffffffff
+                                // src0 after conversion:
+                                //   1.7976922776554302e308 = 0x7fefffff00000000
     v_ceil_f64 v[0:1], 1.7976931348623157e308
 
-Examples of invalid literals:
+    v_add_f16 v1, 65500.0, v2   // ok for f16.
+    v_add_f32 v1, 65600.0, v2   // ok for f32, but would result in overflow for f16.
+
+Examples of disabled conversions:
 
 .. parsed-literal::
 
@@ -1087,25 +1044,35 @@ Examples of invalid literals:
 
     v_add_f16 v1, 65600.0, v2    // overflow
 
-.. _amdgpu_synid_exp_conv:
+.. _amdgpu_synid_rl_conv:
 
-Expressions
-~~~~~~~~~~~
+Conversion of Relocatable Values
+--------------------------------
 
-Expressions operate with and result in 64-bit integers.
+:ref:`Relocatable expressions<amdgpu_synid_relocatable_expression>`
+may be used with 32-bit integer operands and jump targets.
 
-When used as operands they are truncated to
-:ref:`expected operand size<amdgpu_syn_instruction_type>`.
-No data type conversions are performed.
+When the value of a relocatable expression is resolved by a linker, it is
+converted as needed and truncated to the operand size. The conversion depends
+on :ref:`relocation type<amdgpu-relocation-records>` and operand kind.
 
-Examples:
+For example, when a 32-bit operand of an instruction refers a relocatable expression *expr*,
+this reference is evaluated to a 64-bit offset from the address after the
+instruction to the address being referenced, *counted in bytes*.
+Then the value is truncated to 32 bits and encoded as a literal:
 
 .. parsed-literal::
 
-    // GFX9
+    expr = .
+    v_add_co_u32_e32 v0, vcc, expr, v1  // 'expr' operand is evaluated to -4
+                                        // and then truncated to 0xFFFFFFFC
 
-    x = 0.1
-    v_sqrt_f32 v0, x           // v0 = [low 32 bits of 0.1 (double)]
-    v_sqrt_f32 v0, (0.1 + 0)   // the same as above
-    v_sqrt_f32 v0, 0.1         // v0 = [0.1 (double) converted to float]
+As another example, when a branch instruction refers a label,
+this reference is evaluated to an offset from the address after the
+instruction to the label address, *counted in dwords*.
+Then the value is truncated to 16 bits:
+
+.. parsed-literal::
 
+    label:
+    s_branch label  // 'label' operand is evaluated to -1 and truncated to 0xFFFF
diff --git a/llvm/docs/Atomics.rst b/llvm/docs/Atomics.rst
index 60d7090790f15..e6c000057b517 100644
--- a/llvm/docs/Atomics.rst
+++ b/llvm/docs/Atomics.rst
@@ -38,7 +38,7 @@ be moved across a volatile load freely, but not an Acquire load.
 
 This document is intended to provide a guide to anyone either writing a frontend
 for LLVM or working on optimization passes for LLVM with a guide for how to deal
-with instructions with special semantics in the presence of concurrency.  This
+with instructions with special semantics in the presence of concurrency. This
 is not intended to be a precise guide to the semantics; the details can get
 extremely complicated and unreadable, and are not usually necessary.
 
diff --git a/llvm/docs/Bugpoint.rst b/llvm/docs/Bugpoint.rst
index f3bb54cffb54e..19efaf2bdee7b 100644
--- a/llvm/docs/Bugpoint.rst
+++ b/llvm/docs/Bugpoint.rst
@@ -121,6 +121,12 @@ non-obvious ways.  Here are some hints and tips:
   miscompilation.  Programs should be temporarily modified to disable outputs
   that are likely to vary from run to run.
 
+* In the `crash debugger`_, ``bugpoint`` does not distiguish different crashes
+  during reduction. Thus, if new crash or miscompilation happens, ``bugpoint``
+  will continue with the new crash instead. If you would like to stick to
+  particular crash, you should write check scripts to validate the error
+  message, see ``-compile-command`` in :doc:`CommandGuide/bugpoint`.
+
 * In the code generator and miscompilation debuggers, debugging will go faster
   if you manually modify the program or its inputs to reduce the runtime, but
   still exhibit the problem.
diff --git a/llvm/docs/CommandGuide/index.rst b/llvm/docs/CommandGuide/index.rst
index 52ad316459d4c..77fece030eb60 100644
--- a/llvm/docs/CommandGuide/index.rst
+++ b/llvm/docs/CommandGuide/index.rst
@@ -74,3 +74,4 @@ Developer Tools
    llvm-build
    llvm-exegesis
    llvm-pdbutil
+   llvm-locstats
diff --git a/llvm/docs/CommandGuide/llvm-cxxfilt.rst b/llvm/docs/CommandGuide/llvm-cxxfilt.rst
index 1b6f9c665e265..5d257b98895e2 100644
--- a/llvm/docs/CommandGuide/llvm-cxxfilt.rst
+++ b/llvm/docs/CommandGuide/llvm-cxxfilt.rst
@@ -63,7 +63,7 @@ OPTIONS
 
 .. option:: --version
 
-  Display the version of this program.
+  Display the version of the :program:`llvm-cxxfilt` executable.
 
 .. option:: @<FILE>
 
diff --git a/llvm/docs/CommandGuide/llvm-locstats.rst b/llvm/docs/CommandGuide/llvm-locstats.rst
new file mode 100644
index 0000000000000..434b8c8ed2af6
--- /dev/null
+++ b/llvm/docs/CommandGuide/llvm-locstats.rst
@@ -0,0 +1,79 @@
+llvm-locstats - calculate statistics on DWARF debug location
+============================================================
+
+.. program:: llvm-locstats
+
+SYNOPSIS
+--------
+
+:program:`llvm-locstats` [*options*] [*filename*]
+
+DESCRIPTION
+-----------
+
+:program:`llvm-locstats` works like a wrapper around :program:`llvm-dwarfdump`.
+It parses :program:`llvm-dwarfdump` statistics regarding debug location by
+pretty printing it in a more human readable way.
+
+The line 0% shows the number and the percentage of DIEs with no location
+information, but the line 100% shows the information for DIEs where there is
+location information in all code section bytes (where the variable or parameter
+is in the scope). The line 50-59% shows the number and the percentage of DIEs
+where the location information is between 50 and 59 percentage of its scope
+covered.
+
+OPTIONS
+-------
+
+.. option:: -only-variables
+
+            Calculate the location statistics only for local variables.
+
+.. option:: -only-formal-parameters
+
+            Calculate the location statistics only for formal parameters.
+
+.. option:: -ignore-debug-entry-values
+
+            Ignore the location statistics on locations containing the
+            debug entry values DWARF operation.
+
+EXIT STATUS
+-----------
+
+:program:`llvm-locstats` returns 0 if the input file were parsed
+successfully. Otherwise, it returns 1.
+
+OUTPUT EXAMPLE
+--------------
+
+.. code-block:: none
+
+  =================================================
+            Debug Location Statistics
+  =================================================
+        cov%          samples       percentage(~)
+  -------------------------------------------------
+     0%                    1              16%
+     1-9%                  0               0%
+     10-19%                0               0%
+     20-29%                0               0%
+     30-39%                0               0%
+     40-49%                0               0%
+     50-99%                1              16%
+     60-69%                0               0%
+     70-79%                0               0%
+     80-89%                1              16%
+     90-99%                0               0%
+     100%                  3              50%
+  =================================================
+  -the number of debug variables processed: 6
+  -PC ranges covered: 81%
+  -------------------------------------------------
+  -total availability: 83%
+  =================================================
+
+SEE ALSO
+--------
+
+:manpage:`llvm-dwarfdump(1)`
diff --git a/llvm/docs/CommandGuide/llvm-nm.rst b/llvm/docs/CommandGuide/llvm-nm.rst
index 1efa15e2dfa29..71efac7fa7af3 100644
--- a/llvm/docs/CommandGuide/llvm-nm.rst
+++ b/llvm/docs/CommandGuide/llvm-nm.rst
@@ -227,7 +227,8 @@ OPTIONS
 
 .. option:: --version
 
- Display the version of this program. Does not stack with other commands.
+ Display the version of the :program:`llvm-nm` executable. Does not stack with
+ other commands.
 
 .. option:: --without-aliases
 
diff --git a/llvm/docs/CommandGuide/llvm-objcopy.rst b/llvm/docs/CommandGuide/llvm-objcopy.rst
index 04721170f5031..ccdcf13eb597f 100644
--- a/llvm/docs/CommandGuide/llvm-objcopy.rst
+++ b/llvm/docs/CommandGuide/llvm-objcopy.rst
@@ -43,6 +43,10 @@ multiple file formats.
  starts with ".note". Otherwise, it will have type `SHT_PROGBITS`. Can be
  specified multiple times to add multiple sections.
 
+.. option:: --binary-architecture <arch>, -B
+
+ Ignored for compatibility.
+
 .. option:: --disable-deterministic-archives, -U
 
  Use real values for UIDs, GIDs and timestamps when updating archive member
@@ -78,6 +82,11 @@ multiple file formats.
  Remove the specified section from the output. Can be specified multiple times
  to remove multiple sections simultaneously.
 
+.. option:: --set-section-alignment <section>=<align>
+
+ Set the alignment of section ``<section>`` to `<align>``. Can be specified
+ multiple times to update multiple sections.
+
 .. option:: --strip-all-gnu
 
  Remove all symbols, debug sections and relocations from the output. This option
@@ -127,7 +136,11 @@ multiple file formats.
 
 .. option:: --version, -V
 
-  Display the version of this program.
+ Display the version of the :program:`llvm-objcopy` executable.
+
+.. option:: @<FILE>
+
+ Read command-line options and commands from response file `<FILE>`.
 
 COFF-SPECIFIC OPTIONS
 ---------------------
@@ -174,25 +187,8 @@ them.
 
 .. option:: --allow-broken-links
 
- Allow llvm-objcopy to remove sections even if it would leave invalid section
- references. Any invalid sh_link fields will be set to zero.
-
-.. option:: --binary-architecture <arch>, -B
-
- Specify the architecture to use, when transforming an architecture-less format
- (e.g. binary) to another format. Valid options are:
-
- - `aarch64`
- - `arm`
- - `i386`
- - `i386:x86-64`
- - `mips`
- - `powerpc:common64`
- - `riscv:rv32`
- - `riscv:rv64`
- - `sparc`
- - `sparcel`
- - `x86-64`
+ Allow :program:`llvm-objcopy` to remove sections even if it would leave invalid
+ section references. Any invalid sh_link fields will be set to zero.
 
 .. option:: --build-id-link-dir <dir>
 
diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst
index d65e4906025b3..1b8273beb5671 100644
--- a/llvm/docs/CommandGuide/llvm-objdump.rst
+++ b/llvm/docs/CommandGuide/llvm-objdump.rst
@@ -91,7 +91,8 @@ combined with other commands:
 
 .. option:: --version
 
-  Display the version of this program. Does not stack with other commands.
+  Display the version of the :program:`llvm-objdump` executable. Does not stack
+  with other commands.
 
 .. option:: -x, --all-headers
 
diff --git a/llvm/docs/CommandGuide/llvm-readelf.rst b/llvm/docs/CommandGuide/llvm-readelf.rst
index 6aa904755725a..2868d0b598977 100644
--- a/llvm/docs/CommandGuide/llvm-readelf.rst
+++ b/llvm/docs/CommandGuide/llvm-readelf.rst
@@ -28,9 +28,9 @@ OPTIONS
 
  Display the address-significance table.
 
-.. option:: --arm-attributes
+.. option:: --arch-specific, -A
 
- Display the ARM attributes section. Only applicable for ARM architectures.
+ Display architecture-specific information, e.g. the ARM attributes section on ARM.
 
 .. option:: --color
 
@@ -181,7 +181,7 @@ OPTIONS
 
 .. option:: --version
 
- Display the version of this program.
+ Display the version of the :program:`llvm-readelf` executable.
 
 .. option:: --version-info, -V
 
diff --git a/llvm/docs/CommandGuide/llvm-readobj.rst b/llvm/docs/CommandGuide/llvm-readobj.rst
index 30f7976183d89..ea1b22d6d9f56 100644
--- a/llvm/docs/CommandGuide/llvm-readobj.rst
+++ b/llvm/docs/CommandGuide/llvm-readobj.rst
@@ -133,7 +133,7 @@ file formats.
 
 .. option:: --version
 
- Display the version of this program.
+ Display the version of the :program:`llvm-readobj` executable.
 
 .. option:: @<FILE>
 
@@ -144,9 +144,9 @@ ELF SPECIFIC OPTIONS
 
 The following options are implemented only for the ELF file format.
 
-.. option:: --arm-attributes
+.. option:: --arch-specific, -A
 
- Display the ARM attributes section. Only applicable for ARM architectures.
+ Display architecture-specific information, e.g. the ARM attributes section on ARM.
 
 .. option:: --demangle, -C
 
diff --git a/llvm/docs/CommandGuide/llvm-size.rst b/llvm/docs/CommandGuide/llvm-size.rst
index 92dcf4a571c5e..08426db4a3286 100644
--- a/llvm/docs/CommandGuide/llvm-size.rst
+++ b/llvm/docs/CommandGuide/llvm-size.rst
@@ -6,10 +6,193 @@ llvm-size - print size information
 SYNOPSIS
 --------
 
-:program:`llvm-size` [*options*]
+:program:`llvm-size` [*options*] [*input...*]
 
 DESCRIPTION
 -----------
 
-:program:`llvm-size` is a tool that prints size information for object files.
-The goal is to make it a drop-in replacement for GNU's :program:`size`.
+:program:`llvm-size` is a tool that prints size information for binary files.
+It is intended to be a drop-in replacement for GNU's :program:`size`.
+
+The tool prints size information for each ``input`` specified. If no input is
+specified, the program prints size information for ``a.out``. If "``-``" is
+specified as an input file, :program:`llvm-size` reads a file from the standard
+input stream. If an input is an archive, size information will be displayed for
+all its members.
+
+OPTIONS
+-------
+
+.. option:: -A
+
+ Equivalent to :option:`--format` with a value of ``sysv``.
+
+.. option:: --arch=<arch>
+
+ Architecture(s) from Mach-O universal binaries to display information for.
+
+.. option:: -B
+
+ Equivalent to :option:`--format` with a value of ``berkeley``.
+
+.. option:: --common
+
+ Include ELF common symbol sizes in bss size for ``berkeley`` output format, or
+ as a separate section entry for ``sysv`` output. If not specified, these
+ symbols are ignored.
+
+.. option:: -d
+
+ Equivalent to :option:`--radix` with a value of ``10``.
+
+.. option:: -l
+
+ Display verbose address and offset information for segments and sections in
+ Mach-O files in ``darwin`` format.
+
+.. option:: --format=<format>
+
+ Set the output format to the ``<format>`` specified. Available ``<format>``
+ options are ``berkeley`` (the default), ``sysv`` and ``darwin``.
+
+ Berkeley output summarises text, data and bss sizes in each file, as shown
+ below for a typical pair of ELF files:
+
+ .. code-block:: console
+
+  $ llvm-size --format=berkeley test.o test2.o
+     text    data     bss     dec     hex filename
+      182      16       5     203      cb test.elf
+       82       8       1      91      5b test2.o
+
+ For Mach-O files, the output format is slightly different:
+
+ .. code-block:: console
+
+  $ llvm-size --format=berkeley macho.obj macho2.obj
+  __TEXT  __DATA  __OBJC  others  dec     hex
+  4       8       0       0       12      c       macho.obj
+  16      32      0       0       48      30      macho2.obj
+
+ Sysv output displays size and address information for most sections, with each
+ file being listed separately:
+
+ .. code-block:: console
+
+  $ llvm-size --format=sysv test.elf test2.o
+     test.elf  :
+     section       size      addr
+     .eh_frame       92   2097496
+     .text           90   2101248
+     .data           16   2105344
+     .bss             5   2105360
+     .comment       209         0
+     Total          412
+
+     test2.o  :
+     section             size   addr
+     .text                 26      0
+     .data                  8      0
+     .bss                   1      0
+     .comment             106      0
+     .note.GNU-stack        0      0
+     .eh_frame             56      0
+     .llvm_addrsig          2      0
+     Total                199
+
+ ``darwin`` format only affects Mach-O input files. If an input of a different
+ file format is specified, :program:`llvm-size` falls back to ``berkeley``
+ format. When producing ``darwin`` format, the tool displays information about
+ segments and sections:
+
+ .. code-block:: console
+
+  $ llvm-size --format=darwin macho.obj macho2.obj
+     macho.obj:
+     Segment : 12
+             Section (__TEXT, __text): 4
+             Section (__DATA, __data): 8
+             total 12
+     total 12
+     macho2.obj:
+     Segment : 48
+             Section (__TEXT, __text): 16
+             Section (__DATA, __data): 32
+             total 48
+     total 48
+
+.. option:: --help, -h
+
+ Display a summary of command line options.
+
+.. option:: --help-list
+
+ Display an uncategorized summary of command line options.
+
+.. option:: -m
+
+ Equivalent to :option:`--format` with a value of ``darwin``.
+
+.. option:: -o
+
+ Equivalent to :option:`--radix` with a value of ``8``.
+
+.. option:: --radix=<value>
+
+ Display size information in the specified radix. Permitted values are ``8``,
+ ``10`` (the default) and ``16`` for octal, decimal and hexadecimal output
+ respectively.
+
+ Example:
+
+ .. code-block:: console
+
+  $ llvm-size --radix=8 test.o
+     text    data     bss     oct     hex filename
+     0152      04      04     162      72 test.o
+
+  $ llvm-size --radix=10 test.o
+     text    data     bss     dec     hex filename
+      106       4       4     114      72 test.o
+
+  $ llvm-size --radix=16 test.o
+     text    data     bss     dec     hex filename
+     0x6a     0x4     0x4     114      72 test.o
+
+.. option:: --totals, -t
+
+ Applies only to ``berkeley`` output format. Display the totals for all listed
+ fields, in addition to the individual file listings.
+
+ Example:
+
+ .. code-block:: console
+
+  $ llvm-size --totals test.elf test2.o
+     text    data     bss     dec     hex filename
+      182      16       5     203      cb test.elf
+       82       8       1      91      5b test2.o
+      264      24       6     294     126 (TOTALS)
+
+.. option:: --version
+
+ Display the version of the :program:`llvm-size` executable.
+
+.. option:: -x
+
+ Equivalent to :option:`--radix` with a value of ``16``.
+
+.. option:: @<FILE>
+
+ Read command-line options from response file ``<FILE>``.
+
+EXIT STATUS
+-----------
+
+:program:`llvm-size` exits with a non-zero exit code if there is an error.
+Otherwise, it exits with code 0.
+
+BUGS
+----
+
+To report bugs, please visit <http://llvm.org/bugs/>.
diff --git a/llvm/docs/CommandGuide/llvm-strings.rst b/llvm/docs/CommandGuide/llvm-strings.rst
index f6307d4a90569..f2d04c4190b85 100644
--- a/llvm/docs/CommandGuide/llvm-strings.rst
+++ b/llvm/docs/CommandGuide/llvm-strings.rst
@@ -6,10 +6,125 @@ llvm-strings - print strings
 SYNOPSIS
 --------
 
-:program:`llvm-strings` [*options*]
+:program:`llvm-strings` [*options*] [*input...*]
 
 DESCRIPTION
 -----------
 
-:program:`llvm-strings` is a tool that prints strings in files. The goal is to
-make it a drop-in replacement for GNU's :program:`strings`.
+:program:`llvm-strings` is a tool intended as a drop-in replacement for GNU's
+:program:`strings`, which looks for printable strings in files and writes them
+to the standard output stream. A printable string is any sequence of four (by
+default) or more printable ASCII characters. The end of the file, or any other
+byte, terminates the current sequence.
+
+:program:`llvm-strings` looks for strings in each ``input`` file specified.
+Unlike GNU :program:`strings` it looks in the entire input file, regardless of
+file format, rather than restricting the search to certain sections of object
+files. If "``-``" is specified as an ``input``, or no ``input`` is specified,
+the program reads from the standard input stream.
+
+EXAMPLE
+-------
+
+.. code-block:: console
+
+ $ cat input.txt
+ bars
+ foo
+ wibble blob
+ $ llvm-strings input.txt
+ bars
+ wibble blob
+
+OPTIONS
+-------
+
+.. option:: --all, -a
+
+ Silently ignored. Present for GNU :program:`strings` compatibility.
+
+.. option:: --bytes=<length>, -n
+
+ Set the minimum number of printable ASCII characters required for a sequence of
+ bytes to be considered a string. The default value is 4.
+
+.. option:: --help, -h
+
+ Display a summary of command line options.
+
+.. option:: --help-list
+
+ Display an uncategorized summary of command line options.
+
+.. option:: --print-file-name, -f
+
+ Display the name of the containing file before each string.
+
+ Example:
+
+ .. code-block:: console
+
+  $ llvm-strings --print-file-name test.o test.elf
+  test.o: _Z5hellov
+  test.o: some_bss
+  test.o: test.cpp
+  test.o: main
+  test.elf: test.cpp
+  test.elf: test2.cpp
+  test.elf: _Z5hellov
+  test.elf: main
+  test.elf: some_bss
+
+.. option:: --radix=<radix>, -t
+
+ Display the offset within the file of each string, before the string and using
+ the specified radix. Valid ``<radix>`` values are ``o``, ``d`` and ``x`` for
+ octal, decimal and hexadecimal respectively.
+
+ Example:
+
+ .. code-block:: console
+
+  $ llvm-strings --radix=o test.o
+      1054 _Z5hellov
+      1066 .rela.text
+      1101 .comment
+      1112 some_bss
+      1123 .bss
+      1130 test.cpp
+      1141 main
+  $ llvm-strings --radix=d test.o
+      556 _Z5hellov
+      566 .rela.text
+      577 .comment
+      586 some_bss
+      595 .bss
+      600 test.cpp
+      609 main
+  $ llvm-strings -t x test.o
+      22c _Z5hellov
+      236 .rela.text
+      241 .comment
+      24a some_bss
+      253 .bss
+      258 test.cpp
+      261 main
+
+.. option:: --version
+
+ Display the version of the :program:`llvm-strings` executable.
+
+.. option:: @<FILE>
+
+ Read command-line options from response file ``<FILE>``.
+
+EXIT STATUS
+-----------
+
+:program:`llvm-strings` exits with a non-zero exit code if there is an error.
+Otherwise, it exits with code 0.
+
+BUGS
+----
+
+To report bugs, please visit <http://llvm.org/bugs/>.
diff --git a/llvm/docs/CommandGuide/llvm-strip.rst b/llvm/docs/CommandGuide/llvm-strip.rst
index a4f497ae0e7ea..41529bf08cc14 100644
--- a/llvm/docs/CommandGuide/llvm-strip.rst
+++ b/llvm/docs/CommandGuide/llvm-strip.rst
@@ -13,10 +13,10 @@ DESCRIPTION
 
 :program:`llvm-strip` is a tool to strip sections and symbols from object files.
 If no other stripping or remove options are specified, :option:`--strip-all`
-will be enabled by default.
+will be enabled.
 
-The input files are modified in-place. If "-" is specified for the input file,
-the input is read from the program's standard input stream.
+By default, the input files are modified in-place. If "-" is specified for the
+input file, the input is read from the program's standard input stream.
 
 If the input is an archive, any requested operations will be applied to each
 archive member individually.
@@ -52,7 +52,7 @@ multiple file formats.
 
 .. option::  --no-strip-all
 
- Disable --strip-all.
+ Disable :option:`--strip-all`.
 
 .. option::  -o <file>
 
@@ -84,11 +84,7 @@ multiple file formats.
 
 .. option:: --strip-debug, -g
 
- Remove all debug sections.
-
-.. option:: --strip-sections
-
- Remove all section headers and all sections not in segments.
+ Remove all debug sections from the output.
 
 .. option:: --strip-symbol <symbol>, -N
 
@@ -102,7 +98,11 @@ multiple file formats.
 
 .. option:: --version, -V
 
- Display the version of this program.
+ Display the version of the :program:`llvm-strip` executable.
+
+.. option:: @<FILE>
+
+ Read command-line options and commands from response file `<FILE>`.
 
 COFF-SPECIFIC OPTIONS
 ---------------------
@@ -125,8 +125,8 @@ them.
 
 .. option:: --allow-broken-links
 
- Allow llvm-strip to remove sections even if it would leave invalid section
- references. Any invalid sh_link fields will be set to zero.
+ Allow :program:`llvm-strip` to remove sections even if it would leave invalid
+ section references. Any invalid sh_link fields will be set to zero.
 
 .. option:: --discard-locals, -X
 
@@ -143,12 +143,18 @@ them.
 
 .. option:: --keep-symbol <symbol>, -K
 
- Do not remove symbols named ``<symbol>``. Can be specified multiple times to
- keep multiple symbols.
+ When removing symbols from the output, do not remove symbols named
+ ``<symbol>``. Can be specified multiple times to keep multiple symbols.
 
 .. option::  --preserve-dates, -p
 
- Preserve access and modification timestamps.
+ Preserve access and modification timestamps in the output.
+
+.. option:: --strip-sections
+
+ Remove from the output all section headers and all section data not within
+ segments. Note that many tools will not be able to use an object without
+ section headers.
 
 EXIT STATUS
 -----------
diff --git a/llvm/docs/DependenceGraphs/cycle.png b/llvm/docs/DependenceGraphs/cycle.png
new file mode 100644
index 0000000000000..8099c4166544d
Binary files /dev/null and b/llvm/docs/DependenceGraphs/cycle.png differ
diff --git a/llvm/docs/DependenceGraphs/cycle_pi.png b/llvm/docs/DependenceGraphs/cycle_pi.png
new file mode 100644
index 0000000000000..ce69fd36a0799
Binary files /dev/null and b/llvm/docs/DependenceGraphs/cycle_pi.png differ
diff --git a/llvm/docs/DependenceGraphs/index.rst b/llvm/docs/DependenceGraphs/index.rst
new file mode 100644
index 0000000000000..c53ee9051d5fc
--- /dev/null
+++ b/llvm/docs/DependenceGraphs/index.rst
@@ -0,0 +1,140 @@
+=========================
+Dependence Graphs in LLVM
+=========================
+
+.. contents::
+   :local:
+
+Introduction
+============
+Dependence graphs are useful tools in compilers for analyzing relationships
+between various program elements to help guide optimizations. The ideas
+behind these graphs are described in papers [1]_ and [2]_.
+
+The implementation of these ideas in LLVM may be slightly different than
+what is mentioned in the papers. These differences are documented in
+the `implementation details <implementation-details_>`_.
+
+.. _DataDependenceGraph:
+
+Data Dependence Graph
+=====================
+In its simplest form the Data Dependence Graph (or DDG) represents data
+dependencies between individual instructions. Each node in such a graph
+represents a single instruction and is referred to as an "atomic" node.
+It is also possible to combine some atomic nodes that have a simple
+def-use dependency between them into larger nodes that contain multiple-
+instructions.
+
+As described in [1]_ the DDG uses graph abstraction to group nodes
+that are part of a strongly connected component of the graph 
+into special nodes called pi-blocks. pi-blocks represent cycles of data
+dependency that prevent reordering transformations. Since any strongly
+connected component of the graph is a maximal subgraph of all the nodes
+that form a cycle, pi-blocks are at most one level deep. In other words,
+no pi-blocks are nested inside another pi-block, resulting in a 
+hierarchical representation that is at most one level deep.
+
+
+For example, consider the following:
+
+.. code-block:: c++
+
+  for (int i = 1; i < n; i++) {
+    b[i] = c[i] + b[i-1];
+  }
+
+This code contains a statement that has a loop carried dependence on
+itself creating a cycle in the DDG. The figure bellow illustrates
+how the cycle of dependency is carried through multiple def-use relations
+and a memory access dependency.
+
+.. image:: cycle.png
+
+The DDG corresponding to this example would have a pi-block that contains
+all the nodes participating in the cycle, as shown bellow:
+
+.. image:: cycle_pi.png
+
+Program Dependence Graph
+========================
+
+The Program Dependence Graph (or PDG) has a similar structure as the
+DDG, but it is capable of representing both data dependencies and
+control-flow dependencies between program elements such as
+instructions, groups of instructions, basic blocks or groups of
+basic blocks.
+
+High-Level Design
+=================
+
+The DDG and the PDG are both directed graphs and they extend the
+``DirectedGraph`` class. Each implementation extends its corresponding
+node and edge types resulting in the inheritance relationship depicted
+in the UML diagram bellow:
+
+.. image:: uml_nodes_and_edges.png
+
+Graph Construction
+------------------
+
+The graph build algorithm considers dependencies between elements of
+a given set of instructions or basic blocks. Any dependencies coming
+into or going out of instructions that do not belong to that range
+are ignored. The steps in the build algorithm for the DDG are very
+similar to the steps in the build algorithm for the PDG. As such,
+one of the design goals is to reuse the build algorithm code to
+allow creation of both DDG and PDG representations while allowing
+the two implementations to define their own distinct and independent
+node and edge types. This is achieved by using the well-known builder
+design pattern to isolate the construction of the dependence graph
+from its concrete representation.
+
+The following UML diagram depicts the overall structure of the design
+pattern as it applies to the dependence graph implementation.
+
+.. image:: uml_builder_pattern.png
+
+Notice that the common code for building the two types of graphs are
+provided in the ``DependenceGraphBuilder`` class, while the ``DDGBuilder``
+and ``PDGBuilder`` control some aspects of how the graph is constructed
+by the way of overriding virtual methods defined in ``DependenceGraphBuilder``.
+
+Note also that the steps and the names used in this diagram are for
+illustrative purposes and may be different from those in the actual
+implementation.
+
+Design Trade-offs
+-----------------
+
+Advantages:
+^^^^^^^^^^^
+  - Builder allows graph construction code to be reused for DDG and PDG.
+  - Builder allows us to create DDG and PDG as separate graphs.
+  - DDG nodes and edges are completely disjoint from PDG nodes and edges allowing them to change easily and independently.
+
+Disadvantages:
+^^^^^^^^^^^^^^
+  - Builder may be perceived as over-engineering at first.
+  - There are some similarities between DDG nodes and edges compared to PDG nodes and edges, but there is little reuse of the class definitions.
+
+    - This is tolerable given that the node and edge types are fairly simple and there is little code reuse opportunity anyway.
+
+
+.. _implementation-details:
+
+Implementation Details
+======================
+
+The current implementation of DDG differs slightly from the dependence
+graph described in [1]_ in the following ways:
+
+  1. The graph nodes in the paper represent three main program components, namely *assignment statements*, *for loop headers* and *while loop headers*. In this implementation, DDG nodes naturally represent LLVM IR instructions. An assignment statement in this implementation typically involves a node representing the ``store`` instruction along with a number of individual nodes computing the right-hand-side of the assignment that connect to the ``store`` node via a def-use edge.  The loop header instructions are not represented as special nodes in this implementation because they have limited uses and can be easily identified, for example, through ``LoopAnalysis``.
+  2. The paper describes five types of dependency edges between nodes namely *loop dependency*, *flow-*, *anti-*, *output-*, and *input-* dependencies. In this implementation *memory* edges represent the *flow-*, *anti-*, *output-*, and *input-* dependencies. However, *loop dependencies* are not made explicit, because they mainly represent association between a loop structure and the program elements inside the loop and this association is fairly obvious in LLVM IR itself. 
+  3. The paper describes two types of pi-blocks; *recurrences* whose bodies are SCCs and *IN* nodes whose bodies are not part of any SCC. In this impelmentation, pi-blocks are only created for *recurrences*. *IN* nodes remain as simple DDG nodes in the graph.
+
+
+References
+----------
+.. [1] "D. J. Kuck, R. H. Kuhn, D. A. Padua, B. Leasure, and M. Wolfe (1981). DEPENDENCE GRAPHS AND COMPILER OPTIMIZATIONS."
+.. [2] "J. FERRANTE (IBM), K. J. OTTENSTEIN (Michigan Technological University) and JOE D. WARREN (Rice University), 1987. The Program Dependence Graph and Its Use in Optimization."
diff --git a/llvm/docs/DependenceGraphs/uml_builder_pattern.png b/llvm/docs/DependenceGraphs/uml_builder_pattern.png
new file mode 100644
index 0000000000000..1651f9141de51
Binary files /dev/null and b/llvm/docs/DependenceGraphs/uml_builder_pattern.png differ
diff --git a/llvm/docs/DependenceGraphs/uml_nodes_and_edges.png b/llvm/docs/DependenceGraphs/uml_nodes_and_edges.png
new file mode 100644
index 0000000000000..c0ffacafca709
Binary files /dev/null and b/llvm/docs/DependenceGraphs/uml_nodes_and_edges.png differ
diff --git a/llvm/docs/Frontend/PerformanceTips.rst b/llvm/docs/Frontend/PerformanceTips.rst
index 142d262eb6573..749fbd8085879 100644
--- a/llvm/docs/Frontend/PerformanceTips.rst
+++ b/llvm/docs/Frontend/PerformanceTips.rst
@@ -9,219 +9,220 @@ Performance Tips for Frontend Authors
 Abstract
 ========
 
-The intended audience of this document is developers of language frontends 
-targeting LLVM IR. This document is home to a collection of tips on how to 
-generate IR that optimizes well.  
+The intended audience of this document is developers of language frontends
+targeting LLVM IR. This document is home to a collection of tips on how to
+generate IR that optimizes well.
 
 IR Best Practices
 =================
 
-As with any optimizer, LLVM has its strengths and weaknesses.  In some cases, 
-surprisingly small changes in the source IR can have a large effect on the 
-generated code.  
+As with any optimizer, LLVM has its strengths and weaknesses.  In some cases,
+surprisingly small changes in the source IR can have a large effect on the
+generated code.
 
-Beyond the specific items on the list below, it's worth noting that the most 
-mature frontend for LLVM is Clang.  As a result, the further your IR gets from what Clang might emit, the less likely it is to be effectively optimized.  It 
+Beyond the specific items on the list below, it's worth noting that the most
+mature frontend for LLVM is Clang.  As a result, the further your IR gets from
+what Clang might emit, the less likely it is to be effectively optimized. It
 can often be useful to write a quick C program with the semantics you're trying
-to model and see what decisions Clang's IRGen makes about what IR to emit.  
-Studying Clang's CodeGen directory can also be a good source of ideas.  Note 
-that Clang and LLVM are explicitly version locked so you'll need to make sure 
-you're using a Clang built from the same svn revision or release as the LLVM 
-library you're using.  As always, it's *strongly* recommended that you track 
+to model and see what decisions Clang's IRGen makes about what IR to emit.
+Studying Clang's CodeGen directory can also be a good source of ideas.  Note
+that Clang and LLVM are explicitly version locked so you'll need to make sure
+you're using a Clang built from the same svn revision or release as the LLVM
+library you're using.  As always, it's *strongly* recommended that you track
 tip of tree development, particularly during bring up of a new project.
 
 The Basics
 ^^^^^^^^^^^
 
-#. Make sure that your Modules contain both a data layout specification and 
+#. Make sure that your Modules contain both a data layout specification and
    target triple. Without these pieces, non of the target specific optimization
    will be enabled.  This can have a major effect on the generated code quality.
 
 #. For each function or global emitted, use the most private linkage type
-   possible (private, internal or linkonce_odr preferably).  Doing so will 
+   possible (private, internal or linkonce_odr preferably).  Doing so will
    make LLVM's inter-procedural optimizations much more effective.
 
 #. Avoid high in-degree basic blocks (e.g. basic blocks with dozens or hundreds
-   of predecessors).  Among other issues, the register allocator is known to 
-   perform badly with confronted with such structures.  The only exception to 
+   of predecessors).  Among other issues, the register allocator is known to
+   perform badly with confronted with such structures.  The only exception to
    this guidance is that a unified return block with high in-degree is fine.
 
 Use of allocas
 ^^^^^^^^^^^^^^
 
-An alloca instruction can be used to represent a function scoped stack slot, 
-but can also represent dynamic frame expansion.  When representing function 
-scoped variables or locations, placing alloca instructions at the beginning of 
-the entry block should be preferred.   In particular, place them before any 
-call instructions. Call instructions might get inlined and replaced with 
-multiple basic blocks. The end result is that a following alloca instruction 
+An alloca instruction can be used to represent a function scoped stack slot,
+but can also represent dynamic frame expansion.  When representing function
+scoped variables or locations, placing alloca instructions at the beginning of
+the entry block should be preferred.   In particular, place them before any
+call instructions. Call instructions might get inlined and replaced with
+multiple basic blocks. The end result is that a following alloca instruction
 would no longer be in the entry basic block afterward.
 
 The SROA (Scalar Replacement Of Aggregates) and Mem2Reg passes only attempt
-to eliminate alloca instructions that are in the entry basic block.  Given 
-SSA is the canonical form expected by much of the optimizer; if allocas can 
-not be eliminated by Mem2Reg or SROA, the optimizer is likely to be less 
+to eliminate alloca instructions that are in the entry basic block.  Given
+SSA is the canonical form expected by much of the optimizer; if allocas can
+not be eliminated by Mem2Reg or SROA, the optimizer is likely to be less
 effective than it could be.
 
 Avoid loads and stores of large aggregate type
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 LLVM currently does not optimize well loads and stores of large :ref:`aggregate
-types <t_aggregate>` (i.e. structs and arrays).  As an alternative, consider 
+types <t_aggregate>` (i.e. structs and arrays).  As an alternative, consider
 loading individual fields from memory.
 
-Aggregates that are smaller than the largest (performant) load or store 
-instruction supported by the targeted hardware are well supported.  These can 
-be an effective way to represent collections of small packed fields.  
+Aggregates that are smaller than the largest (performant) load or store
+instruction supported by the targeted hardware are well supported.  These can
+be an effective way to represent collections of small packed fields.
 
 Prefer zext over sext when legal
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-On some architectures (X86_64 is one), sign extension can involve an extra 
+On some architectures (X86_64 is one), sign extension can involve an extra
 instruction whereas zero extension can be folded into a load.  LLVM will try to
-replace a sext with a zext when it can be proven safe, but if you have 
-information in your source language about the range of a integer value, it can 
-be profitable to use a zext rather than a sext.  
+replace a sext with a zext when it can be proven safe, but if you have
+information in your source language about the range of a integer value, it can
+be profitable to use a zext rather than a sext.
 
-Alternatively, you can :ref:`specify the range of the value using metadata 
+Alternatively, you can :ref:`specify the range of the value using metadata
 <range-metadata>` and LLVM can do the sext to zext conversion for you.
 
 Zext GEP indices to machine register width
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Internally, LLVM often promotes the width of GEP indices to machine register
-width.  When it does so, it will default to using sign extension (sext) 
-operations for safety.  If your source language provides information about 
-the range of the index, you may wish to manually extend indices to machine 
+width.  When it does so, it will default to using sign extension (sext)
+operations for safety.  If your source language provides information about
+the range of the index, you may wish to manually extend indices to machine
 register width using a zext instruction.
 
 When to specify alignment
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 LLVM will always generate correct code if you don’t specify alignment, but may
-generate inefficient code.  For example, if you are targeting MIPS (or older 
-ARM ISAs) then the hardware does not handle unaligned loads and stores, and 
-so you will enter a trap-and-emulate path if you do a load or store with 
-lower-than-natural alignment.  To avoid this, LLVM will emit a slower 
-sequence of loads, shifts and masks (or load-right + load-left on MIPS) for 
-all cases where the load / store does not have a sufficiently high alignment 
+generate inefficient code.  For example, if you are targeting MIPS (or older
+ARM ISAs) then the hardware does not handle unaligned loads and stores, and
+so you will enter a trap-and-emulate path if you do a load or store with
+lower-than-natural alignment.  To avoid this, LLVM will emit a slower
+sequence of loads, shifts and masks (or load-right + load-left on MIPS) for
+all cases where the load / store does not have a sufficiently high alignment
 in the IR.
 
-The alignment is used to guarantee the alignment on allocas and globals, 
-though in most cases this is unnecessary (most targets have a sufficiently 
-high default alignment that they’ll be fine).  It is also used to provide a 
+The alignment is used to guarantee the alignment on allocas and globals,
+though in most cases this is unnecessary (most targets have a sufficiently
+high default alignment that they’ll be fine).  It is also used to provide a
 contract to the back end saying ‘either this load/store has this alignment, or
-it is undefined behavior’.  This means that the back end is free to emit 
-instructions that rely on that alignment (and mid-level optimizers are free to 
-perform transforms that require that alignment).  For x86, it doesn’t make 
-much difference, as almost all instructions are alignment-independent.  For 
+it is undefined behavior’.  This means that the back end is free to emit
+instructions that rely on that alignment (and mid-level optimizers are free to
+perform transforms that require that alignment).  For x86, it doesn’t make
+much difference, as almost all instructions are alignment-independent.  For
 MIPS, it can make a big difference.
 
-Note that if your loads and stores are atomic, the backend will be unable to 
-lower an under aligned access into a sequence of natively aligned accesses.  
+Note that if your loads and stores are atomic, the backend will be unable to
+lower an under aligned access into a sequence of natively aligned accesses.
 As a result, alignment is mandatory for atomic loads and stores.
 
 Other Things to Consider
 ^^^^^^^^^^^^^^^^^^^^^^^^
 
-#. Use ptrtoint/inttoptr sparingly (they interfere with pointer aliasing 
+#. Use ptrtoint/inttoptr sparingly (they interfere with pointer aliasing
    analysis), prefer GEPs
 
-#. Prefer globals over inttoptr of a constant address - this gives you 
-   dereferencability information.  In MCJIT, use getSymbolAddress to provide 
+#. Prefer globals over inttoptr of a constant address - this gives you
+   dereferencability information.  In MCJIT, use getSymbolAddress to provide
    actual address.
 
-#. Be wary of ordered and atomic memory operations.  They are hard to optimize 
+#. Be wary of ordered and atomic memory operations.  They are hard to optimize
    and may not be well optimized by the current optimizer.  Depending on your
    source language, you may consider using fences instead.
 
-#. If calling a function which is known to throw an exception (unwind), use 
-   an invoke with a normal destination which contains an unreachable 
-   instruction.  This form conveys to the optimizer that the call returns 
+#. If calling a function which is known to throw an exception (unwind), use
+   an invoke with a normal destination which contains an unreachable
+   instruction.  This form conveys to the optimizer that the call returns
    abnormally.  For an invoke which neither returns normally or requires unwind
-   code in the current function, you can use a noreturn call instruction if 
+   code in the current function, you can use a noreturn call instruction if
    desired.  This is generally not required because the optimizer will convert
    an invoke with an unreachable unwind destination to a call instruction.
 
-#. Use profile metadata to indicate statically known cold paths, even if 
-   dynamic profiling information is not available.  This can make a large 
+#. Use profile metadata to indicate statically known cold paths, even if
+   dynamic profiling information is not available.  This can make a large
    difference in code placement and thus the performance of tight loops.
 
 #. When generating code for loops, try to avoid terminating the header block of
-   the loop earlier than necessary.  If the terminator of the loop header 
+   the loop earlier than necessary.  If the terminator of the loop header
    block is a loop exiting conditional branch, the effectiveness of LICM will
-   be limited for loads not in the header.  (This is due to the fact that LLVM 
-   may not know such a load is safe to speculatively execute and thus can't 
-   lift an otherwise loop invariant load unless it can prove the exiting 
-   condition is not taken.)  It can be profitable, in some cases, to emit such 
-   instructions into the header even if they are not used along a rarely 
-   executed path that exits the loop.  This guidance specifically does not 
+   be limited for loads not in the header.  (This is due to the fact that LLVM
+   may not know such a load is safe to speculatively execute and thus can't
+   lift an otherwise loop invariant load unless it can prove the exiting
+   condition is not taken.)  It can be profitable, in some cases, to emit such
+   instructions into the header even if they are not used along a rarely
+   executed path that exits the loop.  This guidance specifically does not
    apply if the condition which terminates the loop header is itself invariant,
    or can be easily discharged by inspecting the loop index variables.
 
-#. In hot loops, consider duplicating instructions from small basic blocks 
-   which end in highly predictable terminators into their successor blocks.  
-   If a hot successor block contains instructions which can be vectorized 
+#. In hot loops, consider duplicating instructions from small basic blocks
+   which end in highly predictable terminators into their successor blocks.
+   If a hot successor block contains instructions which can be vectorized
    with the duplicated ones, this can provide a noticeable throughput
-   improvement.  Note that this is not always profitable and does involve a 
+   improvement.  Note that this is not always profitable and does involve a
    potentially large increase in code size.
 
 #. When checking a value against a constant, emit the check using a consistent
    comparison type.  The GVN pass *will* optimize redundant equalities even if
    the type of comparison is inverted, but GVN only runs late in the pipeline.
-   As a result, you may miss the opportunity to run other important 
-   optimizations.  Improvements to EarlyCSE to remove this issue are tracked in 
+   As a result, you may miss the opportunity to run other important
+   optimizations.  Improvements to EarlyCSE to remove this issue are tracked in
    Bug 23333.
 
-#. Avoid using arithmetic intrinsics unless you are *required* by your source 
-   language specification to emit a particular code sequence.  The optimizer 
+#. Avoid using arithmetic intrinsics unless you are *required* by your source
+   language specification to emit a particular code sequence.  The optimizer
    is quite good at reasoning about general control flow and arithmetic, it is
-   not anywhere near as strong at reasoning about the various intrinsics.  If 
-   profitable for code generation purposes, the optimizer will likely form the 
-   intrinsics itself late in the optimization pipeline.  It is *very* rarely 
+   not anywhere near as strong at reasoning about the various intrinsics.  If
+   profitable for code generation purposes, the optimizer will likely form the
+   intrinsics itself late in the optimization pipeline.  It is *very* rarely
    profitable to emit these directly in the language frontend.  This item
    explicitly includes the use of the :ref:`overflow intrinsics <int_overflow>`.
 
-#. Avoid using the :ref:`assume intrinsic <int_assume>` until you've 
-   established that a) there's no other way to express the given fact and b) 
-   that fact is critical for optimization purposes.  Assumes are a great 
-   prototyping mechanism, but they can have negative effects on both compile 
-   time and optimization effectiveness.  The former is fixable with enough 
+#. Avoid using the :ref:`assume intrinsic <int_assume>` until you've
+   established that a) there's no other way to express the given fact and b)
+   that fact is critical for optimization purposes.  Assumes are a great
+   prototyping mechanism, but they can have negative effects on both compile
+   time and optimization effectiveness.  The former is fixable with enough
    effort, but the later is fairly fundamental to their designed purpose.
 
 
 Describing Language Specific Properties
 =======================================
 
-When translating a source language to LLVM, finding ways to express concepts 
-and guarantees available in your source language which are not natively 
-provided by LLVM IR will greatly improve LLVM's ability to optimize your code. 
+When translating a source language to LLVM, finding ways to express concepts
+and guarantees available in your source language which are not natively
+provided by LLVM IR will greatly improve LLVM's ability to optimize your code.
 As an example, C/C++'s ability to mark every add as "no signed wrap (nsw)" goes
-a long way to assisting the optimizer in reasoning about loop induction 
-variables and thus generating more optimal code for loops.  
+a long way to assisting the optimizer in reasoning about loop induction
+variables and thus generating more optimal code for loops.
 
-The LLVM LangRef includes a number of mechanisms for annotating the IR with 
-additional semantic information.  It is *strongly* recommended that you become 
-highly familiar with this document.  The list below is intended to highlight a 
+The LLVM LangRef includes a number of mechanisms for annotating the IR with
+additional semantic information.  It is *strongly* recommended that you become
+highly familiar with this document.  The list below is intended to highlight a
 couple of items of particular interest, but is by no means exhaustive.
 
 Restricted Operation Semantics
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-#. Add nsw/nuw flags as appropriate.  Reasoning about overflow is 
-   generally hard for an optimizer so providing these facts from the frontend 
-   can be very impactful.  
+#. Add nsw/nuw flags as appropriate.  Reasoning about overflow is
+   generally hard for an optimizer so providing these facts from the frontend
+   can be very impactful.
 
-#. Use fast-math flags on floating point operations if legal.  If you don't 
-   need strict IEEE floating point semantics, there are a number of additional 
-   optimizations that can be performed.  This can be highly impactful for 
+#. Use fast-math flags on floating point operations if legal.  If you don't
+   need strict IEEE floating point semantics, there are a number of additional
+   optimizations that can be performed.  This can be highly impactful for
    floating point intensive computations.
 
 Describing Aliasing Properties
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-#. Add noalias/align/dereferenceable/nonnull to function arguments and return 
+#. Add noalias/align/dereferenceable/nonnull to function arguments and return
    values as appropriate
 
-#. Use pointer aliasing metadata, especially tbaa metadata, to communicate 
+#. Use pointer aliasing metadata, especially tbaa metadata, to communicate
    otherwise-non-deducible pointer aliasing facts
 
 #. Use inbounds on geps.  This can help to disambiguate some aliasing queries.
@@ -232,37 +233,37 @@ Modeling Memory Effects
 
 #. Mark functions as readnone/readonly/argmemonly or noreturn/nounwind when
    known.  The optimizer will try to infer these flags, but may not always be
-   able to.  Manual annotations are particularly important for external 
+   able to.  Manual annotations are particularly important for external
    functions that the optimizer can not analyze.
 
-#. Use the lifetime.start/lifetime.end and invariant.start/invariant.end 
-   intrinsics where possible.  Common profitable uses are for stack like data 
-   structures (thus allowing dead store elimination) and for describing 
-   life times of allocas (thus allowing smaller stack sizes).  
+#. Use the lifetime.start/lifetime.end and invariant.start/invariant.end
+   intrinsics where possible.  Common profitable uses are for stack like data
+   structures (thus allowing dead store elimination) and for describing
+   life times of allocas (thus allowing smaller stack sizes).
 
 #. Mark invariant locations using !invariant.load and TBAA's constant flags
 
 Pass Ordering
 ^^^^^^^^^^^^^
 
-One of the most common mistakes made by new language frontend projects is to 
+One of the most common mistakes made by new language frontend projects is to
 use the existing -O2 or -O3 pass pipelines as is.  These pass pipelines make a
-good starting point for an optimizing compiler for any language, but they have 
-been carefully tuned for C and C++, not your target language.  You will almost 
-certainly need to use a custom pass order to achieve optimal performance.  A 
+good starting point for an optimizing compiler for any language, but they have
+been carefully tuned for C and C++, not your target language.  You will almost
+certainly need to use a custom pass order to achieve optimal performance.  A
 couple specific suggestions:
 
-#. For languages with numerous rarely executed guard conditions (e.g. null 
-   checks, type checks, range checks) consider adding an extra execution or 
-   two of LoopUnswith and LICM to your pass order.  The standard pass order, 
-   which is tuned for C and C++ applications, may not be sufficient to remove 
+#. For languages with numerous rarely executed guard conditions (e.g. null
+   checks, type checks, range checks) consider adding an extra execution or
+   two of LoopUnswith and LICM to your pass order.  The standard pass order,
+   which is tuned for C and C++ applications, may not be sufficient to remove
    all dischargeable checks from loops.
 
-#. If you language uses range checks, consider using the IRCE pass.  It is not 
+#. If you language uses range checks, consider using the IRCE pass.  It is not
    currently part of the standard pass order.
 
-#. A useful sanity check to run is to run your optimized IR back through the 
-   -O2 pipeline again.  If you see noticeable improvement in the resulting IR, 
+#. A useful sanity check to run is to run your optimized IR back through the
+   -O2 pipeline again.  If you see noticeable improvement in the resulting IR,
    you likely need to adjust your pass order.
 
 
@@ -271,16 +272,16 @@ I Still Can't Find What I'm Looking For
 
 If you didn't find what you were looking for above, consider proposing an piece
 of metadata which provides the optimization hint you need.  Such extensions are
-relatively common and are generally well received by the community.  You will 
-need to ensure that your proposal is sufficiently general so that it benefits 
+relatively common and are generally well received by the community.  You will
+need to ensure that your proposal is sufficiently general so that it benefits
 others if you wish to contribute it upstream.
 
-You should also consider describing the problem you're facing on `llvm-dev 
-<http://lists.llvm.org/mailman/listinfo/llvm-dev>`_ and asking for advice.  
-It's entirely possible someone has encountered your problem before and can 
-give good advice.  If there are multiple interested parties, that also 
+You should also consider describing the problem you're facing on `llvm-dev
+<http://lists.llvm.org/mailman/listinfo/llvm-dev>`_ and asking for advice.
+It's entirely possible someone has encountered your problem before and can
+give good advice.  If there are multiple interested parties, that also
 increases the chances that a metadata extension would be well received by the
-community as a whole.  
+community as a whole.
 
 Adding to this document
 =======================
@@ -289,8 +290,8 @@ If you run across a case that you feel deserves to be covered here, please send
 a patch to `llvm-commits
 <http://lists.llvm.org/mailman/listinfo/llvm-commits>`_ for review.
 
-If you have questions on these items, please direct them to `llvm-dev 
-<http://lists.llvm.org/mailman/listinfo/llvm-dev>`_.  The more relevant 
-context you are able to give to your question, the more likely it is to be 
+If you have questions on these items, please direct them to `llvm-dev
+<http://lists.llvm.org/mailman/listinfo/llvm-dev>`_.  The more relevant
+context you are able to give to your question, the more likely it is to be
 answered.
 
diff --git a/llvm/docs/GettingInvolved.rst b/llvm/docs/GettingInvolved.rst
new file mode 100644
index 0000000000000..343b226231ad6
--- /dev/null
+++ b/llvm/docs/GettingInvolved.rst
@@ -0,0 +1,206 @@
+Getting Involved
+================
+
+LLVM welcomes contributions of all kinds. To get started, please review the following topics:
+
+.. contents::
+   :local:
+
+.. toctree::
+   :hidden:
+
+   Contributing
+   DeveloperPolicy
+   SphinxQuickstartTemplate
+   Phabricator
+   HowToSubmitABug
+   BugLifeCycle
+   CodingStandards
+
+:doc:`Contributing`
+   An overview on how to contribute to LLVM.
+
+:doc:`DeveloperPolicy`
+   The LLVM project's policy towards developers and their contributions.
+
+:doc:`SphinxQuickstartTemplate`
+  A template + tutorial for writing new Sphinx documentation. It is meant
+  to be read in source form.
+
+:doc:`Phabricator`
+   Describes how to use the Phabricator code review tool hosted on
+   http://reviews.llvm.org/ and its command line interface, Arcanist.
+
+:doc:`HowToSubmitABug`
+   Instructions for properly submitting information about any bugs you run into
+   in the LLVM system.
+
+:doc:`BugLifeCycle`
+   Describes how bugs are reported, triaged and closed.
+
+:doc:`CodingStandards`
+  Details the LLVM coding standards and provides useful information on writing
+  efficient C++ code.
+
+.. _development-process:
+
+Development Process
+-------------------
+
+Information about LLVM's development process.
+
+.. toctree::
+   :hidden:
+
+   Projects
+   LLVMBuild
+   HowToReleaseLLVM
+   Packaging
+   ReleaseProcess
+   HowToAddABuilder
+   ReleaseNotes
+
+:doc:`Projects`
+  How-to guide and templates for new projects that *use* the LLVM
+  infrastructure.  The templates (directory organization, Makefiles, and test
+  tree) allow the project code to be located outside (or inside) the ``llvm/``
+  tree, while using LLVM header files and libraries.
+
+:doc:`LLVMBuild`
+  Describes the LLVMBuild organization and files used by LLVM to specify
+  component descriptions.
+
+:doc:`HowToReleaseLLVM`
+  This is a guide to preparing LLVM releases. Most developers can ignore it.
+
+:doc:`ReleaseProcess`
+  This is a guide to validate a new release, during the release process. Most developers can ignore it.
+
+:doc:`HowToAddABuilder`
+   Instructions for adding new builder to LLVM buildbot master.
+
+:doc:`Packaging`
+   Advice on packaging LLVM into a distribution.
+
+:doc:`Release notes for the current release <ReleaseNotes>`
+   This describes new features, known bugs, and other limitations.
+
+.. _mailing-lists:
+
+Mailing Lists
+-------------
+
+If you can't find what you need in these docs, try consulting the mailing
+lists.
+
+`Developer's List (llvm-dev)`__
+  This list is for people who want to be included in technical discussions of
+  LLVM. People post to this list when they have questions about writing code
+  for or using the LLVM tools. It is relatively low volume.
+
+  .. __: http://lists.llvm.org/mailman/listinfo/llvm-dev
+
+`Commits Archive (llvm-commits)`__
+  This list contains all commit messages that are made when LLVM developers
+  commit code changes to the repository. It also serves as a forum for
+  patch review (i.e. send patches here). It is useful for those who want to
+  stay on the bleeding edge of LLVM development. This list is very high
+  volume.
+
+  .. __: http://lists.llvm.org/pipermail/llvm-commits/
+
+`Bugs & Patches Archive (llvm-bugs)`__
+  This list gets emailed every time a bug is opened and closed. It is
+  higher volume than the LLVM-dev list.
+
+  .. __: http://lists.llvm.org/pipermail/llvm-bugs/
+
+`Test Results Archive (llvm-testresults)`__
+  A message is automatically sent to this list by every active nightly tester
+  when it completes.  As such, this list gets email several times each day,
+  making it a high volume list.
+
+  .. __: http://lists.llvm.org/pipermail/llvm-testresults/
+
+`LLVM Announcements List (llvm-announce)`__
+  This is a low volume list that provides important announcements regarding
+  LLVM.  It gets email about once a month.
+
+  .. __: http://lists.llvm.org/mailman/listinfo/llvm-announce
+
+IRC
+---
+
+Users and developers of the LLVM project (including subprojects such as Clang)
+can be found in #llvm on `irc.oftc.net <irc://irc.oftc.net/llvm>`_.
+
+This channel has several bots.
+
+* Buildbot reporters
+
+  * llvmbb - Bot for the main LLVM buildbot master.
+    http://lab.llvm.org:8011/console
+  * smooshlab - Apple's internal buildbot master.
+
+* robot - Bugzilla linker. %bug <number>
+
+* clang-bot - A `geordi <http://www.eelis.net/geordi/>`_ instance running
+  near-trunk clang instead of gcc.
+
+.. _meetups-social-events:
+
+Meetups and social events
+-------------------------
+
+.. toctree::
+   :hidden:
+
+   MeetupGuidelines
+
+Besides developer `meetings and conferences <https://llvm.org/devmtg/>`_,
+there are several user groups called
+`LLVM Socials <https://www.meetup.com/pro/llvm/>`_. We greatly encourage you to
+join one in your city. Or start a new one if there is none:
+
+:doc:`MeetupGuidelines`
+
+.. _community-proposals:
+
+Community wide proposals
+------------------------
+
+Proposals for massive changes in how the community behaves and how the work flow
+can be better.
+
+.. toctree::
+   :hidden:
+
+   CodeOfConduct
+   Proposals/GitHubMove
+   BugpointRedesign
+   Proposals/LLVMLibC
+   Proposals/TestSuite
+   Proposals/VariableNames
+   Proposals/VectorizationPlan
+
+:doc:`CodeOfConduct`
+   Proposal to adopt a code of conduct on the LLVM social spaces (lists, events,
+   IRC, etc).
+
+:doc:`Proposals/GitHubMove`
+   Proposal to move from SVN/Git to GitHub.
+
+:doc:`BugpointRedesign`
+   Design doc for a redesign of the Bugpoint tool.
+
+:doc:`Proposals/LLVMLibC`
+   Proposal to add a libc implementation under the LLVM project.
+
+:doc:`Proposals/TestSuite`
+   Proposals for additional benchmarks/programs for llvm's test-suite.
+
+:doc:`Proposals/VariableNames`
+   Proposal to change the variable names coding standard.
+
+:doc:`Proposals/VectorizationPlan`
+   Proposal to model the process and upgrade the infrastructure of LLVM's Loop Vectorizer.
\ No newline at end of file
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 03b017c94b56d..2d4c57b796811 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1786,7 +1786,8 @@ example:
     requires strict floating-point semantics.  LLVM will not attempt any
     optimizations that require assumptions about the floating-point rounding
     mode or that might alter the state of floating-point status flags that
-    might otherwise be set or cleared by calling this function.
+    might otherwise be set or cleared by calling this function. LLVM will
+    not introduce any new floating-point instructions that may trap.
 ``"thunk"``
     This attribute indicates that the function will delegate to some other
     function with a tail call. The prototype of a thunk should not be used for
@@ -3825,6 +3826,8 @@ AArch64:
 - ``w``: A 32, 64, or 128-bit floating-point, SIMD or SVE vector register.
 - ``x``: Like w, but restricted to registers 0 to 15 inclusive.
 - ``y``: Like w, but restricted to SVE vector registers Z0 to Z7 inclusive.
+- ``Upl``: One of the low eight SVE predicate registers (P0 to P7)
+- ``Upa``: Any of the SVE predicate registers (P0 to P15)
 
 AMDGPU:
 
@@ -3859,12 +3862,12 @@ ARM and ARM's Thumb2 mode:
   as ``r``.
 - ``h``: In Thumb2 mode, a high 32-bit GPR register (``r8-r15``). In ARM mode,
   invalid.
-- ``w``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s31``,
-  ``d0-d31``, or ``q0-q15``.
-- ``x``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s15``,
-  ``d0-d7``, or ``q0-q3``.
-- ``t``: A low floating-point/SIMD register: ``s0-s31``, ``d0-d16``, or
-  ``q0-q8``.
+- ``w``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s31``, ``d0-d31``, or ``q0-q15``, respectively.
+- ``t``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s31``, ``d0-d15``, or ``q0-q7``, respectively.
+- ``x``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s15``, ``d0-d7``, or ``q0-q3``, respectively.
 
 ARM's Thumb1 mode:
 
@@ -3879,12 +3882,12 @@ ARM's Thumb1 mode:
 - ``r``: A low 32-bit GPR register (``r0-r7``).
 - ``l``: A low 32-bit GPR register (``r0-r7``).
 - ``h``: A high GPR register (``r0-r7``).
-- ``w``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s31``,
-  ``d0-d31``, or ``q0-q15``.
-- ``x``: A 32, 64, or 128-bit floating-point/SIMD register: ``s0-s15``,
-  ``d0-d7``, or ``q0-q3``.
-- ``t``: A low floating-point/SIMD register: ``s0-s31``, ``d0-d16``, or
-  ``q0-q8``.
+- ``w``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s31``, ``d0-d31``, or ``q0-q15``, respectively.
+- ``t``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s31``, ``d0-d15``, or ``q0-q7``, respectively.
+- ``x``: A 32, 64, or 128-bit floating-point/SIMD register in the ranges
+  ``s0-s15``, ``d0-d7``, or ``q0-q3``, respectively.
 
 
 Hexagon:
@@ -7068,7 +7071,7 @@ Syntax:
 ::
 
       <result> = callbr [cconv] [ret attrs] [addrspace(<num>)] <ty>|<fnty> <fnptrval>(<function args>) [fn attrs]
-                    [operand bundles] to label <normal label> or jump [other labels]
+                    [operand bundles] to label <normal label> [other labels]
 
 Overview:
 """""""""
@@ -7112,7 +7115,8 @@ This instruction requires several arguments:
 #. '``normal label``': the label reached when the called function
    executes a '``ret``' instruction.
 #. '``other labels``': the labels reached when a callee transfers control
-   to a location other than the normal '``normal label``'
+   to a location other than the normal '``normal label``'. The blockaddress
+   constant for these should also be in the list of '``function args``'.
 #. The optional :ref:`function attributes <fnattrs>` list.
 #. The optional :ref:`operand bundles <opbundles>` list.
 
@@ -7134,7 +7138,7 @@ Example:
 .. code-block:: text
 
       callbr void asm "", "r,x"(i32 %x, i8 *blockaddress(@foo, %fail))
-                  to label %normal or jump [label %fail]
+                  to label %normal [label %fail]
 
 .. _i_resume:
 
@@ -10065,7 +10069,7 @@ Syntax:
 
 ::
 
-      <result> = phi <ty> [ <val0>, <label0>], ...
+      <result> = phi [fast-math-flags] <ty> [ <val0>, <label0>], ...
 
 Overview:
 """""""""
@@ -10092,6 +10096,12 @@ deemed to occur on the edge from the corresponding predecessor block to
 the current block (but after any definition of an '``invoke``'
 instruction's return value on the same edge).
 
+The optional ``fast-math-flags`` marker indicates that the phi has one
+or more :ref:`fast-math-flags <fastmath>`. These are optimization hints
+to enable otherwise unsafe floating-point optimizations. Fast-math-flags
+are only valid for phis that return a floating-point scalar or vector
+type.
+
 Semantics:
 """"""""""
 
@@ -13951,12 +13961,12 @@ The expression:
 
       %0 = call float @llvm.fmuladd.f32(%a, %b, %c)
 
-is equivalent to the expression a \* b + c, except that rounding will
-not be performed between the multiplication and addition steps if the
-code generator fuses the operations. Fusion is not guaranteed, even if
-the target platform supports it. If a fused multiply-add is required, the
-corresponding :ref:`llvm.fma <int_fma>` intrinsic function should be used
-instead. This never sets errno, just as '``llvm.fma.*``'.
+is equivalent to the expression a \* b + c, except that it is unspecified
+whether rounding will be performed between the multiplication and addition
+steps. Fusion is not guaranteed, even if the target platform supports it.
+If a fused multiply-add is required, the corresponding
+:ref:`llvm.fma <int_fma>` intrinsic function should be used instead.
+This never sets errno, just as '``llvm.fma.*``'.
 
 Examples:
 """""""""
@@ -15064,12 +15074,21 @@ Constrained FP intrinsics are used to support non-default rounding modes and
 accurately preserve exception behavior without compromising LLVM's ability to
 optimize FP code when the default behavior is used.
 
-Each of these intrinsics corresponds to a normal floating-point operation.  The
-first two arguments and the return value are the same as the corresponding FP
+If any FP operation in a function is constrained then they all must be
+constrained. This is required for correct LLVM IR. Optimizations that
+move code around can create miscompiles if mixing of constrained and normal
+operations is done. The correct way to mix constrained and less constrained
+operations is to use the rounding mode and exception handling metadata to
+mark constrained intrinsics as having LLVM's default behavior.
+
+Each of these intrinsics corresponds to a normal floating-point operation. The
+data arguments and the return value are the same as the corresponding FP
 operation.
 
-The third argument is a metadata argument specifying the rounding mode to be
-assumed. This argument must be one of the following strings:
+The rounding mode argument is a metadata string specifying what 
+assumptions, if any, the optimizer can make when transforming constant 
+values. Some constrained FP intrinsics omit this argument. If required 
+by the intrinsic, this argument must be one of the following strings:
 
 ::
 
@@ -15099,9 +15118,9 @@ the specified rounding mode, but this is not guaranteed.  Using a specific
 non-dynamic rounding mode which does not match the actual rounding mode at
 runtime results in undefined behavior.
 
-The fourth argument to the constrained floating-point intrinsics specifies the
-required exception behavior.  This argument must be one of the following
-strings:
+The exception behavior argument is a metadata string describing the floating
+point exception semantics that required for the intrinsic. This argument
+must be one of the following strings:
 
 ::
 
@@ -15137,6 +15156,14 @@ example, a series of FP operations that each may raise exceptions may be
 vectorized into a single instruction that raises each unique exception a single
 time.
 
+Proper :ref:`function attributes <fnattrs>` usage is required for the
+constrained intrinsics to function correctly.
+
+All function *calls* done in a function that uses constrained floating
+point intrinsics must have the ``strictfp`` attribute.
+
+All function *definitions* that use constrained floating point intrinsics
+must have the ``strictfp`` attribute.
 
 '``llvm.experimental.constrained.fadd``' Intrinsic
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/llvm/docs/LibFuzzer.rst b/llvm/docs/LibFuzzer.rst
index 49f72f63ee3eb..e38489014e69e 100644
--- a/llvm/docs/LibFuzzer.rst
+++ b/llvm/docs/LibFuzzer.rst
@@ -590,7 +590,7 @@ User-supplied mutators
 ----------------------
 
 LibFuzzer allows to use custom (user-supplied) mutators, see
-`Structure-Aware Fuzzing <https://github.com/google/fuzzer-test-suite/blob/master/tutorial/structure-aware-fuzzing.md>`_
+`Structure-Aware Fuzzing <https://github.com/google/fuzzing/blob/master/docs/structure-aware-fuzzing.md>`_
 for more details.
 
 Startup initialization
diff --git a/llvm/docs/ProgrammingDocumentation.rst b/llvm/docs/ProgrammingDocumentation.rst
index a51150898534d..6a4d7aa25ba78 100644
--- a/llvm/docs/ProgrammingDocumentation.rst
+++ b/llvm/docs/ProgrammingDocumentation.rst
@@ -7,9 +7,8 @@ For developers of applications which use LLVM as a library.
    :hidden:
 
    Atomics
-   CodingStandards
    CommandLine
-   CompilerWriterInfo
+   CommandGuide/index
    ExtendingLLVM
    HowToSetUpLLVMStyleRTTI
    ProgrammersManual
@@ -20,10 +19,6 @@ For developers of applications which use LLVM as a library.
    OptBisect
    GwpAsan
 
-:doc:`LLVM Language Reference Manual <LangRef>`
-  Defines the LLVM intermediate representation and the assembly form of the
-  different nodes.
-
 :doc:`Atomics`
   Information about LLVM's concurrency model.
 
@@ -34,13 +29,6 @@ For developers of applications which use LLVM as a library.
 :doc:`Extensions`
   LLVM-specific extensions to tools and formats LLVM seeks compatibility with.
 
-:doc:`CommandLine`
-  Provides information on using the command line parsing library.
-
-:doc:`CodingStandards`
-  Details the LLVM coding standards and provides useful information on writing
-  efficient C++ code.
-
 :doc:`HowToSetUpLLVMStyleRTTI`
   How to make ``isa<>``, ``dyn_cast<>``, etc. available for clients of your
   class hierarchy.
@@ -48,28 +36,32 @@ For developers of applications which use LLVM as a library.
 :doc:`ExtendingLLVM`
   Look here to see how to add instructions and intrinsics to LLVM.
 
-`Doxygen generated documentation <http://llvm.org/doxygen/>`_
-  (`classes <http://llvm.org/doxygen/inherits.html>`_)
+:doc:`ScudoHardenedAllocator`
+  A library that implements a security-hardened `malloc()`.
 
-`Documentation for Go bindings <http://godoc.org/llvm.org/llvm/bindings/go/llvm>`_
+:doc:`GwpAsan`
+  A sampled heap memory error detection toolkit designed for production use.
 
-`Github Source Repository Browser <http://github.com/llvm/llvm-project//>`_
-   ..
+============
+Command Line
+============
 
-:doc:`CompilerWriterInfo`
-  A list of helpful links for compiler writers.
+:doc:`CommandLine`
+  Provides information on using the command line parsing library.
 
-:doc:`LibFuzzer`
-  A library for writing in-process guided fuzzers.
+:doc:`OptBisect`
+  A command line option for debugging optimization-induced failures.
 
-:doc:`FuzzingLLVM`
-  Information on writing and using Fuzzers to find bugs in LLVM.
+:doc:`LLVM Command Guide <CommandGuide/index>`
+   A reference manual for the LLVM command line utilities ("man" pages for LLVM
+   tools).
 
-:doc:`ScudoHardenedAllocator`
-  A library that implements a security-hardened `malloc()`.
+=========
+LibFuzzer
+=========
 
-:doc:`OptBisect`
-  A command line option for debugging optimization-induced failures.
+:doc:`LibFuzzer`
+  A library for writing in-process guided fuzzers.
 
-:doc:`GwpAsan`
-  A sampled heap memory error detection toolkit designed for production use.
\ No newline at end of file
+:doc:`FuzzingLLVM`
+  Information on writing and using Fuzzers to find bugs in LLVM.
\ No newline at end of file
diff --git a/llvm/docs/Reference.rst b/llvm/docs/Reference.rst
new file mode 100644
index 0000000000000..49ff6b6e0e2db
--- /dev/null
+++ b/llvm/docs/Reference.rst
@@ -0,0 +1,47 @@
+Reference
+=========
+
+LLVM and API reference documentation.
+
+.. toctree::
+   :hidden:
+
+   LangRef
+   TestingGuide
+   CompilerWriterInfo
+   MIRLangRef
+   NVPTXUsage
+   AMDGPUUsage
+
+==============
+LLVM Reference
+==============
+
+:doc:`LLVM Language Reference Manual <LangRef>`
+  Defines the LLVM intermediate representation and the assembly form of the
+  different nodes.
+
+:doc:`LLVM Testing Infrastructure Guide <TestingGuide>`
+   A reference manual for using the LLVM testing infrastructure.
+
+:doc:`CompilerWriterInfo`
+  A list of helpful links for compiler writers.
+
+:doc:`Machine IR (MIR) Format Reference Manual <MIRLangRef>`
+   A reference manual for the MIR serialization format, which is used to test
+   LLVM's code generation passes.
+
+:doc:`NVPTXUsage`
+   This document describes using the NVPTX backend to compile GPU kernels.
+
+:doc:`AMDGPUUsage`
+   This document describes using the AMDGPU backend to compile GPU kernels.
+
+=============
+API Reference
+=============
+
+`Doxygen generated documentation <http://llvm.org/doxygen/>`_
+  (`classes <http://llvm.org/doxygen/inherits.html>`_)
+
+`Documentation for Go bindings <http://godoc.org/llvm.org/llvm/bindings/go/llvm>`_
diff --git a/llvm/docs/ReferenceDocumentation.rst b/llvm/docs/ReferenceDocumentation.rst
deleted file mode 100644
index d72c4978da894..0000000000000
--- a/llvm/docs/ReferenceDocumentation.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-Reference Documentation
-========================
-
-LLVM and API reference documentation.
-
-.. toctree::
-   :hidden:
-
-   CommandGuide/index
-   TestingGuide
-
-:doc:`LLVM Language Reference Manual <LangRef>`
-  Defines the LLVM intermediate representation and the assembly form of the
-  different nodes.
-
-:doc:`LLVM Command Guide <CommandGuide/index>`
-   A reference manual for the LLVM command line utilities ("man" pages for LLVM
-   tools).
-
-:doc:`LLVM Testing Infrastructure Guide <TestingGuide>`
-   A reference manual for using the LLVM testing infrastructure.
-
-`Doxygen generated documentation <http://llvm.org/doxygen/>`_
-  (`classes <http://llvm.org/doxygen/inherits.html>`_)
-
-`Documentation for Go bindings <http://godoc.org/llvm.org/llvm/bindings/go/llvm>`_
-
-`Github Source Repository Browser <http://github.com/llvm/llvm-project//>`_
-   ..
-
diff --git a/llvm/docs/ReleaseProcess.rst b/llvm/docs/ReleaseProcess.rst
index 5822360cd1dfd..0cd455fdb79a6 100644
--- a/llvm/docs/ReleaseProcess.rst
+++ b/llvm/docs/ReleaseProcess.rst
@@ -90,6 +90,9 @@ The important flags to get right are:
 * On the final candidate you just need Release builds, and that's the binary
   directory you'll have to pack.
 
+* On macOS, you must export ``MACOSX_DEPLOYMENT_TARGET=10.9`` before running
+  the script.
+
 This script builds three phases of Clang+LLVM twice each (Release and
 Release+Asserts), so use screen or nohup to avoid headaches, since it'll take
 a long time.
diff --git a/llvm/docs/SubsystemDocumentation.rst b/llvm/docs/SubsystemDocumentation.rst
index 005d541ceb38c..69764cbd2822a 100644
--- a/llvm/docs/SubsystemDocumentation.rst
+++ b/llvm/docs/SubsystemDocumentation.rst
@@ -14,7 +14,6 @@ For API clients and LLVM developers.
    BlockFrequencyTerminology
    BranchWeightMetadata
    Bugpoint
-   BugpointRedesign
    CodeGenerator
    ExceptionHandling
    AddingConstrainedIntrinsics
@@ -33,8 +32,6 @@ For API clients and LLVM developers.
    GarbageCollection
    WritingAnLLVMPass
    HowToUseAttributes
-   NVPTXUsage
-   AMDGPUUsage
    StackMaps
    InAlloca
    BigEndianNEON
@@ -44,7 +41,6 @@ For API clients and LLVM developers.
    TypeMetadata
    TransformMetadata
    FaultMaps
-   MIRLangRef
    Coroutines
    GlobalISel
    XRay
@@ -55,6 +51,7 @@ For API clients and LLVM developers.
    SpeculativeLoadHardening
    StackSafetyAnalysis
    LoopTerminology
+   DependenceGraphs/index
 
 :doc:`WritingAnLLVMPass`
    Information on how to write LLVM transformations and analyses.
@@ -67,10 +64,6 @@ For API clients and LLVM developers.
    working on retargetting LLVM to a new architecture, designing a new codegen
    pass, or enhancing existing components.
 
-:doc:`Machine IR (MIR) Format Reference Manual <MIRLangRef>`
-   A reference manual for the MIR serialization format, which is used to test
-   LLVM's code generation passes.
-
 :doc:`TableGen <TableGen/index>`
    Describes the TableGen tool, which is used heavily by the LLVM code
    generator.
@@ -82,10 +75,6 @@ For API clients and LLVM developers.
 :doc:`MemorySSA`
    Information about the MemorySSA utility in LLVM, as well as how to use it.
 
-:doc:`GarbageCollection`
-   The interfaces source-language compilers should use for compiling GC'd
-   programs.
-
 :doc:`Source Level Debugging with LLVM <SourceLevelDebugging>`
    This document describes the design and philosophy behind the LLVM
    source-level debugger.
@@ -105,9 +94,6 @@ For API clients and LLVM developers.
    Automatic bug finder and test-case reducer description and usage
    information.
 
-:doc:`BugpointRedesign`
-   Design doc for a redesign of the Bugpoint tool.
-
 :doc:`BitCodeFormat`
    This describes the file format and encoding used for LLVM "bc" files.
 
@@ -148,12 +134,6 @@ For API clients and LLVM developers.
 :doc:`HowToUseAttributes`
   Answers some questions about the new Attributes infrastructure.
 
-:doc:`NVPTXUsage`
-   This document describes using the NVPTX backend to compile GPU kernels.
-
-:doc:`AMDGPUUsage`
-   This document describes using the AMDGPU backend to compile GPU kernels.
-
 :doc:`StackMaps`
   LLVM support for mapping instruction addresses to the location of
   values and allowing code to be patched.
@@ -165,10 +145,6 @@ For API clients and LLVM developers.
 :doc:`CoverageMappingFormat`
   This describes the format and encoding used for LLVM’s code coverage mapping.
 
-:doc:`Statepoints`
-  This describes a set of experimental extensions for garbage
-  collection support.
-
 :doc:`MergeFunctions`
   Describes functions merging optimization.
 
@@ -207,4 +183,20 @@ For API clients and LLVM developers.
   variables.
 
 :doc:`LoopTerminology`
-  A document describing Loops and associated terms as used in LLVM.
\ No newline at end of file
+  A document describing Loops and associated terms as used in LLVM.
+
+:doc:`Dependence Graphs <DependenceGraphs/index>`
+  A description of the design of the various dependence graphs such as
+  the DDG (Data Dependence Graph).
+
+==================
+Garbage Collection
+==================
+
+:doc:`GarbageCollection`
+   The interfaces source-language compilers should use for compiling GC'd
+   programs.
+
+:doc:`Statepoints`
+  This describes a set of experimental extensions for garbage
+  collection support.
diff --git a/llvm/docs/UserGuides.rst b/llvm/docs/UserGuides.rst
index 26ce6f927e2d7..d75769a793899 100644
--- a/llvm/docs/UserGuides.rst
+++ b/llvm/docs/UserGuides.rst
@@ -18,16 +18,9 @@ intermediate LLVM representation.
    HowToBuildWithPGO
    HowToCrossCompileBuiltinsOnArm
    HowToCrossCompileLLVM
-   GettingStarted
-   GettingStartedVS
-   FAQ
-   Lexicon
-   HowToAddABuilder
    yaml2obj
    MarkdownQuickstartTemplate
    Phabricator
-   tutorial/index
-   ReleaseNotes
    Passes
    YamlIO
    GetElementPtr
@@ -42,57 +35,60 @@ intermediate LLVM representation.
    BuildingADistribution
    Remarks
 
-:doc:`GettingStarted`
-   Discusses how to get up and running quickly with the LLVM infrastructure.
-   Everything from unpacking and compilation of the distribution to execution
-   of some tools.
+Building, Packaging, and Distributing LLVM
+------------------------------------------
+
+How to build, package, and distribute LLVM.
+
+=====
+CMake
+=====
+
+:doc:`BuildingADistribution`
+  A best-practices guide for using LLVM's CMake build system to package and
+  distribute LLVM-based tools.
 
 :doc:`CMake`
    An addendum to the main Getting Started guide for those using the `CMake
    build system <http://www.cmake.org>`_.
 
+=====
+Clang
+=====
+
 :doc:`HowToBuildOnARM`
    Notes on building and testing LLVM/Clang on ARM.
 
 :doc:`HowToBuildWithPGO`
     Notes on building LLVM/Clang with PGO.
 
-:doc:`HowToCrossCompileBuiltinsOnArm`
-   Notes on cross-building and testing the compiler-rt builtins for Arm.
-
 :doc:`HowToCrossCompileLLVM`
    Notes on cross-building and testing LLVM/Clang.
 
-:doc:`GettingStartedVS`
-   An addendum to the main Getting Started guide for those using Visual Studio
-   on Windows.
-
-:doc:`tutorial/index`
-   Tutorials about using LLVM. Includes a tutorial about making a custom
-   language with LLVM.
-
-:doc:`Passes`
-   A list of optimizations and analyses implemented in LLVM.
+`How to build the C, C++, ObjC, and ObjC++ front end`__
+   Instructions for building the clang front-end from source.
 
-:doc:`FAQ`
-   A list of common questions and problems and their solutions.
+   .. __: http://clang.llvm.org/get_started.html
 
-:doc:`Release notes for the current release <ReleaseNotes>`
-   This describes new features, known bugs, and other limitations.
+======
+Docker
+======
 
-:doc:`TestSuiteGuide`
-  Describes how to compile and run the test-suite benchmarks.
+:doc:`Docker`
+   A reference for using Dockerfiles provided with LLVM.
 
-`How to build the C, C++, ObjC, and ObjC++ front end`__
-   Instructions for building the clang front-end from source.
+=================
+Additional Topics
+=================
 
-   .. __: http://clang.llvm.org/get_started.html
+:doc:`HowToCrossCompileBuiltinsOnArm`
+   Notes on cross-building and testing the compiler-rt builtins for Arm.
 
-:doc:`Lexicon`
-   Definition of acronyms, terms and concepts used in LLVM.
+:doc:`Passes`
+   A list of optimizations and analyses implemented in LLVM.
 
-:doc:`HowToAddABuilder`
-   Instructions for adding new builder to LLVM buildbot master.
+:doc:`TestSuiteGuide`
+  Describes how to compile and run the test-suite benchmarks.
 
 :doc:`YamlIO`
    A reference guide for using LLVM's YAML I/O library.
@@ -105,12 +101,5 @@ intermediate LLVM representation.
    A collection of tips for frontend authors on how to generate IR
    which LLVM is able to effectively optimize.
 
-:doc:`Docker`
-   A reference for using Dockerfiles provided with LLVM.
-
-:doc:`BuildingADistribution`
-  A best-practices guide for using LLVM's CMake build system to package and
-  distribute LLVM-based tools.
-
 :doc:`Remarks`
    A reference on the implementation of remarks in LLVM.
\ No newline at end of file
diff --git a/llvm/docs/_templates/indexsidebar.html b/llvm/docs/_templates/indexsidebar.html
index 416174279ce48..1a7b7df3ef6ef 100644
--- a/llvm/docs/_templates/indexsidebar.html
+++ b/llvm/docs/_templates/indexsidebar.html
@@ -1,7 +1,21 @@
 {# This template defines sidebar which can be used to provide common links on
    all documentation pages. #}
 
-<h3>Bugs</h3>
+<h3>Getting Involved</h3>
 
-<p>LLVM bugs should be reported to
-  <a href="http://llvm.org/bugs">Bugzilla</a>.</p>
+<ul class="want-points">
+    <li><a href="https://llvm.org/docs/Contributing.html">Contributing to LLVM</a></li>
+    <li><a href="https://llvm.org/docs/HowToSubmitABug.html">How to submit a bug</a></li>
+    <li><a href="https://llvm.org/docs/GettingInvolved.html#mailing-lists">Mailing Lists</a></li>
+    <li><a href="https://llvm.org/docs/GettingInvolved.html#irc">IRC</a></li>
+    <li><a href="https://llvm.org/docs/GettingInvolved.html#meetups-and-social-events">Meetups and social events</a></li>
+</ul>
+
+<h3>Additional Links</h3>
+
+<ul class="want-points">
+    <li><a href="https://llvm.org/docs/FAQ.html">FAQ</a></li>
+    <li><a href="https://llvm.org/docs/Lexicon.html">Glossary</a></li>
+    <li><a href="https://llvm.org/pubs">Publications</a></li>
+    <li><a href="https://github.com/llvm/llvm-project//">Github Repository</a></li>
+</ul>
\ No newline at end of file
diff --git a/llvm/docs/conf.py b/llvm/docs/conf.py
index a0357cf3cb8ac..c92ede3ea44b6 100644
--- a/llvm/docs/conf.py
+++ b/llvm/docs/conf.py
@@ -99,7 +99,7 @@
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-html_theme_options = { "nosidebar": True }
+html_theme_options = { "nosidebar": False }
 
 # Add any paths that contain custom themes here, relative to this directory.
 html_theme_path = ["_themes"]
@@ -134,7 +134,14 @@
 #html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-html_sidebars = {'index': 'indexsidebar.html'}
+
+html_sidebars = {
+    '**': [
+        'indexsidebar.html',
+        'sourcelink.html',
+        'searchbox.html',
+    ]
+}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
diff --git a/llvm/docs/index.rst b/llvm/docs/index.rst
index 944ca8bdfdba2..17a0706a196bd 100644
--- a/llvm/docs/index.rst
+++ b/llvm/docs/index.rst
@@ -21,10 +21,8 @@ Several introductory papers and presentations.
 .. toctree::
    :hidden:
 
-   LangRef
-
-:doc:`LangRef`
-  Defines the LLVM intermediate representation.
+   FAQ
+   Lexicon
 
 `Introduction to the LLVM Compiler`__
   Presentation providing a users introduction to LLVM.
@@ -47,9 +45,6 @@ Several introductory papers and presentations.
 
   .. __: http://llvm.org/pubs/2002-12-LattnerMSThesis.html
 
-`Publications mentioning LLVM <http://llvm.org/pubs>`_
-   ..
-
 Documentation
 =============
 
@@ -58,9 +53,10 @@ Getting Started, How-tos, Developer Guides, and Tutorials.
 .. toctree::
    :hidden:
 
-   Programming Documentation
-   Subsystem Documentation
-   User Guides
+   UserGuides
+   ProgrammingDocumentation
+   Reference
+   SubsystemDocumentation
 
 :doc:`UserGuides`
   For those new to the LLVM system.
@@ -71,194 +67,47 @@ Getting Started, How-tos, Developer Guides, and Tutorials.
 :doc:`SubsystemDocumentation`
   For API clients and LLVM developers.
 
-:doc:`ReferenceDocumentation`
+:doc:`Reference`
   LLVM and API reference documentation.
 
-Community
-=========
-
-LLVM has a thriving community of friendly and helpful developers.
-The two primary communication mechanisms in the LLVM community are mailing
-lists and IRC.
-
-Getting Involved
-----------------
-
-LLVM welcomes contributions of all kinds. To get started, please review the following topics:
-
-.. toctree::
-   :hidden:
-
-   Contributing
-   DeveloperPolicy
-   SphinxQuickstartTemplate
-   Phabricator
-   HowToSubmitABug
-   BugLifeCycle
-
-:doc:`Contributing`
-   An overview on how to contribute to LLVM.
-
-:doc:`DeveloperPolicy`
-   The LLVM project's policy towards developers and their contributions.
-
-:doc:`SphinxQuickstartTemplate`
-  A template + tutorial for writing new Sphinx documentation. It is meant
-  to be read in source form.
-
-:doc:`Phabricator`
-   Describes how to use the Phabricator code review tool hosted on
-   http://reviews.llvm.org/ and its command line interface, Arcanist.
-
-:doc:`HowToSubmitABug`
-   Instructions for properly submitting information about any bugs you run into
-   in the LLVM system.
-
-:doc:`BugLifeCycle`
-   Describes how bugs are reported, triaged and closed.
-
-Development Process
--------------------
-
-Information about LLVM's development process.
-
-.. toctree::
-   :hidden:
-
-   Projects
-   LLVMBuild
-   HowToReleaseLLVM
-   Packaging
-   ReleaseProcess
-
-:doc:`Projects`
-  How-to guide and templates for new projects that *use* the LLVM
-  infrastructure.  The templates (directory organization, Makefiles, and test
-  tree) allow the project code to be located outside (or inside) the ``llvm/``
-  tree, while using LLVM header files and libraries.
-
-:doc:`LLVMBuild`
-  Describes the LLVMBuild organization and files used by LLVM to specify
-  component descriptions.
-
-:doc:`HowToReleaseLLVM`
-  This is a guide to preparing LLVM releases. Most developers can ignore it.
-
-:doc:`ReleaseProcess`
-  This is a guide to validate a new release, during the release process. Most developers can ignore it.
-
-:doc:`Packaging`
-   Advice on packaging LLVM into a distribution.
-
-Mailing Lists
--------------
-
-If you can't find what you need in these docs, try consulting the mailing
-lists.
-
-`Developer's List (llvm-dev)`__
-  This list is for people who want to be included in technical discussions of
-  LLVM. People post to this list when they have questions about writing code
-  for or using the LLVM tools. It is relatively low volume.
-
-  .. __: http://lists.llvm.org/mailman/listinfo/llvm-dev
-
-`Commits Archive (llvm-commits)`__
-  This list contains all commit messages that are made when LLVM developers
-  commit code changes to the repository. It also serves as a forum for
-  patch review (i.e. send patches here). It is useful for those who want to
-  stay on the bleeding edge of LLVM development. This list is very high
-  volume.
-
-  .. __: http://lists.llvm.org/pipermail/llvm-commits/
-
-`Bugs & Patches Archive (llvm-bugs)`__
-  This list gets emailed every time a bug is opened and closed. It is
-  higher volume than the LLVM-dev list.
-
-  .. __: http://lists.llvm.org/pipermail/llvm-bugs/
-
-`Test Results Archive (llvm-testresults)`__
-  A message is automatically sent to this list by every active nightly tester
-  when it completes.  As such, this list gets email several times each day,
-  making it a high volume list.
-
-  .. __: http://lists.llvm.org/pipermail/llvm-testresults/
-
-`LLVM Announcements List (llvm-announce)`__
-  This is a low volume list that provides important announcements regarding
-  LLVM.  It gets email about once a month.
-
-  .. __: http://lists.llvm.org/mailman/listinfo/llvm-announce
-
-IRC
----
-
-Users and developers of the LLVM project (including subprojects such as Clang)
-can be found in #llvm on `irc.oftc.net <irc://irc.oftc.net/llvm>`_.
-
-This channel has several bots.
-
-* Buildbot reporters
-
-  * llvmbb - Bot for the main LLVM buildbot master.
-    http://lab.llvm.org:8011/console
-  * smooshlab - Apple's internal buildbot master.
-
-* robot - Bugzilla linker. %bug <number>
-
-* clang-bot - A `geordi <http://www.eelis.net/geordi/>`_ instance running
-  near-trunk clang instead of gcc.
-
-Meetups and social events
+Getting Started/Tutorials
 -------------------------
 
 .. toctree::
    :hidden:
 
-   MeetupGuidelines
+   GettingStarted
+   tutorial/index
+   GettingStartedVS
+
+:doc:`GettingStarted`
+   Discusses how to get up and running quickly with the LLVM infrastructure.
+   Everything from unpacking and compilation of the distribution to execution
+   of some tools.
 
-Besides developer `meetings and conferences <https://llvm.org/devmtg/>`_,
-there are several user groups called
-`LLVM Socials <https://www.meetup.com/pro/llvm/>`_. We greatly encourage you to
-join one in your city. Or start a new one if there is none:
+:doc:`tutorial/index`
+   Tutorials about using LLVM. Includes a tutorial about making a custom
+   language with LLVM.
 
-:doc:`MeetupGuidelines`
+:doc:`GettingStartedVS`
+   An addendum to the main Getting Started guide for those using Visual Studio
+   on Windows.
 
-Community wide proposals
-------------------------
+Community
+=========
 
-Proposals for massive changes in how the community behaves and how the work flow
-can be better.
+LLVM welcomes contributions of all kinds. To learn more, see the following articles:
 
 .. toctree::
    :hidden:
 
-   CodeOfConduct
-   Proposals/GitHubMove
-   Proposals/LLVMLibC
-   Proposals/TestSuite
-   Proposals/VariableNames
-   Proposals/VectorizationPlan
-
-:doc:`CodeOfConduct`
-   Proposal to adopt a code of conduct on the LLVM social spaces (lists, events,
-   IRC, etc).
-
-:doc:`Proposals/GitHubMove`
-   Proposal to move from SVN/Git to GitHub.
-
-:doc:`Proposals/LLVMLibC`
-   Proposal to add a libc implementation under the LLVM project.
-
-:doc:`Proposals/TestSuite`
-   Proposals for additional benchmarks/programs for llvm's test-suite.
-
-:doc:`Proposals/VariableNames`
-   Proposal to change the variable names coding standard.
+   GettingInvolved
 
-:doc:`Proposals/VectorizationPlan`
-   Proposal to model the process and upgrade the infrastructure of LLVM's Loop Vectorizer.
+* :doc:`GettingInvolved`
+* :ref:`development-process`
+* :ref:`mailing-lists`
+* :ref:`meetups-social-events`
+* :ref:`community-proposals`
 
 Indices and tables
 ==================
diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h
index cac2f297056dc..815541c65d8cc 100644
--- a/llvm/include/llvm-c/Core.h
+++ b/llvm/include/llvm-c/Core.h
@@ -370,9 +370,13 @@ typedef enum {
     LLVMAtomicRMWBinOpUMax, /**< Sets the value if it's greater than the
                              original using an unsigned comparison and return
                              the old one */
-    LLVMAtomicRMWBinOpUMin /**< Sets the value if it's greater than the
-                             original using an unsigned comparison  and return
-                             the old one */
+    LLVMAtomicRMWBinOpUMin, /**< Sets the value if it's greater than the
+                              original using an unsigned comparison and return
+                              the old one */
+    LLVMAtomicRMWBinOpFAdd, /**< Add a floating point value and return the
+                              old one */
+    LLVMAtomicRMWBinOpFSub /**< Subtract a floating point value and return the
+                             old one */
 } LLVMAtomicRMWBinOp;
 
 typedef enum {
@@ -1571,6 +1575,8 @@ LLVMTypeRef LLVMX86MMXType(void);
       macro(ResumeInst)                     \
       macro(CleanupReturnInst)              \
       macro(CatchReturnInst)                \
+      macro(CatchSwitchInst)                \
+      macro(CallBrInst)                     \
       macro(FuncletPadInst)                 \
         macro(CatchPadInst)                 \
         macro(CleanupPadInst)               \
@@ -1592,7 +1598,10 @@ LLVMTypeRef LLVMX86MMXType(void);
           macro(ZExtInst)                   \
         macro(ExtractValueInst)             \
         macro(LoadInst)                     \
-        macro(VAArgInst)
+        macro(VAArgInst)                    \
+      macro(AtomicCmpXchgInst)              \
+      macro(AtomicRMWInst)                  \
+      macro(FenceInst)
 
 /**
  * @defgroup LLVMCCoreValueGeneral General APIs
@@ -3807,8 +3816,12 @@ LLVMValueRef LLVMBuildGlobalStringPtr(LLVMBuilderRef B, const char *Str,
                                       const char *Name);
 LLVMBool LLVMGetVolatile(LLVMValueRef MemoryAccessInst);
 void LLVMSetVolatile(LLVMValueRef MemoryAccessInst, LLVMBool IsVolatile);
+LLVMBool LLVMGetWeak(LLVMValueRef CmpXchgInst);
+void LLVMSetWeak(LLVMValueRef CmpXchgInst, LLVMBool IsWeak);
 LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemoryAccessInst);
 void LLVMSetOrdering(LLVMValueRef MemoryAccessInst, LLVMAtomicOrdering Ordering);
+LLVMAtomicRMWBinOp LLVMGetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst);
+void LLVMSetAtomicRMWBinOp(LLVMValueRef AtomicRMWInst, LLVMAtomicRMWBinOp BinOp);
 
 /* Casts */
 LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef, LLVMValueRef Val,
diff --git a/llvm/include/llvm-c/DebugInfo.h b/llvm/include/llvm-c/DebugInfo.h
index 33c8110a863c0..c96faa2fbba4e 100644
--- a/llvm/include/llvm-c/DebugInfo.h
+++ b/llvm/include/llvm-c/DebugInfo.h
@@ -32,7 +32,7 @@ typedef enum {
   LLVMDIFlagPublic = 3,
   LLVMDIFlagFwdDecl = 1 << 2,
   LLVMDIFlagAppleBlock = 1 << 3,
-  LLVMDIFlagBlockByrefStruct = 1 << 4,
+  LLVMDIFlagReservedBit4 = 1 << 4,
   LLVMDIFlagVirtual = 1 << 5,
   LLVMDIFlagArtificial = 1 << 6,
   LLVMDIFlagExplicit = 1 << 7,
diff --git a/llvm/include/llvm-c/Transforms/IPO.h b/llvm/include/llvm-c/Transforms/IPO.h
index 748b6db05e3c0..51d007581283d 100644
--- a/llvm/include/llvm-c/Transforms/IPO.h
+++ b/llvm/include/llvm-c/Transforms/IPO.h
@@ -34,6 +34,9 @@ void LLVMAddArgumentPromotionPass(LLVMPassManagerRef PM);
 /** See llvm::createConstantMergePass function. */
 void LLVMAddConstantMergePass(LLVMPassManagerRef PM);
 
+/** See llvm::createMergeFunctionsPass function. */
+void LLVMAddMergeFunctionsPass(LLVMPassManagerRef PM);
+
 /** See llvm::createCalledValuePropagationPass function. */
 void LLVMAddCalledValuePropagationPass(LLVMPassManagerRef PM);
 
diff --git a/llvm/include/llvm-c/Transforms/Scalar.h b/llvm/include/llvm-c/Transforms/Scalar.h
index 031cf98b2df26..484ac90ab5d59 100644
--- a/llvm/include/llvm-c/Transforms/Scalar.h
+++ b/llvm/include/llvm-c/Transforms/Scalar.h
@@ -35,6 +35,9 @@ extern "C" {
 /** See llvm::createAggressiveDCEPass function. */
 void LLVMAddAggressiveDCEPass(LLVMPassManagerRef PM);
 
+/** See llvm::createDeadCodeEliminationPass function. */
+void LLVMAddDCEPass(LLVMPassManagerRef PM);
+
 /** See llvm::createBitTrackingDCEPass function. */
 void LLVMAddBitTrackingDCEPass(LLVMPassManagerRef PM);
 
diff --git a/llvm/include/llvm-c/lto.h b/llvm/include/llvm-c/lto.h
index 2467722b19541..41e6067cf44f5 100644
--- a/llvm/include/llvm-c/lto.h
+++ b/llvm/include/llvm-c/lto.h
@@ -44,7 +44,7 @@ typedef bool lto_bool_t;
  * @{
  */
 
-#define LTO_API_VERSION 24
+#define LTO_API_VERSION 25
 
 /**
  * \since prior to LTO_API_VERSION=3
@@ -550,6 +550,56 @@ extern void
 lto_codegen_set_should_embed_uselists(lto_code_gen_t cg,
                                       lto_bool_t ShouldEmbedUselists);
 
+/** Opaque reference to an LTO input file */
+typedef struct LLVMOpaqueLTOInput *lto_input_t;
+
+/**
+  * Creates an LTO input file from a buffer. The path
+  * argument is used for diagnotics as this function
+  * otherwise does not know which file the given buffer
+  * is associated with.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern lto_input_t lto_input_create(const void *buffer,
+                                    size_t buffer_size,
+                                    const char *path);
+
+/**
+  * Frees all memory internally allocated by the LTO input file.
+  * Upon return the lto_module_t is no longer valid.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern void lto_input_dispose(lto_input_t input);
+
+/**
+  * Returns the number of dependent library specifiers
+  * for the given LTO input file.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input);
+
+/**
+  * Returns the ith dependent library specifier
+  * for the given LTO input file. The returned
+  * string is not null-terminated.
+  *
+  * \since LTO_API_VERSION=24
+  */
+extern const char * lto_input_get_dependent_library(lto_input_t input,
+                                                    size_t index,
+                                                    size_t *size);
+
+/**
+ * Returns the list of libcall symbols that can be generated by LTO
+ * that might not be visible from the symbol table of bitcode files.
+ *
+ * \since prior to LTO_API_VERSION=25
+ */
+extern const char *const *lto_runtime_lib_symbols_list(size_t *size);
+
 /**
  * @} // endgoup LLVMCLTO
  * @defgroup LLVMCTLTO ThinLTO
@@ -846,48 +896,6 @@ thinlto_codegen_set_cache_size_megabytes(thinlto_code_gen_t cg,
 extern void thinlto_codegen_set_cache_size_files(thinlto_code_gen_t cg,
                                                  unsigned max_size_files);
 
-/** Opaque reference to an LTO input file */
-typedef struct LLVMOpaqueLTOInput *lto_input_t;
-
-/**
-  * Creates an LTO input file from a buffer. The path
-  * argument is used for diagnotics as this function
-  * otherwise does not know which file the given buffer
-  * is associated with.
-  *
-  * \since LTO_API_VERSION=24
-  */
-extern lto_input_t lto_input_create(const void *buffer,
-                                    size_t buffer_size,
-                                    const char *path);
-
-/**
-  * Frees all memory internally allocated by the LTO input file.
-  * Upon return the lto_module_t is no longer valid.
-  *
-  * \since LTO_API_VERSION=24
-  */
-extern void lto_input_dispose(lto_input_t input);
-
-/**
-  * Returns the number of dependent library specifiers
-  * for the given LTO input file.
-  *
-  * \since LTO_API_VERSION=24
-  */
-extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input);
-
-/**
-  * Returns the ith dependent library specifier
-  * for the given LTO input file. The returned
-  * string is not null-terminated.
-  *
-  * \since LTO_API_VERSION=24
-  */
-extern const char * lto_input_get_dependent_library(lto_input_t input,
-                                                    size_t index,
-                                                    size_t *size);
-
 /**
  * @} // endgroup LLVMCTLTO_CACHING
  */
diff --git a/llvm/include/llvm/ADT/APInt.h b/llvm/include/llvm/ADT/APInt.h
index 987ca636cef7d..8dce5a621bb35 100644
--- a/llvm/include/llvm/ADT/APInt.h
+++ b/llvm/include/llvm/ADT/APInt.h
@@ -1503,9 +1503,11 @@ class LLVM_NODISCARD APInt {
 
   /// Insert the bits from a smaller APInt starting at bitPosition.
   void insertBits(const APInt &SubBits, unsigned bitPosition);
+  void insertBits(uint64_t SubBits, unsigned bitPosition, unsigned numBits);
 
   /// Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
   APInt extractBits(unsigned numBits, unsigned bitPosition) const;
+  uint64_t extractBitsAsZExtValue(unsigned numBits, unsigned bitPosition) const;
 
   /// @}
   /// \name Value Characterization Functions
diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index 8a586fc267096..51fa4844f307d 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -391,6 +391,16 @@ class StringMap : public StringMapImpl {
     return try_emplace(KV.first, std::move(KV.second));
   }
 
+  /// Inserts an element or assigns to the current element if the key already
+  /// exists. The return type is the same as try_emplace.
+  template <typename V>
+  std::pair<iterator, bool> insert_or_assign(StringRef Key, V &&Val) {
+    auto Ret = try_emplace(Key, std::forward<V>(Val));
+    if (!Ret.second)
+      Ret.first->second = std::forward<V>(Val);
+    return Ret;
+  }
+
   /// Emplace a new element for the specified key into the map if the key isn't
   /// already in the map. The bool component of the returned pair is true
   /// if and only if the insertion takes place, and the iterator component of
diff --git a/llvm/include/llvm/Analysis/DDG.h b/llvm/include/llvm/Analysis/DDG.h
new file mode 100644
index 0000000000000..0e1eb9d2cda35
--- /dev/null
+++ b/llvm/include/llvm/Analysis/DDG.h
@@ -0,0 +1,430 @@
+//===- llvm/Analysis/DDG.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the Data-Dependence Graph (DDG).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_DDG_H
+#define LLVM_ANALYSIS_DDG_H
+
+#include "llvm/ADT/DirectedGraph.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/DependenceGraphBuilder.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/IR/Instructions.h"
+#include <unordered_map>
+
+namespace llvm {
+class DDGNode;
+class DDGEdge;
+using DDGNodeBase = DGNode<DDGNode, DDGEdge>;
+using DDGEdgeBase = DGEdge<DDGNode, DDGEdge>;
+using DDGBase = DirectedGraph<DDGNode, DDGEdge>;
+class LPMUpdater;
+
+/// Data Dependence Graph Node
+/// The graph can represent the following types of nodes:
+/// 1. Single instruction node containing just one instruction.
+/// 2. Multiple instruction node where two or more instructions from
+///    the same basic block are merged into one node.
+/// 3. Root node is a special node that connects to all components such that
+///    there is always a path from it to any node in the graph.
+class DDGNode : public DDGNodeBase {
+public:
+  using InstructionListType = SmallVectorImpl<Instruction *>;
+
+  enum class NodeKind {
+    Unknown,
+    SingleInstruction,
+    MultiInstruction,
+    Root,
+  };
+
+  DDGNode() = delete;
+  DDGNode(const NodeKind K) : DDGNodeBase(), Kind(K) {}
+  DDGNode(const DDGNode &N) : DDGNodeBase(N), Kind(N.Kind) {}
+  DDGNode(DDGNode &&N) : DDGNodeBase(std::move(N)), Kind(N.Kind) {}
+  virtual ~DDGNode() = 0;
+
+  DDGNode &operator=(const DDGNode &N) {
+    DGNode::operator=(N);
+    Kind = N.Kind;
+    return *this;
+  }
+
+  DDGNode &operator=(DDGNode &&N) {
+    DGNode::operator=(std::move(N));
+    Kind = N.Kind;
+    return *this;
+  }
+
+  /// Getter for the kind of this node.
+  NodeKind getKind() const { return Kind; }
+
+  /// Collect a list of instructions, in \p IList, for which predicate \p Pred
+  /// evaluates to true when iterating over instructions of this node. Return
+  /// true if at least one instruction was collected, and false otherwise.
+  bool collectInstructions(llvm::function_ref<bool(Instruction *)> const &Pred,
+                           InstructionListType &IList) const;
+
+protected:
+  /// Setter for the kind of this node.
+  void setKind(NodeKind K) { Kind = K; }
+
+private:
+  NodeKind Kind;
+};
+
+/// Subclass of DDGNode representing the root node of the graph.
+/// There should only be one such node in a given graph.
+class RootDDGNode : public DDGNode {
+public:
+  RootDDGNode() : DDGNode(NodeKind::Root) {}
+  RootDDGNode(const RootDDGNode &N) = delete;
+  RootDDGNode(RootDDGNode &&N) : DDGNode(std::move(N)) {}
+  ~RootDDGNode() {}
+
+  /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc.
+  static bool classof(const DDGNode *N) {
+    return N->getKind() == NodeKind::Root;
+  }
+  static bool classof(const RootDDGNode *N) { return true; }
+};
+
+/// Subclass of DDGNode representing single or multi-instruction nodes.
+class SimpleDDGNode : public DDGNode {
+public:
+  SimpleDDGNode() = delete;
+  SimpleDDGNode(Instruction &I);
+  SimpleDDGNode(const SimpleDDGNode &N);
+  SimpleDDGNode(SimpleDDGNode &&N);
+  ~SimpleDDGNode();
+
+  SimpleDDGNode &operator=(const SimpleDDGNode &N) {
+    DDGNode::operator=(N);
+    InstList = N.InstList;
+    return *this;
+  }
+
+  SimpleDDGNode &operator=(SimpleDDGNode &&N) {
+    DDGNode::operator=(std::move(N));
+    InstList = std::move(N.InstList);
+    return *this;
+  }
+
+  /// Get the list of instructions in this node.
+  const InstructionListType &getInstructions() const {
+    assert(!InstList.empty() && "Instruction List is empty.");
+    return InstList;
+  }
+  InstructionListType &getInstructions() {
+    return const_cast<InstructionListType &>(
+        static_cast<const SimpleDDGNode *>(this)->getInstructions());
+  }
+
+  /// Get the first/last instruction in the node.
+  Instruction *getFirstInstruction() const { return getInstructions().front(); }
+  Instruction *getLastInstruction() const { return getInstructions().back(); }
+
+  /// Define classof to be able to use isa<>, cast<>, dyn_cast<>, etc.
+  static bool classof(const DDGNode *N) {
+    return N->getKind() == NodeKind::SingleInstruction ||
+           N->getKind() == NodeKind::MultiInstruction;
+  }
+  static bool classof(const SimpleDDGNode *N) { return true; }
+
+private:
+  /// Append the list of instructions in \p Input to this node.
+  void appendInstructions(const InstructionListType &Input) {
+    setKind((InstList.size() == 0 && Input.size() == 1)
+                ? NodeKind::SingleInstruction
+                : NodeKind::MultiInstruction);
+    InstList.insert(InstList.end(), Input.begin(), Input.end());
+  }
+  void appendInstructions(const SimpleDDGNode &Input) {
+    appendInstructions(Input.getInstructions());
+  }
+
+  /// List of instructions associated with a single or multi-instruction node.
+  SmallVector<Instruction *, 2> InstList;
+};
+
+/// Data Dependency Graph Edge.
+/// An edge in the DDG can represent a def-use relationship or
+/// a memory dependence based on the result of DependenceAnalysis.
+/// A rooted edge connects the root node to one of the components
+/// of the graph.
+class DDGEdge : public DDGEdgeBase {
+public:
+  /// The kind of edge in the DDG
+  enum class EdgeKind { Unknown, RegisterDefUse, MemoryDependence, Rooted };
+
+  explicit DDGEdge(DDGNode &N) = delete;
+  DDGEdge(DDGNode &N, EdgeKind K) : DDGEdgeBase(N), Kind(K) {}
+  DDGEdge(const DDGEdge &E) : DDGEdgeBase(E), Kind(E.getKind()) {}
+  DDGEdge(DDGEdge &&E) : DDGEdgeBase(std::move(E)), Kind(E.Kind) {}
+  DDGEdge &operator=(const DDGEdge &E) {
+    DDGEdgeBase::operator=(E);
+    Kind = E.Kind;
+    return *this;
+  }
+
+  DDGEdge &operator=(DDGEdge &&E) {
+    DDGEdgeBase::operator=(std::move(E));
+    Kind = E.Kind;
+    return *this;
+  }
+
+  /// Get the edge kind
+  EdgeKind getKind() const { return Kind; };
+
+  /// Return true if this is a def-use edge, and false otherwise.
+  bool isDefUse() const { return Kind == EdgeKind::RegisterDefUse; }
+
+  /// Return true if this is a memory dependence edge, and false otherwise.
+  bool isMemoryDependence() const { return Kind == EdgeKind::MemoryDependence; }
+
+  /// Return true if this is an edge stemming from the root node, and false
+  /// otherwise.
+  bool isRooted() const { return Kind == EdgeKind::Rooted; }
+
+private:
+  EdgeKind Kind;
+};
+
+/// Encapsulate some common data and functionality needed for different
+/// variations of data dependence graphs.
+template <typename NodeType> class DependenceGraphInfo {
+public:
+  using DependenceList = SmallVector<std::unique_ptr<Dependence>, 1>;
+
+  DependenceGraphInfo() = delete;
+  DependenceGraphInfo(const DependenceGraphInfo &G) = delete;
+  DependenceGraphInfo(const std::string &N, const DependenceInfo &DepInfo)
+      : Name(N), DI(DepInfo), Root(nullptr) {}
+  DependenceGraphInfo(DependenceGraphInfo &&G)
+      : Name(std::move(G.Name)), DI(std::move(G.DI)), Root(G.Root) {}
+  virtual ~DependenceGraphInfo() {}
+
+  /// Return the label that is used to name this graph.
+  const StringRef getName() const { return Name; }
+
+  /// Return the root node of the graph.
+  NodeType &getRoot() const {
+    assert(Root && "Root node is not available yet. Graph construction may "
+                   "still be in progress\n");
+    return *Root;
+  }
+
+protected:
+  // Name of the graph.
+  std::string Name;
+
+  // Store a copy of DependenceInfo in the graph, so that individual memory
+  // dependencies don't need to be stored. Instead when the dependence is
+  // queried it is recomputed using @DI.
+  const DependenceInfo DI;
+
+  // A special node in the graph that has an edge to every connected component of
+  // the graph, to ensure all nodes are reachable in a graph walk.
+  NodeType *Root = nullptr;
+};
+
+using DDGInfo = DependenceGraphInfo<DDGNode>;
+
+/// Data Dependency Graph
+class DataDependenceGraph : public DDGBase, public DDGInfo {
+  friend class DDGBuilder;
+
+public:
+  using NodeType = DDGNode;
+  using EdgeType = DDGEdge;
+
+  DataDependenceGraph() = delete;
+  DataDependenceGraph(const DataDependenceGraph &G) = delete;
+  DataDependenceGraph(DataDependenceGraph &&G)
+      : DDGBase(std::move(G)), DDGInfo(std::move(G)) {}
+  DataDependenceGraph(Function &F, DependenceInfo &DI);
+  DataDependenceGraph(const Loop &L, DependenceInfo &DI);
+  ~DataDependenceGraph();
+
+protected:
+  /// Add node \p N to the graph, if it's not added yet, and keep track of
+  /// the root node. Return true if node is successfully added.
+  bool addNode(NodeType &N);
+
+};
+
+/// Concrete implementation of a pure data dependence graph builder. This class
+/// provides custom implementation for the pure-virtual functions used in the
+/// generic dependence graph build algorithm.
+///
+/// For information about time complexity of the build algorithm see the
+/// comments near the declaration of AbstractDependenceGraphBuilder.
+class DDGBuilder : public AbstractDependenceGraphBuilder<DataDependenceGraph> {
+public:
+  DDGBuilder(DataDependenceGraph &G, DependenceInfo &D,
+             const BasicBlockListType &BBs)
+      : AbstractDependenceGraphBuilder(G, D, BBs) {}
+  DDGNode &createRootNode() final override {
+    auto *RN = new RootDDGNode();
+    assert(RN && "Failed to allocate memory for DDG root node.");
+    Graph.addNode(*RN);
+    return *RN;
+  }
+  DDGNode &createFineGrainedNode(Instruction &I) final override {
+    auto *SN = new SimpleDDGNode(I);
+    assert(SN && "Failed to allocate memory for simple DDG node.");
+    Graph.addNode(*SN);
+    return *SN;
+  }
+  DDGEdge &createDefUseEdge(DDGNode &Src, DDGNode &Tgt) final override {
+    auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::RegisterDefUse);
+    assert(E && "Failed to allocate memory for edge");
+    Graph.connect(Src, Tgt, *E);
+    return *E;
+  }
+  DDGEdge &createMemoryEdge(DDGNode &Src, DDGNode &Tgt) final override {
+    auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::MemoryDependence);
+    assert(E && "Failed to allocate memory for edge");
+    Graph.connect(Src, Tgt, *E);
+    return *E;
+  }
+  DDGEdge &createRootedEdge(DDGNode &Src, DDGNode &Tgt) final override {
+    auto *E = new DDGEdge(Tgt, DDGEdge::EdgeKind::Rooted);
+    assert(E && "Failed to allocate memory for edge");
+    assert(isa<RootDDGNode>(Src) && "Expected root node");
+    Graph.connect(Src, Tgt, *E);
+    return *E;
+  }
+
+};
+
+raw_ostream &operator<<(raw_ostream &OS, const DDGNode &N);
+raw_ostream &operator<<(raw_ostream &OS, const DDGNode::NodeKind K);
+raw_ostream &operator<<(raw_ostream &OS, const DDGEdge &E);
+raw_ostream &operator<<(raw_ostream &OS, const DDGEdge::EdgeKind K);
+raw_ostream &operator<<(raw_ostream &OS, const DataDependenceGraph &G);
+
+//===--------------------------------------------------------------------===//
+// DDG Analysis Passes
+//===--------------------------------------------------------------------===//
+
+/// Analysis pass that builds the DDG for a loop.
+class DDGAnalysis : public AnalysisInfoMixin<DDGAnalysis> {
+public:
+  using Result = std::unique_ptr<DataDependenceGraph>;
+  Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR);
+
+private:
+  friend AnalysisInfoMixin<DDGAnalysis>;
+  static AnalysisKey Key;
+};
+
+/// Textual printer pass for the DDG of a loop.
+class DDGAnalysisPrinterPass : public PassInfoMixin<DDGAnalysisPrinterPass> {
+public:
+  explicit DDGAnalysisPrinterPass(raw_ostream &OS) : OS(OS) {}
+  PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
+                        LoopStandardAnalysisResults &AR, LPMUpdater &U);
+
+private:
+  raw_ostream &OS;
+};
+
+//===--------------------------------------------------------------------===//
+// GraphTraits specializations for the DDG
+//===--------------------------------------------------------------------===//
+
+/// non-const versions of the grapth trait specializations for DDG
+template <> struct GraphTraits<DDGNode *> {
+  using NodeRef = DDGNode *;
+
+  static DDGNode *DDGGetTargetNode(DGEdge<DDGNode, DDGEdge> *P) {
+    return &P->getTargetNode();
+  }
+
+  // Provide a mapped iterator so that the GraphTrait-based implementations can
+  // find the target nodes without having to explicitly go through the edges.
+  using ChildIteratorType =
+      mapped_iterator<DDGNode::iterator, decltype(&DDGGetTargetNode)>;
+  using ChildEdgeIteratorType = DDGNode::iterator;
+
+  static NodeRef getEntryNode(NodeRef N) { return N; }
+  static ChildIteratorType child_begin(NodeRef N) {
+    return ChildIteratorType(N->begin(), &DDGGetTargetNode);
+  }
+  static ChildIteratorType child_end(NodeRef N) {
+    return ChildIteratorType(N->end(), &DDGGetTargetNode);
+  }
+
+  static ChildEdgeIteratorType child_edge_begin(NodeRef N) {
+    return N->begin();
+  }
+  static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); }
+};
+
+template <>
+struct GraphTraits<DataDependenceGraph *> : public GraphTraits<DDGNode *> {
+  using nodes_iterator = DataDependenceGraph::iterator;
+  static NodeRef getEntryNode(DataDependenceGraph *DG) {
+    return &DG->getRoot();
+  }
+  static nodes_iterator nodes_begin(DataDependenceGraph *DG) {
+    return DG->begin();
+  }
+  static nodes_iterator nodes_end(DataDependenceGraph *DG) { return DG->end(); }
+};
+
+/// const versions of the grapth trait specializations for DDG
+template <> struct GraphTraits<const DDGNode *> {
+  using NodeRef = const DDGNode *;
+
+  static const DDGNode *DDGGetTargetNode(const DGEdge<DDGNode, DDGEdge> *P) {
+    return &P->getTargetNode();
+  }
+
+  // Provide a mapped iterator so that the GraphTrait-based implementations can
+  // find the target nodes without having to explicitly go through the edges.
+  using ChildIteratorType =
+      mapped_iterator<DDGNode::const_iterator, decltype(&DDGGetTargetNode)>;
+  using ChildEdgeIteratorType = DDGNode::const_iterator;
+
+  static NodeRef getEntryNode(NodeRef N) { return N; }
+  static ChildIteratorType child_begin(NodeRef N) {
+    return ChildIteratorType(N->begin(), &DDGGetTargetNode);
+  }
+  static ChildIteratorType child_end(NodeRef N) {
+    return ChildIteratorType(N->end(), &DDGGetTargetNode);
+  }
+
+  static ChildEdgeIteratorType child_edge_begin(NodeRef N) {
+    return N->begin();
+  }
+  static ChildEdgeIteratorType child_edge_end(NodeRef N) { return N->end(); }
+};
+
+template <>
+struct GraphTraits<const DataDependenceGraph *>
+    : public GraphTraits<const DDGNode *> {
+  using nodes_iterator = DataDependenceGraph::const_iterator;
+  static NodeRef getEntryNode(const DataDependenceGraph *DG) {
+    return &DG->getRoot();
+  }
+  static nodes_iterator nodes_begin(const DataDependenceGraph *DG) {
+    return DG->begin();
+  }
+  static nodes_iterator nodes_end(const DataDependenceGraph *DG) {
+    return DG->end();
+  }
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_DDG_H
diff --git a/llvm/include/llvm/Analysis/DependenceGraphBuilder.h b/llvm/include/llvm/Analysis/DependenceGraphBuilder.h
new file mode 100644
index 0000000000000..5f4bdb47043b5
--- /dev/null
+++ b/llvm/include/llvm/Analysis/DependenceGraphBuilder.h
@@ -0,0 +1,119 @@
+//===- llvm/Analysis/DependenceGraphBuilder.h -------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a builder interface that can be used to populate dependence
+// graphs such as DDG and PDG.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H
+#define LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H
+
+#include "llvm/ADT/EquivalenceClasses.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Instructions.h"
+
+namespace llvm {
+
+/// This abstract builder class defines a set of high-level steps for creating
+/// DDG-like graphs. The client code is expected to inherit from this class and
+/// define concrete implementation for each of the pure virtual functions used
+/// in the high-level algorithm.
+template <class GraphType> class AbstractDependenceGraphBuilder {
+protected:
+  using BasicBlockListType = SmallVectorImpl<BasicBlock *>;
+
+private:
+  using NodeType = typename GraphType::NodeType;
+  using EdgeType = typename GraphType::EdgeType;
+
+public:
+  using ClassesType = EquivalenceClasses<BasicBlock *>;
+  using NodeListType = SmallVector<NodeType *, 4>;
+
+  AbstractDependenceGraphBuilder(GraphType &G, DependenceInfo &D,
+                                 const BasicBlockListType &BBs)
+      : Graph(G), DI(D), BBList(BBs) {}
+  virtual ~AbstractDependenceGraphBuilder() {}
+
+  /// The main entry to the graph construction algorithm. It starts by
+  /// creating nodes in increasing order of granularity and then
+  /// adds def-use and memory edges.
+  ///
+  /// The algorithmic complexity of this implementation is O(V^2 * I^2), where V
+  /// is the number of vertecies (nodes) and I is the number of instructions in
+  /// each node. The total number of instructions, N, is equal to V * I,
+  /// therefore the worst-case time complexity is O(N^2). The average time
+  /// complexity is O((N^2)/2).
+  void populate() {
+    createFineGrainedNodes();
+    createDefUseEdges();
+    createMemoryDependencyEdges();
+    createAndConnectRootNode();
+  }
+
+  /// Create fine grained nodes. These are typically atomic nodes that
+  /// consist of a single instruction.
+  void createFineGrainedNodes();
+
+  /// Analyze the def-use chains and create edges from the nodes containing
+  /// definitions to the nodes containing the uses.
+  void createDefUseEdges();
+
+  /// Analyze data dependencies that exist between memory loads or stores,
+  /// in the graph nodes and create edges between them.
+  void createMemoryDependencyEdges();
+
+  /// Create a root node and add edges such that each node in the graph is
+  /// reachable from the root.
+  void createAndConnectRootNode();
+
+protected:
+  /// Create the root node of the graph.
+  virtual NodeType &createRootNode() = 0;
+
+  /// Create an atomic node in the graph given a single instruction.
+  virtual NodeType &createFineGrainedNode(Instruction &I) = 0;
+
+  /// Create a def-use edge going from \p Src to \p Tgt.
+  virtual EdgeType &createDefUseEdge(NodeType &Src, NodeType &Tgt) = 0;
+
+  /// Create a memory dependence edge going from \p Src to \p Tgt.
+  virtual EdgeType &createMemoryEdge(NodeType &Src, NodeType &Tgt) = 0;
+
+  /// Create a rooted edge going from \p Src to \p Tgt .
+  virtual EdgeType &createRootedEdge(NodeType &Src, NodeType &Tgt) = 0;
+
+  /// Deallocate memory of edge \p E.
+  virtual void destroyEdge(EdgeType &E) { delete &E; }
+
+  /// Deallocate memory of node \p N.
+  virtual void destroyNode(NodeType &N) { delete &N; }
+
+  /// Map types to map instructions to nodes used when populating the graph.
+  using InstToNodeMap = DenseMap<Instruction *, NodeType *>;
+
+  /// Reference to the graph that gets built by a concrete implementation of
+  /// this builder.
+  GraphType &Graph;
+
+  /// Dependence information used to create memory dependence edges in the
+  /// graph.
+  DependenceInfo &DI;
+
+  /// The list of basic blocks to consider when building the graph.
+  const BasicBlockListType &BBList;
+
+  /// A mapping from instructions to the corresponding nodes in the graph.
+  InstToNodeMap IMap;
+};
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_DEPENDENCE_GRAPH_BUILDER_H
diff --git a/llvm/include/llvm/Analysis/InstructionSimplify.h b/llvm/include/llvm/Analysis/InstructionSimplify.h
index db92130c86328..a5ffca13046b7 100644
--- a/llvm/include/llvm/Analysis/InstructionSimplify.h
+++ b/llvm/include/llvm/Analysis/InstructionSimplify.h
@@ -142,6 +142,13 @@ Value *SimplifyFSubInst(Value *LHS, Value *RHS, FastMathFlags FMF,
 Value *SimplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF,
                         const SimplifyQuery &Q);
 
+/// Given operands for the multiplication of a FMA, fold the result or return
+/// null. In contrast to SimplifyFMulInst, this function will not perform
+/// simplifications whose unrounded results differ when rounded to the argument
+/// type.
+Value *SimplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF,
+                       const SimplifyQuery &Q);
+
 /// Given operands for a Mul, fold the result or return null.
 Value *SimplifyMulInst(Value *LHS, Value *RHS, const SimplifyQuery &Q);
 
diff --git a/llvm/include/llvm/Analysis/LoopInfoImpl.h b/llvm/include/llvm/Analysis/LoopInfoImpl.h
index 4c33dac9e21e1..8b11e848a195b 100644
--- a/llvm/include/llvm/Analysis/LoopInfoImpl.h
+++ b/llvm/include/llvm/Analysis/LoopInfoImpl.h
@@ -85,9 +85,9 @@ template <class BlockT, class LoopT>
 bool LoopBase<BlockT, LoopT>::hasDedicatedExits() const {
   // Each predecessor of each exit block of a normal loop is contained
   // within the loop.
-  SmallVector<BlockT *, 4> ExitBlocks;
-  getExitBlocks(ExitBlocks);
-  for (BlockT *EB : ExitBlocks)
+  SmallVector<BlockT *, 4> UniqueExitBlocks;
+  getUniqueExitBlocks(UniqueExitBlocks);
+  for (BlockT *EB : UniqueExitBlocks)
     for (BlockT *Predecessor : children<Inverse<BlockT *>>(EB))
       if (!contains(Predecessor))
         return false;
@@ -200,8 +200,6 @@ BlockT *LoopBase<BlockT, LoopT>::getLoopPredecessor() const {
     }
   }
 
-  // Make sure there is only one exit out of the preheader.
-  assert(Out && "Header of loop has no predecessors from outside loop?");
   return Out;
 }
 
diff --git a/llvm/include/llvm/Analysis/MemoryBuiltins.h b/llvm/include/llvm/Analysis/MemoryBuiltins.h
index 5c201f3591669..a89d76b9e5bd9 100644
--- a/llvm/include/llvm/Analysis/MemoryBuiltins.h
+++ b/llvm/include/llvm/Analysis/MemoryBuiltins.h
@@ -104,6 +104,11 @@ bool isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI);
 bool isOpNewLikeFn(const Value *V, const TargetLibraryInfo *TLI,
                      bool LookThroughBitCast = false);
 
+/// Tests if a value is a call or invoke to a library function that
+/// allocates memory (strdup, strndup).
+bool isStrdupLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                     bool LookThroughBitCast = false);
+
 //===----------------------------------------------------------------------===//
 //  malloc Call Utility Functions.
 //
diff --git a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
index f309d344b8d15..6693e40ccf223 100644
--- a/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
+++ b/llvm/include/llvm/Analysis/ProfileSummaryInfo.h
@@ -52,6 +52,15 @@ class ProfileSummaryInfo {
   // because the number of profile counts required to reach the hot
   // percentile is above a huge threshold.
   Optional<bool> HasHugeWorkingSetSize;
+  // True if the working set size of the code is considered large,
+  // because the number of profile counts required to reach the hot
+  // percentile is above a large threshold.
+  Optional<bool> HasLargeWorkingSetSize;
+  // Compute the threshold for a given cutoff.
+  Optional<uint64_t> computeThreshold(int PercentileCutoff);
+  // The map that caches the threshold values. The keys are the percentile
+  // cutoff values and the values are the corresponding threshold values.
+  DenseMap<int, uint64_t> ThresholdCache;
 
 public:
   ProfileSummaryInfo(Module &M) : M(M) {}
@@ -96,6 +105,8 @@ class ProfileSummaryInfo {
                                      bool AllowSynthetic = false);
   /// Returns true if the working set size of the code is considered huge.
   bool hasHugeWorkingSetSize();
+  /// Returns true if the working set size of the code is considered large.
+  bool hasLargeWorkingSetSize();
   /// Returns true if \p F has hot function entry.
   bool isFunctionEntryHot(const Function *F);
   /// Returns true if \p F contains hot code.
@@ -104,14 +115,26 @@ class ProfileSummaryInfo {
   bool isFunctionEntryCold(const Function *F);
   /// Returns true if \p F contains only cold code.
   bool isFunctionColdInCallGraph(const Function *F, BlockFrequencyInfo &BFI);
+  /// Returns true if \p F contains hot code with regard to a given hot
+  /// percentile cutoff value.
+  bool isFunctionHotInCallGraphNthPercentile(int PercentileCutoff,
+                                             const Function *F,
+                                             BlockFrequencyInfo &BFI);
   /// Returns true if count \p C is considered hot.
   bool isHotCount(uint64_t C);
   /// Returns true if count \p C is considered cold.
   bool isColdCount(uint64_t C);
+  /// Returns true if count \p C is considered hot with regard to a given
+  /// hot percentile cutoff value.
+  bool isHotCountNthPercentile(int PercentileCutoff, uint64_t C);
   /// Returns true if BasicBlock \p BB is considered hot.
   bool isHotBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI);
   /// Returns true if BasicBlock \p BB is considered cold.
   bool isColdBlock(const BasicBlock *BB, BlockFrequencyInfo *BFI);
+  /// Returns true if BasicBlock \p BB is considered hot with regard to a given
+  /// hot percentile cutoff value.
+  bool isHotBlockNthPercentile(int PercentileCutoff,
+                               const BasicBlock *BB, BlockFrequencyInfo *BFI);
   /// Returns true if CallSite \p CS is considered hot.
   bool isHotCallSite(const CallSite &CS, BlockFrequencyInfo *BFI);
   /// Returns true if Callsite \p CS is considered cold.
diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
index ce98a145fe0e2..b4d727449fbe0 100644
--- a/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
+++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpander.h
@@ -77,9 +77,13 @@ namespace llvm {
     /// Phis that complete an IV chain. Reuse
     DenseSet<AssertingVH<PHINode>> ChainedPhis;
 
-    /// When true, expressions are expanded in "canonical" form. In particular,
-    /// addrecs are expanded as arithmetic based on a canonical induction
-    /// variable. When false, expression are expanded in a more literal form.
+    /// When true, SCEVExpander tries to expand expressions in "canonical" form.
+    /// When false, expressions are expanded in a more literal form.
+    ///
+    /// In "canonical" form addrecs are expanded as arithmetic based on a
+    /// canonical induction variable. Note that CanonicalMode doesn't guarantee
+    /// that all expressions are expanded in "canonical" form. For some
+    /// expressions literal mode can be preferred.
     bool CanonicalMode;
 
     /// When invoked from LSR, the expander is in "strength reduction" mode. The
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 70b69e9940ac0..6da2d7f43bc42 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -580,9 +580,9 @@ class TargetTransformInfo {
   bool isLegalMaskedLoad(Type *DataType) const;
 
   /// Return true if the target supports nontemporal store.
-  bool isLegalNTStore(Type *DataType, llvm::Align Alignment) const;
+  bool isLegalNTStore(Type *DataType, Align Alignment) const;
   /// Return true if the target supports nontemporal load.
-  bool isLegalNTLoad(Type *DataType, llvm::Align Alignment) const;
+  bool isLegalNTLoad(Type *DataType, Align Alignment) const;
 
   /// Return true if the target supports masked scatter.
   bool isLegalMaskedScatter(Type *DataType) const;
@@ -641,12 +641,6 @@ class TargetTransformInfo {
   /// Return true if this type is legal.
   bool isTypeLegal(Type *Ty) const;
 
-  /// Returns the target's jmp_buf alignment in bytes.
-  unsigned getJumpBufAlignment() const;
-
-  /// Returns the target's jmp_buf size in bytes.
-  unsigned getJumpBufSize() const;
-
   /// Return true if switches should be turned into lookup tables for the
   /// target.
   bool shouldBuildLookupTables() const;
@@ -1202,8 +1196,8 @@ class TargetTransformInfo::Concept {
   virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0;
   virtual bool isLegalMaskedStore(Type *DataType) = 0;
   virtual bool isLegalMaskedLoad(Type *DataType) = 0;
-  virtual bool isLegalNTStore(Type *DataType, llvm::Align Alignment) = 0;
-  virtual bool isLegalNTLoad(Type *DataType, llvm::Align Alignment) = 0;
+  virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
+  virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
   virtual bool isLegalMaskedScatter(Type *DataType) = 0;
   virtual bool isLegalMaskedGather(Type *DataType) = 0;
   virtual bool isLegalMaskedCompressStore(Type *DataType) = 0;
@@ -1219,8 +1213,6 @@ class TargetTransformInfo::Concept {
   virtual bool isProfitableToHoist(Instruction *I) = 0;
   virtual bool useAA() = 0;
   virtual bool isTypeLegal(Type *Ty) = 0;
-  virtual unsigned getJumpBufAlignment() = 0;
-  virtual unsigned getJumpBufSize() = 0;
   virtual bool shouldBuildLookupTables() = 0;
   virtual bool shouldBuildLookupTablesForConstant(Constant *C) = 0;
   virtual bool useColdCCForColdCall(Function &F) = 0;
@@ -1479,10 +1471,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   bool isLegalMaskedLoad(Type *DataType) override {
     return Impl.isLegalMaskedLoad(DataType);
   }
-  bool isLegalNTStore(Type *DataType, llvm::Align Alignment) override {
+  bool isLegalNTStore(Type *DataType, Align Alignment) override {
     return Impl.isLegalNTStore(DataType, Alignment);
   }
-  bool isLegalNTLoad(Type *DataType, llvm::Align Alignment) override {
+  bool isLegalNTLoad(Type *DataType, Align Alignment) override {
     return Impl.isLegalNTLoad(DataType, Alignment);
   }
   bool isLegalMaskedScatter(Type *DataType) override {
@@ -1523,8 +1515,6 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
   }
   bool useAA() override { return Impl.useAA(); }
   bool isTypeLegal(Type *Ty) override { return Impl.isTypeLegal(Ty); }
-  unsigned getJumpBufAlignment() override { return Impl.getJumpBufAlignment(); }
-  unsigned getJumpBufSize() override { return Impl.getJumpBufSize(); }
   bool shouldBuildLookupTables() override {
     return Impl.shouldBuildLookupTables();
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index cc41488f517d3..2f1011799f137 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -247,14 +247,14 @@ class TargetTransformInfoImplBase {
 
   bool isLegalMaskedLoad(Type *DataType) { return false; }
 
-  bool isLegalNTStore(Type *DataType, llvm::Align Alignment) {
+  bool isLegalNTStore(Type *DataType, Align Alignment) {
     // By default, assume nontemporal memory stores are available for stores
     // that are aligned and have a size that is a power of 2.
     unsigned DataSize = DL.getTypeStoreSize(DataType);
     return Alignment >= DataSize && isPowerOf2_32(DataSize);
   }
 
-  bool isLegalNTLoad(Type *DataType, llvm::Align Alignment) {
+  bool isLegalNTLoad(Type *DataType, Align Alignment) {
     // By default, assume nontemporal memory loads are available for loads that
     // are aligned and have a size that is a power of 2.
     unsigned DataSize = DL.getTypeStoreSize(DataType);
@@ -294,10 +294,6 @@ class TargetTransformInfoImplBase {
 
   bool isTypeLegal(Type *Ty) { return false; }
 
-  unsigned getJumpBufAlignment() { return 0; }
-
-  unsigned getJumpBufSize() { return 0; }
-
   bool shouldBuildLookupTables() { return true; }
   bool shouldBuildLookupTablesForConstant(Constant *C) { return true; }
 
diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h
index 0a3fab3fb71fe..33b064fcf9d2f 100644
--- a/llvm/include/llvm/Analysis/ValueTracking.h
+++ b/llvm/include/llvm/Analysis/ValueTracking.h
@@ -620,12 +620,12 @@ class Value;
   SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS,
                                          Instruction::CastOps *CastOp = nullptr,
                                          unsigned Depth = 0);
+
   inline SelectPatternResult
-  matchSelectPattern(const Value *V, const Value *&LHS, const Value *&RHS,
-                     Instruction::CastOps *CastOp = nullptr) {
-    Value *L = const_cast<Value*>(LHS);
-    Value *R = const_cast<Value*>(RHS);
-    auto Result = matchSelectPattern(const_cast<Value*>(V), L, R);
+  matchSelectPattern(const Value *V, const Value *&LHS, const Value *&RHS) {
+    Value *L = const_cast<Value *>(LHS);
+    Value *R = const_cast<Value *>(RHS);
+    auto Result = matchSelectPattern(const_cast<Value *>(V), L, R);
     LHS = L;
     RHS = R;
     return Result;
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index bf0063b3f870d..9811b41256953 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -20,6 +20,117 @@
 
 namespace llvm {
 
+/// Describes the type of Parameters
+enum class VFParamKind {
+  Vector,            // No semantic information.
+  OMP_Linear,        // declare simd linear(i)
+  OMP_LinearRef,     // declare simd linear(ref(i))
+  OMP_LinearVal,     // declare simd linear(val(i))
+  OMP_LinearUVal,    // declare simd linear(uval(i))
+  OMP_LinearPos,     // declare simd linear(i:c) uniform(c)
+  OMP_LinearValPos,  // declare simd linear(val(i:c)) uniform(c)
+  OMP_LinearRefPos,  // declare simd linear(ref(i:c)) uniform(c)
+  OMP_LinearUValPos, // declare simd linear(uval(i:c)) uniform(c
+  OMP_Uniform,       // declare simd uniform(i)
+  GlobalPredicate,   // Global logical predicate that acts on all lanes
+                     // of the input and output mask concurrently. For
+                     // example, it is implied by the `M` token in the
+                     // Vector Function ABI mangled name.
+  Unknown
+};
+
+/// Describes the type of Instruction Set Architecture
+enum class VFISAKind {
+  AdvancedSIMD, // AArch64 Advanced SIMD (NEON)
+  SVE,          // AArch64 Scalable Vector Extension
+  SSE,          // x86 SSE
+  AVX,          // x86 AVX
+  AVX2,         // x86 AVX2
+  AVX512,       // x86 AVX512
+  Unknown       // Unknown ISA
+};
+
+/// Encapsulates information needed to describe a parameter.
+///
+/// The description of the parameter is not linked directly to
+/// OpenMP or any other vector function description. This structure
+/// is extendible to handle other paradigms that describe vector
+/// functions and their parameters.
+struct VFParameter {
+  unsigned ParamPos;         // Parameter Position in Scalar Function.
+  VFParamKind ParamKind;     // Kind of Parameter.
+  int LinearStepOrPos = 0;   // Step or Position of the Parameter.
+  Align Alignment = Align(); // Optional aligment in bytes, defaulted to 1.
+
+  // Comparison operator.
+  bool operator==(const VFParameter &Other) const {
+    return std::tie(ParamPos, ParamKind, LinearStepOrPos, Alignment) ==
+           std::tie(Other.ParamPos, Other.ParamKind, Other.LinearStepOrPos,
+                    Other.Alignment);
+  }
+};
+
+/// Contains the information about the kind of vectorization
+/// available.
+///
+/// This object in independent on the paradigm used to
+/// represent vector functions. in particular, it is not attached to
+/// any target-specific ABI.
+struct VFShape {
+  unsigned VF;     // Vectorization factor.
+  bool IsScalable; // True if the function is a scalable function.
+  VFISAKind ISA;   // Instruction Set Architecture.
+  SmallVector<VFParameter, 8> Parameters; // List of parameter informations.
+  // Comparison operator.
+  bool operator==(const VFShape &Other) const {
+    return std::tie(VF, IsScalable, ISA, Parameters) ==
+           std::tie(Other.VF, Other.IsScalable, Other.ISA, Other.Parameters);
+  }
+};
+
+/// Holds the VFShape for a specific scalar to vector function mapping.
+struct VFInfo {
+  VFShape Shape;        // Classification of the vector function.
+  StringRef ScalarName; // Scalar Function Name.
+  StringRef VectorName; // Vector Function Name associated to this VFInfo.
+
+  // Comparison operator.
+  bool operator==(const VFInfo &Other) const {
+    return std::tie(Shape, ScalarName, VectorName) ==
+           std::tie(Shape, Other.ScalarName, Other.VectorName);
+  }
+};
+
+namespace VFABI {
+/// Function to contruct a VFInfo out of a mangled names in the
+/// following format:
+///
+/// <VFABI_name>{(<redirection>)}
+///
+/// where <VFABI_name> is the name of the vector function, mangled according
+/// to the rules described in the Vector Function ABI of the target vector
+/// extentsion (or <isa> from now on). The <VFABI_name> is in the following
+/// format:
+///
+/// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]
+///
+/// This methods support demangling rules for the following <isa>:
+///
+/// * AArch64: https://developer.arm.com/docs/101129/latest
+///
+/// * x86 (libmvec): https://sourceware.org/glibc/wiki/libmvec and
+///  https://sourceware.org/glibc/wiki/libmvec?action=AttachFile&do=view&target=VectorABI.txt
+///
+///
+///
+/// \param MangledName -> input string in the format
+/// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)].
+Optional<VFInfo> tryDemangleForVFABI(StringRef MangledName);
+
+/// Retrieve the `VFParamKind` from a string token.
+VFParamKind getVFParamKindFromString(const StringRef Token);
+} // end namespace VFABI
+
 template <typename T> class ArrayRef;
 class DemandedBits;
 class GetElementPtrInst;
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index 72f36c46a4a8f..21cdc6a88d01d 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -180,6 +180,59 @@ enum SourceLanguage {
   DW_LANG_hi_user = 0xffff
 };
 
+inline bool isCPlusPlus(SourceLanguage S) {
+  // Deliberately enumerate all the language options so we get a warning when
+  // new language options are added (-Wswitch) that'll hopefully help keep this
+  // switch up-to-date when new C++ versions are added.
+  switch (S) {
+  case DW_LANG_C_plus_plus:
+  case DW_LANG_C_plus_plus_03:
+  case DW_LANG_C_plus_plus_11:
+  case DW_LANG_C_plus_plus_14:
+    return true;
+  case DW_LANG_C89:
+  case DW_LANG_C:
+  case DW_LANG_Ada83:
+  case DW_LANG_Cobol74:
+  case DW_LANG_Cobol85:
+  case DW_LANG_Fortran77:
+  case DW_LANG_Fortran90:
+  case DW_LANG_Pascal83:
+  case DW_LANG_Modula2:
+  case DW_LANG_Java:
+  case DW_LANG_C99:
+  case DW_LANG_Ada95:
+  case DW_LANG_Fortran95:
+  case DW_LANG_PLI:
+  case DW_LANG_ObjC:
+  case DW_LANG_ObjC_plus_plus:
+  case DW_LANG_UPC:
+  case DW_LANG_D:
+  case DW_LANG_Python:
+  case DW_LANG_OpenCL:
+  case DW_LANG_Go:
+  case DW_LANG_Modula3:
+  case DW_LANG_Haskell:
+  case DW_LANG_OCaml:
+  case DW_LANG_Rust:
+  case DW_LANG_C11:
+  case DW_LANG_Swift:
+  case DW_LANG_Julia:
+  case DW_LANG_Dylan:
+  case DW_LANG_Fortran03:
+  case DW_LANG_Fortran08:
+  case DW_LANG_RenderScript:
+  case DW_LANG_BLISS:
+  case DW_LANG_Mips_Assembler:
+  case DW_LANG_GOOGLE_RenderScript:
+  case DW_LANG_BORLAND_Delphi:
+  case DW_LANG_lo_user:
+  case DW_LANG_hi_user:
+    return false;
+  }
+  llvm_unreachable("Invalid source language");
+}
+
 enum CaseSensitivity {
   // Identifier case codes
   DW_ID_case_sensitive = 0x00,
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index a06051f5bf6df..a4580da5aec93 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -111,6 +111,10 @@ class AsmPrinter : public MachineFunctionPass {
   /// of each call to runOnMachineFunction().
   MCSymbol *CurrentFnSym = nullptr;
 
+  /// The symbol for the current function descriptor on AIX. This is created
+  /// at the beginning of each call to SetupMachineFunction().
+  MCSymbol *CurrentFnDescSym = nullptr;
+
   /// The symbol used to represent the start of the current function for the
   /// purpose of calculating its size (e.g. using the .size directive). By
   /// default, this is equal to CurrentFnSym.
@@ -304,7 +308,7 @@ class AsmPrinter : public MachineFunctionPass {
 
   /// This should be called when a new MachineFunction is being processed from
   /// runOnMachineFunction.
-  void SetupMachineFunction(MachineFunction &MF);
+  virtual void SetupMachineFunction(MachineFunction &MF);
 
   /// This method emits the body and trailer for a function.
   void EmitFunctionBody();
@@ -346,7 +350,7 @@ class AsmPrinter : public MachineFunctionPass {
   /// global value is specified, and if that global has an explicit alignment
   /// requested, it will override the alignment request if required for
   /// correctness.
-  void EmitAlignment(llvm::Align Align, const GlobalObject *GV = nullptr) const;
+  void EmitAlignment(Align Alignment, const GlobalObject *GV = nullptr) const;
 
   /// Lower the specified LLVM Constant to an MCExpr.
   virtual const MCExpr *lowerConstant(const Constant *CV);
@@ -414,6 +418,10 @@ class AsmPrinter : public MachineFunctionPass {
 
   virtual void EmitFunctionEntryLabel();
 
+  virtual void EmitFunctionDescriptor() {
+    llvm_unreachable("Function descriptor is target-specific.");
+  }
+
   virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV);
 
   /// Targets can override this to change how global constants that are part of
@@ -635,8 +643,8 @@ class AsmPrinter : public MachineFunctionPass {
   void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
 
   /// Return the alignment for the specified \p GV.
-  static llvm::Align getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
-                                    llvm::Align InAlign = llvm::Align(1));
+  static Align getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+                              Align InAlign = Align::None());
 
 private:
   /// Private state for PrintSpecial()
diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h
index 56da9b198017d..a30ca638ee6da 100644
--- a/llvm/include/llvm/CodeGen/CallingConvLower.h
+++ b/llvm/include/llvm/CodeGen/CallingConvLower.h
@@ -424,18 +424,18 @@ class CCState {
   /// AllocateStack - Allocate a chunk of stack space with the specified size
   /// and alignment.
   unsigned AllocateStack(unsigned Size, unsigned Alignment) {
-    const llvm::Align Align(Alignment);
-    StackOffset = alignTo(StackOffset, Align);
+    const Align CheckedAlignment(Alignment);
+    StackOffset = alignTo(StackOffset, CheckedAlignment);
     unsigned Result = StackOffset;
     StackOffset += Size;
-    MaxStackArgAlign = std::max(Align, MaxStackArgAlign);
-    ensureMaxAlignment(Align);
+    MaxStackArgAlign = std::max(CheckedAlignment, MaxStackArgAlign);
+    ensureMaxAlignment(CheckedAlignment);
     return Result;
   }
 
-  void ensureMaxAlignment(llvm::Align Align) {
+  void ensureMaxAlignment(Align Alignment) {
     if (!AnalyzingMustTailForwardedRegs)
-      MF.getFrameInfo().ensureMaxAlignment(Align.value());
+      MF.getFrameInfo().ensureMaxAlignment(Alignment.value());
   }
 
   /// Version of AllocateStack with extra register to be shadowed.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
index 6e54e70c4dc88..dddb267f0c674 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CallLowering.h
@@ -95,6 +95,10 @@ class CallLowering {
     /// optimization.
     bool IsTailCall = false;
 
+    /// True if the call was lowered as a tail call. This is consumed by the
+    /// legalizer. This allows the legalizer to lower libcalls as tail calls.
+    bool LoweredTailCall = false;
+
     /// True if the call is to a vararg function.
     bool IsVarArg = false;
   };
@@ -216,7 +220,8 @@ class CallLowering {
   ///
   /// Used to check if arguments are suitable for tail call lowering.
   bool analyzeArgInfo(CCState &CCState, SmallVectorImpl<ArgInfo> &Args,
-                      CCAssignFn &Fn) const;
+                      CCAssignFn &AssignFnFixed,
+                      CCAssignFn &AssignFnVarArg) const;
 
   /// \returns True if the calling convention for a callee and its caller pass
   /// results in the same way. Typically used for tail call eligibility checks.
@@ -224,12 +229,18 @@ class CallLowering {
   /// \p Info is the CallLoweringInfo for the call.
   /// \p MF is the MachineFunction for the caller.
   /// \p InArgs contains the results of the call.
-  /// \p CalleeAssignFn is the CCAssignFn to be used for the callee.
-  /// \p CallerAssignFn is the CCAssignFn to be used for the caller.
+  /// \p CalleeAssignFnFixed is the CCAssignFn to be used for the callee for
+  /// fixed arguments.
+  /// \p CalleeAssignFnVarArg is similar, but for varargs.
+  /// \p CallerAssignFnFixed is the CCAssignFn to be used for the caller for
+  /// fixed arguments.
+  /// \p CallerAssignFnVarArg is similar, but for varargs.
   bool resultsCompatible(CallLoweringInfo &Info, MachineFunction &MF,
                          SmallVectorImpl<ArgInfo> &InArgs,
-                         CCAssignFn &CalleeAssignFn,
-                         CCAssignFn &CallerAssignFn) const;
+                         CCAssignFn &CalleeAssignFnFixed,
+                         CCAssignFn &CalleeAssignFnVarArg,
+                         CCAssignFn &CallerAssignFnFixed,
+                         CCAssignFn &CallerAssignFnVarArg) const;
 
 public:
   CallLowering(const TargetLowering *TLI) : TLI(TLI) {}
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 14834b83b9df1..4dd5958c1425e 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -63,11 +63,11 @@ class CombinerHelper {
   bool matchCombineCopy(MachineInstr &MI);
   void applyCombineCopy(MachineInstr &MI);
 
-  /// \brief \returns true if \p DefMI precedes \p UseMI or they are the same
+  /// Returns true if \p DefMI precedes \p UseMI or they are the same
   /// instruction. Both must be in the same basic block.
   bool isPredecessor(MachineInstr &DefMI, MachineInstr &UseMI);
 
-  /// \brief \returns true if \p DefMI dominates \p UseMI. By definition an
+  /// Returns true if \p DefMI dominates \p UseMI. By definition an
   /// instruction dominates itself.
   ///
   /// If we haven't been provided with a MachineDominatorTree during
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
index 16529cc129736..ad645a46bbe6c 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerInfo.h
@@ -31,7 +31,7 @@ class CombinerInfo {
                bool MinSize)
       : IllegalOpsAllowed(AllowIllegalOps),
         LegalizeIllegalOps(ShouldLegalizeIllegal), LInfo(LInfo),
-        EnableOpt(OptEnabled), EnableOptSize(OptSize), EnableMinSize(OptSize) {
+        EnableOpt(OptEnabled), EnableOptSize(OptSize), EnableMinSize(MinSize) {
     assert(((AllowIllegalOps || !LegalizeIllegalOps) || LInfo) &&
            "Expecting legalizerInfo when illegalops not allowed");
   }
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
index e4877b7ad6391..fd3dc743000bb 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h
@@ -220,6 +220,11 @@ enum {
   /// - OpIdx - Operand index
   GIM_CheckIsMBB,
 
+  /// Check the specified operand is an Imm
+  /// - InsnID - Instruction ID
+  /// - OpIdx - Operand index
+  GIM_CheckIsImm,
+
   /// Check if the specified operand is safe to fold into the current
   /// instruction.
   /// - InsnID - Instruction ID
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
index bcb1df64a465f..08f2f54bcf90f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h
@@ -690,7 +690,19 @@ bool InstructionSelector::executeMatchTable(
       }
       break;
     }
-
+    case GIM_CheckIsImm: {
+      int64_t InsnID = MatchTable[CurrentIdx++];
+      int64_t OpIdx = MatchTable[CurrentIdx++];
+      DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
+                      dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
+                             << "]->getOperand(" << OpIdx << "))\n");
+      assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
+      if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
+        if (handleReject() == RejectAndGiveUp)
+          return false;
+      }
+      break;
+    }
     case GIM_CheckIsSafeToFold: {
       int64_t InsnID = MatchTable[CurrentIdx++];
       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h
index c3b472e6555c2..290a2381d9c96 100644
--- a/llvm/include/llvm/CodeGen/LiveInterval.h
+++ b/llvm/include/llvm/CodeGen/LiveInterval.h
@@ -189,6 +189,10 @@ namespace llvm {
         return start == Other.start && end == Other.end;
       }
 
+      bool operator!=(const Segment &Other) const {
+        return !(*this == Other);
+      }
+
       void dump() const;
     };
 
diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
index 040a1e078d855..ccdde78a0b22e 100644
--- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h
@@ -105,7 +105,7 @@ class MachineBasicBlock
 
   /// Alignment of the basic block. One if the basic block does not need to be
   /// aligned.
-  llvm::Align Alignment;
+  Align Alignment;
 
   /// Indicate that this basic block is entered via an exception handler.
   bool IsEHPad = false;
@@ -372,17 +372,11 @@ class MachineBasicBlock
   /// \see getBeginClobberMask()
   const uint32_t *getEndClobberMask(const TargetRegisterInfo *TRI) const;
 
-  /// Return alignment of the basic block. The alignment is specified as
-  /// log2(bytes).
-  /// FIXME: Remove the Log versions once migration to llvm::Align is over.
-  unsigned getLogAlignment() const { return Log2(Alignment); }
-  llvm::Align getAlignment() const { return Alignment; }
-
-  /// Set alignment of the basic block. The alignment is specified as
-  /// log2(bytes).
-  /// FIXME: Remove the Log versions once migration to llvm::Align is over.
-  void setLogAlignment(unsigned A) { Alignment = llvm::Align(1ULL << A); }
-  void setAlignment(llvm::Align A) { Alignment = A; }
+  /// Return alignment of the basic block.
+  Align getAlignment() const { return Alignment; }
+
+  /// Set alignment of the basic block.
+  void setAlignment(Align A) { Alignment = A; }
 
   /// Returns true if the block is a landing pad. That is this basic block is
   /// entered via an exception handler.
diff --git a/llvm/include/llvm/CodeGen/MachineDominators.h b/llvm/include/llvm/CodeGen/MachineDominators.h
index d2200080b8978..e4d7a02f8c48e 100644
--- a/llvm/include/llvm/CodeGen/MachineDominators.h
+++ b/llvm/include/llvm/CodeGen/MachineDominators.h
@@ -44,6 +44,8 @@ using MachineDomTreeNode = DomTreeNodeBase<MachineBasicBlock>;
 /// compute a normal dominator tree.
 ///
 class MachineDominatorTree : public MachineFunctionPass {
+  using DomTreeT = DomTreeBase<MachineBasicBlock>;
+
   /// Helper structure used to hold all the basic blocks
   /// involved in the split of a critical edge.
   struct CriticalEdge {
@@ -65,8 +67,8 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// such as BB == elt.NewBB.
   mutable SmallSet<MachineBasicBlock *, 32> NewBBs;
 
-  /// The DominatorTreeBase that is used to compute a normal dominator tree
-  std::unique_ptr<DomTreeBase<MachineBasicBlock>> DT;
+  /// The DominatorTreeBase that is used to compute a normal dominator tree.
+  std::unique_ptr<DomTreeT> DT;
 
   /// Apply all the recorded critical edges to the DT.
   /// This updates the underlying DT information in a way that uses
@@ -80,8 +82,8 @@ class MachineDominatorTree : public MachineFunctionPass {
 
   MachineDominatorTree();
 
-  DomTreeBase<MachineBasicBlock> &getBase() {
-    if (!DT) DT.reset(new DomTreeBase<MachineBasicBlock>());
+  DomTreeT &getBase() {
+    if (!DT) DT.reset(new DomTreeT());
     applySplitCriticalEdges();
     return *DT;
   }
@@ -92,31 +94,30 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// multiple blocks if we are computing post dominators.  For forward
   /// dominators, this will always be a single block (the entry node).
   ///
-  inline const SmallVectorImpl<MachineBasicBlock*> &getRoots() const {
+  const SmallVectorImpl<MachineBasicBlock*> &getRoots() const {
     applySplitCriticalEdges();
     return DT->getRoots();
   }
 
-  inline MachineBasicBlock *getRoot() const {
+  MachineBasicBlock *getRoot() const {
     applySplitCriticalEdges();
     return DT->getRoot();
   }
 
-  inline MachineDomTreeNode *getRootNode() const {
+  MachineDomTreeNode *getRootNode() const {
     applySplitCriticalEdges();
     return DT->getRootNode();
   }
 
   bool runOnMachineFunction(MachineFunction &F) override;
 
-  inline bool dominates(const MachineDomTreeNode* A,
-                        const MachineDomTreeNode* B) const {
+  bool dominates(const MachineDomTreeNode *A,
+                 const MachineDomTreeNode *B) const {
     applySplitCriticalEdges();
     return DT->dominates(A, B);
   }
 
-  inline bool dominates(const MachineBasicBlock* A,
-                        const MachineBasicBlock* B) const {
+  bool dominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const {
     applySplitCriticalEdges();
     return DT->dominates(A, B);
   }
@@ -133,36 +134,30 @@ class MachineDominatorTree : public MachineFunctionPass {
     for (; &*I != A && &*I != B; ++I)
       /*empty*/ ;
 
-    //if(!DT.IsPostDominators) {
-      // A dominates B if it is found first in the basic block.
-      return &*I == A;
-    //} else {
-    //  // A post-dominates B if B is found first in the basic block.
-    //  return &*I == B;
-    //}
+    return &*I == A;
   }
 
-  inline bool properlyDominates(const MachineDomTreeNode* A,
-                                const MachineDomTreeNode* B) const {
+  bool properlyDominates(const MachineDomTreeNode *A,
+                         const MachineDomTreeNode *B) const {
     applySplitCriticalEdges();
     return DT->properlyDominates(A, B);
   }
 
-  inline bool properlyDominates(const MachineBasicBlock* A,
-                                const MachineBasicBlock* B) const {
+  bool properlyDominates(const MachineBasicBlock *A,
+                         const MachineBasicBlock *B) const {
     applySplitCriticalEdges();
     return DT->properlyDominates(A, B);
   }
 
   /// findNearestCommonDominator - Find nearest common dominator basic block
   /// for basic block A and B. If there is no such block then return NULL.
-  inline MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A,
-                                                       MachineBasicBlock *B) {
+  MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A,
+                                                MachineBasicBlock *B) {
     applySplitCriticalEdges();
     return DT->findNearestCommonDominator(A, B);
   }
 
-  inline MachineDomTreeNode *operator[](MachineBasicBlock *BB) const {
+  MachineDomTreeNode *operator[](MachineBasicBlock *BB) const {
     applySplitCriticalEdges();
     return DT->getNode(BB);
   }
@@ -170,7 +165,7 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// getNode - return the (Post)DominatorTree node for the specified basic
   /// block.  This is the same as using operator[] on this class.
   ///
-  inline MachineDomTreeNode *getNode(MachineBasicBlock *BB) const {
+  MachineDomTreeNode *getNode(MachineBasicBlock *BB) const {
     applySplitCriticalEdges();
     return DT->getNode(BB);
   }
@@ -178,8 +173,8 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// addNewBlock - Add a new node to the dominator tree information.  This
   /// creates a new node as a child of DomBB dominator node,linking it into
   /// the children list of the immediate dominator.
-  inline MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB,
-                                         MachineBasicBlock *DomBB) {
+  MachineDomTreeNode *addNewBlock(MachineBasicBlock *BB,
+                                  MachineBasicBlock *DomBB) {
     applySplitCriticalEdges();
     return DT->addNewBlock(BB, DomBB);
   }
@@ -187,14 +182,14 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// changeImmediateDominator - This method is used to update the dominator
   /// tree information when a node's immediate dominator changes.
   ///
-  inline void changeImmediateDominator(MachineBasicBlock *N,
-                                       MachineBasicBlock* NewIDom) {
+  void changeImmediateDominator(MachineBasicBlock *N,
+                                MachineBasicBlock *NewIDom) {
     applySplitCriticalEdges();
     DT->changeImmediateDominator(N, NewIDom);
   }
 
-  inline void changeImmediateDominator(MachineDomTreeNode *N,
-                                       MachineDomTreeNode* NewIDom) {
+  void changeImmediateDominator(MachineDomTreeNode *N,
+                                MachineDomTreeNode *NewIDom) {
     applySplitCriticalEdges();
     DT->changeImmediateDominator(N, NewIDom);
   }
@@ -202,14 +197,14 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// eraseNode - Removes a node from  the dominator tree. Block must not
   /// dominate any other blocks. Removes node from its immediate dominator's
   /// children list. Deletes dominator node associated with basic block BB.
-  inline void eraseNode(MachineBasicBlock *BB) {
+  void eraseNode(MachineBasicBlock *BB) {
     applySplitCriticalEdges();
     DT->eraseNode(BB);
   }
 
   /// splitBlock - BB is split and now it has one successor. Update dominator
   /// tree to reflect this change.
-  inline void splitBlock(MachineBasicBlock* NewBB) {
+  void splitBlock(MachineBasicBlock* NewBB) {
     applySplitCriticalEdges();
     DT->splitBlock(NewBB);
   }
diff --git a/llvm/include/llvm/CodeGen/MachineFrameInfo.h b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
index b4584ece52d65..01fc50d14a7fd 100644
--- a/llvm/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineFrameInfo.h
@@ -181,7 +181,7 @@ class MachineFrameInfo {
 
     uint8_t SSPLayout;
 
-    StackObject(uint64_t Size, llvm::Align Alignment, int64_t SPOffset,
+    StackObject(uint64_t Size, Align Alignment, int64_t SPOffset,
                 bool IsImmutable, bool IsSpillSlot, const AllocaInst *Alloca,
                 bool IsAliased, uint8_t StackID = 0)
         : SPOffset(SPOffset), Size(Size), Alignment(Alignment),
@@ -419,7 +419,9 @@ class MachineFrameInfo {
 
   /// Required alignment of the local object blob,
   /// which is the strictest alignment of any object in it.
-  void setLocalFrameMaxAlign(Align Align) { LocalFrameMaxAlign = Align; }
+  void setLocalFrameMaxAlign(Align Alignment) {
+    LocalFrameMaxAlign = Alignment;
+  }
 
   /// Return the required alignment of the local object blob.
   Align getLocalFrameMaxAlign() const { return LocalFrameMaxAlign; }
@@ -564,7 +566,7 @@ class MachineFrameInfo {
   unsigned getMaxAlignment() const { return MaxAlignment.value(); }
 
   /// Make sure the function is at least Align bytes aligned.
-  void ensureMaxAlignment(llvm::Align Align);
+  void ensureMaxAlignment(Align Alignment);
   /// FIXME: Remove this once transition to Align is over.
   inline void ensureMaxAlignment(unsigned Align) {
     ensureMaxAlignment(assumeAligned(Align));
@@ -732,9 +734,9 @@ class MachineFrameInfo {
 
   /// Create a new statically sized stack object, returning
   /// a nonnegative identifier to represent it.
-  int CreateStackObject(uint64_t Size, llvm::Align Alignment, bool isSpillSlot,
+  int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot,
                         const AllocaInst *Alloca = nullptr, uint8_t ID = 0);
-  /// FIXME: Remove this function when transition to llvm::Align is over.
+  /// FIXME: Remove this function when transition to Align is over.
   inline int CreateStackObject(uint64_t Size, unsigned Alignment,
                                bool isSpillSlot,
                                const AllocaInst *Alloca = nullptr,
@@ -745,8 +747,8 @@ class MachineFrameInfo {
 
   /// Create a new statically sized stack object that represents a spill slot,
   /// returning a nonnegative identifier to represent it.
-  int CreateSpillStackObject(uint64_t Size, llvm::Align Alignment);
-  /// FIXME: Remove this function when transition to llvm::Align is over.
+  int CreateSpillStackObject(uint64_t Size, Align Alignment);
+  /// FIXME: Remove this function when transition to Align is over.
   inline int CreateSpillStackObject(uint64_t Size, unsigned Alignment) {
     return CreateSpillStackObject(Size, assumeAligned(Alignment));
   }
@@ -760,9 +762,8 @@ class MachineFrameInfo {
   /// Notify the MachineFrameInfo object that a variable sized object has been
   /// created.  This must be created whenever a variable sized object is
   /// created, whether or not the index returned is actually used.
-  int CreateVariableSizedObject(llvm::Align Alignment,
-                                const AllocaInst *Alloca);
-  /// FIXME: Remove this function when transition to llvm::Align is over.
+  int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca);
+  /// FIXME: Remove this function when transition to Align is over.
   int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca) {
     return CreateVariableSizedObject(assumeAligned(Alignment), Alloca);
   }
diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h
index be685afdba066..adf3f4c0e2f7e 100644
--- a/llvm/include/llvm/CodeGen/MachineFunction.h
+++ b/llvm/include/llvm/CodeGen/MachineFunction.h
@@ -277,7 +277,7 @@ class MachineFunction {
   unsigned FunctionNumber;
 
   /// Alignment - The alignment of the function.
-  llvm::Align Alignment;
+  Align Alignment;
 
   /// ExposesReturnsTwice - True if the function calls setjmp or related
   /// functions with attribute "returns twice", but doesn't have
@@ -509,13 +509,13 @@ class MachineFunction {
   WinEHFuncInfo *getWinEHFuncInfo() { return WinEHInfo; }
 
   /// getAlignment - Return the alignment of the function.
-  llvm::Align getAlignment() const { return Alignment; }
+  Align getAlignment() const { return Alignment; }
 
   /// setAlignment - Set the alignment of the function.
-  void setAlignment(llvm::Align A) { Alignment = A; }
+  void setAlignment(Align A) { Alignment = A; }
 
   /// ensureAlignment - Make sure the function is at least A bytes aligned.
-  void ensureAlignment(llvm::Align A) {
+  void ensureAlignment(Align A) {
     if (Alignment < A)
       Alignment = A;
   }
diff --git a/llvm/include/llvm/CodeGen/MachineLoopUtils.h b/llvm/include/llvm/CodeGen/MachineLoopUtils.h
new file mode 100644
index 0000000000000..41379b75d00a6
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/MachineLoopUtils.h
@@ -0,0 +1,41 @@
+//=- MachineLoopUtils.h - Helper functions for manipulating loops -*- C++ -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H
+#define LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H
+
+namespace llvm {
+class MachineBasicBlock;
+class MachineRegisterInfo;
+class TargetInstrInfo;
+
+enum LoopPeelDirection {
+  LPD_Front, ///< Peel the first iteration of the loop.
+  LPD_Back   ///< Peel the last iteration of the loop.
+};
+
+/// Peels a single block loop. Loop must have two successors, one of which
+/// must be itself. Similarly it must have two predecessors, one of which must
+/// be itself.
+///
+/// The loop block is copied and inserted into the CFG such that two copies of
+/// the loop follow on from each other. The copy is inserted either before or
+/// after the loop based on Direction.
+///
+/// Phis are updated and an unconditional branch inserted at the end of the
+/// clone so as to execute a single iteration.
+///
+/// The trip count of Loop is not updated.
+MachineBasicBlock *PeelSingleBlockLoop(LoopPeelDirection Direction,
+                                       MachineBasicBlock *Loop,
+                                       MachineRegisterInfo &MRI,
+                                       const TargetInstrInfo *TII);
+
+} // namespace llvm
+
+#endif // LLVM_LIB_CODEGEN_MACHINELOOPUTILS_H
diff --git a/llvm/include/llvm/CodeGen/MachineMemOperand.h b/llvm/include/llvm/CodeGen/MachineMemOperand.h
index 65f706302bc2e..33a48a235e18c 100644
--- a/llvm/include/llvm/CodeGen/MachineMemOperand.h
+++ b/llvm/include/llvm/CodeGen/MachineMemOperand.h
@@ -293,8 +293,6 @@ class MachineMemOperand {
 
   /// Support for operator<<.
   /// @{
-  void print(raw_ostream &OS) const;
-  void print(raw_ostream &OS, ModuleSlotTracker &MST) const;
   void print(raw_ostream &OS, ModuleSlotTracker &MST,
              SmallVectorImpl<StringRef> &SSNs, const LLVMContext &Context,
              const MachineFrameInfo *MFI, const TargetInstrInfo *TII) const;
@@ -319,11 +317,6 @@ class MachineMemOperand {
   }
 };
 
-inline raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO) {
-  MRO.print(OS);
-  return OS;
-}
-
 } // End llvm namespace
 
 #endif
diff --git a/llvm/include/llvm/CodeGen/MachineModuleInfo.h b/llvm/include/llvm/CodeGen/MachineModuleInfo.h
index 4ff5c7fd013a1..6902dada2423a 100644
--- a/llvm/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineModuleInfo.h
@@ -33,6 +33,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Pass.h"
@@ -74,7 +75,10 @@ class MachineModuleInfoImpl {
 /// made by different debugging and exception handling schemes and reformated
 /// for specific use.
 ///
-class MachineModuleInfo : public ImmutablePass {
+class MachineModuleInfo {
+  friend class MachineModuleInfoWrapperPass;
+  friend class MachineModuleAnalysis;
+
   const LLVMTargetMachine &TM;
 
   /// This is the MCContext used for the entire code generator.
@@ -140,15 +144,17 @@ class MachineModuleInfo : public ImmutablePass {
   const Function *LastRequest = nullptr; ///< Used for shortcut/cache.
   MachineFunction *LastResult = nullptr; ///< Used for shortcut/cache.
 
-public:
-  static char ID; // Pass identification, replacement for typeid
+  MachineModuleInfo &operator=(MachineModuleInfo &&MMII) = delete;
 
+public:
   explicit MachineModuleInfo(const LLVMTargetMachine *TM = nullptr);
-  ~MachineModuleInfo() override;
 
-  // Initialization and Finalization
-  bool doInitialization(Module &) override;
-  bool doFinalization(Module &) override;
+  MachineModuleInfo(MachineModuleInfo &&MMII);
+
+  ~MachineModuleInfo();
+
+  void initialize();
+  void finalize();
 
   const LLVMTargetMachine &getTarget() const { return TM; }
 
@@ -254,6 +260,38 @@ class MachineModuleInfo : public ImmutablePass {
   /// \}
 }; // End class MachineModuleInfo
 
+class MachineModuleInfoWrapperPass : public ImmutablePass {
+  MachineModuleInfo MMI;
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  explicit MachineModuleInfoWrapperPass(const LLVMTargetMachine *TM = nullptr);
+
+  // Initialization and Finalization
+  bool doInitialization(Module &) override;
+  bool doFinalization(Module &) override;
+
+  MachineModuleInfo &getMMI() { return MMI; }
+  const MachineModuleInfo &getMMI() const { return MMI; }
+};
+
+/// An analysis that produces \c MachineInfo for a module.
+class MachineModuleAnalysis : public AnalysisInfoMixin<MachineModuleAnalysis> {
+  friend AnalysisInfoMixin<MachineModuleAnalysis>;
+  static AnalysisKey Key;
+
+  const LLVMTargetMachine *TM;
+
+public:
+  /// Provide the result type for this analysis pass.
+  using Result = MachineModuleInfo;
+
+  MachineModuleAnalysis(const LLVMTargetMachine *TM) : TM(TM) {}
+
+  /// Run the analysis pass and produce machine module information.
+  MachineModuleInfo run(Module &M, ModuleAnalysisManager &);
+};
+
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_MACHINEMODULEINFO_H
diff --git a/llvm/include/llvm/CodeGen/MachinePostDominators.h b/llvm/include/llvm/CodeGen/MachinePostDominators.h
index b67e6b52ac8fe..cb258b5e7b217 100644
--- a/llvm/include/llvm/CodeGen/MachinePostDominators.h
+++ b/llvm/include/llvm/CodeGen/MachinePostDominators.h
@@ -16,68 +16,76 @@
 
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include <memory>
 
 namespace llvm {
 
 ///
-/// PostDominatorTree Class - Concrete subclass of DominatorTree that is used
-/// to compute the post-dominator tree.
+/// MachinePostDominatorTree - an analysis pass wrapper for DominatorTree
+/// used to compute the post-dominator tree for MachineFunctions.
 ///
-struct MachinePostDominatorTree : public MachineFunctionPass {
-private:
- PostDomTreeBase<MachineBasicBlock> *DT;
+class MachinePostDominatorTree : public MachineFunctionPass {
+  using PostDomTreeT = PostDomTreeBase<MachineBasicBlock>;
+  std::unique_ptr<PostDomTreeT> PDT;
 
 public:
   static char ID;
 
   MachinePostDominatorTree();
 
-  ~MachinePostDominatorTree() override;
-
   FunctionPass *createMachinePostDominatorTreePass();
 
   const SmallVectorImpl<MachineBasicBlock *> &getRoots() const {
-    return DT->getRoots();
+    return PDT->getRoots();
   }
 
-  MachineDomTreeNode *getRootNode() const {
-    return DT->getRootNode();
-  }
+  MachineDomTreeNode *getRootNode() const { return PDT->getRootNode(); }
 
   MachineDomTreeNode *operator[](MachineBasicBlock *BB) const {
-    return DT->getNode(BB);
+    return PDT->getNode(BB);
   }
 
   MachineDomTreeNode *getNode(MachineBasicBlock *BB) const {
-    return DT->getNode(BB);
+    return PDT->getNode(BB);
   }
 
   bool dominates(const MachineDomTreeNode *A,
                  const MachineDomTreeNode *B) const {
-    return DT->dominates(A, B);
+    return PDT->dominates(A, B);
   }
 
   bool dominates(const MachineBasicBlock *A, const MachineBasicBlock *B) const {
-    return DT->dominates(A, B);
+    return PDT->dominates(A, B);
   }
 
   bool properlyDominates(const MachineDomTreeNode *A,
                          const MachineDomTreeNode *B) const {
-    return DT->properlyDominates(A, B);
+    return PDT->properlyDominates(A, B);
   }
 
   bool properlyDominates(const MachineBasicBlock *A,
                          const MachineBasicBlock *B) const {
-    return DT->properlyDominates(A, B);
+    return PDT->properlyDominates(A, B);
+  }
+
+  bool isVirtualRoot(const MachineDomTreeNode *Node) const {
+    return PDT->isVirtualRoot(Node);
   }
 
   MachineBasicBlock *findNearestCommonDominator(MachineBasicBlock *A,
-                                                MachineBasicBlock *B) {
-    return DT->findNearestCommonDominator(A, B);
+                                                MachineBasicBlock *B) const {
+    return PDT->findNearestCommonDominator(A, B);
   }
 
+  /// Returns the nearest common dominator of the given blocks.
+  /// If that tree node is a virtual root, a nullptr will be returned.
+  MachineBasicBlock *
+  findNearestCommonDominator(ArrayRef<MachineBasicBlock *> Blocks) const;
+
   bool runOnMachineFunction(MachineFunction &MF) override;
   void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void releaseMemory() override { PDT.reset(nullptr); }
+  void verifyAnalysis() const override;
   void print(llvm::raw_ostream &OS, const Module *M = nullptr) const override;
 };
 } //end of namespace llvm
diff --git a/llvm/include/llvm/CodeGen/MachineRegionInfo.h b/llvm/include/llvm/CodeGen/MachineRegionInfo.h
index 6d9fb9b9100a8..eeb69fef2c6b9 100644
--- a/llvm/include/llvm/CodeGen/MachineRegionInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineRegionInfo.h
@@ -22,7 +22,7 @@
 
 namespace llvm {
 
-struct MachinePostDominatorTree;
+class MachinePostDominatorTree;
 class MachineRegion;
 class MachineRegionNode;
 class MachineRegionInfo;
diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h
index 75a334f61ad05..333367943ac06 100644
--- a/llvm/include/llvm/CodeGen/MachineScheduler.h
+++ b/llvm/include/llvm/CodeGen/MachineScheduler.h
@@ -100,6 +100,7 @@ namespace llvm {
 
 extern cl::opt<bool> ForceTopDown;
 extern cl::opt<bool> ForceBottomUp;
+extern cl::opt<bool> VerifyScheduling;
 
 class LiveIntervals;
 class MachineDominatorTree;
diff --git a/llvm/include/llvm/CodeGen/ModuloSchedule.h b/llvm/include/llvm/CodeGen/ModuloSchedule.h
index 530cefd1c81a7..81a9b63b64ca3 100644
--- a/llvm/include/llvm/CodeGen/ModuloSchedule.h
+++ b/llvm/include/llvm/CodeGen/ModuloSchedule.h
@@ -62,7 +62,10 @@
 
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineLoopUtils.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include <deque>
 #include <vector>
 
 namespace llvm {
@@ -141,9 +144,7 @@ class ModuloSchedule {
   /// Return the rescheduled instructions in order.
   ArrayRef<MachineInstr *> getInstructions() { return ScheduledInstrs; }
 
-  void dump() {
-    print(dbgs());
-  }
+  void dump() { print(dbgs()); }
   void print(raw_ostream &OS);
 };
 
@@ -168,6 +169,7 @@ class ModuloScheduleExpander {
   MachineBasicBlock *BB;
   MachineBasicBlock *Preheader;
   MachineBasicBlock *NewKernel = nullptr;
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> LoopInfo;
 
   /// Map for each register and the max difference between its uses and def.
   /// The first element in the pair is the max difference in stages. The
@@ -268,9 +270,6 @@ class ModuloScheduleExpander {
 
 /// A reimplementation of ModuloScheduleExpander. It works by generating a
 /// standalone kernel loop and peeling out the prologs and epilogs.
-///
-/// FIXME: This implementation cannot yet generate valid code. It can generate
-/// a correct kernel but cannot peel out prologs and epilogs.
 class PeelingModuloScheduleExpander {
   ModuloSchedule &Schedule;
   MachineFunction &MF;
@@ -279,17 +278,70 @@ class PeelingModuloScheduleExpander {
   const TargetInstrInfo *TII;
   LiveIntervals *LIS;
 
+  /// The original loop block that gets rewritten in-place.
   MachineBasicBlock *BB;
+  /// The original loop preheader.
   MachineBasicBlock *Preheader;
+  /// All prolog and epilog blocks.
+  SmallVector<MachineBasicBlock *, 4> Prologs, Epilogs;
+  /// For every block, the stages that are produced.
+  DenseMap<MachineBasicBlock *, BitVector> LiveStages;
+  /// For every block, the stages that are available. A stage can be available
+  /// but not produced (in the epilog) or produced but not available (in the
+  /// prolog).
+  DenseMap<MachineBasicBlock *, BitVector> AvailableStages;
+
+  /// CanonicalMIs and BlockMIs form a bidirectional map between any of the
+  /// loop kernel clones.
+  DenseMap<MachineInstr *, MachineInstr *> CanonicalMIs;
+  DenseMap<std::pair<MachineBasicBlock *, MachineInstr *>, MachineInstr *>
+      BlockMIs;
+
+  /// State passed from peelKernel to peelPrologAndEpilogs().
+  std::deque<MachineBasicBlock *> PeeledFront, PeeledBack;
+
 public:
   PeelingModuloScheduleExpander(MachineFunction &MF, ModuloSchedule &S,
                                 LiveIntervals *LIS)
       : Schedule(S), MF(MF), ST(MF.getSubtarget()), MRI(MF.getRegInfo()),
         TII(ST.getInstrInfo()), LIS(LIS) {}
 
+  void expand();
+
   /// Runs ModuloScheduleExpander and treats it as a golden input to validate
   /// aspects of the code generated by PeelingModuloScheduleExpander.
   void validateAgainstModuloScheduleExpander();
+
+protected:
+  /// Converts BB from the original loop body to the rewritten, pipelined
+  /// steady-state.
+  void rewriteKernel();
+
+private:
+  /// Peels one iteration of the rewritten kernel (BB) in the specified
+  /// direction.
+  MachineBasicBlock *peelKernel(LoopPeelDirection LPD);
+  /// Peel the kernel forwards and backwards to produce prologs and epilogs,
+  /// and stitch them together.
+  void peelPrologAndEpilogs();
+  /// All prolog and epilog blocks are clones of the kernel, so any produced
+  /// register in one block has an corollary in all other blocks.
+  Register getEquivalentRegisterIn(Register Reg, MachineBasicBlock *BB);
+  /// Change all users of MI, if MI is predicated out
+  /// (LiveStages[MI->getParent()] == false).
+  void rewriteUsesOf(MachineInstr *MI);
+  /// Insert branches between prologs, kernel and epilogs.
+  void fixupBranches();
+  /// Create a poor-man's LCSSA by cloning only the PHIs from the kernel block
+  /// to a block dominated by all prologs and epilogs. This allows us to treat
+  /// the loop exiting block as any other kernel clone.
+  MachineBasicBlock *CreateLCSSAExitingBlock();
+  /// Helper to get the stage of an instruction in the schedule.
+  unsigned getStage(MachineInstr *MI) {
+    if (CanonicalMIs.count(MI))
+      MI = CanonicalMIs[MI];
+    return Schedule.getStage(MI);
+  }
 };
 
 /// Expander that simply annotates each scheduled instruction with a post-instr
diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 0ddcca0932ec0..eaa55c911b812 100644
--- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -374,6 +374,9 @@ namespace llvm {
     /// Returns a mask for which lanes get read/written by the given (register)
     /// machine operand.
     LaneBitmask getLaneMaskForMO(const MachineOperand &MO) const;
+
+    /// Returns true if the def register in \p MO has no uses.
+    bool deadDefHasNoUse(const MachineOperand &MO);
   };
 
   /// Creates a new SUnit and return a ptr to it.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 8b8436f9eaab6..be6e2bd7d6526 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -388,7 +388,11 @@ class SelectionDAG {
     Node->OperandList = nullptr;
   }
   void CreateTopologicalOrder(std::vector<SDNode*>& Order);
+
 public:
+  // Maximum depth for recursive analysis such as computeKnownBits, etc.
+  static constexpr unsigned MaxRecursionDepth = 6;
+
   explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level);
   SelectionDAG(const SelectionDAG &) = delete;
   SelectionDAG &operator=(const SelectionDAG &) = delete;
@@ -1037,7 +1041,7 @@ class SelectionDAG {
     unsigned Align = 0,
     MachineMemOperand::Flags Flags
     = MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
-    unsigned Size = 0,
+    uint64_t Size = 0,
     const AAMDNodes &AAInfo = AAMDNodes());
 
   SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
diff --git a/llvm/include/llvm/CodeGen/StackProtector.h b/llvm/include/llvm/CodeGen/StackProtector.h
index 2bdf4425e24a3..ed52db3e6269b 100644
--- a/llvm/include/llvm/CodeGen/StackProtector.h
+++ b/llvm/include/llvm/CodeGen/StackProtector.h
@@ -61,6 +61,12 @@ class StackProtector : public FunctionPass {
   /// protection when -fstack-protection is used.
   unsigned SSPBufferSize = 0;
 
+  /// VisitedPHIs - The set of PHI nodes visited when determining
+  /// if a variable's reference has been taken.  This set
+  /// is maintained to ensure we don't visit the same PHI node multiple
+  /// times.
+  SmallPtrSet<const PHINode *, 16> VisitedPHIs;
+
   // A prologue is generated.
   bool HasPrologue = false;
 
diff --git a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
index 31b5f794d90c6..b8adcf759b197 100644
--- a/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
+++ b/llvm/include/llvm/CodeGen/SwitchLoweringUtils.h
@@ -212,13 +212,14 @@ struct BitTestBlock {
   BitTestInfo Cases;
   BranchProbability Prob;
   BranchProbability DefaultProb;
+  bool OmitRangeCheck;
 
   BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, bool E,
                bool CR, MachineBasicBlock *P, MachineBasicBlock *D,
                BitTestInfo C, BranchProbability Pr)
       : First(std::move(F)), Range(std::move(R)), SValue(SV), Reg(Rg),
         RegVT(RgVT), Emitted(E), ContiguousRange(CR), Parent(P), Default(D),
-        Cases(std::move(C)), Prob(Pr) {}
+        Cases(std::move(C)), Prob(Pr), OmitRangeCheck(false) {}
 };
 
 /// Return the range of values within a range.
diff --git a/llvm/include/llvm/CodeGen/TargetCallingConv.h b/llvm/include/llvm/CodeGen/TargetCallingConv.h
index d25a9a24591fc..360fc51bd0415 100644
--- a/llvm/include/llvm/CodeGen/TargetCallingConv.h
+++ b/llvm/include/llvm/CodeGen/TargetCallingConv.h
@@ -126,7 +126,7 @@ namespace ISD {
       return A ? A->value() : 0;
     }
     void setByValAlign(unsigned A) {
-      ByValAlign = encode(llvm::Align(A));
+      ByValAlign = encode(Align(A));
       assert(getByValAlign() == A && "bitfield overflow");
     }
 
@@ -135,7 +135,7 @@ namespace ISD {
       return A ? A->value() : 0;
     }
     void setOrigAlign(unsigned A) {
-      OrigAlign = encode(llvm::Align(A));
+      OrigAlign = encode(Align(A));
       assert(getOrigAlign() == A && "bitfield overflow");
     }
 
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 9384bbcca33a5..9afd85237621f 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/MachineCombinerPattern.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineOperand.h"
 #include "llvm/CodeGen/MachineOutliner.h"
@@ -421,7 +422,8 @@ class TargetInstrInfo : public MCInstrInfo {
   ///     findCommutedOpIndices(MI, Op1, Op2);
   /// can be interpreted as a query asking to find an operand that would be
   /// commutable with the operand#1.
-  virtual bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+  virtual bool findCommutedOpIndices(const MachineInstr &MI,
+                                     unsigned &SrcOpIdx1,
                                      unsigned &SrcOpIdx2) const;
 
   /// A pair composed of a register and a sub-register index.
@@ -661,6 +663,50 @@ class TargetInstrInfo : public MCInstrInfo {
                         BytesAdded);
   }
 
+  /// Object returned by analyzeLoopForPipelining. Allows software pipelining
+  /// implementations to query attributes of the loop being pipelined and to
+  /// apply target-specific updates to the loop once pipelining is complete.
+  class PipelinerLoopInfo {
+  public:
+    virtual ~PipelinerLoopInfo();
+    /// Return true if the given instruction should not be pipelined and should
+    /// be ignored. An example could be a loop comparison, or induction variable
+    /// update with no users being pipelined.
+    virtual bool shouldIgnoreForPipelining(const MachineInstr *MI) const = 0;
+
+    /// Create a condition to determine if the trip count of the loop is greater
+    /// than TC.
+    ///
+    /// If the trip count is statically known to be greater than TC, return
+    /// true. If the trip count is statically known to be not greater than TC,
+    /// return false. Otherwise return nullopt and fill out Cond with the test
+    /// condition.
+    virtual Optional<bool>
+    createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
+                                    SmallVectorImpl<MachineOperand> &Cond) = 0;
+
+    /// Modify the loop such that the trip count is
+    /// OriginalTC + TripCountAdjust.
+    virtual void adjustTripCount(int TripCountAdjust) = 0;
+
+    /// Called when the loop's preheader has been modified to NewPreheader.
+    virtual void setPreheader(MachineBasicBlock *NewPreheader) = 0;
+
+    /// Called when the loop is being removed. Any instructions in the preheader
+    /// should be removed.
+    ///
+    /// Once this function is called, no other functions on this object are
+    /// valid; the loop has been removed.
+    virtual void disposed() = 0;
+  };
+
+  /// Analyze loop L, which must be a single-basic-block loop, and if the
+  /// conditions can be understood enough produce a PipelinerLoopInfo object.
+  virtual std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+    return nullptr;
+  }
+
   /// Analyze the loop code, return true if it cannot be understoo. Upon
   /// success, this function returns false and returns information about the
   /// induction variable and compare instruction used at the end.
@@ -1560,8 +1606,7 @@ class TargetInstrInfo : public MCInstrInfo {
   /// function.
   virtual bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const {
+                                  const MachineInstr &MIb) const {
     assert((MIa.mayLoad() || MIa.mayStore()) &&
            "MIa must load from or modify a memory location");
     assert((MIb.mayLoad() || MIb.mayStore()) &&
@@ -1638,6 +1683,28 @@ class TargetInstrInfo : public MCInstrInfo {
     return false;
   }
 
+  /// During PHI eleimination lets target to make necessary checks and
+  /// insert the copy to the PHI destination register in a target specific
+  /// manner.
+  virtual MachineInstr *createPHIDestinationCopy(
+      MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
+      const DebugLoc &DL, Register Src, Register Dst) const {
+    return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst)
+        .addReg(Src);
+  }
+
+  /// During PHI eleimination lets target to make necessary checks and
+  /// insert the copy to the PHI destination register in a target specific
+  /// manner.
+  virtual MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
+                                            MachineBasicBlock::iterator InsPt,
+                                            const DebugLoc &DL, Register Src,
+                                            Register SrcSubReg,
+                                            Register Dst) const {
+    return BuildMI(MBB, InsPt, DL, get(TargetOpcode::COPY), Dst)
+        .addReg(Src, 0, SrcSubReg);
+  }
+
   /// Returns a \p outliner::OutlinedFunction struct containing target-specific
   /// information for a set of outlining candidates.
   virtual outliner::OutlinedFunction getOutliningCandidateInfo(
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index cf90f3e5c6aee..999c0ea30223e 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -73,6 +73,7 @@ class Constant;
 class FastISel;
 class FunctionLoweringInfo;
 class GlobalValue;
+class GISelKnownBits;
 class IntrinsicInst;
 struct KnownBits;
 class LLVMContext;
@@ -836,9 +837,9 @@ class TargetLoweringBase {
     PointerUnion<const Value *, const PseudoSourceValue *> ptrVal;
 
     int          offset = 0;       // offset off of ptrVal
-    unsigned     size = 0;         // the size of the memory location
+    uint64_t     size = 0;         // the size of the memory location
                                    // (taken from memVT if zero)
-    MaybeAlign align = Align(1);   // alignment
+    MaybeAlign align = Align::None(); // alignment
 
     MachineMemOperand::Flags flags = MachineMemOperand::MONone;
     IntrinsicInfo() = default;
@@ -1470,11 +1471,30 @@ class TargetLoweringBase {
     return false;
   }
 
+  /// This function returns true if the memory access is aligned or if the
+  /// target allows this specific unaligned memory access. If the access is
+  /// allowed, the optional final parameter returns if the access is also fast
+  /// (as defined by the target).
+  bool allowsMemoryAccessForAlignment(
+      LLVMContext &Context, const DataLayout &DL, EVT VT,
+      unsigned AddrSpace = 0, unsigned Alignment = 1,
+      MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
+      bool *Fast = nullptr) const;
+
+  /// Return true if the memory access of this type is aligned or if the target
+  /// allows this specific unaligned access for the given MachineMemOperand.
+  /// If the access is allowed, the optional final parameter returns if the
+  /// access is also fast (as defined by the target).
+  bool allowsMemoryAccessForAlignment(LLVMContext &Context,
+                                      const DataLayout &DL, EVT VT,
+                                      const MachineMemOperand &MMO,
+                                      bool *Fast = nullptr) const;
+
   /// Return true if the target supports a memory access of this type for the
   /// given address space and alignment. If the access is allowed, the optional
   /// final parameter returns if the access is also fast (as defined by the
   /// target).
-  bool
+  virtual bool
   allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
                      unsigned AddrSpace = 0, unsigned Alignment = 1,
                      MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
@@ -1577,18 +1597,18 @@ class TargetLoweringBase {
   }
 
   /// Return the minimum stack alignment of an argument.
-  llvm::Align getMinStackArgumentAlignment() const {
+  Align getMinStackArgumentAlignment() const {
     return MinStackArgumentAlignment;
   }
 
   /// Return the minimum function alignment.
-  llvm::Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
+  Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
 
   /// Return the preferred function alignment.
-  llvm::Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
+  Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
 
   /// Return the preferred loop alignment.
-  virtual llvm::Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const {
+  virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const {
     return PrefLoopAlignment;
   }
 
@@ -2101,24 +2121,24 @@ class TargetLoweringBase {
   }
 
   /// Set the target's minimum function alignment.
-  void setMinFunctionAlignment(llvm::Align Align) {
-    MinFunctionAlignment = Align;
+  void setMinFunctionAlignment(Align Alignment) {
+    MinFunctionAlignment = Alignment;
   }
 
   /// Set the target's preferred function alignment.  This should be set if
   /// there is a performance benefit to higher-than-minimum alignment
-  void setPrefFunctionAlignment(llvm::Align Align) {
-    PrefFunctionAlignment = Align;
+  void setPrefFunctionAlignment(Align Alignment) {
+    PrefFunctionAlignment = Alignment;
   }
 
   /// Set the target's preferred loop alignment. Default alignment is one, it
   /// means the target does not care about loop alignment. The target may also
   /// override getPrefLoopAlignment to provide per-loop values.
-  void setPrefLoopAlignment(llvm::Align Align) { PrefLoopAlignment = Align; }
+  void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; }
 
   /// Set the minimum stack alignment of an argument.
-  void setMinStackArgumentAlignment(llvm::Align Align) {
-    MinStackArgumentAlignment = Align;
+  void setMinStackArgumentAlignment(Align Alignment) {
+    MinStackArgumentAlignment = Alignment;
   }
 
   /// Set the maximum atomic operation size supported by the
@@ -2680,18 +2700,18 @@ class TargetLoweringBase {
   Sched::Preference SchedPreferenceInfo;
 
   /// The minimum alignment that any argument on the stack needs to have.
-  llvm::Align MinStackArgumentAlignment;
+  Align MinStackArgumentAlignment;
 
   /// The minimum function alignment (used when optimizing for size, and to
   /// prevent explicitly provided alignment from leading to incorrect code).
-  llvm::Align MinFunctionAlignment;
+  Align MinFunctionAlignment;
 
   /// The preferred function alignment (used when alignment unspecified and
   /// optimizing for speed).
-  llvm::Align PrefFunctionAlignment;
+  Align PrefFunctionAlignment;
 
   /// The preferred loop alignment (in log2 bot in bytes).
-  llvm::Align PrefLoopAlignment;
+  Align PrefLoopAlignment;
 
   /// Size in bits of the maximum atomics size the backend supports.
   /// Accesses larger than this will be expanded by AtomicExpandPass.
@@ -3148,7 +3168,8 @@ class TargetLowering : public TargetLoweringBase {
   /// or one and return them in the KnownZero/KnownOne bitsets. The DemandedElts
   /// argument allows us to only collect the known bits that are shared by the
   /// requested vector elements. This is for GISel.
-  virtual void computeKnownBitsForTargetInstr(Register R, KnownBits &Known,
+  virtual void computeKnownBitsForTargetInstr(GISelKnownBits &Analysis,
+                                              Register R, KnownBits &Known,
                                               const APInt &DemandedElts,
                                               const MachineRegisterInfo &MRI,
                                               unsigned Depth = 0) const;
@@ -3365,6 +3386,18 @@ class TargetLowering : public TargetLoweringBase {
     llvm_unreachable("Not Implemented");
   }
 
+  /// Return 1 if we can compute the negated form of the specified expression
+  /// for the same cost as the expression itself, or 2 if we can compute the
+  /// negated form more cheaply than the expression itself. Else return 0.
+  virtual char isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
+                                  bool LegalOperations, bool ForCodeSize,
+                                  unsigned Depth = 0) const;
+
+  /// If isNegatibleForFree returns true, return the newly negated expression.
+  virtual SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG,
+                                       bool LegalOperations, bool ForCodeSize,
+                                       unsigned Depth = 0) const;
+
   //===--------------------------------------------------------------------===//
   // Lowering methods - These methods must be implemented by targets so that
   // the SelectionDAGBuilder code knows how to lower these.
@@ -3650,8 +3683,8 @@ class TargetLowering : public TargetLoweringBase {
   /// Return the register ID of the name passed in. Used by named register
   /// global variables extension. There is no target-independent behaviour
   /// so the default action is to bail.
-  virtual unsigned getRegisterByName(const char* RegName, EVT VT,
-                                     SelectionDAG &DAG) const {
+  virtual Register getRegisterByName(const char* RegName, EVT VT,
+                                     const MachineFunction &MF) const {
     report_fatal_error("Named registers not implemented for this target");
   }
 
diff --git a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
index b24828152e5b9..56018eca8c273 100644
--- a/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetSubtargetInfo.h
@@ -110,8 +110,6 @@ class TargetSubtargetInfo : public MCSubtargetInfo {
     return nullptr;
   }
 
-  virtual unsigned getHwMode() const { return 0; }
-
   /// Target can subclass this hook to select a different DAG scheduler.
   virtual RegisterScheduler::FunctionPassCtor
       getDAGScheduler(CodeGenOpt::Level) const {
diff --git a/llvm/include/llvm/CodeGen/ValueTypes.td b/llvm/include/llvm/CodeGen/ValueTypes.td
index 3293c29a05fb9..16df565bc8b8d 100644
--- a/llvm/include/llvm/CodeGen/ValueTypes.td
+++ b/llvm/include/llvm/CodeGen/ValueTypes.td
@@ -40,131 +40,132 @@ def v16i1  : ValueType<16,  18>;   //  16 x i1 vector value
 def v32i1  : ValueType<32 , 19>;   //  32 x i1 vector value
 def v64i1  : ValueType<64 , 20>;   //  64 x i1 vector value
 def v128i1 : ValueType<128, 21>;   // 128 x i1 vector value
-def v512i1 : ValueType<512, 22>;   // 512 x i1 vector value
-def v1024i1: ValueType<1024,23>;   //1024 x i1 vector value
-
-def v1i8   : ValueType<8,   24>;   //  1 x i8  vector value
-def v2i8   : ValueType<16 , 25>;   //  2 x i8  vector value
-def v4i8   : ValueType<32 , 26>;   //  4 x i8  vector value
-def v8i8   : ValueType<64 , 27>;   //  8 x i8  vector value
-def v16i8  : ValueType<128, 28>;   // 16 x i8  vector value
-def v32i8  : ValueType<256, 29>;   // 32 x i8  vector value
-def v64i8  : ValueType<512, 30>;   // 64 x i8  vector value
-def v128i8 : ValueType<1024,31>;   //128 x i8  vector value
-def v256i8 : ValueType<2048,32>;   //256 x i8  vector value
-
-def v1i16  : ValueType<16 , 33>;   //  1 x i16 vector value
-def v2i16  : ValueType<32 , 34>;   //  2 x i16 vector value
-def v3i16  : ValueType<48 , 35>;   //  3 x i16 vector value
-def v4i16  : ValueType<64 , 36>;   //  4 x i16 vector value
-def v8i16  : ValueType<128, 37>;   //  8 x i16 vector value
-def v16i16 : ValueType<256, 38>;   // 16 x i16 vector value
-def v32i16 : ValueType<512, 39>;   // 32 x i16 vector value
-def v64i16 : ValueType<1024,40>;   // 64 x i16 vector value
-def v128i16: ValueType<2048,41>;   //128 x i16 vector value
-
-def v1i32    : ValueType<32 , 42>;   //  1 x i32 vector value
-def v2i32    : ValueType<64 , 43>;   //  2 x i32 vector value
-def v3i32    : ValueType<96 , 44>;   //  3 x i32 vector value
-def v4i32    : ValueType<128, 45>;   //  4 x i32 vector value
-def v5i32    : ValueType<160, 46>;   //  5 x i32 vector value
-def v8i32    : ValueType<256, 47>;   //  8 x i32 vector value
-def v16i32   : ValueType<512, 48>;   // 16 x i32 vector value
-def v32i32   : ValueType<1024,49>;   // 32 x i32 vector value
-def v64i32   : ValueType<2048,50>;   // 64 x i32 vector value
-def v128i32  : ValueType<4096,51>;   // 128 x i32 vector value
-def v256i32  : ValueType<8182,52>;   // 256 x i32 vector value
-def v512i32  : ValueType<16384,53>;  // 512 x i32 vector value
-def v1024i32 : ValueType<32768,54>;  // 1024 x i32 vector value
-def v2048i32 : ValueType<65536,55>;  // 2048 x i32 vector value
-
-def v1i64  : ValueType<64 , 56>;   //  1 x i64 vector value
-def v2i64  : ValueType<128, 57>;   //  2 x i64 vector value
-def v4i64  : ValueType<256, 58>;   //  4 x i64 vector value
-def v8i64  : ValueType<512, 59>;   //  8 x i64 vector value
-def v16i64 : ValueType<1024,60>;   // 16 x i64 vector value
-def v32i64 : ValueType<2048,61>;   // 32 x i64 vector value
-
-def v1i128 : ValueType<128, 62>;   //  1 x i128 vector value
-
-def nxv1i1  : ValueType<1,   63>;  // n x  1 x i1  vector value
-def nxv2i1  : ValueType<2,   64>;  // n x  2 x i1  vector value
-def nxv4i1  : ValueType<4,   65>;  // n x  4 x i1  vector value
-def nxv8i1  : ValueType<8,   66>;  // n x  8 x i1  vector value
-def nxv16i1 : ValueType<16,  67>;  // n x 16 x i1  vector value
-def nxv32i1 : ValueType<32,  68>;  // n x 32 x i1  vector value
-
-def nxv1i8  : ValueType<8,   69>;  // n x  1 x i8  vector value
-def nxv2i8  : ValueType<16,  70>;  // n x  2 x i8  vector value
-def nxv4i8  : ValueType<32,  71>;  // n x  4 x i8  vector value
-def nxv8i8  : ValueType<64,  72>;  // n x  8 x i8  vector value
-def nxv16i8 : ValueType<128, 73>;  // n x 16 x i8  vector value
-def nxv32i8 : ValueType<256, 74>;  // n x 32 x i8  vector value
-
-def nxv1i16 : ValueType<16,  75>;  // n x  1 x i16 vector value
-def nxv2i16 : ValueType<32,  76>;  // n x  2 x i16 vector value
-def nxv4i16 : ValueType<64,  77>;  // n x  4 x i16 vector value
-def nxv8i16 : ValueType<128, 78>;  // n x  8 x i16 vector value
-def nxv16i16: ValueType<256, 79>;  // n x 16 x i16 vector value
-def nxv32i16: ValueType<512, 80>;  // n x 32 x i16 vector value
-
-def nxv1i32 : ValueType<32,  81>;  // n x  1 x i32 vector value
-def nxv2i32 : ValueType<64,  82>;  // n x  2 x i32 vector value
-def nxv4i32 : ValueType<128, 83>;  // n x  4 x i32 vector value
-def nxv8i32 : ValueType<256, 84>;  // n x  8 x i32 vector value
-def nxv16i32: ValueType<512, 85>;  // n x 16 x i32 vector value
-def nxv32i32: ValueType<1024,86>;  // n x 32 x i32 vector value
-
-def nxv1i64 : ValueType<64,  87>;  // n x  1 x i64 vector value
-def nxv2i64 : ValueType<128, 88>;  // n x  2 x i64 vector value
-def nxv4i64 : ValueType<256, 89>;  // n x  4 x i64 vector value
-def nxv8i64 : ValueType<512, 90>;  // n x  8 x i64 vector value
-def nxv16i64: ValueType<1024,91>;  // n x 16 x i64 vector value
-def nxv32i64: ValueType<2048,92>;  // n x 32 x i64 vector value
-
-def v2f16    : ValueType<32 , 93>;    //    2 x f16 vector value
-def v3f16    : ValueType<48 , 94>;    //    3 x f16 vector value
-def v4f16    : ValueType<64 , 95>;    //    4 x f16 vector value
-def v8f16    : ValueType<128, 96>;    //    8 x f16 vector value
-def v16f16   : ValueType<256, 97>;    //    8 x f16 vector value
-def v32f16   : ValueType<512, 98>;    //    8 x f16 vector value
-def v1f32    : ValueType<32 , 99>;    //    1 x f32 vector value
-def v2f32    : ValueType<64 , 100>;   //    2 x f32 vector value
-def v3f32    : ValueType<96 , 101>;   //    3 x f32 vector value
-def v4f32    : ValueType<128, 102>;   //    4 x f32 vector value
-def v5f32    : ValueType<160, 103>;   //    5 x f32 vector value
-def v8f32    : ValueType<256, 104>;   //    8 x f32 vector value
-def v16f32   : ValueType<512,  105>;  //   16 x f32 vector value
-def v32f32   : ValueType<1024, 106>;  //   32 x f32 vector value
-def v64f32   : ValueType<2048, 107>;  //   64 x f32 vector value
-def v128f32  : ValueType<4096, 108>;  //  128 x f32 vector value
-def v256f32  : ValueType<8182, 109>;  //  256 x f32 vector value
-def v512f32  : ValueType<16384, 110>; //  512 x f32 vector value
-def v1024f32 : ValueType<32768, 111>; // 1024 x f32 vector value
-def v2048f32 : ValueType<65536, 112>; // 2048 x f32 vector value
-def v1f64    : ValueType<64, 113>;    //    1 x f64 vector value
-def v2f64    : ValueType<128, 114>;   //    2 x f64 vector value
-def v4f64    : ValueType<256, 115>;   //    4 x f64 vector value
-def v8f64    : ValueType<512, 116>;   //    8 x f64 vector value
-
-def nxv2f16  : ValueType<32 , 117>; // n x  2 x f16 vector value
-def nxv4f16  : ValueType<64 , 118>; // n x  4 x f16 vector value
-def nxv8f16  : ValueType<128, 119>; // n x  8 x f16 vector value
-def nxv1f32  : ValueType<32 , 120>; // n x  1 x f32 vector value
-def nxv2f32  : ValueType<64 , 121>; // n x  2 x f32 vector value
-def nxv4f32  : ValueType<128, 122>; // n x  4 x f32 vector value
-def nxv8f32  : ValueType<256, 123>; // n x  8 x f32 vector value
-def nxv16f32 : ValueType<512, 124>; // n x 16 x f32 vector value
-def nxv1f64  : ValueType<64,  125>; // n x  1 x f64 vector value
-def nxv2f64  : ValueType<128, 126>; // n x  2 x f64 vector value
-def nxv4f64  : ValueType<256, 127>; // n x  4 x f64 vector value
-def nxv8f64  : ValueType<512, 128>; // n x  8 x f64 vector value
-
-def x86mmx : ValueType<64 , 129>;   // X86 MMX value
-def FlagVT : ValueType<0  , 130>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 131>;   // Produces no value
-def untyped: ValueType<8  , 132>;   // Produces an untyped value
-def exnref: ValueType<0, 133>;      // WebAssembly's exnref type
+def v256i1 : ValueType<256, 22>;   // 256 x i1 vector value
+def v512i1 : ValueType<512, 23>;   // 512 x i1 vector value
+def v1024i1: ValueType<1024,24>;   //1024 x i1 vector value
+
+def v1i8   : ValueType<8,   25>;   //  1 x i8  vector value
+def v2i8   : ValueType<16 , 26>;   //  2 x i8  vector value
+def v4i8   : ValueType<32 , 27>;   //  4 x i8  vector value
+def v8i8   : ValueType<64 , 28>;   //  8 x i8  vector value
+def v16i8  : ValueType<128, 29>;   // 16 x i8  vector value
+def v32i8  : ValueType<256, 30>;   // 32 x i8  vector value
+def v64i8  : ValueType<512, 31>;   // 64 x i8  vector value
+def v128i8 : ValueType<1024,32>;   //128 x i8  vector value
+def v256i8 : ValueType<2048,33>;   //256 x i8  vector value
+
+def v1i16  : ValueType<16 , 34>;   //  1 x i16 vector value
+def v2i16  : ValueType<32 , 35>;   //  2 x i16 vector value
+def v3i16  : ValueType<48 , 36>;   //  3 x i16 vector value
+def v4i16  : ValueType<64 , 37>;   //  4 x i16 vector value
+def v8i16  : ValueType<128, 38>;   //  8 x i16 vector value
+def v16i16 : ValueType<256, 39>;   // 16 x i16 vector value
+def v32i16 : ValueType<512, 40>;   // 32 x i16 vector value
+def v64i16 : ValueType<1024,41>;   // 64 x i16 vector value
+def v128i16: ValueType<2048,42>;   //128 x i16 vector value
+
+def v1i32    : ValueType<32 , 43>;   //  1 x i32 vector value
+def v2i32    : ValueType<64 , 44>;   //  2 x i32 vector value
+def v3i32    : ValueType<96 , 45>;   //  3 x i32 vector value
+def v4i32    : ValueType<128, 46>;   //  4 x i32 vector value
+def v5i32    : ValueType<160, 47>;   //  5 x i32 vector value
+def v8i32    : ValueType<256, 48>;   //  8 x i32 vector value
+def v16i32   : ValueType<512, 49>;   // 16 x i32 vector value
+def v32i32   : ValueType<1024,50>;   // 32 x i32 vector value
+def v64i32   : ValueType<2048,51>;   // 64 x i32 vector value
+def v128i32  : ValueType<4096,52>;   // 128 x i32 vector value
+def v256i32  : ValueType<8182,53>;   // 256 x i32 vector value
+def v512i32  : ValueType<16384,54>;  // 512 x i32 vector value
+def v1024i32 : ValueType<32768,55>;  // 1024 x i32 vector value
+def v2048i32 : ValueType<65536,56>;  // 2048 x i32 vector value
+
+def v1i64  : ValueType<64 , 57>;   //  1 x i64 vector value
+def v2i64  : ValueType<128, 58>;   //  2 x i64 vector value
+def v4i64  : ValueType<256, 59>;   //  4 x i64 vector value
+def v8i64  : ValueType<512, 60>;   //  8 x i64 vector value
+def v16i64 : ValueType<1024,61>;   // 16 x i64 vector value
+def v32i64 : ValueType<2048,62>;   // 32 x i64 vector value
+
+def v1i128 : ValueType<128, 63>;   //  1 x i128 vector value
+
+def v2f16    : ValueType<32 , 64>;    //    2 x f16 vector value
+def v3f16    : ValueType<48 , 65>;    //    3 x f16 vector value
+def v4f16    : ValueType<64 , 66>;    //    4 x f16 vector value
+def v8f16    : ValueType<128, 67>;    //    8 x f16 vector value
+def v16f16   : ValueType<256, 68>;    //    8 x f16 vector value
+def v32f16   : ValueType<512, 69>;    //    8 x f16 vector value
+def v1f32    : ValueType<32 , 70>;    //    1 x f32 vector value
+def v2f32    : ValueType<64 , 71>;    //    2 x f32 vector value
+def v3f32    : ValueType<96 , 72>;    //    3 x f32 vector value
+def v4f32    : ValueType<128, 73>;    //    4 x f32 vector value
+def v5f32    : ValueType<160, 74>;    //    5 x f32 vector value
+def v8f32    : ValueType<256, 75>;    //    8 x f32 vector value
+def v16f32   : ValueType<512,  76>;   //   16 x f32 vector value
+def v32f32   : ValueType<1024, 77>;   //   32 x f32 vector value
+def v64f32   : ValueType<2048, 78>;   //   64 x f32 vector value
+def v128f32  : ValueType<4096, 79>;   //  128 x f32 vector value
+def v256f32  : ValueType<8182, 80>;   //  256 x f32 vector value
+def v512f32  : ValueType<16384, 81>;  //  512 x f32 vector value
+def v1024f32 : ValueType<32768, 82>;  // 1024 x f32 vector value
+def v2048f32 : ValueType<65536, 83>;  // 2048 x f32 vector value
+def v1f64    : ValueType<64, 84>;     //    1 x f64 vector value
+def v2f64    : ValueType<128, 85>;    //    2 x f64 vector value
+def v4f64    : ValueType<256, 86>;    //    4 x f64 vector value
+def v8f64    : ValueType<512, 87>;    //    8 x f64 vector value
+
+def nxv1i1  : ValueType<1,   88>;  // n x  1 x i1  vector value
+def nxv2i1  : ValueType<2,   89>;  // n x  2 x i1  vector value
+def nxv4i1  : ValueType<4,   90>;  // n x  4 x i1  vector value
+def nxv8i1  : ValueType<8,   91>;  // n x  8 x i1  vector value
+def nxv16i1 : ValueType<16,  92>;  // n x 16 x i1  vector value
+def nxv32i1 : ValueType<32,  93>;  // n x 32 x i1  vector value
+
+def nxv1i8  : ValueType<8,   94>;  // n x  1 x i8  vector value
+def nxv2i8  : ValueType<16,  95>;  // n x  2 x i8  vector value
+def nxv4i8  : ValueType<32,  96>;  // n x  4 x i8  vector value
+def nxv8i8  : ValueType<64,  97>;  // n x  8 x i8  vector value
+def nxv16i8 : ValueType<128, 98>;  // n x 16 x i8  vector value
+def nxv32i8 : ValueType<256, 99>;  // n x 32 x i8  vector value
+
+def nxv1i16 : ValueType<16,  100>; // n x  1 x i16 vector value
+def nxv2i16 : ValueType<32,  101>; // n x  2 x i16 vector value
+def nxv4i16 : ValueType<64,  102>; // n x  4 x i16 vector value
+def nxv8i16 : ValueType<128, 103>; // n x  8 x i16 vector value
+def nxv16i16: ValueType<256, 104>; // n x 16 x i16 vector value
+def nxv32i16: ValueType<512, 105>; // n x 32 x i16 vector value
+
+def nxv1i32 : ValueType<32,  106>; // n x  1 x i32 vector value
+def nxv2i32 : ValueType<64,  107>; // n x  2 x i32 vector value
+def nxv4i32 : ValueType<128, 108>; // n x  4 x i32 vector value
+def nxv8i32 : ValueType<256, 109>; // n x  8 x i32 vector value
+def nxv16i32: ValueType<512, 110>; // n x 16 x i32 vector value
+def nxv32i32: ValueType<1024,111>; // n x 32 x i32 vector value
+
+def nxv1i64 : ValueType<64,  112>; // n x  1 x i64 vector value
+def nxv2i64 : ValueType<128, 113>; // n x  2 x i64 vector value
+def nxv4i64 : ValueType<256, 114>; // n x  4 x i64 vector value
+def nxv8i64 : ValueType<512, 115>; // n x  8 x i64 vector value
+def nxv16i64: ValueType<1024,116>; // n x 16 x i64 vector value
+def nxv32i64: ValueType<2048,117>; // n x 32 x i64 vector value
+
+def nxv2f16  : ValueType<32 , 118>; // n x  2 x f16 vector value
+def nxv4f16  : ValueType<64 , 119>; // n x  4 x f16 vector value
+def nxv8f16  : ValueType<128, 120>; // n x  8 x f16 vector value
+def nxv1f32  : ValueType<32 , 121>; // n x  1 x f32 vector value
+def nxv2f32  : ValueType<64 , 122>; // n x  2 x f32 vector value
+def nxv4f32  : ValueType<128, 123>; // n x  4 x f32 vector value
+def nxv8f32  : ValueType<256, 124>; // n x  8 x f32 vector value
+def nxv16f32 : ValueType<512, 125>; // n x 16 x f32 vector value
+def nxv1f64  : ValueType<64,  126>; // n x  1 x f64 vector value
+def nxv2f64  : ValueType<128, 127>; // n x  2 x f64 vector value
+def nxv4f64  : ValueType<256, 128>; // n x  4 x f64 vector value
+def nxv8f64  : ValueType<512, 129>; // n x  8 x f64 vector value
+
+def x86mmx : ValueType<64 , 130>;   // X86 MMX value
+def FlagVT : ValueType<0  , 131>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 132>;   // Produces no value
+def untyped: ValueType<8  , 133>;   // Produces an untyped value
+def exnref: ValueType<0, 134>;      // WebAssembly's exnref type
 def token  : ValueType<0  , 248>;   // TokenTy
 def MetadataVT: ValueType<0, 249>;  // Metadata
 
diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
index ee8131359aac7..ed5c143818e6b 100644
--- a/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
+++ b/llvm/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def
@@ -489,6 +489,11 @@ CV_REGISTER(ARM_NQ15, 415)
 
 #if defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64)
 
+// arm64intr.h from MSVC defines ARM64_FPSR, which conflicts with
+// these declarations.
+#pragma push_macro("ARM64_FPSR")
+#undef ARM64_FPSR
+
 // ARM64 registers
 
 CV_REGISTER(ARM64_NOREG, 0)
@@ -677,4 +682,6 @@ CV_REGISTER(ARM64_Q31, 211)
 
 CV_REGISTER(ARM64_FPSR, 220)
 
+#pragma pop_macro("ARM64_FPSR")
+
 #endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64)
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
index dcc47346fa4d5..f65a20af0cb1c 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h
@@ -68,7 +68,7 @@ class DWARFDebugLoc {
   /// Return the location list at the given offset or nullptr.
   LocationList const *getLocationListAtOffset(uint64_t Offset) const;
 
-  static Expected<LocationList>
+  Expected<LocationList>
   parseOneLocationList(const DWARFDataExtractor &Data, uint64_t *Offset);
 };
 
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
index 9bc011726247a..2f72c642a2d51 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h
@@ -53,8 +53,7 @@ class DWARFDebugRangeList {
       assert(AddressSize == 4 || AddressSize == 8);
       if (AddressSize == 4)
         return StartAddress == -1U;
-      else
-        return StartAddress == -1ULL;
+      return StartAddress == -1ULL;
     }
   };
 
diff --git a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
index 3a72b117d6953..63e18bb2ecd54 100644
--- a/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
+++ b/llvm/include/llvm/DebugInfo/GSYM/FunctionInfo.h
@@ -21,15 +21,68 @@ namespace llvm {
 class raw_ostream;
 namespace gsym {
 
-/// Function information in GSYM files encodes information for one
-/// contiguous address range. The name of the function is encoded as
-/// a string table offset and allows multiple functions with the same
-/// name to share the name string in the string table. Line tables are
-/// stored in a sorted vector of gsym::LineEntry objects and are split
-/// into line tables for each function. If a function has a discontiguous
-/// range, it will be split into two gsym::FunctionInfo objects. If the
-/// function has inline functions, the information will be encoded in
-/// the "Inline" member, see gsym::InlineInfo for more information.
+/// Function information in GSYM files encodes information for one contiguous
+/// address range. If a function has discontiguous address ranges, they will
+/// need to be encoded using multiple FunctionInfo objects.
+///
+/// ENCODING
+///
+/// The function information gets the function start address as an argument
+/// to the FunctionInfo::decode(...) function. This information is calculated
+/// from the GSYM header and an address offset from the GSYM address offsets
+/// table. The encoded FunctionInfo information must be alinged to a 4 byte
+/// boundary.
+///
+/// The encoded data for a FunctionInfo starts with fixed data that all
+/// function info objects have:
+///
+/// ENCODING  NAME        DESCRIPTION
+/// ========= =========== ====================================================
+/// uint32_t  Size        The size in bytes of this function.
+/// uint32_t  Name        The string table offset of the function name.
+///
+/// The optional data in a FunctionInfo object follows this fixed information
+/// and consists of a stream of tuples that consist of:
+///
+/// ENCODING  NAME        DESCRIPTION
+/// ========= =========== ====================================================
+/// uint32_t  InfoType    An "InfoType" enumeration that describes the type
+///                       of optional data that is encoded.
+/// uint32_t  InfoLength  The size in bytes of the encoded data that
+///                       immediately follows this length if this value is
+///                       greater than zero.
+/// uint8_t[] InfoData    Encoded bytes that represent the data for the
+///                       "InfoType". These bytes are only present if
+///                       "InfoLength" is greater than zero.
+///
+/// The "InfoType" is an enumeration:
+///
+///   enum InfoType {
+///     EndOfList = 0u,
+///     LineTableInfo = 1u,
+///     InlineInfo = 2u
+///   };
+///
+/// This stream of tuples is terminated by a "InfoType" whose value is
+/// InfoType::EndOfList and a zero for "InfoLength". This signifies the end of
+/// the optional information list. This format allows us to add new optional
+/// information data to a FunctionInfo object over time and allows older
+/// clients to still parse the format and skip over any data that they don't
+/// understand or want to parse.
+///
+/// So the function information encoding essientially looks like:
+///
+/// struct {
+///   uint32_t Size;
+///   uint32_t Name;
+///   struct {
+///     uint32_t InfoType;
+///     uint32_t InfoLength;
+///     uint8_t InfoData[InfoLength];
+///   }[N];
+/// }
+///
+/// Where "N" is the number of tuples.
 struct FunctionInfo {
   AddressRange Range;
   uint32_t Name; ///< String table offset in the string table.
@@ -39,23 +92,54 @@ struct FunctionInfo {
   FunctionInfo(uint64_t Addr = 0, uint64_t Size = 0, uint32_t N = 0)
       : Range(Addr, Addr + Size), Name(N) {}
 
+  /// Query if a FunctionInfo has rich debug info.
+  ///
+  /// \returns A bool that indicates if this object has something else than
+  /// range and name. When converting information from a symbol table and from
+  /// debug info, we might end up with multiple FunctionInfo objects for the
+  /// same range and we need to be able to tell which one is the better object
+  /// to use.
   bool hasRichInfo() const {
-    /// Returns whether we have something else than range and name. When
-    /// converting information from a symbol table and from debug info, we
-    /// might end up with multiple FunctionInfo objects for the same range
-    /// and we need to be able to tell which one is the better object to use.
     return OptLineTable.hasValue() || Inline.hasValue();
   }
 
+  /// Query if a FunctionInfo object is valid.
+  ///
+  /// Address and size can be zero and there can be no line entries for a
+  /// symbol so the only indication this entry is valid is if the name is
+  /// not zero. This can happen when extracting information from symbol
+  /// tables that do not encode symbol sizes. In that case only the
+  /// address and name will be filled in.
+  ///
+  /// \returns A boolean indicating if this FunctionInfo is valid.
   bool isValid() const {
-    /// Address and size can be zero and there can be no line entries for a
-    /// symbol so the only indication this entry is valid is if the name is
-    /// not zero. This can happen when extracting information from symbol
-    /// tables that do not encode symbol sizes. In that case only the
-    /// address and name will be filled in.
     return Name != 0;
   }
 
+  /// Decode an object from a binary data stream.
+  ///
+  /// \param Data The binary stream to read the data from. This object must
+  /// have the data for the object starting at offset zero. The data
+  /// can contain more data than needed.
+  ///
+  /// \param BaseAddr The FunctionInfo's start address and will be used as the
+  /// base address when decoding any contained information like the line table
+  /// and the inline info.
+  ///
+  /// \returns An FunctionInfo or an error describing the issue that was
+  /// encountered during decoding.
+  static llvm::Expected<FunctionInfo> decode(DataExtractor &Data,
+                                             uint64_t BaseAddr);
+
+  /// Encode this object into FileWriter stream.
+  ///
+  /// \param O The binary stream to write the data to at the current file
+  /// position.
+  ///
+  /// \returns An error object that indicates failure or the offset of the
+  /// function info that was successfully written into the stream.
+  llvm::Expected<uint64_t> encode(FileWriter &O) const;
+
   uint64_t startAddress() const { return Range.Start; }
   uint64_t endAddress() const { return Range.End; }
   uint64_t size() const { return Range.size(); }
diff --git a/llvm/include/llvm/DebugInfo/GSYM/Header.h b/llvm/include/llvm/DebugInfo/GSYM/Header.h
new file mode 100644
index 0000000000000..b81f68c10782e
--- /dev/null
+++ b/llvm/include/llvm/DebugInfo/GSYM/Header.h
@@ -0,0 +1,124 @@
+//===- Header.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_GSYM_HEADER_H
+#define LLVM_DEBUGINFO_GSYM_HEADER_H
+
+#include "llvm/Support/Error.h"
+
+#include <cstddef>
+#include <cstdint>
+
+namespace llvm {
+class raw_ostream;
+class DataExtractor;
+
+namespace gsym {
+class FileWriter;
+
+constexpr uint32_t GSYM_MAGIC = 0x4753594d; // 'GSYM'
+constexpr uint32_t GSYM_VERSION = 1;
+constexpr size_t GSYM_MAX_UUID_SIZE = 20;
+
+/// The GSYM header.
+///
+/// The GSYM header is found at the start of a stand alone GSYM file, or as
+/// the first bytes in a section when GSYM is contained in a section of an
+/// executable file (ELF, mach-o, COFF).
+///
+/// The structure is encoded exactly as it appears in the structure definition
+/// with no gaps between members. Alignment should not change from system to
+/// system as the members were laid out so that they shouldn't align
+/// differently on different architectures.
+///
+/// When endianness of the system loading a GSYM file matches, the file can
+/// be mmap'ed in and a pointer to the header can be cast to the first bytes
+/// of the file (stand alone GSYM file) or section data (GSYM in a section).
+/// When endianness is swapped, the Header::decode() function should be used to
+/// decode the header.
+struct Header {
+  /// The magic bytes should be set to GSYM_MAGIC. This helps detect if a file
+  /// is a GSYM file by scanning the first 4 bytes of a file or section.
+  /// This value might appear byte swapped
+  uint32_t Magic;
+  /// The version can number determines how the header is decoded and how each
+  /// InfoType in FunctionInfo is encoded/decoded. As version numbers increase,
+  /// "Magic" and "Version" members should always appear at offset zero and 4
+  /// respectively to ensure clients figure out if they can parse the format.
+  uint16_t Version;
+  /// The size in bytes of each address offset in the address offsets table.
+  uint8_t AddrOffSize;
+  /// The size in bytes of the UUID encoded in the "UUID" member.
+  uint8_t UUIDSize;
+  /// The 64 bit base address that all address offsets in the address offsets
+  /// table are relative to. Storing a full 64 bit address allows our address
+  /// offsets table to be smaller on disk.
+  uint64_t BaseAddress;
+  /// The number of addresses stored in the address offsets table.
+  uint32_t NumAddresses;
+  /// The file relative offset of the start of the string table for strings
+  /// contained in the GSYM file. If the GSYM in contained in a stand alone
+  /// file this will be the file offset of the start of the string table. If
+  /// the GSYM is contained in a section within an executable file, this can
+  /// be the offset of the first string used in the GSYM file and can possibly
+  /// span one or more executable string tables. This allows the strings to
+  /// share string tables in an ELF or mach-o file.
+  uint32_t StrtabOffset;
+  /// The size in bytes of the string table. For a stand alone GSYM file, this
+  /// will be the exact size in bytes of the string table. When the GSYM data
+  /// is in a section within an executable file, this size can span one or more
+  /// sections that contains strings. This allows any strings that are already
+  /// stored in the executable file to be re-used, and any extra strings could
+  /// be added to another string table and the string table offset and size
+  /// can be set to span all needed string tables.
+  uint32_t StrtabSize;
+  /// The UUID of the original executable file. This is stored to allow
+  /// matching a GSYM file to an executable file when symbolication is
+  /// required. Only the first "UUIDSize" bytes of the UUID are valid. Any
+  /// bytes in the UUID value that appear after the first UUIDSize bytes should
+  /// be set to zero.
+  uint8_t UUID[GSYM_MAX_UUID_SIZE];
+
+  /// Check if a header is valid.
+  ///
+  /// \returns True if the header is valid and if the version is supported.
+  bool isValid() const {
+    if (Magic != GSYM_MAGIC)
+      return false;
+    if (Version != GSYM_VERSION)
+      return false;
+    return true;
+  }
+
+  /// Decode an object from a binary data stream.
+  ///
+  /// \param Data The binary stream to read the data from. This object must
+  /// have the data for the object starting at offset zero. The data
+  /// can contain more data than needed.
+  ///
+  /// \returns A Header or an error describing the issue that was
+  /// encountered during decoding.
+  static llvm::Expected<Header> decode(DataExtractor &Data);
+
+  /// Encode this object into FileWriter stream.
+  ///
+  /// \param O The binary stream to write the data to at the current file
+  /// position.
+  ///
+  /// \returns An error object that indicates success or failure of the
+  /// encoding process.
+  llvm::Error encode(FileWriter &O) const;
+};
+
+bool operator==(const Header &LHS, const Header &RHS);
+raw_ostream &operator<<(raw_ostream &OS, const llvm::gsym::Header &H);
+
+} // namespace gsym
+} // namespace llvm
+
+#endif // #ifndef LLVM_DEBUGINFO_GSYM_HEADER_H
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangle.h b/llvm/include/llvm/Demangle/MicrosoftDemangle.h
index 382e79401c437..c6f26061bedde 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangle.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangle.h
@@ -158,6 +158,7 @@ class Demangler {
                                     QualifiedNameNode *QN);
   SymbolNode *demangleDeclarator(StringView &MangledName);
   SymbolNode *demangleMD5Name(StringView &MangledName);
+  SymbolNode *demangleTypeinfoName(StringView &MangledName);
 
   VariableSymbolNode *demangleVariableEncoding(StringView &MangledName,
                                                StorageClass SC);
diff --git a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
index 3d47471f0ef0e..37abede6b0310 100644
--- a/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
+++ b/llvm/include/llvm/Demangle/MicrosoftDemangleNodes.h
@@ -303,8 +303,6 @@ struct TypeNode : public Node {
     outputPost(OS, Flags);
   }
 
-  void outputQuals(bool SpaceBefore, bool SpaceAfter) const;
-
   Qualifiers Quals = Q_None;
 };
 
diff --git a/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h
index caea3bea997ea..ee9c2cc69c30e 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/RPCUtils.h
@@ -1417,7 +1417,7 @@ class MultiThreadedRPCEndpoint
     auto FutureResult = Promise.get_future();
 
     if (auto Err = this->template appendCallAsync<Func>(
-            [Promise = std::move(Promise)](ErrorReturn RetOrErr) {
+            [Promise = std::move(Promise)](ErrorReturn RetOrErr) mutable {
               Promise.set_value(std::move(RetOrErr));
               return Error::success();
             },
diff --git a/llvm/include/llvm/IR/AutoUpgrade.h b/llvm/include/llvm/IR/AutoUpgrade.h
index 99ab68977a4f8..66f38e5b55d10 100644
--- a/llvm/include/llvm/IR/AutoUpgrade.h
+++ b/llvm/include/llvm/IR/AutoUpgrade.h
@@ -87,6 +87,10 @@ namespace llvm {
   /// Upgrade the loop attachment metadata node.
   MDNode *upgradeInstructionLoopAttachment(MDNode &N);
 
+  /// Upgrade the datalayout string by adding a section for address space
+  /// pointers.
+  std::string UpgradeDataLayoutString(StringRef DL, StringRef Triple);
+
 } // End llvm namespace
 
 #endif
diff --git a/llvm/include/llvm/IR/BasicBlock.h b/llvm/include/llvm/IR/BasicBlock.h
index 69555af50e1fe..d594145f8636c 100644
--- a/llvm/include/llvm/IR/BasicBlock.h
+++ b/llvm/include/llvm/IR/BasicBlock.h
@@ -192,6 +192,11 @@ class BasicBlock final : public Value, // Basic blocks are data objects also
                                  std::function<bool(Instruction &)>>>
   instructionsWithoutDebug();
 
+  /// Return the size of the basic block ignoring debug instructions
+  filter_iterator<BasicBlock::const_iterator,
+                  std::function<bool(const Instruction &)>>::difference_type
+  sizeWithoutDebug() const;
+
   /// Unlink 'this' from the containing function, but do not delete it.
   void removeFromParent();
 
diff --git a/llvm/include/llvm/IR/ConstantRange.h b/llvm/include/llvm/IR/ConstantRange.h
index 91f3f31abe17c..ce4b851cb4572 100644
--- a/llvm/include/llvm/IR/ConstantRange.h
+++ b/llvm/include/llvm/IR/ConstantRange.h
@@ -330,6 +330,14 @@ class LLVM_NODISCARD ConstantRange {
   /// from an addition of a value in this range and a value in \p Other.
   ConstantRange add(const ConstantRange &Other) const;
 
+  /// Return a new range representing the possible values resulting
+  /// from an addition with wrap type \p NoWrapKind of a value in this
+  /// range and a value in \p Other.
+  /// If the result range is disjoint, the preferred range is determined by the
+  /// \p PreferredRangeType.
+  ConstantRange addWithNoWrap(const ConstantRange &Other, unsigned NoWrapKind,
+                              PreferredRangeType RangeType = Smallest) const;
+
   /// Return a new range representing the possible values resulting from a
   /// known NSW addition of a value in this range and \p Other constant.
   ConstantRange addWithNoSignedWrap(const APInt &Other) const;
diff --git a/llvm/include/llvm/IR/DataLayout.h b/llvm/include/llvm/IR/DataLayout.h
index 918463d3987ff..b33cb497d6e40 100644
--- a/llvm/include/llvm/IR/DataLayout.h
+++ b/llvm/include/llvm/IR/DataLayout.h
@@ -72,11 +72,11 @@ struct LayoutAlignElem {
   /// Alignment type from \c AlignTypeEnum
   unsigned AlignType : 8;
   unsigned TypeBitWidth : 24;
-  unsigned ABIAlign : 16;
-  unsigned PrefAlign : 16;
+  Align ABIAlign;
+  Align PrefAlign;
 
-  static LayoutAlignElem get(AlignTypeEnum align_type, unsigned abi_align,
-                             unsigned pref_align, uint32_t bit_width);
+  static LayoutAlignElem get(AlignTypeEnum align_type, Align abi_align,
+                             Align pref_align, uint32_t bit_width);
 
   bool operator==(const LayoutAlignElem &rhs) const;
 };
@@ -88,15 +88,15 @@ struct LayoutAlignElem {
 /// \note The unusual order of elements in the structure attempts to reduce
 /// padding and make the structure slightly more cache friendly.
 struct PointerAlignElem {
-  unsigned ABIAlign;
-  unsigned PrefAlign;
+  Align ABIAlign;
+  Align PrefAlign;
   uint32_t TypeByteWidth;
   uint32_t AddressSpace;
   uint32_t IndexWidth;
 
   /// Initializer
-  static PointerAlignElem get(uint32_t AddressSpace, unsigned ABIAlign,
-                              unsigned PrefAlign, uint32_t TypeByteWidth,
+  static PointerAlignElem get(uint32_t AddressSpace, Align ABIAlign,
+                              Align PrefAlign, uint32_t TypeByteWidth,
                               uint32_t IndexWidth);
 
   bool operator==(const PointerAlignElem &rhs) const;
@@ -173,16 +173,15 @@ class DataLayout {
   /// well-defined bitwise representation.
   SmallVector<unsigned, 8> NonIntegralAddressSpaces;
 
-  void setAlignment(AlignTypeEnum align_type, unsigned abi_align,
-                    unsigned pref_align, uint32_t bit_width);
-  unsigned getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width,
-                            bool ABIAlign, Type *Ty) const;
-  void setPointerAlignment(uint32_t AddrSpace, unsigned ABIAlign,
-                           unsigned PrefAlign, uint32_t TypeByteWidth,
-                           uint32_t IndexWidth);
+  void setAlignment(AlignTypeEnum align_type, Align abi_align, Align pref_align,
+                    uint32_t bit_width);
+  Align getAlignmentInfo(AlignTypeEnum align_type, uint32_t bit_width,
+                         bool ABIAlign, Type *Ty) const;
+  void setPointerAlignment(uint32_t AddrSpace, Align ABIAlign, Align PrefAlign,
+                           uint32_t TypeByteWidth, uint32_t IndexWidth);
 
   /// Internal helper method that returns requested alignment for type.
-  unsigned getAlignment(Type *Ty, bool abi_or_pref) const;
+  Align getAlignment(Type *Ty, bool abi_or_pref) const;
 
   /// Parses a target data specification string. Assert if the string is
   /// malformed.
@@ -262,11 +261,15 @@ class DataLayout {
   bool isIllegalInteger(uint64_t Width) const { return !isLegalInteger(Width); }
 
   /// Returns true if the given alignment exceeds the natural stack alignment.
-  bool exceedsNaturalStackAlignment(llvm::Align Align) const {
-    return StackNaturalAlign && (Align > StackNaturalAlign);
+  bool exceedsNaturalStackAlignment(Align Alignment) const {
+    return StackNaturalAlign && (Alignment > StackNaturalAlign);
+  }
+
+  Align getStackAlignment() const {
+    assert(StackNaturalAlign && "StackNaturalAlign must be defined");
+    return *StackNaturalAlign;
   }
 
-  unsigned getStackAlignment() const { return StackNaturalAlign ? StackNaturalAlign->value() : 0; }
   unsigned getAllocaAddrSpace() const { return AllocaAddrSpace; }
 
   /// Returns the alignment of function pointers, which may or may not be
@@ -345,12 +348,12 @@ class DataLayout {
   }
 
   /// Layout pointer alignment
-  unsigned getPointerABIAlignment(unsigned AS) const;
+  Align getPointerABIAlignment(unsigned AS) const;
 
   /// Return target's alignment for stack-based pointers
   /// FIXME: The defaults need to be removed once all of
   /// the backends/clients are updated.
-  unsigned getPointerPrefAlignment(unsigned AS = 0) const;
+  Align getPointerPrefAlignment(unsigned AS = 0) const;
 
   /// Layout pointer size
   /// FIXME: The defaults need to be removed once all of
@@ -486,7 +489,7 @@ class DataLayout {
 
   /// Returns the minimum ABI-required alignment for an integer type of
   /// the specified bitwidth.
-  unsigned getABIIntegerTypeAlignment(unsigned BitWidth) const;
+  Align getABIIntegerTypeAlignment(unsigned BitWidth) const;
 
   /// Returns the preferred stack/global alignment for the specified
   /// type.
@@ -558,7 +561,7 @@ inline LLVMTargetDataRef wrap(const DataLayout *P) {
 /// based on the DataLayout structure.
 class StructLayout {
   uint64_t StructSize;
-  unsigned StructAlignment;
+  Align StructAlignment;
   unsigned IsPadded : 1;
   unsigned NumElements : 31;
   uint64_t MemberOffsets[1]; // variable sized array!
@@ -568,7 +571,7 @@ class StructLayout {
 
   uint64_t getSizeInBits() const { return 8 * StructSize; }
 
-  unsigned getAlignment() const { return StructAlignment; }
+  Align getAlignment() const { return StructAlignment; }
 
   /// Returns whether the struct has padding or not between its fields.
   /// NB: Padding in nested element is not taken into account.
diff --git a/llvm/include/llvm/IR/DebugInfoFlags.def b/llvm/include/llvm/IR/DebugInfoFlags.def
index c6c696523fbb5..f90c580f10ef1 100644
--- a/llvm/include/llvm/IR/DebugInfoFlags.def
+++ b/llvm/include/llvm/IR/DebugInfoFlags.def
@@ -31,7 +31,8 @@ HANDLE_DI_FLAG(2, Protected)
 HANDLE_DI_FLAG(3, Public)
 HANDLE_DI_FLAG((1 << 2), FwdDecl)
 HANDLE_DI_FLAG((1 << 3), AppleBlock)
-HANDLE_DI_FLAG((1 << 4), BlockByrefStruct)
+// Used to be BlockByRef, can be reused for anything except DICompositeType.
+HANDLE_DI_FLAG((1 << 4), ReservedBit4)
 HANDLE_DI_FLAG((1 << 5), Virtual)
 HANDLE_DI_FLAG((1 << 6), Artificial)
 HANDLE_DI_FLAG((1 << 7), Explicit)
diff --git a/llvm/include/llvm/IR/DebugInfoMetadata.h b/llvm/include/llvm/IR/DebugInfoMetadata.h
index 1f0533d98f63a..b94640f177122 100644
--- a/llvm/include/llvm/IR/DebugInfoMetadata.h
+++ b/llvm/include/llvm/IR/DebugInfoMetadata.h
@@ -650,7 +650,6 @@ class DIType : public DIScope {
   }
   bool isForwardDecl() const { return getFlags() & FlagFwdDecl; }
   bool isAppleBlockExtension() const { return getFlags() & FlagAppleBlock; }
-  bool isBlockByrefStruct() const { return getFlags() & FlagBlockByrefStruct; }
   bool isVirtual() const { return getFlags() & FlagVirtual; }
   bool isArtificial() const { return getFlags() & FlagArtificial; }
   bool isObjectPointer() const { return getFlags() & FlagObjectPointer; }
diff --git a/llvm/include/llvm/IR/DerivedTypes.h b/llvm/include/llvm/IR/DerivedTypes.h
index 3c1d4278905fc..4cac62150a2bd 100644
--- a/llvm/include/llvm/IR/DerivedTypes.h
+++ b/llvm/include/llvm/IR/DerivedTypes.h
@@ -62,6 +62,11 @@ class IntegerType : public Type {
   /// Get or create an IntegerType instance.
   static IntegerType *get(LLVMContext &C, unsigned NumBits);
 
+  /// Returns type twice as wide the input type.
+  IntegerType *getExtendedType() const {
+    return Type::getIntNTy(getContext(), 2 * getScalarSizeInBits());
+  }
+
   /// Get the number of bits in this IntegerType
   unsigned getBitWidth() const { return getSubclassData(); }
 
@@ -470,21 +475,47 @@ class VectorType : public SequentialType {
   /// This static method is like getInteger except that the element types are
   /// twice as wide as the elements in the input type.
   static VectorType *getExtendedElementVectorType(VectorType *VTy) {
-    unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
-    Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2);
-    return VectorType::get(EltTy, VTy->getElementCount());
+    assert(VTy->isIntOrIntVectorTy() && "VTy expected to be a vector of ints.");
+    auto *EltTy = cast<IntegerType>(VTy->getElementType());
+    return VectorType::get(EltTy->getExtendedType(), VTy->getElementCount());
   }
 
-  /// This static method is like getInteger except that the element types are
-  /// half as wide as the elements in the input type.
+  // This static method gets a VectorType with the same number of elements as
+  // the input type, and the element type is an integer or float type which
+  // is half as wide as the elements in the input type.
   static VectorType *getTruncatedElementVectorType(VectorType *VTy) {
-    unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
-    assert((EltBits & 1) == 0 &&
-           "Cannot truncate vector element with odd bit-width");
-    Type *EltTy = IntegerType::get(VTy->getContext(), EltBits / 2);
+    Type *EltTy;
+    if (VTy->getElementType()->isFloatingPointTy()) {
+      switch(VTy->getElementType()->getTypeID()) {
+      case DoubleTyID:
+        EltTy = Type::getFloatTy(VTy->getContext());
+        break;
+      case FloatTyID:
+        EltTy = Type::getHalfTy(VTy->getContext());
+        break;
+      default:
+        llvm_unreachable("Cannot create narrower fp vector element type");
+      }
+    } else {
+      unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits();
+      assert((EltBits & 1) == 0 &&
+             "Cannot truncate vector element with odd bit-width");
+      EltTy = IntegerType::get(VTy->getContext(), EltBits / 2);
+    }
     return VectorType::get(EltTy, VTy->getElementCount());
   }
 
+  // This static method returns a VectorType with a smaller number of elements
+  // of a larger type than the input element type. For example, a <16 x i8>
+  // subdivided twice would return <4 x i32>
+  static VectorType *getSubdividedVectorType(VectorType *VTy, int NumSubdivs) {
+    for (int i = 0; i < NumSubdivs; ++i) {
+      VTy = VectorType::getDoubleElementsVectorType(VTy);
+      VTy = VectorType::getTruncatedElementVectorType(VTy);
+    }
+    return VTy;
+  }
+
   /// This static method returns a VectorType with half as many elements as the
   /// input type and the same element type.
   static VectorType *getHalfElementsVectorType(VectorType *VTy) {
@@ -577,6 +608,16 @@ class PointerType : public Type {
   }
 };
 
+Type *Type::getExtendedType() const {
+  assert(
+      isIntOrIntVectorTy() &&
+      "Original type expected to be a vector of integers or a scalar integer.");
+  if (auto *VTy = dyn_cast<VectorType>(this))
+    return VectorType::getExtendedElementVectorType(
+        const_cast<VectorType *>(VTy));
+  return cast<IntegerType>(this)->getExtendedType();
+}
+
 unsigned Type::getPointerAddressSpace() const {
   return cast<PointerType>(getScalarType())->getAddressSpace();
 }
diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h
index 4cabadb0e4516..ec469982d378b 100644
--- a/llvm/include/llvm/IR/DiagnosticInfo.h
+++ b/llvm/include/llvm/IR/DiagnosticInfo.h
@@ -74,9 +74,10 @@ enum DiagnosticKind {
   DK_LastMachineRemark = DK_MachineOptimizationRemarkAnalysis,
   DK_MIRParser,
   DK_PGOProfile,
+  DK_MisExpect,
   DK_Unsupported,
-  DK_FirstPluginKind,
-  DK_MisExpect
+  DK_FirstPluginKind // Must be last value to work with
+                     // getNextAvailablePluginDiagnosticKind
 };
 
 /// Get the next available kind ID for a plugin diagnostic.
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index e5dcea1d4c0e9..fa053fa70291e 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -1612,19 +1612,19 @@ class IRBuilder : public IRBuilderBase, public Inserter {
   LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
                               const char *Name) {
     LoadInst *LI = CreateLoad(Ty, Ptr, Name);
-    LI->setAlignment(Align);
+    LI->setAlignment(MaybeAlign(Align));
     return LI;
   }
   LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
                               const Twine &Name = "") {
     LoadInst *LI = CreateLoad(Ty, Ptr, Name);
-    LI->setAlignment(Align);
+    LI->setAlignment(MaybeAlign(Align));
     return LI;
   }
   LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align,
                               bool isVolatile, const Twine &Name = "") {
     LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name);
-    LI->setAlignment(Align);
+    LI->setAlignment(MaybeAlign(Align));
     return LI;
   }
 
@@ -2231,7 +2231,10 @@ class IRBuilder : public IRBuilderBase, public Inserter {
 
   PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues,
                      const Twine &Name = "") {
-    return Insert(PHINode::Create(Ty, NumReservedValues), Name);
+    PHINode *Phi = PHINode::Create(Ty, NumReservedValues);
+    if (isa<FPMathOperator>(Phi))
+      Phi = cast<PHINode>(setFPAttrs(Phi, nullptr /* MDNode* */, FMF));
+    return Insert(Phi, Name);
   }
 
   CallInst *CreateCall(FunctionType *FTy, Value *Callee,
diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h
index c99fd78426300..71cf9fc38d831 100644
--- a/llvm/include/llvm/IR/Instructions.h
+++ b/llvm/include/llvm/IR/Instructions.h
@@ -110,9 +110,11 @@ class AllocaInst : public UnaryInstruction {
   /// Return the alignment of the memory that is being allocated by the
   /// instruction.
   unsigned getAlignment() const {
-    return (1u << (getSubclassDataFromInstruction() & 31)) >> 1;
+    if (const auto MA = decodeMaybeAlign(getSubclassDataFromInstruction() & 31))
+      return MA->value();
+    return 0;
   }
-  void setAlignment(unsigned Align);
+  void setAlignment(MaybeAlign Align);
 
   /// Return true if this alloca is in the entry block of the function and is a
   /// constant size. If so, the code generator will fold it into the
@@ -238,10 +240,13 @@ class LoadInst : public UnaryInstruction {
 
   /// Return the alignment of the access that is being performed.
   unsigned getAlignment() const {
-    return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
+    if (const auto MA =
+            decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31))
+      return MA->value();
+    return 0;
   }
 
-  void setAlignment(unsigned Align);
+  void setAlignment(MaybeAlign Align);
 
   /// Returns the ordering constraint of this load instruction.
   AtomicOrdering getOrdering() const {
@@ -363,10 +368,15 @@ class StoreInst : public Instruction {
 
   /// Return the alignment of the access that is being performed
   unsigned getAlignment() const {
-    return (1 << ((getSubclassDataFromInstruction() >> 1) & 31)) >> 1;
+    if (const auto MA =
+            decodeMaybeAlign((getSubclassDataFromInstruction() >> 1) & 31))
+      return MA->value();
+    return 0;
   }
 
+  // FIXME: Remove once migration to Align is over.
   void setAlignment(unsigned Align);
+  void setAlignment(MaybeAlign Align);
 
   /// Returns the ordering constraint of this store instruction.
   AtomicOrdering getOrdering() const {
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index 46e68a7cfcc01..9e4ebd915afce 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -100,7 +100,8 @@ namespace Intrinsic {
       Integer, Vector, Pointer, Struct,
       Argument, ExtendArgument, TruncArgument, HalfVecArgument,
       SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt,
-      VecElementArgument, ScalableVecArgument
+      VecElementArgument, ScalableVecArgument, Subdivide2Argument,
+      Subdivide4Argument, VecOfBitcastsToInt
     } Kind;
 
     union {
@@ -125,14 +126,17 @@ namespace Intrinsic {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
              Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == PtrToElt || Kind == VecElementArgument);
+             Kind == PtrToElt || Kind == VecElementArgument ||
+             Kind == Subdivide2Argument || Kind == Subdivide4Argument ||
+             Kind == VecOfBitcastsToInt);
       return Argument_Info >> 3;
     }
     ArgKind getArgumentKind() const {
       assert(Kind == Argument || Kind == ExtendArgument ||
              Kind == TruncArgument || Kind == HalfVecArgument ||
              Kind == SameVecWidthArgument || Kind == PtrToArgument ||
-             Kind == VecElementArgument);
+             Kind == VecElementArgument || Kind == Subdivide2Argument ||
+             Kind == Subdivide4Argument || Kind == VecOfBitcastsToInt);
       return (ArgKind)(Argument_Info & 7);
     }
 
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 514f0ccd3d23b..2d2e2a6f3d6f7 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -187,6 +187,16 @@ class LLVMVectorElementType<int num> : LLVMMatchType<num>;
 // vector type, but change the element count to be half as many
 class LLVMHalfElementsVectorType<int num> : LLVMMatchType<num>;
 
+// Match the type of another intrinsic parameter that is expected to be a
+// vector type (i.e. <N x iM>) but with each element subdivided to
+// form a vector with more elements that are smaller than the original.
+class LLVMSubdivide2VectorType<int num> : LLVMMatchType<num>;
+class LLVMSubdivide4VectorType<int num> : LLVMMatchType<num>;
+
+// Match the element count and bit width of another intrinsic parameter, but
+// change the element type to an integer.
+class LLVMVectorOfBitcastsToInt<int num> : LLVMMatchType<num>;
+
 def llvm_void_ty       : LLVMType<isVoid>;
 let isAny = 1 in {
   def llvm_any_ty        : LLVMType<Any>;
@@ -1130,6 +1140,8 @@ def int_load_relative: Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty],
 
 def int_hwasan_check_memaccess :
   Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly, ImmArg<2>]>;
+def int_hwasan_check_memaccess_shortgranules :
+  Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly, ImmArg<2>]>;
 
 // Xray intrinsics
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index a81127e75e216..e73f5b8b2b006 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -768,6 +768,18 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
                  LLVMMatchType<0>],
                 [IntrNoMem]>;
 
+  class AdvSIMD_SVE_CNT_Intrinsic
+    : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
+                [LLVMVectorOfBitcastsToInt<0>,
+                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
+                 llvm_anyvector_ty],
+                [IntrNoMem]>;
+
+  class AdvSIMD_SVE_PUNPKHI_Intrinsic
+    : Intrinsic<[LLVMHalfElementsVectorType<0>],
+                [llvm_anyvector_ty],
+                [IntrNoMem]>;
+
   // This class of intrinsics are not intended to be useful within LLVM IR but
   // are instead here to support some of the more regid parts of the ACLE.
   class Builtin_SVCVT<string name, LLVMType OUT, LLVMType IN>
@@ -787,9 +799,22 @@ let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
 
+//
+// Counting bits
+//
+
+def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
+
 //
 // Floating-point comparisons
 //
 
 def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>;
+
+//
+// Predicate operations
+//
+
+def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
+def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 }
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index 896170b4be579..ab6ee7f92dd1d 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -899,7 +899,10 @@ class AMDGPURawBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
   [llvm_v4i32_ty,     // rsrc(SGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
-   llvm_i32_ty],      // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
   [IntrReadMem, ImmArg<3>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<0>;
 def int_amdgcn_raw_buffer_load_format : AMDGPURawBufferLoad<llvm_anyfloat_ty>;
@@ -911,7 +914,10 @@ class AMDGPUStructBufferLoad<LLVMType data_ty = llvm_any_ty> : Intrinsic <
    llvm_i32_ty,       // vindex(VGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
-   llvm_i32_ty],      // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
   [IntrReadMem, ImmArg<4>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<0>;
 def int_amdgcn_struct_buffer_load_format : AMDGPUStructBufferLoad<llvm_anyfloat_ty>;
@@ -923,7 +929,10 @@ class AMDGPURawBufferStore<LLVMType data_ty = llvm_any_ty> : Intrinsic <
    llvm_v4i32_ty,     // rsrc(SGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
-   llvm_i32_ty],      // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
   [IntrWriteMem, ImmArg<4>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<1>;
 def int_amdgcn_raw_buffer_store_format : AMDGPURawBufferStore<llvm_anyfloat_ty>;
@@ -936,7 +945,10 @@ class AMDGPUStructBufferStore<LLVMType data_ty = llvm_any_ty> : Intrinsic <
    llvm_i32_ty,       // vindex(VGPR)
    llvm_i32_ty,       // offset(VGPR/imm, included in bounds checking and swizzling)
    llvm_i32_ty,       // soffset(SGPR/imm, excluded from bounds checking and swizzling)
-   llvm_i32_ty],      // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+   llvm_i32_ty],      // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
   [IntrWriteMem, ImmArg<5>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<1>;
 def int_amdgcn_struct_buffer_store_format : AMDGPUStructBufferStore<llvm_anyfloat_ty>;
@@ -1050,7 +1062,10 @@ def int_amdgcn_raw_tbuffer_load : Intrinsic <
      llvm_i32_ty,     // offset(VGPR/imm, included in bounds checking and swizzling)
      llvm_i32_ty,     // soffset(SGPR/imm, excluded from bounds checking and swizzling)
      llvm_i32_ty,     // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt)
-     llvm_i32_ty],    // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+     llvm_i32_ty],    // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
     [IntrReadMem, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<0>;
 
@@ -1061,7 +1076,10 @@ def int_amdgcn_raw_tbuffer_store : Intrinsic <
      llvm_i32_ty,    // offset(VGPR/imm, included in bounds checking and swizzling)
      llvm_i32_ty,    // soffset(SGPR/imm, excluded from bounds checking and swizzling)
      llvm_i32_ty,    // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt)
-     llvm_i32_ty],   // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+     llvm_i32_ty],   // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                     //                                       bit 1 = slc,
+                     //                                       bit 2 = dlc on gfx10+),
+                     //                      swizzled buffer (bit 3 = swz))
     [IntrWriteMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<1>;
 
@@ -1072,7 +1090,10 @@ def int_amdgcn_struct_tbuffer_load : Intrinsic <
      llvm_i32_ty,     // offset(VGPR/imm, included in bounds checking and swizzling)
      llvm_i32_ty,     // soffset(SGPR/imm, excluded from bounds checking and swizzling)
      llvm_i32_ty,     // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt)
-     llvm_i32_ty],    // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+     llvm_i32_ty],    // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                      //                                       bit 1 = slc,
+                      //                                       bit 2 = dlc on gfx10+),
+                      //                      swizzled buffer (bit 3 = swz))
     [IntrReadMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<0>;
 
@@ -1084,7 +1105,10 @@ def int_amdgcn_struct_tbuffer_store : Intrinsic <
      llvm_i32_ty,    // offset(VGPR/imm, included in bounds checking and swizzling)
      llvm_i32_ty,    // soffset(SGPR/imm, excluded from bounds checking and swizzling)
      llvm_i32_ty,    // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt)
-     llvm_i32_ty],   // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+)
+     llvm_i32_ty],   // auxiliary data (imm, cachepolicy     (bit 0 = glc,
+                     //                                       bit 1 = slc,
+                     //                                       bit 2 = dlc on gfx10+),
+                     //                      swizzled buffer (bit 3 = swz))
     [IntrWriteMem, ImmArg<5>, ImmArg<6>], "", [SDNPMemOperand]>,
   AMDGPURsrcIntrinsic<1>;
 
diff --git a/llvm/include/llvm/IR/IntrinsicsMips.td b/llvm/include/llvm/IR/IntrinsicsMips.td
index 6393a9ca35d55..bfcdd80a52d57 100644
--- a/llvm/include/llvm/IR/IntrinsicsMips.td
+++ b/llvm/include/llvm/IR/IntrinsicsMips.td
@@ -1260,16 +1260,16 @@ def int_mips_insve_d : GCCBuiltin<"__builtin_msa_insve_d">,
 
 def int_mips_ld_b : GCCBuiltin<"__builtin_msa_ld_b">,
   Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>;
+  [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_h : GCCBuiltin<"__builtin_msa_ld_h">,
   Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>;
+  [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_w : GCCBuiltin<"__builtin_msa_ld_w">,
   Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>;
+  [IntrReadMem, IntrArgMemOnly]>;
 def int_mips_ld_d : GCCBuiltin<"__builtin_msa_ld_d">,
   Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty],
-  [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>;
+  [IntrReadMem, IntrArgMemOnly]>;
 
 def int_mips_ldi_b : GCCBuiltin<"__builtin_msa_ldi_b">,
   Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>;
@@ -1684,16 +1684,16 @@ def int_mips_srlri_d : GCCBuiltin<"__builtin_msa_srlri_d">,
 
 def int_mips_st_b : GCCBuiltin<"__builtin_msa_st_b">,
   Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty, llvm_i32_ty],
-  [IntrArgMemOnly, ImmArg<2>]>;
+  [IntrArgMemOnly]>;
 def int_mips_st_h : GCCBuiltin<"__builtin_msa_st_h">,
   Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty, llvm_i32_ty],
-  [IntrArgMemOnly, ImmArg<2>]>;
+  [IntrArgMemOnly]>;
 def int_mips_st_w : GCCBuiltin<"__builtin_msa_st_w">,
   Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty],
-  [IntrArgMemOnly, ImmArg<2>]>;
+  [IntrArgMemOnly]>;
 def int_mips_st_d : GCCBuiltin<"__builtin_msa_st_d">,
   Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty],
-  [IntrArgMemOnly, ImmArg<2>]>;
+  [IntrArgMemOnly]>;
 
 def int_mips_subs_s_b : GCCBuiltin<"__builtin_msa_subs_s_b">,
   Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
diff --git a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
index 73d190b0b550e..6b5d0cbf799ab 100644
--- a/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
+++ b/llvm/include/llvm/IR/IntrinsicsWebAssembly.td
@@ -117,6 +117,31 @@ def int_wasm_qfms :
   Intrinsic<[llvm_anyvector_ty],
             [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
             [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_narrow_signed :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty, LLVMMatchType<1>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_narrow_unsigned :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty, LLVMMatchType<1>],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_widen_low_signed :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_widen_high_signed :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_widen_low_unsigned :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+def int_wasm_widen_high_unsigned :
+  Intrinsic<[llvm_anyvector_ty],
+            [llvm_anyvector_ty],
+            [IntrNoMem, IntrSpeculatable]>;
+
 
 //===----------------------------------------------------------------------===//
 // Bulk memory intrinsics
diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index 236d312d7d786..5796686dd79fa 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -2091,16 +2091,20 @@ let TargetPrefix = "x86" in {  // All intrinsics start with "llvm.x86.".
               Intrinsic<[llvm_ptr_ty], [], []>;
   def int_x86_lwpins32 :
               GCCBuiltin<"__builtin_ia32_lwpins32">,
-              Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+              Intrinsic<[llvm_i8_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                        [ImmArg<2>]>;
   def int_x86_lwpins64 :
               GCCBuiltin<"__builtin_ia32_lwpins64">,
-              Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>;
+              Intrinsic<[llvm_i8_ty], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty],
+                        [ImmArg<2>]>;
   def int_x86_lwpval32 :
               GCCBuiltin<"__builtin_ia32_lwpval32">,
-              Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>;
+              Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
+                        [ImmArg<2>]>;
   def int_x86_lwpval64 :
               GCCBuiltin<"__builtin_ia32_lwpval64">,
-              Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty], []>;
+              Intrinsic<[], [llvm_i64_ty, llvm_i32_ty, llvm_i32_ty],
+                        [ImmArg<2>]>;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/Operator.h b/llvm/include/llvm/IR/Operator.h
index 8199c65ca8a0f..608f33c3ed549 100644
--- a/llvm/include/llvm/IR/Operator.h
+++ b/llvm/include/llvm/IR/Operator.h
@@ -379,13 +379,17 @@ class FPMathOperator : public Operator {
       return false;
 
     switch (Opcode) {
+    // FIXME: To clean up and correct the semantics of fast-math-flags, FCmp
+    //        should not be treated as a math op, but the other opcodes should.
+    //        This would make things consistent with Select/PHI (FP value type
+    //        determines whether they are math ops and, therefore, capable of
+    //        having fast-math-flags).
     case Instruction::FCmp:
       return true;
     // non math FP Operators (no FMF)
     case Instruction::ExtractElement:
     case Instruction::ShuffleVector:
     case Instruction::InsertElement:
-    case Instruction::PHI:
       return false;
     default:
       return V->getType()->isFPOrFPVectorTy();
diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 4be032bcb983c..73b182c87b800 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -88,6 +88,11 @@ inline class_match<UndefValue> m_Undef() { return class_match<UndefValue>(); }
 /// Match an arbitrary Constant and ignore it.
 inline class_match<Constant> m_Constant() { return class_match<Constant>(); }
 
+/// Match an arbitrary basic block value and ignore it.
+inline class_match<BasicBlock> m_BasicBlock() {
+  return class_match<BasicBlock>();
+}
+
 /// Inverting matcher
 template <typename Ty> struct match_unless {
   Ty M;
@@ -563,6 +568,12 @@ inline bind_ty<Constant> m_Constant(Constant *&C) { return C; }
 /// Match a ConstantFP, capturing the value if we match.
 inline bind_ty<ConstantFP> m_ConstantFP(ConstantFP *&C) { return C; }
 
+/// Match a basic block value, capturing it if we match.
+inline bind_ty<BasicBlock> m_BasicBlock(BasicBlock *&V) { return V; }
+inline bind_ty<const BasicBlock> m_BasicBlock(const BasicBlock *&V) {
+  return V;
+}
+
 /// Match a specified Value*.
 struct specificval_ty {
   const Value *Val;
@@ -656,6 +667,32 @@ inline specific_intval m_SpecificInt(uint64_t V) { return specific_intval(V); }
 /// ConstantInts wider than 64-bits.
 inline bind_const_intval_ty m_ConstantInt(uint64_t &V) { return V; }
 
+/// Match a specified basic block value.
+struct specific_bbval {
+  BasicBlock *Val;
+
+  specific_bbval(BasicBlock *Val) : Val(Val) {}
+
+  template <typename ITy> bool match(ITy *V) {
+    const auto *BB = dyn_cast<BasicBlock>(V);
+    return BB && BB == Val;
+  }
+};
+
+/// Match a specific basic block value.
+inline specific_bbval m_SpecificBB(BasicBlock *BB) {
+  return specific_bbval(BB);
+}
+
+/// A commutative-friendly version of m_Specific().
+inline deferredval_ty<BasicBlock> m_Deferred(BasicBlock *const &BB) {
+  return BB;
+}
+inline deferredval_ty<const BasicBlock>
+m_Deferred(const BasicBlock *const &BB) {
+  return BB;
+}
+
 //===----------------------------------------------------------------------===//
 // Matcher for any binary operator.
 //
@@ -1282,6 +1319,12 @@ m_ZExtOrSelf(const OpTy &Op) {
   return m_CombineOr(m_ZExt(Op), Op);
 }
 
+template <typename OpTy>
+inline match_combine_or<CastClass_match<OpTy, Instruction::SExt>, OpTy>
+m_SExtOrSelf(const OpTy &Op) {
+  return m_CombineOr(m_SExt(Op), Op);
+}
+
 template <typename OpTy>
 inline match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
                         CastClass_match<OpTy, Instruction::SExt>>
@@ -1289,6 +1332,15 @@ m_ZExtOrSExt(const OpTy &Op) {
   return m_CombineOr(m_ZExt(Op), m_SExt(Op));
 }
 
+template <typename OpTy>
+inline match_combine_or<
+    match_combine_or<CastClass_match<OpTy, Instruction::ZExt>,
+                     CastClass_match<OpTy, Instruction::SExt>>,
+    OpTy>
+m_ZExtOrSExtOrSelf(const OpTy &Op) {
+  return m_CombineOr(m_ZExtOrSExt(Op), Op);
+}
+
 /// Matches UIToFP.
 template <typename OpTy>
 inline CastClass_match<OpTy, Instruction::UIToFP> m_UIToFP(const OpTy &Op) {
@@ -1334,27 +1386,34 @@ struct br_match {
 
 inline br_match m_UnconditionalBr(BasicBlock *&Succ) { return br_match(Succ); }
 
-template <typename Cond_t> struct brc_match {
+template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
+struct brc_match {
   Cond_t Cond;
-  BasicBlock *&T, *&F;
+  TrueBlock_t T;
+  FalseBlock_t F;
 
-  brc_match(const Cond_t &C, BasicBlock *&t, BasicBlock *&f)
+  brc_match(const Cond_t &C, const TrueBlock_t &t, const FalseBlock_t &f)
       : Cond(C), T(t), F(f) {}
 
   template <typename OpTy> bool match(OpTy *V) {
     if (auto *BI = dyn_cast<BranchInst>(V))
-      if (BI->isConditional() && Cond.match(BI->getCondition())) {
-        T = BI->getSuccessor(0);
-        F = BI->getSuccessor(1);
-        return true;
-      }
+      if (BI->isConditional() && Cond.match(BI->getCondition()))
+        return T.match(BI->getSuccessor(0)) && F.match(BI->getSuccessor(1));
     return false;
   }
 };
 
 template <typename Cond_t>
-inline brc_match<Cond_t> m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) {
-  return brc_match<Cond_t>(C, T, F);
+inline brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>
+m_Br(const Cond_t &C, BasicBlock *&T, BasicBlock *&F) {
+  return brc_match<Cond_t, bind_ty<BasicBlock>, bind_ty<BasicBlock>>(
+      C, m_BasicBlock(T), m_BasicBlock(F));
+}
+
+template <typename Cond_t, typename TrueBlock_t, typename FalseBlock_t>
+inline brc_match<Cond_t, TrueBlock_t, FalseBlock_t>
+m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
+  return brc_match<Cond_t, TrueBlock_t, FalseBlock_t>(C, T, F);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/IR/RemarkStreamer.h b/llvm/include/llvm/IR/RemarkStreamer.h
index 7b2fc95d15e56..2abf6f99cb085 100644
--- a/llvm/include/llvm/IR/RemarkStreamer.h
+++ b/llvm/include/llvm/IR/RemarkStreamer.h
@@ -25,12 +25,12 @@
 namespace llvm {
 /// Streamer for remarks.
 class RemarkStreamer {
-  /// The filename that the remark diagnostics are emitted to.
-  const std::string Filename;
   /// The regex used to filter remarks based on the passes that emit them.
   Optional<Regex> PassFilter;
   /// The object used to serialize the remarks to a specific format.
   std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer;
+  /// The filename that the remark diagnostics are emitted to.
+  const Optional<std::string> Filename;
 
   /// Convert diagnostics into remark objects.
   /// The lifetime of the members of the result is bound to the lifetime of
@@ -38,10 +38,12 @@ class RemarkStreamer {
   remarks::Remark toRemark(const DiagnosticInfoOptimizationBase &Diag);
 
 public:
-  RemarkStreamer(StringRef Filename,
-                 std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer);
+  RemarkStreamer(std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer,
+                 Optional<StringRef> Filename = None);
   /// Return the filename that the remark diagnostics are emitted to.
-  StringRef getFilename() const { return Filename; }
+  Optional<StringRef> getFilename() const {
+    return Filename ? Optional<StringRef>(*Filename) : None;
+  }
   /// Return stream that the remark diagnostics are emitted to.
   raw_ostream &getStream() { return RemarkSerializer->OS; }
   /// Return the serializer used for this stream.
@@ -84,13 +86,21 @@ struct RemarkSetupFormatError : RemarkSetupErrorInfo<RemarkSetupFormatError> {
   using RemarkSetupErrorInfo<RemarkSetupFormatError>::RemarkSetupErrorInfo;
 };
 
-/// Setup optimization remarks.
+/// Setup optimization remarks that output to a file.
 Expected<std::unique_ptr<ToolOutputFile>>
 setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
                          StringRef RemarksPasses, StringRef RemarksFormat,
                          bool RemarksWithHotness,
                          unsigned RemarksHotnessThreshold = 0);
 
+/// Setup optimization remarks that output directly to a raw_ostream.
+/// \p OS is managed by the caller and should be open for writing as long as \p
+/// Context is streaming remarks to it.
+Error setupOptimizationRemarks(LLVMContext &Context, raw_ostream &OS,
+                               StringRef RemarksPasses, StringRef RemarksFormat,
+                               bool RemarksWithHotness,
+                               unsigned RemarksHotnessThreshold = 0);
+
 } // end namespace llvm
 
 #endif // LLVM_IR_REMARKSTREAMER_H
diff --git a/llvm/include/llvm/IR/Type.h b/llvm/include/llvm/IR/Type.h
index f2aa49030aaae..34271aae49c30 100644
--- a/llvm/include/llvm/IR/Type.h
+++ b/llvm/include/llvm/IR/Type.h
@@ -378,6 +378,10 @@ class Type {
     return ContainedTys[0];
   }
 
+  /// Given scalar/vector integer type, returns a type with elements twice as
+  /// wide as in the original type. For vectors, preserves element count.
+  inline Type *getExtendedType() const;
+
   /// Get the address space of this pointer or pointer vector type.
   inline unsigned getPointerAddressSpace() const;
 
diff --git a/llvm/include/llvm/IR/User.h b/llvm/include/llvm/IR/User.h
index 19d87c5c621d9..850ee72a03877 100644
--- a/llvm/include/llvm/IR/User.h
+++ b/llvm/include/llvm/IR/User.h
@@ -111,7 +111,7 @@ class User : public Value {
 #endif
   }
   /// Placement delete - required by std, called if the ctor throws.
-  void operator delete(void *Usr, unsigned, bool) {
+  void operator delete(void *Usr, unsigned, unsigned) {
     // Note: If a subclass manipulates the information which is required to calculate the
     // Usr memory pointer, e.g. NumUserOperands, the operator delete of that subclass has
     // to restore the changed information to the original value, since the dtor of that class
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 05fdbb1e26e40..777aea78e7fe6 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -266,7 +266,7 @@ void initializeMachineDominatorTreePass(PassRegistry&);
 void initializeMachineFunctionPrinterPassPass(PassRegistry&);
 void initializeMachineLICMPass(PassRegistry&);
 void initializeMachineLoopInfoPass(PassRegistry&);
-void initializeMachineModuleInfoPass(PassRegistry&);
+void initializeMachineModuleInfoWrapperPassPass(PassRegistry &);
 void initializeMachineOptimizationRemarkEmitterPassPass(PassRegistry&);
 void initializeMachineOutlinerPass(PassRegistry&);
 void initializeMachinePipelinerPass(PassRegistry&);
diff --git a/llvm/include/llvm/LTO/LTO.h b/llvm/include/llvm/LTO/LTO.h
index 4bf93c5ee4450..407af5e8de5d5 100644
--- a/llvm/include/llvm/LTO/LTO.h
+++ b/llvm/include/llvm/LTO/LTO.h
@@ -298,6 +298,10 @@ class LTO {
   /// Cache) for each task identifier.
   Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
 
+  /// Static method that returns a list of libcall symbols that can be generated
+  /// by LTO but might not be visible from bitcode symbol table.
+  static ArrayRef<const char*> getRuntimeLibcallSymbols();
+
 private:
   Config Conf;
 
diff --git a/llvm/include/llvm/MC/MCAsmInfo.h b/llvm/include/llvm/MC/MCAsmInfo.h
index 95c5662835a23..3261c483e0d80 100644
--- a/llvm/include/llvm/MC/MCAsmInfo.h
+++ b/llvm/include/llvm/MC/MCAsmInfo.h
@@ -317,6 +317,10 @@ class MCAsmInfo {
   /// Defaults to false.
   bool HasLinkOnceDirective = false;
 
+  /// True if we have a .lglobl directive, which is used to emit the information
+  /// of a static symbol into the symbol table. Defaults to false.
+  bool HasDotLGloblDirective = false;
+
   /// This attribute, if not MCSA_Invalid, is used to declare a symbol as having
   /// hidden visibility.  Defaults to MCSA_Hidden.
   MCSymbolAttr HiddenVisibilityAttr = MCSA_Hidden;
@@ -392,6 +396,9 @@ class MCAsmInfo {
   // %hi(), and similar unary operators.
   bool HasMipsExpressions = false;
 
+  // If true, emit function descriptor symbol on AIX.
+  bool NeedsFunctionDescriptors = false;
+
 public:
   explicit MCAsmInfo();
   virtual ~MCAsmInfo();
@@ -565,6 +572,8 @@ class MCAsmInfo {
 
   bool hasLinkOnceDirective() const { return HasLinkOnceDirective; }
 
+  bool hasDotLGloblDirective() const { return HasDotLGloblDirective; }
+
   MCSymbolAttr getHiddenVisibilityAttr() const { return HiddenVisibilityAttr; }
 
   MCSymbolAttr getHiddenDeclarationVisibilityAttr() const {
@@ -647,6 +656,7 @@ class MCAsmInfo {
   bool canRelaxRelocations() const { return RelaxELFRelocations; }
   void setRelaxELFRelocations(bool V) { RelaxELFRelocations = V; }
   bool hasMipsExpressions() const { return HasMipsExpressions; }
+  bool needsFunctionDescriptors() const { return NeedsFunctionDescriptors; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h
index 2a72ba7398a7a..4a3bacc954e0f 100644
--- a/llvm/include/llvm/MC/MCAsmInfoXCOFF.h
+++ b/llvm/include/llvm/MC/MCAsmInfoXCOFF.h
@@ -18,6 +18,11 @@ class MCAsmInfoXCOFF : public MCAsmInfo {
 
 protected:
   MCAsmInfoXCOFF();
+
+public:
+  // Return true only when the identifier Name does not need quotes to be
+  // syntactically correct for XCOFF.
+  bool isValidUnquotedName(StringRef Name) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCDirectives.h b/llvm/include/llvm/MC/MCDirectives.h
index 4029264c20263..ea79e68674e53 100644
--- a/llvm/include/llvm/MC/MCDirectives.h
+++ b/llvm/include/llvm/MC/MCDirectives.h
@@ -28,6 +28,7 @@ enum MCSymbolAttr {
   MCSA_ELF_TypeNoType,      ///< .type _foo, STT_NOTYPE  # aka @notype
   MCSA_ELF_TypeGnuUniqueObject, /// .type _foo, @gnu_unique_object
   MCSA_Global,              ///< .globl
+  MCSA_LGlobal,             ///< .lglobl (XCOFF)
   MCSA_Hidden,              ///< .hidden (ELF)
   MCSA_IndirectSymbol,      ///< .indirect_symbol (MachO)
   MCSA_Internal,            ///< .internal (ELF)
diff --git a/llvm/include/llvm/MC/MCLinkerOptimizationHint.h b/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
index f2a1364ad8841..003491f32f754 100644
--- a/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
+++ b/llvm/include/llvm/MC/MCLinkerOptimizationHint.h
@@ -61,6 +61,7 @@ static inline int MCLOHNameToId(StringRef Name) {
     MCLOHCaseNameToId(AdrpAdd)
     MCLOHCaseNameToId(AdrpLdrGot)
     .Default(-1);
+#undef MCLOHCaseNameToId
 }
 
 static inline StringRef MCLOHIdToName(MCLOHType Kind) {
@@ -76,6 +77,7 @@ static inline StringRef MCLOHIdToName(MCLOHType Kind) {
     MCLOHCaseIdToName(AdrpLdrGot);
   }
   return StringRef();
+#undef MCLOHCaseIdToName
 }
 
 static inline int MCLOHIdToNbArgs(MCLOHType Kind) {
diff --git a/llvm/include/llvm/MC/MCRegisterInfo.h b/llvm/include/llvm/MC/MCRegisterInfo.h
index 33ce3bd6aaf07..c7dc56ea588e9 100644
--- a/llvm/include/llvm/MC/MCRegisterInfo.h
+++ b/llvm/include/llvm/MC/MCRegisterInfo.h
@@ -395,13 +395,9 @@ class MCRegisterInfo {
   /// debugging info.
   int getDwarfRegNum(MCRegister RegNum, bool isEH) const;
 
-  /// Map a dwarf register back to a target register.
-  int getLLVMRegNum(unsigned RegNum, bool isEH) const;
-
-  /// Map a DWARF EH register back to a target register (same as
-  /// getLLVMRegNum(RegNum, true)) but return -1 if there is no mapping,
-  /// rather than asserting that there must be one.
-  int getLLVMRegNumFromEH(unsigned RegNum) const;
+  /// Map a dwarf register back to a target register. Returns None is there is
+  /// no mapping.
+  Optional<unsigned> getLLVMRegNum(unsigned RegNum, bool isEH) const;
 
   /// Map a target EH register number to an equivalent DWARF register
   /// number.
diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h
index e97274a960295..d057feda87d8b 100644
--- a/llvm/include/llvm/MC/MCSection.h
+++ b/llvm/include/llvm/MC/MCSection.h
@@ -59,7 +59,7 @@ class MCSection {
   MCSymbol *Begin;
   MCSymbol *End = nullptr;
   /// The alignment requirement of this section.
-  llvm::Align Alignment;
+  Align Alignment;
   /// The section index in the assemblers section list.
   unsigned Ordinal = 0;
   /// The index of this section in the layout order.
@@ -119,7 +119,7 @@ class MCSection {
   bool hasEnded() const;
 
   unsigned getAlignment() const { return Alignment.value(); }
-  void setAlignment(llvm::Align Value) { Alignment = Value; }
+  void setAlignment(Align Value) { Alignment = Value; }
 
   unsigned getOrdinal() const { return Ordinal; }
   void setOrdinal(unsigned Value) { Ordinal = Value; }
diff --git a/llvm/include/llvm/MC/MCSubtargetInfo.h b/llvm/include/llvm/MC/MCSubtargetInfo.h
index 9490a6ecedadb..23a5999b359cb 100644
--- a/llvm/include/llvm/MC/MCSubtargetInfo.h
+++ b/llvm/include/llvm/MC/MCSubtargetInfo.h
@@ -221,6 +221,8 @@ class MCSubtargetInfo {
     auto Found = std::lower_bound(ProcDesc.begin(), ProcDesc.end(), CPU);
     return Found != ProcDesc.end() && StringRef(Found->Key) == CPU;
   }
+
+  virtual unsigned getHwMode() const { return 0; }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h
index 566aa7ca8eb49..98ecd24669262 100644
--- a/llvm/include/llvm/MC/MCSymbolXCOFF.h
+++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h
@@ -35,14 +35,14 @@ class MCSymbolXCOFF : public MCSymbol {
     return StorageClass.getValue();
   }
 
-  void setContainingCsect(const MCSectionXCOFF *C) {
+  void setContainingCsect(MCSectionXCOFF *C) {
     assert((!ContainingCsect || ContainingCsect == C) &&
            "Trying to set a containing csect that doesn't match the one that"
            "this symbol is already mapped to.");
     ContainingCsect = C;
   }
 
-  const MCSectionXCOFF *getContainingCsect() const {
+  MCSectionXCOFF *getContainingCsect() const {
     assert(ContainingCsect &&
            "Trying to get containing csect but none was set.");
     return ContainingCsect;
@@ -50,7 +50,7 @@ class MCSymbolXCOFF : public MCSymbol {
 
 private:
   Optional<XCOFF::StorageClass> StorageClass;
-  const MCSectionXCOFF *ContainingCsect = nullptr;
+  MCSectionXCOFF *ContainingCsect = nullptr;
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
index 6ca103808aa81..0dd5d3322aa18 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/LSUnit.h
@@ -209,8 +209,10 @@ class LSUnitBase : public HardwareUnit {
 
   unsigned getUsedLQEntries() const { return UsedLQEntries; }
   unsigned getUsedSQEntries() const { return UsedSQEntries; }
-  unsigned assignLQSlot() { return UsedLQEntries++; }
-  unsigned assignSQSlot() { return UsedSQEntries++; }
+  void acquireLQSlot() { ++UsedLQEntries; }
+  void acquireSQSlot() { ++UsedSQEntries; }
+  void releaseLQSlot() { --UsedLQEntries; }
+  void releaseSQSlot() { --UsedSQEntries; }
 
   bool assumeNoAlias() const { return NoAlias; }
 
diff --git a/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h b/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
index 74c71a73dffd1..6c196757e571e 100644
--- a/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
+++ b/llvm/include/llvm/MCA/HardwareUnits/Scheduler.h
@@ -68,7 +68,7 @@ class DefaultSchedulerStrategy : public SchedulerStrategy {
 /// instructions from the dispatch stage, until the write-back stage.
 ///
 class Scheduler : public HardwareUnit {
-  LSUnit &LSU;
+  LSUnitBase &LSU;
 
   // Instruction selection strategy for this Scheduler.
   std::unique_ptr<SchedulerStrategy> Strategy;
@@ -154,15 +154,15 @@ class Scheduler : public HardwareUnit {
   bool promoteToPendingSet(SmallVectorImpl<InstRef> &Pending);
 
 public:
-  Scheduler(const MCSchedModel &Model, LSUnit &Lsu)
+  Scheduler(const MCSchedModel &Model, LSUnitBase &Lsu)
       : Scheduler(Model, Lsu, nullptr) {}
 
-  Scheduler(const MCSchedModel &Model, LSUnit &Lsu,
+  Scheduler(const MCSchedModel &Model, LSUnitBase &Lsu,
             std::unique_ptr<SchedulerStrategy> SelectStrategy)
       : Scheduler(std::make_unique<ResourceManager>(Model), Lsu,
                   std::move(SelectStrategy)) {}
 
-  Scheduler(std::unique_ptr<ResourceManager> RM, LSUnit &Lsu,
+  Scheduler(std::unique_ptr<ResourceManager> RM, LSUnitBase &Lsu,
             std::unique_ptr<SchedulerStrategy> SelectStrategy)
       : LSU(Lsu), Resources(std::move(RM)), BusyResourceUnits(0),
         NumDispatchedToThePendingSet(0), HadTokenStall(false) {
diff --git a/llvm/include/llvm/Object/COFF.h b/llvm/include/llvm/Object/COFF.h
index a7427cd097243..b91ee5887fec7 100644
--- a/llvm/include/llvm/Object/COFF.h
+++ b/llvm/include/llvm/Object/COFF.h
@@ -314,7 +314,10 @@ class COFFSymbolRef {
     return CS16 ? CS16->Name.Offset : CS32->Name.Offset;
   }
 
-  uint32_t getValue() const { return CS16 ? CS16->Value : CS32->Value; }
+  uint32_t getValue() const {
+    assert(isSet() && "COFFSymbolRef points to nothing!");
+    return CS16 ? CS16->Value : CS32->Value;
+  }
 
   int32_t getSectionNumber() const {
     assert(isSet() && "COFFSymbolRef points to nothing!");
diff --git a/llvm/include/llvm/Object/ELF.h b/llvm/include/llvm/Object/ELF.h
index 5b7ad184fc890..28b00c8413de7 100644
--- a/llvm/include/llvm/Object/ELF.h
+++ b/llvm/include/llvm/Object/ELF.h
@@ -150,15 +150,16 @@ class ELFFile {
 
   static Expected<ELFFile> create(StringRef Object);
 
+  bool isLE() const {
+    return getHeader()->getDataEncoding() == ELF::ELFDATA2LSB;
+  }
+
   bool isMipsELF64() const {
     return getHeader()->e_machine == ELF::EM_MIPS &&
            getHeader()->getFileClass() == ELF::ELFCLASS64;
   }
 
-  bool isMips64EL() const {
-    return isMipsELF64() &&
-           getHeader()->getDataEncoding() == ELF::ELFDATA2LSB;
-  }
+  bool isMips64EL() const { return isMipsELF64() && isLE(); }
 
   Expected<Elf_Shdr_Range> sections() const;
 
diff --git a/llvm/include/llvm/Object/ELFObjectFile.h b/llvm/include/llvm/Object/ELFObjectFile.h
index 8dac02c56dcd7..774d8c3ef3c5b 100644
--- a/llvm/include/llvm/Object/ELFObjectFile.h
+++ b/llvm/include/llvm/Object/ELFObjectFile.h
@@ -239,6 +239,10 @@ template <class ELFT> class ELFObjectFile : public ELFObjectFileBase {
   using Elf_Rela = typename ELFT::Rela;
   using Elf_Dyn = typename ELFT::Dyn;
 
+  SectionRef toSectionRef(const Elf_Shdr *Sec) const {
+    return SectionRef(toDRI(Sec), this);
+  }
+
 private:
   ELFObjectFile(MemoryBufferRef Object, ELFFile<ELFT> EF,
                 const Elf_Shdr *DotDynSymSec, const Elf_Shdr *DotSymtabSec,
diff --git a/llvm/include/llvm/Object/MachOUniversal.h b/llvm/include/llvm/Object/MachOUniversal.h
index 4eb06526f7d9d..490d73f4c7ed6 100644
--- a/llvm/include/llvm/Object/MachOUniversal.h
+++ b/llvm/include/llvm/Object/MachOUniversal.h
@@ -159,8 +159,14 @@ class MachOUniversalBinary : public Binary {
     return V->isMachOUniversalBinary();
   }
 
-  Expected<std::unique_ptr<MachOObjectFile>>
+  Expected<ObjectForArch>
   getObjectForArch(StringRef ArchName) const;
+
+  Expected<std::unique_ptr<MachOObjectFile>>
+  getMachOObjectForArch(StringRef ArchName) const;
+
+  Expected<std::unique_ptr<Archive>>
+  getArchiveForArch(StringRef ArchName) const;
 };
 
 }
diff --git a/llvm/include/llvm/ObjectYAML/ELFYAML.h b/llvm/include/llvm/ObjectYAML/ELFYAML.h
index 7854800991b67..1662d06bf91e3 100644
--- a/llvm/include/llvm/ObjectYAML/ELFYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ELFYAML.h
@@ -25,6 +25,8 @@
 namespace llvm {
 namespace ELFYAML {
 
+StringRef dropUniqueSuffix(StringRef S);
+
 // These types are invariant across 32/64-bit ELF, so for simplicity just
 // directly give them their exact sizes. We don't need to worry about
 // endianness because these are just the types in the YAMLIO structures,
@@ -117,6 +119,11 @@ struct DynamicEntry {
   llvm::yaml::Hex64 Val;
 };
 
+struct StackSizeEntry {
+  llvm::yaml::Hex64 Address;
+  llvm::yaml::Hex64 Size;
+};
+
 struct Section {
   enum class SectionKind {
     Dynamic,
@@ -124,8 +131,10 @@ struct Section {
     RawContent,
     Relocation,
     NoBits,
+    Hash,
     Verdef,
     Verneed,
+    StackSizes,
     SymtabShndxSection,
     Symver,
     MipsABIFlags
@@ -163,6 +172,22 @@ struct Section {
   Optional<llvm::yaml::Hex64> ShSize;
 };
 
+struct StackSizesSection : Section {
+  Optional<yaml::BinaryRef> Content;
+  Optional<llvm::yaml::Hex64> Size;
+  Optional<std::vector<StackSizeEntry>> Entries;
+
+  StackSizesSection() : Section(SectionKind::StackSizes) {}
+
+  static bool classof(const Section *S) {
+    return S->Kind == SectionKind::StackSizes;
+  }
+
+  static bool nameMatches(StringRef Name) {
+    return Name == ".stack_sizes";
+  }
+};
+
 struct DynamicSection : Section {
   std::vector<DynamicEntry> Entries;
   Optional<yaml::BinaryRef> Content;
@@ -196,6 +221,17 @@ struct NoBitsSection : Section {
   }
 };
 
+struct HashSection : Section {
+  Optional<yaml::BinaryRef> Content;
+  Optional<llvm::yaml::Hex64> Size;
+  Optional<std::vector<uint32_t>> Bucket;
+  Optional<std::vector<uint32_t>> Chain;
+
+  HashSection() : Section(SectionKind::Hash) {}
+
+  static bool classof(const Section *S) { return S->Kind == SectionKind::Hash; }
+};
+
 struct VernauxEntry {
   uint32_t Hash;
   uint16_t Flags;
@@ -326,6 +362,7 @@ struct Object {
 } // end namespace ELFYAML
 } // end namespace llvm
 
+LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::StackSizeEntry)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::DynamicEntry)
 LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader)
 LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr<llvm::ELFYAML::Section>)
@@ -461,6 +498,10 @@ struct MappingTraits<ELFYAML::Symbol> {
   static StringRef validate(IO &IO, ELFYAML::Symbol &Symbol);
 };
 
+template <> struct MappingTraits<ELFYAML::StackSizeEntry> {
+  static void mapping(IO &IO, ELFYAML::StackSizeEntry &Rel);
+};
+
 template <> struct MappingTraits<ELFYAML::DynamicEntry> {
   static void mapping(IO &IO, ELFYAML::DynamicEntry &Rel);
 };
diff --git a/llvm/include/llvm/Pass.h b/llvm/include/llvm/Pass.h
index 329f7eaba73dd..1d53ae32cf37c 100644
--- a/llvm/include/llvm/Pass.h
+++ b/llvm/include/llvm/Pass.h
@@ -306,6 +306,9 @@ class FunctionPass : public Pass {
 };
 
 //===----------------------------------------------------------------------===//
+/// Deprecated - do not create new passes as BasicBlockPasses. Use FunctionPass
+/// with a loop over the BasicBlocks instead.
+//
 /// BasicBlockPass class - This class is used to implement most local
 /// optimizations.  Optimizations should subclass this class if they
 /// meet the following constraints:
@@ -338,6 +341,8 @@ class BasicBlockPass : public Pass {
   /// do any post processing needed after all passes have run.
   virtual bool doFinalization(Function &);
 
+  void preparePassManager(PMStack &PMS) override;
+
   void assignPassManager(PMStack &PMS, PassManagerType T) override;
 
   ///  Return what kind of Pass Manager can manage this pass.
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 7284a67ba4a05..0dd0c7ec80654 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -301,7 +301,12 @@ class CounterMappingContext {
 struct FunctionRecord {
   /// Raw function name.
   std::string Name;
-  /// Associated files.
+  /// Mapping from FileID (i.e. vector index) to filename. Used to support
+  /// macro expansions within a function in which the macro and function are
+  /// defined in separate files.
+  ///
+  /// TODO: Uniquing filenames across all function records may be a performance
+  /// optimization.
   std::vector<std::string> Filenames;
   /// Regions in the function along with their counts.
   std::vector<CountedRegion> CountedRegions;
@@ -508,6 +513,7 @@ class CoverageData {
 class CoverageMapping {
   DenseMap<size_t, DenseSet<size_t>> RecordProvenance;
   std::vector<FunctionRecord> Functions;
+  DenseMap<size_t, SmallVector<unsigned, 0>> FilenameHash2RecordIndices;
   std::vector<std::pair<std::string, uint64_t>> FuncHashMismatches;
 
   CoverageMapping() = default;
@@ -516,6 +522,13 @@ class CoverageMapping {
   Error loadFunctionRecord(const CoverageMappingRecord &Record,
                            IndexedInstrProfReader &ProfileReader);
 
+  /// Look up the indices for function records which are at least partially
+  /// defined in the specified file. This is guaranteed to return a superset of
+  /// such records: extra records not in the file may be included if there is
+  /// a hash collision on the filename. Clients must be robust to collisions.
+  ArrayRef<unsigned>
+  getImpreciseRecordIndicesForFilename(StringRef Filename) const;
+
 public:
   CoverageMapping(const CoverageMapping &) = delete;
   CoverageMapping &operator=(const CoverageMapping &) = delete;
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
index 5f88cacdfcbb1..6fcd8a09a4940 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h
@@ -30,8 +30,7 @@ class CoverageFilenamesSectionWriter {
   ArrayRef<StringRef> Filenames;
 
 public:
-  CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames)
-      : Filenames(Filenames) {}
+  CoverageFilenamesSectionWriter(ArrayRef<StringRef> Filenames);
 
   /// Write encoded filenames to the given output stream.
   void write(raw_ostream &OS);
diff --git a/llvm/include/llvm/ProfileData/InstrProf.h b/llvm/include/llvm/ProfileData/InstrProf.h
index a229020b8da85..c26f769499920 100644
--- a/llvm/include/llvm/ProfileData/InstrProf.h
+++ b/llvm/include/llvm/ProfileData/InstrProf.h
@@ -93,10 +93,6 @@ inline StringRef getInstrProfValuesVarPrefix() { return "__profvp_"; }
 /// Return the name of value profile node array variables:
 inline StringRef getInstrProfVNodesVarName() { return "__llvm_prf_vnodes"; }
 
-/// Return the name prefix of the COMDAT group for instrumentation variables
-/// associated with a COMDAT function.
-inline StringRef getInstrProfComdatPrefix() { return "__profv_"; }
-
 /// Return the name of the variable holding the strings (possibly compressed)
 /// of all function's PGO names.
 inline StringRef getInstrProfNamesVarName() {
@@ -634,8 +630,8 @@ struct OverlapStats {
     FuncHash = Hash;
   }
 
-  Error accumuateCounts(const std::string &BaseFilename,
-                        const std::string &TestFilename, bool IsCS);
+  Error accumulateCounts(const std::string &BaseFilename,
+                         const std::string &TestFilename, bool IsCS);
   void addOneMismatch(const CountSumOrPercent &MismatchFunc);
   void addOneUnique(const CountSumOrPercent &UniqueFunc);
 
@@ -772,7 +768,7 @@ struct InstrProfRecord {
   void clearValueData() { ValueData = nullptr; }
 
   /// Compute the sums of all counts and store in Sum.
-  void accumuateCounts(CountSumOrPercent &Sum) const;
+  void accumulateCounts(CountSumOrPercent &Sum) const;
 
   /// Compute the overlap b/w this IntrprofRecord and Other.
   void overlap(InstrProfRecord &Other, OverlapStats &Overlap,
diff --git a/llvm/include/llvm/ProfileData/InstrProfReader.h b/llvm/include/llvm/ProfileData/InstrProfReader.h
index 3eb84a835f97f..f5f552672bf03 100644
--- a/llvm/include/llvm/ProfileData/InstrProfReader.h
+++ b/llvm/include/llvm/ProfileData/InstrProfReader.h
@@ -92,7 +92,7 @@ class InstrProfReader {
   virtual InstrProfSymtab &getSymtab() = 0;
 
   /// Compute the sum of counts and return in Sum.
-  void accumuateCounts(CountSumOrPercent &Sum, bool IsCS);
+  void accumulateCounts(CountSumOrPercent &Sum, bool IsCS);
 
 protected:
   std::unique_ptr<InstrProfSymtab> Symtab;
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 157c79f6772fc..9ad740ed804eb 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -125,6 +125,22 @@ enum SecType {
   SecLBRProfile = SecFuncProfileFirst
 };
 
+static inline std::string getSecName(SecType Type) {
+  switch (Type) {
+  case SecInValid:
+    return "InvalidSection";
+  case SecProfSummary:
+    return "ProfileSummarySection";
+  case SecNameTable:
+    return "NameTableSection";
+  case SecProfileSymbolList:
+    return "ProfileSymbolListSection";
+  case SecLBRProfile:
+    return "LBRProfileSection";
+  }
+  llvm_unreachable("A SecType has no name for output");
+}
+
 // Entry type of section header table used by SampleProfileExtBinaryBaseReader
 // and SampleProfileExtBinaryBaseWriter.
 struct SecHdrTableEntry {
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
index d690c6d6edb81..761dbde059569 100644
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -330,6 +330,11 @@ class SampleProfileReader {
     return nullptr;
   };
 
+  /// It includes all the names that have samples either in outline instance
+  /// or inline instance.
+  virtual std::vector<StringRef> *getNameTable() { return nullptr; }
+  virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) { return false; };
+
 protected:
   /// Map every function to its associated profile.
   ///
@@ -387,6 +392,10 @@ class SampleProfileReaderBinary : public SampleProfileReader {
   /// Read sample profiles from the associated file.
   std::error_code read() override;
 
+  /// It includes all the names that have samples either in outline instance
+  /// or inline instance.
+  virtual std::vector<StringRef> *getNameTable() override { return &NameTable; }
+
 protected:
   /// Read a numeric value of type T from the profile.
   ///
@@ -496,6 +505,12 @@ class SampleProfileReaderExtBinaryBase : public SampleProfileReaderBinary {
 
   /// Read sample profiles in extensible format from the associated file.
   std::error_code read() override;
+
+  /// Get the total size of all \p Type sections.
+  uint64_t getSectionSize(SecType Type);
+  /// Get the total size of header and all sections.
+  uint64_t getFileSize();
+  virtual bool dumpSectionInfo(raw_ostream &OS = dbgs()) override;
 };
 
 class SampleProfileReaderExtBinary : public SampleProfileReaderExtBinaryBase {
diff --git a/llvm/include/llvm/Remarks/BitstreamRemarkSerializer.h b/llvm/include/llvm/Remarks/BitstreamRemarkSerializer.h
index 5e5fd0eb99a33..62a175a1db0bb 100644
--- a/llvm/include/llvm/Remarks/BitstreamRemarkSerializer.h
+++ b/llvm/include/llvm/Remarks/BitstreamRemarkSerializer.h
@@ -148,6 +148,10 @@ struct BitstreamRemarkSerializer : public RemarkSerializer {
   std::unique_ptr<MetaSerializer>
   metaSerializer(raw_ostream &OS,
                  Optional<StringRef> ExternalFilename = None) override;
+
+  static bool classof(const RemarkSerializer *S) {
+    return S->SerializerFormat == Format::Bitstream;
+  }
 };
 
 /// Serializer of metadata for bitstream remarks.
diff --git a/llvm/include/llvm/Remarks/RemarkSerializer.h b/llvm/include/llvm/Remarks/RemarkSerializer.h
index cea2afb2d378b..35752cd5f6fb8 100644
--- a/llvm/include/llvm/Remarks/RemarkSerializer.h
+++ b/llvm/include/llvm/Remarks/RemarkSerializer.h
@@ -36,6 +36,8 @@ struct MetaSerializer;
 /// This is the base class for a remark serializer.
 /// It includes support for using a string table while emitting.
 struct RemarkSerializer {
+  /// The format of the serializer.
+  Format SerializerFormat;
   /// The open raw_ostream that the remark diagnostics are emitted to.
   raw_ostream &OS;
   /// The serialization mode.
@@ -44,8 +46,9 @@ struct RemarkSerializer {
   /// The table can be serialized to be consumed after the compilation.
   Optional<StringTable> StrTab;
 
-  RemarkSerializer(raw_ostream &OS, SerializerMode Mode)
-      : OS(OS), Mode(Mode), StrTab() {}
+  RemarkSerializer(Format SerializerFormat, raw_ostream &OS,
+                   SerializerMode Mode)
+      : SerializerFormat(SerializerFormat), OS(OS), Mode(Mode), StrTab() {}
 
   /// This is just an interface.
   virtual ~RemarkSerializer() = default;
@@ -71,7 +74,8 @@ struct MetaSerializer {
 
 /// Create a remark serializer.
 Expected<std::unique_ptr<RemarkSerializer>>
-createRemarkSerializer(Format RemarksFormat, SerializerMode Mode, raw_ostream &OS);
+createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
+                       raw_ostream &OS);
 
 /// Create a remark serializer that uses a pre-filled string table.
 Expected<std::unique_ptr<RemarkSerializer>>
diff --git a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
index 77b78380c0725..f1213beab15da 100644
--- a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
+++ b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
@@ -34,12 +34,22 @@ struct YAMLRemarkSerializer : public RemarkSerializer {
   /// The YAML streamer.
   yaml::Output YAMLOutput;
 
-  YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode);
+  YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
+                       Optional<StringTable> StrTab = None);
 
   void emit(const Remark &Remark) override;
   std::unique_ptr<MetaSerializer>
   metaSerializer(raw_ostream &OS,
                  Optional<StringRef> ExternalFilename = None) override;
+
+  static bool classof(const RemarkSerializer *S) {
+    return S->SerializerFormat == Format::YAML;
+  }
+
+protected:
+  YAMLRemarkSerializer(Format SerializerFormat, raw_ostream &OS,
+                       SerializerMode Mode,
+                       Optional<StringTable> StrTab = None);
 };
 
 struct YAMLMetaSerializer : public MetaSerializer {
@@ -60,15 +70,13 @@ struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
   bool DidEmitMeta = false;
 
   YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
-      : YAMLRemarkSerializer(OS, Mode) {
-    // Having a string table set up enables the serializer to use it.
+      : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode) {
+    // We always need a string table for this type of serializer.
     StrTab.emplace();
   }
   YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
-                             StringTable StrTabIn)
-      : YAMLRemarkSerializer(OS, Mode) {
-    StrTab = std::move(StrTabIn);
-  }
+                             StringTable StrTab)
+      : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode, std::move(StrTab)) {}
 
   /// Override to emit the metadata if necessary.
   void emit(const Remark &Remark) override;
@@ -76,16 +84,20 @@ struct YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
   std::unique_ptr<MetaSerializer>
   metaSerializer(raw_ostream &OS,
                  Optional<StringRef> ExternalFilename = None) override;
+
+  static bool classof(const RemarkSerializer *S) {
+    return S->SerializerFormat == Format::YAMLStrTab;
+  }
 };
 
 struct YAMLStrTabMetaSerializer : public YAMLMetaSerializer {
   /// The string table is part of the metadata.
-  StringTable StrTab;
+  const StringTable &StrTab;
 
   YAMLStrTabMetaSerializer(raw_ostream &OS,
                            Optional<StringRef> ExternalFilename,
-                           StringTable StrTab)
-      : YAMLMetaSerializer(OS, ExternalFilename), StrTab(std::move(StrTab)) {}
+                           const StringTable &StrTab)
+      : YAMLMetaSerializer(OS, ExternalFilename), StrTab(StrTab) {}
 
   void emit() override;
 };
diff --git a/llvm/include/llvm/Support/Alignment.h b/llvm/include/llvm/Support/Alignment.h
index 07a69e12c8a57..3d8a4235b0e65 100644
--- a/llvm/include/llvm/Support/Alignment.h
+++ b/llvm/include/llvm/Support/Alignment.h
@@ -55,7 +55,7 @@ struct Align {
 
 public:
   /// Default is byte-aligned.
-  Align() = default;
+  constexpr Align() = default;
   /// Do not perform checks in case of copy/move construct/assign, because the
   /// checks have been performed when building `Other`.
   Align(const Align &Other) = default;
@@ -73,6 +73,13 @@ struct Align {
   /// This is a hole in the type system and should not be abused.
   /// Needed to interact with C for instance.
   uint64_t value() const { return uint64_t(1) << ShiftValue; }
+
+  /// Returns a default constructed Align which corresponds to no alignment.
+  /// This is useful to test for unalignment as it conveys clear semantic.
+  /// `if (A != Align::None())`
+  /// would be better than
+  /// `if (A > Align(1))`
+  constexpr static const Align None() { return Align(); }
 };
 
 /// Treats the value 0 as a 1, so Align is always at least 1.
@@ -135,8 +142,8 @@ inline uint64_t alignTo(uint64_t Size, MaybeAlign A) {
 
 /// Returns the offset to the next integer (mod 2**64) that is greater than
 /// or equal to \p Value and is a multiple of \p Align.
-inline uint64_t offsetToAlignment(uint64_t Value, llvm::Align Align) {
-  return alignTo(Value, Align) - Value;
+inline uint64_t offsetToAlignment(uint64_t Value, Align Alignment) {
+  return alignTo(Value, Alignment) - Value;
 }
 
 /// Returns the log2 of the alignment.
@@ -326,6 +333,14 @@ inline MaybeAlign operator/(MaybeAlign Lhs, uint64_t Divisor) {
   return Lhs ? Lhs.getValue() / Divisor : MaybeAlign();
 }
 
+inline Align max(MaybeAlign Lhs, Align Rhs) {
+  return Lhs && *Lhs > Rhs ? *Lhs : Rhs;
+}
+
+inline Align max(Align Lhs, MaybeAlign Rhs) {
+  return Rhs && *Rhs > Lhs ? *Rhs : Lhs;
+}
+
 #undef ALIGN_CHECK_ISPOSITIVE
 #undef ALIGN_CHECK_ISSET
 
diff --git a/llvm/include/llvm/Support/Compiler.h b/llvm/include/llvm/Support/Compiler.h
index 210a6f3386236..cb7e57d4cd21c 100644
--- a/llvm/include/llvm/Support/Compiler.h
+++ b/llvm/include/llvm/Support/Compiler.h
@@ -137,10 +137,18 @@
 #endif
 
 /// LLVM_NODISCARD - Warn if a type or return value is discarded.
+
+// Use the 'nodiscard' attribute in C++17 or newer mode.
 #if __cplusplus > 201402L && LLVM_HAS_CPP_ATTRIBUTE(nodiscard)
 #define LLVM_NODISCARD [[nodiscard]]
 #elif LLVM_HAS_CPP_ATTRIBUTE(clang::warn_unused_result)
 #define LLVM_NODISCARD [[clang::warn_unused_result]]
+// Clang in C++14 mode claims that it has the 'nodiscard' attribute, but also
+// warns in the pedantic mode that 'nodiscard' is a C++17 extension (PR33518).
+// Use the 'nodiscard' attribute in C++14 mode only with GCC.
+// TODO: remove this workaround when PR33518 is resolved.
+#elif defined(__GNUC__) && LLVM_HAS_CPP_ATTRIBUTE(nodiscard)
+#define LLVM_NODISCARD [[nodiscard]]
 #else
 #define LLVM_NODISCARD
 #endif
diff --git a/llvm/include/llvm/Support/DataExtractor.h b/llvm/include/llvm/Support/DataExtractor.h
index bd337f23925c9..f590a1e104fb1 100644
--- a/llvm/include/llvm/Support/DataExtractor.h
+++ b/llvm/include/llvm/Support/DataExtractor.h
@@ -82,6 +82,11 @@ class DataExtractor {
   /// valid.
   DataExtractor(StringRef Data, bool IsLittleEndian, uint8_t AddressSize)
     : Data(Data), IsLittleEndian(IsLittleEndian), AddressSize(AddressSize) {}
+  DataExtractor(ArrayRef<uint8_t> Data, bool IsLittleEndian,
+                uint8_t AddressSize)
+      : Data(StringRef(reinterpret_cast<const char *>(Data.data()),
+                       Data.size())),
+        IsLittleEndian(IsLittleEndian), AddressSize(AddressSize) {}
 
   /// Get the data pointed to by this extractor.
   StringRef getData() const { return Data; }
diff --git a/llvm/include/llvm/Support/FileCheck.h b/llvm/include/llvm/Support/FileCheck.h
index 34301f92bcb5c..5c6585ed76f75 100644
--- a/llvm/include/llvm/Support/FileCheck.h
+++ b/llvm/include/llvm/Support/FileCheck.h
@@ -13,14 +13,10 @@
 #ifndef LLVM_SUPPORT_FILECHECK_H
 #define LLVM_SUPPORT_FILECHECK_H
 
-#include "llvm/ADT/Optional.h"
-#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
-#include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Regex.h"
 #include "llvm/Support/SourceMgr.h"
-#include <map>
 #include <string>
 #include <vector>
 
@@ -41,216 +37,7 @@ struct FileCheckRequest {
 };
 
 //===----------------------------------------------------------------------===//
-// Numeric substitution handling code.
-//===----------------------------------------------------------------------===//
-
-/// Base class representing the AST of a given expression.
-class FileCheckExpressionAST {
-public:
-  virtual ~FileCheckExpressionAST() = default;
-
-  /// Evaluates and \returns the value of the expression represented by this
-  /// AST or an error if evaluation fails.
-  virtual Expected<uint64_t> eval() const = 0;
-};
-
-/// Class representing an unsigned literal in the AST of an expression.
-class FileCheckExpressionLiteral : public FileCheckExpressionAST {
-private:
-  /// Actual value of the literal.
-  uint64_t Value;
-
-public:
-  /// Constructs a literal with the specified value.
-  FileCheckExpressionLiteral(uint64_t Val) : Value(Val) {}
-
-  /// \returns the literal's value.
-  Expected<uint64_t> eval() const { return Value; }
-};
-
-/// Class to represent an undefined variable error, which quotes that
-/// variable's name when printed.
-class FileCheckUndefVarError : public ErrorInfo<FileCheckUndefVarError> {
-private:
-  StringRef VarName;
-
-public:
-  static char ID;
-
-  FileCheckUndefVarError(StringRef VarName) : VarName(VarName) {}
-
-  StringRef getVarName() const { return VarName; }
-
-  std::error_code convertToErrorCode() const override {
-    return inconvertibleErrorCode();
-  }
-
-  /// Print name of variable associated with this error.
-  void log(raw_ostream &OS) const override {
-    OS << "\"";
-    OS.write_escaped(VarName) << "\"";
-  }
-};
-
-/// Class representing a numeric variable and its associated current value.
-class FileCheckNumericVariable {
-private:
-  /// Name of the numeric variable.
-  StringRef Name;
-
-  /// Value of numeric variable, if defined, or None otherwise.
-  Optional<uint64_t> Value;
-
-  /// Line number where this variable is defined, or None if defined before
-  /// input is parsed. Used to determine whether a variable is defined on the
-  /// same line as a given use.
-  Optional<size_t> DefLineNumber;
-
-public:
-  /// Constructor for a variable \p Name defined at line \p DefLineNumber or
-  /// defined before input is parsed if \p DefLineNumber is None.
-  explicit FileCheckNumericVariable(StringRef Name,
-                                    Optional<size_t> DefLineNumber = None)
-      : Name(Name), DefLineNumber(DefLineNumber) {}
-
-  /// \returns name of this numeric variable.
-  StringRef getName() const { return Name; }
-
-  /// \returns this variable's value.
-  Optional<uint64_t> getValue() const { return Value; }
-
-  /// Sets value of this numeric variable to \p NewValue.
-  void setValue(uint64_t NewValue) { Value = NewValue; }
-
-  /// Clears value of this numeric variable, regardless of whether it is
-  /// currently defined or not.
-  void clearValue() { Value = None; }
-
-  /// \returns the line number where this variable is defined, if any, or None
-  /// if defined before input is parsed.
-  Optional<size_t> getDefLineNumber() { return DefLineNumber; }
-};
-
-/// Class representing the use of a numeric variable in the AST of an
-/// expression.
-class FileCheckNumericVariableUse : public FileCheckExpressionAST {
-private:
-  /// Name of the numeric variable.
-  StringRef Name;
-
-  /// Pointer to the class instance for the variable this use is about.
-  FileCheckNumericVariable *NumericVariable;
-
-public:
-  FileCheckNumericVariableUse(StringRef Name,
-                              FileCheckNumericVariable *NumericVariable)
-      : Name(Name), NumericVariable(NumericVariable) {}
-
-  /// \returns the value of the variable referenced by this instance.
-  Expected<uint64_t> eval() const;
-};
-
-/// Type of functions evaluating a given binary operation.
-using binop_eval_t = uint64_t (*)(uint64_t, uint64_t);
-
-/// Class representing a single binary operation in the AST of an expression.
-class FileCheckASTBinop : public FileCheckExpressionAST {
-private:
-  /// Left operand.
-  std::unique_ptr<FileCheckExpressionAST> LeftOperand;
-
-  /// Right operand.
-  std::unique_ptr<FileCheckExpressionAST> RightOperand;
-
-  /// Pointer to function that can evaluate this binary operation.
-  binop_eval_t EvalBinop;
-
-public:
-  FileCheckASTBinop(binop_eval_t EvalBinop,
-                    std::unique_ptr<FileCheckExpressionAST> LeftOp,
-                    std::unique_ptr<FileCheckExpressionAST> RightOp)
-      : EvalBinop(EvalBinop) {
-    LeftOperand = std::move(LeftOp);
-    RightOperand = std::move(RightOp);
-  }
-
-  /// Evaluates the value of the binary operation represented by this AST,
-  /// using EvalBinop on the result of recursively evaluating the operands.
-  /// \returns the expression value or an error if an undefined numeric
-  /// variable is used in one of the operands.
-  Expected<uint64_t> eval() const;
-};
-
-class FileCheckPatternContext;
-
-/// Class representing a substitution to perform in the RegExStr string.
-class FileCheckSubstitution {
-protected:
-  /// Pointer to a class instance holding, among other things, the table with
-  /// the values of live string variables at the start of any given CHECK line.
-  /// Used for substituting string variables with the text they were defined
-  /// as. Expressions are linked to the numeric variables they use at
-  /// parse time and directly access the value of the numeric variable to
-  /// evaluate their value.
-  FileCheckPatternContext *Context;
-
-  /// The string that needs to be substituted for something else. For a
-  /// string variable this is its name, otherwise this is the whole expression.
-  StringRef FromStr;
-
-  // Index in RegExStr of where to do the substitution.
-  size_t InsertIdx;
-
-public:
-  FileCheckSubstitution(FileCheckPatternContext *Context, StringRef VarName,
-                        size_t InsertIdx)
-      : Context(Context), FromStr(VarName), InsertIdx(InsertIdx) {}
-
-  virtual ~FileCheckSubstitution() = default;
-
-  /// \returns the string to be substituted for something else.
-  StringRef getFromString() const { return FromStr; }
-
-  /// \returns the index where the substitution is to be performed in RegExStr.
-  size_t getIndex() const { return InsertIdx; }
-
-  /// \returns a string containing the result of the substitution represented
-  /// by this class instance or an error if substitution failed.
-  virtual Expected<std::string> getResult() const = 0;
-};
-
-class FileCheckStringSubstitution : public FileCheckSubstitution {
-public:
-  FileCheckStringSubstitution(FileCheckPatternContext *Context,
-                              StringRef VarName, size_t InsertIdx)
-      : FileCheckSubstitution(Context, VarName, InsertIdx) {}
-
-  /// \returns the text that the string variable in this substitution matched
-  /// when defined, or an error if the variable is undefined.
-  Expected<std::string> getResult() const override;
-};
-
-class FileCheckNumericSubstitution : public FileCheckSubstitution {
-private:
-  /// Pointer to the class representing the expression whose value is to be
-  /// substituted.
-  std::unique_ptr<FileCheckExpressionAST> ExpressionAST;
-
-public:
-  FileCheckNumericSubstitution(FileCheckPatternContext *Context, StringRef Expr,
-                               std::unique_ptr<FileCheckExpressionAST> ExprAST,
-                               size_t InsertIdx)
-      : FileCheckSubstitution(Context, Expr, InsertIdx) {
-    ExpressionAST = std::move(ExprAST);
-  }
-
-  /// \returns a string containing the result of evaluating the expression in
-  /// this substitution, or an error if evaluation failed.
-  Expected<std::string> getResult() const override;
-};
-
-//===----------------------------------------------------------------------===//
-// Pattern handling code.
+// Summary of a FileCheck diagnostic.
 //===----------------------------------------------------------------------===//
 
 namespace Check {
@@ -294,340 +81,6 @@ class FileCheckType {
 };
 } // namespace Check
 
-struct FileCheckDiag;
-
-/// Class holding the FileCheckPattern global state, shared by all patterns:
-/// tables holding values of variables and whether they are defined or not at
-/// any given time in the matching process.
-class FileCheckPatternContext {
-  friend class FileCheckPattern;
-
-private:
-  /// When matching a given pattern, this holds the value of all the string
-  /// variables defined in previous patterns. In a pattern, only the last
-  /// definition for a given variable is recorded in this table.
-  /// Back-references are used for uses after any the other definition.
-  StringMap<StringRef> GlobalVariableTable;
-
-  /// Map of all string variables defined so far. Used at parse time to detect
-  /// a name conflict between a numeric variable and a string variable when
-  /// the former is defined on a later line than the latter.
-  StringMap<bool> DefinedVariableTable;
-
-  /// When matching a given pattern, this holds the pointers to the classes
-  /// representing the numeric variables defined in previous patterns. When
-  /// matching a pattern all definitions for that pattern are recorded in the
-  /// NumericVariableDefs table in the FileCheckPattern instance of that
-  /// pattern.
-  StringMap<FileCheckNumericVariable *> GlobalNumericVariableTable;
-
-  /// Pointer to the class instance representing the @LINE pseudo variable for
-  /// easily updating its value.
-  FileCheckNumericVariable *LineVariable = nullptr;
-
-  /// Vector holding pointers to all parsed numeric variables. Used to
-  /// automatically free them once they are guaranteed to no longer be used.
-  std::vector<std::unique_ptr<FileCheckNumericVariable>> NumericVariables;
-
-  /// Vector holding pointers to all substitutions. Used to automatically free
-  /// them once they are guaranteed to no longer be used.
-  std::vector<std::unique_ptr<FileCheckSubstitution>> Substitutions;
-
-public:
-  /// \returns the value of string variable \p VarName or an error if no such
-  /// variable has been defined.
-  Expected<StringRef> getPatternVarValue(StringRef VarName);
-
-  /// Defines string and numeric variables from definitions given on the
-  /// command line, passed as a vector of [#]VAR=VAL strings in
-  /// \p CmdlineDefines. \returns an error list containing diagnostics against
-  /// \p SM for all definition parsing failures, if any, or Success otherwise.
-  Error defineCmdlineVariables(std::vector<std::string> &CmdlineDefines,
-                               SourceMgr &SM);
-
-  /// Create @LINE pseudo variable. Value is set when pattern are being
-  /// matched.
-  void createLineVariable();
-
-  /// Undefines local variables (variables whose name does not start with a '$'
-  /// sign), i.e. removes them from GlobalVariableTable and from
-  /// GlobalNumericVariableTable and also clears the value of numeric
-  /// variables.
-  void clearLocalVars();
-
-private:
-  /// Makes a new numeric variable and registers it for destruction when the
-  /// context is destroyed.
-  template <class... Types>
-  FileCheckNumericVariable *makeNumericVariable(Types... args);
-
-  /// Makes a new string substitution and registers it for destruction when the
-  /// context is destroyed.
-  FileCheckSubstitution *makeStringSubstitution(StringRef VarName,
-                                                size_t InsertIdx);
-
-  /// Makes a new numeric substitution and registers it for destruction when
-  /// the context is destroyed.
-  FileCheckSubstitution *
-  makeNumericSubstitution(StringRef ExpressionStr,
-                          std::unique_ptr<FileCheckExpressionAST> ExpressionAST,
-                          size_t InsertIdx);
-};
-
-/// Class to represent an error holding a diagnostic with location information
-/// used when printing it.
-class FileCheckErrorDiagnostic : public ErrorInfo<FileCheckErrorDiagnostic> {
-private:
-  SMDiagnostic Diagnostic;
-
-public:
-  static char ID;
-
-  FileCheckErrorDiagnostic(SMDiagnostic &&Diag) : Diagnostic(Diag) {}
-
-  std::error_code convertToErrorCode() const override {
-    return inconvertibleErrorCode();
-  }
-
-  /// Print diagnostic associated with this error when printing the error.
-  void log(raw_ostream &OS) const override { Diagnostic.print(nullptr, OS); }
-
-  static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg) {
-    return make_error<FileCheckErrorDiagnostic>(
-        SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg));
-  }
-
-  static Error get(const SourceMgr &SM, StringRef Buffer, const Twine &ErrMsg) {
-    return get(SM, SMLoc::getFromPointer(Buffer.data()), ErrMsg);
-  }
-};
-
-class FileCheckNotFoundError : public ErrorInfo<FileCheckNotFoundError> {
-public:
-  static char ID;
-
-  std::error_code convertToErrorCode() const override {
-    return inconvertibleErrorCode();
-  }
-
-  /// Print diagnostic associated with this error when printing the error.
-  void log(raw_ostream &OS) const override {
-    OS << "String not found in input";
-  }
-};
-
-class FileCheckPattern {
-  SMLoc PatternLoc;
-
-  /// A fixed string to match as the pattern or empty if this pattern requires
-  /// a regex match.
-  StringRef FixedStr;
-
-  /// A regex string to match as the pattern or empty if this pattern requires
-  /// a fixed string to match.
-  std::string RegExStr;
-
-  /// Entries in this vector represent a substitution of a string variable or
-  /// an expression in the RegExStr regex at match time. For example, in the
-  /// case of a CHECK directive with the pattern "foo[[bar]]baz[[#N+1]]",
-  /// RegExStr will contain "foobaz" and we'll get two entries in this vector
-  /// that tells us to insert the value of string variable "bar" at offset 3
-  /// and the value of expression "N+1" at offset 6.
-  std::vector<FileCheckSubstitution *> Substitutions;
-
-  /// Maps names of string variables defined in a pattern to the number of
-  /// their parenthesis group in RegExStr capturing their last definition.
-  ///
-  /// E.g. for the pattern "foo[[bar:.*]]baz([[bar]][[QUUX]][[bar:.*]])",
-  /// RegExStr will be "foo(.*)baz(\1<quux value>(.*))" where <quux value> is
-  /// the value captured for QUUX on the earlier line where it was defined, and
-  /// VariableDefs will map "bar" to the third parenthesis group which captures
-  /// the second definition of "bar".
-  ///
-  /// Note: uses std::map rather than StringMap to be able to get the key when
-  /// iterating over values.
-  std::map<StringRef, unsigned> VariableDefs;
-
-  /// Structure representing the definition of a numeric variable in a pattern.
-  /// It holds the pointer to the class representing the numeric variable whose
-  /// value is being defined and the number of the parenthesis group in
-  /// RegExStr to capture that value.
-  struct FileCheckNumericVariableMatch {
-    /// Pointer to class representing the numeric variable whose value is being
-    /// defined.
-    FileCheckNumericVariable *DefinedNumericVariable;
-
-    /// Number of the parenthesis group in RegExStr that captures the value of
-    /// this numeric variable definition.
-    unsigned CaptureParenGroup;
-  };
-
-  /// Holds the number of the parenthesis group in RegExStr and pointer to the
-  /// corresponding FileCheckNumericVariable class instance of all numeric
-  /// variable definitions. Used to set the matched value of all those
-  /// variables.
-  StringMap<FileCheckNumericVariableMatch> NumericVariableDefs;
-
-  /// Pointer to a class instance holding the global state shared by all
-  /// patterns:
-  /// - separate tables with the values of live string and numeric variables
-  ///   respectively at the start of any given CHECK line;
-  /// - table holding whether a string variable has been defined at any given
-  ///   point during the parsing phase.
-  FileCheckPatternContext *Context;
-
-  Check::FileCheckType CheckTy;
-
-  /// Line number for this CHECK pattern or None if it is an implicit pattern.
-  /// Used to determine whether a variable definition is made on an earlier
-  /// line to the one with this CHECK.
-  Optional<size_t> LineNumber;
-
-public:
-  FileCheckPattern(Check::FileCheckType Ty, FileCheckPatternContext *Context,
-                   Optional<size_t> Line = None)
-      : Context(Context), CheckTy(Ty), LineNumber(Line) {}
-
-  /// \returns the location in source code.
-  SMLoc getLoc() const { return PatternLoc; }
-
-  /// \returns the pointer to the global state for all patterns in this
-  /// FileCheck instance.
-  FileCheckPatternContext *getContext() const { return Context; }
-
-  /// \returns whether \p C is a valid first character for a variable name.
-  static bool isValidVarNameStart(char C);
-
-  /// Parsing information about a variable.
-  struct VariableProperties {
-    StringRef Name;
-    bool IsPseudo;
-  };
-
-  /// Parses the string at the start of \p Str for a variable name. \returns
-  /// a VariableProperties structure holding the variable name and whether it
-  /// is the name of a pseudo variable, or an error holding a diagnostic
-  /// against \p SM if parsing fail. If parsing was successful, also strips
-  /// \p Str from the variable name.
-  static Expected<VariableProperties> parseVariable(StringRef &Str,
-                                                    const SourceMgr &SM);
-  /// Parses \p Expr for a numeric substitution block at line \p LineNumber,
-  /// or before input is parsed if \p LineNumber is None. Parameter
-  /// \p IsLegacyLineExpr indicates whether \p Expr should be a legacy @LINE
-  /// expression and \p Context points to the class instance holding the live
-  /// string and numeric variables. \returns a pointer to the class instance
-  /// representing the AST of the expression whose value must be substitued, or
-  /// an error holding a diagnostic against \p SM if parsing fails. If
-  /// substitution was successful, sets \p DefinedNumericVariable to point to
-  /// the class representing the numeric variable defined in this numeric
-  /// substitution block, or None if this block does not define any variable.
-  static Expected<std::unique_ptr<FileCheckExpressionAST>>
-  parseNumericSubstitutionBlock(
-      StringRef Expr,
-      Optional<FileCheckNumericVariable *> &DefinedNumericVariable,
-      bool IsLegacyLineExpr, Optional<size_t> LineNumber,
-      FileCheckPatternContext *Context, const SourceMgr &SM);
-  /// Parses the pattern in \p PatternStr and initializes this FileCheckPattern
-  /// instance accordingly.
-  ///
-  /// \p Prefix provides which prefix is being matched, \p Req describes the
-  /// global options that influence the parsing such as whitespace
-  /// canonicalization, \p SM provides the SourceMgr used for error reports.
-  /// \returns true in case of an error, false otherwise.
-  bool parsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM,
-                    const FileCheckRequest &Req);
-  /// Matches the pattern string against the input buffer \p Buffer
-  ///
-  /// \returns the position that is matched or an error indicating why matching
-  /// failed. If there is a match, updates \p MatchLen with the size of the
-  /// matched string.
-  ///
-  /// The GlobalVariableTable StringMap in the FileCheckPatternContext class
-  /// instance provides the current values of FileCheck string variables and
-  /// is updated if this match defines new values. Likewise, the
-  /// GlobalNumericVariableTable StringMap in the same class provides the
-  /// current values of FileCheck numeric variables and is updated if this
-  /// match defines new numeric values.
-  Expected<size_t> match(StringRef Buffer, size_t &MatchLen,
-                         const SourceMgr &SM) const;
-  /// Prints the value of successful substitutions or the name of the undefined
-  /// string or numeric variables preventing a successful substitution.
-  void printSubstitutions(const SourceMgr &SM, StringRef Buffer,
-                          SMRange MatchRange = None) const;
-  void printFuzzyMatch(const SourceMgr &SM, StringRef Buffer,
-                       std::vector<FileCheckDiag> *Diags) const;
-
-  bool hasVariable() const {
-    return !(Substitutions.empty() && VariableDefs.empty());
-  }
-
-  Check::FileCheckType getCheckTy() const { return CheckTy; }
-
-  int getCount() const { return CheckTy.getCount(); }
-
-private:
-  bool AddRegExToRegEx(StringRef RS, unsigned &CurParen, SourceMgr &SM);
-  void AddBackrefToRegEx(unsigned BackrefNum);
-  /// Computes an arbitrary estimate for the quality of matching this pattern
-  /// at the start of \p Buffer; a distance of zero should correspond to a
-  /// perfect match.
-  unsigned computeMatchDistance(StringRef Buffer) const;
-  /// Finds the closing sequence of a regex variable usage or definition.
-  ///
-  /// \p Str has to point in the beginning of the definition (right after the
-  /// opening sequence). \p SM holds the SourceMgr used for error repporting.
-  ///  \returns the offset of the closing sequence within Str, or npos if it
-  /// was not found.
-  size_t FindRegexVarEnd(StringRef Str, SourceMgr &SM);
-
-  /// Parses \p Expr for the name of a numeric variable to be defined at line
-  /// \p LineNumber, or before input is parsed if \p LineNumber is None.
-  /// \returns a pointer to the class instance representing that variable,
-  /// creating it if needed, or an error holding a diagnostic against \p SM
-  /// should defining such a variable be invalid.
-  static Expected<FileCheckNumericVariable *> parseNumericVariableDefinition(
-      StringRef &Expr, FileCheckPatternContext *Context,
-      Optional<size_t> LineNumber, const SourceMgr &SM);
-  /// Parses \p Name as a (pseudo if \p IsPseudo is true) numeric variable use
-  /// at line \p LineNumber, or before input is parsed if \p LineNumber is
-  /// None. Parameter \p Context points to the class instance holding the live
-  /// string and numeric variables. \returns the pointer to the class instance
-  /// representing that variable if successful, or an error holding a
-  /// diagnostic against \p SM otherwise.
-  static Expected<std::unique_ptr<FileCheckNumericVariableUse>>
-  parseNumericVariableUse(StringRef Name, bool IsPseudo,
-                          Optional<size_t> LineNumber,
-                          FileCheckPatternContext *Context,
-                          const SourceMgr &SM);
-  enum class AllowedOperand { LineVar, Literal, Any };
-  /// Parses \p Expr for use of a numeric operand at line \p LineNumber, or
-  /// before input is parsed if \p LineNumber is None. Accepts both literal
-  /// values and numeric variables, depending on the value of \p AO. Parameter
-  /// \p Context points to the class instance holding the live string and
-  /// numeric variables. \returns the class representing that operand in the
-  /// AST of the expression or an error holding a diagnostic against \p SM
-  /// otherwise.
-  static Expected<std::unique_ptr<FileCheckExpressionAST>>
-  parseNumericOperand(StringRef &Expr, AllowedOperand AO,
-                      Optional<size_t> LineNumber,
-                      FileCheckPatternContext *Context, const SourceMgr &SM);
-  /// Parses \p Expr for a binary operation at line \p LineNumber, or before
-  /// input is parsed if \p LineNumber is None. The left operand of this binary
-  /// operation is given in \p LeftOp and \p IsLegacyLineExpr indicates whether
-  /// we are parsing a legacy @LINE expression. Parameter \p Context points to
-  /// the class instance holding the live string and numeric variables.
-  /// \returns the class representing the binary operation in the AST of the
-  /// expression, or an error holding a diagnostic against \p SM otherwise.
-  static Expected<std::unique_ptr<FileCheckExpressionAST>>
-  parseBinop(StringRef &Expr, std::unique_ptr<FileCheckExpressionAST> LeftOp,
-             bool IsLegacyLineExpr, Optional<size_t> LineNumber,
-             FileCheckPatternContext *Context, const SourceMgr &SM);
-};
-
-//===----------------------------------------------------------------------===//
-/// Summary of a FileCheck diagnostic.
-//===----------------------------------------------------------------------===//
-
 struct FileCheckDiag {
   /// What is the FileCheck directive for this diagnostic?
   Check::FileCheckType CheckTy;
@@ -677,61 +130,20 @@ struct FileCheckDiag {
                 SMLoc CheckLoc, MatchType MatchTy, SMRange InputRange);
 };
 
-//===----------------------------------------------------------------------===//
-// Check Strings.
-//===----------------------------------------------------------------------===//
-
-/// A check that we found in the input file.
-struct FileCheckString {
-  /// The pattern to match.
-  FileCheckPattern Pat;
-
-  /// Which prefix name this check matched.
-  StringRef Prefix;
-
-  /// The location in the match file that the check string was specified.
-  SMLoc Loc;
-
-  /// All of the strings that are disallowed from occurring between this match
-  /// string and the previous one (or start of file).
-  std::vector<FileCheckPattern> DagNotStrings;
-
-  FileCheckString(const FileCheckPattern &P, StringRef S, SMLoc L)
-      : Pat(P), Prefix(S), Loc(L) {}
-
-  /// Matches check string and its "not strings" and/or "dag strings".
-  size_t Check(const SourceMgr &SM, StringRef Buffer, bool IsLabelScanMode,
-               size_t &MatchLen, FileCheckRequest &Req,
-               std::vector<FileCheckDiag> *Diags) const;
-
-  /// Verifies that there is a single line in the given \p Buffer. Errors are
-  /// reported against \p SM.
-  bool CheckNext(const SourceMgr &SM, StringRef Buffer) const;
-  /// Verifies that there is no newline in the given \p Buffer. Errors are
-  /// reported against \p SM.
-  bool CheckSame(const SourceMgr &SM, StringRef Buffer) const;
-  /// Verifies that none of the strings in \p NotStrings are found in the given
-  /// \p Buffer. Errors are reported against \p SM and diagnostics recorded in
-  /// \p Diags according to the verbosity level set in \p Req.
-  bool CheckNot(const SourceMgr &SM, StringRef Buffer,
-                const std::vector<const FileCheckPattern *> &NotStrings,
-                const FileCheckRequest &Req,
-                std::vector<FileCheckDiag> *Diags) const;
-  /// Matches "dag strings" and their mixed "not strings".
-  size_t CheckDag(const SourceMgr &SM, StringRef Buffer,
-                  std::vector<const FileCheckPattern *> &NotStrings,
-                  const FileCheckRequest &Req,
-                  std::vector<FileCheckDiag> *Diags) const;
-};
+class FileCheckPatternContext;
+struct FileCheckString;
 
 /// FileCheck class takes the request and exposes various methods that
 /// use information from the request.
 class FileCheck {
   FileCheckRequest Req;
-  FileCheckPatternContext PatternContext;
+  std::unique_ptr<FileCheckPatternContext> PatternContext;
+  // C++17 TODO: make this a plain std::vector.
+  std::unique_ptr<std::vector<FileCheckString>> CheckStrings;
 
 public:
-  FileCheck(FileCheckRequest Req) : Req(Req) {}
+  explicit FileCheck(FileCheckRequest Req);
+  ~FileCheck();
 
   // Combines the check prefixes into a single regex so that we can efficiently
   // scan for any of the set.
@@ -741,13 +153,11 @@ class FileCheck {
   Regex buildCheckPrefixRegex();
 
   /// Reads the check file from \p Buffer and records the expected strings it
-  /// contains in the \p CheckStrings vector. Errors are reported against
-  /// \p SM.
+  /// contains. Errors are reported against \p SM.
   ///
   /// Only expected strings whose prefix is one of those listed in \p PrefixRE
   /// are recorded. \returns true in case of an error, false otherwise.
-  bool ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
-                     std::vector<FileCheckString> &CheckStrings);
+  bool readCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE);
 
   bool ValidateCheckPrefixes();
 
@@ -757,13 +167,14 @@ class FileCheck {
                              SmallVectorImpl<char> &OutputBuffer);
 
   /// Checks the input to FileCheck provided in the \p Buffer against the
-  /// \p CheckStrings read from the check file and record diagnostics emitted
+  /// expected strings read from the check file and record diagnostics emitted
   /// in \p Diags. Errors are recorded against \p SM.
   ///
   /// \returns false if the input fails to satisfy the checks.
-  bool CheckInput(SourceMgr &SM, StringRef Buffer,
-                  ArrayRef<FileCheckString> CheckStrings,
+  bool checkInput(SourceMgr &SM, StringRef Buffer,
                   std::vector<FileCheckDiag> *Diags = nullptr);
 };
+
 } // namespace llvm
+
 #endif
diff --git a/llvm/include/llvm/Support/FileUtilities.h b/llvm/include/llvm/Support/FileUtilities.h
index 715ed80d5377a..04efdced32a4a 100644
--- a/llvm/include/llvm/Support/FileUtilities.h
+++ b/llvm/include/llvm/Support/FileUtilities.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 
@@ -75,10 +76,40 @@ namespace llvm {
     void releaseFile() { DeleteIt = false; }
   };
 
+  enum class atomic_write_error {
+    failed_to_create_uniq_file = 0,
+    output_stream_error,
+    failed_to_rename_temp_file
+  };
+
+  class AtomicFileWriteError : public llvm::ErrorInfo<AtomicFileWriteError> {
+  public:
+    AtomicFileWriteError(atomic_write_error Error) : Error(Error) {}
+
+    void log(raw_ostream &OS) const override;
+
+    const atomic_write_error Error;
+    static char ID;
+
+  private:
+    // Users are not expected to use error_code.
+    std::error_code convertToErrorCode() const override {
+      return llvm::inconvertibleErrorCode();
+    }
+  };
+
+  // atomic_write_error + whatever the Writer can return
+
   /// Creates a unique file with name according to the given \p TempPathModel,
   /// writes content of \p Buffer to the file and renames it to \p FinalPath.
+  ///
+  /// \returns \c AtomicFileWriteError in case of error.
   llvm::Error writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
                                   StringRef Buffer);
+
+  llvm::Error
+  writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
+                      std::function<llvm::Error(llvm::raw_ostream &)> Writer);
 } // End llvm namespace
 
 #endif
diff --git a/llvm/include/llvm/Support/MachineValueType.h b/llvm/include/llvm/Support/MachineValueType.h
index f15475b118bd7..bc617e3b90e6f 100644
--- a/llvm/include/llvm/Support/MachineValueType.h
+++ b/llvm/include/llvm/Support/MachineValueType.h
@@ -65,156 +65,162 @@ namespace llvm {
       v32i1          =  19,   //   32 x i1
       v64i1          =  20,   //   64 x i1
       v128i1         =  21,   //  128 x i1
-      v512i1         =  22,   //  512 x i1
-      v1024i1        =  23,   // 1024 x i1
-
-      v1i8           =  24,   //  1 x i8
-      v2i8           =  25,   //  2 x i8
-      v4i8           =  26,   //  4 x i8
-      v8i8           =  27,   //  8 x i8
-      v16i8          =  28,   // 16 x i8
-      v32i8          =  29,   // 32 x i8
-      v64i8          =  30,   // 64 x i8
-      v128i8         =  31,   //128 x i8
-      v256i8         =  32,   //256 x i8
-
-      v1i16          =  33,   //  1 x i16
-      v2i16          =  34,   //  2 x i16
-      v3i16          =  35,   //  3 x i16
-      v4i16          =  36,   //  4 x i16
-      v8i16          =  37,   //  8 x i16
-      v16i16         =  38,   // 16 x i16
-      v32i16         =  39,   // 32 x i16
-      v64i16         =  40,   // 64 x i16
-      v128i16        =  41,   //128 x i16
-
-      v1i32          =  42,   //    1 x i32
-      v2i32          =  43,   //    2 x i32
-      v3i32          =  44,   //    3 x i32
-      v4i32          =  45,   //    4 x i32
-      v5i32          =  46,   //    5 x i32
-      v8i32          =  47,   //    8 x i32
-      v16i32         =  48,   //   16 x i32
-      v32i32         =  49,   //   32 x i32
-      v64i32         =  50,   //   64 x i32
-      v128i32        =  51,   //  128 x i32
-      v256i32        =  52,   //  256 x i32
-      v512i32        =  53,   //  512 x i32
-      v1024i32       =  54,   // 1024 x i32
-      v2048i32       =  55,   // 2048 x i32
-
-      v1i64          =  56,   //  1 x i64
-      v2i64          =  57,   //  2 x i64
-      v4i64          =  58,   //  4 x i64
-      v8i64          =  59,   //  8 x i64
-      v16i64         =  60,   // 16 x i64
-      v32i64         =  61,   // 32 x i64
-
-      v1i128         =  62,   //  1 x i128
-
-      // Scalable integer types
-      nxv1i1         =  63,   // n x  1 x i1
-      nxv2i1         =  64,   // n x  2 x i1
-      nxv4i1         =  65,   // n x  4 x i1
-      nxv8i1         =  66,   // n x  8 x i1
-      nxv16i1        =  67,   // n x 16 x i1
-      nxv32i1        =  68,   // n x 32 x i1
-
-      nxv1i8         =  69,   // n x  1 x i8
-      nxv2i8         =  70,   // n x  2 x i8
-      nxv4i8         =  71,   // n x  4 x i8
-      nxv8i8         =  72,   // n x  8 x i8
-      nxv16i8        =  73,   // n x 16 x i8
-      nxv32i8        =  74,   // n x 32 x i8
-
-      nxv1i16        =  75,   // n x  1 x i16
-      nxv2i16        =  76,   // n x  2 x i16
-      nxv4i16        =  77,   // n x  4 x i16
-      nxv8i16        =  78,   // n x  8 x i16
-      nxv16i16       =  79,   // n x 16 x i16
-      nxv32i16       =  80,   // n x 32 x i16
-
-      nxv1i32        =  81,   // n x  1 x i32
-      nxv2i32        =  82,   // n x  2 x i32
-      nxv4i32        =  83,   // n x  4 x i32
-      nxv8i32        =  84,   // n x  8 x i32
-      nxv16i32       =  85,   // n x 16 x i32
-      nxv32i32       =  86,   // n x 32 x i32
-
-      nxv1i64        =  87,   // n x  1 x i64
-      nxv2i64        =  88,   // n x  2 x i64
-      nxv4i64        =  89,   // n x  4 x i64
-      nxv8i64        =  90,   // n x  8 x i64
-      nxv16i64       =  91,   // n x 16 x i64
-      nxv32i64       =  92,   // n x 32 x i64
-
-      FIRST_INTEGER_VECTOR_VALUETYPE = v1i1,
-      LAST_INTEGER_VECTOR_VALUETYPE = nxv32i64,
-
-      FIRST_INTEGER_SCALABLE_VALUETYPE = nxv1i1,
-      LAST_INTEGER_SCALABLE_VALUETYPE = nxv32i64,
-
-      v2f16          =  93,   //    2 x f16
-      v3f16          =  94,   //    3 x f16
-      v4f16          =  95,   //    4 x f16
-      v8f16          =  96,   //    8 x f16
-      v16f16         =  97,   //   16 x f16
-      v32f16         =  98,   //   32 x f16
-      v1f32          =  99,   //    1 x f32
-      v2f32          =  100,  //    2 x f32
-      v3f32          =  101,  //    3 x f32
-      v4f32          =  102,  //    4 x f32
-      v5f32          =  103,  //    5 x f32
-      v8f32          =  104,  //    8 x f32
-      v16f32         =  105,  //   16 x f32
-      v32f32         =  106,  //   32 x f32
-      v64f32         =  107,  //   64 x f32
-      v128f32        =  108,  //  128 x f32
-      v256f32        =  109,  //  256 x f32
-      v512f32        =  110,  //  512 x f32
-      v1024f32       =  111,  // 1024 x f32
-      v2048f32       =  112,  // 2048 x f32
-      v1f64          =  113,  //    1 x f64
-      v2f64          =  114,  //    2 x f64
-      v4f64          =  115,  //    4 x f64
-      v8f64          =  116,  //    8 x f64
-
-      nxv2f16        =  117,  // n x  2 x f16
-      nxv4f16        =  118,  // n x  4 x f16
-      nxv8f16        =  119,  // n x  8 x f16
-      nxv1f32        =  120,  // n x  1 x f32
-      nxv2f32        =  121,  // n x  2 x f32
-      nxv4f32        =  122,  // n x  4 x f32
-      nxv8f32        =  123,  // n x  8 x f32
-      nxv16f32       =  124,  // n x 16 x f32
-      nxv1f64        =  125,  // n x  1 x f64
-      nxv2f64        =  126,  // n x  2 x f64
-      nxv4f64        =  127,  // n x  4 x f64
-      nxv8f64        =  128,  // n x  8 x f64
-
-      FIRST_FP_VECTOR_VALUETYPE = v2f16,
-      LAST_FP_VECTOR_VALUETYPE = nxv8f64,
-
-      FIRST_FP_SCALABLE_VALUETYPE = nxv2f16,
-      LAST_FP_SCALABLE_VALUETYPE = nxv8f64,
+      v256i1         =  22,   //  256 x i1
+      v512i1         =  23,   //  512 x i1
+      v1024i1        =  24,   // 1024 x i1
+
+      v1i8           =  25,   //  1 x i8
+      v2i8           =  26,   //  2 x i8
+      v4i8           =  27,   //  4 x i8
+      v8i8           =  28,   //  8 x i8
+      v16i8          =  29,   // 16 x i8
+      v32i8          =  30,   // 32 x i8
+      v64i8          =  31,   // 64 x i8
+      v128i8         =  32,   //128 x i8
+      v256i8         =  33,   //256 x i8
+
+      v1i16          =  34,   //  1 x i16
+      v2i16          =  35,   //  2 x i16
+      v3i16          =  36,   //  3 x i16
+      v4i16          =  37,   //  4 x i16
+      v8i16          =  38,   //  8 x i16
+      v16i16         =  39,   // 16 x i16
+      v32i16         =  40,   // 32 x i16
+      v64i16         =  41,   // 64 x i16
+      v128i16        =  42,   //128 x i16
+
+      v1i32          =  43,   //    1 x i32
+      v2i32          =  44,   //    2 x i32
+      v3i32          =  45,   //    3 x i32
+      v4i32          =  46,   //    4 x i32
+      v5i32          =  47,   //    5 x i32
+      v8i32          =  48,   //    8 x i32
+      v16i32         =  49,   //   16 x i32
+      v32i32         =  50,   //   32 x i32
+      v64i32         =  51,   //   64 x i32
+      v128i32        =  52,   //  128 x i32
+      v256i32        =  53,   //  256 x i32
+      v512i32        =  54,   //  512 x i32
+      v1024i32       =  55,   // 1024 x i32
+      v2048i32       =  56,   // 2048 x i32
+
+      v1i64          =  57,   //  1 x i64
+      v2i64          =  58,   //  2 x i64
+      v4i64          =  59,   //  4 x i64
+      v8i64          =  60,   //  8 x i64
+      v16i64         =  61,   // 16 x i64
+      v32i64         =  62,   // 32 x i64
+
+      v1i128         =  63,   //  1 x i128
+
+      FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
+      LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE = v1i128,
+
+      v2f16          =  64,   //    2 x f16
+      v3f16          =  65,   //    3 x f16
+      v4f16          =  66,   //    4 x f16
+      v8f16          =  67,   //    8 x f16
+      v16f16         =  68,   //   16 x f16
+      v32f16         =  69,   //   32 x f16
+      v1f32          =  70,   //    1 x f32
+      v2f32          =  71,   //    2 x f32
+      v3f32          =  72,   //    3 x f32
+      v4f32          =  73,   //    4 x f32
+      v5f32          =  74,   //    5 x f32
+      v8f32          =  75,   //    8 x f32
+      v16f32         =  76,   //   16 x f32
+      v32f32         =  77,   //   32 x f32
+      v64f32         =  78,   //   64 x f32
+      v128f32        =  79,   //  128 x f32
+      v256f32        =  80,   //  256 x f32
+      v512f32        =  81,   //  512 x f32
+      v1024f32       =  82,   // 1024 x f32
+      v2048f32       =  83,   // 2048 x f32
+      v1f64          =  84,   //    1 x f64
+      v2f64          =  85,   //    2 x f64
+      v4f64          =  86,   //    4 x f64
+      v8f64          =  87,   //    8 x f64
+
+      FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE = v2f16,
+      LAST_FP_FIXEDLEN_VECTOR_VALUETYPE = v8f64,
+
+      FIRST_FIXEDLEN_VECTOR_VALUETYPE = v1i1,
+      LAST_FIXEDLEN_VECTOR_VALUETYPE = v8f64,
+
+      nxv1i1         =  88,   // n x  1 x i1
+      nxv2i1         =  89,   // n x  2 x i1
+      nxv4i1         =  90,   // n x  4 x i1
+      nxv8i1         =  91,   // n x  8 x i1
+      nxv16i1        =  92,   // n x 16 x i1
+      nxv32i1        =  93,   // n x 32 x i1
+
+      nxv1i8         =  94,   // n x  1 x i8
+      nxv2i8         =  95,   // n x  2 x i8
+      nxv4i8         =  96,   // n x  4 x i8
+      nxv8i8         =  97,   // n x  8 x i8
+      nxv16i8        =  98,   // n x 16 x i8
+      nxv32i8        =  99,   // n x 32 x i8
+
+      nxv1i16        =  100,  // n x  1 x i16
+      nxv2i16        =  101,  // n x  2 x i16
+      nxv4i16        =  102,  // n x  4 x i16
+      nxv8i16        =  103,  // n x  8 x i16
+      nxv16i16       =  104,  // n x 16 x i16
+      nxv32i16       =  105,  // n x 32 x i16
+
+      nxv1i32        =  106,  // n x  1 x i32
+      nxv2i32        =  107,  // n x  2 x i32
+      nxv4i32        =  108,  // n x  4 x i32
+      nxv8i32        =  109,  // n x  8 x i32
+      nxv16i32       =  110,  // n x 16 x i32
+      nxv32i32       =  111,  // n x 32 x i32
+
+      nxv1i64        =  112,  // n x  1 x i64
+      nxv2i64        =  113,  // n x  2 x i64
+      nxv4i64        =  114,  // n x  4 x i64
+      nxv8i64        =  115,  // n x  8 x i64
+      nxv16i64       =  116,  // n x 16 x i64
+      nxv32i64       =  117,  // n x 32 x i64
+
+      FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
+      LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE = nxv32i64,
+
+      nxv2f16        =  118,  // n x  2 x f16
+      nxv4f16        =  119,  // n x  4 x f16
+      nxv8f16        =  120,  // n x  8 x f16
+      nxv1f32        =  121,  // n x  1 x f32
+      nxv2f32        =  122,  // n x  2 x f32
+      nxv4f32        =  123,  // n x  4 x f32
+      nxv8f32        =  124,  // n x  8 x f32
+      nxv16f32       =  125,  // n x 16 x f32
+      nxv1f64        =  126,  // n x  1 x f64
+      nxv2f64        =  127,  // n x  2 x f64
+      nxv4f64        =  128,  // n x  4 x f64
+      nxv8f64        =  129,  // n x  8 x f64
+
+      FIRST_FP_SCALABLE_VECTOR_VALUETYPE = nxv2f16,
+      LAST_FP_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
+
+      FIRST_SCALABLE_VECTOR_VALUETYPE = nxv1i1,
+      LAST_SCALABLE_VECTOR_VALUETYPE = nxv8f64,
 
       FIRST_VECTOR_VALUETYPE = v1i1,
       LAST_VECTOR_VALUETYPE  = nxv8f64,
 
-      x86mmx         =  129,   // This is an X86 MMX value
+      x86mmx         =  130,   // This is an X86 MMX value
 
-      Glue           =  130,   // This glues nodes together during pre-RA sched
+      Glue           =  131,   // This glues nodes together during pre-RA sched
 
-      isVoid         =  131,   // This has no value
+      isVoid         =  132,   // This has no value
 
-      Untyped        =  132,   // This value takes a register, but has
+      Untyped        =  133,   // This value takes a register, but has
                                // unspecified type.  The register class
                                // will be determined by the opcode.
 
-      exnref         =  133,   // WebAssembly's exnref type
+      exnref         =  134,   // WebAssembly's exnref type
 
       FIRST_VALUETYPE = 1,     // This is always the beginning of the list.
-      LAST_VALUETYPE =  134,   // This always remains at the end of the list.
+      LAST_VALUETYPE =  135,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -278,16 +284,20 @@ namespace llvm {
     bool isFloatingPoint() const {
       return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
                SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
+              (SimpleTy >= MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE));
     }
 
     /// Return true if this is an integer or a vector integer type.
     bool isInteger() const {
       return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
                SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE));
+              (SimpleTy >= MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE));
     }
 
     /// Return true if this is an integer, not including vectors.
@@ -305,10 +315,13 @@ namespace llvm {
     /// Return true if this is a vector value type where the
     /// runtime length is machine dependent
     bool isScalableVector() const {
-      return ((SimpleTy >= MVT::FIRST_INTEGER_SCALABLE_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_SCALABLE_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_SCALABLE_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_SCALABLE_VALUETYPE));
+      return (SimpleTy >= MVT::FIRST_SCALABLE_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_SCALABLE_VECTOR_VALUETYPE);
+    }
+
+    bool isFixedLengthVector() const {
+      return (SimpleTy >= MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE);
     }
 
     /// Return true if this is a 16-bit vector type.
@@ -346,7 +359,7 @@ namespace llvm {
       return (SimpleTy == MVT::v16f16 || SimpleTy == MVT::v8f32 ||
               SimpleTy == MVT::v4f64  || SimpleTy == MVT::v32i8 ||
               SimpleTy == MVT::v16i16 || SimpleTy == MVT::v8i32 ||
-              SimpleTy == MVT::v4i64);
+              SimpleTy == MVT::v4i64  || SimpleTy == MVT::v256i1);
     }
 
     /// Return true if this is a 512-bit vector type.
@@ -420,6 +433,7 @@ namespace llvm {
       case v32i1:
       case v64i1:
       case v128i1:
+      case v256i1:
       case v512i1:
       case v1024i1:
       case nxv1i1:
@@ -542,6 +556,7 @@ namespace llvm {
       case v512i1:
       case v512i32:
       case v512f32: return 512;
+      case v256i1:
       case v256i8:
       case v256i32:
       case v256f32: return 256;
@@ -753,6 +768,7 @@ namespace llvm {
       case nxv2f64: return 128;
       case v5i32:
       case v5f32: return 160;
+      case v256i1:
       case v32i8:
       case v16i16:
       case v8i32:
@@ -892,6 +908,7 @@ namespace llvm {
         if (NumElements == 32)   return MVT::v32i1;
         if (NumElements == 64)   return MVT::v64i1;
         if (NumElements == 128)  return MVT::v128i1;
+        if (NumElements == 256)  return MVT::v256i1;
         if (NumElements == 512)  return MVT::v512i1;
         if (NumElements == 1024) return MVT::v1024i1;
         break;
@@ -1104,26 +1121,40 @@ namespace llvm {
                        (MVT::SimpleValueType)(MVT::LAST_VECTOR_VALUETYPE + 1));
     }
 
-    static mvt_range integer_vector_valuetypes() {
+    static mvt_range fixedlen_vector_valuetypes() {
+      return mvt_range(
+               MVT::FIRST_FIXEDLEN_VECTOR_VALUETYPE,
+               (MVT::SimpleValueType)(MVT::LAST_FIXEDLEN_VECTOR_VALUETYPE + 1));
+    }
+
+    static mvt_range scalable_vector_valuetypes() {
       return mvt_range(
-          MVT::FIRST_INTEGER_VECTOR_VALUETYPE,
-          (MVT::SimpleValueType)(MVT::LAST_INTEGER_VECTOR_VALUETYPE + 1));
+               MVT::FIRST_SCALABLE_VECTOR_VALUETYPE,
+               (MVT::SimpleValueType)(MVT::LAST_SCALABLE_VECTOR_VALUETYPE + 1));
     }
 
-    static mvt_range fp_vector_valuetypes() {
+    static mvt_range integer_fixedlen_vector_valuetypes() {
       return mvt_range(
-          MVT::FIRST_FP_VECTOR_VALUETYPE,
-          (MVT::SimpleValueType)(MVT::LAST_FP_VECTOR_VALUETYPE + 1));
+       MVT::FIRST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE,
+       (MVT::SimpleValueType)(MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE + 1));
+    }
+
+    static mvt_range fp_fixedlen_vector_valuetypes() {
+      return mvt_range(
+          MVT::FIRST_FP_FIXEDLEN_VECTOR_VALUETYPE,
+          (MVT::SimpleValueType)(MVT::LAST_FP_FIXEDLEN_VECTOR_VALUETYPE + 1));
     }
 
     static mvt_range integer_scalable_vector_valuetypes() {
-      return mvt_range(MVT::FIRST_INTEGER_SCALABLE_VALUETYPE,
-              (MVT::SimpleValueType)(MVT::LAST_INTEGER_SCALABLE_VALUETYPE + 1));
+      return mvt_range(
+       MVT::FIRST_INTEGER_SCALABLE_VECTOR_VALUETYPE,
+       (MVT::SimpleValueType)(MVT::LAST_INTEGER_SCALABLE_VECTOR_VALUETYPE + 1));
     }
 
     static mvt_range fp_scalable_vector_valuetypes() {
-      return mvt_range(MVT::FIRST_FP_SCALABLE_VALUETYPE,
-                   (MVT::SimpleValueType)(MVT::LAST_FP_SCALABLE_VALUETYPE + 1));
+      return mvt_range(
+            MVT::FIRST_FP_SCALABLE_VECTOR_VALUETYPE,
+            (MVT::SimpleValueType)(MVT::LAST_FP_SCALABLE_VECTOR_VALUETYPE + 1));
     }
     /// @}
   };
diff --git a/llvm/include/llvm/Support/OnDiskHashTable.h b/llvm/include/llvm/Support/OnDiskHashTable.h
index c4c0ac97bdb88..11dc0de0f354c 100644
--- a/llvm/include/llvm/Support/OnDiskHashTable.h
+++ b/llvm/include/llvm/Support/OnDiskHashTable.h
@@ -208,8 +208,7 @@ template <typename Info> class OnDiskChainedHashTableGenerator {
 
     // Pad with zeros so that we can start the hashtable at an aligned address.
     offset_type TableOff = Out.tell();
-    uint64_t N =
-        llvm::offsetToAlignment(TableOff, llvm::Align(alignof(offset_type)));
+    uint64_t N = offsetToAlignment(TableOff, Align(alignof(offset_type)));
     TableOff += N;
     while (N--)
       LE.write<uint8_t>(0);
diff --git a/llvm/include/llvm/Support/Regex.h b/llvm/include/llvm/Support/Regex.h
index ca5e8cd79a64b..b2620ab4cfc93 100644
--- a/llvm/include/llvm/Support/Regex.h
+++ b/llvm/include/llvm/Support/Regex.h
@@ -57,8 +57,8 @@ namespace llvm {
     Regex(Regex &&regex);
     ~Regex();
 
-    /// isValid - returns the error encountered during regex compilation, or
-    /// matching, if any.
+    /// isValid - returns the error encountered during regex compilation, if
+    /// any.
     bool isValid(std::string &Error) const;
     bool isValid() const { return !error; }
 
@@ -73,8 +73,12 @@ namespace llvm {
     /// with references to the matched group expressions (inside \p String),
     /// the first group is always the entire pattern.
     ///
+    /// \param Error - If non-null, any errors in the matching will be recorded
+    /// as a non-empty string. If there is no error, it will be an empty string.
+    ///
     /// This returns true on a successful match.
-    bool match(StringRef String, SmallVectorImpl<StringRef> *Matches = nullptr);
+    bool match(StringRef String, SmallVectorImpl<StringRef> *Matches = nullptr,
+               std::string *Error = nullptr) const;
 
     /// sub - Return the result of replacing the first match of the regex in
     /// \p String with the \p Repl string. Backreferences like "\0" in the
@@ -85,9 +89,9 @@ namespace llvm {
     ///
     /// \param Error If non-null, any errors in the substitution (invalid
     /// backreferences, trailing backslashes) will be recorded as a non-empty
-    /// string.
+    /// string. If there is no error, it will be an empty string.
     std::string sub(StringRef Repl, StringRef String,
-                    std::string *Error = nullptr);
+                    std::string *Error = nullptr) const;
 
     /// If this function returns true, ^Str$ is an extended regular
     /// expression that matches Str and only Str.
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 31c9e851daed9..627831ea7cb37 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -730,9 +730,10 @@ class RedirectingFileSystem : public vfs::FileSystem {
 
   StringRef getExternalContentsPrefixDir() const;
 
+  void dump(raw_ostream &OS) const;
+  void dumpEntry(raw_ostream &OS, Entry *E, int NumSpaces = 0) const;
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   LLVM_DUMP_METHOD void dump() const;
-  LLVM_DUMP_METHOD void dumpEntry(Entry *E, int NumSpaces = 0) const;
 #endif
 };
 
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
new file mode 100644
index 0000000000000..065e28eca8a69
--- /dev/null
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -0,0 +1,17 @@
+//===- Combine.td - Combine rule definitions ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Declare GlobalISel combine rules and provide mechanisms to opt-out.
+//
+//===----------------------------------------------------------------------===//
+
+// Declares a combiner helper class
+class GICombinerHelper<string classname> {
+  // The class name to use in the generated output.
+  string Classname = classname;
+}
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index fc63696142cc9..93a6135928828 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -443,6 +443,15 @@ class InstructionEncoding {
   bit hasCompleteDecoder = 1;
 }
 
+// Allows specifying an InstructionEncoding by HwMode. If an Instruction specifies
+// an EncodingByHwMode, its Inst and Size members are ignored and Ts are used
+// to encode and decode based on HwMode.
+class EncodingByHwMode<list<HwMode> Ms = [], list<InstructionEncoding> Ts = []>
+    : HwModeSelect<Ms> {
+  // The length of this list must be the same as the length of Ms.
+  list<InstructionEncoding> Objects = Ts;
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction set description - These classes correspond to the C++ classes in
 // the Target/TargetInstrInfo.h file.
@@ -454,6 +463,10 @@ class Instruction : InstructionEncoding {
   dag InOperandList;        // An dag containing the MI use operand list.
   string AsmString = "";    // The .s format to print the instruction with.
 
+  // Allows specifying a canonical InstructionEncoding by HwMode. If non-empty,
+  // the Inst member of this Instruction is ignored.
+  EncodingByHwMode EncodingInfos;
+
   // Pattern - Set to the DAG pattern for this instruction, if we know of one,
   // otherwise, uninitialized.
   list<dag> Pattern;
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 0bf5e30afaeae..285c0ec0fb905 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -25,7 +25,7 @@ namespace llvm {
 
 class Function;
 class GlobalValue;
-class MachineModuleInfo;
+class MachineModuleInfoWrapperPass;
 class Mangler;
 class MCAsmInfo;
 class MCContext;
@@ -157,7 +157,7 @@ class TargetMachine {
   /// The LLVM Module owns a DataLayout that is used for the target independent
   /// optimizations and code generation. This hook provides a target specific
   /// check on the validity of this DataLayout.
-  virtual bool isCompatibleDataLayout(const DataLayout &Candidate) const {
+  bool isCompatibleDataLayout(const DataLayout &Candidate) const {
     return DL == Candidate;
   }
 
@@ -284,12 +284,13 @@ class TargetMachine {
   /// emitted.  Typically this will involve several steps of code generation.
   /// This method should return true if emission of this file type is not
   /// supported, or false on success.
-  /// \p MMI is an optional parameter that, if set to non-nullptr,
+  /// \p MMIWP is an optional parameter that, if set to non-nullptr,
   /// will be used to set the MachineModuloInfo for this PM.
-  virtual bool addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &,
-                                   raw_pwrite_stream *, CodeGenFileType,
-                                   bool /*DisableVerify*/ = true,
-                                   MachineModuleInfo *MMI = nullptr) {
+  virtual bool
+  addPassesToEmitFile(PassManagerBase &, raw_pwrite_stream &,
+                      raw_pwrite_stream *, CodeGenFileType,
+                      bool /*DisableVerify*/ = true,
+                      MachineModuleInfoWrapperPass *MMIWP = nullptr) {
     return true;
   }
 
@@ -341,12 +342,13 @@ class LLVMTargetMachine : public TargetMachine {
 
   /// Add passes to the specified pass manager to get the specified file
   /// emitted.  Typically this will involve several steps of code generation.
-  /// \p MMI is an optional parameter that, if set to non-nullptr,
-  /// will be used to set the MachineModuloInfofor this PM.
-  bool addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out,
-                           raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
-                           bool DisableVerify = true,
-                           MachineModuleInfo *MMI = nullptr) override;
+  /// \p MMIWP is an optional parameter that, if set to non-nullptr,
+  /// will be used to set the MachineModuloInfo for this PM.
+  bool
+  addPassesToEmitFile(PassManagerBase &PM, raw_pwrite_stream &Out,
+                      raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
+                      bool DisableVerify = true,
+                      MachineModuleInfoWrapperPass *MMIWP = nullptr) override;
 
   /// Add passes to the specified pass manager to get machine code emitted with
   /// the MCJIT. This method returns true if machine code is not supported. It
@@ -365,7 +367,7 @@ class LLVMTargetMachine : public TargetMachine {
   /// Adds an AsmPrinter pass to the pipeline that prints assembly or
   /// machine code from the MI representation.
   bool addAsmPrinter(PassManagerBase &PM, raw_pwrite_stream &Out,
-                     raw_pwrite_stream *DwoOut, CodeGenFileType FileTYpe,
+                     raw_pwrite_stream *DwoOut, CodeGenFileType FileType,
                      MCContext &Context);
 
   /// True if the target uses physical regs at Prolog/Epilog insertion
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index f42faad736027..8856cf003af00 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -848,6 +848,11 @@ class ImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
   bit IsAPFloat = 0;
 }
 
+// Convenience wrapper for ImmLeaf to use timm/TargetConstant instead
+// of imm/Constant.
+class TImmLeaf<ValueType vt, code pred, SDNodeXForm xform = NOOP_SDNodeXForm,
+  SDNode ImmNode = timm> : ImmLeaf<vt, pred, xform, ImmNode>;
+
 // An ImmLeaf except that Imm is an APInt. This is useful when you need to
 // zero-extend the immediate instead of sign-extend it.
 //
diff --git a/llvm/include/llvm/TextAPI/MachO/Architecture.h b/llvm/include/llvm/TextAPI/MachO/Architecture.h
index 055baeb0c0f0c..3898cbada68f7 100644
--- a/llvm/include/llvm/TextAPI/MachO/Architecture.h
+++ b/llvm/include/llvm/TextAPI/MachO/Architecture.h
@@ -14,6 +14,7 @@
 #define LLVM_TEXTAPI_MACHO_ARCHITECTURE_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/Triple.h"
 #include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
@@ -39,6 +40,9 @@ StringRef getArchitectureName(Architecture Arch);
 /// Convert an architecture slice to a CPU Type and Subtype pair.
 std::pair<uint32_t, uint32_t> getCPUTypeFromArchitecture(Architecture Arch);
 
+/// Convert a target to an architecture slice.
+Architecture mapToArchitecture(const llvm::Triple &Target);
+
 raw_ostream &operator<<(raw_ostream &OS, Architecture Arch);
 
 } // end namespace MachO.
diff --git a/llvm/include/llvm/TextAPI/MachO/ArchitectureSet.h b/llvm/include/llvm/TextAPI/MachO/ArchitectureSet.h
index d8dfc7f1af214..6e4ede6275b4f 100644
--- a/llvm/include/llvm/TextAPI/MachO/ArchitectureSet.h
+++ b/llvm/include/llvm/TextAPI/MachO/ArchitectureSet.h
@@ -59,6 +59,10 @@ class ArchitectureSet {
 
   ArchSetType rawValue() const { return ArchSet; }
 
+  bool hasX86() const {
+    return has(AK_i386) || has(AK_x86_64) || has(AK_x86_64h);
+  }
+
   template <typename Ty>
   class arch_iterator
       : public std::iterator<std::forward_iterator_tag, Architecture, size_t> {
diff --git a/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h b/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
index 323b05095856a..8bb6b5ff06665 100644
--- a/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
+++ b/llvm/include/llvm/TextAPI/MachO/InterfaceFile.h
@@ -26,21 +26,13 @@
 #include "llvm/TextAPI/MachO/Architecture.h"
 #include "llvm/TextAPI/MachO/ArchitectureSet.h"
 #include "llvm/TextAPI/MachO/PackedVersion.h"
+#include "llvm/TextAPI/MachO/Platform.h"
 #include "llvm/TextAPI/MachO/Symbol.h"
+#include "llvm/TextAPI/MachO/Target.h"
 
 namespace llvm {
 namespace MachO {
 
-/// Defines the list of MachO platforms.
-enum class PlatformKind : unsigned {
-  unknown,
-  macOS = MachO::PLATFORM_MACOS,
-  iOS = MachO::PLATFORM_IOS,
-  tvOS = MachO::PLATFORM_TVOS,
-  watchOS = MachO::PLATFORM_WATCHOS,
-  bridgeOS = MachO::PLATFORM_BRIDGEOS,
-};
-
 /// Defines a list of Objective-C constraints.
 enum class ObjCConstraintType : unsigned {
   /// No constraint.
@@ -89,29 +81,42 @@ class InterfaceFileRef {
 
   InterfaceFileRef(StringRef InstallName) : InstallName(InstallName) {}
 
-  InterfaceFileRef(StringRef InstallName, ArchitectureSet Archs)
-      : InstallName(InstallName), Architectures(Archs) {}
+  InterfaceFileRef(StringRef InstallName, const TargetList Targets)
+      : InstallName(InstallName), Targets(std::move(Targets)) {}
 
   StringRef getInstallName() const { return InstallName; };
-  void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; }
-  ArchitectureSet getArchitectures() const { return Architectures; }
-  bool hasArchitecture(Architecture Arch) const {
-    return Architectures.has(Arch);
+
+  void addTarget(const Target &Target);
+  template <typename RangeT> void addTargets(RangeT &&Targets) {
+    for (const auto &Target : Targets)
+      addTarget(Target(Target));
   }
 
+  using const_target_iterator = TargetList::const_iterator;
+  using const_target_range = llvm::iterator_range<const_target_iterator>;
+  const_target_range targets() const { return {Targets}; }
+
+  ArchitectureSet getArchitectures() const {
+    return mapToArchitectureSet(Targets);
+  }
+
+  PlatformSet getPlatforms() const { return mapToPlatformSet(Targets); }
+
   bool operator==(const InterfaceFileRef &O) const {
-    return std::tie(InstallName, Architectures) ==
-           std::tie(O.InstallName, O.Architectures);
+    return std::tie(InstallName, Targets) == std::tie(O.InstallName, O.Targets);
+  }
+
+  bool operator!=(const InterfaceFileRef &O) const {
+    return std::tie(InstallName, Targets) != std::tie(O.InstallName, O.Targets);
   }
 
   bool operator<(const InterfaceFileRef &O) const {
-    return std::tie(InstallName, Architectures) <
-           std::tie(O.InstallName, O.Architectures);
+    return std::tie(InstallName, Targets) < std::tie(O.InstallName, O.Targets);
   }
 
 private:
   std::string InstallName;
-  ArchitectureSet Architectures;
+  TargetList Targets;
 };
 
 } // end namespace MachO.
@@ -170,27 +175,43 @@ class InterfaceFile {
   /// \return The file type.
   FileType getFileType() const { return FileKind; }
 
-  /// Set the platform.
-  void setPlatform(PlatformKind Platform_) { Platform = Platform_; }
+  /// Get the architectures.
+  ///
+  /// \return The applicable architectures.
+  ArchitectureSet getArchitectures() const {
+    return mapToArchitectureSet(Targets);
+  }
 
-  /// Get the platform.
-  PlatformKind getPlatform() const { return Platform; }
+  /// Get the platforms.
+  ///
+  /// \return The applicable platforms.
+  PlatformSet getPlatforms() const { return mapToPlatformSet(Targets); }
 
-  /// Specify the set of supported architectures by this file.
-  void setArchitectures(ArchitectureSet Architectures_) {
-    Architectures = Architectures_;
-  }
+  /// Set and add target.
+  ///
+  /// \param Target the target to add into.
+  void addTarget(const Target &Target);
 
-  /// Add the set of supported architectures by this file.
-  void addArchitectures(ArchitectureSet Architectures_) {
-    Architectures |= Architectures_;
+  /// Set and add targets.
+  ///
+  /// Add the subset of llvm::triples that is supported by Tapi
+  ///
+  /// \param Targets the collection of targets.
+  template <typename RangeT> void addTargets(RangeT &&Targets) {
+    for (const auto &Target_ : Targets)
+      addTarget(Target(Target_));
   }
 
-  /// Add supported architecture by this file..
-  void addArch(Architecture Arch) { Architectures.set(Arch); }
+  using const_target_iterator = TargetList::const_iterator;
+  using const_target_range = llvm::iterator_range<const_target_iterator>;
+  const_target_range targets() const { return {Targets}; }
 
-  /// Get the set of supported architectures.
-  ArchitectureSet getArchitectures() const { return Architectures; }
+  using const_filtered_target_iterator =
+      llvm::filter_iterator<const_target_iterator,
+                            std::function<bool(const Target &)>>;
+  using const_filtered_target_range =
+      llvm::iterator_range<const_filtered_target_iterator>;
+  const_filtered_target_range targets(ArchitectureSet Archs) const;
 
   /// Set the install name of the library.
   void setInstallName(StringRef InstallName_) { InstallName = InstallName_; }
@@ -244,11 +265,18 @@ class InterfaceFile {
   /// Check if this file was generated during InstallAPI.
   bool isInstallAPI() const { return IsInstallAPI; }
 
-  /// Set the parent umbrella framework.
-  void setParentUmbrella(StringRef Parent) { ParentUmbrella = Parent; }
+  /// Set the parent umbrella frameworks.
+  /// \param Target_ The target applicable to Parent
+  /// \param Parent  The name of Parent
+  void addParentUmbrella(const Target &Target_, StringRef Parent);
+  const std::vector<std::pair<Target, std::string>> &umbrellas() const {
+    return ParentUmbrellas;
+  }
 
   /// Get the parent umbrella framework.
-  StringRef getParentUmbrella() const { return ParentUmbrella; }
+  const std::vector<std::pair<Target, std::string>> getParentUmbrellas() const {
+    return ParentUmbrellas;
+  }
 
   /// Add an allowable client.
   ///
@@ -257,9 +285,9 @@ class InterfaceFile {
   /// that is being generated needs to match one of the allowable clients or the
   /// linker refuses to link this library.
   ///
-  /// \param Name The name of the client that is allowed to link this library.
-  /// \param Architectures The set of architecture for which this applies.
-  void addAllowableClient(StringRef Name, ArchitectureSet Architectures);
+  /// \param InstallName The name of the client that is allowed to link this library.
+  /// \param Target The target triple for which this applies.
+  void addAllowableClient(StringRef InstallName, const Target &Target);
 
   /// Get the list of allowable clients.
   ///
@@ -271,9 +299,8 @@ class InterfaceFile {
   /// Add a re-exported library.
   ///
   /// \param InstallName The name of the library to re-export.
-  /// \param Architectures The set of architecture for which this applies.
-  void addReexportedLibrary(StringRef InstallName,
-                            ArchitectureSet Architectures);
+  /// \param Target The target triple for which this applies.
+  void addReexportedLibrary(StringRef InstallName, const Target &Target);
 
   /// Get the list of re-exported libraries.
   ///
@@ -282,27 +309,27 @@ class InterfaceFile {
     return ReexportedLibraries;
   }
 
-  /// Add an architecture/UUID pair.
+  /// Add an Target/UUID pair.
   ///
-  /// \param Arch The architecture for which this applies.
+  /// \param Target The target triple for which this applies.
   /// \param UUID The UUID of the library for the specified architecture.
-  void addUUID(Architecture Arch, StringRef UUID);
+  void addUUID(const Target &Target, StringRef UUID);
 
-  /// Add an architecture/UUID pair.
+  /// Add an Target/UUID pair.
   ///
-  /// \param Arch The architecture for which this applies.
+  /// \param Target The target triple for which this applies.
   /// \param UUID The UUID of the library for the specified architecture.
-  void addUUID(Architecture Arch, uint8_t UUID[16]);
+  void addUUID(const Target &Target, uint8_t UUID[16]);
 
-  /// Get the list of architecture/UUID pairs.
+  /// Get the list of Target/UUID pairs.
   ///
-  /// \return Returns a list of architecture/UUID pairs.
-  const std::vector<std::pair<Architecture, std::string>> &uuids() const {
+  /// \return Returns a list of Target/UUID pairs.
+  const std::vector<std::pair<Target, std::string>> &uuids() const {
     return UUIDs;
   }
 
   /// Add a symbol to the symbols list or extend an existing one.
-  void addSymbol(SymbolKind Kind, StringRef Name, ArchitectureSet Architectures,
+  void addSymbol(SymbolKind Kind, StringRef Name, const TargetList &Targets,
                  SymbolFlags Flags = SymbolFlags::None);
 
   using SymbolMapType = DenseMap<SymbolsMapKey, Symbol *>;
@@ -362,10 +389,9 @@ class InterfaceFile {
     return StringRef(reinterpret_cast<const char *>(Ptr), String.size());
   }
 
+  TargetList Targets;
   std::string Path;
   FileType FileKind;
-  PlatformKind Platform;
-  ArchitectureSet Architectures;
   std::string InstallName;
   PackedVersion CurrentVersion;
   PackedVersion CompatibilityVersion;
@@ -374,10 +400,10 @@ class InterfaceFile {
   bool IsAppExtensionSafe{false};
   bool IsInstallAPI{false};
   ObjCConstraintType ObjcConstraint = ObjCConstraintType::None;
-  std::string ParentUmbrella;
+  std::vector<std::pair<Target, std::string>> ParentUmbrellas;
   std::vector<InterfaceFileRef> AllowableClients;
   std::vector<InterfaceFileRef> ReexportedLibraries;
-  std::vector<std::pair<Architecture, std::string>> UUIDs;
+  std::vector<std::pair<Target, std::string>> UUIDs;
   SymbolMapType Symbols;
 };
 
diff --git a/llvm/include/llvm/TextAPI/MachO/Platform.h b/llvm/include/llvm/TextAPI/MachO/Platform.h
new file mode 100644
index 0000000000000..a22aae9b7dce7
--- /dev/null
+++ b/llvm/include/llvm/TextAPI/MachO/Platform.h
@@ -0,0 +1,45 @@
+//===- llvm/TextAPI/MachO/Platform.h - Platform -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the Platforms supported by Tapi and helpers.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_TEXTAPI_MACHO_PLATFORM_H
+#define LLVM_TEXTAPI_MACHO_PLATFORM_H
+
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/BinaryFormat/MachO.h"
+
+namespace llvm {
+namespace MachO {
+
+/// Defines the list of MachO platforms.
+enum class PlatformKind : unsigned {
+  unknown,
+  macOS = MachO::PLATFORM_MACOS,
+  iOS = MachO::PLATFORM_IOS,
+  tvOS = MachO::PLATFORM_TVOS,
+  watchOS = MachO::PLATFORM_WATCHOS,
+  bridgeOS = MachO::PLATFORM_BRIDGEOS,
+  macCatalyst = MachO::PLATFORM_MACCATALYST,
+  iOSSimulator = MachO::PLATFORM_IOSSIMULATOR,
+  tvOSSimulator = MachO::PLATFORM_TVOSSIMULATOR,
+  watchOSSimulator = MachO::PLATFORM_WATCHOSSIMULATOR
+};
+
+using PlatformSet = SmallSet<PlatformKind, 3>;
+
+PlatformKind mapToPlatformKind(PlatformKind Platform, bool WantSim);
+PlatformKind mapToPlatformKind(const Triple &Target);
+PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets);
+StringRef getPlatformName(PlatformKind Platform);
+
+} // end namespace MachO.
+} // end namespace llvm.
+
+#endif // LLVM_TEXTAPI_MACHO_PLATFORM_H
\ No newline at end of file
diff --git a/llvm/include/llvm/TextAPI/MachO/Symbol.h b/llvm/include/llvm/TextAPI/MachO/Symbol.h
index 3c7ff5e0f4ea3..b6444fbd78ff7 100644
--- a/llvm/include/llvm/TextAPI/MachO/Symbol.h
+++ b/llvm/include/llvm/TextAPI/MachO/Symbol.h
@@ -14,6 +14,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TextAPI/MachO/ArchitectureSet.h"
+#include "llvm/TextAPI/MachO/Target.h"
 
 namespace llvm {
 namespace MachO {
@@ -49,16 +50,18 @@ enum class SymbolKind : uint8_t {
   ObjectiveCInstanceVariable,
 };
 
+using TargetList = SmallVector<Target, 20>;
 class Symbol {
 public:
-  constexpr Symbol(SymbolKind Kind, StringRef Name,
-                   ArchitectureSet Architectures, SymbolFlags Flags)
-      : Name(Name), Architectures(Architectures), Kind(Kind), Flags(Flags) {}
+  Symbol(SymbolKind Kind, StringRef Name, TargetList Targets, SymbolFlags Flags)
+      : Name(Name), Targets(std::move(Targets)), Kind(Kind), Flags(Flags) {}
 
+  void addTarget(Target target) { Targets.emplace_back(target); }
   SymbolKind getKind() const { return Kind; }
   StringRef getName() const { return Name; }
-  ArchitectureSet getArchitectures() const { return Architectures; }
-  void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; }
+  ArchitectureSet getArchitectures() const {
+    return mapToArchitectureSet(Targets);
+  }
   SymbolFlags getFlags() const { return Flags; }
 
   bool isWeakDefined() const {
@@ -78,6 +81,17 @@ class Symbol {
     return (Flags & SymbolFlags::Undefined) == SymbolFlags::Undefined;
   }
 
+  using const_target_iterator = TargetList::const_iterator;
+  using const_target_range = llvm::iterator_range<const_target_iterator>;
+  const_target_range targets() const { return {Targets}; }
+
+  using const_filtered_target_iterator =
+      llvm::filter_iterator<const_target_iterator,
+                            std::function<bool(const Target &)>>;
+  using const_filtered_target_range =
+      llvm::iterator_range<const_filtered_target_iterator>;
+  const_filtered_target_range targets(ArchitectureSet architectures) const;
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump(raw_ostream &OS) const;
   void dump() const { dump(llvm::errs()); }
@@ -85,7 +99,7 @@ class Symbol {
 
 private:
   StringRef Name;
-  ArchitectureSet Architectures;
+  TargetList Targets;
   SymbolKind Kind;
   SymbolFlags Flags;
 };
diff --git a/llvm/include/llvm/TextAPI/MachO/Target.h b/llvm/include/llvm/TextAPI/MachO/Target.h
new file mode 100644
index 0000000000000..74e900d812f52
--- /dev/null
+++ b/llvm/include/llvm/TextAPI/MachO/Target.h
@@ -0,0 +1,66 @@
+//===- llvm/TextAPI/Target.h - TAPI Target ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TEXTAPI_MACHO_TARGET_H
+#define LLVM_TEXTAPI_MACHO_TARGET_H
+
+#include "llvm/ADT/Triple.h"
+#include "llvm/Support/Error.h"
+#include "llvm/TextAPI/MachO/Architecture.h"
+#include "llvm/TextAPI/MachO/ArchitectureSet.h"
+#include "llvm/TextAPI/MachO/Platform.h"
+
+namespace llvm {
+namespace MachO {
+
+// This is similar to a llvm Triple, but the triple doesn't have all the
+// information we need. For example there is no enum value for x86_64h. The
+// only way to get that information is to parse the triple string.
+class Target {
+public:
+  Target() = default;
+  Target(Architecture Arch, PlatformKind Platform)
+      : Arch(Arch), Platform(Platform) {}
+  explicit Target(const llvm::Triple &Triple)
+      : Arch(mapToArchitecture(Triple)), Platform(mapToPlatformKind(Triple)) {}
+
+  operator std::string() const;
+
+  Architecture Arch;
+  PlatformKind Platform;
+};
+
+inline bool operator==(const Target &LHS, const Target &RHS) {
+  return std::tie(LHS.Arch, LHS.Platform) == std::tie(RHS.Arch, RHS.Platform);
+}
+
+inline bool operator!=(const Target &LHS, const Target &RHS) {
+  return std::tie(LHS.Arch, LHS.Platform) != std::tie(RHS.Arch, RHS.Platform);
+}
+
+inline bool operator<(const Target &LHS, const Target &RHS) {
+  return std::tie(LHS.Arch, LHS.Platform) < std::tie(RHS.Arch, RHS.Platform);
+}
+
+inline bool operator==(const Target &LHS, const Architecture &RHS) {
+  return LHS.Arch == RHS;
+}
+
+inline bool operator!=(const Target &LHS, const Architecture &RHS) {
+  return LHS.Arch != RHS;
+}
+
+PlatformSet mapToPlatformSet(ArrayRef<Target> Targets);
+ArchitectureSet mapToArchitectureSet(ArrayRef<Target> Targets);
+
+raw_ostream &operator<<(raw_ostream &OS, const Target &Target);
+
+} // namespace MachO
+} // namespace llvm
+
+#endif // LLVM_TEXTAPI_MACHO_TARGET_H
diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index 84aab8f6fd04c..cf72d44867343 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -96,8 +96,12 @@
 #ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 #define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H
 
-#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/MapVector.h"
+#include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/CallSite.h"
 #include "llvm/IR/PassManager.h"
 
@@ -518,6 +522,31 @@ class SubsumingPositionIterator {
   iterator end() { return IRPositions.end(); }
 };
 
+/// Wrapper for FunctoinAnalysisManager.
+struct AnalysisGetter {
+  template <typename Analysis>
+  typename Analysis::Result *getAnalysis(const Function &F) {
+    if (!MAM || !F.getParent())
+      return nullptr;
+    auto &FAM = MAM->getResult<FunctionAnalysisManagerModuleProxy>(
+                       const_cast<Module &>(*F.getParent()))
+                    .getManager();
+    return &FAM.getResult<Analysis>(const_cast<Function &>(F));
+  }
+
+  template <typename Analysis>
+  typename Analysis::Result *getAnalysis(const Module &M) {
+    if (!MAM)
+      return nullptr;
+    return &MAM->getResult<Analysis>(const_cast<Module &>(M));
+  }
+  AnalysisGetter(ModuleAnalysisManager &MAM) : MAM(&MAM) {}
+  AnalysisGetter() {}
+
+private:
+  ModuleAnalysisManager *MAM = nullptr;
+};
+
 /// Data structure to hold cached (LLVM-IR) information.
 ///
 /// All attributes are given an InformationCache object at creation time to
@@ -531,7 +560,20 @@ class SubsumingPositionIterator {
 /// reusable, it is advised to inherit from the InformationCache and cast the
 /// instance down in the abstract attributes.
 struct InformationCache {
-  InformationCache(const DataLayout &DL) : DL(DL) {}
+  InformationCache(const Module &M, AnalysisGetter &AG)
+      : DL(M.getDataLayout()), AG(AG) {
+
+    CallGraph *CG = AG.getAnalysis<CallGraphAnalysis>(M);
+    if (!CG)
+      return;
+
+    DenseMap<const Function *, unsigned> SccSize;
+    for (scc_iterator<CallGraph *> I = scc_begin(CG); !I.isAtEnd(); ++I) {
+      for (CallGraphNode *Node : *I)
+        SccSize[Node->getFunction()] = I->size();
+    }
+    SccSizeOpt = std::move(SccSize);
+  }
 
   /// A map type from opcodes to instructions with this opcode.
   using OpcodeInstMapTy = DenseMap<unsigned, SmallVector<Instruction *, 32>>;
@@ -550,6 +592,26 @@ struct InformationCache {
     return FuncRWInstsMap[&F];
   }
 
+  /// Return TargetLibraryInfo for function \p F.
+  TargetLibraryInfo *getTargetLibraryInfoForFunction(const Function &F) {
+    return AG.getAnalysis<TargetLibraryAnalysis>(F);
+  }
+
+  /// Return AliasAnalysis Result for function \p F.
+  AAResults *getAAResultsForFunction(const Function &F) {
+    return AG.getAnalysis<AAManager>(F);
+  }
+
+  /// Return SCC size on call graph for function \p F.
+  unsigned getSccSize(const Function &F) {
+    if (!SccSizeOpt.hasValue())
+      return 0;
+    return (SccSizeOpt.getValue())[&F];
+  }
+
+  /// Return datalayout used in the module.
+  const DataLayout &getDL() { return DL; }
+
 private:
   /// A map type from functions to opcode to instruction maps.
   using FuncInstOpcodeMapTy = DenseMap<const Function *, OpcodeInstMapTy>;
@@ -567,6 +629,12 @@ struct InformationCache {
   /// The datalayout used in the module.
   const DataLayout &DL;
 
+  /// Getters for analysis.
+  AnalysisGetter &AG;
+
+  /// Cache result for scc size in the call graph
+  Optional<DenseMap<const Function *, unsigned>> SccSizeOpt;
+
   /// Give the Attributor access to the members so
   /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them.
   friend struct Attributor;
@@ -697,13 +765,20 @@ struct Attributor {
   /// various places.
   void identifyDefaultAbstractAttributes(Function &F);
 
+  /// Initialize the information cache for queries regarding function \p F.
+  ///
+  /// This method needs to be called for all function that might be looked at
+  /// through the information cache interface *prior* to looking at them.
+  void initializeInformationCache(Function &F);
+
   /// Mark the internal function \p F as live.
   ///
   /// This will trigger the identification and initialization of attributes for
   /// \p F.
   void markLiveInternalFunction(const Function &F) {
     assert(F.hasInternalLinkage() &&
-            "Only internal linkage is assumed dead initially.");
+           "Only internal linkage is assumed dead initially.");
+
     identifyDefaultAbstractAttributes(const_cast<Function &>(F));
   }
 
@@ -1725,6 +1800,30 @@ struct AAValueSimplify : public StateWrapper<BooleanState, AbstractAttribute>,
   static const char ID;
 };
 
+struct AAHeapToStack : public StateWrapper<BooleanState, AbstractAttribute>,
+                       public IRPosition {
+  AAHeapToStack(const IRPosition &IRP) : IRPosition(IRP) {}
+
+  /// Returns true if HeapToStack conversion is assumed to be possible.
+  bool isAssumedHeapToStack() const { return getAssumed(); }
+
+  /// Returns true if HeapToStack conversion is known to be possible.
+  bool isKnownHeapToStack() const { return getKnown(); }
+
+  /// Return an IR position, see struct IRPosition.
+  ///
+  ///{
+  IRPosition &getIRPosition() { return *this; }
+  const IRPosition &getIRPosition() const { return *this; }
+  ///}
+
+  /// Create an abstract attribute view for the position \p IRP.
+  static AAHeapToStack &createForPosition(const IRPosition &IRP, Attributor &A);
+
+  /// Unique ID (due to the unique address)
+  static const char ID;
+};
+
 } // end namespace llvm
 
 #endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H
diff --git a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
index 73668844590d7..93395759d603e 100644
--- a/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
+++ b/llvm/include/llvm/Transforms/IPO/HotColdSplitting.h
@@ -17,6 +17,42 @@
 namespace llvm {
 
 class Module;
+class ProfileSummaryInfo;
+class BlockFrequencyInfo;
+class TargetTransformInfo;
+class OptimizationRemarkEmitter;
+class AssumptionCache;
+class DominatorTree;
+
+/// A sequence of basic blocks.
+///
+/// A 0-sized SmallVector is slightly cheaper to move than a std::vector.
+using BlockSequence = SmallVector<BasicBlock *, 0>;
+
+class HotColdSplitting {
+public:
+  HotColdSplitting(ProfileSummaryInfo *ProfSI,
+                   function_ref<BlockFrequencyInfo *(Function &)> GBFI,
+                   function_ref<TargetTransformInfo &(Function &)> GTTI,
+                   std::function<OptimizationRemarkEmitter &(Function &)> *GORE,
+                   function_ref<AssumptionCache *(Function &)> LAC)
+      : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE), LookupAC(LAC) {}
+  bool run(Module &M);
+
+private:
+  bool isFunctionCold(const Function &F) const;
+  bool shouldOutlineFrom(const Function &F) const;
+  bool outlineColdRegions(Function &F, bool HasProfileSummary);
+  Function *extractColdRegion(const BlockSequence &Region, DominatorTree &DT,
+                              BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
+                              OptimizationRemarkEmitter &ORE,
+                              AssumptionCache *AC, unsigned Count);
+  ProfileSummaryInfo *PSI;
+  function_ref<BlockFrequencyInfo *(Function &)> GetBFI;
+  function_ref<TargetTransformInfo &(Function &)> GetTTI;
+  std::function<OptimizationRemarkEmitter &(Function &)> *GetORE;
+  function_ref<AssumptionCache *(Function &)> LookupAC;
+};
 
 /// Pass to outline cold regions.
 class HotColdSplittingPass : public PassInfoMixin<HotColdSplittingPass> {
diff --git a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
index b30d098acabe0..22435e4ed1e5b 100644
--- a/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
+++ b/llvm/include/llvm/Transforms/IPO/WholeProgramDevirt.h
@@ -251,7 +251,7 @@ void runWholeProgramDevirtOnIndex(
 /// devirt target names for any locals that were exported.
 void updateIndexWPDForExports(
     ModuleSummaryIndex &Summary,
-    StringMap<FunctionImporter::ExportSetTy> &ExportLists,
+    function_ref<bool(StringRef, GlobalValue::GUID)> isExported,
     std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap);
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h
index b6055639e8a8f..74cbf84b64b20 100644
--- a/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h
+++ b/llvm/include/llvm/Transforms/Scalar/CallSiteSplitting.h
@@ -9,13 +9,8 @@
 #ifndef LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H
 #define LLVM_TRANSFORMS_SCALAR_CALLSITESPLITTING__H
 
-#include "llvm/ADT/SetVector.h"
-#include "llvm/Analysis/AssumptionCache.h"
-#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/Support/Compiler.h"
-#include <vector>
 
 namespace llvm {
 
diff --git a/llvm/include/llvm/Transforms/Scalar/Float2Int.h b/llvm/include/llvm/Transforms/Scalar/Float2Int.h
index 06aeb83225273..f04b98a19d82c 100644
--- a/llvm/include/llvm/Transforms/Scalar/Float2Int.h
+++ b/llvm/include/llvm/Transforms/Scalar/Float2Int.h
@@ -17,6 +17,7 @@
 #include "llvm/ADT/EquivalenceClasses.h"
 #include "llvm/ADT/MapVector.h"
 #include "llvm/IR/ConstantRange.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/PassManager.h"
 
@@ -26,10 +27,11 @@ class Float2IntPass : public PassInfoMixin<Float2IntPass> {
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
   // Glue for old PM.
-  bool runImpl(Function &F);
+  bool runImpl(Function &F, const DominatorTree &DT);
 
 private:
-  void findRoots(Function &F, SmallPtrSet<Instruction *, 8> &Roots);
+  void findRoots(Function &F, const DominatorTree &DT,
+                 SmallPtrSet<Instruction *, 8> &Roots);
   void seen(Instruction *I, ConstantRange R);
   ConstantRange badRange();
   ConstantRange unknownRange();
diff --git a/llvm/include/llvm/Transforms/Scalar/GVNExpression.h b/llvm/include/llvm/Transforms/Scalar/GVNExpression.h
index 3dc4515f85a15..1600d1af32426 100644
--- a/llvm/include/llvm/Transforms/Scalar/GVNExpression.h
+++ b/llvm/include/llvm/Transforms/Scalar/GVNExpression.h
@@ -323,7 +323,7 @@ class CallExpression final : public MemoryExpression {
 class LoadExpression final : public MemoryExpression {
 private:
   LoadInst *Load;
-  unsigned Alignment;
+  MaybeAlign Alignment;
 
 public:
   LoadExpression(unsigned NumOperands, LoadInst *L,
@@ -333,7 +333,8 @@ class LoadExpression final : public MemoryExpression {
   LoadExpression(enum ExpressionType EType, unsigned NumOperands, LoadInst *L,
                  const MemoryAccess *MemoryLeader)
       : MemoryExpression(NumOperands, EType, MemoryLeader), Load(L) {
-    Alignment = L ? L->getAlignment() : 0;
+    if (L)
+      Alignment = MaybeAlign(L->getAlignment());
   }
 
   LoadExpression() = delete;
@@ -348,8 +349,8 @@ class LoadExpression final : public MemoryExpression {
   LoadInst *getLoadInst() const { return Load; }
   void setLoadInst(LoadInst *L) { Load = L; }
 
-  unsigned getAlignment() const { return Alignment; }
-  void setAlignment(unsigned Align) { Alignment = Align; }
+  MaybeAlign getAlignment() const { return Alignment; }
+  void setAlignment(MaybeAlign Align) { Alignment = Align; }
 
   bool equals(const Expression &Other) const override;
   bool exactlyEquals(const Expression &Other) const override {
diff --git a/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h
index 951dc90d82345..afeb1f1da029e 100644
--- a/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h
+++ b/llvm/include/llvm/Transforms/Scalar/LoopUnrollPass.h
@@ -63,6 +63,7 @@ struct LoopUnrollOptions {
   Optional<bool> AllowRuntime;
   Optional<bool> AllowUpperBound;
   Optional<bool> AllowProfileBasedPeeling;
+  Optional<unsigned> FullUnrollMaxCount;
   int OptLevel;
 
   /// If false, use a cost model to determine whether unrolling of a loop is
@@ -117,6 +118,12 @@ struct LoopUnrollOptions {
     AllowProfileBasedPeeling = O;
     return *this;
   }
+
+  // Sets the max full unroll count.
+  LoopUnrollOptions &setFullUnrollMaxCount(unsigned O) {
+    FullUnrollMaxCount = O;
+    return *this;
+  }
 };
 
 /// Loop unroll pass that will support both full and partial unrolling.
diff --git a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
index 726723ae23b65..3d15b2a7bf2ae 100644
--- a/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/BuildLibCalls.h
@@ -50,6 +50,11 @@ namespace llvm {
   Value *emitStrLen(Value *Ptr, IRBuilder<> &B, const DataLayout &DL,
                     const TargetLibraryInfo *TLI);
 
+  /// Emit a call to the strdup function to the builder, for the specified
+  /// pointer. Ptr is required to be some pointer type, and the return value has
+  /// 'i8*' type.
+  Value *emitStrDup(Value *Ptr, IRBuilder<> &B, const TargetLibraryInfo *TLI);
+
   /// Emit a call to the strnlen function to the builder, for the specified
   /// pointer. Ptr is required to be some pointer type, MaxLen must be of size_t
   /// type, and the return value has 'intptr_t' type.
diff --git a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
index 9d79ee1633f68..8ff5172339326 100644
--- a/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
+++ b/llvm/include/llvm/Transforms/Utils/CodeExtractor.h
@@ -110,7 +110,10 @@ class Value;
     ///
     /// Based on the blocks used when constructing the code extractor,
     /// determine whether it is eligible for extraction.
-    bool isEligible() const { return !Blocks.empty(); }
+    /// 
+    /// Checks that varargs handling (with vastart and vaend) is only done in
+    /// the outlined blocks.
+    bool isEligible() const;
 
     /// Compute the set of input values and output values for the code.
     ///
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 5c5e0aef34cdf..9fcb2f64d79b8 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -412,8 +412,7 @@ void removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU = nullptr);
 /// Remove all blocks that can not be reached from the function's entry.
 ///
 /// Returns true if any basic block was removed.
-bool removeUnreachableBlocks(Function &F, LazyValueInfo *LVI = nullptr,
-                             DomTreeUpdater *DTU = nullptr,
+bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr,
                              MemorySSAUpdater *MSSAU = nullptr);
 
 /// Combine the metadata of two instructions so that K can replace J. Some
diff --git a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
index 6966a75da0d91..88c2ef787ad81 100644
--- a/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
+++ b/llvm/include/llvm/Transforms/Utils/SimplifyLibCalls.h
@@ -160,6 +160,7 @@ class LibCallSimplifier {
   Value *optimizeStrRChr(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStrCmp(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStrNCmp(CallInst *CI, IRBuilder<> &B);
+  Value *optimizeStrNDup(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStrCpy(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStpCpy(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStrNCpy(CallInst *CI, IRBuilder<> &B);
@@ -170,15 +171,17 @@ class LibCallSimplifier {
   Value *optimizeStrCSpn(CallInst *CI, IRBuilder<> &B);
   Value *optimizeStrStr(CallInst *CI, IRBuilder<> &B);
   Value *optimizeMemChr(CallInst *CI, IRBuilder<> &B);
+  Value *optimizeMemRChr(CallInst *CI, IRBuilder<> &B);
   Value *optimizeMemCmp(CallInst *CI, IRBuilder<> &B);
   Value *optimizeBCmp(CallInst *CI, IRBuilder<> &B);
   Value *optimizeMemCmpBCmpCommon(CallInst *CI, IRBuilder<> &B);
   Value *optimizeMemPCpy(CallInst *CI, IRBuilder<> &B);
-  Value *optimizeMemCpy(CallInst *CI, IRBuilder<> &B, bool isIntrinsic = false);
-  Value *optimizeMemMove(CallInst *CI, IRBuilder<> &B, bool isIntrinsic = false);
-  Value *optimizeMemSet(CallInst *CI, IRBuilder<> &B, bool isIntrinsic = false);
+  Value *optimizeMemCpy(CallInst *CI, IRBuilder<> &B);
+  Value *optimizeMemMove(CallInst *CI, IRBuilder<> &B);
+  Value *optimizeMemSet(CallInst *CI, IRBuilder<> &B);
   Value *optimizeRealloc(CallInst *CI, IRBuilder<> &B);
   Value *optimizeWcslen(CallInst *CI, IRBuilder<> &B);
+  Value *optimizeBCopy(CallInst *CI, IRBuilder<> &B);
   // Wrapper for all String/Memory Library Call Optimizations
   Value *optimizeStringMemoryLibCall(CallInst *CI, IRBuilder<> &B);
 
diff --git a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
index f3651323c2545..02b81b4b7ee26 100644
--- a/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
+++ b/llvm/include/llvm/Transforms/Utils/UnrollLoop.h
@@ -114,8 +114,8 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI,
                         DominatorTree &DT, LoopInfo *LI, ScalarEvolution &SE,
                         const SmallPtrSetImpl<const Value *> &EphValues,
                         OptimizationRemarkEmitter *ORE, unsigned &TripCount,
-                        unsigned MaxTripCount, unsigned &TripMultiple,
-                        unsigned LoopSize,
+                        unsigned MaxTripCount, bool MaxOrZero,
+                        unsigned &TripMultiple, unsigned LoopSize,
                         TargetTransformInfo::UnrollingPreferences &UP,
                         bool &UseUpperBound);
 
@@ -133,7 +133,8 @@ TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
     Optional<unsigned> UserThreshold, Optional<unsigned> UserCount,
     Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
     Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling,
-    Optional<bool> UserAllowProfileBasedPeeling);
+    Optional<bool> UserAllowProfileBasedPeeling,
+    Optional<unsigned> UserFullUnrollMaxCount);
 
 unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
                              bool &NotDuplicatable, bool &Convergent,
diff --git a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
index ac6afb761d4d5..32ccc8a463805 100644
--- a/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
+++ b/llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h
@@ -24,7 +24,6 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/IR/PassManager.h"
-#include "llvm/IR/ValueHandle.h"
 
 namespace llvm {
 
@@ -60,8 +59,8 @@ extern cl::opt<bool> RunSLPVectorization;
 struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
   using StoreList = SmallVector<StoreInst *, 8>;
   using StoreListMap = MapVector<Value *, StoreList>;
-  using WeakTrackingVHList = SmallVector<WeakTrackingVH, 8>;
-  using WeakTrackingVHListMap = MapVector<Value *, WeakTrackingVHList>;
+  using GEPList = SmallVector<GetElementPtrInst *, 8>;
+  using GEPListMap = MapVector<Value *, GEPList>;
 
   ScalarEvolution *SE = nullptr;
   TargetTransformInfo *TTI = nullptr;
@@ -131,7 +130,7 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
 
   /// Tries to vectorize constructs started from CmpInst, InsertValueInst or
   /// InsertElementInst instructions.
-  bool vectorizeSimpleInstructions(SmallVectorImpl<WeakVH> &Instructions,
+  bool vectorizeSimpleInstructions(SmallVectorImpl<Instruction *> &Instructions,
                                    BasicBlock *BB, slpvectorizer::BoUpSLP &R);
 
   /// Scan the basic block and look for patterns that are likely to start
@@ -147,7 +146,7 @@ struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
   StoreListMap Stores;
 
   /// The getelementptr instructions in a basic block organized by base pointer.
-  WeakTrackingVHListMap GEPs;
+  GEPListMap GEPs;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Analysis/AssumptionCache.cpp b/llvm/lib/Analysis/AssumptionCache.cpp
index 7d6429a0fec1c..129944743c5ef 100644
--- a/llvm/lib/Analysis/AssumptionCache.cpp
+++ b/llvm/lib/Analysis/AssumptionCache.cpp
@@ -130,7 +130,10 @@ void AssumptionCache::unregisterAssumption(CallInst *CI) {
     if (AVI != AffectedValues.end())
       AffectedValues.erase(AVI);
   }
-  remove_if(AssumeHandles, [CI](WeakTrackingVH &VH) { return CI == VH; });
+
+  AssumeHandles.erase(
+      remove_if(AssumeHandles, [CI](WeakTrackingVH &VH) { return CI == VH; }),
+      AssumeHandles.end());
 }
 
 void AssumptionCache::AffectedValueCallbackVH::deleted() {
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 1dc9544120e78..5ecc8fe6f7359 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -22,9 +22,11 @@ add_llvm_library(LLVMAnalysis
   CostModel.cpp
   CodeMetrics.cpp
   ConstantFolding.cpp
+  DDG.cpp
   Delinearization.cpp
   DemandedBits.cpp
   DependenceAnalysis.cpp
+  DependenceGraphBuilder.cpp
   DivergenceAnalysis.cpp
   DomPrinter.cpp
   DomTreeUpdater.cpp
@@ -95,6 +97,7 @@ add_llvm_library(LLVMAnalysis
   ValueLatticeUtils.cpp
   ValueTracking.cpp
   VectorUtils.cpp
+  VFABIDemangling.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Analysis
diff --git a/llvm/lib/Analysis/ConstantFolding.cpp b/llvm/lib/Analysis/ConstantFolding.cpp
index 7b3d81577802d..8dbcf7034fdaf 100644
--- a/llvm/lib/Analysis/ConstantFolding.cpp
+++ b/llvm/lib/Analysis/ConstantFolding.cpp
@@ -93,6 +93,9 @@ static Constant *foldConstVectorToAPInt(APInt &Result, Type *DestTy,
 /// This always returns a non-null constant, but it may be a
 /// ConstantExpr if unfoldable.
 Constant *FoldBitCast(Constant *C, Type *DestTy, const DataLayout &DL) {
+  assert(CastInst::castIsValid(Instruction::BitCast, C, DestTy) &&
+         "Invalid constantexpr bitcast!");
+
   // Catch the obvious splat cases.
   if (C->isNullValue() && !DestTy->isX86_MMXTy())
     return Constant::getNullValue(DestTy);
@@ -521,8 +524,23 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy,
       return nullptr;
 
     C = FoldBitCast(C, MapTy->getPointerTo(AS), DL);
-    if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL))
-      return FoldBitCast(Res, LoadTy, DL);
+    if (Constant *Res = FoldReinterpretLoadFromConstPtr(C, MapTy, DL)) {
+      if (Res->isNullValue() && !LoadTy->isX86_MMXTy())
+        // Materializing a zero can be done trivially without a bitcast
+        return Constant::getNullValue(LoadTy);
+      Type *CastTy = LoadTy->isPtrOrPtrVectorTy() ? DL.getIntPtrType(LoadTy) : LoadTy;
+      Res = FoldBitCast(Res, CastTy, DL);
+      if (LoadTy->isPtrOrPtrVectorTy()) {
+        // For vector of pointer, we needed to first convert to a vector of integer, then do vector inttoptr
+        if (Res->isNullValue() && !LoadTy->isX86_MMXTy())
+          return Constant::getNullValue(LoadTy);
+        if (DL.isNonIntegralPointerType(LoadTy->getScalarType()))
+          // Be careful not to replace a load of an addrspace value with an inttoptr here
+          return nullptr;
+        Res = ConstantExpr::getCast(Instruction::IntToPtr, Res, LoadTy);
+      }
+      return Res;
+    }
     return nullptr;
   }
 
@@ -1038,7 +1056,7 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode,
     return ConstantExpr::getExtractElement(Ops[0], Ops[1]);
   case Instruction::ExtractValue:
     return ConstantExpr::getExtractValue(
-        Ops[0], dyn_cast<ExtractValueInst>(InstOrCE)->getIndices());
+        Ops[0], cast<ExtractValueInst>(InstOrCE)->getIndices());
   case Instruction::InsertElement:
     return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]);
   case Instruction::ShuffleVector:
@@ -1490,6 +1508,7 @@ bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
            Name == "fmod" || Name == "fmodf";
   case 'l':
     return Name == "log" || Name == "logf" ||
+           Name == "log2" || Name == "log2f" ||
            Name == "log10" || Name == "log10f";
   case 'n':
     return Name == "nearbyint" || Name == "nearbyintf";
@@ -1872,6 +1891,14 @@ static Constant *ConstantFoldScalarCall1(StringRef Name,
       if (V > 0.0 && TLI->has(Func))
         return ConstantFoldFP(log, V, Ty);
       break;
+    case LibFunc_log2:
+    case LibFunc_log2f:
+    case LibFunc_log2_finite:
+    case LibFunc_log2f_finite:
+      if (V > 0.0 && TLI->has(Func))
+        // TODO: What about hosts that lack a C99 library?
+        return ConstantFoldFP(Log2, V, Ty);
+      break;
     case LibFunc_log10:
     case LibFunc_log10f:
     case LibFunc_log10_finite:
diff --git a/llvm/lib/Analysis/DDG.cpp b/llvm/lib/Analysis/DDG.cpp
new file mode 100644
index 0000000000000..b5c3c761ad98f
--- /dev/null
+++ b/llvm/lib/Analysis/DDG.cpp
@@ -0,0 +1,203 @@
+//===- DDG.cpp - Data Dependence Graph -------------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The implementation for the data dependence graph.
+//===----------------------------------------------------------------------===//
+#include "llvm/Analysis/DDG.h"
+#include "llvm/Analysis/LoopInfo.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "ddg"
+
+template class llvm::DGEdge<DDGNode, DDGEdge>;
+template class llvm::DGNode<DDGNode, DDGEdge>;
+template class llvm::DirectedGraph<DDGNode, DDGEdge>;
+
+//===--------------------------------------------------------------------===//
+// DDGNode implementation
+//===--------------------------------------------------------------------===//
+DDGNode::~DDGNode() {}
+
+bool DDGNode::collectInstructions(
+    llvm::function_ref<bool(Instruction *)> const &Pred,
+    InstructionListType &IList) const {
+  assert(IList.empty() && "Expected the IList to be empty on entry.");
+  if (isa<SimpleDDGNode>(this)) {
+    for (auto *I : cast<const SimpleDDGNode>(this)->getInstructions())
+      if (Pred(I))
+        IList.push_back(I);
+  } else
+    llvm_unreachable("unimplemented type of node");
+  return !IList.empty();
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const DDGNode::NodeKind K) {
+  const char *Out;
+  switch (K) {
+  case DDGNode::NodeKind::SingleInstruction:
+    Out = "single-instruction";
+    break;
+  case DDGNode::NodeKind::MultiInstruction:
+    Out = "multi-instruction";
+    break;
+  case DDGNode::NodeKind::Root:
+    Out = "root";
+    break;
+  case DDGNode::NodeKind::Unknown:
+    Out = "??";
+    break;
+  }
+  OS << Out;
+  return OS;
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const DDGNode &N) {
+  OS << "Node Address:" << &N << ":" << N.getKind() << "\n";
+  if (isa<SimpleDDGNode>(N)) {
+    OS << " Instructions:\n";
+    for (auto *I : cast<const SimpleDDGNode>(N).getInstructions())
+      OS.indent(2) << *I << "\n";
+  } else if (!isa<RootDDGNode>(N))
+    llvm_unreachable("unimplemented type of node");
+
+  OS << (N.getEdges().empty() ? " Edges:none!\n" : " Edges:\n");
+  for (auto &E : N.getEdges())
+    OS.indent(2) << *E;
+  return OS;
+}
+
+//===--------------------------------------------------------------------===//
+// SimpleDDGNode implementation
+//===--------------------------------------------------------------------===//
+
+SimpleDDGNode::SimpleDDGNode(Instruction &I)
+  : DDGNode(NodeKind::SingleInstruction), InstList() {
+  assert(InstList.empty() && "Expected empty list.");
+  InstList.push_back(&I);
+}
+
+SimpleDDGNode::SimpleDDGNode(const SimpleDDGNode &N)
+    : DDGNode(N), InstList(N.InstList) {
+  assert(((getKind() == NodeKind::SingleInstruction && InstList.size() == 1) ||
+          (getKind() == NodeKind::MultiInstruction && InstList.size() > 1)) &&
+         "constructing from invalid simple node.");
+}
+
+SimpleDDGNode::SimpleDDGNode(SimpleDDGNode &&N)
+    : DDGNode(std::move(N)), InstList(std::move(N.InstList)) {
+  assert(((getKind() == NodeKind::SingleInstruction && InstList.size() == 1) ||
+          (getKind() == NodeKind::MultiInstruction && InstList.size() > 1)) &&
+         "constructing from invalid simple node.");
+}
+
+SimpleDDGNode::~SimpleDDGNode() { InstList.clear(); }
+
+//===--------------------------------------------------------------------===//
+// DDGEdge implementation
+//===--------------------------------------------------------------------===//
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const DDGEdge::EdgeKind K) {
+  const char *Out;
+  switch (K) {
+  case DDGEdge::EdgeKind::RegisterDefUse:
+    Out = "def-use";
+    break;
+  case DDGEdge::EdgeKind::MemoryDependence:
+    Out = "memory";
+    break;
+  case DDGEdge::EdgeKind::Rooted:
+    Out = "rooted";
+    break;
+  case DDGEdge::EdgeKind::Unknown:
+    Out = "??";
+    break;
+  }
+  OS << Out;
+  return OS;
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const DDGEdge &E) {
+  OS << "[" << E.getKind() << "] to " << &E.getTargetNode() << "\n";
+  return OS;
+}
+
+//===--------------------------------------------------------------------===//
+// DataDependenceGraph implementation
+//===--------------------------------------------------------------------===//
+using BasicBlockListType = SmallVector<BasicBlock *, 8>;
+
+DataDependenceGraph::DataDependenceGraph(Function &F, DependenceInfo &D)
+    : DependenceGraphInfo(F.getName().str(), D) {
+  BasicBlockListType BBList;
+  for (auto &BB : F.getBasicBlockList())
+    BBList.push_back(&BB);
+  DDGBuilder(*this, D, BBList).populate();
+}
+
+DataDependenceGraph::DataDependenceGraph(const Loop &L, DependenceInfo &D)
+    : DependenceGraphInfo(Twine(L.getHeader()->getParent()->getName() + "." +
+                                L.getHeader()->getName())
+                              .str(),
+                          D) {
+  BasicBlockListType BBList;
+  for (BasicBlock *BB : L.blocks())
+    BBList.push_back(BB);
+  DDGBuilder(*this, D, BBList).populate();
+}
+
+DataDependenceGraph::~DataDependenceGraph() {
+  for (auto *N : Nodes) {
+    for (auto *E : *N)
+      delete E;
+    delete N;
+  }
+}
+
+bool DataDependenceGraph::addNode(DDGNode &N) {
+  if (!DDGBase::addNode(N))
+    return false;
+
+  // In general, if the root node is already created and linked, it is not safe
+  // to add new nodes since they may be unreachable by the root.
+  // TODO: Allow adding Pi-block nodes after root is created. Pi-blocks are an
+  // exception because they represent components that are already reachable by
+  // root.
+  assert(!Root && "Root node is already added. No more nodes can be added.");
+  if (isa<RootDDGNode>(N))
+    Root = &N;
+
+  return true;
+}
+
+raw_ostream &llvm::operator<<(raw_ostream &OS, const DataDependenceGraph &G) {
+  for (auto *Node : G)
+    OS << *Node << "\n";
+  return OS;
+}
+
+//===--------------------------------------------------------------------===//
+// DDG Analysis Passes
+//===--------------------------------------------------------------------===//
+
+/// DDG as a loop pass.
+DDGAnalysis::Result DDGAnalysis::run(Loop &L, LoopAnalysisManager &AM,
+                                     LoopStandardAnalysisResults &AR) {
+  Function *F = L.getHeader()->getParent();
+  DependenceInfo DI(F, &AR.AA, &AR.SE, &AR.LI);
+  return std::make_unique<DataDependenceGraph>(L, DI);
+}
+AnalysisKey DDGAnalysis::Key;
+
+PreservedAnalyses DDGAnalysisPrinterPass::run(Loop &L, LoopAnalysisManager &AM,
+                                              LoopStandardAnalysisResults &AR,
+                                              LPMUpdater &U) {
+  OS << "'DDG' for loop '" << L.getHeader()->getName() << "':\n";
+  OS << *AM.getResult<DDGAnalysis>(L, AR);
+  return PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Analysis/DependenceGraphBuilder.cpp b/llvm/lib/Analysis/DependenceGraphBuilder.cpp
new file mode 100644
index 0000000000000..ed1d8351b2f03
--- /dev/null
+++ b/llvm/lib/Analysis/DependenceGraphBuilder.cpp
@@ -0,0 +1,228 @@
+//===- DependenceGraphBuilder.cpp ------------------------------------------==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This file implements common steps of the build algorithm for construction
+// of dependence graphs such as DDG and PDG.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/DependenceGraphBuilder.h"
+#include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DDG.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "dgb"
+
+STATISTIC(TotalGraphs, "Number of dependence graphs created.");
+STATISTIC(TotalDefUseEdges, "Number of def-use edges created.");
+STATISTIC(TotalMemoryEdges, "Number of memory dependence edges created.");
+STATISTIC(TotalFineGrainedNodes, "Number of fine-grained nodes created.");
+STATISTIC(TotalConfusedEdges,
+          "Number of confused memory dependencies between two nodes.");
+STATISTIC(TotalEdgeReversals,
+          "Number of times the source and sink of dependence was reversed to "
+          "expose cycles in the graph.");
+
+using InstructionListType = SmallVector<Instruction *, 2>;
+
+//===--------------------------------------------------------------------===//
+// AbstractDependenceGraphBuilder implementation
+//===--------------------------------------------------------------------===//
+
+template <class G>
+void AbstractDependenceGraphBuilder<G>::createFineGrainedNodes() {
+  ++TotalGraphs;
+  assert(IMap.empty() && "Expected empty instruction map at start");
+  for (BasicBlock *BB : BBList)
+    for (Instruction &I : *BB) {
+      auto &NewNode = createFineGrainedNode(I);
+      IMap.insert(std::make_pair(&I, &NewNode));
+      ++TotalFineGrainedNodes;
+    }
+}
+
+template <class G>
+void AbstractDependenceGraphBuilder<G>::createAndConnectRootNode() {
+  // Create a root node that connects to every connected component of the graph.
+  // This is done to allow graph iterators to visit all the disjoint components
+  // of the graph, in a single walk.
+  //
+  // This algorithm works by going through each node of the graph and for each
+  // node N, do a DFS starting from N. A rooted edge is established between the
+  // root node and N (if N is not yet visited). All the nodes reachable from N
+  // are marked as visited and are skipped in the DFS of subsequent nodes.
+  //
+  // Note: This algorithm tries to limit the number of edges out of the root
+  // node to some extent, but there may be redundant edges created depending on
+  // the iteration order. For example for a graph {A -> B}, an edge from the
+  // root node is added to both nodes if B is visited before A. While it does
+  // not result in minimal number of edges, this approach saves compile-time
+  // while keeping the number of edges in check.
+  auto &RootNode = createRootNode();
+  df_iterator_default_set<const NodeType *, 4> Visited;
+  for (auto *N : Graph) {
+    if (*N == RootNode)
+      continue;
+    for (auto I : depth_first_ext(N, Visited))
+      if (I == N)
+        createRootedEdge(RootNode, *N);
+  }
+}
+
+template <class G> void AbstractDependenceGraphBuilder<G>::createDefUseEdges() {
+  for (NodeType *N : Graph) {
+    InstructionListType SrcIList;
+    N->collectInstructions([](const Instruction *I) { return true; }, SrcIList);
+
+    // Use a set to mark the targets that we link to N, so we don't add
+    // duplicate def-use edges when more than one instruction in a target node
+    // use results of instructions that are contained in N.
+    SmallPtrSet<NodeType *, 4> VisitedTargets;
+
+    for (Instruction *II : SrcIList) {
+      for (User *U : II->users()) {
+        Instruction *UI = dyn_cast<Instruction>(U);
+        if (!UI)
+          continue;
+        NodeType *DstNode = nullptr;
+        if (IMap.find(UI) != IMap.end())
+          DstNode = IMap.find(UI)->second;
+
+        // In the case of loops, the scope of the subgraph is all the
+        // basic blocks (and instructions within them) belonging to the loop. We
+        // simply ignore all the edges coming from (or going into) instructions
+        // or basic blocks outside of this range.
+        if (!DstNode) {
+          LLVM_DEBUG(
+              dbgs()
+              << "skipped def-use edge since the sink" << *UI
+              << " is outside the range of instructions being considered.\n");
+          continue;
+        }
+
+        // Self dependencies are ignored because they are redundant and
+        // uninteresting.
+        if (DstNode == N) {
+          LLVM_DEBUG(dbgs()
+                     << "skipped def-use edge since the sink and the source ("
+                     << N << ") are the same.\n");
+          continue;
+        }
+
+        if (VisitedTargets.insert(DstNode).second) {
+          createDefUseEdge(*N, *DstNode);
+          ++TotalDefUseEdges;
+        }
+      }
+    }
+  }
+}
+
+template <class G>
+void AbstractDependenceGraphBuilder<G>::createMemoryDependencyEdges() {
+  using DGIterator = typename G::iterator;
+  auto isMemoryAccess = [](const Instruction *I) {
+    return I->mayReadOrWriteMemory();
+  };
+  for (DGIterator SrcIt = Graph.begin(), E = Graph.end(); SrcIt != E; ++SrcIt) {
+    InstructionListType SrcIList;
+    (*SrcIt)->collectInstructions(isMemoryAccess, SrcIList);
+    if (SrcIList.empty())
+      continue;
+
+    for (DGIterator DstIt = SrcIt; DstIt != E; ++DstIt) {
+      if (**SrcIt == **DstIt)
+        continue;
+      InstructionListType DstIList;
+      (*DstIt)->collectInstructions(isMemoryAccess, DstIList);
+      if (DstIList.empty())
+        continue;
+      bool ForwardEdgeCreated = false;
+      bool BackwardEdgeCreated = false;
+      for (Instruction *ISrc : SrcIList) {
+        for (Instruction *IDst : DstIList) {
+          auto D = DI.depends(ISrc, IDst, true);
+          if (!D)
+            continue;
+
+          // If we have a dependence with its left-most non-'=' direction
+          // being '>' we need to reverse the direction of the edge, because
+          // the source of the dependence cannot occur after the sink. For
+          // confused dependencies, we will create edges in both directions to
+          // represent the possibility of a cycle.
+
+          auto createConfusedEdges = [&](NodeType &Src, NodeType &Dst) {
+            if (!ForwardEdgeCreated) {
+              createMemoryEdge(Src, Dst);
+              ++TotalMemoryEdges;
+            }
+            if (!BackwardEdgeCreated) {
+              createMemoryEdge(Dst, Src);
+              ++TotalMemoryEdges;
+            }
+            ForwardEdgeCreated = BackwardEdgeCreated = true;
+            ++TotalConfusedEdges;
+          };
+
+          auto createForwardEdge = [&](NodeType &Src, NodeType &Dst) {
+            if (!ForwardEdgeCreated) {
+              createMemoryEdge(Src, Dst);
+              ++TotalMemoryEdges;
+            }
+            ForwardEdgeCreated = true;
+          };
+
+          auto createBackwardEdge = [&](NodeType &Src, NodeType &Dst) {
+            if (!BackwardEdgeCreated) {
+              createMemoryEdge(Dst, Src);
+              ++TotalMemoryEdges;
+            }
+            BackwardEdgeCreated = true;
+          };
+
+          if (D->isConfused())
+            createConfusedEdges(**SrcIt, **DstIt);
+          else if (D->isOrdered() && !D->isLoopIndependent()) {
+            bool ReversedEdge = false;
+            for (unsigned Level = 1; Level <= D->getLevels(); ++Level) {
+              if (D->getDirection(Level) == Dependence::DVEntry::EQ)
+                continue;
+              else if (D->getDirection(Level) == Dependence::DVEntry::GT) {
+                createBackwardEdge(**SrcIt, **DstIt);
+                ReversedEdge = true;
+                ++TotalEdgeReversals;
+                break;
+              } else if (D->getDirection(Level) == Dependence::DVEntry::LT)
+                break;
+              else {
+                createConfusedEdges(**SrcIt, **DstIt);
+                break;
+              }
+            }
+            if (!ReversedEdge)
+              createForwardEdge(**SrcIt, **DstIt);
+          } else
+            createForwardEdge(**SrcIt, **DstIt);
+
+          // Avoid creating duplicate edges.
+          if (ForwardEdgeCreated && BackwardEdgeCreated)
+            break;
+        }
+
+        // If we've created edges in both directions, there is no more
+        // unique edge that we can create between these two nodes, so we
+        // can exit early.
+        if (ForwardEdgeCreated && BackwardEdgeCreated)
+          break;
+      }
+    }
+  }
+}
+
+template class llvm::AbstractDependenceGraphBuilder<DataDependenceGraph>;
+template class llvm::DependenceGraphInfo<DDGNode>;
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp
index ce285f82f720f..6fb600114bc61 100644
--- a/llvm/lib/Analysis/IVDescriptors.cpp
+++ b/llvm/lib/Analysis/IVDescriptors.cpp
@@ -300,7 +300,8 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind,
       ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr);
       if (!ReduxDesc.isRecurrence())
         return false;
-      if (isa<FPMathOperator>(ReduxDesc.getPatternInst()))
+      // FIXME: FMF is allowed on phi, but propagation is not handled correctly.
+      if (isa<FPMathOperator>(ReduxDesc.getPatternInst()) && !IsAPhi)
         FMF &= ReduxDesc.getPatternInst()->getFastMathFlags();
     }
 
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index c6e5c1c1ace93..57dee459fc2cb 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -1453,19 +1453,6 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) {
   // Maximum valid cost increased in this function.
   int CostUpperBound = INT_MAX - InlineConstants::InstrCost - 1;
 
-  // Exit early for a large switch, assuming one case needs at least one
-  // instruction.
-  // FIXME: This is not true for a bit test, but ignore such case for now to
-  // save compile-time.
-  int64_t CostLowerBound =
-      std::min((int64_t)CostUpperBound,
-               (int64_t)SI.getNumCases() * InlineConstants::InstrCost + Cost);
-
-  if (CostLowerBound > Threshold && !ComputeFullInlineCost) {
-    addCost((int64_t)SI.getNumCases() * InlineConstants::InstrCost);
-    return false;
-  }
-
   unsigned JumpTableSize = 0;
   unsigned NumCaseCluster =
       TTI.getEstimatedNumberOfCaseClusters(SI, JumpTableSize);
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index defa3c3ea5671..cb8987721700b 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -1382,23 +1382,56 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
 
   ICmpInst::Predicate UnsignedPred;
 
-  // Y = (A - B);  Y >= A && Y != 0  --> Y >= A  iff B != 0
-  // Y = (A - B);  Y <  A || Y == 0  --> Y <  A  iff B != 0
   Value *A, *B;
-  if (match(Y, m_Sub(m_Value(A), m_Value(B))) &&
-      match(UnsignedICmp,
-            m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) {
-    if (UnsignedICmp->getOperand(0) != Y)
-      UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);
-
-    if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&
-        EqPred == ICmpInst::ICMP_NE &&
-        isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
-      return UnsignedICmp;
-    if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&
-        EqPred == ICmpInst::ICMP_EQ &&
-        isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
-      return UnsignedICmp;
+  // Y = (A - B);
+  if (match(Y, m_Sub(m_Value(A), m_Value(B)))) {
+    if (match(UnsignedICmp,
+              m_c_ICmp(UnsignedPred, m_Specific(A), m_Specific(B))) &&
+        ICmpInst::isUnsigned(UnsignedPred)) {
+      if (UnsignedICmp->getOperand(0) != A)
+        UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);
+
+      // A >=/<= B || (A - B) != 0  <-->  true
+      if ((UnsignedPred == ICmpInst::ICMP_UGE ||
+           UnsignedPred == ICmpInst::ICMP_ULE) &&
+          EqPred == ICmpInst::ICMP_NE && !IsAnd)
+        return ConstantInt::getTrue(UnsignedICmp->getType());
+      // A </> B && (A - B) == 0  <-->  false
+      if ((UnsignedPred == ICmpInst::ICMP_ULT ||
+           UnsignedPred == ICmpInst::ICMP_UGT) &&
+          EqPred == ICmpInst::ICMP_EQ && IsAnd)
+        return ConstantInt::getFalse(UnsignedICmp->getType());
+
+      // A </> B && (A - B) != 0  <-->  A </> B
+      // A </> B || (A - B) != 0  <-->  (A - B) != 0
+      if (EqPred == ICmpInst::ICMP_NE && (UnsignedPred == ICmpInst::ICMP_ULT ||
+                                          UnsignedPred == ICmpInst::ICMP_UGT))
+        return IsAnd ? UnsignedICmp : ZeroICmp;
+
+      // A <=/>= B && (A - B) == 0  <-->  (A - B) == 0
+      // A <=/>= B || (A - B) == 0  <-->  A <=/>= B
+      if (EqPred == ICmpInst::ICMP_EQ && (UnsignedPred == ICmpInst::ICMP_ULE ||
+                                          UnsignedPred == ICmpInst::ICMP_UGE))
+        return IsAnd ? ZeroICmp : UnsignedICmp;
+    }
+
+    // Given  Y = (A - B)
+    //   Y >= A && Y != 0  --> Y >= A  iff B != 0
+    //   Y <  A || Y == 0  --> Y <  A  iff B != 0
+    if (match(UnsignedICmp,
+              m_c_ICmp(UnsignedPred, m_Specific(Y), m_Specific(A)))) {
+      if (UnsignedICmp->getOperand(0) != Y)
+        UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);
+
+      if (UnsignedPred == ICmpInst::ICMP_UGE && IsAnd &&
+          EqPred == ICmpInst::ICMP_NE &&
+          isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+        return UnsignedICmp;
+      if (UnsignedPred == ICmpInst::ICMP_ULT && !IsAnd &&
+          EqPred == ICmpInst::ICMP_EQ &&
+          isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+        return UnsignedICmp;
+    }
   }
 
   if (match(UnsignedICmp, m_ICmp(UnsignedPred, m_Value(X), m_Specific(Y))) &&
@@ -1422,25 +1455,27 @@ static Value *simplifyUnsignedRangeCheck(ICmpInst *ZeroICmp,
       isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
     return IsAnd ? UnsignedICmp : ZeroICmp;
 
+  // X >= Y && Y == 0  -->  Y == 0
+  // X >= Y || Y == 0  -->  X >= Y
+  if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ)
+    return IsAnd ? ZeroICmp : UnsignedICmp;
+
   // X > Y && Y == 0  -->  Y == 0  iff X != 0
   // X > Y || Y == 0  -->  X > Y   iff X != 0
   if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
       isKnownNonZero(X, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
     return IsAnd ? ZeroICmp : UnsignedICmp;
 
-  // X >= Y || Y != 0  -->  true
-  // X >= Y || Y == 0  -->  X >= Y
-  if (UnsignedPred == ICmpInst::ICMP_UGE && !IsAnd) {
-    if (EqPred == ICmpInst::ICMP_NE)
-      return getTrue(UnsignedICmp->getType());
-    return UnsignedICmp;
-  }
-
   // X < Y && Y == 0  -->  false
   if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_EQ &&
       IsAnd)
     return getFalse(UnsignedICmp->getType());
 
+  // X >= Y || Y != 0  -->  true
+  if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_NE &&
+      !IsAnd)
+    return getTrue(UnsignedICmp->getType());
+
   return nullptr;
 }
 
@@ -4447,14 +4482,16 @@ static Constant *propagateNaN(Constant *In) {
   return In;
 }
 
-static Constant *simplifyFPBinop(Value *Op0, Value *Op1) {
-  if (isa<UndefValue>(Op0) || isa<UndefValue>(Op1))
-    return ConstantFP::getNaN(Op0->getType());
+/// Perform folds that are common to any floating-point operation. This implies
+/// transforms based on undef/NaN because the operation itself makes no
+/// difference to the result.
+static Constant *simplifyFPOp(ArrayRef<Value *> Ops) {
+  if (any_of(Ops, [](Value *V) { return isa<UndefValue>(V); }))
+    return ConstantFP::getNaN(Ops[0]->getType());
 
-  if (match(Op0, m_NaN()))
-    return propagateNaN(cast<Constant>(Op0));
-  if (match(Op1, m_NaN()))
-    return propagateNaN(cast<Constant>(Op1));
+  for (Value *V : Ops)
+    if (match(V, m_NaN()))
+      return propagateNaN(cast<Constant>(V));
 
   return nullptr;
 }
@@ -4466,7 +4503,7 @@ static Value *SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   if (Constant *C = foldOrCommuteConstant(Instruction::FAdd, Op0, Op1, Q))
     return C;
 
-  if (Constant *C = simplifyFPBinop(Op0, Op1))
+  if (Constant *C = simplifyFPOp({Op0, Op1}))
     return C;
 
   // fadd X, -0 ==> X
@@ -4513,7 +4550,7 @@ static Value *SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   if (Constant *C = foldOrCommuteConstant(Instruction::FSub, Op0, Op1, Q))
     return C;
 
-  if (Constant *C = simplifyFPBinop(Op0, Op1))
+  if (Constant *C = simplifyFPOp({Op0, Op1}))
     return C;
 
   // fsub X, +0 ==> X
@@ -4553,23 +4590,27 @@ static Value *SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   return nullptr;
 }
 
-/// Given the operands for an FMul, see if we can fold the result
-static Value *SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
-                               const SimplifyQuery &Q, unsigned MaxRecurse) {
-  if (Constant *C = foldOrCommuteConstant(Instruction::FMul, Op0, Op1, Q))
-    return C;
-
-  if (Constant *C = simplifyFPBinop(Op0, Op1))
+static Value *SimplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,
+                              const SimplifyQuery &Q, unsigned MaxRecurse) {
+  if (Constant *C = simplifyFPOp({Op0, Op1}))
     return C;
 
   // fmul X, 1.0 ==> X
   if (match(Op1, m_FPOne()))
     return Op0;
 
+  // fmul 1.0, X ==> X
+  if (match(Op0, m_FPOne()))
+    return Op1;
+
   // fmul nnan nsz X, 0 ==> 0
   if (FMF.noNaNs() && FMF.noSignedZeros() && match(Op1, m_AnyZeroFP()))
     return ConstantFP::getNullValue(Op0->getType());
 
+  // fmul nnan nsz 0, X ==> 0
+  if (FMF.noNaNs() && FMF.noSignedZeros() && match(Op0, m_AnyZeroFP()))
+    return ConstantFP::getNullValue(Op1->getType());
+
   // sqrt(X) * sqrt(X) --> X, if we can:
   // 1. Remove the intermediate rounding (reassociate).
   // 2. Ignore non-zero negative numbers because sqrt would produce NAN.
@@ -4582,6 +4623,16 @@ static Value *SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   return nullptr;
 }
 
+/// Given the operands for an FMul, see if we can fold the result
+static Value *SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
+                               const SimplifyQuery &Q, unsigned MaxRecurse) {
+  if (Constant *C = foldOrCommuteConstant(Instruction::FMul, Op0, Op1, Q))
+    return C;
+
+  // Now apply simplifications that do not require rounding.
+  return SimplifyFMAFMul(Op0, Op1, FMF, Q, MaxRecurse);
+}
+
 Value *llvm::SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF,
                               const SimplifyQuery &Q) {
   return ::SimplifyFAddInst(Op0, Op1, FMF, Q, RecursionLimit);
@@ -4598,12 +4649,17 @@ Value *llvm::SimplifyFMulInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   return ::SimplifyFMulInst(Op0, Op1, FMF, Q, RecursionLimit);
 }
 
+Value *llvm::SimplifyFMAFMul(Value *Op0, Value *Op1, FastMathFlags FMF,
+                             const SimplifyQuery &Q) {
+  return ::SimplifyFMAFMul(Op0, Op1, FMF, Q, RecursionLimit);
+}
+
 static Value *SimplifyFDivInst(Value *Op0, Value *Op1, FastMathFlags FMF,
                                const SimplifyQuery &Q, unsigned) {
   if (Constant *C = foldOrCommuteConstant(Instruction::FDiv, Op0, Op1, Q))
     return C;
 
-  if (Constant *C = simplifyFPBinop(Op0, Op1))
+  if (Constant *C = simplifyFPOp({Op0, Op1}))
     return C;
 
   // X / 1.0 -> X
@@ -4648,7 +4704,7 @@ static Value *SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF,
   if (Constant *C = foldOrCommuteConstant(Instruction::FRem, Op0, Op1, Q))
     return C;
 
-  if (Constant *C = simplifyFPBinop(Op0, Op1))
+  if (Constant *C = simplifyFPOp({Op0, Op1}))
     return C;
 
   // Unlike fdiv, the result of frem always matches the sign of the dividend.
@@ -5130,6 +5186,15 @@ static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) {
     }
     return nullptr;
   }
+  case Intrinsic::fma:
+  case Intrinsic::fmuladd: {
+    Value *Op0 = Call->getArgOperand(0);
+    Value *Op1 = Call->getArgOperand(1);
+    Value *Op2 = Call->getArgOperand(2);
+    if (Value *V = simplifyFPOp({ Op0, Op1, Op2 }))
+      return V;
+    return nullptr;
+  }
   default:
     return nullptr;
   }
diff --git a/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp b/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp
index 7a12f597969de..7de9d2cbfddba 100644
--- a/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp
+++ b/llvm/lib/Analysis/LegacyDivergenceAnalysis.cpp
@@ -257,9 +257,8 @@ void DivergencePropagator::computeInfluenceRegion(
 void DivergencePropagator::exploreDataDependency(Value *V) {
   // Follow def-use chains of V.
   for (User *U : V->users()) {
-    Instruction *UserInst = cast<Instruction>(U);
-    if (!TTI.isAlwaysUniform(U) && DV.insert(UserInst).second)
-      Worklist.push_back(UserInst);
+    if (!TTI.isAlwaysUniform(U) && DV.insert(U).second)
+      Worklist.push_back(U);
   }
 }
 
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 7f24a2c297a8e..3d8f77675f3a2 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1212,8 +1212,8 @@ bool llvm::isConsecutiveAccess(Value *A, Value *B, const DataLayout &DL,
   const SCEV *OffsetSCEVA = SE.getConstant(OffsetA);
   const SCEV *OffsetSCEVB = SE.getConstant(OffsetB);
   const SCEV *OffsetDeltaSCEV = SE.getMinusSCEV(OffsetSCEVB, OffsetSCEVA);
-  const SCEVConstant *OffsetDeltaC = dyn_cast<SCEVConstant>(OffsetDeltaSCEV);
-  const APInt &OffsetDelta = OffsetDeltaC->getAPInt();
+  const APInt &OffsetDelta = cast<SCEVConstant>(OffsetDeltaSCEV)->getAPInt();
+
   // Check if they are based on the same pointer. That makes the offsets
   // sufficient.
   if (PtrA == PtrB)
diff --git a/llvm/lib/Analysis/LoopInfo.cpp b/llvm/lib/Analysis/LoopInfo.cpp
index 419f013c0e7d2..1dc63a3c3338f 100644
--- a/llvm/lib/Analysis/LoopInfo.cpp
+++ b/llvm/lib/Analysis/LoopInfo.cpp
@@ -362,18 +362,16 @@ bool Loop::isAuxiliaryInductionVariable(PHINode &AuxIndVar,
 BranchInst *Loop::getLoopGuardBranch() const {
   assert(isLoopSimplifyForm() && "Only valid for loop in simplify form");
   BasicBlock *Preheader = getLoopPreheader();
-  BasicBlock *Latch = getLoopLatch();
-  assert(Preheader && Latch &&
+  assert(Preheader && getLoopLatch() &&
          "Expecting a loop with valid preheader and latch");
-  assert(isLoopExiting(Latch) && "Only valid for rotated loop");
+  assert(isLoopExiting(getLoopLatch()) && "Only valid for rotated loop");
 
-  Instruction *LatchTI = Latch->getTerminator();
-  if (!LatchTI || LatchTI->getNumSuccessors() != 2)
+  // Disallow loops with more than one unique exit block, as we do not verify
+  // that GuardOtherSucc post dominates all exit blocks.
+  BasicBlock *ExitFromLatch = getUniqueExitBlock();
+  if (!ExitFromLatch)
     return nullptr;
 
-  BasicBlock *ExitFromLatch = (LatchTI->getSuccessor(0) == getHeader())
-                                  ? LatchTI->getSuccessor(1)
-                                  : LatchTI->getSuccessor(0);
   BasicBlock *ExitFromLatchSucc = ExitFromLatch->getUniqueSuccessor();
   if (!ExitFromLatchSucc)
     return nullptr;
diff --git a/llvm/lib/Analysis/MemoryBuiltins.cpp b/llvm/lib/Analysis/MemoryBuiltins.cpp
index 52f31238227d5..172c86eb46466 100644
--- a/llvm/lib/Analysis/MemoryBuiltins.cpp
+++ b/llvm/lib/Analysis/MemoryBuiltins.cpp
@@ -307,6 +307,13 @@ bool llvm::isOpNewLikeFn(const Value *V, const TargetLibraryInfo *TLI,
   return getAllocationData(V, OpNewLike, TLI, LookThroughBitCast).hasValue();
 }
 
+/// Tests if a value is a call or invoke to a library function that
+/// allocates memory (strdup, strndup).
+bool llvm::isStrdupLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                          bool LookThroughBitCast) {
+  return getAllocationData(V, StrDupLike, TLI, LookThroughBitCast).hasValue();
+}
+
 /// extractMallocCall - Returns the corresponding CallInst if the instruction
 /// is a malloc call.  Since CallInst::CreateMalloc() only creates calls, we
 /// ignore InvokeInst here.
@@ -553,9 +560,9 @@ STATISTIC(ObjectVisitorArgument,
 STATISTIC(ObjectVisitorLoad,
           "Number of load instructions with unsolved size and offset");
 
-APInt ObjectSizeOffsetVisitor::align(APInt Size, uint64_t Align) {
-  if (Options.RoundToAlign && Align)
-    return APInt(IntTyBits, alignTo(Size.getZExtValue(), llvm::Align(Align)));
+APInt ObjectSizeOffsetVisitor::align(APInt Size, uint64_t Alignment) {
+  if (Options.RoundToAlign && Alignment)
+    return APInt(IntTyBits, alignTo(Size.getZExtValue(), Align(Alignment)));
   return Size;
 }
 
diff --git a/llvm/lib/Analysis/MemorySSA.cpp b/llvm/lib/Analysis/MemorySSA.cpp
index db12fcee3ef75..cfb8b7e7dcb55 100644
--- a/llvm/lib/Analysis/MemorySSA.cpp
+++ b/llvm/lib/Analysis/MemorySSA.cpp
@@ -873,6 +873,7 @@ template <class AliasAnalysisType> class ClobberWalker {
         if (!DefChainEnd)
           for (auto *MA : def_chain(const_cast<MemoryAccess *>(Target)))
             DefChainEnd = MA;
+        assert(DefChainEnd && "Failed to find dominating phi/liveOnEntry");
 
         // If any of the terminated paths don't dominate the phi we'll try to
         // optimize, we need to figure out what they are and quit.
@@ -1736,9 +1737,15 @@ MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I,
   // FIXME: Replace this special casing with a more accurate modelling of
   // assume's control dependency.
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
-    if (II->getIntrinsicID() == Intrinsic::assume || isa<DbgInfoIntrinsic>(II))
+    if (II->getIntrinsicID() == Intrinsic::assume)
       return nullptr;
 
+  // Using a nonstandard AA pipelines might leave us with unexpected modref
+  // results for I, so add a check to not model instructions that may not read
+  // from or write to memory. This is necessary for correctness.
+  if (!I->mayReadFromMemory() && !I->mayWriteToMemory())
+    return nullptr;
+
   bool Def, Use;
   if (Template) {
     Def = dyn_cast_or_null<MemoryDef>(Template) != nullptr;
diff --git a/llvm/lib/Analysis/MemorySSAUpdater.cpp b/llvm/lib/Analysis/MemorySSAUpdater.cpp
index b29be09bda752..d103c3a8b831f 100644
--- a/llvm/lib/Analysis/MemorySSAUpdater.cpp
+++ b/llvm/lib/Analysis/MemorySSAUpdater.cpp
@@ -71,11 +71,19 @@ MemoryAccess *MemorySSAUpdater::getPreviousDefRecursive(
     // Recurse to get the values in our predecessors for placement of a
     // potential phi node. This will insert phi nodes if we cycle in order to
     // break the cycle and have an operand.
-    for (auto *Pred : predecessors(BB))
-      if (MSSA->DT->isReachableFromEntry(Pred))
-        PhiOps.push_back(getPreviousDefFromEnd(Pred, CachedPreviousDef));
-      else
+    bool UniqueIncomingAccess = true;
+    MemoryAccess *SingleAccess = nullptr;
+    for (auto *Pred : predecessors(BB)) {
+      if (MSSA->DT->isReachableFromEntry(Pred)) {
+        auto *IncomingAccess = getPreviousDefFromEnd(Pred, CachedPreviousDef);
+        if (!SingleAccess)
+          SingleAccess = IncomingAccess;
+        else if (IncomingAccess != SingleAccess)
+          UniqueIncomingAccess = false;
+        PhiOps.push_back(IncomingAccess);
+      } else
         PhiOps.push_back(MSSA->getLiveOnEntryDef());
+    }
 
     // Now try to simplify the ops to avoid placing a phi.
     // This may return null if we never created a phi yet, that's okay
@@ -84,7 +92,9 @@ MemoryAccess *MemorySSAUpdater::getPreviousDefRecursive(
     // See if we can avoid the phi by simplifying it.
     auto *Result = tryRemoveTrivialPhi(Phi, PhiOps);
     // If we couldn't simplify, we may have to create a phi
-    if (Result == Phi) {
+    if (Result == Phi && UniqueIncomingAccess && SingleAccess)
+      Result = SingleAccess;
+    else if (Result == Phi && !(UniqueIncomingAccess && SingleAccess)) {
       if (!Phi)
         Phi = MSSA->createMemoryPhi(BB);
 
@@ -293,7 +303,12 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) {
 
   // See if we had a local def, and if not, go hunting.
   MemoryAccess *DefBefore = getPreviousDef(MD);
-  bool DefBeforeSameBlock = DefBefore->getBlock() == MD->getBlock();
+  bool DefBeforeSameBlock = false;
+  if (DefBefore->getBlock() == MD->getBlock() &&
+      !(isa<MemoryPhi>(DefBefore) &&
+        std::find(InsertedPHIs.begin(), InsertedPHIs.end(), DefBefore) !=
+            InsertedPHIs.end()))
+    DefBeforeSameBlock = true;
 
   // There is a def before us, which means we can replace any store/phi uses
   // of that thing with us, since we are in the way of whatever was there
@@ -313,10 +328,10 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) {
   // and that def is now our defining access.
   MD->setDefiningAccess(DefBefore);
 
-  // Remember the index where we may insert new phis below.
-  unsigned NewPhiIndex = InsertedPHIs.size();
-
   SmallVector<WeakVH, 8> FixupList(InsertedPHIs.begin(), InsertedPHIs.end());
+
+  // Remember the index where we may insert new phis.
+  unsigned NewPhiIndex = InsertedPHIs.size();
   if (!DefBeforeSameBlock) {
     // If there was a local def before us, we must have the same effect it
     // did. Because every may-def is the same, any phis/etc we would create, it
@@ -332,50 +347,54 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) {
 
     // If this is the first def in the block and this insert is in an arbitrary
     // place, compute IDF and place phis.
+    SmallPtrSet<BasicBlock *, 2> DefiningBlocks;
+
+    // If this is the last Def in the block, also compute IDF based on MD, since
+    // this may a new Def added, and we may need additional Phis.
     auto Iter = MD->getDefsIterator();
     ++Iter;
     auto IterEnd = MSSA->getBlockDefs(MD->getBlock())->end();
-    if (Iter == IterEnd) {
-      ForwardIDFCalculator IDFs(*MSSA->DT);
-      SmallVector<BasicBlock *, 32> IDFBlocks;
-      SmallPtrSet<BasicBlock *, 2> DefiningBlocks;
+    if (Iter == IterEnd)
       DefiningBlocks.insert(MD->getBlock());
-      IDFs.setDefiningBlocks(DefiningBlocks);
-      IDFs.calculate(IDFBlocks);
-      SmallVector<AssertingVH<MemoryPhi>, 4> NewInsertedPHIs;
-      for (auto *BBIDF : IDFBlocks) {
-        auto *MPhi = MSSA->getMemoryAccess(BBIDF);
-        if (!MPhi) {
-          MPhi = MSSA->createMemoryPhi(BBIDF);
-          NewInsertedPHIs.push_back(MPhi);
-        }
-        // Add the phis created into the IDF blocks to NonOptPhis, so they are
-        // not optimized out as trivial by the call to getPreviousDefFromEnd
-        // below. Once they are complete, all these Phis are added to the
-        // FixupList, and removed from NonOptPhis inside fixupDefs().
-        // Existing Phis in IDF may need fixing as well, and potentially be
-        // trivial before this insertion, hence add all IDF Phis. See PR43044.
-        NonOptPhis.insert(MPhi);
-      }
 
-      for (auto &MPhi : NewInsertedPHIs) {
-        auto *BBIDF = MPhi->getBlock();
-        for (auto *Pred : predecessors(BBIDF)) {
-          DenseMap<BasicBlock *, TrackingVH<MemoryAccess>> CachedPreviousDef;
-          MPhi->addIncoming(getPreviousDefFromEnd(Pred, CachedPreviousDef),
-                            Pred);
-        }
+    for (const auto &VH : InsertedPHIs)
+      if (const auto *RealPHI = cast_or_null<MemoryPhi>(VH))
+        DefiningBlocks.insert(RealPHI->getBlock());
+    ForwardIDFCalculator IDFs(*MSSA->DT);
+    SmallVector<BasicBlock *, 32> IDFBlocks;
+    IDFs.setDefiningBlocks(DefiningBlocks);
+    IDFs.calculate(IDFBlocks);
+    SmallVector<AssertingVH<MemoryPhi>, 4> NewInsertedPHIs;
+    for (auto *BBIDF : IDFBlocks) {
+      auto *MPhi = MSSA->getMemoryAccess(BBIDF);
+      if (!MPhi) {
+        MPhi = MSSA->createMemoryPhi(BBIDF);
+        NewInsertedPHIs.push_back(MPhi);
       }
-
-      // Re-take the index where we're adding the new phis, because the above
-      // call to getPreviousDefFromEnd, may have inserted into InsertedPHIs.
-      NewPhiIndex = InsertedPHIs.size();
-      for (auto &MPhi : NewInsertedPHIs) {
-        InsertedPHIs.push_back(&*MPhi);
-        FixupList.push_back(&*MPhi);
+      // Add the phis created into the IDF blocks to NonOptPhis, so they are not
+      // optimized out as trivial by the call to getPreviousDefFromEnd below.
+      // Once they are complete, all these Phis are added to the FixupList, and
+      // removed from NonOptPhis inside fixupDefs(). Existing Phis in IDF may
+      // need fixing as well, and potentially be trivial before this insertion,
+      // hence add all IDF Phis. See PR43044.
+      NonOptPhis.insert(MPhi);
+    }
+    for (auto &MPhi : NewInsertedPHIs) {
+      auto *BBIDF = MPhi->getBlock();
+      for (auto *Pred : predecessors(BBIDF)) {
+        DenseMap<BasicBlock *, TrackingVH<MemoryAccess>> CachedPreviousDef;
+        MPhi->addIncoming(getPreviousDefFromEnd(Pred, CachedPreviousDef), Pred);
       }
     }
 
+    // Re-take the index where we're adding the new phis, because the above call
+    // to getPreviousDefFromEnd, may have inserted into InsertedPHIs.
+    NewPhiIndex = InsertedPHIs.size();
+    for (auto &MPhi : NewInsertedPHIs) {
+      InsertedPHIs.push_back(&*MPhi);
+      FixupList.push_back(&*MPhi);
+    }
+
     FixupList.push_back(MD);
   }
 
@@ -617,7 +636,7 @@ void MemorySSAUpdater::updatePhisWhenInsertingUniqueBackedgeBlock(
 
   // If NewMPhi is a trivial phi, remove it. Its use in the header MPhi will be
   // replaced with the unique value.
-  tryRemoveTrivialPhi(MPhi);
+  tryRemoveTrivialPhi(NewMPhi);
 }
 
 void MemorySSAUpdater::updateForClonedLoop(const LoopBlocksRPO &LoopBlocks,
@@ -823,6 +842,9 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
       } else {
         // Single predecessor, BB cannot be dead. GetLastDef of Pred.
         assert(Count == 1 && Pred && "Single predecessor expected.");
+        // BB can be unreachable though, return LoE if that is the case.
+        if (!DT.getNode(BB))
+          return MSSA->getLiveOnEntryDef();
         BB = Pred;
       }
     };
@@ -1063,7 +1085,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef<CFGUpdate> Updates,
         for (; UI != E;) {
           Use &U = *UI;
           ++UI;
-          MemoryAccess *Usr = dyn_cast<MemoryAccess>(U.getUser());
+          MemoryAccess *Usr = cast<MemoryAccess>(U.getUser());
           if (MemoryPhi *UsrPhi = dyn_cast<MemoryPhi>(Usr)) {
             BasicBlock *DominatedBlock = UsrPhi->getIncomingBlock(U);
             if (!DT.dominates(DominatingBlock, DominatedBlock))
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index dce19d6d546e2..b99b75715025b 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -45,6 +45,13 @@ static cl::opt<unsigned> ProfileSummaryHugeWorkingSetSizeThreshold(
              " blocks required to reach the -profile-summary-cutoff-hot"
              " percentile exceeds this count."));
 
+static cl::opt<unsigned> ProfileSummaryLargeWorkingSetSizeThreshold(
+    "profile-summary-large-working-set-size-threshold", cl::Hidden,
+    cl::init(12500), cl::ZeroOrMore,
+    cl::desc("The code working set size is considered large if the number of"
+             " blocks required to reach the -profile-summary-cutoff-hot"
+             " percentile exceeds this count."));
+
 // The next two options override the counts derived from summary computation and
 // are useful for debugging purposes.
 static cl::opt<int> ProfileSummaryHotCount(
@@ -186,6 +193,31 @@ bool ProfileSummaryInfo::isFunctionColdInCallGraph(const Function *F,
   return true;
 }
 
+// Like isFunctionHotInCallGraph but for a given cutoff.
+bool ProfileSummaryInfo::isFunctionHotInCallGraphNthPercentile(
+    int PercentileCutoff, const Function *F, BlockFrequencyInfo &BFI) {
+  if (!F || !computeSummary())
+    return false;
+  if (auto FunctionCount = F->getEntryCount())
+    if (isHotCountNthPercentile(PercentileCutoff, FunctionCount.getCount()))
+      return true;
+
+  if (hasSampleProfile()) {
+    uint64_t TotalCallCount = 0;
+    for (const auto &BB : *F)
+      for (const auto &I : BB)
+        if (isa<CallInst>(I) || isa<InvokeInst>(I))
+          if (auto CallCount = getProfileCount(&I, nullptr))
+            TotalCallCount += CallCount.getValue();
+    if (isHotCountNthPercentile(PercentileCutoff, TotalCallCount))
+      return true;
+  }
+  for (const auto &BB : *F)
+    if (isHotBlockNthPercentile(PercentileCutoff, &BB, &BFI))
+      return true;
+  return false;
+}
+
 /// Returns true if the function's entry is a cold. If it returns false, it
 /// either means it is not cold or it is unknown whether it is cold or not (for
 /// example, no profile data is available).
@@ -222,6 +254,23 @@ void ProfileSummaryInfo::computeThresholds() {
          "Cold count threshold cannot exceed hot count threshold!");
   HasHugeWorkingSetSize =
       HotEntry.NumCounts > ProfileSummaryHugeWorkingSetSizeThreshold;
+  HasLargeWorkingSetSize =
+      HotEntry.NumCounts > ProfileSummaryLargeWorkingSetSizeThreshold;
+}
+
+Optional<uint64_t> ProfileSummaryInfo::computeThreshold(int PercentileCutoff) {
+  if (!computeSummary())
+    return None;
+  auto iter = ThresholdCache.find(PercentileCutoff);
+  if (iter != ThresholdCache.end()) {
+    return iter->second;
+  }
+  auto &DetailedSummary = Summary->getDetailedSummary();
+  auto &Entry =
+      getEntryForPercentile(DetailedSummary, PercentileCutoff);
+  uint64_t CountThreshold = Entry.MinCount;
+  ThresholdCache[PercentileCutoff] = CountThreshold;
+  return CountThreshold;
 }
 
 bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
@@ -230,6 +279,12 @@ bool ProfileSummaryInfo::hasHugeWorkingSetSize() {
   return HasHugeWorkingSetSize && HasHugeWorkingSetSize.getValue();
 }
 
+bool ProfileSummaryInfo::hasLargeWorkingSetSize() {
+  if (!HasLargeWorkingSetSize)
+    computeThresholds();
+  return HasLargeWorkingSetSize && HasLargeWorkingSetSize.getValue();
+}
+
 bool ProfileSummaryInfo::isHotCount(uint64_t C) {
   if (!HotCountThreshold)
     computeThresholds();
@@ -242,6 +297,11 @@ bool ProfileSummaryInfo::isColdCount(uint64_t C) {
   return ColdCountThreshold && C <= ColdCountThreshold.getValue();
 }
 
+bool ProfileSummaryInfo::isHotCountNthPercentile(int PercentileCutoff, uint64_t C) {
+  auto CountThreshold = computeThreshold(PercentileCutoff);
+  return CountThreshold && C >= CountThreshold.getValue();
+}
+
 uint64_t ProfileSummaryInfo::getOrCompHotCountThreshold() {
   if (!HotCountThreshold)
     computeThresholds();
@@ -265,6 +325,13 @@ bool ProfileSummaryInfo::isColdBlock(const BasicBlock *BB,
   return Count && isColdCount(*Count);
 }
 
+bool ProfileSummaryInfo::isHotBlockNthPercentile(int PercentileCutoff,
+                                                 const BasicBlock *BB,
+                                                 BlockFrequencyInfo *BFI) {
+  auto Count = BFI->getBlockProfileCount(BB);
+  return Count && isHotCountNthPercentile(PercentileCutoff, *Count);
+}
+
 bool ProfileSummaryInfo::isHotCallSite(const CallSite &CS,
                                        BlockFrequencyInfo *BFI) {
   auto C = getProfileCount(CS.getInstruction(), BFI);
diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp
index 2cac4a090ff04..c29fc5dbccfb8 100644
--- a/llvm/lib/Analysis/ScalarEvolution.cpp
+++ b/llvm/lib/Analysis/ScalarEvolution.cpp
@@ -148,6 +148,7 @@ STATISTIC(NumBruteForceTripCountsComputed,
 
 static cl::opt<unsigned>
 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
+                        cl::ZeroOrMore,
                         cl::desc("Maximum number of iterations SCEV will "
                                  "symbolically execute a constant "
                                  "derived loop"),
@@ -4991,7 +4992,7 @@ const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
   // overflow.
   if (auto *BEInst = dyn_cast<Instruction>(BEValueV))
     if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L))
-      (void)getAddRecExpr(getAddExpr(StartVal, Accum, Flags), Accum, L, Flags);
+      (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags);
 
   return PHISCEV;
 }
diff --git a/llvm/lib/Analysis/ScalarEvolutionExpander.cpp b/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
index e8a95d35482c0..8070882c150ff 100644
--- a/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionExpander.cpp
@@ -1486,7 +1486,18 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) {
 }
 
 Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) {
-  if (!CanonicalMode) return expandAddRecExprLiterally(S);
+  // In canonical mode we compute the addrec as an expression of a canonical IV
+  // using evaluateAtIteration and expand the resulting SCEV expression. This
+  // way we avoid introducing new IVs to carry on the comutation of the addrec
+  // throughout the loop.
+  //
+  // For nested addrecs evaluateAtIteration might need a canonical IV of a
+  // type wider than the addrec itself. Emitting a canonical IV of the
+  // proper type might produce non-legal types, for example expanding an i64
+  // {0,+,2,+,1} addrec would need an i65 canonical IV. To avoid this just fall
+  // back to non-canonical mode for nested addrecs.
+  if (!CanonicalMode || (S->getNumOperands() > 2))
+    return expandAddRecExprLiterally(S);
 
   Type *Ty = SE.getEffectiveSCEVType(S->getType());
   const Loop *L = S->getLoop();
@@ -2094,11 +2105,10 @@ SCEVExpander::getRelatedExistingExpansion(const SCEV *S, const Instruction *At,
   for (BasicBlock *BB : ExitingBlocks) {
     ICmpInst::Predicate Pred;
     Instruction *LHS, *RHS;
-    BasicBlock *TrueBB, *FalseBB;
 
     if (!match(BB->getTerminator(),
                m_Br(m_ICmp(Pred, m_Instruction(LHS), m_Instruction(RHS)),
-                    TrueBB, FalseBB)))
+                    m_BasicBlock(), m_BasicBlock())))
       continue;
 
     if (SE.getSCEV(LHS) == S && SE.DT.dominates(LHS, At))
diff --git a/llvm/lib/Analysis/SyncDependenceAnalysis.cpp b/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
index 3cf248a311428..8447dc87069db 100644
--- a/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
+++ b/llvm/lib/Analysis/SyncDependenceAnalysis.cpp
@@ -218,9 +218,11 @@ struct DivergencePropagator {
   template <typename SuccessorIterable>
   std::unique_ptr<ConstBlockSet>
   computeJoinPoints(const BasicBlock &RootBlock,
-                    SuccessorIterable NodeSuccessors, const Loop *ParentLoop, const BasicBlock * PdBoundBlock) {
+                    SuccessorIterable NodeSuccessors, const Loop *ParentLoop) {
     assert(JoinBlocks);
 
+    LLVM_DEBUG(dbgs() << "SDA:computeJoinPoints. Parent loop: " << (ParentLoop ? ParentLoop->getName() : "<null>") << "\n" );
+
     // bootstrap with branch targets
     for (const auto *SuccBlock : NodeSuccessors) {
       DefMap.emplace(SuccBlock, SuccBlock);
@@ -228,13 +230,19 @@ struct DivergencePropagator {
       if (ParentLoop && !ParentLoop->contains(SuccBlock)) {
         // immediate loop exit from node.
         ReachedLoopExits.insert(SuccBlock);
-        continue;
       } else {
         // regular successor
         PendingUpdates.insert(SuccBlock);
       }
     }
 
+    LLVM_DEBUG(
+      dbgs() << "SDA: rpo order:\n";
+      for (const auto * RpoBlock : FuncRPOT) {
+        dbgs() << "- " << RpoBlock->getName() << "\n";
+      }
+    );
+
     auto ItBeginRPO = FuncRPOT.begin();
 
     // skip until term (TODO RPOT won't let us start at @term directly)
@@ -245,16 +253,18 @@ struct DivergencePropagator {
 
     // propagate definitions at the immediate successors of the node in RPO
     auto ItBlockRPO = ItBeginRPO;
-    while (++ItBlockRPO != ItEndRPO && *ItBlockRPO != PdBoundBlock) {
+    while ((++ItBlockRPO != ItEndRPO) &&
+           !PendingUpdates.empty()) {
       const auto *Block = *ItBlockRPO;
+      LLVM_DEBUG(dbgs() << "SDA::joins. visiting " << Block->getName() << "\n");
 
-      // skip @block if not pending update
+      // skip Block if not pending update
       auto ItPending = PendingUpdates.find(Block);
       if (ItPending == PendingUpdates.end())
         continue;
       PendingUpdates.erase(ItPending);
 
-      // propagate definition at @block to its successors
+      // propagate definition at Block to its successors
       auto ItDef = DefMap.find(Block);
       const auto *DefBlock = ItDef->second;
       assert(DefBlock);
@@ -278,6 +288,8 @@ struct DivergencePropagator {
       }
     }
 
+    LLVM_DEBUG(dbgs() << "SDA::joins. After propagation:\n"; printDefs(dbgs()));
+
     // We need to know the definition at the parent loop header to decide
     // whether the definition at the header is different from the definition at
     // the loop exits, which would indicate a divergent loop exits.
@@ -292,24 +304,17 @@ struct DivergencePropagator {
     // |
     // proper exit from both loops
     //
-    // D post-dominates B as it is the only proper exit from the "A loop".
-    // If C has a divergent branch, propagation will therefore stop at D.
-    // That implies that B will never receive a definition.
-    // But that definition can only be the same as at D (D itself in thise case)
-    // because all paths to anywhere have to pass through D.
-    //
-    const BasicBlock *ParentLoopHeader =
-        ParentLoop ? ParentLoop->getHeader() : nullptr;
-    if (ParentLoop && ParentLoop->contains(PdBoundBlock)) {
-      DefMap[ParentLoopHeader] = DefMap[PdBoundBlock];
-    }
-
     // analyze reached loop exits
     if (!ReachedLoopExits.empty()) {
+      const BasicBlock *ParentLoopHeader =
+          ParentLoop ? ParentLoop->getHeader() : nullptr;
+
       assert(ParentLoop);
-      const auto *HeaderDefBlock = DefMap[ParentLoopHeader];
+      auto ItHeaderDef = DefMap.find(ParentLoopHeader);
+      const auto *HeaderDefBlock = (ItHeaderDef == DefMap.end()) ? nullptr : ItHeaderDef->second;
+
       LLVM_DEBUG(printDefs(dbgs()));
-      assert(HeaderDefBlock && "no definition in header of carrying loop");
+      assert(HeaderDefBlock && "no definition at header of carrying loop");
 
       for (const auto *ExitBlock : ReachedLoopExits) {
         auto ItExitDef = DefMap.find(ExitBlock);
@@ -339,19 +344,10 @@ const ConstBlockSet &SyncDependenceAnalysis::join_blocks(const Loop &Loop) {
     return *ItCached->second;
   }
 
-  // dont propagte beyond the immediate post dom of the loop
-  const auto *PdNode = PDT.getNode(const_cast<BasicBlock *>(Loop.getHeader()));
-  const auto *IpdNode = PdNode->getIDom();
-  const auto *PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr;
-  while (PdBoundBlock && Loop.contains(PdBoundBlock)) {
-    IpdNode = IpdNode->getIDom();
-    PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr;
-  }
-
   // compute all join points
   DivergencePropagator Propagator{FuncRPOT, DT, PDT, LI};
   auto JoinBlocks = Propagator.computeJoinPoints<const LoopExitVec &>(
-      *Loop.getHeader(), LoopExits, Loop.getParentLoop(), PdBoundBlock);
+      *Loop.getHeader(), LoopExits, Loop.getParentLoop());
 
   auto ItInserted = CachedLoopExitJoins.emplace(&Loop, std::move(JoinBlocks));
   assert(ItInserted.second);
@@ -370,16 +366,11 @@ SyncDependenceAnalysis::join_blocks(const Instruction &Term) {
   if (ItCached != CachedBranchJoins.end())
     return *ItCached->second;
 
-  // dont propagate beyond the immediate post dominator of the branch
-  const auto *PdNode = PDT.getNode(const_cast<BasicBlock *>(Term.getParent()));
-  const auto *IpdNode = PdNode->getIDom();
-  const auto *PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr;
-
   // compute all join points
   DivergencePropagator Propagator{FuncRPOT, DT, PDT, LI};
   const auto &TermBlock = *Term.getParent();
   auto JoinBlocks = Propagator.computeJoinPoints<succ_const_range>(
-      TermBlock, successors(Term.getParent()), LI.getLoopFor(&TermBlock), PdBoundBlock);
+      TermBlock, successors(Term.getParent()), LI.getLoopFor(&TermBlock));
 
   auto ItInserted = CachedBranchJoins.emplace(&Term, std::move(JoinBlocks));
   assert(ItInserted.second);
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index bce2c700de989..f3d20ce984dbd 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -9,6 +9,7 @@
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TargetTransformInfoImpl.h"
 #include "llvm/IR/CallSite.h"
+#include "llvm/IR/CFG.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
@@ -59,11 +60,7 @@ bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
   SmallVector<BasicBlock *, 4> ExitingBlocks;
   L->getExitingBlocks(ExitingBlocks);
 
-  for (SmallVectorImpl<BasicBlock *>::iterator I = ExitingBlocks.begin(),
-                                               IE = ExitingBlocks.end();
-       I != IE; ++I) {
-    BasicBlock *BB = *I;
-
+  for (BasicBlock *BB : ExitingBlocks) {
     // If we pass the updated counter back through a phi, we need to know
     // which latch the updated value will be coming from.
     if (!L->isLoopLatch(BB)) {
@@ -97,13 +94,11 @@ bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
     // For this to be true, we must dominate all blocks with backedges. Such
     // blocks are in-loop predecessors to the header block.
     bool NotAlways = false;
-    for (pred_iterator PI = pred_begin(L->getHeader()),
-                       PIE = pred_end(L->getHeader());
-         PI != PIE; ++PI) {
-      if (!L->contains(*PI))
+    for (BasicBlock *Pred : predecessors(L->getHeader())) {
+      if (!L->contains(Pred))
         continue;
 
-      if (!DT.dominates(*I, *PI)) {
+      if (!DT.dominates(BB, Pred)) {
         NotAlways = true;
         break;
       }
@@ -127,7 +122,7 @@ bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE,
 
     // Note that this block may not be the loop latch block, even if the loop
     // has a latch block.
-    ExitBlock = *I;
+    ExitBlock = BB;
     ExitCount = EC;
     break;
   }
@@ -302,12 +297,11 @@ bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const {
 }
 
 bool TargetTransformInfo::isLegalNTStore(Type *DataType,
-                                         llvm::Align Alignment) const {
+                                         Align Alignment) const {
   return TTIImpl->isLegalNTStore(DataType, Alignment);
 }
 
-bool TargetTransformInfo::isLegalNTLoad(Type *DataType,
-                                        llvm::Align Alignment) const {
+bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
   return TTIImpl->isLegalNTLoad(DataType, Alignment);
 }
 
@@ -369,14 +363,6 @@ bool TargetTransformInfo::isTypeLegal(Type *Ty) const {
   return TTIImpl->isTypeLegal(Ty);
 }
 
-unsigned TargetTransformInfo::getJumpBufAlignment() const {
-  return TTIImpl->getJumpBufAlignment();
-}
-
-unsigned TargetTransformInfo::getJumpBufSize() const {
-  return TTIImpl->getJumpBufSize();
-}
-
 bool TargetTransformInfo::shouldBuildLookupTables() const {
   return TTIImpl->shouldBuildLookupTables();
 }
diff --git a/llvm/lib/Analysis/VFABIDemangling.cpp b/llvm/lib/Analysis/VFABIDemangling.cpp
new file mode 100644
index 0000000000000..6fd8ae63f5f07
--- /dev/null
+++ b/llvm/lib/Analysis/VFABIDemangling.cpp
@@ -0,0 +1,418 @@
+//===- VFABIDemangling.cpp - Vector Function ABI demangling utilities. ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/VectorUtils.h"
+
+using namespace llvm;
+
+namespace {
+/// Utilities for the Vector Function ABI name parser.
+
+/// Return types for the parser functions.
+enum class ParseRet {
+  OK,   // Found.
+  None, // Not found.
+  Error // Syntax error.
+};
+
+/// Extracts the `<isa>` information from the mangled string, and
+/// sets the `ISA` accordingly.
+ParseRet tryParseISA(StringRef &MangledName, VFISAKind &ISA) {
+  if (MangledName.empty())
+    return ParseRet::Error;
+
+  ISA = StringSwitch<VFISAKind>(MangledName.take_front(1))
+            .Case("n", VFISAKind::AdvancedSIMD)
+            .Case("s", VFISAKind::SVE)
+            .Case("b", VFISAKind::SSE)
+            .Case("c", VFISAKind::AVX)
+            .Case("d", VFISAKind::AVX2)
+            .Case("e", VFISAKind::AVX512)
+            .Default(VFISAKind::Unknown);
+
+  MangledName = MangledName.drop_front(1);
+
+  return ParseRet::OK;
+}
+
+/// Extracts the `<mask>` information from the mangled string, and
+/// sets `IsMasked` accordingly. The input string `MangledName` is
+/// left unmodified.
+ParseRet tryParseMask(StringRef &MangledName, bool &IsMasked) {
+  if (MangledName.consume_front("M")) {
+    IsMasked = true;
+    return ParseRet::OK;
+  }
+
+  if (MangledName.consume_front("N")) {
+    IsMasked = false;
+    return ParseRet::OK;
+  }
+
+  return ParseRet::Error;
+}
+
+/// Extract the `<vlen>` information from the mangled string, and
+/// sets `VF` accordingly. A `<vlen> == "x"` token is interpreted as a scalable
+/// vector length. On success, the `<vlen>` token is removed from
+/// the input string `ParseString`.
+///
+ParseRet tryParseVLEN(StringRef &ParseString, unsigned &VF, bool &IsScalable) {
+  if (ParseString.consume_front("x")) {
+    VF = 0;
+    IsScalable = true;
+    return ParseRet::OK;
+  }
+
+  if (ParseString.consumeInteger(10, VF))
+    return ParseRet::Error;
+
+  IsScalable = false;
+  return ParseRet::OK;
+}
+
+/// The function looks for the following strings at the beginning of
+/// the input string `ParseString`:
+///
+///  <token> <number>
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `Pos` to
+/// <number>, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+///
+/// The function expects <token> to be one of "ls", "Rs", "Us" or
+/// "Ls".
+ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString,
+                                            VFParamKind &PKind, int &Pos,
+                                            const StringRef Token) {
+  if (ParseString.consume_front(Token)) {
+    PKind = VFABI::getVFParamKindFromString(Token);
+    if (ParseString.consumeInteger(10, Pos))
+      return ParseRet::Error;
+    return ParseRet::OK;
+  }
+
+  return ParseRet::None;
+}
+
+/// The function looks for the following stringt at the beginning of
+/// the input string `ParseString`:
+///
+///  <token> <number>
+///
+/// <token> is one of "ls", "Rs", "Us" or "Ls".
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `StepOrPos` to
+/// <number>, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+ParseRet tryParseLinearWithRuntimeStep(StringRef &ParseString,
+                                       VFParamKind &PKind, int &StepOrPos) {
+  ParseRet Ret;
+
+  // "ls" <RuntimeStepPos>
+  Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "ls");
+  if (Ret != ParseRet::None)
+    return Ret;
+
+  // "Rs" <RuntimeStepPos>
+  Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Rs");
+  if (Ret != ParseRet::None)
+    return Ret;
+
+  // "Ls" <RuntimeStepPos>
+  Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Ls");
+  if (Ret != ParseRet::None)
+    return Ret;
+
+  // "Us" <RuntimeStepPos>
+  Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Us");
+  if (Ret != ParseRet::None)
+    return Ret;
+
+  return ParseRet::None;
+}
+
+/// The function looks for the following strings at the beginning of
+/// the input string `ParseString`:
+///
+///  <token> {"n"} <number>
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `LinearStep` to
+/// <number>, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+///
+/// The function expects <token> to be one of "l", "R", "U" or
+/// "L".
+ParseRet tryParseCompileTimeLinearToken(StringRef &ParseString,
+                                        VFParamKind &PKind, int &LinearStep,
+                                        const StringRef Token) {
+  if (ParseString.consume_front(Token)) {
+    PKind = VFABI::getVFParamKindFromString(Token);
+    const bool Negate = ParseString.consume_front("n");
+    if (ParseString.consumeInteger(10, LinearStep))
+      LinearStep = 1;
+    if (Negate)
+      LinearStep *= -1;
+    return ParseRet::OK;
+  }
+
+  return ParseRet::None;
+}
+
+/// The function looks for the following strings at the beginning of
+/// the input string `ParseString`:
+///
+/// ["l" | "R" | "U" | "L"] {"n"} <number>
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `LinearStep` to
+/// <number>, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+ParseRet tryParseLinearWithCompileTimeStep(StringRef &ParseString,
+                                           VFParamKind &PKind, int &StepOrPos) {
+  // "l" {"n"} <CompileTimeStep>
+  if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "l") ==
+      ParseRet::OK)
+    return ParseRet::OK;
+
+  // "R" {"n"} <CompileTimeStep>
+  if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "R") ==
+      ParseRet::OK)
+    return ParseRet::OK;
+
+  // "L" {"n"} <CompileTimeStep>
+  if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "L") ==
+      ParseRet::OK)
+    return ParseRet::OK;
+
+  // "U" {"n"} <CompileTimeStep>
+  if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "U") ==
+      ParseRet::OK)
+    return ParseRet::OK;
+
+  return ParseRet::None;
+}
+
+/// The function looks for the following strings at the beginning of
+/// the input string `ParseString`:
+///
+/// "u" <number>
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `Pos` to
+/// <number>, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+ParseRet tryParseUniform(StringRef &ParseString, VFParamKind &PKind, int &Pos) {
+  // "u" <Pos>
+  const char *UniformToken = "u";
+  if (ParseString.consume_front(UniformToken)) {
+    PKind = VFABI::getVFParamKindFromString(UniformToken);
+    if (ParseString.consumeInteger(10, Pos))
+      return ParseRet::Error;
+
+    return ParseRet::OK;
+  }
+  return ParseRet::None;
+}
+
+/// Looks into the <parameters> part of the mangled name in search
+/// for valid paramaters at the beginning of the string
+/// `ParseString`.
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `StepOrPos`
+/// accordingly, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+ParseRet tryParseParameter(StringRef &ParseString, VFParamKind &PKind,
+                           int &StepOrPos) {
+  if (ParseString.consume_front("v")) {
+    PKind = VFParamKind::Vector;
+    StepOrPos = 0;
+    return ParseRet::OK;
+  }
+
+  const ParseRet HasLinearRuntime =
+      tryParseLinearWithRuntimeStep(ParseString, PKind, StepOrPos);
+  if (HasLinearRuntime != ParseRet::None)
+    return HasLinearRuntime;
+
+  const ParseRet HasLinearCompileTime =
+      tryParseLinearWithCompileTimeStep(ParseString, PKind, StepOrPos);
+  if (HasLinearCompileTime != ParseRet::None)
+    return HasLinearCompileTime;
+
+  const ParseRet HasUniform = tryParseUniform(ParseString, PKind, StepOrPos);
+  if (HasUniform != ParseRet::None)
+    return HasUniform;
+
+  return ParseRet::None;
+}
+
+/// Looks into the <parameters> part of the mangled name in search
+/// of a valid 'aligned' clause. The function should be invoked
+/// after parsing a parameter via `tryParseParameter`.
+///
+/// On success, it removes the parsed parameter from `ParseString`,
+/// sets `PKind` to the correspondent enum value, sets `StepOrPos`
+/// accordingly, and return success.  On a syntax error, it return a
+/// parsing error. If nothing is parsed, it returns None.
+ParseRet tryParseAlign(StringRef &ParseString, Align &Alignment) {
+  uint64_t Val;
+  //    "a" <number>
+  if (ParseString.consume_front("a")) {
+    if (ParseString.consumeInteger(10, Val))
+      return ParseRet::Error;
+
+    if (!isPowerOf2_64(Val))
+      return ParseRet::Error;
+
+    Alignment = Align(Val);
+
+    return ParseRet::OK;
+  }
+
+  return ParseRet::None;
+}
+} // namespace
+
+// Format of the ABI name:
+// _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]
+Optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName) {
+  // Assume there is no custom name <redirection>, and therefore the
+  // vector name consists of
+  // _ZGV<isa><mask><vlen><parameters>_<scalarname>.
+  StringRef VectorName = MangledName;
+
+  // Parse the fixed size part of the manled name
+  if (!MangledName.consume_front("_ZGV"))
+    return None;
+
+  // Extract ISA. An unknow ISA is also supported, so we accept all
+  // values.
+  VFISAKind ISA;
+  if (tryParseISA(MangledName, ISA) != ParseRet::OK)
+    return None;
+
+  // Extract <mask>.
+  bool IsMasked;
+  if (tryParseMask(MangledName, IsMasked) != ParseRet::OK)
+    return None;
+
+  // Parse the variable size, starting from <vlen>.
+  unsigned VF;
+  bool IsScalable;
+  if (tryParseVLEN(MangledName, VF, IsScalable) != ParseRet::OK)
+    return None;
+
+  // Parse the <parameters>.
+  ParseRet ParamFound;
+  SmallVector<VFParameter, 8> Parameters;
+  do {
+    const unsigned ParameterPos = Parameters.size();
+    VFParamKind PKind;
+    int StepOrPos;
+    ParamFound = tryParseParameter(MangledName, PKind, StepOrPos);
+
+    // Bail off if there is a parsing error in the parsing of the parameter.
+    if (ParamFound == ParseRet::Error)
+      return None;
+
+    if (ParamFound == ParseRet::OK) {
+      Align Alignment;
+      // Look for the alignment token "a <number>".
+      const ParseRet AlignFound = tryParseAlign(MangledName, Alignment);
+      // Bail off if there is a syntax error in the align token.
+      if (AlignFound == ParseRet::Error)
+        return None;
+
+      // Add the parameter.
+      Parameters.push_back({ParameterPos, PKind, StepOrPos, Alignment});
+    }
+  } while (ParamFound == ParseRet::OK);
+
+  // A valid MangledName mus have at least one valid entry in the
+  // <parameters>.
+  if (Parameters.empty())
+    return None;
+
+  // Check for the <scalarname> and the optional <redirection>, which
+  // are separated from the prefix with "_"
+  if (!MangledName.consume_front("_"))
+    return None;
+
+  // The rest of the string must be in the format:
+  // <scalarname>[(<redirection>)]
+  const StringRef ScalarName =
+      MangledName.take_while([](char In) { return In != '('; });
+
+  if (ScalarName.empty())
+    return None;
+
+  // Reduce MangledName to [(<redirection>)].
+  MangledName = MangledName.ltrim(ScalarName);
+  // Find the optional custom name redirection.
+  if (MangledName.consume_front("(")) {
+    if (!MangledName.consume_back(")"))
+      return None;
+    // Update the vector variant with the one specified by the user.
+    VectorName = MangledName;
+    // If the vector name is missing, bail out.
+    if (VectorName.empty())
+      return None;
+  }
+
+  // When <mask> is "M", we need to add a parameter that is used as
+  // global predicate for the function.
+  if (IsMasked) {
+    const unsigned Pos = Parameters.size();
+    Parameters.push_back({Pos, VFParamKind::GlobalPredicate});
+  }
+
+  // Asserts for parameters of type `VFParamKind::GlobalPredicate`, as
+  // prescribed by the Vector Function ABI specifications supported by
+  // this parser:
+  // 1. Uniqueness.
+  // 2. Must be the last in the parameter list.
+  const auto NGlobalPreds = std::count_if(
+      Parameters.begin(), Parameters.end(), [](const VFParameter PK) {
+        return PK.ParamKind == VFParamKind::GlobalPredicate;
+      });
+  assert(NGlobalPreds < 2 && "Cannot have more than one global predicate.");
+  if (NGlobalPreds)
+    assert(Parameters.back().ParamKind == VFParamKind::GlobalPredicate &&
+           "The global predicate must be the last parameter");
+
+  const VFShape Shape({VF, IsScalable, ISA, Parameters});
+  return VFInfo({Shape, ScalarName, VectorName});
+}
+
+VFParamKind VFABI::getVFParamKindFromString(const StringRef Token) {
+  const VFParamKind ParamKind = StringSwitch<VFParamKind>(Token)
+                                    .Case("v", VFParamKind::Vector)
+                                    .Case("l", VFParamKind::OMP_Linear)
+                                    .Case("R", VFParamKind::OMP_LinearRef)
+                                    .Case("L", VFParamKind::OMP_LinearVal)
+                                    .Case("U", VFParamKind::OMP_LinearUVal)
+                                    .Case("ls", VFParamKind::OMP_LinearPos)
+                                    .Case("Ls", VFParamKind::OMP_LinearValPos)
+                                    .Case("Rs", VFParamKind::OMP_LinearRefPos)
+                                    .Case("Us", VFParamKind::OMP_LinearUValPos)
+                                    .Case("u", VFParamKind::OMP_Uniform)
+                                    .Default(VFParamKind::Unknown);
+
+  if (ParamKind != VFParamKind::Unknown)
+    return ParamKind;
+
+  // This function should never be invoked with an invalid input.
+  llvm_unreachable("This fuction should be invoken only on parameters"
+                   " that have a textual representation in the mangled name"
+                   " of the Vector Function ABI");
+}
diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index 55c3e0228b245..f0b87810ef9a1 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1055,7 +1055,7 @@ static void computeKnownBitsFromOperator(const Operator *I, KnownBits &Known,
     break;
   }
   case Instruction::Select: {
-    const Value *LHS, *RHS;
+    const Value *LHS = nullptr, *RHS = nullptr;
     SelectPatternFlavor SPF = matchSelectPattern(I, LHS, RHS).Flavor;
     if (SelectPatternResult::isMinOrMax(SPF)) {
       computeKnownBits(RHS, Known, Depth + 1, Q);
@@ -2307,7 +2307,7 @@ static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
          cast<Operator>(Select)->getOpcode() == Instruction::Select &&
          "Input should be a Select!");
 
-  const Value *LHS, *RHS, *LHS2, *RHS2;
+  const Value *LHS = nullptr, *RHS = nullptr;
   SelectPatternFlavor SPF = matchSelectPattern(Select, LHS, RHS).Flavor;
   if (SPF != SPF_SMAX && SPF != SPF_SMIN)
     return false;
@@ -2315,6 +2315,7 @@ static bool isSignedMinMaxClamp(const Value *Select, const Value *&In,
   if (!match(RHS, m_APInt(CLow)))
     return false;
 
+  const Value *LHS2 = nullptr, *RHS2 = nullptr;
   SelectPatternFlavor SPF2 = matchSelectPattern(LHS, LHS2, RHS2).Flavor;
   if (getInverseMinMaxFlavor(SPF) != SPF2)
     return false;
@@ -2391,253 +2392,256 @@ static unsigned ComputeNumSignBitsImpl(const Value *V, unsigned Depth,
   if (Depth == MaxDepth)
     return 1;  // Limit search depth.
 
-  const Operator *U = dyn_cast<Operator>(V);
-  switch (Operator::getOpcode(V)) {
-  default: break;
-  case Instruction::SExt:
-    Tmp = TyBits - U->getOperand(0)->getType()->getScalarSizeInBits();
-    return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q) + Tmp;
+  if (auto *U = dyn_cast<Operator>(V)) {
+    switch (Operator::getOpcode(V)) {
+    default: break;
+    case Instruction::SExt:
+      Tmp = TyBits - U->getOperand(0)->getType()->getScalarSizeInBits();
+      return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q) + Tmp;
 
-  case Instruction::SDiv: {
-    const APInt *Denominator;
-    // sdiv X, C -> adds log(C) sign bits.
-    if (match(U->getOperand(1), m_APInt(Denominator))) {
+    case Instruction::SDiv: {
+      const APInt *Denominator;
+      // sdiv X, C -> adds log(C) sign bits.
+      if (match(U->getOperand(1), m_APInt(Denominator))) {
 
-      // Ignore non-positive denominator.
-      if (!Denominator->isStrictlyPositive())
-        break;
+        // Ignore non-positive denominator.
+        if (!Denominator->isStrictlyPositive())
+          break;
 
-      // Calculate the incoming numerator bits.
-      unsigned NumBits = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+        // Calculate the incoming numerator bits.
+        unsigned NumBits = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
 
-      // Add floor(log(C)) bits to the numerator bits.
-      return std::min(TyBits, NumBits + Denominator->logBase2());
+        // Add floor(log(C)) bits to the numerator bits.
+        return std::min(TyBits, NumBits + Denominator->logBase2());
+      }
+      break;
     }
-    break;
-  }
 
-  case Instruction::SRem: {
-    const APInt *Denominator;
-    // srem X, C -> we know that the result is within [-C+1,C) when C is a
-    // positive constant.  This let us put a lower bound on the number of sign
-    // bits.
-    if (match(U->getOperand(1), m_APInt(Denominator))) {
-
-      // Ignore non-positive denominator.
-      if (!Denominator->isStrictlyPositive())
-        break;
+    case Instruction::SRem: {
+      const APInt *Denominator;
+      // srem X, C -> we know that the result is within [-C+1,C) when C is a
+      // positive constant.  This let us put a lower bound on the number of sign
+      // bits.
+      if (match(U->getOperand(1), m_APInt(Denominator))) {
 
-      // Calculate the incoming numerator bits. SRem by a positive constant
-      // can't lower the number of sign bits.
-      unsigned NumrBits =
-          ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-
-      // Calculate the leading sign bit constraints by examining the
-      // denominator.  Given that the denominator is positive, there are two
-      // cases:
-      //
-      //  1. the numerator is positive.  The result range is [0,C) and [0,C) u<
-      //     (1 << ceilLogBase2(C)).
-      //
-      //  2. the numerator is negative.  Then the result range is (-C,0] and
-      //     integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
-      //
-      // Thus a lower bound on the number of sign bits is `TyBits -
-      // ceilLogBase2(C)`.
+        // Ignore non-positive denominator.
+        if (!Denominator->isStrictlyPositive())
+          break;
 
-      unsigned ResBits = TyBits - Denominator->ceilLogBase2();
-      return std::max(NumrBits, ResBits);
+        // Calculate the incoming numerator bits. SRem by a positive constant
+        // can't lower the number of sign bits.
+        unsigned NumrBits = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+
+        // Calculate the leading sign bit constraints by examining the
+        // denominator.  Given that the denominator is positive, there are two
+        // cases:
+        //
+        //  1. the numerator is positive. The result range is [0,C) and [0,C) u<
+        //     (1 << ceilLogBase2(C)).
+        //
+        //  2. the numerator is negative. Then the result range is (-C,0] and
+        //     integers in (-C,0] are either 0 or >u (-1 << ceilLogBase2(C)).
+        //
+        // Thus a lower bound on the number of sign bits is `TyBits -
+        // ceilLogBase2(C)`.
+
+        unsigned ResBits = TyBits - Denominator->ceilLogBase2();
+        return std::max(NumrBits, ResBits);
+      }
+      break;
     }
-    break;
-  }
 
-  case Instruction::AShr: {
-    Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-    // ashr X, C   -> adds C sign bits.  Vectors too.
-    const APInt *ShAmt;
-    if (match(U->getOperand(1), m_APInt(ShAmt))) {
-      if (ShAmt->uge(TyBits))
-        break;  // Bad shift.
-      unsigned ShAmtLimited = ShAmt->getZExtValue();
-      Tmp += ShAmtLimited;
-      if (Tmp > TyBits) Tmp = TyBits;
-    }
-    return Tmp;
-  }
-  case Instruction::Shl: {
-    const APInt *ShAmt;
-    if (match(U->getOperand(1), m_APInt(ShAmt))) {
-      // shl destroys sign bits.
+    case Instruction::AShr: {
       Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-      if (ShAmt->uge(TyBits) ||      // Bad shift.
-          ShAmt->uge(Tmp)) break;    // Shifted all sign bits out.
-      Tmp2 = ShAmt->getZExtValue();
-      return Tmp - Tmp2;
+      // ashr X, C   -> adds C sign bits.  Vectors too.
+      const APInt *ShAmt;
+      if (match(U->getOperand(1), m_APInt(ShAmt))) {
+        if (ShAmt->uge(TyBits))
+          break; // Bad shift.
+        unsigned ShAmtLimited = ShAmt->getZExtValue();
+        Tmp += ShAmtLimited;
+        if (Tmp > TyBits) Tmp = TyBits;
+      }
+      return Tmp;
     }
-    break;
-  }
-  case Instruction::And:
-  case Instruction::Or:
-  case Instruction::Xor:    // NOT is handled here.
-    // Logical binary ops preserve the number of sign bits at the worst.
-    Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-    if (Tmp != 1) {
-      Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
-      FirstAnswer = std::min(Tmp, Tmp2);
-      // We computed what we know about the sign bits as our first
-      // answer. Now proceed to the generic code that uses
-      // computeKnownBits, and pick whichever answer is better.
+    case Instruction::Shl: {
+      const APInt *ShAmt;
+      if (match(U->getOperand(1), m_APInt(ShAmt))) {
+        // shl destroys sign bits.
+        Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+        if (ShAmt->uge(TyBits) ||   // Bad shift.
+            ShAmt->uge(Tmp)) break; // Shifted all sign bits out.
+        Tmp2 = ShAmt->getZExtValue();
+        return Tmp - Tmp2;
+      }
+      break;
     }
-    break;
-
-  case Instruction::Select: {
-    // If we have a clamp pattern, we know that the number of sign bits will be
-    // the minimum of the clamp min/max range.
-    const Value *X;
-    const APInt *CLow, *CHigh;
-    if (isSignedMinMaxClamp(U, X, CLow, CHigh))
-      return std::min(CLow->getNumSignBits(), CHigh->getNumSignBits());
-
-    Tmp = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
-    if (Tmp == 1) break;
-    Tmp2 = ComputeNumSignBits(U->getOperand(2), Depth + 1, Q);
-    return std::min(Tmp, Tmp2);
-  }
-
-  case Instruction::Add:
-    // Add can have at most one carry bit.  Thus we know that the output
-    // is, at worst, one more bit than the inputs.
-    Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-    if (Tmp == 1) break;
-
-    // Special case decrementing a value (ADD X, -1):
-    if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1)))
-      if (CRHS->isAllOnesValue()) {
-        KnownBits Known(TyBits);
-        computeKnownBits(U->getOperand(0), Known, Depth + 1, Q);
-
-        // If the input is known to be 0 or 1, the output is 0/-1, which is all
-        // sign bits set.
-        if ((Known.Zero | 1).isAllOnesValue())
-          return TyBits;
-
-        // If we are subtracting one from a positive number, there is no carry
-        // out of the result.
-        if (Known.isNonNegative())
-          return Tmp;
+    case Instruction::And:
+    case Instruction::Or:
+    case Instruction::Xor: // NOT is handled here.
+      // Logical binary ops preserve the number of sign bits at the worst.
+      Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      if (Tmp != 1) {
+        Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
+        FirstAnswer = std::min(Tmp, Tmp2);
+        // We computed what we know about the sign bits as our first
+        // answer. Now proceed to the generic code that uses
+        // computeKnownBits, and pick whichever answer is better.
       }
+      break;
 
-    Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
-    if (Tmp2 == 1) break;
-    return std::min(Tmp, Tmp2)-1;
+    case Instruction::Select: {
+      // If we have a clamp pattern, we know that the number of sign bits will
+      // be the minimum of the clamp min/max range.
+      const Value *X;
+      const APInt *CLow, *CHigh;
+      if (isSignedMinMaxClamp(U, X, CLow, CHigh))
+        return std::min(CLow->getNumSignBits(), CHigh->getNumSignBits());
+
+      Tmp = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
+      if (Tmp == 1) break;
+      Tmp2 = ComputeNumSignBits(U->getOperand(2), Depth + 1, Q);
+      return std::min(Tmp, Tmp2);
+    }
 
-  case Instruction::Sub:
-    Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
-    if (Tmp2 == 1) break;
-
-    // Handle NEG.
-    if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0)))
-      if (CLHS->isNullValue()) {
-        KnownBits Known(TyBits);
-        computeKnownBits(U->getOperand(1), Known, Depth + 1, Q);
-        // If the input is known to be 0 or 1, the output is 0/-1, which is all
-        // sign bits set.
-        if ((Known.Zero | 1).isAllOnesValue())
-          return TyBits;
-
-        // If the input is known to be positive (the sign bit is known clear),
-        // the output of the NEG has the same number of sign bits as the input.
-        if (Known.isNonNegative())
-          return Tmp2;
-
-        // Otherwise, we treat this like a SUB.
-      }
+    case Instruction::Add:
+      // Add can have at most one carry bit.  Thus we know that the output
+      // is, at worst, one more bit than the inputs.
+      Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      if (Tmp == 1) break;
+
+      // Special case decrementing a value (ADD X, -1):
+      if (const auto *CRHS = dyn_cast<Constant>(U->getOperand(1)))
+        if (CRHS->isAllOnesValue()) {
+          KnownBits Known(TyBits);
+          computeKnownBits(U->getOperand(0), Known, Depth + 1, Q);
+
+          // If the input is known to be 0 or 1, the output is 0/-1, which is
+          // all sign bits set.
+          if ((Known.Zero | 1).isAllOnesValue())
+            return TyBits;
+
+          // If we are subtracting one from a positive number, there is no carry
+          // out of the result.
+          if (Known.isNonNegative())
+            return Tmp;
+        }
 
-    // Sub can have at most one carry bit.  Thus we know that the output
-    // is, at worst, one more bit than the inputs.
-    Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-    if (Tmp == 1) break;
-    return std::min(Tmp, Tmp2)-1;
+      Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
+      if (Tmp2 == 1) break;
+      return std::min(Tmp, Tmp2) - 1;
 
-  case Instruction::Mul: {
-    // The output of the Mul can be at most twice the valid bits in the inputs.
-    unsigned SignBitsOp0 = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-    if (SignBitsOp0 == 1) break;
-    unsigned SignBitsOp1 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
-    if (SignBitsOp1 == 1) break;
-    unsigned OutValidBits =
-        (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
-    return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
-  }
+    case Instruction::Sub:
+      Tmp2 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
+      if (Tmp2 == 1) break;
+
+      // Handle NEG.
+      if (const auto *CLHS = dyn_cast<Constant>(U->getOperand(0)))
+        if (CLHS->isNullValue()) {
+          KnownBits Known(TyBits);
+          computeKnownBits(U->getOperand(1), Known, Depth + 1, Q);
+          // If the input is known to be 0 or 1, the output is 0/-1, which is
+          // all sign bits set.
+          if ((Known.Zero | 1).isAllOnesValue())
+            return TyBits;
+
+          // If the input is known to be positive (the sign bit is known clear),
+          // the output of the NEG has the same number of sign bits as the
+          // input.
+          if (Known.isNonNegative())
+            return Tmp2;
+
+          // Otherwise, we treat this like a SUB.
+        }
 
-  case Instruction::PHI: {
-    const PHINode *PN = cast<PHINode>(U);
-    unsigned NumIncomingValues = PN->getNumIncomingValues();
-    // Don't analyze large in-degree PHIs.
-    if (NumIncomingValues > 4) break;
-    // Unreachable blocks may have zero-operand PHI nodes.
-    if (NumIncomingValues == 0) break;
-
-    // Take the minimum of all incoming values.  This can't infinitely loop
-    // because of our depth threshold.
-    Tmp = ComputeNumSignBits(PN->getIncomingValue(0), Depth + 1, Q);
-    for (unsigned i = 1, e = NumIncomingValues; i != e; ++i) {
-      if (Tmp == 1) return Tmp;
-      Tmp = std::min(
-          Tmp, ComputeNumSignBits(PN->getIncomingValue(i), Depth + 1, Q));
+      // Sub can have at most one carry bit.  Thus we know that the output
+      // is, at worst, one more bit than the inputs.
+      Tmp = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      if (Tmp == 1) break;
+      return std::min(Tmp, Tmp2) - 1;
+
+    case Instruction::Mul: {
+      // The output of the Mul can be at most twice the valid bits in the
+      // inputs.
+      unsigned SignBitsOp0 = ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+      if (SignBitsOp0 == 1) break;
+      unsigned SignBitsOp1 = ComputeNumSignBits(U->getOperand(1), Depth + 1, Q);
+      if (SignBitsOp1 == 1) break;
+      unsigned OutValidBits =
+          (TyBits - SignBitsOp0 + 1) + (TyBits - SignBitsOp1 + 1);
+      return OutValidBits > TyBits ? 1 : TyBits - OutValidBits + 1;
     }
-    return Tmp;
-  }
 
-  case Instruction::Trunc:
-    // FIXME: it's tricky to do anything useful for this, but it is an important
-    // case for targets like X86.
-    break;
+    case Instruction::PHI: {
+      const PHINode *PN = cast<PHINode>(U);
+      unsigned NumIncomingValues = PN->getNumIncomingValues();
+      // Don't analyze large in-degree PHIs.
+      if (NumIncomingValues > 4) break;
+      // Unreachable blocks may have zero-operand PHI nodes.
+      if (NumIncomingValues == 0) break;
+
+      // Take the minimum of all incoming values.  This can't infinitely loop
+      // because of our depth threshold.
+      Tmp = ComputeNumSignBits(PN->getIncomingValue(0), Depth + 1, Q);
+      for (unsigned i = 1, e = NumIncomingValues; i != e; ++i) {
+        if (Tmp == 1) return Tmp;
+        Tmp = std::min(
+            Tmp, ComputeNumSignBits(PN->getIncomingValue(i), Depth + 1, Q));
+      }
+      return Tmp;
+    }
 
-  case Instruction::ExtractElement:
-    // Look through extract element. At the moment we keep this simple and skip
-    // tracking the specific element. But at least we might find information
-    // valid for all elements of the vector (for example if vector is sign
-    // extended, shifted, etc).
-    return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
-
-  case Instruction::ShuffleVector: {
-    // TODO: This is copied almost directly from the SelectionDAG version of
-    //       ComputeNumSignBits. It would be better if we could share common
-    //       code. If not, make sure that changes are translated to the DAG.
-
-    // Collect the minimum number of sign bits that are shared by every vector
-    // element referenced by the shuffle.
-    auto *Shuf = cast<ShuffleVectorInst>(U);
-    int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
-    int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
-    APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
-    for (int i = 0; i != NumMaskElts; ++i) {
-      int M = Shuf->getMaskValue(i);
-      assert(M < NumElts * 2 && "Invalid shuffle mask constant");
-      // For undef elements, we don't know anything about the common state of
-      // the shuffle result.
-      if (M == -1)
-        return 1;
-      if (M < NumElts)
-        DemandedLHS.setBit(M % NumElts);
-      else
-        DemandedRHS.setBit(M % NumElts);
+    case Instruction::Trunc:
+      // FIXME: it's tricky to do anything useful for this, but it is an
+      // important case for targets like X86.
+      break;
+
+    case Instruction::ExtractElement:
+      // Look through extract element. At the moment we keep this simple and
+      // skip tracking the specific element. But at least we might find
+      // information valid for all elements of the vector (for example if vector
+      // is sign extended, shifted, etc).
+      return ComputeNumSignBits(U->getOperand(0), Depth + 1, Q);
+
+    case Instruction::ShuffleVector: {
+      // TODO: This is copied almost directly from the SelectionDAG version of
+      //       ComputeNumSignBits. It would be better if we could share common
+      //       code. If not, make sure that changes are translated to the DAG.
+
+      // Collect the minimum number of sign bits that are shared by every vector
+      // element referenced by the shuffle.
+      auto *Shuf = cast<ShuffleVectorInst>(U);
+      int NumElts = Shuf->getOperand(0)->getType()->getVectorNumElements();
+      int NumMaskElts = Shuf->getMask()->getType()->getVectorNumElements();
+      APInt DemandedLHS(NumElts, 0), DemandedRHS(NumElts, 0);
+      for (int i = 0; i != NumMaskElts; ++i) {
+        int M = Shuf->getMaskValue(i);
+        assert(M < NumElts * 2 && "Invalid shuffle mask constant");
+        // For undef elements, we don't know anything about the common state of
+        // the shuffle result.
+        if (M == -1)
+          return 1;
+        if (M < NumElts)
+          DemandedLHS.setBit(M % NumElts);
+        else
+          DemandedRHS.setBit(M % NumElts);
+      }
+      Tmp = std::numeric_limits<unsigned>::max();
+      if (!!DemandedLHS)
+        Tmp = ComputeNumSignBits(Shuf->getOperand(0), Depth + 1, Q);
+      if (!!DemandedRHS) {
+        Tmp2 = ComputeNumSignBits(Shuf->getOperand(1), Depth + 1, Q);
+        Tmp = std::min(Tmp, Tmp2);
+      }
+      // If we don't know anything, early out and try computeKnownBits
+      // fall-back.
+      if (Tmp == 1)
+        break;
+      assert(Tmp <= V->getType()->getScalarSizeInBits() &&
+             "Failed to determine minimum sign bits");
+      return Tmp;
     }
-    Tmp = std::numeric_limits<unsigned>::max();
-    if (!!DemandedLHS)
-      Tmp = ComputeNumSignBits(Shuf->getOperand(0), Depth + 1, Q);
-    if (!!DemandedRHS) {
-      Tmp2 = ComputeNumSignBits(Shuf->getOperand(1), Depth + 1, Q);
-      Tmp = std::min(Tmp, Tmp2);
     }
-    // If we don't know anything, early out and try computeKnownBits fall-back.
-    if (Tmp == 1)
-      break;
-    assert(Tmp <= V->getType()->getScalarSizeInBits() &&
-           "Failed to determine minimum sign bits");
-    return Tmp;
-  }
   }
 
   // Finally, if we can prove that the top bits of the result are 0's or 1's,
@@ -2662,8 +2666,6 @@ static unsigned ComputeNumSignBitsImpl(const Value *V, unsigned Depth,
 /// through SExt instructions only if LookThroughSExt is true.
 bool llvm::ComputeMultiple(Value *V, unsigned Base, Value *&Multiple,
                            bool LookThroughSExt, unsigned Depth) {
-  const unsigned MaxDepth = 6;
-
   assert(V && "No Value?");
   assert(Depth <= MaxDepth && "Limit Search Depth");
   assert(V->getType()->isIntegerTy() && "Not integer or pointer type!");
@@ -4575,12 +4577,12 @@ static SelectPatternResult matchMinMaxOfMinMax(CmpInst::Predicate Pred,
   // TODO: Allow FP min/max with nnan/nsz.
   assert(CmpInst::isIntPredicate(Pred) && "Expected integer comparison");
 
-  Value *A, *B;
+  Value *A = nullptr, *B = nullptr;
   SelectPatternResult L = matchSelectPattern(TVal, A, B, nullptr, Depth + 1);
   if (!SelectPatternResult::isMinOrMax(L.Flavor))
     return {SPF_UNKNOWN, SPNB_NA, false};
 
-  Value *C, *D;
+  Value *C = nullptr, *D = nullptr;
   SelectPatternResult R = matchSelectPattern(FVal, C, D, nullptr, Depth + 1);
   if (L.Flavor != R.Flavor)
     return {SPF_UNKNOWN, SPNB_NA, false};
@@ -5631,7 +5633,7 @@ static void setLimitsForIntrinsic(const IntrinsicInst &II, APInt &Lower,
 
 static void setLimitsForSelectPattern(const SelectInst &SI, APInt &Lower,
                                       APInt &Upper, const InstrInfoQuery &IIQ) {
-  const Value *LHS, *RHS;
+  const Value *LHS = nullptr, *RHS = nullptr;
   SelectPatternResult R = matchSelectPattern(&SI, LHS, RHS);
   if (R.Flavor == SPF_UNKNOWN)
     return;
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index 3edb0322c1096..f54794c59e3fe 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -832,7 +832,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses(
                                     /*Assume=*/true, /*ShouldCheckWrap=*/false);
 
       const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr);
-      PointerType *PtrTy = dyn_cast<PointerType>(Ptr->getType());
+      PointerType *PtrTy = cast<PointerType>(Ptr->getType());
       uint64_t Size = DL.getTypeAllocSize(PtrTy->getElementType());
 
       // An alignment of 0 means target ABI alignment.
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index eb66a7c82b558..5ea0b7d39c170 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -5802,7 +5802,19 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB,
   case lltok::kw_extractelement: return ParseExtractElement(Inst, PFS);
   case lltok::kw_insertelement:  return ParseInsertElement(Inst, PFS);
   case lltok::kw_shufflevector:  return ParseShuffleVector(Inst, PFS);
-  case lltok::kw_phi:            return ParsePHI(Inst, PFS);
+  case lltok::kw_phi: {
+    FastMathFlags FMF = EatFastMathFlagsIfPresent();
+    int Res = ParsePHI(Inst, PFS);
+    if (Res != 0)
+      return Res;
+    if (FMF.any()) {
+      if (!Inst->getType()->isFPOrFPVectorTy())
+        return Error(Loc, "fast-math-flags specified for phi without "
+                          "floating-point scalar or vector return type");
+      Inst->setFastMathFlags(FMF);
+    }
+    return 0;
+  }
   case lltok::kw_landingpad:     return ParseLandingPad(Inst, PFS);
   // Call.
   case lltok::kw_call:     return ParseCall(Inst, PFS, CallInst::TCK_None);
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index af906071225bf..9c377552b4a36 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -3648,6 +3648,11 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit,
       break;
     }
     Record.clear();
+
+    // Upgrade data layout string.
+    std::string DL = llvm::UpgradeDataLayoutString(
+        TheModule->getDataLayoutStr(), TheModule->getTargetTriple());
+    TheModule->setDataLayout(DL);
   }
 }
 
@@ -4624,31 +4629,48 @@ Error BitcodeReader::parseFunctionBody(Function *F) {
       InstructionList.push_back(I);
       break;
     case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...]
-      if (Record.size() < 1 || ((Record.size()-1)&1))
+      if (Record.size() < 1)
         return error("Invalid record");
+      // The first record specifies the type.
       FullTy = getFullyStructuredTypeByID(Record[0]);
       Type *Ty = flattenPointerTypes(FullTy);
       if (!Ty)
         return error("Invalid record");
 
-      PHINode *PN = PHINode::Create(Ty, (Record.size()-1)/2);
+      // Phi arguments are pairs of records of [value, basic block].
+      // There is an optional final record for fast-math-flags if this phi has a
+      // floating-point type.
+      size_t NumArgs = (Record.size() - 1) / 2;
+      if ((Record.size() - 1) % 2 == 1 && !Ty->isFloatingPointTy())
+        return error("Invalid record");
+
+      PHINode *PN = PHINode::Create(Ty, NumArgs);
       InstructionList.push_back(PN);
 
-      for (unsigned i = 0, e = Record.size()-1; i != e; i += 2) {
+      for (unsigned i = 0; i != NumArgs; i++) {
         Value *V;
         // With the new function encoding, it is possible that operands have
         // negative IDs (for forward references).  Use a signed VBR
         // representation to keep the encoding small.
         if (UseRelativeIDs)
-          V = getValueSigned(Record, 1+i, NextValueNo, Ty);
+          V = getValueSigned(Record, i * 2 + 1, NextValueNo, Ty);
         else
-          V = getValue(Record, 1+i, NextValueNo, Ty);
-        BasicBlock *BB = getBasicBlock(Record[2+i]);
+          V = getValue(Record, i * 2 + 1, NextValueNo, Ty);
+        BasicBlock *BB = getBasicBlock(Record[i * 2 + 2]);
         if (!V || !BB)
           return error("Invalid record");
         PN->addIncoming(V, BB);
       }
       I = PN;
+
+      // If there are an even number of records, the final record must be FMF.
+      if (Record.size() % 2 == 0) {
+        assert(isa<FPMathOperator>(I) && "Unexpected phi type");
+        FastMathFlags FMF = getDecodedFastMathFlags(Record[Record.size() - 1]);
+        if (FMF.any())
+          I->setFastMathFlags(FMF);
+      }
+
       break;
     }
 
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index fa50ec5766a75..4da51dda8b747 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -515,7 +515,7 @@ class MetadataLoader::MetadataLoaderImpl {
       GV.getMetadata(LLVMContext::MD_dbg, MDs);
       GV.eraseMetadata(LLVMContext::MD_dbg);
       for (auto *MD : MDs)
-        if (auto *DGV = dyn_cast_or_null<DIGlobalVariable>(MD)) {
+        if (auto *DGV = dyn_cast<DIGlobalVariable>(MD)) {
           auto *DGVE = DIGlobalVariableExpression::getDistinct(
               Context, DGV, DIExpression::get(Context, {}));
           GV.addMetadata(LLVMContext::MD_dbg, *DGVE);
@@ -987,7 +987,7 @@ void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata(
   assert(ID >= MDStringRef.size() && "Unexpected lazy-loading of MDString");
   // Lookup first if the metadata hasn't already been loaded.
   if (auto *MD = MetadataList.lookup(ID)) {
-    auto *N = dyn_cast_or_null<MDNode>(MD);
+    auto *N = cast<MDNode>(MD);
     if (!N->isTemporary())
       return;
   }
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 0eea6dc87ce2f..2818e843b50b6 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -2880,6 +2880,11 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I,
       pushValueSigned(PN.getIncomingValue(i), InstID, Vals64);
       Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i)));
     }
+
+    uint64_t Flags = getOptimizationFlags(&I);
+    if (Flags != 0)
+      Vals64.push_back(Flags);
+
     // Emit a Vals64 vector and exit.
     Stream.EmitRecord(Code, Vals64, AbbrevToUse);
     Vals64.clear();
diff --git a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 18fcee6239e4a..f64b775a8b775 100644
--- a/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -790,7 +790,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
         CriticalPathSU = SU;
       }
     }
-
+    assert(CriticalPathSU && "Failed to find SUnit critical path");
     CriticalPathMI = CriticalPathSU->getInstr();
   }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index ca223f78c166c..513361e134178 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -91,10 +91,12 @@
 #include "llvm/MC/MCSectionCOFF.h"
 #include "llvm/MC/MCSectionELF.h"
 #include "llvm/MC/MCSectionMachO.h"
+#include "llvm/MC/MCSectionXCOFF.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCSymbolELF.h"
+#include "llvm/MC/MCSymbolXCOFF.h"
 #include "llvm/MC/MCTargetOptions.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/MC/SectionKind.h"
@@ -161,29 +163,28 @@ static gcp_map_type &getGCMap(void *&P) {
 
 /// getGVAlignment - Return the alignment to use for the specified global
 /// value.  This rounds up to the preferred alignment if possible and legal.
-llvm::Align AsmPrinter::getGVAlignment(const GlobalValue *GV,
-                                       const DataLayout &DL,
-                                       llvm::Align InAlign) {
-  llvm::Align Align;
+Align AsmPrinter::getGVAlignment(const GlobalValue *GV, const DataLayout &DL,
+                                 Align InAlign) {
+  Align Alignment;
   if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
-    Align = llvm::Align(DL.getPreferredAlignment(GVar));
+    Alignment = Align(DL.getPreferredAlignment(GVar));
 
   // If InAlign is specified, round it to it.
-  if (InAlign > Align)
-    Align = InAlign;
+  if (InAlign > Alignment)
+    Alignment = InAlign;
 
   // If the GV has a specified alignment, take it into account.
-  const llvm::MaybeAlign GVAlign(GV->getAlignment());
+  const MaybeAlign GVAlign(GV->getAlignment());
   if (!GVAlign)
-    return Align;
+    return Alignment;
 
   assert(GVAlign && "GVAlign must be set");
 
   // If the GVAlign is larger than NumBits, or if we are required to obey
   // NumBits because the GV has an assigned section, obey it.
-  if (*GVAlign > Align || GV->hasSection())
-    Align = *GVAlign;
-  return Align;
+  if (*GVAlign > Alignment || GV->hasSection())
+    Alignment = *GVAlign;
+  return Alignment;
 }
 
 AsmPrinter::AsmPrinter(TargetMachine &tm, std::unique_ptr<MCStreamer> Streamer)
@@ -249,13 +250,14 @@ const MCSection *AsmPrinter::getCurrentSection() const {
 void AsmPrinter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
   MachineFunctionPass::getAnalysisUsage(AU);
-  AU.addRequired<MachineModuleInfo>();
+  AU.addRequired<MachineModuleInfoWrapperPass>();
   AU.addRequired<MachineOptimizationRemarkEmitterPass>();
   AU.addRequired<GCModuleInfo>();
 }
 
 bool AsmPrinter::doInitialization(Module &M) {
-  MMI = getAnalysisIfAvailable<MachineModuleInfo>();
+  auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+  MMI = MMIWP ? &MMIWP->getMMI() : nullptr;
 
   // Initialize TargetLoweringObjectFile.
   const_cast<TargetLoweringObjectFile&>(getObjFileLowering())
@@ -426,7 +428,10 @@ void AsmPrinter::EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const {
     OutStreamer->EmitSymbolAttribute(GVSym, MCSA_Global);
     return;
   case GlobalValue::PrivateLinkage:
+    return;
   case GlobalValue::InternalLinkage:
+    if (MAI->hasDotLGloblDirective())
+      OutStreamer->EmitSymbolAttribute(GVSym, MCSA_LGlobal);
     return;
   case GlobalValue::AppendingLinkage:
   case GlobalValue::AvailableExternallyLinkage:
@@ -502,7 +507,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   // If the alignment is specified, we *must* obey it.  Overaligning a global
   // with a specified alignment is a prompt way to break globals emitted to
   // sections and expected to be contiguous (e.g. ObjC metadata).
-  const llvm::Align Align = getGVAlignment(GV, DL);
+  const Align Alignment = getGVAlignment(GV, DL);
 
   for (const HandlerInfo &HI : Handlers) {
     NamedRegionTimer T(HI.TimerName, HI.TimerDescription,
@@ -518,7 +523,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     const bool SupportsAlignment =
         getObjFileLowering().getCommDirectiveSupportsAlignment();
     OutStreamer->EmitCommonSymbol(GVSym, Size,
-                                  SupportsAlignment ? Align.value() : 0);
+                                  SupportsAlignment ? Alignment.value() : 0);
     return;
   }
 
@@ -533,7 +538,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       Size = 1; // zerofill of 0 bytes is undefined.
     EmitLinkage(GV, GVSym);
     // .zerofill __DATA, __bss, _foo, 400, 5
-    OutStreamer->EmitZerofill(TheSection, GVSym, Size, Align.value());
+    OutStreamer->EmitZerofill(TheSection, GVSym, Size, Alignment.value());
     return;
   }
 
@@ -552,7 +557,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     // Prefer to simply fall back to .local / .comm in this case.
     if (MAI->getLCOMMDirectiveAlignmentType() != LCOMM::NoAlignment) {
       // .lcomm _foo, 42
-      OutStreamer->EmitLocalCommonSymbol(GVSym, Size, Align.value());
+      OutStreamer->EmitLocalCommonSymbol(GVSym, Size, Alignment.value());
       return;
     }
 
@@ -562,7 +567,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     const bool SupportsAlignment =
         getObjFileLowering().getCommDirectiveSupportsAlignment();
     OutStreamer->EmitCommonSymbol(GVSym, Size,
-                                  SupportsAlignment ? Align.value() : 0);
+                                  SupportsAlignment ? Alignment.value() : 0);
     return;
   }
 
@@ -583,11 +588,11 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
 
     if (GVKind.isThreadBSS()) {
       TheSection = getObjFileLowering().getTLSBSSSection();
-      OutStreamer->EmitTBSSSymbol(TheSection, MangSym, Size, Align.value());
+      OutStreamer->EmitTBSSSymbol(TheSection, MangSym, Size, Alignment.value());
     } else if (GVKind.isThreadData()) {
       OutStreamer->SwitchSection(TheSection);
 
-      EmitAlignment(Align, GV);
+      EmitAlignment(Alignment, GV);
       OutStreamer->EmitLabel(MangSym);
 
       EmitGlobalConstant(GV->getParent()->getDataLayout(),
@@ -623,7 +628,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   OutStreamer->SwitchSection(TheSection);
 
   EmitLinkage(GV, EmittedInitSym);
-  EmitAlignment(Align, GV);
+  EmitAlignment(Alignment, GV);
 
   OutStreamer->EmitLabel(EmittedInitSym);
 
@@ -662,6 +667,10 @@ void AsmPrinter::EmitFunctionHeader() {
   OutStreamer->SwitchSection(getObjFileLowering().SectionForGlobal(&F, TM));
   EmitVisibility(CurrentFnSym, F.getVisibility());
 
+  if (MAI->needsFunctionDescriptors() &&
+      F.getLinkage() != GlobalValue::InternalLinkage)
+    EmitLinkage(&F, CurrentFnDescSym);
+
   EmitLinkage(&F, CurrentFnSym);
   if (MAI->hasFunctionAlignment())
     EmitAlignment(MF->getAlignment(), &F);
@@ -697,8 +706,13 @@ void AsmPrinter::EmitFunctionHeader() {
     }
   }
 
-  // Emit the CurrentFnSym.  This is a virtual function to allow targets to
-  // do their wild and crazy things as required.
+  // Emit the function descriptor. This is a virtual function to allow targets
+  // to emit their specific function descriptor.
+  if (MAI->needsFunctionDescriptors())
+    EmitFunctionDescriptor();
+
+  // Emit the CurrentFnSym. This is a virtual function to allow targets to do
+  // their wild and crazy things as required.
   EmitFunctionEntryLabel();
 
   // If the function had address-taken blocks that got deleted, then we have
@@ -1349,15 +1363,18 @@ void AsmPrinter::emitRemarksSection(Module &M) {
     return;
   remarks::RemarkSerializer &RemarkSerializer = RS->getSerializer();
 
-  StringRef FilenameRef = RS->getFilename();
-  SmallString<128> Filename = FilenameRef;
-  sys::fs::make_absolute(Filename);
-  assert(!Filename.empty() && "The filename can't be empty.");
+  Optional<SmallString<128>> Filename;
+  if (Optional<StringRef> FilenameRef = RS->getFilename()) {
+    Filename = *FilenameRef;
+    sys::fs::make_absolute(*Filename);
+    assert(!Filename->empty() && "The filename can't be empty.");
+  }
 
   std::string Buf;
   raw_string_ostream OS(Buf);
   std::unique_ptr<remarks::MetaSerializer> MetaSerializer =
-      RemarkSerializer.metaSerializer(OS, StringRef(Filename));
+      Filename ? RemarkSerializer.metaSerializer(OS, StringRef(*Filename))
+               : RemarkSerializer.metaSerializer(OS);
   MetaSerializer->emit();
 
   // Switch to the right section: .remarks/__remarks.
@@ -1418,7 +1435,7 @@ bool AsmPrinter::doFinalization(Module &M) {
       OutStreamer->SwitchSection(TLOF.getDataSection());
       const DataLayout &DL = M.getDataLayout();
 
-      EmitAlignment(llvm::Align(DL.getPointerSize()));
+      EmitAlignment(Align(DL.getPointerSize()));
       for (const auto &Stub : Stubs) {
         OutStreamer->EmitLabel(Stub.first);
         OutStreamer->EmitSymbolValue(Stub.second.getPointer(),
@@ -1445,7 +1462,7 @@ bool AsmPrinter::doFinalization(Module &M) {
                 COFF::IMAGE_SCN_LNK_COMDAT,
             SectionKind::getReadOnly(), Stub.first->getName(),
             COFF::IMAGE_COMDAT_SELECT_ANY));
-        EmitAlignment(llvm::Align(DL.getPointerSize()));
+        EmitAlignment(Align(DL.getPointerSize()));
         OutStreamer->EmitSymbolAttribute(Stub.first, MCSA_Global);
         OutStreamer->EmitLabel(Stub.first);
         OutStreamer->EmitSymbolValue(Stub.second.getPointer(),
@@ -1641,8 +1658,27 @@ MCSymbol *AsmPrinter::getCurExceptionSym() {
 
 void AsmPrinter::SetupMachineFunction(MachineFunction &MF) {
   this->MF = &MF;
+
   // Get the function symbol.
-  CurrentFnSym = getSymbol(&MF.getFunction());
+  if (MAI->needsFunctionDescriptors()) {
+    assert(TM.getTargetTriple().isOSAIX() && "Function descriptor is only"
+                                             " supported on AIX.");
+    assert(CurrentFnDescSym && "The function descriptor symbol needs to be"
+		                           " initalized first.");
+
+    // Get the function entry point symbol.
+    CurrentFnSym =
+        OutContext.getOrCreateSymbol("." + CurrentFnDescSym->getName());
+
+    const Function &F = MF.getFunction();
+    MCSectionXCOFF *FnEntryPointSec =
+        cast<MCSectionXCOFF>(getObjFileLowering().SectionForGlobal(&F, TM));
+    // Set the containing csect.
+    cast<MCSymbolXCOFF>(CurrentFnSym)->setContainingCsect(FnEntryPointSec);
+  } else {
+    CurrentFnSym = getSymbol(&MF.getFunction());
+  }
+
   CurrentFnSymForSize = CurrentFnSym;
   CurrentFnBegin = nullptr;
   CurExceptionSym = nullptr;
@@ -1727,7 +1763,7 @@ void AsmPrinter::EmitConstantPool() {
 
       if (CurSection != CPSections[i].S) {
         OutStreamer->SwitchSection(CPSections[i].S);
-        EmitAlignment(llvm::Align(CPSections[i].Alignment));
+        EmitAlignment(Align(CPSections[i].Alignment));
         CurSection = CPSections[i].S;
         Offset = 0;
       }
@@ -1774,7 +1810,7 @@ void AsmPrinter::EmitJumpTableInfo() {
     OutStreamer->SwitchSection(ReadOnlySection);
   }
 
-  EmitAlignment(llvm::Align(MJTI->getEntryAlignment(DL)));
+  EmitAlignment(Align(MJTI->getEntryAlignment(DL)));
 
   // Jump tables in code sections are marked with a data_region directive
   // where that's supported.
@@ -1987,10 +2023,10 @@ void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List,
   }
 
   // Emit the function pointers in the target-specific order
-  const llvm::Align Align = llvm::Align(DL.getPointerPrefAlignment());
   llvm::stable_sort(Structors, [](const Structor &L, const Structor &R) {
     return L.Priority < R.Priority;
   });
+  const Align Align = DL.getPointerPrefAlignment();
   for (Structor &S : Structors) {
     const TargetLoweringObjectFile &Obj = getObjFileLowering();
     const MCSymbol *KeySym = nullptr;
@@ -2114,18 +2150,17 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
 // two boundary.  If a global value is specified, and if that global has
 // an explicit alignment requested, it will override the alignment request
 // if required for correctness.
-void AsmPrinter::EmitAlignment(llvm::Align Align,
-                               const GlobalObject *GV) const {
+void AsmPrinter::EmitAlignment(Align Alignment, const GlobalObject *GV) const {
   if (GV)
-    Align = getGVAlignment(GV, GV->getParent()->getDataLayout(), Align);
+    Alignment = getGVAlignment(GV, GV->getParent()->getDataLayout(), Alignment);
 
-  if (Align == 1)
+  if (Alignment == Align::None())
     return; // 1-byte aligned: no need to emit alignment.
 
   if (getCurrentSection()->getKind().isText())
-    OutStreamer->EmitCodeAlignment(Align.value());
+    OutStreamer->EmitCodeAlignment(Alignment.value());
   else
-    OutStreamer->EmitValueToAlignment(Align.value());
+    OutStreamer->EmitValueToAlignment(Alignment.value());
 }
 
 //===----------------------------------------------------------------------===//
@@ -2441,6 +2476,7 @@ static void emitGlobalConstantStruct(const DataLayout &DL,
 }
 
 static void emitGlobalConstantFP(APFloat APF, Type *ET, AsmPrinter &AP) {
+  assert(ET && "Unknown float type");
   APInt API = APF.bitcastToAPInt();
 
   // First print a comment with what we think the original floating-point value
@@ -2448,11 +2484,7 @@ static void emitGlobalConstantFP(APFloat APF, Type *ET, AsmPrinter &AP) {
   if (AP.isVerbose()) {
     SmallString<8> StrVal;
     APF.toString(StrVal);
-
-    if (ET)
-      ET->print(AP.OutStreamer->GetCommentOS());
-    else
-      AP.OutStreamer->GetCommentOS() << "Printing <null> Type";
+    ET->print(AP.OutStreamer->GetCommentOS());
     AP.OutStreamer->GetCommentOS() << ' ' << StrVal << '\n';
   }
 
@@ -2900,9 +2932,9 @@ void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) {
   }
 
   // Emit an alignment directive for this block, if needed.
-  const llvm::Align Align = MBB.getAlignment();
-  if (Align > 1)
-    EmitAlignment(Align);
+  const Align Alignment = MBB.getAlignment();
+  if (Alignment != Align::None())
+    EmitAlignment(Alignment);
   MCCodePaddingContext Context;
   setupCodePaddingContext(MBB, Context);
   OutStreamer->EmitCodePaddingBasicBlockStart(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h b/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
index 789291771b5a1..a062baf7698a7 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DebugLocStream.h
@@ -38,14 +38,10 @@ class DebugLocStream {
         : CU(CU), EntryOffset(EntryOffset) {}
   };
   struct Entry {
-    const MCSymbol *BeginSym;
-    const MCSymbol *EndSym;
+    const MCSymbol *Begin;
+    const MCSymbol *End;
     size_t ByteOffset;
     size_t CommentOffset;
-    Entry(const MCSymbol *BeginSym, const MCSymbol *EndSym, size_t ByteOffset,
-          size_t CommentOffset)
-        : BeginSym(BeginSym), EndSym(EndSym), ByteOffset(ByteOffset),
-          CommentOffset(CommentOffset) {}
   };
 
 private:
@@ -93,7 +89,7 @@ class DebugLocStream {
   /// Until the next call, bytes added to the stream will be added to this
   /// entry.
   void startEntry(const MCSymbol *BeginSym, const MCSymbol *EndSym) {
-    Entries.emplace_back(BeginSym, EndSym, DWARFBytes.size(), Comments.size());
+    Entries.push_back({BeginSym, EndSym, DWARFBytes.size(), Comments.size()});
   }
 
   /// Finalize a .debug_loc entry, deleting if it's empty.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index f859a02033b4b..69c4d3fb5b449 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -326,14 +326,13 @@ void DwarfCompileUnit::addRange(RangeSpan Range) {
   // emitted into and the subprogram was contained within. If these are the
   // same then extend our current range, otherwise add this as a new range.
   if (CURanges.empty() || !SameAsPrevCU ||
-      (&CURanges.back().getEnd()->getSection() !=
-       &Range.getEnd()->getSection())) {
+      (&CURanges.back().End->getSection() !=
+       &Range.End->getSection())) {
     CURanges.push_back(Range);
-    DD->addSectionLabel(Range.getStart());
     return;
   }
 
-  CURanges.back().setEnd(Range.getEnd());
+  CURanges.back().End = Range.End;
 }
 
 void DwarfCompileUnit::initStmtList() {
@@ -507,7 +506,7 @@ void DwarfCompileUnit::attachRangesOrLowHighPC(
   if (Ranges.size() == 1 || !DD->useRangesSection()) {
     const RangeSpan &Front = Ranges.front();
     const RangeSpan &Back = Ranges.back();
-    attachLowHighPC(Die, Front.getStart(), Back.getEnd());
+    attachLowHighPC(Die, Front.Begin, Back.End);
   } else
     addScopeRangeList(Die, std::move(Ranges));
 }
@@ -517,8 +516,8 @@ void DwarfCompileUnit::attachRangesOrLowHighPC(
   SmallVector<RangeSpan, 2> List;
   List.reserve(Ranges.size());
   for (const InsnRange &R : Ranges)
-    List.push_back(RangeSpan(DD->getLabelBeforeInsn(R.first),
-                             DD->getLabelAfterInsn(R.second)));
+    List.push_back(
+        {DD->getLabelBeforeInsn(R.first), DD->getLabelAfterInsn(R.second)});
   attachRangesOrLowHighPC(Die, std::move(List));
 }
 
@@ -1165,16 +1164,8 @@ void DwarfCompileUnit::addGlobalTypeUnitType(const DIType *Ty,
   GlobalTypes.insert(std::make_pair(std::move(FullName), &getUnitDie()));
 }
 
-/// addVariableAddress - Add DW_AT_location attribute for a
-/// DbgVariable based on provided MachineLocation.
 void DwarfCompileUnit::addVariableAddress(const DbgVariable &DV, DIE &Die,
                                           MachineLocation Location) {
-  // addBlockByrefAddress is obsolete and will be removed soon.
-  // The clang frontend always generates block byref variables with a
-  // complex expression that encodes exactly what addBlockByrefAddress
-  // would do.
-  assert((!DV.isBlockByrefVariable() || DV.hasComplexAddress()) &&
-         "block byref variable without a complex expression");
   if (DV.hasComplexAddress())
     addComplexAddress(DV, Die, dwarf::DW_AT_location, Location);
   else
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 9be64c42f0cd2..aeca172cfdb4e 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -198,54 +198,8 @@ bool DebugLocDwarfExpression::isFrameRegister(const TargetRegisterInfo &TRI,
   return false;
 }
 
-bool DbgVariable::isBlockByrefVariable() const {
-  assert(getVariable() && "Invalid complex DbgVariable!");
-  return getVariable()->getType()->isBlockByrefStruct();
-}
-
 const DIType *DbgVariable::getType() const {
-  DIType *Ty = getVariable()->getType();
-  // FIXME: isBlockByrefVariable should be reformulated in terms of complex
-  // addresses instead.
-  if (Ty->isBlockByrefStruct()) {
-    /* Byref variables, in Blocks, are declared by the programmer as
-       "SomeType VarName;", but the compiler creates a
-       __Block_byref_x_VarName struct, and gives the variable VarName
-       either the struct, or a pointer to the struct, as its type.  This
-       is necessary for various behind-the-scenes things the compiler
-       needs to do with by-reference variables in blocks.
-
-       However, as far as the original *programmer* is concerned, the
-       variable should still have type 'SomeType', as originally declared.
-
-       The following function dives into the __Block_byref_x_VarName
-       struct to find the original type of the variable.  This will be
-       passed back to the code generating the type for the Debug
-       Information Entry for the variable 'VarName'.  'VarName' will then
-       have the original type 'SomeType' in its debug information.
-
-       The original type 'SomeType' will be the type of the field named
-       'VarName' inside the __Block_byref_x_VarName struct.
-
-       NOTE: In order for this to not completely fail on the debugger
-       side, the Debug Information Entry for the variable VarName needs to
-       have a DW_AT_location that tells the debugger how to unwind through
-       the pointers and __Block_byref_x_VarName struct to find the actual
-       value of the variable.  The function addBlockByrefType does this.  */
-    DIType *subType = Ty;
-    uint16_t tag = Ty->getTag();
-
-    if (tag == dwarf::DW_TAG_pointer_type)
-      subType = cast<DIDerivedType>(Ty)->getBaseType();
-
-    auto Elements = cast<DICompositeType>(subType)->getElements();
-    for (unsigned i = 0, N = Elements.size(); i < N; ++i) {
-      auto *DT = cast<DIDerivedType>(Elements[i]);
-      if (getName() == DT->getName())
-        return DT->getBaseType();
-    }
-  }
-  return Ty;
+  return getVariable()->getType();
 }
 
 /// Get .debug_loc entry for the instruction range starting at MI.
@@ -1144,7 +1098,7 @@ void DwarfDebug::finalizeModuleInfo() {
         // 2.17.3).
         U.addUInt(U.getUnitDie(), dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr, 0);
       else
-        U.setBaseAddress(TheCU.getRanges().front().getStart());
+        U.setBaseAddress(TheCU.getRanges().front().Begin);
       U.attachRangesOrLowHighPC(U.getUnitDie(), TheCU.takeRanges());
     }
 
@@ -1803,6 +1757,9 @@ void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) {
   if (SP->getUnit()->getEmissionKind() == DICompileUnit::NoDebug)
     return;
 
+  SectionLabels.insert(std::make_pair(&Asm->getFunctionBegin()->getSection(),
+                                      Asm->getFunctionBegin()));
+
   DwarfCompileUnit &CU = getOrCreateDwarfCompileUnit(SP->getUnit());
 
   // Set DwarfDwarfCompileUnitID in MCContext to the Compile Unit this function
@@ -1850,7 +1807,7 @@ void DwarfDebug::endFunctionImpl(const MachineFunction *MF) {
   collectEntityInfo(TheCU, SP, Processed);
 
   // Add the range of this function to the list of ranges for the CU.
-  TheCU.addRange(RangeSpan(Asm->getFunctionBegin(), Asm->getFunctionEnd()));
+  TheCU.addRange({Asm->getFunctionBegin(), Asm->getFunctionEnd()});
 
   // Under -gmlt, skip building the subprogram if there are no inlined
   // subroutines inside it. But with -fdebug-info-for-profiling, the subprogram
@@ -2032,9 +1989,10 @@ static dwarf::PubIndexEntryDescriptor computeIndexValue(DwarfUnit *CU,
   case dwarf::DW_TAG_union_type:
   case dwarf::DW_TAG_enumeration_type:
     return dwarf::PubIndexEntryDescriptor(
-        dwarf::GIEK_TYPE, CU->getLanguage() != dwarf::DW_LANG_C_plus_plus
-                              ? dwarf::GIEL_STATIC
-                              : dwarf::GIEL_EXTERNAL);
+        dwarf::GIEK_TYPE,
+        dwarf::isCPlusPlus((dwarf::SourceLanguage)CU->getLanguage())
+            ? dwarf::GIEL_EXTERNAL
+            : dwarf::GIEL_STATIC);
   case dwarf::DW_TAG_typedef:
   case dwarf::DW_TAG_base_type:
   case dwarf::DW_TAG_subrange_type:
@@ -2367,12 +2325,12 @@ void DwarfDebug::emitDebugLoc() {
           Asm->OutStreamer->AddComment("DW_LLE_offset_pair");
           Asm->OutStreamer->EmitIntValue(dwarf::DW_LLE_offset_pair, 1);
           Asm->OutStreamer->AddComment("  starting offset");
-          Asm->EmitLabelDifferenceAsULEB128(Entry.BeginSym, Base);
+          Asm->EmitLabelDifferenceAsULEB128(Entry.Begin, Base);
           Asm->OutStreamer->AddComment("  ending offset");
-          Asm->EmitLabelDifferenceAsULEB128(Entry.EndSym, Base);
+          Asm->EmitLabelDifferenceAsULEB128(Entry.End, Base);
         } else {
-          Asm->EmitLabelDifference(Entry.BeginSym, Base, Size);
-          Asm->EmitLabelDifference(Entry.EndSym, Base, Size);
+          Asm->EmitLabelDifference(Entry.Begin, Base, Size);
+          Asm->EmitLabelDifference(Entry.End, Base, Size);
         }
 
         emitDebugLocEntryLocation(Entry, CU);
@@ -2388,12 +2346,12 @@ void DwarfDebug::emitDebugLoc() {
         Asm->OutStreamer->AddComment("DW_LLE_startx_length");
         Asm->emitInt8(dwarf::DW_LLE_startx_length);
         Asm->OutStreamer->AddComment("  start idx");
-        Asm->EmitULEB128(AddrPool.getIndex(Entry.BeginSym));
+        Asm->EmitULEB128(AddrPool.getIndex(Entry.Begin));
         Asm->OutStreamer->AddComment("  length");
-        Asm->EmitLabelDifferenceAsULEB128(Entry.EndSym, Entry.BeginSym);
+        Asm->EmitLabelDifferenceAsULEB128(Entry.End, Entry.Begin);
       } else {
-        Asm->OutStreamer->EmitSymbolValue(Entry.BeginSym, Size);
-        Asm->OutStreamer->EmitSymbolValue(Entry.EndSym, Size);
+        Asm->OutStreamer->EmitSymbolValue(Entry.Begin, Size);
+        Asm->OutStreamer->EmitSymbolValue(Entry.End, Size);
       }
 
       emitDebugLocEntryLocation(Entry, CU);
@@ -2428,9 +2386,9 @@ void DwarfDebug::emitDebugLocDWO() {
       // Ideally/in v5, this could use SectionLabels to reuse existing addresses
       // in the address pool to minimize object size/relocations.
       Asm->emitInt8(dwarf::DW_LLE_startx_length);
-      unsigned idx = AddrPool.getIndex(Entry.BeginSym);
+      unsigned idx = AddrPool.getIndex(Entry.Begin);
       Asm->EmitULEB128(idx);
-      Asm->EmitLabelDifference(Entry.EndSym, Entry.BeginSym, 4);
+      Asm->EmitLabelDifference(Entry.End, Entry.Begin, 4);
 
       emitDebugLocEntryLocation(Entry, List.CU);
     }
@@ -2555,8 +2513,8 @@ void DwarfDebug::emitDebugARanges() {
     unsigned TupleSize = PtrSize * 2;
 
     // 7.20 in the Dwarf specs requires the table to be aligned to a tuple.
-    unsigned Padding = offsetToAlignment(sizeof(int32_t) + ContentSize,
-                                         llvm::Align(TupleSize));
+    unsigned Padding =
+        offsetToAlignment(sizeof(int32_t) + ContentSize, Align(TupleSize));
 
     ContentSize += Padding;
     ContentSize += (List.size() + 1) * TupleSize;
@@ -2612,7 +2570,7 @@ static void emitRangeList(DwarfDebug &DD, AsmPrinter *Asm,
   auto Size = Asm->MAI->getCodePointerSize();
 
   for (const RangeSpan &Range : List.getRanges())
-    SectionRanges[&Range.getStart()->getSection()].push_back(&Range);
+    SectionRanges[&Range.Begin->getSection()].push_back(&Range);
 
   const DwarfCompileUnit &CU = List.getCU();
   const MCSymbol *CUBase = CU.getBaseAddress();
@@ -2628,11 +2586,8 @@ static void emitRangeList(DwarfDebug &DD, AsmPrinter *Asm,
     if (!Base && (P.second.size() > 1 || DwarfVersion < 5) &&
         (CU.getCUNode()->getRangesBaseAddress() || DwarfVersion >= 5)) {
       BaseIsSet = true;
-      // FIXME/use care: This may not be a useful base address if it's not
-      // the lowest address/range in this object.
-      Base = P.second.front()->getStart();
+      Base = DD.getSectionLabel(&P.second.front()->Begin->getSection());
       if (DwarfVersion >= 5) {
-        Base = DD.getSectionLabel(&Base->getSection());
         Asm->OutStreamer->AddComment("DW_RLE_base_addressx");
         Asm->OutStreamer->EmitIntValue(dwarf::DW_RLE_base_addressx, 1);
         Asm->OutStreamer->AddComment("  base address index");
@@ -2650,8 +2605,8 @@ static void emitRangeList(DwarfDebug &DD, AsmPrinter *Asm,
     }
 
     for (const auto *RS : P.second) {
-      const MCSymbol *Begin = RS->getStart();
-      const MCSymbol *End = RS->getEnd();
+      const MCSymbol *Begin = RS->Begin;
+      const MCSymbol *End = RS->End;
       assert(Begin && "Range without a begin symbol?");
       assert(End && "Range without an end symbol?");
       if (Base) {
@@ -3075,10 +3030,6 @@ uint16_t DwarfDebug::getDwarfVersion() const {
   return Asm->OutStreamer->getContext().getDwarfVersion();
 }
 
-void DwarfDebug::addSectionLabel(const MCSymbol *Sym) {
-  SectionLabels.insert(std::make_pair(&Sym->getSection(), Sym));
-}
-
 const MCSymbol *DwarfDebug::getSectionLabel(const MCSection *S) {
   return SectionLabels.find(S)->second;
 }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
index 843caf1ab184f..c8c511f67c2a0 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -216,7 +216,6 @@ class DbgVariable : public DbgEntity {
     return !FrameIndexExprs.empty();
   }
 
-  bool isBlockByrefVariable() const;
   const DIType *getType() const;
 
   static bool classof(const DbgEntity *N) {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
index 52e5109dbf905..ea42e60344d9e 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp
@@ -246,8 +246,8 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
   // a call site parameter expression and if that expression is just a register
   // location, emit it with addBReg and offset 0, because we should emit a DWARF
   // expression representing a value, rather than a location.
-  if (!isMemoryLocation() && !HasComplexExpression && (!isParameterValue() ||
-                                                       isEntryValue())) {
+  if (!isMemoryLocation() && !HasComplexExpression &&
+      (!isParameterValue() || isEntryValue())) {
     for (auto &Reg : DwarfRegs) {
       if (Reg.DwarfRegNo >= 0)
         addReg(Reg.DwarfRegNo, Reg.Comment);
@@ -413,6 +413,9 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor,
       break;
     case dwarf::DW_OP_deref:
       assert(!isRegisterLocation());
+      // For more detailed explanation see llvm.org/PR43343.
+      assert(!isParameterValue() && "Parameter entry values should not be "
+                                    "dereferenced due to safety reasons.");
       if (!isMemoryLocation() && ::isMemoryLocation(ExprCursor))
         // Turning this into a memory location description makes the deref
         // implicit.
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h b/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h
index 244678ce9dc12..25ed8da970a40 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfFile.h
@@ -32,15 +32,9 @@ class LexicalScope;
 class MCSection;
 
 // Data structure to hold a range for range lists.
-class RangeSpan {
-public:
-  RangeSpan(MCSymbol *S, MCSymbol *E) : Start(S), End(E) {}
-  const MCSymbol *getStart() const { return Start; }
-  const MCSymbol *getEnd() const { return End; }
-  void setEnd(const MCSymbol *E) { End = E; }
-
-private:
-  const MCSymbol *Start, *End;
+struct RangeSpan {
+  const MCSymbol *Begin;
+  const MCSymbol *End;
 };
 
 class RangeSpanList {
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
index 834ea358736fd..be743a52cc63b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.cpp
@@ -722,7 +722,7 @@ std::string DwarfUnit::getParentContextString(const DIScope *Context) const {
     return "";
 
   // FIXME: Decide whether to implement this for non-C++ languages.
-  if (getLanguage() != dwarf::DW_LANG_C_plus_plus)
+  if (!dwarf::isCPlusPlus((dwarf::SourceLanguage)getLanguage()))
     return "";
 
   std::string CS;
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
index a376d5e75f675..a188002722d2f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfUnit.h
@@ -216,15 +216,6 @@ class DwarfUnit : public DIEUnit {
   /// Add thrown types.
   void addThrownTypes(DIE &Die, DINodeArray ThrownTypes);
 
-  // FIXME: Should be reformulated in terms of addComplexAddress.
-  /// Start with the address based on the location provided, and generate the
-  /// DWARF information necessary to find the actual Block variable (navigating
-  /// the Block struct) based on the starting location.  Add the DWARF
-  /// information to the die.  Obsolete, please use addComplexAddress instead.
-  void addBlockByrefAddress(const DbgVariable &DV, DIE &Die,
-                            dwarf::Attribute Attribute,
-                            const MachineLocation &Location);
-
   /// Add a new type attribute to the specified entity.
   ///
   /// This takes and attribute parameter because DW_AT_friend attributes are
diff --git a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
index 689d55b3caac1..31dfaaac836eb 100644
--- a/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/EHStreamer.cpp
@@ -426,7 +426,7 @@ MCSymbol *EHStreamer::emitExceptionTable() {
   // EHABI). In this case LSDASection will be NULL.
   if (LSDASection)
     Asm->OutStreamer->SwitchSection(LSDASection);
-  Asm->EmitAlignment(llvm::Align(4));
+  Asm->EmitAlignment(Align(4));
 
   // Emit the LSDA.
   MCSymbol *GCCETSym =
@@ -602,11 +602,11 @@ MCSymbol *EHStreamer::emitExceptionTable() {
   }
 
   if (HaveTTData) {
-    Asm->EmitAlignment(llvm::Align(4));
+    Asm->EmitAlignment(Align(4));
     emitTypeInfos(TTypeEncoding, TTBaseLabel);
   }
 
-  Asm->EmitAlignment(llvm::Align(4));
+  Asm->EmitAlignment(Align(4));
   return GCCETSym;
 }
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
index aac08041d6365..3849644d15844 100644
--- a/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp
@@ -72,7 +72,7 @@ void ErlangGCPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
      **/
 
     // Align to address width.
-    AP.EmitAlignment(IntPtrSize == 4 ? llvm::Align(4) : llvm::Align(8));
+    AP.EmitAlignment(IntPtrSize == 4 ? Align(4) : Align(8));
 
     // Emit PointCount.
     OS.AddComment("safe point count");
diff --git a/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
index 824d0cbcb0cf9..b4eda5fa8c581 100644
--- a/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
@@ -129,7 +129,7 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
     report_fatal_error(" Too much descriptor for ocaml GC");
   }
   AP.emitInt16(NumDescriptors);
-  AP.EmitAlignment(IntPtrSize == 4 ? llvm::Align(4) : llvm::Align(8));
+  AP.EmitAlignment(IntPtrSize == 4 ? Align(4) : Align(8));
 
   for (GCModuleInfo::FuncInfoVec::iterator I = Info.funcinfo_begin(),
                                            IE = Info.funcinfo_end();
@@ -180,7 +180,7 @@ void OcamlGCMetadataPrinter::finishAssembly(Module &M, GCModuleInfo &Info,
         AP.emitInt16(K->StackOffset);
       }
 
-      AP.EmitAlignment(IntPtrSize == 4 ? llvm::Align(4) : llvm::Align(8));
+      AP.EmitAlignment(IntPtrSize == 4 ? Align(4) : Align(8));
     }
   }
 }
diff --git a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
index 155e91ce61a1b..0398675577cd5 100644
--- a/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/WinException.cpp
@@ -982,8 +982,7 @@ void WinException::emitExceptHandlerTable(const MachineFunction *MF) {
   OS.EmitValueToAlignment(4);
   OS.EmitLabel(LSDALabel);
 
-  const Function *Per =
-      dyn_cast<Function>(F.getPersonalityFn()->stripPointerCasts());
+  const auto *Per = cast<Function>(F.getPersonalityFn()->stripPointerCasts());
   StringRef PerName = Per->getName();
   int BaseState = -1;
   if (PerName == "_except_handler4") {
diff --git a/llvm/lib/CodeGen/AtomicExpandPass.cpp b/llvm/lib/CodeGen/AtomicExpandPass.cpp
index dc7eaf6a5fe7e..cc3379f13b4d6 100644
--- a/llvm/lib/CodeGen/AtomicExpandPass.cpp
+++ b/llvm/lib/CodeGen/AtomicExpandPass.cpp
@@ -382,7 +382,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) {
   Value *NewAddr = Builder.CreateBitCast(Addr, PT);
 
   auto *NewLI = Builder.CreateLoad(NewTy, NewAddr);
-  NewLI->setAlignment(LI->getAlignment());
+  NewLI->setAlignment(MaybeAlign(LI->getAlignment()));
   NewLI->setVolatile(LI->isVolatile());
   NewLI->setAtomic(LI->getOrdering(), LI->getSyncScopeID());
   LLVM_DEBUG(dbgs() << "Replaced " << *LI << " with " << *NewLI << "\n");
@@ -1376,7 +1376,7 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
   Builder.SetInsertPoint(BB);
   LoadInst *InitLoaded = Builder.CreateLoad(ResultTy, Addr);
   // Atomics require at least natural alignment.
-  InitLoaded->setAlignment(ResultTy->getPrimitiveSizeInBits() / 8);
+  InitLoaded->setAlignment(MaybeAlign(ResultTy->getPrimitiveSizeInBits() / 8));
   Builder.CreateBr(LoopBB);
 
   // Start the main loop block now that we've taken care of the preliminaries.
@@ -1711,7 +1711,7 @@ bool AtomicExpand::expandAtomicOpToLibcall(
   // 'expected' argument, if present.
   if (CASExpected) {
     AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType());
-    AllocaCASExpected->setAlignment(AllocaAlignment);
+    AllocaCASExpected->setAlignment(MaybeAlign(AllocaAlignment));
     unsigned AllocaAS =  AllocaCASExpected->getType()->getPointerAddressSpace();
 
     AllocaCASExpected_i8 =
@@ -1730,7 +1730,7 @@ bool AtomicExpand::expandAtomicOpToLibcall(
       Args.push_back(IntValue);
     } else {
       AllocaValue = AllocaBuilder.CreateAlloca(ValueOperand->getType());
-      AllocaValue->setAlignment(AllocaAlignment);
+      AllocaValue->setAlignment(MaybeAlign(AllocaAlignment));
       AllocaValue_i8 =
           Builder.CreateBitCast(AllocaValue, Type::getInt8PtrTy(Ctx));
       Builder.CreateLifetimeStart(AllocaValue_i8, SizeVal64);
@@ -1742,7 +1742,7 @@ bool AtomicExpand::expandAtomicOpToLibcall(
   // 'ret' argument.
   if (!CASExpected && HasResult && !UseSizedLibcall) {
     AllocaResult = AllocaBuilder.CreateAlloca(I->getType());
-    AllocaResult->setAlignment(AllocaAlignment);
+    AllocaResult->setAlignment(MaybeAlign(AllocaAlignment));
     unsigned AllocaAS =  AllocaResult->getType()->getPointerAddressSpace();
     AllocaResult_i8 =
       Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx, AllocaAS));
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index a0a072be868b3..6c997a73023bc 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -129,9 +129,10 @@ bool BranchFolderPass::runOnMachineFunction(MachineFunction &MF) {
       getAnalysis<MachineBlockFrequencyInfo>());
   BranchFolder Folder(EnableTailMerge, /*CommonHoist=*/true, MBBFreqInfo,
                       getAnalysis<MachineBranchProbabilityInfo>());
-  return Folder.OptimizeFunction(MF, MF.getSubtarget().getInstrInfo(),
-                                 MF.getSubtarget().getRegisterInfo(),
-                                 getAnalysisIfAvailable<MachineModuleInfo>());
+  auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+  return Folder.OptimizeFunction(
+      MF, MF.getSubtarget().getInstrInfo(), MF.getSubtarget().getRegisterInfo(),
+      MMIWP ? &MMIWP->getMMI() : nullptr);
 }
 
 BranchFolder::BranchFolder(bool defaultEnableTailMerge, bool CommonHoist,
diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp
index 3abfd147b0965..6efdc9efa9684 100644
--- a/llvm/lib/CodeGen/BranchRelaxation.cpp
+++ b/llvm/lib/CodeGen/BranchRelaxation.cpp
@@ -65,17 +65,17 @@ class BranchRelaxation : public MachineFunctionPass {
     /// block.
     unsigned postOffset(const MachineBasicBlock &MBB) const {
       const unsigned PO = Offset + Size;
-      const llvm::Align Align = MBB.getAlignment();
-      if (Align == 1)
+      const Align Alignment = MBB.getAlignment();
+      if (Alignment == 1)
         return PO;
 
-      const llvm::Align ParentAlign = MBB.getParent()->getAlignment();
-      if (Align <= ParentAlign)
-        return PO + offsetToAlignment(PO, Align);
+      const Align ParentAlign = MBB.getParent()->getAlignment();
+      if (Alignment <= ParentAlign)
+        return PO + offsetToAlignment(PO, Alignment);
 
       // The alignment of this MBB is larger than the function's alignment, so we
       // can't tell whether or not it will insert nops. Assume that it will.
-      return PO + Align.value() + offsetToAlignment(PO, Align);
+      return PO + Alignment.value() + offsetToAlignment(PO, Alignment);
     }
   };
 
@@ -127,9 +127,8 @@ void BranchRelaxation::verify() {
 #ifndef NDEBUG
   unsigned PrevNum = MF->begin()->getNumber();
   for (MachineBasicBlock &MBB : *MF) {
-    unsigned LogAlign = MBB.getLogAlignment();
-    unsigned Num = MBB.getNumber();
-    assert(BlockInfo[Num].Offset % (1u << LogAlign) == 0);
+    const unsigned Num = MBB.getNumber();
+    assert(isAligned(MBB.getAlignment(), BlockInfo[Num].Offset));
     assert(!Num || BlockInfo[PrevNum].postOffset(MBB) <= BlockInfo[Num].Offset);
     assert(BlockInfo[Num].Size == computeBlockSize(MBB));
     PrevNum = Num;
diff --git a/llvm/lib/CodeGen/BreakFalseDeps.cpp b/llvm/lib/CodeGen/BreakFalseDeps.cpp
index 382f76da3218c..9b6da64e043b0 100644
--- a/llvm/lib/CodeGen/BreakFalseDeps.cpp
+++ b/llvm/lib/CodeGen/BreakFalseDeps.cpp
@@ -177,6 +177,7 @@ void BreakFalseDeps::processDefs(MachineInstr *MI) {
   assert(!MI->isDebugInstr() && "Won't process debug values");
 
   // Break dependence on undef uses. Do this before updating LiveRegs below.
+  // This can remove a false dependence with no additional instructions.
   unsigned OpNum;
   unsigned Pref = TII->getUndefRegClearance(*MI, OpNum, TRI);
   if (Pref) {
@@ -188,6 +189,11 @@ void BreakFalseDeps::processDefs(MachineInstr *MI) {
       UndefReads.push_back(std::make_pair(MI, OpNum));
   }
 
+  // The code below allows the target to create a new instruction to break the
+  // dependence. That opposes the goal of minimizing size, so bail out now.
+  if (MF->getFunction().hasMinSize())
+    return;
+
   const MCInstrDesc &MCID = MI->getDesc();
   for (unsigned i = 0,
     e = MI->isVariadic() ? MI->getNumOperands() : MCID.getNumDefs();
@@ -208,6 +214,11 @@ void BreakFalseDeps::processUndefReads(MachineBasicBlock *MBB) {
   if (UndefReads.empty())
     return;
 
+  // The code below allows the target to create a new instruction to break the
+  // dependence. That opposes the goal of minimizing size, so bail out now.
+  if (MF->getFunction().hasMinSize())
+    return;
+
   // Collect this block's live out register units.
   LiveRegSet.init(*TRI);
   // We do not need to care about pristine registers as they are just preserved
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt
index 3cf0c60108efa..50b469d6d936a 100644
--- a/llvm/lib/CodeGen/CMakeLists.txt
+++ b/llvm/lib/CodeGen/CMakeLists.txt
@@ -80,6 +80,7 @@ add_llvm_library(LLVMCodeGen
   MachineInstr.cpp
   MachineLICM.cpp
   MachineLoopInfo.cpp
+  MachineLoopUtils.cpp
   MachineModuleInfo.cpp
   MachineModuleInfoImpls.cpp
   MachineOperand.cpp
diff --git a/llvm/lib/CodeGen/CallingConvLower.cpp b/llvm/lib/CodeGen/CallingConvLower.cpp
index 39eabd926903e..a397039180a42 100644
--- a/llvm/lib/CodeGen/CallingConvLower.cpp
+++ b/llvm/lib/CodeGen/CallingConvLower.cpp
@@ -43,17 +43,18 @@ CCState::CCState(CallingConv::ID CC, bool isVarArg, MachineFunction &mf,
 void CCState::HandleByVal(unsigned ValNo, MVT ValVT, MVT LocVT,
                           CCValAssign::LocInfo LocInfo, int MinSize,
                           int MinAlignment, ISD::ArgFlagsTy ArgFlags) {
-  llvm::Align MinAlign(MinAlignment);
-  llvm::Align Align(ArgFlags.getByValAlign());
+  Align MinAlign(MinAlignment);
+  Align Alignment(ArgFlags.getByValAlign());
   unsigned Size  = ArgFlags.getByValSize();
   if (MinSize > (int)Size)
     Size = MinSize;
-  if (MinAlign > Align)
-    Align = MinAlign;
-  ensureMaxAlignment(Align);
-  MF.getSubtarget().getTargetLowering()->HandleByVal(this, Size, Align.value());
+  if (MinAlign > Alignment)
+    Alignment = MinAlign;
+  ensureMaxAlignment(Alignment);
+  MF.getSubtarget().getTargetLowering()->HandleByVal(this, Size,
+                                                     Alignment.value());
   Size = unsigned(alignTo(Size, MinAlign));
-  unsigned Offset = AllocateStack(Size, Align.value());
+  unsigned Offset = AllocateStack(Size, Alignment.value());
   addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
 }
 
@@ -198,7 +199,7 @@ static bool isValueTypeInRegForCC(CallingConv::ID CC, MVT VT) {
 void CCState::getRemainingRegParmsForType(SmallVectorImpl<MCPhysReg> &Regs,
                                           MVT VT, CCAssignFn Fn) {
   unsigned SavedStackOffset = StackOffset;
-  llvm::Align SavedMaxStackArgAlign = MaxStackArgAlign;
+  Align SavedMaxStackArgAlign = MaxStackArgAlign;
   unsigned NumLocs = Locs.size();
 
   // Set the 'inreg' flag if it is used for this calling convention.
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index d369160b299a5..ad9525f927e89 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -65,7 +65,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachineFunctionPrinterPassPass(Registry);
   initializeMachineLICMPass(Registry);
   initializeMachineLoopInfoPass(Registry);
-  initializeMachineModuleInfoPass(Registry);
+  initializeMachineModuleInfoWrapperPassPass(Registry);
   initializeMachineOptimizationRemarkEmitterPassPass(Registry);
   initializeMachineOutlinerPass(Registry);
   initializeMachinePipelinerPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index b3a6e284f4f9c..ee2a35d626b8e 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -1812,7 +1812,7 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) {
       AllocaInst *AI;
       if ((AI = dyn_cast<AllocaInst>(Val)) && AI->getAlignment() < PrefAlign &&
           DL->getTypeAllocSize(AI->getAllocatedType()) >= MinSize + Offset2)
-        AI->setAlignment(PrefAlign);
+        AI->setAlignment(MaybeAlign(PrefAlign));
       // Global variables can only be aligned if they are defined in this
       // object (i.e. they are uniquely initialized in this object), and
       // over-aligning global variables that have an explicit section is
@@ -3332,7 +3332,7 @@ class AddressingModeCombiner {
         // So the values are different and does not match. So we need them to
         // match. (But we register no more than one match per PHI node, so that
         // we won't later try to replace them twice.)
-        if (!MatchedPHIs.insert(FirstPhi).second)
+        if (MatchedPHIs.insert(FirstPhi).second)
           Matcher.insert({ FirstPhi, SecondPhi });
         // But me must check it.
         WorkList.push_back({ FirstPhi, SecondPhi });
diff --git a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
index 22575d526b3ce..702e7e244bcec 100644
--- a/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/llvm/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -457,6 +457,7 @@ BreakAntiDependencies(const std::vector<SUnit> &SUnits,
     if (!Max || SU->getDepth() + SU->Latency > Max->getDepth() + Max->Latency)
       Max = SU;
   }
+  assert(Max && "Failed to find bottom of the critical path");
 
 #ifndef NDEBUG
   {
diff --git a/llvm/lib/CodeGen/GCRootLowering.cpp b/llvm/lib/CodeGen/GCRootLowering.cpp
index 90571d090bfbc..0dc0a5bce7475 100644
--- a/llvm/lib/CodeGen/GCRootLowering.cpp
+++ b/llvm/lib/CodeGen/GCRootLowering.cpp
@@ -249,7 +249,7 @@ GCMachineCodeAnalysis::GCMachineCodeAnalysis() : MachineFunctionPass(ID) {}
 void GCMachineCodeAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
   MachineFunctionPass::getAnalysisUsage(AU);
   AU.setPreservesAll();
-  AU.addRequired<MachineModuleInfo>();
+  AU.addRequired<MachineModuleInfoWrapperPass>();
   AU.addRequired<GCModuleInfo>();
 }
 
@@ -310,7 +310,7 @@ bool GCMachineCodeAnalysis::runOnMachineFunction(MachineFunction &MF) {
     return false;
 
   FI = &getAnalysis<GCModuleInfo>().getFunctionInfo(MF.getFunction());
-  MMI = &getAnalysis<MachineModuleInfo>();
+  MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
   TII = MF.getSubtarget().getInstrInfo();
 
   // Find the size of the stack frame.  There may be no correct static frame
diff --git a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
index 9280c9efc93bc..d05e2091c71bd 100644
--- a/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CallLowering.cpp
@@ -379,10 +379,12 @@ bool CallLowering::handleAssignments(CCState &CCInfo,
 }
 
 bool CallLowering::analyzeArgInfo(CCState &CCState,
-                                     SmallVectorImpl<ArgInfo> &Args,
-                                     CCAssignFn &Fn) const {
+                                  SmallVectorImpl<ArgInfo> &Args,
+                                  CCAssignFn &AssignFnFixed,
+                                  CCAssignFn &AssignFnVarArg) const {
   for (unsigned i = 0, e = Args.size(); i < e; ++i) {
     MVT VT = MVT::getVT(Args[i].Ty);
+    CCAssignFn &Fn = Args[i].IsFixed ? AssignFnFixed : AssignFnVarArg;
     if (Fn(i, VT, VT, CCValAssign::Full, Args[i].Flags[0], CCState)) {
       // Bail out on anything we can't handle.
       LLVM_DEBUG(dbgs() << "Cannot analyze " << EVT(VT).getEVTString()
@@ -396,8 +398,10 @@ bool CallLowering::analyzeArgInfo(CCState &CCState,
 bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
                                      MachineFunction &MF,
                                      SmallVectorImpl<ArgInfo> &InArgs,
-                                     CCAssignFn &CalleeAssignFn,
-                                     CCAssignFn &CallerAssignFn) const {
+                                     CCAssignFn &CalleeAssignFnFixed,
+                                     CCAssignFn &CalleeAssignFnVarArg,
+                                     CCAssignFn &CallerAssignFnFixed,
+                                     CCAssignFn &CallerAssignFnVarArg) const {
   const Function &F = MF.getFunction();
   CallingConv::ID CalleeCC = Info.CallConv;
   CallingConv::ID CallerCC = F.getCallingConv();
@@ -407,12 +411,14 @@ bool CallLowering::resultsCompatible(CallLoweringInfo &Info,
 
   SmallVector<CCValAssign, 16> ArgLocs1;
   CCState CCInfo1(CalleeCC, false, MF, ArgLocs1, F.getContext());
-  if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFn))
+  if (!analyzeArgInfo(CCInfo1, InArgs, CalleeAssignFnFixed,
+                      CalleeAssignFnVarArg))
     return false;
 
   SmallVector<CCValAssign, 16> ArgLocs2;
   CCState CCInfo2(CallerCC, false, MF, ArgLocs2, F.getContext());
-  if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFn))
+  if (!analyzeArgInfo(CCInfo2, InArgs, CallerAssignFnFixed,
+                      CalleeAssignFnVarArg))
     return false;
 
   // We need the argument locations to match up exactly. If there's more in
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 76e82a90a3239..974b500d4abb5 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -673,10 +673,8 @@ static bool findGISelOptimalMemOpLowering(
       // SDAGisms map cleanly to GISel concepts.
       if (NewTy.isVector())
         NewTy = NewTy.getSizeInBits() > 64 ? LLT::scalar(64) : LLT::scalar(32);
+      NewTy = LLT::scalar(PowerOf2Floor(NewTy.getSizeInBits() - 1));
       unsigned NewTySize = NewTy.getSizeInBytes();
-
-      NewTy = LLT::scalar(PowerOf2Floor(NewTy.getSizeInBits()-1));
-      NewTySize = NewTy.getSizeInBytes();
       assert(NewTySize > 0 && "Could not find appropriate type");
 
       // If the new LLT cannot cover all of the remaining bits, then consider
@@ -868,7 +866,7 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst,
   bool DstAlignCanChange = false;
   MachineFrameInfo &MFI = MF.getFrameInfo();
   bool OptSize = shouldLowerMemFuncForSize(MF);
-  unsigned Align = MinAlign(DstAlign, SrcAlign);
+  unsigned Alignment = MinAlign(DstAlign, SrcAlign);
 
   MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
   if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
@@ -887,7 +885,8 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst,
   MachinePointerInfo SrcPtrInfo = SrcMMO.getPointerInfo();
 
   if (!findGISelOptimalMemOpLowering(
-          MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Align), SrcAlign,
+          MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Alignment),
+          SrcAlign,
           /*IsMemset=*/false,
           /*ZeroMemset=*/false, /*MemcpyStrSrc=*/false,
           /*AllowOverlap=*/!IsVolatile, DstPtrInfo.getAddrSpace(),
@@ -903,16 +902,16 @@ bool CombinerHelper::optimizeMemcpy(MachineInstr &MI, Register Dst,
     // realignment.
     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
     if (!TRI->needsStackRealignment(MF))
-      while (NewAlign > Align &&
-             DL.exceedsNaturalStackAlignment(llvm::Align(NewAlign)))
-          NewAlign /= 2;
+      while (NewAlign > Alignment &&
+             DL.exceedsNaturalStackAlignment(Align(NewAlign)))
+        NewAlign /= 2;
 
-    if (NewAlign > Align) {
+    if (NewAlign > Alignment) {
       unsigned FI = FIDef->getOperand(1).getIndex();
       // Give the stack frame object a larger alignment if needed.
       if (MFI.getObjectAlignment(FI) < NewAlign)
         MFI.setObjectAlignment(FI, NewAlign);
-      Align = NewAlign;
+      Alignment = NewAlign;
     }
   }
 
@@ -975,7 +974,7 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
   bool DstAlignCanChange = false;
   MachineFrameInfo &MFI = MF.getFrameInfo();
   bool OptSize = shouldLowerMemFuncForSize(MF);
-  unsigned Align = MinAlign(DstAlign, SrcAlign);
+  unsigned Alignment = MinAlign(DstAlign, SrcAlign);
 
   MachineInstr *FIDef = getOpcodeDef(TargetOpcode::G_FRAME_INDEX, Dst, MRI);
   if (FIDef && !MFI.isFixedObjectIndex(FIDef->getOperand(1).getIndex()))
@@ -993,7 +992,8 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
   // to a bug in it's findOptimalMemOpLowering implementation. For now do the
   // same thing here.
   if (!findGISelOptimalMemOpLowering(
-          MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Align), SrcAlign,
+          MemOps, Limit, KnownLen, (DstAlignCanChange ? 0 : Alignment),
+          SrcAlign,
           /*IsMemset=*/false,
           /*ZeroMemset=*/false, /*MemcpyStrSrc=*/false,
           /*AllowOverlap=*/false, DstPtrInfo.getAddrSpace(),
@@ -1009,16 +1009,16 @@ bool CombinerHelper::optimizeMemmove(MachineInstr &MI, Register Dst,
     // realignment.
     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
     if (!TRI->needsStackRealignment(MF))
-      while (NewAlign > Align &&
-             DL.exceedsNaturalStackAlignment(llvm::Align(NewAlign)))
-          NewAlign /= 2;
+      while (NewAlign > Alignment &&
+             DL.exceedsNaturalStackAlignment(Align(NewAlign)))
+        NewAlign /= 2;
 
-    if (NewAlign > Align) {
+    if (NewAlign > Alignment) {
       unsigned FI = FIDef->getOperand(1).getIndex();
       // Give the stack frame object a larger alignment if needed.
       if (MFI.getObjectAlignment(FI) < NewAlign)
         MFI.setObjectAlignment(FI, NewAlign);
-      Align = NewAlign;
+      Alignment = NewAlign;
     }
   }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
index c0391f7771f8e..8f9b7ddeabf20 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelKnownBits.cpp
@@ -119,7 +119,8 @@ void GISelKnownBits::computeKnownBitsImpl(Register R, KnownBits &Known,
 
   switch (Opcode) {
   default:
-    TL.computeKnownBitsForTargetInstr(R, Known, DemandedElts, MRI, Depth);
+    TL.computeKnownBitsForTargetInstr(*this, R, Known, DemandedElts, MRI,
+                                      Depth);
     break;
   case TargetOpcode::COPY: {
     MachineOperand Dst = MI.getOperand(0);
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 66082df0108b0..37ac96e52905e 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -589,8 +589,8 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
     Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
     Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
   } else {
-    assert(CB.PredInfo.Pred == CmpInst::ICMP_ULE &&
-           "Can only handle ULE ranges");
+    assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
+           "Can only handle SLE ranges");
 
     const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
     const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
@@ -599,7 +599,7 @@ void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
       Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
       Cond =
-          MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0);
+          MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
     } else {
       const LLT &CmpTy = MRI->getType(CmpOpReg);
       auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS);
@@ -729,7 +729,7 @@ bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
     MHS = nullptr;
   } else {
     // Check I->Low <= Cond <= I->High.
-    Pred = CmpInst::ICMP_ULE;
+    Pred = CmpInst::ICMP_SLE;
     LHS = I->Low;
     MHS = Cond;
     RHS = I->High;
@@ -1143,6 +1143,11 @@ bool IRTranslator::translateMemFunc(const CallInst &CI,
     DstAlign = std::max<unsigned>(MSI->getDestAlignment(), 1);
   }
 
+  // We need to propagate the tail call flag from the IR inst as an argument.
+  // Otherwise, we have to pessimize and assume later that we cannot tail call
+  // any memory intrinsics.
+  ICall.addImm(CI.isTailCall() ? 1 : 0);
+
   // Create mem operands to store the alignment and volatile info.
   auto VolFlag = IsVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone;
   ICall.addMemOperand(MF->getMachineMemOperand(
@@ -1173,7 +1178,7 @@ void IRTranslator::getStackGuard(Register DstReg,
                MachineMemOperand::MODereferenceable;
   MachineMemOperand *MemRef =
       MF->getMachineMemOperand(MPInfo, Flags, DL->getPointerSizeInBits() / 8,
-                               DL->getPointerABIAlignment(0));
+                               DL->getPointerABIAlignment(0).value());
   MIB.setMemRefs({MemRef});
 }
 
@@ -1564,7 +1569,9 @@ bool IRTranslator::translateCallSite(const ImmutableCallSite &CS,
     Args.push_back(getOrCreateVRegs(*Arg));
   }
 
-  MF->getFrameInfo().setHasCalls(true);
+  // We don't set HasCalls on MFI here yet because call lowering may decide to
+  // optimize into tail calls. Instead, we defer that to selection where a final
+  // scan is done to check if any instructions are calls.
   bool Success =
       CLI->lowerCall(MIRBuilder, CS, Res, Args, SwiftErrorVReg,
                      [&]() { return getOrCreateVReg(*CS.getCalledValue()); });
@@ -1617,14 +1624,29 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   if (isa<FPMathOperator>(CI))
     MIB->copyIRFlags(CI);
 
-  for (auto &Arg : CI.arg_operands()) {
+  for (auto &Arg : enumerate(CI.arg_operands())) {
     // Some intrinsics take metadata parameters. Reject them.
-    if (isa<MetadataAsValue>(Arg))
-      return false;
-    ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg);
-    if (VRegs.size() > 1)
+    if (isa<MetadataAsValue>(Arg.value()))
       return false;
-    MIB.addUse(VRegs[0]);
+
+    // If this is required to be an immediate, don't materialize it in a
+    // register.
+    if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
+      if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
+        // imm arguments are more convenient than cimm (and realistically
+        // probably sufficient), so use them.
+        assert(CI->getBitWidth() <= 64 &&
+               "large intrinsic immediates not handled");
+        MIB.addImm(CI->getSExtValue());
+      } else {
+        MIB.addFPImm(cast<ConstantFP>(Arg.value()));
+      }
+    } else {
+      ArrayRef<Register> VRegs = getOrCreateVRegs(*Arg.value());
+      if (VRegs.size() > 1)
+        return false;
+      MIB.addUse(VRegs[0]);
+    }
   }
 
   // Add a MachineMemOperand if it is a target mem intrinsic.
@@ -2171,6 +2193,20 @@ void IRTranslator::finalizeFunction() {
   FuncInfo.clear();
 }
 
+/// Returns true if a BasicBlock \p BB within a variadic function contains a
+/// variadic musttail call.
+static bool checkForMustTailInVarArgFn(bool IsVarArg, const BasicBlock &BB) {
+  if (!IsVarArg)
+    return false;
+
+  // Walk the block backwards, because tail calls usually only appear at the end
+  // of a block.
+  return std::any_of(BB.rbegin(), BB.rend(), [](const Instruction &I) {
+    const auto *CI = dyn_cast<CallInst>(&I);
+    return CI && CI->isMustTailCall();
+  });
+}
+
 bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   MF = &CurMF;
   const Function &F = MF->getFunction();
@@ -2232,6 +2268,9 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   SwiftError.setFunction(CurMF);
   SwiftError.createEntriesInEntryBlock(DbgLoc);
 
+  bool IsVarArg = F.isVarArg();
+  bool HasMustTailInVarArgFn = false;
+
   // Create all blocks, in IR order, to preserve the layout.
   for (const BasicBlock &BB: F) {
     auto *&MBB = BBToMBB[&BB];
@@ -2241,8 +2280,13 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
 
     if (BB.hasAddressTaken())
       MBB->setHasAddressTaken();
+
+    if (!HasMustTailInVarArgFn)
+      HasMustTailInVarArgFn = checkForMustTailInVarArgFn(IsVarArg, BB);
   }
 
+  MF->getFrameInfo().setHasMustTailInVarArgFunc(HasMustTailInVarArgFn);
+
   // Make our arguments/constants entry block fallthrough to the IR entry block.
   EntryBB->addSuccessor(&getMBB(F.front()));
 
diff --git a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
index bf441afc9dc80..7c4fd2d140d36 100644
--- a/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InstructionSelect.cpp
@@ -17,7 +17,9 @@
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
@@ -223,6 +225,22 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) {
   auto &TLI = *MF.getSubtarget().getTargetLowering();
   TLI.finalizeLowering(MF);
 
+  // Determine if there are any calls in this machine function. Ported from
+  // SelectionDAG.
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+  for (const auto &MBB : MF) {
+    if (MFI.hasCalls() && MF.hasInlineAsm())
+      break;
+
+    for (const auto &MI : MBB) {
+      if ((MI.isCall() && !MI.isReturn()) || MI.isStackAligningInlineAsm())
+        MFI.setHasCalls(true);
+      if (MI.isInlineAsm())
+        MF.setHasInlineAsm(true);
+    }
+  }
+
+
   LLVM_DEBUG({
     dbgs() << "Rules covered by selecting function: " << MF.getName() << ":";
     for (auto RuleID : CoverageInfo.covered())
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index fbb817e127a20..be3e58e165029 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -327,6 +327,35 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
   llvm_unreachable("Unknown libcall function");
 }
 
+/// True if an instruction is in tail position in its caller. Intended for
+/// legalizing libcalls as tail calls when possible.
+static bool isLibCallInTailPosition(MachineInstr &MI) {
+  const Function &F = MI.getParent()->getParent()->getFunction();
+
+  // Conservatively require the attributes of the call to match those of
+  // the return. Ignore NoAlias and NonNull because they don't affect the
+  // call sequence.
+  AttributeList CallerAttrs = F.getAttributes();
+  if (AttrBuilder(CallerAttrs, AttributeList::ReturnIndex)
+          .removeAttribute(Attribute::NoAlias)
+          .removeAttribute(Attribute::NonNull)
+          .hasAttributes())
+    return false;
+
+  // It's not safe to eliminate the sign / zero extension of the return value.
+  if (CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::ZExt) ||
+      CallerAttrs.hasAttribute(AttributeList::ReturnIndex, Attribute::SExt))
+    return false;
+
+  // Only tail call if the following instruction is a standard return.
+  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
+  MachineInstr *Next = MI.getNextNode();
+  if (!Next || TII.isTailCall(*Next) || !Next->isReturn())
+    return false;
+
+  return true;
+}
+
 LegalizerHelper::LegalizeResult
 llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
                     const CallLowering::ArgInfo &Result,
@@ -335,8 +364,6 @@ llvm::createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
   auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
   const char *Name = TLI.getLibcallName(Libcall);
 
-  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
-
   CallLowering::CallLoweringInfo Info;
   Info.CallConv = TLI.getLibcallCallingConv(Libcall);
   Info.Callee = MachineOperand::CreateES(Name);
@@ -368,7 +395,8 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
   auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
 
   SmallVector<CallLowering::ArgInfo, 3> Args;
-  for (unsigned i = 1; i < MI.getNumOperands(); i++) {
+  // Add all the args, except for the last which is an imm denoting 'tail'.
+  for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
     Register Reg = MI.getOperand(i).getReg();
 
     // Need derive an IR type for call lowering.
@@ -401,16 +429,30 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
   const char *Name = TLI.getLibcallName(RTLibcall);
 
   MIRBuilder.setInstr(MI);
-  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
 
   CallLowering::CallLoweringInfo Info;
   Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
   Info.Callee = MachineOperand::CreateES(Name);
   Info.OrigRet = CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx));
+  Info.IsTailCall = MI.getOperand(MI.getNumOperands() - 1).getImm() == 1 &&
+                    isLibCallInTailPosition(MI);
+
   std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
   if (!CLI.lowerCall(MIRBuilder, Info))
     return LegalizerHelper::UnableToLegalize;
 
+  if (Info.LoweredTailCall) {
+    assert(Info.IsTailCall && "Lowered tail call when it wasn't a tail call?");
+    // We must have a return following the call to get past
+    // isLibCallInTailPosition.
+    assert(MI.getNextNode() && MI.getNextNode()->isReturn() &&
+           "Expected instr following MI to be a return?");
+
+    // We lowered a tail call, so the call is now the return from the block.
+    // Delete the old return.
+    MI.getNextNode()->eraseFromParent();
+  }
+
   return LegalizerHelper::Legalized;
 }
 
@@ -888,7 +930,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI,
       for (unsigned j = 1; j < MI.getNumOperands(); j += 2)
         MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1));
     }
-    MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
+    MIRBuilder.setInsertPt(MBB, MBB.getFirstNonPHI());
     MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
     Observer.changedInstr(MI);
     MI.eraseFromParent();
@@ -1580,13 +1622,15 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
 
   case TargetOpcode::G_FPTOSI:
   case TargetOpcode::G_FPTOUI:
-    if (TypeIdx != 0)
-      return UnableToLegalize;
     Observer.changingInstr(MI);
-    widenScalarDst(MI, WideTy);
+
+    if (TypeIdx == 0)
+      widenScalarDst(MI, WideTy);
+    else
+      widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT);
+
     Observer.changedInstr(MI);
     return Legalized;
-
   case TargetOpcode::G_SITOFP:
     if (TypeIdx != 1)
       return UnableToLegalize;
diff --git a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
index 8dd9969212845..f882ecbf5db32 100644
--- a/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/Localizer.cpp
@@ -220,5 +220,6 @@ bool Localizer::runOnMachineFunction(MachineFunction &MF) {
   LocalizedSetVecT LocalizedInstrs;
 
   bool Changed = localizeInterBlock(MF, LocalizedInstrs);
-  return Changed |= localizeIntraBlock(LocalizedInstrs);
+  Changed |= localizeIntraBlock(LocalizedInstrs);
+  return Changed;
 }
diff --git a/llvm/lib/CodeGen/IfConversion.cpp b/llvm/lib/CodeGen/IfConversion.cpp
index 99f59f2f946af..82acc9e932ef1 100644
--- a/llvm/lib/CodeGen/IfConversion.cpp
+++ b/llvm/lib/CodeGen/IfConversion.cpp
@@ -356,8 +356,10 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
   if (!PreRegAlloc) {
     // Tail merge tend to expose more if-conversion opportunities.
     BranchFolder BF(true, false, MBFI, *MBPI);
-    BFChange = BF.OptimizeFunction(MF, TII, ST.getRegisterInfo(),
-                                   getAnalysisIfAvailable<MachineModuleInfo>());
+    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+    BFChange = BF.OptimizeFunction(
+        MF, TII, ST.getRegisterInfo(),
+        MMIWP ? &MMIWP->getMMI() : nullptr);
   }
 
   LLVM_DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum << ") \'"
@@ -496,8 +498,10 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
 
   if (MadeChange && IfCvtBranchFold) {
     BranchFolder BF(false, false, MBFI, *MBPI);
-    BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(),
-                        getAnalysisIfAvailable<MachineModuleInfo>());
+    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+    BF.OptimizeFunction(
+        MF, TII, MF.getSubtarget().getRegisterInfo(),
+        MMIWP ? &MMIWP->getMMI() : nullptr);
   }
 
   MadeChange |= BFChange;
@@ -569,6 +573,9 @@ bool IfConverter::ValidTriangle(BBInfo &TrueBBI, BBInfo &FalseBBI,
                                 bool FalseBranch, unsigned &Dups,
                                 BranchProbability Prediction) const {
   Dups = 0;
+  if (TrueBBI.BB == FalseBBI.BB)
+    return false;
+
   if (TrueBBI.IsBeingAnalyzed || TrueBBI.IsDone)
     return false;
 
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index 9525da849e2a6..770c4952d169d 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -940,8 +940,8 @@ struct VectorInfo {
   /// \param V input value
   /// \param Result result polynomial
   static void computePolynomial(Value &V, Polynomial &Result) {
-    if (isa<BinaryOperator>(&V))
-      computePolynomialBinOp(*dyn_cast<BinaryOperator>(&V), Result);
+    if (auto *BO = dyn_cast<BinaryOperator>(&V))
+      computePolynomialBinOp(*BO, Result);
     else
       Result = Polynomial(&V);
   }
diff --git a/llvm/lib/CodeGen/LLVMTargetMachine.cpp b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
index cd70792146d2b..1c362aec6e674 100644
--- a/llvm/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/llvm/lib/CodeGen/LLVMTargetMachine.cpp
@@ -96,14 +96,15 @@ LLVMTargetMachine::getTargetTransformInfo(const Function &F) {
 /// addPassesToX helper drives creation and initialization of TargetPassConfig.
 static TargetPassConfig *
 addPassesToGenerateCode(LLVMTargetMachine &TM, PassManagerBase &PM,
-                        bool DisableVerify, MachineModuleInfo &MMI) {
+                        bool DisableVerify,
+                        MachineModuleInfoWrapperPass &MMIWP) {
   // Targets may override createPassConfig to provide a target-specific
   // subclass.
   TargetPassConfig *PassConfig = TM.createPassConfig(PM);
   // Set PassConfig options provided by TargetMachine.
   PassConfig->setDisableVerify(DisableVerify);
   PM.add(PassConfig);
-  PM.add(&MMI);
+  PM.add(&MMIWP);
 
   if (PassConfig->addISelPasses())
     return nullptr;
@@ -186,17 +187,15 @@ bool LLVMTargetMachine::addAsmPrinter(PassManagerBase &PM,
   return false;
 }
 
-bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
-                                            raw_pwrite_stream &Out,
-                                            raw_pwrite_stream *DwoOut,
-                                            CodeGenFileType FileType,
-                                            bool DisableVerify,
-                                            MachineModuleInfo *MMI) {
+bool LLVMTargetMachine::addPassesToEmitFile(
+    PassManagerBase &PM, raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut,
+    CodeGenFileType FileType, bool DisableVerify,
+    MachineModuleInfoWrapperPass *MMIWP) {
   // Add common CodeGen passes.
-  if (!MMI)
-    MMI = new MachineModuleInfo(this);
+  if (!MMIWP)
+    MMIWP = new MachineModuleInfoWrapperPass(this);
   TargetPassConfig *PassConfig =
-      addPassesToGenerateCode(*this, PM, DisableVerify, *MMI);
+      addPassesToGenerateCode(*this, PM, DisableVerify, *MMIWP);
   if (!PassConfig)
     return true;
 
@@ -206,12 +205,13 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
       // testing to be meaningful, we need to ensure that the symbols created
       // are MCSymbolXCOFF variants, which requires that
       // the TargetLoweringObjectFile instance has been initialized.
-      MCContext &Ctx = MMI->getContext();
+      MCContext &Ctx = MMIWP->getMMI().getContext();
       const_cast<TargetLoweringObjectFile &>(*this->getObjFileLowering())
           .Initialize(Ctx, *this);
     }
     PM.add(createPrintMIRPass(Out));
-  } else if (addAsmPrinter(PM, Out, DwoOut, FileType, MMI->getContext()))
+  } else if (addAsmPrinter(PM, Out, DwoOut, FileType,
+                           MMIWP->getMMI().getContext()))
     return true;
 
   PM.add(createFreeMachineFunctionPass());
@@ -227,15 +227,15 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM, MCContext *&Ctx,
                                           raw_pwrite_stream &Out,
                                           bool DisableVerify) {
   // Add common CodeGen passes.
-  MachineModuleInfo *MMI = new MachineModuleInfo(this);
+  MachineModuleInfoWrapperPass *MMIWP = new MachineModuleInfoWrapperPass(this);
   TargetPassConfig *PassConfig =
-      addPassesToGenerateCode(*this, PM, DisableVerify, *MMI);
+      addPassesToGenerateCode(*this, PM, DisableVerify, *MMIWP);
   if (!PassConfig)
     return true;
   assert(TargetPassConfig::willCompleteCodeGenPipeline() &&
          "Cannot emit MC with limited codegen pipeline");
 
-  Ctx = &MMI->getContext();
+  Ctx = &MMIWP->getMMI().getContext();
   if (Options.MCOptions.MCSaveTempLabels)
     Ctx->setAllowTemporaryLabels(false);
 
diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
index f8c4dd6655997..0a628d53923ae 100644
--- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp
@@ -641,7 +641,7 @@ bool MIParser::parseBasicBlockDefinition(
     return error(Loc, Twine("redefinition of machine basic block with id #") +
                           Twine(ID));
   if (Alignment)
-    MBB->setLogAlignment(Log2_32(Alignment));
+    MBB->setAlignment(Align(Alignment));
   if (HasAddressTaken)
     MBB->setHasAddressTaken();
   MBB->setIsEHPad(IsLandingPad);
diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
index 72d3d1d14e935..55fac93d8991f 100644
--- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
+++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp
@@ -355,7 +355,7 @@ bool MIRParserImpl::initializeCallSiteInfo(
     if (MILoc.Offset >= CallB->size())
       return error(Twine(MF.getName()) +
                    Twine(" call instruction offset out of range.") +
-                   "Unable to reference instruction at bb: " +
+                   " Unable to reference instruction at bb: " +
                    Twine(MILoc.BlockNum) + " at offset:" + Twine(MILoc.Offset));
     auto CallI = std::next(CallB->instr_begin(), MILoc.Offset);
     if (!CallI->isCall(MachineInstr::IgnoreBundle))
@@ -393,7 +393,7 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF,
   }
 
   if (YamlMF.Alignment)
-    MF.setAlignment(llvm::Align(YamlMF.Alignment));
+    MF.setAlignment(Align(YamlMF.Alignment));
   MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice);
   MF.setHasWinCFI(YamlMF.HasWinCFI);
 
diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp
index 415f28b094bcf..1a4e21ac06a9a 100644
--- a/llvm/lib/CodeGen/MIRPrinter.cpp
+++ b/llvm/lib/CodeGen/MIRPrinter.cpp
@@ -629,10 +629,9 @@ void MIPrinter::print(const MachineBasicBlock &MBB) {
     OS << "landing-pad";
     HasAttributes = true;
   }
-  if (MBB.getLogAlignment()) {
+  if (MBB.getAlignment() != Align::None()) {
     OS << (HasAttributes ? ", " : " (");
-    OS << "align "
-       << (1UL << MBB.getLogAlignment());
+    OS << "align " << MBB.getAlignment().value();
     HasAttributes = true;
   }
   if (HasAttributes)
diff --git a/llvm/lib/CodeGen/MIRVRegNamerUtils.h b/llvm/lib/CodeGen/MIRVRegNamerUtils.h
index 06d16f80cd030..c5b52a9685380 100644
--- a/llvm/lib/CodeGen/MIRVRegNamerUtils.h
+++ b/llvm/lib/CodeGen/MIRVRegNamerUtils.h
@@ -1,3 +1,4 @@
+
 //===------------ MIRVRegNamerUtils.h - MIR VReg Renaming Utilities -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
@@ -13,6 +14,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#ifndef LLVM_LIB_CODEGEN_MIRVREGNAMERUTILS_H
+#define LLVM_LIB_CODEGEN_MIRVREGNAMERUTILS_H
+
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
@@ -23,7 +27,7 @@
 
 #include <queue>
 
-using namespace llvm;
+namespace llvm {
 
 /// NamedVRegCursor - The cursor is an object that keeps track of what the next
 /// vreg name should be. It does book keeping to determine when to skip the
@@ -81,3 +85,7 @@ class NamedVRegCursor {
   // @return changed
   bool renameVRegs(MachineBasicBlock *MBB);
 };
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp
index bd2ab45f6696e..854bef3aab050 100644
--- a/llvm/lib/CodeGen/MachineBasicBlock.cpp
+++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp
@@ -326,9 +326,9 @@ void MachineBasicBlock::print(raw_ostream &OS, ModuleSlotTracker &MST,
     OS << "landing-pad";
     HasAttributes = true;
   }
-  if (getLogAlignment()) {
+  if (getAlignment() != Align::None()) {
     OS << (HasAttributes ? ", " : " (");
-    OS << "align " << getLogAlignment();
+    OS << "align " << Log2(getAlignment());
     HasAttributes = true;
   }
   if (HasAttributes)
diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index babc05195a1a5..27438ecf0adc8 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -2807,7 +2807,7 @@ void MachineBlockPlacement::alignBlocks() {
     if (!L)
       continue;
 
-    const llvm::Align Align = TLI->getPrefLoopAlignment(L);
+    const Align Align = TLI->getPrefLoopAlignment(L);
     if (Align == 1)
       continue; // Don't care about loop alignment.
 
@@ -2832,7 +2832,7 @@ void MachineBlockPlacement::alignBlocks() {
     // Force alignment if all the predecessors are jumps. We already checked
     // that the block isn't cold above.
     if (!LayoutPred->isSuccessor(ChainBB)) {
-      ChainBB->setLogAlignment(Log2(Align));
+      ChainBB->setAlignment(Align);
       continue;
     }
 
@@ -2844,7 +2844,7 @@ void MachineBlockPlacement::alignBlocks() {
         MBPI->getEdgeProbability(LayoutPred, ChainBB);
     BlockFrequency LayoutEdgeFreq = MBFI->getBlockFreq(LayoutPred) * LayoutProb;
     if (LayoutEdgeFreq <= (Freq * ColdProb))
-      ChainBB->setLogAlignment(Log2(Align));
+      ChainBB->setAlignment(Align);
   }
 }
 
@@ -3082,8 +3082,9 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
     BranchFolder BF(/*EnableTailMerge=*/true, /*CommonHoist=*/false, *MBFI,
                     *MBPI, TailMergeSize);
 
+    auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
     if (BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(),
-                            getAnalysisIfAvailable<MachineModuleInfo>(), MLI,
+                            MMIWP ? &MMIWP->getMMI() : nullptr, MLI,
                             /*AfterPlacement=*/true)) {
       // Redo the layout if tail merging creates/removes/moves blocks.
       BlockToChain.clear();
@@ -3109,14 +3110,14 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) {
   if (AlignAllBlock)
     // Align all of the blocks in the function to a specific alignment.
     for (MachineBasicBlock &MBB : MF)
-      MBB.setLogAlignment(AlignAllBlock);
+      MBB.setAlignment(Align(1ULL << AlignAllBlock));
   else if (AlignAllNonFallThruBlocks) {
     // Align all of the blocks that have no fall-through predecessors to a
     // specific alignment.
     for (auto MBI = std::next(MF.begin()), MBE = MF.end(); MBI != MBE; ++MBI) {
       auto LayoutPred = std::prev(MBI);
       if (!LayoutPred->isSuccessor(&*MBI))
-        MBI->setLogAlignment(AlignAllNonFallThruBlocks);
+        MBI->setAlignment(Align(1ULL << AlignAllNonFallThruBlocks));
     }
   }
   if (ViewBlockLayoutWithBFI != GVDT_None &&
diff --git a/llvm/lib/CodeGen/MachineDominators.cpp b/llvm/lib/CodeGen/MachineDominators.cpp
index 1dfba8638c227..706c706d75274 100644
--- a/llvm/lib/CodeGen/MachineDominators.cpp
+++ b/llvm/lib/CodeGen/MachineDominators.cpp
@@ -18,12 +18,15 @@
 
 using namespace llvm;
 
+namespace llvm {
 // Always verify dominfo if expensive checking is enabled.
 #ifdef EXPENSIVE_CHECKS
-static bool VerifyMachineDomInfo = true;
+bool VerifyMachineDomInfo = true;
 #else
-static bool VerifyMachineDomInfo = false;
+bool VerifyMachineDomInfo = false;
 #endif
+} // namespace llvm
+
 static cl::opt<bool, true> VerifyMachineDomInfoX(
     "verify-machine-dom-info", cl::location(VerifyMachineDomInfo), cl::Hidden,
     cl::desc("Verify machine dominator info (time consuming)"));
@@ -64,21 +67,11 @@ void MachineDominatorTree::releaseMemory() {
 }
 
 void MachineDominatorTree::verifyAnalysis() const {
-  if (DT && VerifyMachineDomInfo) {
-    MachineFunction &F = *getRoot()->getParent();
-
-    DomTreeBase<MachineBasicBlock> OtherDT;
-    OtherDT.recalculate(F);
-    if (getRootNode()->getBlock() != OtherDT.getRootNode()->getBlock() ||
-        DT->compare(OtherDT)) {
-      errs() << "MachineDominatorTree for function " << F.getName()
-            << " is not up to date!\nComputed:\n";
-      DT->print(errs());
-      errs() << "\nActual:\n";
-      OtherDT.print(errs());
+  if (DT && VerifyMachineDomInfo)
+    if (!DT->verify(DomTreeT::VerificationLevel::Basic)) {
+      errs() << "MachineDominatorTree verification failed\n";
       abort();
     }
-  }
 }
 
 void MachineDominatorTree::print(raw_ostream &OS, const Module*) const {
diff --git a/llvm/lib/CodeGen/MachineFrameInfo.cpp b/llvm/lib/CodeGen/MachineFrameInfo.cpp
index e045e4ed41442..604f5145b1a0f 100644
--- a/llvm/lib/CodeGen/MachineFrameInfo.cpp
+++ b/llvm/lib/CodeGen/MachineFrameInfo.cpp
@@ -28,26 +28,26 @@
 
 using namespace llvm;
 
-void MachineFrameInfo::ensureMaxAlignment(llvm::Align Align) {
+void MachineFrameInfo::ensureMaxAlignment(Align Alignment) {
   if (!StackRealignable)
-    assert(Align <= StackAlignment &&
-           "For targets without stack realignment, Align is out of limit!");
-  if (MaxAlignment < Align) MaxAlignment = Align;
+    assert(Alignment <= StackAlignment &&
+           "For targets without stack realignment, Alignment is out of limit!");
+  if (MaxAlignment < Alignment)
+    MaxAlignment = Alignment;
 }
 
 /// Clamp the alignment if requested and emit a warning.
-static inline llvm::Align clampStackAlignment(bool ShouldClamp,
-                                              llvm::Align Align,
-                                              llvm::Align StackAlign) {
-  if (!ShouldClamp || Align <= StackAlign)
-    return Align;
-  LLVM_DEBUG(dbgs() << "Warning: requested alignment " << Align.value()
-                    << " exceeds the stack alignment " << StackAlign.value()
+static inline Align clampStackAlignment(bool ShouldClamp, Align Alignment,
+                                        Align StackAlignment) {
+  if (!ShouldClamp || Alignment <= StackAlignment)
+    return Alignment;
+  LLVM_DEBUG(dbgs() << "Warning: requested alignment " << Alignment.value()
+                    << " exceeds the stack alignment " << StackAlignment.value()
                     << " when stack realignment is off" << '\n');
-  return StackAlign;
+  return StackAlignment;
 }
 
-int MachineFrameInfo::CreateStackObject(uint64_t Size, llvm::Align Alignment,
+int MachineFrameInfo::CreateStackObject(uint64_t Size, Align Alignment,
                                         bool IsSpillSlot,
                                         const AllocaInst *Alloca,
                                         uint8_t StackID) {
@@ -62,8 +62,7 @@ int MachineFrameInfo::CreateStackObject(uint64_t Size, llvm::Align Alignment,
   return Index;
 }
 
-int MachineFrameInfo::CreateSpillStackObject(uint64_t Size,
-                                             llvm::Align Alignment) {
+int MachineFrameInfo::CreateSpillStackObject(uint64_t Size, Align Alignment) {
   Alignment = clampStackAlignment(!StackRealignable, Alignment, StackAlignment);
   CreateStackObject(Size, Alignment, true);
   int Index = (int)Objects.size() - NumFixedObjects - 1;
@@ -71,7 +70,7 @@ int MachineFrameInfo::CreateSpillStackObject(uint64_t Size,
   return Index;
 }
 
-int MachineFrameInfo::CreateVariableSizedObject(llvm::Align Alignment,
+int MachineFrameInfo::CreateVariableSizedObject(Align Alignment,
                                                 const AllocaInst *Alloca) {
   HasVarSizedObjects = true;
   Alignment = clampStackAlignment(!StackRealignable, Alignment, StackAlignment);
@@ -89,8 +88,8 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset,
   // object is 16-byte aligned. Note that unlike the non-fixed case, if the
   // stack needs realignment, we can't assume that the stack will in fact be
   // aligned.
-  llvm::Align Alignment =
-      commonAlignment(ForcedRealign ? llvm::Align() : StackAlignment, SPOffset);
+  Align Alignment =
+      commonAlignment(ForcedRealign ? Align::None() : StackAlignment, SPOffset);
   Alignment = clampStackAlignment(!StackRealignable, Alignment, StackAlignment);
   Objects.insert(Objects.begin(),
                  StackObject(Size, Alignment, SPOffset, IsImmutable,
@@ -102,8 +101,8 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset,
 int MachineFrameInfo::CreateFixedSpillStackObject(uint64_t Size,
                                                   int64_t SPOffset,
                                                   bool IsImmutable) {
-  llvm::Align Alignment =
-      commonAlignment(ForcedRealign ? llvm::Align() : StackAlignment, SPOffset);
+  Align Alignment =
+      commonAlignment(ForcedRealign ? Align::None() : StackAlignment, SPOffset);
   Alignment = clampStackAlignment(!StackRealignable, Alignment, StackAlignment);
   Objects.insert(Objects.begin(),
                  StackObject(Size, Alignment, SPOffset, IsImmutable,
diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 832895e2c92e7..79380339cc2d7 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -182,7 +182,7 @@ void MachineFunction::init() {
                          STI->getTargetLowering()->getPrefFunctionAlignment());
 
   if (AlignAllFunctions)
-    Alignment = llvm::Align(1ULL << AlignAllFunctions);
+    Alignment = Align(1ULL << AlignAllFunctions);
 
   JumpTableInfo = nullptr;
 
@@ -883,13 +883,13 @@ unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
   // alignment.
   switch (getEntryKind()) {
   case MachineJumpTableInfo::EK_BlockAddress:
-    return TD.getPointerABIAlignment(0);
+    return TD.getPointerABIAlignment(0).value();
   case MachineJumpTableInfo::EK_GPRel64BlockAddress:
-    return TD.getABIIntegerTypeAlignment(64);
+    return TD.getABIIntegerTypeAlignment(64).value();
   case MachineJumpTableInfo::EK_GPRel32BlockAddress:
   case MachineJumpTableInfo::EK_LabelDifference32:
   case MachineJumpTableInfo::EK_Custom32:
-    return TD.getABIIntegerTypeAlignment(32);
+    return TD.getABIIntegerTypeAlignment(32).value();
   case MachineJumpTableInfo::EK_Inline:
     return 1;
   }
diff --git a/llvm/lib/CodeGen/MachineFunctionPass.cpp b/llvm/lib/CodeGen/MachineFunctionPass.cpp
index 0da4cf3fc90c8..03149aa7db4a9 100644
--- a/llvm/lib/CodeGen/MachineFunctionPass.cpp
+++ b/llvm/lib/CodeGen/MachineFunctionPass.cpp
@@ -41,7 +41,7 @@ bool MachineFunctionPass::runOnFunction(Function &F) {
   if (F.hasAvailableExternallyLinkage())
     return false;
 
-  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfo>();
+  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
   MachineFunction &MF = MMI.getOrCreateMachineFunction(F);
 
   MachineFunctionProperties &MFProps = MF.getProperties();
@@ -101,8 +101,8 @@ bool MachineFunctionPass::runOnFunction(Function &F) {
 }
 
 void MachineFunctionPass::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.addRequired<MachineModuleInfo>();
-  AU.addPreserved<MachineModuleInfo>();
+  AU.addRequired<MachineModuleInfoWrapperPass>();
+  AU.addPreserved<MachineModuleInfoWrapperPass>();
 
   // MachineFunctionPass preserves all LLVM IR passes, but there's no
   // high-level way to express this. Instead, just list a bunch of
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 779f6086b8c15..5a7cbc394cc6d 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1205,7 +1205,7 @@ bool MachineInstr::mayAlias(AliasAnalysis *AA, const MachineInstr &Other,
     return false;
 
   // Let the target decide if memory accesses cannot possibly overlap.
-  if (TII->areMemAccessesTriviallyDisjoint(*this, Other, AA))
+  if (TII->areMemAccessesTriviallyDisjoint(*this, Other))
     return false;
 
   // FIXME: Need to handle multiple memory operands to support all targets.
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 15c38da2fb908..6a898ff6ef88d 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -153,7 +153,6 @@ namespace {
       AU.addRequired<MachineDominatorTree>();
       AU.addRequired<AAResultsWrapperPass>();
       AU.addPreserved<MachineLoopInfo>();
-      AU.addPreserved<MachineDominatorTree>();
       MachineFunctionPass::getAnalysisUsage(AU);
     }
 
diff --git a/llvm/lib/CodeGen/MachineLoopUtils.cpp b/llvm/lib/CodeGen/MachineLoopUtils.cpp
new file mode 100644
index 0000000000000..e074b76082f0e
--- /dev/null
+++ b/llvm/lib/CodeGen/MachineLoopUtils.cpp
@@ -0,0 +1,132 @@
+//=- MachineLoopUtils.cpp - Functions for manipulating loops ----------------=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachineLoopUtils.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+using namespace llvm;
+
+namespace {
+// MI's parent and BB are clones of each other. Find the equivalent copy of MI
+// in BB.
+MachineInstr &findEquivalentInstruction(MachineInstr &MI,
+                                        MachineBasicBlock *BB) {
+  MachineBasicBlock *PB = MI.getParent();
+  unsigned Offset = std::distance(PB->instr_begin(), MachineBasicBlock::instr_iterator(MI));
+  return *std::next(BB->instr_begin(), Offset);
+}
+} // namespace
+
+MachineBasicBlock *llvm::PeelSingleBlockLoop(LoopPeelDirection Direction,
+                                             MachineBasicBlock *Loop,
+                                             MachineRegisterInfo &MRI,
+                                             const TargetInstrInfo *TII) {
+  MachineFunction &MF = *Loop->getParent();
+  MachineBasicBlock *Preheader = *Loop->pred_begin();
+  if (Preheader == Loop)
+    Preheader = *std::next(Loop->pred_begin());
+  MachineBasicBlock *Exit = *Loop->succ_begin();
+  if (Exit == Loop)
+    Exit = *std::next(Loop->succ_begin());
+
+  MachineBasicBlock *NewBB = MF.CreateMachineBasicBlock(Loop->getBasicBlock());
+  if (Direction == LPD_Front)
+    MF.insert(Loop->getIterator(), NewBB);
+  else
+    MF.insert(std::next(Loop->getIterator()), NewBB);
+
+  // FIXME: Add DenseMapInfo trait for Register so we can use it as a key.
+  DenseMap<unsigned, Register> Remaps;
+  auto InsertPt = NewBB->end();
+  for (MachineInstr &MI : *Loop) {
+    MachineInstr *NewMI = MF.CloneMachineInstr(&MI);
+    NewBB->insert(InsertPt, NewMI);
+    for (MachineOperand &MO : NewMI->defs()) {
+      Register OrigR = MO.getReg();
+      if (OrigR.isPhysical())
+        continue;
+      Register &R = Remaps[OrigR];
+      R = MRI.createVirtualRegister(MRI.getRegClass(OrigR));
+      MO.setReg(R);
+
+      if (Direction == LPD_Back) {
+        // Replace all uses outside the original loop with the new register.
+        // FIXME: is the use_iterator stable enough to mutate register uses
+        // while iterating?
+        SmallVector<MachineOperand *, 4> Uses;
+        for (auto &Use : MRI.use_operands(OrigR))
+          if (Use.getParent()->getParent() != Loop)
+            Uses.push_back(&Use);
+        for (auto *Use : Uses) {
+          MRI.constrainRegClass(R, MRI.getRegClass(Use->getReg()));
+          Use->setReg(R);
+        }
+      }
+    }
+  }
+
+  for (auto I = NewBB->getFirstNonPHI(); I != NewBB->end(); ++I)
+    for (MachineOperand &MO : I->uses())
+      if (MO.isReg() && Remaps.count(MO.getReg()))
+        MO.setReg(Remaps[MO.getReg()]);
+
+  for (auto I = NewBB->begin(); I->isPHI(); ++I) {
+    MachineInstr &MI = *I;
+    unsigned LoopRegIdx = 3, InitRegIdx = 1;
+    if (MI.getOperand(2).getMBB() != Preheader)
+      std::swap(LoopRegIdx, InitRegIdx);
+    MachineInstr &OrigPhi = findEquivalentInstruction(MI, Loop);
+    assert(OrigPhi.isPHI());
+    if (Direction == LPD_Front) {
+      // When peeling front, we are only left with the initial value from the
+      // preheader.
+      Register R = MI.getOperand(LoopRegIdx).getReg();
+      if (Remaps.count(R))
+        R = Remaps[R];
+      OrigPhi.getOperand(InitRegIdx).setReg(R);
+      MI.RemoveOperand(LoopRegIdx + 1);
+      MI.RemoveOperand(LoopRegIdx + 0);
+    } else {
+      // When peeling back, the initial value is the loop-carried value from
+      // the original loop.
+      Register LoopReg = OrigPhi.getOperand(LoopRegIdx).getReg();
+      MI.getOperand(LoopRegIdx).setReg(LoopReg);
+      MI.RemoveOperand(InitRegIdx + 1);
+      MI.RemoveOperand(InitRegIdx + 0);
+    }
+  }
+
+  DebugLoc DL;
+  if (Direction == LPD_Front) {
+    Preheader->replaceSuccessor(Loop, NewBB);
+    NewBB->addSuccessor(Loop);
+    Loop->replacePhiUsesWith(Preheader, NewBB);
+    if (TII->removeBranch(*Preheader) > 0)
+      TII->insertBranch(*Preheader, NewBB, nullptr, {}, DL);
+    TII->removeBranch(*NewBB);
+    TII->insertBranch(*NewBB, Loop, nullptr, {}, DL);
+  } else {
+    Loop->replaceSuccessor(Exit, NewBB);
+    Exit->replacePhiUsesWith(Loop, NewBB);
+    NewBB->addSuccessor(Exit);
+
+    MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+    SmallVector<MachineOperand, 4> Cond;
+    bool CanAnalyzeBr = !TII->analyzeBranch(*Loop, TBB, FBB, Cond);
+    (void)CanAnalyzeBr;
+    assert(CanAnalyzeBr && "Must be able to analyze the loop branch!");
+    TII->removeBranch(*Loop);
+    TII->insertBranch(*Loop, TBB == Exit ? NewBB : TBB,
+                      FBB == Exit ? NewBB : FBB, Cond, DL);
+    if (TII->removeBranch(*NewBB) > 0)
+      TII->insertBranch(*NewBB, Exit, nullptr, {}, DL);
+  }
+
+  return NewBB;
+}
diff --git a/llvm/lib/CodeGen/MachineModuleInfo.cpp b/llvm/lib/CodeGen/MachineModuleInfo.cpp
index 3dca5a619bc4e..50a9251780e69 100644
--- a/llvm/lib/CodeGen/MachineModuleInfo.cpp
+++ b/llvm/lib/CodeGen/MachineModuleInfo.cpp
@@ -36,11 +36,6 @@
 using namespace llvm;
 using namespace llvm::dwarf;
 
-// Handle the Pass registration stuff necessary to use DataLayout's.
-INITIALIZE_PASS(MachineModuleInfo, "machinemoduleinfo",
-                "Machine Module Information", false, false)
-char MachineModuleInfo::ID = 0;
-
 // Out of line virtual method.
 MachineModuleInfoImpl::~MachineModuleInfoImpl() = default;
 
@@ -193,27 +188,15 @@ void MMIAddrLabelMapCallbackPtr::allUsesReplacedWith(Value *V2) {
   Map->UpdateForRAUWBlock(cast<BasicBlock>(getValPtr()), cast<BasicBlock>(V2));
 }
 
-MachineModuleInfo::MachineModuleInfo(const LLVMTargetMachine *TM)
-    : ImmutablePass(ID), TM(*TM),
-      Context(TM->getMCAsmInfo(), TM->getMCRegisterInfo(),
-              TM->getObjFileLowering(), nullptr, nullptr, false) {
-  initializeMachineModuleInfoPass(*PassRegistry::getPassRegistry());
-}
-
-MachineModuleInfo::~MachineModuleInfo() = default;
-
-bool MachineModuleInfo::doInitialization(Module &M) {
+void MachineModuleInfo::initialize() {
   ObjFileMMI = nullptr;
   CurCallSite = 0;
   UsesMSVCFloatingPoint = UsesMorestackAddr = false;
   HasSplitStack = HasNosplitStack = false;
   AddrLabelSymbols = nullptr;
-  TheModule = &M;
-  DbgInfoAvailable = !llvm::empty(M.debug_compile_units());
-  return false;
 }
 
-bool MachineModuleInfo::doFinalization(Module &M) {
+void MachineModuleInfo::finalize() {
   Personalities.clear();
 
   delete AddrLabelSymbols;
@@ -223,10 +206,30 @@ bool MachineModuleInfo::doFinalization(Module &M) {
 
   delete ObjFileMMI;
   ObjFileMMI = nullptr;
+}
 
-  return false;
+MachineModuleInfo::MachineModuleInfo(MachineModuleInfo &&MMI)
+    : TM(std::move(MMI.TM)),
+      Context(MMI.TM.getMCAsmInfo(), MMI.TM.getMCRegisterInfo(),
+              MMI.TM.getObjFileLowering(), nullptr, nullptr, false) {
+  ObjFileMMI = MMI.ObjFileMMI;
+  CurCallSite = MMI.CurCallSite;
+  UsesMSVCFloatingPoint = MMI.UsesMSVCFloatingPoint;
+  UsesMorestackAddr = MMI.UsesMorestackAddr;
+  HasSplitStack = MMI.HasSplitStack;
+  HasNosplitStack = MMI.HasNosplitStack;
+  AddrLabelSymbols = MMI.AddrLabelSymbols;
+  TheModule = MMI.TheModule;
 }
 
+MachineModuleInfo::MachineModuleInfo(const LLVMTargetMachine *TM)
+    : TM(*TM), Context(TM->getMCAsmInfo(), TM->getMCRegisterInfo(),
+                       TM->getObjFileLowering(), nullptr, nullptr, false) {
+  initialize();
+}
+
+MachineModuleInfo::~MachineModuleInfo() { finalize(); }
+
 //===- Address of Block Management ----------------------------------------===//
 
 ArrayRef<MCSymbol *>
@@ -305,12 +308,13 @@ class FreeMachineFunction : public FunctionPass {
   FreeMachineFunction() : FunctionPass(ID) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<MachineModuleInfo>();
-    AU.addPreserved<MachineModuleInfo>();
+    AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addPreserved<MachineModuleInfoWrapperPass>();
   }
 
   bool runOnFunction(Function &F) override {
-    MachineModuleInfo &MMI = getAnalysis<MachineModuleInfo>();
+    MachineModuleInfo &MMI =
+        getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
     MMI.deleteMachineFunctionFor(F);
     return true;
   }
@@ -327,3 +331,36 @@ char FreeMachineFunction::ID;
 FunctionPass *llvm::createFreeMachineFunctionPass() {
   return new FreeMachineFunction();
 }
+
+MachineModuleInfoWrapperPass::MachineModuleInfoWrapperPass(
+    const LLVMTargetMachine *TM)
+    : ImmutablePass(ID), MMI(TM) {
+  initializeMachineModuleInfoWrapperPassPass(*PassRegistry::getPassRegistry());
+}
+
+// Handle the Pass registration stuff necessary to use DataLayout's.
+INITIALIZE_PASS(MachineModuleInfoWrapperPass, "machinemoduleinfo",
+                "Machine Module Information", false, false)
+char MachineModuleInfoWrapperPass::ID = 0;
+
+bool MachineModuleInfoWrapperPass::doInitialization(Module &M) {
+  MMI.initialize();
+  MMI.TheModule = &M;
+  MMI.DbgInfoAvailable = !empty(M.debug_compile_units());
+  return false;
+}
+
+bool MachineModuleInfoWrapperPass::doFinalization(Module &M) {
+  MMI.finalize();
+  return false;
+}
+
+AnalysisKey MachineModuleAnalysis::Key;
+
+MachineModuleInfo MachineModuleAnalysis::run(Module &M,
+                                             ModuleAnalysisManager &) {
+  MachineModuleInfo MMI(TM);
+  MMI.TheModule = &M;
+  MMI.DbgInfoAvailable = !empty(M.debug_compile_units());
+  return MMI;
+}
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index c7e804f6260a6..8eccfb85a9461 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -429,12 +429,10 @@ static void printCFIRegister(unsigned DwarfReg, raw_ostream &OS,
     return;
   }
 
-  int Reg = TRI->getLLVMRegNum(DwarfReg, true);
-  if (Reg == -1) {
+  if (Optional<unsigned> Reg = TRI->getLLVMRegNum(DwarfReg, true))
+    OS << printReg(*Reg, TRI);
+  else
     OS << "<badreg>";
-    return;
-  }
-  OS << printReg(Reg, TRI);
 }
 
 static void printIRBlockReference(raw_ostream &OS, const BasicBlock &BB,
@@ -1067,17 +1065,6 @@ uint64_t MachineMemOperand::getAlignment() const {
   return MinAlign(getBaseAlignment(), getOffset());
 }
 
-void MachineMemOperand::print(raw_ostream &OS) const {
-  ModuleSlotTracker DummyMST(nullptr);
-  print(OS, DummyMST);
-}
-
-void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST) const {
-  SmallVector<StringRef, 0> SSNs;
-  LLVMContext Ctx;
-  print(OS, MST, SSNs, Ctx, nullptr, nullptr);
-}
-
 void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
                               SmallVectorImpl<StringRef> &SSNs,
                               const LLVMContext &Context,
diff --git a/llvm/lib/CodeGen/MachineOutliner.cpp b/llvm/lib/CodeGen/MachineOutliner.cpp
index 80a235aeaa5cc..533cce57adc84 100644
--- a/llvm/lib/CodeGen/MachineOutliner.cpp
+++ b/llvm/lib/CodeGen/MachineOutliner.cpp
@@ -846,8 +846,8 @@ struct MachineOutliner : public ModulePass {
   StringRef getPassName() const override { return "Machine Outliner"; }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<MachineModuleInfo>();
-    AU.addPreserved<MachineModuleInfo>();
+    AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addPreserved<MachineModuleInfoWrapperPass>();
     AU.setPreservesAll();
     ModulePass::getAnalysisUsage(AU);
   }
@@ -1128,7 +1128,7 @@ MachineOutliner::createOutlinedFunction(Module &M, OutlinedFunction &OF,
   IRBuilder<> Builder(EntryBB);
   Builder.CreateRetVoid();
 
-  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfo>();
+  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
   MachineFunction &MF = MMI.getOrCreateMachineFunction(*F);
   MachineBasicBlock &MBB = *MF.CreateMachineBasicBlock();
   const TargetSubtargetInfo &STI = MF.getSubtarget();
@@ -1421,7 +1421,7 @@ bool MachineOutliner::runOnModule(Module &M) {
   if (M.empty())
     return false;
 
-  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfo>();
+  MachineModuleInfo &MMI = getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
 
   // If the user passed -enable-machine-outliner=always or
   // -enable-machine-outliner, the pass will run on all functions in the module.
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index fc16ebf66f5d9..89c9f6093a975 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -326,7 +326,7 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) {
 
   LI.LoopInductionVar = nullptr;
   LI.LoopCompare = nullptr;
-  if (TII->analyzeLoop(L, LI.LoopInductionVar, LI.LoopCompare)) {
+  if (!TII->analyzeLoopForPipelining(L.getTopBlock())) {
     LLVM_DEBUG(
         dbgs() << "Unable to analyzeLoop, can NOT pipeline current Loop\n");
     NumFailLoop++;
@@ -557,10 +557,7 @@ void SwingSchedulerDAG::schedule() {
   // The experimental code generator can't work if there are InstChanges.
   if (ExperimentalCodeGen && NewInstrChanges.empty()) {
     PeelingModuloScheduleExpander MSE(MF, MS, &LIS);
-    // Experimental code generation isn't complete yet, but it can partially
-    // validate the code it generates against the original
-    // ModuloScheduleExpander.
-    MSE.validateAgainstModuloScheduleExpander();
+    MSE.expand();
   } else {
     ModuloScheduleExpander MSE(MF, MS, LIS, std::move(NewInstrChanges));
     MSE.expand();
@@ -700,7 +697,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) {
               TII->getMemOperandWithOffset(MI, BaseOp2, Offset2, TRI)) {
             if (BaseOp1->isIdenticalTo(*BaseOp2) &&
                 (int)Offset1 < (int)Offset2) {
-              assert(TII->areMemAccessesTriviallyDisjoint(LdMI, MI, AA) &&
+              assert(TII->areMemAccessesTriviallyDisjoint(LdMI, MI) &&
                      "What happened to the chain edge?");
               SDep Dep(Load, SDep::Barrier);
               Dep.setLatency(1);
diff --git a/llvm/lib/CodeGen/MachinePostDominators.cpp b/llvm/lib/CodeGen/MachinePostDominators.cpp
index 7f220ed1fd8f3..f4daff667e86d 100644
--- a/llvm/lib/CodeGen/MachinePostDominators.cpp
+++ b/llvm/lib/CodeGen/MachinePostDominators.cpp
@@ -17,7 +17,9 @@ using namespace llvm;
 
 namespace llvm {
 template class DominatorTreeBase<MachineBasicBlock, true>; // PostDomTreeBase
-}
+
+extern bool VerifyMachineDomInfo;
+} // namespace llvm
 
 char MachinePostDominatorTree::ID = 0;
 
@@ -25,33 +27,52 @@ char MachinePostDominatorTree::ID = 0;
 INITIALIZE_PASS(MachinePostDominatorTree, "machinepostdomtree",
                 "MachinePostDominator Tree Construction", true, true)
 
-MachinePostDominatorTree::MachinePostDominatorTree() : MachineFunctionPass(ID) {
+MachinePostDominatorTree::MachinePostDominatorTree()
+    : MachineFunctionPass(ID), PDT(nullptr) {
   initializeMachinePostDominatorTreePass(*PassRegistry::getPassRegistry());
-  DT = new PostDomTreeBase<MachineBasicBlock>();
 }
 
-FunctionPass *
-MachinePostDominatorTree::createMachinePostDominatorTreePass() {
+FunctionPass *MachinePostDominatorTree::createMachinePostDominatorTreePass() {
   return new MachinePostDominatorTree();
 }
 
-bool
-MachinePostDominatorTree::runOnMachineFunction(MachineFunction &F) {
-  DT->recalculate(F);
+bool MachinePostDominatorTree::runOnMachineFunction(MachineFunction &F) {
+  PDT = std::make_unique<PostDomTreeT>();
+  PDT->recalculate(F);
   return false;
 }
 
-MachinePostDominatorTree::~MachinePostDominatorTree() {
-  delete DT;
-}
-
-void
-MachinePostDominatorTree::getAnalysisUsage(AnalysisUsage &AU) const {
+void MachinePostDominatorTree::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
 
-void
-MachinePostDominatorTree::print(llvm::raw_ostream &OS, const Module *M) const {
-  DT->print(OS);
+MachineBasicBlock *MachinePostDominatorTree::findNearestCommonDominator(
+    ArrayRef<MachineBasicBlock *> Blocks) const {
+  assert(!Blocks.empty());
+
+  MachineBasicBlock *NCD = Blocks.front();
+  for (MachineBasicBlock *BB : Blocks.drop_front()) {
+    NCD = PDT->findNearestCommonDominator(NCD, BB);
+
+    // Stop when the root is reached.
+    if (PDT->isVirtualRoot(PDT->getNode(NCD)))
+      return nullptr;
+  }
+
+  return NCD;
+}
+
+void MachinePostDominatorTree::verifyAnalysis() const {
+  if (PDT && VerifyMachineDomInfo)
+    if (!PDT->verify(PostDomTreeT::VerificationLevel::Basic)) {
+      errs() << "MachinePostDominatorTree verification failed\n";
+
+      abort();
+    }
+}
+
+void MachinePostDominatorTree::print(llvm::raw_ostream &OS,
+                                     const Module *M) const {
+  PDT->print(OS);
 }
diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index a46ded954af38..f0721ea3b76de 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -82,6 +82,10 @@ cl::opt<bool>
 DumpCriticalPathLength("misched-dcpl", cl::Hidden,
                        cl::desc("Print critical path length to stdout"));
 
+cl::opt<bool> VerifyScheduling(
+    "verify-misched", cl::Hidden,
+    cl::desc("Verify machine instrs before and after machine scheduling"));
+
 } // end namespace llvm
 
 #ifndef NDEBUG
@@ -122,9 +126,6 @@ static cl::opt<bool> EnableMemOpCluster("misched-cluster", cl::Hidden,
                                         cl::desc("Enable memop clustering."),
                                         cl::init(true));
 
-static cl::opt<bool> VerifyScheduling("verify-misched", cl::Hidden,
-  cl::desc("Verify machine instrs before and after machine scheduling"));
-
 // DAG subtrees must have at least this many nodes.
 static const unsigned MinSubtreeSize = 8;
 
@@ -198,6 +199,7 @@ char &llvm::MachineSchedulerID = MachineScheduler::ID;
 INITIALIZE_PASS_BEGIN(MachineScheduler, DEBUG_TYPE,
                       "Machine Instruction Scheduler", false, false)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
 INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
 INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
@@ -210,7 +212,7 @@ MachineScheduler::MachineScheduler() : MachineSchedulerBase(ID) {
 
 void MachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
-  AU.addRequiredID(MachineDominatorsID);
+  AU.addRequired<MachineDominatorTree>();
   AU.addRequired<MachineLoopInfo>();
   AU.addRequired<AAResultsWrapperPass>();
   AU.addRequired<TargetPassConfig>();
@@ -234,7 +236,7 @@ PostMachineScheduler::PostMachineScheduler() : MachineSchedulerBase(ID) {
 
 void PostMachineScheduler::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesCFG();
-  AU.addRequiredID(MachineDominatorsID);
+  AU.addRequired<MachineDominatorTree>();
   AU.addRequired<MachineLoopInfo>();
   AU.addRequired<TargetPassConfig>();
   MachineFunctionPass::getAnalysisUsage(AU);
@@ -985,7 +987,7 @@ void ScheduleDAGMILive::enterRegion(MachineBasicBlock *bb,
          "ShouldTrackLaneMasks requires ShouldTrackPressure");
 }
 
-// Setup the register pressure trackers for the top scheduled top and bottom
+// Setup the register pressure trackers for the top scheduled and bottom
 // scheduled regions.
 void ScheduleDAGMILive::initRegPressure() {
   VRegUses.clear();
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 8f0d436dfa505..27a2e7023f222 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -115,15 +115,12 @@ namespace {
     bool runOnMachineFunction(MachineFunction &MF) override;
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
-      AU.setPreservesCFG();
       MachineFunctionPass::getAnalysisUsage(AU);
       AU.addRequired<AAResultsWrapperPass>();
       AU.addRequired<MachineDominatorTree>();
       AU.addRequired<MachinePostDominatorTree>();
       AU.addRequired<MachineLoopInfo>();
       AU.addRequired<MachineBranchProbabilityInfo>();
-      AU.addPreserved<MachineDominatorTree>();
-      AU.addPreserved<MachinePostDominatorTree>();
       AU.addPreserved<MachineLoopInfo>();
       if (UseBlockFreqInfo)
         AU.addRequired<MachineBlockFrequencyInfo>();
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 727ce0af02996..d23f6c8e5477f 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1368,6 +1368,20 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) {
         break;
       }
     }
+    switch (IntrID) {
+    case Intrinsic::memcpy:
+      if (MI->getNumOperands() != 5)
+        report("Expected memcpy intrinsic to have 5 operands", MI);
+      break;
+    case Intrinsic::memmove:
+      if (MI->getNumOperands() != 5)
+        report("Expected memmove intrinsic to have 5 operands", MI);
+      break;
+    case Intrinsic::memset:
+      if (MI->getNumOperands() != 5)
+        report("Expected memset intrinsic to have 5 operands", MI);
+      break;
+    }
     break;
   }
   case TargetOpcode::G_SEXT_INREG: {
diff --git a/llvm/lib/CodeGen/ModuloSchedule.cpp b/llvm/lib/CodeGen/ModuloSchedule.cpp
index 55fa2aff29331..30aa81487c8b1 100644
--- a/llvm/lib/CodeGen/ModuloSchedule.cpp
+++ b/llvm/lib/CodeGen/ModuloSchedule.cpp
@@ -10,6 +10,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineLoopUtils.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
 #include "llvm/MC/MCContext.h"
@@ -105,6 +106,9 @@ void ModuloScheduleExpander::expand() {
 }
 
 void ModuloScheduleExpander::generatePipelinedLoop() {
+  LoopInfo = TII->analyzeLoopForPipelining(BB);
+  assert(LoopInfo && "Must be able to analyze loop!");
+
   // Create a new basic block for the kernel and add it to the CFG.
   MachineBasicBlock *KernelBB = MF.CreateMachineBasicBlock(BB->getBasicBlock());
 
@@ -847,10 +851,6 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
                                          MBBVectorTy &EpilogBBs,
                                          ValueMapTy *VRMap) {
   assert(PrologBBs.size() == EpilogBBs.size() && "Prolog/Epilog mismatch");
-  MachineInstr *IndVar;
-  MachineInstr *Cmp;
-  if (TII->analyzeLoop(*Schedule.getLoop(), IndVar, Cmp))
-    llvm_unreachable("Must be able to analyze loop!");
   MachineBasicBlock *LastPro = KernelBB;
   MachineBasicBlock *LastEpi = KernelBB;
 
@@ -858,32 +858,20 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
   // to the first prolog and the last epilog blocks.
   SmallVector<MachineInstr *, 4> PrevInsts;
   unsigned MaxIter = PrologBBs.size() - 1;
-  unsigned LC = UINT_MAX;
-  unsigned LCMin = UINT_MAX;
   for (unsigned i = 0, j = MaxIter; i <= MaxIter; ++i, --j) {
     // Add branches to the prolog that go to the corresponding
     // epilog, and the fall-thru prolog/kernel block.
     MachineBasicBlock *Prolog = PrologBBs[j];
     MachineBasicBlock *Epilog = EpilogBBs[i];
-    // We've executed one iteration, so decrement the loop count and check for
-    // the loop end.
-    SmallVector<MachineOperand, 4> Cond;
-    // Check if the LOOP0 has already been removed. If so, then there is no need
-    // to reduce the trip count.
-    if (LC != 0)
-      LC = TII->reduceLoopCount(*Prolog, PreheaderBB, IndVar, *Cmp, Cond,
-                                PrevInsts, j, MaxIter);
-
-    // Record the value of the first trip count, which is used to determine if
-    // branches and blocks can be removed for constant trip counts.
-    if (LCMin == UINT_MAX)
-      LCMin = LC;
 
+    SmallVector<MachineOperand, 4> Cond;
+    Optional<bool> StaticallyGreater =
+        LoopInfo->createTripCountGreaterCondition(j + 1, *Prolog, Cond);
     unsigned numAdded = 0;
-    if (Register::isVirtualRegister(LC)) {
+    if (!StaticallyGreater.hasValue()) {
       Prolog->addSuccessor(Epilog);
       numAdded = TII->insertBranch(*Prolog, Epilog, LastPro, Cond, DebugLoc());
-    } else if (j >= LCMin) {
+    } else if (*StaticallyGreater == false) {
       Prolog->addSuccessor(Epilog);
       Prolog->removeSuccessor(LastPro);
       LastEpi->removeSuccessor(Epilog);
@@ -894,10 +882,12 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
         LastEpi->clear();
         LastEpi->eraseFromParent();
       }
+      if (LastPro == KernelBB) {
+        LoopInfo->disposed();
+        NewKernel = nullptr;
+      }
       LastPro->clear();
       LastPro->eraseFromParent();
-      if (LastPro == KernelBB)
-        NewKernel = nullptr;
     } else {
       numAdded = TII->insertBranch(*Prolog, LastPro, nullptr, Cond, DebugLoc());
       removePhis(Epilog, Prolog);
@@ -909,6 +899,11 @@ void ModuloScheduleExpander::addBranches(MachineBasicBlock &PreheaderBB,
          I != E && numAdded > 0; ++I, --numAdded)
       updateInstruction(&*I, false, j, 0, VRMap);
   }
+
+  if (NewKernel) {
+    LoopInfo->setPreheader(PrologBBs[MaxIter]);
+    LoopInfo->adjustTripCount(-(MaxIter + 1));
+  }
 }
 
 /// Return true if we can compute the amount the instruction changes
@@ -1307,6 +1302,7 @@ void KernelRewriter::rewrite() {
     if (!FirstMI)
       FirstMI = MI;
   }
+  assert(FirstMI && "Failed to find first MI in schedule");
 
   // At this point all of the scheduled instructions are between FirstMI
   // and the end of the block. Kill from the first non-phi to FirstMI.
@@ -1569,6 +1565,266 @@ class KernelOperandInfo {
 };
 } // namespace
 
+MachineBasicBlock *
+PeelingModuloScheduleExpander::peelKernel(LoopPeelDirection LPD) {
+  MachineBasicBlock *NewBB = PeelSingleBlockLoop(LPD, BB, MRI, TII);
+  if (LPD == LPD_Front)
+    PeeledFront.push_back(NewBB);
+  else
+    PeeledBack.push_front(NewBB);
+  for (auto I = BB->begin(), NI = NewBB->begin(); !I->isTerminator();
+       ++I, ++NI) {
+    CanonicalMIs[&*I] = &*I;
+    CanonicalMIs[&*NI] = &*I;
+    BlockMIs[{NewBB, &*I}] = &*NI;
+    BlockMIs[{BB, &*I}] = &*I;
+  }
+  return NewBB;
+}
+
+void PeelingModuloScheduleExpander::peelPrologAndEpilogs() {
+  BitVector LS(Schedule.getNumStages(), true);
+  BitVector AS(Schedule.getNumStages(), true);
+  LiveStages[BB] = LS;
+  AvailableStages[BB] = AS;
+
+  // Peel out the prologs.
+  LS.reset();
+  for (int I = 0; I < Schedule.getNumStages() - 1; ++I) {
+    LS[I] = 1;
+    Prologs.push_back(peelKernel(LPD_Front));
+    LiveStages[Prologs.back()] = LS;
+    AvailableStages[Prologs.back()] = LS;
+  }
+
+  // Create a block that will end up as the new loop exiting block (dominated by
+  // all prologs and epilogs). It will only contain PHIs, in the same order as
+  // BB's PHIs. This gives us a poor-man's LCSSA with the inductive property
+  // that the exiting block is a (sub) clone of BB. This in turn gives us the
+  // property that any value deffed in BB but used outside of BB is used by a
+  // PHI in the exiting block.
+  MachineBasicBlock *ExitingBB = CreateLCSSAExitingBlock();
+
+  // Push out the epilogs, again in reverse order.
+  // We can't assume anything about the minumum loop trip count at this point,
+  // so emit a fairly complex epilog:
+  //  K[0, 1, 2]     // Kernel runs stages 0, 1, 2
+  //  E0[2] <- P1    // Epilog runs stage 2 only, so the state after is [0].
+  //  E1[1, 2] <- P0 // Epilog 1 moves the last item from stage 0 to stage 2.
+  //
+  // This creates a single-successor single-predecessor sequence of blocks for
+  // each epilog, which are kept this way for simplicity at this stage and
+  // cleaned up by the optimizer later.
+  for (int I = 1; I <= Schedule.getNumStages() - 1; ++I) {
+    Epilogs.push_back(nullptr);
+    for (int J = Schedule.getNumStages() - 1; J >= I; --J) {
+      LS.reset();
+      LS[J] = 1;
+      Epilogs.back() = peelKernel(LPD_Back);
+      LiveStages[Epilogs.back()] = LS;
+      AvailableStages[Epilogs.back()] = AS;
+    }
+  }
+
+  // Now we've defined all the prolog and epilog blocks as a fallthrough
+  // sequence, add the edges that will be followed if the loop trip count is
+  // lower than the number of stages (connecting prologs directly with epilogs).
+  auto PI = Prologs.begin();
+  auto EI = Epilogs.begin();
+  assert(Prologs.size() == Epilogs.size());
+  for (; PI != Prologs.end(); ++PI, ++EI) {
+    MachineBasicBlock *Pred = *(*EI)->pred_begin();
+    (*PI)->addSuccessor(*EI);
+    for (MachineInstr &MI : (*EI)->phis()) {
+      Register Reg = MI.getOperand(1).getReg();
+      MachineInstr *Use = MRI.getUniqueVRegDef(Reg);
+      if (Use && Use->getParent() == Pred)
+        Reg = getEquivalentRegisterIn(Reg, *PI);
+      MI.addOperand(MachineOperand::CreateReg(Reg, /*isDef=*/false));
+      MI.addOperand(MachineOperand::CreateMBB(*PI));
+    }
+  }
+
+  // Create a list of all blocks in order.
+  SmallVector<MachineBasicBlock *, 8> Blocks;
+  llvm::copy(PeeledFront, std::back_inserter(Blocks));
+  Blocks.push_back(BB);
+  llvm::copy(PeeledBack, std::back_inserter(Blocks));
+
+  // Iterate in reverse order over all instructions, remapping as we go.
+  for (MachineBasicBlock *B : reverse(Blocks)) {
+    for (auto I = B->getFirstInstrTerminator()->getReverseIterator();
+         I != std::next(B->getFirstNonPHI()->getReverseIterator());) {
+      MachineInstr *MI = &*I++;
+      rewriteUsesOf(MI);
+    }
+  }
+  // Now all remapping has been done, we're free to optimize the generated code.
+  for (MachineBasicBlock *B : reverse(Blocks))
+    EliminateDeadPhis(B, MRI, LIS);
+  EliminateDeadPhis(ExitingBB, MRI, LIS);
+}
+
+MachineBasicBlock *PeelingModuloScheduleExpander::CreateLCSSAExitingBlock() {
+  MachineFunction &MF = *BB->getParent();
+  MachineBasicBlock *Exit = *BB->succ_begin();
+  if (Exit == BB)
+    Exit = *std::next(BB->succ_begin());
+
+  MachineBasicBlock *NewBB = MF.CreateMachineBasicBlock(BB->getBasicBlock());
+  MF.insert(std::next(BB->getIterator()), NewBB);
+
+  // Clone all phis in BB into NewBB and rewrite.
+  for (MachineInstr &MI : BB->phis()) {
+    auto RC = MRI.getRegClass(MI.getOperand(0).getReg());
+    Register OldR = MI.getOperand(3).getReg();
+    Register R = MRI.createVirtualRegister(RC);
+    SmallVector<MachineInstr *, 4> Uses;
+    for (MachineInstr &Use : MRI.use_instructions(OldR))
+      if (Use.getParent() != BB)
+        Uses.push_back(&Use);
+    for (MachineInstr *Use : Uses)
+      Use->substituteRegister(OldR, R, /*SubIdx=*/0,
+                              *MRI.getTargetRegisterInfo());
+    MachineInstr *NI = BuildMI(NewBB, DebugLoc(), TII->get(TargetOpcode::PHI), R)
+        .addReg(OldR)
+        .addMBB(BB);
+    BlockMIs[{NewBB, &MI}] = NI;
+    CanonicalMIs[NI] = &MI;
+  }
+  BB->replaceSuccessor(Exit, NewBB);
+  Exit->replacePhiUsesWith(BB, NewBB);
+  NewBB->addSuccessor(Exit);
+
+  MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
+  SmallVector<MachineOperand, 4> Cond;
+  bool CanAnalyzeBr = !TII->analyzeBranch(*BB, TBB, FBB, Cond);
+  (void)CanAnalyzeBr;
+  assert(CanAnalyzeBr && "Must be able to analyze the loop branch!");
+  TII->removeBranch(*BB);
+  TII->insertBranch(*BB, TBB == Exit ? NewBB : TBB, FBB == Exit ? NewBB : FBB,
+                    Cond, DebugLoc());
+  TII->insertUnconditionalBranch(*NewBB, Exit, DebugLoc());
+  return NewBB;
+}
+
+Register
+PeelingModuloScheduleExpander::getEquivalentRegisterIn(Register Reg,
+                                                       MachineBasicBlock *BB) {
+  MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
+  unsigned OpIdx = MI->findRegisterDefOperandIdx(Reg);
+  return BlockMIs[{BB, CanonicalMIs[MI]}]->getOperand(OpIdx).getReg();
+}
+
+void PeelingModuloScheduleExpander::rewriteUsesOf(MachineInstr *MI) {
+  if (MI->isPHI()) {
+    // This is an illegal PHI. The loop-carried (desired) value is operand 3,
+    // and it is produced by this block.
+    Register PhiR = MI->getOperand(0).getReg();
+    Register R = MI->getOperand(3).getReg();
+    int RMIStage = getStage(MRI.getUniqueVRegDef(R));
+    if (RMIStage != -1 && !AvailableStages[MI->getParent()].test(RMIStage))
+      R = MI->getOperand(1).getReg();
+    MRI.setRegClass(R, MRI.getRegClass(PhiR));
+    MRI.replaceRegWith(PhiR, R);
+    if (LIS)
+      LIS->RemoveMachineInstrFromMaps(*MI);
+    MI->eraseFromParent();
+    return;
+  }
+
+  int Stage = getStage(MI);
+  if (Stage == -1 || LiveStages.count(MI->getParent()) == 0 ||
+      LiveStages[MI->getParent()].test(Stage))
+    // Instruction is live, no rewriting to do.
+    return;
+
+  for (MachineOperand &DefMO : MI->defs()) {
+    SmallVector<std::pair<MachineInstr *, Register>, 4> Subs;
+    for (MachineInstr &UseMI : MRI.use_instructions(DefMO.getReg())) {
+      // Only PHIs can use values from this block by construction.
+      // Match with the equivalent PHI in B.
+      assert(UseMI.isPHI());
+      Register Reg = getEquivalentRegisterIn(UseMI.getOperand(0).getReg(),
+                                             MI->getParent());
+      Subs.emplace_back(&UseMI, Reg);
+    }
+    for (auto &Sub : Subs)
+      Sub.first->substituteRegister(DefMO.getReg(), Sub.second, /*SubIdx=*/0,
+                                    *MRI.getTargetRegisterInfo());
+  }
+  if (LIS)
+    LIS->RemoveMachineInstrFromMaps(*MI);
+  MI->eraseFromParent();
+}
+
+void PeelingModuloScheduleExpander::fixupBranches() {
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo> Info =
+      TII->analyzeLoopForPipelining(BB);
+  assert(Info);
+
+  // Work outwards from the kernel.
+  bool KernelDisposed = false;
+  int TC = Schedule.getNumStages() - 1;
+  for (auto PI = Prologs.rbegin(), EI = Epilogs.rbegin(); PI != Prologs.rend();
+       ++PI, ++EI, --TC) {
+    MachineBasicBlock *Prolog = *PI;
+    MachineBasicBlock *Fallthrough = *Prolog->succ_begin();
+    MachineBasicBlock *Epilog = *EI;
+    SmallVector<MachineOperand, 4> Cond;
+    Optional<bool> StaticallyGreater =
+        Info->createTripCountGreaterCondition(TC, *Prolog, Cond);
+    if (!StaticallyGreater.hasValue()) {
+      LLVM_DEBUG(dbgs() << "Dynamic: TC > " << TC << "\n");
+      // Dynamically branch based on Cond.
+      TII->removeBranch(*Prolog);
+      TII->insertBranch(*Prolog, Epilog, Fallthrough, Cond, DebugLoc());
+    } else if (*StaticallyGreater == false) {
+      LLVM_DEBUG(dbgs() << "Static-false: TC > " << TC << "\n");
+      // Prolog never falls through; branch to epilog and orphan interior
+      // blocks. Leave it to unreachable-block-elim to clean up.
+      Prolog->removeSuccessor(Fallthrough);
+      for (MachineInstr &P : Fallthrough->phis()) {
+        P.RemoveOperand(2);
+        P.RemoveOperand(1);
+      }
+      TII->removeBranch(*Prolog);
+      TII->insertUnconditionalBranch(*Prolog, Epilog, DebugLoc());
+      KernelDisposed = true;
+    } else {
+      LLVM_DEBUG(dbgs() << "Static-true: TC > " << TC << "\n");
+      // Prolog always falls through; remove incoming values in epilog.
+      Prolog->removeSuccessor(Epilog);
+      for (MachineInstr &P : Epilog->phis()) {
+        P.RemoveOperand(4);
+        P.RemoveOperand(3);
+      }
+    }
+  }
+
+  if (!KernelDisposed) {
+    Info->adjustTripCount(-(Schedule.getNumStages() - 1));
+    Info->setPreheader(Prologs.back());
+  } else {
+    Info->disposed();
+  }
+}
+
+void PeelingModuloScheduleExpander::rewriteKernel() {
+  KernelRewriter KR(*Schedule.getLoop(), Schedule);
+  KR.rewrite();
+}
+
+void PeelingModuloScheduleExpander::expand() {
+  BB = Schedule.getLoop()->getTopBlock();
+  Preheader = Schedule.getLoop()->getLoopPreheader();
+  LLVM_DEBUG(Schedule.dump());
+
+  rewriteKernel();
+  peelPrologAndEpilogs();
+  fixupBranches();
+}
+
 void PeelingModuloScheduleExpander::validateAgainstModuloScheduleExpander() {
   BB = Schedule.getLoop()->getTopBlock();
   Preheader = Schedule.getLoop()->getLoopPreheader();
@@ -1598,6 +1854,7 @@ void PeelingModuloScheduleExpander::validateAgainstModuloScheduleExpander() {
   // Now run the new expansion algorithm.
   KernelRewriter KR(*Schedule.getLoop(), Schedule);
   KR.rewrite();
+  peelPrologAndEpilogs();
 
   // Collect all illegal phis that the new algorithm created. We'll give these
   // to KernelOperandInfo.
@@ -1667,6 +1924,7 @@ void PeelingModuloScheduleExpander::validateAgainstModuloScheduleExpander() {
 //  "Stage=%d Cycle=%d".
 //===----------------------------------------------------------------------===//
 
+namespace {
 class ModuloScheduleTest : public MachineFunctionPass {
 public:
   static char ID;
@@ -1684,6 +1942,7 @@ class ModuloScheduleTest : public MachineFunctionPass {
     MachineFunctionPass::getAnalysisUsage(AU);
   }
 };
+} // namespace
 
 char ModuloScheduleTest::ID = 0;
 
diff --git a/llvm/lib/CodeGen/PHIElimination.cpp b/llvm/lib/CodeGen/PHIElimination.cpp
index 2b3ce890d984d..4dd4c4b1084e5 100644
--- a/llvm/lib/CodeGen/PHIElimination.cpp
+++ b/llvm/lib/CodeGen/PHIElimination.cpp
@@ -31,7 +31,9 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/Pass.h"
@@ -183,6 +185,11 @@ bool PHIElimination::runOnMachineFunction(MachineFunction &MF) {
     MF.DeleteMachineInstr(I.first);
   }
 
+  // TODO: we should use the incremental DomTree updater here.
+  if (Changed)
+    if (auto *MDT = getAnalysisIfAvailable<MachineDominatorTree>())
+      MDT->getBase().recalculate(MF);
+
   LoweredPHIs.clear();
   ImpDefs.clear();
   VRegPHIUseCount.clear();
@@ -252,11 +259,12 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
   // Insert a register to register copy at the top of the current block (but
   // after any remaining phi nodes) which copies the new incoming register
   // into the phi node destination.
+  MachineInstr *PHICopy = nullptr;
   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
   if (allPhiOperandsUndefined(*MPhi, *MRI))
     // If all sources of a PHI node are implicit_def or undef uses, just emit an
     // implicit_def instead of a copy.
-    BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(),
+    PHICopy = BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(),
             TII->get(TargetOpcode::IMPLICIT_DEF), DestReg);
   else {
     // Can we reuse an earlier PHI node? This only happens for critical edges,
@@ -273,15 +281,13 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
       const TargetRegisterClass *RC = MF.getRegInfo().getRegClass(DestReg);
       entry = IncomingReg = MF.getRegInfo().createVirtualRegister(RC);
     }
-    BuildMI(MBB, AfterPHIsIt, MPhi->getDebugLoc(),
-            TII->get(TargetOpcode::COPY), DestReg)
-      .addReg(IncomingReg);
+    // Give the target possiblity to handle special cases fallthrough otherwise
+    PHICopy = TII->createPHIDestinationCopy(MBB, AfterPHIsIt, MPhi->getDebugLoc(),
+                                  IncomingReg, DestReg);
   }
 
   // Update live variable information if there is any.
   if (LV) {
-    MachineInstr &PHICopy = *std::prev(AfterPHIsIt);
-
     if (IncomingReg) {
       LiveVariables::VarInfo &VI = LV->getVarInfo(IncomingReg);
 
@@ -302,7 +308,7 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
       // killed.  Note that because the value is defined in several places (once
       // each for each incoming block), the "def" block and instruction fields
       // for the VarInfo is not filled in.
-      LV->addVirtualRegisterKilled(IncomingReg, PHICopy);
+      LV->addVirtualRegisterKilled(IncomingReg, *PHICopy);
     }
 
     // Since we are going to be deleting the PHI node, if it is the last use of
@@ -312,15 +318,14 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
 
     // If the result is dead, update LV.
     if (isDead) {
-      LV->addVirtualRegisterDead(DestReg, PHICopy);
+      LV->addVirtualRegisterDead(DestReg, *PHICopy);
       LV->removeVirtualRegisterDead(DestReg, *MPhi);
     }
   }
 
   // Update LiveIntervals for the new copy or implicit def.
   if (LIS) {
-    SlotIndex DestCopyIndex =
-        LIS->InsertMachineInstrInMaps(*std::prev(AfterPHIsIt));
+    SlotIndex DestCopyIndex = LIS->InsertMachineInstrInMaps(*PHICopy);
 
     SlotIndex MBBStartIndex = LIS->getMBBStartIdx(&MBB);
     if (IncomingReg) {
@@ -406,9 +411,9 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
           if (DefMI->isImplicitDef())
             ImpDefs.insert(DefMI);
       } else {
-        NewSrcInstr = BuildMI(opBlock, InsertPos, MPhi->getDebugLoc(),
-                            TII->get(TargetOpcode::COPY), IncomingReg)
-                        .addReg(SrcReg, 0, SrcSubReg);
+        NewSrcInstr =
+            TII->createPHISourceCopy(opBlock, InsertPos, MPhi->getDebugLoc(),
+                                     SrcReg, SrcSubReg, IncomingReg);
       }
     }
 
@@ -457,7 +462,7 @@ void PHIElimination::LowerPHINode(MachineBasicBlock &MBB,
           }
         } else {
           // We just inserted this copy.
-          KillInst = std::prev(InsertPos);
+          KillInst = NewSrcInstr;
         }
       }
       assert(KillInst->readsRegister(SrcReg) && "Cannot find kill instruction");
diff --git a/llvm/lib/CodeGen/PatchableFunction.cpp b/llvm/lib/CodeGen/PatchableFunction.cpp
index 9d7605f078f97..529fde84e39a3 100644
--- a/llvm/lib/CodeGen/PatchableFunction.cpp
+++ b/llvm/lib/CodeGen/PatchableFunction.cpp
@@ -78,7 +78,7 @@ bool PatchableFunction::runOnMachineFunction(MachineFunction &MF) {
     MIB.add(MO);
 
   FirstActualI->eraseFromParent();
-  MF.ensureAlignment(llvm::Align(16));
+  MF.ensureAlignment(Align(16));
   return true;
 }
 
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 2752e186875c1..0d2f6f99ca96d 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -76,7 +76,7 @@ static bool lowerObjCCall(Function &F, const char *NewFn,
   }
 
   for (auto I = F.use_begin(), E = F.use_end(); I != E;) {
-    auto *CI = dyn_cast<CallInst>(I->getUser());
+    auto *CI = cast<CallInst>(I->getUser());
     assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
     ++I;
 
diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
index af07f8f948f95..735af2455d724 100644
--- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -373,6 +373,13 @@ LaneBitmask ScheduleDAGInstrs::getLaneMaskForMO(const MachineOperand &MO) const
   return TRI->getSubRegIndexLaneMask(SubReg);
 }
 
+bool ScheduleDAGInstrs::deadDefHasNoUse(const MachineOperand &MO) {
+  auto RegUse = CurrentVRegUses.find(MO.getReg());
+  if (RegUse == CurrentVRegUses.end())
+    return true;
+  return (RegUse->LaneMask & getLaneMaskForMO(MO)).none();
+}
+
 /// Adds register output and data dependencies from this SUnit to instructions
 /// that occur later in the same scheduling region if they read from or write to
 /// the virtual register defined at OperIdx.
@@ -393,6 +400,18 @@ void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
     // earlier instruction.
     KillLaneMask = IsKill ? LaneBitmask::getAll() : DefLaneMask;
 
+    if (MO.getSubReg() != 0 && MO.isUndef()) {
+      // There may be other subregister defs on the same instruction of the same
+      // register in later operands. The lanes of other defs will now be live
+      // after this instruction, so these should not be treated as killed by the
+      // instruction even though they appear to be killed in this one operand.
+      for (int I = OperIdx + 1, E = MI->getNumOperands(); I != E; ++I) {
+        const MachineOperand &OtherMO = MI->getOperand(I);
+        if (OtherMO.isReg() && OtherMO.isDef() && OtherMO.getReg() == Reg)
+          KillLaneMask &= ~getLaneMaskForMO(OtherMO);
+      }
+    }
+
     // Clear undef flag, we'll re-add it later once we know which subregister
     // Def is first.
     MO.setIsUndef(false);
@@ -402,8 +421,7 @@ void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
   }
 
   if (MO.isDead()) {
-    assert(CurrentVRegUses.find(Reg) == CurrentVRegUses.end() &&
-           "Dead defs should have no uses");
+    assert(deadDefHasNoUse(MO) && "Dead defs should have no uses");
   } else {
     // Add data dependence to all uses we found so far.
     const TargetSubtargetInfo &ST = MF.getSubtarget();
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a7e506855481a..93248cb078cd6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -540,11 +540,11 @@ namespace {
     SDValue MatchBSwapHWordLow(SDNode *N, SDValue N0, SDValue N1,
                                bool DemandHighBits = true);
     SDValue MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1);
-    SDNode *MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
+    SDValue MatchRotatePosNeg(SDValue Shifted, SDValue Pos, SDValue Neg,
                               SDValue InnerPos, SDValue InnerNeg,
                               unsigned PosOpcode, unsigned NegOpcode,
                               const SDLoc &DL);
-    SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
+    SDValue MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL);
     SDValue MatchLoadCombine(SDNode *N);
     SDValue MatchStoreCombine(StoreSDNode *N);
     SDValue ReduceLoadWidth(SDNode *N);
@@ -785,251 +785,6 @@ void DAGCombiner::deleteAndRecombine(SDNode *N) {
   DAG.DeleteNode(N);
 }
 
-/// Return 1 if we can compute the negated form of the specified expression for
-/// the same cost as the expression itself, or 2 if we can compute the negated
-/// form more cheaply than the expression itself.
-static char isNegatibleForFree(SDValue Op, bool LegalOperations,
-                               const TargetLowering &TLI,
-                               const TargetOptions *Options,
-                               bool ForCodeSize,
-                               unsigned Depth = 0) {
-  // fneg is removable even if it has multiple uses.
-  if (Op.getOpcode() == ISD::FNEG)
-    return 2;
-
-  // Don't allow anything with multiple uses unless we know it is free.
-  EVT VT = Op.getValueType();
-  const SDNodeFlags Flags = Op->getFlags();
-  if (!Op.hasOneUse() &&
-      !(Op.getOpcode() == ISD::FP_EXTEND &&
-        TLI.isFPExtFree(VT, Op.getOperand(0).getValueType())))
-    return 0;
-
-  // Don't recurse exponentially.
-  if (Depth > 6)
-    return 0;
-
-  switch (Op.getOpcode()) {
-  default: return false;
-  case ISD::ConstantFP: {
-    if (!LegalOperations)
-      return 1;
-
-    // Don't invert constant FP values after legalization unless the target says
-    // the negated constant is legal.
-    return TLI.isOperationLegal(ISD::ConstantFP, VT) ||
-           TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
-                            ForCodeSize);
-  }
-  case ISD::BUILD_VECTOR: {
-    // Only permit BUILD_VECTOR of constants.
-    if (llvm::any_of(Op->op_values(), [&](SDValue N) {
-          return !N.isUndef() && !isa<ConstantFPSDNode>(N);
-        }))
-      return 0;
-    if (!LegalOperations)
-      return 1;
-    if (TLI.isOperationLegal(ISD::ConstantFP, VT) &&
-        TLI.isOperationLegal(ISD::BUILD_VECTOR, VT))
-      return 1;
-    return llvm::all_of(Op->op_values(), [&](SDValue N) {
-      return N.isUndef() ||
-             TLI.isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
-                              ForCodeSize);
-    });
-  }
-  case ISD::FADD:
-    if (!Options->NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
-      return 0;
-
-    // After operation legalization, it might not be legal to create new FSUBs.
-    if (LegalOperations && !TLI.isOperationLegalOrCustom(ISD::FSUB, VT))
-      return 0;
-
-    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
-    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
-                                    Options, ForCodeSize, Depth + 1))
-      return V;
-    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
-    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
-                              ForCodeSize, Depth + 1);
-  case ISD::FSUB:
-    // We can't turn -(A-B) into B-A when we honor signed zeros.
-    if (!Options->NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
-      return 0;
-
-    // fold (fneg (fsub A, B)) -> (fsub B, A)
-    return 1;
-
-  case ISD::FMUL:
-  case ISD::FDIV:
-    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
-    if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
-                                    Options, ForCodeSize, Depth + 1))
-      return V;
-
-    // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
-    if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
-      if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
-        return 0;
-
-    return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options,
-                              ForCodeSize, Depth + 1);
-
-  case ISD::FMA:
-  case ISD::FMAD: {
-    if (!Options->NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
-      return 0;
-
-    // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
-    // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
-    char V2 = isNegatibleForFree(Op.getOperand(2), LegalOperations, TLI,
-                                 Options, ForCodeSize, Depth + 1);
-    if (!V2)
-      return 0;
-
-    // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
-    char V0 = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
-                                 Options, ForCodeSize, Depth + 1);
-    char V1 = isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI,
-                                 Options, ForCodeSize, Depth + 1);
-    char V01 = std::max(V0, V1);
-    return V01 ? std::max(V01, V2) : 0;
-  }
-
-  case ISD::FP_EXTEND:
-  case ISD::FP_ROUND:
-  case ISD::FSIN:
-    return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options,
-                              ForCodeSize, Depth + 1);
-  }
-}
-
-/// If isNegatibleForFree returns true, return the newly negated expression.
-static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG,
-                                    bool LegalOperations, bool ForCodeSize,
-                                    unsigned Depth = 0) {
-  // fneg is removable even if it has multiple uses.
-  if (Op.getOpcode() == ISD::FNEG)
-    return Op.getOperand(0);
-
-  assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree");
-  const TargetOptions &Options = DAG.getTarget().Options;
-  const SDNodeFlags Flags = Op->getFlags();
-
-  switch (Op.getOpcode()) {
-  default: llvm_unreachable("Unknown code");
-  case ISD::ConstantFP: {
-    APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
-    V.changeSign();
-    return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
-  }
-  case ISD::BUILD_VECTOR: {
-    SmallVector<SDValue, 4> Ops;
-    for (SDValue C : Op->op_values()) {
-      if (C.isUndef()) {
-        Ops.push_back(C);
-        continue;
-      }
-      APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
-      V.changeSign();
-      Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType()));
-    }
-    return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops);
-  }
-  case ISD::FADD:
-    assert((Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
-           "Expected NSZ fp-flag");
-
-    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
-    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
-                           DAG.getTargetLoweringInfo(), &Options, ForCodeSize,
-                           Depth + 1))
-      return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
-                         GetNegatedExpression(Op.getOperand(0), DAG,
-                                              LegalOperations, ForCodeSize,
-                                              Depth + 1),
-                         Op.getOperand(1), Flags);
-    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
-    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
-                       GetNegatedExpression(Op.getOperand(1), DAG,
-                                            LegalOperations, ForCodeSize,
-                                            Depth + 1),
-                       Op.getOperand(0), Flags);
-  case ISD::FSUB:
-    // fold (fneg (fsub 0, B)) -> B
-    if (ConstantFPSDNode *N0CFP =
-            isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true))
-      if (N0CFP->isZero())
-        return Op.getOperand(1);
-
-    // fold (fneg (fsub A, B)) -> (fsub B, A)
-    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
-                       Op.getOperand(1), Op.getOperand(0), Flags);
-
-  case ISD::FMUL:
-  case ISD::FDIV:
-    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
-    if (isNegatibleForFree(Op.getOperand(0), LegalOperations,
-                           DAG.getTargetLoweringInfo(), &Options, ForCodeSize,
-                           Depth + 1))
-      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
-                         GetNegatedExpression(Op.getOperand(0), DAG,
-                                              LegalOperations, ForCodeSize,
-                                              Depth + 1),
-                         Op.getOperand(1), Flags);
-
-    // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
-    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
-                       Op.getOperand(0),
-                       GetNegatedExpression(Op.getOperand(1), DAG,
-                                            LegalOperations, ForCodeSize,
-                                            Depth + 1), Flags);
-
-  case ISD::FMA:
-  case ISD::FMAD: {
-    assert((Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros()) &&
-           "Expected NSZ fp-flag");
-
-    SDValue Neg2 = GetNegatedExpression(Op.getOperand(2), DAG, LegalOperations,
-                                        ForCodeSize, Depth + 1);
-
-    char V0 = isNegatibleForFree(Op.getOperand(0), LegalOperations,
-                                 DAG.getTargetLoweringInfo(), &Options,
-                                 ForCodeSize, Depth + 1);
-    char V1 = isNegatibleForFree(Op.getOperand(1), LegalOperations,
-                                 DAG.getTargetLoweringInfo(), &Options,
-                                 ForCodeSize, Depth + 1);
-    if (V0 >= V1) {
-      // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
-      SDValue Neg0 = GetNegatedExpression(
-          Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1);
-      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Neg0,
-                         Op.getOperand(1), Neg2, Flags);
-    }
-
-    // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
-    SDValue Neg1 = GetNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
-                                        ForCodeSize, Depth + 1);
-    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
-                       Op.getOperand(0), Neg1, Neg2, Flags);
-  }
-
-  case ISD::FP_EXTEND:
-  case ISD::FSIN:
-    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
-                       GetNegatedExpression(Op.getOperand(0), DAG,
-                                            LegalOperations, ForCodeSize,
-                                            Depth + 1));
-  case ISD::FP_ROUND:
-    return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
-                       GetNegatedExpression(Op.getOperand(0), DAG,
-                                            LegalOperations, ForCodeSize,
-                                            Depth + 1),
-                       Op.getOperand(1));
-  }
-}
-
 // APInts must be the same size for most operations, this helper
 // function zero extends the shorter of the pair so that they match.
 // We provide an Offset so that we can create bitwidths that won't overflow.
@@ -3384,6 +3139,18 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     }
   }
 
+  if (TLI.isOperationLegalOrCustom(ISD::ADDCARRY, VT)) {
+    // (sub Carry, X)  ->  (addcarry (sub 0, X), 0, Carry)
+    if (SDValue Carry = getAsCarry(TLI, N0)) {
+      SDValue X = N1;
+      SDValue Zero = DAG.getConstant(0, DL, VT);
+      SDValue NegX = DAG.getNode(ISD::SUB, DL, VT, Zero, X);
+      return DAG.getNode(ISD::ADDCARRY, DL,
+                         DAG.getVTList(VT, Carry.getValueType()), NegX, Zero,
+                         Carry);
+    }
+  }
+
   return SDValue();
 }
 
@@ -4939,7 +4706,7 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
   if (LDST->getMemoryVT().getSizeInBits() < MemVT.getSizeInBits())
     return false;
 
-  // Ensure that this isn't going to produce an unsupported unaligned access.
+  // Ensure that this isn't going to produce an unsupported memory access.
   if (ShAmt &&
       !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
                               LDST->getAddressSpace(), ShAmt / 8,
@@ -6049,8 +5816,8 @@ SDValue DAGCombiner::visitOR(SDNode *N) {
       return V;
 
   // See if this is some rotate idiom.
-  if (SDNode *Rot = MatchRotate(N0, N1, SDLoc(N)))
-    return SDValue(Rot, 0);
+  if (SDValue Rot = MatchRotate(N0, N1, SDLoc(N)))
+    return Rot;
 
   if (SDValue Load = MatchLoadCombine(N))
     return Load;
@@ -6335,7 +6102,7 @@ static bool matchRotateSub(SDValue Pos, SDValue Neg, unsigned EltSize,
 // to both (PosOpcode Shifted, Pos) and (NegOpcode Shifted, Neg), with the
 // former being preferred if supported.  InnerPos and InnerNeg are Pos and
 // Neg with outer conversions stripped away.
-SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
+SDValue DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
                                        SDValue Neg, SDValue InnerPos,
                                        SDValue InnerNeg, unsigned PosOpcode,
                                        unsigned NegOpcode, const SDLoc &DL) {
@@ -6350,32 +6117,33 @@ SDNode *DAGCombiner::MatchRotatePosNeg(SDValue Shifted, SDValue Pos,
   if (matchRotateSub(InnerPos, InnerNeg, VT.getScalarSizeInBits(), DAG)) {
     bool HasPos = TLI.isOperationLegalOrCustom(PosOpcode, VT);
     return DAG.getNode(HasPos ? PosOpcode : NegOpcode, DL, VT, Shifted,
-                       HasPos ? Pos : Neg).getNode();
+                       HasPos ? Pos : Neg);
   }
 
-  return nullptr;
+  return SDValue();
 }
 
 // MatchRotate - Handle an 'or' of two operands.  If this is one of the many
 // idioms for rotate, and if the target supports rotation instructions, generate
 // a rot[lr].
-SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
+SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
   // Must be a legal type.  Expanded 'n promoted things won't work with rotates.
   EVT VT = LHS.getValueType();
-  if (!TLI.isTypeLegal(VT)) return nullptr;
+  if (!TLI.isTypeLegal(VT))
+    return SDValue();
 
   // The target must have at least one rotate flavor.
   bool HasROTL = hasOperation(ISD::ROTL, VT);
   bool HasROTR = hasOperation(ISD::ROTR, VT);
-  if (!HasROTL && !HasROTR) return nullptr;
+  if (!HasROTL && !HasROTR)
+    return SDValue();
 
   // Check for truncated rotate.
   if (LHS.getOpcode() == ISD::TRUNCATE && RHS.getOpcode() == ISD::TRUNCATE &&
       LHS.getOperand(0).getValueType() == RHS.getOperand(0).getValueType()) {
     assert(LHS.getValueType() == RHS.getValueType());
-    if (SDNode *Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
-      return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(),
-                         SDValue(Rot, 0)).getNode();
+    if (SDValue Rot = MatchRotate(LHS.getOperand(0), RHS.getOperand(0), DL)) {
+      return DAG.getNode(ISD::TRUNCATE, SDLoc(LHS), LHS.getValueType(), Rot);
     }
   }
 
@@ -6390,7 +6158,7 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
 
   // If neither side matched a rotate half, bail
   if (!LHSShift && !RHSShift)
-    return nullptr;
+    return SDValue();
 
   // InstCombine may have combined a constant shl, srl, mul, or udiv with one
   // side of the rotate, so try to handle that here. In all cases we need to
@@ -6413,15 +6181,15 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
 
   // If a side is still missing, nothing else we can do.
   if (!RHSShift || !LHSShift)
-    return nullptr;
+    return SDValue();
 
   // At this point we've matched or extracted a shift op on each side.
 
   if (LHSShift.getOperand(0) != RHSShift.getOperand(0))
-    return nullptr;   // Not shifting the same value.
+    return SDValue(); // Not shifting the same value.
 
   if (LHSShift.getOpcode() == RHSShift.getOpcode())
-    return nullptr;   // Shifts must disagree.
+    return SDValue(); // Shifts must disagree.
 
   // Canonicalize shl to left side in a shl/srl pair.
   if (RHSShift.getOpcode() == ISD::SHL) {
@@ -6465,13 +6233,13 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
       Rot = DAG.getNode(ISD::AND, DL, VT, Rot, Mask);
     }
 
-    return Rot.getNode();
+    return Rot;
   }
 
   // If there is a mask here, and we have a variable shift, we can't be sure
   // that we're masking out the right stuff.
   if (LHSMask.getNode() || RHSMask.getNode())
-    return nullptr;
+    return SDValue();
 
   // If the shift amount is sign/zext/any-extended just peel it off.
   SDValue LExtOp0 = LHSShiftAmt;
@@ -6488,17 +6256,17 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
     RExtOp0 = RHSShiftAmt.getOperand(0);
   }
 
-  SDNode *TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
+  SDValue TryL = MatchRotatePosNeg(LHSShiftArg, LHSShiftAmt, RHSShiftAmt,
                                    LExtOp0, RExtOp0, ISD::ROTL, ISD::ROTR, DL);
   if (TryL)
     return TryL;
 
-  SDNode *TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
+  SDValue TryR = MatchRotatePosNeg(RHSShiftArg, RHSShiftAmt, LHSShiftAmt,
                                    RExtOp0, LExtOp0, ISD::ROTR, ISD::ROTL, DL);
   if (TryR)
     return TryR;
 
-  return nullptr;
+  return SDValue();
 }
 
 namespace {
@@ -12039,17 +11807,17 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
 
   // fold (fadd A, (fneg B)) -> (fsub A, B)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
-      isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize) == 2)
-    return DAG.getNode(ISD::FSUB, DL, VT, N0,
-                       GetNegatedExpression(N1, DAG, LegalOperations,
-                                            ForCodeSize), Flags);
+      TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize) == 2)
+    return DAG.getNode(
+        ISD::FSUB, DL, VT, N0,
+        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
 
   // fold (fadd (fneg A), B) -> (fsub B, A)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
-      isNegatibleForFree(N0, LegalOperations, TLI, &Options, ForCodeSize) == 2)
-    return DAG.getNode(ISD::FSUB, DL, VT, N1,
-                       GetNegatedExpression(N0, DAG, LegalOperations,
-                                            ForCodeSize), Flags);
+      TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize) == 2)
+    return DAG.getNode(
+        ISD::FSUB, DL, VT, N1,
+        TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize), Flags);
 
   auto isFMulNegTwo = [](SDValue FMul) {
     if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL)
@@ -12228,16 +11996,16 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
   if (N0CFP && N0CFP->isZero()) {
     if (N0CFP->isNegative() ||
         (Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros())) {
-      if (isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize))
-        return GetNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
+      if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
+        return TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
       if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))
         return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags);
     }
   }
 
   if (((Options.UnsafeFPMath && Options.NoSignedZerosFPMath) ||
-      (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros()))
-      && N1.getOpcode() == ISD::FADD) {
+       (Flags.hasAllowReassociation() && Flags.hasNoSignedZeros())) &&
+      N1.getOpcode() == ISD::FADD) {
     // X - (X + Y) -> -Y
     if (N0 == N1->getOperand(0))
       return DAG.getNode(ISD::FNEG, DL, VT, N1->getOperand(1), Flags);
@@ -12247,10 +12015,10 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
   }
 
   // fold (fsub A, (fneg B)) -> (fadd A, B)
-  if (isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize))
-    return DAG.getNode(ISD::FADD, DL, VT, N0,
-                       GetNegatedExpression(N1, DAG, LegalOperations,
-                                            ForCodeSize), Flags);
+  if (TLI.isNegatibleForFree(N1, DAG, LegalOperations, ForCodeSize))
+    return DAG.getNode(
+        ISD::FADD, DL, VT, N0,
+        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
 
   // FSUB -> FMA combines:
   if (SDValue Fused = visitFSUBForFMACombine(N)) {
@@ -12264,11 +12032,10 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
 /// Return true if both inputs are at least as cheap in negated form and at
 /// least one input is strictly cheaper in negated form.
 bool DAGCombiner::isCheaperToUseNegatedFPOps(SDValue X, SDValue Y) {
-  const TargetOptions &Options = DAG.getTarget().Options;
-  if (char LHSNeg = isNegatibleForFree(X, LegalOperations, TLI, &Options,
-                                   ForCodeSize))
-    if (char RHSNeg = isNegatibleForFree(Y, LegalOperations, TLI, &Options,
-                                         ForCodeSize))
+  if (char LHSNeg =
+          TLI.isNegatibleForFree(X, DAG, LegalOperations, ForCodeSize))
+    if (char RHSNeg =
+            TLI.isNegatibleForFree(Y, DAG, LegalOperations, ForCodeSize))
       // Both negated operands are at least as cheap as their counterparts.
       // Check to see if at least one is cheaper negated.
       if (LHSNeg == 2 || RHSNeg == 2)
@@ -12349,8 +12116,10 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
 
   // -N0 * -N1 --> N0 * N1
   if (isCheaperToUseNegatedFPOps(N0, N1)) {
-    SDValue NegN0 = GetNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
-    SDValue NegN1 = GetNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
+    SDValue NegN0 =
+        TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
+    SDValue NegN1 =
+        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
     return DAG.getNode(ISD::FMUL, DL, VT, NegN0, NegN1, Flags);
   }
 
@@ -12432,8 +12201,10 @@ SDValue DAGCombiner::visitFMA(SDNode *N) {
 
   // (-N0 * -N1) + N2 --> (N0 * N1) + N2
   if (isCheaperToUseNegatedFPOps(N0, N1)) {
-    SDValue NegN0 = GetNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
-    SDValue NegN1 = GetNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
+    SDValue NegN0 =
+        TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
+    SDValue NegN1 =
+        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize);
     return DAG.getNode(ISD::FMA, DL, VT, NegN0, NegN1, N2, Flags);
   }
 
@@ -12694,8 +12465,8 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
   if (isCheaperToUseNegatedFPOps(N0, N1))
     return DAG.getNode(
         ISD::FDIV, SDLoc(N), VT,
-        GetNegatedExpression(N0, DAG, LegalOperations, ForCodeSize),
-        GetNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
+        TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize),
+        TLI.getNegatedExpression(N1, DAG, LegalOperations, ForCodeSize), Flags);
 
   return SDValue();
 }
@@ -13249,9 +13020,8 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
   if (isConstantFPBuildVectorOrConstantFP(N0))
     return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0);
 
-  if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(),
-                         &DAG.getTarget().Options, ForCodeSize))
-    return GetNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
+  if (TLI.isNegatibleForFree(N0, DAG, LegalOperations, ForCodeSize))
+    return TLI.getNegatedExpression(N0, DAG, LegalOperations, ForCodeSize);
 
   // Transform fneg(bitconvert(x)) -> bitconvert(x ^ sign) to avoid loading
   // constant pool values.
@@ -16832,12 +16602,12 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT,
   SDValue From[] = { SDValue(EVE, 0), SDValue(OriginalLoad, 1) };
   SDValue To[] = { Load, Chain };
   DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
+  // Make sure to revisit this node to clean it up; it will usually be dead.
+  AddToWorklist(EVE);
   // Since we're explicitly calling ReplaceAllUses, add the new node to the
   // worklist explicitly as well.
-  AddToWorklist(Load.getNode());
   AddUsersToWorklist(Load.getNode()); // Add users too
-  // Make sure to revisit this node to clean it up; it will usually be dead.
-  AddToWorklist(EVE);
+  AddToWorklist(Load.getNode());
   ++OpsNarrowed;
   return SDValue(EVE, 0);
 }
@@ -18977,7 +18747,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) {
   // build_vector.
   if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) {
     int SplatIndex = SVN->getSplatIndex();
-    if (TLI.isExtractVecEltCheap(VT, SplatIndex) &&
+    if (N0.hasOneUse() && TLI.isExtractVecEltCheap(VT, SplatIndex) &&
         TLI.isBinOp(N0.getOpcode()) && N0.getNode()->getNumValues() == 1) {
       // splat (vector_bo L, R), Index -->
       // splat (scalar_bo (extelt L, Index), (extelt R, Index))
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index f8352ca2fe628..8904283ff8dae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1677,11 +1677,11 @@ bool FastISel::selectInstruction(const Instruction *I) {
 /// (fall-through) successor, and update the CFG.
 void FastISel::fastEmitBranch(MachineBasicBlock *MSucc,
                               const DebugLoc &DbgLoc) {
-  if (FuncInfo.MBB->getBasicBlock()->size() > 1 &&
+  if (FuncInfo.MBB->getBasicBlock()->sizeWithoutDebug() > 1 &&
       FuncInfo.MBB->isLayoutSuccessor(MSucc)) {
-    // For more accurate line information if this is the only instruction
-    // in the block then emit it, otherwise we have the unconditional
-    // fall-through case, which needs no instructions.
+    // For more accurate line information if this is the only non-debug
+    // instruction in the block then emit it, otherwise we have the
+    // unconditional fall-through case, which needs no instructions.
   } else {
     // The unconditional branch case.
     TII.insertBranch(*FuncInfo.MBB, MSucc, nullptr,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 3d43295d7df86..0efcaaa19cbde 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -503,8 +503,8 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
       // expand it.
       EVT MemVT = ST->getMemoryVT();
       const DataLayout &DL = DAG.getDataLayout();
-      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
-                                  *ST->getMemOperand())) {
+      if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
+                                              *ST->getMemOperand())) {
         LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
         ReplaceNode(SDValue(ST, 0), Result);
@@ -618,8 +618,8 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
       EVT MemVT = ST->getMemoryVT();
       // If this is an unaligned store and the target doesn't support it,
       // expand it.
-      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
-                                  *ST->getMemOperand())) {
+      if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
+                                              *ST->getMemOperand())) {
         SDValue Result = TLI.expandUnalignedStore(ST, DAG);
         ReplaceNode(SDValue(ST, 0), Result);
       }
@@ -679,8 +679,8 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
       const DataLayout &DL = DAG.getDataLayout();
       // If this is an unaligned load and the target doesn't support it,
       // expand it.
-      if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
-                                  *LD->getMemOperand())) {
+      if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
+                                              *LD->getMemOperand())) {
         std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
       }
       break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index eadc388fc9d5c..560b5729e3def 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -514,15 +514,15 @@ void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
   if (Cond.getValueType().isVector()) {
     if (SDValue Res = WidenVSELECTAndMask(N))
       std::tie(CL, CH) = DAG.SplitVector(Res->getOperand(0), dl);
-    // It seems to improve code to generate two narrow SETCCs as opposed to
-    // splitting a wide result vector.
-    else if (Cond.getOpcode() == ISD::SETCC)
-      SplitVecRes_SETCC(Cond.getNode(), CL, CH);
     // Check if there are already splitted versions of the vector available and
     // use those instead of splitting the mask operand again.
     else if (getTypeAction(Cond.getValueType()) ==
              TargetLowering::TypeSplitVector)
       GetSplitVector(Cond, CL, CH);
+    // It seems to improve code to generate two narrow SETCCs as opposed to
+    // splitting a wide result vector.
+    else if (Cond.getOpcode() == ISD::SETCC)
+      SplitVecRes_SETCC(Cond.getNode(), CL, CH);
     else
       std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
   }
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 96884bad54a9e..0c55ff73c3b74 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1898,7 +1898,7 @@ SDValue SelectionDAG::expandVAArg(SDNode *Node) {
   EVT VT = Node->getValueType(0);
   SDValue Tmp1 = Node->getOperand(0);
   SDValue Tmp2 = Node->getOperand(1);
-  const llvm::MaybeAlign MA(Node->getConstantOperandVal(3));
+  const MaybeAlign MA(Node->getConstantOperandVal(3));
 
   SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
                                Tmp2, MachinePointerInfo(V));
@@ -2380,15 +2380,39 @@ SDValue SelectionDAG::getSplatValue(SDValue V) {
 /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that
 /// is less than the element bit-width of the shift node, return it.
 static const APInt *getValidShiftAmountConstant(SDValue V) {
+  unsigned BitWidth = V.getScalarValueSizeInBits();
   if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1))) {
     // Shifting more than the bitwidth is not valid.
     const APInt &ShAmt = SA->getAPIntValue();
-    if (ShAmt.ult(V.getScalarValueSizeInBits()))
+    if (ShAmt.ult(BitWidth))
       return &ShAmt;
   }
   return nullptr;
 }
 
+/// If a SHL/SRA/SRL node has constant vector shift amounts that are all less
+/// than the element bit-width of the shift node, return the minimum value.
+static const APInt *getValidMinimumShiftAmountConstant(SDValue V) {
+  unsigned BitWidth = V.getScalarValueSizeInBits();
+  auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
+  if (!BV)
+    return nullptr;
+  const APInt *MinShAmt = nullptr;
+  for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
+    auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
+    if (!SA)
+      return nullptr;
+    // Shifting more than the bitwidth is not valid.
+    const APInt &ShAmt = SA->getAPIntValue();
+    if (ShAmt.uge(BitWidth))
+      return nullptr;
+    if (MinShAmt && MinShAmt->ule(ShAmt))
+      continue;
+    MinShAmt = &ShAmt;
+  }
+  return MinShAmt;
+}
+
 /// Determine which bits of Op are known to be either zero or one and return
 /// them in Known. For vectors, the known bits are those that are shared by
 /// every vector element.
@@ -2422,7 +2446,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     return Known;
   }
 
-  if (Depth >= 6)
+  if (Depth >= MaxRecursionDepth)
     return Known;  // Limit search depth.
 
   KnownBits Known2;
@@ -2567,14 +2591,13 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
     SDValue Src = Op.getOperand(0);
     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    APInt DemandedSrc = APInt::getAllOnesValue(NumSrcElts);
     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
       // Offset the demanded elts by the subvector index.
       uint64_t Idx = SubIdx->getZExtValue();
-      APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
-      Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
-    } else {
-      Known = computeKnownBits(Src, Depth + 1);
+      DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
     }
+    Known = computeKnownBits(Src, DemandedSrc, Depth + 1);
     break;
   }
   case ISD::SCALAR_TO_VECTOR: {
@@ -2785,25 +2808,9 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts,
       Known.One.lshrInPlace(Shift);
       // High bits are known zero.
       Known.Zero.setHighBits(Shift);
-    } else if (auto *BV = dyn_cast<BuildVectorSDNode>(Op.getOperand(1))) {
-      // If the shift amount is a vector of constants see if we can bound
-      // the number of upper zero bits.
-      unsigned ShiftAmountMin = BitWidth;
-      for (unsigned i = 0; i != BV->getNumOperands(); ++i) {
-        if (auto *C = dyn_cast<ConstantSDNode>(BV->getOperand(i))) {
-          const APInt &ShAmt = C->getAPIntValue();
-          if (ShAmt.ult(BitWidth)) {
-            ShiftAmountMin = std::min<unsigned>(ShiftAmountMin,
-                                                ShAmt.getZExtValue());
-            continue;
-          }
-        }
-        // Don't know anything.
-        ShiftAmountMin = 0;
-        break;
-      }
-
-      Known.Zero.setHighBits(ShiftAmountMin);
+    } else if (const APInt *ShMinAmt = getValidMinimumShiftAmountConstant(Op)) {
+      // Minimum shift high bits are known zero.
+      Known.Zero.setHighBits(ShMinAmt->getZExtValue());
     }
     break;
   case ISD::SRA:
@@ -3412,7 +3419,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
     return Val.getNumSignBits();
   }
 
-  if (Depth >= 6)
+  if (Depth >= MaxRecursionDepth)
     return 1;  // Limit search depth.
 
   if (!DemandedElts)
@@ -3814,13 +3821,13 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
     SDValue Src = Op.getOperand(0);
     ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
     unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    APInt DemandedSrc = APInt::getAllOnesValue(NumSrcElts);
     if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
       // Offset the demanded elts by the subvector index.
       uint64_t Idx = SubIdx->getZExtValue();
-      APInt DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
-      return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
+      DemandedSrc = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
     }
-    return ComputeNumSignBits(Src, Depth + 1);
+    return ComputeNumSignBits(Src, DemandedSrc, Depth + 1);
   }
   case ISD::CONCAT_VECTORS: {
     // Determine the minimum number of sign bits across all demanded
@@ -3973,7 +3980,7 @@ bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const
   if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
     return true;
 
-  if (Depth >= 6)
+  if (Depth >= MaxRecursionDepth)
     return false; // Limit search depth.
 
   // TODO: Handle vectors.
@@ -5750,7 +5757,7 @@ static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl,
 
 static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
                                        SDValue Chain, SDValue Dst, SDValue Src,
-                                       uint64_t Size, unsigned Align,
+                                       uint64_t Size, unsigned Alignment,
                                        bool isVol, bool AlwaysInline,
                                        MachinePointerInfo DstPtrInfo,
                                        MachinePointerInfo SrcPtrInfo) {
@@ -5775,15 +5782,15 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
   if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
   unsigned SrcAlign = DAG.InferPtrAlignment(Src);
-  if (Align > SrcAlign)
-    SrcAlign = Align;
+  if (Alignment > SrcAlign)
+    SrcAlign = Alignment;
   ConstantDataArraySlice Slice;
   bool CopyFromConstant = isMemSrcFromConstant(Src, Slice);
   bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
   unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
 
   if (!TLI.findOptimalMemOpLowering(
-          MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align),
+          MemOps, Limit, Size, (DstAlignCanChange ? 0 : Alignment),
           (isZeroConstant ? 0 : SrcAlign), /*IsMemset=*/false,
           /*ZeroMemset=*/false, /*MemcpyStrSrc=*/CopyFromConstant,
           /*AllowOverlap=*/!isVol, DstPtrInfo.getAddrSpace(),
@@ -5798,15 +5805,15 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
     // realignment.
     const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
     if (!TRI->needsStackRealignment(MF))
-      while (NewAlign > Align &&
-             DL.exceedsNaturalStackAlignment(llvm::Align(NewAlign)))
-          NewAlign /= 2;
+      while (NewAlign > Alignment &&
+             DL.exceedsNaturalStackAlignment(Align(NewAlign)))
+        NewAlign /= 2;
 
-    if (NewAlign > Align) {
+    if (NewAlign > Alignment) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI.getObjectAlignment(FI->getIndex()) < NewAlign)
         MFI.setObjectAlignment(FI->getIndex(), NewAlign);
-      Align = NewAlign;
+      Alignment = NewAlign;
     }
   }
 
@@ -5849,10 +5856,9 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
       }
       Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
       if (Value.getNode()) {
-        Store = DAG.getStore(Chain, dl, Value,
-                             DAG.getMemBasePlusOffset(Dst, DstOff, dl),
-                             DstPtrInfo.getWithOffset(DstOff), Align,
-                             MMOFlags);
+        Store = DAG.getStore(
+            Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
+            DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags);
         OutChains.push_back(Store);
       }
     }
@@ -5880,7 +5886,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl,
 
       Store = DAG.getTruncStore(
           Chain, dl, Value, DAG.getMemBasePlusOffset(Dst, DstOff, dl),
-          DstPtrInfo.getWithOffset(DstOff), VT, Align, MMOFlags);
+          DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags);
       OutStoreChains.push_back(Store);
     }
     SrcOff += VTSize;
@@ -6547,7 +6553,7 @@ SDValue SelectionDAG::getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl) {
 SDValue SelectionDAG::getMemIntrinsicNode(
     unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
     EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align,
-    MachineMemOperand::Flags Flags, unsigned Size, const AAMDNodes &AAInfo) {
+    MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
   if (Align == 0)  // Ensure that codegen never sees alignment 0
     Align = getEVTAlignment(MemVT);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ce3255c081be3..c6587188bc005 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -2619,24 +2619,18 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
                                              MachineBasicBlock *SwitchBB) {
   SDLoc dl = getCurSDLoc();
 
-  // Subtract the minimum value
+  // Subtract the minimum value.
   SDValue SwitchOp = getValue(B.SValue);
   EVT VT = SwitchOp.getValueType();
-  SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
-                            DAG.getConstant(B.First, dl, VT));
-
-  // Check range
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  SDValue RangeCmp = DAG.getSetCC(
-      dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
-                                 Sub.getValueType()),
-      Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
+  SDValue RangeSub =
+      DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT));
 
   // Determine the type of the test operands.
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   bool UsePtrType = false;
-  if (!TLI.isTypeLegal(VT))
+  if (!TLI.isTypeLegal(VT)) {
     UsePtrType = true;
-  else {
+  } else {
     for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
       if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
         // Switch table case range are encoded into series of masks.
@@ -2645,6 +2639,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
         break;
       }
   }
+  SDValue Sub = RangeSub;
   if (UsePtrType) {
     VT = TLI.getPointerTy(DAG.getDataLayout());
     Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
@@ -2656,20 +2651,29 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
 
   MachineBasicBlock* MBB = B.Cases[0].ThisBB;
 
-  addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
+  if (!B.OmitRangeCheck)
+    addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
   addSuccessorWithProb(SwitchBB, MBB, B.Prob);
   SwitchBB->normalizeSuccProbs();
 
-  SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
-                                MVT::Other, CopyTo, RangeCmp,
-                                DAG.getBasicBlock(B.Default));
+  SDValue Root = CopyTo;
+  if (!B.OmitRangeCheck) {
+    // Conditional branch to the default block.
+    SDValue RangeCmp = DAG.getSetCC(dl,
+        TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
+                               RangeSub.getValueType()),
+        RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()),
+        ISD::SETUGT);
+
+    Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp,
+                       DAG.getBasicBlock(B.Default));
+  }
 
   // Avoid emitting unnecessary branches to the next block.
   if (MBB != NextBlock(SwitchBB))
-    BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
-                          DAG.getBasicBlock(MBB));
+    Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB));
 
-  DAG.setRoot(BrRange);
+  DAG.setRoot(Root);
 }
 
 /// visitBitTestCase - this function produces one "bit test"
@@ -4768,8 +4772,22 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
 
   // Add all operands of the call to the operand list.
   for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
-    SDValue Op = getValue(I.getArgOperand(i));
-    Ops.push_back(Op);
+    const Value *Arg = I.getArgOperand(i);
+    if (!I.paramHasAttr(i, Attribute::ImmArg)) {
+      Ops.push_back(getValue(Arg));
+      continue;
+    }
+
+    // Use TargetConstant instead of a regular constant for immarg.
+    EVT VT = TLI.getValueType(*DL, Arg->getType(), true);
+    if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) {
+      assert(CI->getBitWidth() <= 64 &&
+             "large intrinsic immediates not handled");
+      Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT));
+    } else {
+      Ops.push_back(
+          DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT));
+    }
   }
 
   SmallVector<EVT, 4> ValueVTs;
@@ -5955,65 +5973,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
   case Intrinsic::masked_compressstore:
     visitMaskedStore(I, true /* IsCompressing */);
     return;
-  case Intrinsic::x86_mmx_pslli_w:
-  case Intrinsic::x86_mmx_pslli_d:
-  case Intrinsic::x86_mmx_pslli_q:
-  case Intrinsic::x86_mmx_psrli_w:
-  case Intrinsic::x86_mmx_psrli_d:
-  case Intrinsic::x86_mmx_psrli_q:
-  case Intrinsic::x86_mmx_psrai_w:
-  case Intrinsic::x86_mmx_psrai_d: {
-    SDValue ShAmt = getValue(I.getArgOperand(1));
-    if (isa<ConstantSDNode>(ShAmt)) {
-      visitTargetIntrinsic(I, Intrinsic);
-      return;
-    }
-    unsigned NewIntrinsic = 0;
-    EVT ShAmtVT = MVT::v2i32;
-    switch (Intrinsic) {
-    case Intrinsic::x86_mmx_pslli_w:
-      NewIntrinsic = Intrinsic::x86_mmx_psll_w;
-      break;
-    case Intrinsic::x86_mmx_pslli_d:
-      NewIntrinsic = Intrinsic::x86_mmx_psll_d;
-      break;
-    case Intrinsic::x86_mmx_pslli_q:
-      NewIntrinsic = Intrinsic::x86_mmx_psll_q;
-      break;
-    case Intrinsic::x86_mmx_psrli_w:
-      NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
-      break;
-    case Intrinsic::x86_mmx_psrli_d:
-      NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
-      break;
-    case Intrinsic::x86_mmx_psrli_q:
-      NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
-      break;
-    case Intrinsic::x86_mmx_psrai_w:
-      NewIntrinsic = Intrinsic::x86_mmx_psra_w;
-      break;
-    case Intrinsic::x86_mmx_psrai_d:
-      NewIntrinsic = Intrinsic::x86_mmx_psra_d;
-      break;
-    default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
-    }
-
-    // The vector shift intrinsics with scalars uses 32b shift amounts but
-    // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
-    // to be zero.
-    // We must do this early because v2i32 is not a legal type.
-    SDValue ShOps[2];
-    ShOps[0] = ShAmt;
-    ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
-    ShAmt =  DAG.getBuildVector(ShAmtVT, sdl, ShOps);
-    EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
-    ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
-    Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
-                       DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
-                       getValue(I.getArgOperand(0)), ShAmt);
-    setValue(&I, Res);
-    return;
-  }
   case Intrinsic::powi:
     setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
                             getValue(I.getArgOperand(1)), DAG));
@@ -10209,8 +10168,6 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
         break;
       }
       case CC_BitTests: {
-        // FIXME: If Fallthrough is unreachable, skip the range check.
-
         // FIXME: Optimize away range check based on pivot comparisons.
         BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex];
 
@@ -10231,6 +10188,11 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
           BTB->DefaultProb -= DefaultProb / 2;
         }
 
+        if (FallthroughUnreachable) {
+          // Skip the range check if the fallthrough block is unreachable.
+          BTB->OmitRangeCheck = true;
+        }
+
         // If we're in the right place, emit the bit test header right now.
         if (CurMBB == SwitchMBB) {
           visitBitTestHeader(*BTB, SwitchMBB);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 10d98616f6327..121de700f6ba4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -2231,9 +2231,9 @@ void SelectionDAGISel::Select_READ_REGISTER(SDNode *Op) {
   SDLoc dl(Op);
   MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
   const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
-  unsigned Reg =
+  Register Reg =
       TLI->getRegisterByName(RegStr->getString().data(), Op->getValueType(0),
-                             *CurDAG);
+                             CurDAG->getMachineFunction());
   SDValue New = CurDAG->getCopyFromReg(
                         Op->getOperand(0), dl, Reg, Op->getValueType(0));
   New->setNodeId(-1);
@@ -2245,9 +2245,9 @@ void SelectionDAGISel::Select_WRITE_REGISTER(SDNode *Op) {
   SDLoc dl(Op);
   MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(Op->getOperand(1));
   const MDString *RegStr = dyn_cast<MDString>(MD->getMD()->getOperand(0));
-  unsigned Reg = TLI->getRegisterByName(RegStr->getString().data(),
+  Register Reg = TLI->getRegisterByName(RegStr->getString().data(),
                                         Op->getOperand(2).getValueType(),
-                                        *CurDAG);
+                                        CurDAG->getMachineFunction());
   SDValue New = CurDAG->getCopyToReg(
                         Op->getOperand(0), dl, Reg, Op->getOperand(2));
   New->setNodeId(-1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 4fc9f2465d30e..0163ec35951e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -585,7 +585,7 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedBits(
     SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
     SelectionDAG &DAG, unsigned Depth) const {
   // Limit search depth.
-  if (Depth >= 6)
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
     return SDValue();
 
   // Ignore UNDEFs.
@@ -798,12 +798,15 @@ bool TargetLowering::SimplifyDemandedBits(
   } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) {
     // Not demanding any bits/elts from Op.
     return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
-  } else if (Depth >= 6) { // Limit search depth.
+  } else if (Depth >= SelectionDAG::MaxRecursionDepth) {
+    // Limit search depth.
     return false;
   }
 
   KnownBits Known2, KnownOut;
   switch (Op.getOpcode()) {
+  case ISD::TargetConstant:
+    llvm_unreachable("Can't simplify this node");
   case ISD::SCALAR_TO_VECTOR: {
     if (!DemandedElts[0])
       return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
@@ -907,6 +910,21 @@ bool TargetLowering::SimplifyDemandedBits(
     }
     break;
   }
+  case ISD::EXTRACT_SUBVECTOR: {
+    // If index isn't constant, assume we need all the source vector elements.
+    SDValue Src = Op.getOperand(0);
+    ConstantSDNode *SubIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
+    unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
+    APInt SrcElts = APInt::getAllOnesValue(NumSrcElts);
+    if (SubIdx && SubIdx->getAPIntValue().ule(NumSrcElts - NumElts)) {
+      // Offset the demanded elts by the subvector index.
+      uint64_t Idx = SubIdx->getZExtValue();
+      SrcElts = DemandedElts.zextOrSelf(NumSrcElts).shl(Idx);
+    }
+    if (SimplifyDemandedBits(Src, DemandedBits, SrcElts, Known, TLO, Depth + 1))
+      return true;
+    break;
+  }
   case ISD::CONCAT_VECTORS: {
     Known.Zero.setAllBits();
     Known.One.setAllBits();
@@ -1270,7 +1288,7 @@ bool TargetLowering::SimplifyDemandedBits(
       // out) are never demanded.
       // TODO - support non-uniform vector amounts.
       if (Op0.getOpcode() == ISD::SRL) {
-        if ((DemandedBits & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) {
+        if (!DemandedBits.intersects(APInt::getLowBitsSet(BitWidth, ShAmt))) {
           if (ConstantSDNode *SA2 =
                   isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
             if (SA2->getAPIntValue().ult(BitWidth)) {
@@ -1377,7 +1395,8 @@ bool TargetLowering::SimplifyDemandedBits(
       if (Op0.getOpcode() == ISD::SHL) {
         if (ConstantSDNode *SA2 =
                 isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) {
-          if ((DemandedBits & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) {
+          if (!DemandedBits.intersects(
+                  APInt::getHighBitsSet(BitWidth, ShAmt))) {
             if (SA2->getAPIntValue().ult(BitWidth)) {
               unsigned C1 = SA2->getZExtValue();
               unsigned Opc = ISD::SRL;
@@ -2091,7 +2110,7 @@ bool TargetLowering::SimplifyDemandedVectorElts(
   }
 
   // Limit search depth.
-  if (Depth >= 6)
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
     return false;
 
   SDLoc DL(Op);
@@ -2570,8 +2589,9 @@ void TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
 }
 
 void TargetLowering::computeKnownBitsForTargetInstr(
-    Register R, KnownBits &Known, const APInt &DemandedElts,
-    const MachineRegisterInfo &MRI, unsigned Depth) const {
+    GISelKnownBits &Analysis, Register R, KnownBits &Known,
+    const APInt &DemandedElts, const MachineRegisterInfo &MRI,
+    unsigned Depth) const {
   Known.resetAll();
 }
 
@@ -5314,6 +5334,246 @@ verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const {
   return false;
 }
 
+char TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
+                                        bool LegalOperations, bool ForCodeSize,
+                                        unsigned Depth) const {
+  // fneg is removable even if it has multiple uses.
+  if (Op.getOpcode() == ISD::FNEG)
+    return 2;
+
+  // Don't allow anything with multiple uses unless we know it is free.
+  EVT VT = Op.getValueType();
+  const SDNodeFlags Flags = Op->getFlags();
+  const TargetOptions &Options = DAG.getTarget().Options;
+  if (!Op.hasOneUse() && !(Op.getOpcode() == ISD::FP_EXTEND &&
+                           isFPExtFree(VT, Op.getOperand(0).getValueType())))
+    return 0;
+
+  // Don't recurse exponentially.
+  if (Depth > SelectionDAG::MaxRecursionDepth)
+    return 0;
+
+  switch (Op.getOpcode()) {
+  case ISD::ConstantFP: {
+    if (!LegalOperations)
+      return 1;
+
+    // Don't invert constant FP values after legalization unless the target says
+    // the negated constant is legal.
+    return isOperationLegal(ISD::ConstantFP, VT) ||
+           isFPImmLegal(neg(cast<ConstantFPSDNode>(Op)->getValueAPF()), VT,
+                        ForCodeSize);
+  }
+  case ISD::BUILD_VECTOR: {
+    // Only permit BUILD_VECTOR of constants.
+    if (llvm::any_of(Op->op_values(), [&](SDValue N) {
+          return !N.isUndef() && !isa<ConstantFPSDNode>(N);
+        }))
+      return 0;
+    if (!LegalOperations)
+      return 1;
+    if (isOperationLegal(ISD::ConstantFP, VT) &&
+        isOperationLegal(ISD::BUILD_VECTOR, VT))
+      return 1;
+    return llvm::all_of(Op->op_values(), [&](SDValue N) {
+      return N.isUndef() ||
+             isFPImmLegal(neg(cast<ConstantFPSDNode>(N)->getValueAPF()), VT,
+                          ForCodeSize);
+    });
+  }
+  case ISD::FADD:
+    if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+      return 0;
+
+    // After operation legalization, it might not be legal to create new FSUBs.
+    if (LegalOperations && !isOperationLegalOrCustom(ISD::FSUB, VT))
+      return 0;
+
+    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
+    if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
+                                    ForCodeSize, Depth + 1))
+      return V;
+    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
+    return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
+                              ForCodeSize, Depth + 1);
+  case ISD::FSUB:
+    // We can't turn -(A-B) into B-A when we honor signed zeros.
+    if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+      return 0;
+
+    // fold (fneg (fsub A, B)) -> (fsub B, A)
+    return 1;
+
+  case ISD::FMUL:
+  case ISD::FDIV:
+    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y))
+    if (char V = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
+                                    ForCodeSize, Depth + 1))
+      return V;
+
+    // Ignore X * 2.0 because that is expected to be canonicalized to X + X.
+    if (auto *C = isConstOrConstSplatFP(Op.getOperand(1)))
+      if (C->isExactlyValue(2.0) && Op.getOpcode() == ISD::FMUL)
+        return 0;
+
+    return isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
+                              ForCodeSize, Depth + 1);
+
+  case ISD::FMA:
+  case ISD::FMAD: {
+    if (!Options.NoSignedZerosFPMath && !Flags.hasNoSignedZeros())
+      return 0;
+
+    // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
+    // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
+    char V2 = isNegatibleForFree(Op.getOperand(2), DAG, LegalOperations,
+                                 ForCodeSize, Depth + 1);
+    if (!V2)
+      return 0;
+
+    // One of Op0/Op1 must be cheaply negatible, then select the cheapest.
+    char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
+                                 ForCodeSize, Depth + 1);
+    char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
+                                 ForCodeSize, Depth + 1);
+    char V01 = std::max(V0, V1);
+    return V01 ? std::max(V01, V2) : 0;
+  }
+
+  case ISD::FP_EXTEND:
+  case ISD::FP_ROUND:
+  case ISD::FSIN:
+    return isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
+                              ForCodeSize, Depth + 1);
+  }
+
+  return 0;
+}
+
+SDValue TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
+                                             bool LegalOperations,
+                                             bool ForCodeSize,
+                                             unsigned Depth) const {
+  // fneg is removable even if it has multiple uses.
+  if (Op.getOpcode() == ISD::FNEG)
+    return Op.getOperand(0);
+
+  assert(Depth <= SelectionDAG::MaxRecursionDepth &&
+         "getNegatedExpression doesn't match isNegatibleForFree");
+  const SDNodeFlags Flags = Op->getFlags();
+
+  switch (Op.getOpcode()) {
+  case ISD::ConstantFP: {
+    APFloat V = cast<ConstantFPSDNode>(Op)->getValueAPF();
+    V.changeSign();
+    return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType());
+  }
+  case ISD::BUILD_VECTOR: {
+    SmallVector<SDValue, 4> Ops;
+    for (SDValue C : Op->op_values()) {
+      if (C.isUndef()) {
+        Ops.push_back(C);
+        continue;
+      }
+      APFloat V = cast<ConstantFPSDNode>(C)->getValueAPF();
+      V.changeSign();
+      Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType()));
+    }
+    return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops);
+  }
+  case ISD::FADD:
+    assert((DAG.getTarget().Options.NoSignedZerosFPMath ||
+            Flags.hasNoSignedZeros()) &&
+           "Expected NSZ fp-flag");
+
+    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
+    if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
+                           Depth + 1))
+      return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
+                         getNegatedExpression(Op.getOperand(0), DAG,
+                                              LegalOperations, ForCodeSize,
+                                              Depth + 1),
+                         Op.getOperand(1), Flags);
+    // fold (fneg (fadd A, B)) -> (fsub (fneg B), A)
+    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
+                       getNegatedExpression(Op.getOperand(1), DAG,
+                                            LegalOperations, ForCodeSize,
+                                            Depth + 1),
+                       Op.getOperand(0), Flags);
+  case ISD::FSUB:
+    // fold (fneg (fsub 0, B)) -> B
+    if (ConstantFPSDNode *N0CFP =
+            isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true))
+      if (N0CFP->isZero())
+        return Op.getOperand(1);
+
+    // fold (fneg (fsub A, B)) -> (fsub B, A)
+    return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(1), Op.getOperand(0), Flags);
+
+  case ISD::FMUL:
+  case ISD::FDIV:
+    // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y)
+    if (isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations, ForCodeSize,
+                           Depth + 1))
+      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
+                         getNegatedExpression(Op.getOperand(0), DAG,
+                                              LegalOperations, ForCodeSize,
+                                              Depth + 1),
+                         Op.getOperand(1), Flags);
+
+    // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y))
+    return DAG.getNode(
+        Op.getOpcode(), SDLoc(Op), Op.getValueType(), Op.getOperand(0),
+        getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
+                             ForCodeSize, Depth + 1),
+        Flags);
+
+  case ISD::FMA:
+  case ISD::FMAD: {
+    assert((DAG.getTarget().Options.NoSignedZerosFPMath ||
+            Flags.hasNoSignedZeros()) &&
+           "Expected NSZ fp-flag");
+
+    SDValue Neg2 = getNegatedExpression(Op.getOperand(2), DAG, LegalOperations,
+                                        ForCodeSize, Depth + 1);
+
+    char V0 = isNegatibleForFree(Op.getOperand(0), DAG, LegalOperations,
+                                 ForCodeSize, Depth + 1);
+    char V1 = isNegatibleForFree(Op.getOperand(1), DAG, LegalOperations,
+                                 ForCodeSize, Depth + 1);
+    if (V0 >= V1) {
+      // fold (fneg (fma X, Y, Z)) -> (fma (fneg X), Y, (fneg Z))
+      SDValue Neg0 = getNegatedExpression(
+          Op.getOperand(0), DAG, LegalOperations, ForCodeSize, Depth + 1);
+      return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Neg0,
+                         Op.getOperand(1), Neg2, Flags);
+    }
+
+    // fold (fneg (fma X, Y, Z)) -> (fma X, (fneg Y), (fneg Z))
+    SDValue Neg1 = getNegatedExpression(Op.getOperand(1), DAG, LegalOperations,
+                                        ForCodeSize, Depth + 1);
+    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
+                       Op.getOperand(0), Neg1, Neg2, Flags);
+  }
+
+  case ISD::FP_EXTEND:
+  case ISD::FSIN:
+    return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
+                       getNegatedExpression(Op.getOperand(0), DAG,
+                                            LegalOperations, ForCodeSize,
+                                            Depth + 1));
+  case ISD::FP_ROUND:
+    return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(),
+                       getNegatedExpression(Op.getOperand(0), DAG,
+                                            LegalOperations, ForCodeSize,
+                                            Depth + 1),
+                       Op.getOperand(1));
+  }
+
+  llvm_unreachable("Unknown code");
+}
+
 //===----------------------------------------------------------------------===//
 // Legalization Utilities
 //===----------------------------------------------------------------------===//
@@ -6888,24 +7148,19 @@ void TargetLowering::expandSADDSUBO(
 
   SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
 
-  //   LHSSign -> LHS >= 0
-  //   RHSSign -> RHS >= 0
-  //   SumSign -> Result >= 0
-  //
-  //   Add:
-  //   Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
-  //   Sub:
-  //   Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
-  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
-  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
-  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
-                                    IsAdd ? ISD::SETEQ : ISD::SETNE);
-
-  SDValue SumSign = DAG.getSetCC(dl, OType, Result, Zero, ISD::SETGE);
-  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
-
-  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
-  Overflow = DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType);
+  // For an addition, the result should be less than one of the operands (LHS)
+  // if and only if the other operand (RHS) is negative, otherwise there will
+  // be overflow.
+  // For a subtraction, the result should be less than one of the operands
+  // (LHS) if and only if the other operand (RHS) is (non-zero) positive,
+  // otherwise there will be overflow.
+  SDValue ResultLowerThanLHS = DAG.getSetCC(dl, OType, Result, LHS, ISD::SETLT);
+  SDValue ConditionRHS =
+      DAG.getSetCC(dl, OType, RHS, Zero, IsAdd ? ISD::SETLT : ISD::SETGT);
+
+  Overflow = DAG.getBoolExtOrTrunc(
+      DAG.getNode(ISD::XOR, dl, OType, ConditionRHS, ResultLowerThanLHS), dl,
+      ResultType, ResultType);
 }
 
 bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result,
diff --git a/llvm/lib/CodeGen/StackMaps.cpp b/llvm/lib/CodeGen/StackMaps.cpp
index f4e6aa2471c8a..383c91259ffc8 100644
--- a/llvm/lib/CodeGen/StackMaps.cpp
+++ b/llvm/lib/CodeGen/StackMaps.cpp
@@ -155,7 +155,7 @@ StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI,
 
     unsigned Offset = 0;
     unsigned DwarfRegNum = getDwarfRegNum(MOI->getReg(), TRI);
-    unsigned LLVMRegNum = TRI->getLLVMRegNum(DwarfRegNum, false);
+    unsigned LLVMRegNum = *TRI->getLLVMRegNum(DwarfRegNum, false);
     unsigned SubRegIdx = TRI->getSubRegIndex(LLVMRegNum, MOI->getReg());
     if (SubRegIdx)
       Offset = TRI->getSubRegIdxOffset(SubRegIdx);
diff --git a/llvm/lib/CodeGen/StackProtector.cpp b/llvm/lib/CodeGen/StackProtector.cpp
index 809960c7fdf98..5683d1db473c0 100644
--- a/llvm/lib/CodeGen/StackProtector.cpp
+++ b/llvm/lib/CodeGen/StackProtector.cpp
@@ -17,7 +17,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/BranchProbabilityInfo.h"
-#include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/EHPersonalities.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/CodeGen/Passes.h"
@@ -157,6 +156,68 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge,
   return NeedsProtector;
 }
 
+bool StackProtector::HasAddressTaken(const Instruction *AI) {
+  for (const User *U : AI->users()) {
+    const auto *I = cast<Instruction>(U);
+    switch (I->getOpcode()) {
+    case Instruction::Store:
+      if (AI == cast<StoreInst>(I)->getValueOperand())
+        return true;
+      break;
+    case Instruction::AtomicCmpXchg:
+      // cmpxchg conceptually includes both a load and store from the same
+      // location. So, like store, the value being stored is what matters.
+      if (AI == cast<AtomicCmpXchgInst>(I)->getNewValOperand())
+        return true;
+      break;
+    case Instruction::PtrToInt:
+      if (AI == cast<PtrToIntInst>(I)->getOperand(0))
+        return true;
+      break;
+    case Instruction::Call: {
+      // Ignore intrinsics that do not become real instructions.
+      // TODO: Narrow this to intrinsics that have store-like effects.
+      const auto *CI = cast<CallInst>(I);
+      if (!isa<DbgInfoIntrinsic>(CI) && !CI->isLifetimeStartOrEnd())
+        return true;
+      break;
+    }
+    case Instruction::Invoke:
+      return true;
+    case Instruction::BitCast:
+    case Instruction::GetElementPtr:
+    case Instruction::Select:
+    case Instruction::AddrSpaceCast:
+      if (HasAddressTaken(I))
+        return true;
+      break;
+    case Instruction::PHI: {
+      // Keep track of what PHI nodes we have already visited to ensure
+      // they are only visited once.
+      const auto *PN = cast<PHINode>(I);
+      if (VisitedPHIs.insert(PN).second)
+        if (HasAddressTaken(PN))
+          return true;
+      break;
+    }
+    case Instruction::Load:
+    case Instruction::AtomicRMW:
+    case Instruction::Ret:
+      // These instructions take an address operand, but have load-like or
+      // other innocuous behavior that should not trigger a stack protector.
+      // atomicrmw conceptually has both load and store semantics, but the
+      // value being stored must be integer; so if a pointer is being stored,
+      // we'll catch it in the PtrToInt case above.
+      break;
+    default:
+      // Conservatively return true for any instruction that takes an address
+      // operand, but is not handled above.
+      return true;
+    }
+  }
+  return false;
+}
+
 /// Search for the first call to the llvm.stackprotector intrinsic and return it
 /// if present.
 static const CallInst *findStackProtectorIntrinsic(Function &F) {
@@ -264,9 +325,7 @@ bool StackProtector::RequiresStackProtector() {
           continue;
         }
 
-        if (Strong && PointerMayBeCaptured(AI,
-                                           /* ReturnCaptures */ false,
-                                           /* StoreCaptures */ true)) {
+        if (Strong && HasAddressTaken(AI)) {
           ++NumAddrTaken;
           Layout.insert(std::make_pair(AI, MachineFrameInfo::SSPLK_AddrOf));
           ORE.emit([&]() {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 0406649345b36..2108fc6fecf88 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -282,7 +282,7 @@ bool TargetInstrInfo::fixCommutedOpIndices(unsigned &ResultIdx1,
   return true;
 }
 
-bool TargetInstrInfo::findCommutedOpIndices(MachineInstr &MI,
+bool TargetInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
                                             unsigned &SrcOpIdx1,
                                             unsigned &SrcOpIdx2) const {
   assert(!MI.isBundle() &&
@@ -394,7 +394,7 @@ bool TargetInstrInfo::getStackSlotRange(const TargetRegisterClass *RC,
   if (BitOffset < 0 || BitOffset % 8)
     return false;
 
-  Size = BitSize /= 8;
+  Size = BitSize / 8;
   Offset = (unsigned)BitOffset / 8;
 
   assert(TRI->getSpillSize(*RC) >= (Offset + Size) && "bad subregister range");
@@ -1133,18 +1133,6 @@ TargetInstrInfo::describeLoadedValue(const MachineInstr &MI) const {
   } else if (MI.isMoveImmediate()) {
     Op = &MI.getOperand(1);
     return ParamLoadedValue(*Op, Expr);
-  } else if (MI.hasOneMemOperand()) {
-    int64_t Offset;
-    const auto &TRI = MF->getSubtarget().getRegisterInfo();
-    const auto &TII = MF->getSubtarget().getInstrInfo();
-    const MachineOperand *BaseOp;
-
-    if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI))
-      return None;
-
-    Expr = DIExpression::prepend(Expr, DIExpression::DerefAfter, Offset);
-    Op = BaseOp;
-    return ParamLoadedValue(*Op, Expr);
   }
 
   return None;
@@ -1257,3 +1245,5 @@ bool TargetInstrInfo::getInsertSubregInputs(
   InsertedReg.SubIdx = (unsigned)MOSubIdx.getImm();
   return true;
 }
+
+TargetInstrInfo::PipelinerLoopInfo::~PipelinerLoopInfo() {}
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 7c6860eb26c37..bc005a2cc27be 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1266,7 +1266,8 @@ void TargetLoweringBase::computeRegisterProperties(
     case TypePromoteInteger:
       // Try to promote the elements of integer vectors. If no legal
       // promotion was found, fall through to the widen-vector method.
-      for (unsigned nVT = i + 1; nVT <= MVT::LAST_INTEGER_VECTOR_VALUETYPE; ++nVT) {
+      for (unsigned nVT = i + 1;
+           nVT <= MVT::LAST_INTEGER_FIXEDLEN_VECTOR_VALUETYPE; ++nVT) {
         MVT SVT = (MVT::SimpleValueType) nVT;
         // Promote vectors of integers to vectors with the same number
         // of elements, with a wider element type.
@@ -1504,12 +1505,9 @@ unsigned TargetLoweringBase::getByValTypeAlignment(Type *Ty,
   return DL.getABITypeAlignment(Ty);
 }
 
-bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
-                                            const DataLayout &DL, EVT VT,
-                                            unsigned AddrSpace,
-                                            unsigned Alignment,
-                                            MachineMemOperand::Flags Flags,
-                                            bool *Fast) const {
+bool TargetLoweringBase::allowsMemoryAccessForAlignment(
+    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
+    unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
   // Check if the specified alignment is sufficient based on the data layout.
   // TODO: While using the data layout works in practice, a better solution
   // would be to implement this check directly (make this a virtual function).
@@ -1527,6 +1525,21 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
   return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
 }
 
+bool TargetLoweringBase::allowsMemoryAccessForAlignment(
+    LLVMContext &Context, const DataLayout &DL, EVT VT,
+    const MachineMemOperand &MMO, bool *Fast) const {
+  return allowsMemoryAccessForAlignment(Context, DL, VT, MMO.getAddrSpace(),
+                                        MMO.getAlignment(), MMO.getFlags(),
+                                        Fast);
+}
+
+bool TargetLoweringBase::allowsMemoryAccess(
+    LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
+    unsigned Alignment, MachineMemOperand::Flags Flags, bool *Fast) const {
+  return allowsMemoryAccessForAlignment(Context, DL, VT, AddrSpace, Alignment,
+                                        Flags, Fast);
+}
+
 bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
                                             const DataLayout &DL, EVT VT,
                                             const MachineMemOperand &MMO,
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 68a14453e7601..89a98bba65e5c 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -377,7 +377,7 @@ void TargetLoweringObjectFileELF::emitPersonalityValue(
                                                    ELF::SHT_PROGBITS, Flags, 0);
   unsigned Size = DL.getPointerSize();
   Streamer.SwitchSection(Sec);
-  Streamer.EmitValueToAlignment(DL.getPointerABIAlignment(0));
+  Streamer.EmitValueToAlignment(DL.getPointerABIAlignment(0).value());
   Streamer.EmitSymbolAttribute(Label, MCSA_ELF_TypeObject);
   const MCExpr *E = MCConstantExpr::create(Size, getContext());
   Streamer.emitELFSize(Label, E);
diff --git a/llvm/lib/CodeGen/UnreachableBlockElim.cpp b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
index de15e6c814b68..9d00219263677 100644
--- a/llvm/lib/CodeGen/UnreachableBlockElim.cpp
+++ b/llvm/lib/CodeGen/UnreachableBlockElim.cpp
@@ -103,7 +103,8 @@ bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
   df_iterator_default_set<MachineBasicBlock*> Reachable;
   bool ModifiedPHI = false;
 
-  MMI = getAnalysisIfAvailable<MachineModuleInfo>();
+  auto *MMIWP = getAnalysisIfAvailable<MachineModuleInfoWrapperPass>();
+  MMI = MMIWP ? &MMIWP->getMMI() : nullptr;
   MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
   MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
 
diff --git a/llvm/lib/CodeGen/ValueTypes.cpp b/llvm/lib/CodeGen/ValueTypes.cpp
index f56aa84bad096..73b862d51c0f7 100644
--- a/llvm/lib/CodeGen/ValueTypes.cpp
+++ b/llvm/lib/CodeGen/ValueTypes.cpp
@@ -144,6 +144,7 @@ std::string EVT::getEVTString() const {
   case MVT::v32i1:   return "v32i1";
   case MVT::v64i1:   return "v64i1";
   case MVT::v128i1:  return "v128i1";
+  case MVT::v256i1:  return "v256i1";
   case MVT::v512i1:  return "v512i1";
   case MVT::v1024i1: return "v1024i1";
   case MVT::v1i8:    return "v1i8";
@@ -287,6 +288,7 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::v32i1:   return VectorType::get(Type::getInt1Ty(Context), 32);
   case MVT::v64i1:   return VectorType::get(Type::getInt1Ty(Context), 64);
   case MVT::v128i1:  return VectorType::get(Type::getInt1Ty(Context), 128);
+  case MVT::v256i1:  return VectorType::get(Type::getInt1Ty(Context), 256);
   case MVT::v512i1:  return VectorType::get(Type::getInt1Ty(Context), 512);
   case MVT::v1024i1: return VectorType::get(Type::getInt1Ty(Context), 1024);
   case MVT::v1i8:    return VectorType::get(Type::getInt8Ty(Context), 1);
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
index 5507448ff5986..f25431321bbc9 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp
@@ -101,6 +101,7 @@ DWARFDebugLoc::parseOneLocationList(const DWARFDataExtractor &Data,
 
     if (Error Err = C.takeError())
       return std::move(Err);
+
     // The end of any given location list is marked by an end of list entry,
     // which consists of a 0 for the beginning address offset and a 0 for the
     // ending address offset.
@@ -109,9 +110,12 @@ DWARFDebugLoc::parseOneLocationList(const DWARFDataExtractor &Data,
       return LL;
     }
 
-    unsigned Bytes = Data.getU16(C);
-    // A single location description describing the location of the object...
-    Data.getU8(C, E.Loc, Bytes);
+    if (E.Begin != (AddressSize == 4 ? -1U : -1ULL)) {
+      unsigned Bytes = Data.getU16(C);
+      // A single location description describing the location of the object...
+      Data.getU8(C, E.Loc, Bytes);
+    }
+
     LL.Entries.push_back(std::move(E));
   }
 }
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index e897a0eadbf81..5009b1b7b4127 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -218,9 +218,8 @@ static bool prettyPrintRegisterOp(raw_ostream &OS, uint8_t Opcode,
   else
     DwarfRegNum = Opcode - DW_OP_reg0;
 
-  int LLVMRegNum = MRI->getLLVMRegNum(DwarfRegNum, isEH);
-  if (LLVMRegNum >= 0) {
-    if (const char *RegName = MRI->getName(LLVMRegNum)) {
+  if (Optional<unsigned> LLVMRegNum = MRI->getLLVMRegNum(DwarfRegNum, isEH)) {
+    if (const char *RegName = MRI->getName(*LLVMRegNum)) {
       if ((Opcode >= DW_OP_breg0 && Opcode <= DW_OP_breg31) ||
           Opcode == DW_OP_bregx)
         OS << format(" %s%+" PRId64, RegName, Operands[OpNum]);
diff --git a/llvm/lib/DebugInfo/GSYM/CMakeLists.txt b/llvm/lib/DebugInfo/GSYM/CMakeLists.txt
index af9910fb491b9..e05b2c112ce78 100644
--- a/llvm/lib/DebugInfo/GSYM/CMakeLists.txt
+++ b/llvm/lib/DebugInfo/GSYM/CMakeLists.txt
@@ -1,4 +1,5 @@
 add_llvm_library(LLVMDebugInfoGSYM
+  Header.cpp
   FileWriter.cpp
   FunctionInfo.cpp
   InlineInfo.cpp
diff --git a/llvm/lib/DebugInfo/GSYM/FunctionInfo.cpp b/llvm/lib/DebugInfo/GSYM/FunctionInfo.cpp
index 5b8b7ea44fb7a..dff867d66eb97 100644
--- a/llvm/lib/DebugInfo/GSYM/FunctionInfo.cpp
+++ b/llvm/lib/DebugInfo/GSYM/FunctionInfo.cpp
@@ -1,19 +1,147 @@
 //===- FunctionInfo.cpp -----------------------------------------*- C++ -*-===//
 //
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/DebugInfo/GSYM/FunctionInfo.h"
+#include "llvm/DebugInfo/GSYM/FileWriter.h"
+#include "llvm/DebugInfo/GSYM/LineTable.h"
+#include "llvm/DebugInfo/GSYM/InlineInfo.h"
+#include "llvm/Support/DataExtractor.h"
 
 using namespace llvm;
 using namespace gsym;
 
+/// FunctionInfo information type that is used to encode the optional data
+/// that is associated with a FunctionInfo object.
+enum InfoType : uint32_t {
+  EndOfList = 0u,
+  LineTableInfo = 1u,
+  InlineInfo = 2u
+};
+
 raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const FunctionInfo &FI) {
   OS << '[' << HEX64(FI.Range.Start) << '-' << HEX64(FI.Range.End) << "): "
      << "Name=" << HEX32(FI.Name) << '\n' << FI.OptLineTable << FI.Inline;
   return OS;
 }
+
+llvm::Expected<FunctionInfo> FunctionInfo::decode(DataExtractor &Data,
+                                                  uint64_t BaseAddr) {
+  FunctionInfo FI;
+  FI.Range.Start = BaseAddr;
+  uint64_t Offset = 0;
+  if (!Data.isValidOffsetForDataOfSize(Offset, 4))
+    return createStringError(std::errc::io_error,
+        "0x%8.8" PRIx64 ": missing FunctionInfo Size", Offset);
+  FI.Range.End = FI.Range.Start + Data.getU32(&Offset);
+  if (!Data.isValidOffsetForDataOfSize(Offset, 4))
+    return createStringError(std::errc::io_error,
+        "0x%8.8" PRIx64 ": missing FunctionInfo Name", Offset);
+  FI.Name = Data.getU32(&Offset);
+  if (FI.Name == 0)
+    return createStringError(std::errc::io_error,
+        "0x%8.8" PRIx64 ": invalid FunctionInfo Name value 0x%8.8x",
+        Offset - 4, FI.Name);
+  bool Done = false;
+  while (!Done) {
+    if (!Data.isValidOffsetForDataOfSize(Offset, 4))
+      return createStringError(std::errc::io_error,
+          "0x%8.8" PRIx64 ": missing FunctionInfo InfoType value", Offset);
+    const uint32_t IT = Data.getU32(&Offset);
+    if (!Data.isValidOffsetForDataOfSize(Offset, 4))
+      return createStringError(std::errc::io_error,
+          "0x%8.8" PRIx64 ": missing FunctionInfo InfoType length", Offset);
+    const uint32_t InfoLength = Data.getU32(&Offset);
+    if (!Data.isValidOffsetForDataOfSize(Offset, InfoLength))
+      return createStringError(std::errc::io_error,
+          "0x%8.8" PRIx64 ": missing FunctionInfo data for InfoType %u",
+          Offset, IT);
+    DataExtractor InfoData(Data.getData().substr(Offset, InfoLength),
+                           Data.isLittleEndian(),
+                           Data.getAddressSize());
+    switch (IT) {
+      case InfoType::EndOfList:
+        Done = true;
+        break;
+
+      case InfoType::LineTableInfo:
+        if (Expected<LineTable> LT = LineTable::decode(InfoData, BaseAddr))
+          FI.OptLineTable = std::move(LT.get());
+        else
+          return LT.takeError();
+        break;
+
+      case InfoType::InlineInfo:
+        if (Expected<InlineInfo> II = InlineInfo::decode(InfoData, BaseAddr))
+          FI.Inline = std::move(II.get());
+        else
+          return II.takeError();
+        break;
+
+      default:
+        return createStringError(std::errc::io_error,
+                                 "0x%8.8" PRIx64 ": unsupported InfoType %u",
+                                 Offset-8, IT);
+    }
+    Offset += InfoLength;
+  }
+  return std::move(FI);
+}
+
+llvm::Expected<uint64_t> FunctionInfo::encode(FileWriter &O) const {
+  if (!isValid())
+    return createStringError(std::errc::invalid_argument,
+        "attempted to encode invalid FunctionInfo object");
+  // Align FunctionInfo data to a 4 byte alignment.
+  O.alignTo(4);
+  const uint64_t FuncInfoOffset = O.tell();
+  // Write the size in bytes of this function as a uint32_t. This can be zero
+  // if we just have a symbol from a symbol table and that symbol has no size.
+  O.writeU32(size());
+  // Write the name of this function as a uint32_t string table offset.
+  O.writeU32(Name);
+
+  if (OptLineTable.hasValue()) {
+    O.writeU32(InfoType::LineTableInfo);
+    // Write a uint32_t length as zero for now, we will fix this up after
+    // writing the LineTable out with the number of bytes that were written.
+    O.writeU32(0);
+    const auto StartOffset = O.tell();
+    llvm::Error err = OptLineTable->encode(O, Range.Start);
+    if (err)
+      return std::move(err);
+    const off_t Length = O.tell() - StartOffset;
+    if (Length > UINT32_MAX)
+        return createStringError(std::errc::invalid_argument,
+            "LineTable length is greater than UINT32_MAX");
+    // Fixup the size of the LineTable data with the correct size.
+    O.fixup32(static_cast<uint32_t>(Length), StartOffset - 4);
+  }
+
+  // Write out the inline function info if we have any and if it is valid.
+  if (Inline.hasValue()) {
+    O.writeU32(InfoType::InlineInfo);
+    // Write a uint32_t length as zero for now, we will fix this up after
+    // writing the LineTable out with the number of bytes that were written.
+    O.writeU32(0);
+    const auto StartOffset = O.tell();
+    llvm::Error err = Inline->encode(O, Range.Start);
+    if (err)
+      return std::move(err);
+    const off_t Length = O.tell() - StartOffset;
+    if (Length > UINT32_MAX)
+        return createStringError(std::errc::invalid_argument,
+            "InlineInfo length is greater than UINT32_MAX");
+    // Fixup the size of the InlineInfo data with the correct size.
+    O.fixup32(static_cast<uint32_t>(Length), StartOffset - 4);
+  }
+
+  // Terminate the data chunks with and end of list with zero size
+  O.writeU32(InfoType::EndOfList);
+  O.writeU32(0);
+  return FuncInfoOffset;
+}
diff --git a/llvm/lib/DebugInfo/GSYM/Header.cpp b/llvm/lib/DebugInfo/GSYM/Header.cpp
new file mode 100644
index 0000000000000..fbf991682e7f0
--- /dev/null
+++ b/llvm/lib/DebugInfo/GSYM/Header.cpp
@@ -0,0 +1,111 @@
+//===- Header.cpp -----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DebugInfo/GSYM/Header.h"
+#include "llvm/DebugInfo/GSYM/FileWriter.h"
+#include "llvm/Support/DataExtractor.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
+#define HEX8(v) llvm::format_hex(v, 4)
+#define HEX16(v) llvm::format_hex(v, 6)
+#define HEX32(v) llvm::format_hex(v, 10)
+#define HEX64(v) llvm::format_hex(v, 18)
+
+using namespace llvm;
+using namespace gsym;
+
+raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const Header &H) {
+  OS << "Header:\n";
+  OS << "  Magic        = " << HEX32(H.Magic) << "\n";
+  OS << "  Version      = " << HEX16(H.Version) << '\n';
+  OS << "  AddrOffSize  = " << HEX8(H.AddrOffSize) << '\n';
+  OS << "  UUIDSize     = " << HEX8(H.UUIDSize) << '\n';
+  OS << "  BaseAddress  = " << HEX64(H.BaseAddress) << '\n';
+  OS << "  NumAddresses = " << HEX32(H.NumAddresses) << '\n';
+  OS << "  StrtabOffset = " << HEX32(H.StrtabOffset) << '\n';
+  OS << "  StrtabSize   = " << HEX32(H.StrtabSize) << '\n';
+  OS << "  UUID         = ";
+  for (uint8_t I = 0; I < H.UUIDSize; ++I)
+    OS << format_hex_no_prefix(H.UUID[I], 2);
+  OS << '\n';
+  return OS;
+}
+
+/// Check the header and detect any errors.
+static llvm::Error getHeaderError(const Header &H) {
+  if (H.Magic != GSYM_MAGIC)
+    return createStringError(std::errc::invalid_argument,
+                             "invalid GSYM magic 0x%8.8x", H.Magic);
+  if (H.Version != GSYM_VERSION)
+    return createStringError(std::errc::invalid_argument,
+                             "unsupported GSYM version %u", H.Version);
+  switch (H.AddrOffSize) {
+    case 1: break;
+    case 2: break;
+    case 4: break;
+    case 8: break;
+    default:
+        return createStringError(std::errc::invalid_argument,
+                                 "invalid address offset size %u",
+                                 H.AddrOffSize);
+  }
+  if (H.UUIDSize > GSYM_MAX_UUID_SIZE)
+    return createStringError(std::errc::invalid_argument,
+                             "invalid UUID size %u", H.UUIDSize);
+  return Error::success();
+}
+
+llvm::Expected<Header> Header::decode(DataExtractor &Data) {
+  uint64_t Offset = 0;
+  // The header is stored as a single blob of data that has a fixed byte size.
+  if (!Data.isValidOffsetForDataOfSize(Offset, sizeof(Header)))
+    return createStringError(std::errc::invalid_argument,
+                             "not enough data for a gsym::Header");
+  Header H;
+  H.Magic = Data.getU32(&Offset);
+  H.Version = Data.getU16(&Offset);
+  H.AddrOffSize = Data.getU8(&Offset);
+  H.UUIDSize = Data.getU8(&Offset);
+  H.BaseAddress = Data.getU64(&Offset);
+  H.NumAddresses = Data.getU32(&Offset);
+  H.StrtabOffset = Data.getU32(&Offset);
+  H.StrtabSize = Data.getU32(&Offset);
+  Data.getU8(&Offset, H.UUID, GSYM_MAX_UUID_SIZE);
+  llvm::Error Err = getHeaderError(H);
+  if (Err)
+    return std::move(Err);
+  return H;
+}
+
+llvm::Error Header::encode(FileWriter &O) const {
+  // Users must verify the Header is valid prior to calling this funtion.
+  llvm::Error Err = getHeaderError(*this);
+  if (Err)
+    return Err;
+  O.writeU32(Magic);
+  O.writeU16(Version);
+  O.writeU8(AddrOffSize);
+  O.writeU8(UUIDSize);
+  O.writeU64(BaseAddress);
+  O.writeU32(NumAddresses);
+  O.writeU32(StrtabOffset);
+  O.writeU32(StrtabSize);
+  O.writeData(llvm::ArrayRef<uint8_t>(UUID));
+  return Error::success();
+}
+
+bool llvm::gsym::operator==(const Header &LHS, const Header &RHS) {
+  return LHS.Magic == RHS.Magic && LHS.Version == RHS.Version &&
+      LHS.AddrOffSize == RHS.AddrOffSize && LHS.UUIDSize == RHS.UUIDSize &&
+      LHS.BaseAddress == RHS.BaseAddress &&
+      LHS.NumAddresses == RHS.NumAddresses &&
+      LHS.StrtabOffset == RHS.StrtabOffset &&
+      LHS.StrtabSize == RHS.StrtabSize &&
+      memcmp(LHS.UUID, RHS.UUID, LHS.UUIDSize) == 0;
+}
diff --git a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
index d8bc3dc06e685..a8e1d0a619cab 100644
--- a/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
+++ b/llvm/lib/DebugInfo/PDB/UDTLayout.cpp
@@ -84,7 +84,7 @@ VBPtrLayoutItem::VBPtrLayoutItem(const UDTLayoutBase &Parent,
 }
 
 const PDBSymbolData &DataMemberLayoutItem::getDataMember() {
-  return *dyn_cast<PDBSymbolData>(Symbol);
+  return *cast<PDBSymbolData>(Symbol);
 }
 
 bool DataMemberLayoutItem::hasUDTLayout() const { return UdtLayout != nullptr; }
diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
index a5559373b05b8..8e1852bcf8eb8 100644
--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
@@ -401,7 +401,7 @@ LLVMSymbolizer::getOrCreateObject(const std::string &Path,
       return I->second.get();
 
     Expected<std::unique_ptr<ObjectFile>> ObjOrErr =
-        UB->getObjectForArch(ArchName);
+        UB->getMachOObjectForArch(ArchName);
     if (!ObjOrErr) {
       ObjectForUBPathAndArch.emplace(std::make_pair(Path, ArchName),
                                      std::unique_ptr<ObjectFile>());
diff --git a/llvm/lib/Demangle/MicrosoftDemangle.cpp b/llvm/lib/Demangle/MicrosoftDemangle.cpp
index bf7d77638f34e..8c58254d6e0aa 100644
--- a/llvm/lib/Demangle/MicrosoftDemangle.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangle.cpp
@@ -783,8 +783,26 @@ SymbolNode *Demangler::demangleMD5Name(StringView &MangledName) {
   return S;
 }
 
+SymbolNode *Demangler::demangleTypeinfoName(StringView &MangledName) {
+  assert(MangledName.startsWith('.'));
+  MangledName.consumeFront('.');
+
+  TypeNode *T = demangleType(MangledName, QualifierMangleMode::Result);
+  if (Error || !MangledName.empty()) {
+    Error = true;
+    return nullptr;
+  }
+  return synthesizeVariable(Arena, T, "`RTTI Type Descriptor Name'");
+}
+
 // Parser entry point.
 SymbolNode *Demangler::parse(StringView &MangledName) {
+  // Typeinfo names are strings stored in RTTI data. They're not symbol names.
+  // It's still useful to demangle them. They're the only demangled entity
+  // that doesn't start with a "?" but a ".".
+  if (MangledName.startsWith('.'))
+    return demangleTypeinfoName(MangledName);
+
   if (MangledName.startsWith("??@"))
     return demangleMD5Name(MangledName);
 
@@ -2161,7 +2179,7 @@ NodeArrayNode *Demangler::demangleFunctionParameterList(StringView &MangledName,
 
 NodeArrayNode *
 Demangler::demangleTemplateParameterList(StringView &MangledName) {
-  NodeList *Head;
+  NodeList *Head = nullptr;
   NodeList **Current = &Head;
   size_t Count = 0;
 
diff --git a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
index 63ca475ec1fed..10363dc69cda4 100644
--- a/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
+++ b/llvm/lib/Demangle/MicrosoftDemangleNodes.cpp
@@ -120,8 +120,6 @@ std::string Node::toString(OutputFlags Flags) const {
   return {OS.getBuffer()};
 }
 
-void TypeNode::outputQuals(bool SpaceBefore, bool SpaceAfter) const {}
-
 void PrimitiveTypeNode::outputPre(OutputStream &OS, OutputFlags Flags) const {
   switch (PrimKind) {
     OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Void, "void");
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
index a425c61210898..ee7a7cb60bc98 100644
--- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -626,7 +626,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
       break;
     case Type::VectorTyID:
       // if the whole vector is 'undef' just reserve memory for the value.
-      auto* VTy = dyn_cast<VectorType>(C->getType());
+      auto* VTy = cast<VectorType>(C->getType());
       Type *ElemTy = VTy->getElementType();
       unsigned int elemNum = VTy->getNumElements();
       Result.AggregateVal.resize(elemNum);
@@ -925,7 +925,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
         elemNum = CDV->getNumElements();
         ElemTy = CDV->getElementType();
     } else if (CV || CAZ) {
-        VectorType* VTy = dyn_cast<VectorType>(C->getType());
+        auto* VTy = cast<VectorType>(C->getType());
         elemNum = VTy->getNumElements();
         ElemTy = VTy->getElementType();
     } else {
diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index 990fc1342d535..c20d7d1d0faf0 100644
--- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -68,7 +68,7 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const {
     }
   }
 
-  ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+  auto *Priority = cast<ConstantInt>(CS->getOperand(0));
   Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr;
   if (Data && !isa<GlobalValue>(Data))
     Data = nullptr;
diff --git a/llvm/lib/ExecutionEngine/Orc/LLVMBuild.txt b/llvm/lib/ExecutionEngine/Orc/LLVMBuild.txt
index af7ef49f5eb71..11a7c5bf09bd9 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLVMBuild.txt
+++ b/llvm/lib/ExecutionEngine/Orc/LLVMBuild.txt
@@ -18,5 +18,5 @@
 type = Library
 name = OrcJIT
 parent = ExecutionEngine
-required_libraries = Core ExecutionEngine JITLink Object MC RuntimeDyld Support
-                     Target TransformUtils
+required_libraries = Core ExecutionEngine JITLink Object MC Passes RuntimeDyld
+                     Support Target TransformUtils
diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index 18c74172a0711..434fecfb49e1a 100644
--- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -729,17 +729,17 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj,
 
   // Assign the address of each symbol
   for (auto &Sym : SymbolsToAllocate) {
-    uint32_t Align = Sym.getAlignment();
+    uint32_t Alignment = Sym.getAlignment();
     uint64_t Size = Sym.getCommonSize();
     StringRef Name;
     if (auto NameOrErr = Sym.getName())
       Name = *NameOrErr;
     else
       return NameOrErr.takeError();
-    if (Align) {
+    if (Alignment) {
       // This symbol has an alignment requirement.
       uint64_t AlignOffset =
-          offsetToAlignment((uint64_t)Addr, llvm::Align(Align));
+          offsetToAlignment((uint64_t)Addr, Align(Alignment));
       Addr += AlignOffset;
       Offset += AlignOffset;
     }
diff --git a/llvm/lib/IR/AttributeImpl.h b/llvm/lib/IR/AttributeImpl.h
index f989fa3b910e6..54e4463aeb953 100644
--- a/llvm/lib/IR/AttributeImpl.h
+++ b/llvm/lib/IR/AttributeImpl.h
@@ -159,7 +159,7 @@ class StringAttributeImpl : public AttributeImpl {
 };
 
 class TypeAttributeImpl : public EnumAttributeImpl {
-  virtual void anchor();
+  void anchor() override;
 
   Type *Ty;
 
diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp
index e9d49d8a2b5cb..79f580d0e14d9 100644
--- a/llvm/lib/IR/AutoUpgrade.cpp
+++ b/llvm/lib/IR/AutoUpgrade.cpp
@@ -522,7 +522,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
                                         F->arg_begin()->getType());
       return true;
     }
-    Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
+    static const Regex vldRegex("^arm\\.neon\\.vld([1234]|[234]lane)\\.v[a-z0-9]*$");
     if (vldRegex.match(Name)) {
       auto fArgs = F->getFunctionType()->params();
       SmallVector<Type *, 4> Tys(fArgs.begin(), fArgs.end());
@@ -533,7 +533,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
                                "llvm." + Name + ".p0i8", F->getParent());
       return true;
     }
-    Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
+    static const Regex vstRegex("^arm\\.neon\\.vst([1234]|[234]lane)\\.v[a-z0-9]*$");
     if (vstRegex.match(Name)) {
       static const Intrinsic::ID StoreInts[] = {Intrinsic::arm_neon_vst1,
                                                 Intrinsic::arm_neon_vst2,
@@ -598,7 +598,7 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) {
   }
   case 'e': {
     SmallVector<StringRef, 2> Groups;
-    Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
+    static const Regex R("^experimental.vector.reduce.([a-z]+)\\.[fi][0-9]+");
     if (R.match(Name, &Groups)) {
       Intrinsic::ID ID = Intrinsic::not_intrinsic;
       if (Groups[1] == "fadd")
@@ -4113,3 +4113,23 @@ MDNode *llvm::upgradeInstructionLoopAttachment(MDNode &N) {
 
   return MDTuple::get(T->getContext(), Ops);
 }
+
+std::string llvm::UpgradeDataLayoutString(StringRef DL, StringRef TT) {
+  std::string AddrSpaces = "-p270:32:32-p271:32:32-p272:64:64";
+
+  // If X86, and the datalayout matches the expected format, add pointer size
+  // address spaces to the datalayout.
+  Triple::ArchType Arch = Triple(TT).getArch();
+  if ((Arch != llvm::Triple::x86 && Arch != llvm::Triple::x86_64) ||
+      DL.contains(AddrSpaces))
+    return DL;
+
+  SmallVector<StringRef, 4> Groups;
+  Regex R("(e-m:[a-z](-p:32:32)?)(-[if]64:.*$)");
+  if (!R.match(DL, &Groups))
+    return DL;
+
+  SmallString<1024> Buf;
+  std::string Res = (Groups[1] + AddrSpaces + Groups[3]).toStringRef(Buf).str();
+  return Res;
+}
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index 34410712645da..bdee6990f9324 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -107,6 +107,13 @@ BasicBlock::instructionsWithoutDebug() {
   return make_filter_range(*this, Fn);
 }
 
+filter_iterator<BasicBlock::const_iterator,
+                std::function<bool(const Instruction &)>>::difference_type
+BasicBlock::sizeWithoutDebug() const {
+  return std::distance(instructionsWithoutDebug().begin(),
+                       instructionsWithoutDebug().end());
+}
+
 void BasicBlock::removeFromParent() {
   getParent()->getBasicBlockList().remove(getIterator());
 }
diff --git a/llvm/lib/IR/ConstantFold.cpp b/llvm/lib/IR/ConstantFold.cpp
index 835fbb3443b8f..54814bbdaf3ce 100644
--- a/llvm/lib/IR/ConstantFold.cpp
+++ b/llvm/lib/IR/ConstantFold.cpp
@@ -746,7 +746,7 @@ Constant *llvm::ConstantFoldSelectInstruction(Constant *Cond,
                                                     ConstantInt::get(Ty, i));
       Constant *V2Element = ConstantExpr::getExtractElement(V2,
                                                     ConstantInt::get(Ty, i));
-      Constant *Cond = dyn_cast<Constant>(CondV->getOperand(i));
+      auto *Cond = cast<Constant>(CondV->getOperand(i));
       if (V1Element == V2Element) {
         V = V1Element;
       } else if (isa<UndefValue>(Cond)) {
diff --git a/llvm/lib/IR/ConstantRange.cpp b/llvm/lib/IR/ConstantRange.cpp
index 920fdc01a14ff..8ea688ff1f07e 100644
--- a/llvm/lib/IR/ConstantRange.cpp
+++ b/llvm/lib/IR/ConstantRange.cpp
@@ -815,6 +815,57 @@ ConstantRange::add(const ConstantRange &Other) const {
   return X;
 }
 
+ConstantRange ConstantRange::addWithNoWrap(const ConstantRange &Other,
+                                           unsigned NoWrapKind,
+                                           PreferredRangeType RangeType) const {
+  // Calculate the range for "X + Y" which is guaranteed not to wrap(overflow).
+  // (X is from this, and Y is from Other)
+  if (isEmptySet() || Other.isEmptySet())
+    return getEmpty();
+  if (isFullSet() && Other.isFullSet())
+    return getFull();
+
+  using OBO = OverflowingBinaryOperator;
+  ConstantRange Result = add(Other);
+
+  auto addWithNoUnsignedWrap = [this](const ConstantRange &Other) {
+    APInt LMin = getUnsignedMin(), LMax = getUnsignedMax();
+    APInt RMin = Other.getUnsignedMin(), RMax = Other.getUnsignedMax();
+    bool Overflow;
+    APInt NewMin = LMin.uadd_ov(RMin, Overflow);
+    if (Overflow)
+      return getEmpty();
+    APInt NewMax = LMax.uadd_sat(RMax);
+    return getNonEmpty(std::move(NewMin), std::move(NewMax) + 1);
+  };
+
+  auto addWithNoSignedWrap = [this](const ConstantRange &Other) {
+    APInt LMin = getSignedMin(), LMax = getSignedMax();
+    APInt RMin = Other.getSignedMin(), RMax = Other.getSignedMax();
+    if (LMin.isNonNegative()) {
+      bool Overflow;
+      APInt Temp = LMin.sadd_ov(RMin, Overflow);
+      if (Overflow)
+        return getEmpty();
+    }
+    if (LMax.isNegative()) {
+      bool Overflow;
+      APInt Temp = LMax.sadd_ov(RMax, Overflow);
+      if (Overflow)
+        return getEmpty();
+    }
+    APInt NewMin = LMin.sadd_sat(RMin);
+    APInt NewMax = LMax.sadd_sat(RMax);
+    return getNonEmpty(std::move(NewMin), std::move(NewMax) + 1);
+  };
+
+  if (NoWrapKind & OBO::NoSignedWrap)
+    Result = Result.intersectWith(addWithNoSignedWrap(Other), RangeType);
+  if (NoWrapKind & OBO::NoUnsignedWrap)
+    Result = Result.intersectWith(addWithNoUnsignedWrap(Other), RangeType);
+  return Result;
+}
+
 ConstantRange ConstantRange::addWithNoSignedWrap(const APInt &Other) const {
   // Calculate the subset of this range such that "X + Other" is
   // guaranteed not to wrap (overflow) for all X in this subset.
diff --git a/llvm/lib/IR/ConstantsContext.h b/llvm/lib/IR/ConstantsContext.h
index 7614dab9f15d9..1ec9087551f86 100644
--- a/llvm/lib/IR/ConstantsContext.h
+++ b/llvm/lib/IR/ConstantsContext.h
@@ -480,14 +480,16 @@ struct ConstantExprKeyType {
       : Opcode(CE->getOpcode()),
         SubclassOptionalData(CE->getRawSubclassOptionalData()),
         SubclassData(CE->isCompare() ? CE->getPredicate() : 0), Ops(Operands),
-        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()) {}
+        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
+        ExplicitTy(nullptr) {}
 
   ConstantExprKeyType(const ConstantExpr *CE,
                       SmallVectorImpl<Constant *> &Storage)
       : Opcode(CE->getOpcode()),
         SubclassOptionalData(CE->getRawSubclassOptionalData()),
         SubclassData(CE->isCompare() ? CE->getPredicate() : 0),
-        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()) {
+        Indexes(CE->hasIndices() ? CE->getIndices() : ArrayRef<unsigned>()),
+        ExplicitTy(nullptr) {
     assert(Storage.empty() && "Expected empty storage");
     for (unsigned I = 0, E = CE->getNumOperands(); I != E; ++I)
       Storage.push_back(CE->getOperand(I));
@@ -676,9 +678,9 @@ template <class ConstantClass> class ConstantUniqueMap {
     /// Hash once, and reuse it for the lookup and the insertion if needed.
     LookupKeyHashed Lookup(MapInfo::getHashValue(Key), Key);
 
-    auto I = Map.find_as(Lookup);
-    if (I != Map.end())
-      return *I;
+    auto ItMap = Map.find_as(Lookup);
+    if (ItMap != Map.end())
+      return *ItMap;
 
     // Update to the new value.  Optimize for the case when we have a single
     // operand that we're changing, but handle bulk updates efficiently.
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index c8f794b4cc3c9..a599a08998767 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2010,9 +2010,9 @@ void LLVMSetAlignment(LLVMValueRef V, unsigned Bytes) {
   if (GlobalObject *GV = dyn_cast<GlobalObject>(P))
     GV->setAlignment(Bytes);
   else if (AllocaInst *AI = dyn_cast<AllocaInst>(P))
-    AI->setAlignment(Bytes);
+    AI->setAlignment(MaybeAlign(Bytes));
   else if (LoadInst *LI = dyn_cast<LoadInst>(P))
-    LI->setAlignment(Bytes);
+    LI->setAlignment(MaybeAlign(Bytes));
   else if (StoreInst *SI = dyn_cast<StoreInst>(P))
     SI->setAlignment(Bytes);
   else
@@ -3527,6 +3527,47 @@ static LLVMAtomicOrdering mapToLLVMOrdering(AtomicOrdering Ordering) {
   llvm_unreachable("Invalid AtomicOrdering value!");
 }
 
+static AtomicRMWInst::BinOp mapFromLLVMRMWBinOp(LLVMAtomicRMWBinOp BinOp) {
+  switch (BinOp) {
+    case LLVMAtomicRMWBinOpXchg: return AtomicRMWInst::Xchg;
+    case LLVMAtomicRMWBinOpAdd: return AtomicRMWInst::Add;
+    case LLVMAtomicRMWBinOpSub: return AtomicRMWInst::Sub;
+    case LLVMAtomicRMWBinOpAnd: return AtomicRMWInst::And;
+    case LLVMAtomicRMWBinOpNand: return AtomicRMWInst::Nand;
+    case LLVMAtomicRMWBinOpOr: return AtomicRMWInst::Or;
+    case LLVMAtomicRMWBinOpXor: return AtomicRMWInst::Xor;
+    case LLVMAtomicRMWBinOpMax: return AtomicRMWInst::Max;
+    case LLVMAtomicRMWBinOpMin: return AtomicRMWInst::Min;
+    case LLVMAtomicRMWBinOpUMax: return AtomicRMWInst::UMax;
+    case LLVMAtomicRMWBinOpUMin: return AtomicRMWInst::UMin;
+    case LLVMAtomicRMWBinOpFAdd: return AtomicRMWInst::FAdd;
+    case LLVMAtomicRMWBinOpFSub: return AtomicRMWInst::FSub;
+  }
+
+  llvm_unreachable("Invalid LLVMAtomicRMWBinOp value!");
+}
+
+static LLVMAtomicRMWBinOp mapToLLVMRMWBinOp(AtomicRMWInst::BinOp BinOp) {
+  switch (BinOp) {
+    case AtomicRMWInst::Xchg: return LLVMAtomicRMWBinOpXchg;
+    case AtomicRMWInst::Add: return LLVMAtomicRMWBinOpAdd;
+    case AtomicRMWInst::Sub: return LLVMAtomicRMWBinOpSub;
+    case AtomicRMWInst::And: return LLVMAtomicRMWBinOpAnd;
+    case AtomicRMWInst::Nand: return LLVMAtomicRMWBinOpNand;
+    case AtomicRMWInst::Or: return LLVMAtomicRMWBinOpOr;
+    case AtomicRMWInst::Xor: return LLVMAtomicRMWBinOpXor;
+    case AtomicRMWInst::Max: return LLVMAtomicRMWBinOpMax;
+    case AtomicRMWInst::Min: return LLVMAtomicRMWBinOpMin;
+    case AtomicRMWInst::UMax: return LLVMAtomicRMWBinOpUMax;
+    case AtomicRMWInst::UMin: return LLVMAtomicRMWBinOpUMin;
+    case AtomicRMWInst::FAdd: return LLVMAtomicRMWBinOpFAdd;
+    case AtomicRMWInst::FSub: return LLVMAtomicRMWBinOpFSub;
+    default: break;
+  }
+
+  llvm_unreachable("Invalid AtomicRMWBinOp value!");
+}
+
 // TODO: Should this and other atomic instructions support building with
 // "syncscope"?
 LLVMValueRef LLVMBuildFence(LLVMBuilderRef B, LLVMAtomicOrdering Ordering,
@@ -3602,14 +3643,30 @@ LLVMBool LLVMGetVolatile(LLVMValueRef MemAccessInst) {
   Value *P = unwrap<Value>(MemAccessInst);
   if (LoadInst *LI = dyn_cast<LoadInst>(P))
     return LI->isVolatile();
-  return cast<StoreInst>(P)->isVolatile();
+  if (StoreInst *SI = dyn_cast<StoreInst>(P))
+    return SI->isVolatile();
+  if (AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(P))
+    return AI->isVolatile();
+  return cast<AtomicCmpXchgInst>(P)->isVolatile();
 }
 
 void LLVMSetVolatile(LLVMValueRef MemAccessInst, LLVMBool isVolatile) {
   Value *P = unwrap<Value>(MemAccessInst);
   if (LoadInst *LI = dyn_cast<LoadInst>(P))
     return LI->setVolatile(isVolatile);
-  return cast<StoreInst>(P)->setVolatile(isVolatile);
+  if (StoreInst *SI = dyn_cast<StoreInst>(P))
+    return SI->setVolatile(isVolatile);
+  if (AtomicRMWInst *AI = dyn_cast<AtomicRMWInst>(P))
+    return AI->setVolatile(isVolatile);
+  return cast<AtomicCmpXchgInst>(P)->setVolatile(isVolatile);
+}
+
+LLVMBool LLVMGetWeak(LLVMValueRef CmpXchgInst) {
+  return unwrap<AtomicCmpXchgInst>(CmpXchgInst)->isWeak();
+}
+
+void LLVMSetWeak(LLVMValueRef CmpXchgInst, LLVMBool isWeak) {
+  return unwrap<AtomicCmpXchgInst>(CmpXchgInst)->setWeak(isWeak);
 }
 
 LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemAccessInst) {
@@ -3617,8 +3674,10 @@ LLVMAtomicOrdering LLVMGetOrdering(LLVMValueRef MemAccessInst) {
   AtomicOrdering O;
   if (LoadInst *LI = dyn_cast<LoadInst>(P))
     O = LI->getOrdering();
+  else if (StoreInst *SI = dyn_cast<StoreInst>(P))
+    O = SI->getOrdering();
   else
-    O = cast<StoreInst>(P)->getOrdering();
+    O = cast<AtomicRMWInst>(P)->getOrdering();
   return mapToLLVMOrdering(O);
 }
 
@@ -3631,6 +3690,14 @@ void LLVMSetOrdering(LLVMValueRef MemAccessInst, LLVMAtomicOrdering Ordering) {
   return cast<StoreInst>(P)->setOrdering(O);
 }
 
+LLVMAtomicRMWBinOp LLVMGetAtomicRMWBinOp(LLVMValueRef Inst) {
+  return mapToLLVMRMWBinOp(unwrap<AtomicRMWInst>(Inst)->getOperation());
+}
+
+void LLVMSetAtomicRMWBinOp(LLVMValueRef Inst, LLVMAtomicRMWBinOp BinOp) {
+  unwrap<AtomicRMWInst>(Inst)->setOperation(mapFromLLVMRMWBinOp(BinOp));
+}
+
 /*--.. Casts ...............................................................--*/
 
 LLVMValueRef LLVMBuildTrunc(LLVMBuilderRef B, LLVMValueRef Val,
@@ -3849,20 +3916,7 @@ LLVMValueRef LLVMBuildAtomicRMW(LLVMBuilderRef B,LLVMAtomicRMWBinOp op,
                                LLVMValueRef PTR, LLVMValueRef Val,
                                LLVMAtomicOrdering ordering,
                                LLVMBool singleThread) {
-  AtomicRMWInst::BinOp intop;
-  switch (op) {
-    case LLVMAtomicRMWBinOpXchg: intop = AtomicRMWInst::Xchg; break;
-    case LLVMAtomicRMWBinOpAdd: intop = AtomicRMWInst::Add; break;
-    case LLVMAtomicRMWBinOpSub: intop = AtomicRMWInst::Sub; break;
-    case LLVMAtomicRMWBinOpAnd: intop = AtomicRMWInst::And; break;
-    case LLVMAtomicRMWBinOpNand: intop = AtomicRMWInst::Nand; break;
-    case LLVMAtomicRMWBinOpOr: intop = AtomicRMWInst::Or; break;
-    case LLVMAtomicRMWBinOpXor: intop = AtomicRMWInst::Xor; break;
-    case LLVMAtomicRMWBinOpMax: intop = AtomicRMWInst::Max; break;
-    case LLVMAtomicRMWBinOpMin: intop = AtomicRMWInst::Min; break;
-    case LLVMAtomicRMWBinOpUMax: intop = AtomicRMWInst::UMax; break;
-    case LLVMAtomicRMWBinOpUMin: intop = AtomicRMWInst::UMin; break;
-  }
+  AtomicRMWInst::BinOp intop = mapFromLLVMRMWBinOp(op);
   return wrap(unwrap(B)->CreateAtomicRMW(intop, unwrap(PTR), unwrap(Val),
     mapFromLLVMOrdering(ordering), singleThread ? SyncScope::SingleThread
                                                 : SyncScope::System));
diff --git a/llvm/lib/IR/DataLayout.cpp b/llvm/lib/IR/DataLayout.cpp
index d56cd8ea5ab51..b125d1550c697 100644
--- a/llvm/lib/IR/DataLayout.cpp
+++ b/llvm/lib/IR/DataLayout.cpp
@@ -44,7 +44,6 @@ using namespace llvm;
 
 StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
   assert(!ST->isOpaque() && "Cannot get layout of opaque structs");
-  StructAlignment = 0;
   StructSize = 0;
   IsPadded = false;
   NumElements = ST->getNumElements();
@@ -52,10 +51,10 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
   // Loop over each of the elements, placing them in memory.
   for (unsigned i = 0, e = NumElements; i != e; ++i) {
     Type *Ty = ST->getElementType(i);
-    unsigned TyAlign = ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty);
+    const Align TyAlign(ST->isPacked() ? 1 : DL.getABITypeAlignment(Ty));
 
     // Add padding if necessary to align the data element properly.
-    if ((StructSize & (TyAlign-1)) != 0) {
+    if (!isAligned(TyAlign, StructSize)) {
       IsPadded = true;
       StructSize = alignTo(StructSize, TyAlign);
     }
@@ -67,12 +66,9 @@ StructLayout::StructLayout(StructType *ST, const DataLayout &DL) {
     StructSize += DL.getTypeAllocSize(Ty); // Consume space for this data item
   }
 
-  // Empty structures have alignment of 1 byte.
-  if (StructAlignment == 0) StructAlignment = 1;
-
   // Add padding to the end of the struct so that it could be put in an array
   // and all array elements would be aligned correctly.
-  if ((StructSize & (StructAlignment-1)) != 0) {
+  if (!isAligned(StructAlignment, StructSize)) {
     IsPadded = true;
     StructSize = alignTo(StructSize, StructAlignment);
   }
@@ -102,9 +98,8 @@ unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const {
 // LayoutAlignElem, LayoutAlign support
 //===----------------------------------------------------------------------===//
 
-LayoutAlignElem
-LayoutAlignElem::get(AlignTypeEnum align_type, unsigned abi_align,
-                     unsigned pref_align, uint32_t bit_width) {
+LayoutAlignElem LayoutAlignElem::get(AlignTypeEnum align_type, Align abi_align,
+                                     Align pref_align, uint32_t bit_width) {
   assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
   LayoutAlignElem retval;
   retval.AlignType = align_type;
@@ -126,10 +121,9 @@ LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const {
 // PointerAlignElem, PointerAlign support
 //===----------------------------------------------------------------------===//
 
-PointerAlignElem
-PointerAlignElem::get(uint32_t AddressSpace, unsigned ABIAlign,
-                      unsigned PrefAlign, uint32_t TypeByteWidth,
-                      uint32_t IndexWidth) {
+PointerAlignElem PointerAlignElem::get(uint32_t AddressSpace, Align ABIAlign,
+                                       Align PrefAlign, uint32_t TypeByteWidth,
+                                       uint32_t IndexWidth) {
   assert(ABIAlign <= PrefAlign && "Preferred alignment worse than ABI!");
   PointerAlignElem retval;
   retval.AddressSpace = AddressSpace;
@@ -162,18 +156,18 @@ const char *DataLayout::getManglingComponent(const Triple &T) {
 }
 
 static const LayoutAlignElem DefaultAlignments[] = {
-  { INTEGER_ALIGN, 1, 1, 1 },    // i1
-  { INTEGER_ALIGN, 8, 1, 1 },    // i8
-  { INTEGER_ALIGN, 16, 2, 2 },   // i16
-  { INTEGER_ALIGN, 32, 4, 4 },   // i32
-  { INTEGER_ALIGN, 64, 4, 8 },   // i64
-  { FLOAT_ALIGN, 16, 2, 2 },     // half
-  { FLOAT_ALIGN, 32, 4, 4 },     // float
-  { FLOAT_ALIGN, 64, 8, 8 },     // double
-  { FLOAT_ALIGN, 128, 16, 16 },  // ppcf128, quad, ...
-  { VECTOR_ALIGN, 64, 8, 8 },    // v2i32, v1i64, ...
-  { VECTOR_ALIGN, 128, 16, 16 }, // v16i8, v8i16, v4i32, ...
-  { AGGREGATE_ALIGN, 0, 0, 8 }   // struct
+    {INTEGER_ALIGN, 1, Align(1), Align(1)},    // i1
+    {INTEGER_ALIGN, 8, Align(1), Align(1)},    // i8
+    {INTEGER_ALIGN, 16, Align(2), Align(2)},   // i16
+    {INTEGER_ALIGN, 32, Align(4), Align(4)},   // i32
+    {INTEGER_ALIGN, 64, Align(4), Align(8)},   // i64
+    {FLOAT_ALIGN, 16, Align(2), Align(2)},     // half
+    {FLOAT_ALIGN, 32, Align(4), Align(4)},     // float
+    {FLOAT_ALIGN, 64, Align(8), Align(8)},     // double
+    {FLOAT_ALIGN, 128, Align(16), Align(16)},  // ppcf128, quad, ...
+    {VECTOR_ALIGN, 64, Align(8), Align(8)},    // v2i32, v1i64, ...
+    {VECTOR_ALIGN, 128, Align(16), Align(16)}, // v16i8, v8i16, v4i32, ...
+    {AGGREGATE_ALIGN, 0, Align(1), Align(8)}   // struct
 };
 
 void DataLayout::reset(StringRef Desc) {
@@ -194,7 +188,7 @@ void DataLayout::reset(StringRef Desc) {
     setAlignment((AlignTypeEnum)E.AlignType, E.ABIAlign, E.PrefAlign,
                  E.TypeBitWidth);
   }
-  setPointerAlignment(0, 8, 8, 8, 8);
+  setPointerAlignment(0, Align(8), Align(8), 8, 8);
 
   parseSpecifier(Desc);
 }
@@ -320,8 +314,9 @@ void DataLayout::parseSpecifier(StringRef Desc) {
             report_fatal_error("Invalid index size of 0 bytes");
         }
       }
-      setPointerAlignment(AddrSpace, PointerABIAlign, PointerPrefAlign,
-                          PointerMemSize, IndexSize);
+      setPointerAlignment(AddrSpace, assumeAligned(PointerABIAlign),
+                          assumeAligned(PointerPrefAlign), PointerMemSize,
+                          IndexSize);
       break;
     }
     case 'i':
@@ -349,11 +344,16 @@ void DataLayout::parseSpecifier(StringRef Desc) {
         report_fatal_error(
             "Missing alignment specification in datalayout string");
       Split = split(Rest, ':');
-      unsigned ABIAlign = inBytes(getInt(Tok));
+      const unsigned ABIAlign = inBytes(getInt(Tok));
       if (AlignType != AGGREGATE_ALIGN && !ABIAlign)
         report_fatal_error(
             "ABI alignment specification must be >0 for non-aggregate types");
 
+      if (!isUInt<16>(ABIAlign))
+        report_fatal_error("Invalid ABI alignment, must be a 16bit integer");
+      if (ABIAlign != 0 && !isPowerOf2_64(ABIAlign))
+        report_fatal_error("Invalid ABI alignment, must be a power of 2");
+
       // Preferred alignment.
       unsigned PrefAlign = ABIAlign;
       if (!Rest.empty()) {
@@ -361,7 +361,14 @@ void DataLayout::parseSpecifier(StringRef Desc) {
         PrefAlign = inBytes(getInt(Tok));
       }
 
-      setAlignment(AlignType, ABIAlign, PrefAlign, Size);
+      if (!isUInt<16>(PrefAlign))
+        report_fatal_error(
+            "Invalid preferred alignment, must be a 16bit integer");
+      if (PrefAlign != 0 && !isPowerOf2_64(PrefAlign))
+        report_fatal_error("Invalid preferred alignment, must be a power of 2");
+
+      setAlignment(AlignType, assumeAligned(ABIAlign), assumeAligned(PrefAlign),
+                   Size);
 
       break;
     }
@@ -474,20 +481,15 @@ DataLayout::findAlignmentLowerBound(AlignTypeEnum AlignType,
   });
 }
 
-void
-DataLayout::setAlignment(AlignTypeEnum align_type, unsigned abi_align,
-                         unsigned pref_align, uint32_t bit_width) {
+void DataLayout::setAlignment(AlignTypeEnum align_type, Align abi_align,
+                              Align pref_align, uint32_t bit_width) {
+  // AlignmentsTy::ABIAlign and AlignmentsTy::PrefAlign were once stored as
+  // uint16_t, it is unclear if there are requirements for alignment to be less
+  // than 2^16 other than storage. In the meantime we leave the restriction as
+  // an assert. See D67400 for context.
+  assert(Log2(abi_align) < 16 && Log2(pref_align) < 16 && "Alignment too big");
   if (!isUInt<24>(bit_width))
     report_fatal_error("Invalid bit width, must be a 24bit integer");
-  if (!isUInt<16>(abi_align))
-    report_fatal_error("Invalid ABI alignment, must be a 16bit integer");
-  if (!isUInt<16>(pref_align))
-    report_fatal_error("Invalid preferred alignment, must be a 16bit integer");
-  if (abi_align != 0 && !isPowerOf2_64(abi_align))
-    report_fatal_error("Invalid ABI alignment, must be a power of 2");
-  if (pref_align != 0 && !isPowerOf2_64(pref_align))
-    report_fatal_error("Invalid preferred alignment, must be a power of 2");
-
   if (pref_align < abi_align)
     report_fatal_error(
         "Preferred alignment cannot be less than the ABI alignment");
@@ -513,8 +515,8 @@ DataLayout::findPointerLowerBound(uint32_t AddressSpace) {
   });
 }
 
-void DataLayout::setPointerAlignment(uint32_t AddrSpace, unsigned ABIAlign,
-                                     unsigned PrefAlign, uint32_t TypeByteWidth,
+void DataLayout::setPointerAlignment(uint32_t AddrSpace, Align ABIAlign,
+                                     Align PrefAlign, uint32_t TypeByteWidth,
                                      uint32_t IndexWidth) {
   if (PrefAlign < ABIAlign)
     report_fatal_error(
@@ -534,9 +536,8 @@ void DataLayout::setPointerAlignment(uint32_t AddrSpace, unsigned ABIAlign,
 
 /// getAlignmentInfo - Return the alignment (either ABI if ABIInfo = true or
 /// preferred if ABIInfo = false) the layout wants for the specified datatype.
-unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType,
-                                      uint32_t BitWidth, bool ABIInfo,
-                                      Type *Ty) const {
+Align DataLayout::getAlignmentInfo(AlignTypeEnum AlignType, uint32_t BitWidth,
+                                   bool ABIInfo, Type *Ty) const {
   AlignmentsTy::const_iterator I = findAlignmentLowerBound(AlignType, BitWidth);
   // See if we found an exact match. Of if we are looking for an integer type,
   // but don't have an exact match take the next largest integer. This is where
@@ -555,10 +556,11 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType,
   } else if (AlignType == VECTOR_ALIGN) {
     // By default, use natural alignment for vector types. This is consistent
     // with what clang and llvm-gcc do.
-    unsigned Align = getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
-    Align *= cast<VectorType>(Ty)->getNumElements();
-    Align = PowerOf2Ceil(Align);
-    return Align;
+    unsigned Alignment =
+        getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
+    Alignment *= cast<VectorType>(Ty)->getNumElements();
+    Alignment = PowerOf2Ceil(Alignment);
+    return Align(Alignment);
    }
 
   // If we still couldn't find a reasonable default alignment, fall back
@@ -567,9 +569,9 @@ unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType,
   // approximation of reality, and if the user wanted something less
   // less conservative, they should have specified it explicitly in the data
   // layout.
-  unsigned Align = getTypeStoreSize(Ty);
-  Align = PowerOf2Ceil(Align);
-  return Align;
+   unsigned Alignment = getTypeStoreSize(Ty);
+   Alignment = PowerOf2Ceil(Alignment);
+   return Align(Alignment);
 }
 
 namespace {
@@ -630,7 +632,7 @@ const StructLayout *DataLayout::getStructLayout(StructType *Ty) const {
   return L;
 }
 
-unsigned DataLayout::getPointerABIAlignment(unsigned AS) const {
+Align DataLayout::getPointerABIAlignment(unsigned AS) const {
   PointersTy::const_iterator I = findPointerLowerBound(AS);
   if (I == Pointers.end() || I->AddressSpace != AS) {
     I = findPointerLowerBound(0);
@@ -639,7 +641,7 @@ unsigned DataLayout::getPointerABIAlignment(unsigned AS) const {
   return I->ABIAlign;
 }
 
-unsigned DataLayout::getPointerPrefAlignment(unsigned AS) const {
+Align DataLayout::getPointerPrefAlignment(unsigned AS) const {
   PointersTy::const_iterator I = findPointerLowerBound(AS);
   if (I == Pointers.end() || I->AddressSpace != AS) {
     I = findPointerLowerBound(0);
@@ -696,21 +698,18 @@ unsigned DataLayout::getIndexTypeSizeInBits(Type *Ty) const {
   Get the ABI (\a abi_or_pref == true) or preferred alignment (\a abi_or_pref
   == false) for the requested type \a Ty.
  */
-unsigned DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
+Align DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   AlignTypeEnum AlignType;
 
   assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
   switch (Ty->getTypeID()) {
   // Early escape for the non-numeric types.
   case Type::LabelTyID:
-    return (abi_or_pref
-            ? getPointerABIAlignment(0)
-            : getPointerPrefAlignment(0));
+    return abi_or_pref ? getPointerABIAlignment(0) : getPointerPrefAlignment(0);
   case Type::PointerTyID: {
     unsigned AS = cast<PointerType>(Ty)->getAddressSpace();
-    return (abi_or_pref
-            ? getPointerABIAlignment(AS)
-            : getPointerPrefAlignment(AS));
+    return abi_or_pref ? getPointerABIAlignment(AS)
+                       : getPointerPrefAlignment(AS);
     }
   case Type::ArrayTyID:
     return getAlignment(cast<ArrayType>(Ty)->getElementType(), abi_or_pref);
@@ -718,11 +717,11 @@ unsigned DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
   case Type::StructTyID: {
     // Packed structure types always have an ABI alignment of one.
     if (cast<StructType>(Ty)->isPacked() && abi_or_pref)
-      return 1;
+      return Align::None();
 
     // Get the layout annotation... which is lazily created on demand.
     const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
-    unsigned Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
+    const Align Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
     return std::max(Align, Layout->getAlignment());
   }
   case Type::IntegerTyID:
@@ -750,17 +749,17 @@ unsigned DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
 }
 
 unsigned DataLayout::getABITypeAlignment(Type *Ty) const {
-  return getAlignment(Ty, true);
+  return getAlignment(Ty, true).value();
 }
 
 /// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
 /// an integer type of the specified bitwidth.
-unsigned DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const {
+Align DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const {
   return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, nullptr);
 }
 
 unsigned DataLayout::getPrefTypeAlignment(Type *Ty) const {
-  return getAlignment(Ty, false);
+  return getAlignment(Ty, false).value();
 }
 
 IntegerType *DataLayout::getIntPtrType(LLVMContext &C,
diff --git a/llvm/lib/IR/DebugInfoMetadata.cpp b/llvm/lib/IR/DebugInfoMetadata.cpp
index 3482b6168530d..2fe9d33ac763f 100644
--- a/llvm/lib/IR/DebugInfoMetadata.cpp
+++ b/llvm/lib/IR/DebugInfoMetadata.cpp
@@ -1161,6 +1161,7 @@ Optional<DIExpression *> DIExpression::createFragmentExpression(
       Op.appendToVector(Ops);
     }
   }
+  assert(Expr && "Unknown DIExpression");
   Ops.push_back(dwarf::DW_OP_LLVM_fragment);
   Ops.push_back(OffsetInBits);
   Ops.push_back(SizeInBits);
diff --git a/llvm/lib/IR/Function.cpp b/llvm/lib/IR/Function.cpp
index 34aad7ddd0360..3f70d2c904e5c 100644
--- a/llvm/lib/IR/Function.cpp
+++ b/llvm/lib/IR/Function.cpp
@@ -293,7 +293,8 @@ void Function::BuildLazyArguments() const {
 
   // Clear the lazy arguments bit.
   unsigned SDC = getSubclassDataFromValue();
-  const_cast<Function*>(this)->setValueSubclassData(SDC &= ~(1<<0));
+  SDC &= ~(1 << 0);
+  const_cast<Function*>(this)->setValueSubclassData(SDC);
   assert(!hasLazyArguments());
 }
 
@@ -703,7 +704,10 @@ enum IIT_Info {
   IIT_STRUCT8 = 40,
   IIT_F128 = 41,
   IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43
+  IIT_SCALABLE_VEC = 43,
+  IIT_SUBDIVIDE2_ARG = 44,
+  IIT_SUBDIVIDE4_ARG = 45,
+  IIT_VEC_OF_BITCASTS_TO_INT = 46
 };
 
 static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
@@ -868,6 +872,18 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
       DecodeIITType(NextElt, Infos, OutputTable);
     return;
   }
+  case IIT_SUBDIVIDE2_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Subdivide2Argument,
+                                             ArgInfo));
+    return;
+  }
+  case IIT_SUBDIVIDE4_ARG: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::Subdivide4Argument,
+                                             ArgInfo));
+    return;
+  }
   case IIT_VEC_ELEMENT: {
     unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
     OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecElementArgument,
@@ -880,6 +896,12 @@ static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
     DecodeIITType(NextElt, Infos, OutputTable);
     return;
   }
+  case IIT_VEC_OF_BITCASTS_TO_INT: {
+    unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]);
+    OutputTable.push_back(IITDescriptor::get(IITDescriptor::VecOfBitcastsToInt,
+                                             ArgInfo));
+    return;
+  }
   }
   llvm_unreachable("unhandled");
 }
@@ -970,6 +992,14 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
     assert(ITy->getBitWidth() % 2 == 0);
     return IntegerType::get(Context, ITy->getBitWidth() / 2);
   }
+  case IITDescriptor::Subdivide2Argument:
+  case IITDescriptor::Subdivide4Argument: {
+    Type *Ty = Tys[D.getArgumentNumber()];
+    VectorType *VTy = dyn_cast<VectorType>(Ty);
+    assert(VTy && "Expected an argument of Vector Type");
+    int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
+    return VectorType::getSubdividedVectorType(VTy, SubDivs);
+  }
   case IITDescriptor::HalfVecArgument:
     return VectorType::getHalfElementsVectorType(cast<VectorType>(
                                                   Tys[D.getArgumentNumber()]));
@@ -998,6 +1028,12 @@ static Type *DecodeFixedType(ArrayRef<Intrinsic::IITDescriptor> &Infos,
       return VTy->getElementType();
     llvm_unreachable("Expected an argument of Vector Type");
   }
+  case IITDescriptor::VecOfBitcastsToInt: {
+    Type *Ty = Tys[D.getArgumentNumber()];
+    VectorType *VTy = dyn_cast<VectorType>(Ty);
+    assert(VTy && "Expected an argument of Vector Type");
+    return VectorType::getInteger(VTy);
+  }
   case IITDescriptor::VecOfAnyPtrsToElt:
     // Return the overloaded type (which determines the pointers address space)
     return Tys[D.getOverloadArgNumber()];
@@ -1188,8 +1224,9 @@ static bool matchIntrinsicType(
     }
     case IITDescriptor::HalfVecArgument:
       // If this is a forward reference, defer the check for later.
-      return D.getArgumentNumber() >= ArgTys.size() ||
-             !isa<VectorType>(ArgTys[D.getArgumentNumber()]) ||
+      if (D.getArgumentNumber() >= ArgTys.size())
+        return IsDeferredCheck || DeferCheck(Ty);
+      return !isa<VectorType>(ArgTys[D.getArgumentNumber()]) ||
              VectorType::getHalfElementsVectorType(
                      cast<VectorType>(ArgTys[D.getArgumentNumber()])) != Ty;
     case IITDescriptor::SameVecWidthArgument: {
@@ -1269,6 +1306,20 @@ static bool matchIntrinsicType(
       auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);
       return !ReferenceType || Ty != ReferenceType->getElementType();
     }
+    case IITDescriptor::Subdivide2Argument:
+    case IITDescriptor::Subdivide4Argument: {
+      // If this is a forward reference, defer the check for later.
+      if (D.getArgumentNumber() >= ArgTys.size())
+        return IsDeferredCheck || DeferCheck(Ty);
+
+      Type *NewTy = ArgTys[D.getArgumentNumber()];
+      if (auto *VTy = dyn_cast<VectorType>(NewTy)) {
+        int SubDivs = D.Kind == IITDescriptor::Subdivide2Argument ? 1 : 2;
+        NewTy = VectorType::getSubdividedVectorType(VTy, SubDivs);
+        return Ty != NewTy;
+      }
+      return true;
+    }
     case IITDescriptor::ScalableVecArgument: {
       VectorType *VTy = dyn_cast<VectorType>(Ty);
       if (!VTy || !VTy->isScalable())
@@ -1276,6 +1327,15 @@ static bool matchIntrinsicType(
       return matchIntrinsicType(VTy, Infos, ArgTys, DeferredChecks,
                                 IsDeferredCheck);
     }
+    case IITDescriptor::VecOfBitcastsToInt: {
+      if (D.getArgumentNumber() >= ArgTys.size())
+        return IsDeferredCheck || DeferCheck(Ty);
+      auto *ReferenceType = dyn_cast<VectorType>(ArgTys[D.getArgumentNumber()]);
+      auto *ThisArgVecTy = dyn_cast<VectorType>(Ty);
+      if (!ThisArgVecTy || !ReferenceType)
+        return true;
+      return ThisArgVecTy != VectorType::getInteger(ReferenceType);
+    }
   }
   llvm_unreachable("unhandled");
 }
diff --git a/llvm/lib/IR/InlineAsm.cpp b/llvm/lib/IR/InlineAsm.cpp
index 99da7caaccf09..fd732f9eda8b8 100644
--- a/llvm/lib/IR/InlineAsm.cpp
+++ b/llvm/lib/IR/InlineAsm.cpp
@@ -181,6 +181,16 @@ bool InlineAsm::ConstraintInfo::Parse(StringRef Str,
       // FIXME: For now assuming these are 2-character constraints.
       pCodes->push_back(StringRef(I+1, 2));
       I += 3;
+    } else if (*I == '@') {
+      // Multi-letter constraint
+      ++I;
+      unsigned char C = static_cast<unsigned char>(*I);
+      assert(isdigit(C) && "Expected a digit!");
+      int N = C - '0';
+      assert(N > 0 && "Found a zero letter constraint!");
+      ++I;
+      pCodes->push_back(StringRef(I, N));
+      I += N;
     } else {
       // Single letter constraint.
       pCodes->push_back(StringRef(I, 1));
diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp
index 37d3371e2332f..0f000623bdb51 100644
--- a/llvm/lib/IR/Instructions.cpp
+++ b/llvm/lib/IR/Instructions.cpp
@@ -1228,7 +1228,7 @@ AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
   : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertBefore),
     AllocatedType(Ty) {
-  setAlignment(Align);
+  setAlignment(MaybeAlign(Align));
   assert(!Ty->isVoidTy() && "Cannot allocate void!");
   setName(Name);
 }
@@ -1239,18 +1239,21 @@ AllocaInst::AllocaInst(Type *Ty, unsigned AddrSpace, Value *ArraySize,
   : UnaryInstruction(PointerType::get(Ty, AddrSpace), Alloca,
                      getAISize(Ty->getContext(), ArraySize), InsertAtEnd),
       AllocatedType(Ty) {
-  setAlignment(Align);
+  setAlignment(MaybeAlign(Align));
   assert(!Ty->isVoidTy() && "Cannot allocate void!");
   setName(Name);
 }
 
-void AllocaInst::setAlignment(unsigned Align) {
-  assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
-  assert(Align <= MaximumAlignment &&
+void AllocaInst::setAlignment(MaybeAlign Align) {
+  assert((!Align || *Align <= MaximumAlignment) &&
          "Alignment is greater than MaximumAlignment!");
   setInstructionSubclassData((getSubclassDataFromInstruction() & ~31) |
-                             (Log2_32(Align) + 1));
-  assert(getAlignment() == Align && "Alignment representation error!");
+                             encode(Align));
+  if (Align)
+    assert(getAlignment() == Align->value() &&
+           "Alignment representation error!");
+  else
+    assert(getAlignment() == 0 && "Alignment representation error!");
 }
 
 bool AllocaInst::isArrayAllocation() const {
@@ -1314,7 +1317,7 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
     : UnaryInstruction(Ty, Load, Ptr, InsertBef) {
   assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
   setVolatile(isVolatile);
-  setAlignment(Align);
+  setAlignment(MaybeAlign(Align));
   setAtomic(Order, SSID);
   AssertOK();
   setName(Name);
@@ -1326,19 +1329,22 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile,
     : UnaryInstruction(Ty, Load, Ptr, InsertAE) {
   assert(Ty == cast<PointerType>(Ptr->getType())->getElementType());
   setVolatile(isVolatile);
-  setAlignment(Align);
+  setAlignment(MaybeAlign(Align));
   setAtomic(Order, SSID);
   AssertOK();
   setName(Name);
 }
 
-void LoadInst::setAlignment(unsigned Align) {
-  assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
-  assert(Align <= MaximumAlignment &&
+void LoadInst::setAlignment(MaybeAlign Align) {
+  assert((!Align || *Align <= MaximumAlignment) &&
          "Alignment is greater than MaximumAlignment!");
   setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
-                             ((Log2_32(Align)+1)<<1));
-  assert(getAlignment() == Align && "Alignment representation error!");
+                             (encode(Align) << 1));
+  if (Align)
+    assert(getAlignment() == Align->value() &&
+           "Alignment representation error!");
+  else
+    assert(getAlignment() == 0 && "Alignment representation error!");
 }
 
 //===----------------------------------------------------------------------===//
@@ -1413,12 +1419,19 @@ StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile,
 }
 
 void StoreInst::setAlignment(unsigned Align) {
-  assert((Align & (Align-1)) == 0 && "Alignment is not a power of 2!");
-  assert(Align <= MaximumAlignment &&
+  setAlignment(llvm::MaybeAlign(Align));
+}
+
+void StoreInst::setAlignment(MaybeAlign Align) {
+  assert((!Align || *Align <= MaximumAlignment) &&
          "Alignment is greater than MaximumAlignment!");
   setInstructionSubclassData((getSubclassDataFromInstruction() & ~(31 << 1)) |
-                             ((Log2_32(Align)+1) << 1));
-  assert(getAlignment() == Align && "Alignment representation error!");
+                             (encode(Align) << 1));
+  if (Align)
+    assert(getAlignment() == Align->value() &&
+           "Alignment representation error!");
+  else
+    assert(getAlignment() == 0 && "Alignment representation error!");
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index a426c9b4df1e1..d3c948d6de385 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -67,13 +67,12 @@ int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
   // size 1. During the search, we can skip the prefix that we already know is
   // identical. By using strncmp we consider names with differing suffixes to
   // be part of the equal range.
-  size_t CmpStart = 0;
   size_t CmpEnd = 4; // Skip the "llvm" component.
   const char *const *Low = NameTable.begin();
   const char *const *High = NameTable.end();
   const char *const *LastLow = Low;
   while (CmpEnd < Name.size() && High - Low > 0) {
-    CmpStart = CmpEnd;
+    size_t CmpStart = CmpEnd;
     CmpEnd = Name.find('.', CmpStart + 1);
     CmpEnd = CmpEnd == StringRef::npos ? Name.size() : CmpEnd;
     auto Cmp = [CmpStart, CmpEnd](const char *LHS, const char *RHS) {
@@ -107,7 +106,7 @@ Optional<ConstrainedFPIntrinsic::RoundingMode>
 ConstrainedFPIntrinsic::getRoundingMode() const {
   unsigned NumOperands = getNumArgOperands();
   Metadata *MD =
-      dyn_cast<MetadataAsValue>(getArgOperand(NumOperands - 2))->getMetadata();
+      cast<MetadataAsValue>(getArgOperand(NumOperands - 2))->getMetadata();
   if (!MD || !isa<MDString>(MD))
     return None;
   return StrToRoundingMode(cast<MDString>(MD)->getString());
@@ -153,7 +152,7 @@ Optional<ConstrainedFPIntrinsic::ExceptionBehavior>
 ConstrainedFPIntrinsic::getExceptionBehavior() const {
   unsigned NumOperands = getNumArgOperands();
   Metadata *MD =
-      dyn_cast<MetadataAsValue>(getArgOperand(NumOperands - 1))->getMetadata();
+      cast<MetadataAsValue>(getArgOperand(NumOperands - 1))->getMetadata();
   if (!MD || !isa<MDString>(MD))
     return None;
   return StrToExceptionBehavior(cast<MDString>(MD)->getString());
diff --git a/llvm/lib/IR/LegacyPassManager.cpp b/llvm/lib/IR/LegacyPassManager.cpp
index c575d6e782b93..3a03c493100b4 100644
--- a/llvm/lib/IR/LegacyPassManager.cpp
+++ b/llvm/lib/IR/LegacyPassManager.cpp
@@ -1680,7 +1680,6 @@ bool FPPassManager::runOnFunction(Function &F) {
 bool FPPassManager::runOnModule(Module &M) {
   bool Changed = false;
 
-  llvm::TimeTraceScope TimeScope("OptModule", M.getName());
   for (Function &F : M)
     Changed |= runOnFunction(F);
 
@@ -1999,10 +1998,28 @@ void FunctionPass::assignPassManager(PMStack &PMS,
   FPP->add(this);
 }
 
+void BasicBlockPass::preparePassManager(PMStack &PMS) {
+  // Find BBPassManager
+  while (!PMS.empty() &&
+         PMS.top()->getPassManagerType() > PMT_BasicBlockPassManager)
+    PMS.pop();
+
+  // If this pass is destroying high level information that is used
+  // by other passes that are managed by BBPM then do not insert
+  // this pass in current BBPM. Use new BBPassManager.
+  if (PMS.top()->getPassManagerType() == PMT_BasicBlockPassManager &&
+      !PMS.top()->preserveHigherLevelAnalysis(this))
+    PMS.pop();
+}
+
 /// Find appropriate Basic Pass Manager or Call Graph Pass Manager
 /// in the PM Stack and add self into that manager.
 void BasicBlockPass::assignPassManager(PMStack &PMS,
                                        PassManagerType PreferredType) {
+  while (!PMS.empty() &&
+         PMS.top()->getPassManagerType() > PMT_BasicBlockPassManager)
+    PMS.pop();
+
   BBPassManager *BBP;
 
   // Basic Pass Manager is a leaf pass manager. It does not handle
@@ -2018,6 +2035,7 @@ void BasicBlockPass::assignPassManager(PMStack &PMS,
 
     // [1] Create new Basic Block Manager
     BBP = new BBPassManager();
+    BBP->populateInheritedAnalysis(PMS);
 
     // [2] Set up new manager's top level manager
     // Basic Block Pass Manager does not live by itself
diff --git a/llvm/lib/IR/RemarkStreamer.cpp b/llvm/lib/IR/RemarkStreamer.cpp
index 73387ecbac1a7..8a70862de726f 100644
--- a/llvm/lib/IR/RemarkStreamer.cpp
+++ b/llvm/lib/IR/RemarkStreamer.cpp
@@ -22,12 +22,10 @@
 using namespace llvm;
 
 RemarkStreamer::RemarkStreamer(
-    StringRef Filename,
-    std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer)
-    : Filename(Filename), PassFilter(),
-      RemarkSerializer(std::move(RemarkSerializer)) {
-  assert(!Filename.empty() && "This needs to be a real filename.");
-}
+    std::unique_ptr<remarks::RemarkSerializer> RemarkSerializer,
+    Optional<StringRef> FilenameIn)
+    : PassFilter(), RemarkSerializer(std::move(RemarkSerializer)),
+      Filename(FilenameIn ? Optional<std::string>(FilenameIn->str()) : None) {}
 
 Error RemarkStreamer::setFilter(StringRef Filter) {
   Regex R = Regex(Filter);
@@ -137,12 +135,13 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
     return make_error<RemarkSetupFormatError>(std::move(E));
 
   Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer =
-      remarks::createRemarkSerializer(*Format, remarks::SerializerMode::Separate, RemarksFile->os());
+      remarks::createRemarkSerializer(
+          *Format, remarks::SerializerMode::Separate, RemarksFile->os());
   if (Error E = RemarkSerializer.takeError())
     return make_error<RemarkSetupFormatError>(std::move(E));
 
   Context.setRemarkStreamer(std::make_unique<RemarkStreamer>(
-      RemarksFilename, std::move(*RemarkSerializer)));
+      std::move(*RemarkSerializer), RemarksFilename));
 
   if (!RemarksPasses.empty())
     if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses))
@@ -150,3 +149,34 @@ llvm::setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename,
 
   return std::move(RemarksFile);
 }
+
+Error llvm::setupOptimizationRemarks(LLVMContext &Context, raw_ostream &OS,
+                                     StringRef RemarksPasses,
+                                     StringRef RemarksFormat,
+                                     bool RemarksWithHotness,
+                                     unsigned RemarksHotnessThreshold) {
+  if (RemarksWithHotness)
+    Context.setDiagnosticsHotnessRequested(true);
+
+  if (RemarksHotnessThreshold)
+    Context.setDiagnosticsHotnessThreshold(RemarksHotnessThreshold);
+
+  Expected<remarks::Format> Format = remarks::parseFormat(RemarksFormat);
+  if (Error E = Format.takeError())
+    return make_error<RemarkSetupFormatError>(std::move(E));
+
+  Expected<std::unique_ptr<remarks::RemarkSerializer>> RemarkSerializer =
+      remarks::createRemarkSerializer(*Format,
+                                      remarks::SerializerMode::Separate, OS);
+  if (Error E = RemarkSerializer.takeError())
+    return make_error<RemarkSetupFormatError>(std::move(E));
+
+  Context.setRemarkStreamer(
+      std::make_unique<RemarkStreamer>(std::move(*RemarkSerializer)));
+
+  if (!RemarksPasses.empty())
+    if (Error E = Context.getRemarkStreamer()->setFilter(RemarksPasses))
+      return make_error<RemarkSetupPatternError>(std::move(E));
+
+  return Error::success();
+}
diff --git a/llvm/lib/IR/SafepointIRVerifier.cpp b/llvm/lib/IR/SafepointIRVerifier.cpp
index 7f3dea5e6a6d2..c90347ec48fdc 100644
--- a/llvm/lib/IR/SafepointIRVerifier.cpp
+++ b/llvm/lib/IR/SafepointIRVerifier.cpp
@@ -102,11 +102,11 @@ class CFGDeadness {
   }
 
   bool isDeadEdge(const Use *U) const {
-    assert(dyn_cast<Instruction>(U->getUser())->isTerminator() &&
+    assert(cast<Instruction>(U->getUser())->isTerminator() &&
            "edge must be operand of terminator");
     assert(cast_or_null<BasicBlock>(U->get()) &&
            "edge must refer to basic block");
-    assert(!isDeadBlock(dyn_cast<Instruction>(U->getUser())->getParent()) &&
+    assert(!isDeadBlock(cast<Instruction>(U->getUser())->getParent()) &&
            "isDeadEdge() must be applied to edge from live block");
     return DeadEdges.count(U);
   }
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index c66cbddafd9a8..65b98d382cc47 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -665,21 +665,20 @@ uint64_t Value::getPointerDereferenceableBytes(const DataLayout &DL,
 
 unsigned Value::getPointerAlignment(const DataLayout &DL) const {
   assert(getType()->isPointerTy() && "must be pointer");
-
-  unsigned Align = 0;
   if (auto *GO = dyn_cast<GlobalObject>(this)) {
     if (isa<Function>(GO)) {
-      MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign();
-      unsigned Align = FunctionPtrAlign ? FunctionPtrAlign->value() : 0;
+      const MaybeAlign FunctionPtrAlign = DL.getFunctionPtrAlign();
+      const unsigned Align = FunctionPtrAlign ? FunctionPtrAlign->value() : 0;
       switch (DL.getFunctionPtrAlignType()) {
       case DataLayout::FunctionPtrAlignType::Independent:
         return Align;
       case DataLayout::FunctionPtrAlignType::MultipleOfFunctionAlign:
         return std::max(Align, GO->getAlignment());
       }
+      llvm_unreachable("Unhandled FunctionPtrAlignType");
     }
-    Align = GO->getAlignment();
-    if (Align == 0) {
+    const unsigned Align = GO->getAlignment();
+    if (!Align) {
       if (auto *GVar = dyn_cast<GlobalVariable>(GO)) {
         Type *ObjectType = GVar->getValueType();
         if (ObjectType->isSized()) {
@@ -687,39 +686,42 @@ unsigned Value::getPointerAlignment(const DataLayout &DL) const {
           // it the preferred alignment. Otherwise, we have to assume that it
           // may only have the minimum ABI alignment.
           if (GVar->isStrongDefinitionForLinker())
-            Align = DL.getPreferredAlignment(GVar);
+            return DL.getPreferredAlignment(GVar);
           else
-            Align = DL.getABITypeAlignment(ObjectType);
+            return DL.getABITypeAlignment(ObjectType);
         }
       }
     }
+    return Align;
   } else if (const Argument *A = dyn_cast<Argument>(this)) {
-    Align = A->getParamAlignment();
-
+    const unsigned Align = A->getParamAlignment();
     if (!Align && A->hasStructRetAttr()) {
       // An sret parameter has at least the ABI alignment of the return type.
       Type *EltTy = cast<PointerType>(A->getType())->getElementType();
       if (EltTy->isSized())
-        Align = DL.getABITypeAlignment(EltTy);
+        return DL.getABITypeAlignment(EltTy);
     }
+    return Align;
   } else if (const AllocaInst *AI = dyn_cast<AllocaInst>(this)) {
-    Align = AI->getAlignment();
-    if (Align == 0) {
+    const unsigned Align = AI->getAlignment();
+    if (!Align) {
       Type *AllocatedType = AI->getAllocatedType();
       if (AllocatedType->isSized())
-        Align = DL.getPrefTypeAlignment(AllocatedType);
+        return DL.getPrefTypeAlignment(AllocatedType);
     }
+    return Align;
   } else if (const auto *Call = dyn_cast<CallBase>(this)) {
-    Align = Call->getRetAlignment();
-    if (Align == 0 && Call->getCalledFunction())
-      Align = Call->getCalledFunction()->getAttributes().getRetAlignment();
-  } else if (const LoadInst *LI = dyn_cast<LoadInst>(this))
+    const unsigned Align = Call->getRetAlignment();
+    if (!Align && Call->getCalledFunction())
+      return Call->getCalledFunction()->getAttributes().getRetAlignment();
+    return Align;
+  } else if (const LoadInst *LI = dyn_cast<LoadInst>(this)) {
     if (MDNode *MD = LI->getMetadata(LLVMContext::MD_align)) {
       ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(0));
-      Align = CI->getLimitedValue();
+      return CI->getLimitedValue();
     }
-
-  return Align;
+  }
+  return 0;
 }
 
 const Value *Value::DoPHITranslation(const BasicBlock *CurBB,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 18bd74baf177a..6739ef26ed413 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -982,6 +982,9 @@ void Verifier::visitDICompositeType(const DICompositeType &N) {
            N.getRawVTableHolder());
   AssertDI(!hasConflictingReferenceFlags(N.getFlags()),
            "invalid reference flags", &N);
+  unsigned DIBlockByRefStruct = 1 << 4;
+  AssertDI((N.getFlags() & DIBlockByRefStruct) == 0,
+           "DIBlockByRefStruct on DICompositeType is no longer supported", &N);
 
   if (N.isVector()) {
     const DINodeArray Elements = N.getElements();
@@ -2501,6 +2504,15 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) {
         Assert(CBI.getOperand(i) != CBI.getOperand(j),
                "Duplicate callbr destination!", &CBI);
   }
+  {
+    SmallPtrSet<BasicBlock *, 4> ArgBBs;
+    for (Value *V : CBI.args())
+      if (auto *BA = dyn_cast<BlockAddress>(V))
+        ArgBBs.insert(BA->getBasicBlock());
+    for (BasicBlock *BB : CBI.getIndirectDests())
+      Assert(ArgBBs.find(BB) != ArgBBs.end(),
+             "Indirect label missing from arglist.", &CBI);
+  }
 
   visitTerminator(CBI);
 }
@@ -2719,8 +2731,8 @@ void Verifier::visitPtrToIntInst(PtrToIntInst &I) {
          &I);
 
   if (SrcTy->isVectorTy()) {
-    VectorType *VSrc = dyn_cast<VectorType>(SrcTy);
-    VectorType *VDest = dyn_cast<VectorType>(DestTy);
+    VectorType *VSrc = cast<VectorType>(SrcTy);
+    VectorType *VDest = cast<VectorType>(DestTy);
     Assert(VSrc->getNumElements() == VDest->getNumElements(),
            "PtrToInt Vector width mismatch", &I);
   }
@@ -2744,8 +2756,8 @@ void Verifier::visitIntToPtrInst(IntToPtrInst &I) {
   Assert(SrcTy->isVectorTy() == DestTy->isVectorTy(), "IntToPtr type mismatch",
          &I);
   if (SrcTy->isVectorTy()) {
-    VectorType *VSrc = dyn_cast<VectorType>(SrcTy);
-    VectorType *VDest = dyn_cast<VectorType>(DestTy);
+    VectorType *VSrc = cast<VectorType>(SrcTy);
+    VectorType *VDest = cast<VectorType>(DestTy);
     Assert(VSrc->getNumElements() == VDest->getNumElements(),
            "IntToPtr Vector width mismatch", &I);
   }
@@ -4902,11 +4914,6 @@ void Verifier::visitDbgIntrinsic(StringRef Kind, DbgVariableIntrinsic &DII) {
   // This check is redundant with one in visitLocalVariable().
   AssertDI(isType(Var->getRawType()), "invalid type ref", Var,
            Var->getRawType());
-  if (auto *Type = dyn_cast_or_null<DIType>(Var->getRawType()))
-    if (Type->isBlockByrefStruct())
-      AssertDI(DII.getExpression() && DII.getExpression()->getNumElements(),
-               "BlockByRef variable without complex expression", Var, &DII);
-
   verifyFnArgs(DII);
 }
 
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp
index 4696265a23204..6c5858b942a3f 100644
--- a/llvm/lib/LTO/LTO.cpp
+++ b/llvm/lib/LTO/LTO.cpp
@@ -1011,6 +1011,16 @@ Error LTO::runRegularLTO(AddStreamFn AddStream) {
                  std::move(RegularLTO.CombinedModule), ThinLTO.CombinedIndex);
 }
 
+static const char *libcallRoutineNames[] = {
+#define HANDLE_LIBCALL(code, name) name,
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+};
+
+ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
+  return makeArrayRef(libcallRoutineNames);
+}
+
 /// This class defines the interface to the ThinLTO backend.
 class lto::ThinBackendProc {
 protected:
@@ -1294,11 +1304,6 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
     ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
                              ImportLists, ExportLists);
 
-  // Update local devirtualized targets that were exported by cross-module
-  // importing
-  updateIndexWPDForExports(ThinLTO.CombinedIndex, ExportLists,
-                           LocalWPDTargetsMap);
-
   // Figure out which symbols need to be internalized. This also needs to happen
   // at -O0 because summary-based DCE is implemented using internalization, and
   // we must apply DCE consistently with the full LTO module in order to avoid
@@ -1328,6 +1333,12 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
             ExportList->second.count(GUID)) ||
            ExportedGUIDs.count(GUID);
   };
+
+  // Update local devirtualized targets that were exported by cross-module
+  // importing or by other devirtualizations marked in the ExportedGUIDs set.
+  updateIndexWPDForExports(ThinLTO.CombinedIndex, isExported,
+                           LocalWPDTargetsMap);
+
   auto isPrevailing = [&](GlobalValue::GUID GUID,
                           const GlobalValueSummary *S) {
     return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
index 02c0d41d132c9..db3ca90d905cf 100644
--- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -39,6 +39,7 @@
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/SmallVectorMemoryBuffer.h"
@@ -368,23 +369,26 @@ class ModuleCacheEntry {
     // Write to a temporary to avoid race condition
     SmallString<128> TempFilename;
     SmallString<128> CachePath(EntryPath);
-    int TempFD;
     llvm::sys::path::remove_filename(CachePath);
     sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
-    std::error_code EC =
-      sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename);
-    if (EC) {
-      errs() << "Error: " << EC.message() << "\n";
-      report_fatal_error("ThinLTO: Can't get a temporary file");
-    }
-    {
-      raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
-      OS << OutputBuffer.getBuffer();
+
+    if (auto Err = handleErrors(
+            llvm::writeFileAtomically(TempFilename, EntryPath,
+                                      OutputBuffer.getBuffer()),
+            [](const llvm::AtomicFileWriteError &E) {
+              std::string ErrorMsgBuffer;
+              llvm::raw_string_ostream S(ErrorMsgBuffer);
+              E.log(S);
+
+              if (E.Error ==
+                  llvm::atomic_write_error::failed_to_create_uniq_file) {
+                errs() << "Error: " << ErrorMsgBuffer << "\n";
+                report_fatal_error("ThinLTO: Can't get a temporary file");
+              }
+            })) {
+      // FIXME
+      consumeError(std::move(Err));
     }
-    // Rename temp file to final destination; rename is atomic
-    EC = sys::fs::rename(TempFilename, EntryPath);
-    if (EC)
-      sys::fs::remove(TempFilename);
   }
 };
 
diff --git a/llvm/lib/MC/ELFObjectWriter.cpp b/llvm/lib/MC/ELFObjectWriter.cpp
index b237d5620bfce..6f160e491cea5 100644
--- a/llvm/lib/MC/ELFObjectWriter.cpp
+++ b/llvm/lib/MC/ELFObjectWriter.cpp
@@ -337,7 +337,7 @@ class ELFDwoObjectWriter : public ELFObjectWriter {
 } // end anonymous namespace
 
 void ELFWriter::align(unsigned Alignment) {
-  uint64_t Padding = offsetToAlignment(W.OS.tell(), llvm::Align(Alignment));
+  uint64_t Padding = offsetToAlignment(W.OS.tell(), Align(Alignment));
   W.OS.write_zeros(Padding);
 }
 
@@ -638,7 +638,7 @@ void ELFWriter::computeSymbolTable(
   unsigned EntrySize = is64Bit() ? ELF::SYMENTRY_SIZE64 : ELF::SYMENTRY_SIZE32;
   MCSectionELF *SymtabSection =
       Ctx.getELFSection(".symtab", ELF::SHT_SYMTAB, 0, EntrySize, "");
-  SymtabSection->setAlignment(is64Bit() ? llvm::Align(8) : llvm::Align(4));
+  SymtabSection->setAlignment(is64Bit() ? Align(8) : Align(4));
   SymbolTableIndex = addToSectionTable(SymtabSection);
 
   align(SymtabSection->getAlignment());
@@ -736,7 +736,7 @@ void ELFWriter::computeSymbolTable(
     MCSectionELF *SymtabShndxSection =
         Ctx.getELFSection(".symtab_shndx", ELF::SHT_SYMTAB_SHNDX, 0, 4, "");
     SymtabShndxSectionIndex = addToSectionTable(SymtabShndxSection);
-    SymtabShndxSection->setAlignment(llvm::Align(4));
+    SymtabShndxSection->setAlignment(Align(4));
   }
 
   ArrayRef<std::string> FileNames = Asm.getFileNames();
@@ -824,7 +824,7 @@ MCSectionELF *ELFWriter::createRelocationSection(MCContext &Ctx,
   MCSectionELF *RelaSection = Ctx.createELFRelSection(
       RelaSectionName, hasRelocationAddend() ? ELF::SHT_RELA : ELF::SHT_REL,
       Flags, EntrySize, Sec.getGroup(), &Sec);
-  RelaSection->setAlignment(is64Bit() ? llvm::Align(8) : llvm::Align(4));
+  RelaSection->setAlignment(is64Bit() ? Align(8) : Align(4));
   return RelaSection;
 }
 
@@ -911,7 +911,7 @@ void ELFWriter::writeSectionData(const MCAssembler &Asm, MCSection &Sec,
     Section.setFlags(Section.getFlags() | ELF::SHF_COMPRESSED);
     // Alignment field should reflect the requirements of
     // the compressed section header.
-    Section.setAlignment(is64Bit() ? llvm::Align(8) : llvm::Align(4));
+    Section.setAlignment(is64Bit() ? Align(8) : Align(4));
   } else {
     // Add "z" prefix to section name. This is zlib-gnu style.
     MC.renameELFSection(&Section, (".z" + SectionName.drop_front(1)).str());
@@ -1135,7 +1135,7 @@ uint64_t ELFWriter::writeObject(MCAssembler &Asm, const MCAsmLayout &Layout) {
       if (!GroupIdx) {
         MCSectionELF *Group = Ctx.createELFGroupSection(SignatureSymbol);
         GroupIdx = addToSectionTable(Group);
-        Group->setAlignment(llvm::Align(4));
+        Group->setAlignment(Align(4));
         Groups.push_back(Group);
       }
       std::vector<const MCSectionELF *> &Members =
diff --git a/llvm/lib/MC/MCAsmInfoXCOFF.cpp b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
index 13939f6a65f85..65fe8848e20fb 100644
--- a/llvm/lib/MC/MCAsmInfoXCOFF.cpp
+++ b/llvm/lib/MC/MCAsmInfoXCOFF.cpp
@@ -20,5 +20,16 @@ MCAsmInfoXCOFF::MCAsmInfoXCOFF() {
   UseDotAlignForAlignment = true;
   AsciiDirective = nullptr; // not supported
   AscizDirective = nullptr; // not supported
+  NeedsFunctionDescriptors = true;
+  HasDotLGloblDirective = true;
   Data64bitsDirective = "\t.llong\t";
+  SupportsQuotedNames = false;
+}
+
+bool MCAsmInfoXCOFF::isValidUnquotedName(StringRef Name) const {
+  // FIXME: Remove this function when we stop using "TOC[TC0]" as a symbol name.
+  if (Name.equals("TOC[TC0]"))
+    return true;
+
+  return MCAsmInfo::isValidUnquotedName(Name);
 }
diff --git a/llvm/lib/MC/MCAsmStreamer.cpp b/llvm/lib/MC/MCAsmStreamer.cpp
index d2f99efef22ab..dd27d0bc8a526 100644
--- a/llvm/lib/MC/MCAsmStreamer.cpp
+++ b/llvm/lib/MC/MCAsmStreamer.cpp
@@ -650,6 +650,7 @@ bool MCAsmStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_Global: // .globl/.global
     OS << MAI->getGlobalDirective();
     break;
+  case MCSA_LGlobal:        OS << "\t.lglobl\t";          break;
   case MCSA_Hidden:         OS << "\t.hidden\t";          break;
   case MCSA_IndirectSymbol: OS << "\t.indirect_symbol\t"; break;
   case MCSA_Internal:       OS << "\t.internal\t";        break;
@@ -1533,9 +1534,8 @@ void MCAsmStreamer::EmitRegisterName(int64_t Register) {
     // just ones that map to LLVM register numbers and have known names.
     // Fall back to using the original number directly if no name is known.
     const MCRegisterInfo *MRI = getContext().getRegisterInfo();
-    int LLVMRegister = MRI->getLLVMRegNumFromEH(Register);
-    if (LLVMRegister != -1) {
-      InstPrinter->printRegName(OS, LLVMRegister);
+    if (Optional<unsigned> LLVMRegister = MRI->getLLVMRegNum(Register, true)) {
+      InstPrinter->printRegName(OS, *LLVMRegister);
       return;
     }
   }
diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp
index 0e0ef80ed5028..cf42fe85b8e5d 100644
--- a/llvm/lib/MC/MCAssembler.cpp
+++ b/llvm/lib/MC/MCAssembler.cpp
@@ -322,7 +322,7 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
   case MCFragment::FT_Align: {
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
     unsigned Offset = Layout.getFragmentOffset(&AF);
-    unsigned Size = offsetToAlignment(Offset, llvm::Align(AF.getAlignment()));
+    unsigned Size = offsetToAlignment(Offset, Align(AF.getAlignment()));
 
     // Insert extra Nops for code alignment if the target define
     // shouldInsertExtraNopBytesForCodeAlign target hook.
diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp
index 816e742b2631a..fa2133078bfea 100644
--- a/llvm/lib/MC/MCELFStreamer.cpp
+++ b/llvm/lib/MC/MCELFStreamer.cpp
@@ -139,7 +139,7 @@ static void setSectionAlignmentForBundling(const MCAssembler &Assembler,
                                            MCSection *Section) {
   if (Section && Assembler.isBundlingEnabled() && Section->hasInstructions() &&
       Section->getAlignment() < Assembler.getBundleAlignSize())
-    Section->setAlignment(llvm::Align(Assembler.getBundleAlignSize()));
+    Section->setAlignment(Align(Assembler.getBundleAlignSize()));
 }
 
 void MCELFStreamer::ChangeSection(MCSection *Section,
@@ -277,6 +277,9 @@ bool MCELFStreamer::EmitSymbolAttribute(MCSymbol *S, MCSymbolAttr Attribute) {
 
   case MCSA_AltEntry:
     llvm_unreachable("ELF doesn't support the .alt_entry attribute");
+
+  case MCSA_LGlobal:
+    llvm_unreachable("ELF doesn't support the .lglobl attribute");
   }
 
   return true;
@@ -306,7 +309,7 @@ void MCELFStreamer::EmitCommonSymbol(MCSymbol *S, uint64_t Size,
 
     // Update the maximum alignment of the section if necessary.
     if (ByteAlignment > Section.getAlignment())
-      Section.setAlignment(llvm::Align(ByteAlignment));
+      Section.setAlignment(Align(ByteAlignment));
 
     SwitchSection(P.first, P.second);
   } else {
diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp
index 613f255a4ea47..8e558a36b7a1f 100644
--- a/llvm/lib/MC/MCMachOStreamer.cpp
+++ b/llvm/lib/MC/MCMachOStreamer.cpp
@@ -330,6 +330,7 @@ bool MCMachOStreamer::EmitSymbolAttribute(MCSymbol *Sym,
   case MCSA_Protected:
   case MCSA_Weak:
   case MCSA_Local:
+  case MCSA_LGlobal:
     return false;
 
   case MCSA_Global:
diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp
index 1e7b5d711f238..83f6ab8fe332e 100644
--- a/llvm/lib/MC/MCObjectStreamer.cpp
+++ b/llvm/lib/MC/MCObjectStreamer.cpp
@@ -539,7 +539,7 @@ void MCObjectStreamer::EmitValueToAlignment(unsigned ByteAlignment,
   // Update the maximum alignment on the current section if necessary.
   MCSection *CurSec = getCurrentSectionOnly();
   if (ByteAlignment > CurSec->getAlignment())
-    CurSec->setAlignment(llvm::Align(ByteAlignment));
+    CurSec->setAlignment(Align(ByteAlignment));
 }
 
 void MCObjectStreamer::EmitCodeAlignment(unsigned ByteAlignment,
diff --git a/llvm/lib/MC/MCRegisterInfo.cpp b/llvm/lib/MC/MCRegisterInfo.cpp
index 7c7376d906f7d..d491c0eb7e060 100644
--- a/llvm/lib/MC/MCRegisterInfo.cpp
+++ b/llvm/lib/MC/MCRegisterInfo.cpp
@@ -78,29 +78,18 @@ int MCRegisterInfo::getDwarfRegNum(MCRegister RegNum, bool isEH) const {
   return I->ToReg;
 }
 
-int MCRegisterInfo::getLLVMRegNum(unsigned RegNum, bool isEH) const {
+Optional<unsigned> MCRegisterInfo::getLLVMRegNum(unsigned RegNum,
+                                                 bool isEH) const {
   const DwarfLLVMRegPair *M = isEH ? EHDwarf2LRegs : Dwarf2LRegs;
   unsigned Size = isEH ? EHDwarf2LRegsSize : Dwarf2LRegsSize;
 
   if (!M)
-    return -1;
-  DwarfLLVMRegPair Key = { RegNum, 0 };
-  const DwarfLLVMRegPair *I = std::lower_bound(M, M+Size, Key);
-  assert(I != M+Size && I->FromReg == RegNum && "Invalid RegNum");
-  return I->ToReg;
-}
-
-int MCRegisterInfo::getLLVMRegNumFromEH(unsigned RegNum) const {
-  const DwarfLLVMRegPair *M = EHDwarf2LRegs;
-  unsigned Size = EHDwarf2LRegsSize;
-
-  if (!M)
-    return -1;
+    return None;
   DwarfLLVMRegPair Key = { RegNum, 0 };
   const DwarfLLVMRegPair *I = std::lower_bound(M, M+Size, Key);
-  if (I == M+Size || I->FromReg != RegNum)
-    return -1;
-  return I->ToReg;
+  if (I != M + Size && I->FromReg == RegNum)
+    return I->ToReg;
+  return None;
 }
 
 int MCRegisterInfo::getDwarfRegNumFromDwarfEHRegNum(unsigned RegNum) const {
@@ -112,9 +101,8 @@ int MCRegisterInfo::getDwarfRegNumFromDwarfEHRegNum(unsigned RegNum) const {
   // a corresponding LLVM register number at all.  So if we can't map the
   // EH register number to an LLVM register number, assume it's just a
   // valid DWARF register number as is.
-  int LRegNum = getLLVMRegNumFromEH(RegNum);
-  if (LRegNum != -1)
-    return getDwarfRegNum(LRegNum, false);
+  if (Optional<unsigned> LRegNum = getLLVMRegNum(RegNum, true))
+    return getDwarfRegNum(*LRegNum, false);
   return RegNum;
 }
 
diff --git a/llvm/lib/MC/MCSectionXCOFF.cpp b/llvm/lib/MC/MCSectionXCOFF.cpp
index db277521db256..d52959f15f928 100644
--- a/llvm/lib/MC/MCSectionXCOFF.cpp
+++ b/llvm/lib/MC/MCSectionXCOFF.cpp
@@ -15,26 +15,46 @@ using namespace llvm;
 
 MCSectionXCOFF::~MCSectionXCOFF() = default;
 
+static StringRef getMappingClassString(XCOFF::StorageMappingClass SMC) {
+  switch (SMC) {
+  case XCOFF::XMC_DS:
+    return "DS";
+  case XCOFF::XMC_RW:
+    return "RW";
+  case XCOFF::XMC_PR:
+    return "PR";
+  default:
+    report_fatal_error("Unhandled storage-mapping class.");
+  }
+}
+
 void MCSectionXCOFF::PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T,
                                           raw_ostream &OS,
                                           const MCExpr *Subsection) const {
   if (getKind().isText()) {
     if (getMappingClass() != XCOFF::XMC_PR)
-      llvm_unreachable("Unsupported storage-mapping class for .text csect");
+      report_fatal_error("Unhandled storage-mapping class for .text csect");
 
     OS << "\t.csect " << getSectionName() << "["
-       << "PR"
+       << getMappingClassString(getMappingClass())
        << "]" << '\n';
     return;
   }
 
   if (getKind().isData()) {
-    assert(getMappingClass() == XCOFF::XMC_RW &&
-           "Unhandled storage-mapping class for data section.");
-
-    OS << "\t.csect " << getSectionName() << "["
-       << "RW"
-       << "]" << '\n';
+    switch (getMappingClass()) {
+    case XCOFF::XMC_RW:
+    case XCOFF::XMC_DS:
+      OS << "\t.csect " << getSectionName() << "["
+         << getMappingClassString(getMappingClass()) << "]" << '\n';
+      break;
+    case XCOFF::XMC_TC0:
+      OS << "\t.toc\n";
+      break;
+    default:
+      report_fatal_error(
+          "Unhandled storage-mapping class for .data csect.");
+    }
     return;
   }
 
diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp
index fc79cab522cdf..c5a21312140bb 100644
--- a/llvm/lib/MC/MCWinCOFFStreamer.cpp
+++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp
@@ -192,7 +192,7 @@ void MCWinCOFFStreamer::EmitCOFFSafeSEH(MCSymbol const *Symbol) {
   MCSection *SXData = getContext().getObjectFileInfo()->getSXDataSection();
   getAssembler().registerSection(*SXData);
   if (SXData->getAlignment() < 4)
-    SXData->setAlignment(llvm::Align(4));
+    SXData->setAlignment(Align(4));
 
   new MCSymbolIdFragment(Symbol, SXData);
 
@@ -209,7 +209,7 @@ void MCWinCOFFStreamer::EmitCOFFSymbolIndex(MCSymbol const *Symbol) {
   MCSection *Sec = getCurrentSectionOnly();
   getAssembler().registerSection(*Sec);
   if (Sec->getAlignment() < 4)
-    Sec->setAlignment(llvm::Align(4));
+    Sec->setAlignment(Align(4));
 
   new MCSymbolIdFragment(Symbol, getCurrentSectionOnly());
 
diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp
index 03fb03fe1b654..9f6af981aca10 100644
--- a/llvm/lib/MC/MachObjectWriter.cpp
+++ b/llvm/lib/MC/MachObjectWriter.cpp
@@ -127,7 +127,7 @@ uint64_t MachObjectWriter::getPaddingSize(const MCSection *Sec,
   const MCSection &NextSec = *Layout.getSectionOrder()[Next];
   if (NextSec.isVirtualSection())
     return 0;
-  return offsetToAlignment(EndAddr, llvm::Align(NextSec.getAlignment()));
+  return offsetToAlignment(EndAddr, Align(NextSec.getAlignment()));
 }
 
 void MachObjectWriter::writeHeader(MachO::HeaderFileType Type,
@@ -445,8 +445,8 @@ void MachObjectWriter::writeLinkerOptionsLoadCommand(
   }
 
   // Pad to a multiple of the pointer size.
-  W.OS.write_zeros(offsetToAlignment(BytesWritten, is64Bit() ? llvm::Align(8)
-                                                             : llvm::Align(4)));
+  W.OS.write_zeros(
+      offsetToAlignment(BytesWritten, is64Bit() ? Align(8) : Align(4)));
 
   assert(W.OS.tell() - Start == Size);
 }
@@ -835,7 +835,7 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm,
   //
   // FIXME: Is this machine dependent?
   unsigned SectionDataPadding =
-      offsetToAlignment(SectionDataFileSize, llvm::Align(4));
+      offsetToAlignment(SectionDataFileSize, Align(4));
   SectionDataFileSize += SectionDataPadding;
 
   // Write the prolog, starting with the header and load command...
@@ -1000,8 +1000,8 @@ uint64_t MachObjectWriter::writeObject(MCAssembler &Asm,
 #endif
     Asm.getLOHContainer().emit(*this, Layout);
     // Pad to a multiple of the pointer size.
-    W.OS.write_zeros(offsetToAlignment(LOHRawSize, is64Bit() ? llvm::Align(8)
-                                                             : llvm::Align(4)));
+    W.OS.write_zeros(
+        offsetToAlignment(LOHRawSize, is64Bit() ? Align(8) : Align(4)));
     assert(W.OS.tell() - Start == LOHSize);
   }
 
diff --git a/llvm/lib/MC/WasmObjectWriter.cpp b/llvm/lib/MC/WasmObjectWriter.cpp
index 53618887bb50d..c1ff3cc2480cd 100644
--- a/llvm/lib/MC/WasmObjectWriter.cpp
+++ b/llvm/lib/MC/WasmObjectWriter.cpp
@@ -1350,7 +1350,9 @@ uint64_t WasmObjectWriter::writeObject(MCAssembler &Asm,
         report_fatal_error(".size expression must be evaluatable");
 
       auto &DataSection = static_cast<MCSectionWasm &>(WS.getSection());
-      assert(DataSection.isWasmData());
+      if (!DataSection.isWasmData())
+        report_fatal_error("data symbols must live in a data section: " +
+                           WS.getName());
 
       // For each data symbol, export it in the symtab as a reference to the
       // corresponding Wasm data segment.
diff --git a/llvm/lib/MC/XCOFFObjectWriter.cpp b/llvm/lib/MC/XCOFFObjectWriter.cpp
index e831d3e763f74..5d8735b017d3f 100644
--- a/llvm/lib/MC/XCOFFObjectWriter.cpp
+++ b/llvm/lib/MC/XCOFFObjectWriter.cpp
@@ -235,6 +235,9 @@ void XCOFFObjectWriter::executePostLayoutBinding(
         break;
       }
       report_fatal_error("Unhandled mapping of read-write csect to section.");
+    case XCOFF::XMC_TC0:
+      // TODO FIXME Handle emiting the TOC base.
+      break;
     case XCOFF::XMC_BS:
       assert(XCOFF::XTY_CM == MCSec->getCSectType() &&
              "Mapping invalid csect. CSECT with bss storage class must be "
diff --git a/llvm/lib/MCA/HardwareUnits/LSUnit.cpp b/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
index 0465f53ed36e8..973bb908e41a6 100644
--- a/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
+++ b/llvm/lib/MCA/HardwareUnits/LSUnit.cpp
@@ -72,9 +72,9 @@ unsigned LSUnit::dispatch(const InstRef &IR) {
   assert((Desc.MayLoad || Desc.MayStore) && "Not a memory operation!");
 
   if (Desc.MayLoad)
-    assignLQSlot();
+    acquireLQSlot();
   if (Desc.MayStore)
-    assignSQSlot();
+    acquireSQSlot();
 
   if (Desc.MayStore) {
     // Always create a new group for store operations.
@@ -173,13 +173,13 @@ void LSUnitBase::onInstructionExecuted(const InstRef &IR) {
   }
 
   if (IsALoad) {
-    UsedLQEntries--;
+    releaseLQSlot();
     LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex()
                       << " has been removed from the load queue.\n");
   }
 
   if (IsAStore) {
-    UsedSQEntries--;
+    releaseSQSlot();
     LLVM_DEBUG(dbgs() << "[LSUnit]: Instruction idx=" << IR.getSourceIndex()
                       << " has been removed from the store queue.\n");
   }
diff --git a/llvm/lib/Object/Archive.cpp b/llvm/lib/Object/Archive.cpp
index fd9609f65bc0b..148c011d9cd4c 100644
--- a/llvm/lib/Object/Archive.cpp
+++ b/llvm/lib/Object/Archive.cpp
@@ -550,7 +550,7 @@ Archive::Archive(MemoryBufferRef Source, Error &Err)
   } else if (Buffer.startswith(Magic)) {
     IsThin = false;
   } else {
-    Err = make_error<GenericBinaryError>("File too small to be an archive",
+    Err = make_error<GenericBinaryError>("file too small to be an archive",
                                          object_error::invalid_file_type);
     return;
   }
diff --git a/llvm/lib/Object/ArchiveWriter.cpp b/llvm/lib/Object/ArchiveWriter.cpp
index ecd7c850616f8..5234b0e18233b 100644
--- a/llvm/lib/Object/ArchiveWriter.cpp
+++ b/llvm/lib/Object/ArchiveWriter.cpp
@@ -177,7 +177,7 @@ printBSDMemberHeader(raw_ostream &Out, uint64_t Pos, StringRef Name,
                      unsigned UID, unsigned GID, unsigned Perms, uint64_t Size) {
   uint64_t PosAfterHeader = Pos + 60 + Name.size();
   // Pad so that even 64 bit object files are aligned.
-  unsigned Pad = offsetToAlignment(PosAfterHeader, llvm::Align(8));
+  unsigned Pad = offsetToAlignment(PosAfterHeader, Align(8));
   unsigned NameWithPadding = Name.size() + Pad;
   printWithSpacePadding(Out, Twine("#1/") + Twine(NameWithPadding), 16);
   printRestOfMemberHeader(Out, ModTime, UID, GID, Perms,
@@ -244,7 +244,7 @@ struct MemberData {
 
 static MemberData computeStringTable(StringRef Names) {
   unsigned Size = Names.size();
-  unsigned Pad = offsetToAlignment(Size, llvm::Align(2));
+  unsigned Pad = offsetToAlignment(Size, Align(2));
   std::string Header;
   raw_string_ostream Out(Header);
   printWithSpacePadding(Out, "//", 48);
@@ -308,7 +308,7 @@ static void writeSymbolTable(raw_ostream &Out, object::Archive::Kind Kind,
   // least 4-byte aligned for 32-bit content.  Opt for the larger encoding
   // uniformly.
   // We do this for all bsd formats because it simplifies aligning members.
-  const llvm::Align Alignment(isBSDLike(Kind) ? 8 : 2);
+  const Align Alignment(isBSDLike(Kind) ? 8 : 2);
   unsigned Pad = offsetToAlignment(Size, Alignment);
   Size += Pad;
 
@@ -465,9 +465,9 @@ computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
     // uniformly.  This matches the behaviour with cctools and ensures that ld64
     // is happy with archives that we generate.
     unsigned MemberPadding =
-        isDarwin(Kind) ? offsetToAlignment(Data.size(), llvm::Align(8)) : 0;
+        isDarwin(Kind) ? offsetToAlignment(Data.size(), Align(8)) : 0;
     unsigned TailPadding =
-        offsetToAlignment(Data.size() + MemberPadding, llvm::Align(2));
+        offsetToAlignment(Data.size() + MemberPadding, Align(2));
     StringRef Padding = StringRef(PaddingData, MemberPadding + TailPadding);
 
     sys::TimePoint<std::chrono::seconds> ModTime;
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index a10c643b3b973..bf6ffd6c37b95 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -203,7 +203,7 @@ SubtargetFeatures ELFObjectFileBase::getARMFeatures() const {
     default:
       break;
     case ARMBuildAttrs::Not_Allowed:
-      Features.AddFeature("vfp2d16sp", false);
+      Features.AddFeature("vfp2sp", false);
       Features.AddFeature("vfp3d16sp", false);
       Features.AddFeature("vfp4d16sp", false);
       break;
diff --git a/llvm/lib/Object/MachOObjectFile.cpp b/llvm/lib/Object/MachOObjectFile.cpp
index fdba3442af2fa..c0c873f973545 100644
--- a/llvm/lib/Object/MachOObjectFile.cpp
+++ b/llvm/lib/Object/MachOObjectFile.cpp
@@ -2727,11 +2727,11 @@ bool MachOObjectFile::isValidArch(StringRef ArchFlag) {
 }
 
 ArrayRef<StringRef> MachOObjectFile::getValidArchs() {
-  static const std::array<StringRef, 17> validArchs = {
+  static const std::array<StringRef, 17> validArchs = {{
       "i386",   "x86_64", "x86_64h",  "armv4t",  "arm",    "armv5e",
       "armv6",  "armv6m", "armv7",    "armv7em", "armv7k", "armv7m",
       "armv7s", "arm64",  "arm64_32", "ppc",     "ppc64",
-  };
+  }};
 
   return validArchs;
 }
diff --git a/llvm/lib/Object/MachOUniversal.cpp b/llvm/lib/Object/MachOUniversal.cpp
index 8b5cd0518a53b..3e5be913c25fd 100644
--- a/llvm/lib/Object/MachOUniversal.cpp
+++ b/llvm/lib/Object/MachOUniversal.cpp
@@ -210,19 +210,34 @@ MachOUniversalBinary::MachOUniversalBinary(MemoryBufferRef Source, Error &Err)
   Err = Error::success();
 }
 
-Expected<std::unique_ptr<MachOObjectFile>>
+Expected<MachOUniversalBinary::ObjectForArch>
 MachOUniversalBinary::getObjectForArch(StringRef ArchName) const {
   if (Triple(ArchName).getArch() == Triple::ArchType::UnknownArch)
     return make_error<GenericBinaryError>("Unknown architecture "
                                           "named: " +
                                               ArchName,
                                           object_error::arch_not_found);
-
-  for (auto &Obj : objects())
+  for (const auto &Obj : objects())
     if (Obj.getArchFlagName() == ArchName)
-      return Obj.getAsObjectFile();
+      return Obj;
   return make_error<GenericBinaryError>("fat file does not "
                                         "contain " +
                                             ArchName,
                                         object_error::arch_not_found);
 }
+
+Expected<std::unique_ptr<MachOObjectFile>>
+MachOUniversalBinary::getMachOObjectForArch(StringRef ArchName) const {
+  Expected<ObjectForArch> O = getObjectForArch(ArchName);
+  if (!O)
+    return O.takeError();
+  return O->getAsObjectFile();
+}
+
+Expected<std::unique_ptr<Archive>>
+MachOUniversalBinary::getArchiveForArch(StringRef ArchName) const {
+  Expected<ObjectForArch> O = getObjectForArch(ArchName);
+  if (!O)
+    return O.takeError();
+  return O->getAsArchive();
+}
diff --git a/llvm/lib/Object/Object.cpp b/llvm/lib/Object/Object.cpp
index b518e2d02c9c9..b486e9f5c9a8a 100644
--- a/llvm/lib/Object/Object.cpp
+++ b/llvm/lib/Object/Object.cpp
@@ -138,7 +138,7 @@ LLVMBinaryRef LLVMMachOUniversalBinaryCopyObjectForArch(LLVMBinaryRef BR,
                                                         char **ErrorMessage) {
   auto universal = cast<MachOUniversalBinary>(unwrap(BR));
   Expected<std::unique_ptr<ObjectFile>> ObjOrErr(
-      universal->getObjectForArch({Arch, ArchLen}));
+      universal->getMachOObjectForArch({Arch, ArchLen}));
   if (!ObjOrErr) {
     *ErrorMessage = strdup(toString(ObjOrErr.takeError()).c_str());
     return nullptr;
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index bcc44cd6ce335..75e0952deb686 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -105,7 +105,7 @@ Triple ObjectFile::makeTriple() const {
     TheTriple.setObjectFormat(Triple::MachO);
 
   if (isCOFF()) {
-    const auto COFFObj = dyn_cast<COFFObjectFile>(this);
+    const auto COFFObj = cast<COFFObjectFile>(this);
     if (COFFObj->getArch() == Triple::thumb)
       TheTriple.setTriple("thumbv7-windows");
   }
diff --git a/llvm/lib/Object/RelocationResolver.cpp b/llvm/lib/Object/RelocationResolver.cpp
index 90ae4fa4a54ec..ca89f5671b8a7 100644
--- a/llvm/lib/Object/RelocationResolver.cpp
+++ b/llvm/lib/Object/RelocationResolver.cpp
@@ -30,6 +30,7 @@ static bool supportsX86_64(uint64_t Type) {
   case ELF::R_X86_64_DTPOFF32:
   case ELF::R_X86_64_DTPOFF64:
   case ELF::R_X86_64_PC32:
+  case ELF::R_X86_64_PC64:
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
     return true;
@@ -47,6 +48,7 @@ static uint64_t resolveX86_64(RelocationRef R, uint64_t S, uint64_t A) {
   case ELF::R_X86_64_DTPOFF64:
     return S + getELFAddend(R);
   case ELF::R_X86_64_PC32:
+  case ELF::R_X86_64_PC64:
     return S + getELFAddend(R) - R.getOffset();
   case ELF::R_X86_64_32:
   case ELF::R_X86_64_32S:
diff --git a/llvm/lib/Object/TapiFile.cpp b/llvm/lib/Object/TapiFile.cpp
index 4d9fe7d9e8ac1..c409bd8e59951 100644
--- a/llvm/lib/Object/TapiFile.cpp
+++ b/llvm/lib/Object/TapiFile.cpp
@@ -45,13 +45,13 @@ TapiFile::TapiFile(MemoryBufferRef Source, const InterfaceFile &interface,
     if (!Symbol->getArchitectures().has(Arch))
       continue;
 
-    auto Platform = interface.getPlatform();
     switch (Symbol->getKind()) {
     case SymbolKind::GlobalSymbol:
       Symbols.emplace_back(StringRef(), Symbol->getName(), getFlags(Symbol));
       break;
     case SymbolKind::ObjectiveCClass:
-      if (Platform == PlatformKind::macOS && Arch == AK_i386) {
+      if (interface.getPlatforms().count(PlatformKind::macOS) &&
+          Arch == AK_i386) {
         Symbols.emplace_back(ObjC1ClassNamePrefix, Symbol->getName(),
                              getFlags(Symbol));
       } else {
diff --git a/llvm/lib/ObjectYAML/ELFEmitter.cpp b/llvm/lib/ObjectYAML/ELFEmitter.cpp
index 4584e3d73bd76..c85cf4c924f0a 100644
--- a/llvm/lib/ObjectYAML/ELFEmitter.cpp
+++ b/llvm/lib/ObjectYAML/ELFEmitter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ObjectYAML/ELFYAML.h"
 #include "llvm/ObjectYAML/yaml2obj.h"
 #include "llvm/Support/EndianStream.h"
+#include "llvm/Support/LEB128.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/YAMLTraits.h"
@@ -167,6 +168,12 @@ template <class ELFT> class ELFState {
   void writeSectionContent(Elf_Shdr &SHeader,
                            const ELFYAML::DynamicSection &Section,
                            ContiguousBlobAccumulator &CBA);
+  void writeSectionContent(Elf_Shdr &SHeader,
+                           const ELFYAML::StackSizesSection &Section,
+                           ContiguousBlobAccumulator &CBA);
+  void writeSectionContent(Elf_Shdr &SHeader,
+                           const ELFYAML::HashSection &Section,
+                           ContiguousBlobAccumulator &CBA);
   ELFState(ELFYAML::Object &D, yaml::ErrorHandler EH);
 
 public:
@@ -336,7 +343,7 @@ bool ELFState<ELFT>::initImplicitHeader(ContiguousBlobAccumulator &CBA,
   return true;
 }
 
-static StringRef dropUniqueSuffix(StringRef S) {
+StringRef llvm::ELFYAML::dropUniqueSuffix(StringRef S) {
   size_t SuffixPos = S.rfind(" [");
   if (SuffixPos == StringRef::npos)
     return S;
@@ -367,7 +374,8 @@ void ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
     assert(Sec && "It can't be null unless it is an implicit section. But all "
                   "implicit sections should already have been handled above.");
 
-    SHeader.sh_name = DotShStrtab.getOffset(dropUniqueSuffix(Sec->Name));
+    SHeader.sh_name =
+        DotShStrtab.getOffset(ELFYAML::dropUniqueSuffix(Sec->Name));
     SHeader.sh_type = Sec->Type;
     if (Sec->Flags)
       SHeader.sh_flags = *Sec->Flags;
@@ -411,6 +419,10 @@ void ELFState<ELFT>::initSectionHeaders(std::vector<Elf_Shdr> &SHeaders,
       writeSectionContent(SHeader, *S, CBA);
     } else if (auto S = dyn_cast<ELFYAML::VerdefSection>(Sec)) {
       writeSectionContent(SHeader, *S, CBA);
+    } else if (auto S = dyn_cast<ELFYAML::StackSizesSection>(Sec)) {
+      writeSectionContent(SHeader, *S, CBA);
+    } else if (auto S = dyn_cast<ELFYAML::HashSection>(Sec)) {
+      writeSectionContent(SHeader, *S, CBA);
     } else {
       llvm_unreachable("Unknown section type");
     }
@@ -434,19 +446,20 @@ static size_t findFirstNonGlobal(ArrayRef<ELFYAML::Symbol> Symbols) {
   return Symbols.size();
 }
 
-static uint64_t writeRawSectionData(raw_ostream &OS,
-                                    const ELFYAML::RawContentSection &RawSec) {
+static uint64_t writeContent(raw_ostream &OS,
+                             const Optional<yaml::BinaryRef> &Content,
+                             const Optional<llvm::yaml::Hex64> &Size) {
   size_t ContentSize = 0;
-  if (RawSec.Content) {
-    RawSec.Content->writeAsBinary(OS);
-    ContentSize = RawSec.Content->binary_size();
+  if (Content) {
+    Content->writeAsBinary(OS);
+    ContentSize = Content->binary_size();
   }
 
-  if (!RawSec.Size)
+  if (!Size)
     return ContentSize;
 
-  OS.write_zeros(*RawSec.Size - ContentSize);
-  return *RawSec.Size;
+  OS.write_zeros(*Size - ContentSize);
+  return *Size;
 }
 
 template <class ELFT>
@@ -466,7 +479,7 @@ ELFState<ELFT>::toELFSymbols(ArrayRef<ELFYAML::Symbol> Symbols,
     if (Sym.NameIndex)
       Symbol.st_name = *Sym.NameIndex;
     else if (!Sym.Name.empty())
-      Symbol.st_name = Strtab.getOffset(dropUniqueSuffix(Sym.Name));
+      Symbol.st_name = Strtab.getOffset(ELFYAML::dropUniqueSuffix(Sym.Name));
 
     Symbol.setBindingAndType(Sym.Binding, Sym.Type);
     if (!Sym.Section.empty())
@@ -548,7 +561,7 @@ void ELFState<ELFT>::initSymtabSectionHeader(Elf_Shdr &SHeader,
   auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
   if (RawSec && (RawSec->Content || RawSec->Size)) {
     assert(Symbols.empty());
-    SHeader.sh_size = writeRawSectionData(OS, *RawSec);
+    SHeader.sh_size = writeContent(OS, RawSec->Content, RawSec->Size);
     return;
   }
 
@@ -573,7 +586,7 @@ void ELFState<ELFT>::initStrtabSectionHeader(Elf_Shdr &SHeader, StringRef Name,
 
   auto &OS = CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
   if (RawSec && (RawSec->Content || RawSec->Size)) {
-    SHeader.sh_size = writeRawSectionData(OS, *RawSec);
+    SHeader.sh_size = writeContent(OS, RawSec->Content, RawSec->Size);
   } else {
     STB.write(OS);
     SHeader.sh_size = STB.getSize();
@@ -669,7 +682,7 @@ void ELFState<ELFT>::writeSectionContent(
     ContiguousBlobAccumulator &CBA) {
   raw_ostream &OS =
       CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
-  SHeader.sh_size = writeRawSectionData(OS, Section);
+  SHeader.sh_size = writeContent(OS, Section.Content, Section.Size);
 
   if (Section.EntSize)
     SHeader.sh_entsize = *Section.EntSize;
@@ -781,6 +794,53 @@ void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
   SHeader.sh_size = Section.Entries.size() * SHeader.sh_entsize;
 }
 
+template <class ELFT>
+void ELFState<ELFT>::writeSectionContent(
+    Elf_Shdr &SHeader, const ELFYAML::StackSizesSection &Section,
+    ContiguousBlobAccumulator &CBA) {
+  using uintX_t = typename ELFT::uint;
+  raw_ostream &OS =
+      CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
+
+  if (Section.Content || Section.Size) {
+    SHeader.sh_size = writeContent(OS, Section.Content, Section.Size);
+    return;
+  }
+
+  for (const ELFYAML::StackSizeEntry &E : *Section.Entries) {
+    support::endian::write<uintX_t>(OS, E.Address, ELFT::TargetEndianness);
+    SHeader.sh_size += sizeof(uintX_t) + encodeULEB128(E.Size, OS);
+  }
+}
+
+template <class ELFT>
+void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
+                                         const ELFYAML::HashSection &Section,
+                                         ContiguousBlobAccumulator &CBA) {
+  raw_ostream &OS =
+      CBA.getOSAndAlignedOffset(SHeader.sh_offset, SHeader.sh_addralign);
+
+  unsigned Link = 0;
+  if (Section.Link.empty() && SN2I.lookup(".dynsym", Link))
+    SHeader.sh_link = Link;
+
+  if (Section.Content || Section.Size) {
+    SHeader.sh_size = writeContent(OS, Section.Content, Section.Size);
+    return;
+  }
+
+  support::endian::write<uint32_t>(OS, Section.Bucket->size(),
+                                   ELFT::TargetEndianness);
+  support::endian::write<uint32_t>(OS, Section.Chain->size(),
+                                   ELFT::TargetEndianness);
+  for (uint32_t Val : *Section.Bucket)
+    support::endian::write<uint32_t>(OS, Val, ELFT::TargetEndianness);
+  for (uint32_t Val : *Section.Chain)
+    support::endian::write<uint32_t>(OS, Val, ELFT::TargetEndianness);
+
+  SHeader.sh_size = (2 + Section.Bucket->size() + Section.Chain->size()) * 4;
+}
+
 template <class ELFT>
 void ELFState<ELFT>::writeSectionContent(Elf_Shdr &SHeader,
                                          const ELFYAML::VerdefSection &Section,
@@ -936,7 +996,7 @@ template <class ELFT> void ELFState<ELFT>::buildSectionIndex() {
     if (Name.empty())
       continue;
 
-    DotShStrtab.add(dropUniqueSuffix(Name));
+    DotShStrtab.add(ELFYAML::dropUniqueSuffix(Name));
     if (!SN2I.addName(Name, I))
       reportError("repeated section name: '" + Name +
                   "' at YAML section number " + Twine(I));
@@ -961,12 +1021,12 @@ template <class ELFT> void ELFState<ELFT>::buildSymbolIndexes() {
 template <class ELFT> void ELFState<ELFT>::finalizeStrings() {
   // Add the regular symbol names to .strtab section.
   for (const ELFYAML::Symbol &Sym : Doc.Symbols)
-    DotStrtab.add(dropUniqueSuffix(Sym.Name));
+    DotStrtab.add(ELFYAML::dropUniqueSuffix(Sym.Name));
   DotStrtab.finalize();
 
   // Add the dynamic symbol names to .dynstr section.
   for (const ELFYAML::Symbol &Sym : Doc.DynamicSymbols)
-    DotDynstr.add(dropUniqueSuffix(Sym.Name));
+    DotDynstr.add(ELFYAML::dropUniqueSuffix(Sym.Name));
 
   // SHT_GNU_verdef and SHT_GNU_verneed sections might also
   // add strings to .dynstr section.
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index e7adb8fe4e079..0dd6854cfee0a 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -221,6 +221,7 @@ void ScalarEnumerationTraits<ELFYAML::ELF_EM>::enumeration(
   ECase(EM_LANAI);
   ECase(EM_BPF);
 #undef ECase
+  IO.enumFallback<Hex16>(Value);
 }
 
 void ScalarEnumerationTraits<ELFYAML::ELF_ELFCLASS>::enumeration(
@@ -650,8 +651,12 @@ void ScalarEnumerationTraits<ELFYAML::ELF_REL>::enumeration(
   case ELF::EM_BPF:
 #include "llvm/BinaryFormat/ELFRelocs/BPF.def"
     break;
+  case ELF::EM_PPC64:
+#include "llvm/BinaryFormat/ELFRelocs/PowerPC64.def"
+    break;
   default:
-    llvm_unreachable("Unsupported architecture");
+    // Nothing to do.
+    break;
   }
 #undef ELF_RELOC
   IO.enumFallback<Hex32>(Value);
@@ -1012,6 +1017,21 @@ static void sectionMapping(IO &IO, ELFYAML::RawContentSection &Section) {
   IO.mapOptional("Info", Section.Info);
 }
 
+static void sectionMapping(IO &IO, ELFYAML::StackSizesSection &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapOptional("Content", Section.Content);
+  IO.mapOptional("Size", Section.Size);
+  IO.mapOptional("Entries", Section.Entries);
+}
+
+static void sectionMapping(IO &IO, ELFYAML::HashSection &Section) {
+  commonSectionMapping(IO, Section);
+  IO.mapOptional("Content", Section.Content);
+  IO.mapOptional("Bucket", Section.Bucket);
+  IO.mapOptional("Chain", Section.Chain);
+  IO.mapOptional("Size", Section.Size);
+}
+
 static void sectionMapping(IO &IO, ELFYAML::NoBitsSection &Section) {
   commonSectionMapping(IO, Section);
   IO.mapOptional("Size", Section.Size, Hex64(0));
@@ -1111,6 +1131,11 @@ void MappingTraits<std::unique_ptr<ELFYAML::Section>>::mapping(
       Section.reset(new ELFYAML::NoBitsSection());
     sectionMapping(IO, *cast<ELFYAML::NoBitsSection>(Section.get()));
     break;
+  case ELF::SHT_HASH:
+    if (!IO.outputting())
+      Section.reset(new ELFYAML::HashSection());
+    sectionMapping(IO, *cast<ELFYAML::HashSection>(Section.get()));
+    break;
   case ELF::SHT_MIPS_ABIFLAGS:
     if (!IO.outputting())
       Section.reset(new ELFYAML::MipsABIFlags());
@@ -1137,20 +1162,77 @@ void MappingTraits<std::unique_ptr<ELFYAML::Section>>::mapping(
     sectionMapping(IO, *cast<ELFYAML::SymtabShndxSection>(Section.get()));
     break;
   default:
-    if (!IO.outputting())
-      Section.reset(new ELFYAML::RawContentSection());
-    sectionMapping(IO, *cast<ELFYAML::RawContentSection>(Section.get()));
+    if (!IO.outputting()) {
+      StringRef Name;
+      IO.mapOptional("Name", Name, StringRef());
+      Name = ELFYAML::dropUniqueSuffix(Name);
+
+      if (ELFYAML::StackSizesSection::nameMatches(Name))
+        Section = std::make_unique<ELFYAML::StackSizesSection>();
+      else
+        Section = std::make_unique<ELFYAML::RawContentSection>();
+    }
+
+    if (auto S = dyn_cast<ELFYAML::RawContentSection>(Section.get()))
+      sectionMapping(IO, *S);
+    else
+      sectionMapping(IO, *cast<ELFYAML::StackSizesSection>(Section.get()));
   }
 }
 
 StringRef MappingTraits<std::unique_ptr<ELFYAML::Section>>::validate(
     IO &io, std::unique_ptr<ELFYAML::Section> &Section) {
-  const auto *RawSection = dyn_cast<ELFYAML::RawContentSection>(Section.get());
-  if (!RawSection)
+  if (const auto *RawSection =
+          dyn_cast<ELFYAML::RawContentSection>(Section.get())) {
+    if (RawSection->Size && RawSection->Content &&
+        (uint64_t)(*RawSection->Size) < RawSection->Content->binary_size())
+      return "Section size must be greater than or equal to the content size";
+    return {};
+  }
+
+  if (const auto *SS = dyn_cast<ELFYAML::StackSizesSection>(Section.get())) {
+    if (!SS->Entries && !SS->Content && !SS->Size)
+      return ".stack_sizes: one of Content, Entries and Size must be specified";
+
+    if (SS->Size && SS->Content &&
+        (uint64_t)(*SS->Size) < SS->Content->binary_size())
+      return ".stack_sizes: Size must be greater than or equal to the content "
+             "size";
+
+    // We accept Content, Size or both together when there are no Entries.
+    if (!SS->Entries)
+      return {};
+
+    if (SS->Size)
+      return ".stack_sizes: Size and Entries cannot be used together";
+    if (SS->Content)
+      return ".stack_sizes: Content and Entries cannot be used together";
     return {};
-  if (RawSection->Size && RawSection->Content &&
-      (uint64_t)(*RawSection->Size) < RawSection->Content->binary_size())
-    return "Section size must be greater than or equal to the content size";
+  }
+
+  if (const auto *HS = dyn_cast<ELFYAML::HashSection>(Section.get())) {
+    if (!HS->Content && !HS->Bucket && !HS->Chain && !HS->Size)
+      return "one of \"Content\", \"Size\", \"Bucket\" or \"Chain\" must be "
+             "specified";
+
+    if (HS->Content || HS->Size) {
+      if (HS->Size && HS->Content &&
+          (uint64_t)*HS->Size < HS->Content->binary_size())
+        return "\"Size\" must be greater than or equal to the content "
+               "size";
+
+      if (HS->Bucket)
+        return "\"Bucket\" cannot be used with \"Content\" or \"Size\"";
+      if (HS->Chain)
+        return "\"Chain\" cannot be used with \"Content\" or \"Size\"";
+      return {};
+    }
+
+    if ((HS->Bucket && !HS->Chain) || (!HS->Bucket && HS->Chain))
+      return "\"Bucket\" and \"Chain\" must be used together";
+    return {};
+  }
+
   return {};
 }
 
@@ -1179,6 +1261,13 @@ struct NormalizedMips64RelType {
 
 } // end anonymous namespace
 
+void MappingTraits<ELFYAML::StackSizeEntry>::mapping(
+    IO &IO, ELFYAML::StackSizeEntry &E) {
+  assert(IO.getContext() && "The IO context is not initialized");
+  IO.mapOptional("Address", E.Address, Hex64(0));
+  IO.mapRequired("Size", E.Size);
+}
+
 void MappingTraits<ELFYAML::DynamicEntry>::mapping(IO &IO,
                                                    ELFYAML::DynamicEntry &Rel) {
   assert(IO.getContext() && "The IO context is not initialized");
diff --git a/llvm/lib/ObjectYAML/WasmEmitter.cpp b/llvm/lib/ObjectYAML/WasmEmitter.cpp
index e374764bf1396..42c57d42ad786 100644
--- a/llvm/lib/ObjectYAML/WasmEmitter.cpp
+++ b/llvm/lib/ObjectYAML/WasmEmitter.cpp
@@ -518,7 +518,7 @@ void WasmWriter::writeRelocSection(raw_ostream &OS, WasmYAML::Section &Sec,
     writeStringRef("reloc.DATA", OS);
     break;
   case wasm::WASM_SEC_CUSTOM: {
-    auto CustomSection = dyn_cast<WasmYAML::CustomSection>(&Sec);
+    auto *CustomSection = cast<WasmYAML::CustomSection>(&Sec);
     writeStringRef(("reloc." + CustomSection->Name).str(), OS);
     break;
   }
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index 460992207e622..bfa3ecfb28060 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
 #include "llvm/Analysis/CGSCCPassManager.h"
 #include "llvm/Analysis/CallGraph.h"
+#include "llvm/Analysis/DDG.h"
 #include "llvm/Analysis/DemandedBits.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/DominanceFrontier.h"
@@ -52,6 +53,7 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
 #include "llvm/CodeGen/UnreachableBlockElim.h"
 #include "llvm/IR/Dominators.h"
@@ -208,7 +210,7 @@ static cl::opt<bool> EnableSyntheticCounts(
     cl::desc("Run synthetic function entry count generation "
              "pass"));
 
-static Regex DefaultAliasRegex(
+static const Regex DefaultAliasRegex(
     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
 
 // This option is used in simplifying testing SampleFDO optimizations for
@@ -1459,6 +1461,15 @@ Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
       UnrollOpts.setOptLevel(OptLevel);
       continue;
     }
+    if (ParamName.consume_front("full-unroll-max=")) {
+      int Count;
+      if (ParamName.getAsInteger(0, Count))
+        return make_error<StringError>(
+            formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
+            inconvertibleErrorCode());
+      UnrollOpts.setFullUnrollMaxCount(Count);
+      continue;
+    }
 
     bool Enable = !ParamName.consume_front("no-");
     if (ParamName == "partial") {
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index b943e9c75ed43..4cd05ee11f64d 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -281,6 +281,7 @@ FUNCTION_PASS_WITH_PARAMS("mldst-motion",
 #endif
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("access-info", LoopAccessAnalysis())
+LOOP_ANALYSIS("ddg", DDGAnalysis())
 LOOP_ANALYSIS("ivusers", IVUsersAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
 #undef LOOP_ANALYSIS
@@ -303,6 +304,7 @@ LOOP_PASS("irce", IRCEPass())
 LOOP_PASS("unroll-and-jam", LoopUnrollAndJamPass())
 LOOP_PASS("unroll-full", LoopFullUnrollPass())
 LOOP_PASS("print-access-info", LoopAccessInfoPrinterPass(dbgs()))
+LOOP_PASS("print<ddg>", DDGAnalysisPrinterPass(dbgs()))
 LOOP_PASS("print<ivusers>", IVUsersPrinterPass(dbgs()))
 LOOP_PASS("print<loop-cache-cost>", LoopCachePrinterPass(dbgs()))
 LOOP_PASS("loop-predication", LoopPredicationPass())
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 803605cd79fa8..8d5e56e26c0ff 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -194,6 +194,15 @@ void FunctionRecordIterator::skipOtherFiles() {
     *this = FunctionRecordIterator();
 }
 
+ArrayRef<unsigned> CoverageMapping::getImpreciseRecordIndicesForFilename(
+    StringRef Filename) const {
+  size_t FilenameHash = hash_value(Filename);
+  auto RecordIt = FilenameHash2RecordIndices.find(FilenameHash);
+  if (RecordIt == FilenameHash2RecordIndices.end())
+    return {};
+  return RecordIt->second;
+}
+
 Error CoverageMapping::loadFunctionRecord(
     const CoverageMappingRecord &Record,
     IndexedInstrProfReader &ProfileReader) {
@@ -249,6 +258,20 @@ Error CoverageMapping::loadFunctionRecord(
     return Error::success();
 
   Functions.push_back(std::move(Function));
+
+  // Performance optimization: keep track of the indices of the function records
+  // which correspond to each filename. This can be used to substantially speed
+  // up queries for coverage info in a file.
+  unsigned RecordIndex = Functions.size() - 1;
+  for (StringRef Filename : Record.Filenames) {
+    auto &RecordIndices = FilenameHash2RecordIndices[hash_value(Filename)];
+    // Note that there may be duplicates in the filename set for a function
+    // record, because of e.g. macro expansions in the function in which both
+    // the macro and the function are defined in the same file.
+    if (RecordIndices.empty() || RecordIndices.back() != RecordIndex)
+      RecordIndices.push_back(RecordIndex);
+  }
+
   return Error::success();
 }
 
@@ -626,7 +649,12 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
   CoverageData FileCoverage(Filename);
   std::vector<CountedRegion> Regions;
 
-  for (const auto &Function : Functions) {
+  // Look up the function records in the given file. Due to hash collisions on
+  // the filename, we may get back some records that are not in the file.
+  ArrayRef<unsigned> RecordIndices =
+      getImpreciseRecordIndicesForFilename(Filename);
+  for (unsigned RecordIndex : RecordIndices) {
+    const FunctionRecord &Function = Functions[RecordIndex];
     auto MainFileID = findMainViewFileID(Filename, Function);
     auto FileIDs = gatherFileIDs(Filename, Function);
     for (const auto &CR : Function.CountedRegions)
@@ -646,7 +674,12 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
 std::vector<InstantiationGroup>
 CoverageMapping::getInstantiationGroups(StringRef Filename) const {
   FunctionInstantiationSetCollector InstantiationSetCollector;
-  for (const auto &Function : Functions) {
+  // Look up the function records in the given file. Due to hash collisions on
+  // the filename, we may get back some records that are not in the file.
+  ArrayRef<unsigned> RecordIndices =
+      getImpreciseRecordIndicesForFilename(Filename);
+  for (unsigned RecordIndex : RecordIndices) {
+    const FunctionRecord &Function = Functions[RecordIndex];
     auto MainFileID = findMainViewFileID(Filename, Function);
     if (!MainFileID)
       continue;
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index 4e0b911c14224..6fffacfde74d9 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -682,7 +682,7 @@ loadBinaryFormat(std::unique_ptr<Binary> Bin, StringRef Arch) {
   if (auto *Universal = dyn_cast<MachOUniversalBinary>(Bin.get())) {
     // If we have a universal binary, try to look up the object for the
     // appropriate architecture.
-    auto ObjectFileOrErr = Universal->getObjectForArch(Arch);
+    auto ObjectFileOrErr = Universal->getMachOObjectForArch(Arch);
     if (!ObjectFileOrErr)
       return ObjectFileOrErr.takeError();
     OF = std::move(ObjectFileOrErr.get());
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 432b20f217ca8..d75854a60d1ee 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -24,6 +24,16 @@
 using namespace llvm;
 using namespace coverage;
 
+CoverageFilenamesSectionWriter::CoverageFilenamesSectionWriter(
+    ArrayRef<StringRef> Filenames)
+    : Filenames(Filenames) {
+#ifndef NDEBUG
+  StringSet<> NameSet;
+  for (StringRef Name : Filenames)
+    assert(NameSet.insert(Name).second && "Duplicate filename");
+#endif
+}
+
 void CoverageFilenamesSectionWriter::write(raw_ostream &OS) {
   encodeULEB128(Filenames.size(), OS);
   for (const auto &Filename : Filenames) {
diff --git a/llvm/lib/ProfileData/InstrProf.cpp b/llvm/lib/ProfileData/InstrProf.cpp
index 510fd9887d9a3..57d4fbc59f83b 100644
--- a/llvm/lib/ProfileData/InstrProf.cpp
+++ b/llvm/lib/ProfileData/InstrProf.cpp
@@ -478,7 +478,7 @@ Error readPGOFuncNameStrings(StringRef NameStrings, InstrProfSymtab &Symtab) {
   return Error::success();
 }
 
-void InstrProfRecord::accumuateCounts(CountSumOrPercent &Sum) const {
+void InstrProfRecord::accumulateCounts(CountSumOrPercent &Sum) const {
   uint64_t FuncSum = 0;
   Sum.NumEntries += Counts.size();
   for (size_t F = 0, E = Counts.size(); F < E; ++F)
@@ -552,7 +552,7 @@ void InstrProfRecord::overlap(InstrProfRecord &Other, OverlapStats &Overlap,
                               uint64_t ValueCutoff) {
   // FuncLevel CountSum for other should already computed and nonzero.
   assert(FuncLevelOverlap.Test.CountSum >= 1.0f);
-  accumuateCounts(FuncLevelOverlap.Base);
+  accumulateCounts(FuncLevelOverlap.Base);
   bool Mismatch = (Counts.size() != Other.Counts.size());
 
   // Check if the value profiles mismatch.
@@ -1078,12 +1078,10 @@ bool isIRPGOFlagSet(const Module *M) {
   if (!IRInstrVar->hasInitializer())
     return false;
 
-  const Constant *InitVal = IRInstrVar->getInitializer();
+  auto *InitVal = dyn_cast_or_null<ConstantInt>(IRInstrVar->getInitializer());
   if (!InitVal)
     return false;
-
-  return (dyn_cast<ConstantInt>(InitVal)->getZExtValue() &
-          VARIANT_MASK_IR_PROF) != 0;
+  return (InitVal->getZExtValue() & VARIANT_MASK_IR_PROF) != 0;
 }
 
 // Check if we can safely rename this Comdat function.
@@ -1166,9 +1164,9 @@ void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput) {
   }
 }
 
-Error OverlapStats::accumuateCounts(const std::string &BaseFilename,
-                                    const std::string &TestFilename,
-                                    bool IsCS) {
+Error OverlapStats::accumulateCounts(const std::string &BaseFilename,
+                                     const std::string &TestFilename,
+                                     bool IsCS) {
   auto getProfileSum = [IsCS](const std::string &Filename,
                               CountSumOrPercent &Sum) -> Error {
     auto ReaderOrErr = InstrProfReader::create(Filename);
@@ -1176,7 +1174,7 @@ Error OverlapStats::accumuateCounts(const std::string &BaseFilename,
       return E;
     }
     auto Reader = std::move(ReaderOrErr.get());
-    Reader->accumuateCounts(Sum, IsCS);
+    Reader->accumulateCounts(Sum, IsCS);
     return Error::success();
   };
   auto Ret = getProfileSum(BaseFilename, Base);
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 5fb1d9486c16a..23d078a3ddeef 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -783,13 +783,13 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
         SummaryData->get(Summary::TotalNumFunctions));
     return Cur + SummarySize;
   } else {
-    // For older version of profile data, we need to compute on the fly:
-    using namespace IndexedInstrProf;
-
+    // The older versions do not support a profile summary. This just computes
+    // an empty summary, which will not result in accurate hot/cold detection.
+    // We would need to call addRecord for all NamedInstrProfRecords to get the
+    // correct summary. However, this version is old (prior to early 2016) and
+    // has not been supporting an accurate summary for several years.
     InstrProfSummaryBuilder Builder(ProfileSummaryBuilder::DefaultCutoffs);
-    // FIXME: This only computes an empty summary. Need to call addRecord for
-    // all NamedInstrProfRecords to get the correct summary.
-    this->Summary = Builder.getSummary();
+    Summary = Builder.getSummary();
     return Cur;
   }
 }
@@ -907,7 +907,7 @@ Error IndexedInstrProfReader::readNextRecord(NamedInstrProfRecord &Record) {
   return success();
 }
 
-void InstrProfReader::accumuateCounts(CountSumOrPercent &Sum, bool IsCS) {
+void InstrProfReader::accumulateCounts(CountSumOrPercent &Sum, bool IsCS) {
   uint64_t NumFuncs = 0;
   for (const auto &Func : *this) {
     if (isIRLevelProfile()) {
@@ -915,7 +915,7 @@ void InstrProfReader::accumuateCounts(CountSumOrPercent &Sum, bool IsCS) {
       if (FuncIsCS != IsCS)
         continue;
     }
-    Func.accumuateCounts(Sum);
+    Func.accumulateCounts(Sum);
     ++NumFuncs;
   }
   Sum.NumEntries = NumFuncs;
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index 4ca2defd26da9..ccb270e0b719b 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -193,7 +193,7 @@ void InstrProfWriter::overlapRecord(NamedInstrProfRecord &&Other,
                                     const OverlapFuncFilters &FuncFilter) {
   auto Name = Other.Name;
   auto Hash = Other.Hash;
-  Other.accumuateCounts(FuncLevelOverlap.Test);
+  Other.accumulateCounts(FuncLevelOverlap.Test);
   if (FunctionData.find(Name) == FunctionData.end()) {
     Overlap.addOneUnique(FuncLevelOverlap.Test);
     return;
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index f6b33d962616c..07272ebac0a9e 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -667,6 +667,36 @@ std::error_code SampleProfileReaderExtBinaryBase::readHeader() {
   return sampleprof_error::success;
 }
 
+uint64_t SampleProfileReaderExtBinaryBase::getSectionSize(SecType Type) {
+  for (auto &Entry : SecHdrTable) {
+    if (Entry.Type == Type)
+      return Entry.Size;
+  }
+  return 0;
+}
+
+uint64_t SampleProfileReaderExtBinaryBase::getFileSize() {
+  auto &LastEntry = SecHdrTable.back();
+  return LastEntry.Offset + LastEntry.Size;
+}
+
+bool SampleProfileReaderExtBinaryBase::dumpSectionInfo(raw_ostream &OS) {
+  uint64_t TotalSecsSize = 0;
+  for (auto &Entry : SecHdrTable) {
+    OS << getSecName(Entry.Type) << " - Offset: " << Entry.Offset
+       << ", Size: " << Entry.Size << "\n";
+    TotalSecsSize += getSectionSize(Entry.Type);
+  }
+  uint64_t HeaderSize = SecHdrTable.front().Offset;
+  assert(HeaderSize + TotalSecsSize == getFileSize() &&
+         "Size of 'header + sections' doesn't match the total size of profile");
+
+  OS << "Header Size: " << HeaderSize << "\n";
+  OS << "Total Sections Size: " << TotalSecsSize << "\n";
+  OS << "File Size: " << getFileSize() << "\n";
+  return true;
+}
+
 std::error_code SampleProfileReaderBinary::readMagicIdent() {
   // Read and check the magic identifier.
   auto Magic = readNumber<uint64_t>();
diff --git a/llvm/lib/Remarks/BitstreamRemarkSerializer.cpp b/llvm/lib/Remarks/BitstreamRemarkSerializer.cpp
index 03c5697109eb4..d02782c7954da 100644
--- a/llvm/lib/Remarks/BitstreamRemarkSerializer.cpp
+++ b/llvm/lib/Remarks/BitstreamRemarkSerializer.cpp
@@ -326,7 +326,7 @@ StringRef BitstreamRemarkSerializerHelper::getBuffer() {
 
 BitstreamRemarkSerializer::BitstreamRemarkSerializer(raw_ostream &OS,
                                                      SerializerMode Mode)
-    : RemarkSerializer(OS, Mode),
+    : RemarkSerializer(Format::Bitstream, OS, Mode),
       Helper(BitstreamRemarkContainerType::SeparateRemarksFile) {
   assert(Mode == SerializerMode::Separate &&
          "For SerializerMode::Standalone, a pre-filled string table needs to "
@@ -338,7 +338,7 @@ BitstreamRemarkSerializer::BitstreamRemarkSerializer(raw_ostream &OS,
 BitstreamRemarkSerializer::BitstreamRemarkSerializer(raw_ostream &OS,
                                                      SerializerMode Mode,
                                                      StringTable StrTabIn)
-    : RemarkSerializer(OS, Mode),
+    : RemarkSerializer(Format::Bitstream, OS, Mode),
       Helper(Mode == SerializerMode::Separate
                  ? BitstreamRemarkContainerType::SeparateRemarksFile
                  : BitstreamRemarkContainerType::Standalone) {
diff --git a/llvm/lib/Remarks/RemarkSerializer.cpp b/llvm/lib/Remarks/RemarkSerializer.cpp
index caba7bbdc0ebf..ab19c84bbadbd 100644
--- a/llvm/lib/Remarks/RemarkSerializer.cpp
+++ b/llvm/lib/Remarks/RemarkSerializer.cpp
@@ -42,15 +42,13 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
     return createStringError(std::errc::invalid_argument,
                              "Unknown remark serializer format.");
   case Format::YAML:
-    return createStringError(std::errc::invalid_argument,
-                             "Unable to use a string table with the yaml "
-                             "format. Use 'yaml-strtab' instead.");
+    return std::make_unique<YAMLRemarkSerializer>(OS, Mode, std::move(StrTab));
   case Format::YAMLStrTab:
     return std::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode,
-                                                         std::move(StrTab));
+                                                        std::move(StrTab));
   case Format::Bitstream:
     return std::make_unique<BitstreamRemarkSerializer>(OS, Mode,
-                                                        std::move(StrTab));
+                                                       std::move(StrTab));
   }
   llvm_unreachable("Unknown remarks::Format enum");
 }
diff --git a/llvm/lib/Remarks/RemarkStringTable.cpp b/llvm/lib/Remarks/RemarkStringTable.cpp
index 0f8a95acdba64..51156465be513 100644
--- a/llvm/lib/Remarks/RemarkStringTable.cpp
+++ b/llvm/lib/Remarks/RemarkStringTable.cpp
@@ -46,7 +46,6 @@ void StringTable::internalize(Remark &R) {
   if (R.Loc)
     Impl(R.Loc->SourceFilePath);
   for (Argument &Arg : R.Args) {
-    // We need to mutate elements from an ArrayRef here.
     Impl(Arg.Key);
     Impl(Arg.Val);
     if (Arg.Loc)
diff --git a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
index a4d67cc6715a4..66eb06bbc4f5d 100644
--- a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
+++ b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
@@ -56,11 +56,14 @@ template <> struct MappingTraits<remarks::Remark *> {
     else
       llvm_unreachable("Unknown remark type");
 
-    if (Optional<StringTable> &StrTab =
-            reinterpret_cast<YAMLRemarkSerializer *>(io.getContext())->StrTab) {
-      unsigned PassID = StrTab->add(Remark->PassName).first;
-      unsigned NameID = StrTab->add(Remark->RemarkName).first;
-      unsigned FunctionID = StrTab->add(Remark->FunctionName).first;
+    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
+            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
+      assert(Serializer->StrTab.hasValue() &&
+             "YAMLStrTabSerializer with no StrTab.");
+      StringTable &StrTab = *Serializer->StrTab;
+      unsigned PassID = StrTab.add(Remark->PassName).first;
+      unsigned NameID = StrTab.add(Remark->RemarkName).first;
+      unsigned FunctionID = StrTab.add(Remark->FunctionName).first;
       mapRemarkHeader(io, PassID, NameID, Remark->Loc, FunctionID,
                       Remark->Hotness, Remark->Args);
     } else {
@@ -78,9 +81,12 @@ template <> struct MappingTraits<RemarkLocation> {
     unsigned Line = RL.SourceLine;
     unsigned Col = RL.SourceColumn;
 
-    if (Optional<StringTable> &StrTab =
-            reinterpret_cast<YAMLRemarkSerializer *>(io.getContext())->StrTab) {
-      unsigned FileID = StrTab->add(File).first;
+    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
+            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
+      assert(Serializer->StrTab.hasValue() &&
+             "YAMLStrTabSerializer with no StrTab.");
+      StringTable &StrTab = *Serializer->StrTab;
+      unsigned FileID = StrTab.add(File).first;
       io.mapRequired("File", FileID);
     } else {
       io.mapRequired("File", File);
@@ -130,9 +136,12 @@ template <> struct MappingTraits<Argument> {
   static void mapping(IO &io, Argument &A) {
     assert(io.outputting() && "input not yet implemented");
 
-    if (Optional<StringTable> &StrTab =
-            reinterpret_cast<YAMLRemarkSerializer *>(io.getContext())->StrTab) {
-      auto ValueID = StrTab->add(A.Val).first;
+    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
+            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
+      assert(Serializer->StrTab.hasValue() &&
+             "YAMLStrTabSerializer with no StrTab.");
+      StringTable &StrTab = *Serializer->StrTab;
+      auto ValueID = StrTab.add(A.Val).first;
       io.mapRequired(A.Key.data(), ValueID);
     } else if (StringRef(A.Val).count('\n') > 1) {
       StringBlockVal S(A.Val);
@@ -149,8 +158,17 @@ template <> struct MappingTraits<Argument> {
 
 LLVM_YAML_IS_SEQUENCE_VECTOR(Argument)
 
-YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
-    : RemarkSerializer(OS, Mode), YAMLOutput(OS, reinterpret_cast<void *>(this)) {}
+YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
+                                           Optional<StringTable> StrTabIn)
+    : YAMLRemarkSerializer(Format::YAML, OS, Mode, std::move(StrTabIn)) {}
+
+YAMLRemarkSerializer::YAMLRemarkSerializer(Format SerializerFormat,
+                                           raw_ostream &OS, SerializerMode Mode,
+                                           Optional<StringTable> StrTabIn)
+    : RemarkSerializer(SerializerFormat, OS, Mode),
+      YAMLOutput(OS, reinterpret_cast<void *>(this)) {
+  StrTab = std::move(StrTabIn);
+}
 
 void YAMLRemarkSerializer::emit(const Remark &Remark) {
   // Again, YAMLTraits expect a non-const object for inputting, but we're not
@@ -183,7 +201,7 @@ std::unique_ptr<MetaSerializer> YAMLStrTabRemarkSerializer::metaSerializer(
     raw_ostream &OS, Optional<StringRef> ExternalFilename) {
   assert(StrTab);
   return std::make_unique<YAMLStrTabMetaSerializer>(OS, ExternalFilename,
-                                                     std::move(*StrTab));
+                                                    *StrTab);
 }
 
 static void emitMagic(raw_ostream &OS) {
@@ -200,9 +218,9 @@ static void emitVersion(raw_ostream &OS) {
   OS.write(Version.data(), Version.size());
 }
 
-static void emitStrTab(raw_ostream &OS, const Optional<StringTable> &StrTab) {
+static void emitStrTab(raw_ostream &OS, Optional<const StringTable *> StrTab) {
   // Emit the string table in the section.
-  uint64_t StrTabSize = StrTab ? StrTab->SerializedSize : 0;
+  uint64_t StrTabSize = StrTab ? (*StrTab)->SerializedSize : 0;
   // Emit the total size of the string table (the size itself excluded):
   // little-endian uint64_t.
   // Note: even if no string table is used, emit 0.
@@ -210,7 +228,7 @@ static void emitStrTab(raw_ostream &OS, const Optional<StringTable> &StrTab) {
   support::endian::write64le(StrTabSizeBuf.data(), StrTabSize);
   OS.write(StrTabSizeBuf.data(), StrTabSizeBuf.size());
   if (StrTab)
-    StrTab->serialize(OS);
+    (*StrTab)->serialize(OS);
 }
 
 static void emitExternalFile(raw_ostream &OS, StringRef Filename) {
@@ -233,7 +251,7 @@ void YAMLMetaSerializer::emit() {
 void YAMLStrTabMetaSerializer::emit() {
   emitMagic(OS);
   emitVersion(OS);
-  emitStrTab(OS, std::move(StrTab));
+  emitStrTab(OS, &StrTab);
   if (ExternalFilename)
     emitExternalFile(OS, *ExternalFilename);
 }
diff --git a/llvm/lib/SYCL/ASFixer.cpp b/llvm/lib/SYCL/ASFixer.cpp
index 235f2d539f1b7..b54bc8f0d894b 100644
--- a/llvm/lib/SYCL/ASFixer.cpp
+++ b/llvm/lib/SYCL/ASFixer.cpp
@@ -461,7 +461,7 @@ static AllocaInst *createAllocaReplacement(AllocaInst *OldAlloca, Type *NewTy) {
   auto NewAlloca = new AllocaInst(
       NewTy, OldAlloca->getType()->getAddressSpace(), OldAlloca->getArraySize(),
       "new." + OldAlloca->getName(), OldAlloca);
-  NewAlloca->setAlignment(OldAlloca->getAlignment());
+  NewAlloca->setAlignment(MaybeAlign(OldAlloca->getAlignment()));
   return NewAlloca;
 }
 
diff --git a/llvm/lib/Support/APInt.cpp b/llvm/lib/Support/APInt.cpp
index 43173311cd802..758fe8b4f866a 100644
--- a/llvm/lib/Support/APInt.cpp
+++ b/llvm/lib/Support/APInt.cpp
@@ -401,6 +401,33 @@ void APInt::insertBits(const APInt &subBits, unsigned bitPosition) {
   }
 }
 
+void APInt::insertBits(uint64_t subBits, unsigned bitPosition, unsigned numBits) {
+  uint64_t maskBits = maskTrailingOnes<uint64_t>(numBits);
+  subBits &= maskBits;
+  if (isSingleWord()) {
+    U.VAL &= ~(maskBits << bitPosition);
+    U.VAL |= subBits << bitPosition;
+    return;
+  }
+
+  unsigned loBit = whichBit(bitPosition);
+  unsigned loWord = whichWord(bitPosition);
+  unsigned hiWord = whichWord(bitPosition + numBits - 1);
+  if (loWord == hiWord) {
+    U.pVal[loWord] &= ~(maskBits << loBit);
+    U.pVal[loWord] |= subBits << loBit;
+    return;
+  }
+
+  static_assert(8 * sizeof(WordType) <= 64, "This code assumes only two words affected");
+  unsigned wordBits = 8 * sizeof(WordType);
+  U.pVal[loWord] &= ~(maskBits << loBit);
+  U.pVal[loWord] |= subBits << loBit;
+
+  U.pVal[hiWord] &= ~(maskBits >> (wordBits - loBit));
+  U.pVal[hiWord] |= subBits >> (wordBits - loBit);
+}
+
 APInt APInt::extractBits(unsigned numBits, unsigned bitPosition) const {
   assert(numBits > 0 && "Can't extract zero bits");
   assert(bitPosition < BitWidth && (numBits + bitPosition) <= BitWidth &&
@@ -438,6 +465,31 @@ APInt APInt::extractBits(unsigned numBits, unsigned bitPosition) const {
   return Result.clearUnusedBits();
 }
 
+uint64_t APInt::extractBitsAsZExtValue(unsigned numBits,
+                                       unsigned bitPosition) const {
+  assert(numBits > 0 && "Can't extract zero bits");
+  assert(bitPosition < BitWidth && (numBits + bitPosition) <= BitWidth &&
+         "Illegal bit extraction");
+  assert(numBits <= 64 && "Illegal bit extraction");
+
+  uint64_t maskBits = maskTrailingOnes<uint64_t>(numBits);
+  if (isSingleWord())
+    return (U.VAL >> bitPosition) & maskBits;
+
+  unsigned loBit = whichBit(bitPosition);
+  unsigned loWord = whichWord(bitPosition);
+  unsigned hiWord = whichWord(bitPosition + numBits - 1);
+  if (loWord == hiWord)
+    return (U.pVal[loWord] >> loBit) & maskBits;
+
+  static_assert(8 * sizeof(WordType) <= 64, "This code assumes only two words affected");
+  unsigned wordBits = 8 * sizeof(WordType);
+  uint64_t retBits = U.pVal[loWord] >> loBit;
+  retBits |= U.pVal[hiWord] << (wordBits - loBit);
+  retBits &= maskBits;
+  return retBits;
+}
+
 unsigned APInt::getBitsNeeded(StringRef str, uint8_t radix) {
   assert(!str.empty() && "Invalid string length");
   assert((radix == 10 || radix == 8 || radix == 16 || radix == 2 ||
diff --git a/llvm/lib/Support/ARMTargetParser.cpp b/llvm/lib/Support/ARMTargetParser.cpp
index be948cfc95d43..ce5daa7fe58c0 100644
--- a/llvm/lib/Support/ARMTargetParser.cpp
+++ b/llvm/lib/Support/ARMTargetParser.cpp
@@ -176,10 +176,8 @@ bool ARM::getFPUFeatures(unsigned FPUKind, std::vector<StringRef> &Features) {
     // exist).
 
     {"+fpregs", "-fpregs", FPUVersion::VFPV2, FPURestriction::SP_D16},
-    {"+vfp2", "-vfp2", FPUVersion::VFPV2, FPURestriction::None},
-    {"+vfp2d16", "-vfp2d16", FPUVersion::VFPV2, FPURestriction::D16},
-    {"+vfp2d16sp", "-vfp2d16sp", FPUVersion::VFPV2, FPURestriction::SP_D16},
-    {"+vfp2sp", "-vfp2sp", FPUVersion::VFPV2, FPURestriction::None},
+    {"+vfp2", "-vfp2", FPUVersion::VFPV2, FPURestriction::D16},
+    {"+vfp2sp", "-vfp2sp", FPUVersion::VFPV2, FPURestriction::SP_D16},
     {"+vfp3", "-vfp3", FPUVersion::VFPV3, FPURestriction::None},
     {"+vfp3d16", "-vfp3d16", FPUVersion::VFPV3, FPURestriction::D16},
     {"+vfp3d16sp", "-vfp3d16sp", FPUVersion::VFPV3, FPURestriction::SP_D16},
@@ -195,7 +193,7 @@ bool ARM::getFPUFeatures(unsigned FPUKind, std::vector<StringRef> &Features) {
     {"+fp-armv8sp", "-fp-armv8sp", FPUVersion::VFPV5, FPURestriction::None},
     {"+fullfp16", "-fullfp16", FPUVersion::VFPV5_FULLFP16, FPURestriction::SP_D16},
     {"+fp64", "-fp64", FPUVersion::VFPV2, FPURestriction::D16},
-    {"+d32", "-d32", FPUVersion::VFPV2, FPURestriction::None},
+    {"+d32", "-d32", FPUVersion::VFPV3, FPURestriction::None},
   };
 
   for (const auto &Info: FPUFeatureInfoList) {
diff --git a/llvm/lib/Support/FileCheck.cpp b/llvm/lib/Support/FileCheck.cpp
index 5c05af4c190b9..c3f537b35243e 100644
--- a/llvm/lib/Support/FileCheck.cpp
+++ b/llvm/lib/Support/FileCheck.cpp
@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FileCheck.h"
+#include "FileCheckImpl.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/FormatVariadic.h"
@@ -1116,16 +1117,22 @@ void FileCheckPatternContext::createLineVariable() {
   GlobalNumericVariableTable[LineName] = LineVariable;
 }
 
-bool FileCheck::ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
-                              std::vector<FileCheckString> &CheckStrings) {
+FileCheck::FileCheck(FileCheckRequest Req)
+    : Req(Req), PatternContext(std::make_unique<FileCheckPatternContext>()),
+      CheckStrings(std::make_unique<std::vector<FileCheckString>>()) {}
+
+FileCheck::~FileCheck() = default;
+
+bool FileCheck::readCheckFile(SourceMgr &SM, StringRef Buffer,
+                              Regex &PrefixRE) {
   Error DefineError =
-      PatternContext.defineCmdlineVariables(Req.GlobalDefines, SM);
+      PatternContext->defineCmdlineVariables(Req.GlobalDefines, SM);
   if (DefineError) {
     logAllUnhandledErrors(std::move(DefineError), errs());
     return true;
   }
 
-  PatternContext.createLineVariable();
+  PatternContext->createLineVariable();
 
   std::vector<FileCheckPattern> ImplicitNegativeChecks;
   for (const auto &PatternString : Req.ImplicitCheckNot) {
@@ -1141,7 +1148,7 @@ bool FileCheck::ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
     SM.AddNewSourceBuffer(std::move(CmdLine), SMLoc());
 
     ImplicitNegativeChecks.push_back(
-        FileCheckPattern(Check::CheckNot, &PatternContext));
+        FileCheckPattern(Check::CheckNot, PatternContext.get()));
     ImplicitNegativeChecks.back().parsePattern(PatternInBuffer,
                                                "IMPLICIT-CHECK", SM, Req);
   }
@@ -1204,7 +1211,7 @@ bool FileCheck::ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
     SMLoc PatternLoc = SMLoc::getFromPointer(Buffer.data());
 
     // Parse the pattern.
-    FileCheckPattern P(CheckTy, &PatternContext, LineNumber);
+    FileCheckPattern P(CheckTy, PatternContext.get(), LineNumber);
     if (P.parsePattern(Buffer.substr(0, EOL), UsedPrefix, SM, Req))
       return true;
 
@@ -1222,7 +1229,7 @@ bool FileCheck::ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
     // Verify that CHECK-NEXT/SAME/EMPTY lines have at least one CHECK line before them.
     if ((CheckTy == Check::CheckNext || CheckTy == Check::CheckSame ||
          CheckTy == Check::CheckEmpty) &&
-        CheckStrings.empty()) {
+        CheckStrings->empty()) {
       StringRef Type = CheckTy == Check::CheckNext
                            ? "NEXT"
                            : CheckTy == Check::CheckEmpty ? "EMPTY" : "SAME";
@@ -1240,21 +1247,21 @@ bool FileCheck::ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE,
     }
 
     // Okay, add the string we captured to the output vector and move on.
-    CheckStrings.emplace_back(P, UsedPrefix, PatternLoc);
-    std::swap(DagNotMatches, CheckStrings.back().DagNotStrings);
+    CheckStrings->emplace_back(P, UsedPrefix, PatternLoc);
+    std::swap(DagNotMatches, CheckStrings->back().DagNotStrings);
     DagNotMatches = ImplicitNegativeChecks;
   }
 
   // Add an EOF pattern for any trailing CHECK-DAG/-NOTs, and use the first
   // prefix as a filler for the error message.
   if (!DagNotMatches.empty()) {
-    CheckStrings.emplace_back(
-        FileCheckPattern(Check::CheckEOF, &PatternContext, LineNumber + 1),
+    CheckStrings->emplace_back(
+        FileCheckPattern(Check::CheckEOF, PatternContext.get(), LineNumber + 1),
         *Req.CheckPrefixes.begin(), SMLoc::getFromPointer(Buffer.data()));
-    std::swap(DagNotMatches, CheckStrings.back().DagNotStrings);
+    std::swap(DagNotMatches, CheckStrings->back().DagNotStrings);
   }
 
-  if (CheckStrings.empty()) {
+  if (CheckStrings->empty()) {
     errs() << "error: no check strings found with prefix"
            << (Req.CheckPrefixes.size() > 1 ? "es " : " ");
     auto I = Req.CheckPrefixes.begin();
@@ -1712,7 +1719,7 @@ FileCheckString::CheckDag(const SourceMgr &SM, StringRef Buffer,
 
 // A check prefix must contain only alphanumeric, hyphens and underscores.
 static bool ValidateCheckPrefix(StringRef CheckPrefix) {
-  Regex Validator("^[a-zA-Z0-9_-]*$");
+  static const Regex Validator("^[a-zA-Z0-9_-]*$");
   return Validator.match(CheckPrefix);
 }
 
@@ -1916,18 +1923,17 @@ void FileCheckPatternContext::clearLocalVars() {
     GlobalNumericVariableTable.erase(Var);
 }
 
-bool FileCheck::CheckInput(SourceMgr &SM, StringRef Buffer,
-                           ArrayRef<FileCheckString> CheckStrings,
+bool FileCheck::checkInput(SourceMgr &SM, StringRef Buffer,
                            std::vector<FileCheckDiag> *Diags) {
   bool ChecksFailed = false;
 
-  unsigned i = 0, j = 0, e = CheckStrings.size();
+  unsigned i = 0, j = 0, e = CheckStrings->size();
   while (true) {
     StringRef CheckRegion;
     if (j == e) {
       CheckRegion = Buffer;
     } else {
-      const FileCheckString &CheckLabelStr = CheckStrings[j];
+      const FileCheckString &CheckLabelStr = (*CheckStrings)[j];
       if (CheckLabelStr.Pat.getCheckTy() != Check::CheckLabel) {
         ++j;
         continue;
@@ -1950,10 +1956,10 @@ bool FileCheck::CheckInput(SourceMgr &SM, StringRef Buffer,
     // CHECK-LABEL and it would clear variables defined on the command-line
     // before they get used.
     if (i != 0 && Req.EnableVarScope)
-      PatternContext.clearLocalVars();
+      PatternContext->clearLocalVars();
 
     for (; i != j; ++i) {
-      const FileCheckString &CheckStr = CheckStrings[i];
+      const FileCheckString &CheckStr = (*CheckStrings)[i];
 
       // Check each string within the scanned region, including a second check
       // of any final CHECK-LABEL (to verify CHECK-NOT and CHECK-DAG)
diff --git a/llvm/lib/Support/FileCheckImpl.h b/llvm/lib/Support/FileCheckImpl.h
new file mode 100644
index 0000000000000..001b3589d5fdc
--- /dev/null
+++ b/llvm/lib/Support/FileCheckImpl.h
@@ -0,0 +1,621 @@
+//===-- FileCheckImpl.h - Private FileCheck Interface ------------*- C++ -*-==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the private interfaces of FileCheck. Its purpose is to
+// allow unit testing of FileCheck and to separate the interface from the
+// implementation. It is only meant to be used by FileCheck.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_SUPPORT_FILECHECKIMPL_H
+#define LLVM_LIB_SUPPORT_FILECHECKIMPL_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/SourceMgr.h"
+#include <map>
+#include <string>
+#include <vector>
+
+namespace llvm {
+
+//===----------------------------------------------------------------------===//
+// Numeric substitution handling code.
+//===----------------------------------------------------------------------===//
+
+/// Base class representing the AST of a given expression.
+class FileCheckExpressionAST {
+public:
+  virtual ~FileCheckExpressionAST() = default;
+
+  /// Evaluates and \returns the value of the expression represented by this
+  /// AST or an error if evaluation fails.
+  virtual Expected<uint64_t> eval() const = 0;
+};
+
+/// Class representing an unsigned literal in the AST of an expression.
+class FileCheckExpressionLiteral : public FileCheckExpressionAST {
+private:
+  /// Actual value of the literal.
+  uint64_t Value;
+
+public:
+  /// Constructs a literal with the specified value.
+  FileCheckExpressionLiteral(uint64_t Val) : Value(Val) {}
+
+  /// \returns the literal's value.
+  Expected<uint64_t> eval() const { return Value; }
+};
+
+/// Class to represent an undefined variable error, which quotes that
+/// variable's name when printed.
+class FileCheckUndefVarError : public ErrorInfo<FileCheckUndefVarError> {
+private:
+  StringRef VarName;
+
+public:
+  static char ID;
+
+  FileCheckUndefVarError(StringRef VarName) : VarName(VarName) {}
+
+  StringRef getVarName() const { return VarName; }
+
+  std::error_code convertToErrorCode() const override {
+    return inconvertibleErrorCode();
+  }
+
+  /// Print name of variable associated with this error.
+  void log(raw_ostream &OS) const override {
+    OS << "\"";
+    OS.write_escaped(VarName) << "\"";
+  }
+};
+
+/// Class representing a numeric variable and its associated current value.
+class FileCheckNumericVariable {
+private:
+  /// Name of the numeric variable.
+  StringRef Name;
+
+  /// Value of numeric variable, if defined, or None otherwise.
+  Optional<uint64_t> Value;
+
+  /// Line number where this variable is defined, or None if defined before
+  /// input is parsed. Used to determine whether a variable is defined on the
+  /// same line as a given use.
+  Optional<size_t> DefLineNumber;
+
+public:
+  /// Constructor for a variable \p Name defined at line \p DefLineNumber or
+  /// defined before input is parsed if \p DefLineNumber is None.
+  explicit FileCheckNumericVariable(StringRef Name,
+                                    Optional<size_t> DefLineNumber = None)
+      : Name(Name), DefLineNumber(DefLineNumber) {}
+
+  /// \returns name of this numeric variable.
+  StringRef getName() const { return Name; }
+
+  /// \returns this variable's value.
+  Optional<uint64_t> getValue() const { return Value; }
+
+  /// Sets value of this numeric variable to \p NewValue.
+  void setValue(uint64_t NewValue) { Value = NewValue; }
+
+  /// Clears value of this numeric variable, regardless of whether it is
+  /// currently defined or not.
+  void clearValue() { Value = None; }
+
+  /// \returns the line number where this variable is defined, if any, or None
+  /// if defined before input is parsed.
+  Optional<size_t> getDefLineNumber() { return DefLineNumber; }
+};
+
+/// Class representing the use of a numeric variable in the AST of an
+/// expression.
+class FileCheckNumericVariableUse : public FileCheckExpressionAST {
+private:
+  /// Name of the numeric variable.
+  StringRef Name;
+
+  /// Pointer to the class instance for the variable this use is about.
+  FileCheckNumericVariable *NumericVariable;
+
+public:
+  FileCheckNumericVariableUse(StringRef Name,
+                              FileCheckNumericVariable *NumericVariable)
+      : Name(Name), NumericVariable(NumericVariable) {}
+
+  /// \returns the value of the variable referenced by this instance.
+  Expected<uint64_t> eval() const;
+};
+
+/// Type of functions evaluating a given binary operation.
+using binop_eval_t = uint64_t (*)(uint64_t, uint64_t);
+
+/// Class representing a single binary operation in the AST of an expression.
+class FileCheckASTBinop : public FileCheckExpressionAST {
+private:
+  /// Left operand.
+  std::unique_ptr<FileCheckExpressionAST> LeftOperand;
+
+  /// Right operand.
+  std::unique_ptr<FileCheckExpressionAST> RightOperand;
+
+  /// Pointer to function that can evaluate this binary operation.
+  binop_eval_t EvalBinop;
+
+public:
+  FileCheckASTBinop(binop_eval_t EvalBinop,
+                    std::unique_ptr<FileCheckExpressionAST> LeftOp,
+                    std::unique_ptr<FileCheckExpressionAST> RightOp)
+      : EvalBinop(EvalBinop) {
+    LeftOperand = std::move(LeftOp);
+    RightOperand = std::move(RightOp);
+  }
+
+  /// Evaluates the value of the binary operation represented by this AST,
+  /// using EvalBinop on the result of recursively evaluating the operands.
+  /// \returns the expression value or an error if an undefined numeric
+  /// variable is used in one of the operands.
+  Expected<uint64_t> eval() const;
+};
+
+class FileCheckPatternContext;
+
+/// Class representing a substitution to perform in the RegExStr string.
+class FileCheckSubstitution {
+protected:
+  /// Pointer to a class instance holding, among other things, the table with
+  /// the values of live string variables at the start of any given CHECK line.
+  /// Used for substituting string variables with the text they were defined
+  /// as. Expressions are linked to the numeric variables they use at
+  /// parse time and directly access the value of the numeric variable to
+  /// evaluate their value.
+  FileCheckPatternContext *Context;
+
+  /// The string that needs to be substituted for something else. For a
+  /// string variable this is its name, otherwise this is the whole expression.
+  StringRef FromStr;
+
+  // Index in RegExStr of where to do the substitution.
+  size_t InsertIdx;
+
+public:
+  FileCheckSubstitution(FileCheckPatternContext *Context, StringRef VarName,
+                        size_t InsertIdx)
+      : Context(Context), FromStr(VarName), InsertIdx(InsertIdx) {}
+
+  virtual ~FileCheckSubstitution() = default;
+
+  /// \returns the string to be substituted for something else.
+  StringRef getFromString() const { return FromStr; }
+
+  /// \returns the index where the substitution is to be performed in RegExStr.
+  size_t getIndex() const { return InsertIdx; }
+
+  /// \returns a string containing the result of the substitution represented
+  /// by this class instance or an error if substitution failed.
+  virtual Expected<std::string> getResult() const = 0;
+};
+
+class FileCheckStringSubstitution : public FileCheckSubstitution {
+public:
+  FileCheckStringSubstitution(FileCheckPatternContext *Context,
+                              StringRef VarName, size_t InsertIdx)
+      : FileCheckSubstitution(Context, VarName, InsertIdx) {}
+
+  /// \returns the text that the string variable in this substitution matched
+  /// when defined, or an error if the variable is undefined.
+  Expected<std::string> getResult() const override;
+};
+
+class FileCheckNumericSubstitution : public FileCheckSubstitution {
+private:
+  /// Pointer to the class representing the expression whose value is to be
+  /// substituted.
+  std::unique_ptr<FileCheckExpressionAST> ExpressionAST;
+
+public:
+  FileCheckNumericSubstitution(FileCheckPatternContext *Context, StringRef Expr,
+                               std::unique_ptr<FileCheckExpressionAST> ExprAST,
+                               size_t InsertIdx)
+      : FileCheckSubstitution(Context, Expr, InsertIdx) {
+    ExpressionAST = std::move(ExprAST);
+  }
+
+  /// \returns a string containing the result of evaluating the expression in
+  /// this substitution, or an error if evaluation failed.
+  Expected<std::string> getResult() const override;
+};
+
+//===----------------------------------------------------------------------===//
+// Pattern handling code.
+//===----------------------------------------------------------------------===//
+
+struct FileCheckDiag;
+
+/// Class holding the FileCheckPattern global state, shared by all patterns:
+/// tables holding values of variables and whether they are defined or not at
+/// any given time in the matching process.
+class FileCheckPatternContext {
+  friend class FileCheckPattern;
+
+private:
+  /// When matching a given pattern, this holds the value of all the string
+  /// variables defined in previous patterns. In a pattern, only the last
+  /// definition for a given variable is recorded in this table.
+  /// Back-references are used for uses after any the other definition.
+  StringMap<StringRef> GlobalVariableTable;
+
+  /// Map of all string variables defined so far. Used at parse time to detect
+  /// a name conflict between a numeric variable and a string variable when
+  /// the former is defined on a later line than the latter.
+  StringMap<bool> DefinedVariableTable;
+
+  /// When matching a given pattern, this holds the pointers to the classes
+  /// representing the numeric variables defined in previous patterns. When
+  /// matching a pattern all definitions for that pattern are recorded in the
+  /// NumericVariableDefs table in the FileCheckPattern instance of that
+  /// pattern.
+  StringMap<FileCheckNumericVariable *> GlobalNumericVariableTable;
+
+  /// Pointer to the class instance representing the @LINE pseudo variable for
+  /// easily updating its value.
+  FileCheckNumericVariable *LineVariable = nullptr;
+
+  /// Vector holding pointers to all parsed numeric variables. Used to
+  /// automatically free them once they are guaranteed to no longer be used.
+  std::vector<std::unique_ptr<FileCheckNumericVariable>> NumericVariables;
+
+  /// Vector holding pointers to all substitutions. Used to automatically free
+  /// them once they are guaranteed to no longer be used.
+  std::vector<std::unique_ptr<FileCheckSubstitution>> Substitutions;
+
+public:
+  /// \returns the value of string variable \p VarName or an error if no such
+  /// variable has been defined.
+  Expected<StringRef> getPatternVarValue(StringRef VarName);
+
+  /// Defines string and numeric variables from definitions given on the
+  /// command line, passed as a vector of [#]VAR=VAL strings in
+  /// \p CmdlineDefines. \returns an error list containing diagnostics against
+  /// \p SM for all definition parsing failures, if any, or Success otherwise.
+  Error defineCmdlineVariables(std::vector<std::string> &CmdlineDefines,
+                               SourceMgr &SM);
+
+  /// Create @LINE pseudo variable. Value is set when pattern are being
+  /// matched.
+  void createLineVariable();
+
+  /// Undefines local variables (variables whose name does not start with a '$'
+  /// sign), i.e. removes them from GlobalVariableTable and from
+  /// GlobalNumericVariableTable and also clears the value of numeric
+  /// variables.
+  void clearLocalVars();
+
+private:
+  /// Makes a new numeric variable and registers it for destruction when the
+  /// context is destroyed.
+  template <class... Types>
+  FileCheckNumericVariable *makeNumericVariable(Types... args);
+
+  /// Makes a new string substitution and registers it for destruction when the
+  /// context is destroyed.
+  FileCheckSubstitution *makeStringSubstitution(StringRef VarName,
+                                                size_t InsertIdx);
+
+  /// Makes a new numeric substitution and registers it for destruction when
+  /// the context is destroyed.
+  FileCheckSubstitution *
+  makeNumericSubstitution(StringRef ExpressionStr,
+                          std::unique_ptr<FileCheckExpressionAST> ExpressionAST,
+                          size_t InsertIdx);
+};
+
+/// Class to represent an error holding a diagnostic with location information
+/// used when printing it.
+class FileCheckErrorDiagnostic : public ErrorInfo<FileCheckErrorDiagnostic> {
+private:
+  SMDiagnostic Diagnostic;
+
+public:
+  static char ID;
+
+  FileCheckErrorDiagnostic(SMDiagnostic &&Diag) : Diagnostic(Diag) {}
+
+  std::error_code convertToErrorCode() const override {
+    return inconvertibleErrorCode();
+  }
+
+  /// Print diagnostic associated with this error when printing the error.
+  void log(raw_ostream &OS) const override { Diagnostic.print(nullptr, OS); }
+
+  static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg) {
+    return make_error<FileCheckErrorDiagnostic>(
+        SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg));
+  }
+
+  static Error get(const SourceMgr &SM, StringRef Buffer, const Twine &ErrMsg) {
+    return get(SM, SMLoc::getFromPointer(Buffer.data()), ErrMsg);
+  }
+};
+
+class FileCheckNotFoundError : public ErrorInfo<FileCheckNotFoundError> {
+public:
+  static char ID;
+
+  std::error_code convertToErrorCode() const override {
+    return inconvertibleErrorCode();
+  }
+
+  /// Print diagnostic associated with this error when printing the error.
+  void log(raw_ostream &OS) const override {
+    OS << "String not found in input";
+  }
+};
+
+class FileCheckPattern {
+  SMLoc PatternLoc;
+
+  /// A fixed string to match as the pattern or empty if this pattern requires
+  /// a regex match.
+  StringRef FixedStr;
+
+  /// A regex string to match as the pattern or empty if this pattern requires
+  /// a fixed string to match.
+  std::string RegExStr;
+
+  /// Entries in this vector represent a substitution of a string variable or
+  /// an expression in the RegExStr regex at match time. For example, in the
+  /// case of a CHECK directive with the pattern "foo[[bar]]baz[[#N+1]]",
+  /// RegExStr will contain "foobaz" and we'll get two entries in this vector
+  /// that tells us to insert the value of string variable "bar" at offset 3
+  /// and the value of expression "N+1" at offset 6.
+  std::vector<FileCheckSubstitution *> Substitutions;
+
+  /// Maps names of string variables defined in a pattern to the number of
+  /// their parenthesis group in RegExStr capturing their last definition.
+  ///
+  /// E.g. for the pattern "foo[[bar:.*]]baz([[bar]][[QUUX]][[bar:.*]])",
+  /// RegExStr will be "foo(.*)baz(\1<quux value>(.*))" where <quux value> is
+  /// the value captured for QUUX on the earlier line where it was defined, and
+  /// VariableDefs will map "bar" to the third parenthesis group which captures
+  /// the second definition of "bar".
+  ///
+  /// Note: uses std::map rather than StringMap to be able to get the key when
+  /// iterating over values.
+  std::map<StringRef, unsigned> VariableDefs;
+
+  /// Structure representing the definition of a numeric variable in a pattern.
+  /// It holds the pointer to the class representing the numeric variable whose
+  /// value is being defined and the number of the parenthesis group in
+  /// RegExStr to capture that value.
+  struct FileCheckNumericVariableMatch {
+    /// Pointer to class representing the numeric variable whose value is being
+    /// defined.
+    FileCheckNumericVariable *DefinedNumericVariable;
+
+    /// Number of the parenthesis group in RegExStr that captures the value of
+    /// this numeric variable definition.
+    unsigned CaptureParenGroup;
+  };
+
+  /// Holds the number of the parenthesis group in RegExStr and pointer to the
+  /// corresponding FileCheckNumericVariable class instance of all numeric
+  /// variable definitions. Used to set the matched value of all those
+  /// variables.
+  StringMap<FileCheckNumericVariableMatch> NumericVariableDefs;
+
+  /// Pointer to a class instance holding the global state shared by all
+  /// patterns:
+  /// - separate tables with the values of live string and numeric variables
+  ///   respectively at the start of any given CHECK line;
+  /// - table holding whether a string variable has been defined at any given
+  ///   point during the parsing phase.
+  FileCheckPatternContext *Context;
+
+  Check::FileCheckType CheckTy;
+
+  /// Line number for this CHECK pattern or None if it is an implicit pattern.
+  /// Used to determine whether a variable definition is made on an earlier
+  /// line to the one with this CHECK.
+  Optional<size_t> LineNumber;
+
+public:
+  FileCheckPattern(Check::FileCheckType Ty, FileCheckPatternContext *Context,
+                   Optional<size_t> Line = None)
+      : Context(Context), CheckTy(Ty), LineNumber(Line) {}
+
+  /// \returns the location in source code.
+  SMLoc getLoc() const { return PatternLoc; }
+
+  /// \returns the pointer to the global state for all patterns in this
+  /// FileCheck instance.
+  FileCheckPatternContext *getContext() const { return Context; }
+
+  /// \returns whether \p C is a valid first character for a variable name.
+  static bool isValidVarNameStart(char C);
+
+  /// Parsing information about a variable.
+  struct VariableProperties {
+    StringRef Name;
+    bool IsPseudo;
+  };
+
+  /// Parses the string at the start of \p Str for a variable name. \returns
+  /// a VariableProperties structure holding the variable name and whether it
+  /// is the name of a pseudo variable, or an error holding a diagnostic
+  /// against \p SM if parsing fail. If parsing was successful, also strips
+  /// \p Str from the variable name.
+  static Expected<VariableProperties> parseVariable(StringRef &Str,
+                                                    const SourceMgr &SM);
+  /// Parses \p Expr for a numeric substitution block at line \p LineNumber,
+  /// or before input is parsed if \p LineNumber is None. Parameter
+  /// \p IsLegacyLineExpr indicates whether \p Expr should be a legacy @LINE
+  /// expression and \p Context points to the class instance holding the live
+  /// string and numeric variables. \returns a pointer to the class instance
+  /// representing the AST of the expression whose value must be substitued, or
+  /// an error holding a diagnostic against \p SM if parsing fails. If
+  /// substitution was successful, sets \p DefinedNumericVariable to point to
+  /// the class representing the numeric variable defined in this numeric
+  /// substitution block, or None if this block does not define any variable.
+  static Expected<std::unique_ptr<FileCheckExpressionAST>>
+  parseNumericSubstitutionBlock(
+      StringRef Expr,
+      Optional<FileCheckNumericVariable *> &DefinedNumericVariable,
+      bool IsLegacyLineExpr, Optional<size_t> LineNumber,
+      FileCheckPatternContext *Context, const SourceMgr &SM);
+  /// Parses the pattern in \p PatternStr and initializes this FileCheckPattern
+  /// instance accordingly.
+  ///
+  /// \p Prefix provides which prefix is being matched, \p Req describes the
+  /// global options that influence the parsing such as whitespace
+  /// canonicalization, \p SM provides the SourceMgr used for error reports.
+  /// \returns true in case of an error, false otherwise.
+  bool parsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM,
+                    const FileCheckRequest &Req);
+  /// Matches the pattern string against the input buffer \p Buffer
+  ///
+  /// \returns the position that is matched or an error indicating why matching
+  /// failed. If there is a match, updates \p MatchLen with the size of the
+  /// matched string.
+  ///
+  /// The GlobalVariableTable StringMap in the FileCheckPatternContext class
+  /// instance provides the current values of FileCheck string variables and
+  /// is updated if this match defines new values. Likewise, the
+  /// GlobalNumericVariableTable StringMap in the same class provides the
+  /// current values of FileCheck numeric variables and is updated if this
+  /// match defines new numeric values.
+  Expected<size_t> match(StringRef Buffer, size_t &MatchLen,
+                         const SourceMgr &SM) const;
+  /// Prints the value of successful substitutions or the name of the undefined
+  /// string or numeric variables preventing a successful substitution.
+  void printSubstitutions(const SourceMgr &SM, StringRef Buffer,
+                          SMRange MatchRange = None) const;
+  void printFuzzyMatch(const SourceMgr &SM, StringRef Buffer,
+                       std::vector<FileCheckDiag> *Diags) const;
+
+  bool hasVariable() const {
+    return !(Substitutions.empty() && VariableDefs.empty());
+  }
+
+  Check::FileCheckType getCheckTy() const { return CheckTy; }
+
+  int getCount() const { return CheckTy.getCount(); }
+
+private:
+  bool AddRegExToRegEx(StringRef RS, unsigned &CurParen, SourceMgr &SM);
+  void AddBackrefToRegEx(unsigned BackrefNum);
+  /// Computes an arbitrary estimate for the quality of matching this pattern
+  /// at the start of \p Buffer; a distance of zero should correspond to a
+  /// perfect match.
+  unsigned computeMatchDistance(StringRef Buffer) const;
+  /// Finds the closing sequence of a regex variable usage or definition.
+  ///
+  /// \p Str has to point in the beginning of the definition (right after the
+  /// opening sequence). \p SM holds the SourceMgr used for error repporting.
+  ///  \returns the offset of the closing sequence within Str, or npos if it
+  /// was not found.
+  size_t FindRegexVarEnd(StringRef Str, SourceMgr &SM);
+
+  /// Parses \p Expr for the name of a numeric variable to be defined at line
+  /// \p LineNumber, or before input is parsed if \p LineNumber is None.
+  /// \returns a pointer to the class instance representing that variable,
+  /// creating it if needed, or an error holding a diagnostic against \p SM
+  /// should defining such a variable be invalid.
+  static Expected<FileCheckNumericVariable *> parseNumericVariableDefinition(
+      StringRef &Expr, FileCheckPatternContext *Context,
+      Optional<size_t> LineNumber, const SourceMgr &SM);
+  /// Parses \p Name as a (pseudo if \p IsPseudo is true) numeric variable use
+  /// at line \p LineNumber, or before input is parsed if \p LineNumber is
+  /// None. Parameter \p Context points to the class instance holding the live
+  /// string and numeric variables. \returns the pointer to the class instance
+  /// representing that variable if successful, or an error holding a
+  /// diagnostic against \p SM otherwise.
+  static Expected<std::unique_ptr<FileCheckNumericVariableUse>>
+  parseNumericVariableUse(StringRef Name, bool IsPseudo,
+                          Optional<size_t> LineNumber,
+                          FileCheckPatternContext *Context,
+                          const SourceMgr &SM);
+  enum class AllowedOperand { LineVar, Literal, Any };
+  /// Parses \p Expr for use of a numeric operand at line \p LineNumber, or
+  /// before input is parsed if \p LineNumber is None. Accepts both literal
+  /// values and numeric variables, depending on the value of \p AO. Parameter
+  /// \p Context points to the class instance holding the live string and
+  /// numeric variables. \returns the class representing that operand in the
+  /// AST of the expression or an error holding a diagnostic against \p SM
+  /// otherwise.
+  static Expected<std::unique_ptr<FileCheckExpressionAST>>
+  parseNumericOperand(StringRef &Expr, AllowedOperand AO,
+                      Optional<size_t> LineNumber,
+                      FileCheckPatternContext *Context, const SourceMgr &SM);
+  /// Parses \p Expr for a binary operation at line \p LineNumber, or before
+  /// input is parsed if \p LineNumber is None. The left operand of this binary
+  /// operation is given in \p LeftOp and \p IsLegacyLineExpr indicates whether
+  /// we are parsing a legacy @LINE expression. Parameter \p Context points to
+  /// the class instance holding the live string and numeric variables.
+  /// \returns the class representing the binary operation in the AST of the
+  /// expression, or an error holding a diagnostic against \p SM otherwise.
+  static Expected<std::unique_ptr<FileCheckExpressionAST>>
+  parseBinop(StringRef &Expr, std::unique_ptr<FileCheckExpressionAST> LeftOp,
+             bool IsLegacyLineExpr, Optional<size_t> LineNumber,
+             FileCheckPatternContext *Context, const SourceMgr &SM);
+};
+
+//===----------------------------------------------------------------------===//
+// Check Strings.
+//===----------------------------------------------------------------------===//
+
+/// A check that we found in the input file.
+struct FileCheckString {
+  /// The pattern to match.
+  FileCheckPattern Pat;
+
+  /// Which prefix name this check matched.
+  StringRef Prefix;
+
+  /// The location in the match file that the check string was specified.
+  SMLoc Loc;
+
+  /// All of the strings that are disallowed from occurring between this match
+  /// string and the previous one (or start of file).
+  std::vector<FileCheckPattern> DagNotStrings;
+
+  FileCheckString(const FileCheckPattern &P, StringRef S, SMLoc L)
+      : Pat(P), Prefix(S), Loc(L) {}
+
+  /// Matches check string and its "not strings" and/or "dag strings".
+  size_t Check(const SourceMgr &SM, StringRef Buffer, bool IsLabelScanMode,
+               size_t &MatchLen, FileCheckRequest &Req,
+               std::vector<FileCheckDiag> *Diags) const;
+
+  /// Verifies that there is a single line in the given \p Buffer. Errors are
+  /// reported against \p SM.
+  bool CheckNext(const SourceMgr &SM, StringRef Buffer) const;
+  /// Verifies that there is no newline in the given \p Buffer. Errors are
+  /// reported against \p SM.
+  bool CheckSame(const SourceMgr &SM, StringRef Buffer) const;
+  /// Verifies that none of the strings in \p NotStrings are found in the given
+  /// \p Buffer. Errors are reported against \p SM and diagnostics recorded in
+  /// \p Diags according to the verbosity level set in \p Req.
+  bool CheckNot(const SourceMgr &SM, StringRef Buffer,
+                const std::vector<const FileCheckPattern *> &NotStrings,
+                const FileCheckRequest &Req,
+                std::vector<FileCheckDiag> *Diags) const;
+  /// Matches "dag strings" and their mixed "not strings".
+  size_t CheckDag(const SourceMgr &SM, StringRef Buffer,
+                  std::vector<const FileCheckPattern *> &NotStrings,
+                  const FileCheckRequest &Req,
+                  std::vector<FileCheckDiag> *Diags) const;
+};
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/lib/Support/FileUtilities.cpp b/llvm/lib/Support/FileUtilities.cpp
index b933692d92e7c..d11fbb54dc0d8 100644
--- a/llvm/lib/Support/FileUtilities.cpp
+++ b/llvm/lib/Support/FileUtilities.cpp
@@ -12,7 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FileUtilities.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
@@ -266,36 +268,65 @@ int llvm::DiffFilesWithTolerance(StringRef NameA,
   return CompareFailed;
 }
 
-Error llvm::writeFileAtomically(StringRef TempPathModel, StringRef FinalPath,
-                                StringRef Buffer) {
+void llvm::AtomicFileWriteError::log(raw_ostream &OS) const {
+  OS << "atomic_write_error: ";
+  switch (Error) {
+  case atomic_write_error::failed_to_create_uniq_file:
+    OS << "failed_to_create_uniq_file";
+    return;
+  case atomic_write_error::output_stream_error:
+    OS << "output_stream_error";
+    return;
+  case atomic_write_error::failed_to_rename_temp_file:
+    OS << "failed_to_rename_temp_file";
+    return;
+  }
+  llvm_unreachable("unknown atomic_write_error value in "
+                   "failed_to_rename_temp_file::log()");
+}
+
+llvm::Error llvm::writeFileAtomically(StringRef TempPathModel,
+                                      StringRef FinalPath, StringRef Buffer) {
+  return writeFileAtomically(TempPathModel, FinalPath,
+                             [&Buffer](llvm::raw_ostream &OS) {
+                               OS.write(Buffer.data(), Buffer.size());
+                               return llvm::Error::success();
+                             });
+}
+
+llvm::Error llvm::writeFileAtomically(
+    StringRef TempPathModel, StringRef FinalPath,
+    std::function<llvm::Error(llvm::raw_ostream &)> Writer) {
   SmallString<128> GeneratedUniqPath;
   int TempFD;
-  if (const std::error_code Error = sys::fs::createUniqueFile(
-          TempPathModel.str(), TempFD, GeneratedUniqPath)) {
-    return createStringError(
-        Error, "failed to create temporary file with model \"%s\"",
-        TempPathModel.str().c_str());
+  if (sys::fs::createUniqueFile(TempPathModel.str(), TempFD,
+                                GeneratedUniqPath)) {
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::failed_to_create_uniq_file);
   }
+  llvm::FileRemover RemoveTmpFileOnFail(GeneratedUniqPath);
 
   raw_fd_ostream OS(TempFD, /*shouldClose=*/true);
-  OS.write(Buffer.data(), Buffer.size());
-  OS.close();
-  TempFD = -1;
+  if (llvm::Error Err = Writer(OS)) {
+    return Err;
+  }
 
+  OS.close();
   if (OS.has_error()) {
-    const std::error_code Error = OS.error();
     OS.clear_error();
-    return createStringError(Error, "failed to write to \"%s\"",
-                             GeneratedUniqPath.c_str());
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::output_stream_error);
   }
 
   if (const std::error_code Error =
           sys::fs::rename(/*from=*/GeneratedUniqPath.c_str(),
                           /*to=*/FinalPath.str().c_str())) {
-    return createStringError(Error, "failed to rename file \"%s\" to \"%s\"",
-                             GeneratedUniqPath.c_str(),
-                             FinalPath.str().c_str());
+    return llvm::make_error<AtomicFileWriteError>(
+        atomic_write_error::failed_to_rename_temp_file);
   }
 
+  RemoveTmpFileOnFail.releaseFile();
   return Error::success();
 }
+
+char llvm::AtomicFileWriteError::ID;
diff --git a/llvm/lib/Support/Host.cpp b/llvm/lib/Support/Host.cpp
index 0d6fab50424c4..2a473a1994c2b 100644
--- a/llvm/lib/Support/Host.cpp
+++ b/llvm/lib/Support/Host.cpp
@@ -316,7 +316,7 @@ StringRef sys::detail::getHostCPUNameForS390x(StringRef ProcCpuinfoContent) {
         unsigned int Id;
         if (!Lines[I].drop_front(Pos).getAsInteger(10, Id)) {
           if (Id >= 8561 && HaveVectorSupport)
-            return "arch13";
+            return "z15";
           if (Id >= 3906 && HaveVectorSupport)
             return "z14";
           if (Id >= 2964 && HaveVectorSupport)
@@ -1512,6 +1512,17 @@ bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
 
   return true;
 }
+#elif defined(_WIN32) && (defined(__aarch64__) || defined(_M_ARM64))
+bool sys::getHostCPUFeatures(StringMap<bool> &Features) {
+  if (IsProcessorFeaturePresent(PF_ARM_NEON_INSTRUCTIONS_AVAILABLE))
+    Features["neon"] = true;
+  if (IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE))
+    Features["crc"] = true;
+  if (IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE))
+    Features["crypto"] = true;
+
+  return true;
+}
 #else
 bool sys::getHostCPUFeatures(StringMap<bool> &Features) { return false; }
 #endif
diff --git a/llvm/lib/Support/Regex.cpp b/llvm/lib/Support/Regex.cpp
index 4c1b070380244..8da345d4f1404 100644
--- a/llvm/lib/Support/Regex.cpp
+++ b/llvm/lib/Support/Regex.cpp
@@ -52,14 +52,24 @@ Regex::~Regex() {
   }
 }
 
-bool Regex::isValid(std::string &Error) const {
-  if (!error)
-    return true;
+namespace {
 
+/// Utility to convert a regex error code into a human-readable string.
+void RegexErrorToString(int error, struct llvm_regex *preg,
+                        std::string &Error) {
   size_t len = llvm_regerror(error, preg, nullptr, 0);
 
   Error.resize(len - 1);
   llvm_regerror(error, preg, &Error[0], len);
+}
+
+} // namespace
+
+bool Regex::isValid(std::string &Error) const {
+  if (!error)
+    return true;
+
+  RegexErrorToString(error, preg, Error);
   return false;
 }
 
@@ -69,8 +79,14 @@ unsigned Regex::getNumMatches() const {
   return preg->re_nsub;
 }
 
-bool Regex::match(StringRef String, SmallVectorImpl<StringRef> *Matches){
-  if (error)
+bool Regex::match(StringRef String, SmallVectorImpl<StringRef> *Matches,
+                  std::string *Error) const {
+  // Reset error, if given.
+  if (Error && !Error->empty())
+    *Error = "";
+
+  // Check if the regex itself didn't successfully compile.
+  if (Error ? !isValid(*Error) : !isValid())
     return false;
 
   unsigned nmatch = Matches ? preg->re_nsub+1 : 0;
@@ -83,11 +99,13 @@ bool Regex::match(StringRef String, SmallVectorImpl<StringRef> *Matches){
 
   int rc = llvm_regexec(preg, String.data(), nmatch, pm.data(), REG_STARTEND);
 
+  // Failure to match is not an error, it's just a normal return value.
+  // Any other error code is considered abnormal, and is logged in the Error.
   if (rc == REG_NOMATCH)
     return false;
   if (rc != 0) {
-    // regexec can fail due to invalid pattern or running out of memory.
-    error = rc;
+    if (Error)
+      RegexErrorToString(error, preg, *Error);
     return false;
   }
 
@@ -112,14 +130,11 @@ bool Regex::match(StringRef String, SmallVectorImpl<StringRef> *Matches){
 }
 
 std::string Regex::sub(StringRef Repl, StringRef String,
-                       std::string *Error) {
+                       std::string *Error) const {
   SmallVector<StringRef, 8> Matches;
 
-  // Reset error, if given.
-  if (Error && !Error->empty()) *Error = "";
-
   // Return the input if there was no match.
-  if (!match(String, &Matches))
+  if (!match(String, &Matches, Error))
     return String;
 
   // Otherwise splice in the replacement string, starting with the prefix before
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index e4197c18cc99c..5aa420cfc721e 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -1082,20 +1082,19 @@ StringRef RedirectingFileSystem::getExternalContentsPrefixDir() const {
   return ExternalContentsPrefixDir;
 }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const {
+void RedirectingFileSystem::dump(raw_ostream &OS) const {
   for (const auto &Root : Roots)
-    dumpEntry(Root.get());
+    dumpEntry(OS, Root.get());
 }
 
-LLVM_DUMP_METHOD void
-RedirectingFileSystem::dumpEntry(RedirectingFileSystem::Entry *E,
-                                 int NumSpaces) const {
+void RedirectingFileSystem::dumpEntry(raw_ostream &OS,
+                                      RedirectingFileSystem::Entry *E,
+                                      int NumSpaces) const {
   StringRef Name = E->getName();
   for (int i = 0, e = NumSpaces; i < e; ++i)
-    dbgs() << " ";
-  dbgs() << "'" << Name.str().c_str() << "'"
-         << "\n";
+    OS << " ";
+  OS << "'" << Name.str().c_str() << "'"
+     << "\n";
 
   if (E->getKind() == RedirectingFileSystem::EK_Directory) {
     auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(E);
@@ -1103,9 +1102,12 @@ RedirectingFileSystem::dumpEntry(RedirectingFileSystem::Entry *E,
 
     for (std::unique_ptr<Entry> &SubEntry :
          llvm::make_range(DE->contents_begin(), DE->contents_end()))
-      dumpEntry(SubEntry.get(), NumSpaces + 2);
+      dumpEntry(OS, SubEntry.get(), NumSpaces + 2);
   }
 }
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+LLVM_DUMP_METHOD void RedirectingFileSystem::dump() const { dump(dbgs()); }
 #endif
 
 /// A helper class to hold the common YAML parsing state.
@@ -1221,7 +1223,7 @@ class llvm::vfs::RedirectingFileSystemParser {
     }
 
     auto *DE =
-        dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(ParentEntry);
+        cast<RedirectingFileSystem::RedirectingDirectoryEntry>(ParentEntry);
     DE->addContent(std::move(E));
     return DE->getLastContent();
   }
@@ -1232,9 +1234,7 @@ class llvm::vfs::RedirectingFileSystemParser {
     StringRef Name = SrcE->getName();
     switch (SrcE->getKind()) {
     case RedirectingFileSystem::EK_Directory: {
-      auto *DE =
-          dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
-      assert(DE && "Must be a directory");
+      auto *DE = cast<RedirectingFileSystem::RedirectingDirectoryEntry>(SrcE);
       // Empty directories could be present in the YAML as a way to
       // describe a file for a current directory after some of its subdir
       // is parsed. This only leads to redundant walks, ignore it.
@@ -1246,11 +1246,10 @@ class llvm::vfs::RedirectingFileSystemParser {
       break;
     }
     case RedirectingFileSystem::EK_File: {
-      auto *FE = dyn_cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
-      assert(FE && "Must be a file");
       assert(NewParentE && "Parent entry must exist");
-      auto *DE = dyn_cast<RedirectingFileSystem::RedirectingDirectoryEntry>(
-          NewParentE);
+      auto *FE = cast<RedirectingFileSystem::RedirectingFileEntry>(SrcE);
+      auto *DE =
+          cast<RedirectingFileSystem::RedirectingDirectoryEntry>(NewParentE);
       DE->addContent(
           std::make_unique<RedirectingFileSystem::RedirectingFileEntry>(
               Name, FE->getExternalContentsPath(), FE->getUseName()));
diff --git a/llvm/lib/TableGen/Main.cpp b/llvm/lib/TableGen/Main.cpp
index 34195c31a8b84..bf97e37f324aa 100644
--- a/llvm/lib/TableGen/Main.cpp
+++ b/llvm/lib/TableGen/Main.cpp
@@ -99,39 +99,23 @@ int llvm::TableGenMain(char *argv0, TableGenMainFn *MainFn) {
   if (Parser.ParseFile())
     return 1;
 
-  // Write output to memory.
-  std::string OutString;
-  raw_string_ostream Out(OutString);
-  if (MainFn(Out, Records))
-    return 1;
-
-  // Always write the depfile, even if the main output hasn't changed.
-  // If it's missing, Ninja considers the output dirty.  If this was below
-  // the early exit below and someone deleted the .inc.d file but not the .inc
-  // file, tablegen would never write the depfile.
+  std::error_code EC;
+  ToolOutputFile Out(OutputFilename, EC, sys::fs::OF_None);
+  if (EC)
+    return reportError(argv0, "error opening " + OutputFilename + ":" +
+                                  EC.message() + "\n");
   if (!DependFilename.empty()) {
     if (int Ret = createDependencyFile(Parser, argv0))
       return Ret;
   }
 
-  // Only updates the real output file if there are any differences.
-  // This prevents recompilation of all the files depending on it if there
-  // aren't any.
-  if (auto ExistingOrErr = MemoryBuffer::getFile(OutputFilename))
-    if (std::move(ExistingOrErr.get())->getBuffer() == Out.str())
-      return 0;
-
-  std::error_code EC;
-  ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_None);
-  if (EC)
-    return reportError(argv0, "error opening " + OutputFilename + ":" +
-                                  EC.message() + "\n");
-  OutFile.os() << Out.str();
+  if (MainFn(Out.os(), Records))
+    return 1;
 
   if (ErrorsPrinted > 0)
     return reportError(argv0, Twine(ErrorsPrinted) + " errors.\n");
 
   // Declare success.
-  OutFile.keep();
+  Out.keep();
   return 0;
 }
diff --git a/llvm/lib/TableGen/TGParser.cpp b/llvm/lib/TableGen/TGParser.cpp
index a076966aff0cb..c373e2899a5d8 100644
--- a/llvm/lib/TableGen/TGParser.cpp
+++ b/llvm/lib/TableGen/TGParser.cpp
@@ -1147,9 +1147,9 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
       if (!InitList.back()) return nullptr;
 
       // All BinOps require their arguments to be of compatible types.
-      TypedInit *TI = dyn_cast<TypedInit>(InitList.back());
+      RecTy *ListType = cast<TypedInit>(InitList.back())->getType();
       if (!ArgType) {
-        ArgType = TI->getType();
+        ArgType = ListType;
 
         switch (Code) {
         case BinOpInit::LISTCONCAT:
@@ -1198,11 +1198,11 @@ Init *TGParser::ParseOperation(Record *CurRec, RecTy *ItemType) {
         default: llvm_unreachable("other ops have fixed argument types");
         }
       } else {
-        RecTy *Resolved = resolveTypes(ArgType, TI->getType());
+        RecTy *Resolved = resolveTypes(ArgType, ListType);
         if (!Resolved) {
           Error(InitLoc, Twine("expected value of type '") +
-                         ArgType->getAsString() + "', got '" +
-                         TI->getType()->getAsString() + "'");
+                             ArgType->getAsString() + "', got '" +
+                             ListType->getAsString() + "'");
           return nullptr;
         }
         if (Code != BinOpInit::ADD && Code != BinOpInit::AND &&
diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index c55fc24105f72..51bf35d4a161f 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -120,6 +120,7 @@ def FeatureSVE2BitPerm : SubtargetFeature<"sve2-bitperm", "HasSVE2BitPerm", "tru
 
 def FeatureZCRegMove : SubtargetFeature<"zcm", "HasZeroCycleRegMove", "true",
                                         "Has zero-cycle register moves">;
+
 def FeatureZCZeroingGP : SubtargetFeature<"zcz-gp", "HasZeroCycleZeroingGP", "true",
                                         "Has zero-cycle zeroing instructions for generic registers">;
 
@@ -405,6 +406,7 @@ include "AArch64Schedule.td"
 include "AArch64InstrInfo.td"
 include "AArch64SchedPredicates.td"
 include "AArch64SchedPredExynos.td"
+include "AArch64Combine.td"
 
 def AArch64InstrInfo : InstrInfo;
 
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 21048998edfa9..7ea7915c2ca6a 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -99,7 +99,8 @@ class AArch64AsmPrinter : public AsmPrinter {
   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
 
-  std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols;
+  typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
+  std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
   void EmitHwasanMemaccessSymbols(Module &M);
 
@@ -237,8 +238,11 @@ void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
 
 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
   Register Reg = MI.getOperand(0).getReg();
+  bool IsShort =
+      MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
   uint32_t AccessInfo = MI.getOperand(1).getImm();
-  MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}];
+  MCSymbol *&Sym =
+      HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
   if (!Sym) {
     // FIXME: Make this work on non-ELF.
     if (!TM.getTargetTriple().isOSBinFormatELF())
@@ -246,6 +250,8 @@ void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 
     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
                           utostr(AccessInfo);
+    if (IsShort)
+      SymName += "_short";
     Sym = OutContext.getOrCreateSymbol(SymName);
   }
 
@@ -263,15 +269,22 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
   std::unique_ptr<MCSubtargetInfo> STI(
       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
 
-  MCSymbol *HwasanTagMismatchSym =
+  MCSymbol *HwasanTagMismatchV1Sym =
       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
+  MCSymbol *HwasanTagMismatchV2Sym =
+      OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
 
-  const MCSymbolRefExpr *HwasanTagMismatchRef =
-      MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext);
+  const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
+      MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
+  const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
+      MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
 
   for (auto &P : HwasanMemaccessSymbols) {
-    unsigned Reg = P.first.first;
-    uint32_t AccessInfo = P.first.second;
+    unsigned Reg = std::get<0>(P.first);
+    bool IsShort = std::get<1>(P.first);
+    uint32_t AccessInfo = std::get<2>(P.first);
+    const MCSymbolRefExpr *HwasanTagMismatchRef =
+        IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
     MCSymbol *Sym = P.second;
 
     OutStreamer->SwitchSection(OutContext.getELFSection(
@@ -304,82 +317,86 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
             .addReg(Reg)
             .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
         *STI);
-    MCSymbol *HandlePartialSym = OutContext.createTempSymbol();
+    MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
     OutStreamer->EmitInstruction(
         MCInstBuilder(AArch64::Bcc)
             .addImm(AArch64CC::NE)
-            .addExpr(MCSymbolRefExpr::create(HandlePartialSym, OutContext)),
+            .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
+                                             OutContext)),
         *STI);
     MCSymbol *ReturnSym = OutContext.createTempSymbol();
     OutStreamer->EmitLabel(ReturnSym);
     OutStreamer->EmitInstruction(
         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
+    OutStreamer->EmitLabel(HandleMismatchOrPartialSym);
 
-    OutStreamer->EmitLabel(HandlePartialSym);
-    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
-                                     .addReg(AArch64::WZR)
-                                     .addReg(AArch64::W16)
-                                     .addImm(15)
-                                     .addImm(0),
-                                 *STI);
-    MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::Bcc)
-            .addImm(AArch64CC::HI)
-            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
-        *STI);
-
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::ANDXri)
-            .addReg(AArch64::X17)
-            .addReg(Reg)
-            .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
-        *STI);
-    unsigned Size = 1 << (AccessInfo & 0xf);
-    if (Size != 1)
-      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
-                                       .addReg(AArch64::X17)
-                                       .addReg(AArch64::X17)
-                                       .addImm(Size - 1)
+    if (IsShort) {
+      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWri)
+                                       .addReg(AArch64::WZR)
+                                       .addReg(AArch64::W16)
+                                       .addImm(15)
                                        .addImm(0),
                                    *STI);
-    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
-                                     .addReg(AArch64::WZR)
-                                     .addReg(AArch64::W16)
-                                     .addReg(AArch64::W17)
-                                     .addImm(0),
-                                 *STI);
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::Bcc)
-            .addImm(AArch64CC::LS)
-            .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
-        *STI);
-
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::ORRXri)
-            .addReg(AArch64::X16)
-            .addReg(Reg)
-            .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
-        *STI);
-    OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
-                                     .addReg(AArch64::W16)
-                                     .addReg(AArch64::X16)
-                                     .addImm(0),
-                                 *STI);
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::SUBSXrs)
-            .addReg(AArch64::XZR)
-            .addReg(AArch64::X16)
-            .addReg(Reg)
-            .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
-        *STI);
-    OutStreamer->EmitInstruction(
-        MCInstBuilder(AArch64::Bcc)
-            .addImm(AArch64CC::EQ)
-            .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
-        *STI);
+      MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::Bcc)
+              .addImm(AArch64CC::HI)
+              .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
+          *STI);
+
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::ANDXri)
+              .addReg(AArch64::X17)
+              .addReg(Reg)
+              .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
+          *STI);
+      unsigned Size = 1 << (AccessInfo & 0xf);
+      if (Size != 1)
+        OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ADDXri)
+                                         .addReg(AArch64::X17)
+                                         .addReg(AArch64::X17)
+                                         .addImm(Size - 1)
+                                         .addImm(0),
+                                     *STI);
+      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
+                                       .addReg(AArch64::WZR)
+                                       .addReg(AArch64::W16)
+                                       .addReg(AArch64::W17)
+                                       .addImm(0),
+                                   *STI);
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::Bcc)
+              .addImm(AArch64CC::LS)
+              .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
+          *STI);
+
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::ORRXri)
+              .addReg(AArch64::X16)
+              .addReg(Reg)
+              .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
+          *STI);
+      OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBui)
+                                       .addReg(AArch64::W16)
+                                       .addReg(AArch64::X16)
+                                       .addImm(0),
+                                   *STI);
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::SUBSXrs)
+              .addReg(AArch64::XZR)
+              .addReg(AArch64::X16)
+              .addReg(Reg)
+              .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
+          *STI);
+      OutStreamer->EmitInstruction(
+          MCInstBuilder(AArch64::Bcc)
+              .addImm(AArch64CC::EQ)
+              .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
+          *STI);
+
+      OutStreamer->EmitLabel(HandleMismatchSym);
+    }
 
-    OutStreamer->EmitLabel(HandleMismatchSym);
     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::STPXpre)
                                      .addReg(AArch64::SP)
                                      .addReg(AArch64::X0)
@@ -414,16 +431,16 @@ void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
         MCInstBuilder(AArch64::ADRP)
             .addReg(AArch64::X16)
             .addExpr(AArch64MCExpr::create(
-                HwasanTagMismatchRef,
-                AArch64MCExpr::VariantKind::VK_GOT_PAGE, OutContext)),
+                HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
+                OutContext)),
         *STI);
     OutStreamer->EmitInstruction(
         MCInstBuilder(AArch64::LDRXui)
             .addReg(AArch64::X16)
             .addReg(AArch64::X16)
             .addExpr(AArch64MCExpr::create(
-                HwasanTagMismatchRef,
-                AArch64MCExpr::VariantKind::VK_GOT_LO12, OutContext)),
+                HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
+                OutContext)),
         *STI);
     OutStreamer->EmitInstruction(
         MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
@@ -618,6 +635,8 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
     const TargetRegisterClass *RegClass;
     if (AArch64::ZPRRegClass.contains(Reg)) {
       RegClass = &AArch64::ZPRRegClass;
+    } else if (AArch64::PPRRegClass.contains(Reg)) {
+      RegClass = &AArch64::PPRRegClass;
     } else {
       RegClass = &AArch64::FPR128RegClass;
       AltName = AArch64::vreg;
@@ -692,7 +711,7 @@ void AArch64AsmPrinter::EmitJumpTableInfo() {
     if (JTBBs.empty()) continue;
 
     unsigned Size = AFI->getJumpTableEntrySize(JTI);
-    EmitAlignment(llvm::Align(Size));
+    EmitAlignment(Align(Size));
     OutStreamer->EmitLabel(GetJTISymbol(JTI));
 
     for (auto *JTBB : JTBBs)
@@ -1094,6 +1113,7 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
 
   case AArch64::HWASAN_CHECK_MEMACCESS:
+  case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
     LowerHWASAN_CHECK_MEMACCESS(*MI);
     return;
 
diff --git a/llvm/lib/Target/AArch64/AArch64CallLowering.cpp b/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
index 1e3d6e5483f59..45f07fc2ae835 100644
--- a/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallLowering.cpp
@@ -130,9 +130,11 @@ struct CallReturnHandler : public IncomingArgHandler {
 struct OutgoingArgHandler : public CallLowering::ValueHandler {
   OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
                      MachineInstrBuilder MIB, CCAssignFn *AssignFn,
-                     CCAssignFn *AssignFnVarArg, bool IsTailCall = false)
+                     CCAssignFn *AssignFnVarArg, bool IsTailCall = false,
+                     int FPDiff = 0)
       : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB),
-        AssignFnVarArg(AssignFnVarArg), IsTailCall(IsTailCall), StackSize(0) {}
+        AssignFnVarArg(AssignFnVarArg), IsTailCall(IsTailCall), FPDiff(FPDiff),
+        StackSize(0) {}
 
   Register getStackAddress(uint64_t Size, int64_t Offset,
                            MachinePointerInfo &MPO) override {
@@ -141,8 +143,7 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler {
     LLT s64 = LLT::scalar(64);
 
     if (IsTailCall) {
-      // TODO: For -tailcallopt tail calls, Offset will need FPDiff like in
-      // ISelLowering.
+      Offset += FPDiff;
       int FI = MF.getFrameInfo().CreateFixedObject(Size, Offset, true);
       Register FIReg = MRI.createGenericVirtualRegister(p0);
       MIRBuilder.buildFrameIndex(FIReg, FI);
@@ -201,10 +202,18 @@ struct OutgoingArgHandler : public CallLowering::ValueHandler {
   MachineInstrBuilder MIB;
   CCAssignFn *AssignFnVarArg;
   bool IsTailCall;
+
+  /// For tail calls, the byte offset of the call's argument area from the
+  /// callee's. Unused elsewhere.
+  int FPDiff;
   uint64_t StackSize;
 };
 } // namespace
 
+static bool doesCalleeRestoreStack(CallingConv::ID CallConv, bool TailCallOpt) {
+  return CallConv == CallingConv::Fast && TailCallOpt;
+}
+
 void AArch64CallLowering::splitToValueTypes(
     const ArgInfo &OrigArg, SmallVectorImpl<ArgInfo> &SplitArgs,
     const DataLayout &DL, MachineRegisterInfo &MRI, CallingConv::ID CallConv) const {
@@ -359,6 +368,49 @@ bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
   return Success;
 }
 
+/// Helper function to compute forwarded registers for musttail calls. Computes
+/// the forwarded registers, sets MBB liveness, and emits COPY instructions that
+/// can be used to save + restore registers later.
+static void handleMustTailForwardedRegisters(MachineIRBuilder &MIRBuilder,
+                                             CCAssignFn *AssignFn) {
+  MachineBasicBlock &MBB = MIRBuilder.getMBB();
+  MachineFunction &MF = MIRBuilder.getMF();
+  MachineFrameInfo &MFI = MF.getFrameInfo();
+
+  if (!MFI.hasMustTailInVarArgFunc())
+    return;
+
+  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
+  const Function &F = MF.getFunction();
+  assert(F.isVarArg() && "Expected F to be vararg?");
+
+  // Compute the set of forwarded registers. The rest are scratch.
+  SmallVector<CCValAssign, 16> ArgLocs;
+  CCState CCInfo(F.getCallingConv(), /*IsVarArg=*/true, MF, ArgLocs,
+                 F.getContext());
+  SmallVector<MVT, 2> RegParmTypes;
+  RegParmTypes.push_back(MVT::i64);
+  RegParmTypes.push_back(MVT::f128);
+
+  // Later on, we can use this vector to restore the registers if necessary.
+  SmallVectorImpl<ForwardedRegister> &Forwards =
+      FuncInfo->getForwardedMustTailRegParms();
+  CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, AssignFn);
+
+  // Conservatively forward X8, since it might be used for an aggregate
+  // return.
+  if (!CCInfo.isAllocated(AArch64::X8)) {
+    unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
+    Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
+  }
+
+  // Add the forwards to the MachineBasicBlock and MachineFunction.
+  for (const auto &F : Forwards) {
+    MBB.addLiveIn(F.PReg);
+    MIRBuilder.buildCopy(Register(F.VReg), Register(F.PReg));
+  }
+}
+
 bool AArch64CallLowering::lowerFormalArguments(
     MachineIRBuilder &MIRBuilder, const Function &F,
     ArrayRef<ArrayRef<Register>> VRegs) const {
@@ -408,9 +460,21 @@ bool AArch64CallLowering::lowerFormalArguments(
     FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
   }
 
-  // TODO: Port checks for stack to restore for -tailcallopt from ISelLowering.
-  // We need to keep track of the size of function stacks for tail call
-  // optimization. When we tail call, we need to check if the callee's arguments
+  if (doesCalleeRestoreStack(F.getCallingConv(),
+                             MF.getTarget().Options.GuaranteedTailCallOpt)) {
+    // We have a non-standard ABI, so why not make full use of the stack that
+    // we're going to pop? It must be aligned to 16 B in any case.
+    StackOffset = alignTo(StackOffset, 16);
+
+    // If we're expected to restore the stack (e.g. fastcc), then we'll be
+    // adding a multiple of 16.
+    FuncInfo->setArgumentStackToRestore(StackOffset);
+
+    // Our own callers will guarantee that the space is free by giving an
+    // aligned value to CALLSEQ_START.
+  }
+
+  // When we tail call, we need to check if the callee's arguments
   // will fit on the caller's stack. So, whenever we lower formal arguments,
   // we should keep track of this information, since we might lower a tail call
   // in this function later.
@@ -420,6 +484,8 @@ bool AArch64CallLowering::lowerFormalArguments(
   if (Subtarget.hasCustomCallingConv())
     Subtarget.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
 
+  handleMustTailForwardedRegisters(MIRBuilder, AssignFn);
+
   // Move back to the end of the basic block.
   MIRBuilder.setMBB(MBB);
 
@@ -443,6 +509,13 @@ static bool mayTailCallThisCC(CallingConv::ID CC) {
   }
 }
 
+/// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
+/// CC.
+static std::pair<CCAssignFn *, CCAssignFn *>
+getAssignFnsForCC(CallingConv::ID CC, const AArch64TargetLowering &TLI) {
+  return {TLI.CCAssignFnForCall(CC, false), TLI.CCAssignFnForCall(CC, true)};
+}
+
 bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
     CallLoweringInfo &Info, MachineFunction &MF,
     SmallVectorImpl<ArgInfo> &InArgs) const {
@@ -456,11 +529,19 @@ bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
 
   // Check if the caller and callee will handle arguments in the same way.
   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
-  CCAssignFn *CalleeAssignFn = TLI.CCAssignFnForCall(CalleeCC, Info.IsVarArg);
-  CCAssignFn *CallerAssignFn =
-      TLI.CCAssignFnForCall(CallerCC, CallerF.isVarArg());
-
-  if (!resultsCompatible(Info, MF, InArgs, *CalleeAssignFn, *CallerAssignFn))
+  CCAssignFn *CalleeAssignFnFixed;
+  CCAssignFn *CalleeAssignFnVarArg;
+  std::tie(CalleeAssignFnFixed, CalleeAssignFnVarArg) =
+      getAssignFnsForCC(CalleeCC, TLI);
+
+  CCAssignFn *CallerAssignFnFixed;
+  CCAssignFn *CallerAssignFnVarArg;
+  std::tie(CallerAssignFnFixed, CallerAssignFnVarArg) =
+      getAssignFnsForCC(CallerCC, TLI);
+
+  if (!resultsCompatible(Info, MF, InArgs, *CalleeAssignFnFixed,
+                         *CalleeAssignFnVarArg, *CallerAssignFnFixed,
+                         *CallerAssignFnVarArg))
     return false;
 
   // Make sure that the caller and callee preserve all of the same registers.
@@ -487,12 +568,15 @@ bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
   CallingConv::ID CallerCC = CallerF.getCallingConv();
   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
 
+  CCAssignFn *AssignFnFixed;
+  CCAssignFn *AssignFnVarArg;
+  std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
+
   // We have outgoing arguments. Make sure that we can tail call with them.
   SmallVector<CCValAssign, 16> OutLocs;
   CCState OutInfo(CalleeCC, false, MF, OutLocs, CallerF.getContext());
 
-  if (!analyzeArgInfo(OutInfo, OutArgs,
-                      *TLI.CCAssignFnForCall(CalleeCC, Info.IsVarArg))) {
+  if (!analyzeArgInfo(OutInfo, OutArgs, *AssignFnFixed, *AssignFnVarArg)) {
     LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
     return false;
   }
@@ -575,6 +659,11 @@ bool AArch64CallLowering::isEligibleForTailCallOptimization(
     MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
     SmallVectorImpl<ArgInfo> &InArgs,
     SmallVectorImpl<ArgInfo> &OutArgs) const {
+
+  // Must pass all target-independent checks in order to tail call optimize.
+  if (!Info.IsTailCall)
+    return false;
+
   CallingConv::ID CalleeCC = Info.CallConv;
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &CallerF = MF.getFunction();
@@ -639,21 +728,18 @@ bool AArch64CallLowering::isEligibleForTailCallOptimization(
     }
   }
 
-  // If we have -tailcallopt and matching CCs, at this point, we could return
-  // true. However, we don't have full tail call support yet. So, continue
-  // checking. We want to emit a sibling call.
+  // If we have -tailcallopt, then we're done.
+  if (MF.getTarget().Options.GuaranteedTailCallOpt)
+    return canGuaranteeTCO(CalleeCC) && CalleeCC == CallerF.getCallingConv();
+
+  // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
+  // Try to find cases where we can do that.
 
   // I want anyone implementing a new calling convention to think long and hard
   // about this assert.
   assert((!Info.IsVarArg || CalleeCC == CallingConv::C) &&
          "Unexpected variadic calling convention");
 
-  // Before we can musttail varargs, we need to forward parameters like in
-  // r345641. Make sure that we don't enable musttail with varargs without
-  // addressing that!
-  assert(!(Info.IsVarArg && Info.IsMustTailCall) &&
-         "musttail support for varargs not implemented yet!");
-
   // Verify that the incoming and outgoing arguments from the callee are
   // safe to tail call.
   if (!doCallerAndCalleePassArgsTheSameWay(Info, MF, InArgs)) {
@@ -687,26 +773,157 @@ static unsigned getCallOpcode(const Function &CallerF, bool IsIndirect,
   return AArch64::TCRETURNri;
 }
 
-bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
-                                    CallLoweringInfo &Info) const {
+bool AArch64CallLowering::lowerTailCall(
+    MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
+    SmallVectorImpl<ArgInfo> &OutArgs) const {
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &F = MF.getFunction();
   MachineRegisterInfo &MRI = MF.getRegInfo();
-  auto &DL = F.getParent()->getDataLayout();
   const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
+  AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
 
-  if (Info.IsMustTailCall) {
-    // TODO: Until we lower all tail calls, we should fall back on this.
-    LLVM_DEBUG(dbgs() << "Cannot lower musttail calls yet.\n");
+  // True when we're tail calling, but without -tailcallopt.
+  bool IsSibCall = !MF.getTarget().Options.GuaranteedTailCallOpt;
+
+  // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
+  // register class. Until we can do that, we should fall back here.
+  if (F.hasFnAttribute("branch-target-enforcement")) {
+    LLVM_DEBUG(
+        dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
     return false;
   }
 
-  if (Info.IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) {
-    // TODO: Until we lower all tail calls, we should fall back on this.
-    LLVM_DEBUG(dbgs() << "Cannot handle -tailcallopt yet.\n");
+  // Find out which ABI gets to decide where things go.
+  CallingConv::ID CalleeCC = Info.CallConv;
+  CCAssignFn *AssignFnFixed;
+  CCAssignFn *AssignFnVarArg;
+  std::tie(AssignFnFixed, AssignFnVarArg) = getAssignFnsForCC(CalleeCC, TLI);
+
+  MachineInstrBuilder CallSeqStart;
+  if (!IsSibCall)
+    CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
+
+  unsigned Opc = getCallOpcode(F, Info.Callee.isReg(), true);
+  auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
+  MIB.add(Info.Callee);
+
+  // Byte offset for the tail call. When we are sibcalling, this will always
+  // be 0.
+  MIB.addImm(0);
+
+  // Tell the call which registers are clobbered.
+  auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
+  const uint32_t *Mask = TRI->getCallPreservedMask(MF, F.getCallingConv());
+  if (MF.getSubtarget<AArch64Subtarget>().hasCustomCallingConv())
+    TRI->UpdateCustomCallPreservedMask(MF, &Mask);
+  MIB.addRegMask(Mask);
+
+  if (TRI->isAnyArgRegReserved(MF))
+    TRI->emitReservedArgRegCallError(MF);
+
+  // FPDiff is the byte offset of the call's argument area from the callee's.
+  // Stores to callee stack arguments will be placed in FixedStackSlots offset
+  // by this amount for a tail call. In a sibling call it must be 0 because the
+  // caller will deallocate the entire stack and the callee still expects its
+  // arguments to begin at SP+0.
+  int FPDiff = 0;
+
+  // This will be 0 for sibcalls, potentially nonzero for tail calls produced
+  // by -tailcallopt. For sibcalls, the memory operands for the call are
+  // already available in the caller's incoming argument space.
+  unsigned NumBytes = 0;
+  if (!IsSibCall) {
+    // We aren't sibcalling, so we need to compute FPDiff. We need to do this
+    // before handling assignments, because FPDiff must be known for memory
+    // arguments.
+    unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
+    SmallVector<CCValAssign, 16> OutLocs;
+    CCState OutInfo(CalleeCC, false, MF, OutLocs, F.getContext());
+    analyzeArgInfo(OutInfo, OutArgs, *AssignFnFixed, *AssignFnVarArg);
+
+    // The callee will pop the argument stack as a tail call. Thus, we must
+    // keep it 16-byte aligned.
+    NumBytes = alignTo(OutInfo.getNextStackOffset(), 16);
+
+    // FPDiff will be negative if this tail call requires more space than we
+    // would automatically have in our incoming argument space. Positive if we
+    // actually shrink the stack.
+    FPDiff = NumReusableBytes - NumBytes;
+
+    // The stack pointer must be 16-byte aligned at all times it's used for a
+    // memory operation, which in practice means at *all* times and in
+    // particular across call boundaries. Therefore our own arguments started at
+    // a 16-byte aligned SP and the delta applied for the tail call should
+    // satisfy the same constraint.
+    assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
+  }
+
+  const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
+
+  // Do the actual argument marshalling.
+  SmallVector<unsigned, 8> PhysRegs;
+  OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
+                             AssignFnVarArg, true, FPDiff);
+  if (!handleAssignments(MIRBuilder, OutArgs, Handler))
     return false;
+
+  if (Info.IsVarArg && Info.IsMustTailCall) {
+    // Now we know what's being passed to the function. Add uses to the call for
+    // the forwarded registers that we *aren't* passing as parameters. This will
+    // preserve the copies we build earlier.
+    for (const auto &F : Forwards) {
+      Register ForwardedReg = F.PReg;
+      // If the register is already passed, or aliases a register which is
+      // already being passed, then skip it.
+      if (any_of(MIB->uses(), [&ForwardedReg, &TRI](const MachineOperand &Use) {
+            if (!Use.isReg())
+              return false;
+            return TRI->regsOverlap(Use.getReg(), ForwardedReg);
+          }))
+        continue;
+
+      // We aren't passing it already, so we should add it to the call.
+      MIRBuilder.buildCopy(ForwardedReg, Register(F.VReg));
+      MIB.addReg(ForwardedReg, RegState::Implicit);
+    }
+  }
+
+  // If we have -tailcallopt, we need to adjust the stack. We'll do the call
+  // sequence start and end here.
+  if (!IsSibCall) {
+    MIB->getOperand(1).setImm(FPDiff);
+    CallSeqStart.addImm(NumBytes).addImm(0);
+    // End the call sequence *before* emitting the call. Normally, we would
+    // tidy the frame up after the call. However, here, we've laid out the
+    // parameters so that when SP is reset, they will be in the correct
+    // location.
+    MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP).addImm(NumBytes).addImm(0);
   }
 
+  // Now we can add the actual call instruction to the correct basic block.
+  MIRBuilder.insertInstr(MIB);
+
+  // If Callee is a reg, since it is used by a target specific instruction,
+  // it must have a register class matching the constraint of that instruction.
+  if (Info.Callee.isReg())
+    MIB->getOperand(0).setReg(constrainOperandRegClass(
+        MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
+        *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee,
+        0));
+
+  MF.getFrameInfo().setHasTailCall();
+  Info.LoweredTailCall = true;
+  return true;
+}
+
+bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
+                                    CallLoweringInfo &Info) const {
+  MachineFunction &MF = MIRBuilder.getMF();
+  const Function &F = MF.getFunction();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
+  auto &DL = F.getParent()->getDataLayout();
+  const AArch64TargetLowering &TLI = *getTLI<AArch64TargetLowering>();
+
   SmallVector<ArgInfo, 8> OutArgs;
   for (auto &OrigArg : Info.OrigArgs) {
     splitToValueTypes(OrigArg, OutArgs, DL, MRI, Info.CallConv);
@@ -719,44 +936,38 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   if (!Info.OrigRet.Ty->isVoidTy())
     splitToValueTypes(Info.OrigRet, InArgs, DL, MRI, F.getCallingConv());
 
-  bool IsSibCall = Info.IsTailCall && isEligibleForTailCallOptimization(
-                                          MIRBuilder, Info, InArgs, OutArgs);
-  if (IsSibCall)
-    MF.getFrameInfo().setHasTailCall();
+  // If we can lower as a tail call, do that instead.
+  bool CanTailCallOpt =
+      isEligibleForTailCallOptimization(MIRBuilder, Info, InArgs, OutArgs);
+
+  // We must emit a tail call if we have musttail.
+  if (Info.IsMustTailCall && !CanTailCallOpt) {
+    // There are types of incoming/outgoing arguments we can't handle yet, so
+    // it doesn't make sense to actually die here like in ISelLowering. Instead,
+    // fall back to SelectionDAG and let it try to handle this.
+    LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
+    return false;
+  }
+
+  if (CanTailCallOpt)
+    return lowerTailCall(MIRBuilder, Info, OutArgs);
 
   // Find out which ABI gets to decide where things go.
-  CCAssignFn *AssignFnFixed =
-      TLI.CCAssignFnForCall(Info.CallConv, /*IsVarArg=*/false);
-  CCAssignFn *AssignFnVarArg =
-      TLI.CCAssignFnForCall(Info.CallConv, /*IsVarArg=*/true);
+  CCAssignFn *AssignFnFixed;
+  CCAssignFn *AssignFnVarArg;
+  std::tie(AssignFnFixed, AssignFnVarArg) =
+      getAssignFnsForCC(Info.CallConv, TLI);
 
-  // If we have a sibling call, then we don't have to adjust the stack.
-  // Otherwise, we need to adjust it.
   MachineInstrBuilder CallSeqStart;
-  if (!IsSibCall)
-    CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
+  CallSeqStart = MIRBuilder.buildInstr(AArch64::ADJCALLSTACKDOWN);
 
   // Create a temporarily-floating call instruction so we can add the implicit
   // uses of arg registers.
-  unsigned Opc = getCallOpcode(F, Info.Callee.isReg(), IsSibCall);
-
-  // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
-  // register class. Until we can do that, we should fall back here.
-  if (Opc == AArch64::TCRETURNriBTI) {
-    LLVM_DEBUG(
-        dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
-    return false;
-  }
+  unsigned Opc = getCallOpcode(F, Info.Callee.isReg(), false);
 
   auto MIB = MIRBuilder.buildInstrNoInsert(Opc);
   MIB.add(Info.Callee);
 
-  // Add the byte offset for the tail call. We only have sibling calls, so this
-  // is always 0.
-  // TODO: Handle tail calls where we will have a different value here.
-  if (IsSibCall)
-    MIB.addImm(0);
-
   // Tell the call which registers are clobbered.
   auto TRI = MF.getSubtarget<AArch64Subtarget>().getRegisterInfo();
   const uint32_t *Mask = TRI->getCallPreservedMask(MF, F.getCallingConv());
@@ -770,7 +981,7 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   // Do the actual argument marshalling.
   SmallVector<unsigned, 8> PhysRegs;
   OutgoingArgHandler Handler(MIRBuilder, MRI, MIB, AssignFnFixed,
-                             AssignFnVarArg, IsSibCall);
+                             AssignFnVarArg, false);
   if (!handleAssignments(MIRBuilder, OutArgs, Handler))
     return false;
 
@@ -786,11 +997,6 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
         *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Info.Callee,
         0));
 
-  // If we're tail calling, then we're the return from the block. So, we don't
-  // want to copy anything.
-  if (IsSibCall)
-    return true;
-
   // Finally we can copy the returned value back into its virtual-register. In
   // symmetry with the arugments, the physical register must be an
   // implicit-define of the call instruction.
@@ -806,10 +1012,16 @@ bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     MIRBuilder.buildCopy(Info.SwiftErrorVReg, Register(AArch64::X21));
   }
 
+  uint64_t CalleePopBytes =
+      doesCalleeRestoreStack(Info.CallConv,
+                             MF.getTarget().Options.GuaranteedTailCallOpt)
+          ? alignTo(Handler.StackSize, 16)
+          : 0;
+
   CallSeqStart.addImm(Handler.StackSize).addImm(0);
   MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
       .addImm(Handler.StackSize)
-      .addImm(0);
+      .addImm(CalleePopBytes);
 
   return true;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64CallLowering.h b/llvm/lib/Target/AArch64/AArch64CallLowering.h
index 4777a16ac3ebd..b0c601c7062c0 100644
--- a/llvm/lib/Target/AArch64/AArch64CallLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64CallLowering.h
@@ -64,6 +64,9 @@ class AArch64CallLowering: public CallLowering {
                          const DataLayout &DL, MachineRegisterInfo &MRI,
                          CallingConv::ID CallConv) const;
 
+  bool lowerTailCall(MachineIRBuilder &MIRBuilder, CallLoweringInfo &Info,
+                     SmallVectorImpl<ArgInfo> &OutArgs) const;
+
   bool
   doCallerAndCalleePassArgsTheSameWay(CallLoweringInfo &Info,
                                       MachineFunction &MF,
diff --git a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
index 39f42562d5498..a0695cef615f3 100644
--- a/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
+++ b/llvm/lib/Target/AArch64/AArch64CallingConvention.cpp
@@ -40,12 +40,14 @@ static bool finishStackBlock(SmallVectorImpl<CCValAssign> &PendingMembers,
                              MVT LocVT, ISD::ArgFlagsTy &ArgFlags,
                              CCState &State, unsigned SlotAlign) {
   unsigned Size = LocVT.getSizeInBits() / 8;
-  unsigned StackAlign =
+  const Align StackAlign =
       State.getMachineFunction().getDataLayout().getStackAlignment();
-  unsigned Align = std::min(ArgFlags.getOrigAlign(), StackAlign);
+  const Align OrigAlign(ArgFlags.getOrigAlign());
+  const Align Align = std::min(OrigAlign, StackAlign);
 
   for (auto &It : PendingMembers) {
-    It.convertToMem(State.AllocateStack(Size, std::max(Align, SlotAlign)));
+    It.convertToMem(State.AllocateStack(
+        Size, std::max((unsigned)Align.value(), SlotAlign)));
     State.addLoc(It);
     SlotAlign = 1;
   }
diff --git a/llvm/lib/Target/AArch64/AArch64Combine.td b/llvm/lib/Target/AArch64/AArch64Combine.td
new file mode 100644
index 0000000000000..c4658f73b8ddc
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64Combine.td
@@ -0,0 +1,15 @@
+//=- AArch64.td - Define AArch64 Combine Rules ---------------*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//
+//===----------------------------------------------------------------------===//
+
+include "llvm/Target/GlobalISel/Combine.td"
+
+def AArch64PreLegalizerCombinerHelper: GICombinerHelper<
+  "AArch64GenPreLegalizerCombinerHelper">;
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index 4c845ed6f822b..9ce028cb961cf 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -506,7 +506,7 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB,
         unsigned Reg32 = TRI->getSubReg(DstReg, AArch64::sub_32);
         unsigned DstFlags = MI.getOperand(0).getTargetFlags();
         MIB2 = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::LDRWui))
-                   .addDef(Reg32, RegState::Dead)
+                   .addDef(Reg32)
                    .addReg(DstReg, RegState::Kill)
                    .addReg(DstReg, DstFlags | RegState::Implicit);
       } else {
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index cd7e927ac80ca..1f08505f37e79 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -2053,7 +2053,7 @@ static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
 }
 
 static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
-  if (Depth >= 6)
+  if (Depth >= SelectionDAG::MaxRecursionDepth)
     return;
   // Initialize UsefulBits
   if (!Depth) {
@@ -2913,49 +2913,6 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) {
       return;
     break;
 
-  case ISD::EXTRACT_VECTOR_ELT: {
-    // Extracting lane zero is a special case where we can just use a plain
-    // EXTRACT_SUBREG instruction, which will become FMOV. This is easier for
-    // the rest of the compiler, especially the register allocator and copyi
-    // propagation, to reason about, so is preferred when it's possible to
-    // use it.
-    ConstantSDNode *LaneNode = cast<ConstantSDNode>(Node->getOperand(1));
-    // Bail and use the default Select() for non-zero lanes.
-    if (LaneNode->getZExtValue() != 0)
-      break;
-    // If the element type is not the same as the result type, likewise
-    // bail and use the default Select(), as there's more to do than just
-    // a cross-class COPY. This catches extracts of i8 and i16 elements
-    // since they will need an explicit zext.
-    if (VT != Node->getOperand(0).getValueType().getVectorElementType())
-      break;
-    unsigned SubReg;
-    switch (Node->getOperand(0)
-                .getValueType()
-                .getVectorElementType()
-                .getSizeInBits()) {
-    default:
-      llvm_unreachable("Unexpected vector element type!");
-    case 64:
-      SubReg = AArch64::dsub;
-      break;
-    case 32:
-      SubReg = AArch64::ssub;
-      break;
-    case 16:
-      SubReg = AArch64::hsub;
-      break;
-    case 8:
-      llvm_unreachable("unexpected zext-requiring extract element!");
-    }
-    SDValue Extract = CurDAG->getTargetExtractSubreg(SubReg, SDLoc(Node), VT,
-                                                     Node->getOperand(0));
-    LLVM_DEBUG(dbgs() << "ISEL: Custom selection!\n=> ");
-    LLVM_DEBUG(Extract->dumpr(CurDAG));
-    LLVM_DEBUG(dbgs() << "\n");
-    ReplaceNode(Node, Extract.getNode());
-    return;
-  }
   case ISD::Constant: {
     // Materialize zero constants as copies from WZR/XZR.  This allows
     // the coalescer to propagate these into other instructions.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 7d466feeac91e..bec14001ed840 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -307,7 +307,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   // AArch64 lacks both left-rotate and popcount instructions.
   setOperationAction(ISD::ROTL, MVT::i32, Expand);
   setOperationAction(ISD::ROTL, MVT::i64, Expand);
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     setOperationAction(ISD::ROTL, VT, Expand);
     setOperationAction(ISD::ROTR, VT, Expand);
   }
@@ -321,7 +321,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
   setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     setOperationAction(ISD::SDIVREM, VT, Expand);
     setOperationAction(ISD::UDIVREM, VT, Expand);
   }
@@ -641,11 +641,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
   EnableExtLdPromotion = true;
 
   // Set required alignment.
-  setMinFunctionAlignment(llvm::Align(4));
+  setMinFunctionAlignment(Align(4));
   // Set preferred alignments.
-  setPrefLoopAlignment(llvm::Align(1ULL << STI.getPrefLoopLogAlignment()));
-  setPrefFunctionAlignment(
-      llvm::Align(1ULL << STI.getPrefFunctionLogAlignment()));
+  setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
+  setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
 
   // Only change the limit for entries in a jump table if specified by
   // the sub target, but not at the command line.
@@ -754,7 +753,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
     setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
     // Likewise, narrowing and extending vector loads/stores aren't handled
     // directly.
-    for (MVT VT : MVT::vector_valuetypes()) {
+    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
       setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
 
       if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
@@ -770,7 +769,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
       setOperationAction(ISD::BSWAP, VT, Expand);
       setOperationAction(ISD::CTTZ, VT, Expand);
 
-      for (MVT InnerVT : MVT::vector_valuetypes()) {
+      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
         setTruncStoreAction(VT, InnerVT, Expand);
         setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
         setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
@@ -5490,9 +5489,9 @@ SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                                  SelectionDAG &DAG) const {
-  unsigned Reg = MatchRegisterName(RegName);
+Register AArch64TargetLowering::
+getRegisterByName(const char* RegName, EVT VT, const MachineFunction &MF) const {
+  Register Reg = MatchRegisterName(RegName);
   if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
     const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
     unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
@@ -5837,6 +5836,21 @@ const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
   return "r";
 }
 
+enum PredicateConstraint {
+  Upl,
+  Upa,
+  Invalid
+};
+
+static PredicateConstraint parsePredicateConstraint(StringRef Constraint) {
+  PredicateConstraint P = PredicateConstraint::Invalid;
+  if (Constraint == "Upa")
+    P = PredicateConstraint::Upa;
+  if (Constraint == "Upl")
+    P = PredicateConstraint::Upl;
+  return P;
+}
+
 /// getConstraintType - Given a constraint letter, return the type of
 /// constraint it is for this target.
 AArch64TargetLowering::ConstraintType
@@ -5866,7 +5880,9 @@ AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
     case 'S': // A symbolic address
       return C_Other;
     }
-  }
+  } else if (parsePredicateConstraint(Constraint) !=
+             PredicateConstraint::Invalid)
+      return C_RegisterClass;
   return TargetLowering::getConstraintType(Constraint);
 }
 
@@ -5897,6 +5913,10 @@ AArch64TargetLowering::getSingleConstraintMatchWeight(
   case 'z':
     weight = CW_Constant;
     break;
+  case 'U':
+    if (parsePredicateConstraint(constraint) != PredicateConstraint::Invalid)
+      weight = CW_Register;
+    break;
   }
   return weight;
 }
@@ -5941,6 +5961,14 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
         return std::make_pair(0U, &AArch64::ZPR_3bRegClass);
       break;
     }
+  } else {
+    PredicateConstraint PC = parsePredicateConstraint(Constraint);
+    if (PC != PredicateConstraint::Invalid) {
+      assert(VT.isScalableVector());
+      bool restricted = (PC == PredicateConstraint::Upl);
+      return restricted ? std::make_pair(0U, &AArch64::PPR_3bRegClass)
+                          : std::make_pair(0U, &AArch64::PPRRegClass);
+    }
   }
   if (StringRef("{cc}").equals_lower(Constraint))
     return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
@@ -10303,6 +10331,14 @@ static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
     Opcode = AArch64ISD::SQSHLU_I;
     IsRightShift = false;
     break;
+  case Intrinsic::aarch64_neon_sshl:
+  case Intrinsic::aarch64_neon_ushl:
+    // For positive shift amounts we can use SHL, as ushl/sshl perform a regular
+    // left shift for positive shift amounts. Below, we only replace the current
+    // node with VSHL, if this condition is met.
+    Opcode = AArch64ISD::VSHL;
+    IsRightShift = false;
+    break;
   }
 
   if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
@@ -10389,6 +10425,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
   case Intrinsic::aarch64_neon_sqshlu:
   case Intrinsic::aarch64_neon_srshl:
   case Intrinsic::aarch64_neon_urshl:
+  case Intrinsic::aarch64_neon_sshl:
+  case Intrinsic::aarch64_neon_ushl:
     return tryCombineShiftImm(IID, N, DAG);
   case Intrinsic::aarch64_crc32b:
   case Intrinsic::aarch64_crc32cb:
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 60190b2eb9593..462fcabc99491 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -707,8 +707,8 @@ class AArch64TargetLowering : public TargetLowering {
   unsigned combineRepeatedFPDivisors() const override;
 
   ConstraintType getConstraintType(StringRef Constraint) const override;
-  unsigned getRegisterByName(const char* RegName, EVT VT,
-                             SelectionDAG &DAG) const override;
+  Register getRegisterByName(const char* RegName, EVT VT,
+                             const MachineFunction &MF) const override;
 
   /// Examine constraint string and operand type and determine a weight value.
   /// The operand object must already have been set up with the operand type.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 19b10fc0806f4..af61521d4b488 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -685,11 +685,11 @@ def logical_imm64_not : Operand<i64> {
 
 // iXX_imm0_65535 predicates - True if the immediate is in the range [0,65535].
 let ParserMatchClass = AsmImmRange<0, 65535>, PrintMethod = "printImmHex" in {
-def i32_imm0_65535 : Operand<i32>, ImmLeaf<i32, [{
+def i32_imm0_65535 : Operand<i32>, TImmLeaf<i32, [{
   return ((uint32_t)Imm) < 65536;
 }]>;
 
-def i64_imm0_65535 : Operand<i64>, ImmLeaf<i64, [{
+def i64_imm0_65535 : Operand<i64>, TImmLeaf<i64, [{
   return ((uint64_t)Imm) < 65536;
 }]>;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 53fdfa5ea42b5..097a8ba0ae19a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -929,7 +929,7 @@ bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
 }
 
 bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
-    const MachineInstr &MIa, const MachineInstr &MIb, AliasAnalysis *AA) const {
+    const MachineInstr &MIa, const MachineInstr &MIb) const {
   const TargetRegisterInfo *TRI = &getRegisterInfo();
   const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
   int64_t OffsetA = 0, OffsetB = 0;
@@ -2507,6 +2507,17 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     return;
   }
 
+  // Copy a Predicate register by ORRing with itself.
+  if (AArch64::PPRRegClass.contains(DestReg) &&
+      AArch64::PPRRegClass.contains(SrcReg)) {
+    assert(Subtarget.hasSVE() && "Unexpected SVE register.");
+    BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
+      .addReg(SrcReg) // Pg
+      .addReg(SrcReg)
+      .addReg(SrcReg, getKillRegState(KillSrc));
+    return;
+  }
+
   // Copy a Z register by ORRing with itself.
   if (AArch64::ZPRRegClass.contains(DestReg) &&
       AArch64::ZPRRegClass.contains(SrcReg)) {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index d158137bacda2..404acb9f898fd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -56,8 +56,7 @@ class AArch64InstrInfo final : public AArch64GenInstrInfo {
 
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const override;
+                                  const MachineInstr &MIb) const override;
 
   unsigned isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 7123ba043c325..661d089688b33 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -798,7 +798,11 @@ def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins),
 let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in {
 def HWASAN_CHECK_MEMACCESS : Pseudo<
   (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo),
-  [(int_hwasan_check_memaccess X9, GPR64noip:$ptr, (i32 imm:$accessinfo))]>,
+  [(int_hwasan_check_memaccess X9, GPR64noip:$ptr, (i32 timm:$accessinfo))]>,
+  Sched<[]>;
+def HWASAN_CHECK_MEMACCESS_SHORTGRANULES : Pseudo<
+  (outs), (ins GPR64noip:$ptr, i32imm:$accessinfo),
+  [(int_hwasan_check_memaccess_shortgranules X9, GPR64noip:$ptr, (i32 timm:$accessinfo))]>,
   Sched<[]>;
 }
 
@@ -3315,20 +3319,37 @@ defm FNMSUB : ThreeOperandFPData<1, 1, "fnmsub",
 
 // N.b. FMSUB etc have the accumulator at the *end* of (outs), unlike
 // the NEON variant.
+
+// Here we handle first -(a + b*c) for FNMADD:
+
+let Predicates = [HasNEON, HasFullFP16] in
+def : Pat<(f16 (fma (fneg FPR16:$Rn), FPR16:$Rm, FPR16:$Ra)),
+          (FMSUBHrrr FPR16:$Rn, FPR16:$Rm, FPR16:$Ra)>;
+
 def : Pat<(f32 (fma (fneg FPR32:$Rn), FPR32:$Rm, FPR32:$Ra)),
           (FMSUBSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
 
 def : Pat<(f64 (fma (fneg FPR64:$Rn), FPR64:$Rm, FPR64:$Ra)),
           (FMSUBDrrr FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
 
-// We handled -(a + b*c) for FNMADD above, now it's time for "(-a) + (-b)*c" and
-// "(-a) + b*(-c)".
+// Now it's time for "(-a) + (-b)*c"
+
+let Predicates = [HasNEON, HasFullFP16] in
+def : Pat<(f16 (fma (fneg FPR16:$Rn), FPR16:$Rm, (fneg FPR16:$Ra))),
+          (FNMADDHrrr FPR16:$Rn, FPR16:$Rm, FPR16:$Ra)>;
+
 def : Pat<(f32 (fma (fneg FPR32:$Rn), FPR32:$Rm, (fneg FPR32:$Ra))),
           (FNMADDSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
 
 def : Pat<(f64 (fma (fneg FPR64:$Rn), FPR64:$Rm, (fneg FPR64:$Ra))),
           (FNMADDDrrr FPR64:$Rn, FPR64:$Rm, FPR64:$Ra)>;
 
+// And here "(-a) + b*(-c)"
+
+let Predicates = [HasNEON, HasFullFP16] in
+def : Pat<(f16 (fma FPR16:$Rn, (fneg FPR16:$Rm), (fneg FPR16:$Ra))),
+          (FNMADDHrrr FPR16:$Rn, FPR16:$Rm, FPR16:$Ra)>;
+
 def : Pat<(f32 (fma FPR32:$Rn, (fneg FPR32:$Rm), (fneg FPR32:$Ra))),
           (FNMADDSrrr FPR32:$Rn, FPR32:$Rm, FPR32:$Ra)>;
 
@@ -6973,5 +6994,124 @@ def : Pat<(AArch64tcret texternalsym:$dst, (i32 timm:$FPDiff)),
 def MOVMCSym : Pseudo<(outs GPR64:$dst), (ins i64imm:$sym), []>, Sched<[]>;
 def : Pat<(i64 (AArch64LocalRecover mcsym:$sym)), (MOVMCSym mcsym:$sym)>;
 
+// Extracting lane zero is a special case where we can just use a plain
+// EXTRACT_SUBREG instruction, which will become FMOV. This is easier for the
+// rest of the compiler, especially the register allocator and copy propagation,
+// to reason about, so is preferred when it's possible to use it.
+let AddedComplexity = 10 in {
+  def : Pat<(i64 (extractelt (v2i64 V128:$V), (i64 0))), (EXTRACT_SUBREG V128:$V, dsub)>;
+  def : Pat<(i32 (extractelt (v4i32 V128:$V), (i64 0))), (EXTRACT_SUBREG V128:$V, ssub)>;
+  def : Pat<(i32 (extractelt (v2i32 V64:$V), (i64 0))), (EXTRACT_SUBREG V64:$V, ssub)>;
+}
+
+// dot_v4i8
+class mul_v4i8<SDPatternOperator ldop> :
+  PatFrag<(ops node:$Rn, node:$Rm, node:$offset),
+          (mul (ldop (add node:$Rn, node:$offset)),
+               (ldop (add node:$Rm, node:$offset)))>;
+class mulz_v4i8<SDPatternOperator ldop> :
+  PatFrag<(ops node:$Rn, node:$Rm),
+          (mul (ldop node:$Rn), (ldop node:$Rm))>;
+
+def load_v4i8 :
+  OutPatFrag<(ops node:$R),
+             (INSERT_SUBREG
+              (v2i32 (IMPLICIT_DEF)),
+               (i32 (COPY_TO_REGCLASS (LDRWui node:$R, (i64 0)), FPR32)),
+              ssub)>;
+
+class dot_v4i8<Instruction DOT, SDPatternOperator ldop> :
+  Pat<(i32 (add (mul_v4i8<ldop> GPR64sp:$Rn, GPR64sp:$Rm, (i64 3)),
+           (add (mul_v4i8<ldop> GPR64sp:$Rn, GPR64sp:$Rm, (i64 2)),
+           (add (mul_v4i8<ldop> GPR64sp:$Rn, GPR64sp:$Rm, (i64 1)),
+                (mulz_v4i8<ldop> GPR64sp:$Rn, GPR64sp:$Rm))))),
+      (EXTRACT_SUBREG (i64 (DOT (DUPv2i32gpr WZR),
+                                (load_v4i8 GPR64sp:$Rn),
+                                (load_v4i8 GPR64sp:$Rm))),
+                      sub_32)>, Requires<[HasDotProd]>;
+
+// dot_v8i8
+class ee_v8i8<SDPatternOperator extend> :
+  PatFrag<(ops node:$V, node:$K),
+          (v4i16 (extract_subvector (v8i16 (extend node:$V)), node:$K))>;
+
+class mul_v8i8<SDPatternOperator mulop, SDPatternOperator extend> :
+  PatFrag<(ops node:$M, node:$N, node:$K),
+          (mulop (v4i16 (ee_v8i8<extend> node:$M, node:$K)),
+                 (v4i16 (ee_v8i8<extend> node:$N, node:$K)))>;
+
+class idot_v8i8<SDPatternOperator mulop, SDPatternOperator extend> :
+  PatFrag<(ops node:$M, node:$N),
+          (i32 (extractelt
+           (v4i32 (AArch64uaddv
+            (add (mul_v8i8<mulop, extend> node:$M, node:$N, (i64 0)),
+                 (mul_v8i8<mulop, extend> node:$M, node:$N, (i64 4))))),
+           (i64 0)))>;
+
+// vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
+def VADDV_32 : OutPatFrag<(ops node:$R), (ADDPv2i32 node:$R, node:$R)>;
+
+class odot_v8i8<Instruction DOT> :
+  OutPatFrag<(ops node:$Vm, node:$Vn),
+             (EXTRACT_SUBREG
+              (VADDV_32
+               (i64 (DOT (DUPv2i32gpr WZR),
+                         (v8i8 node:$Vm),
+                         (v8i8 node:$Vn)))),
+              sub_32)>;
+
+class dot_v8i8<Instruction DOT, SDPatternOperator mulop,
+                    SDPatternOperator extend> :
+  Pat<(idot_v8i8<mulop, extend> V64:$Vm, V64:$Vn),
+      (odot_v8i8<DOT> V64:$Vm, V64:$Vn)>,
+  Requires<[HasDotProd]>;
+
+// dot_v16i8
+class ee_v16i8<SDPatternOperator extend> :
+  PatFrag<(ops node:$V, node:$K1, node:$K2),
+          (v4i16 (extract_subvector
+           (v8i16 (extend
+            (v8i8 (extract_subvector node:$V, node:$K1)))), node:$K2))>;
+
+class mul_v16i8<SDPatternOperator mulop, SDPatternOperator extend> :
+  PatFrag<(ops node:$M, node:$N, node:$K1, node:$K2),
+          (v4i32
+           (mulop (v4i16 (ee_v16i8<extend> node:$M, node:$K1, node:$K2)),
+                  (v4i16 (ee_v16i8<extend> node:$N, node:$K1, node:$K2))))>;
+
+class idot_v16i8<SDPatternOperator m, SDPatternOperator x> :
+  PatFrag<(ops node:$M, node:$N),
+          (i32 (extractelt
+           (v4i32 (AArch64uaddv
+            (add
+             (add (mul_v16i8<m, x> node:$M, node:$N, (i64 0), (i64 0)),
+                  (mul_v16i8<m, x> node:$M, node:$N, (i64 8), (i64 0))),
+             (add (mul_v16i8<m, x> node:$M, node:$N, (i64 0), (i64 4)),
+                  (mul_v16i8<m, x> node:$M, node:$N, (i64 8), (i64 4)))))),
+           (i64 0)))>;
+
+class odot_v16i8<Instruction DOT> :
+  OutPatFrag<(ops node:$Vm, node:$Vn),
+             (i32 (ADDVv4i32v
+              (DOT (DUPv4i32gpr WZR), node:$Vm, node:$Vn)))>;
+
+class dot_v16i8<Instruction DOT, SDPatternOperator mulop,
+                SDPatternOperator extend> :
+  Pat<(idot_v16i8<mulop, extend> V128:$Vm, V128:$Vn),
+      (odot_v16i8<DOT> V128:$Vm, V128:$Vn)>,
+  Requires<[HasDotProd]>;
+
+let AddedComplexity = 10 in {
+  def : dot_v4i8<SDOTv8i8, sextloadi8>;
+  def : dot_v4i8<UDOTv8i8, zextloadi8>;
+  def : dot_v8i8<SDOTv8i8, AArch64smull, sext>;
+  def : dot_v8i8<UDOTv8i8, AArch64umull, zext>;
+  def : dot_v16i8<SDOTv16i8, AArch64smull, sext>;
+  def : dot_v16i8<UDOTv16i8, AArch64umull, zext>;
+
+  // FIXME: add patterns to generate vector by element dot product.
+  // FIXME: add SVE dot-product patterns.
+}
+
 include "AArch64InstrAtomics.td"
 include "AArch64SVEInstrInfo.td"
diff --git a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
index 8503a0dbb062a..961f38cad1e4a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstructionSelector.cpp
@@ -1018,7 +1018,9 @@ bool AArch64InstructionSelector::selectVectorSHL(
     return false;
 
   unsigned Opc = 0;
-  if (Ty == LLT::vector(4, 32)) {
+  if (Ty == LLT::vector(2, 64)) {
+    Opc = AArch64::USHLv2i64;
+  } else if (Ty == LLT::vector(4, 32)) {
     Opc = AArch64::USHLv4i32;
   } else if (Ty == LLT::vector(2, 32)) {
     Opc = AArch64::USHLv2i32;
@@ -1052,7 +1054,11 @@ bool AArch64InstructionSelector::selectVectorASHR(
   unsigned Opc = 0;
   unsigned NegOpc = 0;
   const TargetRegisterClass *RC = nullptr;
-  if (Ty == LLT::vector(4, 32)) {
+  if (Ty == LLT::vector(2, 64)) {
+    Opc = AArch64::SSHLv2i64;
+    NegOpc = AArch64::NEGv2i64;
+    RC = &AArch64::FPR128RegClass;
+  } else if (Ty == LLT::vector(4, 32)) {
     Opc = AArch64::SSHLv4i32;
     NegOpc = AArch64::NEGv4i32;
     RC = &AArch64::FPR128RegClass;
diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
index 2fd5e0d70a68e..7a1901bd5b1e3 100644
--- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
@@ -82,7 +82,7 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
 
   getActionDefinitionsBuilder(G_BSWAP)
       .legalFor({s32, s64, v4s32, v2s32, v2s64})
-      .clampScalar(0, s16, s64)
+      .clampScalar(0, s32, s64)
       .widenScalarToNextPow2(0);
 
   getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
@@ -124,8 +124,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
         return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
                AmtTy.getSizeInBits() == 32;
       })
-      .legalFor(
-          {{s32, s32}, {s32, s64}, {s64, s64}, {v2s32, v2s32}, {v4s32, v4s32}})
+      .legalFor({{s32, s32},
+                 {s32, s64},
+                 {s64, s64},
+                 {v2s32, v2s32},
+                 {v4s32, v4s32},
+                 {v2s64, v2s64}})
       .clampScalar(1, s32, s64)
       .clampScalar(0, s32, s64)
       .minScalarSameAs(1, 0);
diff --git a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
index 5242e87f0eb2b..a0c4a25bb5b96 100644
--- a/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
@@ -201,8 +201,22 @@ static bool isNarrowStore(unsigned Opc) {
   }
 }
 
+// These instruction set memory tag and either keep memory contents unchanged or
+// set it to zero, ignoring the address part of the source register.
+static bool isTagStore(const MachineInstr &MI) {
+  switch (MI.getOpcode()) {
+  default:
+    return false;
+  case AArch64::STGOffset:
+  case AArch64::STZGOffset:
+  case AArch64::ST2GOffset:
+  case AArch64::STZ2GOffset:
+    return true;
+  }
+}
+
 // Scaling factor for unscaled load or store.
-static int getMemScale(MachineInstr &MI) {
+static int getMemScale(const MachineInstr &MI) {
   switch (MI.getOpcode()) {
   default:
     llvm_unreachable("Opcode has unknown scale!");
@@ -255,6 +269,11 @@ static int getMemScale(MachineInstr &MI) {
   case AArch64::STURQi:
   case AArch64::LDPQi:
   case AArch64::STPQi:
+  case AArch64::STGOffset:
+  case AArch64::STZGOffset:
+  case AArch64::ST2GOffset:
+  case AArch64::STZ2GOffset:
+  case AArch64::STGPi:
     return 16;
   }
 }
@@ -449,6 +468,16 @@ static unsigned getPreIndexedOpcode(unsigned Opc) {
     return AArch64::STPWpre;
   case AArch64::STPXi:
     return AArch64::STPXpre;
+  case AArch64::STGOffset:
+    return AArch64::STGPreIndex;
+  case AArch64::STZGOffset:
+    return AArch64::STZGPreIndex;
+  case AArch64::ST2GOffset:
+    return AArch64::ST2GPreIndex;
+  case AArch64::STZ2GOffset:
+    return AArch64::STZ2GPreIndex;
+  case AArch64::STGPi:
+    return AArch64::STGPpre;
   }
 }
 
@@ -518,6 +547,16 @@ static unsigned getPostIndexedOpcode(unsigned Opc) {
     return AArch64::STPWpost;
   case AArch64::STPXi:
     return AArch64::STPXpost;
+  case AArch64::STGOffset:
+    return AArch64::STGPostIndex;
+  case AArch64::STZGOffset:
+    return AArch64::STZGPostIndex;
+  case AArch64::ST2GOffset:
+    return AArch64::ST2GPostIndex;
+  case AArch64::STZ2GOffset:
+    return AArch64::STZ2GPostIndex;
+  case AArch64::STGPi:
+    return AArch64::STGPpost;
   }
 }
 
@@ -536,10 +575,30 @@ static bool isPairedLdSt(const MachineInstr &MI) {
   case AArch64::STPQi:
   case AArch64::STPWi:
   case AArch64::STPXi:
+  case AArch64::STGPi:
     return true;
   }
 }
 
+// Returns the scale and offset range of pre/post indexed variants of MI.
+static void getPrePostIndexedMemOpInfo(const MachineInstr &MI, int &Scale,
+                                       int &MinOffset, int &MaxOffset) {
+  bool IsPaired = isPairedLdSt(MI);
+  bool IsTagStore = isTagStore(MI);
+  // ST*G and all paired ldst have the same scale in pre/post-indexed variants
+  // as in the "unsigned offset" variant.
+  // All other pre/post indexed ldst instructions are unscaled.
+  Scale = (IsTagStore || IsPaired) ? getMemScale(MI) : 1;
+
+  if (IsPaired) {
+    MinOffset = -64;
+    MaxOffset = 63;
+  } else {
+    MinOffset = -256;
+    MaxOffset = 255;
+  }
+}
+
 static const MachineOperand &getLdStRegOp(const MachineInstr &MI,
                                           unsigned PairedRegOp = 0) {
   assert(PairedRegOp < 2 && "Unexpected register operand idx.");
@@ -618,6 +677,11 @@ static bool isMergeableLdStUpdate(MachineInstr &MI) {
   case AArch64::LDRWui:
   case AArch64::LDRHHui:
   case AArch64::LDRBBui:
+  case AArch64::STGOffset:
+  case AArch64::STZGOffset:
+  case AArch64::ST2GOffset:
+  case AArch64::STZ2GOffset:
+  case AArch64::STGPi:
   // Unscaled instructions.
   case AArch64::STURSi:
   case AArch64::STURDi:
@@ -1328,18 +1392,19 @@ AArch64LoadStoreOpt::mergeUpdateInsn(MachineBasicBlock::iterator I,
   unsigned NewOpc = IsPreIdx ? getPreIndexedOpcode(I->getOpcode())
                              : getPostIndexedOpcode(I->getOpcode());
   MachineInstrBuilder MIB;
+  int Scale, MinOffset, MaxOffset;
+  getPrePostIndexedMemOpInfo(*I, Scale, MinOffset, MaxOffset);
   if (!isPairedLdSt(*I)) {
     // Non-paired instruction.
     MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
               .add(getLdStRegOp(*Update))
               .add(getLdStRegOp(*I))
               .add(getLdStBaseOp(*I))
-              .addImm(Value)
+              .addImm(Value / Scale)
               .setMemRefs(I->memoperands())
               .setMIFlags(I->mergeFlagsWith(*Update));
   } else {
     // Paired instruction.
-    int Scale = getMemScale(*I);
     MIB = BuildMI(*I->getParent(), I, I->getDebugLoc(), TII->get(NewOpc))
               .add(getLdStRegOp(*Update))
               .add(getLdStRegOp(*I, 0))
@@ -1395,28 +1460,21 @@ bool AArch64LoadStoreOpt::isMatchingUpdateInsn(MachineInstr &MemMI,
         MI.getOperand(1).getReg() != BaseReg)
       break;
 
-    bool IsPairedInsn = isPairedLdSt(MemMI);
     int UpdateOffset = MI.getOperand(2).getImm();
     if (MI.getOpcode() == AArch64::SUBXri)
       UpdateOffset = -UpdateOffset;
 
-    // For non-paired load/store instructions, the immediate must fit in a
-    // signed 9-bit integer.
-    if (!IsPairedInsn && (UpdateOffset > 255 || UpdateOffset < -256))
+    // The immediate must be a multiple of the scaling factor of the pre/post
+    // indexed instruction.
+    int Scale, MinOffset, MaxOffset;
+    getPrePostIndexedMemOpInfo(MemMI, Scale, MinOffset, MaxOffset);
+    if (UpdateOffset % Scale != 0)
       break;
 
-    // For paired load/store instructions, the immediate must be a multiple of
-    // the scaling factor.  The scaled offset must also fit into a signed 7-bit
-    // integer.
-    if (IsPairedInsn) {
-      int Scale = getMemScale(MemMI);
-      if (UpdateOffset % Scale != 0)
-        break;
-
-      int ScaledOffset = UpdateOffset / Scale;
-      if (ScaledOffset > 63 || ScaledOffset < -64)
-        break;
-    }
+    // Scaled offset must fit in the instruction immediate.
+    int ScaledOffset = UpdateOffset / Scale;
+    if (ScaledOffset > MaxOffset || ScaledOffset < MinOffset)
+      break;
 
     // If we have a non-zero Offset, we check that it matches the amount
     // we're adding to the register.
@@ -1442,13 +1500,19 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward(
   if (MIUnscaledOffset != UnscaledOffset)
     return E;
 
-  // If the base register overlaps a destination register, we can't
-  // merge the update.
-  bool IsPairedInsn = isPairedLdSt(MemMI);
-  for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
-    Register DestReg = getLdStRegOp(MemMI, i).getReg();
-    if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
-      return E;
+  // If the base register overlaps a source/destination register, we can't
+  // merge the update. This does not apply to tag store instructions which
+  // ignore the address part of the source register.
+  // This does not apply to STGPi as well, which does not have unpredictable
+  // behavior in this case unlike normal stores, and always performs writeback
+  // after reading the source register value.
+  if (!isTagStore(MemMI) && MemMI.getOpcode() != AArch64::STGPi) {
+    bool IsPairedInsn = isPairedLdSt(MemMI);
+    for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
+      Register DestReg = getLdStRegOp(MemMI, i).getReg();
+      if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
+        return E;
+    }
   }
 
   // Track which register units have been modified and used between the first
@@ -1496,11 +1560,13 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward(
     return E;
   // If the base register overlaps a destination register, we can't
   // merge the update.
-  bool IsPairedInsn = isPairedLdSt(MemMI);
-  for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
-    Register DestReg = getLdStRegOp(MemMI, i).getReg();
-    if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
-      return E;
+  if (!isTagStore(MemMI)) {
+    bool IsPairedInsn = isPairedLdSt(MemMI);
+    for (unsigned i = 0, e = IsPairedInsn ? 2 : 1; i != e; ++i) {
+      Register DestReg = getLdStRegOp(MemMI, i).getReg();
+      if (DestReg == BaseReg || TRI->isSubRegister(BaseReg, DestReg))
+        return E;
+    }
   }
 
   // Track which register units have been modified and used between the first
@@ -1659,7 +1725,7 @@ bool AArch64LoadStoreOpt::tryToMergeLdStUpdate
   // however, is not, so adjust here.
   int UnscaledOffset = getLdStOffsetOp(MI).getImm() * getMemScale(MI);
 
-  // Look forward to try to find a post-index instruction. For example,
+  // Look forward to try to find a pre-index instruction. For example,
   // ldr x1, [x0, #64]
   // add x0, x0, #64
   //   merged into:
diff --git a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
index 108970c64e30a..bea75b83517bb 100644
--- a/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
+++ b/llvm/lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp
@@ -27,12 +27,22 @@
 using namespace llvm;
 using namespace MIPatternMatch;
 
+#define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
+#include "AArch64GenGICombiner.inc"
+#undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
+
 namespace {
+#define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
+#include "AArch64GenGICombiner.inc"
+#undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
+
 class AArch64PreLegalizerCombinerInfo : public CombinerInfo {
   GISelKnownBits *KB;
   MachineDominatorTree *MDT;
 
 public:
+  AArch64GenPreLegalizerCombinerHelper Generated;
+
   AArch64PreLegalizerCombinerInfo(bool EnableOpt, bool OptSize, bool MinSize,
                                   GISelKnownBits *KB, MachineDominatorTree *MDT)
       : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
@@ -73,7 +83,7 @@ bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
       // heuristics decide.
       unsigned MaxLen = EnableOpt ? 0 : 32;
       // Try to inline memcpy type calls if optimizations are enabled.
-      return (!EnableOptSize) ? Helper.tryCombineMemCpyFamily(MI, MaxLen)
+      return (!EnableMinSize) ? Helper.tryCombineMemCpyFamily(MI, MaxLen)
                               : false;
     }
     default:
@@ -81,9 +91,16 @@ bool AArch64PreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
     }
   }
 
+  if (Generated.tryCombineAll(Observer, MI, B))
+    return true;
+
   return false;
 }
 
+#define AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
+#include "AArch64GenGICombiner.inc"
+#undef AARCH64PRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
+
 // Pass boilerplate
 // ================
 
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 0813c41dc66c9..d46e905d0fe73 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -97,11 +97,11 @@ let Predicates = [HasSVE] in {
   defm ABS_ZPmZ  : sve_int_un_pred_arit_0<  0b110, "abs", int_aarch64_sve_abs>;
   defm NEG_ZPmZ  : sve_int_un_pred_arit_0<  0b111, "neg", int_aarch64_sve_neg>;
 
-  defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls">;
-  defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz">;
-  defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt">;
-  defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot">;
-  defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not">;
+  defm CLS_ZPmZ  : sve_int_un_pred_arit_1<   0b000, "cls",  null_frag>;
+  defm CLZ_ZPmZ  : sve_int_un_pred_arit_1<   0b001, "clz",  null_frag>;
+  defm CNT_ZPmZ  : sve_int_un_pred_arit_1<   0b010, "cnt",  int_aarch64_sve_cnt>;
+  defm CNOT_ZPmZ : sve_int_un_pred_arit_1<   0b011, "cnot", null_frag>;
+  defm NOT_ZPmZ  : sve_int_un_pred_arit_1<   0b110, "not",  null_frag>;
   defm FABS_ZPmZ : sve_int_un_pred_arit_1_fp<0b100, "fabs">;
   defm FNEG_ZPmZ : sve_int_un_pred_arit_1_fp<0b101, "fneg">;
 
@@ -216,8 +216,8 @@ let Predicates = [HasSVE] in {
   defm UUNPKLO_ZZ : sve_int_perm_unpk<0b10, "uunpklo">;
   defm UUNPKHI_ZZ : sve_int_perm_unpk<0b11, "uunpkhi">;
 
-  def  PUNPKLO_PP : sve_int_perm_punpk<0b0, "punpklo">;
-  def  PUNPKHI_PP : sve_int_perm_punpk<0b1, "punpkhi">;
+  defm PUNPKLO_PP : sve_int_perm_punpk<0b0, "punpklo", int_aarch64_sve_punpklo>;
+  defm PUNPKHI_PP : sve_int_perm_punpk<0b1, "punpkhi", int_aarch64_sve_punpkhi>;
 
   defm MOVPRFX_ZPzZ : sve_int_movprfx_pred_zero<0b000, "movprfx">;
   defm MOVPRFX_ZPmZ : sve_int_movprfx_pred_merge<0b001, "movprfx">;
diff --git a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
index 4dfe743bafa8e..0c52711a8d7e9 100644
--- a/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
+++ b/llvm/lib/Target/AArch64/AArch64StackTagging.cpp
@@ -62,7 +62,7 @@ static cl::opt<bool> ClMergeInit(
 static cl::opt<unsigned> ClScanLimit("stack-tagging-merge-init-scan-limit",
                                      cl::init(40), cl::Hidden);
 
-static constexpr unsigned kTagGranuleSize = 16;
+static const Align kTagGranuleSize = Align(16);
 
 namespace {
 
@@ -458,7 +458,8 @@ Instruction *AArch64StackTagging::insertBaseTaggedPointer(
 }
 
 void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) {
-  unsigned NewAlignment = std::max(Info.AI->getAlignment(), kTagGranuleSize);
+  const Align NewAlignment =
+      max(MaybeAlign(Info.AI->getAlignment()), kTagGranuleSize);
   Info.AI->setAlignment(NewAlignment);
 
   uint64_t Size = Info.AI->getAllocationSizeInBits(*DL).getValue() / 8;
@@ -471,7 +472,7 @@ void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) {
       Info.AI->isArrayAllocation()
           ? ArrayType::get(
                 Info.AI->getAllocatedType(),
-                dyn_cast<ConstantInt>(Info.AI->getArraySize())->getZExtValue())
+                cast<ConstantInt>(Info.AI->getArraySize())->getZExtValue())
           : Info.AI->getAllocatedType();
   Type *PaddingType =
       ArrayType::get(Type::getInt8Ty(F->getContext()), AlignedSize - Size);
@@ -479,7 +480,7 @@ void AArch64StackTagging::alignAndPadAlloca(AllocaInfo &Info) {
   auto *NewAI = new AllocaInst(
       TypeWithPadding, Info.AI->getType()->getAddressSpace(), nullptr, "", Info.AI);
   NewAI->takeName(Info.AI);
-  NewAI->setAlignment(Info.AI->getAlignment());
+  NewAI->setAlignment(MaybeAlign(Info.AI->getAlignment()));
   NewAI->setUsedWithInAlloca(Info.AI->isUsedWithInAlloca());
   NewAI->setSwiftError(Info.AI->isSwiftError());
   NewAI->copyMetadata(*Info.AI);
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 11a4f991fedf0..b3ed96e815bea 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -283,9 +283,11 @@ AArch64TargetMachine::AArch64TargetMachine(const Target &T, const Triple &TT,
     this->Options.TrapUnreachable = true;
   }
 
-  // Enable GlobalISel at or below EnableGlobalISelAt0.
+  // Enable GlobalISel at or below EnableGlobalISelAt0, unless this is
+  // MachO/CodeModel::Large, which GlobalISel does not support.
   if (getOptLevel() <= EnableGlobalISelAtO &&
-      TT.getArch() != Triple::aarch64_32) {
+      TT.getArch() != Triple::aarch64_32 &&
+      !(getCodeModel() == CodeModel::Large && TT.isOSBinFormatMachO())) {
     setGlobalISel(true);
     setGlobalISelAbort(GlobalISelAbortMode::Disable);
   }
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index 94ce04c6e3471..7af0de7f647e0 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -5535,43 +5535,43 @@ unsigned AArch64AsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
   switch (Kind) {
   default:
     return Match_InvalidOperand;
-  case MCK__35_0:
+  case MCK__HASH_0:
     ExpectedVal = 0;
     break;
-  case MCK__35_1:
+  case MCK__HASH_1:
     ExpectedVal = 1;
     break;
-  case MCK__35_12:
+  case MCK__HASH_12:
     ExpectedVal = 12;
     break;
-  case MCK__35_16:
+  case MCK__HASH_16:
     ExpectedVal = 16;
     break;
-  case MCK__35_2:
+  case MCK__HASH_2:
     ExpectedVal = 2;
     break;
-  case MCK__35_24:
+  case MCK__HASH_24:
     ExpectedVal = 24;
     break;
-  case MCK__35_3:
+  case MCK__HASH_3:
     ExpectedVal = 3;
     break;
-  case MCK__35_32:
+  case MCK__HASH_32:
     ExpectedVal = 32;
     break;
-  case MCK__35_4:
+  case MCK__HASH_4:
     ExpectedVal = 4;
     break;
-  case MCK__35_48:
+  case MCK__HASH_48:
     ExpectedVal = 48;
     break;
-  case MCK__35_6:
+  case MCK__HASH_6:
     ExpectedVal = 6;
     break;
-  case MCK__35_64:
+  case MCK__HASH_64:
     ExpectedVal = 64;
     break;
-  case MCK__35_8:
+  case MCK__HASH_8:
     ExpectedVal = 8;
     break;
   }
diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt
index 0da057ea99736..103925d45d510 100644
--- a/llvm/lib/Target/AArch64/CMakeLists.txt
+++ b/llvm/lib/Target/AArch64/CMakeLists.txt
@@ -8,6 +8,8 @@ tablegen(LLVM AArch64GenDAGISel.inc -gen-dag-isel)
 tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler)
 tablegen(LLVM AArch64GenFastISel.inc -gen-fast-isel)
 tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel)
+tablegen(LLVM AArch64GenGICombiner.inc -gen-global-isel-combiner
+              -combiners="AArch64PreLegalizerCombinerHelper")
 tablegen(LLVM AArch64GenInstrInfo.inc -gen-instr-info)
 tablegen(LLVM AArch64GenMCCodeEmitter.inc -gen-emitter)
 tablegen(LLVM AArch64GenMCPseudoLowering.inc -gen-pseudo-lowering)
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index e7593be35247c..21ce5785ea5e1 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -573,7 +573,7 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
       case MCCFIInstruction::OpDefCfa: {
         // Defines a frame pointer.
         unsigned XReg =
-            getXRegFromWReg(MRI.getLLVMRegNum(Inst.getRegister(), true));
+            getXRegFromWReg(*MRI.getLLVMRegNum(Inst.getRegister(), true));
 
         // Other CFA registers than FP are not supported by compact unwind.
         // Fallback on DWARF.
@@ -592,8 +592,8 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
         assert(FPPush.getOperation() == MCCFIInstruction::OpOffset &&
                "Frame pointer not pushed!");
 
-        unsigned LRReg = MRI.getLLVMRegNum(LRPush.getRegister(), true);
-        unsigned FPReg = MRI.getLLVMRegNum(FPPush.getRegister(), true);
+        unsigned LRReg = *MRI.getLLVMRegNum(LRPush.getRegister(), true);
+        unsigned FPReg = *MRI.getLLVMRegNum(FPPush.getRegister(), true);
 
         LRReg = getXRegFromWReg(LRReg);
         FPReg = getXRegFromWReg(FPReg);
@@ -614,14 +614,14 @@ class DarwinAArch64AsmBackend : public AArch64AsmBackend {
       case MCCFIInstruction::OpOffset: {
         // Registers are saved in pairs. We expect there to be two consecutive
         // `.cfi_offset' instructions with the appropriate registers specified.
-        unsigned Reg1 = MRI.getLLVMRegNum(Inst.getRegister(), true);
+        unsigned Reg1 = *MRI.getLLVMRegNum(Inst.getRegister(), true);
         if (i + 1 == e)
           return CU::UNWIND_ARM64_MODE_DWARF;
 
         const MCCFIInstruction &Inst2 = Instrs[++i];
         if (Inst2.getOperation() != MCCFIInstruction::OpOffset)
           return CU::UNWIND_ARM64_MODE_DWARF;
-        unsigned Reg2 = MRI.getLLVMRegNum(Inst2.getRegister(), true);
+        unsigned Reg2 = *MRI.getLLVMRegNum(Inst2.getRegister(), true);
 
         // N.B. The encodings must be in register number order, and the X
         // registers before the D registers.
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 8c8713b464e81..1a9784065d5b5 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -283,6 +283,11 @@ let Predicates = [HasSVE] in {
 // SVE pattern match helpers.
 //===----------------------------------------------------------------------===//
 
+class SVE_1_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
+                   Instruction inst>
+: Pat<(vtd (op vt1:$Op1)),
+      (inst $Op1)>;
+
 class SVE_3_Op_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
                    ValueType vt2, ValueType vt3, Instruction inst>
 : Pat<(vtd (op vt1:$Op1, vt2:$Op2, vt3:$Op3)),
@@ -2871,11 +2876,21 @@ multiclass sve_int_un_pred_arit_0_d<bits<3> opc, string asm> {
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b0 }, asm, ZPR64>;
 }
 
-multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm> {
+multiclass sve_int_un_pred_arit_1<bits<3> opc, string asm,
+                                  SDPatternOperator op> {
   def _B : sve_int_un_pred_arit<0b00, { opc, 0b1 }, asm, ZPR8>;
   def _H : sve_int_un_pred_arit<0b01, { opc, 0b1 }, asm, ZPR16>;
   def _S : sve_int_un_pred_arit<0b10, { opc, 0b1 }, asm, ZPR32>;
   def _D : sve_int_un_pred_arit<0b11, { opc, 0b1 }, asm, ZPR64>;
+
+  def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1,  nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1,  nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1,  nxv2i64, !cast<Instruction>(NAME # _D)>;
+
+  def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
 multiclass sve_int_un_pred_arit_1_fp<bits<3> opc, string asm> {
@@ -4280,6 +4295,14 @@ class sve_int_perm_punpk<bit opc, string asm>
   let Inst{3-0}   = Pd;
 }
 
+multiclass sve_int_perm_punpk<bit opc, string asm, SDPatternOperator op> {
+  def NAME : sve_int_perm_punpk<opc, asm>;
+
+  def : SVE_1_Op_Pat<nxv8i1, op, nxv16i1, !cast<Instruction>(NAME)>;
+  def : SVE_1_Op_Pat<nxv4i1, op, nxv8i1,  !cast<Instruction>(NAME)>;
+  def : SVE_1_Op_Pat<nxv2i1, op, nxv4i1,  !cast<Instruction>(NAME)>;
+}
+
 class sve_int_rdffr_pred<bit s, string asm>
 : I<(outs PPR8:$Pd), (ins PPRAny:$Pg),
   asm, "\t$Pd, $Pg/z",
diff --git a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
index 7bb075c36e798..c27fc7a112ec9 100644
--- a/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
+++ b/llvm/lib/Target/AArch64/Utils/AArch64BaseInfo.cpp
@@ -125,7 +125,7 @@ namespace llvm {
 
 uint32_t AArch64SysReg::parseGenericRegister(StringRef Name) {
   // Try to parse an S<op0>_<op1>_<Cn>_<Cm>_<op2> register name
-  Regex GenericRegPattern("^S([0-3])_([0-7])_C([0-9]|1[0-5])_C([0-9]|1[0-5])_([0-7])$");
+  static const Regex GenericRegPattern("^S([0-3])_([0-7])_C([0-9]|1[0-5])_C([0-9]|1[0-5])_([0-7])$");
 
   std::string UpperName = Name.upper();
   SmallVector<StringRef, 5> Ops;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
index d8d2b6b899a42..e72b3f4fde633 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp
@@ -197,18 +197,12 @@ static bool handleAttr(Function &Parent, const Function &Callee,
 static void copyFeaturesToFunction(Function &Parent, const Function &Callee,
                                    bool &NeedQueuePtr) {
   // X ids unnecessarily propagated to kernels.
-  static const StringRef AttrNames[] = {
-    { "amdgpu-work-item-id-x" },
-    { "amdgpu-work-item-id-y" },
-    { "amdgpu-work-item-id-z" },
-    { "amdgpu-work-group-id-x" },
-    { "amdgpu-work-group-id-y" },
-    { "amdgpu-work-group-id-z" },
-    { "amdgpu-dispatch-ptr" },
-    { "amdgpu-dispatch-id" },
-    { "amdgpu-kernarg-segment-ptr" },
-    { "amdgpu-implicitarg-ptr" }
-  };
+  static constexpr StringLiteral AttrNames[] = {
+      "amdgpu-work-item-id-x",      "amdgpu-work-item-id-y",
+      "amdgpu-work-item-id-z",      "amdgpu-work-group-id-x",
+      "amdgpu-work-group-id-y",     "amdgpu-work-group-id-z",
+      "amdgpu-dispatch-ptr",        "amdgpu-dispatch-id",
+      "amdgpu-kernarg-segment-ptr", "amdgpu-implicitarg-ptr"};
 
   if (handleAttr(Parent, Callee, "amdgpu-queue-ptr"))
     NeedQueuePtr = true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 4b1d9cb5059f2..b83cc7f25284f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -229,7 +229,7 @@ void AMDGPUAsmPrinter::EmitFunctionBodyEnd() {
   // alignment.
   Streamer.EmitValueToAlignment(64, 0, 1, 0);
   if (ReadOnlySection.getAlignment() < 64)
-    ReadOnlySection.setAlignment(llvm::Align(64));
+    ReadOnlySection.setAlignment(Align(64));
 
   const MCSubtargetInfo &STI = MF->getSubtarget();
 
@@ -342,6 +342,8 @@ bool AMDGPUAsmPrinter::doFinalization(Module &M) {
 // Print comments that apply to both callable functions and entry points.
 void AMDGPUAsmPrinter::emitCommonFunctionComments(
   uint32_t NumVGPR,
+  Optional<uint32_t> NumAGPR,
+  uint32_t TotalNumVGPR,
   uint32_t NumSGPR,
   uint64_t ScratchSize,
   uint64_t CodeSize,
@@ -349,6 +351,11 @@ void AMDGPUAsmPrinter::emitCommonFunctionComments(
   OutStreamer->emitRawComment(" codeLenInByte = " + Twine(CodeSize), false);
   OutStreamer->emitRawComment(" NumSgprs: " + Twine(NumSGPR), false);
   OutStreamer->emitRawComment(" NumVgprs: " + Twine(NumVGPR), false);
+  if (NumAGPR) {
+    OutStreamer->emitRawComment(" NumAgprs: " + Twine(*NumAGPR), false);
+    OutStreamer->emitRawComment(" TotalNumVgprs: " + Twine(TotalNumVGPR),
+                                false);
+  }
   OutStreamer->emitRawComment(" ScratchSize: " + Twine(ScratchSize), false);
   OutStreamer->emitRawComment(" MemoryBound: " + Twine(MFI->isMemoryBound()),
                               false);
@@ -417,7 +424,7 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
   // The starting address of all shader programs must be 256 bytes aligned.
   // Regular functions just need the basic required instruction alignment.
-  MF.setAlignment(MFI->isEntryFunction() ? llvm::Align(256) : llvm::Align(4));
+  MF.setAlignment(MFI->isEntryFunction() ? Align(256) : Align(4));
 
   SetupMachineFunction(MF);
 
@@ -474,6 +481,8 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
       SIFunctionResourceInfo &Info = CallGraphResourceInfo[&MF.getFunction()];
       emitCommonFunctionComments(
         Info.NumVGPR,
+        STM.hasMAIInsts() ? Info.NumAGPR : Optional<uint32_t>(),
+        Info.getTotalNumVGPRs(STM),
         Info.getTotalNumSGPRs(MF.getSubtarget<GCNSubtarget>()),
         Info.PrivateSegmentSize,
         getFunctionCodeSize(MF), MFI);
@@ -481,7 +490,11 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
     }
 
     OutStreamer->emitRawComment(" Kernel info:", false);
-    emitCommonFunctionComments(CurrentProgramInfo.NumVGPR,
+    emitCommonFunctionComments(CurrentProgramInfo.NumArchVGPR,
+                               STM.hasMAIInsts()
+                                 ? CurrentProgramInfo.NumAccVGPR
+                                 : Optional<uint32_t>(),
+                               CurrentProgramInfo.NumVGPR,
                                CurrentProgramInfo.NumSGPR,
                                CurrentProgramInfo.ScratchSize,
                                getFunctionCodeSize(MF), MFI);
@@ -592,6 +605,11 @@ int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumSGPRs(
                                                      UsesVCC, UsesFlatScratch);
 }
 
+int32_t AMDGPUAsmPrinter::SIFunctionResourceInfo::getTotalNumVGPRs(
+  const GCNSubtarget &ST) const {
+  return std::max(NumVGPR, NumAGPR);
+}
+
 AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
   const MachineFunction &MF) const {
   SIFunctionResourceInfo Info;
@@ -638,11 +656,18 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
         HighestVGPRReg = Reg;
         break;
       }
-      MCPhysReg AReg = AMDGPU::AGPR0 + TRI.getHWRegIndex(Reg);
-      if (MRI.isPhysRegUsed(AReg)) {
-        HighestVGPRReg = AReg;
-        break;
+    }
+
+    if (ST.hasMAIInsts()) {
+      MCPhysReg HighestAGPRReg = AMDGPU::NoRegister;
+      for (MCPhysReg Reg : reverse(AMDGPU::AGPR_32RegClass.getRegisters())) {
+        if (MRI.isPhysRegUsed(Reg)) {
+          HighestAGPRReg = Reg;
+          break;
+        }
       }
+      Info.NumAGPR = HighestAGPRReg == AMDGPU::NoRegister ? 0 :
+        TRI.getHWRegIndex(HighestAGPRReg) + 1;
     }
 
     MCPhysReg HighestSGPRReg = AMDGPU::NoRegister;
@@ -664,6 +689,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
   }
 
   int32_t MaxVGPR = -1;
+  int32_t MaxAGPR = -1;
   int32_t MaxSGPR = -1;
   uint64_t CalleeFrameSize = 0;
 
@@ -673,6 +699,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
       for (const MachineOperand &MO : MI.operands()) {
         unsigned Width = 0;
         bool IsSGPR = false;
+        bool IsAGPR = false;
 
         if (!MO.isReg())
           continue;
@@ -748,6 +775,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           Width = 1;
         } else if (AMDGPU::AGPR_32RegClass.contains(Reg)) {
           IsSGPR = false;
+          IsAGPR = true;
           Width = 1;
         } else if (AMDGPU::SReg_64RegClass.contains(Reg)) {
           assert(!AMDGPU::TTMP_64RegClass.contains(Reg) &&
@@ -759,6 +787,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           Width = 2;
         } else if (AMDGPU::AReg_64RegClass.contains(Reg)) {
           IsSGPR = false;
+          IsAGPR = true;
           Width = 2;
         } else if (AMDGPU::VReg_96RegClass.contains(Reg)) {
           IsSGPR = false;
@@ -775,6 +804,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           Width = 4;
         } else if (AMDGPU::AReg_128RegClass.contains(Reg)) {
           IsSGPR = false;
+          IsAGPR = true;
           Width = 4;
         } else if (AMDGPU::SReg_256RegClass.contains(Reg)) {
           assert(!AMDGPU::TTMP_256RegClass.contains(Reg) &&
@@ -794,6 +824,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           Width = 16;
         } else if (AMDGPU::AReg_512RegClass.contains(Reg)) {
           IsSGPR = false;
+          IsAGPR = true;
           Width = 16;
         } else if (AMDGPU::SReg_1024RegClass.contains(Reg)) {
           IsSGPR = true;
@@ -803,6 +834,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
           Width = 32;
         } else if (AMDGPU::AReg_1024RegClass.contains(Reg)) {
           IsSGPR = false;
+          IsAGPR = true;
           Width = 32;
         } else {
           llvm_unreachable("Unknown register class");
@@ -811,6 +843,8 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
         int MaxUsed = HWReg + Width - 1;
         if (IsSGPR) {
           MaxSGPR = MaxUsed > MaxSGPR ? MaxUsed : MaxSGPR;
+        } else if (IsAGPR) {
+          MaxAGPR = MaxUsed > MaxAGPR ? MaxUsed : MaxAGPR;
         } else {
           MaxVGPR = MaxUsed > MaxVGPR ? MaxUsed : MaxVGPR;
         }
@@ -832,6 +866,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
             47 - IsaInfo::getNumExtraSGPRs(&ST, true, ST.hasFlatAddressSpace());
           MaxSGPR = std::max(MaxSGPR, MaxSGPRGuess);
           MaxVGPR = std::max(MaxVGPR, 23);
+          MaxAGPR = std::max(MaxAGPR, 23);
 
           CalleeFrameSize = std::max(CalleeFrameSize, UINT64_C(16384));
           Info.UsesVCC = true;
@@ -856,6 +891,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
 
           MaxSGPR = std::max(I->second.NumExplicitSGPR - 1, MaxSGPR);
           MaxVGPR = std::max(I->second.NumVGPR - 1, MaxVGPR);
+          MaxAGPR = std::max(I->second.NumAGPR - 1, MaxAGPR);
           CalleeFrameSize
             = std::max(I->second.PrivateSegmentSize, CalleeFrameSize);
           Info.UsesVCC |= I->second.UsesVCC;
@@ -872,6 +908,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
 
   Info.NumExplicitSGPR = MaxSGPR + 1;
   Info.NumVGPR = MaxVGPR + 1;
+  Info.NumAGPR = MaxAGPR + 1;
   Info.PrivateSegmentSize += CalleeFrameSize;
 
   return Info;
@@ -880,8 +917,11 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage(
 void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
                                         const MachineFunction &MF) {
   SIFunctionResourceInfo Info = analyzeResourceUsage(MF);
+  const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
 
-  ProgInfo.NumVGPR = Info.NumVGPR;
+  ProgInfo.NumArchVGPR = Info.NumVGPR;
+  ProgInfo.NumAccVGPR = Info.NumAGPR;
+  ProgInfo.NumVGPR = Info.getTotalNumVGPRs(STM);
   ProgInfo.NumSGPR = Info.NumExplicitSGPR;
   ProgInfo.ScratchSize = Info.PrivateSegmentSize;
   ProgInfo.VCCUsed = Info.UsesVCC;
@@ -894,7 +934,6 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
     MF.getFunction().getContext().diagnose(DiagStackSize);
   }
 
-  const GCNSubtarget &STM = MF.getSubtarget<GCNSubtarget>();
   const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
 
   // TODO(scott.linder): The calculations related to SGPR/VGPR blocks are
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
index 959104dbc3506..c50c19a4609c6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.h
@@ -43,6 +43,7 @@ class AMDGPUAsmPrinter final : public AsmPrinter {
     // Track the number of explicitly used VGPRs. Special registers reserved at
     // the end are tracked separately.
     int32_t NumVGPR = 0;
+    int32_t NumAGPR = 0;
     int32_t NumExplicitSGPR = 0;
     uint64_t PrivateSegmentSize = 0;
     bool UsesVCC = false;
@@ -51,6 +52,7 @@ class AMDGPUAsmPrinter final : public AsmPrinter {
     bool HasRecursion = false;
 
     int32_t getTotalNumSGPRs(const GCNSubtarget &ST) const;
+    int32_t getTotalNumVGPRs(const GCNSubtarget &ST) const;
   };
 
   SIProgramInfo CurrentProgramInfo;
@@ -77,6 +79,8 @@ class AMDGPUAsmPrinter final : public AsmPrinter {
   void EmitPALMetadata(const MachineFunction &MF,
                        const SIProgramInfo &KernelInfo);
   void emitCommonFunctionComments(uint32_t NumVGPR,
+                                  Optional<uint32_t> NumAGPR,
+                                  uint32_t TotalNumVGPR,
                                   uint32_t NumSGPR,
                                   uint64_t ScratchSize,
                                   uint64_t CodeSize,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
index 944f782db244d..6ee11686f4859 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td
@@ -202,3 +202,6 @@ def : GISelVop2Pat <or, V_OR_B32_e32, i32>;
 foreach Ty = [i64, p0, p1, p4] in {
   defm : SMRD_Pattern <"S_LOAD_DWORDX2",  Ty>;
 }
+
+def gi_as_i32timm : GICustomOperandRenderer<"renderTruncImm32">,
+  GISDNodeXFormEquiv<as_i32timm>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def b/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
index 3b3630bce66f7..85d1ad3491573 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def
@@ -22,15 +22,17 @@ enum PartialMappingIdx {
   PM_SGPR128 = 9,
   PM_SGPR256 = 10,
   PM_SGPR512 = 11,
-  PM_VGPR1  = 12,
-  PM_VGPR16 = 16,
-  PM_VGPR32 = 17,
-  PM_VGPR64 = 18,
-  PM_VGPR128 = 19,
-  PM_VGPR256 = 20,
-  PM_VGPR512 = 21,
-  PM_SGPR96 = 22,
-  PM_VGPR96 = 23
+  PM_SGPR1024 = 12,
+  PM_VGPR1  = 13,
+  PM_VGPR16 = 17,
+  PM_VGPR32 = 18,
+  PM_VGPR64 = 19,
+  PM_VGPR128 = 20,
+  PM_VGPR256 = 21,
+  PM_VGPR512 = 22,
+  PM_VGPR1024 = 23,
+  PM_SGPR96 = 24,
+  PM_VGPR96 = 25
 };
 
 const RegisterBankInfo::PartialMapping PartMappings[] {
@@ -45,6 +47,7 @@ const RegisterBankInfo::PartialMapping PartMappings[] {
   {0, 128, SGPRRegBank},
   {0, 256, SGPRRegBank},
   {0, 512, SGPRRegBank},
+  {0, 1024, SGPRRegBank},
 
   {0, 1,  VGPRRegBank}, // VGPR begin
   {0, 16, VGPRRegBank},
@@ -53,8 +56,9 @@ const RegisterBankInfo::PartialMapping PartMappings[] {
   {0, 128, VGPRRegBank},
   {0, 256, VGPRRegBank},
   {0, 512, VGPRRegBank},
+  {0, 1024, VGPRRegBank},
   {0, 96, SGPRRegBank},
-  {0, 96, VGPRRegBank},
+  {0, 96, VGPRRegBank}
 };
 
 const RegisterBankInfo::ValueMapping ValMappings[] {
@@ -65,30 +69,32 @@ const RegisterBankInfo::ValueMapping ValMappings[] {
   {&PartMappings[1], 1},
 
   // SGPRs
-  {&PartMappings[2], 1},
+  {&PartMappings[2], 1}, // 1
   {nullptr, 0}, // Illegal power of 2 sizes
   {nullptr, 0},
   {nullptr, 0},
-  {&PartMappings[3], 1},
-  {&PartMappings[4], 1},
-  {&PartMappings[5], 1},
-  {&PartMappings[6], 1},
-  {&PartMappings[7], 1},
-  {&PartMappings[8], 1},
-
-    // VGPRs
-  {&PartMappings[9], 1},
+  {&PartMappings[3], 1}, // 16
+  {&PartMappings[4], 1}, // 32
+  {&PartMappings[5], 1}, // 64
+  {&PartMappings[6], 1}, // 128
+  {&PartMappings[7], 1}, // 256
+  {&PartMappings[8], 1}, // 512
+  {&PartMappings[9], 1}, // 1024
+
+  // VGPRs
+  {&PartMappings[10], 1}, // 1
   {nullptr, 0},
   {nullptr, 0},
   {nullptr, 0},
-  {&PartMappings[10], 1},
-  {&PartMappings[11], 1},
-  {&PartMappings[12], 1},
-  {&PartMappings[13], 1},
-  {&PartMappings[14], 1},
-  {&PartMappings[15], 1},
-  {&PartMappings[16], 1},
-  {&PartMappings[17], 1}
+  {&PartMappings[11], 1}, // 16
+  {&PartMappings[12], 1}, // 32
+  {&PartMappings[13], 1}, // 64
+  {&PartMappings[14], 1}, // 128
+  {&PartMappings[15], 1}, // 256
+  {&PartMappings[16], 1}, // 512
+  {&PartMappings[17], 1}, // 1024
+  {&PartMappings[18], 1},
+  {&PartMappings[19], 1}
 };
 
 const RegisterBankInfo::PartialMapping SGPROnly64BreakDown[] {
@@ -116,7 +122,7 @@ const RegisterBankInfo::ValueMapping ValMappingsSGPR64OnlyVGPR32[] {
 enum ValueMappingIdx {
   SCCStartIdx = 0,
   SGPRStartIdx = 2,
-  VGPRStartIdx = 12
+  VGPRStartIdx = 13
 };
 
 const RegisterBankInfo::ValueMapping *getValueMapping(unsigned BankID,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index b2491ebc6f48f..c74a361b2c712 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -186,10 +186,11 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
   bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
                    SDValue &SOffset, SDValue &Offset, SDValue &Offen,
                    SDValue &Idxen, SDValue &Addr64, SDValue &GLC, SDValue &SLC,
-                   SDValue &TFE, SDValue &DLC) const;
+                   SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
                          SDValue &SOffset, SDValue &Offset, SDValue &GLC,
-                         SDValue &SLC, SDValue &TFE, SDValue &DLC) const;
+                         SDValue &SLC, SDValue &TFE, SDValue &DLC,
+                         SDValue &SWZ) const;
   bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                          SDValue &VAddr, SDValue &SOffset, SDValue &Offset,
                          SDValue &SLC) const;
@@ -202,7 +203,7 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
 
   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &SOffset,
                          SDValue &Offset, SDValue &GLC, SDValue &SLC,
-                         SDValue &TFE, SDValue &DLC) const;
+                         SDValue &TFE, SDValue &DLC, SDValue &SWZ) const;
   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
                          SDValue &Offset, SDValue &SLC) const;
   bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
@@ -1313,7 +1314,8 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
                                      SDValue &Offset, SDValue &Offen,
                                      SDValue &Idxen, SDValue &Addr64,
                                      SDValue &GLC, SDValue &SLC,
-                                     SDValue &TFE, SDValue &DLC) const {
+                                     SDValue &TFE, SDValue &DLC,
+                                     SDValue &SWZ) const {
   // Subtarget prefers to use flat instruction
   if (Subtarget->useFlatForGlobal())
     return false;
@@ -1326,6 +1328,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
     SLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
   TFE = CurDAG->getTargetConstant(0, DL, MVT::i1);
   DLC = CurDAG->getTargetConstant(0, DL, MVT::i1);
+  SWZ = CurDAG->getTargetConstant(0, DL, MVT::i1);
 
   Idxen = CurDAG->getTargetConstant(0, DL, MVT::i1);
   Offen = CurDAG->getTargetConstant(0, DL, MVT::i1);
@@ -1405,7 +1408,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                                            SDValue &VAddr, SDValue &SOffset,
                                            SDValue &Offset, SDValue &GLC,
                                            SDValue &SLC, SDValue &TFE,
-                                           SDValue &DLC) const {
+                                           SDValue &DLC, SDValue &SWZ) const {
   SDValue Ptr, Offen, Idxen, Addr64;
 
   // addr64 bit was removed for volcanic islands.
@@ -1413,7 +1416,7 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
     return false;
 
   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
-              GLC, SLC, TFE, DLC))
+              GLC, SLC, TFE, DLC, SWZ))
     return false;
 
   ConstantSDNode *C = cast<ConstantSDNode>(Addr64);
@@ -1435,9 +1438,9 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc,
                                            SDValue &Offset,
                                            SDValue &SLC) const {
   SLC = CurDAG->getTargetConstant(0, SDLoc(Addr), MVT::i1);
-  SDValue GLC, TFE, DLC;
+  SDValue GLC, TFE, DLC, SWZ;
 
-  return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC);
+  return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE, DLC, SWZ);
 }
 
 static bool isStackPtrRelative(const MachinePointerInfo &PtrInfo) {
@@ -1562,13 +1565,14 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFScratchOffset(SDNode *Parent,
 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
                                            SDValue &SOffset, SDValue &Offset,
                                            SDValue &GLC, SDValue &SLC,
-                                           SDValue &TFE, SDValue &DLC) const {
+                                           SDValue &TFE, SDValue &DLC,
+                                           SDValue &SWZ) const {
   SDValue Ptr, VAddr, Offen, Idxen, Addr64;
   const SIInstrInfo *TII =
     static_cast<const SIInstrInfo *>(Subtarget->getInstrInfo());
 
   if (!SelectMUBUF(Addr, Ptr, VAddr, SOffset, Offset, Offen, Idxen, Addr64,
-              GLC, SLC, TFE, DLC))
+              GLC, SLC, TFE, DLC, SWZ))
     return false;
 
   if (!cast<ConstantSDNode>(Offen)->getSExtValue() &&
@@ -1590,16 +1594,16 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
                                            SDValue &Soffset, SDValue &Offset
                                            ) const {
-  SDValue GLC, SLC, TFE, DLC;
+  SDValue GLC, SLC, TFE, DLC, SWZ;
 
-  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC);
+  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
 }
 bool AMDGPUDAGToDAGISel::SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc,
                                            SDValue &Soffset, SDValue &Offset,
                                            SDValue &SLC) const {
-  SDValue GLC, TFE, DLC;
+  SDValue GLC, TFE, DLC, SWZ;
 
-  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC);
+  return SelectMUBUFOffset(Addr, SRsrc, Soffset, Offset, GLC, SLC, TFE, DLC, SWZ);
 }
 
 template <bool IsSigned>
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 6b92ad0599a77..65fa2720b5153 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -134,7 +134,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
     setLoadExtAction(ISD::EXTLOAD, VT, MVT::i32, Expand);
   }
 
-  for (MVT VT : MVT::integer_vector_valuetypes()) {
+  for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
     setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Expand);
     setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Expand);
     setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i8, Expand);
@@ -681,8 +681,9 @@ bool AMDGPUTargetLowering::isLoadBitCastBeneficial(EVT LoadTy, EVT CastTy,
     return false;
 
   bool Fast = false;
-  return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), CastTy,
-                            MMO, &Fast) && Fast;
+  return allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                        CastTy, MMO, &Fast) &&
+         Fast;
 }
 
 // SI+ has instructions for cttz / ctlz for 32-bit values. This is probably also
@@ -718,9 +719,8 @@ bool AMDGPUTargetLowering::isSDNodeAlwaysUniform(const SDNode * N) const {
     break;
     case ISD::LOAD:
     {
-      const LoadSDNode * L = dyn_cast<LoadSDNode>(N);
-      if (L->getMemOperand()->getAddrSpace()
-      == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
+      if (cast<LoadSDNode>(N)->getMemOperand()->getAddrSpace() ==
+          AMDGPUAS::CONSTANT_ADDRESS_32BIT)
         return true;
       return false;
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 23ca49b8a84c4..c5e60ed77be69 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
 #include "llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h"
+#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/Utils.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
@@ -62,6 +63,12 @@ AMDGPUInstructionSelector::AMDGPUInstructionSelector(
 
 const char *AMDGPUInstructionSelector::getName() { return DEBUG_TYPE; }
 
+void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits &KB,
+                                        CodeGenCoverage &CoverageInfo) {
+  MRI = &MF.getRegInfo();
+  InstructionSelector::setupMF(MF, KB, CoverageInfo);
+}
+
 static bool isSCC(Register Reg, const MachineRegisterInfo &MRI) {
   if (Register::isPhysicalRegister(Reg))
     return Reg == AMDGPU::SCC;
@@ -103,8 +110,6 @@ bool AMDGPUInstructionSelector::isVCC(Register Reg,
 bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
   const DebugLoc &DL = I.getDebugLoc();
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   I.setDesc(TII.get(TargetOpcode::COPY));
 
   const MachineOperand &Src = I.getOperand(1);
@@ -112,33 +117,33 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
   Register DstReg = Dst.getReg();
   Register SrcReg = Src.getReg();
 
-  if (isVCC(DstReg, MRI)) {
+  if (isVCC(DstReg, *MRI)) {
     if (SrcReg == AMDGPU::SCC) {
       const TargetRegisterClass *RC
-        = TRI.getConstrainedRegClassForOperand(Dst, MRI);
+        = TRI.getConstrainedRegClassForOperand(Dst, *MRI);
       if (!RC)
         return true;
-      return RBI.constrainGenericRegister(DstReg, *RC, MRI);
+      return RBI.constrainGenericRegister(DstReg, *RC, *MRI);
     }
 
-    if (!isVCC(SrcReg, MRI)) {
+    if (!isVCC(SrcReg, *MRI)) {
       // TODO: Should probably leave the copy and let copyPhysReg expand it.
-      if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), MRI))
+      if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI))
         return false;
 
       BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg)
         .addImm(0)
         .addReg(SrcReg);
 
-      if (!MRI.getRegClassOrNull(SrcReg))
-        MRI.setRegClass(SrcReg, TRI.getConstrainedRegClassForOperand(Src, MRI));
+      if (!MRI->getRegClassOrNull(SrcReg))
+        MRI->setRegClass(SrcReg, TRI.getConstrainedRegClassForOperand(Src, *MRI));
       I.eraseFromParent();
       return true;
     }
 
     const TargetRegisterClass *RC =
-      TRI.getConstrainedRegClassForOperand(Dst, MRI);
-    if (RC && !RBI.constrainGenericRegister(DstReg, *RC, MRI))
+      TRI.getConstrainedRegClassForOperand(Dst, *MRI);
+    if (RC && !RBI.constrainGenericRegister(DstReg, *RC, *MRI))
       return false;
 
     // Don't constrain the source register to a class so the def instruction
@@ -149,8 +154,8 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
     // with size 1. An SReg_32 with size 1 is ambiguous with wave32.
     if (Src.isUndef()) {
       const TargetRegisterClass *SrcRC =
-        TRI.getConstrainedRegClassForOperand(Src, MRI);
-      if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI))
+        TRI.getConstrainedRegClassForOperand(Src, *MRI);
+      if (SrcRC && !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
         return false;
     }
 
@@ -162,26 +167,22 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
       continue;
 
     const TargetRegisterClass *RC =
-            TRI.getConstrainedRegClassForOperand(MO, MRI);
+            TRI.getConstrainedRegClassForOperand(MO, *MRI);
     if (!RC)
       continue;
-    RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
+    RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
   }
   return true;
 }
 
 bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
-  MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-
   const Register DefReg = I.getOperand(0).getReg();
-  const LLT DefTy = MRI.getType(DefReg);
+  const LLT DefTy = MRI->getType(DefReg);
 
   // TODO: Verify this doesn't have insane operands (i.e. VGPR to SGPR copy)
 
   const RegClassOrRegBank &RegClassOrBank =
-    MRI.getRegClassOrRegBank(DefReg);
+    MRI->getRegClassOrRegBank(DefReg);
 
   const TargetRegisterClass *DefRC
     = RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
@@ -197,7 +198,7 @@ bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
       return false;
     }
 
-    DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, MRI);
+    DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, *MRI);
     if (!DefRC) {
       LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
       return false;
@@ -205,7 +206,7 @@ bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const {
   }
 
   I.setDesc(TII.get(TargetOpcode::PHI));
-  return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
+  return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
 }
 
 MachineOperand
@@ -215,9 +216,7 @@ AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
 
   MachineInstr *MI = MO.getParent();
   MachineBasicBlock *BB = MO.getParent()->getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-  Register DstReg = MRI.createVirtualRegister(&SubRC);
+  Register DstReg = MRI->createVirtualRegister(&SubRC);
 
   if (MO.isReg()) {
     unsigned ComposedSubIdx = TRI.composeSubRegIndices(MO.getSubReg(), SubIdx);
@@ -245,10 +244,6 @@ AMDGPUInstructionSelector::getSubOperand64(MachineOperand &MO,
   }
 }
 
-static int64_t getConstant(const MachineInstr *MI) {
-  return MI->getOperand(1).getCImm()->getSExtValue();
-}
-
 static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
   switch (Opc) {
   case AMDGPU::G_AND:
@@ -263,16 +258,13 @@ static unsigned getLogicalBitOpcode(unsigned Opc, bool Is64) {
 }
 
 bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
-  MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   MachineOperand &Dst = I.getOperand(0);
   MachineOperand &Src0 = I.getOperand(1);
   MachineOperand &Src1 = I.getOperand(2);
   Register DstReg = Dst.getReg();
-  unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
+  unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
 
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
   if (DstRB->getID() == AMDGPU::VCCRegBankID) {
     const TargetRegisterClass *RC = TRI.getBoolRC();
     unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(),
@@ -283,12 +275,12 @@ bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
     // The selector for G_ICMP relies on seeing the register bank for the result
     // is VCC. In wave32 if we constrain the registers to SReg_32 here, it will
     // be ambiguous whether it's a scalar or vector bool.
-    if (Src0.isUndef() && !MRI.getRegClassOrNull(Src0.getReg()))
-      MRI.setRegClass(Src0.getReg(), RC);
-    if (Src1.isUndef() && !MRI.getRegClassOrNull(Src1.getReg()))
-      MRI.setRegClass(Src1.getReg(), RC);
+    if (Src0.isUndef() && !MRI->getRegClassOrNull(Src0.getReg()))
+      MRI->setRegClass(Src0.getReg(), RC);
+    if (Src1.isUndef() && !MRI->getRegClassOrNull(Src1.getReg()))
+      MRI->setRegClass(Src1.getReg(), RC);
 
-    return RBI.constrainGenericRegister(DstReg, *RC, MRI);
+    return RBI.constrainGenericRegister(DstReg, *RC, *MRI);
   }
 
   // TODO: Should this allow an SCC bank result, and produce a copy from SCC for
@@ -305,11 +297,10 @@ bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const {
 bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
   MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   Register DstReg = I.getOperand(0).getReg();
   const DebugLoc &DL = I.getDebugLoc();
-  unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
+  unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
   const bool IsSALU = DstRB->getID() == AMDGPU::SGPRRegBankID;
   const bool Sub = I.getOpcode() == TargetOpcode::G_SUB;
 
@@ -334,7 +325,7 @@ bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
 
     const unsigned Opc = Sub ? AMDGPU::V_SUB_I32_e64 : AMDGPU::V_ADD_I32_e64;
 
-    Register UnusedCarry = MRI.createVirtualRegister(TRI.getWaveMaskRegClass());
+    Register UnusedCarry = MRI->createVirtualRegister(TRI.getWaveMaskRegClass());
     MachineInstr *Add
       = BuildMI(*BB, &I, DL, TII.get(Opc), DstReg)
       .addDef(UnusedCarry, RegState::Dead)
@@ -357,8 +348,8 @@ bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
   MachineOperand Hi1(getSubOperand64(I.getOperand(1), HalfRC, AMDGPU::sub1));
   MachineOperand Hi2(getSubOperand64(I.getOperand(2), HalfRC, AMDGPU::sub1));
 
-  Register DstLo = MRI.createVirtualRegister(&HalfRC);
-  Register DstHi = MRI.createVirtualRegister(&HalfRC);
+  Register DstLo = MRI->createVirtualRegister(&HalfRC);
+  Register DstHi = MRI->createVirtualRegister(&HalfRC);
 
   if (IsSALU) {
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_ADD_U32), DstLo)
@@ -369,14 +360,14 @@ bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
       .add(Hi2);
   } else {
     const TargetRegisterClass *CarryRC = TRI.getWaveMaskRegClass();
-    Register CarryReg = MRI.createVirtualRegister(CarryRC);
+    Register CarryReg = MRI->createVirtualRegister(CarryRC);
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADD_I32_e64), DstLo)
       .addDef(CarryReg)
       .add(Lo1)
       .add(Lo2)
       .addImm(0);
     MachineInstr *Addc = BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_ADDC_U32_e64), DstHi)
-      .addDef(MRI.createVirtualRegister(CarryRC), RegState::Dead)
+      .addDef(MRI->createVirtualRegister(CarryRC), RegState::Dead)
       .add(Hi1)
       .add(Hi2)
       .addReg(CarryReg, RegState::Kill)
@@ -393,17 +384,56 @@ bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const {
     .addImm(AMDGPU::sub1);
 
 
-  if (!RBI.constrainGenericRegister(DstReg, RC, MRI))
+  if (!RBI.constrainGenericRegister(DstReg, RC, *MRI))
     return false;
 
   I.eraseFromParent();
   return true;
 }
 
-bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
+bool AMDGPUInstructionSelector::selectG_UADDO_USUBO(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
   MachineFunction *MF = BB->getParent();
   MachineRegisterInfo &MRI = MF->getRegInfo();
+  const DebugLoc &DL = I.getDebugLoc();
+  Register Dst0Reg = I.getOperand(0).getReg();
+  Register Dst1Reg = I.getOperand(1).getReg();
+  const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO;
+
+  if (!isSCC(Dst1Reg, MRI)) {
+    // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned
+    // carry out despite the _i32 name. These were renamed in VI to _U32.
+    // FIXME: We should probably rename the opcodes here.
+    unsigned NewOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64;
+    I.setDesc(TII.get(NewOpc));
+    I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
+    I.addOperand(*MF, MachineOperand::CreateImm(0));
+    return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
+  }
+
+  Register Src0Reg = I.getOperand(2).getReg();
+  Register Src1Reg = I.getOperand(3).getReg();
+  unsigned NewOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32;
+  BuildMI(*BB, &I, DL, TII.get(NewOpc), Dst0Reg)
+    .add(I.getOperand(2))
+    .add(I.getOperand(3));
+  BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg)
+    .addReg(AMDGPU::SCC);
+
+  if (!MRI.getRegClassOrNull(Dst1Reg))
+    MRI.setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass);
+
+  if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, MRI) ||
+      !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, MRI) ||
+      !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, MRI))
+    return false;
+
+  I.eraseFromParent();
+  return true;
+}
+
+bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
+  MachineBasicBlock *BB = I.getParent();
   assert(I.getOperand(2).getImm() % 32 == 0);
   unsigned SubReg = TRI.getSubRegFromChannel(I.getOperand(2).getImm() / 32);
   const DebugLoc &DL = I.getDebugLoc();
@@ -413,10 +443,10 @@ bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
 
   for (const MachineOperand &MO : Copy->operands()) {
     const TargetRegisterClass *RC =
-            TRI.getConstrainedRegClassForOperand(MO, MRI);
+            TRI.getConstrainedRegClassForOperand(MO, *MRI);
     if (!RC)
       continue;
-    RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
+    RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
   }
   I.eraseFromParent();
   return true;
@@ -424,21 +454,19 @@ bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const {
 
 bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
   MachineBasicBlock *BB = MI.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   Register DstReg = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(DstReg);
-  LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
+  LLT DstTy = MRI->getType(DstReg);
+  LLT SrcTy = MRI->getType(MI.getOperand(1).getReg());
 
   const unsigned SrcSize = SrcTy.getSizeInBits();
   if (SrcSize < 32)
     return false;
 
   const DebugLoc &DL = MI.getDebugLoc();
-  const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, TRI);
+  const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI);
   const unsigned DstSize = DstTy.getSizeInBits();
   const TargetRegisterClass *DstRC =
-    TRI.getRegClassForSizeOnBank(DstSize, *DstBank, MRI);
+    TRI.getRegClassForSizeOnBank(DstSize, *DstBank, *MRI);
   if (!DstRC)
     return false;
 
@@ -451,12 +479,12 @@ bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
     MIB.addImm(SubRegs[I]);
 
     const TargetRegisterClass *SrcRC
-      = TRI.getConstrainedRegClassForOperand(Src, MRI);
-    if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, MRI))
+      = TRI.getConstrainedRegClassForOperand(Src, *MRI);
+    if (SrcRC && !RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI))
       return false;
   }
 
-  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI))
+  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
     return false;
 
   MI.eraseFromParent();
@@ -465,25 +493,23 @@ bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const {
 
 bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
   MachineBasicBlock *BB = MI.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   const int NumDst = MI.getNumOperands() - 1;
 
   MachineOperand &Src = MI.getOperand(NumDst);
 
   Register SrcReg = Src.getReg();
   Register DstReg0 = MI.getOperand(0).getReg();
-  LLT DstTy = MRI.getType(DstReg0);
-  LLT SrcTy = MRI.getType(SrcReg);
+  LLT DstTy = MRI->getType(DstReg0);
+  LLT SrcTy = MRI->getType(SrcReg);
 
   const unsigned DstSize = DstTy.getSizeInBits();
   const unsigned SrcSize = SrcTy.getSizeInBits();
   const DebugLoc &DL = MI.getDebugLoc();
-  const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
+  const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
 
   const TargetRegisterClass *SrcRC =
-    TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, MRI);
-  if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI))
+    TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI);
+  if (!SrcRC || !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
     return false;
 
   const unsigned SrcFlags = getUndefRegState(Src.isUndef());
@@ -498,8 +524,8 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const {
       .addReg(SrcReg, SrcFlags, SubRegs[I]);
 
     const TargetRegisterClass *DstRC =
-      TRI.getConstrainedRegClassForOperand(Dst, MRI);
-    if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, MRI))
+      TRI.getConstrainedRegClassForOperand(Dst, *MRI);
+    if (DstRC && !RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI))
       return false;
   }
 
@@ -512,16 +538,13 @@ bool AMDGPUInstructionSelector::selectG_GEP(MachineInstr &I) const {
 }
 
 bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
-  MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   const MachineOperand &MO = I.getOperand(0);
 
   // FIXME: Interface for getConstrainedRegClassForOperand needs work. The
   // regbank check here is to know why getConstrainedRegClassForOperand failed.
-  const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, MRI);
-  if ((!RC && !MRI.getRegBankOrNull(MO.getReg())) ||
-      (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, MRI))) {
+  const TargetRegisterClass *RC = TRI.getConstrainedRegClassForOperand(MO, *MRI);
+  if ((!RC && !MRI->getRegBankOrNull(MO.getReg())) ||
+      (RC && RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI))) {
     I.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
     return true;
   }
@@ -531,14 +554,23 @@ bool AMDGPUInstructionSelector::selectG_IMPLICIT_DEF(MachineInstr &I) const {
 
 bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-  unsigned SubReg = TRI.getSubRegFromChannel(I.getOperand(3).getImm() / 32);
-  DebugLoc DL = I.getDebugLoc();
+  Register Src0Reg = I.getOperand(1).getReg();
+  Register Src1Reg = I.getOperand(2).getReg();
+  LLT Src1Ty = MRI->getType(Src1Reg);
+  if (Src1Ty.getSizeInBits() != 32)
+    return false;
+
+  int64_t Offset = I.getOperand(3).getImm();
+  if (Offset % 32 != 0)
+    return false;
+
+  unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32);
+  const DebugLoc &DL = I.getDebugLoc();
+
   MachineInstr *Ins = BuildMI(*BB, &I, DL, TII.get(TargetOpcode::INSERT_SUBREG))
                                .addDef(I.getOperand(0).getReg())
-                               .addReg(I.getOperand(1).getReg())
-                               .addReg(I.getOperand(2).getReg())
+                               .addReg(Src0Reg)
+                               .addReg(Src1Reg)
                                .addImm(SubReg);
 
   for (const MachineOperand &MO : Ins->operands()) {
@@ -548,22 +580,20 @@ bool AMDGPUInstructionSelector::selectG_INSERT(MachineInstr &I) const {
       continue;
 
     const TargetRegisterClass *RC =
-            TRI.getConstrainedRegClassForOperand(MO, MRI);
+            TRI.getConstrainedRegClassForOperand(MO, *MRI);
     if (!RC)
       continue;
-    RBI.constrainGenericRegister(MO.getReg(), *RC, MRI);
+    RBI.constrainGenericRegister(MO.getReg(), *RC, *MRI);
   }
   I.eraseFromParent();
   return true;
 }
 
 bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
-  unsigned IntrinsicID =  I.getOperand(I.getNumExplicitDefs()).getIntrinsicID();
+  unsigned IntrinsicID = I.getIntrinsicID();
   switch (IntrinsicID) {
   case Intrinsic::amdgcn_if_break: {
     MachineBasicBlock *BB = I.getParent();
-    MachineFunction *MF = BB->getParent();
-    MachineRegisterInfo &MRI = MF->getRegInfo();
 
     // FIXME: Manually selecting to avoid dealiing with the SReg_1 trick
     // SelectionDAG uses for wave32 vs wave64.
@@ -579,8 +609,8 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const {
     I.eraseFromParent();
 
     for (Register Reg : { DstReg, Src0Reg, Src1Reg }) {
-      if (!MRI.getRegClassOrNull(Reg))
-        MRI.setRegClass(Reg, TRI.getWaveMaskRegClass());
+      if (!MRI->getRegClassOrNull(Reg))
+        MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
     }
 
     return true;
@@ -666,17 +696,15 @@ int AMDGPUInstructionSelector::getS_CMPOpcode(CmpInst::Predicate P,
 
 bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   const DebugLoc &DL = I.getDebugLoc();
 
   Register SrcReg = I.getOperand(2).getReg();
-  unsigned Size = RBI.getSizeInBits(SrcReg, MRI, TRI);
+  unsigned Size = RBI.getSizeInBits(SrcReg, *MRI, TRI);
 
   auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate();
 
   Register CCReg = I.getOperand(0).getReg();
-  if (isSCC(CCReg, MRI)) {
+  if (isSCC(CCReg, *MRI)) {
     int Opcode = getS_CMPOpcode(Pred, Size);
     if (Opcode == -1)
       return false;
@@ -687,7 +715,7 @@ bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
       .addReg(AMDGPU::SCC);
     bool Ret =
         constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI) &&
-        RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, MRI);
+        RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32RegClass, *MRI);
     I.eraseFromParent();
     return Ret;
   }
@@ -701,7 +729,7 @@ bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const {
             .add(I.getOperand(2))
             .add(I.getOperand(3));
   RBI.constrainGenericRegister(ICmp->getOperand(0).getReg(),
-                               *TRI.getBoolRC(), MRI);
+                               *TRI.getBoolRC(), *MRI);
   bool Ret = constrainSelectedInstRegOperands(*ICmp, TII, TRI, RBI);
   I.eraseFromParent();
   return Ret;
@@ -725,19 +753,273 @@ buildEXP(const TargetInstrInfo &TII, MachineInstr *Insert, unsigned Tgt,
           .addImm(Enabled);
 }
 
+static bool isZero(Register Reg, MachineRegisterInfo &MRI) {
+  int64_t C;
+  if (mi_match(Reg, MRI, m_ICst(C)) && C == 0)
+    return true;
+
+  // FIXME: matcher should ignore copies
+  return mi_match(Reg, MRI, m_Copy(m_ICst(C))) && C == 0;
+}
+
+static unsigned extractGLC(unsigned AuxiliaryData) {
+  return AuxiliaryData & 1;
+}
+
+static unsigned extractSLC(unsigned AuxiliaryData) {
+  return (AuxiliaryData >> 1) & 1;
+}
+
+static unsigned extractDLC(unsigned AuxiliaryData) {
+  return (AuxiliaryData >> 2) & 1;
+}
+
+static unsigned extractSWZ(unsigned AuxiliaryData) {
+  return (AuxiliaryData >> 3) & 1;
+}
+
+// Returns Base register, constant offset, and offset def point.
+static std::tuple<Register, unsigned, MachineInstr *>
+getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
+  MachineInstr *Def = getDefIgnoringCopies(Reg, MRI);
+  if (!Def)
+    return std::make_tuple(Reg, 0, nullptr);
+
+  if (Def->getOpcode() == AMDGPU::G_CONSTANT) {
+    unsigned Offset;
+    const MachineOperand &Op = Def->getOperand(1);
+    if (Op.isImm())
+      Offset = Op.getImm();
+    else
+      Offset = Op.getCImm()->getZExtValue();
+
+    return std::make_tuple(Register(), Offset, Def);
+  }
+
+  int64_t Offset;
+  if (Def->getOpcode() == AMDGPU::G_ADD) {
+    // TODO: Handle G_OR used for add case
+    if (mi_match(Def->getOperand(1).getReg(), MRI, m_ICst(Offset)))
+      return std::make_tuple(Def->getOperand(0).getReg(), Offset, Def);
+
+    // FIXME: matcher should ignore copies
+    if (mi_match(Def->getOperand(1).getReg(), MRI, m_Copy(m_ICst(Offset))))
+      return std::make_tuple(Def->getOperand(0).getReg(), Offset, Def);
+  }
+
+  return std::make_tuple(Reg, 0, Def);
+}
+
+static unsigned getBufferStoreOpcode(LLT Ty,
+                                     const unsigned MemSize,
+                                     const bool Offen) {
+  const int Size = Ty.getSizeInBits();
+  switch (8 * MemSize) {
+  case 8:
+    return Offen ? AMDGPU::BUFFER_STORE_BYTE_OFFEN_exact :
+                   AMDGPU::BUFFER_STORE_BYTE_OFFSET_exact;
+  case 16:
+    return Offen ? AMDGPU::BUFFER_STORE_SHORT_OFFEN_exact :
+                   AMDGPU::BUFFER_STORE_SHORT_OFFSET_exact;
+  default:
+    unsigned Opc = Offen ? AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact :
+                           AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact;
+    if (Size > 32)
+      Opc = AMDGPU::getMUBUFOpcode(Opc, Size / 32);
+    return Opc;
+  }
+}
+
+static unsigned getBufferStoreFormatOpcode(LLT Ty,
+                                           const unsigned MemSize,
+                                           const bool Offen) {
+  bool IsD16Packed = Ty.getScalarSizeInBits() == 16;
+  bool IsD16Unpacked = 8 * MemSize < Ty.getSizeInBits();
+  int NumElts = Ty.isVector() ? Ty.getNumElements() : 1;
+
+  if (IsD16Packed) {
+    switch (NumElts) {
+    case 1:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_X_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_X_OFFSET_exact;
+    case 2:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XY_OFFSET_exact;
+    case 3:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XYZ_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XYZ_OFFSET_exact;
+    case 4:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XYZW_OFFSET_exact;
+    default:
+      return -1;
+    }
+  }
+
+  if (IsD16Unpacked) {
+    switch (NumElts) {
+    case 1:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_X_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_X_OFFSET_exact;
+    case 2:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFSET_exact;
+    case 3:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XYZ_gfx80_OFFSET_exact;
+    case 4:
+      return Offen ? AMDGPU::BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact :
+                     AMDGPU::BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFSET_exact;
+    default:
+      return -1;
+    }
+  }
+
+  switch (NumElts) {
+  case 1:
+    return Offen ? AMDGPU::BUFFER_STORE_FORMAT_X_OFFEN_exact :
+                   AMDGPU::BUFFER_STORE_FORMAT_X_OFFSET_exact;
+  case 2:
+    return Offen ? AMDGPU::BUFFER_STORE_FORMAT_XY_OFFEN_exact :
+                  AMDGPU::BUFFER_STORE_FORMAT_XY_OFFSET_exact;
+  case 3:
+    return Offen ? AMDGPU::BUFFER_STORE_FORMAT_XYZ_OFFEN_exact :
+                   AMDGPU::BUFFER_STORE_FORMAT_XYZ_OFFSET_exact;
+  case 4:
+    return Offen ? AMDGPU::BUFFER_STORE_FORMAT_XYZW_OFFEN_exact :
+                   AMDGPU::BUFFER_STORE_FORMAT_XYZW_OFFSET_exact;
+  default:
+    return -1;
+  }
+
+  llvm_unreachable("unhandled buffer store");
+}
+
+// TODO: Move this to combiner
+// Returns base register, imm offset, total constant offset.
+std::tuple<Register, unsigned, unsigned>
+AMDGPUInstructionSelector::splitBufferOffsets(MachineIRBuilder &B,
+                                              Register OrigOffset) const {
+  const unsigned MaxImm = 4095;
+  Register BaseReg;
+  unsigned TotalConstOffset;
+  MachineInstr *OffsetDef;
+
+  std::tie(BaseReg, TotalConstOffset, OffsetDef)
+    = getBaseWithConstantOffset(*MRI, OrigOffset);
+
+  unsigned ImmOffset = TotalConstOffset;
+
+  // If the immediate value is too big for the immoffset field, put the value
+  // and -4096 into the immoffset field so that the value that is copied/added
+  // for the voffset field is a multiple of 4096, and it stands more chance
+  // of being CSEd with the copy/add for another similar load/store.f
+  // However, do not do that rounding down to a multiple of 4096 if that is a
+  // negative number, as it appears to be illegal to have a negative offset
+  // in the vgpr, even if adding the immediate offset makes it positive.
+  unsigned Overflow = ImmOffset & ~MaxImm;
+  ImmOffset -= Overflow;
+  if ((int32_t)Overflow < 0) {
+    Overflow += ImmOffset;
+    ImmOffset = 0;
+  }
+
+  if (Overflow != 0) {
+    // In case this is in a waterfall loop, insert offset code at the def point
+    // of the offset, not inside the loop.
+    MachineBasicBlock::iterator OldInsPt = B.getInsertPt();
+    MachineBasicBlock &OldMBB = B.getMBB();
+    B.setInstr(*OffsetDef);
+
+    if (!BaseReg) {
+      BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+      B.buildInstr(AMDGPU::V_MOV_B32_e32)
+        .addDef(BaseReg)
+        .addImm(Overflow);
+    } else {
+      Register OverflowVal = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+      B.buildInstr(AMDGPU::V_MOV_B32_e32)
+        .addDef(OverflowVal)
+        .addImm(Overflow);
+
+      Register NewBaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+      TII.getAddNoCarry(B.getMBB(), B.getInsertPt(), B.getDebugLoc(), NewBaseReg)
+        .addReg(BaseReg)
+        .addReg(OverflowVal, RegState::Kill)
+        .addImm(0);
+      BaseReg = NewBaseReg;
+    }
+
+    B.setInsertPt(OldMBB, OldInsPt);
+  }
+
+  return std::make_tuple(BaseReg, ImmOffset, TotalConstOffset);
+}
+
+bool AMDGPUInstructionSelector::selectStoreIntrinsic(MachineInstr &MI,
+                                                     bool IsFormat) const {
+  MachineIRBuilder B(MI);
+  MachineFunction &MF = B.getMF();
+  Register VData = MI.getOperand(1).getReg();
+  LLT Ty = MRI->getType(VData);
+
+  int Size = Ty.getSizeInBits();
+  if (Size % 32 != 0)
+    return false;
+
+  // FIXME: Verifier should enforce 1 MMO for these intrinsics.
+  MachineMemOperand *MMO = *MI.memoperands_begin();
+  const int MemSize = MMO->getSize();
+
+  Register RSrc = MI.getOperand(2).getReg();
+  Register VOffset = MI.getOperand(3).getReg();
+  Register SOffset = MI.getOperand(4).getReg();
+  unsigned AuxiliaryData = MI.getOperand(5).getImm();
+  unsigned ImmOffset;
+  unsigned TotalOffset;
+
+  std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
+  if (TotalOffset != 0)
+    MMO = MF.getMachineMemOperand(MMO, TotalOffset, MemSize);
+
+  const bool Offen = !isZero(VOffset, *MRI);
+
+  int Opc = IsFormat ? getBufferStoreFormatOpcode(Ty, MemSize, Offen) :
+    getBufferStoreOpcode(Ty, MemSize, Offen);
+  if (Opc == -1)
+    return false;
+
+  MachineInstrBuilder MIB = B.buildInstr(Opc)
+    .addUse(VData);
+
+  if (Offen)
+    MIB.addUse(VOffset);
+
+  MIB.addUse(RSrc)
+     .addUse(SOffset)
+     .addImm(ImmOffset)
+     .addImm(extractGLC(AuxiliaryData))
+     .addImm(extractSLC(AuxiliaryData))
+     .addImm(0) // tfe: FIXME: Remove from inst
+     .addImm(extractDLC(AuxiliaryData))
+     .addImm(extractSWZ(AuxiliaryData))
+     .addMemOperand(MMO);
+
+  MI.eraseFromParent();
+
+  return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI);
+}
+
 bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-
   unsigned IntrinsicID = I.getIntrinsicID();
   switch (IntrinsicID) {
   case Intrinsic::amdgcn_exp: {
-    int64_t Tgt = getConstant(MRI.getVRegDef(I.getOperand(1).getReg()));
-    int64_t Enabled = getConstant(MRI.getVRegDef(I.getOperand(2).getReg()));
-    int64_t Done = getConstant(MRI.getVRegDef(I.getOperand(7).getReg()));
-    int64_t VM = getConstant(MRI.getVRegDef(I.getOperand(8).getReg()));
+    int64_t Tgt = I.getOperand(1).getImm();
+    int64_t Enabled = I.getOperand(2).getImm();
+    int64_t Done = I.getOperand(7).getImm();
+    int64_t VM = I.getOperand(8).getImm();
 
     MachineInstr *Exp = buildEXP(TII, &I, Tgt, I.getOperand(3).getReg(),
                                  I.getOperand(4).getReg(),
@@ -750,13 +1032,13 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
   }
   case Intrinsic::amdgcn_exp_compr: {
     const DebugLoc &DL = I.getDebugLoc();
-    int64_t Tgt = getConstant(MRI.getVRegDef(I.getOperand(1).getReg()));
-    int64_t Enabled = getConstant(MRI.getVRegDef(I.getOperand(2).getReg()));
+    int64_t Tgt = I.getOperand(1).getImm();
+    int64_t Enabled = I.getOperand(2).getImm();
     Register Reg0 = I.getOperand(3).getReg();
     Register Reg1 = I.getOperand(4).getReg();
-    Register Undef = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
-    int64_t Done = getConstant(MRI.getVRegDef(I.getOperand(5).getReg()));
-    int64_t VM = getConstant(MRI.getVRegDef(I.getOperand(6).getReg()));
+    Register Undef = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+    int64_t Done = I.getOperand(5).getImm();
+    int64_t VM = I.getOperand(6).getImm();
 
     BuildMI(*BB, &I, DL, TII.get(AMDGPU::IMPLICIT_DEF), Undef);
     MachineInstr *Exp = buildEXP(TII, &I, Tgt, Reg0, Reg1, Undef, Undef, VM,
@@ -775,10 +1057,14 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     Register Reg = I.getOperand(1).getReg();
     I.eraseFromParent();
 
-    if (!MRI.getRegClassOrNull(Reg))
-      MRI.setRegClass(Reg, TRI.getWaveMaskRegClass());
+    if (!MRI->getRegClassOrNull(Reg))
+      MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
     return true;
   }
+  case Intrinsic::amdgcn_raw_buffer_store:
+    return selectStoreIntrinsic(I, false);
+  case Intrinsic::amdgcn_raw_buffer_store_format:
+    return selectStoreIntrinsic(I, true);
   default:
     return selectImpl(I, *CoverageInfo);
   }
@@ -786,16 +1072,14 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
 
 bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   const DebugLoc &DL = I.getDebugLoc();
 
   Register DstReg = I.getOperand(0).getReg();
-  unsigned Size = RBI.getSizeInBits(DstReg, MRI, TRI);
+  unsigned Size = RBI.getSizeInBits(DstReg, *MRI, TRI);
   assert(Size <= 32 || Size == 64);
   const MachineOperand &CCOp = I.getOperand(1);
   Register CCReg = CCOp.getReg();
-  if (isSCC(CCReg, MRI)) {
+  if (isSCC(CCReg, *MRI)) {
     unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 :
                                          AMDGPU::S_CSELECT_B32;
     MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
@@ -804,8 +1088,8 @@ bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const {
     // The generic constrainSelectedInstRegOperands doesn't work for the scc register
     // bank, because it does not cover the register class that we used to represent
     // for it.  So we need to manually set the register class here.
-    if (!MRI.getRegClassOrNull(CCReg))
-        MRI.setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, MRI));
+    if (!MRI->getRegClassOrNull(CCReg))
+        MRI->setRegClass(CCReg, TRI.getConstrainedRegClassForOperand(CCOp, *MRI));
     MachineInstr *Select = BuildMI(*BB, &I, DL, TII.get(SelectOpcode), DstReg)
             .add(I.getOperand(2))
             .add(I.getOperand(3));
@@ -860,19 +1144,15 @@ static int sizeToSubRegIndex(unsigned Size) {
 }
 
 bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
-  MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-
   Register DstReg = I.getOperand(0).getReg();
   Register SrcReg = I.getOperand(1).getReg();
-  const LLT DstTy = MRI.getType(DstReg);
-  const LLT SrcTy = MRI.getType(SrcReg);
+  const LLT DstTy = MRI->getType(DstReg);
+  const LLT SrcTy = MRI->getType(SrcReg);
   if (!DstTy.isScalar())
     return false;
 
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
-  const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
+  const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
   if (SrcRB != DstRB)
     return false;
 
@@ -880,9 +1160,9 @@ bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
   unsigned SrcSize = SrcTy.getSizeInBits();
 
   const TargetRegisterClass *SrcRC
-    = TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB, MRI);
+    = TRI.getRegClassForSizeOnBank(SrcSize, *SrcRB, *MRI);
   const TargetRegisterClass *DstRC
-    = TRI.getRegClassForSizeOnBank(DstSize, *DstRB, MRI);
+    = TRI.getRegClassForSizeOnBank(DstSize, *DstRB, *MRI);
 
   if (SrcSize > 32) {
     int SubRegIdx = sizeToSubRegIndex(DstSize);
@@ -898,8 +1178,8 @@ bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const {
     I.getOperand(1).setSubReg(SubRegIdx);
   }
 
-  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
-      !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
+  if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) ||
+      !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
     LLVM_DEBUG(dbgs() << "Failed to constrain G_TRUNC\n");
     return false;
   }
@@ -919,20 +1199,18 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
   bool Signed = I.getOpcode() == AMDGPU::G_SEXT;
   const DebugLoc &DL = I.getDebugLoc();
   MachineBasicBlock &MBB = *I.getParent();
-  MachineFunction &MF = *MBB.getParent();
-  MachineRegisterInfo &MRI = MF.getRegInfo();
   const Register DstReg = I.getOperand(0).getReg();
   const Register SrcReg = I.getOperand(1).getReg();
 
-  const LLT DstTy = MRI.getType(DstReg);
-  const LLT SrcTy = MRI.getType(SrcReg);
+  const LLT DstTy = MRI->getType(DstReg);
+  const LLT SrcTy = MRI->getType(SrcReg);
   const LLT S1 = LLT::scalar(1);
   const unsigned SrcSize = SrcTy.getSizeInBits();
   const unsigned DstSize = DstTy.getSizeInBits();
   if (!DstTy.isScalar())
     return false;
 
-  const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, TRI);
+  const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI);
 
   if (SrcBank->getID() == AMDGPU::SCCRegBankID) {
     if (SrcTy != S1 || DstSize > 64) // Invalid
@@ -945,7 +1223,7 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
 
     // FIXME: Create an extra copy to avoid incorrectly constraining the result
     // of the scc producer.
-    Register TmpReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+    Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
     BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), TmpReg)
       .addReg(SrcReg);
     BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
@@ -956,7 +1234,7 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
       .addImm(0)
       .addImm(Signed ? -1 : 1);
     I.eraseFromParent();
-    return RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
+    return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI);
   }
 
   if (SrcBank->getID() == AMDGPU::VCCRegBankID && DstSize <= 32) {
@@ -1002,7 +1280,7 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
   }
 
   if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
-    if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI))
+    if (!RBI.constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, *MRI))
       return false;
 
     if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
@@ -1011,7 +1289,7 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
       BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg)
         .addReg(SrcReg);
       I.eraseFromParent();
-      return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, MRI);
+      return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
     }
 
     const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
@@ -1020,8 +1298,8 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
     // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
     if (DstSize > 32 && SrcSize <= 32) {
       // We need a 64-bit register source, but the high bits don't matter.
-      Register ExtReg = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
-      Register UndefReg = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass);
+      Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
+      Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
       BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
       BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg)
         .addReg(SrcReg)
@@ -1034,7 +1312,7 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
         .addImm(SrcSize << 16);
 
       I.eraseFromParent();
-      return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, MRI);
+      return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass, *MRI);
     }
 
     unsigned Mask;
@@ -1049,16 +1327,57 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const {
     }
 
     I.eraseFromParent();
-    return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, MRI);
+    return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
   }
 
   return false;
 }
 
+static int64_t getFPTrueImmVal(unsigned Size, bool Signed) {
+  switch (Size) {
+  case 16:
+    return Signed ? 0xBC00 : 0x3C00;
+  case 32:
+    return Signed ? 0xbf800000 : 0x3f800000;
+  case 64:
+    return Signed ? 0xbff0000000000000 : 0x3ff0000000000000;
+  default:
+    llvm_unreachable("Invalid FP type size");
+  }
+}
+
+bool AMDGPUInstructionSelector::selectG_SITOFP_UITOFP(MachineInstr &I) const {
+  MachineBasicBlock *MBB = I.getParent();
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+  Register Src = I.getOperand(1).getReg();
+  if (!isSCC(Src, MRI))
+    return selectImpl(I, *CoverageInfo);
+
+  bool Signed = I.getOpcode() == AMDGPU::G_SITOFP;
+  Register DstReg = I.getOperand(0).getReg();
+  const LLT DstTy = MRI.getType(DstReg);
+  const unsigned DstSize = DstTy.getSizeInBits();
+  const DebugLoc &DL = I.getDebugLoc();
+
+  BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC)
+    .addReg(Src);
+
+  unsigned NewOpc =
+    DstSize > 32 ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32;
+  auto MIB = BuildMI(*MBB, I, DL, TII.get(NewOpc), DstReg)
+    .addImm(0)
+    .addImm(getFPTrueImmVal(DstSize, Signed));
+
+  if (!constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI))
+    return false;
+
+  I.eraseFromParent();
+  return true;
+}
+
 bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   MachineOperand &ImmOp = I.getOperand(1);
 
   // The AMDGPU backend only supports Imm operands and not CImm or FPImm.
@@ -1072,12 +1391,12 @@ bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
   Register DstReg = I.getOperand(0).getReg();
   unsigned Size;
   bool IsSgpr;
-  const RegisterBank *RB = MRI.getRegBankOrNull(I.getOperand(0).getReg());
+  const RegisterBank *RB = MRI->getRegBankOrNull(I.getOperand(0).getReg());
   if (RB) {
     IsSgpr = RB->getID() == AMDGPU::SGPRRegBankID;
-    Size = MRI.getType(DstReg).getSizeInBits();
+    Size = MRI->getType(DstReg).getSizeInBits();
   } else {
-    const TargetRegisterClass *RC = TRI.getRegClassForReg(MRI, DstReg);
+    const TargetRegisterClass *RC = TRI.getRegClassForReg(*MRI, DstReg);
     IsSgpr = TRI.isSGPRClass(RC);
     Size = TRI.getRegSizeInBits(*RC);
   }
@@ -1095,8 +1414,8 @@ bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
   DebugLoc DL = I.getDebugLoc();
   const TargetRegisterClass *RC = IsSgpr ? &AMDGPU::SReg_32_XM0RegClass :
                                            &AMDGPU::VGPR_32RegClass;
-  Register LoReg = MRI.createVirtualRegister(RC);
-  Register HiReg = MRI.createVirtualRegister(RC);
+  Register LoReg = MRI->createVirtualRegister(RC);
+  Register HiReg = MRI->createVirtualRegister(RC);
   const APInt &Imm = APInt(Size, I.getOperand(1).getImm());
 
   BuildMI(*BB, &I, DL, TII.get(Opcode), LoReg)
@@ -1116,10 +1435,10 @@ bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
   // work for target independent opcodes
   I.eraseFromParent();
   const TargetRegisterClass *DstRC =
-      TRI.getConstrainedRegClassForOperand(RS->getOperand(0), MRI);
+      TRI.getConstrainedRegClassForOperand(RS->getOperand(0), *MRI);
   if (!DstRC)
     return true;
-  return RBI.constrainGenericRegister(DstReg, *DstRC, MRI);
+  return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI);
 }
 
 static bool isConstant(const MachineInstr &MI) {
@@ -1192,10 +1511,8 @@ bool AMDGPUInstructionSelector::hasVgprParts(ArrayRef<GEPInfo> AddrInfo) const {
 
 void AMDGPUInstructionSelector::initM0(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
 
-  const LLT PtrTy = MRI.getType(I.getOperand(1).getReg());
+  const LLT PtrTy = MRI->getType(I.getOperand(1).getReg());
   unsigned AS = PtrTy.getAddressSpace();
   if ((AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) &&
       STI.ldsRequiresM0Init()) {
@@ -1212,8 +1529,6 @@ bool AMDGPUInstructionSelector::selectG_LOAD_ATOMICRMW(MachineInstr &I) const {
 
 bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   MachineOperand &CondOp = I.getOperand(0);
   Register CondReg = CondOp.getReg();
   const DebugLoc &DL = I.getDebugLoc();
@@ -1227,11 +1542,11 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
   // GlobalISel, we should push that decision into RegBankSelect. Assume for now
   // RegBankSelect knows what it's doing if the branch condition is scc, even
   // though it currently does not.
-  if (isSCC(CondReg, MRI)) {
+  if (isSCC(CondReg, *MRI)) {
     CondPhysReg = AMDGPU::SCC;
     BrOpcode = AMDGPU::S_CBRANCH_SCC1;
     ConstrainRC = &AMDGPU::SReg_32_XM0RegClass;
-  } else if (isVCC(CondReg, MRI)) {
+  } else if (isVCC(CondReg, *MRI)) {
     // FIXME: Do we have to insert an and with exec here, like in SelectionDAG?
     // We sort of know that a VCC producer based on the register bank, that ands
     // inactive lanes with 0. What if there was a logical operation with vcc
@@ -1243,8 +1558,8 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
   } else
     return false;
 
-  if (!MRI.getRegClassOrNull(CondReg))
-    MRI.setRegClass(CondReg, ConstrainRC);
+  if (!MRI->getRegClassOrNull(CondReg))
+    MRI->setRegClass(CondReg, ConstrainRC);
 
   BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CondPhysReg)
     .addReg(CondReg);
@@ -1256,19 +1571,15 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const {
 }
 
 bool AMDGPUInstructionSelector::selectG_FRAME_INDEX(MachineInstr &I) const {
-  MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-
   Register DstReg = I.getOperand(0).getReg();
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
   const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
   I.setDesc(TII.get(IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32));
   if (IsVGPR)
     I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
 
   return RBI.constrainGenericRegister(
-    DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI);
+    DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, *MRI);
 }
 
 bool AMDGPUInstructionSelector::selectG_PTR_MASK(MachineInstr &I) const {
@@ -1276,32 +1587,30 @@ bool AMDGPUInstructionSelector::selectG_PTR_MASK(MachineInstr &I) const {
   const uint64_t Mask = ~((UINT64_C(1) << Align) - 1);
 
   MachineBasicBlock *BB = I.getParent();
-  MachineFunction *MF = BB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
 
   Register DstReg = I.getOperand(0).getReg();
   Register SrcReg = I.getOperand(1).getReg();
 
-  const RegisterBank *DstRB = RBI.getRegBank(DstReg, MRI, TRI);
-  const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, MRI, TRI);
+  const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
+  const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI);
   const bool IsVGPR = DstRB->getID() == AMDGPU::VGPRRegBankID;
   unsigned NewOpc = IsVGPR ? AMDGPU::V_AND_B32_e64 : AMDGPU::S_AND_B32;
   unsigned MovOpc = IsVGPR ? AMDGPU::V_MOV_B32_e32 : AMDGPU::S_MOV_B32;
   const TargetRegisterClass &RegRC
     = IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass;
 
-  LLT Ty = MRI.getType(DstReg);
+  LLT Ty = MRI->getType(DstReg);
 
   const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(Ty, *DstRB,
-                                                                  MRI);
+                                                                  *MRI);
   const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(Ty, *SrcRB,
-                                                                  MRI);
-  if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI) ||
-      !RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI))
+                                                                  *MRI);
+  if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) ||
+      !RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
     return false;
 
   const DebugLoc &DL = I.getDebugLoc();
-  Register ImmReg = MRI.createVirtualRegister(&RegRC);
+  Register ImmReg = MRI->createVirtualRegister(&RegRC);
   BuildMI(*BB, &I, DL, TII.get(MovOpc), ImmReg)
     .addImm(Mask);
 
@@ -1313,9 +1622,9 @@ bool AMDGPUInstructionSelector::selectG_PTR_MASK(MachineInstr &I) const {
     return true;
   }
 
-  Register HiReg = MRI.createVirtualRegister(&RegRC);
-  Register LoReg = MRI.createVirtualRegister(&RegRC);
-  Register MaskLo = MRI.createVirtualRegister(&RegRC);
+  Register HiReg = MRI->createVirtualRegister(&RegRC);
+  Register LoReg = MRI->createVirtualRegister(&RegRC);
+  Register MaskLo = MRI->createVirtualRegister(&RegRC);
 
   BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), LoReg)
     .addReg(SrcReg, 0, AMDGPU::sub0);
@@ -1356,6 +1665,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
     if (selectImpl(I, *CoverageInfo))
       return true;
     return selectG_ADD_SUB(I);
+  case TargetOpcode::G_UADDO:
+  case TargetOpcode::G_USUBO:
+    return selectG_UADDO_USUBO(I);
   case TargetOpcode::G_INTTOPTR:
   case TargetOpcode::G_BITCAST:
     return selectCOPY(I);
@@ -1408,6 +1720,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) {
   case TargetOpcode::G_ZEXT:
   case TargetOpcode::G_ANYEXT:
     return selectG_SZA_EXT(I);
+  case TargetOpcode::G_SITOFP:
+  case TargetOpcode::G_UITOFP:
+    return selectG_SITOFP_UITOFP(I);
   case TargetOpcode::G_BRCOND:
     return selectG_BRCOND(I);
   case TargetOpcode::G_FRAME_INDEX:
@@ -1435,14 +1750,14 @@ AMDGPUInstructionSelector::selectVCSRC(MachineOperand &Root) const {
 
 std::pair<Register, unsigned>
 AMDGPUInstructionSelector::selectVOP3ModsImpl(
-  Register Src, const MachineRegisterInfo &MRI) const {
+  Register Src) const {
   unsigned Mods = 0;
-  MachineInstr *MI = MRI.getVRegDef(Src);
+  MachineInstr *MI = MRI->getVRegDef(Src);
 
   if (MI && MI->getOpcode() == AMDGPU::G_FNEG) {
     Src = MI->getOperand(1).getReg();
     Mods |= SISrcMods::NEG;
-    MI = MRI.getVRegDef(Src);
+    MI = MRI->getVRegDef(Src);
   }
 
   if (MI && MI->getOpcode() == AMDGPU::G_FABS) {
@@ -1465,12 +1780,9 @@ AMDGPUInstructionSelector::selectVSRC0(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
-
   Register Src;
   unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(), MRI);
+  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
 
   return {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
@@ -1482,12 +1794,9 @@ AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectVOP3Mods0Clamp0OMod(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
-
   Register Src;
   unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(), MRI);
+  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
 
   return {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
@@ -1508,12 +1817,9 @@ AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI
-    = Root.getParent()->getParent()->getParent()->getRegInfo();
-
   Register Src;
   unsigned Mods;
-  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg(), MRI);
+  std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg());
 
   return {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); },
@@ -1542,11 +1848,8 @@ AMDGPUInstructionSelector::selectVOP3OpSelMods(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI =
-      Root.getParent()->getParent()->getParent()->getRegInfo();
-
   SmallVector<GEPInfo, 4> AddrInfo;
-  getAddrModeInfo(*Root.getParent(), MRI, AddrInfo);
+  getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo);
 
   if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
     return None;
@@ -1566,11 +1869,8 @@ AMDGPUInstructionSelector::selectSmrdImm(MachineOperand &Root) const {
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectSmrdImm32(MachineOperand &Root) const {
-  MachineRegisterInfo &MRI =
-      Root.getParent()->getParent()->getParent()->getRegInfo();
-
   SmallVector<GEPInfo, 4> AddrInfo;
-  getAddrModeInfo(*Root.getParent(), MRI, AddrInfo);
+  getAddrModeInfo(*Root.getParent(), *MRI, AddrInfo);
 
   if (AddrInfo.empty() || AddrInfo[0].SgprParts.size() != 1)
     return None;
@@ -1591,10 +1891,9 @@ InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
   MachineBasicBlock *MBB = MI->getParent();
-  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
 
   SmallVector<GEPInfo, 4> AddrInfo;
-  getAddrModeInfo(*MI, MRI, AddrInfo);
+  getAddrModeInfo(*MI, *MRI, AddrInfo);
 
   // FIXME: We should shrink the GEP if the offset is known to be <= 32-bits,
   // then we can select all ptr + 32-bit offsets not just immediate offsets.
@@ -1610,7 +1909,7 @@ AMDGPUInstructionSelector::selectSmrdSgpr(MachineOperand &Root) const {
   // failed trying to select this load into one of the _IMM variants since
   // the _IMM Patterns are considered before the _SGPR patterns.
   unsigned PtrReg = GEPInfo.SgprParts[0];
-  Register OffsetReg = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
+  Register OffsetReg = MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
   BuildMI(*MBB, MI, MI->getDebugLoc(), TII.get(AMDGPU::S_MOV_B32), OffsetReg)
           .addImm(GEPInfo.Imm);
   return {{
@@ -1623,8 +1922,6 @@ template <bool Signed>
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
-  MachineBasicBlock *MBB = MI->getParent();
-  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
 
   InstructionSelector::ComplexRendererFns Default = {{
       [=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
@@ -1635,12 +1932,12 @@ AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root) const {
   if (!STI.hasFlatInstOffsets())
     return Default;
 
-  const MachineInstr *OpDef = MRI.getVRegDef(Root.getReg());
+  const MachineInstr *OpDef = MRI->getVRegDef(Root.getReg());
   if (!OpDef || OpDef->getOpcode() != AMDGPU::G_GEP)
     return Default;
 
   Optional<int64_t> Offset =
-    getConstantVRegVal(OpDef->getOperand(2).getReg(), MRI);
+    getConstantVRegVal(OpDef->getOperand(2).getReg(), *MRI);
   if (!Offset.hasValue())
     return Default;
 
@@ -1677,12 +1974,11 @@ AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
   MachineBasicBlock *MBB = MI->getParent();
   MachineFunction *MF = MBB->getParent();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
   const SIMachineFunctionInfo *Info = MF->getInfo<SIMachineFunctionInfo>();
 
   int64_t Offset = 0;
-  if (mi_match(Root.getReg(), MRI, m_ICst(Offset))) {
-    Register HighBits = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+  if (mi_match(Root.getReg(), *MRI, m_ICst(Offset))) {
+    Register HighBits = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
     // TODO: Should this be inside the render function? The iterator seems to
     // move.
@@ -1716,12 +2012,12 @@ AMDGPUInstructionSelector::selectMUBUFScratchOffen(MachineOperand &Root) const {
   // offsets.
   Optional<int> FI;
   Register VAddr = Root.getReg();
-  if (const MachineInstr *RootDef = MRI.getVRegDef(Root.getReg())) {
-    if (isBaseWithConstantOffset(Root, MRI)) {
+  if (const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg())) {
+    if (isBaseWithConstantOffset(Root, *MRI)) {
       const MachineOperand &LHS = RootDef->getOperand(1);
       const MachineOperand &RHS = RootDef->getOperand(2);
-      const MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
-      const MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
+      const MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
+      const MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
       if (LHSDef && RHSDef) {
         int64_t PossibleOffset =
             RHSDef->getOperand(1).getCImm()->getSExtValue();
@@ -1785,10 +2081,9 @@ AMDGPUInstructionSelector::selectMUBUFScratchOffset(
     MachineOperand &Root) const {
   MachineInstr *MI = Root.getParent();
   MachineBasicBlock *MBB = MI->getParent();
-  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
 
   int64_t Offset = 0;
-  if (!mi_match(Root.getReg(), MRI, m_ICst(Offset)) ||
+  if (!mi_match(Root.getReg(), *MRI, m_ICst(Offset)) ||
       !SIInstrInfo::isLegalMUBUFImmOffset(Offset))
     return {};
 
@@ -1811,11 +2106,7 @@ AMDGPUInstructionSelector::selectMUBUFScratchOffset(
 
 InstructionSelector::ComplexRendererFns
 AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
-  MachineInstr *MI = Root.getParent();
-  MachineBasicBlock *MBB = MI->getParent();
-  MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
-
-  const MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
+  const MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
   if (!RootDef) {
     return {{
         [=](MachineInstrBuilder &MIB) { MIB.add(Root); },
@@ -1824,15 +2115,15 @@ AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
   }
 
   int64_t ConstAddr = 0;
-  if (isBaseWithConstantOffset(Root, MRI)) {
+  if (isBaseWithConstantOffset(Root, *MRI)) {
     const MachineOperand &LHS = RootDef->getOperand(1);
     const MachineOperand &RHS = RootDef->getOperand(2);
-    const MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
-    const MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
+    const MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
+    const MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
     if (LHSDef && RHSDef) {
       int64_t PossibleOffset =
         RHSDef->getOperand(1).getCImm()->getSExtValue();
-      if (isDSOffsetLegal(MRI, LHS, PossibleOffset, 16)) {
+      if (isDSOffsetLegal(*MRI, LHS, PossibleOffset, 16)) {
         // (add n0, c0)
         return {{
             [=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
@@ -1844,7 +2135,7 @@ AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
 
 
 
-  } else if (mi_match(Root.getReg(), MRI, m_ICst(ConstAddr))) {
+  } else if (mi_match(Root.getReg(), *MRI, m_ICst(ConstAddr))) {
 
 
   }
@@ -1854,3 +2145,12 @@ AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const {
       [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }
     }};
 }
+
+void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
+                                                 const MachineInstr &MI) const {
+  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
+  assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
+  Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI);
+  assert(CstVal && "Expected constant value");
+  MIB.addImm(CstVal.getValue());
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index 67a5062ca395d..d3c83a6a872a4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -35,6 +35,7 @@ class AMDGPUInstrInfo;
 class AMDGPURegisterBankInfo;
 class GCNSubtarget;
 class MachineInstr;
+class MachineIRBuilder;
 class MachineOperand;
 class MachineRegisterInfo;
 class SIInstrInfo;
@@ -42,6 +43,9 @@ class SIMachineFunctionInfo;
 class SIRegisterInfo;
 
 class AMDGPUInstructionSelector : public InstructionSelector {
+private:
+  MachineRegisterInfo *MRI;
+
 public:
   AMDGPUInstructionSelector(const GCNSubtarget &STI,
                             const AMDGPURegisterBankInfo &RBI,
@@ -50,6 +54,9 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   bool select(MachineInstr &I) override;
   static const char *getName();
 
+  void setupMF(MachineFunction &MF, GISelKnownBits &KB,
+               CodeGenCoverage &CoverageInfo) override;
+
 private:
   struct GEPInfo {
     const MachineInstr &GEP;
@@ -72,9 +79,11 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   bool selectPHI(MachineInstr &I) const;
   bool selectG_TRUNC(MachineInstr &I) const;
   bool selectG_SZA_EXT(MachineInstr &I) const;
+  bool selectG_SITOFP_UITOFP(MachineInstr &I) const;
   bool selectG_CONSTANT(MachineInstr &I) const;
   bool selectG_AND_OR_XOR(MachineInstr &I) const;
   bool selectG_ADD_SUB(MachineInstr &I) const;
+  bool selectG_UADDO_USUBO(MachineInstr &I) const;
   bool selectG_EXTRACT(MachineInstr &I) const;
   bool selectG_MERGE_VALUES(MachineInstr &I) const;
   bool selectG_UNMERGE_VALUES(MachineInstr &I) const;
@@ -82,6 +91,12 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   bool selectG_IMPLICIT_DEF(MachineInstr &I) const;
   bool selectG_INSERT(MachineInstr &I) const;
   bool selectG_INTRINSIC(MachineInstr &I) const;
+
+  std::tuple<Register, unsigned, unsigned>
+  splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset) const;
+
+  bool selectStoreIntrinsic(MachineInstr &MI, bool IsFormat) const;
+
   bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const;
   int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const;
   bool selectG_ICMP(MachineInstr &I) const;
@@ -99,7 +114,7 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   bool selectG_PTR_MASK(MachineInstr &I) const;
 
   std::pair<Register, unsigned>
-  selectVOP3ModsImpl(Register Src, const MachineRegisterInfo &MRI) const;
+  selectVOP3ModsImpl(Register Src) const;
 
   InstructionSelector::ComplexRendererFns
   selectVCSRC(MachineOperand &Root) const;
@@ -149,6 +164,9 @@ class AMDGPUInstructionSelector : public InstructionSelector {
   InstructionSelector::ComplexRendererFns
   selectDS1Addr1Offset(MachineOperand &Root) const;
 
+  void renderTruncImm32(MachineInstrBuilder &MIB,
+                        const MachineInstr &MI) const;
+
   const SIInstrInfo &TII;
   const SIRegisterInfo &TRI;
   const AMDGPURegisterBankInfo &RBI;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 0c7ab7ae32e96..b8b54a2ef1a5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -40,7 +40,7 @@ using namespace LegalityPredicates;
 
 
 static LegalityPredicate isMultiple32(unsigned TypeIdx,
-                                      unsigned MaxSize = 512) {
+                                      unsigned MaxSize = 1024) {
   return [=](const LegalityQuery &Query) {
     const LLT Ty = Query.Types[TypeIdx];
     const LLT EltTy = Ty.getScalarType();
@@ -115,7 +115,7 @@ static LegalityPredicate numElementsNotEven(unsigned TypeIdx) {
   };
 }
 
-// Any combination of 32 or 64-bit elements up to 512 bits, and multiples of
+// Any combination of 32 or 64-bit elements up to 1024 bits, and multiples of
 // v2s16.
 static LegalityPredicate isRegisterType(unsigned TypeIdx) {
   return [=](const LegalityQuery &Query) {
@@ -127,7 +127,7 @@ static LegalityPredicate isRegisterType(unsigned TypeIdx) {
              EltSize == 128 || EltSize == 256;
     }
 
-    return Ty.getSizeInBits() % 32 == 0 && Ty.getSizeInBits() <= 512;
+    return Ty.getSizeInBits() % 32 == 0 && Ty.getSizeInBits() <= 1024;
   };
 }
 
@@ -162,7 +162,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   const LLT S96 = LLT::scalar(96);
   const LLT S128 = LLT::scalar(128);
   const LLT S256 = LLT::scalar(256);
-  const LLT S512 = LLT::scalar(512);
+  const LLT S1024 = LLT::scalar(1024);
 
   const LLT V2S16 = LLT::vector(2, 16);
   const LLT V4S16 = LLT::vector(4, 16);
@@ -182,6 +182,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   const LLT V14S32 = LLT::vector(14, 32);
   const LLT V15S32 = LLT::vector(15, 32);
   const LLT V16S32 = LLT::vector(16, 32);
+  const LLT V32S32 = LLT::vector(32, 32);
 
   const LLT V2S64 = LLT::vector(2, 64);
   const LLT V3S64 = LLT::vector(3, 64);
@@ -190,12 +191,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   const LLT V6S64 = LLT::vector(6, 64);
   const LLT V7S64 = LLT::vector(7, 64);
   const LLT V8S64 = LLT::vector(8, 64);
+  const LLT V16S64 = LLT::vector(16, 64);
 
   std::initializer_list<LLT> AllS32Vectors =
     {V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
-     V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32};
+     V9S32, V10S32, V11S32, V12S32, V13S32, V14S32, V15S32, V16S32, V32S32};
   std::initializer_list<LLT> AllS64Vectors =
-    {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64};
+    {V2S64, V3S64, V4S64, V5S64, V6S64, V7S64, V8S64, V16S64};
 
   const LLT GlobalPtr = GetAddrSpacePtr(AMDGPUAS::GLOBAL_ADDRESS);
   const LLT ConstantPtr = GetAddrSpacePtr(AMDGPUAS::CONSTANT_ADDRESS);
@@ -273,7 +275,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   getActionDefinitionsBuilder({G_UADDO, G_SADDO, G_USUBO, G_SSUBO,
                                G_UADDE, G_SADDE, G_USUBE, G_SSUBE})
     .legalFor({{S32, S1}})
-    .clampScalar(0, S32, S32);
+    .clampScalar(0, S32, S32)
+    .scalarize(0); // TODO: Implement.
 
   getActionDefinitionsBuilder(G_BITCAST)
     .legalForCartesianProduct({S32, V2S16})
@@ -292,7 +295,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalFor({S1, S32, S64, S16, V2S32, V4S32, V2S16, V4S16, GlobalPtr,
                ConstantPtr, LocalPtr, FlatPtr, PrivatePtr})
     .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
-    .clampScalarOrElt(0, S32, S512)
+    .clampScalarOrElt(0, S32, S1024)
     .legalIf(isMultiple32(0))
     .widenScalarToNextPow2(0, 32)
     .clampMaxNumElements(0, S32, 16);
@@ -309,7 +312,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
     .legalIf(isPointer(0));
 
   setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
-  getActionDefinitionsBuilder(G_GLOBAL_VALUE).customFor({LocalPtr});
+  getActionDefinitionsBuilder(G_GLOBAL_VALUE)
+    .customFor({LocalPtr, GlobalPtr, ConstantPtr, Constant32Ptr});
 
 
   auto &FPOpActions = getActionDefinitionsBuilder(
@@ -415,15 +419,22 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
                {S32, S8}, {S128, S32}, {S128, S64}, {S32, LLT::scalar(24)}})
     .scalarize(0);
 
+  // TODO: Split s1->s64 during regbankselect for VALU.
   getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
-    .legalFor({{S32, S32}, {S64, S32}, {S16, S32}})
+    .legalFor({{S32, S32}, {S64, S32}, {S16, S32}, {S32, S1}, {S16, S1}, {S64, S1}})
     .lowerFor({{S32, S64}})
     .customFor({{S64, S64}})
     .scalarize(0);
 
-  getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
-    .legalFor({{S32, S32}, {S32, S64}})
-    .scalarize(0);
+  auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
+    .legalFor({{S32, S32}, {S32, S64}, {S32, S16}});
+  if (ST.has16BitInsts())
+    FPToI.legalFor({{S16, S16}});
+  else
+    FPToI.minScalar(1, S32);
+
+  FPToI.minScalar(0, S32)
+       .scalarize(0);
 
   getActionDefinitionsBuilder(G_INTRINSIC_ROUND)
     .legalFor({S32, S64})
@@ -875,7 +886,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
           return (EltTy.getSizeInBits() == 16 ||
                   EltTy.getSizeInBits() % 32 == 0) &&
                  VecTy.getSizeInBits() % 32 == 0 &&
-                 VecTy.getSizeInBits() <= 512 &&
+                 VecTy.getSizeInBits() <= 1024 &&
                  IdxTy.getSizeInBits() == 32;
         })
       .clampScalar(EltTypeIdx, S32, S64)
@@ -921,8 +932,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
   auto &BuildVector = getActionDefinitionsBuilder(G_BUILD_VECTOR)
     .legalForCartesianProduct(AllS32Vectors, {S32})
     .legalForCartesianProduct(AllS64Vectors, {S64})
-    .clampNumElements(0, V16S32, V16S32)
-    .clampNumElements(0, V2S64, V8S64);
+    .clampNumElements(0, V16S32, V32S32)
+    .clampNumElements(0, V2S64, V16S64);
 
   if (ST.hasScalarPackInsts())
     BuildVector.legalFor({V2S16, S32});
@@ -982,7 +993,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .fewerElementsIf(
         [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
         scalarize(1))
-      .clampScalar(BigTyIdx, S32, S512)
+      .clampScalar(BigTyIdx, S32, S1024)
       .lowerFor({{S16, V2S16}})
       .widenScalarIf(
         [=](const LegalityQuery &Query) {
@@ -1013,7 +1024,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
 
           return BigTy.getSizeInBits() % 16 == 0 &&
                  LitTy.getSizeInBits() % 16 == 0 &&
-                 BigTy.getSizeInBits() <= 512;
+                 BigTy.getSizeInBits() <= 1024;
         })
       // Any vectors left are the wrong size. Scalarize them.
       .scalarize(0)
@@ -1508,6 +1519,62 @@ bool AMDGPULegalizerInfo::legalizeSinCos(
   return true;
 }
 
+bool AMDGPULegalizerInfo::buildPCRelGlobalAddress(
+  Register DstReg, LLT PtrTy,
+  MachineIRBuilder &B, const GlobalValue *GV,
+  unsigned Offset, unsigned GAFlags) const {
+  // In order to support pc-relative addressing, SI_PC_ADD_REL_OFFSET is lowered
+  // to the following code sequence:
+  //
+  // For constant address space:
+  //   s_getpc_b64 s[0:1]
+  //   s_add_u32 s0, s0, $symbol
+  //   s_addc_u32 s1, s1, 0
+  //
+  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
+  //   a fixup or relocation is emitted to replace $symbol with a literal
+  //   constant, which is a pc-relative offset from the encoding of the $symbol
+  //   operand to the global variable.
+  //
+  // For global address space:
+  //   s_getpc_b64 s[0:1]
+  //   s_add_u32 s0, s0, $symbol@{gotpc}rel32@lo
+  //   s_addc_u32 s1, s1, $symbol@{gotpc}rel32@hi
+  //
+  //   s_getpc_b64 returns the address of the s_add_u32 instruction and then
+  //   fixups or relocations are emitted to replace $symbol@*@lo and
+  //   $symbol@*@hi with lower 32 bits and higher 32 bits of a literal constant,
+  //   which is a 64-bit pc-relative offset from the encoding of the $symbol
+  //   operand to the global variable.
+  //
+  // What we want here is an offset from the value returned by s_getpc
+  // (which is the address of the s_add_u32 instruction) to the global
+  // variable, but since the encoding of $symbol starts 4 bytes after the start
+  // of the s_add_u32 instruction, we end up with an offset that is 4 bytes too
+  // small. This requires us to add 4 to the global variable offset in order to
+  // compute the correct address.
+
+  LLT ConstPtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
+
+  Register PCReg = PtrTy.getSizeInBits() != 32 ? DstReg :
+    B.getMRI()->createGenericVirtualRegister(ConstPtrTy);
+
+  MachineInstrBuilder MIB = B.buildInstr(AMDGPU::SI_PC_ADD_REL_OFFSET)
+    .addDef(PCReg);
+
+  MIB.addGlobalAddress(GV, Offset + 4, GAFlags);
+  if (GAFlags == SIInstrInfo::MO_NONE)
+    MIB.addImm(0);
+  else
+    MIB.addGlobalAddress(GV, Offset + 4, GAFlags + 1);
+
+  B.getMRI()->setRegClass(PCReg, &AMDGPU::SReg_64RegClass);
+
+  if (PtrTy.getSizeInBits() == 32)
+    B.buildExtract(DstReg, PCReg, 0);
+  return true;
+ }
+
 bool AMDGPULegalizerInfo::legalizeGlobalValue(
   MachineInstr &MI, MachineRegisterInfo &MRI,
   MachineIRBuilder &B) const {
@@ -1518,10 +1585,9 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
   const GlobalValue *GV = MI.getOperand(1).getGlobal();
   MachineFunction &MF = B.getMF();
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  B.setInstr(MI);
 
   if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
-    B.setInstr(MI);
-
     if (!MFI->isEntryFunction()) {
       const Function &Fn = MF.getFunction();
       DiagnosticInfoUnsupported BadLDSDecl(
@@ -1535,13 +1601,47 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
       MI.eraseFromParent();
       return true;
     }
+
+    const Function &Fn = MF.getFunction();
+    DiagnosticInfoUnsupported BadInit(
+      Fn, "unsupported initializer for address space", MI.getDebugLoc());
+    Fn.getContext().diagnose(BadInit);
+    return true;
+  }
+
+  const SITargetLowering *TLI = ST.getTargetLowering();
+
+  if (TLI->shouldEmitFixup(GV)) {
+    buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0);
+    MI.eraseFromParent();
+    return true;
+  }
+
+  if (TLI->shouldEmitPCReloc(GV)) {
+    buildPCRelGlobalAddress(DstReg, Ty, B, GV, 0, SIInstrInfo::MO_REL32);
+    MI.eraseFromParent();
+    return true;
+  }
+
+  LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
+  Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
+
+  MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
+    MachinePointerInfo::getGOT(MF),
+    MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
+    MachineMemOperand::MOInvariant,
+    8 /*Size*/, 8 /*Align*/);
+
+  buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
+
+  if (Ty.getSizeInBits() == 32) {
+    // Truncate if this is a 32-bit constant adrdess.
+    auto Load = B.buildLoad(PtrTy, GOTAddr, *GOTMMO);
+    B.buildExtract(DstReg, Load, 0);
   } else
-    return false;
+    B.buildLoad(DstReg, GOTAddr, *GOTMMO);
 
-  const Function &Fn = MF.getFunction();
-  DiagnosticInfoUnsupported BadInit(
-    Fn, "unsupported initializer for address space", MI.getDebugLoc());
-  Fn.getContext().diagnose(BadInit);
+  MI.eraseFromParent();
   return true;
 }
 
@@ -1619,9 +1719,14 @@ bool AMDGPULegalizerInfo::loadInputValue(Register DstReg, MachineIRBuilder &B,
     const unsigned Mask = Arg->getMask();
     const unsigned Shift = countTrailingZeros<unsigned>(Mask);
 
-    auto ShiftAmt = B.buildConstant(S32, Shift);
-    auto LShr = B.buildLShr(S32, LiveIn, ShiftAmt);
-    B.buildAnd(DstReg, LShr, B.buildConstant(S32, Mask >> Shift));
+    Register AndMaskSrc = LiveIn;
+
+    if (Shift != 0) {
+      auto ShiftAmt = B.buildConstant(S32, Shift);
+      AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0);
+    }
+
+    B.buildAnd(DstReg, AndMaskSrc, B.buildConstant(S32, Mask >> Shift));
   } else
     B.buildCopy(DstReg, LiveIn);
 
@@ -1750,11 +1855,67 @@ bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
   return true;
 }
 
+/// Handle register layout difference for f16 images for some subtargets.
+Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
+                                             MachineRegisterInfo &MRI,
+                                             Register Reg) const {
+  if (!ST.hasUnpackedD16VMem())
+    return Reg;
+
+  const LLT S16 = LLT::scalar(16);
+  const LLT S32 = LLT::scalar(32);
+  LLT StoreVT = MRI.getType(Reg);
+  assert(StoreVT.isVector() && StoreVT.getElementType() == S16);
+
+  auto Unmerge = B.buildUnmerge(S16, Reg);
+
+  SmallVector<Register, 4> WideRegs;
+  for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
+    WideRegs.push_back(B.buildAnyExt(S32, Unmerge.getReg(I)).getReg(0));
+
+  int NumElts = StoreVT.getNumElements();
+
+  return B.buildBuildVector(LLT::vector(NumElts, S32), WideRegs).getReg(0);
+}
+
+bool AMDGPULegalizerInfo::legalizeRawBufferStore(MachineInstr &MI,
+                                                 MachineRegisterInfo &MRI,
+                                                 MachineIRBuilder &B,
+                                                 bool IsFormat) const {
+  // TODO: Reject f16 format on targets where unsupported.
+  Register VData = MI.getOperand(1).getReg();
+  LLT Ty = MRI.getType(VData);
+
+  B.setInstr(MI);
+
+  const LLT S32 = LLT::scalar(32);
+  const LLT S16 = LLT::scalar(16);
+
+  // Fixup illegal register types for i8 stores.
+  if (Ty == LLT::scalar(8) || Ty == S16) {
+    Register AnyExt = B.buildAnyExt(LLT::scalar(32), VData).getReg(0);
+    MI.getOperand(1).setReg(AnyExt);
+    return true;
+  }
+
+  if (Ty.isVector()) {
+    if (Ty.getElementType() == S16 && Ty.getNumElements() <= 4) {
+      if (IsFormat)
+        MI.getOperand(1).setReg(handleD16VData(B, MRI, VData));
+      return true;
+    }
+
+    return Ty.getElementType() == S32 && Ty.getNumElements() <= 4;
+  }
+
+  return Ty == S32;
+}
+
 bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
                                             MachineRegisterInfo &MRI,
                                             MachineIRBuilder &B) const {
   // Replace the use G_BRCOND with the exec manipulate and branch pseudos.
-  switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+  switch (MI.getIntrinsicID()) {
   case Intrinsic::amdgcn_if: {
     if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) {
       const SIRegisterInfo *TRI
@@ -1842,6 +2003,10 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
     MI.eraseFromParent();
     return true;
   }
+  case Intrinsic::amdgcn_raw_buffer_store:
+    return legalizeRawBufferStore(MI, MRI, B, false);
+  case Intrinsic::amdgcn_raw_buffer_store_format:
+    return legalizeRawBufferStore(MI, MRI, B, true);
   default:
     return true;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
index 99564a04dbb2f..e2edadc9697bf 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h
@@ -16,6 +16,7 @@
 
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "AMDGPUArgumentUsageInfo.h"
+#include "SIInstrInfo.h"
 
 namespace llvm {
 
@@ -58,6 +59,10 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
   bool legalizeSinCos(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &B) const;
 
+  bool buildPCRelGlobalAddress(
+    Register DstReg, LLT PtrTy, MachineIRBuilder &B, const GlobalValue *GV,
+    unsigned Offset, unsigned GAFlags = SIInstrInfo::MO_NONE) const;
+
   bool legalizeGlobalValue(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B) const;
   bool legalizeLoad(MachineInstr &MI, MachineRegisterInfo &MRI,
@@ -83,6 +88,11 @@ class AMDGPULegalizerInfo : public LegalizerInfo {
                               MachineIRBuilder &B) const;
   bool legalizeIsAddrSpace(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &B, unsigned AddrSpace) const;
+
+  Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
+                          Register Reg) const;
+  bool legalizeRawBufferStore(MachineInstr &MI, MachineRegisterInfo &MRI,
+                              MachineIRBuilder &B, bool IsFormat) const;
   bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
                          MachineIRBuilder &B) const override;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
index ce0a9db7c7f41..03f056f0fdce5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp
@@ -1430,8 +1430,8 @@ AllocaInst* AMDGPULibCalls::insertAlloca(CallInst *UI, IRBuilder<> &B,
   B.SetInsertPoint(&*ItNew);
   AllocaInst *Alloc = B.CreateAlloca(RetType, 0,
     std::string(prefix) + UI->getName());
-  Alloc->setAlignment(UCallee->getParent()->getDataLayout()
-                       .getTypeAllocSize(RetType));
+  Alloc->setAlignment(MaybeAlign(
+      UCallee->getParent()->getDataLayout().getTypeAllocSize(RetType)));
   return Alloc;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
index 90ea2467b2d96..5250bf455d719 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp
@@ -30,7 +30,6 @@
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InstVisitor.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
@@ -45,20 +44,13 @@ using namespace llvm;
 
 namespace {
 class LLVM_LIBRARY_VISIBILITY AMDGPUPrintfRuntimeBinding final
-    : public ModulePass,
-      public InstVisitor<AMDGPUPrintfRuntimeBinding> {
+    : public ModulePass {
 
 public:
   static char ID;
 
   explicit AMDGPUPrintfRuntimeBinding();
 
-  void visitCallSite(CallSite CS) {
-    Function *F = CS.getCalledFunction();
-    if (F && F->hasName() && F->getName() == "printf")
-      Printfs.push_back(CS.getInstruction());
-  }
-
 private:
   bool runOnModule(Module &M) override;
   void getConversionSpecifiers(SmallVectorImpl<char> &OpConvSpecifiers,
@@ -80,7 +72,7 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUPrintfRuntimeBinding final
 
   const DataLayout *TD;
   const DominatorTree *DT;
-  SmallVector<Value *, 32> Printfs;
+  SmallVector<CallInst *, 32> Printfs;
 };
 } // namespace
 
@@ -162,9 +154,7 @@ bool AMDGPUPrintfRuntimeBinding::lowerPrintfForGpu(
   // NB: This is important for this string size to be divizable by 4
   const char NonLiteralStr[4] = "???";
 
-  for (auto P : Printfs) {
-    CallInst *CI = dyn_cast<CallInst>(P);
-
+  for (auto CI : Printfs) {
     unsigned NumOps = CI->getNumArgOperands();
 
     SmallString<16> OpConvSpecifiers;
@@ -565,10 +555,8 @@ bool AMDGPUPrintfRuntimeBinding::lowerPrintfForGpu(
   }
 
   // erase the printf calls
-  for (auto P : Printfs) {
-    CallInst *CI = dyn_cast<CallInst>(P);
+  for (auto CI : Printfs)
     CI->eraseFromParent();
-  }
 
   Printfs.clear();
   return true;
@@ -579,7 +567,16 @@ bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) {
   if (TT.getArch() == Triple::r600)
     return false;
 
-  visit(M);
+  auto PrintfFunction = M.getFunction("printf");
+  if (!PrintfFunction)
+    return false;
+
+  for (auto &U : PrintfFunction->uses()) {
+    if (auto *CI = dyn_cast<CallInst>(U.getUser())) {
+      if (CI->isCallee(&U))
+        Printfs.push_back(CI);
+    }
+  }
 
   if (Printfs.empty())
     return false;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 50977afba5f39..badcd77aaef1f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -20,6 +20,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBank.h"
 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
 #include "llvm/CodeGen/TargetRegisterInfo.h"
@@ -33,6 +34,7 @@
 #include "AMDGPUGenRegisterBankInfo.def"
 
 using namespace llvm;
+using namespace MIPatternMatch;
 
 namespace {
 
@@ -84,9 +86,11 @@ class ApplyRegBankMapping final : public GISelChangeObserver {
 };
 
 }
-AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const TargetRegisterInfo &TRI)
+AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST)
     : AMDGPUGenRegisterBankInfo(),
-      TRI(static_cast<const SIRegisterInfo*>(&TRI)) {
+      Subtarget(ST),
+      TRI(Subtarget.getRegisterInfo()),
+      TII(Subtarget.getInstrInfo()) {
 
   // HACK: Until this is fully tablegen'd.
   static bool AlreadyInit = false;
@@ -210,7 +214,7 @@ AMDGPURegisterBankInfo::addMappingFromTable(
 RegisterBankInfo::InstructionMappings
 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(
     const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
-  switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+  switch (MI.getIntrinsicID()) {
   case Intrinsic::amdgcn_readlane: {
     static const OpRegBankEntry<3> Table[2] = {
       // Perfectly legal.
@@ -251,7 +255,7 @@ RegisterBankInfo::InstructionMappings
 AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(
     const MachineInstr &MI, const MachineRegisterInfo &MRI) const {
 
-  switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+  switch (MI.getIntrinsicID()) {
   case Intrinsic::amdgcn_buffer_load: {
     static const OpRegBankEntry<3> Table[4] = {
       // Perfectly legal.
@@ -339,7 +343,21 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
 
   InstructionMappings AltMappings;
   switch (MI.getOpcode()) {
-  case TargetOpcode::G_CONSTANT:
+  case TargetOpcode::G_CONSTANT: {
+    unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI);
+    if (Size == 1) {
+      static const OpRegBankEntry<1> Table[4] = {
+        { { AMDGPU::VGPRRegBankID }, 1 },
+        { { AMDGPU::SGPRRegBankID }, 1 },
+        { { AMDGPU::VCCRegBankID }, 1 },
+        { { AMDGPU::SCCRegBankID }, 1 }
+      };
+
+      return addMappingFromTable<1>(MI, MRI, { 0 }, Table);
+    }
+
+    LLVM_FALLTHROUGH;
+  }
   case TargetOpcode::G_FCONSTANT:
   case TargetOpcode::G_FRAME_INDEX:
   case TargetOpcode::G_GLOBAL_VALUE: {
@@ -348,7 +366,7 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
       { { AMDGPU::SGPRRegBankID }, 1 }
     };
 
-    return addMappingFromTable<1>(MI, MRI, { 0 }, Table);
+    return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table);
   }
   case TargetOpcode::G_AND:
   case TargetOpcode::G_OR:
@@ -429,8 +447,9 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings(
     unsigned PtrSize = PtrTy.getSizeInBits();
     unsigned AS = PtrTy.getAddressSpace();
     LLT LoadTy = MRI.getType(MI.getOperand(0).getReg());
-    if (isInstrUniformNonExtLoadAlign4(MI) &&
-        (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)) {
+    if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
+         AS != AMDGPUAS::PRIVATE_ADDRESS) &&
+        isInstrUniformNonExtLoadAlign4(MI)) {
       const InstructionMapping &SSMapping = getInstructionMapping(
           1, 1, getOperandsMapping(
                     {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size),
@@ -638,8 +657,10 @@ static LLT getHalfSizedType(LLT Ty) {
 ///
 /// There is additional complexity to try for compare values to identify the
 /// unique values used.
-void AMDGPURegisterBankInfo::executeInWaterfallLoop(
-  MachineInstr &MI, MachineRegisterInfo &MRI,
+bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
+  MachineIRBuilder &B,
+  MachineInstr &MI,
+  MachineRegisterInfo &MRI,
   ArrayRef<unsigned> OpIndices) const {
   MachineFunction *MF = MI.getParent()->getParent();
   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
@@ -662,9 +683,8 @@ void AMDGPURegisterBankInfo::executeInWaterfallLoop(
 
   // No operands need to be replaced, so no need to loop.
   if (SGPROperandRegs.empty())
-    return;
+    return false;
 
-  MachineIRBuilder B(MI);
   SmallVector<Register, 4> ResultRegs;
   SmallVector<Register, 4> InitResultRegs;
   SmallVector<Register, 4> PhiRegs;
@@ -922,6 +942,18 @@ void AMDGPURegisterBankInfo::executeInWaterfallLoop(
   B.buildInstr(AMDGPU::S_MOV_B64_term)
     .addDef(AMDGPU::EXEC)
     .addReg(SaveExecReg);
+
+  // Restore the insert point before the original instruction.
+  B.setInsertPt(MBB, MBB.end());
+
+  return true;
+}
+
+bool AMDGPURegisterBankInfo::executeInWaterfallLoop(
+  MachineInstr &MI, MachineRegisterInfo &MRI,
+  ArrayRef<unsigned> OpIndices) const {
+  MachineIRBuilder B(MI);
+  return executeInWaterfallLoop(B, MI, MRI, OpIndices);
 }
 
 // Legalize an operand that must be an SGPR by inserting a readfirstlane.
@@ -1031,6 +1063,33 @@ bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI,
   return true;
 }
 
+bool AMDGPURegisterBankInfo::applyMappingImage(
+    MachineInstr &MI, const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
+    MachineRegisterInfo &MRI, int RsrcIdx) const {
+  const int NumDefs = MI.getNumExplicitDefs();
+
+  // The reported argument index is relative to the IR intrinsic call arguments,
+  // so we need to shift by the number of defs and the intrinsic ID.
+  RsrcIdx += NumDefs + 1;
+
+  // Insert copies to VGPR arguments.
+  applyDefaultMapping(OpdMapper);
+
+  // Fixup any SGPR arguments.
+  SmallVector<unsigned, 4> SGPRIndexes;
+  for (int I = NumDefs, NumOps = MI.getNumOperands(); I != NumOps; ++I) {
+    if (!MI.getOperand(I).isReg())
+      continue;
+
+    // If this intrinsic has a sampler, it immediately follows rsrc.
+    if (I == RsrcIdx || I == RsrcIdx + 1)
+      SGPRIndexes.push_back(I);
+  }
+
+  executeInWaterfallLoop(MI, MRI, SGPRIndexes);
+  return true;
+}
+
 // For cases where only a single copy is inserted for matching register banks.
 // Replace the register in the instruction operand
 static void substituteSimpleCopyRegs(
@@ -1042,6 +1101,184 @@ static void substituteSimpleCopyRegs(
   }
 }
 
+/// Handle register layout difference for f16 images for some subtargets.
+Register AMDGPURegisterBankInfo::handleD16VData(MachineIRBuilder &B,
+                                                MachineRegisterInfo &MRI,
+                                                Register Reg) const {
+  if (!Subtarget.hasUnpackedD16VMem())
+    return Reg;
+
+  const LLT S16 = LLT::scalar(16);
+  LLT StoreVT = MRI.getType(Reg);
+  if (!StoreVT.isVector() || StoreVT.getElementType() != S16)
+    return Reg;
+
+  auto Unmerge = B.buildUnmerge(S16, Reg);
+
+
+  SmallVector<Register, 4> WideRegs;
+  for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
+    WideRegs.push_back(Unmerge.getReg(I));
+
+  const LLT S32 = LLT::scalar(32);
+  int NumElts = StoreVT.getNumElements();
+
+  return B.buildMerge(LLT::vector(NumElts, S32), WideRegs).getReg(0);
+}
+
+static std::pair<Register, unsigned>
+getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) {
+  int64_t Const;
+  if (mi_match(Reg, MRI, m_ICst(Const)))
+    return std::make_pair(Register(), Const);
+
+  Register Base;
+  if (mi_match(Reg, MRI, m_GAdd(m_Reg(Base), m_ICst(Const))))
+    return std::make_pair(Base, Const);
+
+  // TODO: Handle G_OR used for add case
+  return std::make_pair(Reg, 0);
+}
+
+std::pair<Register, unsigned>
+AMDGPURegisterBankInfo::splitBufferOffsets(MachineIRBuilder &B,
+                                           Register OrigOffset) const {
+  const unsigned MaxImm = 4095;
+  Register BaseReg;
+  unsigned ImmOffset;
+  const LLT S32 = LLT::scalar(32);
+
+  std::tie(BaseReg, ImmOffset) = getBaseWithConstantOffset(*B.getMRI(),
+                                                           OrigOffset);
+
+  unsigned C1 = 0;
+  if (ImmOffset != 0) {
+    // If the immediate value is too big for the immoffset field, put the value
+    // and -4096 into the immoffset field so that the value that is copied/added
+    // for the voffset field is a multiple of 4096, and it stands more chance
+    // of being CSEd with the copy/add for another similar load/store.
+    // However, do not do that rounding down to a multiple of 4096 if that is a
+    // negative number, as it appears to be illegal to have a negative offset
+    // in the vgpr, even if adding the immediate offset makes it positive.
+    unsigned Overflow = ImmOffset & ~MaxImm;
+    ImmOffset -= Overflow;
+    if ((int32_t)Overflow < 0) {
+      Overflow += ImmOffset;
+      ImmOffset = 0;
+    }
+
+    C1 = ImmOffset;
+    if (Overflow != 0) {
+      if (!BaseReg)
+        BaseReg = B.buildConstant(S32, Overflow).getReg(0);
+      else {
+        auto OverflowVal = B.buildConstant(S32, Overflow);
+        BaseReg = B.buildAdd(S32, BaseReg, OverflowVal).getReg(0);
+      }
+    }
+  }
+
+  if (!BaseReg)
+    BaseReg = B.buildConstant(S32, 0).getReg(0);
+
+  return {BaseReg, C1};
+}
+
+static bool isZero(Register Reg, MachineRegisterInfo &MRI) {
+  int64_t C;
+  return mi_match(Reg, MRI, m_ICst(C)) && C == 0;
+}
+
+static unsigned extractGLC(unsigned CachePolicy) {
+  return CachePolicy & 1;
+}
+
+static unsigned extractSLC(unsigned CachePolicy) {
+  return (CachePolicy >> 1) & 1;
+}
+
+static unsigned extractDLC(unsigned CachePolicy) {
+  return (CachePolicy >> 2) & 1;
+}
+
+MachineInstr *
+AMDGPURegisterBankInfo::selectStoreIntrinsic(MachineIRBuilder &B,
+                                             MachineInstr &MI) const {
+   MachineRegisterInfo &MRI = *B.getMRI();
+  executeInWaterfallLoop(B, MI, MRI, {2, 4});
+
+  // FIXME: DAG lowering brokenly changes opcode based on FP vs. integer.
+
+  Register VData = MI.getOperand(1).getReg();
+  LLT Ty = MRI.getType(VData);
+
+  int EltSize = Ty.getScalarSizeInBits();
+  int Size = Ty.getSizeInBits();
+
+  // FIXME: Broken integer truncstore.
+  if (EltSize != 32)
+    report_fatal_error("unhandled intrinsic store");
+
+  // FIXME: Verifier should enforce 1 MMO for these intrinsics.
+  const int MemSize = (*MI.memoperands_begin())->getSize();
+
+
+  Register RSrc = MI.getOperand(2).getReg();
+  Register VOffset = MI.getOperand(3).getReg();
+  Register SOffset = MI.getOperand(4).getReg();
+  unsigned CachePolicy = MI.getOperand(5).getImm();
+
+  unsigned ImmOffset;
+  std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+
+  const bool Offen = !isZero(VOffset, MRI);
+
+  unsigned Opc = AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact;
+  switch (8 * MemSize) {
+  case 8:
+    Opc = Offen ? AMDGPU::BUFFER_STORE_BYTE_OFFEN_exact :
+                  AMDGPU::BUFFER_STORE_BYTE_OFFSET_exact;
+    break;
+  case 16:
+    Opc = Offen ? AMDGPU::BUFFER_STORE_SHORT_OFFEN_exact :
+                  AMDGPU::BUFFER_STORE_SHORT_OFFSET_exact;
+    break;
+  default:
+    Opc = Offen ? AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact :
+                  AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact;
+    if (Size > 32)
+      Opc = AMDGPU::getMUBUFOpcode(Opc, Size / 32);
+    break;
+  }
+
+
+  // Set the insertion point back to the instruction in case it was moved into a
+  // loop.
+  B.setInstr(MI);
+
+  MachineInstrBuilder MIB = B.buildInstr(Opc)
+    .addUse(VData);
+
+  if (Offen)
+    MIB.addUse(VOffset);
+
+  MIB.addUse(RSrc)
+     .addUse(SOffset)
+     .addImm(ImmOffset)
+     .addImm(extractGLC(CachePolicy))
+     .addImm(extractSLC(CachePolicy))
+     .addImm(0) // tfe: FIXME: Remove from inst
+     .addImm(extractDLC(CachePolicy))
+     .cloneMemRefs(MI);
+
+  // FIXME: We need a way to report failure from applyMappingImpl.
+  // Insert constrain copies before inserting the loop.
+  if (!constrainSelectedInstRegOperands(*MIB, *TII, *TRI, *this))
+    report_fatal_error("failed to constrain selected store intrinsic");
+
+  return MIB;
+}
+
 void AMDGPURegisterBankInfo::applyMappingImpl(
     const OperandsMapper &OpdMapper) const {
   MachineInstr &MI = OpdMapper.getMI();
@@ -1372,7 +1609,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
     executeInWaterfallLoop(MI, MRI, { 2 });
     return;
   case AMDGPU::G_INTRINSIC: {
-    switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+    switch (MI.getIntrinsicID()) {
     case Intrinsic::amdgcn_s_buffer_load: {
       // FIXME: Move to G_INTRINSIC_W_SIDE_EFFECTS
       executeInWaterfallLoop(MI, MRI, { 2, 3 });
@@ -1405,7 +1642,8 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
     break;
   }
   case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
-    switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+    auto IntrID = MI.getIntrinsicID();
+    switch (IntrID) {
     case Intrinsic::amdgcn_buffer_load: {
       executeInWaterfallLoop(MI, MRI, { 2 });
       return;
@@ -1424,9 +1662,39 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
       constrainOpWithReadfirstlane(MI, MRI, 2); // M0
       return;
     }
-    default:
+    case Intrinsic::amdgcn_raw_buffer_load:
+    case Intrinsic::amdgcn_raw_buffer_load_format:
+    case Intrinsic::amdgcn_raw_tbuffer_load:
+    case Intrinsic::amdgcn_raw_buffer_store:
+    case Intrinsic::amdgcn_raw_buffer_store_format:
+    case Intrinsic::amdgcn_raw_tbuffer_store: {
+      applyDefaultMapping(OpdMapper);
+      executeInWaterfallLoop(MI, MRI, {2, 4});
+      return;
+    }
+    case Intrinsic::amdgcn_struct_buffer_load:
+    case Intrinsic::amdgcn_struct_buffer_store:
+    case Intrinsic::amdgcn_struct_tbuffer_load:
+    case Intrinsic::amdgcn_struct_tbuffer_store: {
+      applyDefaultMapping(OpdMapper);
+      executeInWaterfallLoop(MI, MRI, {2, 5});
+      return;
+    }
+    default: {
+      if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
+              AMDGPU::lookupRsrcIntrinsic(IntrID)) {
+        // Non-images can have complications from operands that allow both SGPR
+        // and VGPR. For now it's too complicated to figure out the final opcode
+        // to derive the register bank from the MCInstrDesc.
+        if (RSrcIntrin->IsImage) {
+          applyMappingImage(MI, OpdMapper, MRI, RSrcIntrin->RsrcArg);
+          return;
+        }
+      }
+
       break;
     }
+    }
     break;
   }
   case AMDGPU::G_LOAD:
@@ -1531,6 +1799,45 @@ AMDGPURegisterBankInfo::getDefaultMappingAllVGPR(const MachineInstr &MI) const {
                                MI.getNumOperands());
 }
 
+const RegisterBankInfo::InstructionMapping &
+AMDGPURegisterBankInfo::getImageMapping(const MachineRegisterInfo &MRI,
+                                        const MachineInstr &MI,
+                                        int RsrcIdx) const {
+  // The reported argument index is relative to the IR intrinsic call arguments,
+  // so we need to shift by the number of defs and the intrinsic ID.
+  RsrcIdx += MI.getNumExplicitDefs() + 1;
+
+  const int NumOps = MI.getNumOperands();
+  SmallVector<const ValueMapping *, 8> OpdsMapping(NumOps);
+
+  // TODO: Should packed/unpacked D16 difference be reported here as part of
+  // the value mapping?
+  for (int I = 0; I != NumOps; ++I) {
+    if (!MI.getOperand(I).isReg())
+      continue;
+
+    Register OpReg = MI.getOperand(I).getReg();
+    unsigned Size = getSizeInBits(OpReg, MRI, *TRI);
+
+    // FIXME: Probably need a new intrinsic register bank searchable table to
+    // handle arbitrary intrinsics easily.
+    //
+    // If this has a sampler, it immediately follows rsrc.
+    const bool MustBeSGPR = I == RsrcIdx || I == RsrcIdx + 1;
+
+    if (MustBeSGPR) {
+      // If this must be an SGPR, so we must report whatever it is as legal.
+      unsigned NewBank = getRegBankID(OpReg, MRI, *TRI, AMDGPU::SGPRRegBankID);
+      OpdsMapping[I] = AMDGPU::getValueMapping(NewBank, Size);
+    } else {
+      // Some operands must be VGPR, and these are easy to copy to.
+      OpdsMapping[I] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
+    }
+  }
+
+  return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), NumOps);
+}
+
 const RegisterBankInfo::InstructionMapping &
 AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
 
@@ -1547,8 +1854,9 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const {
   const ValueMapping *ValMapping;
   const ValueMapping *PtrMapping;
 
-  if (isInstrUniformNonExtLoadAlign4(MI) &&
-      (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS)) {
+  if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS &&
+       AS != AMDGPUAS::PRIVATE_ADDRESS) &&
+      isInstrUniformNonExtLoadAlign4(MI)) {
     // We have a uniform instruction so we want to use an SMRD load
     ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
     PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize);
@@ -1577,11 +1885,31 @@ AMDGPURegisterBankInfo::getRegBankID(Register Reg,
   return Bank ? Bank->getID() : Default;
 }
 
+
 static unsigned regBankUnion(unsigned RB0, unsigned RB1) {
   return (RB0 == AMDGPU::SGPRRegBankID && RB1 == AMDGPU::SGPRRegBankID) ?
     AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID;
 }
 
+const RegisterBankInfo::ValueMapping *
+AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg,
+                                         const MachineRegisterInfo &MRI,
+                                         const TargetRegisterInfo &TRI) const {
+  // Lie and claim anything is legal, even though this needs to be an SGPR
+  // applyMapping will have to deal with it as a waterfall loop.
+  unsigned Bank = getRegBankID(Reg, MRI, TRI, AMDGPU::SGPRRegBankID);
+  unsigned Size = getSizeInBits(Reg, MRI, TRI);
+  return AMDGPU::getValueMapping(Bank, Size);
+}
+
+const RegisterBankInfo::ValueMapping *
+AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg,
+                                         const MachineRegisterInfo &MRI,
+                                         const TargetRegisterInfo &TRI) const {
+  unsigned Size = getSizeInBits(Reg, MRI, TRI);
+  return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
+}
+
 ///
 /// This function must return a legal mapping, because
 /// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called
@@ -1748,7 +2076,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
 
     LLVM_FALLTHROUGH;
   }
-
   case AMDGPU::G_GEP:
   case AMDGPU::G_ADD:
   case AMDGPU::G_SUB:
@@ -1764,8 +2091,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_SADDE:
   case AMDGPU::G_USUBE:
   case AMDGPU::G_SSUBE:
-  case AMDGPU::G_UMULH:
-  case AMDGPU::G_SMULH:
   case AMDGPU::G_SMIN:
   case AMDGPU::G_SMAX:
   case AMDGPU::G_UMIN:
@@ -1783,6 +2108,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_FMAD:
   case AMDGPU::G_FSQRT:
   case AMDGPU::G_FFLOOR:
+  case AMDGPU::G_FCEIL:
+  case AMDGPU::G_FRINT:
   case AMDGPU::G_SITOFP:
   case AMDGPU::G_UITOFP:
   case AMDGPU::G_FPTRUNC:
@@ -1797,6 +2124,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_INTRINSIC_TRUNC:
   case AMDGPU::G_INTRINSIC_ROUND:
     return getDefaultMappingVOP(MI);
+  case AMDGPU::G_UMULH:
+  case AMDGPU::G_SMULH: {
+    if (MF.getSubtarget<GCNSubtarget>().hasScalarMulHiInsts() &&
+        isSALUMapping(MI))
+      return getDefaultMappingSOP(MI);
+    return getDefaultMappingVOP(MI);
+  }
   case AMDGPU::G_IMPLICIT_DEF: {
     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
@@ -1804,13 +2138,19 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   }
   case AMDGPU::G_FCONSTANT:
   case AMDGPU::G_CONSTANT:
-  case AMDGPU::G_FRAME_INDEX:
   case AMDGPU::G_GLOBAL_VALUE:
   case AMDGPU::G_BLOCK_ADDR: {
     unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
     OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
     break;
   }
+  case AMDGPU::G_FRAME_INDEX: {
+    // TODO: This should be the same as other constants, but eliminateFrameIndex
+    // currently assumes VALU uses.
+    unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
+    OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size);
+    break;
+  }
   case AMDGPU::G_INSERT: {
     unsigned BankID = isSALUMapping(MI) ? AMDGPU::SGPRRegBankID :
                                           AMDGPU::VGPRRegBankID;
@@ -2016,7 +2356,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     break;
   }
   case AMDGPU::G_INTRINSIC: {
-    switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
+    switch (MI.getIntrinsicID()) {
     default:
       return getInvalidInstructionMapping();
     case Intrinsic::amdgcn_div_fmas:
@@ -2070,6 +2410,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     case Intrinsic::amdgcn_wwm:
     case Intrinsic::amdgcn_wqm:
       return getDefaultMappingVOP(MI);
+    case Intrinsic::amdgcn_ds_swizzle:
     case Intrinsic::amdgcn_ds_permute:
     case Intrinsic::amdgcn_ds_bpermute:
     case Intrinsic::amdgcn_update_dpp:
@@ -2191,9 +2532,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
     break;
   }
   case AMDGPU::G_INTRINSIC_W_SIDE_EFFECTS: {
-    switch (MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID()) {
-    default:
-      return getInvalidInstructionMapping();
+    auto IntrID = MI.getIntrinsicID();
+    switch (IntrID) {
     case Intrinsic::amdgcn_s_getreg:
     case Intrinsic::amdgcn_s_memtime:
     case Intrinsic::amdgcn_s_memrealtime:
@@ -2233,18 +2573,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
       break;
     case Intrinsic::amdgcn_exp:
-      OpdsMapping[0] = nullptr; // IntrinsicID
-      // FIXME: These are immediate values which can't be read from registers.
-      OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
-      OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
       // FIXME: Could we support packed types here?
       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[4] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[5] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
       OpdsMapping[6] = AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, 32);
-      // FIXME: These are immediate values which can't be read from registers.
-      OpdsMapping[7] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
-      OpdsMapping[8] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32);
       break;
     case Intrinsic::amdgcn_buffer_load: {
       Register RSrc = MI.getOperand(2).getReg();   // SGPR
@@ -2280,7 +2613,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[2] = AMDGPU::getValueMapping(Bank, 32);
       break;
     }
-    case Intrinsic::amdgcn_end_cf: {
+    case Intrinsic::amdgcn_end_cf:
+    case Intrinsic::amdgcn_init_exec: {
       unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
       OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
@@ -2292,6 +2626,64 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
       OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, WaveSize);
       break;
     }
+    case Intrinsic::amdgcn_kill: {
+      OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, 1);
+      break;
+    }
+    case Intrinsic::amdgcn_raw_buffer_load:
+    case Intrinsic::amdgcn_raw_tbuffer_load: {
+      // FIXME: Should make intrinsic ID the last operand of the instruction,
+      // then this would be the same as store
+      OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+      OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+      break;
+    }
+    case Intrinsic::amdgcn_raw_buffer_store:
+    case Intrinsic::amdgcn_raw_buffer_store_format:
+    case Intrinsic::amdgcn_raw_tbuffer_store: {
+      OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+      OpdsMapping[4] = getSGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+      break;
+    }
+    case Intrinsic::amdgcn_struct_buffer_load:
+    case Intrinsic::amdgcn_struct_tbuffer_load: {
+      OpdsMapping[0] = getVGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+      OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+      OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
+      break;
+    }
+    case Intrinsic::amdgcn_struct_buffer_store:
+    case Intrinsic::amdgcn_struct_tbuffer_store: {
+      OpdsMapping[1] = getVGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI);
+      OpdsMapping[4] = getVGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI);
+      OpdsMapping[5] = getSGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI);
+      break;
+    }
+    case Intrinsic::amdgcn_init_exec_from_input: {
+      unsigned Size = getSizeInBits(MI.getOperand(1).getReg(), MRI, *TRI);
+      OpdsMapping[1] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
+      OpdsMapping[2] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
+      break;
+    }
+    default:
+      if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
+              AMDGPU::lookupRsrcIntrinsic(IntrID)) {
+        // Non-images can have complications from operands that allow both SGPR
+        // and VGPR. For now it's too complicated to figure out the final opcode
+        // to derive the register bank from the MCInstrDesc.
+        if (RSrcIntrin->IsImage)
+          return getImageMapping(MRI, MI, RSrcIntrin->RsrcArg);
+      }
+
+      return getInvalidInstructionMapping();
     }
     break;
   }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
index f3a96e2a6128a..584b23c0c2204 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h
@@ -23,7 +23,9 @@
 namespace llvm {
 
 class LLT;
+class GCNSubtarget;
 class MachineIRBuilder;
+class SIInstrInfo;
 class SIRegisterInfo;
 class TargetRegisterInfo;
 
@@ -36,9 +38,15 @@ class AMDGPUGenRegisterBankInfo : public RegisterBankInfo {
 #include "AMDGPUGenRegisterBank.inc"
 };
 class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
+  const GCNSubtarget &Subtarget;
   const SIRegisterInfo *TRI;
+  const SIInstrInfo *TII;
 
-  void executeInWaterfallLoop(MachineInstr &MI,
+  bool executeInWaterfallLoop(MachineIRBuilder &B,
+                              MachineInstr &MI,
+                              MachineRegisterInfo &MRI,
+                              ArrayRef<unsigned> OpIndices) const;
+  bool executeInWaterfallLoop(MachineInstr &MI,
                               MachineRegisterInfo &MRI,
                               ArrayRef<unsigned> OpIndices) const;
 
@@ -47,6 +55,19 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
   bool applyMappingWideLoad(MachineInstr &MI,
                             const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
                             MachineRegisterInfo &MRI) const;
+  bool
+  applyMappingImage(MachineInstr &MI,
+                    const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper,
+                    MachineRegisterInfo &MRI, int RSrcIdx) const;
+
+  Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI,
+                          Register Reg) const;
+
+  std::pair<Register, unsigned>
+  splitBufferOffsets(MachineIRBuilder &B, Register Offset) const;
+
+  MachineInstr *selectStoreIntrinsic(MachineIRBuilder &B,
+                                     MachineInstr &MI) const;
 
   /// See RegisterBankInfo::applyMapping.
   void applyMappingImpl(const OperandsMapper &OpdMapper) const override;
@@ -58,6 +79,16 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
                         const TargetRegisterInfo &TRI,
                         unsigned Default = AMDGPU::VGPRRegBankID) const;
 
+  // Return a value mapping for an operand that is required to be an SGPR.
+  const ValueMapping *getSGPROpMapping(Register Reg,
+                                       const MachineRegisterInfo &MRI,
+                                       const TargetRegisterInfo &TRI) const;
+
+  // Return a value mapping for an operand that is required to be a VGPR.
+  const ValueMapping *getVGPROpMapping(Register Reg,
+                                       const MachineRegisterInfo &MRI,
+                                       const TargetRegisterInfo &TRI) const;
+
   /// Split 64-bit value \p Reg into two 32-bit halves and populate them into \p
   /// Regs. This appropriately sets the regbank of the new registers.
   void split64BitValueForMapping(MachineIRBuilder &B,
@@ -90,8 +121,13 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo {
   const InstructionMapping &getDefaultMappingVOP(const MachineInstr &MI) const;
   const InstructionMapping &getDefaultMappingAllVGPR(
     const MachineInstr &MI) const;
+
+  const InstructionMapping &getImageMapping(const MachineRegisterInfo &MRI,
+                                            const MachineInstr &MI,
+                                            int RsrcIdx) const;
+
 public:
-  AMDGPURegisterBankInfo(const TargetRegisterInfo &TRI);
+  AMDGPURegisterBankInfo(const GCNSubtarget &STI);
 
   unsigned copyCost(const RegisterBank &A, const RegisterBank &B,
                     unsigned Size) const override;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
index 2ecb186cc88f6..00f53b1575770 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td
@@ -7,11 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 def SGPRRegBank : RegisterBank<"SGPR",
-  [SReg_32, SReg_64, SReg_128, SReg_256, SReg_512]
+  [SReg_32, SReg_64, SReg_128, SReg_256, SReg_512, SReg_1024]
 >;
 
 def VGPRRegBank : RegisterBank<"VGPR",
-  [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512]
+  [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512, VReg_1024]
 >;
 
 def SCCRegBank : RegisterBank <"SCC", [SReg_32, SCC_CLASS]>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 3e556b46fa525..cf89af68d6f80 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -283,7 +283,7 @@ GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
   MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(this);
   CallLoweringInfo.reset(new AMDGPUCallLowering(*getTargetLowering()));
   Legalizer.reset(new AMDGPULegalizerInfo(*this, TM));
-  RegBankInfo.reset(new AMDGPURegisterBankInfo(*getRegisterInfo()));
+  RegBankInfo.reset(new AMDGPURegisterBankInfo(*this));
   InstSelector.reset(new AMDGPUInstructionSelector(
   *this, *static_cast<AMDGPURegisterBankInfo *>(RegBankInfo.get()), TM));
 }
@@ -599,7 +599,7 @@ unsigned GCNSubtarget::getOccupancyWithNumVGPRs(unsigned VGPRs) const {
   if (VGPRs < Granule)
     return MaxWaves;
   unsigned RoundedRegs = ((VGPRs + Granule - 1) / Granule) * Granule;
-  return std::min(getTotalNumVGPRs() / RoundedRegs, MaxWaves);
+  return std::min(std::max(getTotalNumVGPRs() / RoundedRegs, 1u), MaxWaves);
 }
 
 unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
index c9d305a5bba8b..bf7cf86bc42db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h
@@ -555,6 +555,10 @@ class GCNSubtarget : public AMDGPUGenSubtargetInfo,
     return GFX9Insts;
   }
 
+  bool hasScalarMulHiInsts() const {
+    return GFX9Insts;
+  }
+
   TrapHandlerAbi getTrapHandlerAbi() const {
     return isAmdHsaOS() ? TrapHandlerAbiHsa : TrapHandlerAbiNone;
   }
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index b8e91d3f3642a..94d1d350dfd2d 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -143,6 +143,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     ImmTyDLC,
     ImmTyGLC,
     ImmTySLC,
+    ImmTySWZ,
     ImmTyTFE,
     ImmTyD16,
     ImmTyClampSI,
@@ -328,6 +329,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
   bool isDLC() const { return isImmTy(ImmTyDLC); }
   bool isGLC() const { return isImmTy(ImmTyGLC); }
   bool isSLC() const { return isImmTy(ImmTySLC); }
+  bool isSWZ() const { return isImmTy(ImmTySWZ); }
   bool isTFE() const { return isImmTy(ImmTyTFE); }
   bool isD16() const { return isImmTy(ImmTyD16); }
   bool isFORMAT() const { return isImmTy(ImmTyFORMAT) && isUInt<8>(getImm()); }
@@ -820,6 +822,7 @@ class AMDGPUOperand : public MCParsedAsmOperand {
     case ImmTyDLC: OS << "DLC"; break;
     case ImmTyGLC: OS << "GLC"; break;
     case ImmTySLC: OS << "SLC"; break;
+    case ImmTySWZ: OS << "SWZ"; break;
     case ImmTyTFE: OS << "TFE"; break;
     case ImmTyD16: OS << "D16"; break;
     case ImmTyFORMAT: OS << "FORMAT"; break;
@@ -1054,11 +1057,23 @@ class AMDGPUAsmParser : public MCTargetAsmParser {
                            std::string &CollectString);
 
   bool AddNextRegisterToList(unsigned& Reg, unsigned& RegWidth,
-                             RegisterKind RegKind, unsigned Reg1,
-                             unsigned RegNum);
+                             RegisterKind RegKind, unsigned Reg1);
   bool ParseAMDGPURegister(RegisterKind& RegKind, unsigned& Reg,
-                           unsigned& RegNum, unsigned& RegWidth,
-                           unsigned *DwordRegIndex);
+                           unsigned& RegNum, unsigned& RegWidth);
+  unsigned ParseRegularReg(RegisterKind &RegKind,
+                           unsigned &RegNum,
+                           unsigned &RegWidth);
+  unsigned ParseSpecialReg(RegisterKind &RegKind,
+                           unsigned &RegNum,
+                           unsigned &RegWidth);
+  unsigned ParseRegList(RegisterKind &RegKind,
+                        unsigned &RegNum,
+                        unsigned &RegWidth);
+  bool ParseRegRange(unsigned& Num, unsigned& Width);
+  unsigned getRegularReg(RegisterKind RegKind,
+                         unsigned RegNum,
+                         unsigned RegWidth);
+
   bool isRegister();
   bool isRegister(const AsmToken &Token, const AsmToken &NextToken) const;
   Optional<StringRef> getGprCountSymbolName(RegisterKind RegKind);
@@ -1953,7 +1968,7 @@ static unsigned getSpecialRegForName(StringRef RegName) {
     .Case("tba_lo", AMDGPU::TBA_LO)
     .Case("tba_hi", AMDGPU::TBA_HI)
     .Case("null", AMDGPU::SGPR_NULL)
-    .Default(0);
+    .Default(AMDGPU::NoRegister);
 }
 
 bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
@@ -1968,8 +1983,7 @@ bool AMDGPUAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
 }
 
 bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
-                                            RegisterKind RegKind, unsigned Reg1,
-                                            unsigned RegNum) {
+                                            RegisterKind RegKind, unsigned Reg1) {
   switch (RegKind) {
   case IS_SPECIAL:
     if (Reg == AMDGPU::EXEC_LO && Reg1 == AMDGPU::EXEC_HI) {
@@ -2017,14 +2031,37 @@ bool AMDGPUAsmParser::AddNextRegisterToList(unsigned &Reg, unsigned &RegWidth,
   }
 }
 
-static const StringRef Registers[] = {
-  { "v" },
-  { "s" },
-  { "ttmp" },
-  { "acc" },
-  { "a" },
+struct RegInfo {
+  StringLiteral Name;
+  RegisterKind Kind;
+};
+
+static constexpr RegInfo RegularRegisters[] = {
+  {{"v"},    IS_VGPR},
+  {{"s"},    IS_SGPR},
+  {{"ttmp"}, IS_TTMP},
+  {{"acc"},  IS_AGPR},
+  {{"a"},    IS_AGPR},
 };
 
+static bool isRegularReg(RegisterKind Kind) {
+  return Kind == IS_VGPR ||
+         Kind == IS_SGPR ||
+         Kind == IS_TTMP ||
+         Kind == IS_AGPR;
+}
+
+static const RegInfo* getRegularRegInfo(StringRef Str) {
+  for (const RegInfo &Reg : RegularRegisters)
+    if (Str.startswith(Reg.Name))
+      return &Reg;
+  return nullptr;
+}
+
+static bool getRegNum(StringRef Str, unsigned& Num) {
+  return !Str.getAsInteger(10, Num);
+}
+
 bool
 AMDGPUAsmParser::isRegister(const AsmToken &Token,
                             const AsmToken &NextToken) const {
@@ -2038,24 +2075,24 @@ AMDGPUAsmParser::isRegister(const AsmToken &Token,
 
   // A single register like s0 or a range of registers like s[0:1]
 
-  StringRef RegName = Token.getString();
-
-  for (StringRef Reg : Registers) {
-    if (RegName.startswith(Reg)) {
-      if (Reg.size() < RegName.size()) {
-        unsigned RegNum;
-        // A single register with an index: rXX
-        if (!RegName.substr(Reg.size()).getAsInteger(10, RegNum))
-          return true;
-      } else {
-        // A range of registers: r[XX:YY].
-        if (NextToken.is(AsmToken::LBrac))
-          return true;
-      }
+  StringRef Str = Token.getString();
+  const RegInfo *Reg = getRegularRegInfo(Str);
+  if (Reg) {
+    StringRef RegName = Reg->Name;
+    StringRef RegSuffix = Str.substr(RegName.size());
+    if (!RegSuffix.empty()) {
+      unsigned Num;
+      // A single register with an index: rXX
+      if (getRegNum(RegSuffix, Num))
+        return true;
+    } else {
+      // A range of registers: r[XX:YY].
+      if (NextToken.is(AsmToken::LBrac))
+        return true;
     }
   }
 
-  return getSpecialRegForName(RegName);
+  return getSpecialRegForName(Str) != AMDGPU::NoRegister;
 }
 
 bool
@@ -2064,137 +2101,161 @@ AMDGPUAsmParser::isRegister()
   return isRegister(getToken(), peekToken());
 }
 
-bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind, unsigned &Reg,
-                                          unsigned &RegNum, unsigned &RegWidth,
-                                          unsigned *DwordRegIndex) {
-  if (DwordRegIndex) { *DwordRegIndex = 0; }
+unsigned
+AMDGPUAsmParser::getRegularReg(RegisterKind RegKind,
+                               unsigned RegNum,
+                               unsigned RegWidth) {
+
+  assert(isRegularReg(RegKind));
+
+  unsigned AlignSize = 1;
+  if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
+    // SGPR and TTMP registers must be aligned.
+    // Max required alignment is 4 dwords.
+    AlignSize = std::min(RegWidth, 4u);
+  }
+
+  if (RegNum % AlignSize != 0)
+    return AMDGPU::NoRegister;
+
+  unsigned RegIdx = RegNum / AlignSize;
+  int RCID = getRegClass(RegKind, RegWidth);
+  if (RCID == -1)
+    return AMDGPU::NoRegister;
+
   const MCRegisterInfo *TRI = getContext().getRegisterInfo();
-  if (getLexer().is(AsmToken::Identifier)) {
-    StringRef RegName = Parser.getTok().getString();
-    if ((Reg = getSpecialRegForName(RegName))) {
-      Parser.Lex();
-      RegKind = IS_SPECIAL;
-    } else {
-      unsigned RegNumIndex = 0;
-      if (RegName[0] == 'v') {
-        RegNumIndex = 1;
-        RegKind = IS_VGPR;
-      } else if (RegName[0] == 's') {
-        RegNumIndex = 1;
-        RegKind = IS_SGPR;
-      } else if (RegName[0] == 'a') {
-        RegNumIndex = RegName.startswith("acc") ? 3 : 1;
-        RegKind = IS_AGPR;
-      } else if (RegName.startswith("ttmp")) {
-        RegNumIndex = strlen("ttmp");
-        RegKind = IS_TTMP;
-      } else {
-        return false;
-      }
-      if (RegName.size() > RegNumIndex) {
-        // Single 32-bit register: vXX.
-        if (RegName.substr(RegNumIndex).getAsInteger(10, RegNum))
-          return false;
-        Parser.Lex();
-        RegWidth = 1;
-      } else {
-        // Range of registers: v[XX:YY]. ":YY" is optional.
-        Parser.Lex();
-        int64_t RegLo, RegHi;
-        if (getLexer().isNot(AsmToken::LBrac))
-          return false;
-        Parser.Lex();
+  const MCRegisterClass RC = TRI->getRegClass(RCID);
+  if (RegIdx >= RC.getNumRegs())
+    return AMDGPU::NoRegister;
 
-        if (getParser().parseAbsoluteExpression(RegLo))
-          return false;
+  return RC.getRegister(RegIdx);
+}
 
-        const bool isRBrace = getLexer().is(AsmToken::RBrac);
-        if (!isRBrace && getLexer().isNot(AsmToken::Colon))
-          return false;
-        Parser.Lex();
+bool
+AMDGPUAsmParser::ParseRegRange(unsigned& Num, unsigned& Width) {
+  int64_t RegLo, RegHi;
+  if (!trySkipToken(AsmToken::LBrac))
+    return false;
 
-        if (isRBrace) {
-          RegHi = RegLo;
-        } else {
-          if (getParser().parseAbsoluteExpression(RegHi))
-            return false;
+  if (!parseExpr(RegLo))
+    return false;
 
-          if (getLexer().isNot(AsmToken::RBrac))
-            return false;
-          Parser.Lex();
-        }
-        RegNum = (unsigned) RegLo;
-        RegWidth = (RegHi - RegLo) + 1;
-      }
-    }
-  } else if (getLexer().is(AsmToken::LBrac)) {
-    // List of consecutive registers: [s0,s1,s2,s3]
-    Parser.Lex();
-    if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, nullptr))
+  if (trySkipToken(AsmToken::Colon)) {
+    if (!parseExpr(RegHi))
       return false;
-    if (RegWidth != 1)
-      return false;
-    RegisterKind RegKind1;
-    unsigned Reg1, RegNum1, RegWidth1;
-    do {
-      if (getLexer().is(AsmToken::Comma)) {
-        Parser.Lex();
-      } else if (getLexer().is(AsmToken::RBrac)) {
-        Parser.Lex();
-        break;
-      } else if (ParseAMDGPURegister(RegKind1, Reg1, RegNum1, RegWidth1, nullptr)) {
-        if (RegWidth1 != 1) {
-          return false;
-        }
-        if (RegKind1 != RegKind) {
-          return false;
-        }
-        if (!AddNextRegisterToList(Reg, RegWidth, RegKind1, Reg1, RegNum1)) {
-          return false;
-        }
-      } else {
-        return false;
-      }
-    } while (true);
   } else {
-    return false;
+    RegHi = RegLo;
   }
-  switch (RegKind) {
-  case IS_SPECIAL:
+
+  if (!trySkipToken(AsmToken::RBrac))
+    return false;
+
+  if (!isUInt<32>(RegLo) || !isUInt<32>(RegHi) || RegLo > RegHi)
+    return false;
+
+  Num = static_cast<unsigned>(RegLo);
+  Width = (RegHi - RegLo) + 1;
+  return true;
+}
+
+unsigned
+AMDGPUAsmParser::ParseSpecialReg(RegisterKind &RegKind,
+                                 unsigned &RegNum,
+                                 unsigned &RegWidth) {
+  assert(isToken(AsmToken::Identifier));
+  unsigned Reg = getSpecialRegForName(getTokenStr());
+  if (Reg) {
     RegNum = 0;
     RegWidth = 1;
-    break;
-  case IS_VGPR:
-  case IS_SGPR:
-  case IS_AGPR:
-  case IS_TTMP:
-  {
-    unsigned Size = 1;
-    if (RegKind == IS_SGPR || RegKind == IS_TTMP) {
-      // SGPR and TTMP registers must be aligned. Max required alignment is 4 dwords.
-      Size = std::min(RegWidth, 4u);
-    }
-    if (RegNum % Size != 0)
-      return false;
-    if (DwordRegIndex) { *DwordRegIndex = RegNum; }
-    RegNum = RegNum / Size;
-    int RCID = getRegClass(RegKind, RegWidth);
-    if (RCID == -1)
-      return false;
-    const MCRegisterClass RC = TRI->getRegClass(RCID);
-    if (RegNum >= RC.getNumRegs())
-      return false;
-    Reg = RC.getRegister(RegNum);
-    break;
+    RegKind = IS_SPECIAL;
+    lex(); // skip register name
+  }
+  return Reg;
+}
+
+unsigned
+AMDGPUAsmParser::ParseRegularReg(RegisterKind &RegKind,
+                                 unsigned &RegNum,
+                                 unsigned &RegWidth) {
+  assert(isToken(AsmToken::Identifier));
+  StringRef RegName = getTokenStr();
+
+  const RegInfo *RI = getRegularRegInfo(RegName);
+  if (!RI)
+    return AMDGPU::NoRegister;
+  lex(); // skip register name
+
+  RegKind = RI->Kind;
+  StringRef RegSuffix = RegName.substr(RI->Name.size());
+  if (!RegSuffix.empty()) {
+    // Single 32-bit register: vXX.
+    if (!getRegNum(RegSuffix, RegNum))
+      return AMDGPU::NoRegister;
+    RegWidth = 1;
+  } else {
+    // Range of registers: v[XX:YY]. ":YY" is optional.
+    if (!ParseRegRange(RegNum, RegWidth))
+      return AMDGPU::NoRegister;
   }
 
-  default:
-    llvm_unreachable("unexpected register kind");
+  return getRegularReg(RegKind, RegNum, RegWidth);
+}
+
+unsigned
+AMDGPUAsmParser::ParseRegList(RegisterKind &RegKind,
+                              unsigned &RegNum,
+                              unsigned &RegWidth) {
+  unsigned Reg = AMDGPU::NoRegister;
+
+  if (!trySkipToken(AsmToken::LBrac))
+    return AMDGPU::NoRegister;
+
+  // List of consecutive registers, e.g.: [s0,s1,s2,s3]
+
+  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth))
+    return AMDGPU::NoRegister;
+  if (RegWidth != 1)
+    return AMDGPU::NoRegister;
+
+  for (; trySkipToken(AsmToken::Comma); ) {
+    RegisterKind NextRegKind;
+    unsigned NextReg, NextRegNum, NextRegWidth;
+
+    if (!ParseAMDGPURegister(NextRegKind, NextReg, NextRegNum, NextRegWidth))
+      return AMDGPU::NoRegister;
+    if (NextRegWidth != 1)
+      return AMDGPU::NoRegister;
+    if (NextRegKind != RegKind)
+      return AMDGPU::NoRegister;
+    if (!AddNextRegisterToList(Reg, RegWidth, RegKind, NextReg))
+      return AMDGPU::NoRegister;
   }
 
-  if (!subtargetHasRegister(*TRI, Reg))
-    return false;
-  return true;
+  if (!trySkipToken(AsmToken::RBrac))
+    return AMDGPU::NoRegister;
+
+  if (isRegularReg(RegKind))
+    Reg = getRegularReg(RegKind, RegNum, RegWidth);
+
+  return Reg;
+}
+
+bool AMDGPUAsmParser::ParseAMDGPURegister(RegisterKind &RegKind,
+                                          unsigned &Reg,
+                                          unsigned &RegNum,
+                                          unsigned &RegWidth) {
+  Reg = AMDGPU::NoRegister;
+
+  if (isToken(AsmToken::Identifier)) {
+    Reg = ParseSpecialReg(RegKind, RegNum, RegWidth);
+    if (Reg == AMDGPU::NoRegister)
+      Reg = ParseRegularReg(RegKind, RegNum, RegWidth);
+  } else {
+    Reg = ParseRegList(RegKind, RegNum, RegWidth);
+  }
+
+  const MCRegisterInfo *TRI = getContext().getRegisterInfo();
+  return Reg != AMDGPU::NoRegister && subtargetHasRegister(*TRI, Reg);
 }
 
 Optional<StringRef>
@@ -2250,18 +2311,18 @@ std::unique_ptr<AMDGPUOperand> AMDGPUAsmParser::parseRegister() {
   SMLoc StartLoc = Tok.getLoc();
   SMLoc EndLoc = Tok.getEndLoc();
   RegisterKind RegKind;
-  unsigned Reg, RegNum, RegWidth, DwordRegIndex;
+  unsigned Reg, RegNum, RegWidth;
 
-  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth, &DwordRegIndex)) {
+  if (!ParseAMDGPURegister(RegKind, Reg, RegNum, RegWidth)) {
     //FIXME: improve error messages (bug 41303).
     Error(StartLoc, "not a valid operand.");
     return nullptr;
   }
   if (AMDGPU::IsaInfo::hasCodeObjectV3(&getSTI())) {
-    if (!updateGprCountSymbols(RegKind, DwordRegIndex, RegWidth))
+    if (!updateGprCountSymbols(RegKind, RegNum, RegWidth))
       return nullptr;
   } else
-    KernelScope.usesRegister(RegKind, DwordRegIndex, RegWidth);
+    KernelScope.usesRegister(RegKind, RegNum, RegWidth);
   return AMDGPUOperand::CreateReg(this, Reg, StartLoc, EndLoc);
 }
 
@@ -3242,6 +3303,7 @@ bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
 
   const int OpIndices[] = { Src0Idx, Src1Idx };
 
+  unsigned NumExprs = 0;
   unsigned NumLiterals = 0;
   uint32_t LiteralValue;
 
@@ -3249,19 +3311,21 @@ bool AMDGPUAsmParser::validateSOPLiteral(const MCInst &Inst) const {
     if (OpIdx == -1) break;
 
     const MCOperand &MO = Inst.getOperand(OpIdx);
-    if (MO.isImm() &&
-        // Exclude special imm operands (like that used by s_set_gpr_idx_on)
-        AMDGPU::isSISrcOperand(Desc, OpIdx) &&
-        !isInlineConstant(Inst, OpIdx)) {
-      uint32_t Value = static_cast<uint32_t>(MO.getImm());
-      if (NumLiterals == 0 || LiteralValue != Value) {
-        LiteralValue = Value;
-        ++NumLiterals;
+    // Exclude special imm operands (like that used by s_set_gpr_idx_on)
+    if (AMDGPU::isSISrcOperand(Desc, OpIdx)) {
+      if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
+        uint32_t Value = static_cast<uint32_t>(MO.getImm());
+        if (NumLiterals == 0 || LiteralValue != Value) {
+          LiteralValue = Value;
+          ++NumLiterals;
+        }
+      } else if (MO.isExpr()) {
+        ++NumExprs;
       }
     }
   }
 
-  return NumLiterals <= 1;
+  return NumLiterals + NumExprs <= 1;
 }
 
 bool AMDGPUAsmParser::validateOpSel(const MCInst &Inst) {
@@ -3297,6 +3361,7 @@ bool AMDGPUAsmParser::validateVOP3Literal(const MCInst &Inst) const {
 
   const int OpIndices[] = { Src0Idx, Src1Idx, Src2Idx };
 
+  unsigned NumExprs = 0;
   unsigned NumLiterals = 0;
   uint32_t LiteralValue;
 
@@ -3304,21 +3369,26 @@ bool AMDGPUAsmParser::validateVOP3Literal(const MCInst &Inst) const {
     if (OpIdx == -1) break;
 
     const MCOperand &MO = Inst.getOperand(OpIdx);
-    if (!MO.isImm() || !AMDGPU::isSISrcOperand(Desc, OpIdx))
+    if (!MO.isImm() && !MO.isExpr())
+      continue;
+    if (!AMDGPU::isSISrcOperand(Desc, OpIdx))
       continue;
 
     if (OpIdx == Src2Idx && (Desc.TSFlags & SIInstrFlags::IsMAI) &&
         getFeatureBits()[AMDGPU::FeatureMFMAInlineLiteralBug])
       return false;
 
-    if (!isInlineConstant(Inst, OpIdx)) {
+    if (MO.isImm() && !isInlineConstant(Inst, OpIdx)) {
       uint32_t Value = static_cast<uint32_t>(MO.getImm());
       if (NumLiterals == 0 || LiteralValue != Value) {
         LiteralValue = Value;
         ++NumLiterals;
       }
+    } else if (MO.isExpr()) {
+      ++NumExprs;
     }
   }
+  NumLiterals += NumExprs;
 
   return !NumLiterals ||
          (NumLiterals == 1 && getFeatureBits()[AMDGPU::FeatureVOP3Literal]);
@@ -5970,6 +6040,7 @@ static const OptionalOperand AMDGPUOptionalOperandTable[] = {
   {"format",  AMDGPUOperand::ImmTyFORMAT, false, nullptr},
   {"glc",     AMDGPUOperand::ImmTyGLC, true, nullptr},
   {"slc",     AMDGPUOperand::ImmTySLC, true, nullptr},
+  {"swz",     AMDGPUOperand::ImmTySWZ, true, nullptr},
   {"tfe",     AMDGPUOperand::ImmTyTFE, true, nullptr},
   {"d16",     AMDGPUOperand::ImmTyD16, true, nullptr},
   {"high",    AMDGPUOperand::ImmTyHigh, true, nullptr},
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 1af12721b64cf..c9e8abad7c3c5 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -7,13 +7,13 @@
 //===----------------------------------------------------------------------===//
 
 def MUBUFAddr32 : ComplexPattern<i64, 9, "SelectMUBUFAddr32">;
-def MUBUFAddr64 : ComplexPattern<i64, 8, "SelectMUBUFAddr64">;
+def MUBUFAddr64 : ComplexPattern<i64, 9, "SelectMUBUFAddr64">;
 def MUBUFAddr64Atomic : ComplexPattern<i64, 5, "SelectMUBUFAddr64">;
 
 def MUBUFScratchOffen : ComplexPattern<i64, 4, "SelectMUBUFScratchOffen", [], [SDNPWantParent]>;
 def MUBUFScratchOffset : ComplexPattern<i64, 3, "SelectMUBUFScratchOffset", [], [SDNPWantParent], 20>;
 
-def MUBUFOffset : ComplexPattern<i64, 7, "SelectMUBUFOffset">;
+def MUBUFOffset : ComplexPattern<i64, 8, "SelectMUBUFOffset">;
 def MUBUFOffsetNoGLC : ComplexPattern<i64, 3, "SelectMUBUFOffset">;
 def MUBUFOffsetAtomic : ComplexPattern<i64, 4, "SelectMUBUFOffset">;
 
@@ -54,6 +54,17 @@ class MTBUFAddr64Table <bit is_addr64, string Name> {
 // MTBUF classes
 //===----------------------------------------------------------------------===//
 
+class MTBUFGetBaseOpcode<string Op> {
+  string ret = !subst("FORMAT_XY", "FORMAT_X",
+    !subst("FORMAT_XYZ", "FORMAT_X",
+    !subst("FORMAT_XYZW", "FORMAT_X", Op)));
+}
+
+class getMTBUFElements<string Op> {
+  int ret = 1;
+}
+
+
 class MTBUF_Pseudo <string opName, dag outs, dag ins,
                     string asmOps, list<dag> pattern=[]> :
   InstSI<outs, ins, "", pattern>,
@@ -67,6 +78,9 @@ class MTBUF_Pseudo <string opName, dag outs, dag ins,
   string Mnemonic = opName;
   string AsmOperands = asmOps;
 
+  Instruction Opcode = !cast<Instruction>(NAME);
+  Instruction BaseOpcode = !cast<Instruction>(MTBUFGetBaseOpcode<NAME>.ret);
+
   let VM_CNT = 1;
   let EXP_CNT = 1;
   let MTBUF = 1;
@@ -90,6 +104,7 @@ class MTBUF_Pseudo <string opName, dag outs, dag ins,
   bits<1> has_offset  = 1;
   bits<1> has_slc     = 1;
   bits<1> has_tfe     = 1;
+  bits<4> elements    = 0;
 }
 
 class MTBUF_Real <MTBUF_Pseudo ps> :
@@ -126,17 +141,17 @@ class getMTBUFInsDA<list<RegisterClass> vdataList,
   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
   dag InsNoData = !if(!empty(vaddrList),
     (ins                    SReg_128:$srsrc, SCSrc_b32:$soffset,
-         offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc),
+         offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
     (ins vaddrClass:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset,
-         offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc)
+         offset:$offset, FORMAT:$format, GLC:$glc, SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
   );
   dag InsData = !if(!empty(vaddrList),
     (ins vdataClass:$vdata,                    SReg_128:$srsrc,
          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
-         SLC:$slc, TFE:$tfe, DLC:$dlc),
+         SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz),
     (ins vdataClass:$vdata, vaddrClass:$vaddr, SReg_128:$srsrc,
          SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, GLC:$glc,
-         SLC:$slc, TFE:$tfe, DLC:$dlc)
+         SLC:$slc, TFE:$tfe, DLC:$dlc, SWZ:$swz)
   );
   dag ret = !if(!empty(vdataList), InsNoData, InsData);
 }
@@ -181,51 +196,54 @@ class MTBUF_SetupAddr<int addrKind> {
 class MTBUF_Load_Pseudo <string opName,
                          int addrKind,
                          RegisterClass vdataClass,
+                         int elems,
                          list<dag> pattern=[],
                          // Workaround bug bz30254
                          int addrKindCopy = addrKind>
   : MTBUF_Pseudo<opName,
                  (outs vdataClass:$vdata),
                  getMTBUFIns<addrKindCopy>.ret,
-                 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
+                 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
                  pattern>,
     MTBUF_SetupAddr<addrKindCopy> {
   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
   let mayLoad = 1;
   let mayStore = 0;
+  let elements = elems;
 }
 
 multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
-                              ValueType load_vt = i32,
+                              int elems, ValueType load_vt = i32,
                               SDPatternOperator ld = null_frag> {
 
-  def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
+  def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
     [(set load_vt:$vdata,
      (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i8:$format,
-                      i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
+                      i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
     MTBUFAddr64Table<0, NAME>;
 
-  def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
+  def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
     [(set load_vt:$vdata,
      (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset,
-                      i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)))]>,
+                      i8:$format, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)))]>,
     MTBUFAddr64Table<1, NAME>;
 
-  def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
-  def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
-  def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+  def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
+  def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
+  def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
-    def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
-    def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
-    def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+    def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
+    def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
+    def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
+    def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
   }
 }
 
 class MTBUF_Store_Pseudo <string opName,
                           int addrKind,
                           RegisterClass vdataClass,
+                          int elems,
                           list<dag> pattern=[],
                           // Workaround bug bz30254
                           int addrKindCopy = addrKind,
@@ -233,39 +251,40 @@ class MTBUF_Store_Pseudo <string opName,
   : MTBUF_Pseudo<opName,
                  (outs),
                  getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
-                 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
+                 " $vdata, " # getMTBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
                  pattern>,
     MTBUF_SetupAddr<addrKindCopy> {
   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
   let mayLoad = 0;
   let mayStore = 1;
+  let elements = elems;
 }
 
 multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
-                               ValueType store_vt = i32,
+                               int elems, ValueType store_vt = i32,
                                SDPatternOperator st = null_frag> {
 
-  def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
+  def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems,
     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
                                        i16:$offset, i8:$format, i1:$glc,
-                                       i1:$slc, i1:$tfe, i1:$dlc))]>,
+                                       i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
     MTBUFAddr64Table<0, NAME>;
 
-  def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
+  def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems,
     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
                                        i16:$offset, i8:$format, i1:$glc,
-                                       i1:$slc, i1:$tfe, i1:$dlc))]>,
+                                       i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
     MTBUFAddr64Table<1, NAME>;
 
-  def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
-  def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
-  def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+  def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
+  def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
+  def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass>;
-    def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass>;
-    def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass>;
-    def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+    def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
+    def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
+    def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
+    def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
   }
 }
 
@@ -393,7 +412,7 @@ class getMUBUFInsDA<list<RegisterClass> vdataList,
   );
   dag ret = !con(
               !if(!empty(vdataList), InsNoData, InsData),
-              !if(isLds, (ins DLC:$dlc), (ins TFE:$tfe, DLC:$dlc))
+              !if(isLds, (ins DLC:$dlc, SWZ:$swz), (ins TFE:$tfe, DLC:$dlc,SWZ:$swz))
              );
 }
 
@@ -465,7 +484,7 @@ class MUBUF_Load_Pseudo <string opName,
                  !con(getMUBUFIns<addrKindCopy, [], isLds>.ret,
                       !if(HasTiedDest, (ins getVregSrcForVT<vdata_vt>.ret:$vdata_in), (ins))),
                  " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc" #
-                   !if(isLds, " lds", "$tfe") # "$dlc",
+                   !if(isLds, " lds", "$tfe") # "$dlc" # "$swz",
                  pattern>,
     MUBUF_SetupAddr<addrKindCopy> {
   let PseudoInstr = opName # !if(isLds, "_lds", "") #
@@ -483,15 +502,15 @@ class MUBUF_Load_Pseudo <string opName,
 }
 
 class MUBUF_Offset_Load_Pat <Instruction inst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> : Pat <
-  (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
-  (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
+  (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
+  (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 >;
 
 class MUBUF_Addr64_Load_Pat <Instruction inst,
                             ValueType load_vt = i32,
                             SDPatternOperator ld = null_frag> : Pat <
-  (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
-  (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))
+  (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
+  (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))
 >;
 
 multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPatternOperator ld = null_frag> {
@@ -542,7 +561,7 @@ class MUBUF_Store_Pseudo <string opName,
   : MUBUF_Pseudo<opName,
                  (outs),
                  getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret]>.ret,
-                 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc",
+                 " $vdata, " # getMUBUFAsmOps<addrKindCopy>.ret # "$glc$slc$tfe$dlc$swz",
                  pattern>,
     MUBUF_SetupAddr<addrKindCopy> {
   let PseudoInstr = opName # "_" # getAddrName<addrKindCopy>.ret;
@@ -558,12 +577,12 @@ multiclass MUBUF_Pseudo_Stores<string opName,
 
   def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, store_vt,
     [(st store_vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
+                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
     MUBUFAddr64Table<0, NAME>;
 
   def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, store_vt,
     [(st store_vt:$vdata, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
-                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))]>,
+                                       i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))]>,
     MUBUFAddr64Table<1, NAME>;
 
   def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, store_vt>;
@@ -581,8 +600,8 @@ multiclass MUBUF_Pseudo_Stores<string opName,
 class MUBUF_Pseudo_Store_Lds<string opName>
   : MUBUF_Pseudo<opName,
                  (outs),
-                 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc),
-                 " $srsrc, $soffset$offset lds$glc$slc"> {
+                 (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, GLC:$glc, SLC:$slc, SWZ:$swz),
+                 " $srsrc, $soffset$offset lds$glc$slc$swz"> {
   let mayLoad = 0;
   let mayStore = 1;
   let maybeAtomic = 1;
@@ -1065,35 +1084,35 @@ defm BUFFER_ATOMIC_PK_ADD_F16 : MUBUF_Pseudo_Atomics_NO_RTN <
 // MTBUF Instructions
 //===----------------------------------------------------------------------===//
 
-defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32>;
-defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64>;
-defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96>;
-defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128>;
-defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32>;
-defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64>;
-defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96>;
-defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128>;
+defm TBUFFER_LOAD_FORMAT_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_x",     VGPR_32,  1>;
+defm TBUFFER_LOAD_FORMAT_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xy",    VReg_64,  2>;
+defm TBUFFER_LOAD_FORMAT_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyz",   VReg_96,  3>;
+defm TBUFFER_LOAD_FORMAT_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_xyzw",  VReg_128, 4>;
+defm TBUFFER_STORE_FORMAT_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_x",    VGPR_32,  1>;
+defm TBUFFER_STORE_FORMAT_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_xy",   VReg_64,  2>;
+defm TBUFFER_STORE_FORMAT_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyz",  VReg_96,  3>;
+defm TBUFFER_STORE_FORMAT_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_xyzw", VReg_128, 4>;
 
 let SubtargetPredicate = HasUnpackedD16VMem, D16Buf = 1 in {
-  defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
-  defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64>;
-  defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96>;
-  defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128>;
-  defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
-  defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64>;
-  defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96>;
-  defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128>;
+  defm TBUFFER_LOAD_FORMAT_D16_X_gfx80     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32,  1>;
+  defm TBUFFER_LOAD_FORMAT_D16_XY_gfx80    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VReg_64,  2>;
+  defm TBUFFER_LOAD_FORMAT_D16_XYZ_gfx80   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_96,  3>;
+  defm TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_128, 4>;
+  defm TBUFFER_STORE_FORMAT_D16_X_gfx80    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32,  1>;
+  defm TBUFFER_STORE_FORMAT_D16_XY_gfx80   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VReg_64,  2>;
+  defm TBUFFER_STORE_FORMAT_D16_XYZ_gfx80  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_96,  3>;
+  defm TBUFFER_STORE_FORMAT_D16_XYZW_gfx80 : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_128, 4>;
 } // End HasUnpackedD16VMem.
 
 let SubtargetPredicate = HasPackedD16VMem, D16Buf = 1 in {
-  defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32>;
-  defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32>;
-  defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64>;
-  defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64>;
-  defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32>;
-  defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32>;
-  defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64>;
-  defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64>;
+  defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_x",     VGPR_32, 1>;
+  defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xy",    VGPR_32, 2>;
+  defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyz",   VReg_64, 3>;
+  defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Pseudo_Loads  <"tbuffer_load_format_d16_xyzw",  VReg_64, 4>;
+  defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_x",    VGPR_32, 1>;
+  defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xy",   VGPR_32, 2>;
+  defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyz",  VReg_64, 3>;
+  defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Pseudo_Stores <"tbuffer_store_format_d16_xyzw", VReg_64, 4>;
 } // End HasPackedD16VMem.
 
 let SubtargetPredicate = isGFX7Plus in {
@@ -1128,6 +1147,10 @@ def extract_dlc : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant((N->getZExtValue() >> 2) & 1, SDLoc(N), MVT::i8);
 }]>;
 
+def extract_swz : SDNodeXForm<imm, [{
+  return CurDAG->getTargetConstant((N->getZExtValue() >> 3) & 1, SDLoc(N), MVT::i8);
+}]>;
+
 //===----------------------------------------------------------------------===//
 // buffer_load/store_format patterns
 //===----------------------------------------------------------------------===//
@@ -1135,33 +1158,37 @@ def extract_dlc : SDNodeXForm<imm, [{
 multiclass MUBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
                                   string opcode> {
   def : GCNPat<
-    (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, 0)),
+    (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
+              timm:$auxiliary, 0)),
     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, 0)),
+    (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$auxiliary, 0)),
     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, imm)),
+    (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
+              timm:$auxiliary, timm)),
     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, imm)),
+    (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$auxiliary, timm)),
     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
       $rsrc, $soffset, (as_i16imm $offset),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 }
 
@@ -1210,36 +1237,40 @@ defm : MUBUF_LoadIntrinsicPat<SIbuffer_load_ushort,  i32, "BUFFER_LOAD_USHORT">;
 multiclass MUBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
                                    string opcode> {
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, 0),
+    (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
+              timm:$auxiliary, 0),
     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset, (as_i16imm $offset),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, 0),
+    (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$auxiliary, 0),
     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
-      (as_i16imm $offset), (extract_glc $cachepolicy),
-      (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (as_i16imm $offset), (extract_glc $auxiliary),
+      (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, imm),
+    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
+              timm:$auxiliary, timm),
     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
-      (as_i16imm $offset), (extract_glc $cachepolicy),
-      (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      (as_i16imm $offset), (extract_glc $auxiliary),
+      (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$cachepolicy, imm),
+    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$auxiliary, timm),
     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_exact)
       $vdata,
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
-      $rsrc, $soffset, (as_i16imm $offset), (extract_glc $cachepolicy),
-      (extract_slc $cachepolicy), 0,  (extract_dlc $cachepolicy))
+      $rsrc, $soffset, (as_i16imm $offset), (extract_glc $auxiliary),
+      (extract_slc $auxiliary), 0,  (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 }
 
@@ -1291,32 +1322,32 @@ multiclass BufferAtomicPatterns<SDPatternOperator name, ValueType vt,
                                 string opcode> {
   def : GCNPat<
     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
-          0, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, 0)),
+          0, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, 0)),
     (!cast<MUBUF_Pseudo>(opcode # _OFFSET_RTN) $vdata_in, $rsrc, $soffset,
                                         (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
-          0, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, imm)),
+          0, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, timm)),
     (!cast<MUBUF_Pseudo>(opcode # _IDXEN_RTN) $vdata_in, $vindex, $rsrc, $soffset,
                                        (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (vt (name vt:$vdata_in, v4i32:$rsrc, 0,
-          i32:$voffset, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, 0)),
+          i32:$voffset, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, 0)),
     (!cast<MUBUF_Pseudo>(opcode # _OFFEN_RTN) $vdata_in, $voffset, $rsrc, $soffset,
                                        (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (vt (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
-          i32:$voffset, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, imm)),
+          i32:$voffset, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, timm)),
     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN_RTN)
       $vdata_in,
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
@@ -1353,32 +1384,32 @@ multiclass BufferAtomicPatterns_NO_RTN<SDPatternOperator name, ValueType vt,
                                        string opcode> {
   def : GCNPat<
     (name vt:$vdata_in, v4i32:$rsrc, 0,
-          0, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, 0),
+          0, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, 0),
     (!cast<MUBUF_Pseudo>(opcode # _OFFSET) $vdata_in, $rsrc, $soffset,
                                         (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
-          0, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, imm),
+          0, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, timm),
     (!cast<MUBUF_Pseudo>(opcode # _IDXEN) $vdata_in, $vindex, $rsrc, $soffset,
                                        (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (name vt:$vdata_in, v4i32:$rsrc, 0,
-          i32:$voffset, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, 0),
+          i32:$voffset, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, 0),
     (!cast<MUBUF_Pseudo>(opcode # _OFFEN) $vdata_in, $voffset, $rsrc, $soffset,
                                        (as_i16imm $offset), (extract_slc $cachepolicy))
   >;
 
   def : GCNPat<
     (name vt:$vdata_in, v4i32:$rsrc, i32:$vindex,
-          i32:$voffset, i32:$soffset, imm:$offset,
-          imm:$cachepolicy, imm),
+          i32:$voffset, i32:$soffset, timm:$offset,
+          timm:$cachepolicy, timm),
     (!cast<MUBUF_Pseudo>(opcode # _BOTHEN)
       $vdata_in,
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
@@ -1392,8 +1423,8 @@ defm : BufferAtomicPatterns_NO_RTN<SIbuffer_atomic_pk_fadd, v2f16, "BUFFER_ATOMI
 def : GCNPat<
   (SIbuffer_atomic_cmpswap
       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
-      0, i32:$soffset, imm:$offset,
-      imm:$cachepolicy, 0),
+      0, i32:$soffset, timm:$offset,
+      timm:$cachepolicy, 0),
   (EXTRACT_SUBREG
     (BUFFER_ATOMIC_CMPSWAP_OFFSET_RTN
       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
@@ -1404,8 +1435,8 @@ def : GCNPat<
 def : GCNPat<
   (SIbuffer_atomic_cmpswap
       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
-      0, i32:$soffset, imm:$offset,
-      imm:$cachepolicy, imm),
+      0, i32:$soffset, timm:$offset,
+      timm:$cachepolicy, timm),
   (EXTRACT_SUBREG
     (BUFFER_ATOMIC_CMPSWAP_IDXEN_RTN
       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
@@ -1416,8 +1447,8 @@ def : GCNPat<
 def : GCNPat<
   (SIbuffer_atomic_cmpswap
       i32:$data, i32:$cmp, v4i32:$rsrc, 0,
-      i32:$voffset, i32:$soffset, imm:$offset,
-      imm:$cachepolicy, 0),
+      i32:$voffset, i32:$soffset, timm:$offset,
+      timm:$cachepolicy, 0),
   (EXTRACT_SUBREG
     (BUFFER_ATOMIC_CMPSWAP_OFFEN_RTN
       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
@@ -1428,8 +1459,8 @@ def : GCNPat<
 def : GCNPat<
   (SIbuffer_atomic_cmpswap
       i32:$data, i32:$cmp, v4i32:$rsrc, i32:$vindex,
-      i32:$voffset, i32:$soffset, imm:$offset,
-      imm:$cachepolicy, imm),
+      i32:$voffset, i32:$soffset, timm:$offset,
+      timm:$cachepolicy, timm),
   (EXTRACT_SUBREG
     (BUFFER_ATOMIC_CMPSWAP_BOTHEN_RTN
       (REG_SEQUENCE VReg_64, $data, sub0, $cmp, sub1),
@@ -1441,8 +1472,8 @@ def : GCNPat<
 class MUBUFLoad_PatternADDR64 <MUBUF_Pseudo Instr_ADDR64, ValueType vt,
                               PatFrag constant_ld> : GCNPat <
      (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
-                                   i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
-     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
+                                   i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
+     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
   >;
 
 multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
@@ -1450,12 +1481,12 @@ multiclass MUBUFLoad_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Ins
   def : GCNPat <
      (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
                                    i16:$offset, i1:$slc))),
-     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
+     (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
   >;
 
   def : GCNPat <
     (vt (atomic_ld (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset))),
-    (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
+    (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
   >;
 }
 
@@ -1476,8 +1507,8 @@ multiclass MUBUFLoad_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
 
   def : GCNPat <
     (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                          i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc))),
-    (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
+                          i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz))),
+    (Instr_OFFSET $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
   >;
 }
 
@@ -1500,12 +1531,12 @@ multiclass MUBUFScratchLoadPat <MUBUF_Pseudo InstrOffen,
   def : GCNPat <
     (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
                                i32:$soffset, u16imm:$offset))),
-    (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
+    (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
   >;
 
   def : GCNPat <
     (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))),
-    (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0)
+    (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
   >;
 }
 
@@ -1515,12 +1546,12 @@ multiclass MUBUFScratchLoadPat_D16 <MUBUF_Pseudo InstrOffen,
                                 ValueType vt, PatFrag ld_frag> {
   def : GCNPat <
     (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in),
-    (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
+    (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
   >;
 
   def : GCNPat <
     (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in),
-    (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, $in)
+    (InstrOffset $srsrc, $soffset, $offset, 0, 0, 0, 0, 0, $in)
   >;
 }
 
@@ -1560,16 +1591,16 @@ defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D1
 
 multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
                                       ValueType vt, PatFrag atomic_st> {
-  // Store follows atomic op convention so address is forst
+  // Store follows atomic op convention so address is first
   def : GCNPat <
      (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset,
                                    i16:$offset, i1:$slc), vt:$val),
-     (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0)
+     (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset, 0, $slc, 0, 0, 0)
   >;
 
   def : GCNPat <
     (atomic_st (MUBUFOffsetNoGLC v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val),
-    (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0)
+    (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset), 0, 0, 0, 0, 0)
   >;
 }
 let SubtargetPredicate = isGFX6GFX7 in {
@@ -1583,8 +1614,8 @@ multiclass MUBUFStore_Pattern <MUBUF_Pseudo Instr_OFFSET, ValueType vt,
 
   def : GCNPat <
     (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset,
-                                      i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc)),
-    (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc)
+                                      i16:$offset, i1:$glc, i1:$slc, i1:$tfe, i1:$dlc, i1:$swz)),
+    (Instr_OFFSET $vdata, $srsrc, $soffset, $offset, $glc, $slc, $tfe, $dlc, $swz)
   >;
 }
 
@@ -1598,13 +1629,13 @@ multiclass MUBUFScratchStorePat <MUBUF_Pseudo InstrOffen,
   def : GCNPat <
     (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr,
                                       i32:$soffset, u16imm:$offset)),
-    (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0)
+    (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
   >;
 
   def : GCNPat <
     (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset,
                                        u16imm:$offset)),
-    (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0)
+    (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0, 0, 0, 0)
   >;
 }
 
@@ -1642,37 +1673,41 @@ defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_D16_HI_OFFEN, BUFFER_STORE_BYTE_D
 multiclass MTBUF_LoadIntrinsicPat<SDPatternOperator name, ValueType vt,
                                   string opcode> {
   def : GCNPat<
-    (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
-              imm:$format, imm:$cachepolicy, 0)),
+    (vt (name v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
+              timm:$format, timm:$auxiliary, 0)),
     (!cast<MTBUF_Pseudo>(opcode # _OFFSET) $rsrc, $soffset, (as_i16imm $offset),
       (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
-              imm:$format, imm:$cachepolicy, imm)),
+    (vt (name v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
+              timm:$format, timm:$auxiliary, timm)),
     (!cast<MTBUF_Pseudo>(opcode # _IDXEN) $vindex, $rsrc, $soffset, (as_i16imm $offset),
       (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$format, imm:$cachepolicy, 0)),
+    (vt (name v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$format, timm:$auxiliary, 0)),
     (!cast<MTBUF_Pseudo>(opcode # _OFFEN) $voffset, $rsrc, $soffset, (as_i16imm $offset),
       (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, imm:$offset,
-              imm:$format, imm:$cachepolicy, imm)),
+    (vt (name v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset, timm:$offset,
+              timm:$format, timm:$auxiliary, timm)),
     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN)
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
       $rsrc, $soffset, (as_i16imm $offset),
       (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 }
 
@@ -1700,37 +1735,41 @@ let SubtargetPredicate = HasPackedD16VMem in {
 multiclass MTBUF_StoreIntrinsicPat<SDPatternOperator name, ValueType vt,
                                    string opcode> {
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, imm:$offset,
-          imm:$format, imm:$cachepolicy, 0),
+    (name vt:$vdata, v4i32:$rsrc, 0, 0, i32:$soffset, timm:$offset,
+          timm:$format, timm:$auxiliary, 0),
     (!cast<MTBUF_Pseudo>(opcode # _OFFSET_exact) $vdata, $rsrc, $soffset,
       (as_i16imm $offset), (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, imm:$offset,
-          imm:$format, imm:$cachepolicy, imm),
+    (name vt:$vdata, v4i32:$rsrc, i32:$vindex, 0, i32:$soffset, timm:$offset,
+          timm:$format, timm:$auxiliary, timm),
     (!cast<MTBUF_Pseudo>(opcode # _IDXEN_exact) $vdata, $vindex, $rsrc, $soffset,
       (as_i16imm $offset), (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
-    (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, imm:$offset,
-          imm:$format, imm:$cachepolicy, 0),
+    (name vt:$vdata, v4i32:$rsrc, 0, i32:$voffset, i32:$soffset, timm:$offset,
+          timm:$format, timm:$auxiliary, 0),
     (!cast<MTBUF_Pseudo>(opcode # _OFFEN_exact) $vdata, $voffset, $rsrc, $soffset,
       (as_i16imm $offset), (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 
   def : GCNPat<
     (name vt:$vdata, v4i32:$rsrc, i32:$vindex, i32:$voffset, i32:$soffset,
-          imm:$offset, imm:$format, imm:$cachepolicy, imm),
+          timm:$offset, timm:$format, timm:$auxiliary, timm),
     (!cast<MTBUF_Pseudo>(opcode # _BOTHEN_exact)
       $vdata,
       (REG_SEQUENCE VReg_64, $vindex, sub0, $voffset, sub1),
       $rsrc, $soffset, (as_i16imm $offset), (as_i8imm $format),
-      (extract_glc $cachepolicy), (extract_slc $cachepolicy), 0, (extract_dlc $cachepolicy))
+      (extract_glc $auxiliary), (extract_slc $auxiliary), 0, (extract_dlc $auxiliary),
+      (extract_swz $auxiliary))
   >;
 }
 
@@ -2397,3 +2436,22 @@ def getMUBUFInfoFromBaseOpcodeAndElements : SearchIndex {
   let Table = MUBUFInfoTable;
   let Key = ["BaseOpcode", "elements"];
 }
+
+def MTBUFInfoTable : GenericTable {
+  let FilterClass = "MTBUF_Pseudo";
+  let CppTypeName = "MTBUFInfo";
+  let Fields = ["Opcode", "BaseOpcode", "elements", "has_vaddr", "has_srsrc", "has_soffset"];
+
+  let PrimaryKey = ["Opcode"];
+  let PrimaryKeyName = "getMTBUFOpcodeHelper";
+}
+
+def getMTBUFInfoFromOpcode : SearchIndex {
+  let Table = MTBUFInfoTable;
+  let Key = ["Opcode"];
+}
+
+def getMTBUFInfoFromBaseOpcodeAndElements : SearchIndex {
+  let Table = MTBUFInfoTable;
+  let Key = ["BaseOpcode", "elements"];
+}
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index e1d53ae71a806..86c2db92acb42 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -603,7 +603,7 @@ def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
 //===----------------------------------------------------------------------===//
 
 def : GCNPat <
-  (int_amdgcn_ds_swizzle i32:$src, imm:$offset16),
+  (int_amdgcn_ds_swizzle i32:$src, timm:$offset16),
   (DS_SWIZZLE_B32 $src, (as_i16imm $offset16), (i1 0))
 >;
 
diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
index da4c1cbb56909..973491a70d3c1 100644
--- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp
@@ -160,6 +160,7 @@ void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
       if (TryCand.ResDelta == SchedResourceDelta())
         TryCand.initResourceDelta(Zone.DAG, SchedModel);
       Cand.setBest(TryCand);
+      LLVM_DEBUG(traceCandidate(Cand));
     }
   }
 }
@@ -195,6 +196,15 @@ SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
     assert(BotCand.Reason != NoCand && "failed to find the first candidate");
   } else {
     LLVM_DEBUG(traceCandidate(BotCand));
+#ifndef NDEBUG
+    if (VerifyScheduling) {
+      SchedCandidate TCand;
+      TCand.reset(CandPolicy());
+      pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), TCand);
+      assert(TCand.SU == BotCand.SU &&
+             "Last pick result should correspond to re-picking right now");
+    }
+#endif
   }
 
   // Check if the top Q has a better candidate.
@@ -206,6 +216,15 @@ SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
     assert(TopCand.Reason != NoCand && "failed to find the first candidate");
   } else {
     LLVM_DEBUG(traceCandidate(TopCand));
+#ifndef NDEBUG
+    if (VerifyScheduling) {
+      SchedCandidate TCand;
+      TCand.reset(CandPolicy());
+      pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TCand);
+      assert(TCand.SU == TopCand.SU &&
+           "Last pick result should correspond to re-picking right now");
+    }
+#endif
   }
 
   // Pick best from BotCand and TopCand.
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index a451625439759..d2ea94548dfee 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -196,6 +196,10 @@ void AMDGPUInstPrinter::printSLC(const MCInst *MI, unsigned OpNo,
   printNamedBit(MI, OpNo, O, "slc");
 }
 
+void AMDGPUInstPrinter::printSWZ(const MCInst *MI, unsigned OpNo,
+                                 const MCSubtargetInfo &STI, raw_ostream &O) {
+}
+
 void AMDGPUInstPrinter::printTFE(const MCInst *MI, unsigned OpNo,
                                  const MCSubtargetInfo &STI, raw_ostream &O) {
   printNamedBit(MI, OpNo, O, "tfe");
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
index 0f62f039763ef..66b70831ff9ef 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h
@@ -72,6 +72,8 @@ class AMDGPUInstPrinter : public MCInstPrinter {
                 raw_ostream &O);
   void printSLC(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                 raw_ostream &O);
+  void printSWZ(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
+                raw_ostream &O);
   void printTFE(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
                 raw_ostream &O);
   void printDMask(const MCInst *MI, unsigned OpNo, const MCSubtargetInfo &STI,
diff --git a/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp b/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp
index 42158151b6444..b29cd75f75cfe 100644
--- a/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp
@@ -104,7 +104,7 @@ bool R600AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 
 
   // Functions needs to be cacheline (256B) aligned.
-  MF.ensureAlignment(llvm::Align(256));
+  MF.ensureAlignment(Align(256));
 
   SetupMachineFunction(MF);
 
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 859e99ce4c325..23ef56afc39c9 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -99,7 +99,10 @@ enum : uint64_t {
   FPAtomic = UINT64_C(1) << 53,
 
   // Is a MFMA instruction.
-  IsMAI = UINT64_C(1) << 54
+  IsMAI = UINT64_C(1) << 54,
+
+  // Is a DOT instruction.
+  IsDOT = UINT64_C(1) << 55
 };
 
 // v_cmp_class_* etc. use a 10-bit mask for what operation is checked.
diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
index e8f7fdb57fb9c..1f8b744155d68 100644
--- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp
@@ -142,16 +142,20 @@ static bool isInlineConstantIfFolded(const SIInstrInfo *TII,
   switch (Opc) {
   case AMDGPU::V_MAC_F32_e64:
   case AMDGPU::V_MAC_F16_e64:
-  case AMDGPU::V_FMAC_F32_e64: {
+  case AMDGPU::V_FMAC_F32_e64:
+  case AMDGPU::V_FMAC_F16_e64: {
     // Special case for mac. Since this is replaced with mad when folded into
     // src2, we need to check the legality for the final instruction.
     int Src2Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2);
     if (static_cast<int>(OpNo) == Src2Idx) {
-      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64;
-      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64;
+      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64 ||
+                   Opc == AMDGPU::V_FMAC_F16_e64;
+      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64 ||
+                   Opc == AMDGPU::V_FMAC_F32_e64;
 
       unsigned Opc = IsFMA ?
-        AMDGPU::V_FMA_F32 : (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
+        (IsF32 ? AMDGPU::V_FMA_F32 : AMDGPU::V_FMA_F16_gfx9) :
+        (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
       const MCInstrDesc &MadDesc = TII->get(Opc);
       return TII->isInlineConstant(OpToFold, MadDesc.OpInfo[OpNo].OperandType);
     }
@@ -314,12 +318,15 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList,
     // Special case for v_mac_{f16, f32}_e64 if we are trying to fold into src2
     unsigned Opc = MI->getOpcode();
     if ((Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
-         Opc == AMDGPU::V_FMAC_F32_e64) &&
+         Opc == AMDGPU::V_FMAC_F32_e64 || Opc == AMDGPU::V_FMAC_F16_e64) &&
         (int)OpNo == AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src2)) {
-      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64;
-      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64;
+      bool IsFMA = Opc == AMDGPU::V_FMAC_F32_e64 ||
+                   Opc == AMDGPU::V_FMAC_F16_e64;
+      bool IsF32 = Opc == AMDGPU::V_MAC_F32_e64 ||
+                   Opc == AMDGPU::V_FMAC_F32_e64;
       unsigned NewOpc = IsFMA ?
-        AMDGPU::V_FMA_F32 : (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
+        (IsF32 ? AMDGPU::V_FMA_F32 : AMDGPU::V_FMA_F16_gfx9) :
+        (IsF32 ? AMDGPU::V_MAD_F32 : AMDGPU::V_MAD_F16);
 
       // Check if changing this to a v_mad_{f16, f32} instruction will allow us
       // to fold the operand.
@@ -1086,6 +1093,13 @@ void SIFoldOperands::foldInstOperand(MachineInstr &MI,
     Copy->addImplicitDefUseOperands(*MF);
 
   for (FoldCandidate &Fold : FoldList) {
+    if (Fold.isReg() && Register::isVirtualRegister(Fold.OpToFold->getReg())) {
+      Register Reg = Fold.OpToFold->getReg();
+      MachineInstr *DefMI = Fold.OpToFold->getParent();
+      if (DefMI->readsRegister(AMDGPU::EXEC, TRI) &&
+          execMayBeModifiedBeforeUse(*MRI, Reg, *DefMI, *Fold.UseMI))
+        continue;
+    }
     if (updateOperand(Fold, *TII, *TRI, *ST)) {
       // Clear kill flags.
       if (Fold.isReg()) {
diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
index 45c06ebb547ac..22f035e7f3e69 100644
--- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp
@@ -112,6 +112,7 @@ static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
       .addImm(0) // slc
       .addImm(0) // tfe
       .addImm(0) // dlc
+      .addImm(0) // swz
       .addMemOperand(MMO);
     return;
   }
@@ -132,6 +133,7 @@ static void buildPrologSpill(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
     .addImm(0) // slc
     .addImm(0) // tfe
     .addImm(0) // dlc
+    .addImm(0) // swz
     .addMemOperand(MMO);
 }
 
@@ -157,6 +159,7 @@ static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
       .addImm(0) // slc
       .addImm(0) // tfe
       .addImm(0) // dlc
+      .addImm(0) // swz
       .addMemOperand(MMO);
     return;
   }
@@ -177,6 +180,7 @@ static void buildEpilogReload(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB,
     .addImm(0) // slc
     .addImm(0) // tfe
     .addImm(0) // dlc
+    .addImm(0) // swz
     .addMemOperand(MMO);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 73aa5cd06669f..1883b28f657a8 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -1671,7 +1671,8 @@ static ArgDescriptor allocateVGPR32Input(CCState &CCInfo, unsigned Mask = ~0u,
   assert(Reg != AMDGPU::NoRegister);
 
   MachineFunction &MF = CCInfo.getMachineFunction();
-  MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
+  Register LiveInVReg = MF.addLiveIn(Reg, &AMDGPU::VGPR_32RegClass);
+  MF.getRegInfo().setType(LiveInVReg, LLT::scalar(32));
   return ArgDescriptor::createRegister(Reg, Mask);
 }
 
@@ -2970,9 +2971,9 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI,
                          IsThisReturn ? OutVals[0] : SDValue());
 }
 
-unsigned SITargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                             SelectionDAG &DAG) const {
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+Register SITargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                             const MachineFunction &MF) const {
+  Register Reg = StringSwitch<Register>(RegName)
     .Case("m0", AMDGPU::M0)
     .Case("exec", AMDGPU::EXEC)
     .Case("exec_lo", AMDGPU::EXEC_LO)
@@ -2980,7 +2981,7 @@ unsigned SITargetLowering::getRegisterByName(const char* RegName, EVT VT,
     .Case("flat_scratch", AMDGPU::FLAT_SCR)
     .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO)
     .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI)
-    .Default(AMDGPU::NoRegister);
+    .Default(Register());
 
   if (Reg == AMDGPU::NoRegister) {
     report_fatal_error(Twine("invalid register name \""
@@ -5665,14 +5666,14 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
   SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue});
   unsigned CachePolicy = cast<ConstantSDNode>(GLC)->getZExtValue();
   SDValue Ops[] = {
-      DAG.getEntryNode(),                         // Chain
-      Rsrc,                                       // rsrc
-      DAG.getConstant(0, DL, MVT::i32),           // vindex
-      {},                                         // voffset
-      {},                                         // soffset
-      {},                                         // offset
-      DAG.getConstant(CachePolicy, DL, MVT::i32), // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1),            // idxen
+      DAG.getEntryNode(),                               // Chain
+      Rsrc,                                             // rsrc
+      DAG.getConstant(0, DL, MVT::i32),                 // vindex
+      {},                                               // voffset
+      {},                                               // soffset
+      {},                                               // offset
+      DAG.getTargetConstant(CachePolicy, DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(0, DL, MVT::i1),            // idxen
   };
 
   // Use the alignment to ensure that the required offsets will fit into the
@@ -5681,7 +5682,7 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc,
 
   uint64_t InstOffset = cast<ConstantSDNode>(Ops[5])->getZExtValue();
   for (unsigned i = 0; i < NumLoads; ++i) {
-    Ops[5] = DAG.getConstant(InstOffset + 16 * i, DL, MVT::i32);
+    Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32);
     Loads.push_back(DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD, DL, VTList,
                                             Ops, LoadVT, MMO));
   }
@@ -5893,12 +5894,12 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
         Op.getOperand(1), // Src0
         Op.getOperand(2), // Attrchan
         Op.getOperand(3), // Attr
-        DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+        DAG.getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
         S, // Src2 - holds two f16 values selected by high
-        DAG.getConstant(0, DL, MVT::i32), // $src2_modifiers
+        DAG.getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
         Op.getOperand(4), // high
-        DAG.getConstant(0, DL, MVT::i1), // $clamp
-        DAG.getConstant(0, DL, MVT::i32) // $omod
+        DAG.getTargetConstant(0, DL, MVT::i1), // $clamp
+        DAG.getTargetConstant(0, DL, MVT::i32) // $omod
       };
       return DAG.getNode(AMDGPUISD::INTERP_P1LV_F16, DL, MVT::f32, Ops);
     } else {
@@ -5907,10 +5908,10 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
         Op.getOperand(1), // Src0
         Op.getOperand(2), // Attrchan
         Op.getOperand(3), // Attr
-        DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+        DAG.getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
         Op.getOperand(4), // high
-        DAG.getConstant(0, DL, MVT::i1), // $clamp
-        DAG.getConstant(0, DL, MVT::i32), // $omod
+        DAG.getTargetConstant(0, DL, MVT::i1), // $clamp
+        DAG.getTargetConstant(0, DL, MVT::i32), // $omod
         Glue
       };
       return DAG.getNode(AMDGPUISD::INTERP_P1LL_F16, DL, MVT::f32, Ops);
@@ -5923,11 +5924,11 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       Op.getOperand(2), // Src0
       Op.getOperand(3), // Attrchan
       Op.getOperand(4), // Attr
-      DAG.getConstant(0, DL, MVT::i32), // $src0_modifiers
+      DAG.getTargetConstant(0, DL, MVT::i32), // $src0_modifiers
       Op.getOperand(1), // Src2
-      DAG.getConstant(0, DL, MVT::i32), // $src2_modifiers
+      DAG.getTargetConstant(0, DL, MVT::i32), // $src2_modifiers
       Op.getOperand(5), // high
-      DAG.getConstant(0, DL, MVT::i1), // $clamp
+      DAG.getTargetConstant(0, DL, MVT::i1), // $clamp
       Glue
     };
     return DAG.getNode(AMDGPUISD::INTERP_P2_F16, DL, MVT::f16, Ops);
@@ -6233,8 +6234,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       SDValue(),        // voffset -- will be set by setBufferOffsets
       SDValue(),        // soffset -- will be set by setBufferOffsets
       SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
     };
 
     setBufferOffsets(Op.getOperand(4), DAG, &Ops[3]);
@@ -6270,8 +6271,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Offsets.first,    // voffset
       Op.getOperand(4), // soffset
       Offsets.second,   // offset
-      Op.getOperand(5), // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idxen
+      Op.getOperand(5), // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(0, DL, MVT::i1), // idxen
     };
 
     return lowerIntrinsicLoad(cast<MemSDNode>(Op), IsFormat, DAG, Ops);
@@ -6288,8 +6289,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Offsets.first,    // voffset
       Op.getOperand(5), // soffset
       Offsets.second,   // offset
-      Op.getOperand(6), // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idxen
+      Op.getOperand(6), // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(1, DL, MVT::i1), // idxen
     };
 
     return lowerIntrinsicLoad(cast<MemSDNode>(Op), IsFormat, DAG, Ops);
@@ -6312,9 +6313,9 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(4),  // voffset
       Op.getOperand(5),  // soffset
       Op.getOperand(6),  // offset
-      DAG.getConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
-      DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
+      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1) // idxen
     };
 
     if (LoadVT.getScalarType() == MVT::f16)
@@ -6337,8 +6338,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(4),  // soffset
       Offsets.second,    // offset
       Op.getOperand(5),  // format
-      Op.getOperand(6),  // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idxen
+      Op.getOperand(6),  // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(0, DL, MVT::i1), // idxen
     };
 
     if (LoadVT.getScalarType() == MVT::f16)
@@ -6361,8 +6362,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(5),  // soffset
       Offsets.second,    // offset
       Op.getOperand(6),  // format
-      Op.getOperand(7),  // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idxen
+      Op.getOperand(7),  // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(1, DL, MVT::i1), // idxen
     };
 
     if (LoadVT.getScalarType() == MVT::f16)
@@ -6394,8 +6395,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       SDValue(),        // voffset -- will be set by setBufferOffsets
       SDValue(),        // soffset -- will be set by setBufferOffsets
       SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
     };
     setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
     EVT VT = Op.getValueType();
@@ -6463,7 +6464,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(5), // soffset
       Offsets.second,   // offset
       Op.getOperand(6), // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(0, DL, MVT::i1), // idxen
     };
     EVT VT = Op.getValueType();
 
@@ -6536,7 +6537,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(6), // soffset
       Offsets.second,   // offset
       Op.getOperand(7), // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(1, DL, MVT::i1), // idxen
     };
     EVT VT = Op.getValueType();
 
@@ -6601,8 +6602,8 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       SDValue(),        // voffset -- will be set by setBufferOffsets
       SDValue(),        // soffset -- will be set by setBufferOffsets
       SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
     };
     setBufferOffsets(Op.getOperand(6), DAG, &Ops[5]);
     EVT VT = Op.getValueType();
@@ -6623,7 +6624,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(6), // soffset
       Offsets.second,   // offset
       Op.getOperand(7), // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(0, DL, MVT::i1), // idxen
     };
     EVT VT = Op.getValueType();
     auto *M = cast<MemSDNode>(Op);
@@ -6643,7 +6644,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
       Op.getOperand(7), // soffset
       Offsets.second,   // offset
       Op.getOperand(8), // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(1, DL, MVT::i1), // idxen
     };
     EVT VT = Op.getValueType();
     auto *M = cast<MemSDNode>(Op);
@@ -6805,9 +6806,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Op.getOperand(5),  // voffset
       Op.getOperand(6),  // soffset
       Op.getOperand(7),  // offset
-      DAG.getConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
-      DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idexen
+      DAG.getTargetConstant(Dfmt | (Nfmt << 4), DL, MVT::i32), // format
+      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idexen
     };
     unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 :
                            AMDGPUISD::TBUFFER_STORE_FORMAT;
@@ -6831,8 +6832,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Op.getOperand(6),  // soffset
       Offsets.second,    // offset
       Op.getOperand(7),  // format
-      Op.getOperand(8),  // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idexen
+      Op.getOperand(8),  // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(1, DL, MVT::i1), // idexen
     };
     unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 :
                            AMDGPUISD::TBUFFER_STORE_FORMAT;
@@ -6856,8 +6857,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Op.getOperand(5),  // soffset
       Offsets.second,    // offset
       Op.getOperand(6),  // format
-      Op.getOperand(7),  // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idexen
+      Op.getOperand(7),  // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(0, DL, MVT::i1), // idexen
     };
     unsigned Opc = IsD16 ? AMDGPUISD::TBUFFER_STORE_FORMAT_D16 :
                            AMDGPUISD::TBUFFER_STORE_FORMAT;
@@ -6885,8 +6886,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       SDValue(), // voffset -- will be set by setBufferOffsets
       SDValue(), // soffset -- will be set by setBufferOffsets
       SDValue(), // offset -- will be set by setBufferOffsets
-      DAG.getConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Glc | (Slc << 1), DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
     };
     setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
     unsigned Opc = IntrinsicID == Intrinsic::amdgcn_buffer_store ?
@@ -6930,8 +6931,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Offsets.first,    // voffset
       Op.getOperand(5), // soffset
       Offsets.second,   // offset
-      Op.getOperand(6), // cachepolicy
-      DAG.getConstant(0, DL, MVT::i1), // idxen
+      Op.getOperand(6), // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(0, DL, MVT::i1), // idxen
     };
     unsigned Opc =
         IsFormat ? AMDGPUISD::BUFFER_STORE_FORMAT : AMDGPUISD::BUFFER_STORE;
@@ -6974,8 +6975,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       Offsets.first,    // voffset
       Op.getOperand(6), // soffset
       Offsets.second,   // offset
-      Op.getOperand(7), // cachepolicy
-      DAG.getConstant(1, DL, MVT::i1), // idxen
+      Op.getOperand(7), // cachepolicy, swizzled buffer
+      DAG.getTargetConstant(1, DL, MVT::i1), // idxen
     };
     unsigned Opc = IntrinsicID == Intrinsic::amdgcn_struct_buffer_store ?
                    AMDGPUISD::BUFFER_STORE : AMDGPUISD::BUFFER_STORE_FORMAT;
@@ -7004,8 +7005,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
       SDValue(),        // voffset -- will be set by setBufferOffsets
       SDValue(),        // soffset -- will be set by setBufferOffsets
       SDValue(),        // offset -- will be set by setBufferOffsets
-      DAG.getConstant(Slc << 1, DL, MVT::i32), // cachepolicy
-      DAG.getConstant(IdxEn, DL, MVT::i1), // idxen
+      DAG.getTargetConstant(Slc << 1, DL, MVT::i32), // cachepolicy
+      DAG.getTargetConstant(IdxEn, DL, MVT::i1), // idxen
     };
     setBufferOffsets(Op.getOperand(5), DAG, &Ops[4]);
     EVT VT = Op.getOperand(2).getValueType();
@@ -7083,7 +7084,7 @@ std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
       Overflow += ImmOffset;
       ImmOffset = 0;
     }
-    C1 = cast<ConstantSDNode>(DAG.getConstant(ImmOffset, DL, MVT::i32));
+    C1 = cast<ConstantSDNode>(DAG.getTargetConstant(ImmOffset, DL, MVT::i32));
     if (Overflow) {
       auto OverflowVal = DAG.getConstant(Overflow, DL, MVT::i32);
       if (!N0)
@@ -7097,7 +7098,7 @@ std::pair<SDValue, SDValue> SITargetLowering::splitBufferOffsets(
   if (!N0)
     N0 = DAG.getConstant(0, DL, MVT::i32);
   if (!C1)
-    C1 = cast<ConstantSDNode>(DAG.getConstant(0, DL, MVT::i32));
+    C1 = cast<ConstantSDNode>(DAG.getTargetConstant(0, DL, MVT::i32));
   return {N0, SDValue(C1, 0)};
 }
 
@@ -7114,7 +7115,7 @@ void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
     if (AMDGPU::splitMUBUFOffset(Imm, SOffset, ImmOffset, Subtarget, Align)) {
       Offsets[0] = DAG.getConstant(0, DL, MVT::i32);
       Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
-      Offsets[2] = DAG.getConstant(ImmOffset, DL, MVT::i32);
+      Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
       return;
     }
   }
@@ -7127,13 +7128,13 @@ void SITargetLowering::setBufferOffsets(SDValue CombinedOffset,
                                                 Subtarget, Align)) {
       Offsets[0] = N0;
       Offsets[1] = DAG.getConstant(SOffset, DL, MVT::i32);
-      Offsets[2] = DAG.getConstant(ImmOffset, DL, MVT::i32);
+      Offsets[2] = DAG.getTargetConstant(ImmOffset, DL, MVT::i32);
       return;
     }
   }
   Offsets[0] = CombinedOffset;
   Offsets[1] = DAG.getConstant(0, DL, MVT::i32);
-  Offsets[2] = DAG.getConstant(0, DL, MVT::i32);
+  Offsets[2] = DAG.getTargetConstant(0, DL, MVT::i32);
 }
 
 // Handle 8 bit and 16 bit buffer loads
@@ -7315,8 +7316,8 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   assert(Op.getValueType().getVectorElementType() == MVT::i32 &&
          "Custom lowering for non-i32 vectors hasn't been implemented.");
 
-  if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
-                          *Load->getMemOperand())) {
+  if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                      MemVT, *Load->getMemOperand())) {
     SDValue Ops[2];
     std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG);
     return DAG.getMergeValues(Ops, DL);
@@ -7817,8 +7818,8 @@ SDValue SITargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   assert(VT.isVector() &&
          Store->getValue().getValueType().getScalarType() == MVT::i32);
 
-  if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
-                          *Store->getMemOperand())) {
+  if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                      VT, *Store->getMemOperand())) {
     return expandUnalignedStore(Store, DAG);
   }
 
@@ -10683,9 +10684,9 @@ void SITargetLowering::computeKnownBitsForFrameIndex(const SDValue Op,
   Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex());
 }
 
-llvm::Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
-  const llvm::Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
-  const llvm::Align CacheLineAlign = llvm::Align(64);
+Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
+  const Align PrefAlign = TargetLowering::getPrefLoopAlignment(ML);
+  const Align CacheLineAlign = Align(64);
 
   // Pre-GFX10 target did not benefit from loop alignment
   if (!ML || DisableLoopAlignment ||
@@ -10713,7 +10714,7 @@ llvm::Align SITargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
     // If inner loop block is aligned assume in average half of the alignment
     // size to be added as nops.
     if (MBB != Header)
-      LoopSize += (1 << MBB->getLogAlignment()) / 2;
+      LoopSize += MBB->getAlignment().value() / 2;
 
     for (const MachineInstr &MI : *MBB) {
       LoopSize += TII->getInstSizeInBytes(MI);
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 79cca882af98f..1257457b5d65a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -186,6 +186,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
 
   unsigned isCFIntrinsic(const SDNode *Intr) const;
 
+public:
   /// \returns True if fixup needs to be emitted for given global value \p GV,
   /// false otherwise.
   bool shouldEmitFixup(const GlobalValue *GV) const;
@@ -198,6 +199,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   /// global value \p GV, false otherwise.
   bool shouldEmitPCReloc(const GlobalValue *GV) const;
 
+private:
   // Analyze a combined offset from an amdgcn_buffer_ intrinsic and store the
   // three offsets (voffset, soffset and instoffset) into the SDValue[3] array
   // pointed to by Offsets.
@@ -317,8 +319,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
   SDValue LowerCall(CallLoweringInfo &CLI,
                     SmallVectorImpl<SDValue> &InVals) const override;
 
-  unsigned getRegisterByName(const char* RegName, EVT VT,
-                             SelectionDAG &DAG) const override;
+  Register getRegisterByName(const char* RegName, EVT VT,
+                             const MachineFunction &MF) const override;
 
   MachineBasicBlock *splitKillBlock(MachineInstr &MI,
                                     MachineBasicBlock *BB) const;
@@ -384,7 +386,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
                                     unsigned Depth = 0) const override;
   AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const override;
 
-  llvm::Align getPrefLoopAlignment(MachineLoop *ML) const override;
+  Align getPrefLoopAlignment(MachineLoop *ML) const override;
 
   void allocateHSAUserSGPRs(CCState &CCInfo,
                             MachineFunction &MF,
diff --git a/llvm/lib/Target/AMDGPU/SIInstrFormats.td b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
index 561a16c3e351d..4dcbe92861f23 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrFormats.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrFormats.td
@@ -124,6 +124,9 @@ class InstSI <dag outs, dag ins, string asm = "",
   // This bit indicates that this is one of MFMA instructions.
   field bit IsMAI = 0;
 
+  // This bit indicates that this is one of DOT instructions.
+  field bit IsDOT = 0;
+
   // These need to be kept in sync with the enum in SIInstrFlags.
   let TSFlags{0} = SALU;
   let TSFlags{1} = VALU;
@@ -189,6 +192,8 @@ class InstSI <dag outs, dag ins, string asm = "",
 
   let TSFlags{54} = IsMAI;
 
+  let TSFlags{55} = IsDOT;
+
   let SchedRW = [Write32Bit];
 
   field bits<1> DisableSIDecoder = 0;
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index aae6595f4174f..7a6bb0e20b795 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -1663,7 +1663,8 @@ MachineInstr *SIInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
 // This needs to be implemented because the source modifiers may be inserted
 // between the true commutable operands, and the base
 // TargetInstrInfo::commuteInstruction uses it.
-bool SIInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx0,
+bool SIInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
+                                        unsigned &SrcOpIdx0,
                                         unsigned &SrcOpIdx1) const {
   return findCommutedOpIndices(MI.getDesc(), SrcOpIdx0, SrcOpIdx1);
 }
@@ -2489,8 +2490,7 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa,
 }
 
 bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                                  const MachineInstr &MIb,
-                                                  AliasAnalysis *AA) const {
+                                                  const MachineInstr &MIb) const {
   assert((MIa.mayLoad() || MIa.mayStore()) &&
          "MIa must load from or modify a memory location");
   assert((MIb.mayLoad() || MIb.mayStore()) &&
@@ -4693,6 +4693,8 @@ void SIInstrInfo::legalizeOperands(MachineInstr &MI,
           MIB.addImm(TFE->getImm());
         }
 
+        MIB.addImm(getNamedImmOperand(MI, AMDGPU::OpName::swz));
+
         MIB.cloneMemRefs(MI);
         Addr64 = MIB;
       } else {
@@ -5663,7 +5665,16 @@ const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
       if (RI.hasAGPRs(NewDstRC))
         return nullptr;
 
-      NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
+      switch (Inst.getOpcode()) {
+      case AMDGPU::PHI:
+      case AMDGPU::REG_SEQUENCE:
+      case AMDGPU::INSERT_SUBREG:
+        NewDstRC = RI.getEquivalentAGPRClass(NewDstRC);
+        break;
+      default:
+        NewDstRC = RI.getEquivalentVGPRClass(NewDstRC);
+      }
+
       if (!NewDstRC)
         return nullptr;
     } else {
@@ -6410,3 +6421,40 @@ bool llvm::execMayBeModifiedBeforeAnyUse(const MachineRegisterInfo &MRI,
       return true;
   }
 }
+
+MachineInstr *SIInstrInfo::createPHIDestinationCopy(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator LastPHIIt,
+    const DebugLoc &DL, Register Src, Register Dst) const {
+  auto Cur = MBB.begin();
+  if (Cur != MBB.end())
+    do {
+      if (!Cur->isPHI() && Cur->readsRegister(Dst))
+        return BuildMI(MBB, Cur, DL, get(TargetOpcode::COPY), Dst).addReg(Src);
+      ++Cur;
+    } while (Cur != MBB.end() && Cur != LastPHIIt);
+
+  return TargetInstrInfo::createPHIDestinationCopy(MBB, LastPHIIt, DL, Src,
+                                                   Dst);
+}
+
+MachineInstr *SIInstrInfo::createPHISourceCopy(
+    MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt,
+    const DebugLoc &DL, Register Src, Register SrcSubReg, Register Dst) const {
+  if (InsPt != MBB.end() &&
+      (InsPt->getOpcode() == AMDGPU::SI_IF ||
+       InsPt->getOpcode() == AMDGPU::SI_ELSE ||
+       InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) &&
+      InsPt->definesRegister(Src)) {
+    InsPt++;
+    return BuildMI(MBB, InsPt, InsPt->getDebugLoc(),
+                   get(ST.isWave32() ? AMDGPU::S_MOV_B32_term
+                                     : AMDGPU::S_MOV_B64_term),
+                   Dst)
+        .addReg(Src, 0, SrcSubReg)
+        .addReg(AMDGPU::EXEC, RegState::Implicit);
+  }
+  return TargetInstrInfo::createPHISourceCopy(MBB, InsPt, DL, Src, SrcSubReg,
+                                              Dst);
+}
+
+bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index 2e629c4725668..a1a3962c41300 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -242,7 +242,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return commuteOpcode(MI.getOpcode());
   }
 
-  bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+  bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
                              unsigned &SrcOpIdx2) const override;
 
   bool findCommutedOpIndices(MCInstrDesc Desc, unsigned & SrcOpIdx0,
@@ -303,8 +303,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
 
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const override;
+                                  const MachineInstr &MIb) const override;
 
   bool isFoldableCopy(const MachineInstr &MI) const;
 
@@ -578,6 +577,14 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
     return get(Opcode).TSFlags & SIInstrFlags::IsMAI;
   }
 
+  static bool isDOT(const MachineInstr &MI) {
+    return MI.getDesc().TSFlags & SIInstrFlags::IsDOT;
+  }
+
+  bool isDOT(uint16_t Opcode) const {
+    return get(Opcode).TSFlags & SIInstrFlags::IsDOT;
+  }
+
   static bool isScalarUnit(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & (SIInstrFlags::SALU | SIInstrFlags::SMRD);
   }
@@ -954,6 +961,19 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo {
 
   bool isBasicBlockPrologue(const MachineInstr &MI) const override;
 
+  MachineInstr *createPHIDestinationCopy(MachineBasicBlock &MBB,
+                                         MachineBasicBlock::iterator InsPt,
+                                         const DebugLoc &DL, Register Src,
+                                         Register Dst) const override;
+
+  MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB,
+                                    MachineBasicBlock::iterator InsPt,
+                                    const DebugLoc &DL, Register Src,
+                                    Register SrcSubReg,
+                                    Register Dst) const override;
+
+  bool isWave32() const;
+
   /// Return a partially built integer add instruction without carry.
   /// Caller must add source operands.
   /// For pre-GFX9 it will generate unused carry destination operand.
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index d47be4a14068e..7473a0c64b2f9 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -84,7 +84,7 @@ def SDTtbuffer_load : SDTypeProfile<1, 8,
    SDTCisVT<4, i32>,    // soffset(SGPR)
    SDTCisVT<5, i32>,    // offset(imm)
    SDTCisVT<6, i32>,    // format(imm)
-   SDTCisVT<7, i32>,    // cachecontrol(imm)
+   SDTCisVT<7, i32>,    // cachepolicy, swizzled buffer(imm)
    SDTCisVT<8, i1>      // idxen(imm)
   ]>;
 
@@ -102,7 +102,7 @@ def SDTtbuffer_store : SDTypeProfile<0, 9,
      SDTCisVT<4, i32>,    // soffset(SGPR)
      SDTCisVT<5, i32>,    // offset(imm)
      SDTCisVT<6, i32>,    // format(imm)
-     SDTCisVT<7, i32>,    // cachecontrol(imm)
+     SDTCisVT<7, i32>,    // cachepolicy, swizzled buffer(imm)
      SDTCisVT<8, i1>      // idxen(imm)
     ]>;
 
@@ -119,7 +119,7 @@ def SDTBufferLoad : SDTypeProfile<1, 7,
      SDTCisVT<3, i32>,   // voffset(VGPR)
      SDTCisVT<4, i32>,   // soffset(SGPR)
      SDTCisVT<5, i32>,   // offset(imm)
-     SDTCisVT<6, i32>,   // cachepolicy(imm)
+     SDTCisVT<6, i32>,   // cachepolicy, swizzled buffer(imm)
      SDTCisVT<7, i1>]>;  // idxen(imm)
 
 def SIbuffer_load : SDNode <"AMDGPUISD::BUFFER_LOAD", SDTBufferLoad,
@@ -145,7 +145,7 @@ def SDTBufferStore : SDTypeProfile<0, 8,
      SDTCisVT<3, i32>,   // voffset(VGPR)
      SDTCisVT<4, i32>,   // soffset(SGPR)
      SDTCisVT<5, i32>,   // offset(imm)
-     SDTCisVT<6, i32>,   // cachepolicy(imm)
+     SDTCisVT<6, i32>,   // cachepolicy, swizzled buffer(imm)
      SDTCisVT<7, i1>]>;  // idxen(imm)
 
 def SIbuffer_store : SDNode <"AMDGPUISD::BUFFER_STORE", SDTBufferStore,
@@ -689,6 +689,10 @@ def as_i32imm: SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
+def as_i32timm: SDNodeXForm<timm, [{
+  return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32);
+}]>;
+
 def as_i64imm: SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i64);
 }]>;
@@ -728,6 +732,10 @@ def UIMM16bit : ImmLeaf <i32,
   [{return isUInt<16>(Imm);}]
 >;
 
+def i64imm_32bit : ImmLeaf<i64, [{
+  return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);
+}]>;
+
 class InlineImm <ValueType vt> : PatLeaf <(vt imm), [{
   return isInlineImmediate(N);
 }]>;
@@ -1027,6 +1035,7 @@ def DLC : NamedOperandBit<"DLC", NamedMatchClass<"DLC">>;
 def GLC : NamedOperandBit<"GLC", NamedMatchClass<"GLC">>;
 def SLC : NamedOperandBit<"SLC", NamedMatchClass<"SLC">>;
 def TFE : NamedOperandBit<"TFE", NamedMatchClass<"TFE">>;
+def SWZ : NamedOperandBit<"SWZ", NamedMatchClass<"SWZ">>;
 def UNorm : NamedOperandBit<"UNorm", NamedMatchClass<"UNorm">>;
 def DA : NamedOperandBit<"DA", NamedMatchClass<"DA">>;
 def R128A16 : NamedOperandBit<"R128A16", NamedMatchClass<"R128A16">>;
@@ -2139,6 +2148,7 @@ class VOPProfile <list<ValueType> _ArgVT, bit _EnableF32SrcMods = 0,
   field int NeedPatGen = PatGenMode.NoPattern;
 
   field bit IsMAI = 0;
+  field bit IsDOT = 0;
 
   field Operand Src0PackedMod = !if(HasSrc0FloatMods, PackedF16InputMods, PackedI16InputMods);
   field Operand Src1PackedMod = !if(HasSrc1FloatMods, PackedF16InputMods, PackedI16InputMods);
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index a99977d72d95f..15b9fce5341bf 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -43,8 +43,8 @@ multiclass V_INTERP_P1_F32_m : VINTRP_m <
   (outs VINTRPDst:$vdst),
   (ins VGPR_32:$vsrc, Attr:$attr, AttrChan:$attrchan),
   "v_interp_p1_f32$vdst, $vsrc, $attr$attrchan",
-  [(set f32:$vdst, (AMDGPUinterp_p1 f32:$vsrc, (i32 imm:$attrchan),
-                                               (i32 imm:$attr)))]
+  [(set f32:$vdst, (AMDGPUinterp_p1 f32:$vsrc, (i32 timm:$attrchan),
+                                               (i32 timm:$attr)))]
 >;
 
 let OtherPredicates = [has32BankLDS] in {
@@ -66,8 +66,8 @@ defm V_INTERP_P2_F32 : VINTRP_m <
   (outs VINTRPDst:$vdst),
   (ins VGPR_32:$src0, VGPR_32:$vsrc, Attr:$attr, AttrChan:$attrchan),
   "v_interp_p2_f32$vdst, $vsrc, $attr$attrchan",
-  [(set f32:$vdst, (AMDGPUinterp_p2 f32:$src0, f32:$vsrc, (i32 imm:$attrchan),
-                                                          (i32 imm:$attr)))]>;
+  [(set f32:$vdst, (AMDGPUinterp_p2 f32:$src0, f32:$vsrc, (i32 timm:$attrchan),
+                                                          (i32 timm:$attr)))]>;
 
 } // End DisableEncoding = "$src0", Constraints = "$src0 = $vdst"
 
@@ -76,8 +76,8 @@ defm V_INTERP_MOV_F32 : VINTRP_m <
   (outs VINTRPDst:$vdst),
   (ins InterpSlot:$vsrc, Attr:$attr, AttrChan:$attrchan),
   "v_interp_mov_f32$vdst, $vsrc, $attr$attrchan",
-  [(set f32:$vdst, (AMDGPUinterp_mov (i32 imm:$vsrc), (i32 imm:$attrchan),
-                                     (i32 imm:$attr)))]>;
+  [(set f32:$vdst, (AMDGPUinterp_mov (i32 imm:$vsrc), (i32 timm:$attrchan),
+                                     (i32 timm:$attr)))]>;
 
 } // End Uses = [M0, EXEC]
 
@@ -349,7 +349,8 @@ def SI_INIT_M0 : SPseudoInstSI <(outs), (ins SSrc_b32:$src)> {
 }
 
 def SI_INIT_EXEC : SPseudoInstSI <
-  (outs), (ins i64imm:$src), []> {
+  (outs), (ins i64imm:$src),
+  [(int_amdgcn_init_exec (i64 timm:$src))]> {
   let Defs = [EXEC];
   let usesCustomInserter = 1;
   let isAsCheapAsAMove = 1;
@@ -365,12 +366,20 @@ def SI_INIT_EXEC_LO : SPseudoInstSI <
   let WaveSizePredicate = isWave32;
 }
 
+// FIXME: Wave32 version
 def SI_INIT_EXEC_FROM_INPUT : SPseudoInstSI <
-  (outs), (ins SSrc_b32:$input, i32imm:$shift), []> {
+  (outs), (ins SSrc_b32:$input, i32imm:$shift),
+  [(int_amdgcn_init_exec_from_input i32:$input, (i32 timm:$shift))]> {
   let Defs = [EXEC];
   let usesCustomInserter = 1;
 }
 
+def : GCNPat <
+  (int_amdgcn_init_exec timm:$src),
+  (SI_INIT_EXEC_LO (as_i32imm imm:$src))> {
+  let WaveSizePredicate = isWave32;
+}
+
 // Return for returning shaders to a shader variant epilog.
 def SI_RETURN_TO_EPILOG : SPseudoInstSI <
   (outs), (ins variable_ops), [(AMDGPUreturn_to_epilog)]> {
@@ -609,23 +618,6 @@ def : GCNPat <
   (SI_PC_ADD_REL_OFFSET $ptr_lo, (i32 0))
 >;
 
-def : GCNPat <
-  (int_amdgcn_init_exec i64:$src),
-  (SI_INIT_EXEC (as_i64imm $src))> {
-  let WaveSizePredicate = isWave64;
-}
-
-def : GCNPat <
-  (int_amdgcn_init_exec i64:$src),
-  (SI_INIT_EXEC_LO (as_i32imm $src))> {
-  let WaveSizePredicate = isWave32;
-}
-
-def : GCNPat <
-  (int_amdgcn_init_exec_from_input i32:$input, i32:$shift),
-  (SI_INIT_EXEC_FROM_INPUT (i32 $input), (as_i32imm $shift))
->;
-
 def : GCNPat<
   (AMDGPUtrap timm:$trapid),
   (S_TRAP $trapid)
@@ -743,22 +735,22 @@ def : GCNPat <
 
 def : GCNPat <
   (i32 (fp_to_sint f16:$src)),
-  (V_CVT_I32_F32_e32 (V_CVT_F32_F16_e32 $src))
+  (V_CVT_I32_F32_e32 (V_CVT_F32_F16_e32 VSrc_b32:$src))
 >;
 
 def : GCNPat <
   (i32 (fp_to_uint f16:$src)),
-  (V_CVT_U32_F32_e32 (V_CVT_F32_F16_e32 $src))
+  (V_CVT_U32_F32_e32 (V_CVT_F32_F16_e32 VSrc_b32:$src))
 >;
 
 def : GCNPat <
   (f16 (sint_to_fp i32:$src)),
-  (V_CVT_F16_F32_e32 (V_CVT_F32_I32_e32 $src))
+  (V_CVT_F16_F32_e32 (V_CVT_F32_I32_e32 VSrc_b32:$src))
 >;
 
 def : GCNPat <
   (f16 (uint_to_fp i32:$src)),
-  (V_CVT_F16_F32_e32 (V_CVT_F32_U32_e32 $src))
+  (V_CVT_F16_F32_e32 (V_CVT_F32_U32_e32 VSrc_b32:$src))
 >;
 
 //===----------------------------------------------------------------------===//
@@ -1108,6 +1100,11 @@ def : GCNPat <
   (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000)))
 >;
 
+def : GCNPat <
+  (fneg (f16 VGPR_32:$src)),
+  (V_XOR_B32_e32 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src)
+>;
+
 def : GCNPat <
   (fabs (f16 SReg_32:$src)),
   (S_AND_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00007fff)))
@@ -1118,6 +1115,11 @@ def : GCNPat <
   (S_OR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x00008000))) // Set sign bit
 >;
 
+def : GCNPat <
+  (fneg (fabs (f16 VGPR_32:$src))),
+  (V_OR_B32_e32 (S_MOV_B32 (i32 0x00008000)), VGPR_32:$src) // Set sign bit
+>;
+
 def : GCNPat <
   (fneg (v2f16 SReg_32:$src)),
   (S_XOR_B32 SReg_32:$src, (S_MOV_B32 (i32 0x80008000)))
@@ -1609,7 +1611,7 @@ def : GCNPat <
   (V_CVT_F16_F32_e32 (
       V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                         /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),
-                        $src))
+                        SSrc_i1:$src))
 >;
 
 def : GCNPat <
@@ -1617,35 +1619,35 @@ def : GCNPat <
   (V_CVT_F16_F32_e32 (
       V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                         /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),
-                        $src))
+                        SSrc_i1:$src))
 >;
 
 def : GCNPat <
   (f32 (sint_to_fp i1:$src)),
   (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                         /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),
-                        $src)
+                        SSrc_i1:$src)
 >;
 
 def : GCNPat <
   (f32 (uint_to_fp i1:$src)),
   (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                         /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_ONE),
-                        $src)
+                        SSrc_i1:$src)
 >;
 
 def : GCNPat <
   (f64 (sint_to_fp i1:$src)),
   (V_CVT_F64_I32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                                         /*src1mod*/(i32 0), /*src1*/(i32 -1),
-                                        $src))
+                                        SSrc_i1:$src))
 >;
 
 def : GCNPat <
   (f64 (uint_to_fp i1:$src)),
   (V_CVT_F64_U32_e32 (V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                                         /*src1mod*/(i32 0), /*src1*/(i32 1),
-                                        $src))
+                                        SSrc_i1:$src))
 >;
 
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
index 49f34817724dd..a78b62de7151d 100644
--- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp
@@ -135,6 +135,35 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
     bool DLC1;
     bool UseST64;
     SmallVector<MachineInstr *, 8> InstsToMove;
+    int AddrIdx[5];
+    const MachineOperand *AddrReg[5];
+    unsigned NumAddresses;
+
+    bool hasSameBaseAddress(const MachineInstr &MI) {
+      for (unsigned i = 0; i < NumAddresses; i++) {
+        const MachineOperand &AddrRegNext = MI.getOperand(AddrIdx[i]);
+
+        if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
+          if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
+              AddrReg[i]->getImm() != AddrRegNext.getImm()) {
+            return false;
+          }
+          continue;
+        }
+
+        // Check same base pointer. Be careful of subregisters, which can occur
+        // with vectors of pointers.
+        if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
+            AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
+         return false;
+        }
+      }
+      return true;
+    }
+
+    void setMI(MachineBasicBlock::iterator MI, const SIInstrInfo &TII,
+               const GCNSubtarget &STM);
+    void setPaired(MachineBasicBlock::iterator MI, const SIInstrInfo &TII);
   };
 
   struct BaseRegisters {
@@ -165,9 +194,6 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
   static unsigned getNewOpcode(const CombineInfo &CI);
   static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI);
   const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI);
-  unsigned getOpcodeWidth(const MachineInstr &MI) const;
-  InstClassEnum getInstClass(unsigned Opc) const;
-  unsigned getRegs(unsigned Opc) const;
 
   bool findMatchingInst(CombineInfo &CI);
 
@@ -183,17 +209,17 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
   MachineBasicBlock::iterator mergeBufferStorePair(CombineInfo &CI);
 
   void updateBaseAndOffset(MachineInstr &I, unsigned NewBase,
-                           int32_t NewOffset);
-  unsigned computeBase(MachineInstr &MI, const MemAddress &Addr);
-  MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI);
-  Optional<int32_t> extractConstOffset(const MachineOperand &Op);
-  void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr);
+                           int32_t NewOffset) const;
+  unsigned computeBase(MachineInstr &MI, const MemAddress &Addr) const;
+  MachineOperand createRegOrImm(int32_t Val, MachineInstr &MI) const;
+  Optional<int32_t> extractConstOffset(const MachineOperand &Op) const;
+  void processBaseWithConstOffset(const MachineOperand &Base, MemAddress &Addr) const;
   /// Promotes constant offset to the immediate by adjusting the base. It
   /// tries to use a base from the nearby instructions that allows it to have
   /// a 13bit constant offset which gets promoted to the immediate.
   bool promoteConstantOffsetToImm(MachineInstr &CI,
                                   MemInfoMap &Visited,
-                                  SmallPtrSet<MachineInstr *, 4> &Promoted);
+                                  SmallPtrSet<MachineInstr *, 4> &Promoted) const;
 
 public:
   static char ID;
@@ -216,6 +242,210 @@ class SILoadStoreOptimizer : public MachineFunctionPass {
   }
 };
 
+static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) {
+  const unsigned Opc = MI.getOpcode();
+
+  if (TII.isMUBUF(Opc)) {
+    // FIXME: Handle d16 correctly
+    return AMDGPU::getMUBUFElements(Opc);
+  }
+
+  switch (Opc) {
+  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
+    return 1;
+  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
+    return 2;
+  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
+    return 4;
+  default:
+    return 0;
+  }
+}
+
+static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) {
+  if (TII.isMUBUF(Opc)) {
+    const int baseOpcode = AMDGPU::getMUBUFBaseOpcode(Opc);
+
+    // If we couldn't identify the opcode, bail out.
+    if (baseOpcode == -1) {
+      return UNKNOWN;
+    }
+
+    switch (baseOpcode) {
+    case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
+      return BUFFER_LOAD_OFFEN;
+    case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
+      return BUFFER_LOAD_OFFSET;
+    case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
+      return BUFFER_STORE_OFFEN;
+    case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
+      return BUFFER_STORE_OFFSET;
+    case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
+      return BUFFER_LOAD_OFFEN_exact;
+    case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
+      return BUFFER_LOAD_OFFSET_exact;
+    case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
+      return BUFFER_STORE_OFFEN_exact;
+    case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
+      return BUFFER_STORE_OFFSET_exact;
+    default:
+      return UNKNOWN;
+    }
+  }
+
+  switch (Opc) {
+  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
+  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
+  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
+    return S_BUFFER_LOAD_IMM;
+  case AMDGPU::DS_READ_B32:
+  case AMDGPU::DS_READ_B64:
+  case AMDGPU::DS_READ_B32_gfx9:
+  case AMDGPU::DS_READ_B64_gfx9:
+    return DS_READ;
+  case AMDGPU::DS_WRITE_B32:
+  case AMDGPU::DS_WRITE_B64:
+  case AMDGPU::DS_WRITE_B32_gfx9:
+  case AMDGPU::DS_WRITE_B64_gfx9:
+    return DS_WRITE;
+  default:
+    return UNKNOWN;
+  }
+}
+
+static unsigned getRegs(unsigned Opc, const SIInstrInfo &TII) {
+  if (TII.isMUBUF(Opc)) {
+    unsigned result = 0;
+
+    if (AMDGPU::getMUBUFHasVAddr(Opc)) {
+      result |= VADDR;
+    }
+
+    if (AMDGPU::getMUBUFHasSrsrc(Opc)) {
+      result |= SRSRC;
+    }
+
+    if (AMDGPU::getMUBUFHasSoffset(Opc)) {
+      result |= SOFFSET;
+    }
+
+    return result;
+  }
+
+  switch (Opc) {
+  default:
+    return 0;
+  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
+  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
+  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
+    return SBASE;
+  case AMDGPU::DS_READ_B32:
+  case AMDGPU::DS_READ_B64:
+  case AMDGPU::DS_READ_B32_gfx9:
+  case AMDGPU::DS_READ_B64_gfx9:
+  case AMDGPU::DS_WRITE_B32:
+  case AMDGPU::DS_WRITE_B64:
+  case AMDGPU::DS_WRITE_B32_gfx9:
+  case AMDGPU::DS_WRITE_B64_gfx9:
+    return ADDR;
+  }
+}
+
+
+void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI,
+                                              const SIInstrInfo &TII,
+                                              const GCNSubtarget &STM) {
+  I = MI;
+  unsigned Opc = MI->getOpcode();
+  InstClass = getInstClass(Opc, TII);
+
+  if (InstClass == UNKNOWN)
+    return;
+
+  switch (InstClass) {
+  case DS_READ:
+   EltSize =
+          (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
+                                                                          : 4;
+   break;
+  case DS_WRITE:
+    EltSize =
+          (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
+                                                                            : 4;
+    break;
+  case S_BUFFER_LOAD_IMM:
+    EltSize = AMDGPU::getSMRDEncodedOffset(STM, 4);
+    break;
+  default:
+    EltSize = 4;
+    break;
+  }
+
+  int OffsetIdx =
+      AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset);
+  Offset0 = I->getOperand(OffsetIdx).getImm();
+  Width0 = getOpcodeWidth(*I, TII);
+
+  if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
+    Offset0 &= 0xffff;
+  } else {
+    GLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::glc)->getImm();
+    if (InstClass != S_BUFFER_LOAD_IMM) {
+      SLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::slc)->getImm();
+    }
+    DLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::dlc)->getImm();
+  }
+
+  unsigned AddrOpName[5] = {0};
+  NumAddresses = 0;
+  const unsigned Regs = getRegs(I->getOpcode(), TII);
+
+  if (Regs & ADDR) {
+    AddrOpName[NumAddresses++] = AMDGPU::OpName::addr;
+  }
+
+  if (Regs & SBASE) {
+    AddrOpName[NumAddresses++] = AMDGPU::OpName::sbase;
+  }
+
+  if (Regs & SRSRC) {
+    AddrOpName[NumAddresses++] = AMDGPU::OpName::srsrc;
+  }
+
+  if (Regs & SOFFSET) {
+    AddrOpName[NumAddresses++] = AMDGPU::OpName::soffset;
+  }
+
+  if (Regs & VADDR) {
+    AddrOpName[NumAddresses++] = AMDGPU::OpName::vaddr;
+  }
+
+  for (unsigned i = 0; i < NumAddresses; i++) {
+    AddrIdx[i] = AMDGPU::getNamedOperandIdx(I->getOpcode(), AddrOpName[i]);
+    AddrReg[i] = &I->getOperand(AddrIdx[i]);
+  }
+}
+
+void SILoadStoreOptimizer::CombineInfo::setPaired(MachineBasicBlock::iterator MI,
+                                                  const SIInstrInfo &TII) {
+  Paired = MI;
+  assert(InstClass == getInstClass(Paired->getOpcode(), TII));
+  int OffsetIdx =
+      AMDGPU::getNamedOperandIdx(I->getOpcode(), AMDGPU::OpName::offset);
+  Offset1 = Paired->getOperand(OffsetIdx).getImm();
+  Width1 = getOpcodeWidth(*Paired, TII);
+  if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) {
+    Offset1 &= 0xffff;
+  } else {
+    GLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::glc)->getImm();
+    if (InstClass != S_BUFFER_LOAD_IMM) {
+      SLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::slc)->getImm();
+    }
+    DLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::dlc)->getImm();
+  }
+}
+
+
 } // end anonymous namespace.
 
 INITIALIZE_PASS_BEGIN(SILoadStoreOptimizer, DEBUG_TYPE,
@@ -398,163 +628,30 @@ bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM,
   }
 }
 
-unsigned SILoadStoreOptimizer::getOpcodeWidth(const MachineInstr &MI) const {
-  const unsigned Opc = MI.getOpcode();
-
-  if (TII->isMUBUF(MI)) {
-    // FIXME: Handle d16 correctly
-    return AMDGPU::getMUBUFElements(Opc);
-  }
-
-  switch (Opc) {
-  default:
-    return 0;
-  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
-    return 1;
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
-    return 2;
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
-    return 4;
-  }
-}
-
-InstClassEnum SILoadStoreOptimizer::getInstClass(unsigned Opc) const {
-  if (TII->isMUBUF(Opc)) {
-    const int baseOpcode = AMDGPU::getMUBUFBaseOpcode(Opc);
-
-    // If we couldn't identify the opcode, bail out.
-    if (baseOpcode == -1) {
-      return UNKNOWN;
-    }
-
-    switch (baseOpcode) {
-    default:
-      return UNKNOWN;
-    case AMDGPU::BUFFER_LOAD_DWORD_OFFEN:
-      return BUFFER_LOAD_OFFEN;
-    case AMDGPU::BUFFER_LOAD_DWORD_OFFSET:
-      return BUFFER_LOAD_OFFSET;
-    case AMDGPU::BUFFER_STORE_DWORD_OFFEN:
-      return BUFFER_STORE_OFFEN;
-    case AMDGPU::BUFFER_STORE_DWORD_OFFSET:
-      return BUFFER_STORE_OFFSET;
-    case AMDGPU::BUFFER_LOAD_DWORD_OFFEN_exact:
-      return BUFFER_LOAD_OFFEN_exact;
-    case AMDGPU::BUFFER_LOAD_DWORD_OFFSET_exact:
-      return BUFFER_LOAD_OFFSET_exact;
-    case AMDGPU::BUFFER_STORE_DWORD_OFFEN_exact:
-      return BUFFER_STORE_OFFEN_exact;
-    case AMDGPU::BUFFER_STORE_DWORD_OFFSET_exact:
-      return BUFFER_STORE_OFFSET_exact;
-    }
-  }
-
-  switch (Opc) {
-  default:
-    return UNKNOWN;
-  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
-    return S_BUFFER_LOAD_IMM;
-  case AMDGPU::DS_READ_B32:
-  case AMDGPU::DS_READ_B64:
-  case AMDGPU::DS_READ_B32_gfx9:
-  case AMDGPU::DS_READ_B64_gfx9:
-    return DS_READ;
-  case AMDGPU::DS_WRITE_B32:
-  case AMDGPU::DS_WRITE_B64:
-  case AMDGPU::DS_WRITE_B32_gfx9:
-  case AMDGPU::DS_WRITE_B64_gfx9:
-    return DS_WRITE;
-  }
-}
-
-unsigned SILoadStoreOptimizer::getRegs(unsigned Opc) const {
-  if (TII->isMUBUF(Opc)) {
-    unsigned result = 0;
-
-    if (AMDGPU::getMUBUFHasVAddr(Opc)) {
-      result |= VADDR;
-    }
-
-    if (AMDGPU::getMUBUFHasSrsrc(Opc)) {
-      result |= SRSRC;
-    }
-
-    if (AMDGPU::getMUBUFHasSoffset(Opc)) {
-      result |= SOFFSET;
-    }
-
-    return result;
-  }
-
-  switch (Opc) {
-  default:
-    return 0;
-  case AMDGPU::S_BUFFER_LOAD_DWORD_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM:
-  case AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM:
-    return SBASE;
-  case AMDGPU::DS_READ_B32:
-  case AMDGPU::DS_READ_B64:
-  case AMDGPU::DS_READ_B32_gfx9:
-  case AMDGPU::DS_READ_B64_gfx9:
-  case AMDGPU::DS_WRITE_B32:
-  case AMDGPU::DS_WRITE_B64:
-  case AMDGPU::DS_WRITE_B32_gfx9:
-  case AMDGPU::DS_WRITE_B64_gfx9:
-    return ADDR;
-  }
-}
-
 bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) {
   MachineBasicBlock *MBB = CI.I->getParent();
   MachineBasicBlock::iterator E = MBB->end();
   MachineBasicBlock::iterator MBBI = CI.I;
 
   const unsigned Opc = CI.I->getOpcode();
-  const InstClassEnum InstClass = getInstClass(Opc);
+  const InstClassEnum InstClass = getInstClass(Opc, *TII);
 
   if (InstClass == UNKNOWN) {
     return false;
   }
 
-  const unsigned Regs = getRegs(Opc);
-
-  unsigned AddrOpName[5] = {0};
-  int AddrIdx[5];
-  const MachineOperand *AddrReg[5];
-  unsigned NumAddresses = 0;
-
-  if (Regs & ADDR) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::addr;
-  }
-
-  if (Regs & SBASE) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::sbase;
-  }
-
-  if (Regs & SRSRC) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::srsrc;
-  }
-
-  if (Regs & SOFFSET) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::soffset;
-  }
-
-  if (Regs & VADDR) {
-    AddrOpName[NumAddresses++] = AMDGPU::OpName::vaddr;
-  }
-
-  for (unsigned i = 0; i < NumAddresses; i++) {
-    AddrIdx[i] = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AddrOpName[i]);
-    AddrReg[i] = &CI.I->getOperand(AddrIdx[i]);
+  // Do not merge VMEM buffer instructions with "swizzled" bit set.
+  int Swizzled =
+      AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::swz);
+  if (Swizzled != -1 && CI.I->getOperand(Swizzled).getImm())
+    return false;
 
+  for (unsigned i = 0; i < CI.NumAddresses; i++) {
     // We only ever merge operations with the same base address register, so
     // don't bother scanning forward if there are no other uses.
-    if (AddrReg[i]->isReg() &&
-        (Register::isPhysicalRegister(AddrReg[i]->getReg()) ||
-         MRI->hasOneNonDBGUse(AddrReg[i]->getReg())))
+    if (CI.AddrReg[i]->isReg() &&
+        (Register::isPhysicalRegister(CI.AddrReg[i]->getReg()) ||
+         MRI->hasOneNonDBGUse(CI.AddrReg[i]->getReg())))
       return false;
   }
 
@@ -567,7 +664,7 @@ bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) {
   for (; MBBI != E; ++MBBI) {
     const bool IsDS = (InstClass == DS_READ) || (InstClass == DS_WRITE);
 
-    if ((getInstClass(MBBI->getOpcode()) != InstClass) ||
+    if ((getInstClass(MBBI->getOpcode(), *TII) != InstClass) ||
         (IsDS && (MBBI->getOpcode() != Opc))) {
       // This is not a matching DS instruction, but we can keep looking as
       // long as one of these conditions are met:
@@ -614,50 +711,10 @@ bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) {
                               CI.InstsToMove))
       continue;
 
-    bool Match = true;
-    for (unsigned i = 0; i < NumAddresses; i++) {
-      const MachineOperand &AddrRegNext = MBBI->getOperand(AddrIdx[i]);
-
-      if (AddrReg[i]->isImm() || AddrRegNext.isImm()) {
-        if (AddrReg[i]->isImm() != AddrRegNext.isImm() ||
-            AddrReg[i]->getImm() != AddrRegNext.getImm()) {
-          Match = false;
-          break;
-        }
-        continue;
-      }
-
-      // Check same base pointer. Be careful of subregisters, which can occur
-      // with vectors of pointers.
-      if (AddrReg[i]->getReg() != AddrRegNext.getReg() ||
-          AddrReg[i]->getSubReg() != AddrRegNext.getSubReg()) {
-        Match = false;
-        break;
-      }
-    }
+    bool Match = CI.hasSameBaseAddress(*MBBI);
 
     if (Match) {
-      int OffsetIdx =
-          AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::offset);
-      CI.Offset0 = CI.I->getOperand(OffsetIdx).getImm();
-      CI.Width0 = getOpcodeWidth(*CI.I);
-      CI.Offset1 = MBBI->getOperand(OffsetIdx).getImm();
-      CI.Width1 = getOpcodeWidth(*MBBI);
-      CI.Paired = MBBI;
-
-      if ((CI.InstClass == DS_READ) || (CI.InstClass == DS_WRITE)) {
-        CI.Offset0 &= 0xffff;
-        CI.Offset1 &= 0xffff;
-      } else {
-        CI.GLC0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::glc)->getImm();
-        CI.GLC1 = TII->getNamedOperand(*MBBI, AMDGPU::OpName::glc)->getImm();
-        if (CI.InstClass != S_BUFFER_LOAD_IMM) {
-          CI.SLC0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::slc)->getImm();
-          CI.SLC1 = TII->getNamedOperand(*MBBI, AMDGPU::OpName::slc)->getImm();
-        }
-        CI.DLC0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::dlc)->getImm();
-        CI.DLC1 = TII->getNamedOperand(*MBBI, AMDGPU::OpName::dlc)->getImm();
-      }
+      CI.setPaired(MBBI, *TII);
 
       // Check both offsets fit in the reduced range.
       // We also need to go through the list of instructions that we plan to
@@ -927,7 +984,7 @@ SILoadStoreOptimizer::mergeBufferLoadPair(CombineInfo &CI) {
 
   auto MIB = BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode), DestReg);
 
-  const unsigned Regs = getRegs(Opcode);
+  const unsigned Regs = getRegs(Opcode, *TII);
 
   if (Regs & VADDR)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
@@ -947,6 +1004,7 @@ SILoadStoreOptimizer::mergeBufferLoadPair(CombineInfo &CI) {
       .addImm(CI.SLC0)      // slc
       .addImm(0)            // tfe
       .addImm(CI.DLC0)      // dlc
+      .addImm(0)            // swz
       .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb));
 
   std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI);
@@ -1119,7 +1177,7 @@ SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI) {
   auto MIB = BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode))
                  .addReg(SrcReg, RegState::Kill);
 
-  const unsigned Regs = getRegs(Opcode);
+  const unsigned Regs = getRegs(Opcode, *TII);
 
   if (Regs & VADDR)
     MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr));
@@ -1140,6 +1198,7 @@ SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI) {
       .addImm(CI.SLC0)      // slc
       .addImm(0)            // tfe
       .addImm(CI.DLC0)      // dlc
+      .addImm(0)            // swz
       .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb));
 
   moveInstsAfter(MIB, CI.InstsToMove);
@@ -1151,7 +1210,7 @@ SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI) {
 }
 
 MachineOperand
-SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) {
+SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) const {
   APInt V(32, Val, true);
   if (TII->isInlineConstant(V))
     return MachineOperand::CreateImm(Val);
@@ -1168,7 +1227,7 @@ SILoadStoreOptimizer::createRegOrImm(int32_t Val, MachineInstr &MI) {
 
 // Compute base address using Addr and return the final register.
 unsigned SILoadStoreOptimizer::computeBase(MachineInstr &MI,
-                                           const MemAddress &Addr) {
+                                           const MemAddress &Addr) const {
   MachineBasicBlock *MBB = MI.getParent();
   MachineBasicBlock::iterator MBBI = MI.getIterator();
   DebugLoc DL = MI.getDebugLoc();
@@ -1227,13 +1286,13 @@ unsigned SILoadStoreOptimizer::computeBase(MachineInstr &MI,
 // Update base and offset with the NewBase and NewOffset in MI.
 void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI,
                                                unsigned NewBase,
-                                               int32_t NewOffset) {
+                                               int32_t NewOffset) const {
   TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->setReg(NewBase);
   TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset);
 }
 
 Optional<int32_t>
-SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) {
+SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) const {
   if (Op.isImm())
     return Op.getImm();
 
@@ -1259,7 +1318,7 @@ SILoadStoreOptimizer::extractConstOffset(const MachineOperand &Op) {
 //   %Base:vreg_64 =
 //       REG_SEQUENCE %LO:vgpr_32, %subreg.sub0, %HI:vgpr_32, %subreg.sub1
 void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base,
-                                                      MemAddress &Addr) {
+                                                      MemAddress &Addr) const {
   if (!Base.isReg())
     return;
 
@@ -1314,7 +1373,7 @@ void SILoadStoreOptimizer::processBaseWithConstOffset(const MachineOperand &Base
 bool SILoadStoreOptimizer::promoteConstantOffsetToImm(
     MachineInstr &MI,
     MemInfoMap &Visited,
-    SmallPtrSet<MachineInstr *, 4> &AnchorList) {
+    SmallPtrSet<MachineInstr *, 4> &AnchorList) const {
 
   if (!(MI.mayLoad() ^ MI.mayStore()))
     return false;
@@ -1484,19 +1543,13 @@ bool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
       continue;
     }
 
-    const unsigned Opc = MI.getOpcode();
-
     CombineInfo CI;
-    CI.I = I;
-    CI.InstClass = getInstClass(Opc);
+    CI.setMI(I, *TII, *STM);
 
     switch (CI.InstClass) {
     default:
       break;
     case DS_READ:
-      CI.EltSize =
-          (Opc == AMDGPU::DS_READ_B64 || Opc == AMDGPU::DS_READ_B64_gfx9) ? 8
-                                                                          : 4;
       if (findMatchingInst(CI)) {
         Modified = true;
         I = mergeRead2Pair(CI);
@@ -1505,9 +1558,6 @@ bool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
       }
       continue;
     case DS_WRITE:
-      CI.EltSize =
-          (Opc == AMDGPU::DS_WRITE_B64 || Opc == AMDGPU::DS_WRITE_B64_gfx9) ? 8
-                                                                            : 4;
       if (findMatchingInst(CI)) {
         Modified = true;
         I = mergeWrite2Pair(CI);
@@ -1516,7 +1566,6 @@ bool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
       }
       continue;
     case S_BUFFER_LOAD_IMM:
-      CI.EltSize = AMDGPU::getSMRDEncodedOffset(*STM, 4);
       if (findMatchingInst(CI)) {
         Modified = true;
         I = mergeSBufferLoadImmPair(CI);
@@ -1529,7 +1578,6 @@ bool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
     case BUFFER_LOAD_OFFSET:
     case BUFFER_LOAD_OFFEN_exact:
     case BUFFER_LOAD_OFFSET_exact:
-      CI.EltSize = 4;
       if (findMatchingInst(CI)) {
         Modified = true;
         I = mergeBufferLoadPair(CI);
@@ -1542,7 +1590,6 @@ bool SILoadStoreOptimizer::optimizeBlock(MachineBasicBlock &MBB) {
     case BUFFER_STORE_OFFSET:
     case BUFFER_STORE_OFFEN_exact:
     case BUFFER_STORE_OFFSET_exact:
-      CI.EltSize = 4;
       if (findMatchingInst(CI)) {
         Modified = true;
         I = mergeBufferStorePair(CI);
diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
index 1a51b7ebffa1a..6f9abd3a8d9b9 100644
--- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp
@@ -98,6 +98,8 @@ class SILowerControlFlow : public MachineFunctionPass {
   void emitLoop(MachineInstr &MI);
   void emitEndCf(MachineInstr &MI);
 
+  Register getSaveExec(MachineInstr* MI);
+
   void findMaskOperands(MachineInstr &MI, unsigned OpNo,
                         SmallVectorImpl<MachineOperand> &Src) const;
 
@@ -175,17 +177,31 @@ static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI,
   return true;
 }
 
+Register SILowerControlFlow::getSaveExec(MachineInstr *MI) {
+  MachineBasicBlock *MBB = MI->getParent();
+  MachineOperand &SaveExec = MI->getOperand(0);
+  assert(SaveExec.getSubReg() == AMDGPU::NoSubRegister);
+
+  Register SaveExecReg = SaveExec.getReg();
+  unsigned FalseTermOpc =
+      TII->isWave32() ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term;
+  MachineBasicBlock::iterator I = (MI);
+  MachineBasicBlock::iterator J = std::next(I);
+  if (J != MBB->end() && J->getOpcode() == FalseTermOpc &&
+      J->getOperand(1).isReg() && J->getOperand(1).getReg() == SaveExecReg) {
+    SaveExecReg = J->getOperand(0).getReg();
+    J->eraseFromParent();
+  }
+  return SaveExecReg;
+}
+
 void SILowerControlFlow::emitIf(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   const DebugLoc &DL = MI.getDebugLoc();
   MachineBasicBlock::iterator I(&MI);
-
-  MachineOperand &SaveExec = MI.getOperand(0);
-  MachineOperand &Cond = MI.getOperand(1);
-  assert(SaveExec.getSubReg() == AMDGPU::NoSubRegister &&
-         Cond.getSubReg() == AMDGPU::NoSubRegister);
-
-  Register SaveExecReg = SaveExec.getReg();
+  Register SaveExecReg = getSaveExec(&MI);
+  MachineOperand& Cond = MI.getOperand(1);
+  assert(Cond.getSubReg() == AMDGPU::NoSubRegister);
 
   MachineOperand &ImpDefSCC = MI.getOperand(4);
   assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
@@ -266,8 +282,7 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   const DebugLoc &DL = MI.getDebugLoc();
 
-  Register DstReg = MI.getOperand(0).getReg();
-  assert(MI.getOperand(0).getSubReg() == AMDGPU::NoSubRegister);
+  Register DstReg = getSaveExec(&MI);
 
   bool ExecModified = MI.getOperand(3).getImm() != 0;
   MachineBasicBlock::iterator Start = MBB.begin();
@@ -339,7 +354,7 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) {
 void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
   const DebugLoc &DL = MI.getDebugLoc();
-  auto Dst = MI.getOperand(0).getReg();
+  auto Dst = getSaveExec(&MI);
 
   // Skip ANDing with exec if the break condition is already masked by exec
   // because it is a V_CMP in the same basic block. (We know the break
@@ -400,13 +415,17 @@ void SILowerControlFlow::emitLoop(MachineInstr &MI) {
 
 void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
   MachineBasicBlock &MBB = *MI.getParent();
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+  unsigned CFMask = MI.getOperand(0).getReg();
+  MachineInstr *Def = MRI.getUniqueVRegDef(CFMask);
   const DebugLoc &DL = MI.getDebugLoc();
 
-  MachineBasicBlock::iterator InsPt = MBB.begin();
-  MachineInstr *NewMI =
-      BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec)
-          .addReg(Exec)
-          .add(MI.getOperand(0));
+  MachineBasicBlock::iterator InsPt =
+      Def && Def->getParent() == &MBB ? std::next(MachineBasicBlock::iterator(Def))
+                               : MBB.begin();
+  MachineInstr *NewMI = BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec)
+                            .addReg(Exec)
+                            .add(MI.getOperand(0));
 
   if (LIS)
     LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
index 86242f1fb6458..b454125363562 100644
--- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
+++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp
@@ -589,12 +589,12 @@ void SILowerI1Copies::lowerPhis() {
 
       // Phis in a loop that are observed outside the loop receive a simple but
       // conservatively correct treatment.
-      MachineBasicBlock *PostDomBound = &MBB;
-      for (MachineInstr &Use : MRI->use_instructions(DstReg)) {
-        PostDomBound =
-            PDT->findNearestCommonDominator(PostDomBound, Use.getParent());
-      }
+      std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
+      for (MachineInstr &Use : MRI->use_instructions(DstReg))
+        DomBlocks.push_back(Use.getParent());
 
+      MachineBasicBlock *PostDomBound =
+          PDT->findNearestCommonDominator(DomBlocks);
       unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
 
       SSAUpdater.Initialize(DstReg);
@@ -711,12 +711,12 @@ void SILowerI1Copies::lowerCopiesToI1() {
 
       // Defs in a loop that are observed outside the loop must be transformed
       // into appropriate bit manipulation.
-      MachineBasicBlock *PostDomBound = &MBB;
-      for (MachineInstr &Use : MRI->use_instructions(DstReg)) {
-        PostDomBound =
-            PDT->findNearestCommonDominator(PostDomBound, Use.getParent());
-      }
+      std::vector<MachineBasicBlock *> DomBlocks = {&MBB};
+      for (MachineInstr &Use : MRI->use_instructions(DstReg))
+        DomBlocks.push_back(Use.getParent());
 
+      MachineBasicBlock *PostDomBound =
+          PDT->findNearestCommonDominator(DomBlocks);
       unsigned FoundLoopLevel = LF.findLoop(PostDomBound);
       if (FoundLoopLevel) {
         SSAUpdater.Initialize(DstReg);
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index 94ebe693feb28..7c039a54b57f0 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -41,6 +41,8 @@ struct SIProgramInfo {
     uint64_t ComputePGMRSrc2 = 0;
 
     uint32_t NumVGPR = 0;
+    uint32_t NumArchVGPR = 0;
+    uint32_t NumAccVGPR = 0;
     uint32_t NumSGPR = 0;
     uint32_t LDSSize = 0;
     bool FlatUsed = false;
diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
index 7967d9c9fb92a..f4dd995316dde 100644
--- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp
@@ -416,9 +416,8 @@ void SIRegisterInfo::resolveFrameIndex(MachineInstr &MI, unsigned BaseReg,
   assert(FIOp && FIOp->isFI() && "frame index must be address operand");
   assert(TII->isMUBUF(MI));
   assert(TII->getNamedOperand(MI, AMDGPU::OpName::soffset)->getReg() ==
-         MF->getInfo<SIMachineFunctionInfo>()->getFrameOffsetReg() &&
-         "should only be seeing frame offset relative FrameIndex");
-
+         MF->getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg() &&
+         "should only be seeing stack pointer offset relative FrameIndex");
 
   MachineOperand *OffsetOp = TII->getNamedOperand(MI, AMDGPU::OpName::offset);
   int64_t NewOffset = OffsetOp->getImm() + Offset;
@@ -618,6 +617,7 @@ static bool buildMUBUFOffsetLoadStore(const SIInstrInfo *TII,
           .addImm(0) // slc
           .addImm(0) // tfe
           .addImm(0) // dlc
+          .addImm(0) // swz
           .cloneMemRefs(*MI);
 
   const MachineOperand *VDataIn = TII->getNamedOperand(*MI,
@@ -738,6 +738,7 @@ void SIRegisterInfo::buildSpillLoadStore(MachineBasicBlock::iterator MI,
         .addImm(0) // slc
         .addImm(0) // tfe
         .addImm(0) // dlc
+        .addImm(0) // swz
         .addMemOperand(NewMMO);
 
       if (!IsStore && TmpReg != AMDGPU::NoRegister)
@@ -1975,6 +1976,9 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size,
   case 512:
     return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_512RegClass :
                                                  &AMDGPU::SReg_512RegClass;
+  case 1024:
+    return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_1024RegClass :
+                                                 &AMDGPU::SReg_1024RegClass;
   default:
     if (Size < 32)
       return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VGPR_32RegClass :
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 1b410b6b59124..1a74ebbf81651 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -793,9 +793,18 @@ multiclass SMLoad_Pattern <string Instr, ValueType vt> {
 // selector to prefer those.
 let AddedComplexity = 100 in {
 
-defm : SMRD_Pattern <"S_LOAD_DWORD",    i32>;
-defm : SMRD_Pattern <"S_LOAD_DWORDX2",  v2i32>;
-defm : SMRD_Pattern <"S_LOAD_DWORDX4",  v4i32>;
+foreach vt = Reg32Types.types in {
+defm : SMRD_Pattern <"S_LOAD_DWORD", vt>;
+}
+
+foreach vt = SReg_64.RegTypes in {
+defm : SMRD_Pattern <"S_LOAD_DWORDX2", vt>;
+}
+
+foreach vt = SReg_128.RegTypes in {
+defm : SMRD_Pattern <"S_LOAD_DWORDX4", vt>;
+}
+
 defm : SMRD_Pattern <"S_LOAD_DWORDX8",  v8i32>;
 defm : SMRD_Pattern <"S_LOAD_DWORDX16", v16i32>;
 
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 0eb01434deb26..d31a49f428ee7 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -419,16 +419,16 @@ def S_SUBB_U32 : SOP2_32 <"s_subb_u32",
 
 let isCommutable = 1 in {
 def S_MIN_I32 : SOP2_32 <"s_min_i32",
-  [(set i32:$sdst, (UniformBinFrag<smin> i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (smin i32:$src0, i32:$src1))]
 >;
 def S_MIN_U32 : SOP2_32 <"s_min_u32",
-  [(set i32:$sdst, (UniformBinFrag<umin> i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (umin i32:$src0, i32:$src1))]
 >;
 def S_MAX_I32 : SOP2_32 <"s_max_i32",
-  [(set i32:$sdst, (UniformBinFrag<smax> i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (smax i32:$src0, i32:$src1))]
 >;
 def S_MAX_U32 : SOP2_32 <"s_max_u32",
-  [(set i32:$sdst, (UniformBinFrag<umax> i32:$src0, i32:$src1))]
+  [(set i32:$sdst, (umax i32:$src0, i32:$src1))]
 >;
 } // End isCommutable = 1
 } // End Defs = [SCC]
@@ -1090,7 +1090,7 @@ def S_WAKEUP : SOPP <0x00000003, (ins), "s_wakeup"> {
 
 let mayLoad = 1, mayStore = 1, hasSideEffects = 1 in
 def S_WAITCNT : SOPP <0x0000000c, (ins WAIT_FLAG:$simm16), "s_waitcnt $simm16",
-    [(int_amdgcn_s_waitcnt UIMM16bit:$simm16)]>;
+    [(int_amdgcn_s_waitcnt timm:$simm16)]>;
 def S_SETHALT : SOPP <0x0000000d, (ins i16imm:$simm16), "s_sethalt $simm16">;
 def S_SETKILL : SOPP <0x0000000b, (ins i16imm:$simm16), "s_setkill $simm16">;
 
@@ -1099,7 +1099,7 @@ def S_SETKILL : SOPP <0x0000000b, (ins i16imm:$simm16), "s_setkill $simm16">;
 // maximum reported is 960 cycles, so 960 / 64 = 15 max, so is the
 // maximum really 15 on VI?
 def S_SLEEP : SOPP <0x0000000e, (ins i32imm:$simm16),
-  "s_sleep $simm16", [(int_amdgcn_s_sleep SIMM16bit:$simm16)]> {
+  "s_sleep $simm16", [(int_amdgcn_s_sleep timm:$simm16)]> {
   let hasSideEffects = 1;
   let mayLoad = 1;
   let mayStore = 1;
@@ -1110,10 +1110,10 @@ def S_SETPRIO : SOPP <0x0000000f, (ins i16imm:$simm16), "s_setprio $simm16">;
 let Uses = [EXEC, M0] in {
 // FIXME: Should this be mayLoad+mayStore?
 def S_SENDMSG : SOPP <0x00000010, (ins SendMsgImm:$simm16), "s_sendmsg $simm16",
-  [(int_amdgcn_s_sendmsg (i32 imm:$simm16), M0)]>;
+  [(int_amdgcn_s_sendmsg (i32 timm:$simm16), M0)]>;
 
 def S_SENDMSGHALT : SOPP <0x00000011, (ins SendMsgImm:$simm16), "s_sendmsghalt $simm16",
-  [(int_amdgcn_s_sendmsghalt (i32 imm:$simm16), M0)]>;
+  [(int_amdgcn_s_sendmsghalt (i32 timm:$simm16), M0)]>;
 
 } // End Uses = [EXEC, M0]
 
@@ -1125,13 +1125,13 @@ def S_ICACHE_INV : SOPP <0x00000013, (ins), "s_icache_inv"> {
   let simm16 = 0;
 }
 def S_INCPERFLEVEL : SOPP <0x00000014, (ins i32imm:$simm16), "s_incperflevel $simm16",
-  [(int_amdgcn_s_incperflevel SIMM16bit:$simm16)]> {
+  [(int_amdgcn_s_incperflevel timm:$simm16)]> {
   let hasSideEffects = 1;
   let mayLoad = 1;
   let mayStore = 1;
 }
 def S_DECPERFLEVEL : SOPP <0x00000015, (ins i32imm:$simm16), "s_decperflevel $simm16",
-  [(int_amdgcn_s_decperflevel SIMM16bit:$simm16)]> {
+  [(int_amdgcn_s_decperflevel timm:$simm16)]> {
   let hasSideEffects = 1;
   let mayLoad = 1;
   let mayStore = 1;
@@ -1180,7 +1180,7 @@ let SubtargetPredicate = isGFX10Plus in {
 // S_GETREG_B32 Intrinsic Pattern.
 //===----------------------------------------------------------------------===//
 def : GCNPat <
-  (int_amdgcn_s_getreg imm:$simm16),
+  (int_amdgcn_s_getreg timm:$simm16),
   (S_GETREG_B32 (as_i16imm $simm16))
 >;
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index bb4169788f466..afb2fd987afdc 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -137,10 +137,51 @@ struct MUBUFInfo {
   bool has_soffset;
 };
 
+struct MTBUFInfo {
+  uint16_t Opcode;
+  uint16_t BaseOpcode;
+  uint8_t elements;
+  bool has_vaddr;
+  bool has_srsrc;
+  bool has_soffset;
+};
+
+#define GET_MTBUFInfoTable_DECL
+#define GET_MTBUFInfoTable_IMPL
 #define GET_MUBUFInfoTable_DECL
 #define GET_MUBUFInfoTable_IMPL
 #include "AMDGPUGenSearchableTables.inc"
 
+int getMTBUFBaseOpcode(unsigned Opc) {
+  const MTBUFInfo *Info = getMTBUFInfoFromOpcode(Opc);
+  return Info ? Info->BaseOpcode : -1;
+}
+
+int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements) {
+  const MTBUFInfo *Info = getMTBUFInfoFromBaseOpcodeAndElements(BaseOpc, Elements);
+  return Info ? Info->Opcode : -1;
+}
+
+int getMTBUFElements(unsigned Opc) {
+  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
+  return Info ? Info->elements : 0;
+}
+
+bool getMTBUFHasVAddr(unsigned Opc) {
+  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
+  return Info ? Info->has_vaddr : false;
+}
+
+bool getMTBUFHasSrsrc(unsigned Opc) {
+  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
+  return Info ? Info->has_srsrc : false;
+}
+
+bool getMTBUFHasSoffset(unsigned Opc) {
+  const MTBUFInfo *Info = getMTBUFOpcodeHelper(Opc);
+  return Info ? Info->has_soffset : false;
+}
+
 int getMUBUFBaseOpcode(unsigned Opc) {
   const MUBUFInfo *Info = getMUBUFInfoFromOpcode(Opc);
   return Info ? Info->BaseOpcode : -1;
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index a578fd2bb6a9a..f78dadd447ff5 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -263,6 +263,24 @@ struct MIMGInfo {
 LLVM_READONLY
 const MIMGInfo *getMIMGInfo(unsigned Opc);
 
+LLVM_READONLY
+int getMTBUFBaseOpcode(unsigned Opc);
+
+LLVM_READONLY
+int getMTBUFOpcode(unsigned BaseOpc, unsigned Elements);
+
+LLVM_READONLY
+int getMTBUFElements(unsigned Opc);
+
+LLVM_READONLY
+bool getMTBUFHasVAddr(unsigned Opc);
+
+LLVM_READONLY
+bool getMTBUFHasSrsrc(unsigned Opc);
+
+LLVM_READONLY
+bool getMTBUFHasSoffset(unsigned Opc);
+
 LLVM_READONLY
 int getMUBUFBaseOpcode(unsigned Opc);
 
diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 76abda9218f94..bea0c7bd080d2 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -841,16 +841,16 @@ def V_MOVRELD_B32_V16 : V_MOVRELD_B32_pseudo<VReg_512>;
 let OtherPredicates = [isGFX8GFX9] in {
 
 def : GCNPat <
-  (i32 (int_amdgcn_mov_dpp i32:$src, imm:$dpp_ctrl, imm:$row_mask, imm:$bank_mask,
-                      imm:$bound_ctrl)),
+  (i32 (int_amdgcn_mov_dpp i32:$src, timm:$dpp_ctrl, timm:$row_mask, timm:$bank_mask,
+                      timm:$bound_ctrl)),
   (V_MOV_B32_dpp $src, $src, (as_i32imm $dpp_ctrl),
                        (as_i32imm $row_mask), (as_i32imm $bank_mask),
                        (as_i1imm $bound_ctrl))
 >;
 
 def : GCNPat <
-  (i32 (int_amdgcn_update_dpp i32:$old, i32:$src, imm:$dpp_ctrl, imm:$row_mask,
-                      imm:$bank_mask, imm:$bound_ctrl)),
+  (i32 (int_amdgcn_update_dpp i32:$old, i32:$src, timm:$dpp_ctrl, timm:$row_mask,
+                      timm:$bank_mask, timm:$bound_ctrl)),
   (V_MOV_B32_dpp $old, $src, (as_i32imm $dpp_ctrl),
                        (as_i32imm $row_mask), (as_i32imm $bank_mask),
                        (as_i1imm $bound_ctrl))
@@ -911,21 +911,21 @@ defm V_SCREEN_PARTITION_4SE_B32 : VOP1_Real_gfx9 <0x37>;
 
 let OtherPredicates = [isGFX10Plus] in {
 def : GCNPat <
-  (i32 (int_amdgcn_mov_dpp8 i32:$src, imm:$dpp8)),
+  (i32 (int_amdgcn_mov_dpp8 i32:$src, timm:$dpp8)),
   (V_MOV_B32_dpp8_gfx10 $src, $src, (as_i32imm $dpp8), (i32 DPP8Mode.FI_0))
 >;
 
 def : GCNPat <
-  (i32 (int_amdgcn_mov_dpp i32:$src, imm:$dpp_ctrl, imm:$row_mask, imm:$bank_mask,
-                      imm:$bound_ctrl)),
+  (i32 (int_amdgcn_mov_dpp i32:$src, timm:$dpp_ctrl, timm:$row_mask, timm:$bank_mask,
+                      timm:$bound_ctrl)),
   (V_MOV_B32_dpp_gfx10 $src, $src, (as_i32imm $dpp_ctrl),
                        (as_i32imm $row_mask), (as_i32imm $bank_mask),
                        (as_i1imm $bound_ctrl), (i32 0))
 >;
 
 def : GCNPat <
-  (i32 (int_amdgcn_update_dpp i32:$old, i32:$src, imm:$dpp_ctrl, imm:$row_mask,
-                              imm:$bank_mask, imm:$bound_ctrl)),
+  (i32 (int_amdgcn_update_dpp i32:$old, i32:$src, timm:$dpp_ctrl, timm:$row_mask,
+                              timm:$bank_mask, timm:$bound_ctrl)),
   (V_MOV_B32_dpp_gfx10 $old, $src, (as_i32imm $dpp_ctrl),
                        (as_i32imm $row_mask), (as_i32imm $bank_mask),
                        (as_i1imm $bound_ctrl), (i32 0))
diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index e5ca22c87f9fc..e5f215fbcd719 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -251,7 +251,9 @@ multiclass VOP2eInstAliases<VOP2_Pseudo ps, VOP2_Real inst> {
 
 class VOP_MADAK <ValueType vt> : VOPProfile <[vt, vt, vt, vt]> {
   field Operand ImmOpType = !if(!eq(vt.Size, 32), f32kimm, f16kimm);
-  field dag Ins32 = (ins VCSrc_f32:$src0, VGPR_32:$src1, ImmOpType:$imm);
+  field dag Ins32 = !if(!eq(vt.Size, 32),
+                        (ins VCSrc_f32:$src0, VGPR_32:$src1, ImmOpType:$imm),
+                        (ins VCSrc_f16:$src0, VGPR_32:$src1, ImmOpType:$imm));
   field bit HasExt = 0;
 
   // Hack to stop printing _e64
@@ -653,7 +655,8 @@ defm V_FMAC_F32 : VOP2Inst <"v_fmac_f32", VOP_MAC_F32>;
 let Constraints = "$vdst = $src2",
       DisableEncoding="$src2",
       isConvertibleToThreeAddress = 1,
-      isCommutable = 1 in {
+      isCommutable = 1,
+      IsDOT = 1 in {
   let SubtargetPredicate = HasDot5Insts in
     defm V_DOT2C_F32_F16 : VOP2Inst_e32<"v_dot2c_f32_f16", VOP_DOT_ACC_F32_V2F16>;
   let SubtargetPredicate = HasDot6Insts in
@@ -774,6 +777,23 @@ class ZExt_i16_i1_Pat <SDNode ext> : GCNPat <
                      $src)
 >;
 
+foreach vt = [i16, v2i16] in {
+def : GCNPat <
+  (and vt:$src0, vt:$src1),
+  (V_AND_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
+>;
+
+def : GCNPat <
+  (or vt:$src0, vt:$src1),
+  (V_OR_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
+>;
+
+def : GCNPat <
+  (xor vt:$src0, vt:$src1),
+  (V_XOR_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
+>;
+}
+
 let Predicates = [Has16BitInsts] in {
 
 let Predicates = [Has16BitInsts, isGFX7GFX8GFX9] in {
@@ -796,21 +816,6 @@ defm : Arithmetic_i16_Pats<umin, V_MIN_U16_e64,   1>;
 defm : Arithmetic_i16_Pats<umax, V_MAX_U16_e64,   1>;
 }
 
-def : GCNPat <
-  (and i16:$src0, i16:$src1),
-  (V_AND_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
->;
-
-def : GCNPat <
-  (or i16:$src0, i16:$src1),
-  (V_OR_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
->;
-
-def : GCNPat <
-  (xor i16:$src0, i16:$src1),
-  (V_XOR_B32_e64 VSrc_b32:$src0, VSrc_b32:$src1)
->;
-
 let Predicates = [Has16BitInsts, isGFX7GFX8GFX9] in {
 defm : Bits_OpsRev_i16_Pats<shl, V_LSHLREV_B16_e64>;
 defm : Bits_OpsRev_i16_Pats<srl, V_LSHRREV_B16_e64>;
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 753f63d3a74d6..605425972b1c9 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -112,7 +112,7 @@ class getVOP3ClampPat<VOPProfile P, SDPatternOperator node> {
 
 class getVOP3MAIPat<VOPProfile P, SDPatternOperator node> {
   list<dag> ret = [(set P.DstVT:$vdst, (node P.Src0VT:$src0, P.Src1VT:$src1, P.Src2VT:$src2,
-                                        imm:$cbsz, imm:$abid, imm:$blgp))];
+                                        timm:$cbsz, timm:$abid, timm:$blgp))];
 }
 
 class VOP3Inst<string OpName, VOPProfile P, SDPatternOperator node = null_frag, bit VOP3Only = 0> :
@@ -453,13 +453,13 @@ let FPDPRounding = 1 in {
 def V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fmad>;
 let Uses = [M0, EXEC] in {
 def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>,
-       [(set f16:$vdst, (AMDGPUinterp_p2_f16 f32:$src0, (i32 imm:$attrchan),
-                                                        (i32 imm:$attr),
-                                                        (i32 imm:$src0_modifiers),
+       [(set f16:$vdst, (AMDGPUinterp_p2_f16 f32:$src0, (i32 timm:$attrchan),
+                                                        (i32 timm:$attr),
+                                                        (i32 timm:$src0_modifiers),
                                                         (f32 VRegSrc_32:$src2),
-                                                        (i32 imm:$src2_modifiers),
-                                                        (i1 imm:$high),
-                                                        (i1 imm:$clamp)))]>;
+                                                        (i32 timm:$src2_modifiers),
+                                                        (i1 timm:$high),
+                                                        (i1 timm:$clamp)))]>;
 } // End Uses = [M0, EXEC]
 } // End FPDPRounding = 1
 } // End renamedInGFX9 = 1
@@ -478,21 +478,21 @@ def V_INTERP_P2_F16_gfx9 : VOP3Interp <"v_interp_p2_f16_gfx9", VOP3_INTERP16<[f1
 
 let Uses = [M0, EXEC], FPDPRounding = 1 in {
 def V_INTERP_P1LL_F16 : VOP3Interp <"v_interp_p1ll_f16", VOP3_INTERP16<[f32, f32, i32, untyped]>,
-       [(set f32:$vdst, (AMDGPUinterp_p1ll_f16 f32:$src0, (i32 imm:$attrchan),
-                                                          (i32 imm:$attr),
-                                                          (i32 imm:$src0_modifiers),
-                                                          (i1 imm:$high),
-                                                          (i1 imm:$clamp),
-                                                          (i32 imm:$omod)))]>;
+       [(set f32:$vdst, (AMDGPUinterp_p1ll_f16 f32:$src0, (i32 timm:$attrchan),
+                                                          (i32 timm:$attr),
+                                                          (i32 timm:$src0_modifiers),
+                                                          (i1 timm:$high),
+                                                          (i1 timm:$clamp),
+                                                          (i32 timm:$omod)))]>;
 def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32, i32, f16]>,
-       [(set f32:$vdst, (AMDGPUinterp_p1lv_f16 f32:$src0, (i32 imm:$attrchan),
-                                                          (i32 imm:$attr),
-                                                          (i32 imm:$src0_modifiers),
+       [(set f32:$vdst, (AMDGPUinterp_p1lv_f16 f32:$src0, (i32 timm:$attrchan),
+                                                          (i32 timm:$attr),
+                                                          (i32 timm:$src0_modifiers),
                                                           (f32 VRegSrc_32:$src2),
-                                                          (i32 imm:$src2_modifiers),
-                                                          (i1 imm:$high),
-                                                          (i1 imm:$clamp),
-                                                          (i32 imm:$omod)))]>;
+                                                          (i32 timm:$src2_modifiers),
+                                                          (i1 timm:$high),
+                                                          (i1 timm:$clamp),
+                                                          (i32 timm:$omod)))]>;
 } // End Uses = [M0, EXEC], FPDPRounding = 1
 
 } // End SubtargetPredicate = Has16BitInsts, isCommutable = 1
@@ -642,11 +642,11 @@ let SubtargetPredicate = isGFX10Plus in {
   } // End $vdst = $vdst_in, DisableEncoding $vdst_in
 
   def : GCNPat<
-    (int_amdgcn_permlane16 i32:$vdst_in, i32:$src0, i32:$src1, i32:$src2, imm:$fi, imm:$bc),
+    (int_amdgcn_permlane16 i32:$vdst_in, i32:$src0, i32:$src1, i32:$src2, timm:$fi, timm:$bc),
     (V_PERMLANE16_B32 (as_i1imm $fi), $src0, (as_i1imm $bc), $src1, 0, $src2, $vdst_in)
   >;
   def : GCNPat<
-    (int_amdgcn_permlanex16 i32:$vdst_in, i32:$src0, i32:$src1, i32:$src2, imm:$fi, imm:$bc),
+    (int_amdgcn_permlanex16 i32:$vdst_in, i32:$src0, i32:$src1, i32:$src2, timm:$fi, timm:$bc),
     (V_PERMLANEX16_B32 (as_i1imm $fi), $src0, (as_i1imm $bc), $src1, 0, $src2, $vdst_in)
   >;
 } // End SubtargetPredicate = isGFX10Plus
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index 55ee5f6577cff..0c13f39fec029 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -261,6 +261,7 @@ class SDot2Pat<Instruction Inst> : GCNPat <
   let SubtargetPredicate = !cast<VOP_Pseudo>(Inst).SubtargetPredicate;
 }
 
+let IsDOT = 1 in {
 let SubtargetPredicate = HasDot2Insts in {
 
 def V_DOT2_F32_F16 : VOP3PInst<"v_dot2_f32_f16", VOP3_Profile<VOP_F32_V2F16_V2F16_F32>>;
@@ -277,6 +278,7 @@ def V_DOT4_I32_I8  : VOP3PInst<"v_dot4_i32_i8", VOP3_Profile<VOP_I32_I32_I32_I32
 def V_DOT8_I32_I4  : VOP3PInst<"v_dot8_i32_i4", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_PACKED>>;
 
 } // End SubtargetPredicate = HasDot1Insts
+} // End let IsDOT = 1
 
 multiclass DotPats<SDPatternOperator dot_op,
                    VOP3PInst dot_inst> {
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index c2623c23251fa..f208a1134a5a4 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -147,6 +147,7 @@ class VOP3_Real <VOP_Pseudo ps, int EncodingFamily> :
 
   // copy relevant pseudo op flags
   let SubtargetPredicate = ps.SubtargetPredicate;
+  let OtherPredicates    = ps.OtherPredicates;
   let AsmMatchConverter  = ps.AsmMatchConverter;
   let AsmVariantName     = ps.AsmVariantName;
   let Constraints        = ps.Constraints;
diff --git a/llvm/lib/Target/ARC/ARCMachineFunctionInfo.h b/llvm/lib/Target/ARC/ARCMachineFunctionInfo.h
index 997327fd1b84e..d4dcf9bf285c6 100644
--- a/llvm/lib/Target/ARC/ARCMachineFunctionInfo.h
+++ b/llvm/lib/Target/ARC/ARCMachineFunctionInfo.h
@@ -35,7 +35,7 @@ class ARCFunctionInfo : public MachineFunctionInfo {
       : ReturnStackOffsetSet(false), VarArgsFrameIndex(0),
         ReturnStackOffset(-1U), MaxCallStackReq(0) {
     // Functions are 4-byte aligned.
-    MF.setAlignment(llvm::Align(4));
+    MF.setAlignment(Align(4));
   }
 
   ~ARCFunctionInfo() {}
diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td
index 6616ae5d16064..fed4cb2b93163 100644
--- a/llvm/lib/Target/ARM/ARM.td
+++ b/llvm/lib/Target/ARM/ARM.td
@@ -57,12 +57,15 @@ def FeatureD32            : SubtargetFeature<"d32", "HasD32", "true",
                                              "Extend FP to 32 double registers">;
 
 multiclass VFPver<string name, string query, string description,
-                  list<SubtargetFeature> prev = [],
-                  list<SubtargetFeature> otherimplies = []> {
+                  list<SubtargetFeature> prev,
+                  list<SubtargetFeature> otherimplies,
+                  list<SubtargetFeature> vfp2prev = []> {
   def _D16_SP: SubtargetFeature<
     name#"d16sp", query#"D16SP", "true",
     description#" with only 16 d-registers and no double precision",
-    !foreach(v, prev, !cast<SubtargetFeature>(v # "_D16_SP")) # otherimplies>;
+    !foreach(v, prev, !cast<SubtargetFeature>(v # "_D16_SP")) #
+      !foreach(v, vfp2prev, !cast<SubtargetFeature>(v # "_SP")) #
+      otherimplies>;
   def _SP: SubtargetFeature<
     name#"sp", query#"SP", "true",
     description#" with no double precision",
@@ -72,6 +75,7 @@ multiclass VFPver<string name, string query, string description,
     name#"d16", query#"D16", "true",
     description#" with only 16 d-registers",
     !foreach(v, prev, !cast<SubtargetFeature>(v # "_D16")) #
+      vfp2prev #
       otherimplies # [FeatureFP64, !cast<SubtargetFeature>(NAME # "_D16_SP")]>;
   def "": SubtargetFeature<
     name, query, "true", description,
@@ -80,11 +84,17 @@ multiclass VFPver<string name, string query, string description,
         !cast<SubtargetFeature>(NAME # "_SP")]>;
 }
 
-defm FeatureVFP2: VFPver<"vfp2", "HasVFPv2", "Enable VFP2 instructions",
-                         [], [FeatureFPRegs]>;
+def FeatureVFP2_SP        : SubtargetFeature<"vfp2sp", "HasVFPv2SP", "true",
+                                             "Enable VFP2 instructions with "
+                                             "no double precision",
+                                             [FeatureFPRegs]>;
+
+def FeatureVFP2           : SubtargetFeature<"vfp2", "HasVFPv2", "true",
+                                             "Enable VFP2 instructions",
+                                             [FeatureFP64, FeatureVFP2_SP]>;
 
 defm FeatureVFP3: VFPver<"vfp3", "HasVFPv3", "Enable VFP3 instructions",
-                         [FeatureVFP2]>;
+                         [], [], [FeatureVFP2]>;
 
 def FeatureNEON           : SubtargetFeature<"neon", "HasNEON", "true",
                                              "Enable NEON instructions",
@@ -98,7 +108,7 @@ defm FeatureVFP4: VFPver<"vfp4", "HasVFPv4", "Enable VFP4 instructions",
                          [FeatureVFP3], [FeatureFP16]>;
 
 defm FeatureFPARMv8: VFPver<"fp-armv8", "HasFPARMv8", "Enable ARMv8 FP",
-                         [FeatureVFP4]>;
+                         [FeatureVFP4], []>;
 
 def FeatureFullFP16       : SubtargetFeature<"fullfp16", "HasFullFP16", "true",
                                              "Enable full half-precision "
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 9fcdb2fb75a08..c8c91e53c44e1 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -168,7 +168,7 @@ bool ARMAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   // relatively easy to exceed the thumb branch range within a TU.
   if (! ThumbIndirectPads.empty()) {
     OutStreamer->EmitAssemblerFlag(MCAF_Code16);
-    EmitAlignment(llvm::Align(2));
+    EmitAlignment(Align(2));
     for (std::pair<unsigned, MCSymbol *> &TIP : ThumbIndirectPads) {
       OutStreamer->EmitLabel(TIP.second);
       EmitToStreamer(*OutStreamer, MCInstBuilder(ARM::tBX)
@@ -526,7 +526,7 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
     if (!Stubs.empty()) {
       // Switch with ".non_lazy_symbol_pointer" directive.
       OutStreamer->SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
-      EmitAlignment(llvm::Align(4));
+      EmitAlignment(Align(4));
 
       for (auto &Stub : Stubs)
         emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
@@ -539,7 +539,7 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
     if (!Stubs.empty()) {
       // Switch with ".non_lazy_symbol_pointer" directive.
       OutStreamer->SwitchSection(TLOFMacho.getThreadLocalPointerSection());
-      EmitAlignment(llvm::Align(4));
+      EmitAlignment(Align(4));
 
       for (auto &Stub : Stubs)
         emitNonLazySymbolPointer(*OutStreamer, Stub.first, Stub.second);
@@ -940,7 +940,7 @@ void ARMAsmPrinter::EmitJumpTableAddrs(const MachineInstr *MI) {
 
   // Make sure the Thumb jump table is 4-byte aligned. This will be a nop for
   // ARM mode tables.
-  EmitAlignment(llvm::Align(4));
+  EmitAlignment(Align(4));
 
   // Emit a label for the jump table.
   MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
@@ -986,7 +986,7 @@ void ARMAsmPrinter::EmitJumpTableInsts(const MachineInstr *MI) {
 
   // Make sure the Thumb jump table is 4-byte aligned. This will be a nop for
   // ARM mode tables.
-  EmitAlignment(llvm::Align(4));
+  EmitAlignment(Align(4));
 
   // Emit a label for the jump table.
   MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
@@ -1015,7 +1015,7 @@ void ARMAsmPrinter::EmitJumpTableTBInst(const MachineInstr *MI,
   unsigned JTI = MO1.getIndex();
 
   if (Subtarget->isThumb1Only())
-    EmitAlignment(llvm::Align(4));
+    EmitAlignment(Align(4));
 
   MCSymbol *JTISymbol = GetARMJTIPICJumpTableLabel(JTI);
   OutStreamer->EmitLabel(JTISymbol);
@@ -1058,7 +1058,7 @@ void ARMAsmPrinter::EmitJumpTableTBInst(const MachineInstr *MI,
   OutStreamer->EmitDataRegion(MCDR_DataRegionEnd);
 
   // Make sure the next instruction is 2-byte aligned.
-  EmitAlignment(llvm::Align(2));
+  EmitAlignment(Align(2));
 }
 
 void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 270fefed38055..de53cb3b59d97 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -4337,10 +4337,10 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   }
 
   const MCInstrDesc &UseMCID = get(UseNode->getMachineOpcode());
-  const MachineSDNode *DefMN = dyn_cast<MachineSDNode>(DefNode);
+  auto *DefMN = cast<MachineSDNode>(DefNode);
   unsigned DefAlign = !DefMN->memoperands_empty()
     ? (*DefMN->memoperands_begin())->getAlignment() : 0;
-  const MachineSDNode *UseMN = dyn_cast<MachineSDNode>(UseNode);
+  auto *UseMN = cast<MachineSDNode>(UseNode);
   unsigned UseAlign = !UseMN->memoperands_empty()
     ? (*UseMN->memoperands_begin())->getAlignment() : 0;
   int Latency = getOperandLatency(ItinData, DefMCID, DefIdx, DefAlign,
diff --git a/llvm/lib/Target/ARM/ARMBasicBlockInfo.cpp b/llvm/lib/Target/ARM/ARMBasicBlockInfo.cpp
index a8bdefa9193b1..2b34b1d854827 100644
--- a/llvm/lib/Target/ARM/ARMBasicBlockInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBasicBlockInfo.cpp
@@ -47,7 +47,7 @@ void ARMBasicBlockUtils::computeBlockSize(MachineBasicBlock *MBB) {
   BasicBlockInfo &BBI = BBInfo[MBB->getNumber()];
   BBI.Size = 0;
   BBI.Unalign = 0;
-  BBI.PostAlign = 0;
+  BBI.PostAlign = Align::None();
 
   for (MachineInstr &I : *MBB) {
     BBI.Size += TII->getInstSizeInBytes(I);
@@ -62,8 +62,8 @@ void ARMBasicBlockUtils::computeBlockSize(MachineBasicBlock *MBB) {
 
   // tBR_JTr contains a .align 2 directive.
   if (!MBB->empty() && MBB->back().getOpcode() == ARM::tBR_JTr) {
-    BBI.PostAlign = 2;
-    MBB->getParent()->ensureAlignment(llvm::Align(4));
+    BBI.PostAlign = Align(4);
+    MBB->getParent()->ensureAlignment(Align(4));
   }
 }
 
@@ -126,9 +126,9 @@ void ARMBasicBlockUtils::adjustBBOffsetsAfter(MachineBasicBlock *BB) {
   for(unsigned i = BBNum + 1, e = MF.getNumBlockIDs(); i < e; ++i) {
     // Get the offset and known bits at the end of the layout predecessor.
     // Include the alignment of the current block.
-    unsigned LogAlign = MF.getBlockNumbered(i)->getLogAlignment();
-    unsigned Offset = BBInfo[i - 1].postOffset(LogAlign);
-    unsigned KnownBits = BBInfo[i - 1].postKnownBits(LogAlign);
+    const Align Align = MF.getBlockNumbered(i)->getAlignment();
+    const unsigned Offset = BBInfo[i - 1].postOffset(Align);
+    const unsigned KnownBits = BBInfo[i - 1].postKnownBits(Align);
 
     // This is where block i begins.  Stop if the offset is already correct,
     // and we have updated 2 blocks.  This is the maximum number of blocks
diff --git a/llvm/lib/Target/ARM/ARMBasicBlockInfo.h b/llvm/lib/Target/ARM/ARMBasicBlockInfo.h
index 400bba351cecc..d0f4a02463b7c 100644
--- a/llvm/lib/Target/ARM/ARMBasicBlockInfo.h
+++ b/llvm/lib/Target/ARM/ARMBasicBlockInfo.h
@@ -27,11 +27,11 @@ using BBInfoVector = SmallVectorImpl<BasicBlockInfo>;
 /// unknown offset bits.  This does not include alignment padding caused by
 /// known offset bits.
 ///
-/// @param LogAlign log2(alignment)
+/// @param Alignment alignment
 /// @param KnownBits Number of known low offset bits.
-inline unsigned UnknownPadding(unsigned LogAlign, unsigned KnownBits) {
-  if (KnownBits < LogAlign)
-    return (1u << LogAlign) - (1u << KnownBits);
+inline unsigned UnknownPadding(Align Alignment, unsigned KnownBits) {
+  if (KnownBits < Log2(Alignment))
+    return Alignment.value() - (1ull << KnownBits);
   return 0;
 }
 
@@ -65,10 +65,9 @@ struct BasicBlockInfo {
   /// multiple of 1 << Unalign.
   uint8_t Unalign = 0;
 
-  /// PostAlign - When non-zero, the block terminator contains a .align
-  /// directive, so the end of the block is aligned to 1 << PostAlign
-  /// bytes.
-  uint8_t PostAlign = 0;
+  /// PostAlign - When > 1, the block terminator contains a .align
+  /// directive, so the end of the block is aligned to PostAlign bytes.
+  Align PostAlign;
 
   BasicBlockInfo() = default;
 
@@ -84,16 +83,16 @@ struct BasicBlockInfo {
     return Bits;
   }
 
-  /// Compute the offset immediately following this block.  If LogAlign is
+  /// Compute the offset immediately following this block.  If Align is
   /// specified, return the offset the successor block will get if it has
   /// this alignment.
-  unsigned postOffset(unsigned LogAlign = 0) const {
+  unsigned postOffset(Align Alignment = Align::None()) const {
     unsigned PO = Offset + Size;
-    unsigned LA = std::max(unsigned(PostAlign), LogAlign);
-    if (!LA)
+    const Align PA = std::max(PostAlign, Alignment);
+    if (PA == Align::None())
       return PO;
     // Add alignment padding from the terminator.
-    return PO + UnknownPadding(LA, internalKnownBits());
+    return PO + UnknownPadding(PA, internalKnownBits());
   }
 
   /// Compute the number of known low bits of postOffset.  If this block
@@ -101,9 +100,8 @@ struct BasicBlockInfo {
   /// instruction alignment.  An aligned terminator may increase the number
   /// of know bits.
   /// If LogAlign is given, also consider the alignment of the next block.
-  unsigned postKnownBits(unsigned LogAlign = 0) const {
-    return std::max(std::max(unsigned(PostAlign), LogAlign),
-                    internalKnownBits());
+  unsigned postKnownBits(Align Align = Align::None()) const {
+    return std::max(Log2(std::max(PostAlign, Align)), internalKnownBits());
   }
 };
 
diff --git a/llvm/lib/Target/ARM/ARMCallingConv.cpp b/llvm/lib/Target/ARM/ARMCallingConv.cpp
index 5ede7c67f7c2e..92ebc542b423b 100644
--- a/llvm/lib/Target/ARM/ARMCallingConv.cpp
+++ b/llvm/lib/Target/ARM/ARMCallingConv.cpp
@@ -193,7 +193,7 @@ static bool CC_ARM_AAPCS_Custom_Aggregate(unsigned &ValNo, MVT &ValVT,
   // Try to allocate a contiguous block of registers, each of the correct
   // size to hold one member.
   auto &DL = State.getMachineFunction().getDataLayout();
-  unsigned StackAlign = DL.getStackAlignment();
+  unsigned StackAlign = DL.getStackAlignment().value();
   unsigned Align = std::min(PendingMembers[0].getExtraInfo(), StackAlign);
 
   ArrayRef<MCPhysReg> RegList;
diff --git a/llvm/lib/Target/ARM/ARMCodeGenPrepare.cpp b/llvm/lib/Target/ARM/ARMCodeGenPrepare.cpp
index 9c8da29febf98..1c2c8aef55bb8 100644
--- a/llvm/lib/Target/ARM/ARMCodeGenPrepare.cpp
+++ b/llvm/lib/Target/ARM/ARMCodeGenPrepare.cpp
@@ -179,9 +179,6 @@ class ARMCodeGenPrepare : public FunctionPass {
 }
 
 static bool GenerateSignBits(Value *V) {
-  if (auto *Arg = dyn_cast<Argument>(V))
-    return Arg->hasSExtAttr();
-
   if (!isa<Instruction>(V))
     return false;
 
@@ -843,8 +840,8 @@ bool ARMCodeGenPrepare::isSupportedValue(Value *V) {
     }
   } else if (isa<Constant>(V) && !isa<ConstantExpr>(V)) {
     return isSupportedType(V);
-  } else if (auto *Arg = dyn_cast<Argument>(V))
-    return isSupportedType(V) && !Arg->hasSExtAttr();
+  } else if (isa<Argument>(V))
+    return isSupportedType(V);
 
   return isa<BasicBlock>(V);
 }
diff --git a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
index 548910e5f8c19..24ca25f73e96d 100644
--- a/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/llvm/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -29,6 +29,7 @@
 #include "llvm/CodeGen/LivePhysRegs.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
+#include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
@@ -70,6 +71,7 @@ STATISTIC(NumT2BrShrunk, "Number of Thumb2 immediate branches shrunk");
 STATISTIC(NumCBZ,        "Number of CBZ / CBNZ formed");
 STATISTIC(NumJTMoved,    "Number of jump table destination blocks moved");
 STATISTIC(NumJTInserted, "Number of jump table intermediate blocks inserted");
+STATISTIC(NumLEInserted, "Number of LE backwards branches inserted");
 
 static cl::opt<bool>
 AdjustJumpTableBlocks("arm-adjust-jump-tables", cl::Hidden, cl::init(true),
@@ -213,6 +215,7 @@ namespace {
     const ARMBaseInstrInfo *TII;
     const ARMSubtarget *STI;
     ARMFunctionInfo *AFI;
+    MachineDominatorTree *DT = nullptr;
     bool isThumb;
     bool isThumb1;
     bool isThumb2;
@@ -225,6 +228,11 @@ namespace {
 
     bool runOnMachineFunction(MachineFunction &MF) override;
 
+    void getAnalysisUsage(AnalysisUsage &AU) const override {
+      AU.addRequired<MachineDominatorTree>();
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+
     MachineFunctionProperties getRequiredProperties() const override {
       return MachineFunctionProperties().set(
           MachineFunctionProperties::Property::NoVRegs);
@@ -239,7 +247,7 @@ namespace {
     void doInitialJumpTablePlacement(std::vector<MachineInstr *> &CPEMIs);
     bool BBHasFallthrough(MachineBasicBlock *MBB);
     CPEntry *findConstPoolEntry(unsigned CPI, const MachineInstr *CPEMI);
-    unsigned getCPELogAlign(const MachineInstr *CPEMI);
+    Align getCPEAlign(const MachineInstr *CPEMI);
     void scanFunctionJumpTables();
     void initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs);
     MachineBasicBlock *splitBlockBeforeInstr(MachineInstr *MI);
@@ -328,8 +336,7 @@ LLVM_DUMP_METHOD void ARMConstantIslands::dumpBBs() {
       const BasicBlockInfo &BBI = BBInfo[J];
       dbgs() << format("%08x %bb.%u\t", BBI.Offset, J)
              << " kb=" << unsigned(BBI.KnownBits)
-             << " ua=" << unsigned(BBI.Unalign)
-             << " pa=" << unsigned(BBI.PostAlign)
+             << " ua=" << unsigned(BBI.Unalign) << " pa=" << Log2(BBI.PostAlign)
              << format(" size=%#x\n", BBInfo[J].Size);
     }
   });
@@ -350,6 +357,7 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   isPositionIndependentOrROPI =
       STI->getTargetLowering()->isPositionIndependent() || STI->isROPI();
   AFI = MF->getInfo<ARMFunctionInfo>();
+  DT = &getAnalysis<MachineDominatorTree>();
 
   isThumb = AFI->isThumbFunction();
   isThumb1 = AFI->isThumb1OnlyFunction();
@@ -396,7 +404,7 @@ bool ARMConstantIslands::runOnMachineFunction(MachineFunction &mf) {
   // Functions with jump tables need an alignment of 4 because they use the ADR
   // instruction, which aligns the PC to 4 bytes before adding an offset.
   if (!T2JumpTables.empty())
-    MF->ensureAlignment(llvm::Align(4));
+    MF->ensureAlignment(Align(4));
 
   /// Remove dead constant pool entries.
   MadeChange |= removeUnusedCPEntries();
@@ -485,11 +493,12 @@ ARMConstantIslands::doInitialConstPlacement(std::vector<MachineInstr*> &CPEMIs)
   MachineBasicBlock *BB = MF->CreateMachineBasicBlock();
   MF->push_back(BB);
 
-  // MachineConstantPool measures alignment in bytes. We measure in log2(bytes).
-  unsigned MaxLogAlign = Log2_32(MCP->getConstantPoolAlignment());
+  // MachineConstantPool measures alignment in bytes.
+  const Align MaxAlign(MCP->getConstantPoolAlignment());
+  const unsigned MaxLogAlign = Log2(MaxAlign);
 
   // Mark the basic block as required by the const-pool.
-  BB->setLogAlignment(MaxLogAlign);
+  BB->setAlignment(MaxAlign);
 
   // The function needs to be as aligned as the basic blocks. The linker may
   // move functions around based on their alignment.
@@ -639,29 +648,27 @@ ARMConstantIslands::findConstPoolEntry(unsigned CPI,
   return nullptr;
 }
 
-/// getCPELogAlign - Returns the required alignment of the constant pool entry
-/// represented by CPEMI.  Alignment is measured in log2(bytes) units.
-unsigned ARMConstantIslands::getCPELogAlign(const MachineInstr *CPEMI) {
+/// getCPEAlign - Returns the required alignment of the constant pool entry
+/// represented by CPEMI.
+Align ARMConstantIslands::getCPEAlign(const MachineInstr *CPEMI) {
   switch (CPEMI->getOpcode()) {
   case ARM::CONSTPOOL_ENTRY:
     break;
   case ARM::JUMPTABLE_TBB:
-    return isThumb1 ? 2 : 0;
+    return isThumb1 ? Align(4) : Align(1);
   case ARM::JUMPTABLE_TBH:
-    return isThumb1 ? 2 : 1;
+    return isThumb1 ? Align(4) : Align(2);
   case ARM::JUMPTABLE_INSTS:
-    return 1;
+    return Align(2);
   case ARM::JUMPTABLE_ADDRS:
-    return 2;
+    return Align(4);
   default:
     llvm_unreachable("unknown constpool entry kind");
   }
 
   unsigned CPI = getCombinedIndex(CPEMI);
   assert(CPI < MCP->getConstants().size() && "Invalid constant pool index.");
-  unsigned Align = MCP->getConstants()[CPI].getAlignment();
-  assert(isPowerOf2_32(Align) && "Invalid CPE alignment");
-  return Log2_32(Align);
+  return Align(MCP->getConstants()[CPI].getAlignment());
 }
 
 /// scanFunctionJumpTables - Do a scan of the function, building up
@@ -1014,10 +1021,10 @@ bool ARMConstantIslands::isWaterInRange(unsigned UserOffset,
                                         MachineBasicBlock* Water, CPUser &U,
                                         unsigned &Growth) {
   BBInfoVector &BBInfo = BBUtils->getBBInfo();
-  unsigned CPELogAlign = getCPELogAlign(U.CPEMI);
-  unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset(CPELogAlign);
+  const Align CPEAlign = getCPEAlign(U.CPEMI);
+  const unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset(CPEAlign);
   unsigned NextBlockOffset;
-  llvm::Align NextBlockAlignment;
+  Align NextBlockAlignment;
   MachineFunction::const_iterator NextBlock = Water->getIterator();
   if (++NextBlock == MF->end()) {
     NextBlockOffset = BBInfo[Water->getNumber()].postOffset();
@@ -1041,8 +1048,7 @@ bool ARMConstantIslands::isWaterInRange(unsigned UserOffset,
     // the offset of the instruction. Also account for unknown alignment padding
     // in blocks between CPE and the user.
     if (CPEOffset < UserOffset)
-      UserOffset +=
-          Growth + UnknownPadding(Log2(MF->getAlignment()), CPELogAlign);
+      UserOffset += Growth + UnknownPadding(MF->getAlignment(), Log2(CPEAlign));
   } else
     // CPE fits in existing padding.
     Growth = 0;
@@ -1208,8 +1214,8 @@ bool ARMConstantIslands::findAvailableWater(CPUser &U, unsigned UserOffset,
   // inserting islands between BB0 and BB1 makes other accesses out of range.
   MachineBasicBlock *UserBB = U.MI->getParent();
   BBInfoVector &BBInfo = BBUtils->getBBInfo();
-  unsigned MinNoSplitDisp =
-      BBInfo[UserBB->getNumber()].postOffset(getCPELogAlign(U.CPEMI));
+  const Align CPEAlign = getCPEAlign(U.CPEMI);
+  unsigned MinNoSplitDisp = BBInfo[UserBB->getNumber()].postOffset(CPEAlign);
   if (CloserWater && MinNoSplitDisp > U.getMaxDisp() / 2)
     return false;
   for (water_iterator IP = std::prev(WaterList.end()), B = WaterList.begin();;
@@ -1262,7 +1268,7 @@ void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
   CPUser &U = CPUsers[CPUserIndex];
   MachineInstr *UserMI = U.MI;
   MachineInstr *CPEMI  = U.CPEMI;
-  unsigned CPELogAlign = getCPELogAlign(CPEMI);
+  const Align CPEAlign = getCPEAlign(CPEMI);
   MachineBasicBlock *UserMBB = UserMI->getParent();
   BBInfoVector &BBInfo = BBUtils->getBBInfo();
   const BasicBlockInfo &UserBBI = BBInfo[UserMBB->getNumber()];
@@ -1275,7 +1281,7 @@ void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
     // Size of branch to insert.
     unsigned Delta = isThumb1 ? 2 : 4;
     // Compute the offset where the CPE will begin.
-    unsigned CPEOffset = UserBBI.postOffset(CPELogAlign) + Delta;
+    unsigned CPEOffset = UserBBI.postOffset(CPEAlign) + Delta;
 
     if (isOffsetInRange(UserOffset, CPEOffset, U)) {
       LLVM_DEBUG(dbgs() << "Split at end of " << printMBBReference(*UserMBB)
@@ -1316,11 +1322,11 @@ void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
 
   // Try to split the block so it's fully aligned.  Compute the latest split
   // point where we can add a 4-byte branch instruction, and then align to
-  // LogAlign which is the largest possible alignment in the function.
-  unsigned LogAlign = Log2(MF->getAlignment());
-  assert(LogAlign >= CPELogAlign && "Over-aligned constant pool entry");
+  // Align which is the largest possible alignment in the function.
+  const Align Align = MF->getAlignment();
+  assert(Align >= CPEAlign && "Over-aligned constant pool entry");
   unsigned KnownBits = UserBBI.internalKnownBits();
-  unsigned UPad = UnknownPadding(LogAlign, KnownBits);
+  unsigned UPad = UnknownPadding(Align, KnownBits);
   unsigned BaseInsertOffset = UserOffset + U.getMaxDisp() - UPad;
   LLVM_DEBUG(dbgs() << format("Split in middle of big block before %#x",
                               BaseInsertOffset));
@@ -1331,7 +1337,7 @@ void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
   BaseInsertOffset -= 4;
 
   LLVM_DEBUG(dbgs() << format(", adjusted to %#x", BaseInsertOffset)
-                    << " la=" << LogAlign << " kb=" << KnownBits
+                    << " la=" << Log2(Align) << " kb=" << KnownBits
                     << " up=" << UPad << '\n');
 
   // This could point off the end of the block if we've already got constant
@@ -1384,8 +1390,8 @@ void ARMConstantIslands::createNewWater(unsigned CPUserIndex,
       CPUser &U = CPUsers[CPUIndex];
       if (!isOffsetInRange(Offset, EndInsertOffset, U)) {
         // Shift intertion point by one unit of alignment so it is within reach.
-        BaseInsertOffset -= 1u << LogAlign;
-        EndInsertOffset  -= 1u << LogAlign;
+        BaseInsertOffset -= Align.value();
+        EndInsertOffset -= Align.value();
       }
       // This is overly conservative, as we don't account for CPEMIs being
       // reused within the block, but it doesn't matter much.  Also assume CPEs
@@ -1495,9 +1501,9 @@ bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex,
   // Always align the new block because CP entries can be smaller than 4
   // bytes. Be careful not to decrease the existing alignment, e.g. NewMBB may
   // be an already aligned constant pool block.
-  const unsigned LogAlign = isThumb ? 1 : 2;
-  if (NewMBB->getLogAlignment() < LogAlign)
-    NewMBB->setLogAlignment(LogAlign);
+  const Align Alignment = isThumb ? Align(2) : Align(4);
+  if (NewMBB->getAlignment() < Alignment)
+    NewMBB->setAlignment(Alignment);
 
   // Remove the original WaterList entry; we want subsequent insertions in
   // this vicinity to go after the one we're about to insert.  This
@@ -1526,7 +1532,7 @@ bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex,
   decrementCPEReferenceCount(CPI, CPEMI);
 
   // Mark the basic block as aligned as required by the const-pool entry.
-  NewIsland->setLogAlignment(getCPELogAlign(U.CPEMI));
+  NewIsland->setAlignment(getCPEAlign(U.CPEMI));
 
   // Increase the size of the island block to account for the new entry.
   BBUtils->adjustBBSize(NewIsland, Size);
@@ -1560,10 +1566,11 @@ void ARMConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) {
     BBInfo[CPEBB->getNumber()].Size = 0;
 
     // This block no longer needs to be aligned.
-    CPEBB->setLogAlignment(0);
-  } else
+    CPEBB->setAlignment(Align::None());
+  } else {
     // Entries are sorted by descending alignment, so realign from the front.
-    CPEBB->setLogAlignment(getCPELogAlign(&*CPEBB->begin()));
+    CPEBB->setAlignment(getCPEAlign(&*CPEBB->begin()));
+  }
 
   BBUtils->adjustBBOffsetsAfter(CPEBB);
   // An island has only one predecessor BB and one successor BB. Check if
@@ -1809,16 +1816,10 @@ bool ARMConstantIslands::optimizeThumb2Instructions() {
   return MadeChange;
 }
 
+
 bool ARMConstantIslands::optimizeThumb2Branches() {
-  bool MadeChange = false;
 
-  // The order in which branches appear in ImmBranches is approximately their
-  // order within the function body. By visiting later branches first, we reduce
-  // the distance between earlier forward branches and their targets, making it
-  // more likely that the cbn?z optimization, which can only apply to forward
-  // branches, will succeed.
-  for (unsigned i = ImmBranches.size(); i != 0; --i) {
-    ImmBranch &Br = ImmBranches[i-1];
+  auto TryShrinkBranch = [this](ImmBranch &Br) {
     unsigned Opcode = Br.MI->getOpcode();
     unsigned NewOpc = 0;
     unsigned Scale = 1;
@@ -1846,47 +1847,115 @@ bool ARMConstantIslands::optimizeThumb2Branches() {
         BBUtils->adjustBBSize(MBB, -2);
         BBUtils->adjustBBOffsetsAfter(MBB);
         ++NumT2BrShrunk;
-        MadeChange = true;
+        return true;
       }
     }
+    return false;
+  };
 
-    Opcode = Br.MI->getOpcode();
-    if (Opcode != ARM::tBcc)
-      continue;
+  struct ImmCompare {
+    MachineInstr* MI = nullptr;
+    unsigned NewOpc = 0;
+  };
+
+  auto FindCmpForCBZ = [this](ImmBranch &Br, ImmCompare &ImmCmp,
+                              MachineBasicBlock *DestBB) {
+    ImmCmp.MI = nullptr;
+    ImmCmp.NewOpc = 0;
 
     // If the conditional branch doesn't kill CPSR, then CPSR can be liveout
     // so this transformation is not safe.
     if (!Br.MI->killsRegister(ARM::CPSR))
-      continue;
+      return false;
 
-    NewOpc = 0;
     unsigned PredReg = 0;
+    unsigned NewOpc = 0;
     ARMCC::CondCodes Pred = getInstrPredicate(*Br.MI, PredReg);
     if (Pred == ARMCC::EQ)
       NewOpc = ARM::tCBZ;
     else if (Pred == ARMCC::NE)
       NewOpc = ARM::tCBNZ;
-    if (!NewOpc)
-      continue;
-    MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB();
+    else
+      return false;
+
     // Check if the distance is within 126. Subtract starting offset by 2
     // because the cmp will be eliminated.
     unsigned BrOffset = BBUtils->getOffsetOf(Br.MI) + 4 - 2;
     BBInfoVector &BBInfo = BBUtils->getBBInfo();
     unsigned DestOffset = BBInfo[DestBB->getNumber()].Offset;
     if (BrOffset >= DestOffset || (DestOffset - BrOffset) > 126)
-      continue;
+      return false;
 
     // Search backwards to find a tCMPi8
     auto *TRI = STI->getRegisterInfo();
     MachineInstr *CmpMI = findCMPToFoldIntoCBZ(Br.MI, TRI);
     if (!CmpMI || CmpMI->getOpcode() != ARM::tCMPi8)
+      return false;
+
+    ImmCmp.MI = CmpMI;
+    ImmCmp.NewOpc = NewOpc;
+    return true;
+  };
+
+  auto TryConvertToLE = [this](ImmBranch &Br, ImmCompare &Cmp) {
+    if (Br.MI->getOpcode() != ARM::t2Bcc || !STI->hasLOB() ||
+        STI->hasMinSize())
+      return false;
+
+    MachineBasicBlock *MBB = Br.MI->getParent();
+    MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB();
+    if (BBUtils->getOffsetOf(MBB) < BBUtils->getOffsetOf(DestBB) ||
+        !BBUtils->isBBInRange(Br.MI, DestBB, 4094))
+      return false;
+
+    if (!DT->dominates(DestBB, MBB))
+      return false;
+
+    // We queried for the CBN?Z opcode based upon the 'ExitBB', the opposite
+    // target of Br. So now we need to reverse the condition.
+    Cmp.NewOpc = Cmp.NewOpc == ARM::tCBZ ? ARM::tCBNZ : ARM::tCBZ;
+
+    MachineInstrBuilder MIB = BuildMI(*MBB, Br.MI, Br.MI->getDebugLoc(),
+                                      TII->get(ARM::t2LE));
+    MIB.add(Br.MI->getOperand(0));
+    Br.MI->eraseFromParent();
+    Br.MI = MIB;
+    ++NumLEInserted;
+    return true;
+  };
+
+  bool MadeChange = false;
+
+  // The order in which branches appear in ImmBranches is approximately their
+  // order within the function body. By visiting later branches first, we reduce
+  // the distance between earlier forward branches and their targets, making it
+  // more likely that the cbn?z optimization, which can only apply to forward
+  // branches, will succeed.
+  for (ImmBranch &Br : reverse(ImmBranches)) {
+    MachineBasicBlock *DestBB = Br.MI->getOperand(0).getMBB();
+    MachineBasicBlock *MBB = Br.MI->getParent();
+    MachineBasicBlock *ExitBB = &MBB->back() == Br.MI ?
+      MBB->getFallThrough() :
+      MBB->back().getOperand(0).getMBB();
+
+    ImmCompare Cmp;
+    if (FindCmpForCBZ(Br, Cmp, ExitBB) && TryConvertToLE(Br, Cmp)) {
+      DestBB = ExitBB;
+      MadeChange = true;
+    } else {
+      FindCmpForCBZ(Br, Cmp, DestBB);
+      MadeChange |= TryShrinkBranch(Br);
+    }
+
+    unsigned Opcode = Br.MI->getOpcode();
+    if ((Opcode != ARM::tBcc && Opcode != ARM::t2LE) || !Cmp.NewOpc)
       continue;
 
-    Register Reg = CmpMI->getOperand(0).getReg();
+    Register Reg = Cmp.MI->getOperand(0).getReg();
 
     // Check for Kill flags on Reg. If they are present remove them and set kill
     // on the new CBZ.
+    auto *TRI = STI->getRegisterInfo();
     MachineBasicBlock::iterator KillMI = Br.MI;
     bool RegKilled = false;
     do {
@@ -1896,19 +1965,32 @@ bool ARMConstantIslands::optimizeThumb2Branches() {
         RegKilled = true;
         break;
       }
-    } while (KillMI != CmpMI);
+    } while (KillMI != Cmp.MI);
 
     // Create the new CBZ/CBNZ
-    MachineBasicBlock *MBB = Br.MI->getParent();
-    LLVM_DEBUG(dbgs() << "Fold: " << *CmpMI << " and: " << *Br.MI);
+    LLVM_DEBUG(dbgs() << "Fold: " << *Cmp.MI << " and: " << *Br.MI);
     MachineInstr *NewBR =
-        BuildMI(*MBB, Br.MI, Br.MI->getDebugLoc(), TII->get(NewOpc))
+        BuildMI(*MBB, Br.MI, Br.MI->getDebugLoc(), TII->get(Cmp.NewOpc))
             .addReg(Reg, getKillRegState(RegKilled))
             .addMBB(DestBB, Br.MI->getOperand(0).getTargetFlags());
-    CmpMI->eraseFromParent();
-    Br.MI->eraseFromParent();
-    Br.MI = NewBR;
+
+    Cmp.MI->eraseFromParent();
+    BBInfoVector &BBInfo = BBUtils->getBBInfo();
     BBInfo[MBB->getNumber()].Size -= 2;
+
+    if (Br.MI->getOpcode() == ARM::tBcc) {
+      Br.MI->eraseFromParent();
+      Br.MI = NewBR;
+    } else if (&MBB->back() != Br.MI) {
+      // We've generated an LE and already erased the original conditional
+      // branch. The CBN?Z is now used to branch to the other successor, so an
+      // unconditional branch terminator is now redundant.
+      MachineInstr *LastMI = &MBB->back();
+      if (LastMI != Br.MI) {
+        BBInfo[MBB->getNumber()].Size -= LastMI->getDesc().getSize();
+        LastMI->eraseFromParent();
+      }
+    }
     BBUtils->adjustBBOffsetsAfter(MBB);
     ++NumCBZ;
     MadeChange = true;
diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
index e11399aa25f2d..03681d5634cc5 100644
--- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp
@@ -1503,11 +1503,17 @@ static unsigned EstimateFunctionSizeInBytes(const MachineFunction &MF,
 /// instructions will require a scratch register during their expansion later.
 // FIXME: Move to TII?
 static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
-                                         const TargetFrameLowering *TFI) {
+                                         const TargetFrameLowering *TFI,
+                                         bool &HasNonSPFrameIndex) {
   const ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+  const ARMBaseInstrInfo &TII =
+      *static_cast<const ARMBaseInstrInfo *>(MF.getSubtarget().getInstrInfo());
+  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
   unsigned Limit = (1 << 12) - 1;
   for (auto &MBB : MF) {
     for (auto &MI : MBB) {
+      if (MI.isDebugInstr())
+        continue;
       for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
         if (!MI.getOperand(i).isFI())
           continue;
@@ -1518,13 +1524,29 @@ static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
           Limit = std::min(Limit, (1U << 8) - 1);
           break;
         }
+        // t2ADDri will not require an extra register, it can reuse the
+        // destination.
+        if (MI.getOpcode() == ARM::t2ADDri || MI.getOpcode() == ARM::t2ADDri12)
+          break;
+
+        const MCInstrDesc &MCID = MI.getDesc();
+        const TargetRegisterClass *RegClass = TII.getRegClass(MCID, i, TRI, MF);
+        if (RegClass && !RegClass->contains(ARM::SP))
+          HasNonSPFrameIndex = true;
 
         // Otherwise check the addressing mode.
         switch (MI.getDesc().TSFlags & ARMII::AddrModeMask) {
+        case ARMII::AddrMode_i12:
+        case ARMII::AddrMode2:
+          // Default 12 bit limit.
+          break;
         case ARMII::AddrMode3:
         case ARMII::AddrModeT2_i8:
           Limit = std::min(Limit, (1U << 8) - 1);
           break;
+        case ARMII::AddrMode5FP16:
+          Limit = std::min(Limit, ((1U << 8) - 1) * 2);
+          break;
         case ARMII::AddrMode5:
         case ARMII::AddrModeT2_i8s4:
         case ARMII::AddrModeT2_ldrex:
@@ -1541,8 +1563,17 @@ static unsigned estimateRSStackSizeLimit(MachineFunction &MF,
           // Addressing modes 4 & 6 (load/store) instructions can't encode an
           // immediate offset for stack references.
           return 0;
-        default:
+        case ARMII::AddrModeT2_i7:
+          Limit = std::min(Limit, ((1U << 7) - 1) * 1);
           break;
+        case ARMII::AddrModeT2_i7s2:
+          Limit = std::min(Limit, ((1U << 7) - 1) * 2);
+          break;
+        case ARMII::AddrModeT2_i7s4:
+          Limit = std::min(Limit, ((1U << 7) - 1) * 4);
+          break;
+        default:
+          llvm_unreachable("Unhandled addressing mode in stack size limit calculation");
         }
         break; // At most one FI per instruction
       }
@@ -1784,6 +1815,7 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
   EstimatedStackSize += 16; // For possible paddings.
 
   unsigned EstimatedRSStackSizeLimit, EstimatedRSFixedSizeLimit;
+  bool HasNonSPFrameIndex = false;
   if (AFI->isThumb1OnlyFunction()) {
     // For Thumb1, don't bother to iterate over the function. The only
     // instruction that requires an emergency spill slot is a store to a
@@ -1804,7 +1836,8 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
       EstimatedRSStackSizeLimit = (1U << 8) * 4;
     EstimatedRSFixedSizeLimit = (1U << 5) * 4;
   } else {
-    EstimatedRSStackSizeLimit = estimateRSStackSizeLimit(MF, this);
+    EstimatedRSStackSizeLimit =
+        estimateRSStackSizeLimit(MF, this, HasNonSPFrameIndex);
     EstimatedRSFixedSizeLimit = EstimatedRSStackSizeLimit;
   }
   // Final estimate of whether sp or bp-relative accesses might require
@@ -1830,12 +1863,11 @@ void ARMFrameLowering::determineCalleeSaves(MachineFunction &MF,
       HasFP && (MaxFixedOffset - MaxFPOffset) > (int)EstimatedRSFixedSizeLimit;
 
   bool BigFrameOffsets = HasLargeStack || !HasBPOrFixedSP ||
-                         HasLargeArgumentList;
+                         HasLargeArgumentList || HasNonSPFrameIndex;
   LLVM_DEBUG(dbgs() << "EstimatedLimit: " << EstimatedRSStackSizeLimit
-                    << "; EstimatedStack" << EstimatedStackSize
-                    << "; EstimatedFPStack" << MaxFixedOffset - MaxFPOffset
-                    << "; BigFrameOffsets: " << BigFrameOffsets
-                    << "\n");
+                    << "; EstimatedStack: " << EstimatedStackSize
+                    << "; EstimatedFPStack: " << MaxFixedOffset - MaxFPOffset
+                    << "; BigFrameOffsets: " << BigFrameOffsets << "\n");
   if (BigFrameOffsets ||
       !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) {
     AFI->setHasStackFrame(true);
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index a59a57327d1c9..8f6515c423eb4 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -139,6 +139,8 @@ class ARMDAGToDAGISel : public SelectionDAGISel {
   bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
                                  SDValue &OffImm);
   bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
+  template <unsigned Shift>
+  bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
 
   // Thumb 2 Addressing Modes:
   bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
@@ -1151,6 +1153,28 @@ bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
   return false;
 }
 
+template <unsigned Shift>
+bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
+                                          SDValue &OffImm) {
+  if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
+    int RHSC;
+    if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80,
+                                RHSC)) {
+      Base = N.getOperand(0);
+      if (N.getOpcode() == ISD::SUB)
+        RHSC = -RHSC;
+      OffImm =
+          CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
+      return true;
+    }
+  }
+
+  // Base only.
+  Base = N;
+  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
+  return true;
+}
+
 
 //===----------------------------------------------------------------------===//
 //                        Thumb 2 Addressing Modes
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index d341ee820d364..45bf676338228 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -259,6 +259,12 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::UMAX, VT, Legal);
     setOperationAction(ISD::ABS, VT, Legal);
     setOperationAction(ISD::SETCC, VT, Custom);
+    setOperationAction(ISD::MLOAD, VT, Custom);
+    setOperationAction(ISD::MSTORE, VT, Legal);
+    setOperationAction(ISD::CTLZ, VT, Legal);
+    setOperationAction(ISD::CTTZ, VT, Custom);
+    setOperationAction(ISD::BITREVERSE, VT, Legal);
+    setOperationAction(ISD::BSWAP, VT, Legal);
 
     // No native support for these.
     setOperationAction(ISD::UDIV, VT, Expand);
@@ -304,6 +310,8 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
     setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom);
     setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
     setOperationAction(ISD::SETCC, VT, Custom);
+    setOperationAction(ISD::MLOAD, VT, Custom);
+    setOperationAction(ISD::MSTORE, VT, Legal);
 
     // Pre and Post inc are supported on loads and stores
     for (unsigned im = (unsigned)ISD::PRE_INC;
@@ -696,8 +704,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
   }
 
-  for (MVT VT : MVT::vector_valuetypes()) {
-    for (MVT InnerVT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
+    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
       setTruncStoreAction(VT, InnerVT, Expand);
       addAllExtLoads(VT, InnerVT, Expand);
     }
@@ -890,10 +898,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::SHL);
     setTargetDAGCombine(ISD::SRL);
     setTargetDAGCombine(ISD::SRA);
-    setTargetDAGCombine(ISD::SIGN_EXTEND);
-    setTargetDAGCombine(ISD::ZERO_EXTEND);
-    setTargetDAGCombine(ISD::ANY_EXTEND);
-    setTargetDAGCombine(ISD::STORE);
     setTargetDAGCombine(ISD::FP_TO_SINT);
     setTargetDAGCombine(ISD::FP_TO_UINT);
     setTargetDAGCombine(ISD::FDIV);
@@ -902,7 +906,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     // It is legal to extload from v4i8 to v4i16 or v4i32.
     for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
                    MVT::v2i32}) {
-      for (MVT VT : MVT::integer_vector_valuetypes()) {
+      for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
         setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
         setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
         setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
@@ -914,6 +918,10 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
     setTargetDAGCombine(ISD::BUILD_VECTOR);
     setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
     setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
+    setTargetDAGCombine(ISD::STORE);
+    setTargetDAGCombine(ISD::SIGN_EXTEND);
+    setTargetDAGCombine(ISD::ZERO_EXTEND);
+    setTargetDAGCombine(ISD::ANY_EXTEND);
   }
 
   if (!Subtarget->hasFP64()) {
@@ -1046,7 +1054,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   // ARM does not have ROTL.
   setOperationAction(ISD::ROTL, MVT::i32, Expand);
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     setOperationAction(ISD::ROTL, VT, Expand);
     setOperationAction(ISD::ROTR, VT, Expand);
   }
@@ -1420,16 +1428,14 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
 
   // On ARM arguments smaller than 4 bytes are extended, so all arguments
   // are at least 4 bytes aligned.
-  setMinStackArgumentAlignment(llvm::Align(4));
+  setMinStackArgumentAlignment(Align(4));
 
   // Prefer likely predicted branches to selects on out-of-order cores.
   PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
 
-  setPrefLoopAlignment(
-      llvm::Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
+  setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
 
-  setMinFunctionAlignment(Subtarget->isThumb() ? llvm::Align(2)
-                                               : llvm::Align(4));
+  setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
 
   if (Subtarget->isThumb() || Subtarget->isThumb2())
     setTargetDAGCombine(ISD::ABS);
@@ -3108,12 +3114,12 @@ ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
 
   // Load the current TEB (thread environment block)
   SDValue Ops[] = {Chain,
-                   DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
-                   DAG.getConstant(15, DL, MVT::i32),
-                   DAG.getConstant(0, DL, MVT::i32),
-                   DAG.getConstant(13, DL, MVT::i32),
-                   DAG.getConstant(0, DL, MVT::i32),
-                   DAG.getConstant(2, DL, MVT::i32)};
+                   DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
+                   DAG.getTargetConstant(15, DL, MVT::i32),
+                   DAG.getTargetConstant(0, DL, MVT::i32),
+                   DAG.getTargetConstant(13, DL, MVT::i32),
+                   DAG.getTargetConstant(0, DL, MVT::i32),
+                   DAG.getTargetConstant(2, DL, MVT::i32)};
   SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
                                    DAG.getVTList(MVT::i32, MVT::Other), Ops);
 
@@ -5477,9 +5483,9 @@ SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                              SelectionDAG &DAG) const {
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+Register ARMTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              const MachineFunction &MF) const {
+  Register Reg = StringSwitch<unsigned>(RegName)
                        .Case("sp", ARM::SP)
                        .Default(0);
   if (Reg)
@@ -5800,8 +5806,7 @@ static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
                          const ARMSubtarget *ST) {
   SDLoc dl(N);
   EVT VT = N->getValueType(0);
-  if (VT.isVector()) {
-    assert(ST->hasNEON());
+  if (VT.isVector() && ST->hasNEON()) {
 
     // Compute the least significant set bit: LSB = X & -X
     SDValue X = N->getOperand(0);
@@ -6004,7 +6009,7 @@ static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
     // If the shift amount is greater than 32 or has a greater bitwidth than 64
     // then do the default optimisation
     if (ShAmt->getValueType(0).getSizeInBits() > 64 ||
-        (Con && Con->getZExtValue() >= 32))
+        (Con && (Con->getZExtValue() == 0 || Con->getZExtValue() >= 32)))
       return SDValue();
 
     // Extract the lower 32 bits of the shift amount if it's not an i32
@@ -6938,6 +6943,19 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
   } else
     return SDValue();
 
+  // If this is a single value copied into all lanes (a splat), we can just sign
+  // extend that single value
+  SDValue FirstOp = Op.getOperand(0);
+  if (!isa<ConstantSDNode>(FirstOp) &&
+      std::all_of(std::next(Op->op_begin()), Op->op_end(),
+                  [&FirstOp](SDUse &U) {
+                    return U.get().isUndef() || U.get() == FirstOp;
+                  })) {
+    SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
+                              DAG.getValueType(MVT::i1));
+    return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
+  }
+
   // First create base with bits set where known
   unsigned Bits32 = 0;
   for (unsigned i = 0; i < NumElts; ++i) {
@@ -6950,7 +6968,6 @@ static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
   }
 
   // Add in unknown nodes
-  // FIXME: Handle splats of the same value better.
   SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
                              DAG.getConstant(Bits32, dl, MVT::i32));
   for (unsigned i = 0; i < NumElts; ++i) {
@@ -7465,6 +7482,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
   unsigned EltSize = VT.getScalarSizeInBits();
   if (EltSize >= 32 ||
       ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
+      ShuffleVectorInst::isIdentityMask(M) ||
       isVREVMask(M, VT, 64) ||
       isVREVMask(M, VT, 32) ||
       isVREVMask(M, VT, 16))
@@ -8848,6 +8866,31 @@ static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
       ST->getMemOperand());
 }
 
+static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
+  MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
+  MVT VT = Op.getSimpleValueType();
+  SDValue Mask = N->getMask();
+  SDValue PassThru = N->getPassThru();
+  SDLoc dl(Op);
+
+  if (ISD::isBuildVectorAllZeros(PassThru.getNode()) ||
+      (PassThru->getOpcode() == ARMISD::VMOVIMM &&
+       isNullConstant(PassThru->getOperand(0))))
+    return Op;
+
+  // MVE Masked loads use zero as the passthru value. Here we convert undef to
+  // zero too, and other values are lowered to a select.
+  SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
+                                DAG.getTargetConstant(0, dl, MVT::i32));
+  SDValue NewLoad = DAG.getMaskedLoad(
+      VT, dl, N->getChain(), N->getBasePtr(), Mask, ZeroVec, N->getMemoryVT(),
+      N->getMemOperand(), N->getExtensionType(), N->isExpandingLoad());
+  SDValue Combo = NewLoad;
+  if (!PassThru.isUndef())
+    Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
+  return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
+}
+
 static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
   if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering()))
     // Acquire/Release load/store is not legal for targets without a dmb or
@@ -8866,12 +8909,12 @@ static void ReplaceREADCYCLECOUNTER(SDNode *N,
   // Under Power Management extensions, the cycle-count is:
   //    mrc p15, #0, <Rt>, c9, c13, #0
   SDValue Ops[] = { N->getOperand(0), // Chain
-                    DAG.getConstant(Intrinsic::arm_mrc, DL, MVT::i32),
-                    DAG.getConstant(15, DL, MVT::i32),
-                    DAG.getConstant(0, DL, MVT::i32),
-                    DAG.getConstant(9, DL, MVT::i32),
-                    DAG.getConstant(13, DL, MVT::i32),
-                    DAG.getConstant(0, DL, MVT::i32)
+                    DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
+                    DAG.getTargetConstant(15, DL, MVT::i32),
+                    DAG.getTargetConstant(0, DL, MVT::i32),
+                    DAG.getTargetConstant(9, DL, MVT::i32),
+                    DAG.getTargetConstant(13, DL, MVT::i32),
+                    DAG.getTargetConstant(0, DL, MVT::i32)
   };
 
   SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
@@ -9051,6 +9094,8 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return LowerPredicateLoad(Op, DAG);
   case ISD::STORE:
     return LowerPredicateStore(Op, DAG);
+  case ISD::MLOAD:
+    return LowerMLOAD(Op, DAG);
   case ISD::ATOMIC_LOAD:
   case ISD::ATOMIC_STORE:  return LowerAtomicLoadStore(Op, DAG);
   case ISD::FSINCOS:       return LowerFSINCOS(Op, DAG);
@@ -12595,6 +12640,24 @@ PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
   return Vec;
 }
 
+static SDValue
+PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
+  EVT VT = N->getValueType(0);
+  SDValue Op = N->getOperand(0);
+  SDLoc dl(N);
+
+  // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
+  if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
+    // If the valuetypes are the same, we can remove the cast entirely.
+    if (Op->getOperand(0).getValueType() == VT)
+      return Op->getOperand(0);
+    return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl,
+                           Op->getOperand(0).getValueType(), Op->getOperand(0));
+  }
+
+  return SDValue();
+}
+
 /// PerformInsertEltCombine - Target-specific dag combine xforms for
 /// ISD::INSERT_VECTOR_ELT.
 static SDValue PerformInsertEltCombine(SDNode *N,
@@ -13056,95 +13119,163 @@ static SDValue PerformLOADCombine(SDNode *N,
   return SDValue();
 }
 
-/// PerformSTORECombine - Target-specific dag combine xforms for
-/// ISD::STORE.
-static SDValue PerformSTORECombine(SDNode *N,
-                                   TargetLowering::DAGCombinerInfo &DCI) {
-  StoreSDNode *St = cast<StoreSDNode>(N);
-  if (St->isVolatile())
-    return SDValue();
-
-  // Optimize trunc store (of multiple scalars) to shuffle and store.  First,
-  // pack all of the elements in one place.  Next, store to memory in fewer
-  // chunks.
+// Optimize trunc store (of multiple scalars) to shuffle and store.  First,
+// pack all of the elements in one place.  Next, store to memory in fewer
+// chunks.
+static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
+                                             SelectionDAG &DAG) {
   SDValue StVal = St->getValue();
   EVT VT = StVal.getValueType();
-  if (St->isTruncatingStore() && VT.isVector()) {
-    SelectionDAG &DAG = DCI.DAG;
-    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-    EVT StVT = St->getMemoryVT();
-    unsigned NumElems = VT.getVectorNumElements();
-    assert(StVT != VT && "Cannot truncate to the same type");
-    unsigned FromEltSz = VT.getScalarSizeInBits();
-    unsigned ToEltSz = StVT.getScalarSizeInBits();
+  if (!St->isTruncatingStore() || !VT.isVector())
+    return SDValue();
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  EVT StVT = St->getMemoryVT();
+  unsigned NumElems = VT.getVectorNumElements();
+  assert(StVT != VT && "Cannot truncate to the same type");
+  unsigned FromEltSz = VT.getScalarSizeInBits();
+  unsigned ToEltSz = StVT.getScalarSizeInBits();
+
+  // From, To sizes and ElemCount must be pow of two
+  if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz))
+    return SDValue();
 
-    // From, To sizes and ElemCount must be pow of two
-    if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz)) return SDValue();
+  // We are going to use the original vector elt for storing.
+  // Accumulated smaller vector elements must be a multiple of the store size.
+  if (0 != (NumElems * FromEltSz) % ToEltSz)
+    return SDValue();
 
-    // We are going to use the original vector elt for storing.
-    // Accumulated smaller vector elements must be a multiple of the store size.
-    if (0 != (NumElems * FromEltSz) % ToEltSz) return SDValue();
+  unsigned SizeRatio = FromEltSz / ToEltSz;
+  assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
 
-    unsigned SizeRatio  = FromEltSz / ToEltSz;
-    assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits());
+  // Create a type on which we perform the shuffle.
+  EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
+                                   NumElems * SizeRatio);
+  assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
 
-    // Create a type on which we perform the shuffle.
-    EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
-                                     NumElems*SizeRatio);
-    assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
+  SDLoc DL(St);
+  SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
+  SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
+  for (unsigned i = 0; i < NumElems; ++i)
+    ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
+                                                      : i * SizeRatio;
 
-    SDLoc DL(St);
-    SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
-    SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
-    for (unsigned i = 0; i < NumElems; ++i)
-      ShuffleVec[i] = DAG.getDataLayout().isBigEndian()
-                          ? (i + 1) * SizeRatio - 1
-                          : i * SizeRatio;
-
-    // Can't shuffle using an illegal type.
-    if (!TLI.isTypeLegal(WideVecVT)) return SDValue();
-
-    SDValue Shuff = DAG.getVectorShuffle(WideVecVT, DL, WideVec,
-                                DAG.getUNDEF(WideVec.getValueType()),
-                                ShuffleVec);
-    // At this point all of the data is stored at the bottom of the
-    // register. We now need to save it to mem.
-
-    // Find the largest store unit
-    MVT StoreType = MVT::i8;
-    for (MVT Tp : MVT::integer_valuetypes()) {
-      if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
-        StoreType = Tp;
-    }
-    // Didn't find a legal store type.
-    if (!TLI.isTypeLegal(StoreType))
-      return SDValue();
+  // Can't shuffle using an illegal type.
+  if (!TLI.isTypeLegal(WideVecVT))
+    return SDValue();
 
-    // Bitcast the original vector into a vector of store-size units
-    EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
-            StoreType, VT.getSizeInBits()/EVT(StoreType).getSizeInBits());
-    assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
-    SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
-    SmallVector<SDValue, 8> Chains;
-    SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
-                                        TLI.getPointerTy(DAG.getDataLayout()));
-    SDValue BasePtr = St->getBasePtr();
+  SDValue Shuff = DAG.getVectorShuffle(
+      WideVecVT, DL, WideVec, DAG.getUNDEF(WideVec.getValueType()), ShuffleVec);
+  // At this point all of the data is stored at the bottom of the
+  // register. We now need to save it to mem.
 
-    // Perform one or more big stores into memory.
-    unsigned E = (ToEltSz*NumElems)/StoreType.getSizeInBits();
-    for (unsigned I = 0; I < E; I++) {
-      SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
-                                   StoreType, ShuffWide,
-                                   DAG.getIntPtrConstant(I, DL));
-      SDValue Ch = DAG.getStore(St->getChain(), DL, SubVec, BasePtr,
-                                St->getPointerInfo(), St->getAlignment(),
-                                St->getMemOperand()->getFlags());
-      BasePtr = DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr,
-                            Increment);
-      Chains.push_back(Ch);
-    }
-    return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
+  // Find the largest store unit
+  MVT StoreType = MVT::i8;
+  for (MVT Tp : MVT::integer_valuetypes()) {
+    if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
+      StoreType = Tp;
   }
+  // Didn't find a legal store type.
+  if (!TLI.isTypeLegal(StoreType))
+    return SDValue();
+
+  // Bitcast the original vector into a vector of store-size units
+  EVT StoreVecVT =
+      EVT::getVectorVT(*DAG.getContext(), StoreType,
+                       VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
+  assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
+  SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
+  SmallVector<SDValue, 8> Chains;
+  SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
+                                      TLI.getPointerTy(DAG.getDataLayout()));
+  SDValue BasePtr = St->getBasePtr();
+
+  // Perform one or more big stores into memory.
+  unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
+  for (unsigned I = 0; I < E; I++) {
+    SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
+                                 ShuffWide, DAG.getIntPtrConstant(I, DL));
+    SDValue Ch =
+        DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(),
+                     St->getAlignment(), St->getMemOperand()->getFlags());
+    BasePtr =
+        DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment);
+    Chains.push_back(Ch);
+  }
+  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
+}
+
+// Try taking a single vector store from an truncate (which would otherwise turn
+// into an expensive buildvector) and splitting it into a series of narrowing
+// stores.
+static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
+                                                 SelectionDAG &DAG) {
+  if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
+    return SDValue();
+  SDValue Trunc = St->getValue();
+  if (Trunc->getOpcode() != ISD::TRUNCATE)
+    return SDValue();
+  EVT FromVT = Trunc->getOperand(0).getValueType();
+  EVT ToVT = Trunc.getValueType();
+  if (!ToVT.isVector())
+    return SDValue();
+  assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
+  EVT ToEltVT = ToVT.getVectorElementType();
+  EVT FromEltVT = FromVT.getVectorElementType();
+
+  unsigned NumElements = 0;
+  if (FromEltVT == MVT::i32 && (ToEltVT == MVT::i16 || ToEltVT == MVT::i8))
+    NumElements = 4;
+  if (FromEltVT == MVT::i16 && ToEltVT == MVT::i8)
+    NumElements = 8;
+  if (NumElements == 0 || FromVT.getVectorNumElements() == NumElements ||
+      FromVT.getVectorNumElements() % NumElements != 0)
+    return SDValue();
+
+  SDLoc DL(St);
+  // Details about the old store
+  SDValue Ch = St->getChain();
+  SDValue BasePtr = St->getBasePtr();
+  unsigned Alignment = St->getOriginalAlignment();
+  MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
+  AAMDNodes AAInfo = St->getAAInfo();
+
+  EVT NewFromVT = EVT::getVectorVT(*DAG.getContext(), FromEltVT, NumElements);
+  EVT NewToVT = EVT::getVectorVT(*DAG.getContext(), ToEltVT, NumElements);
+
+  SmallVector<SDValue, 4> Stores;
+  for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
+    unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
+    SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
+
+    SDValue Extract =
+        DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
+                    DAG.getConstant(i * NumElements, DL, MVT::i32));
+    SDValue Store = DAG.getTruncStore(
+        Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
+        NewToVT, Alignment, MMOFlags, AAInfo);
+    Stores.push_back(Store);
+  }
+  return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
+}
+
+/// PerformSTORECombine - Target-specific dag combine xforms for
+/// ISD::STORE.
+static SDValue PerformSTORECombine(SDNode *N,
+                                   TargetLowering::DAGCombinerInfo &DCI,
+                                   const ARMSubtarget *Subtarget) {
+  StoreSDNode *St = cast<StoreSDNode>(N);
+  if (St->isVolatile())
+    return SDValue();
+  SDValue StVal = St->getValue();
+  EVT VT = StVal.getValueType();
+
+  if (Subtarget->hasNEON())
+    if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
+      return Store;
+
+  if (Subtarget->hasMVEIntegerOps())
+    if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
+      return NewToken;
 
   if (!ISD::isNormalStore(St))
     return SDValue();
@@ -13196,7 +13327,7 @@ static SDValue PerformSTORECombine(SDNode *N,
   }
 
   // If this is a legal vector store, try to combine it into a VST1_UPD.
-  if (ISD::isNormalStore(N) && VT.isVector() &&
+  if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
       DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
     return CombineBaseUpdate(N, DCI);
 
@@ -13564,6 +13695,71 @@ static SDValue PerformShiftCombine(SDNode *N,
   return SDValue();
 }
 
+// Look for a sign/zero extend of a larger than legal load. This can be split
+// into two extending loads, which are simpler to deal with than an arbitrary
+// sign extend.
+static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
+  SDValue N0 = N->getOperand(0);
+  if (N0.getOpcode() != ISD::LOAD)
+    return SDValue();
+  LoadSDNode *LD = cast<LoadSDNode>(N0.getNode());
+  if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
+      LD->getExtensionType() != ISD::NON_EXTLOAD)
+    return SDValue();
+  EVT FromVT = LD->getValueType(0);
+  EVT ToVT = N->getValueType(0);
+  if (!ToVT.isVector())
+    return SDValue();
+  assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements());
+  EVT ToEltVT = ToVT.getVectorElementType();
+  EVT FromEltVT = FromVT.getVectorElementType();
+
+  unsigned NumElements = 0;
+  if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
+    NumElements = 4;
+  if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
+    NumElements = 8;
+  if (NumElements == 0 ||
+      FromVT.getVectorNumElements() == NumElements ||
+      FromVT.getVectorNumElements() % NumElements != 0 ||
+      !isPowerOf2_32(NumElements))
+    return SDValue();
+
+  SDLoc DL(LD);
+  // Details about the old load
+  SDValue Ch = LD->getChain();
+  SDValue BasePtr = LD->getBasePtr();
+  unsigned Alignment = LD->getOriginalAlignment();
+  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
+  AAMDNodes AAInfo = LD->getAAInfo();
+
+  ISD::LoadExtType NewExtType =
+      N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
+  SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
+  EVT NewFromVT = FromVT.getHalfNumVectorElementsVT(*DAG.getContext());
+  EVT NewToVT = ToVT.getHalfNumVectorElementsVT(*DAG.getContext());
+  unsigned NewOffset = NewFromVT.getSizeInBits() / 8;
+  SDValue NewPtr = DAG.getObjectPtrOffset(DL, BasePtr, NewOffset);
+
+  // Split the load in half, each side of which is extended separately. This
+  // is good enough, as legalisation will take it from there. They are either
+  // already legal or they will be split further into something that is
+  // legal.
+  SDValue NewLoad1 =
+      DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, BasePtr, Offset,
+                  LD->getPointerInfo(), NewFromVT, Alignment, MMOFlags, AAInfo);
+  SDValue NewLoad2 =
+      DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
+                  LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
+                  Alignment, MMOFlags, AAInfo);
+
+  SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+                                 SDValue(NewLoad1.getNode(), 1),
+                                 SDValue(NewLoad2.getNode(), 1));
+  DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, NewLoad1, NewLoad2);
+}
+
 /// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
 /// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
 static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
@@ -13601,6 +13797,10 @@ static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
     }
   }
 
+  if (ST->hasMVEIntegerOps())
+    if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
+      return NewLoad;
+
   return SDValue();
 }
 
@@ -14106,7 +14306,7 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
   case ARMISD::BFI:     return PerformBFICombine(N, DCI);
   case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
   case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
-  case ISD::STORE:      return PerformSTORECombine(N, DCI);
+  case ISD::STORE:      return PerformSTORECombine(N, DCI, Subtarget);
   case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
   case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
   case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
@@ -14135,6 +14335,8 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
     return PerformVLDCombine(N, DCI);
   case ARMISD::BUILD_VECTOR:
     return PerformARMBUILD_VECTORCombine(N, DCI);
+  case ARMISD::PREDICATE_CAST:
+    return PerformPREDICATE_CASTCombine(N, DCI);
   case ARMISD::SMULWB: {
     unsigned BitWidth = N->getValueType(0).getSizeInBits();
     APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
@@ -16705,7 +16907,8 @@ ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy,
 
   // Avoid over-aligning vector parameters. It would require realigning the
   // stack and waste space for no real benefit.
-  return std::min(DL.getABITypeAlignment(ArgTy), DL.getStackAlignment());
+  return std::min(DL.getABITypeAlignment(ArgTy),
+                  (unsigned)DL.getStackAlignment().value());
 }
 
 /// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index d7586bca2ba00..b8ce4d65f757e 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -725,8 +725,8 @@ class VectorType;
     void lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
                   SelectionDAG &DAG) const;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                           SmallVectorImpl<SDNode *> &Created) const override;
diff --git a/llvm/lib/Target/ARM/ARMInstrFormats.td b/llvm/lib/Target/ARM/ARMInstrFormats.td
index 043b54a7deb5d..dd14ebda70725 100644
--- a/llvm/lib/Target/ARM/ARMInstrFormats.td
+++ b/llvm/lib/Target/ARM/ARMInstrFormats.td
@@ -408,6 +408,8 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   // mnemonic (when not in an IT block) or preclude it (when in an IT block).
   bit thumbArithFlagSetting = 0;
 
+  bit invalidForTailPredication = 0;
+
   // If this is a pseudo instruction, mark it isCodeGenOnly.
   let isCodeGenOnly = !eq(!cast<string>(f), "Pseudo");
 
@@ -419,6 +421,7 @@ class InstTemplate<AddrMode am, int sz, IndexMode im,
   let TSFlags{14}    = canXformTo16Bit;
   let TSFlags{18-15} = D.Value;
   let TSFlags{19}    = thumbArithFlagSetting;
+  let TSFlags{20}    = invalidForTailPredication;
 
   let Constraints = cstr;
   let Itinerary = itin;
@@ -462,6 +465,7 @@ class AsmPseudoInst<string asm, dag iops, dag oops = (outs)>
   let isCodeGenOnly = 0; // So we get asm matcher for it.
   let AsmString = asm;
   let isPseudo = 1;
+  let hasNoSchedulingInfo = 1;
 }
 
 class ARMAsmPseudo<string asm, dag iops, dag oops = (outs)>
diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.td b/llvm/lib/Target/ARM/ARMInstrInfo.td
index 5396ffb7af5ca..e25260d8b4750 100644
--- a/llvm/lib/Target/ARM/ARMInstrInfo.td
+++ b/llvm/lib/Target/ARM/ARMInstrInfo.td
@@ -1967,7 +1967,7 @@ multiclass AI_str1nopc<bit isByte, string opc, InstrItinClass iii,
 /// the function.  The first operand is the ID# for this instruction, the second
 /// is the index into the MachineConstantPool that this is, the third is the
 /// size in bytes of this constant pool entry.
-let hasSideEffects = 0, isNotDuplicable = 1 in
+let hasSideEffects = 0, isNotDuplicable = 1, hasNoSchedulingInfo = 1 in
 def CONSTPOOL_ENTRY :
 PseudoInst<(outs), (ins cpinst_operand:$instid, cpinst_operand:$cpidx,
                     i32imm:$size), NoItinerary, []>;
@@ -4895,14 +4895,13 @@ def SB : AInoP<(outs), (ins), MiscFrm, NoItinerary, "sb", "", []>,
   let hasSideEffects = 1;
 }
 
-let usesCustomInserter = 1, Defs = [CPSR] in {
-
-// Pseudo instruction that combines movs + predicated rsbmi
-// to implement integer ABS
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
+  // Pseudo instruction that combines movs + predicated rsbmi
+  // to implement integer ABS
   def ABS : ARMPseudoInst<(outs GPR:$dst), (ins GPR:$src), 8, NoItinerary, []>;
 }
 
-let usesCustomInserter = 1, Defs = [CPSR] in {
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
     def COPY_STRUCT_BYVAL_I32 : PseudoInst<
       (outs), (ins GPR:$dst, GPR:$src, i32imm:$size, i32imm:$alignment),
       NoItinerary,
@@ -5110,8 +5109,8 @@ def SWPB: AIswp<1, (outs GPRnopc:$Rt),
 def CDP : ABI<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
             c_imm:$CRd, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2),
             NoItinerary, "cdp", "\t$cop, $opc1, $CRd, $CRn, $CRm, $opc2",
-            [(int_arm_cdp imm:$cop, imm:$opc1, imm:$CRd, imm:$CRn,
-                          imm:$CRm, imm:$opc2)]>,
+            [(int_arm_cdp timm:$cop, timm:$opc1, timm:$CRd, timm:$CRn,
+                          timm:$CRm, timm:$opc2)]>,
             Requires<[IsARM,PreV8]> {
   bits<4> opc1;
   bits<4> CRn;
@@ -5134,8 +5133,8 @@ def CDP : ABI<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
 def CDP2 : ABXI<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
                c_imm:$CRd, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2),
                NoItinerary, "cdp2\t$cop, $opc1, $CRd, $CRn, $CRm, $opc2",
-               [(int_arm_cdp2 imm:$cop, imm:$opc1, imm:$CRd, imm:$CRn,
-                              imm:$CRm, imm:$opc2)]>,
+               [(int_arm_cdp2 timm:$cop, timm:$opc1, timm:$CRd, timm:$CRn,
+                              timm:$CRm, timm:$opc2)]>,
                Requires<[IsARM,PreV8]> {
   let Inst{31-28} = 0b1111;
   bits<4> opc1;
@@ -5314,15 +5313,15 @@ multiclass LdSt2Cop<bit load, bit Dbit, string asm, list<dag> pattern> {
   }
 }
 
-defm LDC   : LdStCop <1, 0, "ldc", [(int_arm_ldc imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm LDCL  : LdStCop <1, 1, "ldcl", [(int_arm_ldcl imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm LDC2  : LdSt2Cop<1, 0, "ldc2", [(int_arm_ldc2 imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
-defm LDC2L : LdSt2Cop<1, 1, "ldc2l", [(int_arm_ldc2l imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
+defm LDC   : LdStCop <1, 0, "ldc", [(int_arm_ldc timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm LDCL  : LdStCop <1, 1, "ldcl", [(int_arm_ldcl timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm LDC2  : LdSt2Cop<1, 0, "ldc2", [(int_arm_ldc2 timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
+defm LDC2L : LdSt2Cop<1, 1, "ldc2l", [(int_arm_ldc2l timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
 
-defm STC   : LdStCop <0, 0, "stc", [(int_arm_stc imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm STCL  : LdStCop <0, 1, "stcl", [(int_arm_stcl imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm STC2  : LdSt2Cop<0, 0, "stc2", [(int_arm_stc2 imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
-defm STC2L : LdSt2Cop<0, 1, "stc2l", [(int_arm_stc2l imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
+defm STC   : LdStCop <0, 0, "stc", [(int_arm_stc timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm STCL  : LdStCop <0, 1, "stcl", [(int_arm_stcl timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm STC2  : LdSt2Cop<0, 0, "stc2", [(int_arm_stc2 timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
+defm STC2L : LdSt2Cop<0, 1, "stc2l", [(int_arm_stc2l timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[IsARM,PreV8]>;
 
 } // DecoderNamespace = "CoProc"
 
@@ -5358,8 +5357,8 @@ def MCR : MovRCopro<"mcr", 0 /* from ARM core register to coprocessor */,
                     (outs),
                     (ins p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
                          c_imm:$CRm, imm0_7:$opc2),
-                    [(int_arm_mcr imm:$cop, imm:$opc1, GPR:$Rt, imm:$CRn,
-                                  imm:$CRm, imm:$opc2)]>,
+                    [(int_arm_mcr timm:$cop, timm:$opc1, GPR:$Rt, timm:$CRn,
+                                  timm:$CRm, timm:$opc2)]>,
                     ComplexDeprecationPredicate<"MCR">;
 def : ARMInstAlias<"mcr${p} $cop, $opc1, $Rt, $CRn, $CRm",
                    (MCR p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
@@ -5372,8 +5371,8 @@ def : ARMInstAlias<"mrc${p} $cop, $opc1, $Rt, $CRn, $CRm",
                    (MRC GPRwithAPSR:$Rt, p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
                         c_imm:$CRm, 0, pred:$p)>;
 
-def : ARMPat<(int_arm_mrc imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
-             (MRC imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
+def : ARMPat<(int_arm_mrc timm:$cop, timm:$opc1, timm:$CRn, timm:$CRm, timm:$opc2),
+             (MRC p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2)>;
 
 class MovRCopro2<string opc, bit direction, dag oops, dag iops,
                  list<dag> pattern>
@@ -5404,8 +5403,8 @@ def MCR2 : MovRCopro2<"mcr2", 0 /* from ARM core register to coprocessor */,
                       (outs),
                       (ins p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
                            c_imm:$CRm, imm0_7:$opc2),
-                      [(int_arm_mcr2 imm:$cop, imm:$opc1, GPR:$Rt, imm:$CRn,
-                                     imm:$CRm, imm:$opc2)]>,
+                      [(int_arm_mcr2 timm:$cop, timm:$opc1, GPR:$Rt, timm:$CRn,
+                                     timm:$CRm, timm:$opc2)]>,
                       Requires<[IsARM,PreV8]>;
 def : ARMInstAlias<"mcr2 $cop, $opc1, $Rt, $CRn, $CRm",
                    (MCR2 p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
@@ -5419,9 +5418,9 @@ def : ARMInstAlias<"mrc2 $cop, $opc1, $Rt, $CRn, $CRm",
                    (MRC2 GPRwithAPSR:$Rt, p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
                          c_imm:$CRm, 0)>;
 
-def : ARMV5TPat<(int_arm_mrc2 imm:$cop, imm:$opc1, imm:$CRn,
-                              imm:$CRm, imm:$opc2),
-                (MRC2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
+def : ARMV5TPat<(int_arm_mrc2 timm:$cop, timm:$opc1, timm:$CRn,
+                              timm:$CRm, timm:$opc2),
+                (MRC2 p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2)>;
 
 class MovRRCopro<string opc, bit direction, dag oops, dag iops, list<dag>
                  pattern = []>
@@ -5447,8 +5446,8 @@ class MovRRCopro<string opc, bit direction, dag oops, dag iops, list<dag>
 def MCRR : MovRRCopro<"mcrr", 0 /* from ARM core register to coprocessor */,
                       (outs), (ins p_imm:$cop, imm0_15:$opc1, GPRnopc:$Rt,
                       GPRnopc:$Rt2, c_imm:$CRm),
-                      [(int_arm_mcrr imm:$cop, imm:$opc1, GPRnopc:$Rt,
-                                     GPRnopc:$Rt2, imm:$CRm)]>;
+                      [(int_arm_mcrr timm:$cop, timm:$opc1, GPRnopc:$Rt,
+                                     GPRnopc:$Rt2, timm:$CRm)]>;
 def MRRC : MovRRCopro<"mrrc", 1 /* from coprocessor to ARM core register */,
                       (outs GPRnopc:$Rt, GPRnopc:$Rt2),
                       (ins p_imm:$cop, imm0_15:$opc1, c_imm:$CRm), []>;
@@ -5480,8 +5479,8 @@ class MovRRCopro2<string opc, bit direction, dag oops, dag iops,
 def MCRR2 : MovRRCopro2<"mcrr2", 0 /* from ARM core register to coprocessor */,
                         (outs), (ins p_imm:$cop, imm0_15:$opc1, GPRnopc:$Rt,
                         GPRnopc:$Rt2, c_imm:$CRm),
-                        [(int_arm_mcrr2 imm:$cop, imm:$opc1, GPRnopc:$Rt,
-                                        GPRnopc:$Rt2, imm:$CRm)]>;
+                        [(int_arm_mcrr2 timm:$cop, timm:$opc1, GPRnopc:$Rt,
+                                        GPRnopc:$Rt2, timm:$CRm)]>;
 
 def MRRC2 : MovRRCopro2<"mrrc2", 1 /* from coprocessor to ARM core register */,
                        (outs GPRnopc:$Rt, GPRnopc:$Rt2),
@@ -5604,12 +5603,12 @@ def MSRbanked : ABI<0b0001, (outs), (ins banked_reg:$banked, GPRnopc:$Rn),
 
 def win__chkstk : SDNode<"ARMISD::WIN__CHKSTK", SDTNone,
                       [SDNPHasChain, SDNPSideEffect]>;
-let usesCustomInserter = 1, Uses = [R4], Defs = [R4, SP] in
+let usesCustomInserter = 1, Uses = [R4], Defs = [R4, SP], hasNoSchedulingInfo = 1 in
   def WIN__CHKSTK : PseudoInst<(outs), (ins), NoItinerary, [(win__chkstk)]>;
 
 def win__dbzchk : SDNode<"ARMISD::WIN__DBZCHK", SDT_WIN__DBZCHK,
                          [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
-let usesCustomInserter = 1, Defs = [CPSR] in
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in
   def WIN__DBZCHK : PseudoInst<(outs), (ins tGPR:$divisor), NoItinerary,
                                [(win__dbzchk tGPR:$divisor)]>;
 
@@ -6156,10 +6155,10 @@ def : InstAlias<"umull${s}${p} $RdLo, $RdHi, $Rn, $Rm",
 def ITasm : ARMAsmPseudo<"it$mask $cc", (ins it_pred:$cc, it_mask:$mask)>,
          ComplexDeprecationPredicate<"IT">;
 
-let mayLoad = 1, mayStore =1, hasSideEffects = 1 in
+let mayLoad = 1, mayStore =1, hasSideEffects = 1, hasNoSchedulingInfo = 1 in
 def SPACE : PseudoInst<(outs GPR:$Rd), (ins i32imm:$size, GPR:$Rn),
                        NoItinerary,
-                       [(set GPR:$Rd, (int_arm_space imm:$size, GPR:$Rn))]>;
+                       [(set GPR:$Rd, (int_arm_space timm:$size, GPR:$Rn))]>;
 
 //===----------------------------------
 // Atomic cmpxchg for -O0
@@ -6199,4 +6198,5 @@ def CompilerBarrier : PseudoInst<(outs), (ins i32imm:$ordering), NoItinerary,
   let hasSideEffects = 1;
   let Size = 0;
   let AsmString = "@ COMPILER BARRIER";
+  let hasNoSchedulingInfo = 1;
 }
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index 93c976a85e1f3..3ba23d9812cf5 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -160,7 +160,8 @@ class TMemImm7ShiftOffsetAsmOperand<int shift> : AsmOperandClass {
   let RenderMethod = "addMemImmOffsetOperands";
 }
 
-class taddrmode_imm7<int shift> : MemOperand {
+class taddrmode_imm7<int shift> : MemOperand,
+    ComplexPattern<i32, 2, "SelectTAddrModeImm7<"#shift#">", []>  {
   let ParserMatchClass = TMemImm7ShiftOffsetAsmOperand<shift>;
   // They are printed the same way as the T2 imm8 version
   let PrintMethod = "printT2AddrModeImm8Operand<false>";
@@ -452,16 +453,16 @@ def MVE_ASRLr   : MVE_ScalarShiftDRegReg<"asrl",    0b1,  [(set tGPREven:$RdaLo,
                                         tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
 def MVE_ASRLi   : MVE_ScalarShiftDRegImm<"asrl",    0b10, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
                                         (ARMasrl tGPREven:$RdaLo_src,
-                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
+                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
 def MVE_LSLLr   : MVE_ScalarShiftDRegReg<"lsll",    0b0,  [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
                                         (ARMlsll tGPREven:$RdaLo_src,
                                         tGPROdd:$RdaHi_src, rGPR:$Rm))]>;
 def MVE_LSLLi   : MVE_ScalarShiftDRegImm<"lsll",    0b00, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
                                         (ARMlsll tGPREven:$RdaLo_src,
-                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
+                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
 def MVE_LSRL    : MVE_ScalarShiftDRegImm<"lsrl",    0b01, ?, [(set tGPREven:$RdaLo, tGPROdd:$RdaHi,
                                         (ARMlsrl tGPREven:$RdaLo_src,
-                                        tGPROdd:$RdaHi_src, (i32 imm:$imm)))]>;
+                                        tGPROdd:$RdaHi_src, (i32 long_shift:$imm)))]>;
 
 def MVE_SQRSHRL : MVE_ScalarShiftDRegRegWithSat<"sqrshrl", 0b1>;
 def MVE_SQSHLL  : MVE_ScalarShiftDRegImm<"sqshll",  0b11, 0b1>;
@@ -506,6 +507,7 @@ class MVE_VABAV<string suffix, bit U, bits<2> size, list<dag> pattern=[]>
   let Inst{5} = Qm{3};
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b1;
+  let invalidForTailPredication = 1;
 }
 
 def MVE_VABAVs8  : MVE_VABAV<"s8", 0b0, 0b00>;
@@ -532,6 +534,7 @@ class MVE_VADDV<string iname, string suffix, dag iops, string cstr,
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VADDV_A<string suffix, bit U, bits<2> size,
@@ -582,6 +585,7 @@ class MVE_VADDLV<string iname, string suffix, dag iops, string cstr,
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VADDLV_A<string suffix, bit U, list<dag> pattern=[]> {
@@ -619,6 +623,7 @@ class MVE_VMINMAXNMV<string iname, string suffix, bit sz,
   let Inst{0} = 0b0;
 
   let Predicates = [HasMVEFloat];
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VMINMAXNMV_fty<string iname, bit bit_7, list<dag> pattern=[]> {
@@ -655,6 +660,7 @@ class MVE_VMINMAXV<string iname, string suffix, bit U, bits<2> size,
   let Inst{6-5} = 0b00;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = 0b0;
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VMINMAXV_ty<string iname, bit bit_7, list<dag> pattern=[]> {
@@ -727,6 +733,7 @@ class MVE_VMLAMLSDAV<string iname, string suffix, dag iops, string cstr,
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = bit_0;
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VMLAMLSDAV_A<string iname, string x, string suffix,
@@ -802,6 +809,7 @@ class MVE_VMLALDAVBase<string iname, string suffix, dag iops, string cstr,
   let Inst{5} = A;
   let Inst{3-1} = Qm{2-0};
   let Inst{0} = bit_0;
+  let invalidForTailPredication = 1;
 }
 
 multiclass MVE_VMLALDAVBase_A<string iname, string x, string suffix,
@@ -1057,6 +1065,13 @@ def MVE_VREV32_16 : MVE_VREV<"vrev32", "16", 0b01, 0b01>;
 
 def MVE_VREV16_8  : MVE_VREV<"vrev16", "8", 0b00, 0b10>;
 
+let Predicates = [HasMVEInt] in {
+  def : Pat<(v8i16 (bswap (v8i16 MQPR:$src))),
+            (v8i16 (MVE_VREV16_8 (v8i16 MQPR:$src)))>;
+  def : Pat<(v4i32 (bswap (v4i32 MQPR:$src))),
+            (v4i32 (MVE_VREV32_8 (v4i32 MQPR:$src)))>;
+}
+
 let Predicates = [HasMVEInt] in {
   def : Pat<(v4i32 (ARMvrev64 (v4i32 MQPR:$src))),
             (v4i32 (MVE_VREV64_32 (v4i32 MQPR:$src)))>;
@@ -1756,6 +1771,15 @@ def MVE_VCLZs8  : MVE_VCLSCLZ<"vclz", "i8",  0b00, 0b1>;
 def MVE_VCLZs16 : MVE_VCLSCLZ<"vclz", "i16", 0b01, 0b1>;
 def MVE_VCLZs32 : MVE_VCLSCLZ<"vclz", "i32", 0b10, 0b1>;
 
+let Predicates = [HasMVEInt] in {
+  def : Pat<(v16i8 ( ctlz (v16i8 MQPR:$val1))),
+            (v16i8 ( MVE_VCLZs8 (v16i8 MQPR:$val1)))>;
+  def : Pat<(v4i32 ( ctlz (v4i32 MQPR:$val1))),
+            (v4i32 ( MVE_VCLZs32 (v4i32 MQPR:$val1)))>;
+  def : Pat<(v8i16 ( ctlz (v8i16 MQPR:$val1))),
+            (v8i16 ( MVE_VCLZs16 (v8i16 MQPR:$val1)))>;
+}
+
 class MVE_VABSNEG_int<string iname, string suffix, bits<2> size, bit negate,
                       list<dag> pattern=[]>
   : MVEIntSingleSrc<iname, suffix, size, pattern> {
@@ -3763,6 +3787,17 @@ def MVE_VBRSR8  : MVE_VBRSR<"vbrsr", "8", 0b00>;
 def MVE_VBRSR16 : MVE_VBRSR<"vbrsr", "16", 0b01>;
 def MVE_VBRSR32 : MVE_VBRSR<"vbrsr", "32", 0b10>;
 
+let Predicates = [HasMVEInt] in {
+  def : Pat<(v16i8 ( bitreverse (v16i8 MQPR:$val1))),
+            (v16i8 ( MVE_VBRSR8 (v16i8 MQPR:$val1), (t2MOVi (i32 8)) ))>;
+
+  def : Pat<(v4i32 ( bitreverse (v4i32 MQPR:$val1))),
+            (v4i32 ( MVE_VBRSR32 (v4i32 MQPR:$val1), (t2MOVi (i32 32)) ))>;
+
+  def : Pat<(v8i16 ( bitreverse (v8i16 MQPR:$val1))),
+            (v8i16 ( MVE_VBRSR16 (v8i16 MQPR:$val1), (t2MOVi (i32 16)) ))>;
+}
+
 class MVE_VMUL_qr_int<string iname, string suffix,
                       bits<2> size, list<dag> pattern=[]>
   : MVE_qDest_rSrc<iname, suffix, "", pattern> {
@@ -3951,6 +3986,7 @@ def MVE_VDWDUPu8  : MVE_VxWDUP<"vdwdup", "u8",  0b00, 0b1>;
 def MVE_VDWDUPu16 : MVE_VxWDUP<"vdwdup", "u16", 0b01, 0b1>;
 def MVE_VDWDUPu32 : MVE_VxWDUP<"vdwdup", "u32", 0b10, 0b1>;
 
+let hasSideEffects = 1 in
 class MVE_VCTP<string suffix, bits<2> size, list<dag> pattern=[]>
   : MVE_p<(outs VCCR:$P0), (ins rGPR:$Rn), NoItinerary, "vctp", suffix,
           "$Rn", vpred_n, "", pattern> {
@@ -4526,7 +4562,7 @@ class MVE_VPT<string suffix, bits<2> size, dag iops, string asm, list<dag> patte
   let Inst{7} = fc{0};
   let Inst{4} = 0b0;
 
-  let Defs = [VPR, P0];
+  let Defs = [VPR];
 }
 
 class MVE_VPTt1<string suffix, bits<2> size, dag iops>
@@ -4542,7 +4578,7 @@ class MVE_VPTt1<string suffix, bits<2> size, dag iops>
 
 class MVE_VPTt1i<string suffix, bits<2> size>
  : MVE_VPTt1<suffix, size,
-           (ins vpt_mask:$Mk, pred_basic_i:$fc, MQPR:$Qn, MQPR:$Qm)> {
+           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_i:$fc)> {
   let Inst{12} = 0b0;
   let Inst{0} = 0b0;
 }
@@ -4553,7 +4589,7 @@ def MVE_VPTv16i8 : MVE_VPTt1i<"i8", 0b00>;
 
 class MVE_VPTt1u<string suffix, bits<2> size>
  : MVE_VPTt1<suffix, size,
-           (ins vpt_mask:$Mk, pred_basic_u:$fc, MQPR:$Qn, MQPR:$Qm)> {
+           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_u:$fc)> {
   let Inst{12} = 0b0;
   let Inst{0} = 0b1;
 }
@@ -4564,7 +4600,7 @@ def MVE_VPTv16u8 : MVE_VPTt1u<"u8", 0b00>;
 
 class MVE_VPTt1s<string suffix, bits<2> size>
  : MVE_VPTt1<suffix, size,
-           (ins vpt_mask:$Mk, pred_basic_s:$fc, MQPR:$Qn, MQPR:$Qm)> {
+           (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_s:$fc)> {
   let Inst{12} = 0b1;
 }
 
@@ -4586,7 +4622,7 @@ class MVE_VPTt2<string suffix, bits<2> size, dag iops>
 
 class MVE_VPTt2i<string suffix, bits<2> size>
   : MVE_VPTt2<suffix, size,
-            (ins vpt_mask:$Mk, pred_basic_i:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
+            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_i:$fc)> {
   let Inst{12} = 0b0;
   let Inst{5} = 0b0;
 }
@@ -4597,7 +4633,7 @@ def MVE_VPTv16i8r : MVE_VPTt2i<"i8", 0b00>;
 
 class MVE_VPTt2u<string suffix, bits<2> size>
   : MVE_VPTt2<suffix, size,
-            (ins vpt_mask:$Mk, pred_basic_u:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
+            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_u:$fc)> {
   let Inst{12} = 0b0;
   let Inst{5} = 0b1;
 }
@@ -4608,7 +4644,7 @@ def MVE_VPTv16u8r : MVE_VPTt2u<"u8", 0b00>;
 
 class MVE_VPTt2s<string suffix, bits<2> size>
   : MVE_VPTt2<suffix, size,
-            (ins vpt_mask:$Mk, pred_basic_s:$fc, MQPR:$Qn, GPRwithZR:$Rm)> {
+            (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_s:$fc)> {
   let Inst{12} = 0b1;
 }
 
@@ -4637,12 +4673,12 @@ class MVE_VPTf<string suffix, bit size, dag iops, string asm, list<dag> pattern=
   let Inst{7} = fc{0};
   let Inst{4} = 0b0;
 
-  let Defs = [P0];
+  let Defs = [VPR];
   let Predicates = [HasMVEFloat];
 }
 
 class MVE_VPTft1<string suffix, bit size>
-  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, pred_basic_fp:$fc, MQPR:$Qn, MQPR:$Qm),
+  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, MQPR:$Qm, pred_basic_fp:$fc),
           "$fc, $Qn, $Qm"> {
   bits<3> fc;
   bits<4> Qm;
@@ -4657,7 +4693,7 @@ def MVE_VPTv4f32         : MVE_VPTft1<"f32", 0b0>;
 def MVE_VPTv8f16         : MVE_VPTft1<"f16", 0b1>;
 
 class MVE_VPTft2<string suffix, bit size>
-  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, pred_basic_fp:$fc, MQPR:$Qn, GPRwithZR:$Rm),
+  : MVE_VPTf<suffix, size, (ins vpt_mask:$Mk, MQPR:$Qn, GPRwithZR:$Rm, pred_basic_fp:$fc),
           "$fc, $Qn, $Rm"> {
   bits<3> fc;
   bits<4> Rm;
@@ -4683,7 +4719,7 @@ def MVE_VPST : MVE_MI<(outs ), (ins vpt_mask:$Mk), NoItinerary,
   let Unpredictable{7} = 0b1;
   let Unpredictable{5} = 0b1;
 
-  let Defs = [P0];
+  let Defs = [VPR];
 }
 
 def MVE_VPSEL : MVE_p<(outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm), NoItinerary,
@@ -4892,6 +4928,10 @@ class MVE_vector_store_typed<ValueType Ty, Instruction RegImmInst,
                              PatFrag StoreKind, int shift>
   : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr),
         (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr)>;
+class MVE_vector_maskedstore_typed<ValueType Ty, Instruction RegImmInst,
+                                   PatFrag StoreKind, int shift>
+  : Pat<(StoreKind (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, VCCR:$pred),
+        (RegImmInst (Ty MQPR:$val), t2addrmode_imm7<shift>:$addr, (i32 1), VCCR:$pred)>;
 
 multiclass MVE_vector_store<Instruction RegImmInst, PatFrag StoreKind,
                             int shift> {
@@ -4908,6 +4948,10 @@ class MVE_vector_load_typed<ValueType Ty, Instruction RegImmInst,
                             PatFrag LoadKind, int shift>
   : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr)),
         (Ty (RegImmInst t2addrmode_imm7<shift>:$addr))>;
+class MVE_vector_maskedload_typed<ValueType Ty, Instruction RegImmInst,
+                                  PatFrag LoadKind, int shift>
+  : Pat<(Ty (LoadKind t2addrmode_imm7<shift>:$addr, VCCR:$pred, (Ty NEONimmAllZerosV))),
+        (Ty (RegImmInst t2addrmode_imm7<shift>:$addr, (i32 1), VCCR:$pred))>;
 
 multiclass MVE_vector_load<Instruction RegImmInst, PatFrag LoadKind,
                            int shift> {
@@ -4953,6 +4997,28 @@ def aligned16_post_store : PatFrag<(ops node:$val, node:$ptr, node:$offset),
   return cast<StoreSDNode>(N)->getAlignment() >= 2;
 }]>;
 
+def alignedmaskedload32 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
+                                  (masked_ld node:$ptr, node:$pred, node:$passthru), [{
+  return cast<MaskedLoadSDNode>(N)->getAlignment() >= 4;
+}]>;
+def alignedmaskedload16 : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
+                                  (masked_ld node:$ptr, node:$pred, node:$passthru), [{
+  return cast<MaskedLoadSDNode>(N)->getAlignment() >= 2;
+}]>;
+def maskedload : PatFrag<(ops node:$ptr, node:$pred, node:$passthru),
+                         (masked_ld node:$ptr, node:$pred, node:$passthru)>;
+
+def alignedmaskedstore32 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
+                                   (masked_st node:$val, node:$ptr, node:$pred), [{
+  return cast<MaskedStoreSDNode>(N)->getAlignment() >= 4;
+}]>;
+def alignedmaskedstore16 : PatFrag<(ops node:$val, node:$ptr, node:$pred),
+                                   (masked_st node:$val, node:$ptr, node:$pred), [{
+  return cast<MaskedStoreSDNode>(N)->getAlignment() >= 2;
+}]>;
+def maskedstore : PatFrag<(ops node:$val, node:$ptr, node:$pred),
+                          (masked_st node:$val, node:$ptr, node:$pred)>;
+
 let Predicates = [HasMVEInt, IsLE] in {
   // Stores
   defm : MVE_vector_store<MVE_VSTRBU8, byte_alignedstore, 0>;
@@ -4971,6 +5037,26 @@ let Predicates = [HasMVEInt, IsLE] in {
   defm : MVE_vector_offset_store<MVE_VSTRHU16_post, aligned16_post_store, 1>;
   defm : MVE_vector_offset_store<MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
   defm : MVE_vector_offset_store<MVE_VSTRWU32_post, aligned32_post_store, 2>;
+
+  // Unaligned masked stores (aligned are below)
+  def : Pat<(maskedstore (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 MQPR:$val, t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v4f32 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 MQPR:$val, t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 MQPR:$val, t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v8f16 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 MQPR:$val, t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+
+  // Unaligned masked loads
+  def : Pat<(v4i32 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v4i32 NEONimmAllZerosV))),
+            (v4i32 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>;
+  def : Pat<(v4f32 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v4f32 NEONimmAllZerosV))),
+            (v4f32 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>;
+  def : Pat<(v8i16 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v8i16 NEONimmAllZerosV))),
+            (v8i16 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>;
+  def : Pat<(v8f16 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v8f16 NEONimmAllZerosV))),
+            (v8f16 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred))>;
 }
 
 let Predicates = [HasMVEInt, IsBE] in {
@@ -5025,8 +5111,41 @@ let Predicates = [HasMVEInt, IsBE] in {
   def : MVE_vector_offset_store_typed<v4i32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
   def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_pre, aligned32_pre_store, 2>;
   def : MVE_vector_offset_store_typed<v4f32, MVE_VSTRWU32_post, aligned32_post_store, 2>;
+
+  // Unaligned masked stores (aligned are below)
+  def : Pat<(maskedstore (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v4f32 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 (MVE_VREV32_8 MQPR:$val), t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  def : Pat<(maskedstore (v8f16 MQPR:$val), t2addrmode_imm7<0>:$addr, VCCR:$pred),
+            (MVE_VSTRBU8 (MVE_VREV16_8 MQPR:$val), t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)>;
+  // Unaligned masked loads
+  def : Pat<(v4i32 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v4i32 NEONimmAllZerosV))),
+            (v4i32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)))>;
+  def : Pat<(v4f32 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v4f32 NEONimmAllZerosV))),
+            (v4f32 (MVE_VREV32_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)))>;
+  def : Pat<(v8i16 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v8i16 NEONimmAllZerosV))),
+            (v8i16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)))>;
+  def : Pat<(v8f16 (maskedload t2addrmode_imm7<0>:$addr, VCCR:$pred, (v8f16 NEONimmAllZerosV))),
+            (v8f16 (MVE_VREV16_8 (MVE_VLDRBU8 t2addrmode_imm7<0>:$addr, (i32 1), VCCR:$pred)))>;
 }
 
+let Predicates = [HasMVEInt] in {
+  // Aligned masked store, shared between LE and BE
+  def : MVE_vector_maskedstore_typed<v16i8, MVE_VSTRBU8, maskedstore, 0>;
+  def : MVE_vector_maskedstore_typed<v8i16, MVE_VSTRHU16, alignedmaskedstore16, 1>;
+  def : MVE_vector_maskedstore_typed<v8f16, MVE_VSTRHU16, alignedmaskedstore16, 1>;
+  def : MVE_vector_maskedstore_typed<v4i32, MVE_VSTRWU32, alignedmaskedstore32, 2>;
+  def : MVE_vector_maskedstore_typed<v4f32, MVE_VSTRWU32, alignedmaskedstore32, 2>;
+  // Aligned masked loads
+  def : MVE_vector_maskedload_typed<v16i8, MVE_VLDRBU8, maskedload, 0>;
+  def : MVE_vector_maskedload_typed<v8i16, MVE_VLDRHU16, alignedmaskedload16, 1>;
+  def : MVE_vector_maskedload_typed<v8f16, MVE_VLDRHU16, alignedmaskedload16, 1>;
+  def : MVE_vector_maskedload_typed<v4i32, MVE_VLDRWU32, alignedmaskedload32, 2>;
+  def : MVE_vector_maskedload_typed<v4f32, MVE_VLDRWU32, alignedmaskedload32, 2>;
+}
 
 // Widening/Narrowing Loads/Stores
 
@@ -5040,12 +5159,12 @@ let MinAlignment = 2 in {
 }
 
 let Predicates = [HasMVEInt] in {
-  def : Pat<(truncstorevi8 (v8i16 MQPR:$val), t2addrmode_imm7<0>:$addr),
-            (MVE_VSTRB16 MQPR:$val, t2addrmode_imm7<0>:$addr)>;
-  def : Pat<(truncstorevi8 (v4i32 MQPR:$val), t2addrmode_imm7<0>:$addr),
-            (MVE_VSTRB32 MQPR:$val, t2addrmode_imm7<0>:$addr)>;
-  def : Pat<(truncstorevi16_align2 (v4i32 MQPR:$val), t2addrmode_imm7<1>:$addr),
-            (MVE_VSTRH32 MQPR:$val, t2addrmode_imm7<1>:$addr)>;
+  def : Pat<(truncstorevi8 (v8i16 MQPR:$val), taddrmode_imm7<0>:$addr),
+            (MVE_VSTRB16 MQPR:$val, taddrmode_imm7<0>:$addr)>;
+  def : Pat<(truncstorevi8 (v4i32 MQPR:$val), taddrmode_imm7<0>:$addr),
+            (MVE_VSTRB32 MQPR:$val, taddrmode_imm7<0>:$addr)>;
+  def : Pat<(truncstorevi16_align2 (v4i32 MQPR:$val), taddrmode_imm7<1>:$addr),
+            (MVE_VSTRH32 MQPR:$val, taddrmode_imm7<1>:$addr)>;
 
   def : Pat<(post_truncstvi8 (v8i16 MQPR:$Rt), tGPR:$Rn, t2am_imm7_offset<0>:$addr),
             (MVE_VSTRB16_post MQPR:$Rt, tGPR:$Rn, t2am_imm7_offset<0>:$addr)>;
@@ -5087,9 +5206,9 @@ multiclass MVEExtLoad<string DestLanes, string DestElemBits,
 }
 
 let Predicates = [HasMVEInt] in {
-  defm : MVEExtLoad<"4", "32", "8",  "B", "", t2addrmode_imm7<0>>;
-  defm : MVEExtLoad<"8", "16", "8",  "B", "", t2addrmode_imm7<0>>;
-  defm : MVEExtLoad<"4", "32", "16", "H", "_align2", t2addrmode_imm7<1>>;
+  defm : MVEExtLoad<"4", "32", "8",  "B", "", taddrmode_imm7<0>>;
+  defm : MVEExtLoad<"8", "16", "8",  "B", "", taddrmode_imm7<0>>;
+  defm : MVEExtLoad<"4", "32", "16", "H", "_align2", taddrmode_imm7<1>>;
 }
 
 
diff --git a/llvm/lib/Target/ARM/ARMInstrThumb2.td b/llvm/lib/Target/ARM/ARMInstrThumb2.td
index f8cea50408cf8..ef5d0908b771c 100644
--- a/llvm/lib/Target/ARM/ARMInstrThumb2.td
+++ b/llvm/lib/Target/ARM/ARMInstrThumb2.td
@@ -45,7 +45,8 @@ def mve_shift_imm : AsmOperandClass {
   let RenderMethod = "addImmOperands";
   let DiagnosticString = "operand must be an immediate in the range [1,32]";
 }
-def long_shift : Operand<i32> {
+def long_shift : Operand<i32>,
+                 ImmLeaf<i32, [{ return Imm > 0 && Imm <= 32; }]> {
   let ParserMatchClass = mve_shift_imm;
   let DecoderMethod = "DecodeLongShiftOperand";
 }
@@ -4085,7 +4086,7 @@ def t2LDRpci_pic : PseudoInst<(outs rGPR:$dst), (ins i32imm:$addr, pclabel:$cp),
 
 // Pseudo isntruction that combines movs + predicated rsbmi
 // to implement integer ABS
-let usesCustomInserter = 1, Defs = [CPSR] in {
+let usesCustomInserter = 1, Defs = [CPSR], hasNoSchedulingInfo = 1 in {
 def t2ABS : PseudoInst<(outs rGPR:$dst), (ins rGPR:$src),
                        NoItinerary, []>, Requires<[IsThumb2]>;
 }
@@ -4175,15 +4176,15 @@ multiclass t2LdStCop<bits<4> op31_28, bit load, bit Dbit, string asm, list<dag>
 }
 
 let DecoderNamespace = "Thumb2CoProc" in {
-defm t2LDC   : t2LdStCop<0b1110, 1, 0, "ldc", [(int_arm_ldc imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm t2LDCL  : t2LdStCop<0b1110, 1, 1, "ldcl", [(int_arm_ldcl imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm t2LDC2  : t2LdStCop<0b1111, 1, 0, "ldc2", [(int_arm_ldc2 imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
-defm t2LDC2L : t2LdStCop<0b1111, 1, 1, "ldc2l", [(int_arm_ldc2l imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
+defm t2LDC   : t2LdStCop<0b1110, 1, 0, "ldc", [(int_arm_ldc timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm t2LDCL  : t2LdStCop<0b1110, 1, 1, "ldcl", [(int_arm_ldcl timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm t2LDC2  : t2LdStCop<0b1111, 1, 0, "ldc2", [(int_arm_ldc2 timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
+defm t2LDC2L : t2LdStCop<0b1111, 1, 1, "ldc2l", [(int_arm_ldc2l timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
 
-defm t2STC   : t2LdStCop<0b1110, 0, 0, "stc", [(int_arm_stc imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm t2STCL  : t2LdStCop<0b1110, 0, 1, "stcl", [(int_arm_stcl imm:$cop, imm:$CRd, addrmode5:$addr)]>;
-defm t2STC2  : t2LdStCop<0b1111, 0, 0, "stc2", [(int_arm_stc2 imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
-defm t2STC2L : t2LdStCop<0b1111, 0, 1, "stc2l", [(int_arm_stc2l imm:$cop, imm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
+defm t2STC   : t2LdStCop<0b1110, 0, 0, "stc", [(int_arm_stc timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm t2STCL  : t2LdStCop<0b1110, 0, 1, "stcl", [(int_arm_stcl timm:$cop, timm:$CRd, addrmode5:$addr)]>;
+defm t2STC2  : t2LdStCop<0b1111, 0, 0, "stc2", [(int_arm_stc2 timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
+defm t2STC2L : t2LdStCop<0b1111, 0, 1, "stc2l", [(int_arm_stc2l timm:$cop, timm:$CRd, addrmode5:$addr)]>, Requires<[PreV8,IsThumb2]>;
 }
 
 
@@ -4368,8 +4369,8 @@ def t2MCR : t2MovRCopro<0b1110, "mcr", 0,
            (outs),
            (ins p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
                 c_imm:$CRm, imm0_7:$opc2),
-           [(int_arm_mcr imm:$cop, imm:$opc1, GPR:$Rt, imm:$CRn,
-                         imm:$CRm, imm:$opc2)]>,
+           [(int_arm_mcr timm:$cop, timm:$opc1, GPR:$Rt, timm:$CRn,
+                         timm:$CRm, timm:$opc2)]>,
            ComplexDeprecationPredicate<"MCR">;
 def : t2InstAlias<"mcr${p} $cop, $opc1, $Rt, $CRn, $CRm",
                   (t2MCR p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
@@ -4377,8 +4378,8 @@ def : t2InstAlias<"mcr${p} $cop, $opc1, $Rt, $CRn, $CRm",
 def t2MCR2 : t2MovRCopro<0b1111, "mcr2", 0,
              (outs), (ins p_imm:$cop, imm0_7:$opc1, GPR:$Rt, c_imm:$CRn,
                           c_imm:$CRm, imm0_7:$opc2),
-             [(int_arm_mcr2 imm:$cop, imm:$opc1, GPR:$Rt, imm:$CRn,
-                            imm:$CRm, imm:$opc2)]> {
+             [(int_arm_mcr2 timm:$cop, timm:$opc1, GPR:$Rt, timm:$CRn,
+                            timm:$CRm, timm:$opc2)]> {
   let Predicates = [IsThumb2, PreV8];
 }
 def : t2InstAlias<"mcr2${p} $cop, $opc1, $Rt, $CRn, $CRm",
@@ -4402,24 +4403,24 @@ def : t2InstAlias<"mrc2${p} $cop, $opc1, $Rt, $CRn, $CRm",
                   (t2MRC2 GPRwithAPSR:$Rt, p_imm:$cop, imm0_7:$opc1, c_imm:$CRn,
                           c_imm:$CRm, 0, pred:$p)>;
 
-def : T2v6Pat<(int_arm_mrc  imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
-              (t2MRC imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
+def : T2v6Pat<(int_arm_mrc  timm:$cop, timm:$opc1, timm:$CRn, timm:$CRm, timm:$opc2),
+              (t2MRC p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2)>;
 
-def : T2v6Pat<(int_arm_mrc2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2),
-              (t2MRC2 imm:$cop, imm:$opc1, imm:$CRn, imm:$CRm, imm:$opc2)>;
+def : T2v6Pat<(int_arm_mrc2 timm:$cop, timm:$opc1, timm:$CRn, timm:$CRm, timm:$opc2),
+              (t2MRC2 p_imm:$cop, imm0_7:$opc1, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2)>;
 
 
 /* from ARM core register to coprocessor */
 def t2MCRR : t2MovRRCopro<0b1110, "mcrr", 0, (outs),
                          (ins p_imm:$cop, imm0_15:$opc1, GPR:$Rt, GPR:$Rt2,
                          c_imm:$CRm),
-                        [(int_arm_mcrr imm:$cop, imm:$opc1, GPR:$Rt, GPR:$Rt2,
-                                       imm:$CRm)]>;
+                        [(int_arm_mcrr timm:$cop, timm:$opc1, GPR:$Rt, GPR:$Rt2,
+                                       timm:$CRm)]>;
 def t2MCRR2 : t2MovRRCopro<0b1111, "mcrr2", 0, (outs),
                           (ins p_imm:$cop, imm0_15:$opc1, GPR:$Rt, GPR:$Rt2,
                            c_imm:$CRm),
-                          [(int_arm_mcrr2 imm:$cop, imm:$opc1, GPR:$Rt,
-                                          GPR:$Rt2, imm:$CRm)]> {
+                          [(int_arm_mcrr2 timm:$cop, timm:$opc1, GPR:$Rt,
+                                          GPR:$Rt2, timm:$CRm)]> {
   let Predicates = [IsThumb2, PreV8];
 }
 
@@ -4439,8 +4440,8 @@ def t2MRRC2 : t2MovRRCopro<0b1111, "mrrc2", 1, (outs GPR:$Rt, GPR:$Rt2),
 def t2CDP : T2Cop<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
                  c_imm:$CRd, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2),
                  "cdp", "\t$cop, $opc1, $CRd, $CRn, $CRm, $opc2",
-                 [(int_arm_cdp imm:$cop, imm:$opc1, imm:$CRd, imm:$CRn,
-                               imm:$CRm, imm:$opc2)]> {
+                 [(int_arm_cdp timm:$cop, timm:$opc1, timm:$CRd, timm:$CRn,
+                               timm:$CRm, timm:$opc2)]> {
   let Inst{27-24} = 0b1110;
 
   bits<4> opc1;
@@ -4465,8 +4466,8 @@ def t2CDP : T2Cop<0b1110, (outs), (ins p_imm:$cop, imm0_15:$opc1,
 def t2CDP2 : T2Cop<0b1111, (outs), (ins p_imm:$cop, imm0_15:$opc1,
                    c_imm:$CRd, c_imm:$CRn, c_imm:$CRm, imm0_7:$opc2),
                    "cdp2", "\t$cop, $opc1, $CRd, $CRn, $CRm, $opc2",
-                   [(int_arm_cdp2 imm:$cop, imm:$opc1, imm:$CRd, imm:$CRn,
-                                  imm:$CRm, imm:$opc2)]> {
+                   [(int_arm_cdp2 timm:$cop, timm:$opc1, timm:$CRd, timm:$CRn,
+                                  timm:$CRm, timm:$opc2)]> {
   let Inst{27-24} = 0b1110;
 
   bits<4> opc1;
@@ -5087,6 +5088,7 @@ def t2BF_LabelPseudo
   : t2PseudoInst<(outs ), (ins pclabel:$cp), 0, NoItinerary, []> {
   let isTerminator = 1;
   let Predicates = [IsThumb2, HasV8_1MMainline, HasLOB];
+  let hasNoSchedulingInfo = 1;
 }
 
 def t2BFi : t2BF<(ins bflabel_u4:$b_label, bflabel_s16:$label, pred:$p),
diff --git a/llvm/lib/Target/ARM/ARMInstrVFP.td b/llvm/lib/Target/ARM/ARMInstrVFP.td
index 13351f23939f1..d3380ab7cef77 100644
--- a/llvm/lib/Target/ARM/ARMInstrVFP.td
+++ b/llvm/lib/Target/ARM/ARMInstrVFP.td
@@ -324,7 +324,7 @@ defm : VFPDTAnyInstAlias<"vpop${p}", "$r",
 // However, there is no UAL syntax for them, so we keep them around for
 // (dis)assembly only.
 multiclass vfp_ldstx_mult<string asm, bit L_bit> {
-  let Predicates = [HasFPRegs] in {
+  let Predicates = [HasFPRegs], hasNoSchedulingInfo = 1 in {
   // Unknown precision
   def XIA :
     AXXI4<(outs), (ins GPR:$Rn, pred:$p, dpr_reglist:$regs, variable_ops),
@@ -1732,7 +1732,8 @@ def VTOSHS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1110, 0b1010, 0,
 
 def VTOUHS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1111, 0b1010, 0,
                        (outs SPR:$dst), (ins SPR:$a, fbits16:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".u16.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".u16.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
@@ -1740,7 +1741,8 @@ def VTOUHS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1111, 0b1010, 0,
 
 def VTOSLS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1110, 0b1010, 1,
                        (outs SPR:$dst), (ins SPR:$a, fbits32:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".s32.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".s32.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
@@ -1748,7 +1750,8 @@ def VTOSLS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1110, 0b1010, 1,
 
 def VTOULS : AVConv1XInsS_Encode<0b11101, 0b11, 0b1111, 0b1010, 1,
                        (outs SPR:$dst), (ins SPR:$a, fbits32:$fbits),
-                 IIC_fpCVTSI, "vcvt", ".u32.f32\t$dst, $a, $fbits", []> {
+                 IIC_fpCVTSI, "vcvt", ".u32.f32\t$dst, $a, $fbits", []>,
+             Sched<[WriteFPCVT]> {
   // Some single precision VFP instructions may be executed on both NEON and
   // VFP pipelines on A8.
   let D = VFPNeonA8Domain;
@@ -2618,7 +2621,8 @@ def VSCCLRMD : VFPXI<(outs), (ins pred:$p, fp_dreglist_with_vpr:$regs, variable_
   let Inst{21-16} = 0b011111;
   let Inst{15-12} = regs{11-8};
   let Inst{11-8} = 0b1011;
-  let Inst{7-0} = regs{7-0};
+  let Inst{7-1} = regs{7-1};
+  let Inst{0} = 0;
 
   let DecoderMethod = "DecodeVSCCLRM";
 
diff --git a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
index cfd1f40871077..e1c5a9c3e223b 100644
--- a/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
+++ b/llvm/lib/Target/ARM/ARMLowOverheadLoops.cpp
@@ -34,6 +34,7 @@ using namespace llvm;
 namespace {
 
   class ARMLowOverheadLoops : public MachineFunctionPass {
+    MachineFunction           *MF = nullptr;
     const ARMBaseInstrInfo    *TII = nullptr;
     MachineRegisterInfo       *MRI = nullptr;
     std::unique_ptr<ARMBasicBlockUtils> BBUtils = nullptr;
@@ -51,27 +52,32 @@ namespace {
 
     bool runOnMachineFunction(MachineFunction &MF) override;
 
+    MachineFunctionProperties getRequiredProperties() const override {
+      return MachineFunctionProperties().set(
+          MachineFunctionProperties::Property::NoVRegs);
+    }
+
+    StringRef getPassName() const override {
+      return ARM_LOW_OVERHEAD_LOOPS_NAME;
+    }
+
+  private:
     bool ProcessLoop(MachineLoop *ML);
 
-    bool RevertNonLoops(MachineFunction &MF);
+    MachineInstr * IsSafeToDefineLR(MachineInstr *MI);
+
+    bool RevertNonLoops();
 
     void RevertWhile(MachineInstr *MI) const;
 
-    void RevertLoopDec(MachineInstr *MI) const;
+    bool RevertLoopDec(MachineInstr *MI, bool AllowFlags = false) const;
 
-    void RevertLoopEnd(MachineInstr *MI) const;
+    void RevertLoopEnd(MachineInstr *MI, bool SkipCmp = false) const;
 
     void Expand(MachineLoop *ML, MachineInstr *Start,
-                MachineInstr *Dec, MachineInstr *End, bool Revert);
-
-    MachineFunctionProperties getRequiredProperties() const override {
-      return MachineFunctionProperties().set(
-          MachineFunctionProperties::Property::NoVRegs);
-    }
+                MachineInstr *InsertPt, MachineInstr *Dec,
+                MachineInstr *End, bool Revert);
 
-    StringRef getPassName() const override {
-      return ARM_LOW_OVERHEAD_LOOPS_NAME;
-    }
   };
 }
 
@@ -80,26 +86,28 @@ char ARMLowOverheadLoops::ID = 0;
 INITIALIZE_PASS(ARMLowOverheadLoops, DEBUG_TYPE, ARM_LOW_OVERHEAD_LOOPS_NAME,
                 false, false)
 
-bool ARMLowOverheadLoops::runOnMachineFunction(MachineFunction &MF) {
-  if (!static_cast<const ARMSubtarget&>(MF.getSubtarget()).hasLOB())
+bool ARMLowOverheadLoops::runOnMachineFunction(MachineFunction &mf) {
+  const ARMSubtarget &ST = static_cast<const ARMSubtarget&>(mf.getSubtarget());
+  if (!ST.hasLOB())
     return false;
 
-  LLVM_DEBUG(dbgs() << "ARM Loops on " << MF.getName() << " ------------- \n");
+  MF = &mf;
+  LLVM_DEBUG(dbgs() << "ARM Loops on " << MF->getName() << " ------------- \n");
 
   auto &MLI = getAnalysis<MachineLoopInfo>();
-  MRI = &MF.getRegInfo();
-  TII = static_cast<const ARMBaseInstrInfo*>(
-    MF.getSubtarget().getInstrInfo());
-  BBUtils = std::unique_ptr<ARMBasicBlockUtils>(new ARMBasicBlockUtils(MF));
+  MF->getProperties().set(MachineFunctionProperties::Property::TracksLiveness);
+  MRI = &MF->getRegInfo();
+  TII = static_cast<const ARMBaseInstrInfo*>(ST.getInstrInfo());
+  BBUtils = std::unique_ptr<ARMBasicBlockUtils>(new ARMBasicBlockUtils(*MF));
   BBUtils->computeAllBlockSizes();
-  BBUtils->adjustBBOffsetsAfter(&MF.front());
+  BBUtils->adjustBBOffsetsAfter(&MF->front());
 
   bool Changed = false;
   for (auto ML : MLI) {
     if (!ML->getParentLoop())
       Changed |= ProcessLoop(ML);
   }
-  Changed |= RevertNonLoops(MF);
+  Changed |= RevertNonLoops();
   return Changed;
 }
 
@@ -108,6 +116,100 @@ static bool IsLoopStart(MachineInstr &MI) {
          MI.getOpcode() == ARM::t2WhileLoopStart;
 }
 
+template<typename T>
+static MachineInstr* SearchForDef(MachineInstr *Begin, T End, unsigned Reg) {
+  for(auto &MI : make_range(T(Begin), End)) {
+    for (auto &MO : MI.operands()) {
+      if (!MO.isReg() || !MO.isDef() || MO.getReg() != Reg)
+        continue;
+      return &MI;
+    }
+  }
+  return nullptr;
+}
+
+static MachineInstr* SearchForUse(MachineInstr *Begin,
+                                  MachineBasicBlock::iterator End,
+                                  unsigned Reg) {
+  for(auto &MI : make_range(MachineBasicBlock::iterator(Begin), End)) {
+    for (auto &MO : MI.operands()) {
+      if (!MO.isReg() || !MO.isUse() || MO.getReg() != Reg)
+        continue;
+      return &MI;
+    }
+  }
+  return nullptr;
+}
+
+// Is it safe to define LR with DLS/WLS?
+// LR can defined if it is the operand to start, because it's the same value,
+// or if it's going to be equivalent to the operand to Start.
+MachineInstr *ARMLowOverheadLoops::IsSafeToDefineLR(MachineInstr *Start) {
+
+  auto IsMoveLR = [](MachineInstr *MI, unsigned Reg) {
+    return MI->getOpcode() == ARM::tMOVr &&
+           MI->getOperand(0).getReg() == ARM::LR &&
+           MI->getOperand(1).getReg() == Reg &&
+           MI->getOperand(2).getImm() == ARMCC::AL;
+   };
+
+  MachineBasicBlock *MBB = Start->getParent();
+  unsigned CountReg = Start->getOperand(0).getReg();
+  // Walk forward and backward in the block to find the closest instructions
+  // that define LR. Then also filter them out if they're not a mov lr.
+  MachineInstr *PredLRDef = SearchForDef(Start, MBB->rend(), ARM::LR);
+  if (PredLRDef && !IsMoveLR(PredLRDef, CountReg))
+    PredLRDef = nullptr;
+
+  MachineInstr *SuccLRDef = SearchForDef(Start, MBB->end(), ARM::LR);
+  if (SuccLRDef && !IsMoveLR(SuccLRDef, CountReg))
+    SuccLRDef = nullptr;
+
+  // We've either found one, two or none mov lr instructions... Now figure out
+  // if they are performing the equilvant mov that the Start instruction will.
+  // Do this by scanning forward and backward to see if there's a def of the
+  // register holding the count value. If we find a suitable def, return it as
+  // the insert point. Later, if InsertPt != Start, then we can remove the
+  // redundant instruction.
+  if (SuccLRDef) {
+    MachineBasicBlock::iterator End(SuccLRDef);
+    if (!SearchForDef(Start, End, CountReg)) {
+      return SuccLRDef;
+    } else
+      SuccLRDef = nullptr;
+  }
+  if (PredLRDef) {
+    MachineBasicBlock::reverse_iterator End(PredLRDef);
+    if (!SearchForDef(Start, End, CountReg)) {
+      return PredLRDef;
+    } else
+      PredLRDef = nullptr;
+  }
+
+  // We can define LR because LR already contains the same value.
+  if (Start->getOperand(0).getReg() == ARM::LR)
+    return Start;
+
+  // We've found no suitable LR def and Start doesn't use LR directly. Can we
+  // just define LR anyway? 
+  const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+  LivePhysRegs LiveRegs(*TRI);
+  LiveRegs.addLiveOuts(*MBB);
+
+  // Not if we've haven't found a suitable mov and LR is live out.
+  if (LiveRegs.contains(ARM::LR))
+    return nullptr;
+
+  // If LR is not live out, we can insert the instruction if nothing else
+  // uses LR after it.
+  if (!SearchForUse(Start, MBB->end(), ARM::LR))
+    return Start;
+
+  LLVM_DEBUG(dbgs() << "ARM Loops: Failed to find suitable insertion point for"
+             << " LR\n");
+  return nullptr;
+}
+
 bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
 
   bool Changed = false;
@@ -169,11 +271,13 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
         End = &MI;
       else if (IsLoopStart(MI))
         Start = &MI;
-      else if (MI.getDesc().isCall())
+      else if (MI.getDesc().isCall()) {
         // TODO: Though the call will require LE to execute again, does this
         // mean we should revert? Always executing LE hopefully should be
         // faster than performing a sub,cmp,br or even subs,br.
         Revert = true;
+        LLVM_DEBUG(dbgs() << "ARM Loops: Found call.\n");
+      }
 
       if (!Dec || End)
         continue;
@@ -237,14 +341,20 @@ bool ARMLowOverheadLoops::ProcessLoop(MachineLoop *ML) {
     Revert = true;
   }
 
-  Expand(ML, Start, Dec, End, Revert);
+  MachineInstr *InsertPt = Revert ? nullptr : IsSafeToDefineLR(Start);
+  if (!InsertPt) {
+    LLVM_DEBUG(dbgs() << "ARM Loops: Unable to find safe insertion point.\n");
+    Revert = true;
+  } else
+    LLVM_DEBUG(dbgs() << "ARM Loops: Start insertion point: " << *InsertPt);
+
+  Expand(ML, Start, InsertPt, Dec, End, Revert);
   return true;
 }
 
 // WhileLoopStart holds the exit block, so produce a cmp lr, 0 and then a
 // beq that branches to the exit branch.
-// FIXME: Need to check that we're not trashing the CPSR when generating the
-// cmp. We could also try to generate a cbz if the value in LR is also in
+// TODO: We could also try to generate a cbz if the value in LR is also in
 // another low register.
 void ARMLowOverheadLoops::RevertWhile(MachineInstr *MI) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp: " << *MI);
@@ -255,19 +365,33 @@ void ARMLowOverheadLoops::RevertWhile(MachineInstr *MI) const {
   MIB.addImm(0);
   MIB.addImm(ARMCC::AL);
   MIB.addReg(ARM::NoRegister);
+  
+  MachineBasicBlock *DestBB = MI->getOperand(1).getMBB();
+  unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ?
+    ARM::tBcc : ARM::t2Bcc;
 
-  // TODO: Try to use tBcc instead
-  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2Bcc));
+  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
   MIB.add(MI->getOperand(1));   // branch target
   MIB.addImm(ARMCC::EQ);        // condition code
   MIB.addReg(ARM::CPSR);
   MI->eraseFromParent();
 }
 
-// TODO: Check flags so that we can possibly generate a tSubs or tSub.
-void ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const {
+bool ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI,
+                                        bool AllowFlags) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to sub: " << *MI);
   MachineBasicBlock *MBB = MI->getParent();
+
+  // If nothing uses or defines CPSR between LoopDec and LoopEnd, use a t2SUBS.
+  bool SetFlags = false;
+  if (AllowFlags) {
+    if (auto *Def = SearchForDef(MI, MBB->end(), ARM::CPSR)) {
+      if (!SearchForUse(MI, MBB->end(), ARM::CPSR) &&
+          Def->getOpcode() == ARM::t2LoopEnd)
+        SetFlags = true;
+    }
+  }
+
   MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
                                     TII->get(ARM::t2SUBri));
   MIB.addDef(ARM::LR);
@@ -275,28 +399,39 @@ void ARMLowOverheadLoops::RevertLoopDec(MachineInstr *MI) const {
   MIB.add(MI->getOperand(2));
   MIB.addImm(ARMCC::AL);
   MIB.addReg(0);
-  MIB.addReg(0);
+
+  if (SetFlags) {
+    MIB.addReg(ARM::CPSR);
+    MIB->getOperand(5).setIsDef(true);
+  } else
+    MIB.addReg(0);
+
   MI->eraseFromParent();
+  return SetFlags;
 }
 
 // Generate a subs, or sub and cmp, and a branch instead of an LE.
-// FIXME: Need to check that we're not trashing the CPSR when generating
-// the cmp.
-void ARMLowOverheadLoops::RevertLoopEnd(MachineInstr *MI) const {
+void ARMLowOverheadLoops::RevertLoopEnd(MachineInstr *MI, bool SkipCmp) const {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting to cmp, br: " << *MI);
 
-  // Create cmp
   MachineBasicBlock *MBB = MI->getParent();
-  MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
-                                    TII->get(ARM::t2CMPri));
-  MIB.addReg(ARM::LR);
-  MIB.addImm(0);
-  MIB.addImm(ARMCC::AL);
-  MIB.addReg(ARM::NoRegister);
+  // Create cmp
+  if (!SkipCmp) {
+    MachineInstrBuilder MIB = BuildMI(*MBB, MI, MI->getDebugLoc(),
+                                      TII->get(ARM::t2CMPri));
+    MIB.addReg(ARM::LR);
+    MIB.addImm(0);
+    MIB.addImm(ARMCC::AL);
+    MIB.addReg(ARM::NoRegister);
+  }
+
+  MachineBasicBlock *DestBB = MI->getOperand(1).getMBB();
+  unsigned BrOpc = BBUtils->isBBInRange(MI, DestBB, 254) ?
+    ARM::tBcc : ARM::t2Bcc;
 
-  // TODO Try to use tBcc instead.
   // Create bne
-  MIB = BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(ARM::t2Bcc));
+  MachineInstrBuilder MIB =
+    BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(BrOpc));
   MIB.add(MI->getOperand(1));   // branch target
   MIB.addImm(ARMCC::NE);        // condition code
   MIB.addReg(ARM::CPSR);
@@ -304,33 +439,13 @@ void ARMLowOverheadLoops::RevertLoopEnd(MachineInstr *MI) const {
 }
 
 void ARMLowOverheadLoops::Expand(MachineLoop *ML, MachineInstr *Start,
+                                 MachineInstr *InsertPt,
                                  MachineInstr *Dec, MachineInstr *End,
                                  bool Revert) {
 
-  auto ExpandLoopStart = [this](MachineLoop *ML, MachineInstr *Start) {
-    // The trip count should already been held in LR since the instructions
-    // within the loop can only read and write to LR. So, there should be a
-    // mov to setup the count. WLS/DLS perform this move, so find the original
-    // and delete it - inserting WLS/DLS in its place.
-    MachineBasicBlock *MBB = Start->getParent();
-    MachineInstr *InsertPt = Start;
-    for (auto &I : MRI->def_instructions(ARM::LR)) {
-      if (I.getParent() != MBB)
-        continue;
-
-      // Always execute.
-      if (!I.getOperand(2).isImm() || I.getOperand(2).getImm() != ARMCC::AL)
-        continue;
-
-      // Only handle move reg, if the trip count it will need moving into a reg
-      // before the setup instruction anyway.
-      if (!I.getDesc().isMoveReg() ||
-          !I.getOperand(1).isIdenticalTo(Start->getOperand(0)))
-        continue;
-      InsertPt = &I;
-      break;
-    }
-
+  auto ExpandLoopStart = [this](MachineLoop *ML, MachineInstr *Start,
+                                MachineInstr *InsertPt) {
+    MachineBasicBlock *MBB = InsertPt->getParent();
     unsigned Opc = Start->getOpcode() == ARM::t2DoLoopStart ?
       ARM::t2DLS : ARM::t2WLS;
     MachineInstrBuilder MIB =
@@ -386,21 +501,21 @@ void ARMLowOverheadLoops::Expand(MachineLoop *ML, MachineInstr *Start,
       RevertWhile(Start);
     else
       Start->eraseFromParent();
-    RevertLoopDec(Dec);
-    RevertLoopEnd(End);
+    bool FlagsAlreadySet = RevertLoopDec(Dec, true);
+    RevertLoopEnd(End, FlagsAlreadySet);
   } else {
-    Start = ExpandLoopStart(ML, Start);
+    Start = ExpandLoopStart(ML, Start, InsertPt);
     RemoveDeadBranch(Start);
     End = ExpandLoopEnd(ML, Dec, End);
     RemoveDeadBranch(End);
   }
 }
 
-bool ARMLowOverheadLoops::RevertNonLoops(MachineFunction &MF) {
+bool ARMLowOverheadLoops::RevertNonLoops() {
   LLVM_DEBUG(dbgs() << "ARM Loops: Reverting any remaining pseudos...\n");
   bool Changed = false;
 
-  for (auto &MBB : MF) {
+  for (auto &MBB : *MF) {
     SmallVector<MachineInstr*, 4> Starts;
     SmallVector<MachineInstr*, 4> Decs;
     SmallVector<MachineInstr*, 4> Ends;
diff --git a/llvm/lib/Target/ARM/ARMPredicates.td b/llvm/lib/Target/ARM/ARMPredicates.td
index 0b6b40de80dd6..b008d3e2e2962 100644
--- a/llvm/lib/Target/ARM/ARMPredicates.td
+++ b/llvm/lib/Target/ARM/ARMPredicates.td
@@ -71,7 +71,7 @@ def HasV8_5a         : Predicate<"Subtarget->hasV8_5aOps()">,
                                  AssemblerPredicate<"HasV8_5aOps", "armv8.5a">;
 def NoVFP            : Predicate<"!Subtarget->hasVFP2Base()">;
 def HasVFP2          : Predicate<"Subtarget->hasVFP2Base()">,
-                                 AssemblerPredicate<"FeatureVFP2_D16_SP", "VFP2">;
+                                 AssemblerPredicate<"FeatureVFP2_SP", "VFP2">;
 def HasVFP3          : Predicate<"Subtarget->hasVFP3Base()">,
                                  AssemblerPredicate<"FeatureVFP3_D16_SP", "VFP3">;
 def HasVFP4          : Predicate<"Subtarget->hasVFP4Base()">,
diff --git a/llvm/lib/Target/ARM/ARMScheduleM4.td b/llvm/lib/Target/ARM/ARMScheduleM4.td
index 38c8ea2b4f35d..bfa5fc0d71319 100644
--- a/llvm/lib/Target/ARM/ARMScheduleM4.td
+++ b/llvm/lib/Target/ARM/ARMScheduleM4.td
@@ -18,6 +18,9 @@ def CortexM4Model : SchedMachineModel {
   let PostRAScheduler   = 1;
 
   let CompleteModel = 0;
+  let UnsupportedFeatures = [IsARM, HasNEON, HasDotProd, HasZCZ, HasMVEInt,
+          IsNotMClass, HasDPVFP, HasFPARMv8, HasFullFP16, Has8MSecExt, HasV8,
+          HasV8_3a, HasTrustZone, HasDFB, IsWindows];
 }
 
 
@@ -50,6 +53,7 @@ def : M4UnitL2<WriteMAC16>;
 def : M4UnitL2<WriteDIV>;
 
 def : M4UnitL2I<(instregex "(t|t2)LDM")>;
+def : M4UnitL2I<(instregex "(t|t2)LDR")>;
 
 
 // Stores we use a latency of 1 as they have no outputs
@@ -78,9 +82,20 @@ def : M4UnitL1<WriteNoop>;
 def : M4UnitL1<WritePreLd>;
 def : M4UnitL1I<(instregex "(t|t2)MOV")>;
 def : M4UnitL1I<(instrs COPY)>;
-def : M4UnitL1I<(instregex "t2IT")>;
-def : M4UnitL1I<(instregex "t2SEL", "t2USAD8",
-    "t2(S|Q|SH|U|UQ|UH)(ADD16|ASX|SAX|SUB16|ADD8|SUB8)", "t2USADA8", "(t|t2)REV")>;
+def : M4UnitL1I<(instregex "t2IT", "t2MSR", "t2MRS")>;
+def : M4UnitL1I<(instregex "t2CLREX")>;
+def : M4UnitL1I<(instregex "t2SEL", "t2USAD8", "t2SML[AS]",
+    "t2(S|Q|SH|U|UQ|UH|QD)(ADD|ASX|SAX|SUB)", "t2USADA8", "(t|t2)REV")>;
+
+// These instructions are not of much interest to scheduling as they will not
+// be generated or it is not very useful to schedule them. They are here to make
+// the model more complete.
+def : M4UnitL1I<(instregex "t2CDP", "t2LDC", "t2MCR", "t2MRC", "t2MRRC", "t2STC")>;
+def : M4UnitL1I<(instregex "tCPS", "t2ISB", "t2DSB", "t2DMB", "t2?HINT$")>;
+def : M4UnitL1I<(instregex "t2?UDF$", "tBKPT", "t2DBG")>;
+def : M4UnitL1I<(instregex "t?2?Int_eh_sjlj_", "tADDframe", "t?ADJCALL")>;
+def : M4UnitL1I<(instregex "CMP_SWAP", "JUMPTABLE", "MEMCPY")>;
+def : M4UnitL1I<(instregex "VSETLNi32", "VGETLNi32")>;
 
 def : ReadAdvance<ReadALU, 0>;
 def : ReadAdvance<ReadALUsr, 0>;
@@ -112,6 +127,9 @@ def : M4UnitL1<WriteVST1>;
 def : M4UnitL1<WriteVST2>;
 def : M4UnitL1<WriteVST3>;
 def : M4UnitL1<WriteVST4>;
+def : M4UnitL1I<(instregex "VMOVS", "FCONSTS", "VCMP", "VNEG", "VABS")>;
+def : M4UnitL2I<(instregex "VMOVD")>;
+def : M4UnitL1I<(instregex "VMRS", "VMSR", "FMSTAT")>;
 
 def : ReadAdvance<ReadFPMUL, 0>;
 def : ReadAdvance<ReadFPMAC, 0>;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 0491420c51fbf..a8b56fb06a7ed 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -180,11 +180,9 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
   bool HasVFPv3SP = false;
   bool HasVFPv4SP = false;
   bool HasFPARMv8SP = false;
-  bool HasVFPv2D16 = false;
   bool HasVFPv3D16 = false;
   bool HasVFPv4D16 = false;
   bool HasFPARMv8D16 = false;
-  bool HasVFPv2D16SP = false;
   bool HasVFPv3D16SP = false;
   bool HasVFPv4D16SP = false;
   bool HasFPARMv8D16SP = false;
@@ -606,7 +604,7 @@ class ARMSubtarget : public ARMGenSubtargetInfo {
 
   bool hasARMOps() const { return !NoARM; }
 
-  bool hasVFP2Base() const { return HasVFPv2D16SP; }
+  bool hasVFP2Base() const { return HasVFPv2SP; }
   bool hasVFP3Base() const { return HasVFPv3D16SP; }
   bool hasVFP4Base() const { return HasVFPv4D16SP; }
   bool hasFPARMv8Base() const { return HasFPARMv8D16SP; }
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
index e74b2b1a2babb..e8ac760786f63 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
@@ -36,6 +36,10 @@ using namespace llvm;
 
 #define DEBUG_TYPE "armtti"
 
+static cl::opt<bool> EnableMaskedLoadStores(
+  "enable-arm-maskedldst", cl::Hidden, cl::init(false),
+  cl::desc("Enable the generation of masked loads and stores"));
+
 static cl::opt<bool> DisableLowOverheadLoops(
   "disable-arm-loloops", cl::Hidden, cl::init(false),
   cl::desc("Disable the generation of low-overhead loops"));
@@ -487,6 +491,22 @@ int ARMTTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
   return BaseT::getAddressComputationCost(Ty, SE, Ptr);
 }
 
+bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy) {
+  if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
+    return false;
+
+  if (DataTy->isVectorTy()) {
+    // We don't yet support narrowing or widening masked loads/stores. Expand
+    // them for the moment.
+    unsigned VecWidth = DataTy->getPrimitiveSizeInBits();
+    if (VecWidth != 128)
+      return false;
+  }
+
+  unsigned EltWidth = DataTy->getScalarSizeInBits();
+  return EltWidth == 32 || EltWidth == 16 || EltWidth == 8;
+}
+
 int ARMTTIImpl::getMemcpyCost(const Instruction *I) {
   const MemCpyInst *MI = dyn_cast<MemCpyInst>(I);
   assert(MI && "MemcpyInst expected");
diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
index 507e01885495d..47e98dac9f6bf 100644
--- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
+++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h
@@ -152,6 +152,9 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
     return ST->getMaxInterleaveFactor();
   }
 
+  bool isLegalMaskedLoad(Type *DataTy);
+  bool isLegalMaskedStore(Type *DataTy) { return isLegalMaskedLoad(DataTy); }
+
   int getMemcpyCost(const Instruction *I);
 
   int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
diff --git a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 93b5254084a3b..dd2d4ec118aa8 100644
--- a/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/llvm/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -2487,28 +2487,28 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addModImmNotOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint32_t Enc = ARM_AM::getSOImmVal(~CE->getValue());
     Inst.addOperand(MCOperand::createImm(Enc));
   }
 
   void addModImmNegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint32_t Enc = ARM_AM::getSOImmVal(-CE->getValue());
     Inst.addOperand(MCOperand::createImm(Enc));
   }
 
   void addThumbModImmNeg8_255Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint32_t Val = -CE->getValue();
     Inst.addOperand(MCOperand::createImm(Val));
   }
 
   void addThumbModImmNeg1_7Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint32_t Val = -CE->getValue();
     Inst.addOperand(MCOperand::createImm(Val));
   }
@@ -2531,19 +2531,19 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addFBits16Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(16 - CE->getValue()));
   }
 
   void addFBits32Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(32 - CE->getValue()));
   }
 
   void addFPImmOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     int Val = ARM_AM::getFP32Imm(APInt(32, CE->getValue()));
     Inst.addOperand(MCOperand::createImm(Val));
   }
@@ -2552,7 +2552,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // FIXME: We really want to scale the value here, but the LDRD/STRD
     // instruction don't encode operands that way yet.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
@@ -2560,35 +2560,31 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // FIXME: We really want to scale the value here, but the VSTR/VLDR_VSYSR
     // instruction don't encode operands that way yet.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
   void addImm7Shift0Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    assert(CE != nullptr && "Invalid operand type!");
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
   void addImm7Shift1Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    assert(CE != nullptr && "Invalid operand type!");
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
   void addImm7Shift2Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    assert(CE != nullptr && "Invalid operand type!");
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
   void addImm7Operands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
-    assert(CE != nullptr && "Invalid operand type!");
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
@@ -2596,7 +2592,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() / 4));
   }
 
@@ -2604,7 +2600,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(-(CE->getValue() / 4)));
   }
 
@@ -2612,7 +2608,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate is scaled by four in the encoding and is stored
     // in the MCInst as such. Lop off the low two bits here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() / 4));
   }
 
@@ -2620,7 +2616,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The constant encodes as the immediate-1, and we store in the instruction
     // the bits as encoded, so subtract off one here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() - 1));
   }
 
@@ -2628,7 +2624,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The constant encodes as the immediate-1, and we store in the instruction
     // the bits as encoded, so subtract off one here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() - 1));
   }
 
@@ -2636,7 +2632,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The constant encodes as the immediate, except for 32, which encodes as
     // zero.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Imm = CE->getValue();
     Inst.addOperand(MCOperand::createImm((Imm == 32 ? 0 : Imm)));
   }
@@ -2645,7 +2641,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // An ASR value of 32 encodes as 0, so that's how we want to add it to
     // the instruction as well.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     int Val = CE->getValue();
     Inst.addOperand(MCOperand::createImm(Val == 32 ? 0 : Val));
   }
@@ -2654,7 +2650,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The operand is actually a t2_so_imm, but we have its bitwise
     // negation in the assembly source, so twiddle it here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(~(uint32_t)CE->getValue()));
   }
 
@@ -2662,7 +2658,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The operand is actually a t2_so_imm, but we have its
     // negation in the assembly source, so twiddle it here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(-(uint32_t)CE->getValue()));
   }
 
@@ -2670,7 +2666,7 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The operand is actually an imm0_4095, but we have its
     // negation in the assembly source, so twiddle it here.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(-(uint32_t)CE->getValue()));
   }
 
@@ -2679,9 +2675,7 @@ class ARMOperand : public MCParsedAsmOperand {
       Inst.addOperand(MCOperand::createImm(CE->getValue() >> 2));
       return;
     }
-
-    const MCSymbolRefExpr *SR = dyn_cast<MCSymbolRefExpr>(Imm.Val);
-    assert(SR && "Unknown value type!");
+    const MCSymbolRefExpr *SR = cast<MCSymbolRefExpr>(Imm.Val);
     Inst.addOperand(MCOperand::createExpr(SR));
   }
 
@@ -2693,10 +2687,7 @@ class ARMOperand : public MCParsedAsmOperand {
         Inst.addOperand(MCOperand::createImm(CE->getValue()));
         return;
       }
-
-      const MCSymbolRefExpr *SR = dyn_cast<MCSymbolRefExpr>(Imm.Val);
-
-      assert(SR && "Unknown value type!");
+      const MCSymbolRefExpr *SR = cast<MCSymbolRefExpr>(Imm.Val);
       Inst.addOperand(MCOperand::createExpr(SR));
       return;
     }
@@ -2758,7 +2749,7 @@ class ARMOperand : public MCParsedAsmOperand {
       return;
     }
 
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     int Val = CE->getValue();
     Inst.addOperand(MCOperand::createImm(Val));
   }
@@ -3138,7 +3129,7 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addPowerTwoOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue()));
   }
 
@@ -3233,14 +3224,14 @@ class ARMOperand : public MCParsedAsmOperand {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
     // Mask in that this is an i8 splat.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() | 0xe00));
   }
 
   void addNEONi16splatOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi16splat(Value);
     Inst.addOperand(MCOperand::createImm(Value));
@@ -3249,7 +3240,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi16splatNotOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi16splat(~Value & 0xffff);
     Inst.addOperand(MCOperand::createImm(Value));
@@ -3258,7 +3249,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi32splatOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi32splat(Value);
     Inst.addOperand(MCOperand::createImm(Value));
@@ -3267,7 +3258,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi32splatNotOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = CE->getValue();
     Value = ARM_AM::encodeNEONi32splat(~Value);
     Inst.addOperand(MCOperand::createImm(Value));
@@ -3275,7 +3266,7 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addNEONi8ReplicateOperands(MCInst &Inst, bool Inv) const {
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     assert((Inst.getOpcode() == ARM::VMOVv8i8 ||
             Inst.getOpcode() == ARM::VMOVv16i8) &&
           "All instructions that wants to replicate non-zero byte "
@@ -3306,7 +3297,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi32vmovOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = encodeNeonVMOVImmediate(CE->getValue());
     Inst.addOperand(MCOperand::createImm(Value));
   }
@@ -3318,7 +3309,7 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addNEONvmovi16ReplicateOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     assert((Inst.getOpcode() == ARM::VMOVv4i16 ||
             Inst.getOpcode() == ARM::VMOVv8i16 ||
             Inst.getOpcode() == ARM::VMVNv4i16 ||
@@ -3335,14 +3326,14 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi32vmovNegOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Value = encodeNeonVMOVImmediate(~CE->getValue());
     Inst.addOperand(MCOperand::createImm(Value));
   }
 
   void addNEONvmovi32ReplicateOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     assert((Inst.getOpcode() == ARM::VMOVv2i32 ||
             Inst.getOpcode() == ARM::VMOVv4i32 ||
             Inst.getOpcode() == ARM::VMVNv2i32 ||
@@ -3357,7 +3348,7 @@ class ARMOperand : public MCParsedAsmOperand {
   void addNEONi64splatOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
     // The immediate encodes the type of constant as well as the value.
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     uint64_t Value = CE->getValue();
     unsigned Imm = 0;
     for (unsigned i = 0; i < 8; ++i, Value >>= 8) {
@@ -3368,19 +3359,19 @@ class ARMOperand : public MCParsedAsmOperand {
 
   void addComplexRotationEvenOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm(CE->getValue() / 90));
   }
 
   void addComplexRotationOddOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     Inst.addOperand(MCOperand::createImm((CE->getValue() - 90) / 180));
   }
 
   void addMveSaturateOperands(MCInst &Inst, unsigned N) const {
     assert(N == 1 && "Invalid number of operands!");
-    const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(getImm());
+    const MCConstantExpr *CE = cast<MCConstantExpr>(getImm());
     unsigned Imm = CE->getValue();
     assert((Imm == 48 || Imm == 64) && "Invalid saturate operand");
     Inst.addOperand(MCOperand::createImm(Imm == 48 ? 1 : 0));
@@ -11734,14 +11725,14 @@ bool ARMAsmParser::parseDirectiveArchExtension(SMLoc L) {
     { ARM::AEK_CRYPTO,  {Feature_HasV8Bit},
       {ARM::FeatureCrypto, ARM::FeatureNEON, ARM::FeatureFPARMv8} },
     { ARM::AEK_FP, {Feature_HasV8Bit},
-      {ARM::FeatureVFP2_D16_SP, ARM::FeatureFPARMv8} },
+      {ARM::FeatureVFP2_SP, ARM::FeatureFPARMv8} },
     { (ARM::AEK_HWDIVTHUMB | ARM::AEK_HWDIVARM),
       {Feature_HasV7Bit, Feature_IsNotMClassBit},
       {ARM::FeatureHWDivThumb, ARM::FeatureHWDivARM} },
     { ARM::AEK_MP, {Feature_HasV7Bit, Feature_IsNotMClassBit},
       {ARM::FeatureMP} },
     { ARM::AEK_SIMD, {Feature_HasV8Bit},
-      {ARM::FeatureNEON, ARM::FeatureVFP2_D16_SP, ARM::FeatureFPARMv8} },
+      {ARM::FeatureNEON, ARM::FeatureVFP2_SP, ARM::FeatureFPARMv8} },
     { ARM::AEK_SEC, {Feature_HasV6KBit}, {ARM::FeatureTrustZone} },
     // FIXME: Only available in A-class, isel not predicated
     { ARM::AEK_VIRT, {Feature_HasV7Bit}, {ARM::FeatureVirtualization} },
@@ -11815,19 +11806,19 @@ unsigned ARMAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
   // immediate in the syntax.
   switch (Kind) {
   default: break;
-  case MCK__35_0:
+  case MCK__HASH_0:
     if (Op.isImm())
       if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op.getImm()))
         if (CE->getValue() == 0)
           return Match_Success;
     break;
-  case MCK__35_8:
+  case MCK__HASH_8:
     if (Op.isImm())
       if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op.getImm()))
         if (CE->getValue() == 8)
           return Match_Success;
     break;
-  case MCK__35_16:
+  case MCK__HASH_16:
     if (Op.isImm())
       if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Op.getImm()))
         if (CE->getValue() == 16)
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index 27141f4fc6797..6196881a9b8f8 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -1105,28 +1105,28 @@ uint32_t ARMAsmBackendDarwin::generateCompactUnwindEncoding(
   if (Instrs.empty())
     return 0;
   // Start off assuming CFA is at SP+0.
-  int CFARegister = ARM::SP;
+  unsigned CFARegister = ARM::SP;
   int CFARegisterOffset = 0;
   // Mark savable registers as initially unsaved
   DenseMap<unsigned, int> RegOffsets;
   int FloatRegCount = 0;
   // Process each .cfi directive and build up compact unwind info.
   for (size_t i = 0, e = Instrs.size(); i != e; ++i) {
-    int Reg;
+    unsigned Reg;
     const MCCFIInstruction &Inst = Instrs[i];
     switch (Inst.getOperation()) {
     case MCCFIInstruction::OpDefCfa: // DW_CFA_def_cfa
       CFARegisterOffset = -Inst.getOffset();
-      CFARegister = MRI.getLLVMRegNum(Inst.getRegister(), true);
+      CFARegister = *MRI.getLLVMRegNum(Inst.getRegister(), true);
       break;
     case MCCFIInstruction::OpDefCfaOffset: // DW_CFA_def_cfa_offset
       CFARegisterOffset = -Inst.getOffset();
       break;
     case MCCFIInstruction::OpDefCfaRegister: // DW_CFA_def_cfa_register
-      CFARegister = MRI.getLLVMRegNum(Inst.getRegister(), true);
+      CFARegister = *MRI.getLLVMRegNum(Inst.getRegister(), true);
       break;
     case MCCFIInstruction::OpOffset: // DW_CFA_offset
-      Reg = MRI.getLLVMRegNum(Inst.getRegister(), true);
+      Reg = *MRI.getLLVMRegNum(Inst.getRegister(), true);
       if (ARMMCRegisterClasses[ARM::GPRRegClassID].contains(Reg))
         RegOffsets[Reg] = Inst.getOffset();
       else if (ARMMCRegisterClasses[ARM::DPRRegClassID].contains(Reg)) {
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
index c4daafe8ee97f..6893229580c58 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h
@@ -393,6 +393,10 @@ namespace ARMII {
     // in an IT block).
     ThumbArithFlagSetting = 1 << 19,
 
+    // Whether an instruction should be excluded from an MVE tail-predicated
+    // loop.
+    InvalidForTailPredication = 1 << 20,
+
     //===------------------------------------------------------------------===//
     // Code domain.
     DomainShift   = 15,
diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
index b863517c0cca9..7b30a61e8ccb4 100644
--- a/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
+++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp
@@ -249,12 +249,12 @@ void ARMTargetStreamer::emitTargetAttributes(const MCSubtargetInfo &STI) {
                             : ARM::FK_VFPV3_D16)
                      : (STI.hasFeature(ARM::FeatureFP16) ? ARM::FK_VFPV3XD_FP16
                                                          : ARM::FK_VFPV3XD)));
-    else if (STI.hasFeature(ARM::FeatureVFP2_D16_SP))
+    else if (STI.hasFeature(ARM::FeatureVFP2_SP))
       emitFPU(ARM::FK_VFPV2);
   }
 
   // ABI_HardFP_use attribute to indicate single precision FP.
-  if (STI.hasFeature(ARM::FeatureVFP2_D16_SP) && !STI.hasFeature(ARM::FeatureFP64))
+  if (STI.hasFeature(ARM::FeatureVFP2_SP) && !STI.hasFeature(ARM::FeatureFP64))
     emitAttribute(ARMBuildAttrs::ABI_HardFP_use,
                   ARMBuildAttrs::HardFPSinglePrecision);
 
diff --git a/llvm/lib/Target/ARM/MVETailPredication.cpp b/llvm/lib/Target/ARM/MVETailPredication.cpp
index 0079184156632..4db8ab17c49b5 100644
--- a/llvm/lib/Target/ARM/MVETailPredication.cpp
+++ b/llvm/lib/Target/ARM/MVETailPredication.cpp
@@ -84,7 +84,7 @@ class MVETailPredication : public LoopPass {
 
   /// Is the icmp that generates an i1 vector, based upon a loop counter
   /// and a limit that is defined outside the loop.
-  bool isTailPredicate(Value *Predicate, Value *NumElements);
+  bool isTailPredicate(Instruction *Predicate, Value *NumElements);
 };
 
 } // end namespace
@@ -178,7 +178,7 @@ bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
   return Changed;
 }
 
-bool MVETailPredication::isTailPredicate(Value *V, Value *NumElements) {
+bool MVETailPredication::isTailPredicate(Instruction *I, Value *NumElements) {
   // Look for the following:
 
   // %trip.count.minus.1 = add i32 %N, -1
@@ -206,7 +206,7 @@ bool MVETailPredication::isTailPredicate(Value *V, Value *NumElements) {
   Instruction *Induction = nullptr;
 
   // The vector icmp
-  if (!match(V, m_ICmp(Pred, m_Instruction(Induction),
+  if (!match(I, m_ICmp(Pred, m_Instruction(Induction),
                        m_Instruction(Shuffle))) ||
       Pred != ICmpInst::ICMP_ULE || !L->isLoopInvariant(Shuffle))
     return false;
@@ -390,6 +390,55 @@ Value* MVETailPredication::ComputeElements(Value *TripCount,
   return Expander.expandCodeFor(Elems, Elems->getType(), InsertPt);
 }
 
+// Look through the exit block to see whether there's a duplicate predicate
+// instruction. This can happen when we need to perform a select on values
+// from the last and previous iteration. Instead of doing a straight
+// replacement of that predicate with the vctp, clone the vctp and place it
+// in the block. This means that the VPR doesn't have to be live into the
+// exit block which should make it easier to convert this loop into a proper
+// tail predicated loop.
+static void Cleanup(DenseMap<Instruction*, Instruction*> &NewPredicates,
+                    SetVector<Instruction*> &MaybeDead, Loop *L) {
+  if (BasicBlock *Exit = L->getUniqueExitBlock()) {
+    for (auto &Pair : NewPredicates) {
+      Instruction *OldPred = Pair.first;
+      Instruction *NewPred = Pair.second;
+
+      for (auto &I : *Exit) {
+        if (I.isSameOperationAs(OldPred)) {
+          Instruction *PredClone = NewPred->clone();
+          PredClone->insertBefore(&I);
+          I.replaceAllUsesWith(PredClone);
+          MaybeDead.insert(&I);
+          break;
+        }
+      }
+    }
+  }
+
+  // Drop references and add operands to check for dead.
+  SmallPtrSet<Instruction*, 4> Dead;
+  while (!MaybeDead.empty()) {
+    auto *I = MaybeDead.front();
+    MaybeDead.remove(I);
+    if (I->hasNUsesOrMore(1))
+      continue;
+
+    for (auto &U : I->operands()) {
+      if (auto *OpI = dyn_cast<Instruction>(U))
+        MaybeDead.insert(OpI);
+    }
+    I->dropAllReferences();
+    Dead.insert(I);
+  }
+
+  for (auto *I : Dead)
+    I->eraseFromParent();
+
+  for (auto I : L->blocks())
+    DeleteDeadPHIs(I);
+}
+
 bool MVETailPredication::TryConvert(Value *TripCount) {
   if (!IsPredicatedVectorLoop())
     return false;
@@ -400,13 +449,14 @@ bool MVETailPredication::TryConvert(Value *TripCount) {
   // operand is generated from an induction variable.
   Module *M = L->getHeader()->getModule();
   Type *Ty = IntegerType::get(M->getContext(), 32);
-  SmallPtrSet<Value*, 4> Predicates;
+  SetVector<Instruction*> Predicates;
+  DenseMap<Instruction*, Instruction*> NewPredicates;
 
   for (auto *I : MaskedInsts) {
     Intrinsic::ID ID = I->getIntrinsicID();
     unsigned PredOp = ID == Intrinsic::masked_load ? 2 : 3;
-    Value *Predicate = I->getArgOperand(PredOp);
-    if (Predicates.count(Predicate))
+    auto *Predicate = dyn_cast<Instruction>(I->getArgOperand(PredOp));
+    if (!Predicate || Predicates.count(Predicate))
       continue;
 
     VectorType *VecTy = getVectorType(I);
@@ -441,21 +491,21 @@ bool MVETailPredication::TryConvert(Value *TripCount) {
     case 16: VCTPID = Intrinsic::arm_vctp8; break;
     }
     Function *VCTP = Intrinsic::getDeclaration(M, VCTPID);
-    // TODO: This add likely already exists in the loop.
-    Value *Remaining = Builder.CreateSub(Processed, Factor);
-    Value *TailPredicate = Builder.CreateCall(VCTP, Remaining);
+    Value *TailPredicate = Builder.CreateCall(VCTP, Processed);
     Predicate->replaceAllUsesWith(TailPredicate);
+    NewPredicates[Predicate] = cast<Instruction>(TailPredicate);
 
     // Add the incoming value to the new phi.
+    // TODO: This add likely already exists in the loop.
+    Value *Remaining = Builder.CreateSub(Processed, Factor);
     Processed->addIncoming(Remaining, L->getLoopLatch());
     LLVM_DEBUG(dbgs() << "TP: Insert processed elements phi: "
                << *Processed << "\n"
                << "TP: Inserted VCTP: " << *TailPredicate << "\n");
   }
 
-  for (auto I : L->blocks())
-    DeleteDeadPHIs(I);
-
+  // Now clean up.
+  Cleanup(NewPredicates, Predicates, L);
   return true;
 }
 
diff --git a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
index bfd7d37dfc895..9d986b6167e06 100644
--- a/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
+++ b/llvm/lib/Target/ARM/MVEVPTBlockPass.cpp
@@ -80,6 +80,90 @@ enum VPTMaskValue {
   TETE  = 15  // 0b1111
 };
 
+static unsigned VCMPOpcodeToVPT(unsigned Opcode) {
+  switch (Opcode) {
+  case ARM::MVE_VCMPf32:
+    return ARM::MVE_VPTv4f32;
+  case ARM::MVE_VCMPf16:
+    return ARM::MVE_VPTv8f16;
+  case ARM::MVE_VCMPi8:
+    return ARM::MVE_VPTv16i8;
+  case ARM::MVE_VCMPi16:
+    return ARM::MVE_VPTv8i16;
+  case ARM::MVE_VCMPi32:
+    return ARM::MVE_VPTv4i32;
+  case ARM::MVE_VCMPu8:
+    return ARM::MVE_VPTv16u8;
+  case ARM::MVE_VCMPu16:
+    return ARM::MVE_VPTv8u16;
+  case ARM::MVE_VCMPu32:
+    return ARM::MVE_VPTv4u32;
+  case ARM::MVE_VCMPs8:
+    return ARM::MVE_VPTv16s8;
+  case ARM::MVE_VCMPs16:
+    return ARM::MVE_VPTv8s16;
+  case ARM::MVE_VCMPs32:
+    return ARM::MVE_VPTv4s32;
+
+  case ARM::MVE_VCMPf32r:
+    return ARM::MVE_VPTv4f32r;
+  case ARM::MVE_VCMPf16r:
+    return ARM::MVE_VPTv8f16r;
+  case ARM::MVE_VCMPi8r:
+    return ARM::MVE_VPTv16i8r;
+  case ARM::MVE_VCMPi16r:
+    return ARM::MVE_VPTv8i16r;
+  case ARM::MVE_VCMPi32r:
+    return ARM::MVE_VPTv4i32r;
+  case ARM::MVE_VCMPu8r:
+    return ARM::MVE_VPTv16u8r;
+  case ARM::MVE_VCMPu16r:
+    return ARM::MVE_VPTv8u16r;
+  case ARM::MVE_VCMPu32r:
+    return ARM::MVE_VPTv4u32r;
+  case ARM::MVE_VCMPs8r:
+    return ARM::MVE_VPTv16s8r;
+  case ARM::MVE_VCMPs16r:
+    return ARM::MVE_VPTv8s16r;
+  case ARM::MVE_VCMPs32r:
+    return ARM::MVE_VPTv4s32r;
+
+  default:
+    return 0;
+  }
+}
+
+static MachineInstr *findVCMPToFoldIntoVPST(MachineBasicBlock::iterator MI,
+                                            const TargetRegisterInfo *TRI,
+                                            unsigned &NewOpcode) {
+  // Search backwards to the instruction that defines VPR. This may or not
+  // be a VCMP, we check that after this loop. If we find another instruction
+  // that reads cpsr, we return nullptr.
+  MachineBasicBlock::iterator CmpMI = MI;
+  while (CmpMI != MI->getParent()->begin()) {
+    --CmpMI;
+    if (CmpMI->modifiesRegister(ARM::VPR, TRI))
+      break;
+    if (CmpMI->readsRegister(ARM::VPR, TRI))
+      break;
+  }
+
+  if (CmpMI == MI)
+    return nullptr;
+  NewOpcode = VCMPOpcodeToVPT(CmpMI->getOpcode());
+  if (NewOpcode == 0)
+    return nullptr;
+
+  // Search forward from CmpMI to MI, checking if either register was def'd
+  if (registerDefinedBetween(CmpMI->getOperand(1).getReg(), std::next(CmpMI),
+                             MI, TRI))
+    return nullptr;
+  if (registerDefinedBetween(CmpMI->getOperand(2).getReg(), std::next(CmpMI),
+                             MI, TRI))
+    return nullptr;
+  return &*CmpMI;
+}
+
 bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
   bool Modified = false;
   MachineBasicBlock::instr_iterator MBIter = Block.instr_begin();
@@ -123,26 +207,42 @@ bool MVEVPTBlock::InsertVPTBlocks(MachineBasicBlock &Block) {
       ++MBIter;
     };
 
-    // Create the new VPST
-    MachineInstrBuilder MIBuilder =
-        BuildMI(Block, MI, dl, TII->get(ARM::MVE_VPST));
+    unsigned BlockMask = 0;
     switch (VPTInstCnt) {
     case 1:
-      MIBuilder.addImm(VPTMaskValue::T);
+      BlockMask = VPTMaskValue::T;
       break;
     case 2:
-      MIBuilder.addImm(VPTMaskValue::TT);
+      BlockMask = VPTMaskValue::TT;
       break;
     case 3:
-      MIBuilder.addImm(VPTMaskValue::TTT);
+      BlockMask = VPTMaskValue::TTT;
       break;
     case 4:
-      MIBuilder.addImm(VPTMaskValue::TTTT);
+      BlockMask = VPTMaskValue::TTTT;
       break;
     default:
       llvm_unreachable("Unexpected number of instruction in a VPT block");
     };
 
+    // Search back for a VCMP that can be folded to create a VPT, or else create
+    // a VPST directly
+    MachineInstrBuilder MIBuilder;
+    unsigned NewOpcode;
+    MachineInstr *VCMP = findVCMPToFoldIntoVPST(MI, TRI, NewOpcode);
+    if (VCMP) {
+      LLVM_DEBUG(dbgs() << "  folding VCMP into VPST: "; VCMP->dump());
+      MIBuilder = BuildMI(Block, MI, dl, TII->get(NewOpcode));
+      MIBuilder.addImm(BlockMask);
+      MIBuilder.add(VCMP->getOperand(1));
+      MIBuilder.add(VCMP->getOperand(2));
+      MIBuilder.add(VCMP->getOperand(3));
+      VCMP->eraseFromParent();
+    } else {
+      MIBuilder = BuildMI(Block, MI, dl, TII->get(ARM::MVE_VPST));
+      MIBuilder.addImm(BlockMask);
+    }
+
     finalizeBundle(
         Block, MachineBasicBlock::instr_iterator(MIBuilder.getInstr()), MBIter);
 
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.cpp b/llvm/lib/Target/AVR/AVRISelLowering.cpp
index 6566f618b9559..f12c59b7d8c35 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.cpp
+++ b/llvm/lib/Target/AVR/AVRISelLowering.cpp
@@ -236,7 +236,7 @@ AVRTargetLowering::AVRTargetLowering(const AVRTargetMachine &TM,
   setLibcallName(RTLIB::SIN_F32, "sin");
   setLibcallName(RTLIB::COS_F32, "cos");
 
-  setMinFunctionAlignment(llvm::Align(2));
+  setMinFunctionAlignment(Align(2));
   setMinimumJumpTableEntries(UINT_MAX);
 }
 
@@ -2006,10 +2006,9 @@ void AVRTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
   return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
 }
 
-unsigned AVRTargetLowering::getRegisterByName(const char *RegName,
-                                              EVT VT,
-                                              SelectionDAG &DAG) const {
-  unsigned Reg;
+Register AVRTargetLowering::getRegisterByName(const char *RegName, EVT VT,
+                                              const MachineFunction &MF) const {
+  Register Reg;
 
   if (VT == MVT::i8) {
     Reg = StringSwitch<unsigned>(RegName)
diff --git a/llvm/lib/Target/AVR/AVRISelLowering.h b/llvm/lib/Target/AVR/AVRISelLowering.h
index ed2d0835903c4..6c722fa5414b6 100644
--- a/llvm/lib/Target/AVR/AVRISelLowering.h
+++ b/llvm/lib/Target/AVR/AVRISelLowering.h
@@ -125,8 +125,8 @@ class AVRTargetLowering : public TargetLowering {
                                     std::vector<SDValue> &Ops,
                                     SelectionDAG &DAG) const override;
 
-  unsigned getRegisterByName(const char* RegName, EVT VT,
-                             SelectionDAG &DAG) const override;
+  Register getRegisterByName(const char* RegName, EVT VT,
+                             const MachineFunction &MF) const override;
 
   bool shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL)
     const override {
diff --git a/llvm/lib/Target/AVR/TargetInfo/LLVMBuild.txt b/llvm/lib/Target/AVR/TargetInfo/LLVMBuild.txt
index 6446977632855..7b112a2f00c6a 100644
--- a/llvm/lib/Target/AVR/TargetInfo/LLVMBuild.txt
+++ b/llvm/lib/Target/AVR/TargetInfo/LLVMBuild.txt
@@ -18,5 +18,5 @@
 type = Library
 name = AVRInfo
 parent = AVR
-required_libraries = MC Support
-add_to_library_groups = AVR
\ No newline at end of file
+required_libraries = Support
+add_to_library_groups = AVR
diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
index e4f29fbb3dba2..c682c46fe3163 100644
--- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
+++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp
@@ -375,6 +375,8 @@ void BPFAbstractMemberAccess::traceAICall(CallInst *Call, uint32_t Kind,
         traceGEP(GI, Call, Kind, ParentMeta, ParentAI);
       else
         BaseAICalls[Call] = Kind;
+    } else {
+      BaseAICalls[Call] = Kind;
     }
   }
 }
@@ -406,6 +408,8 @@ void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
         traceGEP(GI, Parent, Kind, ParentMeta, ParentAI);
       else
         BaseAICalls[Parent] = Kind;
+    } else {
+      BaseAICalls[Parent] = Kind;
     }
   }
 }
@@ -436,6 +440,8 @@ void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
         traceGEP(GI, Parent, Kind, ParentMeta, ParentAI);
       else
         BaseAICalls[Parent] = Kind;
+    } else {
+      BaseAICalls[Parent] = Kind;
     }
   }
 }
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 72fe18b9ed0d7..56e0288f26c9f 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -133,8 +133,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
   setBooleanContents(ZeroOrOneBooleanContent);
 
   // Function alignments
-  setMinFunctionAlignment(llvm::Align(8));
-  setPrefFunctionAlignment(llvm::Align(8));
+  setMinFunctionAlignment(Align(8));
+  setPrefFunctionAlignment(Align(8));
 
   if (BPFExpandMemcpyInOrder) {
     // LLVM generic code will try to expand memcpy into load/store pairs at this
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index 3d08ca63a4b05..3ad32347e0a10 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -473,35 +473,29 @@ void BTFDebug::visitStructType(const DICompositeType *CTy, bool IsStruct,
 
 void BTFDebug::visitArrayType(const DICompositeType *CTy, uint32_t &TypeId) {
   // Visit array element type.
-  uint32_t ElemTypeId, ElemSize;
+  uint32_t ElemTypeId;
   const DIType *ElemType = CTy->getBaseType();
   visitTypeEntry(ElemType, ElemTypeId, false, false);
 
-  // Strip qualifiers from element type to get accurate element size.
-  ElemSize = ElemType->getSizeInBits() >> 3;
-
-  if (!CTy->getSizeInBits()) {
-    auto TypeEntry = std::make_unique<BTFTypeArray>(ElemTypeId, 0);
-    ElemTypeId = addType(std::move(TypeEntry), CTy);
-  } else {
-    // Visit array dimensions.
-    DINodeArray Elements = CTy->getElements();
-    for (int I = Elements.size() - 1; I >= 0; --I) {
-      if (auto *Element = dyn_cast_or_null<DINode>(Elements[I]))
-        if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
-          const DISubrange *SR = cast<DISubrange>(Element);
-          auto *CI = SR->getCount().dyn_cast<ConstantInt *>();
-          int64_t Count = CI->getSExtValue();
-
-          auto TypeEntry =
-              std::make_unique<BTFTypeArray>(ElemTypeId, Count);
-          if (I == 0)
-            ElemTypeId = addType(std::move(TypeEntry), CTy);
-          else
-            ElemTypeId = addType(std::move(TypeEntry));
-          ElemSize = ElemSize * Count;
-        }
-    }
+  // Visit array dimensions.
+  DINodeArray Elements = CTy->getElements();
+  for (int I = Elements.size() - 1; I >= 0; --I) {
+    if (auto *Element = dyn_cast_or_null<DINode>(Elements[I]))
+      if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
+        const DISubrange *SR = cast<DISubrange>(Element);
+        auto *CI = SR->getCount().dyn_cast<ConstantInt *>();
+        int64_t Count = CI->getSExtValue();
+
+        // For struct s { int b; char c[]; }, the c[] will be represented
+        // as an array with Count = -1.
+        auto TypeEntry =
+            std::make_unique<BTFTypeArray>(ElemTypeId,
+                Count >= 0 ? Count : 0);
+        if (I == 0)
+          ElemTypeId = addType(std::move(TypeEntry), CTy);
+        else
+          ElemTypeId = addType(std::move(TypeEntry));
+      }
   }
 
   // The array TypeId is the type id of the outermost dimension.
@@ -1160,12 +1154,8 @@ bool BTFDebug::InstLower(const MachineInstr *MI, MCInst &OutMI) {
       const GlobalValue *GVal = MO.getGlobal();
       auto *GVar = dyn_cast<GlobalVariable>(GVal);
       if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
-        MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index);
-        DIType *Ty = dyn_cast<DIType>(MDN);
-        std::string TypeName = Ty->getName();
-        int64_t Imm = AccessOffsets[GVar->getName().str()];
-
         // Emit "mov ri, <imm>" for abstract member accesses.
+        int64_t Imm = AccessOffsets[GVar->getName().str()];
         OutMI.setOpcode(BPF::MOV_ri);
         OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
         OutMI.addOperand(MCOperand::createImm(Imm));
diff --git a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
index 0881bf841f901..590c4a2eb69d4 100644
--- a/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
+++ b/llvm/lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp
@@ -702,7 +702,7 @@ bool HexagonAsmParser::ParseDirectiveFalign(unsigned Size, SMLoc L) {
     // Make sure we have a number (false is returned if expression is a number)
     if (!getParser().parseExpression(Value)) {
       // Make sure this is a number that is in range
-      const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);
+      auto *MCE = cast<MCConstantExpr>(Value);
       uint64_t IntValue = MCE->getValue();
       if (!isUIntN(Size, IntValue) && !isIntN(Size, IntValue))
         return Error(ExprLoc, "literal value out of range (256) for falign");
diff --git a/llvm/lib/Target/Hexagon/HexagonBranchRelaxation.cpp b/llvm/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
index fec081d324ef0..08f7408068798 100644
--- a/llvm/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonBranchRelaxation.cpp
@@ -105,12 +105,11 @@ void HexagonBranchRelaxation::computeOffset(MachineFunction &MF,
   // offset of the current instruction from the start.
   unsigned InstOffset = 0;
   for (auto &B : MF) {
-    if (B.getLogAlignment()) {
+    if (B.getAlignment() != Align::None()) {
       // Although we don't know the exact layout of the final code, we need
       // to account for alignment padding somehow. This heuristic pads each
       // aligned basic block according to the alignment value.
-      int ByteAlign = (1u << B.getLogAlignment()) - 1;
-      InstOffset = (InstOffset + ByteAlign) & ~(ByteAlign);
+      InstOffset = alignTo(InstOffset, B.getAlignment());
     }
     OffsetMap[&B] = InstOffset;
     for (auto &MI : B.instrs()) {
diff --git a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
index 2ce1419e4790f..e4a2ba0ec29cd 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td
@@ -37,12 +37,12 @@ def: Pat<(int_hexagon_F2_sfmax IntRegs:$src1, IntRegs:$src2),
          (F2_sfmax IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vabswsat DoubleRegs:$src1),
          (A2_vabswsat DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asr_i_r IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p DoubleRegs:$src1, u6_0ImmPred:$src2),
-         (S2_asr_i_p DoubleRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_combineri IntRegs:$src1, s32_0ImmPred:$src2),
-         (A4_combineri IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asr_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2),
+         (S2_asr_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_combineri IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A4_combineri IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_nac_sat_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_nac_sat_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_vpmpyh_acc DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -75,8 +75,8 @@ def: Pat<(int_hexagon_A2_vaddws DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vaddws DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_maxup DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_maxup DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmphgti DoubleRegs:$src1, s8_0ImmPred:$src2),
-         (A4_vcmphgti DoubleRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmphgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_vcmphgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_interleave DoubleRegs:$src1),
          (S2_interleave DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vrcmpyi_s0 DoubleRegs:$src1, DoubleRegs:$src2),
@@ -89,10 +89,10 @@ def: Pat<(int_hexagon_C2_cmpgtu IntRegs:$src1, IntRegs:$src2),
          (C2_cmpgtu IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_cmpgtp DoubleRegs:$src1, DoubleRegs:$src2),
          (C2_cmpgtp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmphgtui IntRegs:$src1, u32_0ImmPred:$src2),
-         (A4_cmphgtui IntRegs:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_cmpgti IntRegs:$src1, s32_0ImmPred:$src2),
-         (C2_cmpgti IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmphgtui IntRegs:$src1, u32_0ImmPred_timm:$src2),
+         (A4_cmphgtui IntRegs:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_cmpgti IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (C2_cmpgti IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyi IntRegs:$src1, IntRegs:$src2),
          (M2_mpyi IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_df2uw_chop DoubleRegs:$src1),
@@ -103,12 +103,12 @@ def: Pat<(int_hexagon_M2_mpy_lh_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_lh_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_lh_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_lh_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_lsr_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_lsr_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vrcnegh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_vrcnegh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_extractup DoubleRegs:$src1, u6_0ImmPred:$src2, u6_0ImmPred:$src3),
-         (S2_extractup DoubleRegs:$src1, u6_0ImmPred:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_extractup DoubleRegs:$src1, u6_0ImmPred_timm:$src2, u6_0ImmPred_timm:$src3),
+         (S2_extractup DoubleRegs:$src1, u6_0ImmPred_timm:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S4_ntstbit_r IntRegs:$src1, IntRegs:$src2),
          (S4_ntstbit_r IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_w2sf IntRegs:$src1),
@@ -125,10 +125,10 @@ def: Pat<(int_hexagon_A4_cmpbgt IntRegs:$src1, IntRegs:$src2),
          (A4_cmpbgt IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asr_r_r_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asr_r_r_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_rcmpneqi IntRegs:$src1, s32_0ImmPred:$src2),
-         (A4_rcmpneqi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asl_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_rcmpneqi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A4_rcmpneqi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asl_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_subacc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_subacc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_orp DoubleRegs:$src1, DoubleRegs:$src2),
@@ -137,28 +137,28 @@ def: Pat<(int_hexagon_M2_mpyu_up IntRegs:$src1, IntRegs:$src2),
          (M2_mpyu_up IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_acc_sat_lh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_acc_sat_lh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2),
-         (S2_asr_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asr_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2),
+         (S2_asr_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asr_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_cmpbgtu IntRegs:$src1, IntRegs:$src2),
          (A4_cmpbgtu IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vcmpbeq_any DoubleRegs:$src1, DoubleRegs:$src2),
          (A4_vcmpbeq_any DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmpbgti IntRegs:$src1, s8_0ImmPred:$src2),
-         (A4_cmpbgti IntRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmpbgti IntRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_cmpbgti IntRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_lh_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyd_lh_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asl_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_asl_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_lsr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_lsr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_addsp IntRegs:$src1, DoubleRegs:$src2),
          (A2_addsp IntRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S4_vxsubaddw DoubleRegs:$src1, DoubleRegs:$src2),
          (S4_vxsubaddw DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpheqi DoubleRegs:$src1, s8_0ImmPred:$src2),
-         (A4_vcmpheqi DoubleRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpheqi DoubleRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_vcmpheqi DoubleRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S4_vxsubaddh DoubleRegs:$src1, DoubleRegs:$src2),
          (S4_vxsubaddh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_pmpyw IntRegs:$src1, IntRegs:$src2),
@@ -177,10 +177,10 @@ def: Pat<(int_hexagon_A2_pxorf PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (A2_pxorf PredRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vsubub DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vsubub DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p DoubleRegs:$src1, u6_0ImmPred:$src2),
-         (S2_asl_i_p DoubleRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asl_i_r IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2),
+         (S2_asl_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asl_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrminuw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (A4_vrminuw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffma IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -199,10 +199,10 @@ def: Pat<(int_hexagon_M4_vrmpyoh_s1 DoubleRegs:$src1, DoubleRegs:$src2),
          (M4_vrmpyoh_s1 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_bitsset IntRegs:$src1, IntRegs:$src2),
          (C2_bitsset IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_mpysip IntRegs:$src1, u32_0ImmPred:$src2),
-         (M2_mpysip IntRegs:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_mpysin IntRegs:$src1, u8_0ImmPred:$src2),
-         (M2_mpysin IntRegs:$src1, u8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_mpysip IntRegs:$src1, u32_0ImmPred_timm:$src2),
+         (M2_mpysip IntRegs:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_mpysin IntRegs:$src1, u8_0ImmPred_timm:$src2),
+         (M2_mpysin IntRegs:$src1, u8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_boundscheck IntRegs:$src1, DoubleRegs:$src2),
          (A4_boundscheck IntRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vrmpybuu DoubleRegs:$src1, DoubleRegs:$src2),
@@ -225,10 +225,10 @@ def: Pat<(int_hexagon_F2_conv_ud2df DoubleRegs:$src1),
          (F2_conv_ud2df DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vnavgw DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vnavgw DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asl_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_subi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_subi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asl_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_subi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_subi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vzxthw IntRegs:$src1),
          (S2_vzxthw IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sfadd IntRegs:$src1, IntRegs:$src2),
@@ -241,12 +241,12 @@ def: Pat<(int_hexagon_M2_vmac2su_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$sr
          (M2_vmac2su_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_dpmpyss_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_dpmpyss_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_insert IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3, u5_0ImmPred:$src4),
-         (S2_insert IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3, u5_0ImmPred:$src4)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_insert IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3, u5_0ImmPred_timm:$src4),
+         (S2_insert IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3, u5_0ImmPred_timm:$src4)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_packhl IntRegs:$src1, IntRegs:$src2),
          (S2_packhl IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpwgti DoubleRegs:$src1, s8_0ImmPred:$src2),
-         (A4_vcmpwgti DoubleRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpwgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_vcmpwgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavguwr DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavguwr DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asl_r_r_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -259,8 +259,8 @@ def: Pat<(int_hexagon_M4_and_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_and_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_d2df DoubleRegs:$src1),
          (F2_conv_d2df DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_cmpgtui IntRegs:$src1, u32_0ImmPred:$src2),
-         (C2_cmpgtui IntRegs:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_cmpgtui IntRegs:$src1, u32_0ImmPred_timm:$src2),
+         (C2_cmpgtui IntRegs:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vconj DoubleRegs:$src1),
          (A2_vconj DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsr_r_vw DoubleRegs:$src1, IntRegs:$src2),
@@ -279,8 +279,8 @@ def: Pat<(int_hexagon_C2_any8 PredRegs:$src1),
          (C2_any8 PredRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_togglebit_r IntRegs:$src1, IntRegs:$src2),
          (S2_togglebit_r IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_togglebit_i IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_togglebit_i IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_togglebit_i IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_togglebit_i IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_uw2sf IntRegs:$src1),
          (F2_conv_uw2sf IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vsathb_nopack DoubleRegs:$src1),
@@ -303,10 +303,10 @@ def: Pat<(int_hexagon_C4_or_andn PredRegs:$src1, PredRegs:$src2, PredRegs:$src3)
          (C4_or_andn PredRegs:$src1, PredRegs:$src2, PredRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asl_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asl_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asl_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpwgtui DoubleRegs:$src1, u7_0ImmPred:$src2),
-         (A4_vcmpwgtui DoubleRegs:$src1, u7_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asl_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpwgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2),
+         (A4_vcmpwgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_vrmpyoh_acc_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M4_vrmpyoh_acc_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_vrmpyoh_acc_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
@@ -323,34 +323,34 @@ def: Pat<(int_hexagon_M2_vrcmacr_s0c DoubleRegs:$src1, DoubleRegs:$src2, DoubleR
          (M2_vrcmacr_s0c DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavgwcr DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavgwcr DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asl_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asl_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrmaxw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (A4_vrmaxw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vnavghr DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vnavghr DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_cmpyi_wh DoubleRegs:$src1, IntRegs:$src2),
          (M4_cmpyi_wh DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_tfrsi s32_0ImmPred:$src1),
-         (A2_tfrsi s32_0ImmPred:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_tfrsi s32_0ImmPred_timm:$src1),
+         (A2_tfrsi s32_0ImmPred_timm:$src1)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_svnavgh IntRegs:$src1, IntRegs:$src2),
          (A2_svnavgh IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_lsr_i_r IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_lsr_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmac2 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_vmac2 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmphgtui DoubleRegs:$src1, u7_0ImmPred:$src2),
-         (A4_vcmphgtui DoubleRegs:$src1, u7_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmphgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2),
+         (A4_vcmphgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_svavgh IntRegs:$src1, IntRegs:$src2),
          (A2_svavgh IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_vrmpyeh_acc_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M4_vrmpyeh_acc_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_vrmpyeh_acc_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M4_vrmpyeh_acc_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p DoubleRegs:$src1, u6_0ImmPred:$src2),
-         (S2_lsr_i_p DoubleRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2),
+         (S2_lsr_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_combine_hl IntRegs:$src1, IntRegs:$src2),
          (A2_combine_hl IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_up IntRegs:$src1, IntRegs:$src2),
@@ -381,10 +381,10 @@ def: Pat<(int_hexagon_M2_cmacr_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3
          (M2_cmacr_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_or_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_or_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M4_mpyrr_addi u32_0ImmPred:$src1, IntRegs:$src2, IntRegs:$src3),
-         (M4_mpyrr_addi u32_0ImmPred:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_or_andi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (S4_or_andi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M4_mpyrr_addi u32_0ImmPred_timm:$src1, IntRegs:$src2, IntRegs:$src3),
+         (M4_mpyrr_addi u32_0ImmPred_timm:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_or_andi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (S4_or_andi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_hl_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_sat_hl_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_hl_s1 IntRegs:$src1, IntRegs:$src2),
@@ -453,8 +453,8 @@ def: Pat<(int_hexagon_M2_mpy_rnd_hl_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_rnd_hl_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffms_lib IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (F2_sffms_lib IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C4_cmpneqi IntRegs:$src1, s32_0ImmPred:$src2),
-         (C4_cmpneqi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C4_cmpneqi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (C4_cmpneqi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_and_xor IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_and_xor IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_sat DoubleRegs:$src1),
@@ -469,8 +469,8 @@ def: Pat<(int_hexagon_A2_svavghs IntRegs:$src1, IntRegs:$src2),
          (A2_svavghs IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vrsadub_acc DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (A2_vrsadub_acc DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_bitsclri IntRegs:$src1, u6_0ImmPred:$src2),
-         (C2_bitsclri IntRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_bitsclri IntRegs:$src1, u6_0ImmPred_timm:$src2),
+         (C2_bitsclri IntRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_subh_h16_sat_hh IntRegs:$src1, IntRegs:$src2),
          (A2_subh_h16_sat_hh IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_subh_h16_sat_hl IntRegs:$src1, IntRegs:$src2),
@@ -535,10 +535,10 @@ def: Pat<(int_hexagon_C2_vmux PredRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3
          (C2_vmux PredRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_parityp DoubleRegs:$src1, DoubleRegs:$src2),
          (S2_parityp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_lsr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_lsr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_nac_ll_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyu_nac_ll_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_nac_ll_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -557,30 +557,30 @@ def: Pat<(int_hexagon_M2_cnacsc_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src
          (M2_cnacsc_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cnacsc_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_cnacsc_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_subaddi IntRegs:$src1, s32_0ImmPred:$src2, IntRegs:$src3),
-         (S4_subaddi IntRegs:$src1, s32_0ImmPred:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_subaddi IntRegs:$src1, s32_0ImmPred_timm:$src2, IntRegs:$src3),
+         (S4_subaddi IntRegs:$src1, s32_0ImmPred_timm:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_nac_hl_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyud_nac_hl_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_nac_hl_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyud_nac_hl_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_tstbit_r IntRegs:$src1, IntRegs:$src2),
          (S2_tstbit_r IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_vrcrotate DoubleRegs:$src1, IntRegs:$src2, u2_0ImmPred:$src3),
-         (S4_vrcrotate DoubleRegs:$src1, IntRegs:$src2, u2_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_vrcrotate DoubleRegs:$src1, IntRegs:$src2, u2_0ImmPred_timm:$src3),
+         (S4_vrcrotate DoubleRegs:$src1, IntRegs:$src2, u2_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmachs_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M2_mmachs_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmachs_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M2_mmachs_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_tstbit_i IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_tstbit_i IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_tstbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_tstbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_up_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_up_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_extractu_rp IntRegs:$src1, DoubleRegs:$src2),
          (S2_extractu_rp IntRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmpyuh_rs0 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_mmpyuh_rs0 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2),
-         (S2_lsr_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_lsr_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_rnd_ll_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_rnd_ll_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_rnd_ll_s1 IntRegs:$src1, IntRegs:$src2),
@@ -605,14 +605,14 @@ def: Pat<(int_hexagon_F2_conv_w2df IntRegs:$src1),
          (F2_conv_w2df IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_subh_l16_sat_hl IntRegs:$src1, IntRegs:$src2),
          (A2_subh_l16_sat_hl IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_cmpeqi IntRegs:$src1, s32_0ImmPred:$src2),
-         (C2_cmpeqi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asl_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_cmpeqi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (C2_cmpeqi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asl_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vcnegh DoubleRegs:$src1, IntRegs:$src2),
          (S2_vcnegh DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpweqi DoubleRegs:$src1, s8_0ImmPred:$src2),
-         (A4_vcmpweqi DoubleRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpweqi DoubleRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_vcmpweqi DoubleRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vdmpyrs_s0 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vdmpyrs_s0 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vdmpyrs_s1 DoubleRegs:$src1, DoubleRegs:$src2),
@@ -633,8 +633,8 @@ def: Pat<(int_hexagon_S2_asl_r_r_acc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3
          (S2_asl_r_r_acc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_cl0p DoubleRegs:$src1),
          (S2_cl0p DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_valignib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred:$src3),
-         (S2_valignib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_valignib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred_timm:$src3),
+         (S2_valignib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffixupd IntRegs:$src1, IntRegs:$src2),
          (F2_sffixupd IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_rnd_hl_s1 IntRegs:$src1, IntRegs:$src2),
@@ -653,8 +653,8 @@ def: Pat<(int_hexagon_M2_dpmpyuu_nac_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs
          (M2_dpmpyuu_nac_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmpyul_rs1 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_mmpyul_rs1 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_ntstbit_i IntRegs:$src1, u5_0ImmPred:$src2),
-         (S4_ntstbit_i IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_ntstbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S4_ntstbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffixupr IntRegs:$src1),
          (F2_sffixupr IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asr_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
@@ -669,32 +669,32 @@ def: Pat<(int_hexagon_C2_andn PredRegs:$src1, PredRegs:$src2),
          (C2_andn PredRegs:$src1, PredRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmpy2s_s0pack IntRegs:$src1, IntRegs:$src2),
          (M2_vmpy2s_s0pack IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_addaddi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (S4_addaddi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_addaddi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (S4_addaddi IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_acc_ll_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyd_acc_ll_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_acc_sat_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_acc_sat_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_rcmpeqi IntRegs:$src1, s32_0ImmPred:$src2),
-         (A4_rcmpeqi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_rcmpeqi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A4_rcmpeqi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_xor_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_xor_and IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asl_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asl_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmpyuh_rs1 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_mmpyuh_rs1 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asr_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asr_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_round_ri IntRegs:$src1, u5_0ImmPred:$src2),
-         (A4_round_ri IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_round_ri IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (A4_round_ri IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_max IntRegs:$src1, IntRegs:$src2),
          (A2_max IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_round_rr IntRegs:$src1, IntRegs:$src2),
          (A4_round_rr IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_combineii s8_0ImmPred:$src1, u32_0ImmPred:$src2),
-         (A4_combineii s8_0ImmPred:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_combineir s32_0ImmPred:$src1, IntRegs:$src2),
-         (A4_combineir s32_0ImmPred:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_combineii s8_0ImmPred_timm:$src1, u32_0ImmPred_timm:$src2),
+         (A4_combineii s8_0ImmPred_timm:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_combineir s32_0ImmPred_timm:$src1, IntRegs:$src2),
+         (A4_combineir s32_0ImmPred_timm:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C4_and_orn PredRegs:$src1, PredRegs:$src2, PredRegs:$src3),
          (C4_and_orn PredRegs:$src1, PredRegs:$src2, PredRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vmacbuu DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -703,8 +703,8 @@ def: Pat<(int_hexagon_A4_rcmpeq IntRegs:$src1, IntRegs:$src2),
          (A4_rcmpeq IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_cmpyr_whc DoubleRegs:$src1, IntRegs:$src2),
          (M4_cmpyr_whc DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_lsr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_lsr_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vzxtbh IntRegs:$src1),
          (S2_vzxtbh IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmacuhs_rs1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
@@ -721,8 +721,8 @@ def: Pat<(int_hexagon_M2_cmpyi_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_cmpyi_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asl_r_p_or DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_asl_r_p_or DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_ori_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_ori_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_ori_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_ori_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C4_nbitsset IntRegs:$src1, IntRegs:$src2),
          (C4_nbitsset IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_acc_hh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -745,10 +745,10 @@ def: Pat<(int_hexagon_M2_mpyd_acc_hh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs
          (M2_mpyd_acc_hh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_acc_hh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyd_acc_hh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_sfimm_p u10_0ImmPred:$src1),
-         (F2_sfimm_p u10_0ImmPred:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_sfimm_n u10_0ImmPred:$src1),
-         (F2_sfimm_n u10_0ImmPred:$src1)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_sfimm_p u10_0ImmPred_timm:$src1),
+         (F2_sfimm_p u10_0ImmPred_timm:$src1)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_sfimm_n u10_0ImmPred_timm:$src1),
+         (F2_sfimm_n u10_0ImmPred_timm:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_cmpyr_wh DoubleRegs:$src1, IntRegs:$src2),
          (M4_cmpyr_wh DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_p_and DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
@@ -759,14 +759,14 @@ def: Pat<(int_hexagon_F2_conv_d2sf DoubleRegs:$src1),
          (F2_conv_d2sf DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavguh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavguh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmpbeqi IntRegs:$src1, u8_0ImmPred:$src2),
-         (A4_cmpbeqi IntRegs:$src1, u8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmpbeqi IntRegs:$src1, u8_0ImmPred_timm:$src2),
+         (A4_cmpbeqi IntRegs:$src1, u8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sfcmpuo IntRegs:$src1, IntRegs:$src2),
          (F2_sfcmpuo IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavguw DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavguw DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vsatwh_nopack DoubleRegs:$src1),
          (S2_vsatwh_nopack DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_hh_s0 IntRegs:$src1, IntRegs:$src2),
@@ -783,8 +783,8 @@ def: Pat<(int_hexagon_M4_or_andn IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_or_andn IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_minp DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_minp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_or_andix IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (S4_or_andix IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_or_andix IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (S4_or_andix IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_rnd_lh_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_rnd_lh_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_rnd_lh_s1 IntRegs:$src1, IntRegs:$src2),
@@ -817,16 +817,16 @@ def: Pat<(int_hexagon_S4_extract_rp IntRegs:$src1, DoubleRegs:$src2),
          (S4_extract_rp IntRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_lsl_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C4_cmplteui IntRegs:$src1, u32_0ImmPred:$src2),
-         (C4_cmplteui IntRegs:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_addi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_addi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C4_cmplteui IntRegs:$src1, u32_0ImmPred_timm:$src2),
+         (C4_cmplteui IntRegs:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_addi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_addi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_tfrcpp CtrRegs64:$src1),
          (A4_tfrcpp CtrRegs64:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_svw_trun DoubleRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asr_i_svw_trun DoubleRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmphgti IntRegs:$src1, s32_0ImmPred:$src2),
-         (A4_cmphgti IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_svw_trun DoubleRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asr_i_svw_trun DoubleRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmphgti IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A4_cmphgti IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrminh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (A4_vrminh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrminw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
@@ -837,8 +837,8 @@ def: Pat<(int_hexagon_S2_insertp_rp DoubleRegs:$src1, DoubleRegs:$src2, DoubleRe
          (S2_insertp_rp DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vnavghcr DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vnavghcr DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_subi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_subi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_subi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_subi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_vh DoubleRegs:$src1, IntRegs:$src2),
          (S2_lsl_r_vh DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_hh_s0 IntRegs:$src1, IntRegs:$src2),
@@ -851,14 +851,14 @@ def: Pat<(int_hexagon_S2_asl_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs
          (S2_asl_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_satb IntRegs:$src1),
          (A2_satb IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_insertp DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3, u6_0ImmPred:$src4),
-         (S2_insertp DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3, u6_0ImmPred:$src4)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_insertp DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3, u6_0ImmPred_timm:$src4),
+         (S2_insertp DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3, u6_0ImmPred_timm:$src4)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_rnd_ll_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyd_rnd_ll_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_rnd_ll_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyd_rnd_ll_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_lsr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_lsr_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_extractup_rp DoubleRegs:$src1, DoubleRegs:$src2),
          (S2_extractup_rp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S4_vxaddsubw DoubleRegs:$src1, DoubleRegs:$src2),
@@ -925,8 +925,8 @@ def: Pat<(int_hexagon_M2_cmpyr_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_cmpyr_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_dpmpyss_rnd_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_dpmpyss_rnd_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_muxri PredRegs:$src1, s32_0ImmPred:$src2, IntRegs:$src3),
-         (C2_muxri PredRegs:$src1, s32_0ImmPred:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_muxri PredRegs:$src1, s32_0ImmPred_timm:$src2, IntRegs:$src3),
+         (C2_muxri PredRegs:$src1, s32_0ImmPred_timm:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmac2es_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M2_vmac2es_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmac2es_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
@@ -937,8 +937,8 @@ def: Pat<(int_hexagon_M2_mpyu_lh_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyu_lh_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_lh_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyu_lh_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asl_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asl_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_acc_hl_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyd_acc_hl_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_acc_hl_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -947,8 +947,8 @@ def: Pat<(int_hexagon_S2_asr_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs
          (S2_asr_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vaddw DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vaddw DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vaddh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vaddh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_nac_sat_lh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -957,16 +957,16 @@ def: Pat<(int_hexagon_M2_mpy_nac_sat_lh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs
          (M2_mpy_nac_sat_lh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_cmpeqp DoubleRegs:$src1, DoubleRegs:$src2),
          (C2_cmpeqp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M4_mpyri_addi u32_0ImmPred:$src1, IntRegs:$src2, u6_0ImmPred:$src3),
-         (M4_mpyri_addi u32_0ImmPred:$src1, IntRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_andi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_andi_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_macsip IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3),
-         (M2_macsip IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M4_mpyri_addi u32_0ImmPred_timm:$src1, IntRegs:$src2, u6_0ImmPred_timm:$src3),
+         (M4_mpyri_addi u32_0ImmPred_timm:$src1, IntRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_andi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_andi_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_macsip IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3),
+         (M2_macsip IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_tfrcrr CtrRegs:$src1),
          (A2_tfrcrr CtrRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_macsin IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3),
-         (M2_macsin IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_macsin IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3),
+         (M2_macsin IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_orn PredRegs:$src1, PredRegs:$src2),
          (C2_orn PredRegs:$src1, PredRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_and_andn IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1005,8 +1005,8 @@ def: Pat<(int_hexagon_M2_vrcmpys_acc_s1 DoubleRegs:$src1, DoubleRegs:$src2, IntR
          (M2_vrcmpys_acc_s1 DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_dfcmpge DoubleRegs:$src1, DoubleRegs:$src2),
          (F2_dfcmpge DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_accii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (M2_accii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_accii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (M2_accii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A5_vaddhubs DoubleRegs:$src1, DoubleRegs:$src2),
          (A5_vaddhubs DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vmaxw DoubleRegs:$src1, DoubleRegs:$src2),
@@ -1017,10 +1017,10 @@ def: Pat<(int_hexagon_A2_vmaxh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vmaxh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vsxthw IntRegs:$src1),
          (S2_vsxthw IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_andi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_andi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asl_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_andi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_andi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asl_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_lsl_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_cmpgt IntRegs:$src1, IntRegs:$src2),
@@ -1035,22 +1035,22 @@ def: Pat<(int_hexagon_F2_conv_sf2w IntRegs:$src1),
          (F2_conv_sf2w IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsr_r_p_or DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_lsr_r_p_or DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_sfclass IntRegs:$src1, u5_0ImmPred:$src2),
-         (F2_sfclass IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_sfclass IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (F2_sfclass IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_acc_lh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyud_acc_lh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_xor_andn IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_xor_andn IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_addasl_rrri IntRegs:$src1, IntRegs:$src2, u3_0ImmPred:$src3),
-         (S2_addasl_rrri IntRegs:$src1, IntRegs:$src2, u3_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_addasl_rrri IntRegs:$src1, IntRegs:$src2, u3_0ImmPred_timm:$src3),
+         (S2_addasl_rrri IntRegs:$src1, IntRegs:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vdmpybsu DoubleRegs:$src1, DoubleRegs:$src2),
          (M5_vdmpybsu DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_nac_hh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyu_nac_hh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_nac_hh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyu_nac_hh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_addi IntRegs:$src1, s32_0ImmPred:$src2),
-         (A2_addi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_addi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A2_addi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_addp DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_addp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmpy2s_s1pack IntRegs:$src1, IntRegs:$src2),
@@ -1063,8 +1063,8 @@ def: Pat<(int_hexagon_M2_nacci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_nacci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_shuffeh DoubleRegs:$src1, DoubleRegs:$src2),
          (S2_shuffeh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_lsr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_lsr_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_rnd_hh_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_sat_rnd_hh_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_rnd_hh_s0 IntRegs:$src1, IntRegs:$src2),
@@ -1131,12 +1131,12 @@ def: Pat<(int_hexagon_C2_and PredRegs:$src1, PredRegs:$src2),
          (C2_and PredRegs:$src1, PredRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S5_popcountp DoubleRegs:$src1),
          (S5_popcountp DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_extractp DoubleRegs:$src1, u6_0ImmPred:$src2, u6_0ImmPred:$src3),
-         (S4_extractp DoubleRegs:$src1, u6_0ImmPred:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_extractp DoubleRegs:$src1, u6_0ImmPred_timm:$src2, u6_0ImmPred_timm:$src3),
+         (S4_extractp DoubleRegs:$src1, u6_0ImmPred_timm:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_cl0 IntRegs:$src1),
          (S2_cl0 IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpbgti DoubleRegs:$src1, s8_0ImmPred:$src2),
-         (A4_vcmpbgti DoubleRegs:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpbgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2),
+         (A4_vcmpbgti DoubleRegs:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmacls_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M2_mmacls_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmacls_s0 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
@@ -1167,8 +1167,8 @@ def: Pat<(int_hexagon_M2_maci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_maci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vmaxuh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vmaxuh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_bitspliti IntRegs:$src1, u5_0ImmPred:$src2),
-         (A4_bitspliti IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_bitspliti IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (A4_bitspliti IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vmaxub DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vmaxub DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_hh_s0 IntRegs:$src1, IntRegs:$src2),
@@ -1185,26 +1185,26 @@ def: Pat<(int_hexagon_F2_conv_sf2d IntRegs:$src1),
          (F2_conv_sf2d IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asr_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asr_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_dfimm_n u10_0ImmPred:$src1),
-         (F2_dfimm_n u10_0ImmPred:$src1)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_dfimm_n u10_0ImmPred_timm:$src1),
+         (F2_dfimm_n u10_0ImmPred_timm:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_cmphgt IntRegs:$src1, IntRegs:$src2),
          (A4_cmphgt IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_dfimm_p u10_0ImmPred:$src1),
-         (F2_dfimm_p u10_0ImmPred:$src1)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_dfimm_p u10_0ImmPred_timm:$src1),
+         (F2_dfimm_p u10_0ImmPred_timm:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_acc_lh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyud_acc_lh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vcmpy_s1_sat_r DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vcmpy_s1_sat_r DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M4_mpyri_addr_u2 IntRegs:$src1, u6_2ImmPred:$src2, IntRegs:$src3),
-         (M4_mpyri_addr_u2 IntRegs:$src1, u6_2ImmPred:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M4_mpyri_addr_u2 IntRegs:$src1, u6_2ImmPred_timm:$src2, IntRegs:$src3),
+         (M4_mpyri_addr_u2 IntRegs:$src1, u6_2ImmPred_timm:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vcmpy_s1_sat_i DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vcmpy_s1_sat_i DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_lsl_r_p_nac DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vrmacbuu DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
          (M5_vrmacbuu DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_vspliceib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred:$src3),
-         (S2_vspliceib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_vspliceib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred_timm:$src3),
+         (S2_vspliceib DoubleRegs:$src1, DoubleRegs:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_dpmpyss_acc_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_dpmpyss_acc_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cnacs_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1215,20 +1215,20 @@ def: Pat<(int_hexagon_A2_maxu IntRegs:$src1, IntRegs:$src2),
          (A2_maxu IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_maxp DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_maxp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_andir IntRegs:$src1, s32_0ImmPred:$src2),
-         (A2_andir IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_andir IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A2_andir IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sfrecipa IntRegs:$src1, IntRegs:$src2),
          (F2_sfrecipa IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_combineii s32_0ImmPred:$src1, s8_0ImmPred:$src2),
-         (A2_combineii s32_0ImmPred:$src1, s8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_combineii s32_0ImmPred_timm:$src1, s8_0ImmPred_timm:$src2),
+         (A2_combineii s32_0ImmPred_timm:$src1, s8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_orn IntRegs:$src1, IntRegs:$src2),
          (A4_orn IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmpbgtui IntRegs:$src1, u32_0ImmPred:$src2),
-         (A4_cmpbgtui IntRegs:$src1, u32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmpbgtui IntRegs:$src1, u32_0ImmPred_timm:$src2),
+         (A4_cmpbgtui IntRegs:$src1, u32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsr_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_lsr_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpbeqi DoubleRegs:$src1, u8_0ImmPred:$src2),
-         (A4_vcmpbeqi DoubleRegs:$src1, u8_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpbeqi DoubleRegs:$src1, u8_0ImmPred_timm:$src2),
+         (A4_vcmpbeqi DoubleRegs:$src1, u8_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_r IntRegs:$src1, IntRegs:$src2),
          (S2_lsl_r_r IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_p DoubleRegs:$src1, IntRegs:$src2),
@@ -1251,16 +1251,16 @@ def: Pat<(int_hexagon_A2_satub IntRegs:$src1),
          (A2_satub IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vrcmpys_s1 DoubleRegs:$src1, IntRegs:$src2),
          (M2_vrcmpys_s1 DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_or_ori IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (S4_or_ori IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_or_ori IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (S4_or_ori IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C4_fastcorner9_not PredRegs:$src1, PredRegs:$src2),
          (C4_fastcorner9_not PredRegs:$src1, PredRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_tfrih IntRegs:$src1, u16_0ImmPred:$src2),
-         (A2_tfrih IntRegs:$src1, u16_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_tfril IntRegs:$src1, u16_0ImmPred:$src2),
-         (A2_tfril IntRegs:$src1, u16_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M4_mpyri_addr IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3),
-         (M4_mpyri_addr IntRegs:$src1, IntRegs:$src2, u32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_tfrih IntRegs:$src1, u16_0ImmPred_timm:$src2),
+         (A2_tfrih IntRegs:$src1, u16_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_tfril IntRegs:$src1, u16_0ImmPred_timm:$src2),
+         (A2_tfril IntRegs:$src1, u16_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M4_mpyri_addr IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3),
+         (M4_mpyri_addr IntRegs:$src1, IntRegs:$src2, u32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vtrunehb DoubleRegs:$src1),
          (S2_vtrunehb DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vabsw DoubleRegs:$src1),
@@ -1269,14 +1269,14 @@ def: Pat<(int_hexagon_A2_vabsh DoubleRegs:$src1),
          (A2_vabsh DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sfsub IntRegs:$src1, IntRegs:$src2),
          (F2_sfsub IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_muxii PredRegs:$src1, s32_0ImmPred:$src2, s8_0ImmPred:$src3),
-         (C2_muxii PredRegs:$src1, s32_0ImmPred:$src2, s8_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C2_muxir PredRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (C2_muxir PredRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_muxii PredRegs:$src1, s32_0ImmPred_timm:$src2, s8_0ImmPred_timm:$src3),
+         (C2_muxii PredRegs:$src1, s32_0ImmPred_timm:$src2, s8_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C2_muxir PredRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (C2_muxir PredRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_swiz IntRegs:$src1),
          (A2_swiz IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asr_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cmpyrsc_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_cmpyrsc_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cmpyrsc_s1 IntRegs:$src1, IntRegs:$src2),
@@ -1295,44 +1295,44 @@ def: Pat<(int_hexagon_M2_mpy_nac_sat_ll_s1 IntRegs:$src1, IntRegs:$src2, IntRegs
          (M2_mpy_nac_sat_ll_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_nac_sat_ll_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_nac_sat_ll_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_extract IntRegs:$src1, u5_0ImmPred:$src2, u5_0ImmPred:$src3),
-         (S4_extract IntRegs:$src1, u5_0ImmPred:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_extract IntRegs:$src1, u5_0ImmPred_timm:$src2, u5_0ImmPred_timm:$src3),
+         (S4_extract IntRegs:$src1, u5_0ImmPred_timm:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vcmpweq DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vcmpweq DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_acci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_acci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_lsr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_lsr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_lsr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_lsr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_ud2sf DoubleRegs:$src1),
          (F2_conv_ud2sf DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_tfr IntRegs:$src1),
          (A2_tfr IntRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_subri s32_0ImmPred:$src1, IntRegs:$src2),
-         (A2_subri s32_0ImmPred:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asr_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_subri s32_0ImmPred_timm:$src1, IntRegs:$src2),
+         (A2_subri s32_0ImmPred_timm:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrmaxuw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (A4_vrmaxuw DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vmpybuu IntRegs:$src1, IntRegs:$src2),
          (M5_vmpybuu IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_vrmaxuh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (A4_vrmaxuh DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asl_i_vw DoubleRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asl_i_vw DoubleRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavgw DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavgw DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_brev IntRegs:$src1),
          (S2_brev IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vavgh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavgh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_clrbit_i IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_clrbit_i IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2),
-         (S2_asl_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_lsr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_clrbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_clrbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2),
+         (S2_asl_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_lsr_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_lsl_r_r_nac IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmpyl_rs1 DoubleRegs:$src1, DoubleRegs:$src2),
@@ -1343,8 +1343,8 @@ def: Pat<(int_hexagon_M2_mmpyl_s0 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_mmpyl_s0 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmpyl_s1 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_mmpyl_s1 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_naccii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3),
-         (M2_naccii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_naccii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3),
+         (M2_naccii IntRegs:$src1, IntRegs:$src2, s32_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vrndpackwhs DoubleRegs:$src1),
          (S2_vrndpackwhs DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vtrunewh DoubleRegs:$src1, DoubleRegs:$src2),
@@ -1357,24 +1357,24 @@ def: Pat<(int_hexagon_M2_mpyd_ll_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyd_ll_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M4_mac_up_s1_sat IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M4_mac_up_s1_sat IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_vrcrotate_acc DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3, u2_0ImmPred:$src4),
-         (S4_vrcrotate_acc DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3, u2_0ImmPred:$src4)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_vrcrotate_acc DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3, u2_0ImmPred_timm:$src4),
+         (S4_vrcrotate_acc DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3, u2_0ImmPred_timm:$src4)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_uw2df IntRegs:$src1),
          (F2_conv_uw2df IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vaddubs DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vaddubs DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asr_r_r_acc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asr_r_r_acc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A2_orir IntRegs:$src1, s32_0ImmPred:$src2),
-         (A2_orir IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A2_orir IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A2_orir IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_andp DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_andp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lfsp DoubleRegs:$src1, DoubleRegs:$src2),
          (S2_lfsp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_min IntRegs:$src1, IntRegs:$src2),
          (A2_min IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_M2_mpysmi IntRegs:$src1, m32_0ImmPred:$src2),
-         (M2_mpysmi IntRegs:$src1, m32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_M2_mpysmi IntRegs:$src1, m32_0ImmPred_timm:$src2),
+         (M2_mpysmi IntRegs:$src1, m32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vcmpy_s0_sat_r DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vcmpy_s0_sat_r DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyu_acc_ll_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1397,10 +1397,10 @@ def: Pat<(int_hexagon_M2_mpyd_lh_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyd_lh_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_df2w DoubleRegs:$src1),
          (F2_conv_df2w DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S5_asrhub_sat DoubleRegs:$src1, u4_0ImmPred:$src2),
-         (S5_asrhub_sat DoubleRegs:$src1, u4_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asl_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S5_asrhub_sat DoubleRegs:$src1, u4_0ImmPred_timm:$src2),
+         (S5_asrhub_sat DoubleRegs:$src1, u4_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asl_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_df2d DoubleRegs:$src1),
          (F2_conv_df2d DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mmaculs_s1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3),
@@ -1423,8 +1423,8 @@ def: Pat<(int_hexagon_A2_vavghr DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vavghr DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffma_sc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3, PredRegs:$src4),
          (F2_sffma_sc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3, PredRegs:$src4)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_F2_dfclass DoubleRegs:$src1, u5_0ImmPred:$src2),
-         (F2_dfclass DoubleRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_F2_dfclass DoubleRegs:$src1, u5_0ImmPred_timm:$src2),
+         (F2_dfclass DoubleRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_df2ud DoubleRegs:$src1),
          (F2_conv_df2ud DoubleRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_conv_df2uw DoubleRegs:$src1),
@@ -1433,16 +1433,16 @@ def: Pat<(int_hexagon_M2_cmpyrs_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_cmpyrs_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cmpyrs_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_cmpyrs_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C4_cmpltei IntRegs:$src1, s32_0ImmPred:$src2),
-         (C4_cmpltei IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C4_cmpltei IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (C4_cmpltei IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C4_cmplteu IntRegs:$src1, IntRegs:$src2),
          (C4_cmplteu IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vsubb_map DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vsubub DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_subh_l16_ll IntRegs:$src1, IntRegs:$src2),
          (A2_subh_l16_ll IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r_rnd IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asr_i_r_rnd IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r_rnd IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asr_i_r_rnd IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vrmpy_s0 DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vrmpy_s0 DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_rnd_hh_s1 IntRegs:$src1, IntRegs:$src2),
@@ -1471,14 +1471,14 @@ def: Pat<(int_hexagon_M2_mpyud_hl_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyud_hl_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vrcmpyi_s0c DoubleRegs:$src1, DoubleRegs:$src2),
          (M2_vrcmpyi_s0c DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p_rnd DoubleRegs:$src1, u6_0ImmPred:$src2),
-         (S2_asr_i_p_rnd DoubleRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p_rnd DoubleRegs:$src1, u6_0ImmPred_timm:$src2),
+         (S2_asr_i_p_rnd DoubleRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_addpsat DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_addpsat DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_svaddhs IntRegs:$src1, IntRegs:$src2),
          (A2_svaddhs IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_ori_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_ori_lsr_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_ori_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_ori_lsr_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_rnd_ll_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpy_sat_rnd_ll_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_sat_rnd_ll_s0 IntRegs:$src1, IntRegs:$src2),
@@ -1499,8 +1499,8 @@ def: Pat<(int_hexagon_M2_mpyud_lh_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_mpyud_lh_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_asl_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (S2_asl_r_r_or IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_lsli s6_0ImmPred:$src1, IntRegs:$src2),
-         (S4_lsli s6_0ImmPred:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_lsli s6_0ImmPred_timm:$src1, IntRegs:$src2),
+         (S4_lsli s6_0ImmPred_timm:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsl_r_vw DoubleRegs:$src1, IntRegs:$src2),
          (S2_lsl_r_vw DoubleRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_hh_s1 IntRegs:$src1, IntRegs:$src2),
@@ -1529,8 +1529,8 @@ def: Pat<(int_hexagon_A4_cmpbeq IntRegs:$src1, IntRegs:$src2),
          (A4_cmpbeq IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_negp DoubleRegs:$src1),
          (A2_negp DoubleRegs:$src1)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_r_sat IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_asl_i_r_sat IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_r_sat IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_asl_i_r_sat IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_addh_l16_sat_hl IntRegs:$src1, IntRegs:$src2),
          (A2_addh_l16_sat_hl IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vsatwuh DoubleRegs:$src1),
@@ -1541,10 +1541,10 @@ def: Pat<(int_hexagon_S2_svsathb IntRegs:$src1),
          (S2_svsathb IntRegs:$src1)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_cmpgtup DoubleRegs:$src1, DoubleRegs:$src2),
          (C2_cmpgtup DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cround_ri IntRegs:$src1, u5_0ImmPred:$src2),
-         (A4_cround_ri IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_clbpaddi DoubleRegs:$src1, s6_0ImmPred:$src2),
-         (S4_clbpaddi DoubleRegs:$src1, s6_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cround_ri IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (A4_cround_ri IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_clbpaddi DoubleRegs:$src1, s6_0ImmPred_timm:$src2),
+         (S4_clbpaddi DoubleRegs:$src1, s6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_cround_rr IntRegs:$src1, IntRegs:$src2),
          (A4_cround_rr IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C2_mux PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1563,12 +1563,12 @@ def: Pat<(int_hexagon_A2_vminuh DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vminuh DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_vminub DoubleRegs:$src1, DoubleRegs:$src2),
          (A2_vminub DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_extractu IntRegs:$src1, u5_0ImmPred:$src2, u5_0ImmPred:$src3),
-         (S2_extractu IntRegs:$src1, u5_0ImmPred:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_extractu IntRegs:$src1, u5_0ImmPred_timm:$src2, u5_0ImmPred_timm:$src3),
+         (S2_extractu IntRegs:$src1, u5_0ImmPred_timm:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A2_svsubh IntRegs:$src1, IntRegs:$src2),
          (A2_svsubh IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_clbaddi IntRegs:$src1, s6_0ImmPred:$src2),
-         (S4_clbaddi IntRegs:$src1, s6_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_clbaddi IntRegs:$src1, s6_0ImmPred_timm:$src2),
+         (S4_clbaddi IntRegs:$src1, s6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_F2_sffms IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (F2_sffms IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_vsxtbh IntRegs:$src1),
@@ -1589,16 +1589,16 @@ def: Pat<(int_hexagon_M2_mpy_acc_hh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$sr
          (M2_mpy_acc_hh_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_acc_hh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_acc_hh_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S4_addi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S4_addi_asl_ri u32_0ImmPred:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S4_addi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S4_addi_asl_ri u32_0ImmPred_timm:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_nac_hh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyd_nac_hh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyd_nac_hh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyd_nac_hh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S2_asr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_cmpheqi IntRegs:$src1, s32_0ImmPred:$src2),
-         (A4_cmpheqi IntRegs:$src1, s32_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S2_asr_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_cmpheqi IntRegs:$src1, s32_0ImmPred_timm:$src2),
+         (A4_cmpheqi IntRegs:$src1, s32_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_S2_lsr_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3),
          (S2_lsr_r_p_xor DoubleRegs:$src1, DoubleRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_acc_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1623,8 +1623,8 @@ def: Pat<(int_hexagon_M2_mpyud_nac_lh_s1 DoubleRegs:$src1, IntRegs:$src2, IntReg
          (M2_mpyud_nac_lh_s1 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpyud_nac_lh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpyud_nac_lh_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_round_ri_sat IntRegs:$src1, u5_0ImmPred:$src2),
-         (A4_round_ri_sat IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_round_ri_sat IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (A4_round_ri_sat IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_nac_hl_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mpy_nac_hl_s0 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_mpy_nac_hl_s1 IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
@@ -1637,10 +1637,10 @@ def: Pat<(int_hexagon_M2_mmacls_rs1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRe
          (M2_mmacls_rs1 DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_cmaci_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_cmaci_s0 DoubleRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_setbit_i IntRegs:$src1, u5_0ImmPred:$src2),
-         (S2_setbit_i IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asl_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asl_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_setbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S2_setbit_i IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asl_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asl_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_andn IntRegs:$src1, IntRegs:$src2),
          (A4_andn IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M5_vrmpybsu DoubleRegs:$src1, DoubleRegs:$src2),
@@ -1655,8 +1655,8 @@ def: Pat<(int_hexagon_C2_bitsclr IntRegs:$src1, IntRegs:$src2),
          (C2_bitsclr IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_xor_xacc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_xor_xacc IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_A4_vcmpbgtui DoubleRegs:$src1, u7_0ImmPred:$src2),
-         (A4_vcmpbgtui DoubleRegs:$src1, u7_0ImmPred:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_A4_vcmpbgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2),
+         (A4_vcmpbgtui DoubleRegs:$src1, u7_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_A4_ornp DoubleRegs:$src1, DoubleRegs:$src2),
          (A4_ornp DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_C4_and_or PredRegs:$src1, PredRegs:$src2, PredRegs:$src3),
@@ -1673,14 +1673,14 @@ def: Pat<(int_hexagon_M2_vmpy2su_s1 IntRegs:$src1, IntRegs:$src2),
          (M2_vmpy2su_s1 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
 def: Pat<(int_hexagon_M2_vmpy2su_s0 IntRegs:$src1, IntRegs:$src2),
          (M2_vmpy2su_s0 IntRegs:$src1, IntRegs:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_asr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_asr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_C4_nbitsclri IntRegs:$src1, u6_0ImmPred:$src2),
-         (C4_nbitsclri IntRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2),
-         (S2_lsr_i_vh DoubleRegs:$src1, u4_0ImmPred:$src2)>, Requires<[HasV5]>;
-def: Pat<(int_hexagon_S2_lsr_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S2_lsr_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_asr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_asr_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_C4_nbitsclri IntRegs:$src1, u6_0ImmPred_timm:$src2),
+         (C4_nbitsclri IntRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2),
+         (S2_lsr_i_vh DoubleRegs:$src1, u4_0ImmPred_timm:$src2)>, Requires<[HasV5]>;
+def: Pat<(int_hexagon_S2_lsr_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S2_lsr_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV5]>;
 
 // V55 Scalar Instructions.
 
@@ -1689,30 +1689,30 @@ def: Pat<(int_hexagon_A5_ACS DoubleRegs:$src1, DoubleRegs:$src2, DoubleRegs:$src
 
 // V60 Scalar Instructions.
 
-def: Pat<(int_hexagon_S6_rol_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S6_rol_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S6_rol_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S6_rol_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S6_rol_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S6_rol_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_p DoubleRegs:$src1, u6_0ImmPred:$src2),
-         (S6_rol_i_p DoubleRegs:$src1, u6_0ImmPred:$src2)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S6_rol_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S6_rol_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S6_rol_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r IntRegs:$src1, u5_0ImmPred:$src2),
-         (S6_rol_i_r IntRegs:$src1, u5_0ImmPred:$src2)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3),
-         (S6_rol_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred:$src3)>, Requires<[HasV60]>;
-def: Pat<(int_hexagon_S6_rol_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3),
-         (S6_rol_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S6_rol_i_p_and DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S6_rol_i_r_xacc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S6_rol_i_r_and IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S6_rol_i_r_acc IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S6_rol_i_p_xacc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2),
+         (S6_rol_i_p DoubleRegs:$src1, u6_0ImmPred_timm:$src2)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S6_rol_i_p_nac DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S6_rol_i_p_acc DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S6_rol_i_r_or IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2),
+         (S6_rol_i_r IntRegs:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3),
+         (S6_rol_i_r_nac IntRegs:$src1, IntRegs:$src2, u5_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
+def: Pat<(int_hexagon_S6_rol_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3),
+         (S6_rol_i_p_or DoubleRegs:$src1, DoubleRegs:$src2, u6_0ImmPred_timm:$src3)>, Requires<[HasV60]>;
 
 // V62 Scalar Instructions.
 
@@ -1744,8 +1744,8 @@ def: Pat<(int_hexagon_F2_dfadd DoubleRegs:$src1, DoubleRegs:$src2),
          (F2_dfadd DoubleRegs:$src1, DoubleRegs:$src2)>, Requires<[HasV66]>;
 def: Pat<(int_hexagon_M2_mnaci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3),
          (M2_mnaci IntRegs:$src1, IntRegs:$src2, IntRegs:$src3)>, Requires<[HasV66]>;
-def: Pat<(int_hexagon_S2_mask u5_0ImmPred:$src1, u5_0ImmPred:$src2),
-         (S2_mask u5_0ImmPred:$src1, u5_0ImmPred:$src2)>, Requires<[HasV66]>;
+def: Pat<(int_hexagon_S2_mask u5_0ImmPred_timm:$src1, u5_0ImmPred_timm:$src2),
+         (S2_mask u5_0ImmPred_timm:$src1, u5_0ImmPred_timm:$src2)>, Requires<[HasV66]>;
 
 // V60 HVX Instructions.
 
@@ -1773,10 +1773,10 @@ def: Pat<(int_hexagon_V6_vaddh_dv HvxWR:$src1, HvxWR:$src2),
          (V6_vaddh_dv HvxWR:$src1, HvxWR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vaddh_dv_128B HvxWR:$src1, HvxWR:$src2),
          (V6_vaddh_dv HvxWR:$src1, HvxWR:$src2)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrmpybusi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrmpybusi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrmpybusi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vshufoh HvxVR:$src1, HvxVR:$src2),
          (V6_vshufoh HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vshufoh_128B HvxVR:$src1, HvxVR:$src2),
@@ -1789,10 +1789,10 @@ def: Pat<(int_hexagon_V6_vdmpyhsuisat HvxWR:$src1, IntRegs:$src2),
          (V6_vdmpyhsuisat HvxWR:$src1, IntRegs:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vdmpyhsuisat_128B HvxWR:$src1, IntRegs:$src2),
          (V6_vdmpyhsuisat HvxWR:$src1, IntRegs:$src2)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrsadubi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrsadubi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrsadubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vnavgw HvxVR:$src1, HvxVR:$src2),
          (V6_vnavgw HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vnavgw_128B HvxVR:$src1, HvxVR:$src2),
@@ -2369,10 +2369,10 @@ def: Pat<(int_hexagon_V6_vsubhsat HvxVR:$src1, HvxVR:$src2),
          (V6_vsubhsat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubhsat_128B HvxVR:$src1, HvxVR:$src2),
          (V6_vsubhsat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrmpyubi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrmpyubi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrmpyubi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vabsw HvxVR:$src1),
          (V6_vabsw HvxVR:$src1)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vabsw_128B HvxVR:$src1),
@@ -2489,10 +2489,10 @@ def: Pat<(int_hexagon_V6_vmpybv_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_vmpybv_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vmpybv_acc_128B HvxWR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_vmpybv_acc HvxWR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrsadubi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrsadubi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrsadubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vdmpyhb_dv_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3),
          (V6_vdmpyhb_dv_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vdmpyhb_dv_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3),
@@ -2677,10 +2677,10 @@ def: Pat<(int_hexagon_V6_vaddbnq HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_vaddbnq HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vaddbnq_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_vaddbnq HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vlalignbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vlalignbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlalignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vsatwh HvxVR:$src1, HvxVR:$src2),
          (V6_vsatwh HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsatwh_128B HvxVR:$src1, HvxVR:$src2),
@@ -2721,10 +2721,10 @@ def: Pat<(int_hexagon_V6_veqh_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_veqh_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_veqh_and_128B HvxQR:$src1, HvxVR:$src2, HvxVR:$src3),
          (V6_veqh_and HvxQR:$src1, HvxVR:$src2, HvxVR:$src3)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_valignbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_valignbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_valignbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vaddwsat HvxVR:$src1, HvxVR:$src2),
          (V6_vaddwsat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vaddwsat_128B HvxVR:$src1, HvxVR:$src2),
@@ -2885,10 +2885,10 @@ def: Pat<(int_hexagon_V6_vsubh HvxVR:$src1, HvxVR:$src2),
          (V6_vsubh HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubh_128B HvxVR:$src1, HvxVR:$src2),
          (V6_vsubh HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrmpyubi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3),
-         (V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred:$src3)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrmpyubi_128B HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3),
+         (V6_vrmpyubi HvxWR:$src1, IntRegs:$src2, u1_0ImmPred_timm:$src3)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vminw HvxVR:$src1, HvxVR:$src2),
          (V6_vminw HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vminw_128B HvxVR:$src1, HvxVR:$src2),
@@ -2929,10 +2929,10 @@ def: Pat<(int_hexagon_V6_vsubuhw HvxVR:$src1, HvxVR:$src2),
          (V6_vsubuhw HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubuhw_128B HvxVR:$src1, HvxVR:$src2),
          (V6_vsubuhw HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV60, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vrmpybusi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4),
-         (V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred:$src4)>, Requires<[HasV60, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vrmpybusi_acc_128B HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4),
+         (V6_vrmpybusi_acc HvxWR:$src1, HvxWR:$src2, IntRegs:$src3, u1_0ImmPred_timm:$src4)>, Requires<[HasV60, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vasrw HvxVR:$src1, IntRegs:$src2),
          (V6_vasrw HvxVR:$src1, IntRegs:$src2)>, Requires<[HasV60, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vasrw_128B HvxVR:$src1, IntRegs:$src2),
@@ -3016,10 +3016,10 @@ def: Pat<(int_hexagon_V6_vlsrb HvxVR:$src1, IntRegs:$src2),
          (V6_vlsrb HvxVR:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vlsrb_128B HvxVR:$src1, IntRegs:$src2),
          (V6_vlsrb HvxVR:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV62, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vlutvwhi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV62, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV62, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vlutvwhi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlutvwhi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV62, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vaddububb_sat HvxVR:$src1, HvxVR:$src2),
          (V6_vaddububb_sat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vaddububb_sat_128B HvxVR:$src1, HvxVR:$src2),
@@ -3032,10 +3032,10 @@ def: Pat<(int_hexagon_V6_ldtp0 PredRegs:$src1, IntRegs:$src2),
          (V6_ldtp0 PredRegs:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_ldtp0_128B PredRegs:$src1, IntRegs:$src2),
          (V6_ldtp0 PredRegs:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4),
-         (V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4)>, Requires<[HasV62, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vlutvvb_oracci_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4),
-         (V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4)>, Requires<[HasV62, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4),
+         (V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4)>, Requires<[HasV62, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vlutvvb_oracci_128B HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4),
+         (V6_vlutvvb_oracci HvxVR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4)>, Requires<[HasV62, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vsubuwsat_dv HvxWR:$src1, HvxWR:$src2),
          (V6_vsubuwsat_dv HvxWR:$src1, HvxWR:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubuwsat_dv_128B HvxWR:$src1, HvxWR:$src2),
@@ -3124,10 +3124,10 @@ def: Pat<(int_hexagon_V6_vasrwuhrndsat HvxVR:$src1, HvxVR:$src2, IntRegsLow8:$sr
          (V6_vasrwuhrndsat HvxVR:$src1, HvxVR:$src2, IntRegsLow8:$src3)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vasrwuhrndsat_128B HvxVR:$src1, HvxVR:$src2, IntRegsLow8:$src3),
          (V6_vasrwuhrndsat HvxVR:$src1, HvxVR:$src2, IntRegsLow8:$src3)>, Requires<[HasV62, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV62, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vlutvvbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3),
-         (V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred:$src3)>, Requires<[HasV62, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV62, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vlutvvbi_128B HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3),
+         (V6_vlutvvbi HvxVR:$src1, HvxVR:$src2, u3_0ImmPred_timm:$src3)>, Requires<[HasV62, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vsubuwsat HvxVR:$src1, HvxVR:$src2),
          (V6_vsubuwsat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubuwsat_128B HvxVR:$src1, HvxVR:$src2),
@@ -3188,10 +3188,10 @@ def: Pat<(int_hexagon_V6_ldcnp0 PredRegs:$src1, IntRegs:$src2),
          (V6_ldcnp0 PredRegs:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_ldcnp0_128B PredRegs:$src1, IntRegs:$src2),
          (V6_ldcnp0 PredRegs:$src1, IntRegs:$src2)>, Requires<[HasV62, UseHVX128B]>;
-def: Pat<(int_hexagon_V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4),
-         (V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4)>, Requires<[HasV62, UseHVX64B]>;
-def: Pat<(int_hexagon_V6_vlutvwh_oracci_128B HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4),
-         (V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred:$src4)>, Requires<[HasV62, UseHVX128B]>;
+def: Pat<(int_hexagon_V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4),
+         (V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4)>, Requires<[HasV62, UseHVX64B]>;
+def: Pat<(int_hexagon_V6_vlutvwh_oracci_128B HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4),
+         (V6_vlutvwh_oracci HvxWR:$src1, HvxVR:$src2, HvxVR:$src3, u3_0ImmPred_timm:$src4)>, Requires<[HasV62, UseHVX128B]>;
 def: Pat<(int_hexagon_V6_vsubbsat HvxVR:$src1, HvxVR:$src2),
          (V6_vsubbsat HvxVR:$src1, HvxVR:$src2)>, Requires<[HasV62, UseHVX64B]>;
 def: Pat<(int_hexagon_V6_vsubbsat_128B HvxVR:$src1, HvxVR:$src2),
diff --git a/llvm/lib/Target/Hexagon/HexagonDepOperands.td b/llvm/lib/Target/Hexagon/HexagonDepOperands.td
index fdba7b9712580..8a94d96522ccc 100644
--- a/llvm/lib/Target/Hexagon/HexagonDepOperands.td
+++ b/llvm/lib/Target/Hexagon/HexagonDepOperands.td
@@ -8,120 +8,125 @@
 // Automatically generated file, please consult code owner before editing.
 //===----------------------------------------------------------------------===//
 
+multiclass ImmOpPred<code pred, ValueType vt = i32> {
+  def "" : PatLeaf<(vt imm), pred>;
+  def _timm : PatLeaf<(vt timm), pred>;
+}
+
 def s4_0ImmOperand : AsmOperandClass { let Name = "s4_0Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s4_0Imm : Operand<i32> { let ParserMatchClass = s4_0ImmOperand; let DecoderMethod = "s4_0ImmDecoder"; }
-def s4_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<4, 0>(N->getSExtValue());}]>;
+defm s4_0ImmPred : ImmOpPred<[{ return isShiftedInt<4, 0>(N->getSExtValue());}]>;
 def s29_3ImmOperand : AsmOperandClass { let Name = "s29_3Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s29_3Imm : Operand<i32> { let ParserMatchClass = s29_3ImmOperand; let DecoderMethod = "s29_3ImmDecoder"; }
-def s29_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 3>(N->getSExtValue());}]>;
+defm s29_3ImmPred : ImmOpPred<[{ return isShiftedInt<32, 3>(N->getSExtValue());}]>;
 def u6_0ImmOperand : AsmOperandClass { let Name = "u6_0Imm"; let RenderMethod = "addImmOperands"; }
 def u6_0Imm : Operand<i32> { let ParserMatchClass = u6_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u6_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<6, 0>(N->getSExtValue());}]>;
+defm u6_0ImmPred : ImmOpPred<[{ return isShiftedUInt<6, 0>(N->getSExtValue());}]>;
 def a30_2ImmOperand : AsmOperandClass { let Name = "a30_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def a30_2Imm : Operand<i32> { let ParserMatchClass = a30_2ImmOperand; let DecoderMethod = "brtargetDecoder"; let PrintMethod = "printBrtarget"; }
-def a30_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
+defm a30_2ImmPred : ImmOpPred<[{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
 def u29_3ImmOperand : AsmOperandClass { let Name = "u29_3Imm"; let RenderMethod = "addImmOperands"; }
 def u29_3Imm : Operand<i32> { let ParserMatchClass = u29_3ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u29_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<32, 3>(N->getSExtValue());}]>;
+defm u29_3ImmPred : ImmOpPred<[{ return isShiftedUInt<32, 3>(N->getSExtValue());}]>;
 def s8_0ImmOperand : AsmOperandClass { let Name = "s8_0Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s8_0Imm : Operand<i32> { let ParserMatchClass = s8_0ImmOperand; let DecoderMethod = "s8_0ImmDecoder"; }
-def s8_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<8, 0>(N->getSExtValue());}]>;
+defm s8_0ImmPred : ImmOpPred<[{ return isShiftedInt<8, 0>(N->getSExtValue());}]>;
 def u32_0ImmOperand : AsmOperandClass { let Name = "u32_0Imm"; let RenderMethod = "addImmOperands"; }
 def u32_0Imm : Operand<i32> { let ParserMatchClass = u32_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u32_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<32, 0>(N->getSExtValue());}]>;
+defm u32_0ImmPred : ImmOpPred<[{ return isShiftedUInt<32, 0>(N->getSExtValue());}]>;
 def u4_2ImmOperand : AsmOperandClass { let Name = "u4_2Imm"; let RenderMethod = "addImmOperands"; }
 def u4_2Imm : Operand<i32> { let ParserMatchClass = u4_2ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u4_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<4, 2>(N->getSExtValue());}]>;
+defm u4_2ImmPred : ImmOpPred<[{ return isShiftedUInt<4, 2>(N->getSExtValue());}]>;
 def u3_0ImmOperand : AsmOperandClass { let Name = "u3_0Imm"; let RenderMethod = "addImmOperands"; }
 def u3_0Imm : Operand<i32> { let ParserMatchClass = u3_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u3_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<3, 0>(N->getSExtValue());}]>;
+defm u3_0ImmPred : ImmOpPred<[{ return isShiftedUInt<3, 0>(N->getSExtValue());}]>;
 def b15_2ImmOperand : AsmOperandClass { let Name = "b15_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def b15_2Imm : Operand<OtherVT> { let ParserMatchClass = b15_2ImmOperand; let DecoderMethod = "brtargetDecoder"; let PrintMethod = "printBrtarget"; }
-def b15_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<15, 2>(N->getSExtValue());}]>;
+defm b15_2ImmPred : ImmOpPred<[{ return isShiftedInt<15, 2>(N->getSExtValue());}]>;
 def u11_3ImmOperand : AsmOperandClass { let Name = "u11_3Imm"; let RenderMethod = "addImmOperands"; }
 def u11_3Imm : Operand<i32> { let ParserMatchClass = u11_3ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u11_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<11, 3>(N->getSExtValue());}]>;
+defm u11_3ImmPred : ImmOpPred<[{ return isShiftedUInt<11, 3>(N->getSExtValue());}]>;
 def s4_3ImmOperand : AsmOperandClass { let Name = "s4_3Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s4_3Imm : Operand<i32> { let ParserMatchClass = s4_3ImmOperand; let DecoderMethod = "s4_3ImmDecoder"; }
-def s4_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<4, 3>(N->getSExtValue());}]>;
+defm s4_3ImmPred : ImmOpPred<[{ return isShiftedInt<4, 3>(N->getSExtValue());}]>;
 def m32_0ImmOperand : AsmOperandClass { let Name = "m32_0Imm"; let RenderMethod = "addImmOperands"; }
 def m32_0Imm : Operand<i32> { let ParserMatchClass = m32_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def m32_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 0>(N->getSExtValue());}]>;
+defm m32_0ImmPred : ImmOpPred<[{ return isShiftedInt<32, 0>(N->getSExtValue());}]>;
 def u3_1ImmOperand : AsmOperandClass { let Name = "u3_1Imm"; let RenderMethod = "addImmOperands"; }
 def u3_1Imm : Operand<i32> { let ParserMatchClass = u3_1ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u3_1ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<3, 1>(N->getSExtValue());}]>;
+defm u3_1ImmPred : ImmOpPred<[{ return isShiftedUInt<3, 1>(N->getSExtValue());}]>;
 def u1_0ImmOperand : AsmOperandClass { let Name = "u1_0Imm"; let RenderMethod = "addImmOperands"; }
 def u1_0Imm : Operand<i32> { let ParserMatchClass = u1_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u1_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<1, 0>(N->getSExtValue());}]>;
+defm u1_0ImmPred : ImmOpPred<[{ return isShiftedUInt<1, 0>(N->getSExtValue());}]>;
 def s31_1ImmOperand : AsmOperandClass { let Name = "s31_1Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s31_1Imm : Operand<i32> { let ParserMatchClass = s31_1ImmOperand; let DecoderMethod = "s31_1ImmDecoder"; }
-def s31_1ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 1>(N->getSExtValue());}]>;
+defm s31_1ImmPred : ImmOpPred<[{ return isShiftedInt<32, 1>(N->getSExtValue());}]>;
 def s3_0ImmOperand : AsmOperandClass { let Name = "s3_0Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s3_0Imm : Operand<i32> { let ParserMatchClass = s3_0ImmOperand; let DecoderMethod = "s3_0ImmDecoder"; }
-def s3_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<3, 0>(N->getSExtValue());}]>;
+defm s3_0ImmPred : ImmOpPred<[{ return isShiftedInt<3, 0>(N->getSExtValue());}]>;
 def s30_2ImmOperand : AsmOperandClass { let Name = "s30_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s30_2Imm : Operand<i32> { let ParserMatchClass = s30_2ImmOperand; let DecoderMethod = "s30_2ImmDecoder"; }
-def s30_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
+defm s30_2ImmPred : ImmOpPred<[{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
 def u4_0ImmOperand : AsmOperandClass { let Name = "u4_0Imm"; let RenderMethod = "addImmOperands"; }
 def u4_0Imm : Operand<i32> { let ParserMatchClass = u4_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u4_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<4, 0>(N->getSExtValue());}]>;
+defm u4_0ImmPred : ImmOpPred<[{ return isShiftedUInt<4, 0>(N->getSExtValue());}]>;
 def s6_0ImmOperand : AsmOperandClass { let Name = "s6_0Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s6_0Imm : Operand<i32> { let ParserMatchClass = s6_0ImmOperand; let DecoderMethod = "s6_0ImmDecoder"; }
-def s6_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<6, 0>(N->getSExtValue());}]>;
+defm s6_0ImmPred : ImmOpPred<[{ return isShiftedInt<6, 0>(N->getSExtValue());}]>;
 def u5_3ImmOperand : AsmOperandClass { let Name = "u5_3Imm"; let RenderMethod = "addImmOperands"; }
 def u5_3Imm : Operand<i32> { let ParserMatchClass = u5_3ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u5_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<5, 3>(N->getSExtValue());}]>;
+defm u5_3ImmPred : ImmOpPred<[{ return isShiftedUInt<5, 3>(N->getSExtValue());}]>;
 def s32_0ImmOperand : AsmOperandClass { let Name = "s32_0Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s32_0Imm : Operand<i32> { let ParserMatchClass = s32_0ImmOperand; let DecoderMethod = "s32_0ImmDecoder"; }
-def s32_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 0>(N->getSExtValue());}]>;
+defm s32_0ImmPred : ImmOpPred<[{ return isShiftedInt<32, 0>(N->getSExtValue());}]>;
 def s6_3ImmOperand : AsmOperandClass { let Name = "s6_3Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s6_3Imm : Operand<i32> { let ParserMatchClass = s6_3ImmOperand; let DecoderMethod = "s6_3ImmDecoder"; }
-def s6_3ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<6, 3>(N->getSExtValue());}]>;
+defm s6_3ImmPred : ImmOpPred<[{ return isShiftedInt<6, 3>(N->getSExtValue());}]>;
 def u10_0ImmOperand : AsmOperandClass { let Name = "u10_0Imm"; let RenderMethod = "addImmOperands"; }
 def u10_0Imm : Operand<i32> { let ParserMatchClass = u10_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u10_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<10, 0>(N->getSExtValue());}]>;
+defm u10_0ImmPred : ImmOpPred<[{ return isShiftedUInt<10, 0>(N->getSExtValue());}]>;
 def u31_1ImmOperand : AsmOperandClass { let Name = "u31_1Imm"; let RenderMethod = "addImmOperands"; }
 def u31_1Imm : Operand<i32> { let ParserMatchClass = u31_1ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u31_1ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<32, 1>(N->getSExtValue());}]>;
+defm u31_1ImmPred : ImmOpPred<[{ return isShiftedUInt<32, 1>(N->getSExtValue());}]>;
 def s4_1ImmOperand : AsmOperandClass { let Name = "s4_1Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s4_1Imm : Operand<i32> { let ParserMatchClass = s4_1ImmOperand; let DecoderMethod = "s4_1ImmDecoder"; }
-def s4_1ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<4, 1>(N->getSExtValue());}]>;
+defm s4_1ImmPred : ImmOpPred<[{ return isShiftedInt<4, 1>(N->getSExtValue());}]>;
 def u16_0ImmOperand : AsmOperandClass { let Name = "u16_0Imm"; let RenderMethod = "addImmOperands"; }
 def u16_0Imm : Operand<i32> { let ParserMatchClass = u16_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u16_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<16, 0>(N->getSExtValue());}]>;
+defm u16_0ImmPred : ImmOpPred<[{ return isShiftedUInt<16, 0>(N->getSExtValue());}]>;
 def u6_1ImmOperand : AsmOperandClass { let Name = "u6_1Imm"; let RenderMethod = "addImmOperands"; }
 def u6_1Imm : Operand<i32> { let ParserMatchClass = u6_1ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u6_1ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<6, 1>(N->getSExtValue());}]>;
+defm u6_1ImmPred : ImmOpPred<[{ return isShiftedUInt<6, 1>(N->getSExtValue());}]>;
 def u5_2ImmOperand : AsmOperandClass { let Name = "u5_2Imm"; let RenderMethod = "addImmOperands"; }
 def u5_2Imm : Operand<i32> { let ParserMatchClass = u5_2ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u5_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<5, 2>(N->getSExtValue());}]>;
+defm u5_2ImmPred : ImmOpPred<[{ return isShiftedUInt<5, 2>(N->getSExtValue());}]>;
 def u26_6ImmOperand : AsmOperandClass { let Name = "u26_6Imm"; let RenderMethod = "addImmOperands"; }
 def u26_6Imm : Operand<i32> { let ParserMatchClass = u26_6ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u26_6ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<26, 6>(N->getSExtValue());}]>;
+defm u26_6ImmPred : ImmOpPred<[{ return isShiftedUInt<26, 6>(N->getSExtValue());}]>;
 def u6_2ImmOperand : AsmOperandClass { let Name = "u6_2Imm"; let RenderMethod = "addImmOperands"; }
 def u6_2Imm : Operand<i32> { let ParserMatchClass = u6_2ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u6_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<6, 2>(N->getSExtValue());}]>;
+defm u6_2ImmPred : ImmOpPred<[{ return isShiftedUInt<6, 2>(N->getSExtValue());}]>;
 def u7_0ImmOperand : AsmOperandClass { let Name = "u7_0Imm"; let RenderMethod = "addImmOperands"; }
 def u7_0Imm : Operand<i32> { let ParserMatchClass = u7_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u7_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<7, 0>(N->getSExtValue());}]>;
+defm u7_0ImmPred : ImmOpPred<[{ return isShiftedUInt<7, 0>(N->getSExtValue());}]>;
 def b13_2ImmOperand : AsmOperandClass { let Name = "b13_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def b13_2Imm : Operand<OtherVT> { let ParserMatchClass = b13_2ImmOperand; let DecoderMethod = "brtargetDecoder"; let PrintMethod = "printBrtarget"; }
-def b13_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<13, 2>(N->getSExtValue());}]>;
+defm b13_2ImmPred : ImmOpPred<[{ return isShiftedInt<13, 2>(N->getSExtValue());}]>;
 def u5_0ImmOperand : AsmOperandClass { let Name = "u5_0Imm"; let RenderMethod = "addImmOperands"; }
 def u5_0Imm : Operand<i32> { let ParserMatchClass = u5_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u5_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<5, 0>(N->getSExtValue());}]>;
+defm u5_0ImmPred : ImmOpPred<[{ return isShiftedUInt<5, 0>(N->getSExtValue());}]>;
 def u2_0ImmOperand : AsmOperandClass { let Name = "u2_0Imm"; let RenderMethod = "addImmOperands"; }
 def u2_0Imm : Operand<i32> { let ParserMatchClass = u2_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u2_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<2, 0>(N->getSExtValue());}]>;
+defm u2_0ImmPred : ImmOpPred<[{ return isShiftedUInt<2, 0>(N->getSExtValue());}]>;
 def s4_2ImmOperand : AsmOperandClass { let Name = "s4_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def s4_2Imm : Operand<i32> { let ParserMatchClass = s4_2ImmOperand; let DecoderMethod = "s4_2ImmDecoder"; }
-def s4_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<4, 2>(N->getSExtValue());}]>;
+defm s4_2ImmPred : ImmOpPred<[{ return isShiftedInt<4, 2>(N->getSExtValue());}]>;
 def b30_2ImmOperand : AsmOperandClass { let Name = "b30_2Imm"; let RenderMethod = "addSignedImmOperands"; }
 def b30_2Imm : Operand<OtherVT> { let ParserMatchClass = b30_2ImmOperand; let DecoderMethod = "brtargetDecoder"; let PrintMethod = "printBrtarget"; }
-def b30_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
+defm b30_2ImmPred : ImmOpPred<[{ return isShiftedInt<32, 2>(N->getSExtValue());}]>;
 def u8_0ImmOperand : AsmOperandClass { let Name = "u8_0Imm"; let RenderMethod = "addImmOperands"; }
 def u8_0Imm : Operand<i32> { let ParserMatchClass = u8_0ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u8_0ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<8, 0>(N->getSExtValue());}]>;
+defm u8_0ImmPred : ImmOpPred<[{ return isShiftedUInt<8, 0>(N->getSExtValue());}]>;
 def u30_2ImmOperand : AsmOperandClass { let Name = "u30_2Imm"; let RenderMethod = "addImmOperands"; }
 def u30_2Imm : Operand<i32> { let ParserMatchClass = u30_2ImmOperand; let DecoderMethod = "unsignedImmDecoder"; }
-def u30_2ImmPred : PatLeaf<(i32 imm), [{ return isShiftedUInt<32, 2>(N->getSExtValue());}]>;
+defm u30_2ImmPred : ImmOpPred<[{ return isShiftedUInt<32, 2>(N->getSExtValue());}]>;
diff --git a/llvm/lib/Target/Hexagon/HexagonFixupHwLoops.cpp b/llvm/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
index 12eda8af363b4..d21de8ccb5ab1 100644
--- a/llvm/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFixupHwLoops.cpp
@@ -114,12 +114,11 @@ bool HexagonFixupHwLoops::fixupLoopInstrs(MachineFunction &MF) {
 
   // First pass - compute the offset of each basic block.
   for (const MachineBasicBlock &MBB : MF) {
-    if (MBB.getLogAlignment()) {
+    if (MBB.getAlignment() != Align::None()) {
       // Although we don't know the exact layout of the final code, we need
       // to account for alignment padding somehow. This heuristic pads each
       // aligned basic block according to the alignment value.
-      int ByteAlign = (1u << MBB.getLogAlignment()) - 1;
-      InstOffset = (InstOffset + ByteAlign) & ~(ByteAlign);
+      InstOffset = alignTo(InstOffset, MBB.getAlignment());
     }
 
     BlockToInstOffset[&MBB] = InstOffset;
diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
index 70afec14b51a0..bfa3372d7faf9 100644
--- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp
@@ -1380,7 +1380,7 @@ void HexagonFrameLowering::processFunctionBeforeFrameFinalized(
   Align A = MFI.getLocalFrameMaxAlign();
   assert(A <= 8 && "Unexpected local frame alignment");
   if (A == 1)
-    MFI.setLocalFrameMaxAlign(llvm::Align(8));
+    MFI.setLocalFrameMaxAlign(Align(8));
   MFI.setUseLocalStackAllocationBlock(true);
 
   // Set the physical aligned-stack base address register.
diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
index 605fcfc25559d..4684d8e4781a3 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp
@@ -697,7 +697,7 @@ void HexagonDAGToDAGISel::SelectIntrinsicWOChain(SDNode *N) {
 //
 void HexagonDAGToDAGISel::SelectConstantFP(SDNode *N) {
   SDLoc dl(N);
-  ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N);
+  auto *CN = cast<ConstantFPSDNode>(N);
   APInt A = CN->getValueAPF().bitcastToAPInt();
   if (N->getValueType(0) == MVT::f32) {
     SDValue V = CurDAG->getTargetConstant(A.getZExtValue(), dl, MVT::i32);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
index 26a178c53789e..8a8986e232a05 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -240,12 +240,12 @@ bool HexagonTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
   return true;
 }
 
-unsigned  HexagonTargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                              SelectionDAG &DAG) const {
+Register HexagonTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                                  const MachineFunction &) const {
   // Just support r19, the linux kernel uses it.
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+  Register Reg = StringSwitch<Register>(RegName)
                      .Case("r19", Hexagon::R19)
-                     .Default(0);
+                     .Default(Register());
   if (Reg)
     return Reg;
 
@@ -1235,9 +1235,9 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
       Subtarget(ST) {
   auto &HRI = *Subtarget.getRegisterInfo();
 
-  setPrefLoopAlignment(llvm::Align(16));
-  setMinFunctionAlignment(llvm::Align(4));
-  setPrefFunctionAlignment(llvm::Align(16));
+  setPrefLoopAlignment(Align(16));
+  setMinFunctionAlignment(Align(4));
+  setPrefFunctionAlignment(Align(16));
   setStackPointerRegisterToSaveRestore(HRI.getStackRegister());
   setBooleanContents(TargetLoweringBase::UndefinedBooleanContent);
   setBooleanVectorContents(TargetLoweringBase::UndefinedBooleanContent);
@@ -1439,12 +1439,12 @@ HexagonTargetLowering::HexagonTargetLowering(const TargetMachine &TM,
     ISD::CONCAT_VECTORS,        ISD::VECTOR_SHUFFLE
   };
 
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     for (unsigned VectExpOp : VectExpOps)
       setOperationAction(VectExpOp, VT, Expand);
 
     // Expand all extending loads and truncating stores:
-    for (MVT TargetVT : MVT::vector_valuetypes()) {
+    for (MVT TargetVT : MVT::fixedlen_vector_valuetypes()) {
       if (TargetVT == VT)
         continue;
       setLoadExtAction(ISD::EXTLOAD, TargetVT, VT, Expand);
@@ -1864,26 +1864,33 @@ bool HexagonTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask,
 
 TargetLoweringBase::LegalizeTypeAction
 HexagonTargetLowering::getPreferredVectorAction(MVT VT) const {
-  if (VT.getVectorNumElements() == 1)
-    return TargetLoweringBase::TypeScalarizeVector;
-
-  // Always widen vectors of i1.
+  unsigned VecLen = VT.getVectorNumElements();
   MVT ElemTy = VT.getVectorElementType();
-  if (ElemTy == MVT::i1)
-    return TargetLoweringBase::TypeWidenVector;
+
+  if (VecLen == 1 || VT.isScalableVector())
+    return TargetLoweringBase::TypeScalarizeVector;
 
   if (Subtarget.useHVXOps()) {
+    unsigned HwLen = Subtarget.getVectorLength();
     // If the size of VT is at least half of the vector length,
     // widen the vector. Note: the threshold was not selected in
     // any scientific way.
     ArrayRef<MVT> Tys = Subtarget.getHVXElementTypes();
     if (llvm::find(Tys, ElemTy) != Tys.end()) {
-      unsigned HwWidth = 8*Subtarget.getVectorLength();
+      unsigned HwWidth = 8*HwLen;
       unsigned VecWidth = VT.getSizeInBits();
       if (VecWidth >= HwWidth/2 && VecWidth < HwWidth)
         return TargetLoweringBase::TypeWidenVector;
     }
+    // Split vectors of i1 that correspond to (byte) vector pairs.
+    if (ElemTy == MVT::i1 && VecLen == 2*HwLen)
+      return TargetLoweringBase::TypeSplitVector;
   }
+
+  // Always widen (remaining) vectors of i1.
+  if (ElemTy == MVT::i1)
+    return TargetLoweringBase::TypeWidenVector;
+
   return TargetLoweringBase::TypeSplitVector;
 }
 
@@ -2666,7 +2673,8 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
     DoDefault = true;
 
   if (!AlignLoads) {
-    if (allowsMemoryAccess(Ctx, DL, LN->getMemoryVT(), *LN->getMemOperand()))
+    if (allowsMemoryAccessForAlignment(Ctx, DL, LN->getMemoryVT(),
+                                       *LN->getMemOperand()))
       return Op;
     DoDefault = true;
   }
@@ -2674,7 +2682,8 @@ HexagonTargetLowering::LowerUnalignedLoad(SDValue Op, SelectionDAG &DAG)
     // The PartTy is the equivalent of "getLoadableTypeOfSize(HaveAlign)".
     MVT PartTy = HaveAlign <= 8 ? MVT::getIntegerVT(8 * HaveAlign)
                                 : MVT::getVectorVT(MVT::i8, HaveAlign);
-    DoDefault = allowsMemoryAccess(Ctx, DL, PartTy, *LN->getMemOperand());
+    DoDefault =
+        allowsMemoryAccessForAlignment(Ctx, DL, PartTy, *LN->getMemOperand());
   }
   if (DoDefault) {
     std::pair<SDValue, SDValue> P = expandUnalignedLoad(LN, DAG);
diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
index 642ad1db19c11..75f553bfec7fb 100644
--- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h
+++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h
@@ -229,8 +229,8 @@ namespace HexagonISD {
 
     bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     /// If a physical register, this returns the register that receives the
     /// exception address on entry to an EH pad.
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 97318a8057bdd..767538f92ed60 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -674,86 +674,96 @@ unsigned HexagonInstrInfo::insertBranch(MachineBasicBlock &MBB,
   return 2;
 }
 
-/// Analyze the loop code to find the loop induction variable and compare used
-/// to compute the number of iterations. Currently, we analyze loop that are
-/// controlled using hardware loops.  In this case, the induction variable
-/// instruction is null.  For all other cases, this function returns true, which
-/// means we're unable to analyze it.
-bool HexagonInstrInfo::analyzeLoop(MachineLoop &L,
-                                   MachineInstr *&IndVarInst,
-                                   MachineInstr *&CmpInst) const {
-
-  MachineBasicBlock *LoopEnd = L.getBottomBlock();
-  MachineBasicBlock::iterator I = LoopEnd->getFirstTerminator();
-  // We really "analyze" only hardware loops right now.
-  if (I != LoopEnd->end() && isEndLoopN(I->getOpcode())) {
-    IndVarInst = nullptr;
-    CmpInst = &*I;
-    return false;
+namespace {
+class HexagonPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
+  MachineInstr *Loop, *EndLoop;
+  MachineFunction *MF;
+  const HexagonInstrInfo *TII;
+  int64_t TripCount;
+  Register LoopCount;
+  DebugLoc DL;
+
+public:
+  HexagonPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop)
+      : Loop(Loop), EndLoop(EndLoop), MF(Loop->getParent()->getParent()),
+        TII(MF->getSubtarget<HexagonSubtarget>().getInstrInfo()),
+        DL(Loop->getDebugLoc()) {
+    // Inspect the Loop instruction up-front, as it may be deleted when we call
+    // createTripCountGreaterCondition.
+    TripCount = Loop->getOpcode() == Hexagon::J2_loop0r
+                    ? -1
+                    : Loop->getOperand(1).getImm();
+    if (TripCount == -1)
+      LoopCount = Loop->getOperand(1).getReg();
   }
-  return true;
-}
 
-/// Generate code to reduce the loop iteration by one and check if the loop is
-/// finished. Return the value/register of the new loop count. this function
-/// assumes the nth iteration is peeled first.
-unsigned HexagonInstrInfo::reduceLoopCount(
-    MachineBasicBlock &MBB, MachineBasicBlock &PreHeader, MachineInstr *IndVar,
-    MachineInstr &Cmp, SmallVectorImpl<MachineOperand> &Cond,
-    SmallVectorImpl<MachineInstr *> &PrevInsts, unsigned Iter,
-    unsigned MaxIter) const {
-  // We expect a hardware loop currently. This means that IndVar is set
-  // to null, and the compare is the ENDLOOP instruction.
-  assert((!IndVar) && isEndLoopN(Cmp.getOpcode())
-                   && "Expecting a hardware loop");
-  MachineFunction *MF = MBB.getParent();
-  DebugLoc DL = Cmp.getDebugLoc();
-  SmallPtrSet<MachineBasicBlock *, 8> VisitedBBs;
-  MachineInstr *Loop = findLoopInstr(&MBB, Cmp.getOpcode(),
-                                     Cmp.getOperand(0).getMBB(), VisitedBBs);
-  if (!Loop)
-    return 0;
-  // If the loop trip count is a compile-time value, then just change the
-  // value.
-  if (Loop->getOpcode() == Hexagon::J2_loop0i ||
-      Loop->getOpcode() == Hexagon::J2_loop1i) {
-    int64_t Offset = Loop->getOperand(1).getImm();
-    if (Offset <= 1)
-      Loop->eraseFromParent();
-    else
-      Loop->getOperand(1).setImm(Offset - 1);
-    return Offset - 1;
+  bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
+    // Only ignore the terminator.
+    return MI == EndLoop;
   }
-  // The loop trip count is a run-time value. We generate code to subtract
-  // one from the trip count, and update the loop instruction.
-  assert(Loop->getOpcode() == Hexagon::J2_loop0r && "Unexpected instruction");
-  Register LoopCount = Loop->getOperand(1).getReg();
-  // Check if we're done with the loop.
-  unsigned LoopEnd = createVR(MF, MVT::i1);
-  MachineInstr *NewCmp = BuildMI(&MBB, DL, get(Hexagon::C2_cmpgtui), LoopEnd).
-    addReg(LoopCount).addImm(1);
-  unsigned NewLoopCount = createVR(MF, MVT::i32);
-  MachineInstr *NewAdd = BuildMI(&MBB, DL, get(Hexagon::A2_addi), NewLoopCount).
-    addReg(LoopCount).addImm(-1);
-  const HexagonRegisterInfo &HRI = *Subtarget.getRegisterInfo();
-  // Update the previously generated instructions with the new loop counter.
-  for (SmallVectorImpl<MachineInstr *>::iterator I = PrevInsts.begin(),
-         E = PrevInsts.end(); I != E; ++I)
-    (*I)->substituteRegister(LoopCount, NewLoopCount, 0, HRI);
-  PrevInsts.clear();
-  PrevInsts.push_back(NewCmp);
-  PrevInsts.push_back(NewAdd);
-  // Insert the new loop instruction if this is the last time the loop is
-  // decremented.
-  if (Iter == MaxIter)
-    BuildMI(&MBB, DL, get(Hexagon::J2_loop0r)).
-      addMBB(Loop->getOperand(0).getMBB()).addReg(NewLoopCount);
-  // Delete the old loop instruction.
-  if (Iter == 0)
-    Loop->eraseFromParent();
-  Cond.push_back(MachineOperand::CreateImm(Hexagon::J2_jumpf));
-  Cond.push_back(NewCmp->getOperand(0));
-  return NewLoopCount;
+
+  Optional<bool>
+  createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
+                                  SmallVectorImpl<MachineOperand> &Cond) override {
+    if (TripCount == -1) {
+      // Check if we're done with the loop.
+      unsigned Done = TII->createVR(MF, MVT::i1);
+      MachineInstr *NewCmp = BuildMI(&MBB, DL,
+                                     TII->get(Hexagon::C2_cmpgtui), Done)
+                                 .addReg(LoopCount)
+                                 .addImm(TC);
+      Cond.push_back(MachineOperand::CreateImm(Hexagon::J2_jumpf));
+      Cond.push_back(NewCmp->getOperand(0));
+      return {};
+    }
+
+    return TripCount > TC;
+  }
+
+  void setPreheader(MachineBasicBlock *NewPreheader) override {
+    NewPreheader->splice(NewPreheader->getFirstTerminator(), Loop->getParent(),
+                         Loop);
+  }
+
+  void adjustTripCount(int TripCountAdjust) override {
+    // If the loop trip count is a compile-time value, then just change the
+    // value.
+    if (Loop->getOpcode() == Hexagon::J2_loop0i ||
+        Loop->getOpcode() == Hexagon::J2_loop1i) {
+      int64_t TripCount = Loop->getOperand(1).getImm() + TripCountAdjust;
+      assert(TripCount > 0 && "Can't create an empty or negative loop!");
+      Loop->getOperand(1).setImm(TripCount);
+      return;
+    }
+
+    // The loop trip count is a run-time value. We generate code to subtract
+    // one from the trip count, and update the loop instruction.
+    Register LoopCount = Loop->getOperand(1).getReg();
+    Register NewLoopCount = TII->createVR(MF, MVT::i32);
+    BuildMI(*Loop->getParent(), Loop, Loop->getDebugLoc(),
+            TII->get(Hexagon::A2_addi), NewLoopCount)
+        .addReg(LoopCount)
+        .addImm(TripCountAdjust);
+    Loop->getOperand(1).setReg(NewLoopCount);
+  }
+
+  void disposed() override { Loop->eraseFromParent(); }
+};
+} // namespace
+
+std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+HexagonInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+  // We really "analyze" only hardware loops right now.
+  MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
+
+  if (I != LoopBB->end() && isEndLoopN(I->getOpcode())) {
+    SmallPtrSet<MachineBasicBlock *, 8> VisitedBBs;
+    MachineInstr *LoopInst = findLoopInstr(
+        LoopBB, I->getOpcode(), I->getOperand(0).getMBB(), VisitedBBs);
+    if (LoopInst)
+      return std::make_unique<HexagonPipelinerLoopInfo>(LoopInst, &*I);
+  }
+  return nullptr;
 }
 
 bool HexagonInstrInfo::isProfitableToIfCvt(MachineBasicBlock &MBB,
@@ -1856,8 +1866,7 @@ DFAPacketizer *HexagonInstrInfo::CreateTargetScheduleState(
 //  S2_storeri_io %r29, 132, killed %r1; flags:  mem:ST4[FixedStack1]
 // Currently AA considers the addresses in these instructions to be aliasing.
 bool HexagonInstrInfo::areMemAccessesTriviallyDisjoint(
-    const MachineInstr &MIa, const MachineInstr &MIb,
-    AliasAnalysis *AA) const {
+    const MachineInstr &MIa, const MachineInstr &MIb) const {
   if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
       MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
     return false;
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index e0a999d0f4c43..60298cd666bb0 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -129,21 +129,10 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
                         const DebugLoc &DL,
                         int *BytesAdded = nullptr) const override;
 
-  /// Analyze the loop code, return true if it cannot be understood. Upon
-  /// success, this function returns false and returns information about the
-  /// induction variable and compare instruction used at the end.
-  bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst,
-                   MachineInstr *&CmpInst) const override;
-
-  /// Generate code to reduce the loop iteration by one and check if the loop
-  /// is finished.  Return the value/register of the new loop count.  We need
-  /// this function when peeling off one or more iterations of a loop. This
-  /// function assumes the nth iteration is peeled first.
-  unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineBasicBlock &PreHeader,
-                           MachineInstr *IndVar, MachineInstr &Cmp,
-                           SmallVectorImpl<MachineOperand> &Cond,
-                           SmallVectorImpl<MachineInstr *> &PrevInsts,
-                           unsigned Iter, unsigned MaxIter) const override;
+  /// Analyze loop L, which must be a single-basic-block loop, and if the
+  /// conditions can be understood enough produce a PipelinerLoopInfo object.
+  std::unique_ptr<PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 
   /// Return true if it's profitable to predicate
   /// instructions with accumulated instruction latency of "NumCycles"
@@ -299,8 +288,7 @@ class HexagonInstrInfo : public HexagonGenInstrInfo {
   // memory addresses and false otherwise.
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const override;
+                                  const MachineInstr &MIb) const override;
 
   /// For instructions with a base and offset, return the position of the
   /// base register and offset operands.
diff --git a/llvm/lib/Target/Hexagon/HexagonIntrinsics.td b/llvm/lib/Target/Hexagon/HexagonIntrinsics.td
index cabfd783effa4..c5e3cfd080d6c 100644
--- a/llvm/lib/Target/Hexagon/HexagonIntrinsics.td
+++ b/llvm/lib/Target/Hexagon/HexagonIntrinsics.td
@@ -22,14 +22,14 @@ class T_RP_pat <InstHexagon MI, Intrinsic IntID>
 
 def: Pat<(int_hexagon_A2_add IntRegs:$Rs, IntRegs:$Rt),
          (A2_add IntRegs:$Rs, IntRegs:$Rt)>;
-def: Pat<(int_hexagon_A2_addi IntRegs:$Rs, imm:$s16),
+def: Pat<(int_hexagon_A2_addi IntRegs:$Rs, timm:$s16),
          (A2_addi IntRegs:$Rs, imm:$s16)>;
 def: Pat<(int_hexagon_A2_addp DoubleRegs:$Rs, DoubleRegs:$Rt),
          (A2_addp DoubleRegs:$Rs, DoubleRegs:$Rt)>;
 
 def: Pat<(int_hexagon_A2_sub IntRegs:$Rs, IntRegs:$Rt),
          (A2_sub IntRegs:$Rs, IntRegs:$Rt)>;
-def: Pat<(int_hexagon_A2_subri imm:$s10, IntRegs:$Rs),
+def: Pat<(int_hexagon_A2_subri timm:$s10, IntRegs:$Rs),
          (A2_subri imm:$s10, IntRegs:$Rs)>;
 def: Pat<(int_hexagon_A2_subp DoubleRegs:$Rs, DoubleRegs:$Rt),
          (A2_subp DoubleRegs:$Rs, DoubleRegs:$Rt)>;
@@ -45,26 +45,26 @@ def: Pat<(int_hexagon_M2_dpmpyss_s0 IntRegs:$Rs, IntRegs:$Rt),
 def: Pat<(int_hexagon_M2_dpmpyuu_s0 IntRegs:$Rs, IntRegs:$Rt),
          (M2_dpmpyuu_s0 IntRegs:$Rs, IntRegs:$Rt)>;
 
-def: Pat<(int_hexagon_S2_asl_i_r IntRegs:$Rs, imm:$u5),
+def: Pat<(int_hexagon_S2_asl_i_r IntRegs:$Rs, timm:$u5),
          (S2_asl_i_r IntRegs:$Rs, imm:$u5)>;
-def: Pat<(int_hexagon_S2_lsr_i_r IntRegs:$Rs, imm:$u5),
+def: Pat<(int_hexagon_S2_lsr_i_r IntRegs:$Rs, timm:$u5),
          (S2_lsr_i_r IntRegs:$Rs, imm:$u5)>;
-def: Pat<(int_hexagon_S2_asr_i_r IntRegs:$Rs, imm:$u5),
+def: Pat<(int_hexagon_S2_asr_i_r IntRegs:$Rs, timm:$u5),
          (S2_asr_i_r IntRegs:$Rs, imm:$u5)>;
-def: Pat<(int_hexagon_S2_asl_i_p DoubleRegs:$Rs, imm:$u6),
+def: Pat<(int_hexagon_S2_asl_i_p DoubleRegs:$Rs, timm:$u6),
          (S2_asl_i_p DoubleRegs:$Rs, imm:$u6)>;
-def: Pat<(int_hexagon_S2_lsr_i_p DoubleRegs:$Rs, imm:$u6),
+def: Pat<(int_hexagon_S2_lsr_i_p DoubleRegs:$Rs, timm:$u6),
          (S2_lsr_i_p DoubleRegs:$Rs, imm:$u6)>;
-def: Pat<(int_hexagon_S2_asr_i_p DoubleRegs:$Rs, imm:$u6),
+def: Pat<(int_hexagon_S2_asr_i_p DoubleRegs:$Rs, timm:$u6),
          (S2_asr_i_p DoubleRegs:$Rs, imm:$u6)>;
 
 def: Pat<(int_hexagon_A2_and IntRegs:$Rs, IntRegs:$Rt),
          (A2_and IntRegs:$Rs, IntRegs:$Rt)>;
-def: Pat<(int_hexagon_A2_andir IntRegs:$Rs, imm:$s10),
+def: Pat<(int_hexagon_A2_andir IntRegs:$Rs, timm:$s10),
          (A2_andir IntRegs:$Rs, imm:$s10)>;
 def: Pat<(int_hexagon_A2_or IntRegs:$Rs, IntRegs:$Rt),
          (A2_or IntRegs:$Rs, IntRegs:$Rt)>;
-def: Pat<(int_hexagon_A2_orir IntRegs:$Rs, imm:$s10),
+def: Pat<(int_hexagon_A2_orir IntRegs:$Rs, timm:$s10),
          (A2_orir IntRegs:$Rs, imm:$s10)>;
 def: Pat<(int_hexagon_A2_xor IntRegs:$Rs, IntRegs:$Rt),
          (A2_xor IntRegs:$Rs, IntRegs:$Rt)>;
@@ -99,13 +99,13 @@ def : Pat <(int_hexagon_S5_asrhub_rnd_sat_goodsyntax I64:$Rs, (i32 0)),
            (S2_vsathub I64:$Rs)>;
 }
 
-def : Pat <(int_hexagon_S2_asr_i_r_rnd_goodsyntax I32:$Rs, u5_0ImmPred:$imm),
+def : Pat <(int_hexagon_S2_asr_i_r_rnd_goodsyntax I32:$Rs, u5_0ImmPred_timm:$imm),
            (S2_asr_i_r_rnd I32:$Rs, (UDEC1 u5_0ImmPred:$imm))>;
-def : Pat <(int_hexagon_S2_asr_i_p_rnd_goodsyntax I64:$Rs, u6_0ImmPred:$imm),
+def : Pat <(int_hexagon_S2_asr_i_p_rnd_goodsyntax I64:$Rs, u6_0ImmPred_timm:$imm),
            (S2_asr_i_p_rnd I64:$Rs, (UDEC1 u6_0ImmPred:$imm))>;
-def : Pat <(int_hexagon_S5_vasrhrnd_goodsyntax I64:$Rs, u4_0ImmPred:$imm),
+def : Pat <(int_hexagon_S5_vasrhrnd_goodsyntax I64:$Rs, u4_0ImmPred_timm:$imm),
            (S5_vasrhrnd I64:$Rs, (UDEC1 u4_0ImmPred:$imm))>;
-def : Pat <(int_hexagon_S5_asrhub_rnd_sat_goodsyntax I64:$Rs, u4_0ImmPred:$imm),
+def : Pat <(int_hexagon_S5_asrhub_rnd_sat_goodsyntax I64:$Rs, u4_0ImmPred_timm:$imm),
            (S5_asrhub_rnd_sat I64:$Rs, (UDEC1 u4_0ImmPred:$imm))>;
 
 def ImmExt64: SDNodeXForm<imm, [{
@@ -121,13 +121,13 @@ def ImmExt64: SDNodeXForm<imm, [{
 // To connect the builtin with the instruction, the builtin's operand
 // needs to be extended to the right type.
 
-def : Pat<(int_hexagon_A2_tfrpi imm:$Is),
+def : Pat<(int_hexagon_A2_tfrpi timm:$Is),
           (A2_tfrpi (ImmExt64 $Is))>;
 
-def : Pat <(int_hexagon_C2_cmpgei I32:$src1, s32_0ImmPred:$src2),
+def : Pat <(int_hexagon_C2_cmpgei I32:$src1, s32_0ImmPred_timm:$src2),
            (C2_tfrpr (C2_cmpgti I32:$src1, (SDEC1 s32_0ImmPred:$src2)))>;
 
-def : Pat <(int_hexagon_C2_cmpgeui I32:$src1, u32_0ImmPred:$src2),
+def : Pat <(int_hexagon_C2_cmpgeui I32:$src1, u32_0ImmPred_timm:$src2),
            (C2_tfrpr (C2_cmpgtui I32:$src1, (UDEC1 u32_0ImmPred:$src2)))>;
 
 def : Pat <(int_hexagon_C2_cmpgeui I32:$src, 0),
@@ -142,7 +142,7 @@ def : Pat <(int_hexagon_C2_cmpltu I32:$src1, I32:$src2),
 //===----------------------------------------------------------------------===//
 class S2op_tableidx_pat <Intrinsic IntID, InstHexagon OutputInst,
                          SDNodeXForm XformImm>
-  : Pat <(IntID I32:$src1, I32:$src2, u4_0ImmPred:$src3, u5_0ImmPred:$src4),
+  : Pat <(IntID I32:$src1, I32:$src2, u4_0ImmPred_timm:$src3, u5_0ImmPred_timm:$src4),
          (OutputInst I32:$src1, I32:$src2, u4_0ImmPred:$src3,
                      (XformImm u5_0ImmPred:$src4))>;
 
@@ -197,11 +197,11 @@ class T_stc_pat <InstHexagon MI, Intrinsic IntID, PatLeaf Imm, PatLeaf Val>
   : Pat<(IntID I32:$Rs, Val:$Rt, I32:$Ru, Imm:$s),
         (MI I32:$Rs, Imm:$s, I32:$Ru, Val:$Rt)>;
 
-def: T_stc_pat<S2_storerb_pci, int_hexagon_circ_stb,   s4_0ImmPred, I32>;
-def: T_stc_pat<S2_storerh_pci, int_hexagon_circ_sth,   s4_1ImmPred, I32>;
-def: T_stc_pat<S2_storeri_pci, int_hexagon_circ_stw,   s4_2ImmPred, I32>;
-def: T_stc_pat<S2_storerd_pci, int_hexagon_circ_std,   s4_3ImmPred, I64>;
-def: T_stc_pat<S2_storerf_pci, int_hexagon_circ_sthhi, s4_1ImmPred, I32>;
+def: T_stc_pat<S2_storerb_pci, int_hexagon_circ_stb,   s4_0ImmPred_timm, I32>;
+def: T_stc_pat<S2_storerh_pci, int_hexagon_circ_sth,   s4_1ImmPred_timm, I32>;
+def: T_stc_pat<S2_storeri_pci, int_hexagon_circ_stw,   s4_2ImmPred_timm, I32>;
+def: T_stc_pat<S2_storerd_pci, int_hexagon_circ_std,   s4_3ImmPred_timm, I64>;
+def: T_stc_pat<S2_storerf_pci, int_hexagon_circ_sthhi, s4_1ImmPred_timm, I32>;
 
 multiclass MaskedStore <InstHexagon MI, Intrinsic IntID> {
   def : Pat<(IntID HvxQR:$src1, IntRegs:$src2, HvxVR:$src3),
diff --git a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
index 18be03332f82d..bda3eccac0cd4 100644
--- a/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -632,9 +632,9 @@ Value *PolynomialMultiplyRecognize::getCountIV(BasicBlock *BB) {
     if (!isa<ConstantInt>(InitV) || !cast<ConstantInt>(InitV)->isZero())
       continue;
     Value *IterV = PN->getIncomingValueForBlock(BB);
-    if (!isa<BinaryOperator>(IterV))
-      continue;
     auto *BO = dyn_cast<BinaryOperator>(IterV);
+    if (!BO)
+      continue;
     if (BO->getOpcode() != Instruction::Add)
       continue;
     Value *IncV = nullptr;
@@ -2020,7 +2020,7 @@ bool HexagonLoopIdiomRecognize::processCopyingStore(Loop *CurLoop,
   // See if the pointer expression is an AddRec like {base,+,1} on the current
   // loop, which indicates a strided load.  If we have something else, it's a
   // random load we can't handle.
-  LoadInst *LI = dyn_cast<LoadInst>(SI->getValueOperand());
+  auto *LI = cast<LoadInst>(SI->getValueOperand());
   auto *LoadEv = cast<SCEVAddRecExpr>(SE->getSCEV(LI->getPointerOperand()));
 
   // The trip count of the loop and the base pointer of the addrec SCEV is
diff --git a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
index 14f43e14897d8..9121115020a2b 100644
--- a/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonOptAddrMode.cpp
@@ -543,7 +543,7 @@ bool HexagonOptAddrMode::changeLoad(MachineInstr *OldMI, MachineOperand ImmOp,
 bool HexagonOptAddrMode::changeStore(MachineInstr *OldMI, MachineOperand ImmOp,
                                      unsigned ImmOpNum) {
   bool Changed = false;
-  unsigned OpStart;
+  unsigned OpStart = 0;
   unsigned OpEnd = OldMI->getNumOperands();
   MachineBasicBlock *BB = OldMI->getParent();
   auto UsePos = MachineBasicBlock::iterator(OldMI);
diff --git a/llvm/lib/Target/Hexagon/HexagonPatterns.td b/llvm/lib/Target/Hexagon/HexagonPatterns.td
index c54be2f16f1d6..485e658e1c846 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatterns.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatterns.td
@@ -344,6 +344,13 @@ multiclass MinMax_pats<InstHexagon PickT, InstHexagon PickS,
            (PickS CmpPred:$Vs, CmpPred:$Vt)>;
 }
 
+// Bitcasts between same-size vector types are no-ops, except for the
+// actual type change.
+multiclass NopCast_pat<ValueType Ty1, ValueType Ty2, RegisterClass RC> {
+  def: Pat<(Ty1 (bitconvert (Ty2 RC:$Val))), (Ty1 RC:$Val)>;
+  def: Pat<(Ty2 (bitconvert (Ty1 RC:$Val))), (Ty2 RC:$Val)>;
+}
+
 
 // Frags for commonly used SDNodes.
 def Add: pf2<add>;    def And: pf2<and>;    def Sra: pf2<sra>;
@@ -427,17 +434,18 @@ def: Pat<(f32 (bitconvert I32:$v)), (F32:$v)>;
 def: Pat<(i64 (bitconvert F64:$v)), (I64:$v)>;
 def: Pat<(f64 (bitconvert I64:$v)), (F64:$v)>;
 
-multiclass Cast_pat<ValueType Ta, ValueType Tb, RegisterClass RC> {
-  def: Pat<(Tb (bitconvert (Ta RC:$Rs))), (Tb RC:$Rs)>;
-  def: Pat<(Ta (bitconvert (Tb RC:$Rs))), (Ta RC:$Rs)>;
-}
-
-// Bit convert vector types to integers.
-defm: Cast_pat<v4i8,  i32, IntRegs>;
-defm: Cast_pat<v2i16, i32, IntRegs>;
-defm: Cast_pat<v8i8,  i64, DoubleRegs>;
-defm: Cast_pat<v4i16, i64, DoubleRegs>;
-defm: Cast_pat<v2i32, i64, DoubleRegs>;
+// Bit convert 32- and 64-bit types.
+// All of these are bitcastable to one another: i32, v2i16, v4i8.
+defm: NopCast_pat<i32,   v2i16, IntRegs>;
+defm: NopCast_pat<i32,    v4i8, IntRegs>;
+defm: NopCast_pat<v2i16,  v4i8, IntRegs>;
+// All of these are bitcastable to one another: i64, v2i32, v4i16, v8i8.
+defm: NopCast_pat<i64,   v2i32, DoubleRegs>;
+defm: NopCast_pat<i64,   v4i16, DoubleRegs>;
+defm: NopCast_pat<i64,    v8i8, DoubleRegs>;
+defm: NopCast_pat<v2i32, v4i16, DoubleRegs>;
+defm: NopCast_pat<v2i32,  v8i8, DoubleRegs>;
+defm: NopCast_pat<v4i16,  v8i8, DoubleRegs>;
 
 
 // --(3) Extend/truncate -------------------------------------------------
diff --git a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
index 32e2260de07c0..078a7135c55be 100644
--- a/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
+++ b/llvm/lib/Target/Hexagon/HexagonPatternsHVX.td
@@ -160,23 +160,14 @@ let Predicates = [UseHVX] in {
 
 // Bitcasts between same-size vector types are no-ops, except for the
 // actual type change.
-class Bitcast<ValueType ResTy, ValueType InpTy, RegisterClass RC>
-  : Pat<(ResTy (bitconvert (InpTy RC:$Val))), (ResTy RC:$Val)>;
-
 let Predicates = [UseHVX] in {
-  def: Bitcast<VecI8,   VecI16,  HvxVR>;
-  def: Bitcast<VecI8,   VecI32,  HvxVR>;
-  def: Bitcast<VecI16,  VecI8,   HvxVR>;
-  def: Bitcast<VecI16,  VecI32,  HvxVR>;
-  def: Bitcast<VecI32,  VecI8,   HvxVR>;
-  def: Bitcast<VecI32,  VecI16,  HvxVR>;
-
-  def: Bitcast<VecPI8,  VecPI16, HvxWR>;
-  def: Bitcast<VecPI8,  VecPI32, HvxWR>;
-  def: Bitcast<VecPI16, VecPI8,  HvxWR>;
-  def: Bitcast<VecPI16, VecPI32, HvxWR>;
-  def: Bitcast<VecPI32, VecPI8,  HvxWR>;
-  def: Bitcast<VecPI32, VecPI16, HvxWR>;
+  defm: NopCast_pat<VecI8,   VecI16,  HvxVR>;
+  defm: NopCast_pat<VecI8,   VecI32,  HvxVR>;
+  defm: NopCast_pat<VecI16,  VecI32,  HvxVR>;
+
+  defm: NopCast_pat<VecPI8,  VecPI16, HvxWR>;
+  defm: NopCast_pat<VecPI8,  VecPI32, HvxWR>;
+  defm: NopCast_pat<VecPI16, VecPI32, HvxWR>;
 }
 
 let Predicates = [UseHVX] in {
diff --git a/llvm/lib/Target/Hexagon/HexagonSubtarget.h b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
index 007423ef19028..31157a0065d9b 100644
--- a/llvm/lib/Target/Hexagon/HexagonSubtarget.h
+++ b/llvm/lib/Target/Hexagon/HexagonSubtarget.h
@@ -228,7 +228,7 @@ class HexagonSubtarget : public HexagonGenSubtargetInfo {
   }
 
   bool isHVXVectorType(MVT VecTy, bool IncludeBool = false) const {
-    if (!VecTy.isVector() || !useHVXOps())
+    if (!VecTy.isVector() || !useHVXOps() || VecTy.isScalableVector())
       return false;
     MVT ElemTy = VecTy.getVectorElementType();
     if (!IncludeBool && ElemTy == MVT::i1)
diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
index 38062e8e922c7..ddbc5543348d5 100644
--- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
@@ -45,6 +45,8 @@ bool HexagonTTIImpl::useHVX() const {
 
 bool HexagonTTIImpl::isTypeForHVX(Type *VecTy) const {
   assert(VecTy->isVectorTy());
+  if (cast<VectorType>(VecTy)->isScalable())
+    return false;
   // Avoid types like <2 x i32*>.
   if (!cast<VectorType>(VecTy)->getElementType()->isIntegerTy())
     return false;
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
index 215af5b97a9f5..a799f7f7c0b97 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp
@@ -116,8 +116,8 @@ void HexagonMCELFStreamer::HexagonMCEmitCommonSymbol(MCSymbol *Symbol,
     }
 
     // Update the maximum alignment of the section if necessary.
-    if (llvm::Align(ByteAlignment) > Section.getAlignment())
-      Section.setAlignment(llvm::Align(ByteAlignment));
+    if (Align(ByteAlignment) > Section.getAlignment())
+      Section.setAlignment(Align(ByteAlignment));
 
     SwitchSection(P.first, P.second);
   } else {
diff --git a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
index b629a24a99f3d..ec82e3a41f2ad 100644
--- a/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
+++ b/llvm/lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp
@@ -469,13 +469,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
     else if (isa<LanaiMCExpr>(getImm())) {
 #ifndef NDEBUG
       const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(getImm());
-      assert(SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO);
+      assert(SymbolRefExpr &&
+             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_LO);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      assert(isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
+      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
              cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
                  LanaiMCExpr::VK_Lanai_ABS_LO);
 #endif
@@ -499,13 +500,14 @@ struct LanaiOperand : public MCParsedAsmOperand {
     else if (isa<LanaiMCExpr>(getImm())) {
 #ifndef NDEBUG
       const LanaiMCExpr *SymbolRefExpr = dyn_cast<LanaiMCExpr>(getImm());
-      assert(SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_HI);
+      assert(SymbolRefExpr &&
+             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_ABS_HI);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      assert(isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
+      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
              cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
                  LanaiMCExpr::VK_Lanai_ABS_HI);
 #endif
@@ -544,10 +546,9 @@ struct LanaiOperand : public MCParsedAsmOperand {
     } else if (isa<MCBinaryExpr>(getImm())) {
 #ifndef NDEBUG
       const MCBinaryExpr *BinaryExpr = dyn_cast<MCBinaryExpr>(getImm());
-      const LanaiMCExpr *SymbolRefExpr =
-          dyn_cast<LanaiMCExpr>(BinaryExpr->getLHS());
-      assert(SymbolRefExpr &&
-             SymbolRefExpr->getKind() == LanaiMCExpr::VK_Lanai_None);
+      assert(BinaryExpr && isa<LanaiMCExpr>(BinaryExpr->getLHS()) &&
+             cast<LanaiMCExpr>(BinaryExpr->getLHS())->getKind() ==
+                 LanaiMCExpr::VK_Lanai_None);
 #endif
       Inst.addOperand(MCOperand::createExpr(getImm()));
     } else
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
index 5cd72da0daaa4..43933d062a7ef 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.cpp
@@ -145,8 +145,8 @@ LanaiTargetLowering::LanaiTargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::XOR);
 
   // Function alignments
-  setMinFunctionAlignment(llvm::Align(4));
-  setPrefFunctionAlignment(llvm::Align(4));
+  setMinFunctionAlignment(Align(4));
+  setPrefFunctionAlignment(Align(4));
 
   setJumpIsExpensive(true);
 
@@ -212,10 +212,11 @@ SDValue LanaiTargetLowering::LowerOperation(SDValue Op,
 //                       Lanai Inline Assembly Support
 //===----------------------------------------------------------------------===//
 
-unsigned LanaiTargetLowering::getRegisterByName(const char *RegName, EVT /*VT*/,
-                                                SelectionDAG & /*DAG*/) const {
+Register LanaiTargetLowering::getRegisterByName(
+  const char *RegName, EVT /*VT*/,
+  const MachineFunction & /*MF*/) const {
   // Only unallocatable registers should be matched here.
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+  Register Reg = StringSwitch<unsigned>(RegName)
                      .Case("pc", Lanai::PC)
                      .Case("sp", Lanai::SP)
                      .Case("fp", Lanai::FP)
diff --git a/llvm/lib/Target/Lanai/LanaiISelLowering.h b/llvm/lib/Target/Lanai/LanaiISelLowering.h
index e7b5755e90412..4c35a2c6fb8e6 100644
--- a/llvm/lib/Target/Lanai/LanaiISelLowering.h
+++ b/llvm/lib/Target/Lanai/LanaiISelLowering.h
@@ -90,8 +90,8 @@ class LanaiTargetLowering : public TargetLowering {
   SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
 
-  unsigned getRegisterByName(const char *RegName, EVT VT,
-                             SelectionDAG &DAG) const override;
+  Register getRegisterByName(const char *RegName, EVT VT,
+                             const MachineFunction &MF) const override;
   std::pair<unsigned, const TargetRegisterClass *>
   getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
                                StringRef Constraint, MVT VT) const override;
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
index b490a32c834d4..b950fd0424ef4 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.cpp
@@ -86,8 +86,7 @@ void LanaiInstrInfo::loadRegFromStackSlot(
 }
 
 bool LanaiInstrInfo::areMemAccessesTriviallyDisjoint(
-    const MachineInstr &MIa, const MachineInstr &MIb,
-    AliasAnalysis * /*AA*/) const {
+    const MachineInstr &MIa, const MachineInstr &MIb) const {
   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
 
diff --git a/llvm/lib/Target/Lanai/LanaiInstrInfo.h b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
index d71424aeb0b1b..59a04d2cc3889 100644
--- a/llvm/lib/Target/Lanai/LanaiInstrInfo.h
+++ b/llvm/lib/Target/Lanai/LanaiInstrInfo.h
@@ -36,8 +36,7 @@ class LanaiInstrInfo : public LanaiGenInstrInfo {
   }
 
   bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                       const MachineInstr &MIb,
-                                       AliasAnalysis *AA) const override;
+                                       const MachineInstr &MIb) const override;
 
   unsigned isLoadFromStackSlot(const MachineInstr &MI,
                                int &FrameIndex) const override;
diff --git a/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp b/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
index 3a71a084d1afb..a3b91acdc6d02 100644
--- a/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
+++ b/llvm/lib/Target/MSP430/MSP430AsmPrinter.cpp
@@ -159,8 +159,9 @@ void MSP430AsmPrinter::EmitInstruction(const MachineInstr *MI) {
 void MSP430AsmPrinter::EmitInterruptVectorSection(MachineFunction &ISR) {
   MCSection *Cur = OutStreamer->getCurrentSectionOnly();
   const auto *F = &ISR.getFunction();
-  assert(F->hasFnAttribute("interrupt") &&
-         "Functions with MSP430_INTR CC should have 'interrupt' attribute");
+  if (F->getCallingConv() != CallingConv::MSP430_INTR) {
+    report_fatal_error("Functions with 'interrupt' attribute must have msp430_intrcc CC");
+  }
   StringRef IVIdx = F->getFnAttribute("interrupt").getValueAsString();
   MCSection *IV = OutStreamer->getContext().getELFSection(
     "__interrupt_vector_" + IVIdx,
@@ -174,8 +175,9 @@ void MSP430AsmPrinter::EmitInterruptVectorSection(MachineFunction &ISR) {
 
 bool MSP430AsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   // Emit separate section for an interrupt vector if ISR
-  if (MF.getFunction().getCallingConv() == CallingConv::MSP430_INTR)
+  if (MF.getFunction().hasFnAttribute("interrupt")) {
     EmitInterruptVectorSection(MF);
+  }
 
   SetupMachineFunction(MF);
   EmitFunctionBody();
diff --git a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
index 8faa3da6ec361..a83fd131ac357 100644
--- a/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/llvm/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -327,8 +327,8 @@ MSP430TargetLowering::MSP430TargetLowering(const TargetMachine &TM,
   setLibcallCallingConv(RTLIB::OGT_F64, CallingConv::MSP430_BUILTIN);
   // TODO: __mspabi_srall, __mspabi_srlll, __mspabi_sllll
 
-  setMinFunctionAlignment(llvm::Align(2));
-  setPrefFunctionAlignment(llvm::Align(2));
+  setMinFunctionAlignment(Align(2));
+  setPrefFunctionAlignment(Align(2));
 }
 
 SDValue MSP430TargetLowering::LowerOperation(SDValue Op,
diff --git a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index 31f0d2e91a81c..5106ffde73e6b 100644
--- a/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/llvm/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -1805,9 +1805,8 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isIntN(inMicroMipsMode() ? 17 : 18, Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(
-              Offset.getImm(),
-              (inMicroMipsMode() ? llvm::Align(2) : llvm::Align(4))))
+      if (offsetToAlignment(Offset.getImm(),
+                            (inMicroMipsMode() ? Align(2) : Align(4))))
         return Error(IDLoc, "branch to misaligned address");
       break;
     case Mips::BGEZ:
@@ -1836,9 +1835,8 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isIntN(inMicroMipsMode() ? 17 : 18, Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(
-              Offset.getImm(),
-              (inMicroMipsMode() ? llvm::Align(2) : llvm::Align(4))))
+      if (offsetToAlignment(Offset.getImm(),
+                            (inMicroMipsMode() ? Align(2) : Align(4))))
         return Error(IDLoc, "branch to misaligned address");
       break;
     case Mips::BGEC:    case Mips::BGEC_MMR6:
@@ -1853,7 +1851,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isIntN(18, Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(Offset.getImm(), llvm::Align(4)))
+      if (offsetToAlignment(Offset.getImm(), Align(4)))
         return Error(IDLoc, "branch to misaligned address");
       break;
     case Mips::BLEZC:   case Mips::BLEZC_MMR6:
@@ -1866,7 +1864,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isIntN(18, Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(Offset.getImm(), llvm::Align(4)))
+      if (offsetToAlignment(Offset.getImm(), Align(4)))
         return Error(IDLoc, "branch to misaligned address");
       break;
     case Mips::BEQZC:   case Mips::BEQZC_MMR6:
@@ -1877,7 +1875,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isIntN(23, Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(Offset.getImm(), llvm::Align(4)))
+      if (offsetToAlignment(Offset.getImm(), Align(4)))
         return Error(IDLoc, "branch to misaligned address");
       break;
     case Mips::BEQZ16_MM:
@@ -1890,7 +1888,7 @@ bool MipsAsmParser::processInstruction(MCInst &Inst, SMLoc IDLoc,
         break; // We'll deal with this situation later on when applying fixups.
       if (!isInt<8>(Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(Offset.getImm(), llvm::Align(2)))
+      if (offsetToAlignment(Offset.getImm(), Align(2)))
         return Error(IDLoc, "branch to misaligned address");
       break;
     }
@@ -2871,13 +2869,12 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
                                             bool Is32BitSym, SMLoc IDLoc,
                                             MCStreamer &Out,
                                             const MCSubtargetInfo *STI) {
-  // FIXME: These expansions do not respect -mxgot.
   MipsTargetStreamer &TOut = getTargetStreamer();
   bool UseSrcReg = SrcReg != Mips::NoRegister && SrcReg != Mips::ZERO &&
                    SrcReg != Mips::ZERO_64;
   warnIfNoMacro(IDLoc);
 
-  if (inPicMode() && ABI.IsO32()) {
+  if (inPicMode()) {
     MCValue Res;
     if (!SymExpr->evaluateAsRelocatable(Res, nullptr, nullptr)) {
       Error(IDLoc, "expected relocatable expression");
@@ -2888,50 +2885,41 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       return true;
     }
 
+    bool IsPtr64 = ABI.ArePtrs64bit();
     bool IsLocalSym =
         Res.getSymA()->getSymbol().isInSection() ||
         Res.getSymA()->getSymbol().isTemporary() ||
         (Res.getSymA()->getSymbol().isELF() &&
          cast<MCSymbolELF>(Res.getSymA()->getSymbol()).getBinding() ==
              ELF::STB_LOCAL);
+    bool UseXGOT = STI->getFeatureBits()[Mips::FeatureXGOT] && !IsLocalSym;
 
     // The case where the result register is $25 is somewhat special. If the
     // symbol in the final relocation is external and not modified with a
-    // constant then we must use R_MIPS_CALL16 instead of R_MIPS_GOT16.
+    // constant then we must use R_MIPS_CALL16 instead of R_MIPS_GOT16
+    // or R_MIPS_CALL16 instead of R_MIPS_GOT_DISP in 64-bit case.
     if ((DstReg == Mips::T9 || DstReg == Mips::T9_64) && !UseSrcReg &&
         Res.getConstant() == 0 && !IsLocalSym) {
-      const MCExpr *CallExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_GOT_CALL, SymExpr, getContext());
-      TOut.emitRRX(Mips::LW, DstReg, GPReg, MCOperand::createExpr(CallExpr),
-                   IDLoc, STI);
+      if (UseXGOT) {
+        const MCExpr *CallHiExpr = MipsMCExpr::create(MipsMCExpr::MEK_CALL_HI16,
+                                                      SymExpr, getContext());
+        const MCExpr *CallLoExpr = MipsMCExpr::create(MipsMCExpr::MEK_CALL_LO16,
+                                                      SymExpr, getContext());
+        TOut.emitRX(Mips::LUi, DstReg, MCOperand::createExpr(CallHiExpr), IDLoc,
+                    STI);
+        TOut.emitRRR(IsPtr64 ? Mips::DADDu : Mips::ADDu, DstReg, DstReg, GPReg,
+                     IDLoc, STI);
+        TOut.emitRRX(IsPtr64 ? Mips::LD : Mips::LW, DstReg, DstReg,
+                     MCOperand::createExpr(CallLoExpr), IDLoc, STI);
+      } else {
+        const MCExpr *CallExpr =
+            MipsMCExpr::create(MipsMCExpr::MEK_GOT_CALL, SymExpr, getContext());
+        TOut.emitRRX(IsPtr64 ? Mips::LD : Mips::LW, DstReg, GPReg,
+                     MCOperand::createExpr(CallExpr), IDLoc, STI);
+      }
       return false;
     }
 
-    // The remaining cases are:
-    //   External GOT: lw $tmp, %got(symbol)($gp)
-    //                >addiu $tmp, $tmp, offset
-    //                >addiu $rd, $tmp, $rs
-    //   Local GOT:    lw $tmp, %got(symbol+offset)($gp)
-    //                 addiu $tmp, $tmp, %lo(symbol+offset)($gp)
-    //                >addiu $rd, $tmp, $rs
-    // The addiu's marked with a '>' may be omitted if they are redundant. If
-    // this happens then the last instruction must use $rd as the result
-    // register.
-    const MipsMCExpr *GotExpr = nullptr;
-    const MCExpr *LoExpr = nullptr;
-    if (IsLocalSym) {
-      GotExpr = MipsMCExpr::create(MipsMCExpr::MEK_GOT, SymExpr, getContext());
-      LoExpr = MipsMCExpr::create(MipsMCExpr::MEK_LO, SymExpr, getContext());
-    } else {
-      // External symbols fully resolve the symbol with just the %got(symbol)
-      // but we must still account for any offset to the symbol for expressions
-      // like symbol+8.
-      GotExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_GOT, Res.getSymA(), getContext());
-      if (Res.getConstant() != 0)
-        LoExpr = MCConstantExpr::create(Res.getConstant(), getContext());
-    }
-
     unsigned TmpReg = DstReg;
     if (UseSrcReg &&
         getContext().getRegisterInfo()->isSuperOrSubRegisterEq(DstReg,
@@ -2944,94 +2932,102 @@ bool MipsAsmParser::loadAndAddSymbolAddress(const MCExpr *SymExpr,
       TmpReg = ATReg;
     }
 
-    TOut.emitRRX(Mips::LW, TmpReg, GPReg, MCOperand::createExpr(GotExpr), IDLoc,
-                 STI);
-
-    if (LoExpr)
-      TOut.emitRRX(Mips::ADDiu, TmpReg, TmpReg, MCOperand::createExpr(LoExpr),
+    if (UseXGOT) {
+      // Loading address from XGOT
+      //   External GOT: lui $tmp, %got_hi(symbol)($gp)
+      //                 addu $tmp, $tmp, $gp
+      //                 lw $tmp, %got_lo(symbol)($tmp)
+      //                >addiu $tmp, $tmp, offset
+      //                >addiu $rd, $tmp, $rs
+      // The addiu's marked with a '>' may be omitted if they are redundant. If
+      // this happens then the last instruction must use $rd as the result
+      // register.
+      const MCExpr *CallHiExpr =
+          MipsMCExpr::create(MipsMCExpr::MEK_GOT_HI16, SymExpr, getContext());
+      const MCExpr *CallLoExpr = MipsMCExpr::create(
+          MipsMCExpr::MEK_GOT_LO16, Res.getSymA(), getContext());
+
+      TOut.emitRX(Mips::LUi, TmpReg, MCOperand::createExpr(CallHiExpr), IDLoc,
+                  STI);
+      TOut.emitRRR(IsPtr64 ? Mips::DADDu : Mips::ADDu, TmpReg, TmpReg, GPReg,
                    IDLoc, STI);
+      TOut.emitRRX(IsPtr64 ? Mips::LD : Mips::LW, TmpReg, TmpReg,
+                   MCOperand::createExpr(CallLoExpr), IDLoc, STI);
 
-    if (UseSrcReg)
-      TOut.emitRRR(Mips::ADDu, DstReg, TmpReg, SrcReg, IDLoc, STI);
-
-    return false;
-  }
-
-  if (inPicMode() && ABI.ArePtrs64bit()) {
-    MCValue Res;
-    if (!SymExpr->evaluateAsRelocatable(Res, nullptr, nullptr)) {
-      Error(IDLoc, "expected relocatable expression");
-      return true;
-    }
-    if (Res.getSymB() != nullptr) {
-      Error(IDLoc, "expected relocatable expression with only one symbol");
-      return true;
-    }
+      if (Res.getConstant() != 0)
+        TOut.emitRRX(IsPtr64 ? Mips::DADDiu : Mips::ADDiu, TmpReg, TmpReg,
+                     MCOperand::createExpr(MCConstantExpr::create(
+                         Res.getConstant(), getContext())),
+                     IDLoc, STI);
 
-    // The case where the result register is $25 is somewhat special. If the
-    // symbol in the final relocation is external and not modified with a
-    // constant then we must use R_MIPS_CALL16 instead of R_MIPS_GOT_DISP.
-    if ((DstReg == Mips::T9 || DstReg == Mips::T9_64) && !UseSrcReg &&
-        Res.getConstant() == 0 &&
-        !(Res.getSymA()->getSymbol().isInSection() ||
-          Res.getSymA()->getSymbol().isTemporary() ||
-          (Res.getSymA()->getSymbol().isELF() &&
-           cast<MCSymbolELF>(Res.getSymA()->getSymbol()).getBinding() ==
-               ELF::STB_LOCAL))) {
-      const MCExpr *CallExpr =
-          MipsMCExpr::create(MipsMCExpr::MEK_GOT_CALL, SymExpr, getContext());
-      TOut.emitRRX(Mips::LD, DstReg, GPReg, MCOperand::createExpr(CallExpr),
-                   IDLoc, STI);
+      if (UseSrcReg)
+        TOut.emitRRR(IsPtr64 ? Mips::DADDu : Mips::ADDu, DstReg, TmpReg, SrcReg,
+                     IDLoc, STI);
       return false;
     }
 
-    // The remaining cases are:
-    //   Small offset: ld $tmp, %got_disp(symbol)($gp)
-    //                >daddiu $tmp, $tmp, offset
-    //                >daddu $rd, $tmp, $rs
-    // The daddiu's marked with a '>' may be omitted if they are redundant. If
-    // this happens then the last instruction must use $rd as the result
-    // register.
-    const MipsMCExpr *GotExpr = MipsMCExpr::create(MipsMCExpr::MEK_GOT_DISP,
-                                                   Res.getSymA(),
-                                                   getContext());
+    const MipsMCExpr *GotExpr = nullptr;
     const MCExpr *LoExpr = nullptr;
-    if (Res.getConstant() != 0) {
-      // Symbols fully resolve with just the %got_disp(symbol) but we
-      // must still account for any offset to the symbol for
-      // expressions like symbol+8.
-      LoExpr = MCConstantExpr::create(Res.getConstant(), getContext());
-
-      // FIXME: Offsets greater than 16 bits are not yet implemented.
-      // FIXME: The correct range is a 32-bit sign-extended number.
-      if (Res.getConstant() < -0x8000 || Res.getConstant() > 0x7fff) {
-        Error(IDLoc, "macro instruction uses large offset, which is not "
-                     "currently supported");
-        return true;
-      }
-    }
+    if (IsPtr64) {
+      // The remaining cases are:
+      //   Small offset: ld $tmp, %got_disp(symbol)($gp)
+      //                >daddiu $tmp, $tmp, offset
+      //                >daddu $rd, $tmp, $rs
+      // The daddiu's marked with a '>' may be omitted if they are redundant. If
+      // this happens then the last instruction must use $rd as the result
+      // register.
+      GotExpr = MipsMCExpr::create(MipsMCExpr::MEK_GOT_DISP, Res.getSymA(),
+                                   getContext());
+      if (Res.getConstant() != 0) {
+        // Symbols fully resolve with just the %got_disp(symbol) but we
+        // must still account for any offset to the symbol for
+        // expressions like symbol+8.
+        LoExpr = MCConstantExpr::create(Res.getConstant(), getContext());
 
-    unsigned TmpReg = DstReg;
-    if (UseSrcReg &&
-        getContext().getRegisterInfo()->isSuperOrSubRegisterEq(DstReg,
-                                                               SrcReg)) {
-      // If $rs is the same as $rd, we need to use AT.
-      // If it is not available we exit.
-      unsigned ATReg = getATReg(IDLoc);
-      if (!ATReg)
-        return true;
-      TmpReg = ATReg;
+        // FIXME: Offsets greater than 16 bits are not yet implemented.
+        // FIXME: The correct range is a 32-bit sign-extended number.
+        if (Res.getConstant() < -0x8000 || Res.getConstant() > 0x7fff) {
+          Error(IDLoc, "macro instruction uses large offset, which is not "
+                       "currently supported");
+          return true;
+        }
+      }
+    } else {
+      // The remaining cases are:
+      //   External GOT: lw $tmp, %got(symbol)($gp)
+      //                >addiu $tmp, $tmp, offset
+      //                >addiu $rd, $tmp, $rs
+      //   Local GOT:    lw $tmp, %got(symbol+offset)($gp)
+      //                 addiu $tmp, $tmp, %lo(symbol+offset)($gp)
+      //                >addiu $rd, $tmp, $rs
+      // The addiu's marked with a '>' may be omitted if they are redundant. If
+      // this happens then the last instruction must use $rd as the result
+      // register.
+      if (IsLocalSym) {
+        GotExpr =
+            MipsMCExpr::create(MipsMCExpr::MEK_GOT, SymExpr, getContext());
+        LoExpr = MipsMCExpr::create(MipsMCExpr::MEK_LO, SymExpr, getContext());
+      } else {
+        // External symbols fully resolve the symbol with just the %got(symbol)
+        // but we must still account for any offset to the symbol for
+        // expressions like symbol+8.
+        GotExpr = MipsMCExpr::create(MipsMCExpr::MEK_GOT, Res.getSymA(),
+                                     getContext());
+        if (Res.getConstant() != 0)
+          LoExpr = MCConstantExpr::create(Res.getConstant(), getContext());
+      }
     }
 
-    TOut.emitRRX(Mips::LD, TmpReg, GPReg, MCOperand::createExpr(GotExpr), IDLoc,
-                 STI);
+    TOut.emitRRX(IsPtr64 ? Mips::LD : Mips::LW, TmpReg, GPReg,
+                 MCOperand::createExpr(GotExpr), IDLoc, STI);
 
     if (LoExpr)
-      TOut.emitRRX(Mips::DADDiu, TmpReg, TmpReg, MCOperand::createExpr(LoExpr),
-                   IDLoc, STI);
+      TOut.emitRRX(IsPtr64 ? Mips::DADDiu : Mips::ADDiu, TmpReg, TmpReg,
+                   MCOperand::createExpr(LoExpr), IDLoc, STI);
 
     if (UseSrcReg)
-      TOut.emitRRR(Mips::DADDu, DstReg, TmpReg, SrcReg, IDLoc, STI);
+      TOut.emitRRR(IsPtr64 ? Mips::DADDu : Mips::ADDu, DstReg, TmpReg, SrcReg,
+                   IDLoc, STI);
 
     return false;
   }
@@ -3497,7 +3493,7 @@ bool MipsAsmParser::expandUncondBranchMMPseudo(MCInst &Inst, SMLoc IDLoc,
     } else {
       if (!isInt<17>(Offset.getImm()))
         return Error(IDLoc, "branch target out of range");
-      if (offsetToAlignment(Offset.getImm(), llvm::Align(2)))
+      if (offsetToAlignment(Offset.getImm(), Align(2)))
         return Error(IDLoc, "branch to misaligned address");
       Inst.clear();
       Inst.setOpcode(Mips::BEQ_MM);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
index 4d7e36995ae4f..cca75dfc45c28 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h
@@ -66,9 +66,9 @@ class MipsAsmBackend : public MCAsmBackend {
 
   /// fixupNeedsRelaxation - Target specific predicate for whether a given
   /// fixup requires the associated instruction to be relaxed.
-   bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
-                             const MCRelaxableFragment *DF,
-                             const MCAsmLayout &Layout) const override {
+  bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value,
+                            const MCRelaxableFragment *DF,
+                            const MCAsmLayout &Layout) const override {
     // FIXME.
     llvm_unreachable("RelaxInstruction() unimplemented");
     return false;
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
index a7a07cadcd989..a84ca8ccfb2d1 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h
@@ -15,7 +15,7 @@
 namespace llvm {
 
 // NaCl MIPS sandbox's instruction bundle size.
-static const llvm::Align MIPS_NACL_BUNDLE_ALIGN = llvm::Align(16);
+static const Align MIPS_NACL_BUNDLE_ALIGN = Align(16);
 
 bool isBasePlusOffsetMemoryAccess(unsigned Opcode, unsigned *AddrIdx,
                                   bool *IsStore = nullptr);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
index 874341e412474..3ff9c722484bf 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp
@@ -37,7 +37,7 @@ void MipsRegInfoRecord::EmitMipsOptionRecord() {
         Context.getELFSection(".MIPS.options", ELF::SHT_MIPS_OPTIONS,
                               ELF::SHF_ALLOC | ELF::SHF_MIPS_NOSTRIP, 1, "");
     MCA.registerSection(*Sec);
-    Sec->setAlignment(llvm::Align(8));
+    Sec->setAlignment(Align(8));
     Streamer->SwitchSection(Sec);
 
     Streamer->EmitIntValue(ELF::ODK_REGINFO, 1);  // kind
@@ -55,7 +55,7 @@ void MipsRegInfoRecord::EmitMipsOptionRecord() {
     MCSectionELF *Sec = Context.getELFSection(".reginfo", ELF::SHT_MIPS_REGINFO,
                                               ELF::SHF_ALLOC, 24, "");
     MCA.registerSection(*Sec);
-    Sec->setAlignment(MTS->getABI().IsN32() ? llvm::Align(8) : llvm::Align(4));
+    Sec->setAlignment(MTS->getABI().IsN32() ? Align(8) : Align(4));
     Streamer->SwitchSection(Sec);
 
     Streamer->EmitIntValue(ri_gprmask, 4);
diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
index d3cc29b8d6af9..b6dae9f6dea82 100644
--- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
+++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp
@@ -901,12 +901,9 @@ void MipsTargetELFStreamer::finish() {
   MCSection &BSSSection = *OFI.getBSSSection();
   MCA.registerSection(BSSSection);
 
-  TextSection.setAlignment(
-      llvm::Align(std::max(16u, TextSection.getAlignment())));
-  DataSection.setAlignment(
-      llvm::Align(std::max(16u, DataSection.getAlignment())));
-  BSSSection.setAlignment(
-      llvm::Align(std::max(16u, BSSSection.getAlignment())));
+  TextSection.setAlignment(Align(std::max(16u, TextSection.getAlignment())));
+  DataSection.setAlignment(Align(std::max(16u, DataSection.getAlignment())));
+  BSSSection.setAlignment(Align(std::max(16u, BSSSection.getAlignment())));
 
   if (RoundSectionSizes) {
     // Make sections sizes a multiple of the alignment. This is useful for
@@ -1029,7 +1026,7 @@ void MipsTargetELFStreamer::emitDirectiveEnd(StringRef Name) {
       MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Context);
 
   MCA.registerSection(*Sec);
-  Sec->setAlignment(llvm::Align(4));
+  Sec->setAlignment(Align(4));
 
   OS.PushSection();
 
@@ -1319,7 +1316,7 @@ void MipsTargetELFStreamer::emitMipsAbiFlags() {
   MCSectionELF *Sec = Context.getELFSection(
       ".MIPS.abiflags", ELF::SHT_MIPS_ABIFLAGS, ELF::SHF_ALLOC, 24, "");
   MCA.registerSection(*Sec);
-  Sec->setAlignment(llvm::Align(8));
+  Sec->setAlignment(Align(8));
   OS.SwitchSection(Sec);
 
   OS << ABIFlagsSection;
diff --git a/llvm/lib/Target/Mips/MicroMipsDSPInstrInfo.td b/llvm/lib/Target/Mips/MicroMipsDSPInstrInfo.td
index 5a12568893aff..9a1e47e5ecca4 100644
--- a/llvm/lib/Target/Mips/MicroMipsDSPInstrInfo.td
+++ b/llvm/lib/Target/Mips/MicroMipsDSPInstrInfo.td
@@ -360,7 +360,7 @@ class RDDSP_MM_DESC {
   dag OutOperandList = (outs GPR32Opnd:$rt);
   dag InOperandList = (ins uimm7:$mask);
   string AsmString = !strconcat("rddsp", "\t$rt, $mask");
-  list<dag> Pattern = [(set GPR32Opnd:$rt, (int_mips_rddsp immZExt7:$mask))];
+  list<dag> Pattern = [(set GPR32Opnd:$rt, (int_mips_rddsp timmZExt7:$mask))];
   InstrItinClass Itinerary = NoItinerary;
 }
 
@@ -383,7 +383,7 @@ class WRDSP_MM_DESC {
   dag OutOperandList = (outs);
   dag InOperandList = (ins GPR32Opnd:$rt, uimm7:$mask);
   string AsmString = !strconcat("wrdsp", "\t$rt, $mask");
-  list<dag> Pattern = [(int_mips_wrdsp GPR32Opnd:$rt, immZExt7:$mask)];
+  list<dag> Pattern = [(int_mips_wrdsp GPR32Opnd:$rt, timmZExt7:$mask)];
   InstrItinClass Itinerary = NoItinerary;
   bit isMoveReg = 1;
 }
diff --git a/llvm/lib/Target/Mips/Mips.td b/llvm/lib/Target/Mips/Mips.td
index 52f400f91f817..0e5a58303748f 100644
--- a/llvm/lib/Target/Mips/Mips.td
+++ b/llvm/lib/Target/Mips/Mips.td
@@ -209,6 +209,9 @@ def FeatureMT : SubtargetFeature<"mt", "HasMT", "true", "Mips MT ASE">;
 def FeatureLongCalls : SubtargetFeature<"long-calls", "UseLongCalls", "true",
                                         "Disable use of the jal instruction">;
 
+def FeatureXGOT
+    : SubtargetFeature<"xgot", "UseXGOT", "true", "Assume 32-bit GOT">;
+
 def FeatureUseIndirectJumpsHazard : SubtargetFeature<"use-indirect-jump-hazard",
                                                     "UseIndirectJumpsHazard",
                                                     "true", "Use indirect jump"
diff --git a/llvm/lib/Target/Mips/Mips64InstrInfo.td b/llvm/lib/Target/Mips/Mips64InstrInfo.td
index 7dea56c01c82f..cc15949b0d57f 100644
--- a/llvm/lib/Target/Mips/Mips64InstrInfo.td
+++ b/llvm/lib/Target/Mips/Mips64InstrInfo.td
@@ -16,6 +16,7 @@
 
 // shamt must fit in 6 bits.
 def immZExt6 : ImmLeaf<i32, [{return Imm == (Imm & 0x3f);}]>;
+def timmZExt6 : TImmLeaf<i32, [{return Imm == (Imm & 0x3f);}]>;
 
 // Node immediate fits as 10-bit sign extended on target immediate.
 // e.g. seqi, snei
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.cpp b/llvm/lib/Target/Mips/MipsCallLowering.cpp
index aac29967f599c..2853203398c0a 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsCallLowering.cpp
@@ -358,7 +358,7 @@ bool OutgoingValueHandler::handleSplit(SmallVectorImpl<Register> &VRegs,
   return true;
 }
 
-static bool isSupportedType(Type *T) {
+static bool isSupportedArgumentType(Type *T) {
   if (T->isIntegerTy())
     return true;
   if (T->isPointerTy())
@@ -368,6 +368,18 @@ static bool isSupportedType(Type *T) {
   return false;
 }
 
+static bool isSupportedReturnType(Type *T) {
+  if (T->isIntegerTy())
+    return true;
+  if (T->isPointerTy())
+    return true;
+  if (T->isFloatingPointTy())
+    return true;
+  if (T->isAggregateType())
+    return true;
+  return false;
+}
+
 static CCValAssign::LocInfo determineLocInfo(const MVT RegisterVT, const EVT VT,
                                              const ISD::ArgFlagsTy &Flags) {
   // > does not mean loss of information as type RegisterVT can't hold type VT,
@@ -404,7 +416,7 @@ bool MipsCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
 
   MachineInstrBuilder Ret = MIRBuilder.buildInstrNoInsert(Mips::RetRA);
 
-  if (Val != nullptr && !isSupportedType(Val->getType()))
+  if (Val != nullptr && !isSupportedReturnType(Val->getType()))
     return false;
 
   if (!VRegs.empty()) {
@@ -412,21 +424,13 @@ bool MipsCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
     const Function &F = MF.getFunction();
     const DataLayout &DL = MF.getDataLayout();
     const MipsTargetLowering &TLI = *getTLI<MipsTargetLowering>();
-    LLVMContext &Ctx = Val->getType()->getContext();
-
-    SmallVector<EVT, 4> SplitEVTs;
-    ComputeValueVTs(TLI, DL, Val->getType(), SplitEVTs);
-    assert(VRegs.size() == SplitEVTs.size() &&
-           "For each split Type there should be exactly one VReg.");
 
     SmallVector<ArgInfo, 8> RetInfos;
     SmallVector<unsigned, 8> OrigArgIndices;
 
-    for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
-      ArgInfo CurArgInfo = ArgInfo{VRegs[i], SplitEVTs[i].getTypeForEVT(Ctx)};
-      setArgFlags(CurArgInfo, AttributeList::ReturnIndex, DL, F);
-      splitToValueTypes(CurArgInfo, 0, RetInfos, OrigArgIndices);
-    }
+    ArgInfo ArgRetInfo(VRegs, Val->getType());
+    setArgFlags(ArgRetInfo, AttributeList::ReturnIndex, DL, F);
+    splitToValueTypes(DL, ArgRetInfo, 0, RetInfos, OrigArgIndices);
 
     SmallVector<ISD::OutputArg, 8> Outs;
     subTargetRegTypeForCallingConv(F, RetInfos, OrigArgIndices, Outs);
@@ -454,12 +458,8 @@ bool MipsCallLowering::lowerFormalArguments(
   if (F.arg_empty())
     return true;
 
-  if (F.isVarArg()) {
-    return false;
-  }
-
   for (auto &Arg : F.args()) {
-    if (!isSupportedType(Arg.getType()))
+    if (!isSupportedArgumentType(Arg.getType()))
       return false;
   }
 
@@ -473,7 +473,8 @@ bool MipsCallLowering::lowerFormalArguments(
   for (auto &Arg : F.args()) {
     ArgInfo AInfo(VRegs[i], Arg.getType());
     setArgFlags(AInfo, i + AttributeList::FirstArgIndex, DL, F);
-    splitToValueTypes(AInfo, i, ArgInfos, OrigArgIndices);
+    ArgInfos.push_back(AInfo);
+    OrigArgIndices.push_back(i);
     ++i;
   }
 
@@ -496,6 +497,40 @@ bool MipsCallLowering::lowerFormalArguments(
   if (!Handler.handle(ArgLocs, ArgInfos))
     return false;
 
+  if (F.isVarArg()) {
+    ArrayRef<MCPhysReg> ArgRegs = ABI.GetVarArgRegs();
+    unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
+
+    int VaArgOffset;
+    unsigned RegSize = 4;
+    if (ArgRegs.size() == Idx)
+      VaArgOffset = alignTo(CCInfo.getNextStackOffset(), RegSize);
+    else {
+      VaArgOffset =
+          (int)ABI.GetCalleeAllocdArgSizeInBytes(CCInfo.getCallingConv()) -
+          (int)(RegSize * (ArgRegs.size() - Idx));
+    }
+
+    MachineFrameInfo &MFI = MF.getFrameInfo();
+    int FI = MFI.CreateFixedObject(RegSize, VaArgOffset, true);
+    MF.getInfo<MipsFunctionInfo>()->setVarArgsFrameIndex(FI);
+
+    for (unsigned I = Idx; I < ArgRegs.size(); ++I, VaArgOffset += RegSize) {
+      MIRBuilder.getMBB().addLiveIn(ArgRegs[I]);
+
+      MachineInstrBuilder Copy =
+          MIRBuilder.buildCopy(LLT::scalar(RegSize * 8), Register(ArgRegs[I]));
+      FI = MFI.CreateFixedObject(RegSize, VaArgOffset, true);
+      MachinePointerInfo MPO = MachinePointerInfo::getFixedStack(MF, FI);
+      MachineInstrBuilder FrameIndex =
+          MIRBuilder.buildFrameIndex(LLT::pointer(MPO.getAddrSpace(), 32), FI);
+      MachineMemOperand *MMO =
+          MF.getMachineMemOperand(MPO, MachineMemOperand::MOStore, RegSize,
+                                  /* Alignment */ RegSize);
+      MIRBuilder.buildStore(Copy, FrameIndex, *MMO);
+    }
+  }
+
   return true;
 }
 
@@ -506,7 +541,7 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     return false;
 
   for (auto &Arg : Info.OrigArgs) {
-    if (!isSupportedType(Arg.Ty))
+    if (!isSupportedArgumentType(Arg.Ty))
       return false;
     if (Arg.Flags[0].isByVal())
       return false;
@@ -514,11 +549,12 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
       return false;
   }
 
-  if (!Info.OrigRet.Ty->isVoidTy() && !isSupportedType(Info.OrigRet.Ty))
+  if (!Info.OrigRet.Ty->isVoidTy() && !isSupportedReturnType(Info.OrigRet.Ty))
     return false;
 
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &F = MF.getFunction();
+  const DataLayout &DL = MF.getDataLayout();
   const MipsTargetLowering &TLI = *getTLI<MipsTargetLowering>();
   const MipsTargetMachine &TM =
       static_cast<const MipsTargetMachine &>(MF.getTarget());
@@ -558,7 +594,8 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     Entry.Ty = Arg.Ty;
     FuncOrigArgs.push_back(Entry);
 
-    splitToValueTypes(Arg, i, ArgInfos, OrigArgIndices);
+    ArgInfos.push_back(Arg);
+    OrigArgIndices.push_back(i);
     ++i;
   }
 
@@ -566,7 +603,12 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   subTargetRegTypeForCallingConv(F, ArgInfos, OrigArgIndices, Outs);
 
   SmallVector<CCValAssign, 8> ArgLocs;
-  MipsCCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs,
+  bool IsCalleeVarArg = false;
+  if (Info.Callee.isGlobal()) {
+    const Function *CF = static_cast<const Function *>(Info.Callee.getGlobal());
+    IsCalleeVarArg = CF->isVarArg();
+  }
+  MipsCCState CCInfo(F.getCallingConv(), IsCalleeVarArg, MF, ArgLocs,
                      F.getContext());
 
   CCInfo.AllocateStack(ABI.GetCalleeAllocdArgSizeInBytes(Info.CallConv), 1);
@@ -604,7 +646,7 @@ bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
     ArgInfos.clear();
     SmallVector<unsigned, 8> OrigRetIndices;
 
-    splitToValueTypes(Info.OrigRet, 0, ArgInfos, OrigRetIndices);
+    splitToValueTypes(DL, Info.OrigRet, 0, ArgInfos, OrigRetIndices);
 
     SmallVector<ISD::InputArg, 8> Ins;
     subTargetRegTypeForCallingConv(F, ArgInfos, OrigRetIndices, Ins);
@@ -658,12 +700,21 @@ void MipsCallLowering::subTargetRegTypeForCallingConv(
 }
 
 void MipsCallLowering::splitToValueTypes(
-    const ArgInfo &OrigArg, unsigned OriginalIndex,
+    const DataLayout &DL, const ArgInfo &OrigArg, unsigned OriginalIndex,
     SmallVectorImpl<ArgInfo> &SplitArgs,
     SmallVectorImpl<unsigned> &SplitArgsOrigIndices) const {
 
-  // TODO : perform structure and array split. For now we only deal with
-  // types that pass isSupportedType check.
-  SplitArgs.push_back(OrigArg);
-  SplitArgsOrigIndices.push_back(OriginalIndex);
+  SmallVector<EVT, 4> SplitEVTs;
+  SmallVector<Register, 4> SplitVRegs;
+  const MipsTargetLowering &TLI = *getTLI<MipsTargetLowering>();
+  LLVMContext &Ctx = OrigArg.Ty->getContext();
+
+  ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitEVTs);
+
+  for (unsigned i = 0; i < SplitEVTs.size(); ++i) {
+    ArgInfo Info = ArgInfo{OrigArg.Regs[i], SplitEVTs[i].getTypeForEVT(Ctx)};
+    Info.Flags = OrigArg.Flags;
+    SplitArgs.push_back(Info);
+    SplitArgsOrigIndices.push_back(OriginalIndex);
+  }
 }
diff --git a/llvm/lib/Target/Mips/MipsCallLowering.h b/llvm/lib/Target/Mips/MipsCallLowering.h
index a91fa9ef27b74..a284cf5e26cf5 100644
--- a/llvm/lib/Target/Mips/MipsCallLowering.h
+++ b/llvm/lib/Target/Mips/MipsCallLowering.h
@@ -82,7 +82,8 @@ class MipsCallLowering : public CallLowering {
 
   /// Split structures and arrays, save original argument indices since
   /// Mips calling convention needs info about original argument type.
-  void splitToValueTypes(const ArgInfo &OrigArg, unsigned OriginalIndex,
+  void splitToValueTypes(const DataLayout &DL, const ArgInfo &OrigArg,
+                         unsigned OriginalIndex,
                          SmallVectorImpl<ArgInfo> &SplitArgs,
                          SmallVectorImpl<unsigned> &SplitArgsOrigIndices) const;
 };
diff --git a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
index cf748dcd7a219..f506405217381 100644
--- a/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
+++ b/llvm/lib/Target/Mips/MipsConstantIslandPass.cpp
@@ -222,12 +222,7 @@ namespace {
 
       BasicBlockInfo() = default;
 
-      // FIXME: ignore LogAlign for this patch
-      //
-      unsigned postOffset(unsigned LogAlign = 0) const {
-        unsigned PO = Offset + Size;
-        return PO;
-      }
+      unsigned postOffset() const { return Offset + Size; }
     };
 
     std::vector<BasicBlockInfo> BBInfo;
@@ -376,7 +371,7 @@ namespace {
 
     void doInitialPlacement(std::vector<MachineInstr*> &CPEMIs);
     CPEntry *findConstPoolEntry(unsigned CPI, const MachineInstr *CPEMI);
-    unsigned getCPELogAlign(const MachineInstr &CPEMI);
+    Align getCPEAlign(const MachineInstr &CPEMI);
     void initializeFunctionInfo(const std::vector<MachineInstr*> &CPEMIs);
     unsigned getOffsetOf(MachineInstr *MI) const;
     unsigned getUserOffset(CPUser&) const;
@@ -534,11 +529,11 @@ MipsConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   MF->push_back(BB);
 
   // MachineConstantPool measures alignment in bytes. We measure in log2(bytes).
-  unsigned MaxLogAlign = Log2_32(MCP->getConstantPoolAlignment());
+  const Align MaxAlign(MCP->getConstantPoolAlignment());
 
   // Mark the basic block as required by the const-pool.
   // If AlignConstantIslands isn't set, use 4-byte alignment for everything.
-  BB->setLogAlignment(AlignConstantIslands ? MaxLogAlign : 2);
+  BB->setAlignment(AlignConstantIslands ? MaxAlign : Align(4));
 
   // The function needs to be as aligned as the basic blocks. The linker may
   // move functions around based on their alignment.
@@ -548,7 +543,7 @@ MipsConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   // alignment of all entries as long as BB is sufficiently aligned.  Keep
   // track of the insertion point for each alignment.  We are going to bucket
   // sort the entries as they are created.
-  SmallVector<MachineBasicBlock::iterator, 8> InsPoint(MaxLogAlign + 1,
+  SmallVector<MachineBasicBlock::iterator, 8> InsPoint(Log2(MaxAlign) + 1,
                                                        BB->end());
 
   // Add all of the constants from the constant pool to the end block, use an
@@ -577,7 +572,7 @@ MipsConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
 
     // Ensure that future entries with higher alignment get inserted before
     // CPEMI. This is bucket sort with iterators.
-    for (unsigned a = LogAlign + 1; a <= MaxLogAlign; ++a)
+    for (unsigned a = LogAlign + 1; a <= Log2(MaxAlign); ++a)
       if (InsPoint[a] == InsAt)
         InsPoint[a] = CPEMI;
     // Add a new CPEntry, but no corresponding CPUser yet.
@@ -622,20 +617,18 @@ MipsConstantIslands::CPEntry
   return nullptr;
 }
 
-/// getCPELogAlign - Returns the required alignment of the constant pool entry
+/// getCPEAlign - Returns the required alignment of the constant pool entry
 /// represented by CPEMI.  Alignment is measured in log2(bytes) units.
-unsigned MipsConstantIslands::getCPELogAlign(const MachineInstr &CPEMI) {
+Align MipsConstantIslands::getCPEAlign(const MachineInstr &CPEMI) {
   assert(CPEMI.getOpcode() == Mips::CONSTPOOL_ENTRY);
 
   // Everything is 4-byte aligned unless AlignConstantIslands is set.
   if (!AlignConstantIslands)
-    return 2;
+    return Align(4);
 
   unsigned CPI = CPEMI.getOperand(1).getIndex();
   assert(CPI < MCP->getConstants().size() && "Invalid constant pool index.");
-  unsigned Align = MCP->getConstants()[CPI].getAlignment();
-  assert(isPowerOf2_32(Align) && "Invalid CPE alignment");
-  return Log2_32(Align);
+  return Align(MCP->getConstants()[CPI].getAlignment());
 }
 
 /// initializeFunctionInfo - Do the initial scan of the function, building up
@@ -941,14 +934,13 @@ bool MipsConstantIslands::isOffsetInRange(unsigned UserOffset,
 bool MipsConstantIslands::isWaterInRange(unsigned UserOffset,
                                         MachineBasicBlock* Water, CPUser &U,
                                         unsigned &Growth) {
-  unsigned CPELogAlign = getCPELogAlign(*U.CPEMI);
-  unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset(CPELogAlign);
+  unsigned CPEOffset = BBInfo[Water->getNumber()].postOffset();
   unsigned NextBlockOffset;
-  llvm::Align NextBlockAlignment;
+  Align NextBlockAlignment;
   MachineFunction::const_iterator NextBlock = ++Water->getIterator();
   if (NextBlock == MF->end()) {
     NextBlockOffset = BBInfo[Water->getNumber()].postOffset();
-    NextBlockAlignment = llvm::Align();
+    NextBlockAlignment = Align::None();
   } else {
     NextBlockOffset = BBInfo[NextBlock->getNumber()].Offset;
     NextBlockAlignment = NextBlock->getAlignment();
@@ -1223,7 +1215,6 @@ void MipsConstantIslands::createNewWater(unsigned CPUserIndex,
   CPUser &U = CPUsers[CPUserIndex];
   MachineInstr *UserMI = U.MI;
   MachineInstr *CPEMI  = U.CPEMI;
-  unsigned CPELogAlign = getCPELogAlign(*CPEMI);
   MachineBasicBlock *UserMBB = UserMI->getParent();
   const BasicBlockInfo &UserBBI = BBInfo[UserMBB->getNumber()];
 
@@ -1233,7 +1224,7 @@ void MipsConstantIslands::createNewWater(unsigned CPUserIndex,
     // Size of branch to insert.
     unsigned Delta = 2;
     // Compute the offset where the CPE will begin.
-    unsigned CPEOffset = UserBBI.postOffset(CPELogAlign) + Delta;
+    unsigned CPEOffset = UserBBI.postOffset() + Delta;
 
     if (isOffsetInRange(UserOffset, CPEOffset, U)) {
       LLVM_DEBUG(dbgs() << "Split at end of " << printMBBReference(*UserMBB)
@@ -1259,9 +1250,8 @@ void MipsConstantIslands::createNewWater(unsigned CPUserIndex,
 
   // Try to split the block so it's fully aligned.  Compute the latest split
   // point where we can add a 4-byte branch instruction, and then align to
-  // LogAlign which is the largest possible alignment in the function.
-  unsigned LogAlign = Log2(MF->getAlignment());
-  assert(LogAlign >= CPELogAlign && "Over-aligned constant pool entry");
+  // Align which is the largest possible alignment in the function.
+  const Align Align = MF->getAlignment();
   unsigned BaseInsertOffset = UserOffset + U.getMaxDisp();
   LLVM_DEBUG(dbgs() << format("Split in middle of big block before %#x",
                               BaseInsertOffset));
@@ -1272,7 +1262,7 @@ void MipsConstantIslands::createNewWater(unsigned CPUserIndex,
   BaseInsertOffset -= 4;
 
   LLVM_DEBUG(dbgs() << format(", adjusted to %#x", BaseInsertOffset)
-                    << " la=" << LogAlign << '\n');
+                    << " la=" << Log2(Align) << '\n');
 
   // This could point off the end of the block if we've already got constant
   // pool entries following this block; only the last one is in the water list.
@@ -1297,8 +1287,8 @@ void MipsConstantIslands::createNewWater(unsigned CPUserIndex,
       CPUser &U = CPUsers[CPUIndex];
       if (!isOffsetInRange(Offset, EndInsertOffset, U)) {
         // Shift intertion point by one unit of alignment so it is within reach.
-        BaseInsertOffset -= 1u << LogAlign;
-        EndInsertOffset  -= 1u << LogAlign;
+        BaseInsertOffset -= Align.value();
+        EndInsertOffset -= Align.value();
       }
       // This is overly conservative, as we don't account for CPEMIs being
       // reused within the block, but it doesn't matter much.  Also assume CPEs
@@ -1401,7 +1391,7 @@ bool MipsConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) {
   ++NumCPEs;
 
   // Mark the basic block as aligned as required by the const-pool entry.
-  NewIsland->setLogAlignment(getCPELogAlign(*U.CPEMI));
+  NewIsland->setAlignment(getCPEAlign(*U.CPEMI));
 
   // Increase the size of the island block to account for the new entry.
   BBInfo[NewIsland->getNumber()].Size += Size;
@@ -1433,10 +1423,11 @@ void MipsConstantIslands::removeDeadCPEMI(MachineInstr *CPEMI) {
     BBInfo[CPEBB->getNumber()].Size = 0;
 
     // This block no longer needs to be aligned.
-    CPEBB->setLogAlignment(0);
-  } else
+    CPEBB->setAlignment(Align(1));
+  } else {
     // Entries are sorted by descending alignment, so realign from the front.
-    CPEBB->setLogAlignment(getCPELogAlign(*CPEBB->begin()));
+    CPEBB->setAlignment(getCPEAlign(*CPEBB->begin()));
+  }
 
   adjustBBOffsetsAfter(CPEBB);
   // An island has only one predecessor BB and one successor BB. Check if
@@ -1531,7 +1522,7 @@ MipsConstantIslands::fixupUnconditionalBr(ImmBranch &Br) {
     // We should have a way to back out this alignment restriction if we "can" later.
     // but it is not harmful.
     //
-    DestBB->setLogAlignment(2);
+    DestBB->setAlignment(Align(4));
     Br.MaxDisp = ((1<<24)-1) * 2;
     MI->setDesc(TII->get(Mips::JalB16));
   }
diff --git a/llvm/lib/Target/Mips/MipsDSPInstrInfo.td b/llvm/lib/Target/Mips/MipsDSPInstrInfo.td
index daca8b9070813..d3e68c014fb72 100644
--- a/llvm/lib/Target/Mips/MipsDSPInstrInfo.td
+++ b/llvm/lib/Target/Mips/MipsDSPInstrInfo.td
@@ -12,12 +12,19 @@
 
 // ImmLeaf
 def immZExt1 : ImmLeaf<i32, [{return isUInt<1>(Imm);}]>;
+def timmZExt1 : ImmLeaf<i32, [{return isUInt<1>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immZExt2 : ImmLeaf<i32, [{return isUInt<2>(Imm);}]>;
+def timmZExt2 : ImmLeaf<i32, [{return isUInt<2>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immZExt3 : ImmLeaf<i32, [{return isUInt<3>(Imm);}]>;
+def timmZExt3 : ImmLeaf<i32, [{return isUInt<3>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immZExt4 : ImmLeaf<i32, [{return isUInt<4>(Imm);}]>;
+def timmZExt4 : ImmLeaf<i32, [{return isUInt<4>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immZExt8 : ImmLeaf<i32, [{return isUInt<8>(Imm);}]>;
+def timmZExt8 : ImmLeaf<i32, [{return isUInt<8>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immZExt10 : ImmLeaf<i32, [{return isUInt<10>(Imm);}]>;
+def timmZExt10 : ImmLeaf<i32, [{return isUInt<10>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immSExt6 : ImmLeaf<i32, [{return isInt<6>(Imm);}]>;
+def timmSExt6 : ImmLeaf<i32, [{return isInt<6>(Imm);}], NOOP_SDNodeXForm, timm>;
 def immSExt10 : ImmLeaf<i32, [{return isInt<10>(Imm);}]>;
 
 // Mips-specific dsp nodes
@@ -306,7 +313,7 @@ class PRECR_SRA_PH_W_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
   dag OutOperandList = (outs ROT:$rt);
   dag InOperandList = (ins ROS:$rs, uimm5:$sa, ROS:$src);
   string AsmString = !strconcat(instr_asm, "\t$rt, $rs, $sa");
-  list<dag> Pattern = [(set ROT:$rt, (OpNode ROS:$src, ROS:$rs, immZExt5:$sa))];
+  list<dag> Pattern = [(set ROT:$rt, (OpNode ROS:$src, ROS:$rs, timmZExt5:$sa))];
   InstrItinClass Itinerary = itin;
   string Constraints = "$src = $rt";
   string BaseOpcode = instr_asm;
@@ -443,7 +450,7 @@ class RDDSP_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
   dag OutOperandList = (outs GPR32Opnd:$rd);
   dag InOperandList = (ins uimm10:$mask);
   string AsmString = !strconcat(instr_asm, "\t$rd, $mask");
-  list<dag> Pattern = [(set GPR32Opnd:$rd, (OpNode immZExt10:$mask))];
+  list<dag> Pattern = [(set GPR32Opnd:$rd, (OpNode timmZExt10:$mask))];
   InstrItinClass Itinerary = itin;
   string BaseOpcode = instr_asm;
   bit isMoveReg = 1;
@@ -454,7 +461,7 @@ class WRDSP_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
   dag OutOperandList = (outs);
   dag InOperandList = (ins GPR32Opnd:$rs, uimm10:$mask);
   string AsmString = !strconcat(instr_asm, "\t$rs, $mask");
-  list<dag> Pattern = [(OpNode GPR32Opnd:$rs, immZExt10:$mask)];
+  list<dag> Pattern = [(OpNode GPR32Opnd:$rs, timmZExt10:$mask)];
   InstrItinClass Itinerary = itin;
   string BaseOpcode = instr_asm;
   bit isMoveReg = 1;
@@ -1096,14 +1103,14 @@ class SHRLV_PH_DESC : SHLL_QB_R3_DESC_BASE<"shrlv.ph", int_mips_shrl_ph,
                                            NoItinerary, DSPROpnd>;
 
 // Misc
-class APPEND_DESC : APPEND_DESC_BASE<"append", int_mips_append, uimm5, immZExt5,
+class APPEND_DESC : APPEND_DESC_BASE<"append", int_mips_append, uimm5, timmZExt5,
                                      NoItinerary>;
 
-class BALIGN_DESC : APPEND_DESC_BASE<"balign", int_mips_balign, uimm2, immZExt2,
+class BALIGN_DESC : APPEND_DESC_BASE<"balign", int_mips_balign, uimm2, timmZExt2,
                                      NoItinerary>;
 
 class PREPEND_DESC : APPEND_DESC_BASE<"prepend", int_mips_prepend, uimm5,
-                                      immZExt5, NoItinerary>;
+                                      timmZExt5, NoItinerary>;
 
 // Pseudos.
 def BPOSGE32_PSEUDO : BPOSGE32_PSEUDO_DESC_BASE<int_mips_bposge32,
diff --git a/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
index 42d158f44f3ac..e5997af3bcc53 100644
--- a/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -217,6 +217,51 @@ bool MipsDAGToDAGISel::selectVSplatMaskR(SDValue N, SDValue &Imm) const {
   return false;
 }
 
+/// Convert vector addition with vector subtraction if that allows to encode
+/// constant as an immediate and thus avoid extra 'ldi' instruction.
+/// add X, <-1, -1...> --> sub X, <1, 1...>
+bool MipsDAGToDAGISel::selectVecAddAsVecSubIfProfitable(SDNode *Node) {
+  assert(Node->getOpcode() == ISD::ADD && "Should only get 'add' here.");
+
+  EVT VT = Node->getValueType(0);
+  assert(VT.isVector() && "Should only be called for vectors.");
+
+  SDValue X = Node->getOperand(0);
+  SDValue C = Node->getOperand(1);
+
+  auto *BVN = dyn_cast<BuildVectorSDNode>(C);
+  if (!BVN)
+    return false;
+
+  APInt SplatValue, SplatUndef;
+  unsigned SplatBitSize;
+  bool HasAnyUndefs;
+
+  if (!BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
+                            8, !Subtarget->isLittle()))
+    return false;
+
+  auto IsInlineConstant = [](const APInt &Imm) { return Imm.isIntN(5); };
+
+  if (IsInlineConstant(SplatValue))
+    return false; // Can already be encoded as an immediate.
+
+  APInt NegSplatValue = 0 - SplatValue;
+  if (!IsInlineConstant(NegSplatValue))
+    return false; // Even if we negate it it won't help.
+
+  SDLoc DL(Node);
+
+  SDValue NegC = CurDAG->FoldConstantArithmetic(
+      ISD::SUB, DL, VT, CurDAG->getConstant(0, DL, VT).getNode(), C.getNode());
+  assert(NegC && "Constant-folding failed!");
+  SDValue NewNode = CurDAG->getNode(ISD::SUB, DL, VT, X, NegC);
+
+  ReplaceNode(Node, NewNode.getNode());
+  SelectCode(NewNode.getNode());
+  return true;
+}
+
 /// Select instructions not customized! Used for
 /// expanded, promoted and normal instructions
 void MipsDAGToDAGISel::Select(SDNode *Node) {
@@ -236,6 +281,12 @@ void MipsDAGToDAGISel::Select(SDNode *Node) {
   switch(Opcode) {
   default: break;
 
+  case ISD::ADD:
+    if (Node->getSimpleValueType(0).isVector() &&
+        selectVecAddAsVecSubIfProfitable(Node))
+      return;
+    break;
+
   // Get target GOT address.
   case ISD::GLOBAL_OFFSET_TABLE:
     ReplaceNode(Node, getGlobalBaseReg());
diff --git a/llvm/lib/Target/Mips/MipsISelDAGToDAG.h b/llvm/lib/Target/Mips/MipsISelDAGToDAG.h
index bae3bbf71f3b2..a768589b374bb 100644
--- a/llvm/lib/Target/Mips/MipsISelDAGToDAG.h
+++ b/llvm/lib/Target/Mips/MipsISelDAGToDAG.h
@@ -125,6 +125,11 @@ class MipsDAGToDAGISel : public SelectionDAGISel {
   /// starting at bit zero.
   virtual bool selectVSplatMaskR(SDValue N, SDValue &Imm) const;
 
+  /// Convert vector addition with vector subtraction if that allows to encode
+  /// constant as an immediate and thus avoid extra 'ldi' instruction.
+  /// add X, <-1, -1...> --> sub X, <1, 1...>
+  bool selectVecAddAsVecSubIfProfitable(SDNode *Node);
+
   void Select(SDNode *N) override;
 
   virtual bool trySelect(SDNode *Node) = 0;
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index c352007415b49..bf1b4756b24fa 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -82,10 +82,6 @@ using namespace llvm;
 
 STATISTIC(NumTailCalls, "Number of tail calls");
 
-static cl::opt<bool>
-LargeGOT("mxgot", cl::Hidden,
-         cl::desc("MIPS: Enable GOT larger than 64k."), cl::init(false));
-
 static cl::opt<bool>
 NoZeroDivCheck("mno-check-zero-division", cl::Hidden,
                cl::desc("MIPS: Don't trap on integer division by zero."),
@@ -330,7 +326,7 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
   }
 
   // Set LoadExtAction for f16 vectors to Expand
-  for (MVT VT : MVT::fp_vector_valuetypes()) {
+  for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
     MVT F16VT = MVT::getVectorVT(MVT::f16, VT.getVectorNumElements());
     if (F16VT.isValid())
       setLoadExtAction(ISD::EXTLOAD, VT, F16VT, Expand);
@@ -518,13 +514,12 @@ MipsTargetLowering::MipsTargetLowering(const MipsTargetMachine &TM,
     setLibcallName(RTLIB::SRA_I128, nullptr);
   }
 
-  setMinFunctionAlignment(Subtarget.isGP64bit() ? llvm::Align(8)
-                                                : llvm::Align(4));
+  setMinFunctionAlignment(Subtarget.isGP64bit() ? Align(8) : Align(4));
 
   // The arguments on the stack are defined in terms of 4-byte slots on O32
   // and 8-byte slots on N32/N64.
-  setMinStackArgumentAlignment((ABI.IsN32() || ABI.IsN64()) ? llvm::Align(8)
-                                                            : llvm::Align(4));
+  setMinStackArgumentAlignment((ABI.IsN32() || ABI.IsN64()) ? Align(8)
+                                                            : Align(4));
 
   setStackPointerRegisterToSaveRestore(ABI.IsN64() ? Mips::SP_64 : Mips::SP);
 
@@ -554,8 +549,9 @@ MipsTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
                      !Subtarget.inMicroMipsMode();
 
   // Disable if either of the following is true:
-  // We do not generate PIC, the ABI is not O32, LargeGOT is being used.
-  if (!TM.isPositionIndependent() || !TM.getABI().IsO32() || LargeGOT)
+  // We do not generate PIC, the ABI is not O32, XGOT is being used.
+  if (!TM.isPositionIndependent() || !TM.getABI().IsO32() ||
+      Subtarget.useXGOT())
     UseFastISel = false;
 
   return UseFastISel ? Mips::createFastISel(funcInfo, libInfo) : nullptr;
@@ -1989,7 +1985,7 @@ SDValue MipsTargetLowering::lowerGlobalAddress(SDValue Op,
   if (GV->hasLocalLinkage())
     return getAddrLocal(N, SDLoc(N), Ty, DAG, ABI.IsN32() || ABI.IsN64());
 
-  if (LargeGOT)
+  if (Subtarget.useXGOT())
     return getAddrGlobalLargeGOT(
         N, SDLoc(N), Ty, DAG, MipsII::MO_GOT_HI16, MipsII::MO_GOT_LO16,
         DAG.getEntryNode(),
@@ -2151,7 +2147,7 @@ SDValue MipsTargetLowering::lowerVAARG(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Node->getValueType(0);
   SDValue Chain = Node->getOperand(0);
   SDValue VAListPtr = Node->getOperand(1);
-  const llvm::Align Align =
+  const Align Align =
       llvm::MaybeAlign(Node->getConstantOperandVal(3)).valueOrOne();
   const Value *SV = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
   SDLoc DL(Node);
@@ -2872,7 +2868,7 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT, MVT LocVT,
 #include "MipsGenCallingConv.inc"
 
  CCAssignFn *MipsTargetLowering::CCAssignFnForCall() const{
-   return CC_Mips;
+   return CC_Mips_FixedArg;
  }
 
  CCAssignFn *MipsTargetLowering::CCAssignFnForReturn() const{
@@ -3272,7 +3268,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
       if (InternalLinkage)
         Callee = getAddrLocal(G, DL, Ty, DAG, ABI.IsN32() || ABI.IsN64());
-      else if (LargeGOT) {
+      else if (Subtarget.useXGOT()) {
         Callee = getAddrGlobalLargeGOT(G, DL, Ty, DAG, MipsII::MO_CALL_HI16,
                                        MipsII::MO_CALL_LO16, Chain,
                                        FuncInfo->callPtrInfo(Val));
@@ -3294,7 +3290,7 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     if (!IsPIC) // static
       Callee = DAG.getTargetExternalSymbol(
           Sym, getPointerTy(DAG.getDataLayout()), MipsII::MO_NO_FLAG);
-    else if (LargeGOT) {
+    else if (Subtarget.useXGOT()) {
       Callee = getAddrGlobalLargeGOT(S, DL, Ty, DAG, MipsII::MO_CALL_HI16,
                                      MipsII::MO_CALL_LO16, Chain,
                                      FuncInfo->callPtrInfo(Sym));
@@ -4570,20 +4566,20 @@ MachineBasicBlock *MipsTargetLowering::emitPseudoD_SELECT(MachineInstr &MI,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned MipsTargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                               SelectionDAG &DAG) const {
+Register MipsTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                               const MachineFunction &MF) const {
   // Named registers is expected to be fairly rare. For now, just support $28
   // since the linux kernel uses it.
   if (Subtarget.isGP64bit()) {
-    unsigned Reg = StringSwitch<unsigned>(RegName)
+    Register Reg = StringSwitch<Register>(RegName)
                          .Case("$28", Mips::GP_64)
-                         .Default(0);
+                         .Default(Register());
     if (Reg)
       return Reg;
   } else {
-    unsigned Reg = StringSwitch<unsigned>(RegName)
+    Register Reg = StringSwitch<Register>(RegName)
                          .Case("$28", Mips::GP)
-                         .Default(0);
+                         .Default(Register());
     if (Reg)
       return Reg;
   }
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h
index 2db60e9801f1b..f5daa28cf83bb 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.h
+++ b/llvm/lib/Target/Mips/MipsISelLowering.h
@@ -347,8 +347,8 @@ class TargetRegisterClass;
 
     void HandleByVal(CCState *, unsigned &, unsigned) const override;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     /// If a physical register, this returns the register that receives the
     /// exception address on entry to an EH pad.
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.cpp b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
index fbd56206b2490..6bb25ee5754d3 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.cpp
@@ -677,7 +677,8 @@ MipsInstrInfo::genInstrWithNewOpc(unsigned NewOpc,
   return MIB;
 }
 
-bool MipsInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+bool MipsInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
+                                          unsigned &SrcOpIdx1,
                                           unsigned &SrcOpIdx2) const {
   assert(!MI.isBundle() &&
          "TargetInstrInfo::findCommutedOpIndices() can't handle bundles");
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.h b/llvm/lib/Target/Mips/MipsInstrInfo.h
index a626c0c3fdb8d..092a960b4ba76 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.h
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.h
@@ -148,7 +148,7 @@ class MipsInstrInfo : public MipsGenInstrInfo {
   MachineInstrBuilder genInstrWithNewOpc(unsigned NewOpc,
                                          MachineBasicBlock::iterator I) const;
 
-  bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+  bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
                              unsigned &SrcOpIdx2) const override;
 
   /// Perform target specific instruction verification.
diff --git a/llvm/lib/Target/Mips/MipsInstrInfo.td b/llvm/lib/Target/Mips/MipsInstrInfo.td
index b42d718b7e83d..58167e0f344de 100644
--- a/llvm/lib/Target/Mips/MipsInstrInfo.td
+++ b/llvm/lib/Target/Mips/MipsInstrInfo.td
@@ -1263,6 +1263,7 @@ def immSExt16  : PatLeaf<(imm), [{ return isInt<16>(N->getSExtValue()); }]>;
 
 // Node immediate fits as 7-bit zero extended on target immediate.
 def immZExt7 : PatLeaf<(imm), [{ return isUInt<7>(N->getZExtValue()); }]>;
+def timmZExt7 : PatLeaf<(timm), [{ return isUInt<7>(N->getZExtValue()); }]>;
 
 // Node immediate fits as 16-bit zero extended on target immediate.
 // The LO16 param means that only the lower 16 bits of the node
@@ -1295,6 +1296,7 @@ def immZExt32  : PatLeaf<(imm), [{ return isUInt<32>(N->getZExtValue()); }]>;
 
 // shamt field must fit in 5 bits.
 def immZExt5 : ImmLeaf<i32, [{return Imm == (Imm & 0x1f);}]>;
+def timmZExt5 : TImmLeaf<i32, [{return Imm == (Imm & 0x1f);}]>;
 
 def immZExt5Plus1 : PatLeaf<(imm), [{
   return isUInt<5>(N->getZExtValue() - 1);
diff --git a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
index 524a58b2d3e96..33e2d2ac4c589 100644
--- a/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
+++ b/llvm/lib/Target/Mips/MipsInstructionSelector.cpp
@@ -773,6 +773,29 @@ bool MipsInstructionSelector::select(MachineInstr &I) {
     MI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::SYNC)).addImm(0);
     break;
   }
+  case G_VASTART: {
+    MipsFunctionInfo *FuncInfo = MF.getInfo<MipsFunctionInfo>();
+    int FI = FuncInfo->getVarArgsFrameIndex();
+
+    Register LeaReg = MRI.createVirtualRegister(&Mips::GPR32RegClass);
+    MachineInstr *LEA_ADDiu =
+        BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::LEA_ADDiu))
+            .addDef(LeaReg)
+            .addFrameIndex(FI)
+            .addImm(0);
+    if (!constrainSelectedInstRegOperands(*LEA_ADDiu, TII, TRI, RBI))
+      return false;
+
+    MachineInstr *Store = BuildMI(MBB, I, I.getDebugLoc(), TII.get(Mips::SW))
+                              .addUse(LeaReg)
+                              .addUse(I.getOperand(0).getReg())
+                              .addImm(0);
+    if (!constrainSelectedInstRegOperands(*Store, TII, TRI, RBI))
+      return false;
+
+    I.eraseFromParent();
+    return true;
+  }
   default:
     return false;
   }
diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
index 2c9fc8ea4ad75..3186b1b59c1ba 100644
--- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp
@@ -122,6 +122,9 @@ MipsLegalizerInfo::MipsLegalizerInfo(const MipsSubtarget &ST) {
   getActionDefinitionsBuilder(G_DYN_STACKALLOC)
       .lowerFor({{p0, s32}});
 
+  getActionDefinitionsBuilder(G_VASTART)
+     .legalFor({p0});
+
   // FP instructions
   getActionDefinitionsBuilder(G_FCONSTANT)
       .legalFor({s32, s64});
@@ -252,6 +255,18 @@ bool MipsLegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
     MI.eraseFromParent();
     return constrainSelectedInstRegOperands(*Trap, TII, TRI, RBI);
   }
+  case Intrinsic::vacopy: {
+    Register Tmp = MRI.createGenericVirtualRegister(LLT::pointer(0, 32));
+    MachinePointerInfo MPO;
+    MIRBuilder.buildLoad(Tmp, MI.getOperand(2),
+                         *MI.getMF()->getMachineMemOperand(
+                             MPO, MachineMemOperand::MOLoad, 4, 4));
+    MIRBuilder.buildStore(Tmp, MI.getOperand(1),
+                          *MI.getMF()->getMachineMemOperand(
+                              MPO, MachineMemOperand::MOStore, 4, 4));
+    MI.eraseFromParent();
+    return true;
+  }
   default:
     break;
   }
diff --git a/llvm/lib/Target/Mips/MipsMSAInstrInfo.td b/llvm/lib/Target/Mips/MipsMSAInstrInfo.td
index 907ed9ef746fa..f585d9c1a148a 100644
--- a/llvm/lib/Target/Mips/MipsMSAInstrInfo.td
+++ b/llvm/lib/Target/Mips/MipsMSAInstrInfo.td
@@ -60,6 +60,11 @@ def immZExt2Ptr : ImmLeaf<iPTR, [{return isUInt<2>(Imm);}]>;
 def immZExt3Ptr : ImmLeaf<iPTR, [{return isUInt<3>(Imm);}]>;
 def immZExt4Ptr : ImmLeaf<iPTR, [{return isUInt<4>(Imm);}]>;
 
+def timmZExt1Ptr : TImmLeaf<iPTR, [{return isUInt<1>(Imm);}]>;
+def timmZExt2Ptr : TImmLeaf<iPTR, [{return isUInt<2>(Imm);}]>;
+def timmZExt3Ptr : TImmLeaf<iPTR, [{return isUInt<3>(Imm);}]>;
+def timmZExt4Ptr : TImmLeaf<iPTR, [{return isUInt<4>(Imm);}]>;
+
 // Operands
 
 def immZExt2Lsa : ImmLeaf<i32, [{return isUInt<2>(Imm - 1);}]>;
@@ -1270,7 +1275,7 @@ class MSA_I8_SHF_DESC_BASE<string instr_asm, RegisterOperand ROWD,
   dag OutOperandList = (outs ROWD:$wd);
   dag InOperandList = (ins ROWS:$ws, uimm8:$u8);
   string AsmString = !strconcat(instr_asm, "\t$wd, $ws, $u8");
-  list<dag> Pattern = [(set ROWD:$wd, (MipsSHF immZExt8:$u8, ROWS:$ws))];
+  list<dag> Pattern = [(set ROWD:$wd, (MipsSHF timmZExt8:$u8, ROWS:$ws))];
   InstrItinClass Itinerary = itin;
 }
 
@@ -2299,13 +2304,13 @@ class INSERT_FW_VIDX64_PSEUDO_DESC :
 class INSERT_FD_VIDX64_PSEUDO_DESC :
     MSA_INSERT_VIDX_PSEUDO_BASE<vector_insert, v2f64, MSA128DOpnd, FGR64Opnd, GPR64Opnd>;
 
-class INSVE_B_DESC : MSA_INSVE_DESC_BASE<"insve.b", insve_v16i8, uimm4, immZExt4,
+class INSVE_B_DESC : MSA_INSVE_DESC_BASE<"insve.b", insve_v16i8, uimm4, timmZExt4,
                                          MSA128BOpnd>;
-class INSVE_H_DESC : MSA_INSVE_DESC_BASE<"insve.h", insve_v8i16, uimm3, immZExt3,
+class INSVE_H_DESC : MSA_INSVE_DESC_BASE<"insve.h", insve_v8i16, uimm3, timmZExt3,
                                          MSA128HOpnd>;
-class INSVE_W_DESC : MSA_INSVE_DESC_BASE<"insve.w", insve_v4i32, uimm2, immZExt2,
+class INSVE_W_DESC : MSA_INSVE_DESC_BASE<"insve.w", insve_v4i32, uimm2, timmZExt2,
                                          MSA128WOpnd>;
-class INSVE_D_DESC : MSA_INSVE_DESC_BASE<"insve.d", insve_v2i64, uimm1, immZExt1,
+class INSVE_D_DESC : MSA_INSVE_DESC_BASE<"insve.d", insve_v2i64, uimm1, timmZExt1,
                                          MSA128DOpnd>;
 
 class LD_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
@@ -2518,22 +2523,22 @@ class PCNT_W_DESC : MSA_2R_DESC_BASE<"pcnt.w", ctpop, MSA128WOpnd>;
 class PCNT_D_DESC : MSA_2R_DESC_BASE<"pcnt.d", ctpop, MSA128DOpnd>;
 
 class SAT_S_B_DESC : MSA_BIT_X_DESC_BASE<"sat_s.b", int_mips_sat_s_b, uimm3,
-                                         immZExt3, MSA128BOpnd>;
+                                         timmZExt3, MSA128BOpnd>;
 class SAT_S_H_DESC : MSA_BIT_X_DESC_BASE<"sat_s.h", int_mips_sat_s_h, uimm4,
-                                         immZExt4, MSA128HOpnd>;
+                                         timmZExt4, MSA128HOpnd>;
 class SAT_S_W_DESC : MSA_BIT_X_DESC_BASE<"sat_s.w", int_mips_sat_s_w, uimm5,
-                                         immZExt5, MSA128WOpnd>;
+                                         timmZExt5, MSA128WOpnd>;
 class SAT_S_D_DESC : MSA_BIT_X_DESC_BASE<"sat_s.d", int_mips_sat_s_d, uimm6,
-                                         immZExt6, MSA128DOpnd>;
+                                         timmZExt6, MSA128DOpnd>;
 
 class SAT_U_B_DESC : MSA_BIT_X_DESC_BASE<"sat_u.b", int_mips_sat_u_b, uimm3,
-                                         immZExt3, MSA128BOpnd>;
+                                         timmZExt3, MSA128BOpnd>;
 class SAT_U_H_DESC : MSA_BIT_X_DESC_BASE<"sat_u.h", int_mips_sat_u_h, uimm4,
-                                         immZExt4, MSA128HOpnd>;
+                                         timmZExt4, MSA128HOpnd>;
 class SAT_U_W_DESC : MSA_BIT_X_DESC_BASE<"sat_u.w", int_mips_sat_u_w, uimm5,
-                                         immZExt5, MSA128WOpnd>;
+                                         timmZExt5, MSA128WOpnd>;
 class SAT_U_D_DESC : MSA_BIT_X_DESC_BASE<"sat_u.d", int_mips_sat_u_d, uimm6,
-                                         immZExt6, MSA128DOpnd>;
+                                         timmZExt6, MSA128DOpnd>;
 
 class SHF_B_DESC : MSA_I8_SHF_DESC_BASE<"shf.b", MSA128BOpnd>;
 class SHF_H_DESC : MSA_I8_SHF_DESC_BASE<"shf.h", MSA128HOpnd>;
@@ -2546,16 +2551,16 @@ class SLD_D_DESC : MSA_3R_SLD_DESC_BASE<"sld.d", int_mips_sld_d, MSA128DOpnd>;
 
 class SLDI_B_DESC : MSA_ELM_SLD_DESC_BASE<"sldi.b", int_mips_sldi_b,
                                           MSA128BOpnd, MSA128BOpnd, uimm4,
-                                          immZExt4>;
+                                          timmZExt4>;
 class SLDI_H_DESC : MSA_ELM_SLD_DESC_BASE<"sldi.h", int_mips_sldi_h,
                                           MSA128HOpnd, MSA128HOpnd, uimm3,
-                                          immZExt3>;
+                                          timmZExt3>;
 class SLDI_W_DESC : MSA_ELM_SLD_DESC_BASE<"sldi.w", int_mips_sldi_w,
                                           MSA128WOpnd, MSA128WOpnd, uimm2,
-                                          immZExt2>;
+                                          timmZExt2>;
 class SLDI_D_DESC : MSA_ELM_SLD_DESC_BASE<"sldi.d", int_mips_sldi_d,
                                           MSA128DOpnd, MSA128DOpnd, uimm1,
-                                          immZExt1>;
+                                          timmZExt1>;
 
 class SLL_B_DESC : MSA_3R_DESC_BASE<"sll.b", shl, MSA128BOpnd>;
 class SLL_H_DESC : MSA_3R_DESC_BASE<"sll.h", shl, MSA128HOpnd>;
@@ -2609,13 +2614,13 @@ class SRAR_W_DESC : MSA_3R_DESC_BASE<"srar.w", int_mips_srar_w, MSA128WOpnd>;
 class SRAR_D_DESC : MSA_3R_DESC_BASE<"srar.d", int_mips_srar_d, MSA128DOpnd>;
 
 class SRARI_B_DESC : MSA_BIT_X_DESC_BASE<"srari.b", int_mips_srari_b, uimm3,
-                                         immZExt3, MSA128BOpnd>;
+                                         timmZExt3, MSA128BOpnd>;
 class SRARI_H_DESC : MSA_BIT_X_DESC_BASE<"srari.h", int_mips_srari_h, uimm4,
-                                         immZExt4, MSA128HOpnd>;
+                                         timmZExt4, MSA128HOpnd>;
 class SRARI_W_DESC : MSA_BIT_X_DESC_BASE<"srari.w", int_mips_srari_w, uimm5,
-                                         immZExt5, MSA128WOpnd>;
+                                         timmZExt5, MSA128WOpnd>;
 class SRARI_D_DESC : MSA_BIT_X_DESC_BASE<"srari.d", int_mips_srari_d, uimm6,
-                                         immZExt6, MSA128DOpnd>;
+                                         timmZExt6, MSA128DOpnd>;
 
 class SRL_B_DESC : MSA_3R_DESC_BASE<"srl.b", srl, MSA128BOpnd>;
 class SRL_H_DESC : MSA_3R_DESC_BASE<"srl.h", srl, MSA128HOpnd>;
@@ -2637,13 +2642,13 @@ class SRLR_W_DESC : MSA_3R_DESC_BASE<"srlr.w", int_mips_srlr_w, MSA128WOpnd>;
 class SRLR_D_DESC : MSA_3R_DESC_BASE<"srlr.d", int_mips_srlr_d, MSA128DOpnd>;
 
 class SRLRI_B_DESC : MSA_BIT_X_DESC_BASE<"srlri.b", int_mips_srlri_b, uimm3,
-                                         immZExt3, MSA128BOpnd>;
+                                         timmZExt3, MSA128BOpnd>;
 class SRLRI_H_DESC : MSA_BIT_X_DESC_BASE<"srlri.h", int_mips_srlri_h, uimm4,
-                                         immZExt4, MSA128HOpnd>;
+                                         timmZExt4, MSA128HOpnd>;
 class SRLRI_W_DESC : MSA_BIT_X_DESC_BASE<"srlri.w", int_mips_srlri_w, uimm5,
-                                         immZExt5, MSA128WOpnd>;
+                                         timmZExt5, MSA128WOpnd>;
 class SRLRI_D_DESC : MSA_BIT_X_DESC_BASE<"srlri.d", int_mips_srlri_d, uimm6,
-                                         immZExt6, MSA128DOpnd>;
+                                         timmZExt6, MSA128DOpnd>;
 
 class ST_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
                    ValueType TyNode, RegisterOperand ROWD,
diff --git a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
index 9c76a892cf944..f8ce80b044cc3 100644
--- a/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsRegisterBankInfo.cpp
@@ -403,6 +403,7 @@ MipsRegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case G_SREM:
   case G_UREM:
   case G_BRINDIRECT:
+  case G_VASTART:
     OperandsMapping = &Mips::ValueMappings[Mips::GPRIdx];
     break;
   case G_LOAD: {
diff --git a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
index 4f349618322be..d9354cadc73b7 100644
--- a/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
+++ b/llvm/lib/Target/Mips/MipsSEISelDAGToDAG.cpp
@@ -247,8 +247,8 @@ bool MipsSEDAGToDAGISel::selectAddrFrameIndexOffset(
         Base = Addr.getOperand(0);
         // If base is a FI, additional offset calculation is done in
         // eliminateFrameIndex, otherwise we need to check the alignment
-        const llvm::Align Align(1ULL << ShiftAmount);
-        if (!isAligned(Align, CN->getZExtValue()))
+        const Align Alignment(1ULL << ShiftAmount);
+        if (!isAligned(Alignment, CN->getZExtValue()))
           return false;
       }
 
@@ -720,7 +720,7 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
   }
 
   case ISD::ConstantFP: {
-    ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Node);
+    auto *CN = cast<ConstantFPSDNode>(Node);
     if (Node->getValueType(0) == MVT::f64 && CN->isExactlyValue(+0.0)) {
       if (Subtarget->isGP64bit()) {
         SDValue Zero = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL,
@@ -744,7 +744,7 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
   }
 
   case ISD::Constant: {
-    const ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Node);
+    auto *CN = cast<ConstantSDNode>(Node);
     int64_t Imm = CN->getSExtValue();
     unsigned Size = CN->getValueSizeInBits(0);
 
@@ -970,7 +970,7 @@ bool MipsSEDAGToDAGISel::trySelect(SDNode *Node) {
       break;
     }
 
-    SDNode *Res;
+    SDNode *Res = nullptr;
 
     // If we have a signed 10 bit integer, we can splat it directly.
     //
diff --git a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
index a79c3a089c4d0..5bd234f955bab 100644
--- a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp
@@ -71,8 +71,8 @@ MipsSETargetLowering::MipsSETargetLowering(const MipsTargetMachine &TM,
 
   if (Subtarget.hasDSP() || Subtarget.hasMSA()) {
     // Expand all truncating stores and extending loads.
-    for (MVT VT0 : MVT::vector_valuetypes()) {
-      for (MVT VT1 : MVT::vector_valuetypes()) {
+    for (MVT VT0 : MVT::fixedlen_vector_valuetypes()) {
+      for (MVT VT1 : MVT::fixedlen_vector_valuetypes()) {
         setTruncStoreAction(VT0, VT1, Expand);
         setLoadExtAction(ISD::SEXTLOAD, VT0, VT1, Expand);
         setLoadExtAction(ISD::ZEXTLOAD, VT0, VT1, Expand);
@@ -2596,7 +2596,8 @@ static SDValue lowerVECTOR_SHUFFLE_SHF(SDValue Op, EVT ResTy,
 
   SDLoc DL(Op);
   return DAG.getNode(MipsISD::SHF, DL, ResTy,
-                     DAG.getConstant(Imm, DL, MVT::i32), Op->getOperand(0));
+                     DAG.getTargetConstant(Imm, DL, MVT::i32),
+                     Op->getOperand(0));
 }
 
 /// Determine whether a range fits a regular pattern of values.
diff --git a/llvm/lib/Target/Mips/MipsSERegisterInfo.cpp b/llvm/lib/Target/Mips/MipsSERegisterInfo.cpp
index 1a6382c96fbd9..a48088c289195 100644
--- a/llvm/lib/Target/Mips/MipsSERegisterInfo.cpp
+++ b/llvm/lib/Target/Mips/MipsSERegisterInfo.cpp
@@ -212,7 +212,7 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
     // element size), otherwise it is a 16-bit signed immediate.
     unsigned OffsetBitSize =
         getLoadStoreOffsetSizeInBits(MI.getOpcode(), MI.getOperand(OpNo - 1));
-    const llvm::Align OffsetAlign(getLoadStoreOffsetAlign(MI.getOpcode()));
+    const Align OffsetAlign(getLoadStoreOffsetAlign(MI.getOpcode()));
     if (OffsetBitSize < 16 && isInt<16>(Offset) &&
         (!isIntN(OffsetBitSize, Offset) || !isAligned(OffsetAlign, Offset))) {
       // If we have an offset that needs to fit into a signed n-bit immediate
diff --git a/llvm/lib/Target/Mips/MipsSubtarget.h b/llvm/lib/Target/Mips/MipsSubtarget.h
index 412103433a6f5..396c479cd204a 100644
--- a/llvm/lib/Target/Mips/MipsSubtarget.h
+++ b/llvm/lib/Target/Mips/MipsSubtarget.h
@@ -189,6 +189,9 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
   // Disable use of the `jal` instruction.
   bool UseLongCalls = false;
 
+  // Assume 32-bit GOT.
+  bool UseXGOT = false;
+
   /// The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned stackAlignment;
@@ -323,6 +326,8 @@ class MipsSubtarget : public MipsGenSubtargetInfo {
 
   bool useLongCalls() const { return UseLongCalls; }
 
+  bool useXGOT() const { return UseXGOT; }
+
   bool enableLongBranchPass() const {
     return hasStandardEncoding() || inMicroMipsMode() || allowMixed16_32();
   }
diff --git a/llvm/lib/Target/NVPTX/NVPTX.h b/llvm/lib/Target/NVPTX/NVPTX.h
index 6530c40ea1007..0acbace5f848f 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.h
+++ b/llvm/lib/Target/NVPTX/NVPTX.h
@@ -44,7 +44,7 @@ MachineFunctionPass *createNVPTXPrologEpilogPass();
 MachineFunctionPass *createNVPTXReplaceImageHandlesPass();
 FunctionPass *createNVPTXImageOptimizerPass();
 FunctionPass *createNVPTXLowerArgsPass(const NVPTXTargetMachine *TM);
-BasicBlockPass *createNVPTXLowerAllocaPass();
+FunctionPass *createNVPTXLowerAllocaPass();
 MachineFunctionPass *createNVPTXPeephole();
 MachineFunctionPass *createNVPTXProxyRegErasurePass();
 
diff --git a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index 3dd06c00ccb09..3a45085297192 100644
--- a/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -1397,7 +1397,7 @@ static unsigned int getOpenCLAlignment(const DataLayout &DL, Type *Ty) {
 
   auto *FTy = dyn_cast<FunctionType>(Ty);
   if (FTy)
-    return DL.getPointerPrefAlignment();
+    return DL.getPointerPrefAlignment().value();
   return DL.getPrefTypeAlignment(Ty);
 }
 
@@ -1861,7 +1861,7 @@ void NVPTXAsmPrinter::bufferLEByte(const Constant *CPV, int Bytes,
   case Type::HalfTyID:
   case Type::FloatTyID:
   case Type::DoubleTyID: {
-    const ConstantFP *CFP = dyn_cast<ConstantFP>(CPV);
+    const auto *CFP = cast<ConstantFP>(CPV);
     Type *Ty = CFP->getType();
     if (Ty == Type::getHalfTy(CPV->getContext())) {
       APInt API = CFP->getValueAPF().bitcastToAPInt();
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index c76e06036b294..9acd0bea66fdf 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -480,7 +480,7 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
 
   // Register custom handling for vector loads/stores
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     if (IsPTXVectorType(VT)) {
       setOperationAction(ISD::LOAD, VT, Custom);
       setOperationAction(ISD::STORE, VT, Custom);
@@ -2230,8 +2230,8 @@ SDValue NVPTXTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   if (Op.getValueType() == MVT::v2f16) {
     LoadSDNode *Load = cast<LoadSDNode>(Op);
     EVT MemVT = Load->getMemoryVT();
-    if (!allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
-                            *Load->getMemOperand())) {
+    if (!allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                        MemVT, *Load->getMemOperand())) {
       SDValue Ops[2];
       std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(Load, DAG);
       return DAG.getMergeValues(Ops, SDLoc(Op));
@@ -2273,8 +2273,8 @@ SDValue NVPTXTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   // v2f16 is legal, so we can't rely on legalizer to handle unaligned
   // stores and have to handle it here.
   if (VT == MVT::v2f16 &&
-      !allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
-                          *Store->getMemOperand()))
+      !allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                      VT, *Store->getMemOperand()))
     return expandUnalignedStore(Store, DAG);
 
   if (VT.isVector())
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
index 0743a2986718a..83039241a7c75 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
@@ -103,7 +103,7 @@ bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
   // Do the transformation of an aggr load/copy/set to a loop
   //
   for (LoadInst *LI : AggrLoads) {
-    StoreInst *SI = dyn_cast<StoreInst>(*LI->user_begin());
+    auto *SI = cast<StoreInst>(*LI->user_begin());
     Value *SrcAddr = LI->getOperand(0);
     Value *DstAddr = SI->getOperand(1);
     unsigned NumLoads = DL.getTypeStoreSize(LI->getType());
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
index 76fb9f3fa692b..945b7286b03cc 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerAlloca.cpp
@@ -41,12 +41,12 @@ void initializeNVPTXLowerAllocaPass(PassRegistry &);
 }
 
 namespace {
-class NVPTXLowerAlloca : public BasicBlockPass {
-  bool runOnBasicBlock(BasicBlock &BB) override;
+class NVPTXLowerAlloca : public FunctionPass {
+  bool runOnFunction(Function &F) override;
 
 public:
   static char ID; // Pass identification, replacement for typeid
-  NVPTXLowerAlloca() : BasicBlockPass(ID) {}
+  NVPTXLowerAlloca() : FunctionPass(ID) {}
   StringRef getPassName() const override {
     return "convert address space of alloca'ed memory to local";
   }
@@ -61,58 +61,61 @@ INITIALIZE_PASS(NVPTXLowerAlloca, "nvptx-lower-alloca",
 // =============================================================================
 // Main function for this pass.
 // =============================================================================
-bool NVPTXLowerAlloca::runOnBasicBlock(BasicBlock &BB) {
-  if (skipBasicBlock(BB))
+bool NVPTXLowerAlloca::runOnFunction(Function &F) {
+  if (skipFunction(F))
     return false;
 
   bool Changed = false;
-  for (auto &I : BB) {
-    if (auto allocaInst = dyn_cast<AllocaInst>(&I)) {
-      Changed = true;
-      auto PTy = dyn_cast<PointerType>(allocaInst->getType());
-      auto ETy = PTy->getElementType();
-      auto LocalAddrTy = PointerType::get(ETy, ADDRESS_SPACE_LOCAL);
-      auto NewASCToLocal = new AddrSpaceCastInst(allocaInst, LocalAddrTy, "");
-      auto GenericAddrTy = PointerType::get(ETy, ADDRESS_SPACE_GENERIC);
-      auto NewASCToGeneric = new AddrSpaceCastInst(NewASCToLocal,
-                                                    GenericAddrTy, "");
-      NewASCToLocal->insertAfter(allocaInst);
-      NewASCToGeneric->insertAfter(NewASCToLocal);
-      for (Value::use_iterator UI = allocaInst->use_begin(),
-                                UE = allocaInst->use_end();
-            UI != UE; ) {
-        // Check Load, Store, GEP, and BitCast Uses on alloca and make them
-        // use the converted generic address, in order to expose non-generic
-        // addrspacecast to NVPTXInferAddressSpaces. For other types
-        // of instructions this is unnecessary and may introduce redundant
-        // address cast.
-        const auto &AllocaUse = *UI++;
-        auto LI = dyn_cast<LoadInst>(AllocaUse.getUser());
-        if (LI && LI->getPointerOperand() == allocaInst && !LI->isVolatile()) {
-          LI->setOperand(LI->getPointerOperandIndex(), NewASCToGeneric);
-          continue;
-        }
-        auto SI = dyn_cast<StoreInst>(AllocaUse.getUser());
-        if (SI && SI->getPointerOperand() == allocaInst && !SI->isVolatile()) {
-          SI->setOperand(SI->getPointerOperandIndex(), NewASCToGeneric);
-          continue;
-        }
-        auto GI = dyn_cast<GetElementPtrInst>(AllocaUse.getUser());
-        if (GI && GI->getPointerOperand() == allocaInst) {
-          GI->setOperand(GI->getPointerOperandIndex(), NewASCToGeneric);
-          continue;
-        }
-        auto BI = dyn_cast<BitCastInst>(AllocaUse.getUser());
-        if (BI && BI->getOperand(0) == allocaInst) {
-          BI->setOperand(0, NewASCToGeneric);
-          continue;
+  for (auto &BB : F)
+    for (auto &I : BB) {
+      if (auto allocaInst = dyn_cast<AllocaInst>(&I)) {
+        Changed = true;
+        auto PTy = dyn_cast<PointerType>(allocaInst->getType());
+        auto ETy = PTy->getElementType();
+        auto LocalAddrTy = PointerType::get(ETy, ADDRESS_SPACE_LOCAL);
+        auto NewASCToLocal = new AddrSpaceCastInst(allocaInst, LocalAddrTy, "");
+        auto GenericAddrTy = PointerType::get(ETy, ADDRESS_SPACE_GENERIC);
+        auto NewASCToGeneric =
+            new AddrSpaceCastInst(NewASCToLocal, GenericAddrTy, "");
+        NewASCToLocal->insertAfter(allocaInst);
+        NewASCToGeneric->insertAfter(NewASCToLocal);
+        for (Value::use_iterator UI = allocaInst->use_begin(),
+                                 UE = allocaInst->use_end();
+             UI != UE;) {
+          // Check Load, Store, GEP, and BitCast Uses on alloca and make them
+          // use the converted generic address, in order to expose non-generic
+          // addrspacecast to NVPTXInferAddressSpaces. For other types
+          // of instructions this is unnecessary and may introduce redundant
+          // address cast.
+          const auto &AllocaUse = *UI++;
+          auto LI = dyn_cast<LoadInst>(AllocaUse.getUser());
+          if (LI && LI->getPointerOperand() == allocaInst &&
+              !LI->isVolatile()) {
+            LI->setOperand(LI->getPointerOperandIndex(), NewASCToGeneric);
+            continue;
+          }
+          auto SI = dyn_cast<StoreInst>(AllocaUse.getUser());
+          if (SI && SI->getPointerOperand() == allocaInst &&
+              !SI->isVolatile()) {
+            SI->setOperand(SI->getPointerOperandIndex(), NewASCToGeneric);
+            continue;
+          }
+          auto GI = dyn_cast<GetElementPtrInst>(AllocaUse.getUser());
+          if (GI && GI->getPointerOperand() == allocaInst) {
+            GI->setOperand(GI->getPointerOperandIndex(), NewASCToGeneric);
+            continue;
+          }
+          auto BI = dyn_cast<BitCastInst>(AllocaUse.getUser());
+          if (BI && BI->getOperand(0) == allocaInst) {
+            BI->setOperand(0, NewASCToGeneric);
+            continue;
+          }
         }
       }
     }
-  }
   return Changed;
 }
 
-BasicBlockPass *llvm::createNVPTXLowerAllocaPass() {
+FunctionPass *llvm::createNVPTXLowerAllocaPass() {
   return new NVPTXLowerAlloca();
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
index c5e02e34e25e4..c3c5f6fbcba72 100644
--- a/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXLowerArgs.cpp
@@ -164,7 +164,7 @@ void NVPTXLowerArgs::handleByValParam(Argument *Arg) {
   // Set the alignment to alignment of the byval parameter. This is because,
   // later load/stores assume that alignment, and we are going to replace
   // the use of the byval parameter with this alloca instruction.
-  AllocA->setAlignment(Func->getParamAlignment(Arg->getArgNo()));
+  AllocA->setAlignment(MaybeAlign(Func->getParamAlignment(Arg->getArgNo())));
   Arg->replaceAllUsesWith(AllocA);
 
   Value *ArgInParam = new AddrSpaceCastInst(
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index 2b64bcf3cf88a..db30e16d15478 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -167,7 +167,13 @@ class PPCAIXAsmPrinter : public PPCAsmPrinter {
 
   StringRef getPassName() const override { return "AIX PPC Assembly Printer"; }
 
+  void SetupMachineFunction(MachineFunction &MF) override;
+
   void EmitGlobalVariable(const GlobalVariable *GV) override;
+
+  void EmitFunctionDescriptor() override;
+
+  void EmitEndOfAsmFile(Module &) override;
 };
 
 } // end anonymous namespace
@@ -649,16 +655,19 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     }
   }
   case PPC::LWZtoc: {
-    // Transform %r3 = LWZtoc @min1, %r2
+    assert(!isDarwin && "TOC is an ELF/XCOFF construct.");
+
+    // Transform %rN = LWZtoc @op1, %r2
     LowerPPCMachineInstrToMCInst(MI, TmpInst, *this, isDarwin);
 
-    // Change the opcode to LWZ, and the global address operand to be a
-    // reference to the GOT entry we will synthesize later.
+    // Change the opcode to LWZ.
     TmpInst.setOpcode(PPC::LWZ);
+
     const MachineOperand &MO = MI->getOperand(1);
+    assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) &&
+           "Unexpected operand type for LWZtoc pseudo.");
 
-    // Map symbol -> label of TOC entry
-    assert(MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress());
+    // Map the operand to its corresponding MCSymbol.
     MCSymbol *MOSymbol = nullptr;
     if (MO.isGlobal())
       MOSymbol = getSymbol(MO.getGlobal());
@@ -669,23 +678,44 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     else if (MO.isBlockAddress())
       MOSymbol = GetBlockAddressSymbol(MO.getBlockAddress());
 
-    if (PL == PICLevel::SmallPIC) {
+    const bool IsAIX = TM.getTargetTriple().isOSAIX();
+
+    // Create a reference to the GOT entry for the symbol. The GOT entry will be
+    // synthesized later.
+    if (PL == PICLevel::SmallPIC && !IsAIX) {
       const MCExpr *Exp =
         MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_GOT,
                                 OutContext);
       TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
-    } else {
-      MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol);
+      EmitToStreamer(*OutStreamer, TmpInst);
+      return;
+    }
 
-      const MCExpr *Exp =
-        MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_None,
-                                OutContext);
-      const MCExpr *PB =
-        MCSymbolRefExpr::create(OutContext.getOrCreateSymbol(Twine(".LTOC")),
-                                                             OutContext);
-      Exp = MCBinaryExpr::createSub(Exp, PB, OutContext);
+    // Otherwise use the TOC. 'TOCEntry' is a label used to reference the
+    // storage allocated in the TOC which contains the address of
+    // 'MOSymbol'. Said TOC entry will be synthesized later.
+    MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol);
+    const MCExpr *Exp =
+        MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_None, OutContext);
+
+    // AIX uses the label directly as the lwz displacement operand for
+    // references into the toc section. The displacement value will be generated
+    // relative to the toc-base.
+    if (IsAIX) {
+      assert(
+          TM.getCodeModel() == CodeModel::Small &&
+          "This pseudo should only be selected for 32-bit small code model.");
       TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
+      EmitToStreamer(*OutStreamer, TmpInst);
+      return;
     }
+
+    // Create an explicit subtract expression between the local symbol and
+    // '.LTOC' to manifest the toc-relative offset.
+    const MCExpr *PB = MCSymbolRefExpr::create(
+        OutContext.getOrCreateSymbol(Twine(".LTOC")), OutContext);
+    Exp = MCBinaryExpr::createSub(Exp, PB, OutContext);
+    TmpInst.getOperand(1) = MCOperand::createExpr(Exp);
     EmitToStreamer(*OutStreamer, TmpInst);
     return;
   }
@@ -742,8 +772,7 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     if (MO.isGlobal()) {
       const GlobalValue *GV = MO.getGlobal();
       MOSymbol = getSymbol(GV);
-      unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
-      GlobalToc = (GVFlags & PPCII::MO_NLP_FLAG);
+      GlobalToc = Subtarget->isGVIndirectSymbol(GV);
     } else if (MO.isCPI()) {
       MOSymbol = GetCPISymbol(MO.getIndex());
     } else if (MO.isJTI()) {
@@ -799,8 +828,7 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
       const GlobalValue *GV = MO.getGlobal();
       MOSymbol = getSymbol(GV);
       LLVM_DEBUG(
-          unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
-          assert((GVFlags & PPCII::MO_NLP_FLAG) &&
+          assert((Subtarget->isGVIndirectSymbol(GV)) &&
                  "LDtocL used on symbol that could be accessed directly is "
                  "invalid. Must match ADDIStocHA8."));
       MOSymbol = lookUpOrCreateTOCEntry(MOSymbol);
@@ -827,8 +855,7 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 
     if (MO.isGlobal()) {
       const GlobalValue *GV = MO.getGlobal();
-      LLVM_DEBUG(unsigned char GVFlags = Subtarget->classifyGlobalReference(GV);
-                 assert(!(GVFlags & PPCII::MO_NLP_FLAG) &&
+      LLVM_DEBUG(assert(!(Subtarget->isGVIndirectSymbol(GV)) &&
                         "Interposable definitions must use indirect access."));
       MOSymbol = getSymbol(GV);
     } else if (MO.isCPI()) {
@@ -1607,7 +1634,7 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
     if (!Stubs.empty()) {
       // Switch with ".non_lazy_symbol_pointer" directive.
       OutStreamer->SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
-      EmitAlignment(isPPC64 ? llvm::Align(8) : llvm::Align(4));
+      EmitAlignment(isPPC64 ? Align(8) : Align(4));
 
       for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
         // L_foo$stub:
@@ -1648,6 +1675,18 @@ bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
   return AsmPrinter::doFinalization(M);
 }
 
+void PPCAIXAsmPrinter::SetupMachineFunction(MachineFunction &MF) {
+  // Get the function descriptor symbol.
+  CurrentFnDescSym = getSymbol(&MF.getFunction());
+  // Set the containing csect.
+  MCSectionXCOFF *FnDescSec = OutStreamer->getContext().getXCOFFSection(
+      CurrentFnDescSym->getName(), XCOFF::XMC_DS, XCOFF::XTY_SD,
+      XCOFF::C_HIDEXT, SectionKind::getData());
+  cast<MCSymbolXCOFF>(CurrentFnDescSym)->setContainingCsect(FnDescSec);
+
+  return AsmPrinter::SetupMachineFunction(MF);
+}
+
 void PPCAIXAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   // Early error checking limiting what is supported.
   if (GV->isThreadLocal())
@@ -1697,6 +1736,45 @@ void PPCAIXAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
   EmitGlobalConstant(GV->getParent()->getDataLayout(), GV->getInitializer());
 }
 
+void PPCAIXAsmPrinter::EmitFunctionDescriptor() {
+  const DataLayout &DL = getDataLayout();
+  const unsigned PointerSize = DL.getPointerSizeInBits() == 64 ? 8 : 4;
+
+  MCSectionSubPair Current = OutStreamer->getCurrentSection();
+  // Emit function descriptor.
+  OutStreamer->SwitchSection(
+      cast<MCSymbolXCOFF>(CurrentFnDescSym)->getContainingCsect());
+  OutStreamer->EmitLabel(CurrentFnDescSym);
+  // Emit function entry point address.
+  OutStreamer->EmitValue(MCSymbolRefExpr::create(CurrentFnSym, OutContext),
+                         PointerSize);
+  // Emit TOC base address.
+  MCSymbol *TOCBaseSym = OutContext.getOrCreateSymbol(StringRef("TOC[TC0]"));
+  OutStreamer->EmitValue(MCSymbolRefExpr::create(TOCBaseSym, OutContext),
+                         PointerSize);
+  // Emit a null environment pointer.
+  OutStreamer->EmitIntValue(0, PointerSize);
+
+  OutStreamer->SwitchSection(Current.first, Current.second);
+}
+
+void PPCAIXAsmPrinter::EmitEndOfAsmFile(Module &M) {
+  // If there are no functions in this module, we will never need to reference
+  // the TOC base.
+  if (M.empty())
+    return;
+
+  // Emit TOC base.
+  MCSymbol *TOCBaseSym = OutContext.getOrCreateSymbol(StringRef("TOC[TC0]"));
+  MCSectionXCOFF *TOCBaseSection = OutStreamer->getContext().getXCOFFSection(
+      StringRef("TOC"), XCOFF::XMC_TC0, XCOFF::XTY_SD, XCOFF::C_HIDEXT,
+      SectionKind::getData());
+  cast<MCSymbolXCOFF>(TOCBaseSym)->setContainingCsect(TOCBaseSection);
+  // Switch to section to emit TOC base.
+  OutStreamer->SwitchSection(TOCBaseSection);
+}
+
+
 /// createPPCAsmPrinterPass - Returns a pass that prints the PPC assembly code
 /// for a MachineFunction to the given output stream, in a format that the
 /// Darwin assembler can deal with.
diff --git a/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp b/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
index 6e3c807b67e71..cdff4d383d236 100644
--- a/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
+++ b/llvm/lib/Target/PowerPC/PPCBranchSelector.cpp
@@ -81,20 +81,20 @@ FunctionPass *llvm::createPPCBranchSelectionPass() {
 /// original Offset.
 unsigned PPCBSel::GetAlignmentAdjustment(MachineBasicBlock &MBB,
                                          unsigned Offset) {
-  const llvm::Align Align = MBB.getAlignment();
-  if (Align == 1)
+  const Align Alignment = MBB.getAlignment();
+  if (Alignment == Align::None())
     return 0;
 
-  const llvm::Align ParentAlign = MBB.getParent()->getAlignment();
+  const Align ParentAlign = MBB.getParent()->getAlignment();
 
-  if (Align <= ParentAlign)
-    return offsetToAlignment(Offset, Align);
+  if (Alignment <= ParentAlign)
+    return offsetToAlignment(Offset, Alignment);
 
   // The alignment of this MBB is larger than the function's alignment, so we
   // can't tell whether or not it will insert nops. Assume that it will.
   if (FirstImpreciseBlock < 0)
     FirstImpreciseBlock = MBB.getNumber();
-  return Align.value() + offsetToAlignment(Offset, Align);
+  return Alignment.value() + offsetToAlignment(Offset, Alignment);
 }
 
 /// We need to be careful about the offset of the first block in the function
@@ -178,21 +178,20 @@ int PPCBSel::computeBranchSize(MachineFunction &Fn,
                                const MachineBasicBlock *Dest,
                                unsigned BrOffset) {
   int BranchSize;
-  unsigned MaxLogAlign = 2;
+  Align MaxAlign = Align(4);
   bool NeedExtraAdjustment = false;
   if (Dest->getNumber() <= Src->getNumber()) {
     // If this is a backwards branch, the delta is the offset from the
     // start of this block to this branch, plus the sizes of all blocks
     // from this block to the dest.
     BranchSize = BrOffset;
-    MaxLogAlign = std::max(MaxLogAlign, Src->getLogAlignment());
+    MaxAlign = std::max(MaxAlign, Src->getAlignment());
 
     int DestBlock = Dest->getNumber();
     BranchSize += BlockSizes[DestBlock].first;
     for (unsigned i = DestBlock+1, e = Src->getNumber(); i < e; ++i) {
       BranchSize += BlockSizes[i].first;
-      MaxLogAlign =
-          std::max(MaxLogAlign, Fn.getBlockNumbered(i)->getLogAlignment());
+      MaxAlign = std::max(MaxAlign, Fn.getBlockNumbered(i)->getAlignment());
     }
 
     NeedExtraAdjustment = (FirstImpreciseBlock >= 0) &&
@@ -203,11 +202,10 @@ int PPCBSel::computeBranchSize(MachineFunction &Fn,
     unsigned StartBlock = Src->getNumber();
     BranchSize = BlockSizes[StartBlock].first - BrOffset;
 
-    MaxLogAlign = std::max(MaxLogAlign, Dest->getLogAlignment());
+    MaxAlign = std::max(MaxAlign, Dest->getAlignment());
     for (unsigned i = StartBlock+1, e = Dest->getNumber(); i != e; ++i) {
       BranchSize += BlockSizes[i].first;
-      MaxLogAlign =
-          std::max(MaxLogAlign, Fn.getBlockNumbered(i)->getLogAlignment());
+      MaxAlign = std::max(MaxAlign, Fn.getBlockNumbered(i)->getAlignment());
     }
 
     NeedExtraAdjustment = (FirstImpreciseBlock >= 0) &&
@@ -257,7 +255,7 @@ int PPCBSel::computeBranchSize(MachineFunction &Fn,
   // The computed offset is at most ((1 << alignment) - 4) bytes smaller
   // than actual offset. So we add this number to the offset for safety.
   if (NeedExtraAdjustment)
-    BranchSize += (1 << MaxLogAlign) - 4;
+    BranchSize += MaxAlign.value() - 4;
 
   return BranchSize;
 }
diff --git a/llvm/lib/Target/PowerPC/PPCFastISel.cpp b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
index 1b545bccb4d0d..d8425d89da92e 100644
--- a/llvm/lib/Target/PowerPC/PPCFastISel.cpp
+++ b/llvm/lib/Target/PowerPC/PPCFastISel.cpp
@@ -2093,8 +2093,7 @@ unsigned PPCFastISel::PPCMaterializeGV(const GlobalValue *GV, MVT VT) {
     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::ADDIStocHA8),
             HighPartReg).addReg(PPC::X2).addGlobalAddress(GV);
 
-    unsigned char GVFlags = PPCSubTarget->classifyGlobalReference(GV);
-    if (GVFlags & PPCII::MO_NLP_FLAG) {
+    if (PPCSubTarget->isGVIndirectSymbol(GV)) {
       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(PPC::LDtocL),
               DestReg).addGlobalAddress(GV).addReg(HighPartReg);
     } else {
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 41a223bd5dde5..8d8ffc1199f35 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -139,7 +139,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
   // arguments are at least 4/8 bytes aligned.
   bool isPPC64 = Subtarget.isPPC64();
-  setMinStackArgumentAlignment(isPPC64 ? llvm::Align(8) : llvm::Align(4));
+  setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
 
   // Set up the register classes.
   addRegisterClass(MVT::i32, &PPC::GPRCRegClass);
@@ -551,7 +551,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   if (Subtarget.hasAltivec()) {
     // First set operation action for all vector types to expand. Then we
     // will selectively turn on ones that can be effectively codegen'd.
-    for (MVT VT : MVT::vector_valuetypes()) {
+    for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
       // add/sub are legal for all supported vector VT's.
       setOperationAction(ISD::ADD, VT, Legal);
       setOperationAction(ISD::SUB, VT, Legal);
@@ -652,7 +652,7 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
       setOperationAction(ISD::ROTL, VT, Expand);
       setOperationAction(ISD::ROTR, VT, Expand);
 
-      for (MVT InnerVT : MVT::vector_valuetypes()) {
+      for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
         setTruncStoreAction(VT, InnerVT, Expand);
         setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
         setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
@@ -1179,9 +1179,9 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
     setJumpIsExpensive();
   }
 
-  setMinFunctionAlignment(llvm::Align(4));
+  setMinFunctionAlignment(Align(4));
   if (Subtarget.isDarwin())
-    setPrefFunctionAlignment(llvm::Align(16));
+    setPrefFunctionAlignment(Align(16));
 
   switch (Subtarget.getDarwinDirective()) {
   default: break;
@@ -1198,8 +1198,8 @@ PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
   case PPC::DIR_PWR7:
   case PPC::DIR_PWR8:
   case PPC::DIR_PWR9:
-    setPrefLoopAlignment(llvm::Align(16));
-    setPrefFunctionAlignment(llvm::Align(16));
+    setPrefLoopAlignment(Align(16));
+    setPrefFunctionAlignment(Align(16));
     break;
   }
 
@@ -1405,7 +1405,8 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::EXTRACT_SPE:     return "PPCISD::EXTRACT_SPE";
   case PPCISD::EXTSWSLI:        return "PPCISD::EXTSWSLI";
   case PPCISD::LD_VSX_LH:       return "PPCISD::LD_VSX_LH";
-  case PPCISD::FP_EXTEND_LH:    return "PPCISD::FP_EXTEND_LH";
+  case PPCISD::FP_EXTEND_HALF:  return "PPCISD::FP_EXTEND_HALF";
+  case PPCISD::LD_SPLAT:        return "PPCISD::LD_SPLAT";
   }
   return nullptr;
 }
@@ -1778,10 +1779,10 @@ int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
 
 /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
 /// specifies a splat of a single element that is suitable for input to
-/// VSPLTB/VSPLTH/VSPLTW.
+/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
 bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
-  assert(N->getValueType(0) == MVT::v16i8 &&
-         (EltSize == 1 || EltSize == 2 || EltSize == 4));
+  assert(N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) &&
+         EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
 
   // The consecutive indices need to specify an element, not part of two
   // different elements.  So abandon ship early if this isn't the case.
@@ -2074,10 +2075,11 @@ bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
 }
 
 
-/// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the
-/// specified isSplatShuffleMask VECTOR_SHUFFLE mask.
-unsigned PPC::getVSPLTImmediate(SDNode *N, unsigned EltSize,
-                                SelectionDAG &DAG) {
+/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
+/// appropriate for PPC mnemonics (which have a big endian bias - namely
+/// elements are counted from the left of the vector register).
+unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
+                                         SelectionDAG &DAG) {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
   assert(isSplatShuffleMask(SVOp, EltSize));
   if (DAG.getDataLayout().isLittleEndian())
@@ -8185,6 +8187,18 @@ SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
                      Op0.getOperand(1));
 }
 
+static const SDValue *getNormalLoadInput(const SDValue &Op) {
+  const SDValue *InputLoad = &Op;
+  if (InputLoad->getOpcode() == ISD::BITCAST)
+    InputLoad = &InputLoad->getOperand(0);
+  if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR)
+    InputLoad = &InputLoad->getOperand(0);
+  if (InputLoad->getOpcode() != ISD::LOAD)
+    return nullptr;
+  LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
+  return ISD::isNormalLoad(LD) ? InputLoad : nullptr;
+}
+
 // If this is a case we can't handle, return null and let the default
 // expansion code take care of it.  If we CAN select this case, and if it
 // selects to a single instruction, return Op.  Otherwise, if we can codegen
@@ -8307,6 +8321,34 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
   if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize,
                              HasAnyUndefs, 0, !Subtarget.isLittleEndian()) ||
       SplatBitSize > 32) {
+
+    const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0));
+    // Handle load-and-splat patterns as we have instructions that will do this
+    // in one go.
+    if (InputLoad && DAG.isSplatValue(Op, true)) {
+      LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
+
+      // We have handling for 4 and 8 byte elements.
+      unsigned ElementSize = LD->getMemoryVT().getScalarSizeInBits();
+
+      // Checking for a single use of this load, we have to check for vector
+      // width (128 bits) / ElementSize uses (since each operand of the
+      // BUILD_VECTOR is a separate use of the value.
+      if (InputLoad->getNode()->hasNUsesOfValue(128 / ElementSize, 0) &&
+          ((Subtarget.hasVSX() && ElementSize == 64) ||
+           (Subtarget.hasP9Vector() && ElementSize == 32))) {
+        SDValue Ops[] = {
+          LD->getChain(),    // Chain
+          LD->getBasePtr(),  // Ptr
+          DAG.getValueType(Op.getValueType()) // VT
+        };
+        return
+          DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl,
+                                  DAG.getVTList(Op.getValueType(), MVT::Other),
+                                  Ops, LD->getMemoryVT(), LD->getMemOperand());
+      }
+    }
+
     // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be
     // lowered to VSX instructions under certain conditions.
     // Without VSX, there is no pattern more efficient than expanding the node.
@@ -8792,6 +8834,45 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
 
   unsigned ShiftElts, InsertAtByte;
   bool Swap = false;
+
+  // If this is a load-and-splat, we can do that with a single instruction
+  // in some cases. However if the load has multiple uses, we don't want to
+  // combine it because that will just produce multiple loads.
+  const SDValue *InputLoad = getNormalLoadInput(V1);
+  if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
+      (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) &&
+      InputLoad->hasOneUse()) {
+    bool IsFourByte = PPC::isSplatShuffleMask(SVOp, 4);
+    int SplatIdx =
+      PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG);
+
+    LoadSDNode *LD = cast<LoadSDNode>(*InputLoad);
+    // For 4-byte load-and-splat, we need Power9.
+    if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
+      uint64_t Offset = 0;
+      if (IsFourByte)
+        Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
+      else
+        Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
+      SDValue BasePtr = LD->getBasePtr();
+      if (Offset != 0)
+        BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
+                              BasePtr, DAG.getIntPtrConstant(Offset, dl));
+      SDValue Ops[] = {
+        LD->getChain(),    // Chain
+        BasePtr,           // BasePtr
+        DAG.getValueType(Op.getValueType()) // VT
+      };
+      SDVTList VTL =
+        DAG.getVTList(IsFourByte ? MVT::v4i32 : MVT::v2i64, MVT::Other);
+      SDValue LdSplt =
+        DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL,
+                                Ops, LD->getMemoryVT(), LD->getMemOperand());
+      if (LdSplt.getValueType() != SVOp->getValueType(0))
+        LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt);
+      return LdSplt;
+    }
+  }
   if (Subtarget.hasP9Vector() &&
       PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap,
                            isLittleEndian)) {
@@ -8868,7 +8949,7 @@ SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
 
   if (Subtarget.hasVSX()) {
     if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) {
-      int SplatIdx = PPC::getVSPLTImmediate(SVOp, 4, DAG);
+      int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, 4, DAG);
 
       SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1);
       SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv,
@@ -9913,6 +9994,30 @@ SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
   switch (Op0.getOpcode()) {
   default:
     return SDValue();
+  case ISD::EXTRACT_SUBVECTOR: {
+    assert(Op0.getNumOperands() == 2 &&
+           isa<ConstantSDNode>(Op0->getOperand(1)) &&
+           "Node should have 2 operands with second one being a constant!");
+
+    if (Op0.getOperand(0).getValueType() != MVT::v4f32)
+      return SDValue();
+
+    // Custom lower is only done for high or low doubleword.
+    int Idx = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
+    if (Idx % 2 != 0)
+      return SDValue();
+
+    // Since input is v4f32, at this point Idx is either 0 or 2.
+    // Shift to get the doubleword position we want.
+    int DWord = Idx >> 1;
+
+    // High and low word positions are different on little endian.
+    if (Subtarget.isLittleEndian())
+      DWord ^= 0x1;
+
+    return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64,
+                       Op0.getOperand(0), DAG.getConstant(DWord, dl, MVT::i32));
+  }
   case ISD::FADD:
   case ISD::FMUL:
   case ISD::FSUB: {
@@ -9924,26 +10029,25 @@ SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
         return SDValue();
       // Generate new load node.
       LoadSDNode *LD = cast<LoadSDNode>(LdOp);
-      SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() };
-      NewLoad[i] =
-        DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl,
-                                DAG.getVTList(MVT::v4f32, MVT::Other),
-                                LoadOps, LD->getMemoryVT(),
-                                LD->getMemOperand());
-    }
-    SDValue NewOp = DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32,
-                              NewLoad[0], NewLoad[1],
-                              Op0.getNode()->getFlags());
-    return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewOp);
+      SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
+      NewLoad[i] = DAG.getMemIntrinsicNode(
+          PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps,
+          LD->getMemoryVT(), LD->getMemOperand());
+    }
+    SDValue NewOp =
+        DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0],
+                    NewLoad[1], Op0.getNode()->getFlags());
+    return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewOp,
+                       DAG.getConstant(0, dl, MVT::i32));
   }
   case ISD::LOAD: {
     LoadSDNode *LD = cast<LoadSDNode>(Op0);
-    SDValue LoadOps[] = { LD->getChain(), LD->getBasePtr() };
-    SDValue NewLd =
-      DAG.getMemIntrinsicNode(PPCISD::LD_VSX_LH, dl,
-                              DAG.getVTList(MVT::v4f32, MVT::Other),
-                              LoadOps, LD->getMemoryVT(), LD->getMemOperand());
-    return DAG.getNode(PPCISD::FP_EXTEND_LH, dl, MVT::v2f64, NewLd);
+    SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
+    SDValue NewLd = DAG.getMemIntrinsicNode(
+        PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps,
+        LD->getMemoryVT(), LD->getMemOperand());
+    return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewLd,
+                       DAG.getConstant(0, dl, MVT::i32));
   }
   }
   llvm_unreachable("ERROR:Should return for all cases within swtich.");
@@ -14006,7 +14110,7 @@ void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
   }
 }
 
-llvm::Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
+Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
   switch (Subtarget.getDarwinDirective()) {
   default: break;
   case PPC::DIR_970:
@@ -14027,7 +14131,7 @@ llvm::Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
       // Actual alignment of the loop will depend on the hotness check and other
       // logic in alignBlocks.
       if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
-        return llvm::Align(32);
+        return Align(32);
     }
 
     const PPCInstrInfo *TII = Subtarget.getInstrInfo();
@@ -14043,7 +14147,7 @@ llvm::Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
       }
 
     if (LoopSize > 16 && LoopSize <= 32)
-      return llvm::Align(32);
+      return Align(32);
 
     break;
   }
@@ -14409,8 +14513,8 @@ SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                              SelectionDAG &DAG) const {
+Register PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              const MachineFunction &MF) const {
   bool isPPC64 = Subtarget.isPPC64();
   bool isDarwinABI = Subtarget.isDarwinABI();
 
@@ -14419,12 +14523,12 @@ unsigned PPCTargetLowering::getRegisterByName(const char* RegName, EVT VT,
     report_fatal_error("Invalid register global variable type");
 
   bool is64Bit = isPPC64 && VT == MVT::i64;
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+  Register Reg = StringSwitch<Register>(RegName)
                    .Case("r1", is64Bit ? PPC::X1 : PPC::R1)
-                   .Case("r2", (isDarwinABI || isPPC64) ? 0 : PPC::R2)
-                   .Case("r13", (!isPPC64 && isDarwinABI) ? 0 :
+                   .Case("r2", (isDarwinABI || isPPC64) ? Register() : PPC::R2)
+                   .Case("r13", (!isPPC64 && isDarwinABI) ? Register() :
                                   (is64Bit ? PPC::X13 : PPC::R13))
-                   .Default(0);
+                   .Default(Register());
 
   if (Reg)
     return Reg;
@@ -14451,14 +14555,8 @@ bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
   if (isa<JumpTableSDNode>(GA) || isa<BlockAddressSDNode>(GA))
     return true;
 
-  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA)) {
-    const GlobalValue *GV = G->getGlobal();
-    unsigned char GVFlags = Subtarget.classifyGlobalReference(GV);
-    // The NLP flag indicates that a global access has to use an
-    // extra indirection.
-    if (GVFlags & PPCII::MO_NLP_FLAG)
-      return true;
-  }
+  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(GA))
+    return Subtarget.isGVIndirectSymbol(G->getGlobal());
 
   return false;
 }
@@ -14523,7 +14621,7 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = -VT.getStoreSize()+1;
     Info.size = 2*VT.getStoreSize()-1;
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
@@ -14557,7 +14655,7 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
     Info.size = VT.getStoreSize();
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags = MachineMemOperand::MOLoad;
     return true;
   }
@@ -14609,7 +14707,7 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = -VT.getStoreSize()+1;
     Info.size = 2*VT.getStoreSize()-1;
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
@@ -14642,7 +14740,7 @@ bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     Info.ptrVal = I.getArgOperand(1);
     Info.offset = 0;
     Info.size = VT.getStoreSize();
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags = MachineMemOperand::MOStore;
     return true;
   }
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index dda9df971581f..62922ea2d4c41 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -412,8 +412,9 @@ namespace llvm {
       /// representation.
       QBFLT,
 
-      /// Custom extend v4f32 to v2f64.
-      FP_EXTEND_LH,
+      /// FP_EXTEND_HALF(VECTOR, IDX) - Custom extend upper (IDX=0) half or
+      /// lower (IDX=1) half of v4f32 to v2f64.
+      FP_EXTEND_HALF,
 
       /// CHAIN = STBRX CHAIN, GPRC, Ptr, Type - This is a
       /// byte-swapping store instruction.  It byte-swaps the low "Type" bits of
@@ -465,6 +466,10 @@ namespace llvm {
       /// v2f32 value into the lower half of a VSR register.
       LD_VSX_LH,
 
+      /// VSRC, CHAIN = LD_SPLAT, CHAIN, Ptr - a splatting load memory
+      /// instructions such as LXVDSX, LXVWSX.
+      LD_SPLAT,
+
       /// CHAIN = STXVD2X CHAIN, VSRC, Ptr - Occurs only for little endian.
       /// Maps directly to an stxvd2x instruction that will be preceded by
       /// an xxswapd.
@@ -573,9 +578,11 @@ namespace llvm {
     bool isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
                          unsigned &InsertAtByte, bool &Swap, bool IsLE);
 
-    /// getVSPLTImmediate - Return the appropriate VSPLT* immediate to splat the
-    /// specified isSplatShuffleMask VECTOR_SHUFFLE mask.
-    unsigned getVSPLTImmediate(SDNode *N, unsigned EltSize, SelectionDAG &DAG);
+    /// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
+    /// appropriate for PPC mnemonics (which have a big endian bias - namely
+    /// elements are counted from the left of the vector register).
+    unsigned getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
+                                        SelectionDAG &DAG);
 
     /// get_VSPLTI_elt - If this is a build_vector of constants which can be
     /// formed by using a vspltis[bhw] instruction of the specified element
@@ -726,8 +733,8 @@ namespace llvm {
     SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                           SmallVectorImpl<SDNode *> &Created) const override;
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     void computeKnownBitsForTargetNode(const SDValue Op,
                                        KnownBits &Known,
@@ -735,7 +742,7 @@ namespace llvm {
                                        const SelectionDAG &DAG,
                                        unsigned Depth = 0) const override;
 
-    llvm::Align getPrefLoopAlignment(MachineLoop *ML) const override;
+    Align getPrefLoopAlignment(MachineLoop *ML) const override;
 
     bool shouldInsertFencesForAtomic(const Instruction *I) const override {
       return true;
diff --git a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
index 8176c5120a835..fd3fc2af23279 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -215,21 +215,21 @@ def vsldoi_swapped_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
 
 // VSPLT*_get_imm xform function: convert vector_shuffle mask to VSPLT* imm.
 def VSPLTB_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 1, *CurDAG), SDLoc(N));
+  return getI32Imm(PPC::getSplatIdxForPPCMnemonics(N, 1, *CurDAG), SDLoc(N));
 }]>;
 def vspltb_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isSplatShuffleMask(cast<ShuffleVectorSDNode>(N), 1);
 }], VSPLTB_get_imm>;
 def VSPLTH_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 2, *CurDAG), SDLoc(N));
+  return getI32Imm(PPC::getSplatIdxForPPCMnemonics(N, 2, *CurDAG), SDLoc(N));
 }]>;
 def vsplth_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
   return PPC::isSplatShuffleMask(cast<ShuffleVectorSDNode>(N), 2);
 }], VSPLTH_get_imm>;
 def VSPLTW_get_imm : SDNodeXForm<vector_shuffle, [{
-  return getI32Imm(PPC::getVSPLTImmediate(N, 4, *CurDAG), SDLoc(N));
+  return getI32Imm(PPC::getSplatIdxForPPCMnemonics(N, 4, *CurDAG), SDLoc(N));
 }]>;
 def vspltw_shuffle : PatFrag<(ops node:$lhs, node:$rhs),
                              (vector_shuffle node:$lhs, node:$rhs), [{
@@ -331,7 +331,7 @@ class VXBX_Int_Ty<bits<11> xo, string opc, Intrinsic IntID, ValueType Ty>
 class VXCR_Int_Ty<bits<11> xo, string opc, Intrinsic IntID, ValueType Ty>
   : VXForm_CR<xo, (outs vrrc:$vD), (ins vrrc:$vA, u1imm:$ST, u4imm:$SIX),
               !strconcat(opc, " $vD, $vA, $ST, $SIX"), IIC_VecFP,
-              [(set Ty:$vD, (IntID Ty:$vA, imm:$ST, imm:$SIX))]>;
+              [(set Ty:$vD, (IntID Ty:$vA, timm:$ST, timm:$SIX))]>;
 
 //===----------------------------------------------------------------------===//
 // Instruction Definitions.
@@ -401,10 +401,10 @@ let isCodeGenOnly = 1 in {
 
 def MFVSCR : VXForm_4<1540, (outs vrrc:$vD), (ins),
                       "mfvscr $vD", IIC_LdStStore,
-                      [(set v8i16:$vD, (int_ppc_altivec_mfvscr))]>; 
+                      [(set v8i16:$vD, (int_ppc_altivec_mfvscr))]>;
 def MTVSCR : VXForm_5<1604, (outs), (ins vrrc:$vB),
                       "mtvscr $vB", IIC_LdStLoad,
-                      [(int_ppc_altivec_mtvscr v4i32:$vB)]>; 
+                      [(int_ppc_altivec_mtvscr v4i32:$vB)]>;
 
 let PPC970_Unit = 2, mayLoad = 1, mayStore = 0 in {  // Loads.
 def LVEBX: XForm_1_memOp<31,   7, (outs vrrc:$vD), (ins memrr:$src),
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index e4540d6520efd..dc013c8ff9a04 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -448,7 +448,8 @@ MachineInstr *PPCInstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
   return &MI;
 }
 
-bool PPCInstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+bool PPCInstrInfo::findCommutedOpIndices(const MachineInstr &MI,
+                                         unsigned &SrcOpIdx1,
                                          unsigned &SrcOpIdx2) const {
   // For VSX A-Type FMA instructions, it is the first two operands that can be
   // commuted, however, because the non-encoded tied input operand is listed
@@ -3570,16 +3571,20 @@ bool PPCInstrInfo::transformToImmFormFedByLI(MachineInstr &MI,
       } else {
         // The 32 bit and 64 bit instructions are quite different.
         if (SpecialShift32) {
-          // Left shifts use (N, 0, 31-N), right shifts use (32-N, N, 31).
-          uint64_t SH = RightShift ? 32 - ShAmt : ShAmt;
+          // Left shifts use (N, 0, 31-N).
+          // Right shifts use (32-N, N, 31) if 0 < N < 32.
+          //              use (0, 0, 31)    if N == 0.
+          uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 32 - ShAmt : ShAmt;
           uint64_t MB = RightShift ? ShAmt : 0;
           uint64_t ME = RightShift ? 31 : 31 - ShAmt;
           replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(MB)
             .addImm(ME);
         } else {
-          // Left shifts use (N, 63-N), right shifts use (64-N, N).
-          uint64_t SH = RightShift ? 64 - ShAmt : ShAmt;
+          // Left shifts use (N, 63-N).
+          // Right shifts use (64-N, N) if 0 < N < 64.
+          //              use (0, 0)    if N == 0.
+          uint64_t SH = ShAmt == 0 ? 0 : RightShift ? 64 - ShAmt : ShAmt;
           uint64_t ME = RightShift ? ShAmt : 63 - ShAmt;
           replaceInstrOperandWithImm(MI, III.OpNoForForwarding, SH);
           MachineInstrBuilder(*MI.getParent()->getParent(), MI).addImm(ME);
@@ -3930,21 +3935,99 @@ bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
   return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
 }
 
-bool PPCInstrInfo::analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst,
-                               MachineInstr *&CmpInst) const {
-  MachineBasicBlock *LoopEnd = L.getBottomBlock();
-  MachineBasicBlock::iterator I = LoopEnd->getFirstTerminator();
-  // We really "analyze" only CTR loops right now.
-  if (I != LoopEnd->end() && isBDNZ(I->getOpcode())) {
-    IndVarInst = nullptr;
-    CmpInst = &*I;
-    return false;
+namespace {
+class PPCPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
+  MachineInstr *Loop, *EndLoop, *LoopCount;
+  MachineFunction *MF;
+  const TargetInstrInfo *TII;
+  int64_t TripCount;
+
+public:
+  PPCPipelinerLoopInfo(MachineInstr *Loop, MachineInstr *EndLoop,
+                       MachineInstr *LoopCount)
+      : Loop(Loop), EndLoop(EndLoop), LoopCount(LoopCount),
+        MF(Loop->getParent()->getParent()),
+        TII(MF->getSubtarget().getInstrInfo()) {
+    // Inspect the Loop instruction up-front, as it may be deleted when we call
+    // createTripCountGreaterCondition.
+    if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI)
+      TripCount = LoopCount->getOperand(1).getImm();
+    else
+      TripCount = -1;
   }
-  return true;
+
+  bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
+    // Only ignore the terminator.
+    return MI == EndLoop;
+  }
+
+  Optional<bool>
+  createTripCountGreaterCondition(int TC, MachineBasicBlock &MBB,
+                                  SmallVectorImpl<MachineOperand> &Cond) override {
+    if (TripCount == -1) {
+      // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
+      // so we don't need to generate any thing here.
+      Cond.push_back(MachineOperand::CreateImm(0));
+      Cond.push_back(MachineOperand::CreateReg(
+          MF->getSubtarget<PPCSubtarget>().isPPC64() ? PPC::CTR8 : PPC::CTR,
+          true));
+      return {};
+    }
+
+    return TripCount > TC;
+  }
+
+  void setPreheader(MachineBasicBlock *NewPreheader) override {
+    // Do nothing. We want the LOOP setup instruction to stay in the *old*
+    // preheader, so we can use BDZ in the prologs to adapt the loop trip count.
+  }
+
+  void adjustTripCount(int TripCountAdjust) override {
+    // If the loop trip count is a compile-time value, then just change the
+    // value.
+    if (LoopCount->getOpcode() == PPC::LI8 ||
+        LoopCount->getOpcode() == PPC::LI) {
+      int64_t TripCount = LoopCount->getOperand(1).getImm() + TripCountAdjust;
+      LoopCount->getOperand(1).setImm(TripCount);
+      return;
+    }
+
+    // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
+    // so we don't need to generate any thing here.
+  }
+
+  void disposed() override {
+    Loop->eraseFromParent();
+    // Ensure the loop setup instruction is deleted too.
+    LoopCount->eraseFromParent();
+  }
+};
+} // namespace
+
+std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+PPCInstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
+  // We really "analyze" only hardware loops right now.
+  MachineBasicBlock::iterator I = LoopBB->getFirstTerminator();
+  MachineBasicBlock *Preheader = *LoopBB->pred_begin();
+  if (Preheader == LoopBB)
+    Preheader = *std::next(LoopBB->pred_begin());
+  MachineFunction *MF = Preheader->getParent();
+
+  if (I != LoopBB->end() && isBDNZ(I->getOpcode())) {
+    SmallPtrSet<MachineBasicBlock *, 8> Visited;
+    if (MachineInstr *LoopInst = findLoopInstr(*Preheader, Visited)) {
+      Register LoopCountReg = LoopInst->getOperand(0).getReg();
+      MachineRegisterInfo &MRI = MF->getRegInfo();
+      MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
+      return std::make_unique<PPCPipelinerLoopInfo>(LoopInst, &*I, LoopCount);
+    }
+  }
+  return nullptr;
 }
 
-MachineInstr *
-PPCInstrInfo::findLoopInstr(MachineBasicBlock &PreHeader) const {
+MachineInstr *PPCInstrInfo::findLoopInstr(
+    MachineBasicBlock &PreHeader,
+    SmallPtrSet<MachineBasicBlock *, 8> &Visited) const {
 
   unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
 
@@ -3955,50 +4038,6 @@ PPCInstrInfo::findLoopInstr(MachineBasicBlock &PreHeader) const {
   return nullptr;
 }
 
-unsigned PPCInstrInfo::reduceLoopCount(
-    MachineBasicBlock &MBB, MachineBasicBlock &PreHeader, MachineInstr *IndVar,
-    MachineInstr &Cmp, SmallVectorImpl<MachineOperand> &Cond,
-    SmallVectorImpl<MachineInstr *> &PrevInsts, unsigned Iter,
-    unsigned MaxIter) const {
-  // We expect a hardware loop currently. This means that IndVar is set
-  // to null, and the compare is the ENDLOOP instruction.
-  assert((!IndVar) && isBDNZ(Cmp.getOpcode()) && "Expecting a CTR loop");
-  MachineFunction *MF = MBB.getParent();
-  DebugLoc DL = Cmp.getDebugLoc();
-  MachineInstr *Loop = findLoopInstr(PreHeader);
-  if (!Loop)
-    return 0;
-  Register LoopCountReg = Loop->getOperand(0).getReg();
-  MachineRegisterInfo &MRI = MF->getRegInfo();
-  MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
-
-  if (!LoopCount)
-    return 0;
-  // If the loop trip count is a compile-time value, then just change the
-  // value.
-  if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI) {
-    int64_t Offset = LoopCount->getOperand(1).getImm();
-    if (Offset <= 1) {
-      LoopCount->eraseFromParent();
-      Loop->eraseFromParent();
-      return 0;
-    }
-    LoopCount->getOperand(1).setImm(Offset - 1);
-    return Offset - 1;
-  }
-
-  // The loop trip count is a run-time value.
-  // We need to subtract one from the trip count,
-  // and insert branch later to check if we're done with the loop.
-
-  // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
-  // so we don't need to generate any thing here.
-  Cond.push_back(MachineOperand::CreateImm(0));
-  Cond.push_back(MachineOperand::CreateReg(
-      Subtarget.isPPC64() ? PPC::CTR8 : PPC::CTR, true));
-  return LoopCountReg;
-}
-
 // Return true if get the base operand, byte offset of an instruction and the
 // memory width. Width is the size of memory that is being loaded/stored.
 bool PPCInstrInfo::getMemOperandWithOffsetWidth(
@@ -4025,8 +4064,7 @@ bool PPCInstrInfo::getMemOperandWithOffsetWidth(
 }
 
 bool PPCInstrInfo::areMemAccessesTriviallyDisjoint(
-    const MachineInstr &MIa, const MachineInstr &MIb,
-    AliasAnalysis * /*AA*/) const {
+    const MachineInstr &MIa, const MachineInstr &MIb) const {
   assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
   assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index 5150650439bc6..54c5726dedfd7 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -252,7 +252,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   unsigned isStoreToStackSlot(const MachineInstr &MI,
                               int &FrameIndex) const override;
 
-  bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+  bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
                              unsigned &SrcOpIdx2) const override;
 
   void insertNoop(MachineBasicBlock &MBB,
@@ -370,8 +370,7 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   /// otherwise
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const override;
+                                  const MachineInstr &MIb) const override;
 
   /// GetInstSize - Return the number of bytes of code the specified
   /// instruction may be.  This returns the maximum number of bytes.
@@ -486,26 +485,14 @@ class PPCInstrInfo : public PPCGenInstrInfo {
   /// On PPC, we have two instructions used to set-up the hardware loop
   /// (MTCTRloop, MTCTR8loop) with corresponding endloop (BDNZ, BDNZ8)
   /// instructions to indicate the end of a loop.
-  MachineInstr *findLoopInstr(MachineBasicBlock &PreHeader) const;
-
-  /// Analyze the loop code to find the loop induction variable and compare used
-  /// to compute the number of iterations. Currently, we analyze loop that are
-  /// controlled using hardware loops.  In this case, the induction variable
-  /// instruction is null.  For all other cases, this function returns true,
-  /// which means we're unable to analyze it. \p IndVarInst and \p CmpInst will
-  /// return new values when we can analyze the readonly loop \p L, otherwise,
-  /// nothing got changed
-  bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst,
-                   MachineInstr *&CmpInst) const override;
-  /// Generate code to reduce the loop iteration by one and check if the loop
-  /// is finished.  Return the value/register of the new loop count.  We need
-  /// this function when peeling off one or more iterations of a loop. This
-  /// function assumes the last iteration is peeled first.
-  unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineBasicBlock &PreHeader,
-                           MachineInstr *IndVar, MachineInstr &Cmp,
-                           SmallVectorImpl<MachineOperand> &Cond,
-                           SmallVectorImpl<MachineInstr *> &PrevInsts,
-                           unsigned Iter, unsigned MaxIter) const override;
+  MachineInstr *
+  findLoopInstr(MachineBasicBlock &PreHeader,
+                SmallPtrSet<MachineBasicBlock *, 8> &Visited) const;
+
+  /// Analyze loop L, which must be a single-basic-block loop, and if the
+  /// conditions can be understood enough produce a PipelinerLoopInfo object.
+  std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
+  analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 };
 
 }
diff --git a/llvm/lib/Target/PowerPC/PPCInstrVSX.td b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
index fbae8de4081f7..2f025b5a59d4d 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrVSX.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrVSX.td
@@ -58,8 +58,12 @@ def SDT_PPCldvsxlh : SDTypeProfile<1, 1, [
   SDTCisVT<0, v4f32>, SDTCisPtrTy<1>
 ]>;
 
-def SDT_PPCfpextlh : SDTypeProfile<1, 1, [
-  SDTCisVT<0, v2f64>, SDTCisVT<1, v4f32>
+def SDT_PPCfpexth : SDTypeProfile<1, 2, [
+  SDTCisVT<0, v2f64>, SDTCisVT<1, v4f32>, SDTCisPtrTy<2>
+]>;
+
+def SDT_PPCldsplat : SDTypeProfile<1, 1, [
+  SDTCisVec<0>, SDTCisPtrTy<1>
 ]>;
 
 // Little-endian-specific nodes.
@@ -102,9 +106,11 @@ def PPCuvec2fp: SDNode<"PPCISD::UINT_VEC_TO_FP", SDTVecConv, []>;
 def PPCswapNoChain : SDNode<"PPCISD::SWAP_NO_CHAIN", SDT_PPCxxswapd>;
 def PPCvabsd : SDNode<"PPCISD::VABSD", SDTVabsd, []>;
 
-def PPCfpextlh : SDNode<"PPCISD::FP_EXTEND_LH", SDT_PPCfpextlh, []>;
+def PPCfpexth : SDNode<"PPCISD::FP_EXTEND_HALF", SDT_PPCfpexth, []>;
 def PPCldvsxlh : SDNode<"PPCISD::LD_VSX_LH", SDT_PPCldvsxlh,
                         [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
+def PPCldsplat : SDNode<"PPCISD::LD_SPLAT", SDT_PPCldsplat,
+                        [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 multiclass XX3Form_Rcr<bits<6> opcode, bits<7> xo, string asmbase,
                     string asmstr, InstrItinClass itin, Intrinsic Int,
@@ -1086,7 +1092,8 @@ def : Pat<(v2f64 (PPCuvec2fp v4i32:$C, 0)),
 def : Pat<(v2f64 (PPCuvec2fp v4i32:$C, 1)),
           (v2f64 (XVCVUXWDP (v2i64 (XXMRGLW $C, $C))))>;
 
-def : Pat<(v2f64 (PPCfpextlh v4f32:$C)), (XVCVSPDP (XXMRGHW $C, $C))>;
+def : Pat<(v2f64 (PPCfpexth v4f32:$C, 0)), (XVCVSPDP (XXMRGHW $C, $C))>;
+def : Pat<(v2f64 (PPCfpexth v4f32:$C, 1)), (XVCVSPDP (XXMRGLW $C, $C))>;
 
 // Loads.
 let Predicates = [HasVSX, HasOnlySwappingMemOps] in {
@@ -1505,6 +1512,12 @@ let AddedComplexity = 400 in { // Prefer VSX patterns over non-VSX patterns.
                           AltVSXFMARel;
   }
 
+  // Additional xsnmsubasp patterns: -a*b + c == -(a*b - c)
+  def : Pat<(fma (fneg f32:$A), f32:$B, f32:$C),
+            (XSNMSUBASP $C, $A, $B)>;
+  def : Pat<(fma f32:$A, (fneg f32:$B), f32:$C),
+            (XSNMSUBASP $C, $A, $B)>;
+
   // Single Precision Conversions (FP <-> INT)
   def XSCVSXDSP : XX2Form<60, 312,
                       (outs vssrc:$XT), (ins vsfrc:$XB),
@@ -2861,12 +2874,12 @@ let AddedComplexity = 400, Predicates = [HasP9Vector] in {
                               (outs vsrc:$XT), (ins u7imm:$DCMX, vsrc:$XB),
                               "xvtstdcsp $XT, $XB, $DCMX", IIC_VecFP,
                               [(set v4i32: $XT,
-                               (int_ppc_vsx_xvtstdcsp v4f32:$XB, imm:$DCMX))]>;
+                               (int_ppc_vsx_xvtstdcsp v4f32:$XB, timm:$DCMX))]>;
   def XVTSTDCDP : XX2_RD6_DCMX7_RS6<60, 15, 5,
                               (outs vsrc:$XT), (ins u7imm:$DCMX, vsrc:$XB),
                               "xvtstdcdp $XT, $XB, $DCMX", IIC_VecFP,
                               [(set v2i64: $XT,
-                               (int_ppc_vsx_xvtstdcdp v2f64:$XB, imm:$DCMX))]>;
+                               (int_ppc_vsx_xvtstdcdp v2f64:$XB, timm:$DCMX))]>;
 
   //===--------------------------------------------------------------------===//
 
@@ -3930,6 +3943,10 @@ let AddedComplexity = 400 in {
                                 (XSCVDPUXWSs (XFLOADf32 xoaddr:$A)), VSRC), 1))>;
     def : Pat<(v4f32 (build_vector f32:$A, f32:$A, f32:$A, f32:$A)),
               (v4f32 (XXSPLTW (v4f32 (XSCVDPSPN $A)), 0))>;
+    def : Pat<(v2f64 (PPCldsplat xoaddr:$A)),
+              (v2f64 (LXVDSX xoaddr:$A))>;
+    def : Pat<(v2i64 (PPCldsplat xoaddr:$A)),
+              (v2i64 (LXVDSX xoaddr:$A))>;
 
     // Build vectors of floating point converted to i64.
     def : Pat<(v2i64 (build_vector FltToLong.A, FltToLong.A)),
@@ -4179,6 +4196,10 @@ let AddedComplexity = 400 in {
               (v2i64 (XXPERMDIs (XSCVDPUXDS (COPY_TO_REGCLASS
                                               (DFLOADf32 iaddrX4:$A),
                                               VSFRC)), 0))>;
+    def : Pat<(v4f32 (PPCldsplat xoaddr:$A)),
+              (v4f32 (LXVWSX xoaddr:$A))>;
+    def : Pat<(v4i32 (PPCldsplat xoaddr:$A)),
+              (v4i32 (LXVWSX xoaddr:$A))>;
   }
 
   let Predicates = [IsISA3_0, HasDirectMove, IsBigEndian] in {
diff --git a/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp b/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
index 4d45d96d4479f..d252cfbd26b1d 100644
--- a/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
+++ b/llvm/lib/Target/PowerPC/PPCLoopPreIncPrep.cpp
@@ -63,8 +63,24 @@ static cl::opt<unsigned> MaxVars("ppc-preinc-prep-max-vars",
   cl::desc("Potential PHI threshold for PPC preinc loop prep"));
 
 STATISTIC(PHINodeAlreadyExists, "PHI node already in pre-increment form");
+STATISTIC(UpdFormChainRewritten, "Num of update form chain rewritten");
 
 namespace {
+  struct BucketElement {
+    BucketElement(const SCEVConstant *O, Instruction *I) : Offset(O), Instr(I) {}
+    BucketElement(Instruction *I) : Offset(nullptr), Instr(I) {}
+
+    const SCEVConstant *Offset;
+    Instruction *Instr;
+  };
+
+  struct Bucket {
+    Bucket(const SCEV *B, Instruction *I) : BaseSCEV(B),
+                                            Elements(1, BucketElement(I)) {}
+
+    const SCEV *BaseSCEV;
+    SmallVector<BucketElement, 16> Elements;
+  };
 
   class PPCLoopPreIncPrep : public FunctionPass {
   public:
@@ -85,21 +101,47 @@ namespace {
       AU.addRequired<ScalarEvolutionWrapperPass>();
     }
 
-    bool alreadyPrepared(Loop *L, Instruction* MemI,
-                         const SCEV *BasePtrStartSCEV,
-                         const SCEVConstant *BasePtrIncSCEV);
     bool runOnFunction(Function &F) override;
 
-    bool runOnLoop(Loop *L);
-    void simplifyLoopLatch(Loop *L);
-    bool rotateLoop(Loop *L);
-
   private:
     PPCTargetMachine *TM = nullptr;
+    const PPCSubtarget *ST; 
     DominatorTree *DT;
     LoopInfo *LI;
     ScalarEvolution *SE;
     bool PreserveLCSSA;
+
+    bool runOnLoop(Loop *L);
+
+    /// Check if required PHI node is already exist in Loop \p L.
+    bool alreadyPrepared(Loop *L, Instruction* MemI,
+                         const SCEV *BasePtrStartSCEV,
+                         const SCEVConstant *BasePtrIncSCEV);
+
+    /// Collect condition matched(\p isValidCandidate() returns true)
+    /// candidates in Loop \p L.
+    SmallVector<Bucket, 16>
+    collectCandidates(Loop *L,
+                      std::function<bool(const Instruction *, const Value *)>
+                          isValidCandidate,
+                      unsigned MaxCandidateNum);
+
+    /// Add a candidate to candidates \p Buckets.
+    void addOneCandidate(Instruction *MemI, const SCEV *LSCEV,
+                         SmallVector<Bucket, 16> &Buckets,
+                         unsigned MaxCandidateNum);
+
+    /// Prepare all candidates in \p Buckets for update form.
+    bool updateFormPrep(Loop *L, SmallVector<Bucket, 16> &Buckets);
+
+    /// Prepare for one chain \p BucketChain, find the best base element and
+    /// update all other elements in \p BucketChain accordingly.
+    bool prepareBaseForUpdateFormChain(Bucket &BucketChain);
+
+    /// Rewrite load/store instructions in \p BucketChain according to
+    /// preparation.
+    bool rewriteLoadStores(Loop *L, Bucket &BucketChain,
+                           SmallSet<BasicBlock *, 16> &BBChanged);
   };
 
 } // end anonymous namespace
@@ -111,30 +153,15 @@ INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
 INITIALIZE_PASS_END(PPCLoopPreIncPrep, DEBUG_TYPE, name, false, false)
 
+static const std::string PHINodeNameSuffix    = ".phi";
+static const std::string CastNodeNameSuffix   = ".cast";
+static const std::string GEPNodeIncNameSuffix = ".inc";
+static const std::string GEPNodeOffNameSuffix = ".off";
+
 FunctionPass *llvm::createPPCLoopPreIncPrepPass(PPCTargetMachine &TM) {
   return new PPCLoopPreIncPrep(TM);
 }
 
-namespace {
-
-  struct BucketElement {
-    BucketElement(const SCEVConstant *O, Instruction *I) : Offset(O), Instr(I) {}
-    BucketElement(Instruction *I) : Offset(nullptr), Instr(I) {}
-
-    const SCEVConstant *Offset;
-    Instruction *Instr;
-  };
-
-  struct Bucket {
-    Bucket(const SCEV *B, Instruction *I) : BaseSCEV(B),
-                                            Elements(1, BucketElement(I)) {}
-
-    const SCEV *BaseSCEV;
-    SmallVector<BucketElement, 16> Elements;
-  };
-
-} // end anonymous namespace
-
 static bool IsPtrInBounds(Value *BasePtr) {
   Value *StrippedBasePtr = BasePtr;
   while (BitCastInst *BC = dyn_cast<BitCastInst>(StrippedBasePtr))
@@ -145,6 +172,14 @@ static bool IsPtrInBounds(Value *BasePtr) {
   return false;
 }
 
+static std::string getInstrName(const Value *I, const std::string Suffix) {
+  assert(I && "Invalid paramater!");
+  if (I->hasName())
+    return (I->getName() + Suffix).str();
+  else
+    return ""; 
+}
+
 static Value *GetPointerOperand(Value *MemI) {
   if (LoadInst *LMemI = dyn_cast<LoadInst>(MemI)) {
     return LMemI->getPointerOperand();
@@ -167,6 +202,7 @@ bool PPCLoopPreIncPrep::runOnFunction(Function &F) {
   auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
   DT = DTWP ? &DTWP->getDomTree() : nullptr;
   PreserveLCSSA = mustPreserveAnalysisID(LCSSAID);
+  ST = TM ? TM->getSubtargetImpl(F) : nullptr;
 
   bool MadeChange = false;
 
@@ -177,10 +213,280 @@ bool PPCLoopPreIncPrep::runOnFunction(Function &F) {
   return MadeChange;
 }
 
+void PPCLoopPreIncPrep::addOneCandidate(Instruction *MemI, const SCEV *LSCEV,
+                                        SmallVector<Bucket, 16> &Buckets,
+                                        unsigned MaxCandidateNum) {
+  assert((MemI && GetPointerOperand(MemI)) &&
+         "Candidate should be a memory instruction.");
+  assert(LSCEV && "Invalid SCEV for Ptr value.");
+  bool FoundBucket = false;
+  for (auto &B : Buckets) {
+    const SCEV *Diff = SE->getMinusSCEV(LSCEV, B.BaseSCEV);
+    if (const auto *CDiff = dyn_cast<SCEVConstant>(Diff)) {
+      B.Elements.push_back(BucketElement(CDiff, MemI));
+      FoundBucket = true;
+      break;
+    }
+  }
+
+  if (!FoundBucket) {
+    if (Buckets.size() == MaxCandidateNum)
+      return;
+    Buckets.push_back(Bucket(LSCEV, MemI));
+  }
+}
+
+SmallVector<Bucket, 16> PPCLoopPreIncPrep::collectCandidates(
+    Loop *L,
+    std::function<bool(const Instruction *, const Value *)> isValidCandidate,
+    unsigned MaxCandidateNum) {
+  SmallVector<Bucket, 16> Buckets;
+  for (const auto &BB : L->blocks())
+    for (auto &J : *BB) {
+      Value *PtrValue;
+      Instruction *MemI;
+
+      if (LoadInst *LMemI = dyn_cast<LoadInst>(&J)) {
+        MemI = LMemI;
+        PtrValue = LMemI->getPointerOperand();
+      } else if (StoreInst *SMemI = dyn_cast<StoreInst>(&J)) {
+        MemI = SMemI;
+        PtrValue = SMemI->getPointerOperand();
+      } else if (IntrinsicInst *IMemI = dyn_cast<IntrinsicInst>(&J)) {
+        if (IMemI->getIntrinsicID() == Intrinsic::prefetch) {
+          MemI = IMemI;
+          PtrValue = IMemI->getArgOperand(0);
+        } else continue;
+      } else continue;
+
+      unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace();
+      if (PtrAddrSpace)
+        continue;
+
+      if (L->isLoopInvariant(PtrValue))
+        continue;
+
+      const SCEV *LSCEV = SE->getSCEVAtScope(PtrValue, L);
+      const SCEVAddRecExpr *LARSCEV = dyn_cast<SCEVAddRecExpr>(LSCEV);
+      if (!LARSCEV || LARSCEV->getLoop() != L)
+        continue;
+
+      if (isValidCandidate(&J, PtrValue))
+        addOneCandidate(MemI, LSCEV, Buckets, MaxCandidateNum);
+    }
+  return Buckets;
+}
+
+// TODO: implement a more clever base choosing policy.
+// Currently we always choose an exist load/store offset. This maybe lead to
+// suboptimal code sequences. For example, for one DS chain with offsets
+// {-32769, 2003, 2007, 2011}, we choose -32769 as base offset, and left disp
+// for load/stores are {0, 34772, 34776, 34780}. Though each offset now is a
+// multipler of 4, it cannot be represented by sint16.
+bool PPCLoopPreIncPrep::prepareBaseForUpdateFormChain(Bucket &BucketChain) {
+  // We have a choice now of which instruction's memory operand we use as the
+  // base for the generated PHI. Always picking the first instruction in each
+  // bucket does not work well, specifically because that instruction might
+  // be a prefetch (and there are no pre-increment dcbt variants). Otherwise,
+  // the choice is somewhat arbitrary, because the backend will happily
+  // generate direct offsets from both the pre-incremented and
+  // post-incremented pointer values. Thus, we'll pick the first non-prefetch
+  // instruction in each bucket, and adjust the recurrence and other offsets
+  // accordingly.
+  for (int j = 0, je = BucketChain.Elements.size(); j != je; ++j) {
+    if (auto *II = dyn_cast<IntrinsicInst>(BucketChain.Elements[j].Instr))
+      if (II->getIntrinsicID() == Intrinsic::prefetch)
+        continue;
+
+    // If we'd otherwise pick the first element anyway, there's nothing to do.
+    if (j == 0)
+      break;
+
+    // If our chosen element has no offset from the base pointer, there's
+    // nothing to do.
+    if (!BucketChain.Elements[j].Offset ||
+        BucketChain.Elements[j].Offset->isZero())
+      break;
+
+    const SCEV *Offset = BucketChain.Elements[j].Offset;
+    BucketChain.BaseSCEV = SE->getAddExpr(BucketChain.BaseSCEV, Offset);
+    for (auto &E : BucketChain.Elements) {
+      if (E.Offset)
+        E.Offset = cast<SCEVConstant>(SE->getMinusSCEV(E.Offset, Offset));
+      else
+        E.Offset = cast<SCEVConstant>(SE->getNegativeSCEV(Offset));
+    }
+
+    std::swap(BucketChain.Elements[j], BucketChain.Elements[0]);
+    break;
+  }
+  return true;
+}
+
+bool PPCLoopPreIncPrep::rewriteLoadStores(
+    Loop *L, Bucket &BucketChain, SmallSet<BasicBlock *, 16> &BBChanged) {
+  bool MadeChange = false;
+  const SCEVAddRecExpr *BasePtrSCEV =
+      cast<SCEVAddRecExpr>(BucketChain.BaseSCEV);
+  if (!BasePtrSCEV->isAffine())
+    return MadeChange;
+
+  LLVM_DEBUG(dbgs() << "PIP: Transforming: " << *BasePtrSCEV << "\n");
+
+  assert(BasePtrSCEV->getLoop() == L && "AddRec for the wrong loop?");
+
+  // The instruction corresponding to the Bucket's BaseSCEV must be the first
+  // in the vector of elements.
+  Instruction *MemI = BucketChain.Elements.begin()->Instr;
+  Value *BasePtr = GetPointerOperand(MemI);
+  assert(BasePtr && "No pointer operand");
+
+  Type *I8Ty = Type::getInt8Ty(MemI->getParent()->getContext());
+  Type *I8PtrTy = Type::getInt8PtrTy(MemI->getParent()->getContext(),
+    BasePtr->getType()->getPointerAddressSpace());
+
+  const SCEV *BasePtrStartSCEV = BasePtrSCEV->getStart();
+  if (!SE->isLoopInvariant(BasePtrStartSCEV, L))
+    return MadeChange;
+
+  const SCEVConstant *BasePtrIncSCEV =
+    dyn_cast<SCEVConstant>(BasePtrSCEV->getStepRecurrence(*SE));
+  if (!BasePtrIncSCEV)
+    return MadeChange;
+  BasePtrStartSCEV = SE->getMinusSCEV(BasePtrStartSCEV, BasePtrIncSCEV);
+  if (!isSafeToExpand(BasePtrStartSCEV, *SE))
+    return MadeChange;
+
+  if (alreadyPrepared(L, MemI, BasePtrStartSCEV, BasePtrIncSCEV))
+    return MadeChange;
+
+  LLVM_DEBUG(dbgs() << "PIP: New start is: " << *BasePtrStartSCEV << "\n");
+
+  BasicBlock *Header = L->getHeader();
+  unsigned HeaderLoopPredCount = pred_size(Header);
+  BasicBlock *LoopPredecessor = L->getLoopPredecessor();
+
+  PHINode *NewPHI =
+      PHINode::Create(I8PtrTy, HeaderLoopPredCount,
+                      getInstrName(MemI, PHINodeNameSuffix),
+                      Header->getFirstNonPHI());
+
+  SCEVExpander SCEVE(*SE, Header->getModule()->getDataLayout(), "pistart");
+  Value *BasePtrStart = SCEVE.expandCodeFor(BasePtrStartSCEV, I8PtrTy,
+                                            LoopPredecessor->getTerminator());
+
+  // Note that LoopPredecessor might occur in the predecessor list multiple
+  // times, and we need to add it the right number of times.
+  for (const auto &PI : predecessors(Header)) {
+    if (PI != LoopPredecessor)
+      continue;
+
+    NewPHI->addIncoming(BasePtrStart, LoopPredecessor);
+  }
+
+  Instruction *InsPoint = &*Header->getFirstInsertionPt();
+  GetElementPtrInst *PtrInc = GetElementPtrInst::Create(
+      I8Ty, NewPHI, BasePtrIncSCEV->getValue(),
+      getInstrName(MemI, GEPNodeIncNameSuffix), InsPoint);
+  PtrInc->setIsInBounds(IsPtrInBounds(BasePtr));
+  for (const auto &PI : predecessors(Header)) {
+    if (PI == LoopPredecessor)
+      continue;
+
+    NewPHI->addIncoming(PtrInc, PI);
+  }
+
+  Instruction *NewBasePtr;
+  if (PtrInc->getType() != BasePtr->getType())
+    NewBasePtr = new BitCastInst(PtrInc, BasePtr->getType(),
+      getInstrName(PtrInc, CastNodeNameSuffix), InsPoint);
+  else
+    NewBasePtr = PtrInc;
+
+  if (Instruction *IDel = dyn_cast<Instruction>(BasePtr))
+    BBChanged.insert(IDel->getParent());
+  BasePtr->replaceAllUsesWith(NewBasePtr);
+  RecursivelyDeleteTriviallyDeadInstructions(BasePtr);
+
+  // Keep track of the replacement pointer values we've inserted so that we
+  // don't generate more pointer values than necessary.
+  SmallPtrSet<Value *, 16> NewPtrs;
+  NewPtrs.insert(NewBasePtr);
+
+  for (auto I = std::next(BucketChain.Elements.begin()),
+       IE = BucketChain.Elements.end(); I != IE; ++I) {
+    Value *Ptr = GetPointerOperand(I->Instr);
+    assert(Ptr && "No pointer operand");
+    if (NewPtrs.count(Ptr))
+      continue;
+
+    Instruction *RealNewPtr;
+    if (!I->Offset || I->Offset->getValue()->isZero()) {
+      RealNewPtr = NewBasePtr;
+    } else {
+      Instruction *PtrIP = dyn_cast<Instruction>(Ptr);
+      if (PtrIP && isa<Instruction>(NewBasePtr) &&
+          cast<Instruction>(NewBasePtr)->getParent() == PtrIP->getParent())
+        PtrIP = nullptr;
+      else if (PtrIP && isa<PHINode>(PtrIP))
+        PtrIP = &*PtrIP->getParent()->getFirstInsertionPt();
+      else if (!PtrIP)
+        PtrIP = I->Instr;
+
+      GetElementPtrInst *NewPtr = GetElementPtrInst::Create(
+          I8Ty, PtrInc, I->Offset->getValue(),
+          getInstrName(I->Instr, GEPNodeOffNameSuffix), PtrIP);
+      if (!PtrIP)
+        NewPtr->insertAfter(cast<Instruction>(PtrInc));
+      NewPtr->setIsInBounds(IsPtrInBounds(Ptr));
+      RealNewPtr = NewPtr;
+    }
+
+    if (Instruction *IDel = dyn_cast<Instruction>(Ptr))
+      BBChanged.insert(IDel->getParent());
+
+    Instruction *ReplNewPtr;
+    if (Ptr->getType() != RealNewPtr->getType()) {
+      ReplNewPtr = new BitCastInst(RealNewPtr, Ptr->getType(),
+        getInstrName(Ptr, CastNodeNameSuffix));
+      ReplNewPtr->insertAfter(RealNewPtr);
+    } else
+      ReplNewPtr = RealNewPtr;
+
+    Ptr->replaceAllUsesWith(ReplNewPtr);
+    RecursivelyDeleteTriviallyDeadInstructions(Ptr);
+
+    NewPtrs.insert(RealNewPtr);
+  }
+
+  MadeChange = true;
+  UpdFormChainRewritten++;
+
+  return MadeChange;
+}
+
+bool PPCLoopPreIncPrep::updateFormPrep(Loop *L,
+                                       SmallVector<Bucket, 16> &Buckets) {
+  bool MadeChange = false;
+  if (Buckets.empty())
+    return MadeChange;
+  SmallSet<BasicBlock *, 16> BBChanged;
+  for (auto &Bucket : Buckets)
+    // The base address of each bucket is transformed into a phi and the others
+    // are rewritten based on new base.
+    if (prepareBaseForUpdateFormChain(Bucket))
+      MadeChange |= rewriteLoadStores(L, Bucket, BBChanged);
+  if (MadeChange)
+    for (auto &BB : L->blocks())
+      if (BBChanged.count(BB))
+        DeleteDeadPHIs(BB);
+  return MadeChange;
+}
+
 // In order to prepare for the pre-increment a PHI is added.
 // This function will check to see if that PHI already exists and will return
-//  true if it found an existing PHI with the same start and increment as the
-//  one we wanted to create.
+// true if it found an existing PHI with the same start and increment as the
+// one we wanted to create.
 bool PPCLoopPreIncPrep::alreadyPrepared(Loop *L, Instruction* MemI,
                                         const SCEV *BasePtrStartSCEV,
                                         const SCEVConstant *BasePtrIncSCEV) {
@@ -216,10 +522,10 @@ bool PPCLoopPreIncPrep::alreadyPrepared(Loop *L, Instruction* MemI,
       continue;
 
     if (CurrentPHINode->getNumIncomingValues() == 2) {
-      if ( (CurrentPHINode->getIncomingBlock(0) == LatchBB &&
-            CurrentPHINode->getIncomingBlock(1) == PredBB) ||
-            (CurrentPHINode->getIncomingBlock(1) == LatchBB &&
-            CurrentPHINode->getIncomingBlock(0) == PredBB) ) {
+      if ((CurrentPHINode->getIncomingBlock(0) == LatchBB &&
+           CurrentPHINode->getIncomingBlock(1) == PredBB) ||
+          (CurrentPHINode->getIncomingBlock(1) == LatchBB &&
+           CurrentPHINode->getIncomingBlock(0) == PredBB)) {
         if (PHIBasePtrSCEV->getStart() == BasePtrStartSCEV &&
             PHIBasePtrIncSCEV == BasePtrIncSCEV) {
           // The existing PHI (CurrentPHINode) has the same start and increment
@@ -242,89 +548,6 @@ bool PPCLoopPreIncPrep::runOnLoop(Loop *L) {
 
   LLVM_DEBUG(dbgs() << "PIP: Examining: " << *L << "\n");
 
-  BasicBlock *Header = L->getHeader();
-
-  const PPCSubtarget *ST =
-    TM ? TM->getSubtargetImpl(*Header->getParent()) : nullptr;
-
-  unsigned HeaderLoopPredCount = pred_size(Header);
-
-  // Collect buckets of comparable addresses used by loads and stores.
-  SmallVector<Bucket, 16> Buckets;
-  for (Loop::block_iterator I = L->block_begin(), IE = L->block_end();
-       I != IE; ++I) {
-    for (BasicBlock::iterator J = (*I)->begin(), JE = (*I)->end();
-        J != JE; ++J) {
-      Value *PtrValue;
-      Instruction *MemI;
-
-      if (LoadInst *LMemI = dyn_cast<LoadInst>(J)) {
-        MemI = LMemI;
-        PtrValue = LMemI->getPointerOperand();
-      } else if (StoreInst *SMemI = dyn_cast<StoreInst>(J)) {
-        MemI = SMemI;
-        PtrValue = SMemI->getPointerOperand();
-      } else if (IntrinsicInst *IMemI = dyn_cast<IntrinsicInst>(J)) {
-        if (IMemI->getIntrinsicID() == Intrinsic::prefetch) {
-          MemI = IMemI;
-          PtrValue = IMemI->getArgOperand(0);
-        } else continue;
-      } else continue;
-
-      unsigned PtrAddrSpace = PtrValue->getType()->getPointerAddressSpace();
-      if (PtrAddrSpace)
-        continue;
-
-      // There are no update forms for Altivec vector load/stores.
-      if (ST && ST->hasAltivec() &&
-          PtrValue->getType()->getPointerElementType()->isVectorTy())
-        continue;
-
-      if (L->isLoopInvariant(PtrValue))
-        continue;
-
-      const SCEV *LSCEV = SE->getSCEVAtScope(PtrValue, L);
-      if (const SCEVAddRecExpr *LARSCEV = dyn_cast<SCEVAddRecExpr>(LSCEV)) {
-        if (LARSCEV->getLoop() != L)
-          continue;
-        // See getPreIndexedAddressParts, the displacement for LDU/STDU has to
-        // be 4's multiple (DS-form). For i64 loads/stores when the displacement
-        // fits in a 16-bit signed field but isn't a multiple of 4, it will be
-        // useless and possible to break some original well-form addressing mode
-        // to make this pre-inc prep for it.
-        if (PtrValue->getType()->getPointerElementType()->isIntegerTy(64)) {
-          if (const SCEVConstant *StepConst =
-                  dyn_cast<SCEVConstant>(LARSCEV->getStepRecurrence(*SE))) {
-            const APInt &ConstInt = StepConst->getValue()->getValue();
-            if (ConstInt.isSignedIntN(16) && ConstInt.srem(4) != 0)
-              continue;
-          }
-        }
-      } else {
-        continue;
-      }
-
-      bool FoundBucket = false;
-      for (auto &B : Buckets) {
-        const SCEV *Diff = SE->getMinusSCEV(LSCEV, B.BaseSCEV);
-        if (const auto *CDiff = dyn_cast<SCEVConstant>(Diff)) {
-          B.Elements.push_back(BucketElement(CDiff, MemI));
-          FoundBucket = true;
-          break;
-        }
-      }
-
-      if (!FoundBucket) {
-        if (Buckets.size() == MaxVars)
-          return MadeChange;
-        Buckets.push_back(Bucket(LSCEV, MemI));
-      }
-    }
-  }
-
-  if (Buckets.empty())
-    return MadeChange;
-
   BasicBlock *LoopPredecessor = L->getLoopPredecessor();
   // If there is no loop predecessor, or the loop predecessor's terminator
   // returns a value (which might contribute to determining the loop's
@@ -335,191 +558,48 @@ bool PPCLoopPreIncPrep::runOnLoop(Loop *L) {
     if (LoopPredecessor)
       MadeChange = true;
   }
-  if (!LoopPredecessor)
+  if (!LoopPredecessor) {
+    LLVM_DEBUG(dbgs() << "PIP fails since no predecessor for current loop.\n");
     return MadeChange;
+  }
 
-  LLVM_DEBUG(dbgs() << "PIP: Found " << Buckets.size() << " buckets\n");
-
-  SmallSet<BasicBlock *, 16> BBChanged;
-  for (unsigned i = 0, e = Buckets.size(); i != e; ++i) {
-    // The base address of each bucket is transformed into a phi and the others
-    // are rewritten as offsets of that variable.
-
-    // We have a choice now of which instruction's memory operand we use as the
-    // base for the generated PHI. Always picking the first instruction in each
-    // bucket does not work well, specifically because that instruction might
-    // be a prefetch (and there are no pre-increment dcbt variants). Otherwise,
-    // the choice is somewhat arbitrary, because the backend will happily
-    // generate direct offsets from both the pre-incremented and
-    // post-incremented pointer values. Thus, we'll pick the first non-prefetch
-    // instruction in each bucket, and adjust the recurrence and other offsets
-    // accordingly.
-    for (int j = 0, je = Buckets[i].Elements.size(); j != je; ++j) {
-      if (auto *II = dyn_cast<IntrinsicInst>(Buckets[i].Elements[j].Instr))
-        if (II->getIntrinsicID() == Intrinsic::prefetch)
-          continue;
-
-      // If we'd otherwise pick the first element anyway, there's nothing to do.
-      if (j == 0)
-        break;
-
-      // If our chosen element has no offset from the base pointer, there's
-      // nothing to do.
-      if (!Buckets[i].Elements[j].Offset ||
-          Buckets[i].Elements[j].Offset->isZero())
-        break;
-
-      const SCEV *Offset = Buckets[i].Elements[j].Offset;
-      Buckets[i].BaseSCEV = SE->getAddExpr(Buckets[i].BaseSCEV, Offset);
-      for (auto &E : Buckets[i].Elements) {
-        if (E.Offset)
-          E.Offset = cast<SCEVConstant>(SE->getMinusSCEV(E.Offset, Offset));
-        else
-          E.Offset = cast<SCEVConstant>(SE->getNegativeSCEV(Offset));
-      }
-
-      std::swap(Buckets[i].Elements[j], Buckets[i].Elements[0]);
-      break;
-    }
-
-    const SCEVAddRecExpr *BasePtrSCEV =
-      cast<SCEVAddRecExpr>(Buckets[i].BaseSCEV);
-    if (!BasePtrSCEV->isAffine())
-      continue;
-
-    LLVM_DEBUG(dbgs() << "PIP: Transforming: " << *BasePtrSCEV << "\n");
-    assert(BasePtrSCEV->getLoop() == L &&
-           "AddRec for the wrong loop?");
-
-    // The instruction corresponding to the Bucket's BaseSCEV must be the first
-    // in the vector of elements.
-    Instruction *MemI = Buckets[i].Elements.begin()->Instr;
-    Value *BasePtr = GetPointerOperand(MemI);
-    assert(BasePtr && "No pointer operand");
-
-    Type *I8Ty = Type::getInt8Ty(MemI->getParent()->getContext());
-    Type *I8PtrTy = Type::getInt8PtrTy(MemI->getParent()->getContext(),
-      BasePtr->getType()->getPointerAddressSpace());
-
-    const SCEV *BasePtrStartSCEV = BasePtrSCEV->getStart();
-    if (!SE->isLoopInvariant(BasePtrStartSCEV, L))
-      continue;
-
-    const SCEVConstant *BasePtrIncSCEV =
-      dyn_cast<SCEVConstant>(BasePtrSCEV->getStepRecurrence(*SE));
-    if (!BasePtrIncSCEV)
-      continue;
-    BasePtrStartSCEV = SE->getMinusSCEV(BasePtrStartSCEV, BasePtrIncSCEV);
-    if (!isSafeToExpand(BasePtrStartSCEV, *SE))
-      continue;
-
-    LLVM_DEBUG(dbgs() << "PIP: New start is: " << *BasePtrStartSCEV << "\n");
-
-    if (alreadyPrepared(L, MemI, BasePtrStartSCEV, BasePtrIncSCEV))
-      continue;
-
-    PHINode *NewPHI = PHINode::Create(I8PtrTy, HeaderLoopPredCount,
-      MemI->hasName() ? MemI->getName() + ".phi" : "",
-      Header->getFirstNonPHI());
-
-    SCEVExpander SCEVE(*SE, Header->getModule()->getDataLayout(), "pistart");
-    Value *BasePtrStart = SCEVE.expandCodeFor(BasePtrStartSCEV, I8PtrTy,
-      LoopPredecessor->getTerminator());
-
-    // Note that LoopPredecessor might occur in the predecessor list multiple
-    // times, and we need to add it the right number of times.
-    for (pred_iterator PI = pred_begin(Header), PE = pred_end(Header);
-         PI != PE; ++PI) {
-      if (*PI != LoopPredecessor)
-        continue;
-
-      NewPHI->addIncoming(BasePtrStart, LoopPredecessor);
-    }
-
-    Instruction *InsPoint = &*Header->getFirstInsertionPt();
-    GetElementPtrInst *PtrInc = GetElementPtrInst::Create(
-        I8Ty, NewPHI, BasePtrIncSCEV->getValue(),
-        MemI->hasName() ? MemI->getName() + ".inc" : "", InsPoint);
-    PtrInc->setIsInBounds(IsPtrInBounds(BasePtr));
-    for (pred_iterator PI = pred_begin(Header), PE = pred_end(Header);
-         PI != PE; ++PI) {
-      if (*PI == LoopPredecessor)
-        continue;
-
-      NewPHI->addIncoming(PtrInc, *PI);
-    }
-
-    Instruction *NewBasePtr;
-    if (PtrInc->getType() != BasePtr->getType())
-      NewBasePtr = new BitCastInst(PtrInc, BasePtr->getType(),
-        PtrInc->hasName() ? PtrInc->getName() + ".cast" : "", InsPoint);
-    else
-      NewBasePtr = PtrInc;
-
-    if (Instruction *IDel = dyn_cast<Instruction>(BasePtr))
-      BBChanged.insert(IDel->getParent());
-    BasePtr->replaceAllUsesWith(NewBasePtr);
-    RecursivelyDeleteTriviallyDeadInstructions(BasePtr);
-
-    // Keep track of the replacement pointer values we've inserted so that we
-    // don't generate more pointer values than necessary.
-    SmallPtrSet<Value *, 16> NewPtrs;
-    NewPtrs.insert( NewBasePtr);
-
-    for (auto I = std::next(Buckets[i].Elements.begin()),
-         IE = Buckets[i].Elements.end(); I != IE; ++I) {
-      Value *Ptr = GetPointerOperand(I->Instr);
-      assert(Ptr && "No pointer operand");
-      if (NewPtrs.count(Ptr))
-        continue;
-
-      Instruction *RealNewPtr;
-      if (!I->Offset || I->Offset->getValue()->isZero()) {
-        RealNewPtr = NewBasePtr;
-      } else {
-        Instruction *PtrIP = dyn_cast<Instruction>(Ptr);
-        if (PtrIP && isa<Instruction>(NewBasePtr) &&
-            cast<Instruction>(NewBasePtr)->getParent() == PtrIP->getParent())
-          PtrIP = nullptr;
-        else if (isa<PHINode>(PtrIP))
-          PtrIP = &*PtrIP->getParent()->getFirstInsertionPt();
-        else if (!PtrIP)
-          PtrIP = I->Instr;
-
-        GetElementPtrInst *NewPtr = GetElementPtrInst::Create(
-            I8Ty, PtrInc, I->Offset->getValue(),
-            I->Instr->hasName() ? I->Instr->getName() + ".off" : "", PtrIP);
-        if (!PtrIP)
-          NewPtr->insertAfter(cast<Instruction>(PtrInc));
-        NewPtr->setIsInBounds(IsPtrInBounds(Ptr));
-        RealNewPtr = NewPtr;
+  // Check if a load/store has update form. This lambda is used by function
+  // collectCandidates which can collect candidates for types defined by lambda.
+  auto isUpdateFormCandidate = [&] (const Instruction *I,
+                                    const Value *PtrValue) {
+    assert((PtrValue && I) && "Invalid parameter!");
+    // There are no update forms for Altivec vector load/stores.
+    if (ST && ST->hasAltivec() &&
+        PtrValue->getType()->getPointerElementType()->isVectorTy())
+      return false;
+    // See getPreIndexedAddressParts, the displacement for LDU/STDU has to
+    // be 4's multiple (DS-form). For i64 loads/stores when the displacement
+    // fits in a 16-bit signed field but isn't a multiple of 4, it will be
+    // useless and possible to break some original well-form addressing mode
+    // to make this pre-inc prep for it.
+    if (PtrValue->getType()->getPointerElementType()->isIntegerTy(64)) {
+      const SCEV *LSCEV = SE->getSCEVAtScope(const_cast<Value *>(PtrValue), L);
+      const SCEVAddRecExpr *LARSCEV = dyn_cast<SCEVAddRecExpr>(LSCEV);
+      if (!LARSCEV || LARSCEV->getLoop() != L)
+        return false;
+      if (const SCEVConstant *StepConst =
+              dyn_cast<SCEVConstant>(LARSCEV->getStepRecurrence(*SE))) {
+        const APInt &ConstInt = StepConst->getValue()->getValue();
+        if (ConstInt.isSignedIntN(16) && ConstInt.srem(4) != 0)
+          return false;
       }
-
-      if (Instruction *IDel = dyn_cast<Instruction>(Ptr))
-        BBChanged.insert(IDel->getParent());
-
-      Instruction *ReplNewPtr;
-      if (Ptr->getType() != RealNewPtr->getType()) {
-        ReplNewPtr = new BitCastInst(RealNewPtr, Ptr->getType(),
-          Ptr->hasName() ? Ptr->getName() + ".cast" : "");
-        ReplNewPtr->insertAfter(RealNewPtr);
-      } else
-        ReplNewPtr = RealNewPtr;
-
-      Ptr->replaceAllUsesWith(ReplNewPtr);
-      RecursivelyDeleteTriviallyDeadInstructions(Ptr);
-
-      NewPtrs.insert(RealNewPtr);
     }
+    return true;
+  };
 
-    MadeChange = true;
-  }
+  // Collect buckets of comparable addresses used by loads, stores and prefetch
+  // intrinsic for update form.
+  SmallVector<Bucket, 16> UpdateFormBuckets =
+      collectCandidates(L, isUpdateFormCandidate, MaxVars);
 
-  for (Loop::block_iterator I = L->block_begin(), IE = L->block_end();
-       I != IE; ++I) {
-    if (BBChanged.count(*I))
-      DeleteDeadPHIs(*I);
-  }
+  // Prepare for update form.
+  if (!UpdateFormBuckets.empty())
+    MadeChange |= updateFormPrep(L, UpdateFormBuckets);
 
   return MadeChange;
 }
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index 6aa7528634d3f..b63f008ce866d 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -145,7 +145,8 @@ void PPCSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (isDarwin())
     HasLazyResolverStubs = true;
 
-  if (TargetTriple.isOSNetBSD() || TargetTriple.isOSOpenBSD() ||
+  if ((TargetTriple.isOSFreeBSD() && TargetTriple.getOSMajorVersion() >= 13) ||
+      TargetTriple.isOSNetBSD() || TargetTriple.isOSOpenBSD() ||
       TargetTriple.isMusl())
     SecurePlt = true;
 
@@ -228,18 +229,13 @@ bool PPCSubtarget::enableSubRegLiveness() const {
   return UseSubRegLiveness;
 }
 
-unsigned char
-PPCSubtarget::classifyGlobalReference(const GlobalValue *GV) const {
-  // Note that currently we don't generate non-pic references.
-  // If a caller wants that, this will have to be updated.
-
+bool PPCSubtarget::isGVIndirectSymbol(const GlobalValue *GV) const {
   // Large code model always uses the TOC even for local symbols.
   if (TM.getCodeModel() == CodeModel::Large)
-    return PPCII::MO_PIC_FLAG | PPCII::MO_NLP_FLAG;
-
+    return true;
   if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
-    return PPCII::MO_PIC_FLAG;
-  return PPCII::MO_PIC_FLAG | PPCII::MO_NLP_FLAG;
+    return false;
+  return true;
 }
 
 bool PPCSubtarget::isELFv2ABI() const { return TM.isELFv2ABI(); }
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index 35bb90c6203f7..0ac0bbe2edec9 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -344,9 +344,8 @@ class PPCSubtarget : public PPCGenSubtargetInfo {
 
   bool enableSubRegLiveness() const override;
 
-  /// classifyGlobalReference - Classify a global variable reference for the
-  /// current subtarget accourding to how we should reference it.
-  unsigned char classifyGlobalReference(const GlobalValue *GV) const;
+  /// True if the GV will be accessed via an indirect symbol.
+  bool isGVIndirectSymbol(const GlobalValue *GV) const;
 
   bool isXRaySupported() const override { return IsPPC64 && IsLittleEndian; }
 };
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index 587725db7ea5b..300ba8dc675c9 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -374,8 +374,8 @@ struct RISCVOperand : public MCParsedAsmOperand {
       return true;
     // Given only Imm, ensuring that the actually specified constant is either
     // a signed or unsigned 64-bit number is unfortunately impossible.
-    bool IsInRange = isRV64() ? true : isInt<32>(Imm) || isUInt<32>(Imm);
-    return IsConstantImm && IsInRange && VK == RISCVMCExpr::VK_RISCV_None;
+    return IsConstantImm && VK == RISCVMCExpr::VK_RISCV_None &&
+           (isRV64() || (isInt<32>(Imm) || isUInt<32>(Imm)));
   }
 
   bool isUImmLog2XLen() const {
@@ -741,46 +741,9 @@ struct RISCVOperand : public MCParsedAsmOperand {
 #define GET_MATCHER_IMPLEMENTATION
 #include "RISCVGenAsmMatcher.inc"
 
-// Return the matching FPR64 register for the given FPR32.
-// FIXME: Ideally this function could be removed in favour of using
-// information from TableGen.
-static Register convertFPR32ToFPR64(Register Reg) {
-  switch (Reg) {
-  default:
-    llvm_unreachable("Not a recognised FPR32 register");
-  case RISCV::F0_32: return RISCV::F0_64;
-  case RISCV::F1_32: return RISCV::F1_64;
-  case RISCV::F2_32: return RISCV::F2_64;
-  case RISCV::F3_32: return RISCV::F3_64;
-  case RISCV::F4_32: return RISCV::F4_64;
-  case RISCV::F5_32: return RISCV::F5_64;
-  case RISCV::F6_32: return RISCV::F6_64;
-  case RISCV::F7_32: return RISCV::F7_64;
-  case RISCV::F8_32: return RISCV::F8_64;
-  case RISCV::F9_32: return RISCV::F9_64;
-  case RISCV::F10_32: return RISCV::F10_64;
-  case RISCV::F11_32: return RISCV::F11_64;
-  case RISCV::F12_32: return RISCV::F12_64;
-  case RISCV::F13_32: return RISCV::F13_64;
-  case RISCV::F14_32: return RISCV::F14_64;
-  case RISCV::F15_32: return RISCV::F15_64;
-  case RISCV::F16_32: return RISCV::F16_64;
-  case RISCV::F17_32: return RISCV::F17_64;
-  case RISCV::F18_32: return RISCV::F18_64;
-  case RISCV::F19_32: return RISCV::F19_64;
-  case RISCV::F20_32: return RISCV::F20_64;
-  case RISCV::F21_32: return RISCV::F21_64;
-  case RISCV::F22_32: return RISCV::F22_64;
-  case RISCV::F23_32: return RISCV::F23_64;
-  case RISCV::F24_32: return RISCV::F24_64;
-  case RISCV::F25_32: return RISCV::F25_64;
-  case RISCV::F26_32: return RISCV::F26_64;
-  case RISCV::F27_32: return RISCV::F27_64;
-  case RISCV::F28_32: return RISCV::F28_64;
-  case RISCV::F29_32: return RISCV::F29_64;
-  case RISCV::F30_32: return RISCV::F30_64;
-  case RISCV::F31_32: return RISCV::F31_64;
-  }
+static Register convertFPR64ToFPR32(Register Reg) {
+  assert(Reg >= RISCV::F0_D && Reg <= RISCV::F31_D && "Invalid register");
+  return Reg - RISCV::F0_D + RISCV::F0_F;
 }
 
 unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
@@ -790,16 +753,16 @@ unsigned RISCVAsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp,
     return Match_InvalidOperand;
 
   Register Reg = Op.getReg();
-  bool IsRegFPR32 =
-      RISCVMCRegisterClasses[RISCV::FPR32RegClassID].contains(Reg);
-  bool IsRegFPR32C =
-      RISCVMCRegisterClasses[RISCV::FPR32CRegClassID].contains(Reg);
+  bool IsRegFPR64 =
+      RISCVMCRegisterClasses[RISCV::FPR64RegClassID].contains(Reg);
+  bool IsRegFPR64C =
+      RISCVMCRegisterClasses[RISCV::FPR64CRegClassID].contains(Reg);
 
   // As the parser couldn't differentiate an FPR32 from an FPR64, coerce the
-  // register from FPR32 to FPR64 or FPR32C to FPR64C if necessary.
-  if ((IsRegFPR32 && Kind == MCK_FPR64) ||
-      (IsRegFPR32C && Kind == MCK_FPR64C)) {
-    Op.Reg.RegNum = convertFPR32ToFPR64(Reg);
+  // register from FPR64 to FPR32 or FPR64C to FPR32C if necessary.
+  if ((IsRegFPR64 && Kind == MCK_FPR32) ||
+      (IsRegFPR64C && Kind == MCK_FPR32C)) {
+    Op.Reg.RegNum = convertFPR64ToFPR32(Reg);
     return Match_Success;
   }
   return Match_InvalidOperand;
@@ -986,11 +949,16 @@ bool RISCVAsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
 static bool matchRegisterNameHelper(bool IsRV32E, Register &RegNo,
                                     StringRef Name) {
   RegNo = MatchRegisterName(Name);
-  if (RegNo == 0)
+  // The 32- and 64-bit FPRs have the same asm name. Check that the initial
+  // match always matches the 64-bit variant, and not the 32-bit one.
+  assert(!(RegNo >= RISCV::F0_F && RegNo <= RISCV::F31_F));
+  // The default FPR register class is based on the tablegen enum ordering.
+  static_assert(RISCV::F0_D < RISCV::F0_F, "FPR matching must be updated");
+  if (RegNo == RISCV::NoRegister)
     RegNo = MatchRegisterAltName(Name);
   if (IsRV32E && RegNo >= RISCV::X16 && RegNo <= RISCV::X31)
-    RegNo = 0;
-  return RegNo == 0;
+    RegNo = RISCV::NoRegister;
+  return RegNo == RISCV::NoRegister;
 }
 
 bool RISCVAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
@@ -1036,7 +1004,7 @@ OperandMatchResultTy RISCVAsmParser::parseRegister(OperandVector &Operands,
     Register RegNo;
     matchRegisterNameHelper(isRV32E(), RegNo, Name);
 
-    if (RegNo == 0) {
+    if (RegNo == RISCV::NoRegister) {
       if (HadParens)
         getLexer().UnLex(LParen);
       return MatchOperand_NoMatch;
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index 4b85955a97be7..15943ba42156f 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -57,17 +57,6 @@ extern "C" void LLVMInitializeRISCVDisassembler() {
                                          createRISCVDisassembler);
 }
 
-static const Register GPRDecoderTable[] = {
-  RISCV::X0,  RISCV::X1,  RISCV::X2,  RISCV::X3,
-  RISCV::X4,  RISCV::X5,  RISCV::X6,  RISCV::X7,
-  RISCV::X8,  RISCV::X9,  RISCV::X10, RISCV::X11,
-  RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15,
-  RISCV::X16, RISCV::X17, RISCV::X18, RISCV::X19,
-  RISCV::X20, RISCV::X21, RISCV::X22, RISCV::X23,
-  RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27,
-  RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31
-};
-
 static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, uint64_t RegNo,
                                            uint64_t Address,
                                            const void *Decoder) {
@@ -77,38 +66,21 @@ static DecodeStatus DecodeGPRRegisterClass(MCInst &Inst, uint64_t RegNo,
           .getFeatureBits();
   bool IsRV32E = FeatureBits[RISCV::FeatureRV32E];
 
-  if (RegNo > array_lengthof(GPRDecoderTable) || (IsRV32E && RegNo > 15))
+  if (RegNo >= 32 || (IsRV32E && RegNo >= 16))
     return MCDisassembler::Fail;
 
-  // We must define our own mapping from RegNo to register identifier.
-  // Accessing index RegNo in the register class will work in the case that
-  // registers were added in ascending order, but not in general.
-  Register Reg = GPRDecoderTable[RegNo];
+  Register Reg = RISCV::X0 + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
-static const Register FPR32DecoderTable[] = {
-  RISCV::F0_32,  RISCV::F1_32,  RISCV::F2_32,  RISCV::F3_32,
-  RISCV::F4_32,  RISCV::F5_32,  RISCV::F6_32,  RISCV::F7_32,
-  RISCV::F8_32,  RISCV::F9_32,  RISCV::F10_32, RISCV::F11_32,
-  RISCV::F12_32, RISCV::F13_32, RISCV::F14_32, RISCV::F15_32,
-  RISCV::F16_32, RISCV::F17_32, RISCV::F18_32, RISCV::F19_32,
-  RISCV::F20_32, RISCV::F21_32, RISCV::F22_32, RISCV::F23_32,
-  RISCV::F24_32, RISCV::F25_32, RISCV::F26_32, RISCV::F27_32,
-  RISCV::F28_32, RISCV::F29_32, RISCV::F30_32, RISCV::F31_32
-};
-
 static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, uint64_t RegNo,
                                              uint64_t Address,
                                              const void *Decoder) {
-  if (RegNo > array_lengthof(FPR32DecoderTable))
+  if (RegNo >= 32)
     return MCDisassembler::Fail;
 
-  // We must define our own mapping from RegNo to register identifier.
-  // Accessing index RegNo in the register class will work in the case that
-  // registers were added in ascending order, but not in general.
-  Register Reg = FPR32DecoderTable[RegNo];
+  Register Reg = RISCV::F0_F + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
@@ -116,35 +88,21 @@ static DecodeStatus DecodeFPR32RegisterClass(MCInst &Inst, uint64_t RegNo,
 static DecodeStatus DecodeFPR32CRegisterClass(MCInst &Inst, uint64_t RegNo,
                                               uint64_t Address,
                                               const void *Decoder) {
-  if (RegNo > 8) {
+  if (RegNo >= 8) {
     return MCDisassembler::Fail;
   }
-  Register Reg = FPR32DecoderTable[RegNo + 8];
+  Register Reg = RISCV::F8_F + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
 
-static const Register FPR64DecoderTable[] = {
-  RISCV::F0_64,  RISCV::F1_64,  RISCV::F2_64,  RISCV::F3_64,
-  RISCV::F4_64,  RISCV::F5_64,  RISCV::F6_64,  RISCV::F7_64,
-  RISCV::F8_64,  RISCV::F9_64,  RISCV::F10_64, RISCV::F11_64,
-  RISCV::F12_64, RISCV::F13_64, RISCV::F14_64, RISCV::F15_64,
-  RISCV::F16_64, RISCV::F17_64, RISCV::F18_64, RISCV::F19_64,
-  RISCV::F20_64, RISCV::F21_64, RISCV::F22_64, RISCV::F23_64,
-  RISCV::F24_64, RISCV::F25_64, RISCV::F26_64, RISCV::F27_64,
-  RISCV::F28_64, RISCV::F29_64, RISCV::F30_64, RISCV::F31_64
-};
-
 static DecodeStatus DecodeFPR64RegisterClass(MCInst &Inst, uint64_t RegNo,
                                              uint64_t Address,
                                              const void *Decoder) {
-  if (RegNo > array_lengthof(FPR64DecoderTable))
+  if (RegNo >= 32)
     return MCDisassembler::Fail;
 
-  // We must define our own mapping from RegNo to register identifier.
-  // Accessing index RegNo in the register class will work in the case that
-  // registers were added in ascending order, but not in general.
-  Register Reg = FPR64DecoderTable[RegNo];
+  Register Reg = RISCV::F0_D + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
@@ -152,10 +110,10 @@ static DecodeStatus DecodeFPR64RegisterClass(MCInst &Inst, uint64_t RegNo,
 static DecodeStatus DecodeFPR64CRegisterClass(MCInst &Inst, uint64_t RegNo,
                                               uint64_t Address,
                                               const void *Decoder) {
-  if (RegNo > 8) {
+  if (RegNo >= 8) {
     return MCDisassembler::Fail;
   }
-  Register Reg = FPR64DecoderTable[RegNo + 8];
+  Register Reg = RISCV::F8_D + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
@@ -183,10 +141,10 @@ static DecodeStatus DecodeGPRNoX0X2RegisterClass(MCInst &Inst, uint64_t RegNo,
 static DecodeStatus DecodeGPRCRegisterClass(MCInst &Inst, uint64_t RegNo,
                                             uint64_t Address,
                                             const void *Decoder) {
-  if (RegNo > 8)
+  if (RegNo >= 8)
     return MCDisassembler::Fail;
 
-  Register Reg = GPRDecoderTable[RegNo + 8];
+  Register Reg = RISCV::X8 + RegNo;
   Inst.addOperand(MCOperand::createReg(Reg));
   return MCDisassembler::Success;
 }
diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.td b/llvm/lib/Target/RISCV/RISCVCallingConv.td
index db13e6e8becaa..025454f8fcca6 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.td
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.td
@@ -18,11 +18,11 @@ def CSR_ILP32_LP64
 
 def CSR_ILP32F_LP64F
     : CalleeSavedRegs<(add CSR_ILP32_LP64,
-                       F8_32, F9_32, (sequence "F%u_32", 18, 27))>;
+                       F8_F, F9_F, (sequence "F%u_F", 18, 27))>;
 
 def CSR_ILP32D_LP64D
     : CalleeSavedRegs<(add CSR_ILP32_LP64,
-                       F8_64, F9_64, (sequence "F%u_64", 18, 27))>;
+                       F8_D, F9_D, (sequence "F%u_D", 18, 27))>;
 
 // Needed for implementation of RISCVRegisterInfo::getNoPreservedMask()
 def CSR_NoRegs : CalleeSavedRegs<(add)>;
@@ -43,12 +43,12 @@ def CSR_XLEN_F32_Interrupt: CalleeSavedRegs<(add X1,
     (sequence "X%u", 12, 17),
     (sequence "X%u", 18, 27),
     (sequence "X%u", 28, 31),
-    (sequence "F%u_32", 0, 7),
-    (sequence "F%u_32", 10, 11),
-    (sequence "F%u_32", 12, 17),
-    (sequence "F%u_32", 28, 31),
-    (sequence "F%u_32", 8, 9),
-    (sequence "F%u_32", 18, 27))>;
+    (sequence "F%u_F", 0, 7),
+    (sequence "F%u_F", 10, 11),
+    (sequence "F%u_F", 12, 17),
+    (sequence "F%u_F", 28, 31),
+    (sequence "F%u_F", 8, 9),
+    (sequence "F%u_F", 18, 27))>;
 
 // Same as CSR_Interrupt, but including all 64-bit FP registers.
 def CSR_XLEN_F64_Interrupt: CalleeSavedRegs<(add X1,
@@ -57,9 +57,9 @@ def CSR_XLEN_F64_Interrupt: CalleeSavedRegs<(add X1,
     (sequence "X%u", 12, 17),
     (sequence "X%u", 18, 27),
     (sequence "X%u", 28, 31),
-    (sequence "F%u_64", 0, 7),
-    (sequence "F%u_64", 10, 11),
-    (sequence "F%u_64", 12, 17),
-    (sequence "F%u_64", 28, 31),
-    (sequence "F%u_64", 8, 9),
-    (sequence "F%u_64", 18, 27))>;
+    (sequence "F%u_D", 0, 7),
+    (sequence "F%u_D", 10, 11),
+    (sequence "F%u_D", 12, 17),
+    (sequence "F%u_D", 28, 31),
+    (sequence "F%u_D", 8, 9),
+    (sequence "F%u_D", 18, 27))>;
diff --git a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
index 8439278b4ed53..1a12d9177d2a6 100644
--- a/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelDAGToDAG.cpp
@@ -68,7 +68,7 @@ static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, int64_t Imm,
   RISCVMatInt::InstSeq Seq;
   RISCVMatInt::generateInstSeq(Imm, XLenVT == MVT::i64, Seq);
 
-  SDNode *Result;
+  SDNode *Result = nullptr;
   SDValue SrcReg = CurDAG->getRegister(RISCV::X0, XLenVT);
   for (RISCVMatInt::Inst &Inst : Seq) {
     SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, XLenVT);
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 371e19c2ee0fa..5c20057c9993e 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -198,7 +198,7 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   setBooleanContents(ZeroOrOneBooleanContent);
 
   // Function alignments.
-  const llvm::Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
+  const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
   setMinFunctionAlignment(FunctionAlignment);
   setPrefFunctionAlignment(FunctionAlignment);
 
@@ -1389,12 +1389,12 @@ static const MCPhysReg ArgGPRs[] = {
   RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
 };
 static const MCPhysReg ArgFPR32s[] = {
-  RISCV::F10_32, RISCV::F11_32, RISCV::F12_32, RISCV::F13_32,
-  RISCV::F14_32, RISCV::F15_32, RISCV::F16_32, RISCV::F17_32
+  RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
+  RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
 };
 static const MCPhysReg ArgFPR64s[] = {
-  RISCV::F10_64, RISCV::F11_64, RISCV::F12_64, RISCV::F13_64,
-  RISCV::F14_64, RISCV::F15_64, RISCV::F16_64, RISCV::F17_64
+  RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
+  RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
 };
 
 // Pass a 2*XLEN argument that has been split into two XLEN values through
@@ -1903,8 +1903,7 @@ SDValue RISCVTargetLowering::LowerFormalArguments(
     // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
     // offsets to even-numbered registered remain 2*XLEN-aligned.
     if (Idx % 2) {
-      FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes,
-                                 true);
+      MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
       VarArgsSaveSize += XLenInBytes;
     }
 
@@ -2108,6 +2107,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
             DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
       } else {
         // Second half of f64 is passed in another GPR.
+        assert(RegLo < RISCV::X31 && "Invalid register pair");
         Register RegHigh = RegLo + 1;
         RegsToPass.push_back(std::make_pair(RegHigh, Hi));
       }
@@ -2324,6 +2324,7 @@ RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       SDValue Lo = SplitF64.getValue(0);
       SDValue Hi = SplitF64.getValue(1);
       Register RegLo = VA.getLocReg();
+      assert(RegLo < RISCV::X31 && "Invalid register pair");
       Register RegHi = RegLo + 1;
       Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
       Glue = Chain.getValue(1);
@@ -2511,38 +2512,38 @@ RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
   if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) {
     std::pair<Register, Register> FReg =
         StringSwitch<std::pair<Register, Register>>(Constraint.lower())
-            .Cases("{f0}", "{ft0}", {RISCV::F0_32, RISCV::F0_64})
-            .Cases("{f1}", "{ft1}", {RISCV::F1_32, RISCV::F1_64})
-            .Cases("{f2}", "{ft2}", {RISCV::F2_32, RISCV::F2_64})
-            .Cases("{f3}", "{ft3}", {RISCV::F3_32, RISCV::F3_64})
-            .Cases("{f4}", "{ft4}", {RISCV::F4_32, RISCV::F4_64})
-            .Cases("{f5}", "{ft5}", {RISCV::F5_32, RISCV::F5_64})
-            .Cases("{f6}", "{ft6}", {RISCV::F6_32, RISCV::F6_64})
-            .Cases("{f7}", "{ft7}", {RISCV::F7_32, RISCV::F7_64})
-            .Cases("{f8}", "{fs0}", {RISCV::F8_32, RISCV::F8_64})
-            .Cases("{f9}", "{fs1}", {RISCV::F9_32, RISCV::F9_64})
-            .Cases("{f10}", "{fa0}", {RISCV::F10_32, RISCV::F10_64})
-            .Cases("{f11}", "{fa1}", {RISCV::F11_32, RISCV::F11_64})
-            .Cases("{f12}", "{fa2}", {RISCV::F12_32, RISCV::F12_64})
-            .Cases("{f13}", "{fa3}", {RISCV::F13_32, RISCV::F13_64})
-            .Cases("{f14}", "{fa4}", {RISCV::F14_32, RISCV::F14_64})
-            .Cases("{f15}", "{fa5}", {RISCV::F15_32, RISCV::F15_64})
-            .Cases("{f16}", "{fa6}", {RISCV::F16_32, RISCV::F16_64})
-            .Cases("{f17}", "{fa7}", {RISCV::F17_32, RISCV::F17_64})
-            .Cases("{f18}", "{fs2}", {RISCV::F18_32, RISCV::F18_64})
-            .Cases("{f19}", "{fs3}", {RISCV::F19_32, RISCV::F19_64})
-            .Cases("{f20}", "{fs4}", {RISCV::F20_32, RISCV::F20_64})
-            .Cases("{f21}", "{fs5}", {RISCV::F21_32, RISCV::F21_64})
-            .Cases("{f22}", "{fs6}", {RISCV::F22_32, RISCV::F22_64})
-            .Cases("{f23}", "{fs7}", {RISCV::F23_32, RISCV::F23_64})
-            .Cases("{f24}", "{fs8}", {RISCV::F24_32, RISCV::F24_64})
-            .Cases("{f25}", "{fs9}", {RISCV::F25_32, RISCV::F25_64})
-            .Cases("{f26}", "{fs10}", {RISCV::F26_32, RISCV::F26_64})
-            .Cases("{f27}", "{fs11}", {RISCV::F27_32, RISCV::F27_64})
-            .Cases("{f28}", "{ft8}", {RISCV::F28_32, RISCV::F28_64})
-            .Cases("{f29}", "{ft9}", {RISCV::F29_32, RISCV::F29_64})
-            .Cases("{f30}", "{ft10}", {RISCV::F30_32, RISCV::F30_64})
-            .Cases("{f31}", "{ft11}", {RISCV::F31_32, RISCV::F31_64})
+            .Cases("{f0}", "{ft0}", {RISCV::F0_F, RISCV::F0_D})
+            .Cases("{f1}", "{ft1}", {RISCV::F1_F, RISCV::F1_D})
+            .Cases("{f2}", "{ft2}", {RISCV::F2_F, RISCV::F2_D})
+            .Cases("{f3}", "{ft3}", {RISCV::F3_F, RISCV::F3_D})
+            .Cases("{f4}", "{ft4}", {RISCV::F4_F, RISCV::F4_D})
+            .Cases("{f5}", "{ft5}", {RISCV::F5_F, RISCV::F5_D})
+            .Cases("{f6}", "{ft6}", {RISCV::F6_F, RISCV::F6_D})
+            .Cases("{f7}", "{ft7}", {RISCV::F7_F, RISCV::F7_D})
+            .Cases("{f8}", "{fs0}", {RISCV::F8_F, RISCV::F8_D})
+            .Cases("{f9}", "{fs1}", {RISCV::F9_F, RISCV::F9_D})
+            .Cases("{f10}", "{fa0}", {RISCV::F10_F, RISCV::F10_D})
+            .Cases("{f11}", "{fa1}", {RISCV::F11_F, RISCV::F11_D})
+            .Cases("{f12}", "{fa2}", {RISCV::F12_F, RISCV::F12_D})
+            .Cases("{f13}", "{fa3}", {RISCV::F13_F, RISCV::F13_D})
+            .Cases("{f14}", "{fa4}", {RISCV::F14_F, RISCV::F14_D})
+            .Cases("{f15}", "{fa5}", {RISCV::F15_F, RISCV::F15_D})
+            .Cases("{f16}", "{fa6}", {RISCV::F16_F, RISCV::F16_D})
+            .Cases("{f17}", "{fa7}", {RISCV::F17_F, RISCV::F17_D})
+            .Cases("{f18}", "{fs2}", {RISCV::F18_F, RISCV::F18_D})
+            .Cases("{f19}", "{fs3}", {RISCV::F19_F, RISCV::F19_D})
+            .Cases("{f20}", "{fs4}", {RISCV::F20_F, RISCV::F20_D})
+            .Cases("{f21}", "{fs5}", {RISCV::F21_F, RISCV::F21_D})
+            .Cases("{f22}", "{fs6}", {RISCV::F22_F, RISCV::F22_D})
+            .Cases("{f23}", "{fs7}", {RISCV::F23_F, RISCV::F23_D})
+            .Cases("{f24}", "{fs8}", {RISCV::F24_F, RISCV::F24_D})
+            .Cases("{f25}", "{fs9}", {RISCV::F25_F, RISCV::F25_D})
+            .Cases("{f26}", "{fs10}", {RISCV::F26_F, RISCV::F26_D})
+            .Cases("{f27}", "{fs11}", {RISCV::F27_F, RISCV::F27_D})
+            .Cases("{f28}", "{ft8}", {RISCV::F28_F, RISCV::F28_D})
+            .Cases("{f29}", "{ft9}", {RISCV::F29_F, RISCV::F29_D})
+            .Cases("{f30}", "{ft10}", {RISCV::F30_F, RISCV::F30_D})
+            .Cases("{f31}", "{ft11}", {RISCV::F31_F, RISCV::F31_D})
             .Default({RISCV::NoRegister, RISCV::NoRegister});
     if (FReg.first != RISCV::NoRegister)
       return Subtarget.hasStdExtD()
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
index 1484ba9f0687d..38ba3f9fb24ed 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoA.td
@@ -214,12 +214,12 @@ class PseudoMaskedAMOUMinUMax
 }
 
 class PseudoMaskedAMOPat<Intrinsic intrin, Pseudo AMOInst>
-    : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, imm:$ordering),
+    : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, timm:$ordering),
           (AMOInst GPR:$addr, GPR:$incr, GPR:$mask, imm:$ordering)>;
 
 class PseudoMaskedAMOMinMaxPat<Intrinsic intrin, Pseudo AMOInst>
     : Pat<(intrin GPR:$addr, GPR:$incr, GPR:$mask, GPR:$shiftamt,
-           imm:$ordering),
+           timm:$ordering),
           (AMOInst GPR:$addr, GPR:$incr, GPR:$mask, GPR:$shiftamt,
            imm:$ordering)>;
 
@@ -288,7 +288,7 @@ def PseudoMaskedCmpXchg32
 }
 
 def : Pat<(int_riscv_masked_cmpxchg_i32
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering),
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering),
           (PseudoMaskedCmpXchg32
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering)>;
 
@@ -365,7 +365,7 @@ def PseudoCmpXchg64 : PseudoCmpXchg;
 defm : PseudoCmpXchgPat<"atomic_cmp_swap_64", PseudoCmpXchg64>;
 
 def : Pat<(int_riscv_masked_cmpxchg_i64
-            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering),
+            GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, timm:$ordering),
           (PseudoMaskedCmpXchg32
             GPR:$addr, GPR:$cmpval, GPR:$newval, GPR:$mask, imm:$ordering)>;
 } // Predicates = [HasStdExtA, IsRV64]
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index aa6d06d3d41a7..66557687c0b62 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -26,6 +26,15 @@
 
 using namespace llvm;
 
+static_assert(RISCV::X1 == RISCV::X0 + 1, "Register list not consecutive");
+static_assert(RISCV::X31 == RISCV::X0 + 31, "Register list not consecutive");
+static_assert(RISCV::F1_F == RISCV::F0_F + 1, "Register list not consecutive");
+static_assert(RISCV::F31_F == RISCV::F0_F + 31,
+              "Register list not consecutive");
+static_assert(RISCV::F1_D == RISCV::F0_D + 1, "Register list not consecutive");
+static_assert(RISCV::F31_D == RISCV::F0_D + 31,
+              "Register list not consecutive");
+
 RISCVRegisterInfo::RISCVRegisterInfo(unsigned HwMode)
     : RISCVGenRegisterInfo(RISCV::X1, /*DwarfFlavour*/0, /*EHFlavor*/0,
                            /*PC*/0, HwMode) {}
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
index 9b039212b4d6f..82b37afd0805f 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.td
@@ -165,41 +165,41 @@ def SP : RegisterClass<"RISCV", [XLenVT], 32, (add X2)> {
 
 // Floating point registers
 let RegAltNameIndices = [ABIRegAltName] in {
-  def F0_32  : RISCVReg32<0, "f0", ["ft0"]>, DwarfRegNum<[32]>;
-  def F1_32  : RISCVReg32<1, "f1", ["ft1"]>, DwarfRegNum<[33]>;
-  def F2_32  : RISCVReg32<2, "f2", ["ft2"]>, DwarfRegNum<[34]>;
-  def F3_32  : RISCVReg32<3, "f3", ["ft3"]>, DwarfRegNum<[35]>;
-  def F4_32  : RISCVReg32<4, "f4", ["ft4"]>, DwarfRegNum<[36]>;
-  def F5_32  : RISCVReg32<5, "f5", ["ft5"]>, DwarfRegNum<[37]>;
-  def F6_32  : RISCVReg32<6, "f6", ["ft6"]>, DwarfRegNum<[38]>;
-  def F7_32  : RISCVReg32<7, "f7", ["ft7"]>, DwarfRegNum<[39]>;
-  def F8_32  : RISCVReg32<8, "f8", ["fs0"]>, DwarfRegNum<[40]>;
-  def F9_32  : RISCVReg32<9, "f9", ["fs1"]>, DwarfRegNum<[41]>;
-  def F10_32 : RISCVReg32<10,"f10", ["fa0"]>, DwarfRegNum<[42]>;
-  def F11_32 : RISCVReg32<11,"f11", ["fa1"]>, DwarfRegNum<[43]>;
-  def F12_32 : RISCVReg32<12,"f12", ["fa2"]>, DwarfRegNum<[44]>;
-  def F13_32 : RISCVReg32<13,"f13", ["fa3"]>, DwarfRegNum<[45]>;
-  def F14_32 : RISCVReg32<14,"f14", ["fa4"]>, DwarfRegNum<[46]>;
-  def F15_32 : RISCVReg32<15,"f15", ["fa5"]>, DwarfRegNum<[47]>;
-  def F16_32 : RISCVReg32<16,"f16", ["fa6"]>, DwarfRegNum<[48]>;
-  def F17_32 : RISCVReg32<17,"f17", ["fa7"]>, DwarfRegNum<[49]>;
-  def F18_32 : RISCVReg32<18,"f18", ["fs2"]>, DwarfRegNum<[50]>;
-  def F19_32 : RISCVReg32<19,"f19", ["fs3"]>, DwarfRegNum<[51]>;
-  def F20_32 : RISCVReg32<20,"f20", ["fs4"]>, DwarfRegNum<[52]>;
-  def F21_32 : RISCVReg32<21,"f21", ["fs5"]>, DwarfRegNum<[53]>;
-  def F22_32 : RISCVReg32<22,"f22", ["fs6"]>, DwarfRegNum<[54]>;
-  def F23_32 : RISCVReg32<23,"f23", ["fs7"]>, DwarfRegNum<[55]>;
-  def F24_32 : RISCVReg32<24,"f24", ["fs8"]>, DwarfRegNum<[56]>;
-  def F25_32 : RISCVReg32<25,"f25", ["fs9"]>, DwarfRegNum<[57]>;
-  def F26_32 : RISCVReg32<26,"f26", ["fs10"]>, DwarfRegNum<[58]>;
-  def F27_32 : RISCVReg32<27,"f27", ["fs11"]>, DwarfRegNum<[59]>;
-  def F28_32 : RISCVReg32<28,"f28", ["ft8"]>, DwarfRegNum<[60]>;
-  def F29_32 : RISCVReg32<29,"f29", ["ft9"]>, DwarfRegNum<[61]>;
-  def F30_32 : RISCVReg32<30,"f30", ["ft10"]>, DwarfRegNum<[62]>;
-  def F31_32 : RISCVReg32<31,"f31", ["ft11"]>, DwarfRegNum<[63]>;
+  def F0_F  : RISCVReg32<0, "f0", ["ft0"]>, DwarfRegNum<[32]>;
+  def F1_F  : RISCVReg32<1, "f1", ["ft1"]>, DwarfRegNum<[33]>;
+  def F2_F  : RISCVReg32<2, "f2", ["ft2"]>, DwarfRegNum<[34]>;
+  def F3_F  : RISCVReg32<3, "f3", ["ft3"]>, DwarfRegNum<[35]>;
+  def F4_F  : RISCVReg32<4, "f4", ["ft4"]>, DwarfRegNum<[36]>;
+  def F5_F  : RISCVReg32<5, "f5", ["ft5"]>, DwarfRegNum<[37]>;
+  def F6_F  : RISCVReg32<6, "f6", ["ft6"]>, DwarfRegNum<[38]>;
+  def F7_F  : RISCVReg32<7, "f7", ["ft7"]>, DwarfRegNum<[39]>;
+  def F8_F  : RISCVReg32<8, "f8", ["fs0"]>, DwarfRegNum<[40]>;
+  def F9_F  : RISCVReg32<9, "f9", ["fs1"]>, DwarfRegNum<[41]>;
+  def F10_F : RISCVReg32<10,"f10", ["fa0"]>, DwarfRegNum<[42]>;
+  def F11_F : RISCVReg32<11,"f11", ["fa1"]>, DwarfRegNum<[43]>;
+  def F12_F : RISCVReg32<12,"f12", ["fa2"]>, DwarfRegNum<[44]>;
+  def F13_F : RISCVReg32<13,"f13", ["fa3"]>, DwarfRegNum<[45]>;
+  def F14_F : RISCVReg32<14,"f14", ["fa4"]>, DwarfRegNum<[46]>;
+  def F15_F : RISCVReg32<15,"f15", ["fa5"]>, DwarfRegNum<[47]>;
+  def F16_F : RISCVReg32<16,"f16", ["fa6"]>, DwarfRegNum<[48]>;
+  def F17_F : RISCVReg32<17,"f17", ["fa7"]>, DwarfRegNum<[49]>;
+  def F18_F : RISCVReg32<18,"f18", ["fs2"]>, DwarfRegNum<[50]>;
+  def F19_F : RISCVReg32<19,"f19", ["fs3"]>, DwarfRegNum<[51]>;
+  def F20_F : RISCVReg32<20,"f20", ["fs4"]>, DwarfRegNum<[52]>;
+  def F21_F : RISCVReg32<21,"f21", ["fs5"]>, DwarfRegNum<[53]>;
+  def F22_F : RISCVReg32<22,"f22", ["fs6"]>, DwarfRegNum<[54]>;
+  def F23_F : RISCVReg32<23,"f23", ["fs7"]>, DwarfRegNum<[55]>;
+  def F24_F : RISCVReg32<24,"f24", ["fs8"]>, DwarfRegNum<[56]>;
+  def F25_F : RISCVReg32<25,"f25", ["fs9"]>, DwarfRegNum<[57]>;
+  def F26_F : RISCVReg32<26,"f26", ["fs10"]>, DwarfRegNum<[58]>;
+  def F27_F : RISCVReg32<27,"f27", ["fs11"]>, DwarfRegNum<[59]>;
+  def F28_F : RISCVReg32<28,"f28", ["ft8"]>, DwarfRegNum<[60]>;
+  def F29_F : RISCVReg32<29,"f29", ["ft9"]>, DwarfRegNum<[61]>;
+  def F30_F : RISCVReg32<30,"f30", ["ft10"]>, DwarfRegNum<[62]>;
+  def F31_F : RISCVReg32<31,"f31", ["ft11"]>, DwarfRegNum<[63]>;
 
   foreach Index = 0-31 in {
-    def F#Index#_64 : RISCVReg64<!cast<RISCVReg32>("F"#Index#"_32")>,
+    def F#Index#_D : RISCVReg64<!cast<RISCVReg32>("F"#Index#"_F")>,
       DwarfRegNum<[!add(Index, 32)]>;
   }
 }
@@ -207,29 +207,29 @@ let RegAltNameIndices = [ABIRegAltName] in {
 // The order of registers represents the preferred allocation sequence,
 // meaning caller-save regs are listed before callee-save.
 def FPR32 : RegisterClass<"RISCV", [f32], 32, (add
-    (sequence "F%u_32", 0, 7),
-    (sequence "F%u_32", 10, 17),
-    (sequence "F%u_32", 28, 31),
-    (sequence "F%u_32", 8, 9),
-    (sequence "F%u_32", 18, 27)
+    (sequence "F%u_F", 0, 7),
+    (sequence "F%u_F", 10, 17),
+    (sequence "F%u_F", 28, 31),
+    (sequence "F%u_F", 8, 9),
+    (sequence "F%u_F", 18, 27)
 )>;
 
 def FPR32C : RegisterClass<"RISCV", [f32], 32, (add
-  (sequence "F%u_32", 10, 15),
-  (sequence "F%u_32", 8, 9)
+  (sequence "F%u_F", 10, 15),
+  (sequence "F%u_F", 8, 9)
 )>;
 
 // The order of registers represents the preferred allocation sequence,
 // meaning caller-save regs are listed before callee-save.
 def FPR64 : RegisterClass<"RISCV", [f64], 64, (add
-    (sequence "F%u_64", 0, 7),
-    (sequence "F%u_64", 10, 17),
-    (sequence "F%u_64", 28, 31),
-    (sequence "F%u_64", 8, 9),
-    (sequence "F%u_64", 18, 27)
+    (sequence "F%u_D", 0, 7),
+    (sequence "F%u_D", 10, 17),
+    (sequence "F%u_D", 28, 31),
+    (sequence "F%u_D", 8, 9),
+    (sequence "F%u_D", 18, 27)
 )>;
 
 def FPR64C : RegisterClass<"RISCV", [f64], 64, (add
-  (sequence "F%u_64", 10, 15),
-  (sequence "F%u_64", 8, 9)
+  (sequence "F%u_D", 10, 15),
+  (sequence "F%u_D", 8, 9)
 )>;
diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h
index fa19252f1f15f..7d0373a5253a8 100644
--- a/llvm/lib/Target/RISCV/RISCVSubtarget.h
+++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h
@@ -80,6 +80,7 @@ class RISCVSubtarget : public RISCVGenSubtargetInfo {
   const SelectionDAGTargetInfo *getSelectionDAGInfo() const override {
     return &TSInfo;
   }
+  bool enableMachineScheduler() const override { return true; }
   bool hasStdExtM() const { return HasStdExtM; }
   bool hasStdExtA() const { return HasStdExtA; }
   bool hasStdExtF() const { return HasStdExtF; }
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.cpp b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
index 660c5298d372d..4a2ba00ac6c21 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.cpp
@@ -1016,9 +1016,9 @@ SparcTargetLowering::LowerCall_32(TargetLowering::CallLoweringInfo &CLI,
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned SparcTargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                               SelectionDAG &DAG) const {
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+Register SparcTargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                                const MachineFunction &MF) const {
+  Register Reg = StringSwitch<unsigned>(RegName)
     .Case("i0", SP::I0).Case("i1", SP::I1).Case("i2", SP::I2).Case("i3", SP::I3)
     .Case("i4", SP::I4).Case("i5", SP::I5).Case("i6", SP::I6).Case("i7", SP::I7)
     .Case("o0", SP::O0).Case("o1", SP::O1).Case("o2", SP::O2).Case("o3", SP::O3)
@@ -1438,7 +1438,7 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
       setOperationAction(Op, MVT::v2i32, Expand);
     }
     // Truncating/extending stores/loads are also not supported.
-    for (MVT VT : MVT::integer_vector_valuetypes()) {
+    for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
       setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Expand);
       setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::v2i32, Expand);
       setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Expand);
@@ -1805,7 +1805,7 @@ SparcTargetLowering::SparcTargetLowering(const TargetMachine &TM,
 
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
 
-  setMinFunctionAlignment(llvm::Align(4));
+  setMinFunctionAlignment(Align(4));
 
   computeRegisterProperties(Subtarget->getRegisterInfo());
 }
@@ -2244,7 +2244,7 @@ SDValue SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UL : {
-    SDValue Mask   = DAG.getTargetConstant(1, DL, Result.getValueType());
+    SDValue Mask   = DAG.getConstant(1, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
     SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
@@ -2277,14 +2277,14 @@ SDValue SparcTargetLowering::LowerF128Compare(SDValue LHS, SDValue RHS,
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_LG :  {
-    SDValue Mask   = DAG.getTargetConstant(3, DL, Result.getValueType());
+    SDValue Mask   = DAG.getConstant(3, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
     SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_NE;
     return DAG.getNode(SPISD::CMPICC, DL, MVT::Glue, Result, RHS);
   }
   case SPCC::FCC_UE : {
-    SDValue Mask   = DAG.getTargetConstant(3, DL, Result.getValueType());
+    SDValue Mask   = DAG.getConstant(3, DL, Result.getValueType());
     Result = DAG.getNode(ISD::AND, DL, Result.getValueType(), Result, Mask);
     SDValue RHS    = DAG.getTargetConstant(0, DL, Result.getValueType());
     SPCC = SPCC::ICC_E;
diff --git a/llvm/lib/Target/Sparc/SparcISelLowering.h b/llvm/lib/Target/Sparc/SparcISelLowering.h
index 8d557a4225e54..3d798cec0c160 100644
--- a/llvm/lib/Target/Sparc/SparcISelLowering.h
+++ b/llvm/lib/Target/Sparc/SparcISelLowering.h
@@ -98,8 +98,8 @@ namespace llvm {
       return MVT::i32;
     }
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     /// If a physical register, this returns the register that receives the
     /// exception address on entry to an EH pad.
diff --git a/llvm/lib/Target/SystemZ/CMakeLists.txt b/llvm/lib/Target/SystemZ/CMakeLists.txt
index 585fb67f96f51..03e5e31831f65 100644
--- a/llvm/lib/Target/SystemZ/CMakeLists.txt
+++ b/llvm/lib/Target/SystemZ/CMakeLists.txt
@@ -17,7 +17,6 @@ add_llvm_target(SystemZCodeGen
   SystemZCallingConv.cpp
   SystemZConstantPoolValue.cpp
   SystemZElimCompare.cpp
-  SystemZExpandPseudo.cpp
   SystemZFrameLowering.cpp
   SystemZHazardRecognizer.cpp
   SystemZISelDAGToDAG.cpp
diff --git a/llvm/lib/Target/SystemZ/SystemZ.h b/llvm/lib/Target/SystemZ/SystemZ.h
index 2b0f90182d7f7..88cf589a3f102 100644
--- a/llvm/lib/Target/SystemZ/SystemZ.h
+++ b/llvm/lib/Target/SystemZ/SystemZ.h
@@ -190,7 +190,6 @@ static inline bool isImmHF(uint64_t Val) {
 FunctionPass *createSystemZISelDag(SystemZTargetMachine &TM,
                                    CodeGenOpt::Level OptLevel);
 FunctionPass *createSystemZElimComparePass(SystemZTargetMachine &TM);
-FunctionPass *createSystemZExpandPseudoPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZShortenInstPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLongBranchPass(SystemZTargetMachine &TM);
 FunctionPass *createSystemZLDCleanupPass(SystemZTargetMachine &TM);
diff --git a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
index f03ed2014e4a9..10023e9e169c5 100644
--- a/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZAsmPrinter.cpp
@@ -553,6 +553,12 @@ static unsigned EmitNop(MCContext &OutContext, MCStreamer &OutStreamer,
 void SystemZAsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
                                          SystemZMCInstLower &Lower) {
   MCContext &Ctx = MF->getContext();
+  if (MF->getFunction().getFnAttribute("mnop-mcount")
+                       .getValueAsString() == "true") {
+    EmitNop(Ctx, *OutStreamer, 6, getSubtargetInfo());
+    return;
+  }
+
   MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
   const MCSymbolRefExpr *Op =
       MCSymbolRefExpr::create(fentry, MCSymbolRefExpr::VK_PLT, Ctx);
diff --git a/llvm/lib/Target/SystemZ/SystemZExpandPseudo.cpp b/llvm/lib/Target/SystemZ/SystemZExpandPseudo.cpp
deleted file mode 100644
index 45270da218119..0000000000000
--- a/llvm/lib/Target/SystemZ/SystemZExpandPseudo.cpp
+++ /dev/null
@@ -1,152 +0,0 @@
-//==-- SystemZExpandPseudo.cpp - Expand pseudo instructions -------*- C++ -*-=//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file contains a pass that expands pseudo instructions into target
-// instructions to allow proper scheduling and other late optimizations.  This
-// pass should be run after register allocation but before the post-regalloc
-// scheduling pass.
-//
-//===----------------------------------------------------------------------===//
-
-#include "SystemZ.h"
-#include "SystemZInstrInfo.h"
-#include "SystemZSubtarget.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/MachineInstrBuilder.h"
-using namespace llvm;
-
-#define SYSTEMZ_EXPAND_PSEUDO_NAME "SystemZ pseudo instruction expansion pass"
-
-namespace llvm {
-  void initializeSystemZExpandPseudoPass(PassRegistry&);
-}
-
-namespace {
-class SystemZExpandPseudo : public MachineFunctionPass {
-public:
-  static char ID;
-  SystemZExpandPseudo() : MachineFunctionPass(ID) {
-    initializeSystemZExpandPseudoPass(*PassRegistry::getPassRegistry());
-  }
-
-  const SystemZInstrInfo *TII;
-
-  bool runOnMachineFunction(MachineFunction &Fn) override;
-
-  StringRef getPassName() const override { return SYSTEMZ_EXPAND_PSEUDO_NAME; }
-
-private:
-  bool expandMBB(MachineBasicBlock &MBB);
-  bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                MachineBasicBlock::iterator &NextMBBI);
-  bool expandLOCRMux(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
-                     MachineBasicBlock::iterator &NextMBBI);
-};
-char SystemZExpandPseudo::ID = 0;
-}
-
-INITIALIZE_PASS(SystemZExpandPseudo, "systemz-expand-pseudo",
-                SYSTEMZ_EXPAND_PSEUDO_NAME, false, false)
-
-/// Returns an instance of the pseudo instruction expansion pass.
-FunctionPass *llvm::createSystemZExpandPseudoPass(SystemZTargetMachine &TM) {
-  return new SystemZExpandPseudo();
-}
-
-// MI is a load-register-on-condition pseudo instruction that could not be
-// handled as a single hardware instruction.  Replace it by a branch sequence.
-bool SystemZExpandPseudo::expandLOCRMux(MachineBasicBlock &MBB,
-                                        MachineBasicBlock::iterator MBBI,
-                                        MachineBasicBlock::iterator &NextMBBI) {
-  MachineFunction &MF = *MBB.getParent();
-  const BasicBlock *BB = MBB.getBasicBlock();
-  MachineInstr &MI = *MBBI;
-  DebugLoc DL = MI.getDebugLoc();
-  Register DestReg = MI.getOperand(0).getReg();
-  Register SrcReg = MI.getOperand(2).getReg();
-  unsigned CCValid = MI.getOperand(3).getImm();
-  unsigned CCMask = MI.getOperand(4).getImm();
-
-  LivePhysRegs LiveRegs(TII->getRegisterInfo());
-  LiveRegs.addLiveOuts(MBB);
-  for (auto I = std::prev(MBB.end()); I != MBBI; --I)
-    LiveRegs.stepBackward(*I);
-
-  // Splice MBB at MI, moving the rest of the block into RestMBB.
-  MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB);
-  MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB);
-  RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end());
-  RestMBB->transferSuccessors(&MBB);
-  for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
-    RestMBB->addLiveIn(*I);
-
-  // Create a new block MoveMBB to hold the move instruction.
-  MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB);
-  MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB);
-  MoveMBB->addLiveIn(SrcReg);
-  for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
-    MoveMBB->addLiveIn(*I);
-
-  // At the end of MBB, create a conditional branch to RestMBB if the
-  // condition is false, otherwise fall through to MoveMBB.
-  BuildMI(&MBB, DL, TII->get(SystemZ::BRC))
-    .addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB);
-  MBB.addSuccessor(RestMBB);
-  MBB.addSuccessor(MoveMBB);
-
-  // In MoveMBB, emit an instruction to move SrcReg into DestReg,
-  // then fall through to RestMBB.
-  TII->copyPhysReg(*MoveMBB, MoveMBB->end(), DL, DestReg, SrcReg,
-                   MI.getOperand(2).isKill());
-  MoveMBB->addSuccessor(RestMBB);
-
-  NextMBBI = MBB.end();
-  MI.eraseFromParent();
-  return true;
-}
-
-/// If MBBI references a pseudo instruction that should be expanded here,
-/// do the expansion and return true.  Otherwise return false.
-bool SystemZExpandPseudo::expandMI(MachineBasicBlock &MBB,
-                                   MachineBasicBlock::iterator MBBI,
-                                   MachineBasicBlock::iterator &NextMBBI) {
-  MachineInstr &MI = *MBBI;
-  switch (MI.getOpcode()) {
-  case SystemZ::LOCRMux:
-    return expandLOCRMux(MBB, MBBI, NextMBBI);
-  default:
-    break;
-  }
-  return false;
-}
-
-/// Iterate over the instructions in basic block MBB and expand any
-/// pseudo instructions.  Return true if anything was modified.
-bool SystemZExpandPseudo::expandMBB(MachineBasicBlock &MBB) {
-  bool Modified = false;
-
-  MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
-  while (MBBI != E) {
-    MachineBasicBlock::iterator NMBBI = std::next(MBBI);
-    Modified |= expandMI(MBB, MBBI, NMBBI);
-    MBBI = NMBBI;
-  }
-
-  return Modified;
-}
-
-bool SystemZExpandPseudo::runOnMachineFunction(MachineFunction &MF) {
-  TII = static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo());
-
-  bool Modified = false;
-  for (auto &MBB : MF)
-    Modified |= expandMBB(MBB);
-  return Modified;
-}
-
diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
index 9dc4512255cc3..751034c2d41a1 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp
@@ -346,6 +346,11 @@ class SystemZDAGToDAGISel : public SelectionDAGISel {
       : SelectionDAGISel(TM, OptLevel) {}
 
   bool runOnMachineFunction(MachineFunction &MF) override {
+    const Function &F = MF.getFunction();
+    if (F.getFnAttribute("mnop-mcount").getValueAsString() == "true" &&
+        F.getFnAttribute("fentry-call").getValueAsString() != "true")
+      report_fatal_error("mnop-mcount only supported with fentry-call");
+
     Subtarget = &MF.getSubtarget<SystemZSubtarget>();
     return SelectionDAGISel::runOnMachineFunction(MF);
   }
@@ -1146,7 +1151,7 @@ void SystemZDAGToDAGISel::loadVectorConstant(
   SDLoc DL(Node);
   SmallVector<SDValue, 2> Ops;
   for (unsigned OpVal : VCI.OpVals)
-    Ops.push_back(CurDAG->getConstant(OpVal, DL, MVT::i32));
+    Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32));
   SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops);
 
   if (VCI.VecVT == VT.getSimpleVT())
@@ -1550,8 +1555,8 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) {
       uint64_t ConstCCMask =
         cast<ConstantSDNode>(CCMask.getNode())->getZExtValue();
       // Invert the condition.
-      CCMask = CurDAG->getConstant(ConstCCValid ^ ConstCCMask, SDLoc(Node),
-                                   CCMask.getValueType());
+      CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask,
+                                         SDLoc(Node), CCMask.getValueType());
       SDValue Op4 = Node->getOperand(4);
       SDNode *UpdatedNode =
         CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4);
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index ce1582157e9d2..d69f578735ac0 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -120,9 +120,9 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
   // Instructions are strings of 2-byte aligned 2-byte values.
-  setMinFunctionAlignment(llvm::Align(2));
+  setMinFunctionAlignment(Align(2));
   // For performance reasons we prefer 16-byte alignment.
-  setPrefFunctionAlignment(llvm::Align(16));
+  setPrefFunctionAlignment(Align(16));
 
   // Handle operations that are handled in a similar way for all types.
   for (unsigned I = MVT::FIRST_INTEGER_VALUETYPE;
@@ -258,7 +258,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Promote);
   setOperationAction(ISD::CTLZ, MVT::i64, Legal);
 
-  // On arch13 we have native support for a 64-bit CTPOP.
+  // On z15 we have native support for a 64-bit CTPOP.
   if (Subtarget.hasMiscellaneousExtensions3()) {
     setOperationAction(ISD::CTPOP, MVT::i32, Promote);
     setOperationAction(ISD::CTPOP, MVT::i64, Legal);
@@ -300,14 +300,14 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   // Handle prefetches with PFD or PFDRL.
   setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
 
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     // Assume by default that all vector operations need to be expanded.
     for (unsigned Opcode = 0; Opcode < ISD::BUILTIN_OP_END; ++Opcode)
       if (getOperationAction(Opcode, VT) == Legal)
         setOperationAction(Opcode, VT, Expand);
 
     // Likewise all truncating stores and extending loads.
-    for (MVT InnerVT : MVT::vector_valuetypes()) {
+    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
       setTruncStoreAction(VT, InnerVT, Expand);
       setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
       setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
@@ -333,7 +333,7 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM,
   }
 
   // Handle integer vector types.
-  for (MVT VT : MVT::integer_vector_valuetypes()) {
+  for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
     if (isTypeLegal(VT)) {
       // These operations have direct equivalents.
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal);
@@ -2549,12 +2549,12 @@ static SDValue emitCmp(SelectionDAG &DAG, const SDLoc &DL, Comparison &C) {
   }
   if (C.Opcode == SystemZISD::ICMP)
     return DAG.getNode(SystemZISD::ICMP, DL, MVT::i32, C.Op0, C.Op1,
-                       DAG.getConstant(C.ICmpType, DL, MVT::i32));
+                       DAG.getTargetConstant(C.ICmpType, DL, MVT::i32));
   if (C.Opcode == SystemZISD::TM) {
     bool RegisterOnly = (bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_0) !=
                          bool(C.CCMask & SystemZ::CCMASK_TM_MIXED_MSB_1));
     return DAG.getNode(SystemZISD::TM, DL, MVT::i32, C.Op0, C.Op1,
-                       DAG.getConstant(RegisterOnly, DL, MVT::i32));
+                       DAG.getTargetConstant(RegisterOnly, DL, MVT::i32));
   }
   return DAG.getNode(C.Opcode, DL, MVT::i32, C.Op0, C.Op1);
 }
@@ -2592,10 +2592,10 @@ static void lowerGR128Binary(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
 // in CCValid, so other values can be ignored.
 static SDValue emitSETCC(SelectionDAG &DAG, const SDLoc &DL, SDValue CCReg,
                          unsigned CCValid, unsigned CCMask) {
-  SDValue Ops[] = { DAG.getConstant(1, DL, MVT::i32),
-                    DAG.getConstant(0, DL, MVT::i32),
-                    DAG.getConstant(CCValid, DL, MVT::i32),
-                    DAG.getConstant(CCMask, DL, MVT::i32), CCReg };
+  SDValue Ops[] = {DAG.getConstant(1, DL, MVT::i32),
+                   DAG.getConstant(0, DL, MVT::i32),
+                   DAG.getTargetConstant(CCValid, DL, MVT::i32),
+                   DAG.getTargetConstant(CCMask, DL, MVT::i32), CCReg};
   return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, MVT::i32, Ops);
 }
 
@@ -2757,9 +2757,10 @@ SDValue SystemZTargetLowering::lowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
 
   Comparison C(getCmp(DAG, CmpOp0, CmpOp1, CC, DL));
   SDValue CCReg = emitCmp(DAG, DL, C);
-  return DAG.getNode(SystemZISD::BR_CCMASK, DL, Op.getValueType(),
-                     Op.getOperand(0), DAG.getConstant(C.CCValid, DL, MVT::i32),
-                     DAG.getConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
+  return DAG.getNode(
+      SystemZISD::BR_CCMASK, DL, Op.getValueType(), Op.getOperand(0),
+      DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
+      DAG.getTargetConstant(C.CCMask, DL, MVT::i32), Dest, CCReg);
 }
 
 // Return true if Pos is CmpOp and Neg is the negative of CmpOp,
@@ -2810,8 +2811,9 @@ SDValue SystemZTargetLowering::lowerSELECT_CC(SDValue Op,
   }
 
   SDValue CCReg = emitCmp(DAG, DL, C);
-  SDValue Ops[] = {TrueOp, FalseOp, DAG.getConstant(C.CCValid, DL, MVT::i32),
-                   DAG.getConstant(C.CCMask, DL, MVT::i32), CCReg};
+  SDValue Ops[] = {TrueOp, FalseOp,
+                   DAG.getTargetConstant(C.CCValid, DL, MVT::i32),
+                   DAG.getTargetConstant(C.CCMask, DL, MVT::i32), CCReg};
 
   return DAG.getNode(SystemZISD::SELECT_CCMASK, DL, Op.getValueType(), Ops);
 }
@@ -3898,11 +3900,8 @@ SDValue SystemZTargetLowering::lowerPREFETCH(SDValue Op,
   bool IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
   unsigned Code = IsWrite ? SystemZ::PFD_WRITE : SystemZ::PFD_READ;
   auto *Node = cast<MemIntrinsicSDNode>(Op.getNode());
-  SDValue Ops[] = {
-    Op.getOperand(0),
-    DAG.getConstant(Code, DL, MVT::i32),
-    Op.getOperand(1)
-  };
+  SDValue Ops[] = {Op.getOperand(0), DAG.getTargetConstant(Code, DL, MVT::i32),
+                   Op.getOperand(1)};
   return DAG.getMemIntrinsicNode(SystemZISD::PREFETCH, DL,
                                  Node->getVTList(), Ops,
                                  Node->getMemoryVT(), Node->getMemOperand());
@@ -4244,7 +4243,7 @@ static SDValue getPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
   Op1 = DAG.getNode(ISD::BITCAST, DL, InVT, Op1);
   SDValue Op;
   if (P.Opcode == SystemZISD::PERMUTE_DWORDS) {
-    SDValue Op2 = DAG.getConstant(P.Operand, DL, MVT::i32);
+    SDValue Op2 = DAG.getTargetConstant(P.Operand, DL, MVT::i32);
     Op = DAG.getNode(SystemZISD::PERMUTE_DWORDS, DL, InVT, Op0, Op1, Op2);
   } else if (P.Opcode == SystemZISD::PACK) {
     MVT OutVT = MVT::getVectorVT(MVT::getIntegerVT(P.Operand * 8),
@@ -4269,7 +4268,8 @@ static SDValue getGeneralPermuteNode(SelectionDAG &DAG, const SDLoc &DL,
   unsigned StartIndex, OpNo0, OpNo1;
   if (isShlDoublePermute(Bytes, StartIndex, OpNo0, OpNo1))
     return DAG.getNode(SystemZISD::SHL_DOUBLE, DL, MVT::v16i8, Ops[OpNo0],
-                       Ops[OpNo1], DAG.getConstant(StartIndex, DL, MVT::i32));
+                       Ops[OpNo1],
+                       DAG.getTargetConstant(StartIndex, DL, MVT::i32));
 
   // Fall back on VPERM.  Construct an SDNode for the permute vector.
   SDValue IndexNodes[SystemZ::VectorBytes];
@@ -4767,7 +4767,7 @@ SDValue SystemZTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
       return DAG.getNode(SystemZISD::REPLICATE, DL, VT, Op0.getOperand(Index));
     // Otherwise keep it as a vector-to-vector operation.
     return DAG.getNode(SystemZISD::SPLAT, DL, VT, Op.getOperand(0),
-                       DAG.getConstant(Index, DL, MVT::i32));
+                       DAG.getTargetConstant(Index, DL, MVT::i32));
   }
 
   GeneralShuffle GS(VT);
@@ -6057,8 +6057,8 @@ SDValue SystemZTargetLowering::combineBR_CCMASK(
   if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
     return DAG.getNode(SystemZISD::BR_CCMASK, SDLoc(N), N->getValueType(0),
                        Chain,
-                       DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32),
-                       DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32),
+                       DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
+                       DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
                        N->getOperand(3), CCReg);
   return SDValue();
 }
@@ -6079,10 +6079,9 @@ SDValue SystemZTargetLowering::combineSELECT_CCMASK(
 
   if (combineCCMask(CCReg, CCValidVal, CCMaskVal))
     return DAG.getNode(SystemZISD::SELECT_CCMASK, SDLoc(N), N->getValueType(0),
-                       N->getOperand(0),
-                       N->getOperand(1),
-                       DAG.getConstant(CCValidVal, SDLoc(N), MVT::i32),
-                       DAG.getConstant(CCMaskVal, SDLoc(N), MVT::i32),
+                       N->getOperand(0), N->getOperand(1),
+                       DAG.getTargetConstant(CCValidVal, SDLoc(N), MVT::i32),
+                       DAG.getTargetConstant(CCMaskVal, SDLoc(N), MVT::i32),
                        CCReg);
   return SDValue();
 }
@@ -6564,19 +6563,17 @@ static bool isSelectPseudo(MachineInstr &MI) {
 
 // Helper function, which inserts PHI functions into SinkMBB:
 //   %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
-// where %FalseValue(i) and %TrueValue(i) are taken from the consequent Selects
-// in [MIItBegin, MIItEnd) range.
-static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
-                                 MachineBasicBlock::iterator MIItEnd,
+// where %FalseValue(i) and %TrueValue(i) are taken from Selects.
+static void createPHIsForSelects(SmallVector<MachineInstr*, 8> &Selects,
                                  MachineBasicBlock *TrueMBB,
                                  MachineBasicBlock *FalseMBB,
                                  MachineBasicBlock *SinkMBB) {
   MachineFunction *MF = TrueMBB->getParent();
   const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
 
-  unsigned CCValid = MIItBegin->getOperand(3).getImm();
-  unsigned CCMask = MIItBegin->getOperand(4).getImm();
-  DebugLoc DL = MIItBegin->getDebugLoc();
+  MachineInstr *FirstMI = Selects.front();
+  unsigned CCValid = FirstMI->getOperand(3).getImm();
+  unsigned CCMask = FirstMI->getOperand(4).getImm();
 
   MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
 
@@ -6588,16 +6585,15 @@ static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
   // destination registers, and the registers that went into the PHI.
   DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
 
-  for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd;
-       MIIt = skipDebugInstructionsForward(++MIIt, MIItEnd)) {
-    Register DestReg = MIIt->getOperand(0).getReg();
-    Register TrueReg = MIIt->getOperand(1).getReg();
-    Register FalseReg = MIIt->getOperand(2).getReg();
+  for (auto MI : Selects) {
+    Register DestReg = MI->getOperand(0).getReg();
+    Register TrueReg = MI->getOperand(1).getReg();
+    Register FalseReg = MI->getOperand(2).getReg();
 
     // If this Select we are generating is the opposite condition from
     // the jump we generated, then we have to swap the operands for the
     // PHI that is going to be generated.
-    if (MIIt->getOperand(4).getImm() == (CCValid ^ CCMask))
+    if (MI->getOperand(4).getImm() == (CCValid ^ CCMask))
       std::swap(TrueReg, FalseReg);
 
     if (RegRewriteTable.find(TrueReg) != RegRewriteTable.end())
@@ -6606,6 +6602,7 @@ static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
     if (RegRewriteTable.find(FalseReg) != RegRewriteTable.end())
       FalseReg = RegRewriteTable[FalseReg].second;
 
+    DebugLoc DL = MI->getDebugLoc();
     BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(SystemZ::PHI), DestReg)
       .addReg(TrueReg).addMBB(TrueMBB)
       .addReg(FalseReg).addMBB(FalseMBB);
@@ -6621,36 +6618,61 @@ static void createPHIsForSelects(MachineBasicBlock::iterator MIItBegin,
 MachineBasicBlock *
 SystemZTargetLowering::emitSelect(MachineInstr &MI,
                                   MachineBasicBlock *MBB) const {
+  assert(isSelectPseudo(MI) && "Bad call to emitSelect()");
   const SystemZInstrInfo *TII =
       static_cast<const SystemZInstrInfo *>(Subtarget.getInstrInfo());
 
   unsigned CCValid = MI.getOperand(3).getImm();
   unsigned CCMask = MI.getOperand(4).getImm();
-  DebugLoc DL = MI.getDebugLoc();
 
   // If we have a sequence of Select* pseudo instructions using the
   // same condition code value, we want to expand all of them into
   // a single pair of basic blocks using the same condition.
-  MachineInstr *LastMI = &MI;
-  MachineBasicBlock::iterator NextMIIt = skipDebugInstructionsForward(
-      std::next(MachineBasicBlock::iterator(MI)), MBB->end());
-
-  if (isSelectPseudo(MI))
-    while (NextMIIt != MBB->end() && isSelectPseudo(*NextMIIt) &&
-           NextMIIt->getOperand(3).getImm() == CCValid &&
-           (NextMIIt->getOperand(4).getImm() == CCMask ||
-            NextMIIt->getOperand(4).getImm() == (CCValid ^ CCMask))) {
-      LastMI = &*NextMIIt;
-      NextMIIt = skipDebugInstructionsForward(++NextMIIt, MBB->end());
+  SmallVector<MachineInstr*, 8> Selects;
+  SmallVector<MachineInstr*, 8> DbgValues;
+  Selects.push_back(&MI);
+  unsigned Count = 0;
+  for (MachineBasicBlock::iterator NextMIIt =
+         std::next(MachineBasicBlock::iterator(MI));
+       NextMIIt != MBB->end(); ++NextMIIt) {
+    if (NextMIIt->definesRegister(SystemZ::CC))
+      break;
+    if (isSelectPseudo(*NextMIIt)) {
+      assert(NextMIIt->getOperand(3).getImm() == CCValid &&
+             "Bad CCValid operands since CC was not redefined.");
+      if (NextMIIt->getOperand(4).getImm() == CCMask ||
+          NextMIIt->getOperand(4).getImm() == (CCValid ^ CCMask)) {
+        Selects.push_back(&*NextMIIt);
+        continue;
+      }
+      break;
     }
+    bool User = false;
+    for (auto SelMI : Selects)
+      if (NextMIIt->readsVirtualRegister(SelMI->getOperand(0).getReg())) {
+        User = true;
+        break;
+      }
+    if (NextMIIt->isDebugInstr()) {
+      if (User) {
+        assert(NextMIIt->isDebugValue() && "Unhandled debug opcode.");
+        DbgValues.push_back(&*NextMIIt);
+      }
+    }
+    else if (User || ++Count > 20)
+      break;
+  }
 
+  MachineInstr *LastMI = Selects.back();
+  bool CCKilled =
+      (LastMI->killsRegister(SystemZ::CC) || checkCCKill(*LastMI, MBB));
   MachineBasicBlock *StartMBB = MBB;
-  MachineBasicBlock *JoinMBB  = splitBlockBefore(MI, MBB);
+  MachineBasicBlock *JoinMBB  = splitBlockAfter(LastMI, MBB);
   MachineBasicBlock *FalseMBB = emitBlockAfter(StartMBB);
 
   // Unless CC was killed in the last Select instruction, mark it as
   // live-in to both FalseMBB and JoinMBB.
-  if (!LastMI->killsRegister(SystemZ::CC) && !checkCCKill(*LastMI, JoinMBB)) {
+  if (!CCKilled) {
     FalseMBB->addLiveIn(SystemZ::CC);
     JoinMBB->addLiveIn(SystemZ::CC);
   }
@@ -6659,7 +6681,7 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
   //   BRC CCMask, JoinMBB
   //   # fallthrough to FalseMBB
   MBB = StartMBB;
-  BuildMI(MBB, DL, TII->get(SystemZ::BRC))
+  BuildMI(MBB, MI.getDebugLoc(), TII->get(SystemZ::BRC))
     .addImm(CCValid).addImm(CCMask).addMBB(JoinMBB);
   MBB->addSuccessor(JoinMBB);
   MBB->addSuccessor(FalseMBB);
@@ -6673,12 +6695,14 @@ SystemZTargetLowering::emitSelect(MachineInstr &MI,
   //   %Result = phi [ %FalseReg, FalseMBB ], [ %TrueReg, StartMBB ]
   //  ...
   MBB = JoinMBB;
-  MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
-  MachineBasicBlock::iterator MIItEnd = skipDebugInstructionsForward(
-      std::next(MachineBasicBlock::iterator(LastMI)), MBB->end());
-  createPHIsForSelects(MIItBegin, MIItEnd, StartMBB, FalseMBB, MBB);
+  createPHIsForSelects(Selects, StartMBB, FalseMBB, MBB);
+  for (auto SelMI : Selects)
+    SelMI->eraseFromParent();
+
+  MachineBasicBlock::iterator InsertPos = MBB->getFirstNonPHI();
+  for (auto DbgMI : DbgValues)
+    MBB->splice(InsertPos, StartMBB, DbgMI);
 
-  StartMBB->erase(MIItBegin, MIItEnd);
   return JoinMBB;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
index 2a1d14de3ddf5..c9dbe3da686d7 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td
@@ -2141,17 +2141,17 @@ class FixedCondBranchRXY<CondVariant V, string mnemonic, bits<16> opcode,
 }
 
 class CmpBranchRIEa<string mnemonic, bits<16> opcode,
-                    RegisterOperand cls, Immediate imm>
+                    RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEa<opcode, (outs), (ins cls:$R1, imm:$I2, cond4:$M3),
              mnemonic#"$M3\t$R1, $I2", []>;
 
 class AsmCmpBranchRIEa<string mnemonic, bits<16> opcode,
-                       RegisterOperand cls, Immediate imm>
+                       RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEa<opcode, (outs), (ins cls:$R1, imm:$I2, imm32zx4:$M3),
              mnemonic#"\t$R1, $I2, $M3", []>;
 
 class FixedCmpBranchRIEa<CondVariant V, string mnemonic, bits<16> opcode,
-                          RegisterOperand cls, Immediate imm>
+                          RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEa<opcode, (outs), (ins cls:$R1, imm:$I2),
              mnemonic#V.suffix#"\t$R1, $I2", []> {
   let isAsmParserOnly = V.alternate;
@@ -2159,7 +2159,7 @@ class FixedCmpBranchRIEa<CondVariant V, string mnemonic, bits<16> opcode,
 }
 
 multiclass CmpBranchRIEaPair<string mnemonic, bits<16> opcode,
-                             RegisterOperand cls, Immediate imm> {
+                             RegisterOperand cls, ImmOpWithPattern imm> {
   let isCodeGenOnly = 1 in
     def "" : CmpBranchRIEa<mnemonic, opcode, cls, imm>;
   def Asm : AsmCmpBranchRIEa<mnemonic, opcode, cls, imm>;
@@ -2193,19 +2193,19 @@ multiclass CmpBranchRIEbPair<string mnemonic, bits<16> opcode,
 }
 
 class CmpBranchRIEc<string mnemonic, bits<16> opcode,
-                    RegisterOperand cls, Immediate imm>
+                    RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEc<opcode, (outs),
              (ins cls:$R1, imm:$I2, cond4:$M3, brtarget16:$RI4),
              mnemonic#"$M3\t$R1, $I2, $RI4", []>;
 
 class AsmCmpBranchRIEc<string mnemonic, bits<16> opcode,
-                       RegisterOperand cls, Immediate imm>
+                       RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEc<opcode, (outs),
              (ins cls:$R1, imm:$I2, imm32zx4:$M3, brtarget16:$RI4),
              mnemonic#"\t$R1, $I2, $M3, $RI4", []>;
 
 class FixedCmpBranchRIEc<CondVariant V, string mnemonic, bits<16> opcode,
-                         RegisterOperand cls, Immediate imm>
+                         RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEc<opcode, (outs), (ins cls:$R1, imm:$I2, brtarget16:$RI4),
              mnemonic#V.suffix#"\t$R1, $I2, $RI4", []> {
   let isAsmParserOnly = V.alternate;
@@ -2213,7 +2213,7 @@ class FixedCmpBranchRIEc<CondVariant V, string mnemonic, bits<16> opcode,
 }
 
 multiclass CmpBranchRIEcPair<string mnemonic, bits<16> opcode,
-                            RegisterOperand cls, Immediate imm> {
+                            RegisterOperand cls, ImmOpWithPattern imm> {
   let isCodeGenOnly = 1 in
     def "" : CmpBranchRIEc<mnemonic, opcode, cls, imm>;
   def Asm : AsmCmpBranchRIEc<mnemonic, opcode, cls, imm>;
@@ -2272,19 +2272,19 @@ multiclass CmpBranchRRSPair<string mnemonic, bits<16> opcode,
 }
 
 class CmpBranchRIS<string mnemonic, bits<16> opcode,
-                   RegisterOperand cls, Immediate imm>
+                   RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIS<opcode, (outs),
             (ins cls:$R1, imm:$I2, cond4:$M3, bdaddr12only:$BD4),
             mnemonic#"$M3\t$R1, $I2, $BD4", []>;
 
 class AsmCmpBranchRIS<string mnemonic, bits<16> opcode,
-                      RegisterOperand cls, Immediate imm>
+                      RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIS<opcode, (outs),
             (ins cls:$R1, imm:$I2, imm32zx4:$M3, bdaddr12only:$BD4),
             mnemonic#"\t$R1, $I2, $M3, $BD4", []>;
 
 class FixedCmpBranchRIS<CondVariant V, string mnemonic, bits<16> opcode,
-                        RegisterOperand cls, Immediate imm>
+                        RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIS<opcode, (outs), (ins cls:$R1, imm:$I2, bdaddr12only:$BD4),
             mnemonic#V.suffix#"\t$R1, $I2, $BD4", []> {
   let isAsmParserOnly = V.alternate;
@@ -2292,7 +2292,7 @@ class FixedCmpBranchRIS<CondVariant V, string mnemonic, bits<16> opcode,
 }
 
 multiclass CmpBranchRISPair<string mnemonic, bits<16> opcode,
-                            RegisterOperand cls, Immediate imm> {
+                            RegisterOperand cls, ImmOpWithPattern imm> {
   let isCodeGenOnly = 1 in
     def "" : CmpBranchRIS<mnemonic, opcode, cls, imm>;
   def Asm : AsmCmpBranchRIS<mnemonic, opcode, cls, imm>;
@@ -2585,7 +2585,7 @@ multiclass StoreMultipleVRSaAlign<string mnemonic, bits<16> opcode> {
 // We therefore match the address in the same way as a normal store and
 // only use the StoreSI* instruction if the matched address is suitable.
 class StoreSI<string mnemonic, bits<8> opcode, SDPatternOperator operator,
-              Immediate imm>
+              ImmOpWithPattern imm>
   : InstSI<opcode, (outs), (ins mviaddr12pair:$BD1, imm:$I2),
            mnemonic#"\t$BD1, $I2",
            [(operator imm:$I2, mviaddr12pair:$BD1)]> {
@@ -2593,7 +2593,7 @@ class StoreSI<string mnemonic, bits<8> opcode, SDPatternOperator operator,
 }
 
 class StoreSIY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-               Immediate imm>
+               ImmOpWithPattern imm>
   : InstSIY<opcode, (outs), (ins mviaddr20pair:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2",
             [(operator imm:$I2, mviaddr20pair:$BD1)]> {
@@ -2601,7 +2601,7 @@ class StoreSIY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class StoreSIL<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-               Immediate imm>
+               ImmOpWithPattern imm>
   : InstSIL<opcode, (outs), (ins mviaddr12pair:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2",
             [(operator imm:$I2, mviaddr12pair:$BD1)]> {
@@ -2609,7 +2609,7 @@ class StoreSIL<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 multiclass StoreSIPair<string mnemonic, bits<8> siOpcode, bits<16> siyOpcode,
-                       SDPatternOperator operator, Immediate imm> {
+                       SDPatternOperator operator, ImmOpWithPattern imm> {
   let DispKey = mnemonic in {
     let DispSize = "12" in
       def "" : StoreSI<mnemonic, siOpcode, operator, imm>;
@@ -2665,7 +2665,7 @@ multiclass CondStoreRSYPair<string mnemonic, bits<16> opcode,
   def Asm : AsmCondStoreRSY<mnemonic, opcode, cls, bytes, mode>;
 }
 
-class SideEffectUnaryI<string mnemonic, bits<8> opcode, Immediate imm>
+class SideEffectUnaryI<string mnemonic, bits<8> opcode, ImmOpWithPattern imm>
   : InstI<opcode, (outs), (ins imm:$I1),
           mnemonic#"\t$I1", []>;
 
@@ -2761,13 +2761,13 @@ class UnaryMemRRFc<string mnemonic, bits<16> opcode,
 }
 
 class UnaryRI<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-              RegisterOperand cls, Immediate imm>
+              RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIa<opcode, (outs cls:$R1), (ins imm:$I2),
             mnemonic#"\t$R1, $I2",
             [(set cls:$R1, (operator imm:$I2))]>;
 
 class UnaryRIL<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-               RegisterOperand cls, Immediate imm>
+               RegisterOperand cls, ImmOpWithPattern imm>
   : InstRILa<opcode, (outs cls:$R1), (ins imm:$I2),
              mnemonic#"\t$R1, $I2",
              [(set cls:$R1, (operator imm:$I2))]>;
@@ -2885,14 +2885,14 @@ multiclass UnaryRXPair<string mnemonic, bits<8> rxOpcode, bits<16> rxyOpcode,
 }
 
 class UnaryVRIa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                TypedReg tr, Immediate imm, bits<4> type = 0>
+                TypedReg tr, ImmOpWithPattern imm, bits<4> type = 0>
   : InstVRIa<opcode, (outs tr.op:$V1), (ins imm:$I2),
              mnemonic#"\t$V1, $I2",
-             [(set (tr.vt tr.op:$V1), (operator imm:$I2))]> {
+             [(set (tr.vt tr.op:$V1), (operator (i32 timm:$I2)))]> {
   let M3 = type;
 }
 
-class UnaryVRIaGeneric<string mnemonic, bits<16> opcode, Immediate imm>
+class UnaryVRIaGeneric<string mnemonic, bits<16> opcode, ImmOpWithPattern imm>
   : InstVRIa<opcode, (outs VR128:$V1), (ins imm:$I2, imm32zx4:$M3),
              mnemonic#"\t$V1, $I2, $M3", []>;
 
@@ -3021,7 +3021,7 @@ class SideEffectBinaryRRFc<string mnemonic, bits<16> opcode,
 }
 
 class SideEffectBinaryIE<string mnemonic, bits<16> opcode,
-                         Immediate imm1, Immediate imm2>
+                         ImmOpWithPattern imm1, ImmOpWithPattern imm2>
   : InstIE<opcode, (outs), (ins imm1:$I1, imm2:$I2),
            mnemonic#"\t$I1, $I2", []>;
 
@@ -3030,7 +3030,7 @@ class SideEffectBinarySI<string mnemonic, bits<8> opcode, Operand imm>
            mnemonic#"\t$BD1, $I2", []>;
 
 class SideEffectBinarySIL<string mnemonic, bits<16> opcode,
-                          SDPatternOperator operator, Immediate imm>
+                          SDPatternOperator operator, ImmOpWithPattern imm>
   : InstSIL<opcode, (outs), (ins bdaddr12only:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2", [(operator bdaddr12only:$BD1, imm:$I2)]>;
 
@@ -3165,7 +3165,7 @@ class BinaryRRFc<string mnemonic, bits<16> opcode,
              mnemonic#"\t$R1, $R2, $M3", []>;
 
 class BinaryMemRRFc<string mnemonic, bits<16> opcode,
-                    RegisterOperand cls1, RegisterOperand cls2, Immediate imm>
+                    RegisterOperand cls1, RegisterOperand cls2, ImmOpWithPattern imm>
   : InstRRFc<opcode, (outs cls2:$R2, cls1:$R1), (ins cls1:$R1src, imm:$M3),
             mnemonic#"\t$R1, $R2, $M3", []> {
   let Constraints = "$R1 = $R1src";
@@ -3267,7 +3267,7 @@ multiclass CondBinaryRRFaPair<string mnemonic, bits<16> opcode,
 }
 
 class BinaryRI<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-               RegisterOperand cls, Immediate imm>
+               RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIa<opcode, (outs cls:$R1), (ins cls:$R1src, imm:$I2),
             mnemonic#"\t$R1, $I2",
             [(set cls:$R1, (operator cls:$R1src, imm:$I2))]> {
@@ -3276,14 +3276,14 @@ class BinaryRI<string mnemonic, bits<12> opcode, SDPatternOperator operator,
 }
 
 class BinaryRIE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                RegisterOperand cls, Immediate imm>
+                RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEd<opcode, (outs cls:$R1), (ins cls:$R3, imm:$I2),
              mnemonic#"\t$R1, $R3, $I2",
              [(set cls:$R1, (operator cls:$R3, imm:$I2))]>;
 
 multiclass BinaryRIAndK<string mnemonic, bits<12> opcode1, bits<16> opcode2,
                         SDPatternOperator operator, RegisterOperand cls,
-                        Immediate imm> {
+                        ImmOpWithPattern imm> {
   let NumOpsKey = mnemonic in {
     let NumOpsValue = "3" in
       def K : BinaryRIE<mnemonic##"k", opcode2, operator, cls, imm>,
@@ -3294,7 +3294,7 @@ multiclass BinaryRIAndK<string mnemonic, bits<12> opcode1, bits<16> opcode2,
 }
 
 class CondBinaryRIE<string mnemonic, bits<16> opcode, RegisterOperand cls,
-                    Immediate imm>
+                    ImmOpWithPattern imm>
   : InstRIEg<opcode, (outs cls:$R1),
              (ins cls:$R1src, imm:$I2, cond4:$valid, cond4:$M3),
              mnemonic#"$M3\t$R1, $I2",
@@ -3308,7 +3308,7 @@ class CondBinaryRIE<string mnemonic, bits<16> opcode, RegisterOperand cls,
 // Like CondBinaryRIE, but used for the raw assembly form.  The condition-code
 // mask is the third operand rather than being part of the mnemonic.
 class AsmCondBinaryRIE<string mnemonic, bits<16> opcode, RegisterOperand cls,
-                       Immediate imm>
+                       ImmOpWithPattern imm>
   : InstRIEg<opcode, (outs cls:$R1),
              (ins cls:$R1src, imm:$I2, imm32zx4:$M3),
              mnemonic#"\t$R1, $I2, $M3", []> {
@@ -3318,7 +3318,7 @@ class AsmCondBinaryRIE<string mnemonic, bits<16> opcode, RegisterOperand cls,
 
 // Like CondBinaryRIE, but with a fixed CC mask.
 class FixedCondBinaryRIE<CondVariant V, string mnemonic, bits<16> opcode,
-                         RegisterOperand cls, Immediate imm>
+                         RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIEg<opcode, (outs cls:$R1), (ins cls:$R1src, imm:$I2),
              mnemonic#V.suffix#"\t$R1, $I2", []> {
   let Constraints = "$R1 = $R1src";
@@ -3328,14 +3328,14 @@ class FixedCondBinaryRIE<CondVariant V, string mnemonic, bits<16> opcode,
 }
 
 multiclass CondBinaryRIEPair<string mnemonic, bits<16> opcode,
-                             RegisterOperand cls, Immediate imm> {
+                             RegisterOperand cls, ImmOpWithPattern imm> {
   let isCodeGenOnly = 1 in
     def "" : CondBinaryRIE<mnemonic, opcode, cls, imm>;
   def Asm : AsmCondBinaryRIE<mnemonic, opcode, cls, imm>;
 }
 
 class BinaryRIL<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-                RegisterOperand cls, Immediate imm>
+                RegisterOperand cls, ImmOpWithPattern imm>
   : InstRILa<opcode, (outs cls:$R1), (ins cls:$R1src, imm:$I2),
              mnemonic#"\t$R1, $I2",
              [(set cls:$R1, (operator cls:$R1src, imm:$I2))]> {
@@ -3484,7 +3484,7 @@ class BinaryVRIb<string mnemonic, bits<16> opcode, SDPatternOperator operator,
                  TypedReg tr, bits<4> type>
   : InstVRIb<opcode, (outs tr.op:$V1), (ins imm32zx8:$I2, imm32zx8:$I3),
              mnemonic#"\t$V1, $I2, $I3",
-             [(set (tr.vt tr.op:$V1), (operator imm32zx8:$I2, imm32zx8:$I3))]> {
+             [(set (tr.vt tr.op:$V1), (operator imm32zx8_timm:$I2, imm32zx8_timm:$I3))]> {
   let M4 = type;
 }
 
@@ -3498,7 +3498,7 @@ class BinaryVRIc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   : InstVRIc<opcode, (outs tr1.op:$V1), (ins tr2.op:$V3, imm32zx16:$I2),
              mnemonic#"\t$V1, $V3, $I2",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V3),
-                                                  imm32zx16:$I2))]> {
+                                                  imm32zx16_timm:$I2))]> {
   let M4 = type;
 }
 
@@ -3512,7 +3512,7 @@ class BinaryVRIe<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   : InstVRIe<opcode, (outs tr1.op:$V1), (ins tr2.op:$V2, imm32zx12:$I3),
              mnemonic#"\t$V1, $V2, $I3",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
-                                                  imm32zx12:$I3))]> {
+                                                  imm32zx12_timm:$I3))]> {
   let M4 = type;
   let M5 = m5;
 }
@@ -3715,7 +3715,7 @@ class BinaryVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
   : InstVRX<opcode, (outs VR128:$V1), (ins bdxaddr12only:$XBD2, imm32zx4:$M3),
             mnemonic#"\t$V1, $XBD2, $M3",
             [(set (tr.vt tr.op:$V1), (operator bdxaddr12only:$XBD2,
-                                               imm32zx4:$M3))]> {
+                                               imm32zx4_timm:$M3))]> {
   let mayLoad = 1;
   let AccessBytes = bytes;
 }
@@ -3765,7 +3765,7 @@ class BinaryVSI<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class StoreBinaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
-                     Immediate index>
+                     ImmOpWithPattern index>
   : InstVRV<opcode, (outs), (ins VR128:$V1, bdvaddr12only:$VBD2, index:$M3),
             mnemonic#"\t$V1, $VBD2, $M3", []> {
   let mayStore = 1;
@@ -3774,7 +3774,7 @@ class StoreBinaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
 
 class StoreBinaryVRX<string mnemonic, bits<16> opcode,
                      SDPatternOperator operator, TypedReg tr, bits<5> bytes,
-                     Immediate index>
+                     ImmOpWithPattern index>
   : InstVRX<opcode, (outs), (ins tr.op:$V1, bdxaddr12only:$XBD2, index:$M3),
             mnemonic#"\t$V1, $XBD2, $M3",
             [(operator (tr.vt tr.op:$V1), bdxaddr12only:$XBD2, index:$M3)]> {
@@ -3809,7 +3809,7 @@ class CompareRRE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class CompareRI<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-                RegisterOperand cls, Immediate imm>
+                RegisterOperand cls, ImmOpWithPattern imm>
   : InstRIa<opcode, (outs), (ins cls:$R1, imm:$I2),
             mnemonic#"\t$R1, $I2",
             [(set CC, (operator cls:$R1, imm:$I2))]> {
@@ -3817,7 +3817,7 @@ class CompareRI<string mnemonic, bits<12> opcode, SDPatternOperator operator,
 }
 
 class CompareRIL<string mnemonic, bits<12> opcode, SDPatternOperator operator,
-                 RegisterOperand cls, Immediate imm>
+                 RegisterOperand cls, ImmOpWithPattern imm>
   : InstRILa<opcode, (outs), (ins cls:$R1, imm:$I2),
              mnemonic#"\t$R1, $I2",
              [(set CC, (operator cls:$R1, imm:$I2))]> {
@@ -3924,7 +3924,7 @@ class CompareSSb<string mnemonic, bits<8> opcode>
 }
 
 class CompareSI<string mnemonic, bits<8> opcode, SDPatternOperator operator,
-                SDPatternOperator load, Immediate imm,
+                SDPatternOperator load, ImmOpWithPattern imm,
                 AddressingMode mode = bdaddr12only>
   : InstSI<opcode, (outs), (ins mode:$BD1, imm:$I2),
            mnemonic#"\t$BD1, $I2",
@@ -3934,7 +3934,7 @@ class CompareSI<string mnemonic, bits<8> opcode, SDPatternOperator operator,
 }
 
 class CompareSIL<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                 SDPatternOperator load, Immediate imm>
+                 SDPatternOperator load, ImmOpWithPattern imm>
   : InstSIL<opcode, (outs), (ins bdaddr12only:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2",
             [(set CC, (operator (load bdaddr12only:$BD1), imm:$I2))]> {
@@ -3943,7 +3943,7 @@ class CompareSIL<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class CompareSIY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                 SDPatternOperator load, Immediate imm,
+                 SDPatternOperator load, ImmOpWithPattern imm,
                  AddressingMode mode = bdaddr20only>
   : InstSIY<opcode, (outs), (ins mode:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2",
@@ -3954,7 +3954,7 @@ class CompareSIY<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 
 multiclass CompareSIPair<string mnemonic, bits<8> siOpcode, bits<16> siyOpcode,
                          SDPatternOperator operator, SDPatternOperator load,
-                         Immediate imm> {
+                         ImmOpWithPattern imm> {
   let DispKey = mnemonic in {
     let DispSize = "12" in
       def "" : CompareSI<mnemonic, siOpcode, operator, load, imm, bdaddr12pair>;
@@ -4012,7 +4012,7 @@ class TestRXE<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class TestBinarySIL<string mnemonic, bits<16> opcode,
-                    SDPatternOperator operator, Immediate imm>
+                    SDPatternOperator operator, ImmOpWithPattern imm>
   : InstSIL<opcode, (outs), (ins bdaddr12only:$BD1, imm:$I2),
             mnemonic#"\t$BD1, $I2",
             [(set CC, (operator bdaddr12only:$BD1, imm:$I2))]>;
@@ -4073,7 +4073,7 @@ class SideEffectTernaryMemMemMemRRFb<string mnemonic, bits<16> opcode,
 
 class SideEffectTernaryRRFc<string mnemonic, bits<16> opcode,
                             RegisterOperand cls1, RegisterOperand cls2,
-                            Immediate imm>
+                            ImmOpWithPattern imm>
   : InstRRFc<opcode, (outs), (ins cls1:$R1, cls2:$R2, imm:$M3),
              mnemonic#"\t$R1, $R2, $M3", []>;
 
@@ -4086,7 +4086,7 @@ multiclass SideEffectTernaryRRFcOpt<string mnemonic, bits<16> opcode,
 
 class SideEffectTernaryMemMemRRFc<string mnemonic, bits<16> opcode,
                                   RegisterOperand cls1, RegisterOperand cls2,
-                                  Immediate imm>
+                                  ImmOpWithPattern imm>
   : InstRRFc<opcode, (outs cls1:$R1, cls2:$R2),
              (ins cls1:$R1src, cls2:$R2src, imm:$M3),
              mnemonic#"\t$R1, $R2, $M3", []> {
@@ -4221,7 +4221,7 @@ class TernaryRXF<string mnemonic, bits<16> opcode, SDPatternOperator operator,
 }
 
 class TernaryVRIa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                  TypedReg tr1, TypedReg tr2, Immediate imm, Immediate index>
+                  TypedReg tr1, TypedReg tr2, ImmOpWithPattern imm, ImmOpWithPattern index>
   : InstVRIa<opcode, (outs tr1.op:$V1), (ins tr2.op:$V1src, imm:$I2, index:$M3),
              mnemonic#"\t$V1, $I2, $M3",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V1src),
@@ -4237,7 +4237,7 @@ class TernaryVRId<string mnemonic, bits<16> opcode, SDPatternOperator operator,
              mnemonic#"\t$V1, $V2, $V3, $I4",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
                                                   (tr2.vt tr2.op:$V3),
-                                                  imm32zx8:$I4))]> {
+                                                  imm32zx8_timm:$I4))]> {
   let M5 = type;
 }
 
@@ -4252,8 +4252,8 @@ class TernaryVRRa<string mnemonic, bits<16> opcode, SDPatternOperator operator,
              (ins tr2.op:$V2, imm32zx4:$M4, imm32zx4:$M5),
              mnemonic#"\t$V1, $V2, $M4, $M5",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
-                                                  imm32zx4:$M4,
-                                                  imm32zx4:$M5))],
+                                                  imm32zx4_timm:$M4,
+                                                  imm32zx4_timm:$M5))],
              m4or> {
   let M3 = type;
 }
@@ -4285,13 +4285,13 @@ multiclass TernaryOptVRRbSPair<string mnemonic, bits<16> opcode,
                                TypedReg tr1, TypedReg tr2, bits<4> type,
                                bits<4> modifier = 0> {
   def "" : TernaryVRRb<mnemonic, opcode, operator, tr1, tr2, type,
-                       imm32zx4even, !and (modifier, 14)>;
+                       imm32zx4even_timm, !and (modifier, 14)>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3",
                   (!cast<Instruction>(NAME) tr1.op:$V1, tr2.op:$V2,
                                             tr2.op:$V3, 0)>;
   let Defs = [CC] in
     def S : TernaryVRRb<mnemonic##"s", opcode, operator_cc, tr1, tr2, type,
-                        imm32zx4even, !add(!and (modifier, 14), 1)>;
+                        imm32zx4even_timm, !add(!and (modifier, 14), 1)>;
   def : InstAlias<mnemonic#"s\t$V1, $V2, $V3",
                   (!cast<Instruction>(NAME#"S") tr1.op:$V1, tr2.op:$V2,
                                                 tr2.op:$V3, 0)>;
@@ -4314,7 +4314,7 @@ class TernaryVRRc<string mnemonic, bits<16> opcode, SDPatternOperator operator,
              mnemonic#"\t$V1, $V2, $V3, $M4",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
                                                   (tr2.vt tr2.op:$V3),
-                                                  imm32zx4:$M4))]> {
+                                                  imm32zx4_timm:$M4))]> {
   let M5 = 0;
   let M6 = 0;
 }
@@ -4327,7 +4327,7 @@ class TernaryVRRcFloat<string mnemonic, bits<16> opcode,
              mnemonic#"\t$V1, $V2, $V3, $M6",
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V2),
                                                   (tr2.vt tr2.op:$V3),
-                                                  imm32zx4:$M6))]> {
+                                                  imm32zx4_timm:$M6))]> {
   let M4 = type;
   let M5 = m5;
 }
@@ -4429,7 +4429,7 @@ class TernaryVRSbGeneric<string mnemonic, bits<16> opcode>
 }
 
 class TernaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
-                 Immediate index>
+                 ImmOpWithPattern index>
   : InstVRV<opcode, (outs VR128:$V1),
            (ins VR128:$V1src, bdvaddr12only:$VBD2, index:$M3),
            mnemonic#"\t$V1, $VBD2, $M3", []> {
@@ -4440,7 +4440,7 @@ class TernaryVRV<string mnemonic, bits<16> opcode, bits<5> bytes,
 }
 
 class TernaryVRX<string mnemonic, bits<16> opcode, SDPatternOperator operator,
-                 TypedReg tr1, TypedReg tr2, bits<5> bytes, Immediate index>
+                 TypedReg tr1, TypedReg tr2, bits<5> bytes, ImmOpWithPattern index>
   : InstVRX<opcode, (outs tr1.op:$V1),
            (ins tr2.op:$V1src, bdxaddr12only:$XBD2, index:$M3),
            mnemonic#"\t$V1, $XBD2, $M3",
@@ -4461,7 +4461,7 @@ class QuaternaryVRId<string mnemonic, bits<16> opcode, SDPatternOperator operato
              [(set (tr1.vt tr1.op:$V1), (operator (tr2.vt tr2.op:$V1src),
                                                   (tr2.vt tr2.op:$V2),
                                                   (tr2.vt tr2.op:$V3),
-                                                  imm32zx8:$I4))]> {
+                                                  imm32zx8_timm:$I4))]> {
   let Constraints = "$V1 = $V1src";
   let DisableEncoding = "$V1src";
   let M5 = type;
@@ -4480,7 +4480,7 @@ class QuaternaryVRIf<string mnemonic, bits<16> opcode>
   : InstVRIf<opcode, (outs VR128:$V1),
              (ins VR128:$V2, VR128:$V3,
                   imm32zx8:$I4, imm32zx4:$M5),
-             mnemonic#"\t$V1, $V2, $V3, $I4, $M5", []>;
+            mnemonic#"\t$V1, $V2, $V3, $I4, $M5", []>;
 
 class QuaternaryVRIg<string mnemonic, bits<16> opcode>
   : InstVRIg<opcode, (outs VR128:$V1),
@@ -4491,7 +4491,7 @@ class QuaternaryVRIg<string mnemonic, bits<16> opcode>
 class QuaternaryVRRd<string mnemonic, bits<16> opcode,
                      SDPatternOperator operator, TypedReg tr1, TypedReg tr2,
                      TypedReg tr3, TypedReg tr4, bits<4> type,
-                     SDPatternOperator m6mask = imm32zx4, bits<4> m6or = 0>
+                     SDPatternOperator m6mask = imm32zx4_timm, bits<4> m6or = 0>
   : InstVRRd<opcode, (outs tr1.op:$V1),
              (ins tr2.op:$V2, tr3.op:$V3, tr4.op:$V4, m6mask:$M6),
              mnemonic#"\t$V1, $V2, $V3, $V4, $M6",
@@ -4518,14 +4518,14 @@ multiclass QuaternaryOptVRRdSPair<string mnemonic, bits<16> opcode,
                                 bits<4> modifier = 0> {
   def "" : QuaternaryVRRd<mnemonic, opcode, operator,
                           tr1, tr2, tr2, tr2, type,
-                          imm32zx4even, !and (modifier, 14)>;
+                          imm32zx4even_timm, !and (modifier, 14)>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $V4",
                   (!cast<Instruction>(NAME) tr1.op:$V1, tr2.op:$V2,
                                             tr2.op:$V3, tr2.op:$V4, 0)>;
   let Defs = [CC] in
     def S : QuaternaryVRRd<mnemonic##"s", opcode, operator_cc,
                            tr1, tr2, tr2, tr2, type,
-                           imm32zx4even, !add (!and (modifier, 14), 1)>;
+                           imm32zx4even_timm, !add (!and (modifier, 14), 1)>;
   def : InstAlias<mnemonic#"s\t$V1, $V2, $V3, $V4",
                   (!cast<Instruction>(NAME#"S") tr1.op:$V1, tr2.op:$V2,
                                                 tr2.op:$V3, tr2.op:$V4, 0)>;
@@ -4536,7 +4536,7 @@ multiclass QuaternaryOptVRRdSPairGeneric<string mnemonic, bits<16> opcode> {
     def "" : QuaternaryVRRdGeneric<mnemonic, opcode>;
   def : InstAlias<mnemonic#"\t$V1, $V2, $V3, $V4, $M5",
                   (!cast<Instruction>(NAME) VR128:$V1, VR128:$V2, VR128:$V3,
-                                            VR128:$V4, imm32zx4:$M5, 0)>;
+                                            VR128:$V4, imm32zx4_timm:$M5, 0)>;
 }
 
 class SideEffectQuaternaryRRFa<string mnemonic, bits<16> opcode,
@@ -4638,13 +4638,13 @@ class RotateSelectRIEf<string mnemonic, bits<16> opcode, RegisterOperand cls1,
 class PrefetchRXY<string mnemonic, bits<16> opcode, SDPatternOperator operator>
   : InstRXYb<opcode, (outs), (ins imm32zx4:$M1, bdxaddr20only:$XBD2),
              mnemonic##"\t$M1, $XBD2",
-             [(operator imm32zx4:$M1, bdxaddr20only:$XBD2)]>;
+             [(operator imm32zx4_timm:$M1, bdxaddr20only:$XBD2)]>;
 
 class PrefetchRILPC<string mnemonic, bits<12> opcode,
                     SDPatternOperator operator>
-  : InstRILc<opcode, (outs), (ins imm32zx4:$M1, pcrel32:$RI2),
+  : InstRILc<opcode, (outs), (ins imm32zx4_timm:$M1, pcrel32:$RI2),
              mnemonic##"\t$M1, $RI2",
-             [(operator imm32zx4:$M1, pcrel32:$RI2)]> {
+             [(operator imm32zx4_timm:$M1, pcrel32:$RI2)]> {
   // We want PC-relative addresses to be tried ahead of BD and BDX addresses.
   // However, BDXs have two extra operands and are therefore 6 units more
   // complex.
@@ -4691,7 +4691,7 @@ class Pseudo<dag outs, dag ins, list<dag> pattern>
 
 // Like UnaryRI, but expanded after RA depending on the choice of register.
 class UnaryRIPseudo<SDPatternOperator operator, RegisterOperand cls,
-                    Immediate imm>
+                    ImmOpWithPattern imm>
   : Pseudo<(outs cls:$R1), (ins imm:$I2),
            [(set cls:$R1, (operator imm:$I2))]>;
 
@@ -4720,7 +4720,7 @@ class UnaryRRPseudo<string key, SDPatternOperator operator,
 
 // Like BinaryRI, but expanded after RA depending on the choice of register.
 class BinaryRIPseudo<SDPatternOperator operator, RegisterOperand cls,
-                     Immediate imm>
+                     ImmOpWithPattern imm>
   : Pseudo<(outs cls:$R1), (ins cls:$R1src, imm:$I2),
            [(set cls:$R1, (operator cls:$R1src, imm:$I2))]> {
   let Constraints = "$R1 = $R1src";
@@ -4728,13 +4728,13 @@ class BinaryRIPseudo<SDPatternOperator operator, RegisterOperand cls,
 
 // Like BinaryRIE, but expanded after RA depending on the choice of register.
 class BinaryRIEPseudo<SDPatternOperator operator, RegisterOperand cls,
-                      Immediate imm>
+                      ImmOpWithPattern imm>
   : Pseudo<(outs cls:$R1), (ins cls:$R3, imm:$I2),
            [(set cls:$R1, (operator cls:$R3, imm:$I2))]>;
 
 // Like BinaryRIAndK, but expanded after RA depending on the choice of register.
 multiclass BinaryRIAndKPseudo<string key, SDPatternOperator operator,
-                              RegisterOperand cls, Immediate imm> {
+                              RegisterOperand cls, ImmOpWithPattern imm> {
   let NumOpsKey = key in {
     let NumOpsValue = "3" in
       def K : BinaryRIEPseudo<operator, cls, imm>,
@@ -4764,7 +4764,7 @@ class MemFoldPseudo<string mnemonic, RegisterOperand cls, bits<5> bytes,
 
 // Like CompareRI, but expanded after RA depending on the choice of register.
 class CompareRIPseudo<SDPatternOperator operator, RegisterOperand cls,
-                      Immediate imm>
+                      ImmOpWithPattern imm>
   : Pseudo<(outs), (ins cls:$R1, imm:$I2),
            [(set CC, (operator cls:$R1, imm:$I2))]> {
   let isCompare = 1;
@@ -4783,7 +4783,7 @@ class CompareRXYPseudo<SDPatternOperator operator, RegisterOperand cls,
 }
 
 // Like TestBinarySIL, but expanded later.
-class TestBinarySILPseudo<SDPatternOperator operator, Immediate imm>
+class TestBinarySILPseudo<SDPatternOperator operator, ImmOpWithPattern imm>
   : Pseudo<(outs), (ins bdaddr12only:$BD1, imm:$I2),
            [(set CC, (operator bdaddr12only:$BD1, imm:$I2))]>;
 
@@ -4812,7 +4812,7 @@ class CondBinaryRRFaPseudo<RegisterOperand cls1, RegisterOperand cls2,
 
 // Like CondBinaryRIE, but expanded after RA depending on the choice of
 // register.
-class CondBinaryRIEPseudo<RegisterOperand cls, Immediate imm>
+class CondBinaryRIEPseudo<RegisterOperand cls, ImmOpWithPattern imm>
   : Pseudo<(outs cls:$R1),
            (ins cls:$R1src, imm:$I2, cond4:$valid, cond4:$M3),
            [(set cls:$R1, (z_select_ccmask imm:$I2, cls:$R1src,
@@ -4876,7 +4876,7 @@ class SelectWrapper<ValueType vt, RegisterOperand cls>
   : Pseudo<(outs cls:$dst),
            (ins cls:$src1, cls:$src2, imm32zx4:$valid, imm32zx4:$cc),
            [(set (vt cls:$dst), (z_select_ccmask cls:$src1, cls:$src2,
-                                            imm32zx4:$valid, imm32zx4:$cc))]> {
+                                            imm32zx4_timm:$valid, imm32zx4_timm:$cc))]> {
   let usesCustomInserter = 1;
   let hasNoSchedulingInfo = 1;
   let Uses = [CC];
@@ -4890,12 +4890,12 @@ multiclass CondStores<RegisterOperand cls, SDPatternOperator store,
     def "" : Pseudo<(outs),
                     (ins cls:$new, mode:$addr, imm32zx4:$valid, imm32zx4:$cc),
                     [(store (z_select_ccmask cls:$new, (load mode:$addr),
-                                             imm32zx4:$valid, imm32zx4:$cc),
+                                             imm32zx4_timm:$valid, imm32zx4_timm:$cc),
                             mode:$addr)]>;
     def Inv : Pseudo<(outs),
                      (ins cls:$new, mode:$addr, imm32zx4:$valid, imm32zx4:$cc),
                      [(store (z_select_ccmask (load mode:$addr), cls:$new,
-                                              imm32zx4:$valid, imm32zx4:$cc),
+                                              imm32zx4_timm:$valid, imm32zx4_timm:$cc),
                               mode:$addr)]>;
   }
 }
@@ -4917,11 +4917,11 @@ class AtomicLoadBinary<SDPatternOperator operator, RegisterOperand cls,
 // Specializations of AtomicLoadWBinary.
 class AtomicLoadBinaryReg32<SDPatternOperator operator>
   : AtomicLoadBinary<operator, GR32, (i32 GR32:$src2), GR32>;
-class AtomicLoadBinaryImm32<SDPatternOperator operator, Immediate imm>
+class AtomicLoadBinaryImm32<SDPatternOperator operator, ImmOpWithPattern imm>
   : AtomicLoadBinary<operator, GR32, (i32 imm:$src2), imm>;
 class AtomicLoadBinaryReg64<SDPatternOperator operator>
   : AtomicLoadBinary<operator, GR64, (i64 GR64:$src2), GR64>;
-class AtomicLoadBinaryImm64<SDPatternOperator operator, Immediate imm>
+class AtomicLoadBinaryImm64<SDPatternOperator operator, ImmOpWithPattern imm>
   : AtomicLoadBinary<operator, GR64, (i64 imm:$src2), imm>;
 
 // OPERATOR is ATOMIC_SWAPW or an ATOMIC_LOADW_* operation.  PAT and OPERAND
@@ -4944,7 +4944,7 @@ class AtomicLoadWBinary<SDPatternOperator operator, dag pat,
 // Specializations of AtomicLoadWBinary.
 class AtomicLoadWBinaryReg<SDPatternOperator operator>
   : AtomicLoadWBinary<operator, (i32 GR32:$src2), GR32>;
-class AtomicLoadWBinaryImm<SDPatternOperator operator, Immediate imm>
+class AtomicLoadWBinaryImm<SDPatternOperator operator, ImmOpWithPattern imm>
   : AtomicLoadWBinary<operator, (i32 imm:$src2), imm>;
 
 // A pseudo instruction that is a direct alias of a real instruction.
@@ -4979,7 +4979,7 @@ class StoreAliasVRX<SDPatternOperator operator, TypedReg tr,
 
 // An alias of a BinaryRI, but with different register sizes.
 class BinaryAliasRI<SDPatternOperator operator, RegisterOperand cls,
-                    Immediate imm>
+                    ImmOpWithPattern imm>
   : Alias<4, (outs cls:$R1), (ins cls:$R1src, imm:$I2),
           [(set cls:$R1, (operator cls:$R1src, imm:$I2))]> {
   let Constraints = "$R1 = $R1src";
@@ -4987,7 +4987,7 @@ class BinaryAliasRI<SDPatternOperator operator, RegisterOperand cls,
 
 // An alias of a BinaryRIL, but with different register sizes.
 class BinaryAliasRIL<SDPatternOperator operator, RegisterOperand cls,
-                     Immediate imm>
+                     ImmOpWithPattern imm>
   : Alias<6, (outs cls:$R1), (ins cls:$R1src, imm:$I2),
           [(set cls:$R1, (operator cls:$R1src, imm:$I2))]> {
   let Constraints = "$R1 = $R1src";
@@ -4999,7 +4999,7 @@ class BinaryAliasVRRf<RegisterOperand cls>
 
 // An alias of a CompareRI, but with different register sizes.
 class CompareAliasRI<SDPatternOperator operator, RegisterOperand cls,
-                     Immediate imm>
+                     ImmOpWithPattern imm>
   : Alias<4, (outs), (ins cls:$R1, imm:$I2),
           [(set CC, (operator cls:$R1, imm:$I2))]> {
   let isCompare = 1;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
index dd2950ded099e..bc783608d45b7 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.cpp
@@ -46,22 +46,12 @@ using namespace llvm;
 #include "SystemZGenInstrInfo.inc"
 
 #define DEBUG_TYPE "systemz-II"
-STATISTIC(LOCRMuxJumps, "Number of LOCRMux jump-sequences (lower is better)");
 
 // Return a mask with Count low bits set.
 static uint64_t allOnes(unsigned int Count) {
   return Count == 0 ? 0 : (uint64_t(1) << (Count - 1) << 1) - 1;
 }
 
-// Reg should be a 32-bit GPR.  Return true if it is a high register rather
-// than a low register.
-static bool isHighReg(unsigned int Reg) {
-  if (SystemZ::GRH32BitRegClass.contains(Reg))
-    return true;
-  assert(SystemZ::GR32BitRegClass.contains(Reg) && "Invalid GRX32");
-  return false;
-}
-
 // Pin the vtable to this file.
 void SystemZInstrInfo::anchor() {}
 
@@ -148,7 +138,7 @@ void SystemZInstrInfo::expandRIPseudo(MachineInstr &MI, unsigned LowOpcode,
                                       unsigned HighOpcode,
                                       bool ConvertHigh) const {
   Register Reg = MI.getOperand(0).getReg();
-  bool IsHigh = isHighReg(Reg);
+  bool IsHigh = SystemZ::isHighReg(Reg);
   MI.setDesc(get(IsHigh ? HighOpcode : LowOpcode));
   if (IsHigh && ConvertHigh)
     MI.getOperand(1).setImm(uint32_t(MI.getOperand(1).getImm()));
@@ -163,8 +153,8 @@ void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode,
                                        unsigned HighOpcode) const {
   Register DestReg = MI.getOperand(0).getReg();
   Register SrcReg = MI.getOperand(1).getReg();
-  bool DestIsHigh = isHighReg(DestReg);
-  bool SrcIsHigh = isHighReg(SrcReg);
+  bool DestIsHigh = SystemZ::isHighReg(DestReg);
+  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);
   if (!DestIsHigh && !SrcIsHigh)
     MI.setDesc(get(LowOpcodeK));
   else {
@@ -185,8 +175,9 @@ void SystemZInstrInfo::expandRIEPseudo(MachineInstr &MI, unsigned LowOpcode,
 void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode,
                                        unsigned HighOpcode) const {
   Register Reg = MI.getOperand(0).getReg();
-  unsigned Opcode = getOpcodeForOffset(isHighReg(Reg) ? HighOpcode : LowOpcode,
-                                       MI.getOperand(2).getImm());
+  unsigned Opcode = getOpcodeForOffset(
+      SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode,
+      MI.getOperand(2).getImm());
   MI.setDesc(get(Opcode));
 }
 
@@ -196,92 +187,10 @@ void SystemZInstrInfo::expandRXYPseudo(MachineInstr &MI, unsigned LowOpcode,
 void SystemZInstrInfo::expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode,
                                        unsigned HighOpcode) const {
   Register Reg = MI.getOperand(0).getReg();
-  unsigned Opcode = isHighReg(Reg) ? HighOpcode : LowOpcode;
+  unsigned Opcode = SystemZ::isHighReg(Reg) ? HighOpcode : LowOpcode;
   MI.setDesc(get(Opcode));
 }
 
-// MI is a load-register-on-condition pseudo instruction.  Replace it with
-// LowOpcode if source and destination are both low GR32s and HighOpcode if
-// source and destination are both high GR32s.
-void SystemZInstrInfo::expandLOCRPseudo(MachineInstr &MI, unsigned LowOpcode,
-                                        unsigned HighOpcode) const {
-  Register DestReg = MI.getOperand(0).getReg();
-  Register SrcReg = MI.getOperand(2).getReg();
-  bool DestIsHigh = isHighReg(DestReg);
-  bool SrcIsHigh = isHighReg(SrcReg);
-
-  if (!DestIsHigh && !SrcIsHigh)
-    MI.setDesc(get(LowOpcode));
-  else if (DestIsHigh && SrcIsHigh)
-    MI.setDesc(get(HighOpcode));
-  else
-    LOCRMuxJumps++;
-
-  // If we were unable to implement the pseudo with a single instruction, we
-  // need to convert it back into a branch sequence.  This cannot be done here
-  // since the caller of expandPostRAPseudo does not handle changes to the CFG
-  // correctly.  This change is defered to the SystemZExpandPseudo pass.
-}
-
-// MI is a select pseudo instruction.  Replace it with LowOpcode if source
-// and destination are all low GR32s and HighOpcode if source and destination
-// are all high GR32s.  Otherwise, use the two-operand MixedOpcode.
-void SystemZInstrInfo::expandSELRPseudo(MachineInstr &MI, unsigned LowOpcode,
-                                        unsigned HighOpcode,
-                                        unsigned MixedOpcode) const {
-  Register DestReg = MI.getOperand(0).getReg();
-  Register Src1Reg = MI.getOperand(1).getReg();
-  Register Src2Reg = MI.getOperand(2).getReg();
-  bool DestIsHigh = isHighReg(DestReg);
-  bool Src1IsHigh = isHighReg(Src1Reg);
-  bool Src2IsHigh = isHighReg(Src2Reg);
-
-  // If sources and destination aren't all high or all low, we may be able to
-  // simplify the operation by moving one of the sources to the destination
-  // first.  But only if this doesn't clobber the other source.
-  if (DestReg != Src1Reg && DestReg != Src2Reg) {
-    if (DestIsHigh != Src1IsHigh) {
-      emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src1Reg,
-                    SystemZ::LR, 32, MI.getOperand(1).isKill(),
-                    MI.getOperand(1).isUndef());
-      MI.getOperand(1).setReg(DestReg);
-      Src1Reg = DestReg;
-      Src1IsHigh = DestIsHigh;
-    } else if (DestIsHigh != Src2IsHigh) {
-      emitGRX32Move(*MI.getParent(), MI, MI.getDebugLoc(), DestReg, Src2Reg,
-                    SystemZ::LR, 32, MI.getOperand(2).isKill(),
-                    MI.getOperand(2).isUndef());
-      MI.getOperand(2).setReg(DestReg);
-      Src2Reg = DestReg;
-      Src2IsHigh = DestIsHigh;
-    }
-  }
-
-  // If the destination (now) matches one source, prefer this to be first.
-  if (DestReg != Src1Reg && DestReg == Src2Reg) {
-    commuteInstruction(MI, false, 1, 2);
-    std::swap(Src1Reg, Src2Reg);
-    std::swap(Src1IsHigh, Src2IsHigh);
-  }
-
-  if (!DestIsHigh && !Src1IsHigh && !Src2IsHigh)
-    MI.setDesc(get(LowOpcode));
-  else if (DestIsHigh && Src1IsHigh && Src2IsHigh)
-    MI.setDesc(get(HighOpcode));
-  else {
-    // Given the simplifcation above, we must already have a two-operand case.
-    assert (DestReg == Src1Reg);
-    MI.setDesc(get(MixedOpcode));
-    MI.tieOperands(0, 1);
-    LOCRMuxJumps++;
-  }
-
-  // If we were unable to implement the pseudo with a single instruction, we
-  // need to convert it back into a branch sequence.  This cannot be done here
-  // since the caller of expandPostRAPseudo does not handle changes to the CFG
-  // correctly.  This change is defered to the SystemZExpandPseudo pass.
-}
-
 // MI is an RR-style pseudo instruction that zero-extends the low Size bits
 // of one GRX32 into another.  Replace it with LowOpcode if both operands
 // are low registers, otherwise use RISB[LH]G.
@@ -341,8 +250,8 @@ SystemZInstrInfo::emitGRX32Move(MachineBasicBlock &MBB,
                                 unsigned Size, bool KillSrc,
                                 bool UndefSrc) const {
   unsigned Opcode;
-  bool DestIsHigh = isHighReg(DestReg);
-  bool SrcIsHigh = isHighReg(SrcReg);
+  bool DestIsHigh = SystemZ::isHighReg(DestReg);
+  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);
   if (DestIsHigh && SrcIsHigh)
     Opcode = SystemZ::RISBHH;
   else if (DestIsHigh && !SrcIsHigh)
@@ -1359,15 +1268,6 @@ bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     expandLOCPseudo(MI, SystemZ::LOCHI, SystemZ::LOCHHI);
     return true;
 
-  case SystemZ::LOCRMux:
-    expandLOCRPseudo(MI, SystemZ::LOCR, SystemZ::LOCFHR);
-    return true;
-
-  case SystemZ::SELRMux:
-    expandSELRPseudo(MI, SystemZ::SELR, SystemZ::SELFHR,
-                         SystemZ::LOCRMux);
-    return true;
-
   case SystemZ::STCMux:
     expandRXYPseudo(MI, SystemZ::STC, SystemZ::STCH);
     return true;
@@ -1469,8 +1369,8 @@ bool SystemZInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
     return true;
 
   case SystemZ::RISBMux: {
-    bool DestIsHigh = isHighReg(MI.getOperand(0).getReg());
-    bool SrcIsHigh = isHighReg(MI.getOperand(2).getReg());
+    bool DestIsHigh = SystemZ::isHighReg(MI.getOperand(0).getReg());
+    bool SrcIsHigh = SystemZ::isHighReg(MI.getOperand(2).getReg());
     if (SrcIsHigh == DestIsHigh)
       MI.setDesc(get(DestIsHigh ? SystemZ::RISBHH : SystemZ::RISBLL));
     else {
@@ -1850,8 +1750,7 @@ void SystemZInstrInfo::loadImmediate(MachineBasicBlock &MBB,
 
 bool SystemZInstrInfo::
 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                const MachineInstr &MIb,
-                                AliasAnalysis *AA) const {
+                                const MachineInstr &MIb) const {
 
   if (!MIa.hasOneMemOperand() || !MIb.hasOneMemOperand())
     return false;
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
index 134ed38a41aa0..6dc6e72aa52a0 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZInstrInfo.h
@@ -170,10 +170,6 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
                        unsigned HighOpcode) const;
   void expandLOCPseudo(MachineInstr &MI, unsigned LowOpcode,
                        unsigned HighOpcode) const;
-  void expandLOCRPseudo(MachineInstr &MI, unsigned LowOpcode,
-                        unsigned HighOpcode) const;
-  void expandSELRPseudo(MachineInstr &MI, unsigned LowOpcode,
-                        unsigned HighOpcode, unsigned MixedOpcode) const;
   void expandZExtPseudo(MachineInstr &MI, unsigned LowOpcode,
                         unsigned Size) const;
   void expandLoadStackGuard(MachineInstr *MI) const;
@@ -332,8 +328,7 @@ class SystemZInstrInfo : public SystemZGenInstrInfo {
   // memory addresses and false otherwise.
   bool
   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
-                                  const MachineInstr &MIb,
-                                  AliasAnalysis *AA = nullptr) const override;
+                                  const MachineInstr &MIb) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/SystemZ/SystemZInstrVector.td b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
index 73c5028f5e321..02364bbda5c18 100644
--- a/llvm/lib/Target/SystemZ/SystemZInstrVector.td
+++ b/llvm/lib/Target/SystemZ/SystemZInstrVector.td
@@ -60,7 +60,7 @@ let Predicates = [FeatureVector] in {
     // Generate byte mask.
     def VZERO : InherentVRIa<"vzero", 0xE744, 0>;
     def VONE  : InherentVRIa<"vone", 0xE744, 0xffff>;
-    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16>;
+    def VGBM  : UnaryVRIa<"vgbm", 0xE744, z_byte_mask, v128b, imm32zx16_timm>;
 
     // Generate mask.
     def VGM  : BinaryVRIbGeneric<"vgm", 0xE746>;
@@ -71,10 +71,10 @@ let Predicates = [FeatureVector] in {
 
     // Replicate immediate.
     def VREPI  : UnaryVRIaGeneric<"vrepi", 0xE745, imm32sx16>;
-    def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16, 0>;
-    def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16, 1>;
-    def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16, 2>;
-    def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16, 3>;
+    def VREPIB : UnaryVRIa<"vrepib", 0xE745, z_replicate, v128b, imm32sx16_timm, 0>;
+    def VREPIH : UnaryVRIa<"vrepih", 0xE745, z_replicate, v128h, imm32sx16_timm, 1>;
+    def VREPIF : UnaryVRIa<"vrepif", 0xE745, z_replicate, v128f, imm32sx16_timm, 2>;
+    def VREPIG : UnaryVRIa<"vrepig", 0xE745, z_replicate, v128g, imm32sx16_timm, 3>;
   }
 
   // Load element immediate.
@@ -116,7 +116,7 @@ let Predicates = [FeatureVector] in {
                                (ins bdxaddr12only:$XBD2, imm32zx4:$M3),
                        "lcbb\t$R1, $XBD2, $M3",
                        [(set GR32:$R1, (int_s390_lcbb bdxaddr12only:$XBD2,
-                                                      imm32zx4:$M3))]>;
+                                                      imm32zx4_timm:$M3))]>;
 
   // Load with length.  The number of loaded bytes is only known at run time.
   def VLL : BinaryVRSb<"vll", 0xE737, int_s390_vll, 0>;
@@ -362,9 +362,9 @@ let Predicates = [FeatureVector] in {
   def VREPH : BinaryVRIc<"vreph", 0xE74D, z_splat, v128h, v128h, 1>;
   def VREPF : BinaryVRIc<"vrepf", 0xE74D, z_splat, v128f, v128f, 2>;
   def VREPG : BinaryVRIc<"vrepg", 0xE74D, z_splat, v128g, v128g, 3>;
-  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16:$index)),
+  def : Pat<(v4f32 (z_splat VR128:$vec, imm32zx16_timm:$index)),
             (VREPF VR128:$vec, imm32zx16:$index)>;
-  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16:$index)),
+  def : Pat<(v2f64 (z_splat VR128:$vec, imm32zx16_timm:$index)),
             (VREPG VR128:$vec, imm32zx16:$index)>;
 
   // Select.
@@ -778,7 +778,7 @@ let Predicates = [FeatureVector] in {
 
   // Shift left double by byte.
   def VSLDB : TernaryVRId<"vsldb", 0xE777, z_shl_double, v128b, v128b, 0>;
-  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8:$z),
+  def : Pat<(int_s390_vsldb VR128:$x, VR128:$y, imm32zx8_timm:$z),
             (VSLDB VR128:$x, VR128:$y, imm32zx8:$z)>;
 
   // Shift left double by bit.
diff --git a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
index 452c439e66872..7241112295693 100644
--- a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
@@ -85,9 +85,9 @@ struct MBBInfo {
   // This value never changes.
   uint64_t Size = 0;
 
-  // The minimum alignment of the block, as a log2 value.
+  // The minimum alignment of the block.
   // This value never changes.
-  unsigned LogAlignment = 0;
+  Align Alignment;
 
   // The number of terminators in this block.  This value never changes.
   unsigned NumTerminators = 0;
@@ -179,17 +179,16 @@ const uint64_t MaxForwardRange = 0xfffe;
 // instructions.
 void SystemZLongBranch::skipNonTerminators(BlockPosition &Position,
                                            MBBInfo &Block) {
-  if (Block.LogAlignment > Position.KnownBits) {
+  if (Log2(Block.Alignment) > Position.KnownBits) {
     // When calculating the address of Block, we need to conservatively
     // assume that Block had the worst possible misalignment.
-    Position.Address += ((uint64_t(1) << Block.LogAlignment) -
-                         (uint64_t(1) << Position.KnownBits));
-    Position.KnownBits = Block.LogAlignment;
+    Position.Address +=
+        (Block.Alignment.value() - (uint64_t(1) << Position.KnownBits));
+    Position.KnownBits = Log2(Block.Alignment);
   }
 
   // Align the addresses.
-  uint64_t AlignMask = (uint64_t(1) << Block.LogAlignment) - 1;
-  Position.Address = (Position.Address + AlignMask) & ~AlignMask;
+  Position.Address = alignTo(Position.Address, Block.Alignment);
 
   // Record the block's position.
   Block.Address = Position.Address;
@@ -282,7 +281,7 @@ uint64_t SystemZLongBranch::initMBBInfo() {
     MBBInfo &Block = MBBs[I];
 
     // Record the alignment, for quick access.
-    Block.LogAlignment = MBB->getLogAlignment();
+    Block.Alignment = MBB->getAlignment();
 
     // Calculate the size of the fixed part of the block.
     MachineBasicBlock::iterator MI = MBB->begin();
diff --git a/llvm/lib/Target/SystemZ/SystemZOperands.td b/llvm/lib/Target/SystemZ/SystemZOperands.td
index 56632e1529a2c..b2bab68a6274f 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperands.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperands.td
@@ -21,15 +21,32 @@ class ImmediateTLSAsmOperand<string name>
   let RenderMethod = "addImmTLSOperands";
 }
 
+class ImmediateOp<ValueType vt, string asmop> : Operand<vt> {
+  let PrintMethod = "print"##asmop##"Operand";
+  let DecoderMethod = "decode"##asmop##"Operand";
+  let ParserMatchClass = !cast<AsmOperandClass>(asmop);
+}
+
+class ImmOpWithPattern<ValueType vt, string asmop, code pred, SDNodeXForm xform,
+      SDNode ImmNode = imm> :
+  ImmediateOp<vt, asmop>, PatLeaf<(vt ImmNode), pred, xform>;
+
+// class ImmediatePatLeaf<ValueType vt, code pred,
+//       SDNodeXForm xform, SDNode ImmNode>
+//   : PatLeaf<(vt ImmNode), pred, xform>;
+
+
 // Constructs both a DAG pattern and instruction operand for an immediate
 // of type VT.  PRED returns true if a node is acceptable and XFORM returns
 // the operand value associated with the node.  ASMOP is the name of the
 // associated asm operand, and also forms the basis of the asm print method.
-class Immediate<ValueType vt, code pred, SDNodeXForm xform, string asmop>
-  : PatLeaf<(vt imm), pred, xform>, Operand<vt> {
-  let PrintMethod = "print"##asmop##"Operand";
-  let DecoderMethod = "decode"##asmop##"Operand";
-  let ParserMatchClass = !cast<AsmOperandClass>(asmop);
+multiclass Immediate<ValueType vt, code pred, SDNodeXForm xform, string asmop> {
+  // def "" : ImmediateOp<vt, asmop>,
+  //          PatLeaf<(vt imm), pred, xform>;
+  def "" : ImmOpWithPattern<vt, asmop, pred, xform>;
+
+//  def _timm : PatLeaf<(vt timm), pred, xform>;
+  def _timm : ImmOpWithPattern<vt, asmop, pred, xform, timm>;
 }
 
 // Constructs an asm operand for a PC-relative address.  SIZE says how
@@ -295,87 +312,87 @@ def U48Imm : ImmediateAsmOperand<"U48Imm">;
 
 // Immediates for the lower and upper 16 bits of an i32, with the other
 // bits of the i32 being zero.
-def imm32ll16 : Immediate<i32, [{
+defm imm32ll16 : Immediate<i32, [{
   return SystemZ::isImmLL(N->getZExtValue());
 }], LL16, "U16Imm">;
 
-def imm32lh16 : Immediate<i32, [{
+defm imm32lh16 : Immediate<i32, [{
   return SystemZ::isImmLH(N->getZExtValue());
 }], LH16, "U16Imm">;
 
 // Immediates for the lower and upper 16 bits of an i32, with the other
 // bits of the i32 being one.
-def imm32ll16c : Immediate<i32, [{
+defm imm32ll16c : Immediate<i32, [{
   return SystemZ::isImmLL(uint32_t(~N->getZExtValue()));
 }], LL16, "U16Imm">;
 
-def imm32lh16c : Immediate<i32, [{
+defm imm32lh16c : Immediate<i32, [{
   return SystemZ::isImmLH(uint32_t(~N->getZExtValue()));
 }], LH16, "U16Imm">;
 
 // Short immediates
-def imm32zx1 : Immediate<i32, [{
+defm imm32zx1 : Immediate<i32, [{
   return isUInt<1>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U1Imm">;
 
-def imm32zx2 : Immediate<i32, [{
+defm imm32zx2 : Immediate<i32, [{
   return isUInt<2>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U2Imm">;
 
-def imm32zx3 : Immediate<i32, [{
+defm imm32zx3 : Immediate<i32, [{
   return isUInt<3>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U3Imm">;
 
-def imm32zx4 : Immediate<i32, [{
+defm imm32zx4 : Immediate<i32, [{
   return isUInt<4>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U4Imm">;
 
 // Note: this enforces an even value during code generation only.
 // When used from the assembler, any 4-bit value is allowed.
-def imm32zx4even : Immediate<i32, [{
+defm imm32zx4even : Immediate<i32, [{
   return isUInt<4>(N->getZExtValue());
 }], UIMM8EVEN, "U4Imm">;
 
-def imm32zx6 : Immediate<i32, [{
+defm imm32zx6 : Immediate<i32, [{
   return isUInt<6>(N->getZExtValue());
 }], NOOP_SDNodeXForm, "U6Imm">;
 
-def imm32sx8 : Immediate<i32, [{
+defm imm32sx8 : Immediate<i32, [{
   return isInt<8>(N->getSExtValue());
 }], SIMM8, "S8Imm">;
 
-def imm32zx8 : Immediate<i32, [{
+defm imm32zx8 : Immediate<i32, [{
   return isUInt<8>(N->getZExtValue());
 }], UIMM8, "U8Imm">;
 
-def imm32zx8trunc : Immediate<i32, [{}], UIMM8, "U8Imm">;
+defm imm32zx8trunc : Immediate<i32, [{}], UIMM8, "U8Imm">;
 
-def imm32zx12 : Immediate<i32, [{
+defm imm32zx12 : Immediate<i32, [{
   return isUInt<12>(N->getZExtValue());
 }], UIMM12, "U12Imm">;
 
-def imm32sx16 : Immediate<i32, [{
+defm imm32sx16 : Immediate<i32, [{
   return isInt<16>(N->getSExtValue());
 }], SIMM16, "S16Imm">;
 
-def imm32sx16n : Immediate<i32, [{
+defm imm32sx16n : Immediate<i32, [{
   return isInt<16>(-N->getSExtValue());
 }], NEGSIMM16, "S16Imm">;
 
-def imm32zx16 : Immediate<i32, [{
+defm imm32zx16 : Immediate<i32, [{
   return isUInt<16>(N->getZExtValue());
 }], UIMM16, "U16Imm">;
 
-def imm32sx16trunc : Immediate<i32, [{}], SIMM16, "S16Imm">;
-def imm32zx16trunc : Immediate<i32, [{}], UIMM16, "U16Imm">;
+defm imm32sx16trunc : Immediate<i32, [{}], SIMM16, "S16Imm">;
+defm imm32zx16trunc : Immediate<i32, [{}], UIMM16, "U16Imm">;
 
 // Full 32-bit immediates.  we need both signed and unsigned versions
 // because the assembler is picky.  E.g. AFI requires signed operands
 // while NILF requires unsigned ones.
-def simm32 : Immediate<i32, [{}], SIMM32, "S32Imm">;
-def uimm32 : Immediate<i32, [{}], UIMM32, "U32Imm">;
+defm simm32 : Immediate<i32, [{}], SIMM32, "S32Imm">;
+defm uimm32 : Immediate<i32, [{}], UIMM32, "U32Imm">;
 
-def simm32n : Immediate<i32, [{
+defm simm32n : Immediate<i32, [{
   return isInt<32>(-N->getSExtValue());
 }], NEGSIMM32, "S32Imm">;
 
@@ -387,107 +404,107 @@ def imm32 : ImmLeaf<i32, [{}]>;
 
 // Immediates for 16-bit chunks of an i64, with the other bits of the
 // i32 being zero.
-def imm64ll16 : Immediate<i64, [{
+defm imm64ll16 : Immediate<i64, [{
   return SystemZ::isImmLL(N->getZExtValue());
 }], LL16, "U16Imm">;
 
-def imm64lh16 : Immediate<i64, [{
+defm imm64lh16 : Immediate<i64, [{
   return SystemZ::isImmLH(N->getZExtValue());
 }], LH16, "U16Imm">;
 
-def imm64hl16 : Immediate<i64, [{
+defm imm64hl16 : Immediate<i64, [{
   return SystemZ::isImmHL(N->getZExtValue());
 }], HL16, "U16Imm">;
 
-def imm64hh16 : Immediate<i64, [{
+defm imm64hh16 : Immediate<i64, [{
   return SystemZ::isImmHH(N->getZExtValue());
 }], HH16, "U16Imm">;
 
 // Immediates for 16-bit chunks of an i64, with the other bits of the
 // i32 being one.
-def imm64ll16c : Immediate<i64, [{
+defm imm64ll16c : Immediate<i64, [{
   return SystemZ::isImmLL(uint64_t(~N->getZExtValue()));
 }], LL16, "U16Imm">;
 
-def imm64lh16c : Immediate<i64, [{
+defm imm64lh16c : Immediate<i64, [{
   return SystemZ::isImmLH(uint64_t(~N->getZExtValue()));
 }], LH16, "U16Imm">;
 
-def imm64hl16c : Immediate<i64, [{
+defm imm64hl16c : Immediate<i64, [{
   return SystemZ::isImmHL(uint64_t(~N->getZExtValue()));
 }], HL16, "U16Imm">;
 
-def imm64hh16c : Immediate<i64, [{
+defm imm64hh16c : Immediate<i64, [{
   return SystemZ::isImmHH(uint64_t(~N->getZExtValue()));
 }], HH16, "U16Imm">;
 
 // Immediates for the lower and upper 32 bits of an i64, with the other
 // bits of the i32 being zero.
-def imm64lf32 : Immediate<i64, [{
+defm imm64lf32 : Immediate<i64, [{
   return SystemZ::isImmLF(N->getZExtValue());
 }], LF32, "U32Imm">;
 
-def imm64hf32 : Immediate<i64, [{
+defm imm64hf32 : Immediate<i64, [{
   return SystemZ::isImmHF(N->getZExtValue());
 }], HF32, "U32Imm">;
 
 // Immediates for the lower and upper 32 bits of an i64, with the other
 // bits of the i32 being one.
-def imm64lf32c : Immediate<i64, [{
+defm imm64lf32c : Immediate<i64, [{
   return SystemZ::isImmLF(uint64_t(~N->getZExtValue()));
 }], LF32, "U32Imm">;
 
-def imm64hf32c : Immediate<i64, [{
+defm imm64hf32c : Immediate<i64, [{
   return SystemZ::isImmHF(uint64_t(~N->getZExtValue()));
 }], HF32, "U32Imm">;
 
 // Negated immediates that fit LF32 or LH16.
-def imm64lh16n : Immediate<i64, [{
+defm imm64lh16n : Immediate<i64, [{
   return SystemZ::isImmLH(uint64_t(-N->getZExtValue()));
 }], NEGLH16, "U16Imm">;
 
-def imm64lf32n : Immediate<i64, [{
+defm imm64lf32n : Immediate<i64, [{
   return SystemZ::isImmLF(uint64_t(-N->getZExtValue()));
 }], NEGLF32, "U32Imm">;
 
 // Short immediates.
-def imm64sx8 : Immediate<i64, [{
+defm imm64sx8 : Immediate<i64, [{
   return isInt<8>(N->getSExtValue());
 }], SIMM8, "S8Imm">;
 
-def imm64zx8 : Immediate<i64, [{
+defm imm64zx8 : Immediate<i64, [{
   return isUInt<8>(N->getSExtValue());
 }], UIMM8, "U8Imm">;
 
-def imm64sx16 : Immediate<i64, [{
+defm imm64sx16 : Immediate<i64, [{
   return isInt<16>(N->getSExtValue());
 }], SIMM16, "S16Imm">;
 
-def imm64sx16n : Immediate<i64, [{
+defm imm64sx16n : Immediate<i64, [{
   return isInt<16>(-N->getSExtValue());
 }], NEGSIMM16, "S16Imm">;
 
-def imm64zx16 : Immediate<i64, [{
+defm imm64zx16 : Immediate<i64, [{
   return isUInt<16>(N->getZExtValue());
 }], UIMM16, "U16Imm">;
 
-def imm64sx32 : Immediate<i64, [{
+defm imm64sx32 : Immediate<i64, [{
   return isInt<32>(N->getSExtValue());
 }], SIMM32, "S32Imm">;
 
-def imm64sx32n : Immediate<i64, [{
+defm imm64sx32n : Immediate<i64, [{
   return isInt<32>(-N->getSExtValue());
 }], NEGSIMM32, "S32Imm">;
 
-def imm64zx32 : Immediate<i64, [{
+defm imm64zx32 : Immediate<i64, [{
   return isUInt<32>(N->getZExtValue());
 }], UIMM32, "U32Imm">;
 
-def imm64zx32n : Immediate<i64, [{
+defm imm64zx32n : Immediate<i64, [{
   return isUInt<32>(-N->getSExtValue());
 }], NEGUIMM32, "U32Imm">;
 
-def imm64zx48 : Immediate<i64, [{
+defm imm64zx48 : Immediate<i64, [{
   return isUInt<64>(N->getZExtValue());
 }], UIMM48, "U48Imm">;
 
@@ -637,7 +654,7 @@ def bdvaddr12only     : BDVMode<            "64", "12">;
 //===----------------------------------------------------------------------===//
 
 // A 4-bit condition-code mask.
-def cond4 : PatLeaf<(i32 imm), [{ return (N->getZExtValue() < 16); }]>,
+def cond4 : PatLeaf<(i32 timm), [{ return (N->getZExtValue() < 16); }]>,
             Operand<i32> {
   let PrintMethod = "printCond4Operand";
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZOperators.td b/llvm/lib/Target/SystemZ/SystemZOperators.td
index 15bd12bc98a40..6fe383e64b740 100644
--- a/llvm/lib/Target/SystemZ/SystemZOperators.td
+++ b/llvm/lib/Target/SystemZ/SystemZOperators.td
@@ -472,17 +472,17 @@ def z_subcarry : PatFrag<(ops node:$lhs, node:$rhs),
                               (z_subcarry_1 node:$lhs, node:$rhs, CC)>;
 
 // Signed and unsigned comparisons.
-def z_scmp : PatFrag<(ops node:$a, node:$b), (z_icmp node:$a, node:$b, imm), [{
+def z_scmp : PatFrag<(ops node:$a, node:$b), (z_icmp node:$a, node:$b, timm), [{
   unsigned Type = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
   return Type != SystemZICMP::UnsignedOnly;
 }]>;
-def z_ucmp : PatFrag<(ops node:$a, node:$b), (z_icmp node:$a, node:$b, imm), [{
+def z_ucmp : PatFrag<(ops node:$a, node:$b), (z_icmp node:$a, node:$b, timm), [{
   unsigned Type = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
   return Type != SystemZICMP::SignedOnly;
 }]>;
 
 // Register- and memory-based TEST UNDER MASK.
-def z_tm_reg : PatFrag<(ops node:$a, node:$b), (z_tm node:$a, node:$b, imm)>;
+def z_tm_reg : PatFrag<(ops node:$a, node:$b), (z_tm node:$a, node:$b, timm)>;
 def z_tm_mem : PatFrag<(ops node:$a, node:$b), (z_tm node:$a, node:$b, 0)>;
 
 // Register sign-extend operations.  Sub-32-bit values are represented as i32s.
diff --git a/llvm/lib/Target/SystemZ/SystemZPatterns.td b/llvm/lib/Target/SystemZ/SystemZPatterns.td
index beaf4de285a3a..65300fb47627d 100644
--- a/llvm/lib/Target/SystemZ/SystemZPatterns.td
+++ b/llvm/lib/Target/SystemZ/SystemZPatterns.td
@@ -100,12 +100,12 @@ multiclass CondStores64<Instruction insn, Instruction insninv,
                         SDPatternOperator store, SDPatternOperator load,
                         AddressingMode mode> {
   def : Pat<(store (z_select_ccmask GR64:$new, (load mode:$addr),
-                                    imm32zx4:$valid, imm32zx4:$cc),
+                                    imm32zx4_timm:$valid, imm32zx4_timm:$cc),
                    mode:$addr),
             (insn (EXTRACT_SUBREG GR64:$new, subreg_l32), mode:$addr,
                   imm32zx4:$valid, imm32zx4:$cc)>;
   def : Pat<(store (z_select_ccmask (load mode:$addr), GR64:$new,
-                                    imm32zx4:$valid, imm32zx4:$cc),
+                                    imm32zx4_timm:$valid, imm32zx4_timm:$cc),
                    mode:$addr),
             (insninv (EXTRACT_SUBREG GR64:$new, subreg_l32), mode:$addr,
                      imm32zx4:$valid, imm32zx4:$cc)>;
diff --git a/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp b/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
index 6ef005f813ad4..aaa7f8fc88f50 100644
--- a/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZPostRewrite.cpp
@@ -25,6 +25,7 @@ using namespace llvm;
 
 #define DEBUG_TYPE "systemz-postrewrite"
 STATISTIC(MemFoldCopies, "Number of copies inserted before folded mem ops.");
+STATISTIC(LOCRMuxJumps, "Number of LOCRMux jump-sequences (lower is better)");
 
 namespace llvm {
   void initializeSystemZPostRewritePass(PassRegistry&);
@@ -45,12 +46,20 @@ class SystemZPostRewrite : public MachineFunctionPass {
 
   StringRef getPassName() const override { return SYSTEMZ_POSTREWRITE_NAME; }
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
 private:
+  void selectLOCRMux(MachineBasicBlock &MBB,
+                     MachineBasicBlock::iterator MBBI,
+                     MachineBasicBlock::iterator &NextMBBI,
+                     unsigned LowOpcode,
+                     unsigned HighOpcode);
+  void selectSELRMux(MachineBasicBlock &MBB,
+                     MachineBasicBlock::iterator MBBI,
+                     MachineBasicBlock::iterator &NextMBBI,
+                     unsigned LowOpcode,
+                     unsigned HighOpcode);
+  bool expandCondMove(MachineBasicBlock &MBB,
+                      MachineBasicBlock::iterator MBBI,
+                      MachineBasicBlock::iterator &NextMBBI);
   bool selectMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
                 MachineBasicBlock::iterator &NextMBBI);
   bool selectMBB(MachineBasicBlock &MBB);
@@ -68,11 +77,141 @@ FunctionPass *llvm::createSystemZPostRewritePass(SystemZTargetMachine &TM) {
   return new SystemZPostRewrite();
 }
 
+// MI is a load-register-on-condition pseudo instruction.  Replace it with
+// LowOpcode if source and destination are both low GR32s and HighOpcode if
+// source and destination are both high GR32s. Otherwise, a branch sequence
+// is created.
+void SystemZPostRewrite::selectLOCRMux(MachineBasicBlock &MBB,
+                                       MachineBasicBlock::iterator MBBI,
+                                       MachineBasicBlock::iterator &NextMBBI,
+                                       unsigned LowOpcode,
+                                       unsigned HighOpcode) {
+  Register DestReg = MBBI->getOperand(0).getReg();
+  Register SrcReg = MBBI->getOperand(2).getReg();
+  bool DestIsHigh = SystemZ::isHighReg(DestReg);
+  bool SrcIsHigh = SystemZ::isHighReg(SrcReg);
+
+  if (!DestIsHigh && !SrcIsHigh)
+    MBBI->setDesc(TII->get(LowOpcode));
+  else if (DestIsHigh && SrcIsHigh)
+    MBBI->setDesc(TII->get(HighOpcode));
+  else
+    expandCondMove(MBB, MBBI, NextMBBI);
+}
+
+// MI is a select pseudo instruction.  Replace it with LowOpcode if source
+// and destination are all low GR32s and HighOpcode if source and destination
+// are all high GR32s. Otherwise, a branch sequence is created.
+void SystemZPostRewrite::selectSELRMux(MachineBasicBlock &MBB,
+                                       MachineBasicBlock::iterator MBBI,
+                                       MachineBasicBlock::iterator &NextMBBI,
+                                       unsigned LowOpcode,
+                                       unsigned HighOpcode) {
+  Register DestReg = MBBI->getOperand(0).getReg();
+  Register Src1Reg = MBBI->getOperand(1).getReg();
+  Register Src2Reg = MBBI->getOperand(2).getReg();
+  bool DestIsHigh = SystemZ::isHighReg(DestReg);
+  bool Src1IsHigh = SystemZ::isHighReg(Src1Reg);
+  bool Src2IsHigh = SystemZ::isHighReg(Src2Reg);
+
+  // If sources and destination aren't all high or all low, we may be able to
+  // simplify the operation by moving one of the sources to the destination
+  // first.  But only if this doesn't clobber the other source.
+  if (DestReg != Src1Reg && DestReg != Src2Reg) {
+    if (DestIsHigh != Src1IsHigh) {
+      BuildMI(*MBBI->getParent(), MBBI, MBBI->getDebugLoc(),
+              TII->get(SystemZ::COPY), DestReg)
+        .addReg(MBBI->getOperand(1).getReg(), getRegState(MBBI->getOperand(1)));
+      MBBI->getOperand(1).setReg(DestReg);
+      Src1Reg = DestReg;
+      Src1IsHigh = DestIsHigh;
+    } else if (DestIsHigh != Src2IsHigh) {
+      BuildMI(*MBBI->getParent(), MBBI, MBBI->getDebugLoc(),
+              TII->get(SystemZ::COPY), DestReg)
+        .addReg(MBBI->getOperand(2).getReg(), getRegState(MBBI->getOperand(2)));
+      MBBI->getOperand(2).setReg(DestReg);
+      Src2Reg = DestReg;
+      Src2IsHigh = DestIsHigh;
+    }
+  }
+
+  // If the destination (now) matches one source, prefer this to be first.
+  if (DestReg != Src1Reg && DestReg == Src2Reg) {
+    TII->commuteInstruction(*MBBI, false, 1, 2);
+    std::swap(Src1Reg, Src2Reg);
+    std::swap(Src1IsHigh, Src2IsHigh);
+  }
+
+  if (!DestIsHigh && !Src1IsHigh && !Src2IsHigh)
+    MBBI->setDesc(TII->get(LowOpcode));
+  else if (DestIsHigh && Src1IsHigh && Src2IsHigh)
+    MBBI->setDesc(TII->get(HighOpcode));
+  else
+    // Given the simplification above, we must already have a two-operand case.
+    expandCondMove(MBB, MBBI, NextMBBI);
+}
+
+// Replace MBBI by a branch sequence that performs a conditional move of
+// operand 2 to the destination register. Operand 1 is expected to be the
+// same register as the destination.
+bool SystemZPostRewrite::expandCondMove(MachineBasicBlock &MBB,
+                                        MachineBasicBlock::iterator MBBI,
+                                        MachineBasicBlock::iterator &NextMBBI) {
+  MachineFunction &MF = *MBB.getParent();
+  const BasicBlock *BB = MBB.getBasicBlock();
+  MachineInstr &MI = *MBBI;
+  DebugLoc DL = MI.getDebugLoc();
+  Register DestReg = MI.getOperand(0).getReg();
+  Register SrcReg = MI.getOperand(2).getReg();
+  unsigned CCValid = MI.getOperand(3).getImm();
+  unsigned CCMask = MI.getOperand(4).getImm();
+  assert(DestReg == MI.getOperand(1).getReg() &&
+         "Expected destination and first source operand to be the same.");
+
+  LivePhysRegs LiveRegs(TII->getRegisterInfo());
+  LiveRegs.addLiveOuts(MBB);
+  for (auto I = std::prev(MBB.end()); I != MBBI; --I)
+    LiveRegs.stepBackward(*I);
+
+  // Splice MBB at MI, moving the rest of the block into RestMBB.
+  MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB);
+  MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB);
+  RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end());
+  RestMBB->transferSuccessors(&MBB);
+  for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
+    RestMBB->addLiveIn(*I);
+
+  // Create a new block MoveMBB to hold the move instruction.
+  MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB);
+  MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB);
+  MoveMBB->addLiveIn(SrcReg);
+  for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
+    MoveMBB->addLiveIn(*I);
+
+  // At the end of MBB, create a conditional branch to RestMBB if the
+  // condition is false, otherwise fall through to MoveMBB.
+  BuildMI(&MBB, DL, TII->get(SystemZ::BRC))
+    .addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB);
+  MBB.addSuccessor(RestMBB);
+  MBB.addSuccessor(MoveMBB);
+
+  // In MoveMBB, emit an instruction to move SrcReg into DestReg,
+  // then fall through to RestMBB.
+  BuildMI(*MoveMBB, MoveMBB->end(), DL, TII->get(SystemZ::COPY), DestReg)
+      .addReg(MI.getOperand(2).getReg(), getRegState(MI.getOperand(2)));
+  MoveMBB->addSuccessor(RestMBB);
+
+  NextMBBI = MBB.end();
+  MI.eraseFromParent();
+  LOCRMuxJumps++;
+  return true;
+}
+
 /// If MBBI references a pseudo instruction that should be selected here,
 /// do it and return true.  Otherwise return false.
 bool SystemZPostRewrite::selectMI(MachineBasicBlock &MBB,
-                                MachineBasicBlock::iterator MBBI,
-                                MachineBasicBlock::iterator &NextMBBI) {
+                                  MachineBasicBlock::iterator MBBI,
+                                  MachineBasicBlock::iterator &NextMBBI) {
   MachineInstr &MI = *MBBI;
   unsigned Opcode = MI.getOpcode();
 
@@ -94,6 +233,15 @@ bool SystemZPostRewrite::selectMI(MachineBasicBlock &MBB,
     return true;
   }
 
+  switch (Opcode) {
+  case SystemZ::LOCRMux:
+    selectLOCRMux(MBB, MBBI, NextMBBI, SystemZ::LOCR, SystemZ::LOCFHR);
+    return true;
+  case SystemZ::SELRMux:
+    selectSELRMux(MBB, MBBI, NextMBBI, SystemZ::SELR, SystemZ::SELFHR);
+    return true;
+  }
+
   return false;
 }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZProcessors.td b/llvm/lib/Target/SystemZ/SystemZProcessors.td
index b27c25beb58ca..af33a03005524 100644
--- a/llvm/lib/Target/SystemZ/SystemZProcessors.td
+++ b/llvm/lib/Target/SystemZ/SystemZProcessors.td
@@ -35,5 +35,6 @@ def : ProcessorModel<"z13", Z13Model, Arch11SupportedFeatures.List>;
 def : ProcessorModel<"arch12", Z14Model, Arch12SupportedFeatures.List>;
 def : ProcessorModel<"z14", Z14Model, Arch12SupportedFeatures.List>;
 
-def : ProcessorModel<"arch13", Arch13Model, Arch13SupportedFeatures.List>;
+def : ProcessorModel<"arch13", Z15Model, Arch13SupportedFeatures.List>;
+def : ProcessorModel<"z15", Z15Model, Arch13SupportedFeatures.List>;
 
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
index 4f721ec23e53a..7044efef1ac64 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.h
@@ -28,6 +28,15 @@ inline unsigned even128(bool Is32bit) {
 inline unsigned odd128(bool Is32bit) {
   return Is32bit ? subreg_l32 : subreg_l64;
 }
+
+// Reg should be a 32-bit GPR.  Return true if it is a high register rather
+// than a low register.
+inline bool isHighReg(unsigned int Reg) {
+  if (SystemZ::GRH32BitRegClass.contains(Reg))
+    return true;
+  assert(SystemZ::GR32BitRegClass.contains(Reg) && "Invalid GRX32");
+  return false;
+}
 } // end namespace SystemZ
 
 struct SystemZRegisterInfo : public SystemZGenRegisterInfo {
diff --git a/llvm/lib/Target/SystemZ/SystemZSchedule.td b/llvm/lib/Target/SystemZ/SystemZSchedule.td
index 98eca28022423..119e3ee7c22c6 100644
--- a/llvm/lib/Target/SystemZ/SystemZSchedule.td
+++ b/llvm/lib/Target/SystemZ/SystemZSchedule.td
@@ -59,7 +59,7 @@ def VBU : SchedWrite; // Virtual branching unit
 
 def MCD : SchedWrite; // Millicode
 
-include "SystemZScheduleArch13.td"
+include "SystemZScheduleZ15.td"
 include "SystemZScheduleZ14.td"
 include "SystemZScheduleZ13.td"
 include "SystemZScheduleZEC12.td"
diff --git a/llvm/lib/Target/SystemZ/SystemZScheduleArch13.td b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
similarity index 97%
rename from llvm/lib/Target/SystemZ/SystemZScheduleArch13.td
rename to llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
index 9f82f24d0e8f3..56ceb88f35d4f 100644
--- a/llvm/lib/Target/SystemZ/SystemZScheduleArch13.td
+++ b/llvm/lib/Target/SystemZ/SystemZScheduleZ15.td
@@ -1,4 +1,4 @@
-//-- SystemZScheduleArch13.td - SystemZ Scheduling Definitions ----*- tblgen -*-=//
+//-- SystemZScheduleZ15.td - SystemZ Scheduling Definitions ----*- tblgen -*-=//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,14 +6,14 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file defines the machine model for Arch13 to support instruction
+// This file defines the machine model for Z15 to support instruction
 // scheduling and other instruction cost heuristics.
 //
 // Pseudos expanded right after isel do not need to be modelled here.
 //
 //===----------------------------------------------------------------------===//
 
-def Arch13Model : SchedMachineModel {
+def Z15Model : SchedMachineModel {
 
     let UnsupportedFeatures = Arch13UnsupportedFeatures.List;
 
@@ -27,7 +27,7 @@ def Arch13Model : SchedMachineModel {
     let MispredictPenalty = 20;
 }
 
-let SchedModel = Arch13Model in  {
+let SchedModel = Z15Model in  {
 // These definitions need the SchedModel value. They could be put in a
 // subtarget common include file, but it seems the include system in Tablegen
 // currently (2016) rejects multiple includes of same file.
@@ -73,43 +73,43 @@ let NumMicroOps = 0 in {
 }
 
 // Execution units.
-def Arch13_FXaUnit     : ProcResource<2>;
-def Arch13_FXbUnit     : ProcResource<2>;
-def Arch13_LSUnit      : ProcResource<2>;
-def Arch13_VecUnit     : ProcResource<2>;
-def Arch13_VecFPdUnit  : ProcResource<2> { let BufferSize = 1; /* blocking */ }
-def Arch13_VBUnit      : ProcResource<2>;
-def Arch13_MCD         : ProcResource<1>;
+def Z15_FXaUnit     : ProcResource<2>;
+def Z15_FXbUnit     : ProcResource<2>;
+def Z15_LSUnit      : ProcResource<2>;
+def Z15_VecUnit     : ProcResource<2>;
+def Z15_VecFPdUnit  : ProcResource<2> { let BufferSize = 1; /* blocking */ }
+def Z15_VBUnit      : ProcResource<2>;
+def Z15_MCD         : ProcResource<1>;
 
 // Subtarget specific definitions of scheduling resources.
 let NumMicroOps = 0 in {
-  def : WriteRes<FXa, [Arch13_FXaUnit]>;
-  def : WriteRes<FXb, [Arch13_FXbUnit]>;
-  def : WriteRes<LSU, [Arch13_LSUnit]>;
-  def : WriteRes<VecBF,  [Arch13_VecUnit]>;
-  def : WriteRes<VecDF,  [Arch13_VecUnit]>;
-  def : WriteRes<VecDFX, [Arch13_VecUnit]>;
-  def : WriteRes<VecMul,  [Arch13_VecUnit]>;
-  def : WriteRes<VecStr,  [Arch13_VecUnit]>;
-  def : WriteRes<VecXsPm, [Arch13_VecUnit]>;
+  def : WriteRes<FXa, [Z15_FXaUnit]>;
+  def : WriteRes<FXb, [Z15_FXbUnit]>;
+  def : WriteRes<LSU, [Z15_LSUnit]>;
+  def : WriteRes<VecBF,  [Z15_VecUnit]>;
+  def : WriteRes<VecDF,  [Z15_VecUnit]>;
+  def : WriteRes<VecDFX, [Z15_VecUnit]>;
+  def : WriteRes<VecMul,  [Z15_VecUnit]>;
+  def : WriteRes<VecStr,  [Z15_VecUnit]>;
+  def : WriteRes<VecXsPm, [Z15_VecUnit]>;
   foreach Num = 2-5 in { let ResourceCycles = [Num] in {
-    def : WriteRes<!cast<SchedWrite>("FXa"#Num), [Arch13_FXaUnit]>;
-    def : WriteRes<!cast<SchedWrite>("FXb"#Num), [Arch13_FXbUnit]>;
-    def : WriteRes<!cast<SchedWrite>("LSU"#Num), [Arch13_LSUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecBF"#Num), [Arch13_VecUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecDF"#Num), [Arch13_VecUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecDFX"#Num), [Arch13_VecUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecMul"#Num), [Arch13_VecUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecStr"#Num), [Arch13_VecUnit]>;
-    def : WriteRes<!cast<SchedWrite>("VecXsPm"#Num), [Arch13_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("FXa"#Num), [Z15_FXaUnit]>;
+    def : WriteRes<!cast<SchedWrite>("FXb"#Num), [Z15_FXbUnit]>;
+    def : WriteRes<!cast<SchedWrite>("LSU"#Num), [Z15_LSUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecBF"#Num), [Z15_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecDF"#Num), [Z15_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecDFX"#Num), [Z15_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecMul"#Num), [Z15_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecStr"#Num), [Z15_VecUnit]>;
+    def : WriteRes<!cast<SchedWrite>("VecXsPm"#Num), [Z15_VecUnit]>;
   }}
 
-  def : WriteRes<VecFPd,  [Arch13_VecFPdUnit]> { let ResourceCycles = [30]; }
+  def : WriteRes<VecFPd,  [Z15_VecFPdUnit]> { let ResourceCycles = [30]; }
 
-  def : WriteRes<VBU,     [Arch13_VBUnit]>; // Virtual Branching Unit
+  def : WriteRes<VBU,     [Z15_VBUnit]>; // Virtual Branching Unit
 }
 
-def : WriteRes<MCD, [Arch13_MCD]> { let NumMicroOps = 3;
+def : WriteRes<MCD, [Z15_MCD]> { let NumMicroOps = 3;
                                     let BeginGroup  = 1;
                                     let EndGroup    = 1; }
 
diff --git a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
index a50e6aa597118..47c925dcf730f 100644
--- a/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp
@@ -209,10 +209,10 @@ std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr(
 
   // Now select between End and null, depending on whether the character
   // was found.
-  SDValue Ops[] = {End, DAG.getConstant(0, DL, PtrVT),
-                   DAG.getConstant(SystemZ::CCMASK_SRST, DL, MVT::i32),
-                   DAG.getConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32),
-                   CCReg};
+  SDValue Ops[] = {
+      End, DAG.getConstant(0, DL, PtrVT),
+      DAG.getTargetConstant(SystemZ::CCMASK_SRST, DL, MVT::i32),
+      DAG.getTargetConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32), CCReg};
   End = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, PtrVT, Ops);
   return std::make_pair(End, Chain);
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
index 4604fcae88146..20865037fe385 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp
@@ -184,6 +184,7 @@ class SystemZPassConfig : public TargetPassConfig {
   bool addInstSelector() override;
   bool addILPOpts() override;
   void addPostRewrite() override;
+  void addPostRegAlloc() override;
   void addPreSched2() override;
   void addPreEmitPass() override;
 };
@@ -217,14 +218,14 @@ void SystemZPassConfig::addPostRewrite() {
   addPass(createSystemZPostRewritePass(getSystemZTargetMachine()));
 }
 
-void SystemZPassConfig::addPreSched2() {
+void SystemZPassConfig::addPostRegAlloc() {
   // PostRewrite needs to be run at -O0 also (in which case addPostRewrite()
   // is not called).
   if (getOptLevel() == CodeGenOpt::None)
     addPass(createSystemZPostRewritePass(getSystemZTargetMachine()));
+}
 
-  addPass(createSystemZExpandPseudoPass(getSystemZTargetMachine()));
-
+void SystemZPassConfig::addPreSched2() {
   if (getOptLevel() != CodeGenOpt::None)
     addPass(&IfConverterID);
 }
diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
index 145cf87ef9f59..8d45e67d73c26 100644
--- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
@@ -707,7 +707,7 @@ int SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       // TODO: Fix base implementation which could simplify things a bit here
       // (seems to miss on differentiating on scalar/vector types).
 
-      // Only 64 bit vector conversions are natively supported before arch13.
+      // Only 64 bit vector conversions are natively supported before z15.
       if (DstScalarBits == 64 || ST->hasVectorEnhancements2()) {
         if (SrcScalarBits == DstScalarBits)
           return NumDstVectors;
diff --git a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
index f2f3d20a98e65..6e17693991b03 100644
--- a/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
+++ b/llvm/lib/Target/WebAssembly/MCTargetDesc/WebAssemblyMCTargetDesc.h
@@ -191,6 +191,8 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I32_S:
   case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I64:
   case WebAssembly::ATOMIC_RMW8_U_CMPXCHG_I64_S:
+  case WebAssembly::LOAD_SPLAT_v8x16:
+  case WebAssembly::LOAD_SPLAT_v8x16_S:
     return 0;
   case WebAssembly::LOAD16_S_I32:
   case WebAssembly::LOAD16_S_I32_S:
@@ -240,6 +242,8 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I32_S:
   case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I64:
   case WebAssembly::ATOMIC_RMW16_U_CMPXCHG_I64_S:
+  case WebAssembly::LOAD_SPLAT_v16x8:
+  case WebAssembly::LOAD_SPLAT_v16x8_S:
     return 1;
   case WebAssembly::LOAD_I32:
   case WebAssembly::LOAD_I32_S:
@@ -295,6 +299,8 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   case WebAssembly::ATOMIC_NOTIFY_S:
   case WebAssembly::ATOMIC_WAIT_I32:
   case WebAssembly::ATOMIC_WAIT_I32_S:
+  case WebAssembly::LOAD_SPLAT_v32x4:
+  case WebAssembly::LOAD_SPLAT_v32x4_S:
     return 2;
   case WebAssembly::LOAD_I64:
   case WebAssembly::LOAD_I64_S:
@@ -324,31 +330,25 @@ inline unsigned GetDefaultP2AlignAny(unsigned Opc) {
   case WebAssembly::ATOMIC_RMW_CMPXCHG_I64_S:
   case WebAssembly::ATOMIC_WAIT_I64:
   case WebAssembly::ATOMIC_WAIT_I64_S:
+  case WebAssembly::LOAD_SPLAT_v64x2:
+  case WebAssembly::LOAD_SPLAT_v64x2_S:
+  case WebAssembly::LOAD_EXTEND_S_v8i16:
+  case WebAssembly::LOAD_EXTEND_S_v8i16_S:
+  case WebAssembly::LOAD_EXTEND_U_v8i16:
+  case WebAssembly::LOAD_EXTEND_U_v8i16_S:
+  case WebAssembly::LOAD_EXTEND_S_v4i32:
+  case WebAssembly::LOAD_EXTEND_S_v4i32_S:
+  case WebAssembly::LOAD_EXTEND_U_v4i32:
+  case WebAssembly::LOAD_EXTEND_U_v4i32_S:
+  case WebAssembly::LOAD_EXTEND_S_v2i64:
+  case WebAssembly::LOAD_EXTEND_S_v2i64_S:
+  case WebAssembly::LOAD_EXTEND_U_v2i64:
+  case WebAssembly::LOAD_EXTEND_U_v2i64_S:
     return 3;
-  case WebAssembly::LOAD_v16i8:
-  case WebAssembly::LOAD_v16i8_S:
-  case WebAssembly::LOAD_v8i16:
-  case WebAssembly::LOAD_v8i16_S:
-  case WebAssembly::LOAD_v4i32:
-  case WebAssembly::LOAD_v4i32_S:
-  case WebAssembly::LOAD_v2i64:
-  case WebAssembly::LOAD_v2i64_S:
-  case WebAssembly::LOAD_v4f32:
-  case WebAssembly::LOAD_v4f32_S:
-  case WebAssembly::LOAD_v2f64:
-  case WebAssembly::LOAD_v2f64_S:
-  case WebAssembly::STORE_v16i8:
-  case WebAssembly::STORE_v16i8_S:
-  case WebAssembly::STORE_v8i16:
-  case WebAssembly::STORE_v8i16_S:
-  case WebAssembly::STORE_v4i32:
-  case WebAssembly::STORE_v4i32_S:
-  case WebAssembly::STORE_v2i64:
-  case WebAssembly::STORE_v2i64_S:
-  case WebAssembly::STORE_v4f32:
-  case WebAssembly::STORE_v4f32_S:
-  case WebAssembly::STORE_v2f64:
-  case WebAssembly::STORE_v2f64_S:
+  case WebAssembly::LOAD_V128:
+  case WebAssembly::LOAD_V128_S:
+  case WebAssembly::STORE_V128:
+  case WebAssembly::STORE_V128_S:
     return 4;
   default:
     return -1;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
index 9f94866c08901..c069af9eed623 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGSort.cpp
@@ -317,6 +317,7 @@ static void sortBlocks(MachineFunction &MF, const MachineLoopInfo &MLI,
       // If Next was originally ordered before MBB, and it isn't because it was
       // loop-rotated above the header, it's not preferred.
       if (Next->getNumber() < MBB->getNumber() &&
+          (WasmDisableEHPadSort || !Next->isEHPad()) &&
           (!R || !R->contains(Next) ||
            R->getHeader()->getNumber() < Next->getNumber())) {
         Ready.push(Next);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
index 6c38dead9ae09..6b2094679cea5 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp
@@ -694,8 +694,26 @@ void WebAssemblyCFGStackify::removeUnnecessaryInstrs(MachineFunction &MF) {
   }
 }
 
+// When MBB is split into MBB and Split, we should unstackify defs in MBB that
+// have their uses in Split.
+static void unstackifyVRegsUsedInSplitBB(MachineBasicBlock &MBB,
+                                         MachineBasicBlock &Split,
+                                         WebAssemblyFunctionInfo &MFI,
+                                         MachineRegisterInfo &MRI) {
+  for (auto &MI : Split) {
+    for (auto &MO : MI.explicit_uses()) {
+      if (!MO.isReg() || Register::isPhysicalRegister(MO.getReg()))
+        continue;
+      if (MachineInstr *Def = MRI.getUniqueVRegDef(MO.getReg()))
+        if (Def->getParent() == &MBB)
+          MFI.unstackifyVReg(MO.getReg());
+    }
+  }
+}
+
 bool WebAssemblyCFGStackify::fixUnwindMismatches(MachineFunction &MF) {
   const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
+  auto &MFI = *MF.getInfo<WebAssemblyFunctionInfo>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
 
   // Linearizing the control flow by placing TRY / END_TRY markers can create
@@ -1007,6 +1025,7 @@ bool WebAssemblyCFGStackify::fixUnwindMismatches(MachineFunction &MF) {
     BrDest->insert(BrDest->end(), EndTry->removeFromParent());
     // Take out the handler body from EH pad to the new branch destination BB.
     BrDest->splice(BrDest->end(), EHPad, SplitPos, EHPad->end());
+    unstackifyVRegsUsedInSplitBB(*EHPad, *BrDest, MFI, MRI);
     // Fix predecessor-successor relationship.
     BrDest->transferSuccessors(EHPad);
     EHPad->addSuccessor(BrDest);
@@ -1122,6 +1141,7 @@ bool WebAssemblyCFGStackify::fixUnwindMismatches(MachineFunction &MF) {
       // new nested continuation BB.
       NestedCont->splice(NestedCont->end(), MBB,
                          std::next(RangeEnd->getIterator()), MBB->end());
+      unstackifyVRegsUsedInSplitBB(*MBB, *NestedCont, MFI, MRI);
       registerTryScope(NestedTry, NestedEndTry, NestedEHPad);
 
       // Fix predecessor-successor relationship.
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
index 77217f16a7277..827a7441ff873 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISD.def
@@ -29,6 +29,7 @@ HANDLE_NODETYPE(SHUFFLE)
 HANDLE_NODETYPE(VEC_SHL)
 HANDLE_NODETYPE(VEC_SHR_S)
 HANDLE_NODETYPE(VEC_SHR_U)
+HANDLE_NODETYPE(LOAD_SPLAT)
 HANDLE_NODETYPE(THROW)
 HANDLE_NODETYPE(MEMORY_COPY)
 HANDLE_NODETYPE(MEMORY_FILL)
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
index b0c03c13fe60c..f83a8a984ae0d 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp
@@ -54,6 +54,12 @@ class WebAssemblyDAGToDAGISel final : public SelectionDAGISel {
 
     ForCodeSize = MF.getFunction().hasOptSize();
     Subtarget = &MF.getSubtarget<WebAssemblySubtarget>();
+
+    // Wasm64 is not fully supported right now (and is not specified)
+    if (Subtarget->hasAddr64())
+      report_fatal_error(
+          "64-bit WebAssembly (wasm64) is not currently supported");
+
     return SelectionDAGISel::runOnMachineFunction(MF);
   }
 
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
index 4ac9995db01e9..53ca7d77ca057 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
@@ -205,7 +205,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
     for (auto T : {MVT::i8, MVT::i16, MVT::i32})
       setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
   }
-  for (auto T : MVT::integer_vector_valuetypes())
+  for (auto T : MVT::integer_fixedlen_vector_valuetypes())
     setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
 
   // Dynamic stack allocation: use the default expansion.
@@ -228,7 +228,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   //  - Floating-point extending loads.
   //  - Floating-point truncating stores.
   //  - i1 extending loads.
-  //  - extending/truncating SIMD loads/stores
+  //  - truncating SIMD stores and most extending loads
   setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
   for (auto T : MVT::integer_valuetypes())
@@ -237,7 +237,7 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
   if (Subtarget->hasSIMD128()) {
     for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
                    MVT::v2f64}) {
-      for (auto MemT : MVT::vector_valuetypes()) {
+      for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
         if (MVT(T) != MemT) {
           setTruncStoreAction(T, MemT, Expand);
           for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
@@ -245,6 +245,14 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
         }
       }
     }
+    // But some vector extending loads are legal
+    if (Subtarget->hasUnimplementedSIMD128()) {
+      for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
+        setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
+        setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
+        setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
+      }
+    }
   }
 
   // Don't do anything clever with build_pairs
@@ -259,16 +267,6 @@ WebAssemblyTargetLowering::WebAssemblyTargetLowering(
 
   setMaxAtomicSizeInBitsSupported(64);
 
-  if (Subtarget->hasBulkMemory()) {
-    // Use memory.copy and friends over multiple loads and stores
-    MaxStoresPerMemcpy = 1;
-    MaxStoresPerMemcpyOptSize = 1;
-    MaxStoresPerMemmove = 1;
-    MaxStoresPerMemmoveOptSize = 1;
-    MaxStoresPerMemset = 1;
-    MaxStoresPerMemsetOptSize = 1;
-  }
-
   // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
   // consistent with the f64 and f128 names.
   setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
@@ -550,6 +548,16 @@ bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
   return true;
 }
 
+bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
+  if (!Subtarget->hasUnimplementedSIMD128())
+    return false;
+  MVT ExtT = ExtVal.getSimpleValueType();
+  MVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getSimpleValueType(0);
+  return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
+         (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
+         (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
+}
+
 EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
                                                   LLVMContext &C,
                                                   EVT VT) const {
@@ -1368,7 +1376,16 @@ SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     }
   }
   // Use a splat for the initial vector
-  SDValue Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
+  SDValue Result;
+  // Possibly a load_splat
+  LoadSDNode *SplattedLoad;
+  if (Subtarget->hasUnimplementedSIMD128() &&
+      (SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
+      SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
+    Result = DAG.getNode(WebAssemblyISD::LOAD_SPLAT, DL, VecT, SplatValue);
+  } else {
+    Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
+  }
   // Add replace_lane instructions for other values
   for (size_t I = 0; I < Lanes; ++I) {
     const SDValue &Lane = Op->getOperand(I);
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
index b3c7f3defd5f8..a53e24a055422 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
@@ -63,7 +63,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
                                       MachineMemOperand::Flags Flags,
                                       bool *Fast) const override;
   bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
-
+  bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
   EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
                          EVT VT) const override;
   bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
index fecb895e0d76d..a9a99d38f9f16 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrAtomics.td
@@ -71,12 +71,6 @@ class NotifyPatImmOff<PatFrag operand> :
 def : NotifyPatImmOff<regPlusImm>;
 def : NotifyPatImmOff<or_is_add>;
 
-def NotifyPatGlobalAddr :
-  Pat<(i32 (int_wasm_atomic_notify (regPlusGA I32:$addr,
-                                    (WebAssemblywrapper tglobaladdr:$off)),
-                                   I32:$count)),
-      (ATOMIC_NOTIFY 0, tglobaladdr:$off, I32:$addr, I32:$count)>;
-
 // Select notifys with just a constant offset.
 def NotifyPatOffsetOnly :
   Pat<(i32 (int_wasm_atomic_notify imm:$off, I32:$count)),
@@ -105,13 +99,6 @@ def : WaitPatImmOff<i32, int_wasm_atomic_wait_i32, or_is_add, ATOMIC_WAIT_I32>;
 def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, regPlusImm, ATOMIC_WAIT_I64>;
 def : WaitPatImmOff<i64, int_wasm_atomic_wait_i64, or_is_add, ATOMIC_WAIT_I64>;
 
-class WaitPatGlobalAddr<ValueType ty, Intrinsic kind, NI inst> :
-  Pat<(i32 (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
-                 ty:$exp, I64:$timeout)),
-      (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, I64:$timeout)>;
-def : WaitPatGlobalAddr<i32, int_wasm_atomic_wait_i32, ATOMIC_WAIT_I32>;
-def : WaitPatGlobalAddr<i64, int_wasm_atomic_wait_i64, ATOMIC_WAIT_I64>;
-
 // Select wait_i32, ATOMIC_WAIT_I32s with just a constant offset.
 class WaitPatOffsetOnly<ValueType ty, Intrinsic kind, NI inst> :
   Pat<(i32 (kind imm:$off, ty:$exp, I64:$timeout)),
@@ -164,9 +151,6 @@ def : LoadPatImmOff<i64, atomic_load_64, regPlusImm, ATOMIC_LOAD_I64>;
 def : LoadPatImmOff<i32, atomic_load_32, or_is_add, ATOMIC_LOAD_I32>;
 def : LoadPatImmOff<i64, atomic_load_64, or_is_add, ATOMIC_LOAD_I64>;
 
-def : LoadPatGlobalAddr<i32, atomic_load_32, ATOMIC_LOAD_I32>;
-def : LoadPatGlobalAddr<i64, atomic_load_64, ATOMIC_LOAD_I64>;
-
 // Select loads with just a constant offset.
 def : LoadPatOffsetOnly<i32, atomic_load_32, ATOMIC_LOAD_I32>;
 def : LoadPatOffsetOnly<i64, atomic_load_64, ATOMIC_LOAD_I64>;
@@ -257,16 +241,6 @@ def : LoadPatImmOff<i64, sext_aload_8_64, or_is_add, ATOMIC_LOAD8_U_I64>;
 def : LoadPatImmOff<i64, sext_aload_16_64, or_is_add, ATOMIC_LOAD16_U_I64>;
 // No 32->64 patterns, just use i32.atomic.load and i64.extend_s/i64
 
-def : LoadPatGlobalAddr<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
-def : LoadPatGlobalAddr<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
-def : LoadPatGlobalAddr<i64, zext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatGlobalAddr<i64, zext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-def : LoadPatGlobalAddr<i64, zext_aload_32_64, ATOMIC_LOAD32_U_I64>;
-def : LoadPatGlobalAddr<i32, atomic_load_8, ATOMIC_LOAD8_U_I32>;
-def : LoadPatGlobalAddr<i32, atomic_load_16, ATOMIC_LOAD16_U_I32>;
-def : LoadPatGlobalAddr<i64, sext_aload_8_64, ATOMIC_LOAD8_U_I64>;
-def : LoadPatGlobalAddr<i64, sext_aload_16_64, ATOMIC_LOAD16_U_I64>;
-
 // Extending loads with just a constant offset
 def : LoadPatOffsetOnly<i32, zext_aload_8_32, ATOMIC_LOAD8_U_I32>;
 def : LoadPatOffsetOnly<i32, zext_aload_16_32, ATOMIC_LOAD16_U_I32>;
@@ -326,13 +300,6 @@ def : AStorePatImmOff<i64, atomic_store_64, regPlusImm, ATOMIC_STORE_I64>;
 def : AStorePatImmOff<i32, atomic_store_32, or_is_add, ATOMIC_STORE_I32>;
 def : AStorePatImmOff<i64, atomic_store_64, or_is_add, ATOMIC_STORE_I64>;
 
-class AStorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
-  Pat<(kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
-            ty:$val),
-      (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>;
-def : AStorePatGlobalAddr<i32, atomic_store_32, ATOMIC_STORE_I32>;
-def : AStorePatGlobalAddr<i64, atomic_store_64, ATOMIC_STORE_I64>;
-
 // Select stores with just a constant offset.
 class AStorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
   Pat<(kind imm:$off, ty:$val), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
@@ -387,12 +354,6 @@ def : AStorePatImmOff<i64, trunc_astore_8_64, or_is_add, ATOMIC_STORE8_I64>;
 def : AStorePatImmOff<i64, trunc_astore_16_64, or_is_add, ATOMIC_STORE16_I64>;
 def : AStorePatImmOff<i64, trunc_astore_32_64, or_is_add, ATOMIC_STORE32_I64>;
 
-def : AStorePatGlobalAddr<i32, atomic_store_8, ATOMIC_STORE8_I32>;
-def : AStorePatGlobalAddr<i32, atomic_store_16, ATOMIC_STORE16_I32>;
-def : AStorePatGlobalAddr<i64, trunc_astore_8_64, ATOMIC_STORE8_I64>;
-def : AStorePatGlobalAddr<i64, trunc_astore_16_64, ATOMIC_STORE16_I64>;
-def : AStorePatGlobalAddr<i64, trunc_astore_32_64, ATOMIC_STORE32_I64>;
-
 // Truncating stores with just a constant offset
 def : AStorePatOffsetOnly<i32, atomic_store_8, ATOMIC_STORE8_I32>;
 def : AStorePatOffsetOnly<i32, atomic_store_16, ATOMIC_STORE16_I32>;
@@ -513,11 +474,6 @@ class BinRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
   Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$val)),
       (inst 0, imm:$off, I32:$addr, ty:$val)>;
 
-class BinRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
-  Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
-                ty:$val)),
-      (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>;
-
 // Select binary RMWs with just a constant offset.
 class BinRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
   Pat<(ty (kind imm:$off, ty:$val)),
@@ -538,9 +494,6 @@ multiclass BinRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32,
   def : BinRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>;
   def : BinRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>;
 
-  def : BinRMWPatGlobalAddr<i32, rmw_32, inst_32>;
-  def : BinRMWPatGlobalAddr<i64, rmw_64, inst_64>;
-
   def : BinRMWPatOffsetOnly<i32, rmw_32, inst_32>;
   def : BinRMWPatOffsetOnly<i64, rmw_64, inst_64>;
 
@@ -635,17 +588,6 @@ multiclass BinRMWTruncExtPattern<
   def : BinRMWPatImmOff<i64, sext_bin_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
   def : BinRMWPatImmOff<i64, sext_bin_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
 
-  def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  def : BinRMWPatGlobalAddr<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_16_64<rmw_16>, inst16_64>;
-  def : BinRMWPatGlobalAddr<i64, zext_bin_rmw_32_64<rmw_32>, inst32_64>;
-
-  def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_8_32<rmw_8>, inst8_32>;
-  def : BinRMWPatGlobalAddr<i32, sext_bin_rmw_16_32<rmw_16>, inst16_32>;
-  def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_8_64<rmw_8>, inst8_64>;
-  def : BinRMWPatGlobalAddr<i64, sext_bin_rmw_16_64<rmw_16>, inst16_64>;
-
   // Truncating-extending binary RMWs with just a constant offset
   def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_8_32<rmw_8>, inst8_32>;
   def : BinRMWPatOffsetOnly<i32, zext_bin_rmw_16_32<rmw_16>, inst16_32>;
@@ -745,11 +687,6 @@ class TerRMWPatImmOff<ValueType ty, PatFrag kind, PatFrag operand, NI inst> :
   Pat<(ty (kind (operand I32:$addr, imm:$off), ty:$exp, ty:$new)),
       (inst 0, imm:$off, I32:$addr, ty:$exp, ty:$new)>;
 
-class TerRMWPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
-  Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)),
-                ty:$exp, ty:$new)),
-      (inst 0, tglobaladdr:$off, I32:$addr, ty:$exp, ty:$new)>;
-
 // Select ternary RMWs with just a constant offset.
 class TerRMWPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
   Pat<(ty (kind imm:$off, ty:$exp, ty:$new)),
@@ -770,9 +707,6 @@ multiclass TerRMWPattern<PatFrag rmw_32, PatFrag rmw_64, NI inst_32,
   def : TerRMWPatImmOff<i32, rmw_32, or_is_add, inst_32>;
   def : TerRMWPatImmOff<i64, rmw_64, or_is_add, inst_64>;
 
-  def : TerRMWPatGlobalAddr<i32, rmw_32, inst_32>;
-  def : TerRMWPatGlobalAddr<i64, rmw_64, inst_64>;
-
   def : TerRMWPatOffsetOnly<i32, rmw_32, inst_32>;
   def : TerRMWPatOffsetOnly<i64, rmw_64, inst_64>;
 
@@ -859,17 +793,6 @@ multiclass TerRMWTruncExtPattern<
   def : TerRMWPatImmOff<i64, sext_ter_rmw_8_64<rmw_8>, or_is_add, inst8_64>;
   def : TerRMWPatImmOff<i64, sext_ter_rmw_16_64<rmw_16>, or_is_add, inst16_64>;
 
-  def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  def : TerRMWPatGlobalAddr<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_16_64<rmw_16>, inst16_64>;
-  def : TerRMWPatGlobalAddr<i64, zext_ter_rmw_32_64<rmw_32>, inst32_64>;
-
-  def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_8_32<rmw_8>, inst8_32>;
-  def : TerRMWPatGlobalAddr<i32, sext_ter_rmw_16_32<rmw_16>, inst16_32>;
-  def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_8_64<rmw_8>, inst8_64>;
-  def : TerRMWPatGlobalAddr<i64, sext_ter_rmw_16_64<rmw_16>, inst16_64>;
-
   // Truncating-extending ternary RMWs with just a constant offset
   def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_8_32<rmw_8>, inst8_32>;
   def : TerRMWPatOffsetOnly<i32, zext_ter_rmw_16_32<rmw_16>, inst16_32>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
index f4352e3d12ec8..05735cf6d31f7 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td
@@ -39,7 +39,7 @@ defm MEMORY_INIT :
          (ins i32imm_op:$seg, i32imm_op:$idx, I32:$dest,
               I32:$offset, I32:$size),
          (outs), (ins i32imm_op:$seg, i32imm_op:$idx),
-         [(int_wasm_memory_init (i32 imm:$seg), (i32 imm:$idx), I32:$dest,
+         [(int_wasm_memory_init (i32 timm:$seg), (i32 timm:$idx), I32:$dest,
             I32:$offset, I32:$size
           )],
          "memory.init\t$seg, $idx, $dest, $offset, $size",
@@ -48,7 +48,7 @@ defm MEMORY_INIT :
 let hasSideEffects = 1 in
 defm DATA_DROP :
   BULK_I<(outs), (ins i32imm_op:$seg), (outs), (ins i32imm_op:$seg),
-         [(int_wasm_data_drop (i32 imm:$seg))],
+         [(int_wasm_data_drop (i32 timm:$seg))],
          "data.drop\t$seg", "data.drop\t$seg", 0x09>;
 
 let mayLoad = 1, mayStore = 1 in
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
index 6916b165f9709..eba9b80d32861 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrMemory.td
@@ -37,16 +37,6 @@ def or_is_add : PatFrag<(ops node:$lhs, node:$rhs), (or node:$lhs, node:$rhs),[{
   return (~Known0.Zero & ~Known1.Zero) == 0;
 }]>;
 
-// GlobalAddresses are conceptually unsigned values, so we can also fold them
-// into immediate values as long as the add is 'nuw'.
-// TODO: We'd like to also match GA offsets but there are cases where the
-// register can have a negative value. Find out what more we can do.
-def regPlusGA : PatFrag<(ops node:$addr, node:$off),
-                        (add node:$addr, node:$off),
-                        [{
-  return N->getFlags().hasNoUnsignedWrap();
-}]>;
-
 // We don't need a regPlusES because external symbols never have constant
 // offsets folded into them, so we can just use add.
 
@@ -93,15 +83,6 @@ def : LoadPatImmOff<i64, load, or_is_add, LOAD_I64>;
 def : LoadPatImmOff<f32, load, or_is_add, LOAD_F32>;
 def : LoadPatImmOff<f64, load, or_is_add, LOAD_F64>;
 
-class LoadPatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
-  Pat<(ty (kind (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off)))),
-      (inst 0, tglobaladdr:$off, I32:$addr)>, Requires<[IsNotPIC]>;
-
-def : LoadPatGlobalAddr<i32, load, LOAD_I32>;
-def : LoadPatGlobalAddr<i64, load, LOAD_I64>;
-def : LoadPatGlobalAddr<f32, load, LOAD_F32>;
-def : LoadPatGlobalAddr<f64, load, LOAD_F64>;
-
 // Select loads with just a constant offset.
 class LoadPatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
   Pat<(ty (kind imm:$off)), (inst 0, imm:$off, (CONST_I32 0))>;
@@ -167,18 +148,6 @@ def : LoadPatImmOff<i64, zextloadi16, or_is_add, LOAD16_U_I64>;
 def : LoadPatImmOff<i64, sextloadi32, or_is_add, LOAD32_S_I64>;
 def : LoadPatImmOff<i64, zextloadi32, or_is_add, LOAD32_U_I64>;
 
-def : LoadPatGlobalAddr<i32, sextloadi8, LOAD8_S_I32>;
-def : LoadPatGlobalAddr<i32, zextloadi8, LOAD8_U_I32>;
-def : LoadPatGlobalAddr<i32, sextloadi16, LOAD16_S_I32>;
-def : LoadPatGlobalAddr<i32, zextloadi8, LOAD16_U_I32>;
-
-def : LoadPatGlobalAddr<i64, sextloadi8, LOAD8_S_I64>;
-def : LoadPatGlobalAddr<i64, zextloadi8, LOAD8_U_I64>;
-def : LoadPatGlobalAddr<i64, sextloadi16, LOAD16_S_I64>;
-def : LoadPatGlobalAddr<i64, zextloadi16, LOAD16_U_I64>;
-def : LoadPatGlobalAddr<i64, sextloadi32, LOAD32_S_I64>;
-def : LoadPatGlobalAddr<i64, zextloadi32, LOAD32_U_I64>;
-
 // Select extending loads with just a constant offset.
 def : LoadPatOffsetOnly<i32, sextloadi8, LOAD8_S_I32>;
 def : LoadPatOffsetOnly<i32, zextloadi8, LOAD8_U_I32>;
@@ -224,11 +193,6 @@ def : LoadPatImmOff<i32, extloadi16, or_is_add, LOAD16_U_I32>;
 def : LoadPatImmOff<i64, extloadi8, or_is_add, LOAD8_U_I64>;
 def : LoadPatImmOff<i64, extloadi16, or_is_add, LOAD16_U_I64>;
 def : LoadPatImmOff<i64, extloadi32, or_is_add, LOAD32_U_I64>;
-def : LoadPatGlobalAddr<i32, extloadi8, LOAD8_U_I32>;
-def : LoadPatGlobalAddr<i32, extloadi16, LOAD16_U_I32>;
-def : LoadPatGlobalAddr<i64, extloadi8, LOAD8_U_I64>;
-def : LoadPatGlobalAddr<i64, extloadi16, LOAD16_U_I64>;
-def : LoadPatGlobalAddr<i64, extloadi32, LOAD32_U_I64>;
 
 // Select "don't care" extending loads with just a constant offset.
 def : LoadPatOffsetOnly<i32, extloadi8, LOAD8_U_I32>;
@@ -282,15 +246,6 @@ def : StorePatImmOff<i64, store, or_is_add, STORE_I64>;
 def : StorePatImmOff<f32, store, or_is_add, STORE_F32>;
 def : StorePatImmOff<f64, store, or_is_add, STORE_F64>;
 
-class StorePatGlobalAddr<ValueType ty, PatFrag kind, NI inst> :
-  Pat<(kind ty:$val,
-            (regPlusGA I32:$addr, (WebAssemblywrapper tglobaladdr:$off))),
-      (inst 0, tglobaladdr:$off, I32:$addr, ty:$val)>, Requires<[IsNotPIC]>;
-def : StorePatGlobalAddr<i32, store, STORE_I32>;
-def : StorePatGlobalAddr<i64, store, STORE_I64>;
-def : StorePatGlobalAddr<f32, store, STORE_F32>;
-def : StorePatGlobalAddr<f64, store, STORE_F64>;
-
 // Select stores with just a constant offset.
 class StorePatOffsetOnly<ValueType ty, PatFrag kind, NI inst> :
   Pat<(kind ty:$val, imm:$off), (inst 0, imm:$off, (CONST_I32 0), ty:$val)>;
@@ -333,12 +288,6 @@ def : StorePatImmOff<i64, truncstorei8, or_is_add, STORE8_I64>;
 def : StorePatImmOff<i64, truncstorei16, or_is_add, STORE16_I64>;
 def : StorePatImmOff<i64, truncstorei32, or_is_add, STORE32_I64>;
 
-def : StorePatGlobalAddr<i32, truncstorei8, STORE8_I32>;
-def : StorePatGlobalAddr<i32, truncstorei16, STORE16_I32>;
-def : StorePatGlobalAddr<i64, truncstorei8, STORE8_I64>;
-def : StorePatGlobalAddr<i64, truncstorei16, STORE16_I64>;
-def : StorePatGlobalAddr<i64, truncstorei32, STORE32_I64>;
-
 // Select truncating stores with just a constant offset.
 def : StorePatOffsetOnly<i32, truncstorei8, STORE8_I32>;
 def : StorePatOffsetOnly<i32, truncstorei16, STORE16_I32>;
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
index ff03167595900..a9966af4b586c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyInstrSIMD.td
@@ -40,47 +40,124 @@ def LaneIdx#SIZE : ImmLeaf<i32, "return 0 <= Imm && Imm < "#SIZE#";">;
 //===----------------------------------------------------------------------===//
 
 // Load: v128.load
-multiclass SIMDLoad<ValueType vec_t> {
-  let mayLoad = 1, UseNamedOperandTable = 1 in
-  defm LOAD_#vec_t :
+let mayLoad = 1, UseNamedOperandTable = 1 in
+defm LOAD_V128 :
+  SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+         "v128.load\t$dst, ${off}(${addr})$p2align",
+         "v128.load\t$off$p2align", 0>;
+
+// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
+foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
+def : LoadPatNoOffset<vec_t, load, LOAD_V128>;
+def : LoadPatImmOff<vec_t, load, regPlusImm, LOAD_V128>;
+def : LoadPatImmOff<vec_t, load, or_is_add, LOAD_V128>;
+def : LoadPatOffsetOnly<vec_t, load, LOAD_V128>;
+def : LoadPatGlobalAddrOffOnly<vec_t, load, LOAD_V128>;
+}
+
+// vNxM.load_splat
+multiclass SIMDLoadSplat<string vec, bits<32> simdop> {
+  let mayLoad = 1, UseNamedOperandTable = 1,
+      Predicates = [HasUnimplementedSIMD128] in
+  defm LOAD_SPLAT_#vec :
+    SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+           vec#".load_splat\t$dst, ${off}(${addr})$p2align",
+           vec#".load_splat\t$off$p2align", simdop>;
+}
+
+defm "" : SIMDLoadSplat<"v8x16", 194>;
+defm "" : SIMDLoadSplat<"v16x8", 195>;
+defm "" : SIMDLoadSplat<"v32x4", 196>;
+defm "" : SIMDLoadSplat<"v64x2", 197>;
+
+def wasm_load_splat_t : SDTypeProfile<1, 1, []>;
+def wasm_load_splat : SDNode<"WebAssemblyISD::LOAD_SPLAT", wasm_load_splat_t>;
+
+foreach args = [["v16i8", "i32", "extloadi8"], ["v8i16", "i32", "extloadi16"],
+                ["v4i32", "i32", "load"], ["v2i64", "i64", "load"],
+                ["v4f32", "f32", "load"], ["v2f64", "f64", "load"]] in
+def load_splat_#args[0] :
+  PatFrag<(ops node:$addr), (wasm_load_splat
+            (!cast<ValueType>(args[1]) (!cast<PatFrag>(args[2]) node:$addr)))>;
+
+let Predicates = [HasUnimplementedSIMD128] in
+foreach args = [["v16i8", "v8x16"], ["v8i16", "v16x8"], ["v4i32", "v32x4"],
+                ["v2i64", "v64x2"], ["v4f32", "v32x4"], ["v2f64", "v64x2"]] in {
+def : LoadPatNoOffset<!cast<ValueType>(args[0]),
+                      !cast<PatFrag>("load_splat_"#args[0]),
+                      !cast<NI>("LOAD_SPLAT_"#args[1])>;
+def : LoadPatImmOff<!cast<ValueType>(args[0]),
+                    !cast<PatFrag>("load_splat_"#args[0]),
+                    regPlusImm,
+                    !cast<NI>("LOAD_SPLAT_"#args[1])>;
+def : LoadPatImmOff<!cast<ValueType>(args[0]),
+                    !cast<PatFrag>("load_splat_"#args[0]),
+                    or_is_add,
+                    !cast<NI>("LOAD_SPLAT_"#args[1])>;
+def : LoadPatOffsetOnly<!cast<ValueType>(args[0]),
+                        !cast<PatFrag>("load_splat_"#args[0]),
+                        !cast<NI>("LOAD_SPLAT_"#args[1])>;
+def : LoadPatGlobalAddrOffOnly<!cast<ValueType>(args[0]),
+                               !cast<PatFrag>("load_splat_"#args[0]),
+                               !cast<NI>("LOAD_SPLAT_"#args[1])>;
+}
+
+// Load and extend
+multiclass SIMDLoadExtend<ValueType vec_t, string name, bits<32> simdop> {
+  let mayLoad = 1, UseNamedOperandTable = 1,
+      Predicates = [HasUnimplementedSIMD128] in {
+  defm LOAD_EXTEND_S_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
+           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+           name#"_s\t$dst, ${off}(${addr})$p2align",
+           name#"_s\t$off$p2align", simdop>;
+  defm LOAD_EXTEND_U_#vec_t :
     SIMD_I<(outs V128:$dst), (ins P2Align:$p2align, offset32_op:$off, I32:$addr),
            (outs), (ins P2Align:$p2align, offset32_op:$off), [],
-           "v128.load\t$dst, ${off}(${addr})$p2align",
-           "v128.load\t$off$p2align", 0>;
+           name#"_u\t$dst, ${off}(${addr})$p2align",
+           name#"_u\t$off$p2align", !add(simdop, 1)>;
+  }
 }
 
-foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
-defm "" : SIMDLoad<vec_t>;
+defm "" : SIMDLoadExtend<v8i16, "i16x8.load8x8", 210>;
+defm "" : SIMDLoadExtend<v4i32, "i32x4.load16x4", 212>;
+defm "" : SIMDLoadExtend<v2i64, "i64x2.load32x2", 214>;
 
-// Def load and store patterns from WebAssemblyInstrMemory.td for vector types
-def : LoadPatNoOffset<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
-def : LoadPatImmOff<vec_t, load, regPlusImm, !cast<NI>("LOAD_"#vec_t)>;
-def : LoadPatImmOff<vec_t, load, or_is_add, !cast<NI>("LOAD_"#vec_t)>;
-def : LoadPatGlobalAddr<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
-def : LoadPatOffsetOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
-def : LoadPatGlobalAddrOffOnly<vec_t, load, !cast<NI>("LOAD_"#vec_t)>;
+let Predicates = [HasUnimplementedSIMD128] in
+foreach types = [[v8i16, i8], [v4i32, i16], [v2i64, i32]] in
+foreach exts = [["sextloadv", "_S"],
+                ["zextloadv", "_U"],
+                ["extloadv", "_U"]] in {
+def : LoadPatNoOffset<types[0], !cast<PatFrag>(exts[0]#types[1]),
+                      !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
+def : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), regPlusImm,
+                    !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
+def : LoadPatImmOff<types[0], !cast<PatFrag>(exts[0]#types[1]), or_is_add,
+                    !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
+def : LoadPatOffsetOnly<types[0], !cast<PatFrag>(exts[0]#types[1]),
+                        !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
+def : LoadPatGlobalAddrOffOnly<types[0], !cast<PatFrag>(exts[0]#types[1]),
+                               !cast<NI>("LOAD_EXTEND"#exts[1]#"_"#types[0])>;
 }
 
+
 // Store: v128.store
-multiclass SIMDStore<ValueType vec_t> {
-  let mayStore = 1, UseNamedOperandTable = 1 in
-  defm STORE_#vec_t :
-    SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec),
-           (outs), (ins P2Align:$p2align, offset32_op:$off), [],
-           "v128.store\t${off}(${addr})$p2align, $vec",
-           "v128.store\t$off$p2align", 1>;
-}
+let mayStore = 1, UseNamedOperandTable = 1 in
+defm STORE_V128 :
+  SIMD_I<(outs), (ins P2Align:$p2align, offset32_op:$off, I32:$addr, V128:$vec),
+         (outs), (ins P2Align:$p2align, offset32_op:$off), [],
+         "v128.store\t${off}(${addr})$p2align, $vec",
+         "v128.store\t$off$p2align", 1>;
 
 foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in {
-defm "" : SIMDStore<vec_t>;
-
 // Def load and store patterns from WebAssemblyInstrMemory.td for vector types
-def : StorePatNoOffset<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
-def : StorePatImmOff<vec_t, store, regPlusImm, !cast<NI>("STORE_"#vec_t)>;
-def : StorePatImmOff<vec_t, store, or_is_add, !cast<NI>("STORE_"#vec_t)>;
-def : StorePatGlobalAddr<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
-def : StorePatOffsetOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
-def : StorePatGlobalAddrOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
+def : StorePatNoOffset<vec_t, store, STORE_V128>;
+def : StorePatImmOff<vec_t, store, regPlusImm, STORE_V128>;
+def : StorePatImmOff<vec_t, store, or_is_add, STORE_V128>;
+def : StorePatOffsetOnly<vec_t, store, STORE_V128>;
+def : StorePatGlobalAddrOffOnly<vec_t, store, STORE_V128>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -90,7 +167,7 @@ def : StorePatGlobalAddrOffOnly<vec_t, store, !cast<NI>("STORE_"#vec_t)>;
 // Constant: v128.const
 multiclass ConstVec<ValueType vec_t, dag ops, dag pat, string args> {
   let isMoveImm = 1, isReMaterializable = 1,
-      Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+      Predicates = [HasUnimplementedSIMD128] in
   defm CONST_V128_#vec_t : SIMD_I<(outs V128:$dst), ops, (outs), ops,
                                   [(set V128:$dst, (vec_t pat))],
                                   "v128.const\t$dst, "#args,
@@ -286,7 +363,7 @@ multiclass ExtractLaneExtended<string sign, bits<32> baseInst> {
 }
 
 defm "" : ExtractLaneExtended<"_s", 5>;
-let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+let Predicates = [HasUnimplementedSIMD128] in
 defm "" : ExtractLaneExtended<"_u", 6>;
 defm "" : ExtractLane<v4i32, "i32x4", LaneIdx4, I32, 13>;
 defm "" : ExtractLane<v2i64, "i64x2", LaneIdx2, I64, 16>;
@@ -472,6 +549,11 @@ defm OR : SIMDBitwise<or, "or", 78>;
 defm XOR : SIMDBitwise<xor, "xor", 79>;
 } // isCommutable = 1
 
+// Bitwise logic: v128.andnot
+def andnot : PatFrag<(ops node:$left, node:$right), (and $left, (vnot $right))>;
+let Predicates = [HasUnimplementedSIMD128] in
+defm ANDNOT : SIMDBitwise<andnot, "andnot", 216>;
+
 // Bitwise select: v128.bitselect
 foreach vec_t = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in
   defm BITSELECT_#vec_t :
@@ -655,7 +737,7 @@ defm ABS : SIMDUnaryFP<fabs, "abs", 149>;
 defm NEG : SIMDUnaryFP<fneg, "neg", 150>;
 
 // Square root: sqrt
-let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+let Predicates = [HasUnimplementedSIMD128] in
 defm SQRT : SIMDUnaryFP<fsqrt, "sqrt", 151>;
 
 //===----------------------------------------------------------------------===//
@@ -679,7 +761,7 @@ let isCommutable = 1 in
 defm MUL : SIMDBinaryFP<fmul, "mul", 156>;
 
 // Division: div
-let Predicates = [HasSIMD128, HasUnimplementedSIMD128] in
+let Predicates = [HasUnimplementedSIMD128] in
 defm DIV : SIMDBinaryFP<fdiv, "div", 157>;
 
 // NaN-propagating minimum: min
@@ -712,6 +794,42 @@ defm "" : SIMDConvert<v4i32, v4f32, fp_to_uint, "i32x4.trunc_sat_f32x4_u", 172>;
 defm "" : SIMDConvert<v2i64, v2f64, fp_to_sint, "i64x2.trunc_sat_f64x2_s", 173>;
 defm "" : SIMDConvert<v2i64, v2f64, fp_to_uint, "i64x2.trunc_sat_f64x2_u", 174>;
 
+// Widening operations
+multiclass SIMDWiden<ValueType vec_t, string vec, ValueType arg_t, string arg,
+                     bits<32> baseInst> {
+  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_signed,
+                        vec#".widen_low_"#arg#"_s", baseInst>;
+  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_signed,
+                        vec#".widen_high_"#arg#"_s", !add(baseInst, 1)>;
+  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_low_unsigned,
+                        vec#".widen_low_"#arg#"_u", !add(baseInst, 2)>;
+  defm "" : SIMDConvert<vec_t, arg_t, int_wasm_widen_high_unsigned,
+                        vec#".widen_high_"#arg#"_u", !add(baseInst, 3)>;
+}
+
+defm "" : SIMDWiden<v8i16, "i16x8", v16i8, "i8x16", 202>;
+defm "" : SIMDWiden<v4i32, "i32x4", v8i16, "i16x8", 206>;
+
+// Narrowing operations
+multiclass SIMDNarrow<ValueType vec_t, string vec, ValueType arg_t, string arg,
+                      bits<32> baseInst> {
+  defm NARROW_S_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
+           [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_signed
+             (arg_t V128:$low), (arg_t V128:$high))))],
+           vec#".narrow_"#arg#"_s\t$dst, $low, $high", vec#".narrow_"#arg#"_s",
+           baseInst>;
+  defm NARROW_U_#vec_t :
+    SIMD_I<(outs V128:$dst), (ins V128:$low, V128:$high), (outs), (ins),
+           [(set (vec_t V128:$dst), (vec_t (int_wasm_narrow_unsigned
+             (arg_t V128:$low), (arg_t V128:$high))))],
+           vec#".narrow_"#arg#"_u\t$dst, $low, $high", vec#".narrow_"#arg#"_u",
+           !add(baseInst, 1)>;
+}
+
+defm "" : SIMDNarrow<v16i8, "i8x16", v8i16, "i16x8", 198>;
+defm "" : SIMDNarrow<v8i16, "i16x8", v4i32, "i32x4", 200>;
+
 // Lower llvm.wasm.trunc.saturate.* to saturating instructions
 def : Pat<(v4i32 (int_wasm_trunc_saturate_signed (v4f32 V128:$src))),
           (fp_to_sint_v4i32_v4f32 (v4f32 V128:$src))>;
@@ -736,6 +854,7 @@ def : Pat<(t1 (bitconvert (t2 V128:$v))), (t1 V128:$v)>;
 //===----------------------------------------------------------------------===//
 // Quasi-Fused Multiply- Add and Subtract (QFMA/QFMS)
 //===----------------------------------------------------------------------===//
+
 multiclass SIMDQFM<ValueType vec_t, string vec, bits<32> baseInst> {
   defm QFMA_#vec_t :
     SIMD_I<(outs V128:$dst), (ins V128:$a, V128:$b, V128:$c),
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
index 91e5055ad5977..16e2f4392984c 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyMachineFunctionInfo.h
@@ -101,6 +101,11 @@ class WebAssemblyFunctionInfo final : public MachineFunctionInfo {
       VRegStackified.resize(I + 1);
     VRegStackified.set(I);
   }
+  void unstackifyVReg(unsigned VReg) {
+    auto I = Register::virtReg2Index(VReg);
+    if (I < VRegStackified.size())
+      VRegStackified.reset(I);
+  }
   bool isVRegStackified(unsigned VReg) const {
     auto I = Register::virtReg2Index(VReg);
     if (I >= VRegStackified.size())
diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index ff42996bd6797..f08fcb575bf00 100644
--- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -557,7 +557,7 @@ class DarwinX86AsmBackend : public X86AsmBackend {
 
         // If the frame pointer is other than esp/rsp, we do not have a way to
         // generate a compact unwinding representation, so bail out.
-        if (MRI.getLLVMRegNum(Inst.getRegister(), true) !=
+        if (*MRI.getLLVMRegNum(Inst.getRegister(), true) !=
             (Is64Bit ? X86::RBP : X86::EBP))
           return 0;
 
@@ -605,7 +605,7 @@ class DarwinX86AsmBackend : public X86AsmBackend {
           // unwind encoding.
           return CU::UNWIND_MODE_DWARF;
 
-        unsigned Reg = MRI.getLLVMRegNum(Inst.getRegister(), true);
+        unsigned Reg = *MRI.getLLVMRegNum(Inst.getRegister(), true);
         SavedRegs[SavedRegIdx++] = Reg;
         StackAdjust += OffsetSize;
         InstrOffset += PushInstrSize(Reg);
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 53a05f6a9c42c..d8631aca2734d 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -457,6 +457,10 @@ def FeatureMergeToThreeWayBranch : SubtargetFeature<"merge-to-threeway-branch",
                                         "Merge branches to a three-way "
                                         "conditional branch">;
 
+// Enable use of alias analysis during code generation.
+def FeatureUseAA : SubtargetFeature<"use-aa", "UseAA", "true",
+                                    "Use alias analysis during codegen">;
+
 // Bonnell
 def ProcIntelAtom : SubtargetFeature<"", "X86ProcFamily", "IntelAtom", "">;
 // Silvermont
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 7f6927df9db2a..8d27be30a2774 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -575,7 +575,7 @@ void X86AsmPrinter::EmitStartOfAsmFile(Module &M) {
 
       // Emitting note header.
       int WordSize = TT.isArch64Bit() ? 8 : 4;
-      EmitAlignment(WordSize == 4 ? llvm::Align(4) : llvm::Align(8));
+      EmitAlignment(WordSize == 4 ? Align(4) : Align(8));
       OutStreamer->EmitIntValue(4, 4 /*size*/); // data size for "GNU\0"
       OutStreamer->EmitIntValue(8 + WordSize, 4 /*size*/); // Elf_Prop size
       OutStreamer->EmitIntValue(ELF::NT_GNU_PROPERTY_TYPE_0, 4 /*size*/);
@@ -585,7 +585,7 @@ void X86AsmPrinter::EmitStartOfAsmFile(Module &M) {
       OutStreamer->EmitIntValue(ELF::GNU_PROPERTY_X86_FEATURE_1_AND, 4);
       OutStreamer->EmitIntValue(4, 4);               // data size
       OutStreamer->EmitIntValue(FeatureFlagsAnd, 4); // data
-      EmitAlignment(WordSize == 4 ? llvm::Align(4) : llvm::Align(8)); // padding
+      EmitAlignment(WordSize == 4 ? Align(4) : Align(8)); // padding
 
       OutStreamer->endSection(Nt);
       OutStreamer->SwitchSection(Cur);
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index e1ed5aa5abcce..af3a33ffd4e8b 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2552,8 +2552,8 @@ static unsigned getHiPELiteral(
 
 // Return true if there are no non-ehpad successors to MBB and there are no
 // non-meta instructions between MBBI and MBB.end().
-bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
-                           MachineBasicBlock::const_iterator MBBI) {
+static bool blockEndIsUnreachable(const MachineBasicBlock &MBB,
+                                  MachineBasicBlock::const_iterator MBBI) {
   return std::all_of(
              MBB.succ_begin(), MBB.succ_end(),
              [](const MachineBasicBlock *Succ) { return Succ->isEHPad(); }) &&
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 8e85cd4f24a2e..5b546d42d98a0 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -253,6 +253,11 @@ namespace {
       return tryFoldLoad(P, P, N, Base, Scale, Index, Disp, Segment);
     }
 
+    bool tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
+                          SDValue &Base, SDValue &Scale,
+                          SDValue &Index, SDValue &Disp,
+                          SDValue &Segment);
+
     /// Implement addressing mode selection for inline asm expressions.
     bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                       unsigned ConstraintID,
@@ -510,6 +515,7 @@ namespace {
     bool combineIncDecVector(SDNode *Node);
     bool tryShrinkShlLogicImm(SDNode *N);
     bool tryVPTESTM(SDNode *Root, SDValue Setcc, SDValue Mask);
+    bool tryMatchBitSelect(SDNode *N);
 
     MachineSDNode *emitPCMPISTR(unsigned ROpc, unsigned MOpc, bool MayFoldLoad,
                                 const SDLoc &dl, MVT VT, SDNode *Node);
@@ -879,10 +885,9 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
       case ISD::FRINT:      Imm = 0x4; break;
       }
       SDLoc dl(N);
-      SDValue Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl,
-                                    N->getValueType(0),
-                                    N->getOperand(0),
-                                    CurDAG->getConstant(Imm, dl, MVT::i8));
+      SDValue Res = CurDAG->getNode(
+          X86ISD::VRNDSCALE, dl, N->getValueType(0), N->getOperand(0),
+          CurDAG->getTargetConstant(Imm, dl, MVT::i8));
       --I;
       CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res);
       ++I;
@@ -2581,6 +2586,7 @@ bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
                                   SDValue &Base, SDValue &Scale,
                                   SDValue &Index, SDValue &Disp,
                                   SDValue &Segment) {
+  assert(Root && P && "Unknown root/parent nodes");
   if (!ISD::isNON_EXTLoad(N.getNode()) ||
       !IsProfitableToFold(N, P, Root) ||
       !IsLegalToFold(N, P, Root, OptLevel))
@@ -2590,6 +2596,20 @@ bool X86DAGToDAGISel::tryFoldLoad(SDNode *Root, SDNode *P, SDValue N,
                     N.getOperand(1), Base, Scale, Index, Disp, Segment);
 }
 
+bool X86DAGToDAGISel::tryFoldBroadcast(SDNode *Root, SDNode *P, SDValue N,
+                                       SDValue &Base, SDValue &Scale,
+                                       SDValue &Index, SDValue &Disp,
+                                       SDValue &Segment) {
+  assert(Root && P && "Unknown root/parent nodes");
+  if (N->getOpcode() != X86ISD::VBROADCAST_LOAD ||
+      !IsProfitableToFold(N, P, Root) ||
+      !IsLegalToFold(N, P, Root, OptLevel))
+    return false;
+
+  return selectAddr(N.getNode(),
+                    N.getOperand(1), Base, Scale, Index, Disp, Segment);
+}
+
 /// Return an SDNode that returns the value of the global base register.
 /// Output instructions required to initialize the global base register,
 /// if necessary.
@@ -3441,8 +3461,9 @@ MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
   // TODO: Maybe load folding, greater than 32-bit masks, or a guarantee of LICM
   // hoisting the move immediate would make it worthwhile with a less optimal
   // BEXTR?
-  if (!Subtarget->hasTBM() &&
-      !(Subtarget->hasBMI() && Subtarget->hasFastBEXTR()))
+  bool PreferBEXTR =
+      Subtarget->hasTBM() || (Subtarget->hasBMI() && Subtarget->hasFastBEXTR());
+  if (!PreferBEXTR && !Subtarget->hasBMI2())
     return nullptr;
 
   // Must have a shift right.
@@ -3481,23 +3502,50 @@ MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
   if (Shift + MaskSize > NVT.getSizeInBits())
     return nullptr;
 
-  SDValue New = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
-  unsigned ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
-  unsigned MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
+  // BZHI, if available, is always fast, unlike BEXTR. But even if we decide
+  // that we can't use BEXTR, it is only worthwhile using BZHI if the mask
+  // does not fit into 32 bits. Load folding is not a sufficient reason.
+  if (!PreferBEXTR && MaskSize <= 32)
+    return nullptr;
 
-  // BMI requires the immediate to placed in a register.
-  if (!Subtarget->hasTBM()) {
-    ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr;
-    MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm;
+  SDValue Control;
+  unsigned ROpc, MOpc;
+
+  if (!PreferBEXTR) {
+    assert(Subtarget->hasBMI2() && "We must have BMI2's BZHI then.");
+    // If we can't make use of BEXTR then we can't fuse shift+mask stages.
+    // Let's perform the mask first, and apply shift later. Note that we need to
+    // widen the mask to account for the fact that we'll apply shift afterwards!
+    Control = CurDAG->getTargetConstant(Shift + MaskSize, dl, NVT);
+    ROpc = NVT == MVT::i64 ? X86::BZHI64rr : X86::BZHI32rr;
+    MOpc = NVT == MVT::i64 ? X86::BZHI64rm : X86::BZHI32rm;
     unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
-    New = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, New), 0);
+    Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
+  } else {
+    // The 'control' of BEXTR has the pattern of:
+    // [15...8 bit][ 7...0 bit] location
+    // [ bit count][     shift] name
+    // I.e. 0b000000011'00000001 means  (x >> 0b1) & 0b11
+    Control = CurDAG->getTargetConstant(Shift | (MaskSize << 8), dl, NVT);
+    if (Subtarget->hasTBM()) {
+      ROpc = NVT == MVT::i64 ? X86::BEXTRI64ri : X86::BEXTRI32ri;
+      MOpc = NVT == MVT::i64 ? X86::BEXTRI64mi : X86::BEXTRI32mi;
+    } else {
+      assert(Subtarget->hasBMI() && "We must have BMI1's BEXTR then.");
+      // BMI requires the immediate to placed in a register.
+      ROpc = NVT == MVT::i64 ? X86::BEXTR64rr : X86::BEXTR32rr;
+      MOpc = NVT == MVT::i64 ? X86::BEXTR64rm : X86::BEXTR32rm;
+      unsigned NewOpc = NVT == MVT::i64 ? X86::MOV32ri64 : X86::MOV32ri;
+      Control = SDValue(CurDAG->getMachineNode(NewOpc, dl, NVT, Control), 0);
+    }
   }
 
   MachineSDNode *NewNode;
   SDValue Input = N0->getOperand(0);
   SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
   if (tryFoldLoad(Node, N0.getNode(), Input, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4)) {
-    SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, New, Input.getOperand(0) };
+    SDValue Ops[] = {
+        Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Control, Input.getOperand(0)};
     SDVTList VTs = CurDAG->getVTList(NVT, MVT::i32, MVT::Other);
     NewNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops);
     // Update the chain.
@@ -3505,7 +3553,15 @@ MachineSDNode *X86DAGToDAGISel::matchBEXTRFromAndImm(SDNode *Node) {
     // Record the mem-refs
     CurDAG->setNodeMemRefs(NewNode, {cast<LoadSDNode>(Input)->getMemOperand()});
   } else {
-    NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, New);
+    NewNode = CurDAG->getMachineNode(ROpc, dl, NVT, MVT::i32, Input, Control);
+  }
+
+  if (!PreferBEXTR) {
+    // We still need to apply the shift.
+    SDValue ShAmt = CurDAG->getTargetConstant(Shift, dl, NVT);
+    unsigned NewOpc = NVT == MVT::i64 ? X86::SHR64ri : X86::SHR32ri;
+    NewNode =
+        CurDAG->getMachineNode(NewOpc, dl, NVT, SDValue(NewNode, 0), ShAmt);
   }
 
   return NewNode;
@@ -4062,12 +4118,18 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
   if (CC != ISD::SETEQ && CC != ISD::SETNE)
     return false;
 
-  // See if we're comparing against zero. This should have been canonicalized
-  // to RHS during lowering.
-  if (!ISD::isBuildVectorAllZeros(Setcc.getOperand(1).getNode()))
+  SDValue SetccOp0 = Setcc.getOperand(0);
+  SDValue SetccOp1 = Setcc.getOperand(1);
+
+  // Canonicalize the all zero vector to the RHS.
+  if (ISD::isBuildVectorAllZeros(SetccOp0.getNode()))
+    std::swap(SetccOp0, SetccOp1);
+
+  // See if we're comparing against zero.
+  if (!ISD::isBuildVectorAllZeros(SetccOp1.getNode()))
     return false;
 
-  SDValue N0 = Setcc.getOperand(0);
+  SDValue N0 = SetccOp0;
 
   MVT CmpVT = N0.getSimpleValueType();
   MVT CmpSVT = CmpVT.getVectorElementType();
@@ -4114,13 +4176,14 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
 
   auto findBroadcastedOp = [](SDValue Src, MVT CmpSVT, SDNode *&Parent) {
     // Look through single use bitcasts.
-    if (Src.getOpcode() == ISD::BITCAST && Src.hasOneUse())
-      Src = Src.getOperand(0);
-
-    if (Src.getOpcode() == X86ISD::VBROADCAST && Src.hasOneUse()) {
+    if (Src.getOpcode() == ISD::BITCAST && Src.hasOneUse()) {
       Parent = Src.getNode();
       Src = Src.getOperand(0);
-      if (Src.getSimpleValueType() == CmpSVT)
+    }
+
+    if (Src.getOpcode() == X86ISD::VBROADCAST_LOAD && Src.hasOneUse()) {
+      auto *MemIntr = cast<MemIntrinsicSDNode>(Src);
+      if (MemIntr->getMemoryVT().getSizeInBits() == CmpSVT.getSizeInBits())
         return Src;
     }
 
@@ -4132,17 +4195,18 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
   bool FoldedBCast = false;
   if (!FoldedLoad && CanFoldLoads &&
       (CmpSVT == MVT::i32 || CmpSVT == MVT::i64)) {
-    SDNode *ParentNode = nullptr;
+    SDNode *ParentNode = N0.getNode();
     if ((Load = findBroadcastedOp(Src1, CmpSVT, ParentNode))) {
-      FoldedBCast = tryFoldLoad(Root, ParentNode, Load, Tmp0,
-                                Tmp1, Tmp2, Tmp3, Tmp4);
+      FoldedBCast = tryFoldBroadcast(Root, ParentNode, Load, Tmp0,
+                                     Tmp1, Tmp2, Tmp3, Tmp4);
     }
 
     // Try the other operand.
     if (!FoldedBCast) {
+      SDNode *ParentNode = N0.getNode();
       if ((Load = findBroadcastedOp(Src0, CmpSVT, ParentNode))) {
-        FoldedBCast = tryFoldLoad(Root, ParentNode, Load, Tmp0,
-                                  Tmp1, Tmp2, Tmp3, Tmp4);
+        FoldedBCast = tryFoldBroadcast(Root, ParentNode, Load, Tmp0,
+                                       Tmp1, Tmp2, Tmp3, Tmp4);
         if (FoldedBCast)
           std::swap(Src0, Src1);
       }
@@ -4212,7 +4276,7 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
     // Update the chain.
     ReplaceUses(Load.getValue(1), SDValue(CNode, 1));
     // Record the mem-refs
-    CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(Load)->getMemOperand()});
+    CurDAG->setNodeMemRefs(CNode, {cast<MemSDNode>(Load)->getMemOperand()});
   } else {
     if (IsMasked)
       CNode = CurDAG->getMachineNode(Opc, dl, MaskVT, InMask, Src0, Src1);
@@ -4233,6 +4297,55 @@ bool X86DAGToDAGISel::tryVPTESTM(SDNode *Root, SDValue Setcc,
   return true;
 }
 
+// Try to match the bitselect pattern (or (and A, B), (andn A, C)). Turn it
+// into vpternlog.
+bool X86DAGToDAGISel::tryMatchBitSelect(SDNode *N) {
+  assert(N->getOpcode() == ISD::OR && "Unexpected opcode!");
+
+  MVT NVT = N->getSimpleValueType(0);
+
+  // Make sure we support VPTERNLOG.
+  if (!NVT.isVector() || !Subtarget->hasAVX512())
+    return false;
+
+  // We need VLX for 128/256-bit.
+  if (!(Subtarget->hasVLX() || NVT.is512BitVector()))
+    return false;
+
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+
+  // Canonicalize AND to LHS.
+  if (N1.getOpcode() == ISD::AND)
+    std::swap(N0, N1);
+
+  if (N0.getOpcode() != ISD::AND ||
+      N1.getOpcode() != X86ISD::ANDNP ||
+      !N0.hasOneUse() || !N1.hasOneUse())
+    return false;
+
+  // ANDN is not commutable, use it to pick down A and C.
+  SDValue A = N1.getOperand(0);
+  SDValue C = N1.getOperand(1);
+
+  // AND is commutable, if one operand matches A, the other operand is B.
+  // Otherwise this isn't a match.
+  SDValue B;
+  if (N0.getOperand(0) == A)
+    B = N0.getOperand(1);
+  else if (N0.getOperand(1) == A)
+    B = N0.getOperand(0);
+  else
+    return false;
+
+  SDLoc dl(N);
+  SDValue Imm = CurDAG->getTargetConstant(0xCA, dl, MVT::i8);
+  SDValue Ternlog = CurDAG->getNode(X86ISD::VPTERNLOG, dl, NVT, A, B, C, Imm);
+  ReplaceNode(N, Ternlog.getNode());
+  SelectCode(Ternlog.getNode());
+  return true;
+}
+
 void X86DAGToDAGISel::Select(SDNode *Node) {
   MVT NVT = Node->getSimpleValueType(0);
   unsigned Opcode = Node->getOpcode();
@@ -4391,6 +4504,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     if (tryShrinkShlLogicImm(Node))
       return;
 
+    if (Opcode == ISD::OR && tryMatchBitSelect(Node))
+      return;
+
     LLVM_FALLTHROUGH;
   case ISD::ADD:
   case ISD::SUB: {
@@ -5095,10 +5211,9 @@ void X86DAGToDAGISel::Select(SDNode *Node) {
     case ISD::FRINT:      Imm = 0x4; break;
     }
     SDLoc dl(Node);
-    SDValue Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl,
-                                  Node->getValueType(0),
+    SDValue Res = CurDAG->getNode(X86ISD::VRNDSCALE, dl, Node->getValueType(0),
                                   Node->getOperand(0),
-                                  CurDAG->getConstant(Imm, dl, MVT::i8));
+                                  CurDAG->getTargetConstant(Imm, dl, MVT::i8));
     ReplaceNode(Node, Res.getNode());
     SelectCode(Res.getNode());
     return;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8205886c689b8..466a33cb6c1ec 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -65,12 +65,6 @@ using namespace llvm;
 
 STATISTIC(NumTailCalls, "Number of tail calls");
 
-cl::opt<bool> ExperimentalVectorWideningLegalization(
-    "x86-experimental-vector-widening-legalization", cl::init(true),
-    cl::desc("Enable an experimental vector type legalization through widening "
-             "rather than promotion."),
-    cl::Hidden);
-
 static cl::opt<int> ExperimentalPrefLoopAlignment(
     "x86-experimental-pref-loop-alignment", cl::init(4),
     cl::desc(
@@ -211,7 +205,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   // Integer absolute.
   if (Subtarget.hasCMov()) {
     setOperationAction(ISD::ABS            , MVT::i16  , Custom);
-    setOperationAction(ISD::ABS            , MVT::i32  , Custom); 
+    setOperationAction(ISD::ABS            , MVT::i32  , Custom);
   }
   setOperationAction(ISD::ABS              , MVT::i64  , Custom);
 
@@ -749,7 +743,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   // First set operation action for all vector types to either promote
   // (for widening) or expand (for scalarization). Then we will selectively
   // turn on ones that can be effectively codegen'd.
-  for (MVT VT : MVT::vector_valuetypes()) {
+  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
     setOperationAction(ISD::SDIV, VT, Expand);
     setOperationAction(ISD::UDIV, VT, Expand);
     setOperationAction(ISD::SREM, VT, Expand);
@@ -787,7 +781,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
     setOperationAction(ISD::ANY_EXTEND, VT, Expand);
     setOperationAction(ISD::SELECT_CC, VT, Expand);
-    for (MVT InnerVT : MVT::vector_valuetypes()) {
+    for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
       setTruncStoreAction(InnerVT, VT, Expand);
 
       setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
@@ -857,12 +851,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::UREM, VT, Custom);
     }
 
-    if (!ExperimentalVectorWideningLegalization) {
-      setOperationAction(ISD::MUL,                MVT::v2i16, Custom);
-      setOperationAction(ISD::MUL,                MVT::v2i32, Custom);
-      setOperationAction(ISD::MUL,                MVT::v4i16, Custom);
-    }
-
     setOperationAction(ISD::MUL,                MVT::v2i8,  Custom);
     setOperationAction(ISD::MUL,                MVT::v4i8,  Custom);
     setOperationAction(ISD::MUL,                MVT::v8i8,  Custom);
@@ -901,29 +889,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::UADDSAT,            MVT::v2i64, Custom);
     setOperationAction(ISD::USUBSAT,            MVT::v2i64, Custom);
 
-    if (!ExperimentalVectorWideningLegalization) {
-      // Use widening instead of promotion.
-      for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8,
-                       MVT::v4i16, MVT::v2i16 }) {
-        setOperationAction(ISD::UADDSAT, VT, Custom);
-        setOperationAction(ISD::SADDSAT, VT, Custom);
-        setOperationAction(ISD::USUBSAT, VT, Custom);
-        setOperationAction(ISD::SSUBSAT, VT, Custom);
-      }
-    }
-
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v8i16, Custom);
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4i32, Custom);
     setOperationAction(ISD::INSERT_VECTOR_ELT,  MVT::v4f32, Custom);
 
-    // Provide custom widening for v2f32 setcc. This is really for VLX when
-    // setcc result type returns v2i1/v4i1 vector for v2f32/v4f32 leading to
-    // type legalization changing the result type to v4i1 during widening.
-    // It works fine for SSE2 and is probably faster so no need to qualify with
-    // VLX support.
-    if (!ExperimentalVectorWideningLegalization)
-      setOperationAction(ISD::SETCC,               MVT::v2i32, Custom);
-
     for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
       setOperationAction(ISD::SETCC,              VT, Custom);
       setOperationAction(ISD::CTPOP,              VT, Custom);
@@ -943,21 +912,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
     }
 
-    // We support custom legalizing of sext and anyext loads for specific
-    // memory vector types which we can load as a scalar (or sequence of
-    // scalars) and extend in-register to a legal 128-bit vector type. For sext
-    // loads these must work with a single scalar load.
-    if (!ExperimentalVectorWideningLegalization) {
-      for (MVT VT : MVT::integer_vector_valuetypes()) {
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom);
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom);
-        setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);
-      }
-    }
-
     for (auto VT : { MVT::v2f64, MVT::v2i64 }) {
       setOperationAction(ISD::BUILD_VECTOR,       VT, Custom);
       setOperationAction(ISD::VECTOR_SHUFFLE,     VT, Custom);
@@ -1031,18 +985,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
     setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
 
-    if (ExperimentalVectorWideningLegalization) {
-      setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
+    setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
 
-      setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);
-      setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);
-      setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);
-      setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
-      setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
-      setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
-    } else {
-      setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i64, Custom);
-    }
+    setOperationAction(ISD::TRUNCATE,    MVT::v2i8,  Custom);
+    setOperationAction(ISD::TRUNCATE,    MVT::v2i16, Custom);
+    setOperationAction(ISD::TRUNCATE,    MVT::v2i32, Custom);
+    setOperationAction(ISD::TRUNCATE,    MVT::v4i8,  Custom);
+    setOperationAction(ISD::TRUNCATE,    MVT::v4i16, Custom);
+    setOperationAction(ISD::TRUNCATE,    MVT::v8i8,  Custom);
 
     // In the customized shift lowering, the legal v4i32/v2i64 cases
     // in AVX2 will be recognized.
@@ -1109,22 +1059,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
     }
 
-    if (!ExperimentalVectorWideningLegalization) {
-      // Avoid narrow result types when widening. The legal types are listed
-      // in the next loop.
-      for (MVT VT : MVT::integer_vector_valuetypes()) {
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom);
-        setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom);
-      }
-    }
-
     // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
     for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
       setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8,  Legal);
       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8,  Legal);
-      if (!ExperimentalVectorWideningLegalization)
-        setLoadExtAction(LoadExtOp, MVT::v2i32, MVT::v2i8,  Legal);
       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8,  Legal);
       setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
       setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
@@ -1476,12 +1414,10 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v16i32, Custom);
     setOperationAction(ISD::SIGN_EXTEND,        MVT::v8i64, Custom);
 
-    if (ExperimentalVectorWideningLegalization) {
-      // Need to custom widen this if we don't have AVX512BW.
-      setOperationAction(ISD::ANY_EXTEND,         MVT::v8i8, Custom);
-      setOperationAction(ISD::ZERO_EXTEND,        MVT::v8i8, Custom);
-      setOperationAction(ISD::SIGN_EXTEND,        MVT::v8i8, Custom);
-    }
+    // Need to custom widen this if we don't have AVX512BW.
+    setOperationAction(ISD::ANY_EXTEND,         MVT::v8i8, Custom);
+    setOperationAction(ISD::ZERO_EXTEND,        MVT::v8i8, Custom);
+    setOperationAction(ISD::SIGN_EXTEND,        MVT::v8i8, Custom);
 
     for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
       setOperationAction(ISD::FFLOOR,           VT, Legal);
@@ -1572,10 +1508,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
       setOperationAction(ISD::MGATHER,             VT, Custom);
       setOperationAction(ISD::MSCATTER,            VT, Custom);
     }
-    // Need to custom split v32i16/v64i8 bitcasts.
     if (!Subtarget.hasBWI()) {
+      // Need to custom split v32i16/v64i8 bitcasts.
       setOperationAction(ISD::BITCAST, MVT::v32i16, Custom);
       setOperationAction(ISD::BITCAST, MVT::v64i8,  Custom);
+
+      // Better to split these into two 256-bit ops.
+      setOperationAction(ISD::BITREVERSE, MVT::v8i64, Custom);
+      setOperationAction(ISD::BITREVERSE, MVT::v16i32, Custom);
     }
 
     if (Subtarget.hasVBMI2()) {
@@ -1951,13 +1891,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   MaxLoadsPerMemcmpOptSize = 2;
 
   // Set loop alignment to 2^ExperimentalPrefLoopAlignment bytes (default: 2^4).
-  setPrefLoopAlignment(llvm::Align(1ULL << ExperimentalPrefLoopAlignment));
+  setPrefLoopAlignment(Align(1ULL << ExperimentalPrefLoopAlignment));
 
   // An out-of-order CPU can speculatively execute past a predictable branch,
   // but a conditional move could be stalled by an expensive earlier operation.
   PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
   EnableExtLdPromotion = true;
-  setPrefFunctionAlignment(llvm::Align(16));
+  setPrefFunctionAlignment(Align(16));
 
   verifyIntrinsicTables();
 }
@@ -1985,8 +1925,7 @@ X86TargetLowering::getPreferredVectorAction(MVT VT) const {
   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
     return TypeSplitVector;
 
-  if (ExperimentalVectorWideningLegalization &&
-      VT.getVectorNumElements() != 1 &&
+  if (VT.getVectorNumElements() != 1 &&
       VT.getVectorElementType() != MVT::i1)
     return TypeWidenVector;
 
@@ -1996,8 +1935,16 @@ X86TargetLowering::getPreferredVectorAction(MVT VT) const {
 MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
                                                      CallingConv::ID CC,
                                                      EVT VT) const {
+  // v32i1 vectors should be promoted to v32i8 to match avx2.
   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
     return MVT::v32i8;
+  // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
+  if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
+      Subtarget.hasAVX512() &&
+      (!isPowerOf2_32(VT.getVectorNumElements()) ||
+       (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
+       (VT.getVectorNumElements() > 64 && Subtarget.hasBWI())))
+    return MVT::i8;
   // FIXME: Should we just make these types legal and custom split operations?
   if ((VT == MVT::v32i16 || VT == MVT::v64i8) &&
       Subtarget.hasAVX512() && !Subtarget.hasBWI() && !EnableOldKNLABI)
@@ -2008,8 +1955,16 @@ MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
 unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
                                                           CallingConv::ID CC,
                                                           EVT VT) const {
+  // v32i1 vectors should be promoted to v32i8 to match avx2.
   if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
     return 1;
+  // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
+  if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
+      Subtarget.hasAVX512() &&
+      (!isPowerOf2_32(VT.getVectorNumElements()) ||
+       (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
+       (VT.getVectorNumElements() > 64 && Subtarget.hasBWI())))
+    return VT.getVectorNumElements();
   // FIXME: Should we just make these types legal and custom split operations?
   if ((VT == MVT::v32i16 || VT == MVT::v64i8) &&
       Subtarget.hasAVX512() && !Subtarget.hasBWI() && !EnableOldKNLABI)
@@ -2017,6 +1972,25 @@ unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
   return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
 }
 
+unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv(
+    LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
+    unsigned &NumIntermediates, MVT &RegisterVT) const {
+  // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
+  if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
+      Subtarget.hasAVX512() &&
+      (!isPowerOf2_32(VT.getVectorNumElements()) ||
+       (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
+       (VT.getVectorNumElements() > 64 && Subtarget.hasBWI()))) {
+    RegisterVT = MVT::i8;
+    IntermediateVT = MVT::i1;
+    NumIntermediates = VT.getVectorNumElements();
+    return NumIntermediates;
+  }
+
+  return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT,
+                                              NumIntermediates, RegisterVT);
+}
+
 EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
                                           LLVMContext& Context,
                                           EVT VT) const {
@@ -2114,6 +2088,11 @@ EVT X86TargetLowering::getOptimalMemOpType(
     if (Size >= 16 && (!Subtarget.isUnalignedMem16Slow() ||
                        ((DstAlign == 0 || DstAlign >= 16) &&
                         (SrcAlign == 0 || SrcAlign >= 16)))) {
+      // FIXME: Check if unaligned 64-byte accesses are slow.
+      if (Size >= 64 && Subtarget.hasAVX512() &&
+          (Subtarget.getPreferVectorWidth() >= 512)) {
+        return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
+      }
       // FIXME: Check if unaligned 32-byte accesses are slow.
       if (Size >= 32 && Subtarget.hasAVX() &&
           (Subtarget.getPreferVectorWidth() >= 256)) {
@@ -2457,8 +2436,8 @@ static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
 
 /// Breaks v64i1 value into two registers and adds the new node to the DAG
 static void Passv64i1ArgInRegs(
-    const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg,
-    SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, CCValAssign &VA,
+    const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg,
+    SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, CCValAssign &VA,
     CCValAssign &NextVA, const X86Subtarget &Subtarget) {
   assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
   assert(Subtarget.is32Bit() && "Expecting 32 bit target");
@@ -2591,7 +2570,7 @@ X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
       assert(VA.getValVT() == MVT::v64i1 &&
              "Currently the only custom case is when we split v64i1 to 2 regs");
 
-      Passv64i1ArgInRegs(dl, DAG, Chain, ValToCopy, RegsToPass, VA, RVLocs[++I],
+      Passv64i1ArgInRegs(dl, DAG, ValToCopy, RegsToPass, VA, RVLocs[++I],
                          Subtarget);
 
       assert(2 == RegsToPass.size() &&
@@ -2870,6 +2849,10 @@ SDValue X86TargetLowering::LowerCallResult(
         ((Is64Bit || Ins[InsIndex].Flags.isInReg()) && !Subtarget.hasSSE1())) {
       errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
       VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
+    } else if (CopyVT == MVT::f64 &&
+               (Is64Bit && !Subtarget.hasSSE2())) {
+      errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
+      VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
     }
 
     // If we prefer to use the value in xmm registers, copy it out as f80 and
@@ -3836,8 +3819,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       assert(VA.getValVT() == MVT::v64i1 &&
              "Currently the only custom case is when we split v64i1 to 2 regs");
       // Split v64i1 value into two registers
-      Passv64i1ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++I],
-                         Subtarget);
+      Passv64i1ArgInRegs(dl, DAG, Arg, RegsToPass, VA, ArgLocs[++I], Subtarget);
     } else if (VA.isRegLoc()) {
       RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
       const TargetOptions &Options = DAG.getTarget().Options;
@@ -4823,7 +4805,7 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
       ScalarVT = MVT::i32;
 
     Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags |= MachineMemOperand::MOStore;
     break;
   }
@@ -4836,7 +4818,7 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
                                 IndexVT.getVectorNumElements());
     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags |= MachineMemOperand::MOLoad;
     break;
   }
@@ -4848,7 +4830,7 @@ bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
     unsigned NumElts = std::min(DataVT.getVectorNumElements(),
                                 IndexVT.getVectorNumElements());
     Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
-    Info.align = Align(1);
+    Info.align = Align::None();
     Info.flags |= MachineMemOperand::MOStore;
     break;
   }
@@ -4943,7 +4925,7 @@ bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
 
   // Find the type this will be legalized too. Otherwise we might prematurely
   // convert this to shl+add/sub and then still have to type legalize those ops.
-  // Another choice would be to defer the decision for illegal types until 
+  // Another choice would be to defer the decision for illegal types until
   // after type legalization. But constant splat vectors of i64 can't make it
   // through type legalization on 32-bit targets so we would need to special
   // case vXi64.
@@ -5721,7 +5703,7 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
 
   if (IdxVal == 0) {
     // Zero lower bits of the Vec
-    SDValue ShiftBits = DAG.getConstant(SubVecNumElems, dl, MVT::i8);
+    SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
     Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
                       ZeroIdx);
     Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
@@ -5740,7 +5722,7 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
   if (Vec.isUndef()) {
     assert(IdxVal != 0 && "Unexpected index");
     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
-                         DAG.getConstant(IdxVal, dl, MVT::i8));
+                         DAG.getTargetConstant(IdxVal, dl, MVT::i8));
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
   }
 
@@ -5750,17 +5732,17 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
     unsigned ShiftLeft = NumElems - SubVecNumElems;
     unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
-                         DAG.getConstant(ShiftLeft, dl, MVT::i8));
+                         DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
     if (ShiftRight != 0)
       SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
-                           DAG.getConstant(ShiftRight, dl, MVT::i8));
+                           DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
   }
 
   // Simple case when we put subvector in the upper part
   if (IdxVal + SubVecNumElems == NumElems) {
     SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
-                         DAG.getConstant(IdxVal, dl, MVT::i8));
+                         DAG.getTargetConstant(IdxVal, dl, MVT::i8));
     if (SubVecNumElems * 2 == NumElems) {
       // Special case, use legal zero extending insert_subvector. This allows
       // isel to opimitize when bits are known zero.
@@ -5773,7 +5755,7 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
       Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
                         Undef, Vec, ZeroIdx);
       NumElems = WideOpVT.getVectorNumElements();
-      SDValue ShiftBits = DAG.getConstant(NumElems - IdxVal, dl, MVT::i8);
+      SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
       Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
       Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
     }
@@ -5787,23 +5769,35 @@ static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
 
   // Widen the vector if needed.
   Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
-  // Move the current value of the bit to be replace to the lsbs.
-  Op = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
-                   DAG.getConstant(IdxVal, dl, MVT::i8));
-  // Xor with the new bit.
-  Op = DAG.getNode(ISD::XOR, dl, WideOpVT, Op, SubVec);
-  // Shift to MSB, filling bottom bits with 0.
+
+  // Clear the upper bits of the subvector and move it to its insert position.
   unsigned ShiftLeft = NumElems - SubVecNumElems;
-  Op = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Op,
-                   DAG.getConstant(ShiftLeft, dl, MVT::i8));
-  // Shift to the final position, filling upper bits with 0.
+  SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
+                       DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
   unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
-  Op = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Op,
-                       DAG.getConstant(ShiftRight, dl, MVT::i8));
-  // Xor with original vector leaving the new value.
-  Op = DAG.getNode(ISD::XOR, dl, WideOpVT, Vec, Op);
+  SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
+                       DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
+
+  // Isolate the bits below the insertion point.
+  unsigned LowShift = NumElems - IdxVal;
+  SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
+                            DAG.getTargetConstant(LowShift, dl, MVT::i8));
+  Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
+                    DAG.getTargetConstant(LowShift, dl, MVT::i8));
+
+  // Isolate the bits after the last inserted bit.
+  unsigned HighShift = IdxVal + SubVecNumElems;
+  SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
+                            DAG.getTargetConstant(HighShift, dl, MVT::i8));
+  High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
+                    DAG.getTargetConstant(HighShift, dl, MVT::i8));
+
+  // Now OR all 3 pieces together.
+  Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
+  SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
+
   // Reduce to original width if needed.
-  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
 }
 
 static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
@@ -6148,6 +6142,37 @@ static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
     }
   }
 
+  if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
+      EltSizeInBits <= VT.getScalarSizeInBits()) {
+    auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
+    if (MemIntr->getMemoryVT().getScalarSizeInBits() != VT.getScalarSizeInBits())
+      return false;
+
+    SDValue Ptr = MemIntr->getBasePtr();
+    if (Ptr->getOpcode() == X86ISD::Wrapper ||
+        Ptr->getOpcode() == X86ISD::WrapperRIP)
+      Ptr = Ptr->getOperand(0);
+
+    auto *CNode = dyn_cast<ConstantPoolSDNode>(Ptr);
+    if (!CNode || CNode->isMachineConstantPoolEntry() ||
+        CNode->getOffset() != 0)
+      return false;
+
+    if (const Constant *C = CNode->getConstVal()) {
+      unsigned SrcEltSizeInBits = C->getType()->getScalarSizeInBits();
+      unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
+
+      APInt UndefSrcElts(NumSrcElts, 0);
+      SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
+      if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
+        if (UndefSrcElts[0])
+          UndefSrcElts.setBits(0, NumSrcElts);
+        SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
+        return CastBitData(UndefSrcElts, SrcEltBits);
+      }
+    }
+  }
+
   // Extract constant bits from a subvector broadcast.
   if (Op.getOpcode() == X86ISD::SUBV_BROADCAST) {
     SmallVector<APInt, 16> SubEltBits;
@@ -7599,7 +7624,7 @@ static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
   assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!");
   SDLoc DL(Op);
   SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
-                               DAG.getIntPtrConstant(InsertPSMask, DL));
+                               DAG.getIntPtrConstant(InsertPSMask, DL, true));
   return DAG.getBitcast(VT, Result);
 }
 
@@ -7612,7 +7637,7 @@ static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
   unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
   SrcOp = DAG.getBitcast(ShVT, SrcOp);
   assert(NumBits % 8 == 0 && "Only support byte sized shifts");
-  SDValue ShiftVal = DAG.getConstant(NumBits/8, dl, MVT::i8);
+  SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
   return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
 }
 
@@ -8451,34 +8476,10 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
          "Unexpected type in LowerBUILD_VECTORvXi1!");
 
   SDLoc dl(Op);
-  if (ISD::isBuildVectorAllZeros(Op.getNode()))
-    return Op;
-
-  if (ISD::isBuildVectorAllOnes(Op.getNode()))
+  if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
+      ISD::isBuildVectorAllOnes(Op.getNode()))
     return Op;
 
-  if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
-    if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
-      // Split the pieces.
-      SDValue Lower =
-          DAG.getBuildVector(MVT::v32i1, dl, Op.getNode()->ops().slice(0, 32));
-      SDValue Upper =
-          DAG.getBuildVector(MVT::v32i1, dl, Op.getNode()->ops().slice(32, 32));
-      // We have to manually lower both halves so getNode doesn't try to
-      // reassemble the build_vector.
-      Lower = LowerBUILD_VECTORvXi1(Lower, DAG, Subtarget);
-      Upper = LowerBUILD_VECTORvXi1(Upper, DAG, Subtarget);
-      return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lower, Upper);
-    }
-    SDValue Imm = ConvertI1VectorToInteger(Op, DAG);
-    if (Imm.getValueSizeInBits() == VT.getSizeInBits())
-      return DAG.getBitcast(VT, Imm);
-    SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm);
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,
-                        DAG.getIntPtrConstant(0, dl));
-  }
-
-  // Vector has one or more non-const elements
   uint64_t Immediate = 0;
   SmallVector<unsigned, 16> NonConstIdx;
   bool IsSplat = true;
@@ -8501,29 +8502,40 @@ static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
   }
 
   // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
-  if (IsSplat)
-    return DAG.getSelect(dl, VT, Op.getOperand(SplatIdx),
+  if (IsSplat) {
+    // The build_vector allows the scalar element to be larger than the vector
+    // element type. We need to mask it to use as a condition unless we know
+    // the upper bits are zero.
+    // FIXME: Use computeKnownBits instead of checking specific opcode?
+    SDValue Cond = Op.getOperand(SplatIdx);
+    assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!");
+    if (Cond.getOpcode() != ISD::SETCC)
+      Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
+                         DAG.getConstant(1, dl, MVT::i8));
+    return DAG.getSelect(dl, VT, Cond,
                          DAG.getConstant(1, dl, VT),
                          DAG.getConstant(0, dl, VT));
+  }
 
   // insert elements one by one
   SDValue DstVec;
-  SDValue Imm;
-  if (Immediate) {
-    MVT ImmVT = MVT::getIntegerVT(std::max((int)VT.getSizeInBits(), 8));
-    Imm = DAG.getConstant(Immediate, dl, ImmVT);
-  }
-  else if (HasConstElts)
-    Imm = DAG.getConstant(0, dl, VT);
-  else
-    Imm = DAG.getUNDEF(VT);
-  if (Imm.getValueSizeInBits() == VT.getSizeInBits())
-    DstVec = DAG.getBitcast(VT, Imm);
-  else {
-    SDValue ExtVec = DAG.getBitcast(MVT::v8i1, Imm);
-    DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, ExtVec,
-                         DAG.getIntPtrConstant(0, dl));
-  }
+  if (HasConstElts) {
+    if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
+      SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
+      SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
+      ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
+      ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
+      DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
+    } else {
+      MVT ImmVT = MVT::getIntegerVT(std::max(VT.getSizeInBits(), 8U));
+      SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
+      MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
+      DstVec = DAG.getBitcast(VecVT, Imm);
+      DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
+                           DAG.getIntPtrConstant(0, dl));
+    }
+  } else
+    DstVec = DAG.getUNDEF(VT);
 
   for (unsigned i = 0, e = NonConstIdx.size(); i != e; ++i) {
     unsigned InsertIdx = NonConstIdx[i];
@@ -9400,9 +9412,9 @@ static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
       SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
                                           {4, 5, 6, 7, 4, 5, 6, 7});
       if (Subtarget.hasXOP())
-        return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32,
-                                              LoLo, HiHi, IndicesVec,
-                                              DAG.getConstant(0, DL, MVT::i8)));
+        return DAG.getBitcast(
+            VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
+                            IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
       // Permute Lo and Hi and then select based on index range.
       // This works as VPERMILPS only uses index bits[0:1] to permute elements.
       SDValue Res = DAG.getSelectCC(
@@ -9436,9 +9448,9 @@ static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
       // VPERMIL2PD selects with bit#1 of the index vector, so scale IndicesVec.
       IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
       if (Subtarget.hasXOP())
-        return DAG.getBitcast(VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64,
-                                              LoLo, HiHi, IndicesVec,
-                                              DAG.getConstant(0, DL, MVT::i8)));
+        return DAG.getBitcast(
+            VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
+                            IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
       // Permute Lo and Hi and then select based on index range.
       // This works as VPERMILPD only uses index bit[1] to permute elements.
       SDValue Res = DAG.getSelectCC(
@@ -10009,7 +10021,7 @@ static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
                          DAG.getUNDEF(ShiftVT), SubVec,
                          DAG.getIntPtrConstant(0, dl));
     Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, SubVec,
-                     DAG.getConstant(Idx * SubVecNumElts, dl, MVT::i8));
+                     DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
     return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
                        DAG.getIntPtrConstant(0, dl));
   }
@@ -10402,7 +10414,7 @@ static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
 
 static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
                                           SelectionDAG &DAG) {
-  return DAG.getConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
+  return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
 }
 
 /// Compute whether each element of a shuffle is zeroable.
@@ -11047,7 +11059,7 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
   case MVT::v8i16:
     assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!");
     return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
-                       DAG.getConstant(BlendMask, DL, MVT::i8));
+                       DAG.getTargetConstant(BlendMask, DL, MVT::i8));
   case MVT::v16i16: {
     assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!");
     SmallVector<int, 8> RepeatedMask;
@@ -11059,7 +11071,7 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
         if (RepeatedMask[i] >= 8)
           BlendMask |= 1ull << i;
       return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
-                         DAG.getConstant(BlendMask, DL, MVT::i8));
+                         DAG.getTargetConstant(BlendMask, DL, MVT::i8));
     }
     // Use PBLENDW for lower/upper lanes and then blend lanes.
     // TODO - we should allow 2 PBLENDW here and leave shuffle combine to
@@ -11068,9 +11080,9 @@ static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
     uint64_t HiMask = (BlendMask >> 8) & 0xFF;
     if (LoMask == 0 || LoMask == 255 || HiMask == 0 || HiMask == 255) {
       SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
-                               DAG.getConstant(LoMask, DL, MVT::i8));
+                               DAG.getTargetConstant(LoMask, DL, MVT::i8));
       SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
-                               DAG.getConstant(HiMask, DL, MVT::i8));
+                               DAG.getTargetConstant(HiMask, DL, MVT::i8));
       return DAG.getVectorShuffle(
           MVT::v16i16, DL, Lo, Hi,
           {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31});
@@ -11330,7 +11342,7 @@ static SDValue lowerShuffleAsByteRotateAndPermute(
     SDValue Rotate = DAG.getBitcast(
         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
                         DAG.getBitcast(ByteVT, Lo),
-                        DAG.getConstant(Scale * RotAmt, DL, MVT::i8)));
+                        DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
     SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef);
     for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
       for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
@@ -11537,7 +11549,7 @@ static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
            "512-bit PALIGNR requires BWI instructions");
     return DAG.getBitcast(
         VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
-                        DAG.getConstant(ByteRotation, DL, MVT::i8)));
+                        DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
   }
 
   assert(VT.is128BitVector() &&
@@ -11551,10 +11563,12 @@ static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
   int LoByteShift = 16 - ByteRotation;
   int HiByteShift = ByteRotation;
 
-  SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
-                                DAG.getConstant(LoByteShift, DL, MVT::i8));
-  SDValue HiShift = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
-                                DAG.getConstant(HiByteShift, DL, MVT::i8));
+  SDValue LoShift =
+      DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
+                  DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
+  SDValue HiShift =
+      DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
+                  DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
   return DAG.getBitcast(VT,
                         DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
 }
@@ -11586,7 +11600,7 @@ static SDValue lowerShuffleAsRotate(const SDLoc &DL, MVT VT, SDValue V1,
     return SDValue();
 
   return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
-                     DAG.getConstant(Rotation, DL, MVT::i8));
+                     DAG.getTargetConstant(Rotation, DL, MVT::i8));
 }
 
 /// Try to lower a vector shuffle as a byte shift sequence.
@@ -11625,27 +11639,27 @@ static SDValue lowerVectorShuffleAsByteShiftMask(
   if (ZeroLo == 0) {
     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * ZeroHi, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
   } else if (ZeroHi == 0) {
     unsigned Shift = Mask[ZeroLo] % NumElts;
     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * ZeroLo, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
   } else if (!Subtarget.hasSSSE3()) {
     // If we don't have PSHUFB then its worth avoiding an AND constant mask
     // by performing 3 byte shifts. Shuffle combining can kick in above that.
     // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
     unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
     Shift += Mask[ZeroLo] % NumElts;
     Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * Shift, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
     Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
-                      DAG.getConstant(Scale * ZeroLo, DL, MVT::i8));
+                      DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
   } else
     return SDValue();
 
@@ -11767,7 +11781,7 @@ static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
          "Illegal integer vector type");
   V = DAG.getBitcast(ShiftVT, V);
   V = DAG.getNode(Opcode, DL, ShiftVT, V,
-                  DAG.getConstant(ShiftAmt, DL, MVT::i8));
+                  DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
   return DAG.getBitcast(VT, V);
 }
 
@@ -11901,14 +11915,14 @@ static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
   uint64_t BitLen, BitIdx;
   if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
     return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
-                       DAG.getConstant(BitLen, DL, MVT::i8),
-                       DAG.getConstant(BitIdx, DL, MVT::i8));
+                       DAG.getTargetConstant(BitLen, DL, MVT::i8),
+                       DAG.getTargetConstant(BitIdx, DL, MVT::i8));
 
   if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
     return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
                        V2 ? V2 : DAG.getUNDEF(VT),
-                       DAG.getConstant(BitLen, DL, MVT::i8),
-                       DAG.getConstant(BitIdx, DL, MVT::i8));
+                       DAG.getTargetConstant(BitLen, DL, MVT::i8),
+                       DAG.getTargetConstant(BitIdx, DL, MVT::i8));
 
   return SDValue();
 }
@@ -12005,8 +12019,8 @@ static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
     int LoIdx = Offset * EltBits;
     SDValue Lo = DAG.getBitcast(
         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
-                                DAG.getConstant(EltBits, DL, MVT::i8),
-                                DAG.getConstant(LoIdx, DL, MVT::i8)));
+                                DAG.getTargetConstant(EltBits, DL, MVT::i8),
+                                DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
 
     if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
       return DAG.getBitcast(VT, Lo);
@@ -12014,8 +12028,8 @@ static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
     int HiIdx = (Offset + 1) * EltBits;
     SDValue Hi = DAG.getBitcast(
         MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
-                                DAG.getConstant(EltBits, DL, MVT::i8),
-                                DAG.getConstant(HiIdx, DL, MVT::i8)));
+                                DAG.getTargetConstant(EltBits, DL, MVT::i8),
+                                DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
     return DAG.getBitcast(VT,
                           DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
   }
@@ -12325,9 +12339,9 @@ static SDValue lowerShuffleAsElementInsertion(
       V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
     } else {
       V2 = DAG.getBitcast(MVT::v16i8, V2);
-      V2 = DAG.getNode(
-          X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
-          DAG.getConstant(V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8));
+      V2 = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
+                       DAG.getTargetConstant(
+                           V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8));
       V2 = DAG.getBitcast(VT, V2);
     }
   }
@@ -12759,7 +12773,7 @@ static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
 
   // Insert the V2 element into the desired position.
   return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
-                     DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                     DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
 }
 
 /// Try to lower a shuffle as a permute of the inputs followed by an
@@ -12908,14 +12922,14 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       // If we have AVX, we can use VPERMILPS which will allow folding a load
       // into the shuffle.
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
-                         DAG.getConstant(SHUFPDMask, DL, MVT::i8));
+                         DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
     }
 
     return DAG.getNode(
         X86ISD::SHUFP, DL, MVT::v2f64,
         Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
         Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
-        DAG.getConstant(SHUFPDMask, DL, MVT::i8));
+        DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
   }
   assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!");
   assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!");
@@ -12961,7 +12975,7 @@ static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
 
   unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
   return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
-                     DAG.getConstant(SHUFPDMask, DL, MVT::i8));
+                     DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
 }
 
 /// Handle lowering of 2-lane 64-bit integer shuffles.
@@ -14841,8 +14855,8 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
       if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
         unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
                             ((WidenedMask[1] % 2) << 1);
-      return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
-                         DAG.getConstant(PermMask, DL, MVT::i8));
+        return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
+                           DAG.getTargetConstant(PermMask, DL, MVT::i8));
       }
     }
   }
@@ -14874,7 +14888,7 @@ static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
     V2 = DAG.getUNDEF(VT);
 
   return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
-                     DAG.getConstant(PermMask, DL, MVT::i8));
+                     DAG.getTargetConstant(PermMask, DL, MVT::i8));
 }
 
 /// Lower a vector shuffle by first fixing the 128-bit lanes and then
@@ -15437,11 +15451,18 @@ static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
 }
 
 static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
+                                   bool &ForceV1Zero, bool &ForceV2Zero,
                                    unsigned &ShuffleImm, ArrayRef<int> Mask) {
   int NumElts = VT.getVectorNumElements();
   assert(VT.getScalarSizeInBits() == 64 &&
          (NumElts == 2 || NumElts == 4 || NumElts == 8) &&
          "Unexpected data type for VSHUFPD");
+  assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&
+         "Illegal shuffle mask");
+
+  bool ZeroLane[2] = { true, true };
+  for (int i = 0; i < NumElts; ++i)
+    ZeroLane[i & 1] &= isUndefOrZero(Mask[i]);
 
   // Mask for V8F64: 0/1,  8/9,  2/3,  10/11, 4/5, ..
   // Mask for V4F64; 0/1,  4/5,  2/3,  6/7..
@@ -15449,7 +15470,7 @@ static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
   bool ShufpdMask = true;
   bool CommutableMask = true;
   for (int i = 0; i < NumElts; ++i) {
-    if (Mask[i] == SM_SentinelUndef)
+    if (Mask[i] == SM_SentinelUndef || ZeroLane[i & 1])
       continue;
     if (Mask[i] < 0)
       return false;
@@ -15462,28 +15483,41 @@ static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
     ShuffleImm |= (Mask[i] % 2) << i;
   }
 
-  if (ShufpdMask)
-    return true;
-  if (CommutableMask) {
+  if (!ShufpdMask && !CommutableMask)
+    return false;
+
+  if (!ShufpdMask && CommutableMask)
     std::swap(V1, V2);
-    return true;
-  }
 
-  return false;
+  ForceV1Zero = ZeroLane[0];
+  ForceV2Zero = ZeroLane[1];
+  return true;
 }
 
-static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT,
-                                      ArrayRef<int> Mask, SDValue V1,
-                                      SDValue V2, SelectionDAG &DAG) {
-  assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64)&&
+static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
+                                      SDValue V2, ArrayRef<int> Original,
+                                      const APInt &Zeroable,
+                                      const X86Subtarget &Subtarget,
+                                      SelectionDAG &DAG) {
+  assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
          "Unexpected data type for VSHUFPD");
 
+  SmallVector<int, 64> Mask = createTargetShuffleMask(Original, Zeroable);
+
   unsigned Immediate = 0;
-  if (!matchShuffleWithSHUFPD(VT, V1, V2, Immediate, Mask))
+  bool ForceV1Zero = false, ForceV2Zero = false;
+  if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate,
+                              Mask))
     return SDValue();
 
+  // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
+  if (ForceV1Zero)
+    V1 = getZeroVector(VT, Subtarget, DAG, DL);
+  if (ForceV2Zero)
+    V2 = getZeroVector(VT, Subtarget, DAG, DL);
+
   return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
-                     DAG.getConstant(Immediate, DL, MVT::i8));
+                     DAG.getTargetConstant(Immediate, DL, MVT::i8));
 }
 
 /// Handle lowering of 4-lane 64-bit floating point shuffles.
@@ -15518,7 +15552,7 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
       unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
                               ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
-                         DAG.getConstant(VPERMILPMask, DL, MVT::i8));
+                         DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
     }
 
     // With AVX2 we have direct support for this permutation.
@@ -15551,7 +15585,8 @@ static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Blend;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
-  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, Mask, V1, V2, DAG))
+  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Op;
 
   // If we have one input in place, then we can permute the other input and
@@ -16256,7 +16291,7 @@ static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
   }
 
   return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
-                     DAG.getConstant(PermMask, DL, MVT::i8));
+                     DAG.getTargetConstant(PermMask, DL, MVT::i8));
 }
 
 /// Handle lowering of 8-lane 64-bit floating point shuffles.
@@ -16281,7 +16316,7 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
                               ((Mask[4] == 5) << 4) | ((Mask[5] == 5) << 5) |
                               ((Mask[6] == 7) << 6) | ((Mask[7] == 7) << 7);
       return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
-                         DAG.getConstant(VPERMILPMask, DL, MVT::i8));
+                         DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
     }
 
     SmallVector<int, 4> RepeatedMask;
@@ -16298,7 +16333,8 @@ static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
     return Unpck;
 
   // Check if the blend happens to exactly fit that of SHUFPD.
-  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, Mask, V1, V2, DAG))
+  if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
+                                          Zeroable, Subtarget, DAG))
     return Op;
 
   if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
@@ -16709,7 +16745,7 @@ static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
                             DAG.getUNDEF(WideVT), V1,
                             DAG.getIntPtrConstant(0, DL));
   Res = DAG.getNode(X86ISD::KSHIFTR, DL, WideVT, Res,
-                    DAG.getConstant(ShiftAmt, DL, MVT::i8));
+                    DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
                      DAG.getIntPtrConstant(0, DL));
 }
@@ -16816,13 +16852,13 @@ static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
         int WideElts = WideVT.getVectorNumElements();
         // Shift left to put the original vector in the MSBs of the new size.
         Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
-                          DAG.getConstant(WideElts - NumElts, DL, MVT::i8));
+                          DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
         // Increase the shift amount to account for the left shift.
         ShiftAmt += WideElts - NumElts;
       }
 
       Res = DAG.getNode(Opcode, DL, WideVT, Res,
-                        DAG.getConstant(ShiftAmt, DL, MVT::i8));
+                        DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
       return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
                          DAG.getIntPtrConstant(0, DL));
     }
@@ -17270,7 +17306,7 @@ static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
 
   // Use kshiftr instruction to move to the lower element.
   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
-                    DAG.getConstant(IdxVal, dl, MVT::i8));
+                    DAG.getTargetConstant(IdxVal, dl, MVT::i8));
 
   return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
                      DAG.getIntPtrConstant(0, dl));
@@ -17498,7 +17534,7 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
           (Subtarget.hasAVX2() && EltVT == MVT::i32)) {
         SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
-                           DAG.getConstant(1, dl, MVT::i8));
+                           DAG.getTargetConstant(1, dl, MVT::i8));
       }
     }
 
@@ -17569,12 +17605,12 @@ SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
         // operand form.
         N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
         return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
-                           DAG.getConstant(1, dl, MVT::i8));
+                           DAG.getTargetConstant(1, dl, MVT::i8));
       }
       // Create this as a scalar to vector..
       N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
       return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
-                         DAG.getConstant(IdxVal << 4, dl, MVT::i8));
+                         DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
     }
 
     // PINSR* works with constant index.
@@ -17660,7 +17696,7 @@ static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
 
   // Shift to the LSB.
   Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
-                    DAG.getConstant(IdxVal, dl, MVT::i8));
+                    DAG.getTargetConstant(IdxVal, dl, MVT::i8));
 
   return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
                      DAG.getIntPtrConstant(0, dl));
@@ -18203,8 +18239,8 @@ static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
     APInt APIntShiftAmt;
     if (X86::isConstantSplat(Amt, APIntShiftAmt)) {
       uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits());
-      return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT,
-                         Op0, Op1, DAG.getConstant(ShiftAmt, DL, MVT::i8));
+      return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, Op0,
+                         Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
     }
 
     return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
@@ -18636,7 +18672,7 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
     // Low will be bitcasted right away, so do not bother bitcasting back to its
     // original type.
     Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
-                      VecCstLowBitcast, DAG.getConstant(0xaa, DL, MVT::i8));
+                      VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
     //     uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
     //                                 (uint4) 0x53000000, 0xaa);
     SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
@@ -18644,7 +18680,7 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
     // High will be bitcasted right away, so do not bother bitcasting back to
     // its original type.
     High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
-                       VecCstHighBitcast, DAG.getConstant(0xaa, DL, MVT::i8));
+                       VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
   } else {
     SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
     //     uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
@@ -18950,7 +18986,7 @@ static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
 
   // Custom legalize v8i8->v8i64 on CPUs without avx512bw.
   if (InVT == MVT::v8i8) {
-    if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64)
+    if (VT != MVT::v8i64)
       return SDValue();
 
     In = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op),
@@ -19281,22 +19317,6 @@ SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
       SDValue Lo, Hi;
       std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
 
-      if (!ExperimentalVectorWideningLegalization) {
-        // Without vector widening we need to manually construct X86 specific
-        // nodes and an unpcklqdq.
-        Lo = DAG.getNode(X86ISD::VTRUNC, DL, VT, Lo);
-        Hi = DAG.getNode(X86ISD::VTRUNC, DL, VT, Hi);
-
-        // Manually concat the truncates using a shuffle.
-        unsigned NumElts = VT.getVectorNumElements();
-        SmallVector<int, 16> ShufMask(NumElts);
-        for (unsigned i = 0; i != NumElts / 2; ++i)
-          ShufMask[i] = i;
-        for (unsigned i = NumElts / 2; i != NumElts; ++i)
-          ShufMask[i] = i + (NumElts / 2);
-        return DAG.getVectorShuffle(VT, DL, Lo, Hi, ShufMask);
-      }
-
       EVT LoVT, HiVT;
       std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
 
@@ -19809,7 +19829,7 @@ static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
 static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
                         SelectionDAG &DAG) {
   return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
-                     DAG.getConstant(Cond, dl, MVT::i8), EFLAGS);
+                     DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
 }
 
 /// Helper for matching OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...))
@@ -19912,8 +19932,8 @@ static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
     VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
   }
 
-  X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE, DL,
-                          MVT::i8);
+  X86CC = DAG.getTargetConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE,
+                                DL, MVT::i8);
   return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, VecIns.back(), VecIns.back());
 }
 
@@ -20249,7 +20269,8 @@ X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
   SDLoc DL(N);
   SDValue N0 = N->getOperand(0);
   SDValue Zero = DAG.getConstant(0, DL, VT);
-  SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
+  APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2);
+  SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
 
   // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
   SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);
@@ -20349,8 +20370,8 @@ static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
   if (Src.getValueType() != BitNo.getValueType())
     BitNo = DAG.getNode(ISD::ANY_EXTEND, dl, Src.getValueType(), BitNo);
 
-  X86CC = DAG.getConstant(CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B,
-                          dl, MVT::i8);
+  X86CC = DAG.getTargetConstant(CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B,
+                                dl, MVT::i8);
   return DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo);
 }
 
@@ -20442,13 +20463,6 @@ static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
 
   ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
 
-  // If this is a seteq make sure any build vectors of all zeros are on the RHS.
-  // This helps with vptestm matching.
-  // TODO: Should we just canonicalize the setcc during DAG combine?
-  if ((SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE) &&
-      ISD::isBuildVectorAllZeros(Op0.getNode()))
-    std::swap(Op0, Op1);
-
   // Prefer SETGT over SETLT.
   if (SetCCOpcode == ISD::SETLT) {
     SetCCOpcode = ISD::getSetCCSwappedOperands(SetCCOpcode);
@@ -20593,14 +20607,14 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
       }
 
       SDValue Cmp0 = DAG.getNode(Opc, dl, VT, Op0, Op1,
-                                 DAG.getConstant(CC0, dl, MVT::i8));
+                                 DAG.getTargetConstant(CC0, dl, MVT::i8));
       SDValue Cmp1 = DAG.getNode(Opc, dl, VT, Op0, Op1,
-                                 DAG.getConstant(CC1, dl, MVT::i8));
+                                 DAG.getTargetConstant(CC1, dl, MVT::i8));
       Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
     } else {
       // Handle all other FP comparisons here.
       Cmp = DAG.getNode(Opc, dl, VT, Op0, Op1,
-                        DAG.getConstant(SSECC, dl, MVT::i8));
+                        DAG.getTargetConstant(SSECC, dl, MVT::i8));
     }
 
     // If this is SSE/AVX CMPP, bitcast the result back to integer to match the
@@ -20613,19 +20627,12 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
   }
 
   MVT VTOp0 = Op0.getSimpleValueType();
+  (void)VTOp0;
   assert(VTOp0 == Op1.getSimpleValueType() &&
          "Expected operands with same type!");
   assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
          "Invalid number of packed elements for source and destination!");
 
-  // This is being called by type legalization because v2i32 is marked custom
-  // for result type legalization for v2f32.
-  if (VTOp0 == MVT::v2i32) {
-    assert(!ExperimentalVectorWideningLegalization &&
-           "Should only get here with promote legalization!");
-    return SDValue();
-  }
-
   // The non-AVX512 code below works under the assumption that source and
   // destination types are the same.
   assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&
@@ -20663,7 +20670,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
         ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
 
     return DAG.getNode(Opc, dl, VT, Op0, Op1,
-                       DAG.getConstant(CmpMode, dl, MVT::i8));
+                       DAG.getTargetConstant(CmpMode, dl, MVT::i8));
   }
 
   // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2.
@@ -20905,7 +20912,7 @@ static SDValue EmitKORTEST(SDValue Op0, SDValue Op1, ISD::CondCode CC,
     RHS = Op0.getOperand(1);
   }
 
-  X86CC = DAG.getConstant(X86Cond, dl, MVT::i8);
+  X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
   return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
 }
 
@@ -20950,7 +20957,7 @@ SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
       if (Invert) {
         X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
         CCode = X86::GetOppositeBranchCondition(CCode);
-        X86CC = DAG.getConstant(CCode, dl, MVT::i8);
+        X86CC = DAG.getTargetConstant(CCode, dl, MVT::i8);
       }
 
       return Op0.getOperand(1);
@@ -20964,7 +20971,7 @@ SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
 
   SDValue EFLAGS = EmitCmp(Op0, Op1, CondCode, dl, DAG);
   EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
-  X86CC = DAG.getConstant(CondCode, dl, MVT::i8);
+  X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
   return EFLAGS;
 }
 
@@ -21133,15 +21140,16 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
         cast<CondCodeSDNode>(Cond.getOperand(2))->get(), CondOp0, CondOp1);
 
     if (Subtarget.hasAVX512()) {
-      SDValue Cmp = DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0,
-                                CondOp1, DAG.getConstant(SSECC, DL, MVT::i8));
+      SDValue Cmp =
+          DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
+                      DAG.getTargetConstant(SSECC, DL, MVT::i8));
       assert(!VT.isVector() && "Not a scalar type?");
       return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
     }
 
     if (SSECC < 8 || Subtarget.hasAVX()) {
       SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
-                                DAG.getConstant(SSECC, DL, MVT::i8));
+                                DAG.getTargetConstant(SSECC, DL, MVT::i8));
 
       // If we have AVX, we can use a variable vector select (VBLENDV) instead
       // of 3 logic instructions for size savings and potentially speed.
@@ -21344,7 +21352,7 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     X86::CondCode X86Cond;
     std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
 
-    CC = DAG.getConstant(X86Cond, DL, MVT::i8);
+    CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
     AddTest = false;
   }
 
@@ -21366,7 +21374,7 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
   }
 
   if (AddTest) {
-    CC = DAG.getConstant(X86::COND_NE, DL, MVT::i8);
+    CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
     Cond = EmitCmp(Cond, DAG.getConstant(0, DL, Cond.getValueType()),
                    X86::COND_NE, DL, DAG);
   }
@@ -21382,9 +21390,9 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
         (isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
         (isNullConstant(Op1) || isNullConstant(Op2))) {
-      SDValue Res = DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
-                                DAG.getConstant(X86::COND_B, DL, MVT::i8),
-                                Cond);
+      SDValue Res =
+          DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
+                      DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
       if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
         return DAG.getNOT(DL, Res, Res.getValueType());
       return Res;
@@ -21599,7 +21607,7 @@ static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
 
     unsigned SignExtShift = DestWidth - InSVT.getSizeInBits();
     SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
-                          DAG.getConstant(SignExtShift, dl, MVT::i8));
+                          DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
   }
 
   if (VT == MVT::v2i64) {
@@ -21637,7 +21645,7 @@ static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
 
   // Custom legalize v8i8->v8i64 on CPUs without avx512bw.
   if (InVT == MVT::v8i8) {
-    if (!ExperimentalVectorWideningLegalization || VT != MVT::v8i64)
+    if (VT != MVT::v8i64)
       return SDValue();
 
     In = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op),
@@ -21781,14 +21789,13 @@ static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
     return SDValue();
   }
 
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   assert(StoreVT.isVector() && StoreVT.getSizeInBits() == 64 &&
          "Unexpected VT");
-  if (DAG.getTargetLoweringInfo().getTypeAction(*DAG.getContext(), StoreVT) !=
-        TargetLowering::TypeWidenVector)
-    return SDValue();
+  assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==
+             TargetLowering::TypeWidenVector && "Unexpected type action!");
 
-  MVT WideVT = MVT::getVectorVT(StoreVT.getVectorElementType(),
-                                StoreVT.getVectorNumElements() * 2);
+  EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
   StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
                           DAG.getUNDEF(StoreVT));
 
@@ -21828,11 +21835,10 @@ static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
 
   LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
   SDLoc dl(Ld);
-  EVT MemVT = Ld->getMemoryVT();
 
   // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 loads.
   if (RegVT.getVectorElementType() == MVT::i1) {
-    assert(EVT(RegVT) == MemVT && "Expected non-extending load");
+    assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load");
     assert(RegVT.getVectorNumElements() <= 8 && "Unexpected VT");
     assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
            "Expected AVX512F without AVX512DQI");
@@ -21851,179 +21857,7 @@ static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
     return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
   }
 
-  if (ExperimentalVectorWideningLegalization)
-    return SDValue();
-
-  // Nothing useful we can do without SSE2 shuffles.
-  assert(Subtarget.hasSSE2() && "We only custom lower sext loads with SSE2.");
-
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  unsigned RegSz = RegVT.getSizeInBits();
-
-  ISD::LoadExtType Ext = Ld->getExtensionType();
-
-  assert((Ext == ISD::EXTLOAD || Ext == ISD::SEXTLOAD)
-         && "Only anyext and sext are currently implemented.");
-  assert(MemVT != RegVT && "Cannot extend to the same type");
-  assert(MemVT.isVector() && "Must load a vector from memory");
-
-  unsigned NumElems = RegVT.getVectorNumElements();
-  unsigned MemSz = MemVT.getSizeInBits();
-  assert(RegSz > MemSz && "Register size must be greater than the mem size");
-
-  if (Ext == ISD::SEXTLOAD && RegSz == 256 && !Subtarget.hasInt256()) {
-    // The only way in which we have a legal 256-bit vector result but not the
-    // integer 256-bit operations needed to directly lower a sextload is if we
-    // have AVX1 but not AVX2. In that case, we can always emit a sextload to
-    // a 128-bit vector and a normal sign_extend to 256-bits that should get
-    // correctly legalized. We do this late to allow the canonical form of
-    // sextload to persist throughout the rest of the DAG combiner -- it wants
-    // to fold together any extensions it can, and so will fuse a sign_extend
-    // of an sextload into a sextload targeting a wider value.
-    SDValue Load;
-    if (MemSz == 128) {
-      // Just switch this to a normal load.
-      assert(TLI.isTypeLegal(MemVT) && "If the memory type is a 128-bit type, "
-                                       "it must be a legal 128-bit vector "
-                                       "type!");
-      Load = DAG.getLoad(MemVT, dl, Ld->getChain(), Ld->getBasePtr(),
-                         Ld->getPointerInfo(), Ld->getAlignment(),
-                         Ld->getMemOperand()->getFlags());
-    } else {
-      assert(MemSz < 128 &&
-             "Can't extend a type wider than 128 bits to a 256 bit vector!");
-      // Do an sext load to a 128-bit vector type. We want to use the same
-      // number of elements, but elements half as wide. This will end up being
-      // recursively lowered by this routine, but will succeed as we definitely
-      // have all the necessary features if we're using AVX1.
-      EVT HalfEltVT =
-          EVT::getIntegerVT(*DAG.getContext(), RegVT.getScalarSizeInBits() / 2);
-      EVT HalfVecVT = EVT::getVectorVT(*DAG.getContext(), HalfEltVT, NumElems);
-      Load =
-          DAG.getExtLoad(Ext, dl, HalfVecVT, Ld->getChain(), Ld->getBasePtr(),
-                         Ld->getPointerInfo(), MemVT, Ld->getAlignment(),
-                         Ld->getMemOperand()->getFlags());
-    }
-
-    // Replace chain users with the new chain.
-    assert(Load->getNumValues() == 2 && "Loads must carry a chain!");
-
-    // Finally, do a normal sign-extend to the desired register.
-    SDValue SExt = DAG.getSExtOrTrunc(Load, dl, RegVT);
-    return DAG.getMergeValues({SExt, Load.getValue(1)}, dl);
-  }
-
-  // All sizes must be a power of two.
-  assert(isPowerOf2_32(RegSz * MemSz * NumElems) &&
-         "Non-power-of-two elements are not custom lowered!");
-
-  // Attempt to load the original value using scalar loads.
-  // Find the largest scalar type that divides the total loaded size.
-  MVT SclrLoadTy = MVT::i8;
-  for (MVT Tp : MVT::integer_valuetypes()) {
-    if (TLI.isTypeLegal(Tp) && ((MemSz % Tp.getSizeInBits()) == 0)) {
-      SclrLoadTy = Tp;
-    }
-  }
-
-  // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
-  if (TLI.isTypeLegal(MVT::f64) && SclrLoadTy.getSizeInBits() < 64 &&
-      (64 <= MemSz))
-    SclrLoadTy = MVT::f64;
-
-  // Calculate the number of scalar loads that we need to perform
-  // in order to load our vector from memory.
-  unsigned NumLoads = MemSz / SclrLoadTy.getSizeInBits();
-
-  assert((Ext != ISD::SEXTLOAD || NumLoads == 1) &&
-         "Can only lower sext loads with a single scalar load!");
-
-  unsigned loadRegSize = RegSz;
-  if (Ext == ISD::SEXTLOAD && RegSz >= 256)
-    loadRegSize = 128;
-
-  // If we don't have BWI we won't be able to create the shuffle needed for
-  // v8i8->v8i64.
-  if (Ext == ISD::EXTLOAD && !Subtarget.hasBWI() && RegVT == MVT::v8i64 &&
-      MemVT == MVT::v8i8)
-    loadRegSize = 128;
-
-  // Represent our vector as a sequence of elements which are the
-  // largest scalar that we can load.
-  EVT LoadUnitVecVT = EVT::getVectorVT(
-      *DAG.getContext(), SclrLoadTy, loadRegSize / SclrLoadTy.getSizeInBits());
-
-  // Represent the data using the same element type that is stored in
-  // memory. In practice, we ''widen'' MemVT.
-  EVT WideVecVT =
-      EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
-                       loadRegSize / MemVT.getScalarSizeInBits());
-
-  assert(WideVecVT.getSizeInBits() == LoadUnitVecVT.getSizeInBits() &&
-         "Invalid vector type");
-
-  // We can't shuffle using an illegal type.
-  assert(TLI.isTypeLegal(WideVecVT) &&
-         "We only lower types that form legal widened vector types");
-
-  SmallVector<SDValue, 8> Chains;
-  SDValue Ptr = Ld->getBasePtr();
-  unsigned OffsetInc = SclrLoadTy.getSizeInBits() / 8;
-  SDValue Increment = DAG.getConstant(OffsetInc, dl,
-                                      TLI.getPointerTy(DAG.getDataLayout()));
-  SDValue Res = DAG.getUNDEF(LoadUnitVecVT);
-
-  unsigned Offset = 0;
-  for (unsigned i = 0; i < NumLoads; ++i) {
-    unsigned NewAlign = MinAlign(Ld->getAlignment(), Offset);
-
-    // Perform a single load.
-    SDValue ScalarLoad =
-      DAG.getLoad(SclrLoadTy, dl, Ld->getChain(), Ptr,
-                  Ld->getPointerInfo().getWithOffset(Offset),
-                  NewAlign, Ld->getMemOperand()->getFlags());
-    Chains.push_back(ScalarLoad.getValue(1));
-    // Create the first element type using SCALAR_TO_VECTOR in order to avoid
-    // another round of DAGCombining.
-    if (i == 0)
-      Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LoadUnitVecVT, ScalarLoad);
-    else
-      Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, LoadUnitVecVT, Res,
-                        ScalarLoad, DAG.getIntPtrConstant(i, dl));
-
-    Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment);
-    Offset += OffsetInc;
-  }
-
-  SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
-
-  // Bitcast the loaded value to a vector of the original element type, in
-  // the size of the target vector type.
-  SDValue SlicedVec = DAG.getBitcast(WideVecVT, Res);
-  unsigned SizeRatio = RegSz / MemSz;
-
-  if (Ext == ISD::SEXTLOAD) {
-    SDValue Sext = getExtendInVec(ISD::SIGN_EXTEND, dl, RegVT, SlicedVec, DAG);
-    return DAG.getMergeValues({Sext, TF}, dl);
-  }
-
-  if (Ext == ISD::EXTLOAD && !Subtarget.hasBWI() && RegVT == MVT::v8i64 &&
-      MemVT == MVT::v8i8) {
-    SDValue Sext = getExtendInVec(ISD::ZERO_EXTEND, dl, RegVT, SlicedVec, DAG);
-    return DAG.getMergeValues({Sext, TF}, dl);
-  }
-
-  // Redistribute the loaded elements into the different locations.
-  SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
-  for (unsigned i = 0; i != NumElems; ++i)
-    ShuffleVec[i * SizeRatio] = i;
-
-  SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
-                                       DAG.getUNDEF(WideVecVT), ShuffleVec);
-
-  // Bitcast to the requested type.
-  Shuff = DAG.getBitcast(RegVT, Shuff);
-  return DAG.getMergeValues({Shuff, TF}, dl);
+  return SDValue();
 }
 
 /// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
@@ -22128,7 +21962,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
     if (Inverted)
       X86Cond = X86::GetOppositeBranchCondition(X86Cond);
 
-    CC = DAG.getConstant(X86Cond, dl, MVT::i8);
+    CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
     addTest = false;
   } else {
     unsigned CondOpc;
@@ -22159,7 +21993,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
           X86::CondCode CCode0 =
               (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
           CCode0 = X86::GetOppositeBranchCondition(CCode0);
-          CC = DAG.getConstant(CCode0, dl, MVT::i8);
+          CC = DAG.getTargetConstant(CCode0, dl, MVT::i8);
           SDNode *User = *Op.getNode()->use_begin();
           // Look for an unconditional branch following this conditional branch.
           // We need this because we need to reverse the successors in order
@@ -22177,7 +22011,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
             X86::CondCode CCode1 =
                 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
             CCode1 = X86::GetOppositeBranchCondition(CCode1);
-            CC = DAG.getConstant(CCode1, dl, MVT::i8);
+            CC = DAG.getTargetConstant(CCode1, dl, MVT::i8);
             Cond = Cmp;
             addTest = false;
           }
@@ -22190,7 +22024,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       X86::CondCode CCode =
         (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
       CCode = X86::GetOppositeBranchCondition(CCode);
-      CC = DAG.getConstant(CCode, dl, MVT::i8);
+      CC = DAG.getTargetConstant(CCode, dl, MVT::i8);
       Cond = Cond.getOperand(0).getOperand(1);
       addTest = false;
     } else if (Cond.getOpcode() == ISD::SETCC &&
@@ -22216,10 +22050,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
           SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
                                     Cond.getOperand(0), Cond.getOperand(1));
           Cmp = ConvertCmpIfNecessary(Cmp, DAG);
-          CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
+          CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
           Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
                               Chain, Dest, CC, Cmp);
-          CC = DAG.getConstant(X86::COND_P, dl, MVT::i8);
+          CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
           Cond = Cmp;
           addTest = false;
         }
@@ -22232,10 +22066,10 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
       SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
                                 Cond.getOperand(0), Cond.getOperand(1));
       Cmp = ConvertCmpIfNecessary(Cmp, DAG);
-      CC = DAG.getConstant(X86::COND_NE, dl, MVT::i8);
+      CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
       Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
                           Chain, Dest, CC, Cmp);
-      CC = DAG.getConstant(X86::COND_P, dl, MVT::i8);
+      CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
       Cond = Cmp;
       addTest = false;
     }
@@ -22260,7 +22094,7 @@ SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
 
   if (addTest) {
     X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
-    CC = DAG.getConstant(X86Cond, dl, MVT::i8);
+    CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
     Cond = EmitCmp(Cond, DAG.getConstant(0, dl, Cond.getValueType()),
                    X86Cond, dl, DAG);
   }
@@ -22594,7 +22428,7 @@ static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
   }
 
   return DAG.getNode(Opc, dl, VT, SrcOp,
-                     DAG.getConstant(ShiftAmt, dl, MVT::i8));
+                     DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
 }
 
 /// Handle vector element shifts where the shift amount may or may not be a
@@ -22639,7 +22473,7 @@ static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
       ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
                           MVT::v2i64, ShAmt);
     else {
-      SDValue ByteShift = DAG.getConstant(
+      SDValue ByteShift = DAG.getTargetConstant(
           (128 - AmtTy.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
       ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
       ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
@@ -22937,9 +22771,6 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       SDValue Src2 = Op.getOperand(2);
       SDValue Src3 = Op.getOperand(3);
 
-      if (IntrData->Type == INTR_TYPE_3OP_IMM8)
-        Src3 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Src3);
-
       // We specify 2 possible opcodes for intrinsics with rounding modes.
       // First, we check if the intrinsic may have non-default rounding mode,
       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
@@ -23192,7 +23023,6 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     case CMP_MASK_CC: {
       MVT MaskVT = Op.getSimpleValueType();
       SDValue CC = Op.getOperand(3);
-      CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, CC);
       // We specify 2 possible opcodes for intrinsics with rounding modes.
       // First, we check if the intrinsic may have non-default rounding mode,
       // (IntrData->Opc1 != 0), then we check the rounding mode operand.
@@ -23211,7 +23041,7 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
     case CMP_MASK_SCALAR_CC: {
       SDValue Src1 = Op.getOperand(1);
       SDValue Src2 = Op.getOperand(2);
-      SDValue CC = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op.getOperand(3));
+      SDValue CC = Op.getOperand(3);
       SDValue Mask = Op.getOperand(4);
 
       SDValue Cmp;
@@ -23282,10 +23112,10 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       SDValue FCmp;
       if (isRoundModeCurDirection(Sae))
         FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
-                           DAG.getConstant(CondVal, dl, MVT::i8));
+                           DAG.getTargetConstant(CondVal, dl, MVT::i8));
       else if (isRoundModeSAE(Sae))
         FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
-                           DAG.getConstant(CondVal, dl, MVT::i8), Sae);
+                           DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
       else
         return SDValue();
       // Need to fill with zeros to ensure the bitcast will produce zeroes
@@ -23345,9 +23175,9 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode");
       // Clear the upper bits of the rounding immediate so that the legacy
       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
-      SDValue RoundingMode = DAG.getNode(ISD::AND, dl, MVT::i32,
-                                         Op.getOperand(2),
-                                         DAG.getConstant(0xf, dl, MVT::i32));
+      auto Round = cast<ConstantSDNode>(Op.getOperand(2));
+      SDValue RoundingMode =
+          DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
                          Op.getOperand(1), RoundingMode);
     }
@@ -23355,12 +23185,22 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
       assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode");
       // Clear the upper bits of the rounding immediate so that the legacy
       // intrinsic can't trigger the scaling behavior of VRNDSCALE.
-      SDValue RoundingMode = DAG.getNode(ISD::AND, dl, MVT::i32,
-                                         Op.getOperand(3),
-                                         DAG.getConstant(0xf, dl, MVT::i32));
+      auto Round = cast<ConstantSDNode>(Op.getOperand(3));
+      SDValue RoundingMode =
+          DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
       return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
                          Op.getOperand(1), Op.getOperand(2), RoundingMode);
     }
+    case BEXTRI: {
+      assert(IntrData->Opc0 == X86ISD::BEXTR && "Unexpected opcode");
+
+      // The control is a TargetConstant, but we need to convert it to a
+      // ConstantSDNode.
+      uint64_t Imm = Op.getConstantOperandVal(2);
+      SDValue Control = DAG.getConstant(Imm, dl, Op.getValueType());
+      return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
+                         Op.getOperand(1), Control);
+    }
     // ADC/ADCX/SBB
     case ADX: {
       SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
@@ -23691,6 +23531,61 @@ SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
                                                  MaskVT, Operation);
     return DAG.getMergeValues({Result0, Result1}, DL);
   }
+  case Intrinsic::x86_mmx_pslli_w:
+  case Intrinsic::x86_mmx_pslli_d:
+  case Intrinsic::x86_mmx_pslli_q:
+  case Intrinsic::x86_mmx_psrli_w:
+  case Intrinsic::x86_mmx_psrli_d:
+  case Intrinsic::x86_mmx_psrli_q:
+  case Intrinsic::x86_mmx_psrai_w:
+  case Intrinsic::x86_mmx_psrai_d: {
+    SDLoc DL(Op);
+    SDValue ShAmt = Op.getOperand(2);
+    // If the argument is a constant, convert it to a target constant.
+    if (auto *C = dyn_cast<ConstantSDNode>(ShAmt)) {
+      ShAmt = DAG.getTargetConstant(C->getZExtValue(), DL, MVT::i32);
+      return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
+                         Op.getOperand(0), Op.getOperand(1), ShAmt);
+    }
+
+    unsigned NewIntrinsic;
+    switch (IntNo) {
+    default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
+    case Intrinsic::x86_mmx_pslli_w:
+      NewIntrinsic = Intrinsic::x86_mmx_psll_w;
+      break;
+    case Intrinsic::x86_mmx_pslli_d:
+      NewIntrinsic = Intrinsic::x86_mmx_psll_d;
+      break;
+    case Intrinsic::x86_mmx_pslli_q:
+      NewIntrinsic = Intrinsic::x86_mmx_psll_q;
+      break;
+    case Intrinsic::x86_mmx_psrli_w:
+      NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
+      break;
+    case Intrinsic::x86_mmx_psrli_d:
+      NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
+      break;
+    case Intrinsic::x86_mmx_psrli_q:
+      NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
+      break;
+    case Intrinsic::x86_mmx_psrai_w:
+      NewIntrinsic = Intrinsic::x86_mmx_psra_w;
+      break;
+    case Intrinsic::x86_mmx_psrai_d:
+      NewIntrinsic = Intrinsic::x86_mmx_psra_d;
+      break;
+    }
+
+    // The vector shift intrinsics with scalars uses 32b shift amounts but
+    // the sse2/mmx shift instructions reads 64 bits. Copy the 32 bits to an
+    // MMX register.
+    ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
+    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
+                       DAG.getConstant(NewIntrinsic, DL, MVT::i32),
+                       Op.getOperand(1), ShAmt);
+
+  }
   }
 }
 
@@ -24071,10 +23966,10 @@ static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
 
     // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
     // Otherwise return the value from Rand, which is always 0, casted to i32.
-    SDValue Ops[] = { DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
-                      DAG.getConstant(1, dl, Op->getValueType(1)),
-                      DAG.getConstant(X86::COND_B, dl, MVT::i8),
-                      SDValue(Result.getNode(), 1) };
+    SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
+                     DAG.getConstant(1, dl, Op->getValueType(1)),
+                     DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
+                     SDValue(Result.getNode(), 1)};
     SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
 
     // Return { result, isValid, chain }.
@@ -24275,12 +24170,11 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 // FIXME? Maybe this could be a TableGen attribute on some registers and
 // this table could be generated automatically from RegInfo.
-unsigned X86TargetLowering::getRegisterByName(const char* RegName, EVT VT,
-                                              SelectionDAG &DAG) const {
+Register X86TargetLowering::getRegisterByName(const char* RegName, EVT VT,
+                                              const MachineFunction &MF) const {
   const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
-  const MachineFunction &MF = DAG.getMachineFunction();
 
-  unsigned Reg = StringSwitch<unsigned>(RegName)
+  Register Reg = StringSwitch<unsigned>(RegName)
                        .Case("esp", X86::ESP)
                        .Case("rsp", X86::RSP)
                        .Case("ebp", X86::EBP)
@@ -24294,8 +24188,7 @@ unsigned X86TargetLowering::getRegisterByName(const char* RegName, EVT VT,
 #ifndef NDEBUG
     else {
       const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
-      unsigned FrameReg =
-          RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
+      Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF);
       assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&
              "Invalid Frame Register!");
     }
@@ -24811,12 +24704,9 @@ static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
 
   if (Opc == ISD::CTLZ) {
     // If src is zero (i.e. bsr sets ZF), returns NumBits.
-    SDValue Ops[] = {
-      Op,
-      DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
-      DAG.getConstant(X86::COND_E, dl, MVT::i8),
-      Op.getValue(1)
-    };
+    SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
+                     DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
+                     Op.getValue(1)};
     Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
   }
 
@@ -24844,12 +24734,9 @@ static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
   Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0);
 
   // If src is zero (i.e. bsf sets ZF), returns NumBits.
-  SDValue Ops[] = {
-    Op,
-    DAG.getConstant(NumBits, dl, VT),
-    DAG.getConstant(X86::COND_E, dl, MVT::i8),
-    Op.getValue(1)
-  };
+  SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
+                   DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
+                   Op.getValue(1)};
   return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
 }
 
@@ -24985,7 +24872,7 @@ static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
     SDValue N0 = Op.getOperand(0);
     SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
                               DAG.getConstant(0, DL, VT), N0);
-    SDValue Ops[] = {N0, Neg, DAG.getConstant(X86::COND_GE, DL, MVT::i8),
+    SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_GE, DL, MVT::i8),
                      SDValue(Neg.getNode(), 1)};
     return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
   }
@@ -26034,7 +25921,7 @@ static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
        (VT == MVT::v32i8 && Subtarget.hasInt256())) &&
       !Subtarget.hasXOP()) {
     int NumElts = VT.getVectorNumElements();
-    SDValue Cst8 = DAG.getConstant(8, dl, MVT::i8);
+    SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
 
     // Extend constant shift amount to vXi16 (it doesn't matter if the type
     // isn't legal).
@@ -26306,7 +26193,7 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
       unsigned Op = (Opcode == ISD::ROTL ? X86ISD::VROTLI : X86ISD::VROTRI);
       uint64_t RotateAmt = EltBits[CstSplatIndex].urem(EltSizeInBits);
       return DAG.getNode(Op, DL, VT, R,
-                         DAG.getConstant(RotateAmt, DL, MVT::i8));
+                         DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
     }
 
     // Else, fall-back on VPROLV/VPRORV.
@@ -26327,7 +26214,7 @@ static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
     if (0 <= CstSplatIndex) {
       uint64_t RotateAmt = EltBits[CstSplatIndex].urem(EltSizeInBits);
       return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
-                         DAG.getConstant(RotateAmt, DL, MVT::i8));
+                         DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
     }
 
     // Use general rotate by variable (per-element).
@@ -26564,7 +26451,7 @@ X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
 
   // If this is a canonical idempotent atomicrmw w/no uses, we have a better
   // lowering available in lowerAtomicArith.
-  // TODO: push more cases through this path. 
+  // TODO: push more cases through this path.
   if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand()))
     if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() &&
         AI->use_empty())
@@ -26634,7 +26521,7 @@ bool X86TargetLowering::lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const {
 /// Emit a locked operation on a stack location which does not change any
 /// memory location, but does involve a lock prefix.  Location is chosen to be
 /// a) very likely accessed only by a single thread to minimize cache traffic,
-/// and b) definitely dereferenceable.  Returns the new Chain result.  
+/// and b) definitely dereferenceable.  Returns the new Chain result.
 static SDValue emitLockedStackOp(SelectionDAG &DAG,
                                  const X86Subtarget &Subtarget,
                                  SDValue Chain, SDLoc DL) {
@@ -26643,22 +26530,22 @@ static SDValue emitLockedStackOp(SelectionDAG &DAG,
   // operations issued by the current processor.  As such, the location
   // referenced is not relevant for the ordering properties of the instruction.
   // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
-  // 8.2.3.9  Loads and Stores Are Not Reordered with Locked Instructions 
+  // 8.2.3.9  Loads and Stores Are Not Reordered with Locked Instructions
   // 2) Using an immediate operand appears to be the best encoding choice
   // here since it doesn't require an extra register.
   // 3) OR appears to be very slightly faster than ADD. (Though, the difference
   // is small enough it might just be measurement noise.)
   // 4) When choosing offsets, there are several contributing factors:
   //   a) If there's no redzone, we default to TOS.  (We could allocate a cache
-  //      line aligned stack object to improve this case.) 
+  //      line aligned stack object to improve this case.)
   //   b) To minimize our chances of introducing a false dependence, we prefer
-  //      to offset the stack usage from TOS slightly.  
+  //      to offset the stack usage from TOS slightly.
   //   c) To minimize concerns about cross thread stack usage - in particular,
   //      the idiomatic MyThreadPool.run([&StackVars]() {...}) pattern which
   //      captures state in the TOS frame and accesses it from many threads -
   //      we want to use an offset such that the offset is in a distinct cache
   //      line from the TOS frame.
-  // 
+  //
   // For a general discussion of the tradeoffs and benchmark results, see:
   // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
 
@@ -26711,7 +26598,7 @@ static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
     if (Subtarget.hasMFence())
       return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 
-    SDValue Chain = Op.getOperand(0); 
+    SDValue Chain = Op.getOperand(0);
     return emitLockedStackOp(DAG, Subtarget, Chain, dl);
   }
 
@@ -27087,6 +26974,13 @@ static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
   SDValue In = Op.getOperand(0);
   SDLoc DL(Op);
 
+  // Split v8i64/v16i32 without BWI so that we can still use the PSHUFB
+  // lowering.
+  if (VT == MVT::v8i64 || VT == MVT::v16i32) {
+    assert(!Subtarget.hasBWI() && "BWI should Expand BITREVERSE");
+    return Lower512IntUnary(Op, DAG);
+  }
+
   unsigned NumElts = VT.getVectorNumElements();
   assert(VT.getScalarType() == MVT::i8 &&
          "Only byte vector BITREVERSE supported");
@@ -27194,12 +27088,12 @@ static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
     // seq_cst which isn't SingleThread, everything just needs to be preserved
     // during codegen and then dropped. Note that we expect (but don't assume),
     // that orderings other than seq_cst and acq_rel have been canonicalized to
-    // a store or load. 
+    // a store or load.
     if (AN->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
         AN->getSyncScopeID() == SyncScope::System) {
       // Prefer a locked operation against a stack location to minimize cache
       // traffic.  This assumes that stack locations are very likely to be
-      // accessed only by the owning thread. 
+      // accessed only by the owning thread.
       SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
       assert(!N->hasAnyUseOfValue(0));
       // NOTE: The getUNDEF is needed to give something for the unused result 0.
@@ -27424,12 +27318,13 @@ static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
   SDValue Chain = N->getChain();
   SDValue BasePtr = N->getBasePtr();
 
-  if (VT == MVT::v2f32) {
+  if (VT == MVT::v2f32 || VT == MVT::v2i32) {
     assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
     // If the index is v2i64 and we have VLX we can use xmm for data and index.
     if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
-      Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
-                        DAG.getUNDEF(MVT::v2f32));
+      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+      EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
+      Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
       SDVTList VTs = DAG.getVTList(MVT::v2i1, MVT::Other);
       SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
       SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
@@ -27439,30 +27334,6 @@ static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
     return SDValue();
   }
 
-  if (VT == MVT::v2i32) {
-    assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
-    Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
-                      DAG.getUNDEF(MVT::v2i32));
-    // If the index is v2i64 and we have VLX we can use xmm for data and index.
-    if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
-      SDVTList VTs = DAG.getVTList(MVT::v2i1, MVT::Other);
-      SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
-      SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
-          VTs, Ops, dl, N->getMemoryVT(), N->getMemOperand());
-      return SDValue(NewScatter.getNode(), 1);
-    }
-    // Custom widen all the operands to avoid promotion.
-    EVT NewIndexVT = EVT::getVectorVT(
-        *DAG.getContext(), Index.getValueType().getVectorElementType(), 4);
-    Index = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewIndexVT, Index,
-                        DAG.getUNDEF(Index.getValueType()));
-    Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
-                       DAG.getConstant(0, dl, MVT::v2i1));
-    SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
-    return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(), dl,
-                                Ops, N->getMemOperand(), N->getIndexType());
-  }
-
   MVT IndexVT = Index.getSimpleValueType();
   MVT MaskVT = Mask.getSimpleValueType();
 
@@ -27896,37 +27767,22 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   }
   case ISD::MUL: {
     EVT VT = N->getValueType(0);
-    assert(VT.isVector() && "Unexpected VT");
-    if (getTypeAction(*DAG.getContext(), VT) == TypePromoteInteger &&
-        VT.getVectorNumElements() == 2) {
-      // Promote to a pattern that will be turned into PMULUDQ.
-      SDValue N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64,
-                               N->getOperand(0));
-      SDValue N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v2i64,
-                               N->getOperand(1));
-      SDValue Mul = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64, N0, N1);
-      Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, VT, Mul));
-    } else if (getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
-               VT.getVectorElementType() == MVT::i8) {
-      // Pre-promote these to vXi16 to avoid op legalization thinking all 16
-      // elements are needed.
-      MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
-      SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
-      SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
-      SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
-      Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
-      unsigned NumConcats = 16 / VT.getVectorNumElements();
-      SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
-      ConcatOps[0] = Res;
-      Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
-      Results.push_back(Res);
-    }
+    assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+           VT.getVectorElementType() == MVT::i8 && "Unexpected VT!");
+    // Pre-promote these to vXi16 to avoid op legalization thinking all 16
+    // elements are needed.
+    MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
+    SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
+    SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
+    SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
+    Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
+    unsigned NumConcats = 16 / VT.getVectorNumElements();
+    SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
+    ConcatOps[0] = Res;
+    Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
+    Results.push_back(Res);
     return;
   }
-  case ISD::UADDSAT:
-  case ISD::SADDSAT:
-  case ISD::USUBSAT:
-  case ISD::SSUBSAT:
   case X86ISD::VPMADDWD:
   case X86ISD::AVG: {
     // Legalize types for ISD::UADDSAT/SADDSAT/USUBSAT/SSUBSAT and
@@ -27937,6 +27793,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     EVT InVT = N->getOperand(0).getValueType();
     assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&
            "Expected a VT that divides into 128 bits.");
+    assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+           "Unexpected type action!");
     unsigned NumConcat = 128 / InVT.getSizeInBits();
 
     EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
@@ -27953,9 +27811,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
 
     SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
-    if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
-      Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
-                        DAG.getIntPtrConstant(0, dl));
     Results.push_back(Res);
     return;
   }
@@ -27984,26 +27839,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     Results.push_back(Hi);
     return;
   }
-  case ISD::SETCC: {
-    // Widen v2i32 (setcc v2f32). This is really needed for AVX512VL when
-    // setCC result type is v2i1 because type legalzation will end up with
-    // a v4i1 setcc plus an extend.
-    assert(N->getValueType(0) == MVT::v2i32 && "Unexpected type");
-    if (N->getOperand(0).getValueType() != MVT::v2f32 ||
-        getTypeAction(*DAG.getContext(), MVT::v2i32) == TypeWidenVector)
-      return;
-    SDValue UNDEF = DAG.getUNDEF(MVT::v2f32);
-    SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
-                              N->getOperand(0), UNDEF);
-    SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
-                              N->getOperand(1), UNDEF);
-    SDValue Res = DAG.getNode(ISD::SETCC, dl, MVT::v4i32, LHS, RHS,
-                              N->getOperand(2));
-    Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
-                      DAG.getIntPtrConstant(0, dl));
-    Results.push_back(Res);
-    return;
-  }
   // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
   case X86ISD::FMINC:
   case X86ISD::FMIN:
@@ -28024,7 +27859,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::SREM:
   case ISD::UREM: {
     EVT VT = N->getValueType(0);
-    if (getTypeAction(*DAG.getContext(), VT) == TypeWidenVector) {
+    if (VT.isVector()) {
+      assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+             "Unexpected type action!");
       // If this RHS is a constant splat vector we can widen this and let
       // division/remainder by constant optimize it.
       // TODO: Can we do something for non-splat?
@@ -28042,17 +27879,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
-    if (VT == MVT::v2i32) {
-      // Legalize v2i32 div/rem by unrolling. Otherwise we promote to the
-      // v2i64 and unroll later. But then we create i64 scalar ops which
-      // might be slow in 64-bit mode or require a libcall in 32-bit mode.
-      Results.push_back(DAG.UnrollVectorOp(N));
-      return;
-    }
-
-    if (VT.isVector())
-      return;
-
     LLVM_FALLTHROUGH;
   }
   case ISD::SDIVREM:
@@ -28112,54 +27938,21 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     }
     return;
   }
-  case ISD::SIGN_EXTEND_VECTOR_INREG: {
-    if (ExperimentalVectorWideningLegalization)
-      return;
-
-    EVT VT = N->getValueType(0);
-    SDValue In = N->getOperand(0);
-    EVT InVT = In.getValueType();
-    if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
-        (InVT == MVT::v16i16 || InVT == MVT::v32i8)) {
-      // Custom split this so we can extend i8/i16->i32 invec. This is better
-      // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
-      // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
-      // we allow the sra from the extend to i32 to be shared by the split.
-      EVT ExtractVT = InVT.getHalfNumVectorElementsVT(*DAG.getContext());
-      MVT ExtendVT = MVT::getVectorVT(MVT::i32,
-                                      VT.getVectorNumElements());
-      In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ExtractVT,
-                       In, DAG.getIntPtrConstant(0, dl));
-      In = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, MVT::v4i32, In);
-
-      // Fill a vector with sign bits for each element.
-      SDValue Zero = DAG.getConstant(0, dl, ExtendVT);
-      SDValue SignBits = DAG.getSetCC(dl, ExtendVT, Zero, In, ISD::SETGT);
-
-      EVT LoVT, HiVT;
-      std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
-
-      // Create an unpackl and unpackh to interleave the sign bits then bitcast
-      // to vXi64.
-      SDValue Lo = getUnpackl(DAG, dl, ExtendVT, In, SignBits);
-      Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
-      SDValue Hi = getUnpackh(DAG, dl, ExtendVT, In, SignBits);
-      Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
-
-      SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
-      Results.push_back(Res);
-      return;
-    }
+  case ISD::ANY_EXTEND:
+    // Right now, only MVT::v8i8 has Custom action for an illegal type.
+    // It's intended to custom handle the input type.
+    assert(N->getValueType(0) == MVT::v8i8 &&
+           "Do not know how to legalize this Node");
     return;
-  }
   case ISD::SIGN_EXTEND:
   case ISD::ZERO_EXTEND: {
     EVT VT = N->getValueType(0);
     SDValue In = N->getOperand(0);
     EVT InVT = In.getValueType();
     if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
-        (InVT == MVT::v4i16 || InVT == MVT::v4i8) &&
-        getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector) {
+        (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
+      assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
+             "Unexpected type action!");
       assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode");
       // Custom split this so we can extend i8/i16->i32 invec. This is better
       // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
@@ -28230,27 +28023,9 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     SDValue Src = N->getOperand(0);
     EVT SrcVT = Src.getValueType();
 
-    // Promote these manually to avoid over promotion to v2i64. Type
-    // legalization will revisit the v2i32 operation for more cleanup.
-    if ((VT == MVT::v2i8 || VT == MVT::v2i16) &&
-        getTypeAction(*DAG.getContext(), VT) == TypePromoteInteger) {
-      // AVX512DQ provides instructions that produce a v2i64 result.
-      if (Subtarget.hasDQI())
-        return;
-
-      SDValue Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v2i32, Src);
-      Res = DAG.getNode(N->getOpcode() == ISD::FP_TO_UINT ? ISD::AssertZext
-                                                          : ISD::AssertSext,
-                        dl, MVT::v2i32, Res,
-                        DAG.getValueType(VT.getVectorElementType()));
-      Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
-      Results.push_back(Res);
-      return;
-    }
-
     if (VT.isVector() && VT.getScalarSizeInBits() < 32) {
-      if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
-        return;
+      assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+             "Unexpected type action!");
 
       // Try to create a 128 bit vector, but don't exceed a 32 bit element.
       unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);
@@ -28285,35 +28060,18 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       assert((IsSigned || Subtarget.hasAVX512()) &&
              "Can only handle signed conversion without AVX512");
       assert(Subtarget.hasSSE2() && "Requires at least SSE2!");
-      bool Widenv2i32 =
-        getTypeAction(*DAG.getContext(), MVT::v2i32) == TypeWidenVector;
+      assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+             "Unexpected type action!");
       if (Src.getValueType() == MVT::v2f64) {
-        unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         if (!IsSigned && !Subtarget.hasVLX()) {
-          // If v2i32 is widened, we can defer to the generic legalizer.
-          if (Widenv2i32)
-            return;
-          // Custom widen by doubling to a legal vector with. Isel will
-          // further widen to v8f64.
-          Opc = ISD::FP_TO_UINT;
-          Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64,
-                            Src, DAG.getUNDEF(MVT::v2f64));
+          // If we have VLX we can emit a target specific FP_TO_UINT node,
+          // otherwise we can defer to the generic legalizer which will widen
+          // the input as well. This will be further widened during op
+          // legalization to v8i32<-v8f64.
+          return;
         }
+        unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
         SDValue Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
-        if (!Widenv2i32)
-          Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
-                            DAG.getIntPtrConstant(0, dl));
-        Results.push_back(Res);
-        return;
-      }
-      if (SrcVT == MVT::v2f32 &&
-          getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) {
-        SDValue Idx = DAG.getIntPtrConstant(0, dl);
-        SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
-                                  DAG.getUNDEF(MVT::v2f32));
-        Res = DAG.getNode(IsSigned ? ISD::FP_TO_SINT
-                                   : ISD::FP_TO_UINT, dl, MVT::v4i32, Res);
-        Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res, Idx);
         Results.push_back(Res);
         return;
       }
@@ -28323,6 +28081,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
+    assert(!VT.isVector() && "Vectors should have been handled above!");
+
     if (Subtarget.hasDQI() && VT == MVT::i64 &&
         (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
       assert(!Subtarget.is64Bit() && "i64 should be legal");
@@ -28607,42 +28367,33 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       return;
     }
 
-    if (DstVT.isVector() && SrcVT == MVT::x86mmx &&
-        getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector) {
+    if (DstVT.isVector() && SrcVT == MVT::x86mmx) {
+      assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
+             "Unexpected type action!");
       EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
       SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, WideVT, N->getOperand(0));
       Results.push_back(Res);
       return;
     }
 
-    if (SrcVT != MVT::f64 ||
-        (DstVT != MVT::v2i32 && DstVT != MVT::v4i16 && DstVT != MVT::v8i8) ||
-        getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector)
-      return;
-
-    unsigned NumElts = DstVT.getVectorNumElements();
-    EVT SVT = DstVT.getVectorElementType();
-    EVT WiderVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumElts * 2);
-    SDValue Res;
-    Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, N->getOperand(0));
-    Res = DAG.getBitcast(WiderVT, Res);
-    Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
-                      DAG.getIntPtrConstant(0, dl));
-    Results.push_back(Res);
     return;
   }
   case ISD::MGATHER: {
     EVT VT = N->getValueType(0);
-    if (VT == MVT::v2f32 && (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
+    if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&
+        (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
       auto *Gather = cast<MaskedGatherSDNode>(N);
       SDValue Index = Gather->getIndex();
       if (Index.getValueType() != MVT::v2i64)
         return;
+      assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+             "Unexpected type action!");
+      EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
       SDValue Mask = Gather->getMask();
       assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
-      SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
+      SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
                                      Gather->getPassThru(),
-                                     DAG.getUNDEF(MVT::v2f32));
+                                     DAG.getUNDEF(VT));
       if (!Subtarget.hasVLX()) {
         // We need to widen the mask, but the instruction will only use 2
         // of its elements. So we can use undef.
@@ -28653,67 +28404,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
       SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
                         Gather->getBasePtr(), Index, Gather->getScale() };
       SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
-        DAG.getVTList(MVT::v4f32, Mask.getValueType(), MVT::Other), Ops, dl,
+        DAG.getVTList(WideVT, Mask.getValueType(), MVT::Other), Ops, dl,
         Gather->getMemoryVT(), Gather->getMemOperand());
       Results.push_back(Res);
       Results.push_back(Res.getValue(2));
       return;
     }
-    if (VT == MVT::v2i32) {
-      auto *Gather = cast<MaskedGatherSDNode>(N);
-      SDValue Index = Gather->getIndex();
-      SDValue Mask = Gather->getMask();
-      assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type");
-      SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32,
-                                     Gather->getPassThru(),
-                                     DAG.getUNDEF(MVT::v2i32));
-      // If the index is v2i64 we can use it directly.
-      if (Index.getValueType() == MVT::v2i64 &&
-          (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
-        if (!Subtarget.hasVLX()) {
-          // We need to widen the mask, but the instruction will only use 2
-          // of its elements. So we can use undef.
-          Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
-                             DAG.getUNDEF(MVT::v2i1));
-          Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
-        }
-        SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
-                          Gather->getBasePtr(), Index, Gather->getScale() };
-        SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
-          DAG.getVTList(MVT::v4i32, Mask.getValueType(), MVT::Other), Ops, dl,
-          Gather->getMemoryVT(), Gather->getMemOperand());
-        SDValue Chain = Res.getValue(2);
-        if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
-          Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
-                            DAG.getIntPtrConstant(0, dl));
-        Results.push_back(Res);
-        Results.push_back(Chain);
-        return;
-      }
-      if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector) {
-        EVT IndexVT = Index.getValueType();
-        EVT NewIndexVT = EVT::getVectorVT(*DAG.getContext(),
-                                          IndexVT.getScalarType(), 4);
-        // Otherwise we need to custom widen everything to avoid promotion.
-        Index = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewIndexVT, Index,
-                            DAG.getUNDEF(IndexVT));
-        Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
-                           DAG.getConstant(0, dl, MVT::v2i1));
-        SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
-                          Gather->getBasePtr(), Index, Gather->getScale() };
-        SDValue Res = DAG.getMaskedGather(DAG.getVTList(MVT::v4i32, MVT::Other),
-                                          Gather->getMemoryVT(), dl, Ops,
-                                          Gather->getMemOperand(),
-                                          Gather->getIndexType());
-        SDValue Chain = Res.getValue(1);
-        if (getTypeAction(*DAG.getContext(), MVT::v2i32) != TypeWidenVector)
-          Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i32, Res,
-                            DAG.getIntPtrConstant(0, dl));
-        Results.push_back(Res);
-        Results.push_back(Chain);
-        return;
-      }
-    }
     return;
   }
   case ISD::LOAD: {
@@ -28722,8 +28418,8 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     // cast since type legalization will try to use an i64 load.
     MVT VT = N->getSimpleValueType(0);
     assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT");
-    if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
-      return;
+    assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
+           "Unexpected type action!");
     if (!ISD::isNON_EXTLoad(N))
       return;
     auto *Ld = cast<LoadSDNode>(N);
@@ -28733,11 +28429,10 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
                                 Ld->getPointerInfo(), Ld->getAlignment(),
                                 Ld->getMemOperand()->getFlags());
       SDValue Chain = Res.getValue(1);
-      MVT WideVT = MVT::getVectorVT(LdVT, 2);
-      Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, WideVT, Res);
-      MVT CastVT = MVT::getVectorVT(VT.getVectorElementType(),
-                                    VT.getVectorNumElements() * 2);
-      Res = DAG.getBitcast(CastVT, Res);
+      MVT VecVT = MVT::getVectorVT(LdVT, 2);
+      Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
+      EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
+      Res = DAG.getBitcast(WideVT, Res);
       Results.push_back(Res);
       Results.push_back(Chain);
       return;
@@ -28930,6 +28625,7 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::UNPCKL:             return "X86ISD::UNPCKL";
   case X86ISD::UNPCKH:             return "X86ISD::UNPCKH";
   case X86ISD::VBROADCAST:         return "X86ISD::VBROADCAST";
+  case X86ISD::VBROADCAST_LOAD:    return "X86ISD::VBROADCAST_LOAD";
   case X86ISD::VBROADCASTM:        return "X86ISD::VBROADCASTM";
   case X86ISD::SUBV_BROADCAST:     return "X86ISD::SUBV_BROADCAST";
   case X86ISD::VPERMILPV:          return "X86ISD::VPERMILPV";
@@ -32405,7 +32101,11 @@ static bool matchBinaryPermuteShuffle(
       ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
        (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
        (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
-    if (matchShuffleWithSHUFPD(MaskVT, V1, V2, PermuteImm, Mask)) {
+    bool ForceV1Zero = false, ForceV2Zero = false;
+    if (matchShuffleWithSHUFPD(MaskVT, V1, V2, ForceV1Zero, ForceV2Zero,
+                               PermuteImm, Mask)) {
+      V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
+      V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
       Shuffle = X86ISD::SHUFP;
       ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
       return true;
@@ -32570,7 +32270,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     Res = DAG.getBitcast(ShuffleVT, V1);
     Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
                       DAG.getUNDEF(ShuffleVT),
-                      DAG.getConstant(PermMask, DL, MVT::i8));
+                      DAG.getTargetConstant(PermMask, DL, MVT::i8));
     return DAG.getBitcast(RootVT, Res);
   }
 
@@ -32677,7 +32377,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         return SDValue(); // Nothing to do!
       Res = DAG.getBitcast(ShuffleVT, V1);
       Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
-                        DAG.getConstant(PermuteImm, DL, MVT::i8));
+                        DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
       return DAG.getBitcast(RootVT, Res);
     }
   }
@@ -32707,7 +32407,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     NewV1 = DAG.getBitcast(ShuffleVT, NewV1);
     NewV2 = DAG.getBitcast(ShuffleVT, NewV2);
     Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
-                      DAG.getConstant(PermuteImm, DL, MVT::i8));
+                      DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
     return DAG.getBitcast(RootVT, Res);
   }
 
@@ -32724,8 +32424,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
         return SDValue(); // Nothing to do!
       V1 = DAG.getBitcast(IntMaskVT, V1);
       Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
-                        DAG.getConstant(BitLen, DL, MVT::i8),
-                        DAG.getConstant(BitIdx, DL, MVT::i8));
+                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
+                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
       return DAG.getBitcast(RootVT, Res);
     }
 
@@ -32735,8 +32435,8 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
       V1 = DAG.getBitcast(IntMaskVT, V1);
       V2 = DAG.getBitcast(IntMaskVT, V2);
       Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
-                        DAG.getConstant(BitLen, DL, MVT::i8),
-                        DAG.getConstant(BitIdx, DL, MVT::i8));
+                        DAG.getTargetConstant(BitLen, DL, MVT::i8),
+                        DAG.getTargetConstant(BitIdx, DL, MVT::i8));
       return DAG.getBitcast(RootVT, Res);
     }
   }
@@ -32900,7 +32600,7 @@ static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
     V2 = DAG.getBitcast(MaskVT, V2);
     SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
     Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
-                      DAG.getConstant(M2ZImm, DL, MVT::i8));
+                      DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
     return DAG.getBitcast(RootVT, Res);
   }
 
@@ -33691,6 +33391,19 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
     if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR)
       return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
 
+    // vbroadcast(scalarload X) -> vbroadcast_load X
+    if (!SrcVT.isVector() && Src.hasOneUse() &&
+        ISD::isNormalLoad(Src.getNode())) {
+      LoadSDNode *LN = cast<LoadSDNode>(Src);
+      SDVTList Tys = DAG.getVTList(VT, MVT::Other);
+      SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
+      SDValue BcastLd =
+          DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
+                                  LN->getMemoryVT(), LN->getMemOperand());
+      DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
+      return BcastLd;
+    }
+
     // Share broadcast with the longest vector and extract low subvector (free).
     for (SDNode *User : Src->uses())
       if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
@@ -33719,7 +33432,7 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
         return DAG.getBitcast(
             VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
                             N1.getOperand(0),
-                            DAG.getConstant(BlendMask, DL, MVT::i8)));
+                            DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
       }
     }
     return SDValue();
@@ -33787,12 +33500,12 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
     // If we zero out all elements from Op0 then we don't need to reference it.
     if (((ZeroMask | (1u << DstIdx)) == 0xF) && !Op0.isUndef())
       return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
-                         DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                         DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
 
     // If we zero out the element from Op1 then we don't need to reference it.
     if ((ZeroMask & (1u << DstIdx)) && !Op1.isUndef())
       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
-                         DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                         DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
 
     // Attempt to merge insertps Op1 with an inner target shuffle node.
     SmallVector<int, 8> TargetMask1;
@@ -33803,14 +33516,14 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
         // Zero/UNDEF insertion - zero out element and remove dependency.
         InsertPSMask |= (1u << DstIdx);
         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
-                           DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                           DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
       }
       // Update insertps mask srcidx and reference the source input directly.
       assert(0 <= M && M < 8 && "Shuffle index out of range");
       InsertPSMask = (InsertPSMask & 0x3f) | ((M & 0x3) << 6);
       Op1 = Ops1[M < 4 ? 0 : 1];
       return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
-                         DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                         DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
     }
 
     // Attempt to merge insertps Op0 with an inner target shuffle node.
@@ -33853,20 +33566,26 @@ static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
 
       if (Updated)
         return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
-                           DAG.getConstant(InsertPSMask, DL, MVT::i8));
+                           DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
     }
 
-    // If we're inserting an element from a vbroadcast of a load, fold the
+    // If we're inserting an element from a vbroadcast load, fold the
     // load into the X86insertps instruction. We need to convert the scalar
     // load to a vector and clear the source lane of the INSERTPS control.
-    if (Op1.getOpcode() == X86ISD::VBROADCAST && Op1.hasOneUse() &&
-        Op1.getOperand(0).hasOneUse() &&
-        !Op1.getOperand(0).getValueType().isVector() &&
-        ISD::isNormalLoad(Op1.getOperand(0).getNode()))
-      return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
-                         DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
-                                     Op1.getOperand(0)),
-                         DAG.getConstant(InsertPSMask & 0x3f, DL, MVT::i8));
+    if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
+      auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
+      if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
+        SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
+                                   MemIntr->getBasePtr(),
+                                   MemIntr->getMemOperand());
+        SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
+                           DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
+                                       Load),
+                           DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
+        DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
+        return Insert;
+      }
+    }
 
     return SDValue();
   }
@@ -34222,14 +33941,14 @@ static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
 
   // When the operands of a horizontal math op are identical, the low half of
   // the result is the same as the high half. If a target shuffle is also
-  // replicating low and high halves, we don't need the shuffle.
+  // replicating low and high halves (and without changing the type/length of
+  // the vector), we don't need the shuffle.
   if (Opcode == X86ISD::MOVDDUP || Opcode == X86ISD::VBROADCAST) {
-    if (HOp.getScalarValueSizeInBits() == 64) {
+    if (HOp.getScalarValueSizeInBits() == 64 && HOp.getValueType() == VT) {
       // movddup (hadd X, X) --> hadd X, X
       // broadcast (extract_vec_elt (hadd X, X), 0) --> hadd X, X
       assert((HOp.getValueType() == MVT::v2f64 ||
-        HOp.getValueType() == MVT::v4f64) && HOp.getValueType() == VT &&
-        "Unexpected type for h-op");
+              HOp.getValueType() == MVT::v4f64) && "Unexpected type for h-op");
       return updateHOp(HOp, DAG);
     }
     return SDValue();
@@ -34309,67 +34028,6 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
       return HAddSub;
   }
 
-  // During Type Legalization, when promoting illegal vector types,
-  // the backend might introduce new shuffle dag nodes and bitcasts.
-  //
-  // This code performs the following transformation:
-  // fold: (shuffle (bitcast (BINOP A, B)), Undef, <Mask>) ->
-  //       (shuffle (BINOP (bitcast A), (bitcast B)), Undef, <Mask>)
-  //
-  // We do this only if both the bitcast and the BINOP dag nodes have
-  // one use. Also, perform this transformation only if the new binary
-  // operation is legal. This is to avoid introducing dag nodes that
-  // potentially need to be further expanded (or custom lowered) into a
-  // less optimal sequence of dag nodes.
-  if (!ExperimentalVectorWideningLegalization &&
-      !DCI.isBeforeLegalize() && DCI.isBeforeLegalizeOps() &&
-      N->getOpcode() == ISD::VECTOR_SHUFFLE &&
-      N->getOperand(0).getOpcode() == ISD::BITCAST &&
-      N->getOperand(1).isUndef() && N->getOperand(0).hasOneUse()) {
-    SDValue N0 = N->getOperand(0);
-    SDValue N1 = N->getOperand(1);
-
-    SDValue BC0 = N0.getOperand(0);
-    EVT SVT = BC0.getValueType();
-    unsigned Opcode = BC0.getOpcode();
-    unsigned NumElts = VT.getVectorNumElements();
-
-    if (BC0.hasOneUse() && SVT.isVector() &&
-        SVT.getVectorNumElements() * 2 == NumElts &&
-        TLI.isOperationLegal(Opcode, VT)) {
-      bool CanFold = false;
-      switch (Opcode) {
-      default : break;
-      case ISD::ADD:
-      case ISD::SUB:
-      case ISD::MUL:
-        // isOperationLegal lies for integer ops on floating point types.
-        CanFold = VT.isInteger();
-        break;
-      case ISD::FADD:
-      case ISD::FSUB:
-      case ISD::FMUL:
-        // isOperationLegal lies for floating point ops on integer types.
-        CanFold = VT.isFloatingPoint();
-        break;
-      }
-
-      unsigned SVTNumElts = SVT.getVectorNumElements();
-      ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-      for (unsigned i = 0, e = SVTNumElts; i != e && CanFold; ++i)
-        CanFold = SVOp->getMaskElt(i) == (int)(i * 2);
-      for (unsigned i = SVTNumElts, e = NumElts; i != e && CanFold; ++i)
-        CanFold = SVOp->getMaskElt(i) < 0;
-
-      if (CanFold) {
-        SDValue BC00 = DAG.getBitcast(VT, BC0.getOperand(0));
-        SDValue BC01 = DAG.getBitcast(VT, BC0.getOperand(1));
-        SDValue NewBinOp = DAG.getNode(BC0.getOpcode(), dl, VT, BC00, BC01);
-        return DAG.getVectorShuffle(VT, dl, NewBinOp, N1, SVOp->getMask());
-      }
-    }
-  }
-
   // Attempt to combine into a vector load/broadcast.
   if (SDValue LD = combineToConsecutiveLoads(VT, N, dl, DAG, Subtarget, true))
     return LD;
@@ -34465,54 +34123,6 @@ static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-
-  // Look for a truncating shuffle to v2i32 of a PMULUDQ where one of the
-  // operands is an extend from v2i32 to v2i64. Turn it into a pmulld.
-  // FIXME: This can probably go away once we default to widening legalization.
-  if (!ExperimentalVectorWideningLegalization &&
-      Subtarget.hasSSE41() && VT == MVT::v4i32 &&
-      N->getOpcode() == ISD::VECTOR_SHUFFLE &&
-      N->getOperand(0).getOpcode() == ISD::BITCAST &&
-      N->getOperand(0).getOperand(0).getOpcode() == X86ISD::PMULUDQ) {
-    SDValue BC = N->getOperand(0);
-    SDValue MULUDQ = BC.getOperand(0);
-    ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
-    ArrayRef<int> Mask = SVOp->getMask();
-    if (BC.hasOneUse() && MULUDQ.hasOneUse() &&
-        Mask[0] == 0 && Mask[1] == 2 && Mask[2] == -1 && Mask[3] == -1) {
-      SDValue Op0 = MULUDQ.getOperand(0);
-      SDValue Op1 = MULUDQ.getOperand(1);
-      if (Op0.getOpcode() == ISD::BITCAST &&
-          Op0.getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
-          Op0.getOperand(0).getValueType() == MVT::v4i32) {
-        ShuffleVectorSDNode *SVOp0 =
-          cast<ShuffleVectorSDNode>(Op0.getOperand(0));
-        ArrayRef<int> Mask2 = SVOp0->getMask();
-        if (Mask2[0] == 0 && Mask2[1] == -1 &&
-            Mask2[2] == 1 && Mask2[3] == -1) {
-          Op0 = SVOp0->getOperand(0);
-          Op1 = DAG.getBitcast(MVT::v4i32, Op1);
-          Op1 = DAG.getVectorShuffle(MVT::v4i32, dl, Op1, Op1, Mask);
-          return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1);
-        }
-      }
-      if (Op1.getOpcode() == ISD::BITCAST &&
-          Op1.getOperand(0).getOpcode() == ISD::VECTOR_SHUFFLE &&
-          Op1.getOperand(0).getValueType() == MVT::v4i32) {
-        ShuffleVectorSDNode *SVOp1 =
-          cast<ShuffleVectorSDNode>(Op1.getOperand(0));
-        ArrayRef<int> Mask2 = SVOp1->getMask();
-        if (Mask2[0] == 0 && Mask2[1] == -1 &&
-            Mask2[2] == 1 && Mask2[3] == -1) {
-          Op0 = DAG.getBitcast(MVT::v4i32, Op0);
-          Op0 = DAG.getVectorShuffle(MVT::v4i32, dl, Op0, Op0, Mask);
-          Op1 = SVOp1->getOperand(0);
-          return DAG.getNode(ISD::MUL, dl, MVT::v4i32, Op0, Op1);
-        }
-      }
-    }
-  }
-
   return SDValue();
 }
 
@@ -34577,29 +34187,82 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
     // TODO convert SrcUndef to KnownUndef.
     break;
   }
-  case X86ISD::KSHIFTL:
-  case X86ISD::KSHIFTR: {
+  case X86ISD::KSHIFTL: {
     SDValue Src = Op.getOperand(0);
     auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
     assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
     unsigned ShiftAmt = Amt->getZExtValue();
-    bool ShiftLeft = (X86ISD::KSHIFTL == Opc);
 
-    APInt DemandedSrc =
-        ShiftLeft ? DemandedElts.lshr(ShiftAmt) : DemandedElts.shl(ShiftAmt);
+    if (ShiftAmt == 0)
+      return TLO.CombineTo(Op, Src);
+
+    // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
+    // single shift.  We can do this if the bottom bits (which are shifted
+    // out) are never demanded.
+    if (Src.getOpcode() == X86ISD::KSHIFTR) {
+      if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) {
+        unsigned C1 = Src.getConstantOperandVal(1);
+        unsigned Opc = X86ISD::KSHIFTL;
+        int Diff = ShiftAmt - C1;
+        if (Diff < 0) {
+          Diff = -Diff;
+          Opc = X86ISD::KSHIFTR;
+        }
+
+        SDLoc dl(Op);
+        SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
+        return TLO.CombineTo(
+            Op, TLO.DAG.getNode(Opc, dl, VT, Src.getOperand(0), NewSA));
+      }
+    }
+
+    APInt DemandedSrc = DemandedElts.lshr(ShiftAmt);
     if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
                                    Depth + 1))
       return true;
 
-    if (ShiftLeft) {
-      KnownUndef = KnownUndef.shl(ShiftAmt);
-      KnownZero = KnownZero.shl(ShiftAmt);
-      KnownZero.setLowBits(ShiftAmt);
-    } else {
-      KnownUndef = KnownUndef.lshr(ShiftAmt);
-      KnownZero = KnownZero.lshr(ShiftAmt);
-      KnownZero.setHighBits(ShiftAmt);
+    KnownUndef <<= ShiftAmt;
+    KnownZero <<= ShiftAmt;
+    KnownZero.setLowBits(ShiftAmt);
+    break;
+  }
+  case X86ISD::KSHIFTR: {
+    SDValue Src = Op.getOperand(0);
+    auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
+    assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount");
+    unsigned ShiftAmt = Amt->getZExtValue();
+
+    if (ShiftAmt == 0)
+      return TLO.CombineTo(Op, Src);
+
+    // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
+    // single shift.  We can do this if the top bits (which are shifted
+    // out) are never demanded.
+    if (Src.getOpcode() == X86ISD::KSHIFTL) {
+      if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) {
+        unsigned C1 = Src.getConstantOperandVal(1);
+        unsigned Opc = X86ISD::KSHIFTR;
+        int Diff = ShiftAmt - C1;
+        if (Diff < 0) {
+          Diff = -Diff;
+          Opc = X86ISD::KSHIFTL;
+        }
+
+        SDLoc dl(Op);
+        SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
+        return TLO.CombineTo(
+            Op, TLO.DAG.getNode(Opc, dl, VT, Src.getOperand(0), NewSA));
+      }
     }
+
+    APInt DemandedSrc = DemandedElts.shl(ShiftAmt);
+    if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
+                                   Depth + 1))
+      return true;
+
+    KnownUndef.lshrInPlace(ShiftAmt);
+    KnownZero.lshrInPlace(ShiftAmt);
+    KnownZero.setHighBits(ShiftAmt);
     break;
   }
   case X86ISD::CVTSI2P:
@@ -34989,7 +34652,7 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
             unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
             SDValue NewShift = TLO.DAG.getNode(
                 NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
-                TLO.DAG.getConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
+                TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
             return TLO.CombineTo(Op, NewShift);
           }
         }
@@ -35635,8 +35298,8 @@ static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
       unsigned ShufMask = (NumElts > 2 ? 0 : 0x44);
       return DAG.getNode(
           ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
-          DAG.getConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32), Splat,
-          DAG.getConstant(ShufMask, DL, MVT::i8));
+          DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32),
+          Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
     }
     Ops.append(NumElts, Splat);
   } else {
@@ -36251,6 +35914,23 @@ static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
       return DAG.getBitcast(VT, SrcOp);
   }
 
+  // If we're extracting a single element from a broadcast load and there are
+  // no other users, just create a single load.
+  if (SrcBC.getOpcode() == X86ISD::VBROADCAST_LOAD && SrcBC.hasOneUse()) {
+    auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
+    unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
+    if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
+        VT.getSizeInBits() == SrcBCWidth) {
+      SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
+                                 MemIntr->getBasePtr(),
+                                 MemIntr->getPointerInfo(),
+                                 MemIntr->getAlignment(),
+                                 MemIntr->getMemOperand()->getFlags());
+      DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
+      return Load;
+    }
+  }
+
   // Handle extract(truncate(x)) for 0'th index.
   // TODO: Treat this as a faux shuffle?
   // TODO: When can we use this for general indices?
@@ -36392,7 +36072,7 @@ static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG) {
   }
 
   // TODO: This switch could include FNEG and the x86-specific FP logic ops
-  // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid 
+  // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid
   // missed load folding and fma+fneg combining.
   switch (Vec.getOpcode()) {
   case ISD::FMA: // Begin 3 operands
@@ -36457,7 +36137,7 @@ static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
 
   SDLoc DL(ExtElt);
 
-  if (ExperimentalVectorWideningLegalization && VecVT == MVT::v8i8) {
+  if (VecVT == MVT::v8i8) {
     // Pad with undef.
     Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Rdx,
                       DAG.getUNDEF(VecVT));
@@ -37163,8 +36843,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
   // Since SKX these selects have a proper lowering.
   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() &&
       CondVT.getVectorElementType() == MVT::i1 &&
-      (ExperimentalVectorWideningLegalization ||
-       VT.getVectorNumElements() > 4) &&
       (VT.getVectorElementType() == MVT::i8 ||
        VT.getVectorElementType() == MVT::i16)) {
     Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
@@ -37747,8 +37425,8 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
   // We can't always do this as FCMOV only supports a subset of X86 cond.
   if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
     if (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC)) {
-      SDValue Ops[] = {FalseOp, TrueOp, DAG.getConstant(CC, DL, MVT::i8),
-        Flags};
+      SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
+                       Flags};
       return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
     }
   }
@@ -37868,8 +37546,8 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
 
       if (CC == X86::COND_E &&
           CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
-        SDValue Ops[] = { FalseOp, Cond.getOperand(0),
-                          DAG.getConstant(CC, DL, MVT::i8), Cond };
+        SDValue Ops[] = {FalseOp, Cond.getOperand(0),
+                         DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
         return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
       }
     }
@@ -37903,10 +37581,11 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
         CC1 = X86::GetOppositeBranchCondition(CC1);
       }
 
-      SDValue LOps[] = {FalseOp, TrueOp, DAG.getConstant(CC0, DL, MVT::i8),
-        Flags};
+      SDValue LOps[] = {FalseOp, TrueOp,
+                        DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
       SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
-      SDValue Ops[] = {LCMOV, TrueOp, DAG.getConstant(CC1, DL, MVT::i8), Flags};
+      SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
+                       Flags};
       SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
       return CMOV;
     }
@@ -37938,9 +37617,9 @@ static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
       EVT VT = N->getValueType(0);
       // This should constant fold.
       SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
-      SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
-                                 DAG.getConstant(X86::COND_NE, DL, MVT::i8),
-                                 Cond);
+      SDValue CMov =
+          DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
+                      DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
       return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
     }
   }
@@ -38040,98 +37719,45 @@ static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
   if ((NumElts % 2) != 0)
     return SDValue();
 
-  unsigned RegSize = 128;
-  MVT OpsVT = MVT::getVectorVT(MVT::i16, RegSize / 16);
   EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
 
   // Shrink the operands of mul.
   SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
   SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
 
-  if (ExperimentalVectorWideningLegalization ||
-      NumElts >= OpsVT.getVectorNumElements()) {
-    // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
-    // lower part is needed.
-    SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
-    if (Mode == MULU8 || Mode == MULS8)
-      return DAG.getNode((Mode == MULU8) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND,
-                         DL, VT, MulLo);
-
-    MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2);
-    // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
-    // the higher part is also needed.
-    SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
-                                ReducedVT, NewN0, NewN1);
-
-    // Repack the lower part and higher part result of mul into a wider
-    // result.
-    // Generate shuffle functioning as punpcklwd.
-    SmallVector<int, 16> ShuffleMask(NumElts);
-    for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
-      ShuffleMask[2 * i] = i;
-      ShuffleMask[2 * i + 1] = i + NumElts;
-    }
-    SDValue ResLo =
-        DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
-    ResLo = DAG.getBitcast(ResVT, ResLo);
-    // Generate shuffle functioning as punpckhwd.
-    for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
-      ShuffleMask[2 * i] = i + NumElts / 2;
-      ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
-    }
-    SDValue ResHi =
-        DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
-    ResHi = DAG.getBitcast(ResVT, ResHi);
-    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
-  }
-
-  // When VT.getVectorNumElements() < OpsVT.getVectorNumElements(), we want
-  // to legalize the mul explicitly because implicit legalization for type
-  // <4 x i16> to <4 x i32> sometimes involves unnecessary unpack
-  // instructions which will not exist when we explicitly legalize it by
-  // extending <4 x i16> to <8 x i16> (concatenating the <4 x i16> val with
-  // <4 x i16> undef).
-  //
-  // Legalize the operands of mul.
-  // FIXME: We may be able to handle non-concatenated vectors by insertion.
-  unsigned ReducedSizeInBits = ReducedVT.getSizeInBits();
-  if ((RegSize % ReducedSizeInBits) != 0)
-    return SDValue();
-
-  SmallVector<SDValue, 16> Ops(RegSize / ReducedSizeInBits,
-                               DAG.getUNDEF(ReducedVT));
-  Ops[0] = NewN0;
-  NewN0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
-  Ops[0] = NewN1;
-  NewN1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, OpsVT, Ops);
-
-  if (Mode == MULU8 || Mode == MULS8) {
-    // Generate lower part of mul: pmullw. For MULU8/MULS8, only the lower
-    // part is needed.
-    SDValue Mul = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
-
-    // convert the type of mul result to VT.
-    MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
-    SDValue Res = DAG.getNode(Mode == MULU8 ? ISD::ZERO_EXTEND_VECTOR_INREG
-                                            : ISD::SIGN_EXTEND_VECTOR_INREG,
-                              DL, ResVT, Mul);
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
-                       DAG.getIntPtrConstant(0, DL));
-  }
+  // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
+  // lower part is needed.
+  SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
+  if (Mode == MULU8 || Mode == MULS8)
+    return DAG.getNode((Mode == MULU8) ? ISD::ZERO_EXTEND : ISD::SIGN_EXTEND,
+                       DL, VT, MulLo);
 
-  // Generate the lower and higher part of mul: pmulhw/pmulhuw. For
-  // MULU16/MULS16, both parts are needed.
-  SDValue MulLo = DAG.getNode(ISD::MUL, DL, OpsVT, NewN0, NewN1);
+  MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2);
+  // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
+  // the higher part is also needed.
   SDValue MulHi = DAG.getNode(Mode == MULS16 ? ISD::MULHS : ISD::MULHU, DL,
-                              OpsVT, NewN0, NewN1);
+                              ReducedVT, NewN0, NewN1);
 
   // Repack the lower part and higher part result of mul into a wider
-  // result. Make sure the type of mul result is VT.
-  MVT ResVT = MVT::getVectorVT(MVT::i32, RegSize / 32);
-  SDValue Res = getUnpackl(DAG, DL, OpsVT, MulLo, MulHi);
-  Res = DAG.getBitcast(ResVT, Res);
-  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
-                     DAG.getIntPtrConstant(0, DL));
+  // result.
+  // Generate shuffle functioning as punpcklwd.
+  SmallVector<int, 16> ShuffleMask(NumElts);
+  for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
+    ShuffleMask[2 * i] = i;
+    ShuffleMask[2 * i + 1] = i + NumElts;
+  }
+  SDValue ResLo =
+      DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
+  ResLo = DAG.getBitcast(ResVT, ResLo);
+  // Generate shuffle functioning as punpckhwd.
+  for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
+    ShuffleMask[2 * i] = i + NumElts / 2;
+    ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
+  }
+  SDValue ResHi =
+      DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
+  ResHi = DAG.getBitcast(ResVT, ResHi);
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
 }
 
 static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
@@ -38239,8 +37865,7 @@ static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
   // Make sure the vXi16 type is legal. This covers the AVX512 without BWI case.
   // Also allow v2i32 if it will be widened.
   MVT WVT = MVT::getVectorVT(MVT::i16, 2 * VT.getVectorNumElements());
-  if (!((ExperimentalVectorWideningLegalization && VT == MVT::v2i32) ||
-        DAG.getTargetLoweringInfo().isTypeLegal(WVT)))
+  if (VT != MVT::v2i32 && !DAG.getTargetLoweringInfo().isTypeLegal(WVT))
     return SDValue();
 
   SDValue N0 = N->getOperand(0);
@@ -38793,7 +38418,7 @@ static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
     if (NewShiftVal >= NumBitsPerElt)
       NewShiftVal = NumBitsPerElt - 1;
     return DAG.getNode(X86ISD::VSRAI, SDLoc(N), VT, N0.getOperand(0),
-                       DAG.getConstant(NewShiftVal, SDLoc(N), MVT::i8));
+                       DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
   }
 
   // We can decode 'whole byte' logical bit shifts as shuffles.
@@ -38913,7 +38538,7 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
           if (Subtarget.hasAVX512()) {
             SDValue FSetCC =
                 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
-                            DAG.getConstant(x86cc, DL, MVT::i8));
+                            DAG.getTargetConstant(x86cc, DL, MVT::i8));
             // Need to fill with zeros to ensure the bitcast will produce zeroes
             // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
             SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
@@ -38922,10 +38547,9 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
             return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
                                       N->getSimpleValueType(0));
           }
-          SDValue OnesOrZeroesF = DAG.getNode(X86ISD::FSETCC, DL,
-                                              CMP00.getValueType(), CMP00, CMP01,
-                                              DAG.getConstant(x86cc, DL,
-                                                              MVT::i8));
+          SDValue OnesOrZeroesF =
+              DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
+                          CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
 
           bool is64BitFP = (CMP00.getValueType() == MVT::f64);
           MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
@@ -39119,7 +38743,7 @@ static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
 
   SDLoc DL(N);
   unsigned ShiftVal = SplatVal.countTrailingOnes();
-  SDValue ShAmt = DAG.getConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
+  SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
   SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
   return DAG.getBitcast(N->getValueType(0), Shift);
 }
@@ -39431,7 +39055,7 @@ static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
                                      const X86Subtarget &Subtarget) {
   assert(N->getOpcode() == ISD::OR && "Unexpected Opcode");
 
-  EVT VT = N->getValueType(0);
+  MVT VT = N->getSimpleValueType(0);
   if (!VT.isVector() || (VT.getScalarSizeInBits() % 8) != 0)
     return SDValue();
 
@@ -39440,10 +39064,12 @@ static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
   if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
     return SDValue();
 
-  // On XOP we'll lower to PCMOV so accept one use, otherwise only
-  // do this if either mask has multiple uses already.
-  if (!(Subtarget.hasXOP() || !N0.getOperand(1).hasOneUse() ||
-        !N1.getOperand(1).hasOneUse()))
+  // On XOP we'll lower to PCMOV so accept one use. With AVX512, we can use
+  // VPTERNLOG. Otherwise only do this if either mask has multiple uses already.
+  bool UseVPTERNLOG = (Subtarget.hasAVX512() && VT.is512BitVector()) ||
+                      Subtarget.hasVLX();
+  if (!(Subtarget.hasXOP() || UseVPTERNLOG ||
+        !N0.getOperand(1).hasOneUse() || !N1.getOperand(1).hasOneUse()))
     return SDValue();
 
   // Attempt to extract constant byte masks.
@@ -40496,93 +40122,7 @@ static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
         return Blend;
   }
 
-  if (ExperimentalVectorWideningLegalization)
-    return SDValue();
-
-  if (Mld->getExtensionType() != ISD::EXTLOAD)
-    return SDValue();
-
-  // Resolve extending loads.
-  EVT VT = Mld->getValueType(0);
-  unsigned NumElems = VT.getVectorNumElements();
-  EVT LdVT = Mld->getMemoryVT();
-  SDLoc dl(Mld);
-
-  assert(LdVT != VT && "Cannot extend to the same type");
-  unsigned ToSz = VT.getScalarSizeInBits();
-  unsigned FromSz = LdVT.getScalarSizeInBits();
-  // From/To sizes and ElemCount must be pow of two.
-  assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
-    "Unexpected size for extending masked load");
-
-  unsigned SizeRatio  = ToSz / FromSz;
-  assert(SizeRatio * NumElems * FromSz == VT.getSizeInBits());
-
-  // Create a type on which we perform the shuffle.
-  EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
-          LdVT.getScalarType(), NumElems*SizeRatio);
-  assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
-
-  // Convert PassThru value.
-  SDValue WidePassThru = DAG.getBitcast(WideVecVT, Mld->getPassThru());
-  if (!Mld->getPassThru().isUndef()) {
-    SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
-    for (unsigned i = 0; i != NumElems; ++i)
-      ShuffleVec[i] = i * SizeRatio;
-
-    // Can't shuffle using an illegal type.
-    assert(DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT) &&
-           "WideVecVT should be legal");
-    WidePassThru = DAG.getVectorShuffle(WideVecVT, dl, WidePassThru,
-                                    DAG.getUNDEF(WideVecVT), ShuffleVec);
-  }
-
-  // Prepare the new mask.
-  SDValue NewMask;
-  SDValue Mask = Mld->getMask();
-  if (Mask.getValueType() == VT) {
-    // Mask and original value have the same type.
-    NewMask = DAG.getBitcast(WideVecVT, Mask);
-    SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
-    for (unsigned i = 0; i != NumElems; ++i)
-      ShuffleVec[i] = i * SizeRatio;
-    for (unsigned i = NumElems; i != NumElems * SizeRatio; ++i)
-      ShuffleVec[i] = NumElems * SizeRatio;
-    NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
-                                   DAG.getConstant(0, dl, WideVecVT),
-                                   ShuffleVec);
-  } else {
-    assert(Mask.getValueType().getVectorElementType() == MVT::i1);
-    unsigned WidenNumElts = NumElems*SizeRatio;
-    unsigned MaskNumElts = VT.getVectorNumElements();
-    EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(),  MVT::i1,
-                                     WidenNumElts);
-
-    unsigned NumConcat = WidenNumElts / MaskNumElts;
-    SDValue ZeroVal = DAG.getConstant(0, dl, Mask.getValueType());
-    SmallVector<SDValue, 16> Ops(NumConcat, ZeroVal);
-    Ops[0] = Mask;
-    NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
-  }
-
-  SDValue WideLd = DAG.getMaskedLoad(WideVecVT, dl, Mld->getChain(),
-                                     Mld->getBasePtr(), NewMask, WidePassThru,
-                                     Mld->getMemoryVT(), Mld->getMemOperand(),
-                                     ISD::NON_EXTLOAD);
-
-  SDValue SlicedVec = DAG.getBitcast(WideVecVT, WideLd);
-  SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
-  for (unsigned i = 0; i != NumElems; ++i)
-    ShuffleVec[i * SizeRatio] = i;
-
-  // Can't shuffle using an illegal type.
-  assert(DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT) &&
-         "WideVecVT should be legal");
-  SlicedVec = DAG.getVectorShuffle(WideVecVT, dl, SlicedVec,
-                                   DAG.getUNDEF(WideVecVT), ShuffleVec);
-  SlicedVec = DAG.getBitcast(VT, SlicedVec);
-
-  return DCI.CombineTo(N, SlicedVec, WideLd.getValue(1), true);
+  return SDValue();
 }
 
 /// If exactly one element of the mask is set for a non-truncating masked store,
@@ -40620,111 +40160,34 @@ static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   EVT VT = Mst->getValue().getValueType();
-  EVT StVT = Mst->getMemoryVT();
   SDLoc dl(Mst);
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
-  if (!Mst->isTruncatingStore()) {
-    if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
-      return ScalarStore;
-
-    // If the mask value has been legalized to a non-boolean vector, try to
-    // simplify ops leading up to it. We only demand the MSB of each lane.
-    SDValue Mask = Mst->getMask();
-    if (Mask.getScalarValueSizeInBits() != 1) {
-      APInt DemandedMask(APInt::getSignMask(VT.getScalarSizeInBits()));
-      if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
-        return SDValue(N, 0);
-    }
-
-    SDValue Value = Mst->getValue();
-    if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
-        TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
-                              Mst->getMemoryVT())) {
-      return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
-                                Mst->getBasePtr(), Mask,
-                                Mst->getMemoryVT(), Mst->getMemOperand(), true);
-    }
-
-    return SDValue();
-  }
-
-  if (ExperimentalVectorWideningLegalization)
-    return SDValue();
-
-  // Resolve truncating stores.
-  unsigned NumElems = VT.getVectorNumElements();
-
-  assert(StVT != VT && "Cannot truncate to the same type");
-  unsigned FromSz = VT.getScalarSizeInBits();
-  unsigned ToSz = StVT.getScalarSizeInBits();
-
-  // The truncating store is legal in some cases. For example
-  // vpmovqb, vpmovqw, vpmovqd, vpmovdb, vpmovdw
-  // are designated for truncate store.
-  // In this case we don't need any further transformations.
-  if (TLI.isTruncStoreLegal(VT, StVT))
+  if (Mst->isTruncatingStore())
     return SDValue();
 
-  // From/To sizes and ElemCount must be pow of two.
-  assert (isPowerOf2_32(NumElems * FromSz * ToSz) &&
-    "Unexpected size for truncating masked store");
-  // We are going to use the original vector elt for storing.
-  // Accumulated smaller vector elements must be a multiple of the store size.
-  assert (((NumElems * FromSz) % ToSz) == 0 &&
-          "Unexpected ratio for truncating masked store");
-
-  unsigned SizeRatio  = FromSz / ToSz;
-  assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
-
-  // Create a type on which we perform the shuffle.
-  EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
-          StVT.getScalarType(), NumElems*SizeRatio);
-
-  assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
+  if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
+    return ScalarStore;
 
-  SDValue WideVec = DAG.getBitcast(WideVecVT, Mst->getValue());
-  SmallVector<int, 16> ShuffleVec(NumElems * SizeRatio, -1);
-  for (unsigned i = 0; i != NumElems; ++i)
-    ShuffleVec[i] = i * SizeRatio;
-
-  // Can't shuffle using an illegal type.
-  assert(DAG.getTargetLoweringInfo().isTypeLegal(WideVecVT) &&
-         "WideVecVT should be legal");
-
-  SDValue TruncatedVal = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
-                                              DAG.getUNDEF(WideVecVT),
-                                              ShuffleVec);
-
-  SDValue NewMask;
+  // If the mask value has been legalized to a non-boolean vector, try to
+  // simplify ops leading up to it. We only demand the MSB of each lane.
   SDValue Mask = Mst->getMask();
-  if (Mask.getValueType() == VT) {
-    // Mask and original value have the same type.
-    NewMask = DAG.getBitcast(WideVecVT, Mask);
-    for (unsigned i = 0; i != NumElems; ++i)
-      ShuffleVec[i] = i * SizeRatio;
-    for (unsigned i = NumElems; i != NumElems*SizeRatio; ++i)
-      ShuffleVec[i] = NumElems*SizeRatio;
-    NewMask = DAG.getVectorShuffle(WideVecVT, dl, NewMask,
-                                   DAG.getConstant(0, dl, WideVecVT),
-                                   ShuffleVec);
-  } else {
-    assert(Mask.getValueType().getVectorElementType() == MVT::i1);
-    unsigned WidenNumElts = NumElems*SizeRatio;
-    unsigned MaskNumElts = VT.getVectorNumElements();
-    EVT NewMaskVT = EVT::getVectorVT(*DAG.getContext(),  MVT::i1,
-                                     WidenNumElts);
+  if (Mask.getScalarValueSizeInBits() != 1) {
+    APInt DemandedMask(APInt::getSignMask(VT.getScalarSizeInBits()));
+    if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
+      return SDValue(N, 0);
+  }
 
-    unsigned NumConcat = WidenNumElts / MaskNumElts;
-    SDValue ZeroVal = DAG.getConstant(0, dl, Mask.getValueType());
-    SmallVector<SDValue, 16> Ops(NumConcat, ZeroVal);
-    Ops[0] = Mask;
-    NewMask = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewMaskVT, Ops);
+  SDValue Value = Mst->getValue();
+  if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
+      TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
+                            Mst->getMemoryVT())) {
+    return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
+                              Mst->getBasePtr(), Mask,
+                              Mst->getMemoryVT(), Mst->getMemOperand(), true);
   }
 
-  return DAG.getMaskedStore(Mst->getChain(), dl, TruncatedVal,
-                            Mst->getBasePtr(), NewMask, StVT,
-                            Mst->getMemOperand(), false);
+  return SDValue();
 }
 
 static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
@@ -40852,41 +40315,6 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
                              MVT::v16i8, St->getMemOperand());
   }
 
-  // Look for a truncating store to a less than 128 bit vector that has been
-  // truncated from an any_extend_inreg from a 128 bit vector with the same
-  // element size. We can use a 64/32/16-bit extractelement and store that.
-  // Disabling this when widening legalization is in effect since the trunc
-  // store would have been unlikely to be created in that case. Only doing this
-  // when truncstore is legal since it would otherwise be decomposed below and
-  // then combined away.
-  if (St->isTruncatingStore() && TLI.isTruncStoreLegal(VT, StVT) &&
-      StoredVal.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG &&
-      StoredVal.getValueType().is128BitVector() &&
-      !ExperimentalVectorWideningLegalization) {
-    EVT OrigVT = StoredVal.getOperand(0).getValueType();
-    if (OrigVT.is128BitVector() &&
-        OrigVT.getVectorElementType() == StVT.getVectorElementType()) {
-      unsigned StoreSize = StVT.getSizeInBits();
-      assert((128 % StoreSize == 0) && "Unexpected store size!");
-      MVT IntVT = MVT::getIntegerVT(StoreSize);
-      MVT CastVT = MVT::getVectorVT(IntVT, 128 / StoreSize);
-      StoredVal = DAG.getBitcast(CastVT, StoredVal.getOperand(0));
-      // Use extract_store for the 64-bit case to support 32-bit targets.
-      if (IntVT == MVT::i64) {
-        SDVTList Tys = DAG.getVTList(MVT::Other);
-        SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
-        return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops,
-                                       IntVT, St->getMemOperand());
-      }
-
-      // Otherwise just use an extract and store.
-      StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, IntVT, StoredVal,
-                              DAG.getIntPtrConstant(0, dl));
-      return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
-                          St->getMemOperand());
-    }
-  }
-
   // Optimize trunc store (of multiple scalars) to shuffle and store.
   // First, pack all of the elements in one place. Next, store to memory
   // in fewer chunks.
@@ -40913,85 +40341,7 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
                                St->getMemoryVT(), St->getMemOperand(), DAG);
     }
 
-    if (ExperimentalVectorWideningLegalization)
-      return SDValue();
-
-    unsigned NumElems = VT.getVectorNumElements();
-    assert(StVT != VT && "Cannot truncate to the same type");
-    unsigned FromSz = VT.getScalarSizeInBits();
-    unsigned ToSz = StVT.getScalarSizeInBits();
-
-    // The truncating store is legal in some cases. For example
-    // vpmovqb, vpmovqw, vpmovqd, vpmovdb, vpmovdw
-    // are designated for truncate store.
-    // In this case we don't need any further transformations.
-    if (TLI.isTruncStoreLegalOrCustom(VT, StVT))
-      return SDValue();
-
-    // From, To sizes and ElemCount must be pow of two
-    if (!isPowerOf2_32(NumElems * FromSz * ToSz)) return SDValue();
-    // We are going to use the original vector elt for storing.
-    // Accumulated smaller vector elements must be a multiple of the store size.
-    if (0 != (NumElems * FromSz) % ToSz) return SDValue();
-
-    unsigned SizeRatio  = FromSz / ToSz;
-
-    assert(SizeRatio * NumElems * ToSz == VT.getSizeInBits());
-
-    // Create a type on which we perform the shuffle
-    EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(),
-            StVT.getScalarType(), NumElems*SizeRatio);
-
-    assert(WideVecVT.getSizeInBits() == VT.getSizeInBits());
-
-    SDValue WideVec = DAG.getBitcast(WideVecVT, St->getValue());
-    SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
-    for (unsigned i = 0; i != NumElems; ++i)
-      ShuffleVec[i] = i * SizeRatio;
-
-    // Can't shuffle using an illegal type.
-    if (!TLI.isTypeLegal(WideVecVT))
-      return SDValue();
-
-    SDValue Shuff = DAG.getVectorShuffle(WideVecVT, dl, WideVec,
-                                         DAG.getUNDEF(WideVecVT),
-                                         ShuffleVec);
-    // At this point all of the data is stored at the bottom of the
-    // register. We now need to save it to mem.
-
-    // Find the largest store unit
-    MVT StoreType = MVT::i8;
-    for (MVT Tp : MVT::integer_valuetypes()) {
-      if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToSz)
-        StoreType = Tp;
-    }
-
-    // On 32bit systems, we can't save 64bit integers. Try bitcasting to F64.
-    if (TLI.isTypeLegal(MVT::f64) && StoreType.getSizeInBits() < 64 &&
-        (64 <= NumElems * ToSz))
-      StoreType = MVT::f64;
-
-    // Bitcast the original vector into a vector of store-size units
-    EVT StoreVecVT = EVT::getVectorVT(*DAG.getContext(),
-            StoreType, VT.getSizeInBits()/StoreType.getSizeInBits());
-    assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits());
-    SDValue ShuffWide = DAG.getBitcast(StoreVecVT, Shuff);
-    SmallVector<SDValue, 8> Chains;
-    SDValue Ptr = St->getBasePtr();
-
-    // Perform one or more big stores into memory.
-    for (unsigned i=0, e=(ToSz*NumElems)/StoreType.getSizeInBits(); i!=e; ++i) {
-      SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
-                                   StoreType, ShuffWide,
-                                   DAG.getIntPtrConstant(i, dl));
-      SDValue Ch =
-          DAG.getStore(St->getChain(), dl, SubVec, Ptr, St->getPointerInfo(),
-                       St->getAlignment(), St->getMemOperand()->getFlags());
-      Ptr = DAG.getMemBasePlusOffset(Ptr, StoreType.getStoreSize(), dl);
-      Chains.push_back(Ch);
-    }
-
-    return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
+    return SDValue();
   }
 
   // Turn load->store of MMX types into GPR load/stores.  This avoids clobbering
@@ -41007,9 +40357,7 @@ static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
   bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
   bool F64IsLegal =
       !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2();
-  if (((VT.isVector() && !VT.isFloatingPoint() &&
-        !ExperimentalVectorWideningLegalization) ||
-       (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit())) &&
+  if ((VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit()) &&
       isa<LoadSDNode>(St->getValue()) &&
       cast<LoadSDNode>(St->getValue())->isSimple() &&
       St->getChain().hasOneUse() && St->isSimple()) {
@@ -41524,9 +40872,7 @@ static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
 
   // Only handle vXi16 types that are at least 128-bits unless they will be
   // widened.
-  if (!VT.isVector() || VT.getVectorElementType() != MVT::i16 ||
-      (!ExperimentalVectorWideningLegalization &&
-       VT.getVectorNumElements() < 8))
+  if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)
     return SDValue();
 
   // Input type should be vXi32.
@@ -42640,127 +41986,6 @@ combineToExtendBoolVectorInReg(SDNode *N, SelectionDAG &DAG,
                      DAG.getConstant(EltSizeInBits - 1, DL, VT));
 }
 
-/// Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or
-/// ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating
-/// with UNDEFs) of the input to vectors of the same size as the target type
-/// which then extends the lowest elements.
-static SDValue combineToExtendVectorInReg(SDNode *N, SelectionDAG &DAG,
-                                          TargetLowering::DAGCombinerInfo &DCI,
-                                          const X86Subtarget &Subtarget) {
-  if (ExperimentalVectorWideningLegalization)
-    return SDValue();
-
-  unsigned Opcode = N->getOpcode();
-  // TODO - add ANY_EXTEND support.
-  if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND)
-    return SDValue();
-  if (!DCI.isBeforeLegalizeOps())
-    return SDValue();
-  if (!Subtarget.hasSSE2())
-    return SDValue();
-
-  SDValue N0 = N->getOperand(0);
-  EVT VT = N->getValueType(0);
-  EVT SVT = VT.getScalarType();
-  EVT InVT = N0.getValueType();
-  EVT InSVT = InVT.getScalarType();
-
-  // FIXME: Generic DAGCombiner previously had a bug that would cause a
-  // sign_extend of setcc to sometimes return the original node and tricked it
-  // into thinking CombineTo was used which prevented the target combines from
-  // running.
-  // Earlying out here to avoid regressions like this
-  //  (v4i32 (sext (v4i1 (setcc (v4i16)))))
-  // Becomes
-  //  (v4i32 (sext_invec (v8i16 (concat (v4i16 (setcc (v4i16))), undef))))
-  // Type legalized to
-  //  (v4i32 (sext_invec (v8i16 (trunc_invec (v4i32 (setcc (v4i32)))))))
-  // Leading to a packssdw+pmovsxwd
-  // We could write a DAG combine to fix this, but really we shouldn't be
-  // creating sext_invec that's forcing v8i16 into the DAG.
-  if (N0.getOpcode() == ISD::SETCC)
-    return SDValue();
-
-  // Input type must be a vector and we must be extending legal integer types.
-  if (!VT.isVector() || VT.getVectorNumElements() < 2)
-    return SDValue();
-  if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)
-    return SDValue();
-  if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
-    return SDValue();
-
-  // If the input/output types are both legal then we have at least AVX1 and
-  // we will be able to use SIGN_EXTEND/ZERO_EXTEND directly.
-  if (DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
-      DAG.getTargetLoweringInfo().isTypeLegal(InVT))
-    return SDValue();
-
-  SDLoc DL(N);
-
-  auto ExtendVecSize = [&DAG](const SDLoc &DL, SDValue N, unsigned Size) {
-    EVT SrcVT = N.getValueType();
-    EVT DstVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
-                                 Size / SrcVT.getScalarSizeInBits());
-    SmallVector<SDValue, 8> Opnds(Size / SrcVT.getSizeInBits(),
-                                  DAG.getUNDEF(SrcVT));
-    Opnds[0] = N;
-    return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Opnds);
-  };
-
-  // If target-size is less than 128-bits, extend to a type that would extend
-  // to 128 bits, extend that and extract the original target vector.
-  if (VT.getSizeInBits() < 128 && !(128 % VT.getSizeInBits())) {
-    unsigned Scale = 128 / VT.getSizeInBits();
-    EVT ExVT =
-        EVT::getVectorVT(*DAG.getContext(), SVT, 128 / SVT.getSizeInBits());
-    SDValue Ex = ExtendVecSize(DL, N0, Scale * InVT.getSizeInBits());
-    SDValue SExt = DAG.getNode(Opcode, DL, ExVT, Ex);
-    return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SExt,
-                       DAG.getIntPtrConstant(0, DL));
-  }
-
-  // If target-size is 128-bits (or 256-bits on AVX target), then convert to
-  // ISD::*_EXTEND_VECTOR_INREG which ensures lowering to X86ISD::V*EXT.
-  // Also use this if we don't have SSE41 to allow the legalizer do its job.
-  if (!Subtarget.hasSSE41() || VT.is128BitVector() ||
-      (VT.is256BitVector() && Subtarget.hasAVX()) ||
-      (VT.is512BitVector() && Subtarget.useAVX512Regs())) {
-    SDValue ExOp = ExtendVecSize(DL, N0, VT.getSizeInBits());
-    Opcode = getOpcode_EXTEND_VECTOR_INREG(Opcode);
-    return DAG.getNode(Opcode, DL, VT, ExOp);
-  }
-
-  auto SplitAndExtendInReg = [&](unsigned SplitSize) {
-    unsigned NumVecs = VT.getSizeInBits() / SplitSize;
-    unsigned NumSubElts = SplitSize / SVT.getSizeInBits();
-    EVT SubVT = EVT::getVectorVT(*DAG.getContext(), SVT, NumSubElts);
-    EVT InSubVT = EVT::getVectorVT(*DAG.getContext(), InSVT, NumSubElts);
-
-    unsigned IROpc = getOpcode_EXTEND_VECTOR_INREG(Opcode);
-    SmallVector<SDValue, 8> Opnds;
-    for (unsigned i = 0, Offset = 0; i != NumVecs; ++i, Offset += NumSubElts) {
-      SDValue SrcVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InSubVT, N0,
-                                   DAG.getIntPtrConstant(Offset, DL));
-      SrcVec = ExtendVecSize(DL, SrcVec, SplitSize);
-      SrcVec = DAG.getNode(IROpc, DL, SubVT, SrcVec);
-      Opnds.push_back(SrcVec);
-    }
-    return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Opnds);
-  };
-
-  // On pre-AVX targets, split into 128-bit nodes of
-  // ISD::*_EXTEND_VECTOR_INREG.
-  if (!Subtarget.hasAVX() && !(VT.getSizeInBits() % 128))
-    return SplitAndExtendInReg(128);
-
-  // On pre-AVX512 targets, split into 256-bit nodes of
-  // ISD::*_EXTEND_VECTOR_INREG.
-  if (!Subtarget.useAVX512Regs() && !(VT.getSizeInBits() % 256))
-    return SplitAndExtendInReg(256);
-
-  return SDValue();
-}
-
 // Attempt to combine a (sext/zext (setcc)) to a setcc with a xmm/ymm/zmm
 // result type.
 static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
@@ -42831,9 +42056,6 @@ static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
     return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT));
   }
 
-  if (SDValue V = combineToExtendVectorInReg(N, DAG, DCI, Subtarget))
-    return V;
-
   if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
     return V;
 
@@ -42969,9 +42191,6 @@ static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
     if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
       return V;
 
-  if (SDValue V = combineToExtendVectorInReg(N, DAG, DCI, Subtarget))
-    return V;
-
   if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
     return V;
 
@@ -43148,8 +42367,6 @@ static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
   // go through type promotion to a 128-bit vector.
   if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&
       VT.getVectorElementType() == MVT::i1 &&
-      (ExperimentalVectorWideningLegalization ||
-       VT.getVectorNumElements() > 4) &&
       (OpVT.getVectorElementType() == MVT::i8 ||
        OpVT.getVectorElementType() == MVT::i16)) {
     SDValue Setcc = DAG.getNode(ISD::SETCC, DL, OpVT, LHS, RHS,
@@ -43213,30 +42430,87 @@ static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
+                                       TargetLowering::DAGCombinerInfo &DCI) {
+  // With vector masks we only demand the upper bit of the mask.
+  SDValue Mask = cast<X86MaskedGatherScatterSDNode>(N)->getMask();
+  if (Mask.getScalarValueSizeInBits() != 1) {
+    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+    APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
+    if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
+      return SDValue(N, 0);
+  }
+
+  return SDValue();
+}
+
 static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
-                                    TargetLowering::DAGCombinerInfo &DCI,
-                                    const X86Subtarget &Subtarget) {
+                                    TargetLowering::DAGCombinerInfo &DCI) {
   SDLoc DL(N);
+  auto *GorS = cast<MaskedGatherScatterSDNode>(N);
+  SDValue Chain = GorS->getChain();
+  SDValue Index = GorS->getIndex();
+  SDValue Mask = GorS->getMask();
+  SDValue Base = GorS->getBasePtr();
+  SDValue Scale = GorS->getScale();
+
+  // Shrink constant indices if they are larger than 32-bits.
+  // Only do this before legalize types since v2i64 could become v2i32.
+  // FIXME: We could check that the type is legal if we're after legalize types,
+  // but then we would need to construct test cases where that happens.
+  // FIXME: We could support more than just constant vectors, but we need to
+  // careful with costing. A truncate that can be optimized out would be fine.
+  // Otherwise we might only want to create a truncate if it avoids a split.
+  if (DCI.isBeforeLegalize()) {
+    if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
+      unsigned IndexWidth = Index.getScalarValueSizeInBits();
+      if (BV->isConstant() && IndexWidth > 32 &&
+          DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
+        unsigned NumElts = Index.getValueType().getVectorNumElements();
+        EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
+        Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
+        if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
+          SDValue Ops[] = { Chain, Gather->getPassThru(),
+                            Mask, Base, Index, Scale } ;
+          return DAG.getMaskedGather(Gather->getVTList(),
+                                     Gather->getMemoryVT(), DL, Ops,
+                                     Gather->getMemOperand(),
+                                     Gather->getIndexType());
+        }
+        auto *Scatter = cast<MaskedScatterSDNode>(GorS);
+        SDValue Ops[] = { Chain, Scatter->getValue(),
+                          Mask, Base, Index, Scale };
+        return DAG.getMaskedScatter(Scatter->getVTList(),
+                                    Scatter->getMemoryVT(), DL,
+                                    Ops, Scatter->getMemOperand(),
+                                    Scatter->getIndexType());
+      }
+    }
+  }
 
   if (DCI.isBeforeLegalizeOps()) {
-    SDValue Index = N->getOperand(4);
     // Remove any sign extends from 32 or smaller to larger than 32.
     // Only do this before LegalizeOps in case we need the sign extend for
     // legalization.
-    if (Index.getOpcode() == ISD::SIGN_EXTEND) {
-      if (Index.getScalarValueSizeInBits() > 32 &&
-          Index.getOperand(0).getScalarValueSizeInBits() <= 32) {
-        SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
-        NewOps[4] = Index.getOperand(0);
-        SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
-        if (Res == N) {
-          // The original sign extend has less users, add back to worklist in
-          // case it needs to be removed
-          DCI.AddToWorklist(Index.getNode());
-          DCI.AddToWorklist(N);
-        }
-        return SDValue(Res, 0);
-      }
+    if (Index.getOpcode() == ISD::SIGN_EXTEND &&
+        Index.getScalarValueSizeInBits() > 32 &&
+        Index.getOperand(0).getScalarValueSizeInBits() <= 32) {
+      Index = Index.getOperand(0);
+      if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
+        SDValue Ops[] = { Chain, Gather->getPassThru(),
+                          Mask, Base, Index, Scale } ;
+        return DAG.getMaskedGather(Gather->getVTList(),
+                                   Gather->getMemoryVT(), DL, Ops,
+                                   Gather->getMemOperand(),
+                                   Gather->getIndexType());
+      }
+      auto *Scatter = cast<MaskedScatterSDNode>(GorS);
+      SDValue Ops[] = { Chain, Scatter->getValue(),
+                        Mask, Base, Index, Scale };
+      return DAG.getMaskedScatter(Scatter->getVTList(),
+                                  Scatter->getMemoryVT(), DL,
+                                  Ops, Scatter->getMemOperand(),
+                                  Scatter->getIndexType());
     }
 
     // Make sure the index is either i32 or i64
@@ -43246,38 +42520,51 @@ static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
       EVT IndexVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
                                    Index.getValueType().getVectorNumElements());
       Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
-      SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
-      NewOps[4] = Index;
-      SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
-      if (Res == N)
-        DCI.AddToWorklist(N);
-      return SDValue(Res, 0);
+      if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
+        SDValue Ops[] = { Chain, Gather->getPassThru(),
+                          Mask, Base, Index, Scale } ;
+        return DAG.getMaskedGather(Gather->getVTList(),
+                                   Gather->getMemoryVT(), DL, Ops,
+                                   Gather->getMemOperand(),
+                                   Gather->getIndexType());
+      }
+      auto *Scatter = cast<MaskedScatterSDNode>(GorS);
+      SDValue Ops[] = { Chain, Scatter->getValue(),
+                        Mask, Base, Index, Scale };
+      return DAG.getMaskedScatter(Scatter->getVTList(),
+                                  Scatter->getMemoryVT(), DL,
+                                  Ops, Scatter->getMemOperand(),
+                                  Scatter->getIndexType());
     }
 
     // Try to remove zero extends from 32->64 if we know the sign bit of
     // the input is zero.
     if (Index.getOpcode() == ISD::ZERO_EXTEND &&
         Index.getScalarValueSizeInBits() == 64 &&
-        Index.getOperand(0).getScalarValueSizeInBits() == 32) {
-      if (DAG.SignBitIsZero(Index.getOperand(0))) {
-        SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
-        NewOps[4] = Index.getOperand(0);
-        SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
-        if (Res == N) {
-          // The original sign extend has less users, add back to worklist in
-          // case it needs to be removed
-          DCI.AddToWorklist(Index.getNode());
-          DCI.AddToWorklist(N);
-        }
-        return SDValue(Res, 0);
-      }
-    }
-  }
-
-  // With AVX2 we only demand the upper bit of the mask.
-  if (!Subtarget.hasAVX512()) {
+        Index.getOperand(0).getScalarValueSizeInBits() == 32 &&
+        DAG.SignBitIsZero(Index.getOperand(0))) {
+      Index = Index.getOperand(0);
+      if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
+        SDValue Ops[] = { Chain, Gather->getPassThru(),
+                          Mask, Base, Index, Scale } ;
+        return DAG.getMaskedGather(Gather->getVTList(),
+                                   Gather->getMemoryVT(), DL, Ops,
+                                   Gather->getMemOperand(),
+                                   Gather->getIndexType());
+      }
+      auto *Scatter = cast<MaskedScatterSDNode>(GorS);
+      SDValue Ops[] = { Chain, Scatter->getValue(),
+                        Mask, Base, Index, Scale };
+      return DAG.getMaskedScatter(Scatter->getVTList(),
+                                  Scatter->getMemoryVT(), DL,
+                                  Ops, Scatter->getMemOperand(),
+                                  Scatter->getIndexType());
+    }
+  }
+
+  // With vector masks we only demand the upper bit of the mask.
+  if (Mask.getScalarValueSizeInBits() != 1) {
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-    SDValue Mask = N->getOperand(2);
     APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
     if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
       return SDValue(N, 0);
@@ -43311,7 +42598,7 @@ static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
   // Make sure to not keep references to operands, as combineSetCCEFLAGS can
   // RAUW them under us.
   if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
-    SDValue Cond = DAG.getConstant(CC, DL, MVT::i8);
+    SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
                        N->getOperand(1), Cond, Flags);
   }
@@ -43730,12 +43017,12 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
     SDLoc DL(N);
     EVT VT = N->getValueType(0);
     SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
-    SDValue Res1 = DAG.getNode(ISD::AND, DL, VT,
-                               DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                                           DAG.getConstant(X86::COND_B, DL,
-                                                           MVT::i8),
-                                           N->getOperand(2)),
-                               DAG.getConstant(1, DL, VT));
+    SDValue Res1 =
+        DAG.getNode(ISD::AND, DL, VT,
+                    DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
+                                DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
+                                N->getOperand(2)),
+                    DAG.getConstant(1, DL, VT));
     return DCI.CombineTo(N, Res1, CarryOut);
   }
 
@@ -43795,7 +43082,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
       // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax
       //  0 - SETB  -->  0 -  (CF) --> CF ? -1 : 0 --> SBB %eax, %eax
       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                         DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                         DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
                          Y.getOperand(1));
     }
 
@@ -43813,7 +43100,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
             EFLAGS.getOperand(1), EFLAGS.getOperand(0));
         SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
         return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                           DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                           DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
                            NewEFLAGS);
       }
     }
@@ -43873,7 +43160,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
       SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
       SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                         DAG.getConstant(X86::COND_B, DL, MVT::i8),
+                         DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
                          SDValue(Neg.getNode(), 1));
     }
 
@@ -43886,7 +43173,7 @@ static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
       SDValue One = DAG.getConstant(1, DL, ZVT);
       SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
       return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
-                         DAG.getConstant(X86::COND_B, DL, MVT::i8), Cmp1);
+                         DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cmp1);
     }
   }
 
@@ -44026,11 +43313,7 @@ static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
   // result to v2i32 which will be removed by type legalization. If we/ widen
   // narrow vectors then we bitcast to v4i32 and extract v2i32.
   MVT ResVT = MVT::getVectorVT(MVT::i32, Sad.getValueSizeInBits() / 32);
-  if (ExperimentalVectorWideningLegalization ||
-      VT.getSizeInBits() >= ResVT.getSizeInBits())
-    Sad = DAG.getNode(ISD::BITCAST, DL, ResVT, Sad);
-  else
-    Sad = DAG.getNode(ISD::TRUNCATE, DL, VT, Sad);
+  Sad = DAG.getNode(ISD::BITCAST, DL, ResVT, Sad);
 
   if (VT.getSizeInBits() > ResVT.getSizeInBits()) {
     // Fill the upper elements with zero to match the add width.
@@ -44039,8 +43322,7 @@ static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
     SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, DL, ResVT));
     Ops[0] = Sad;
     Sad = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops);
-  } else if (ExperimentalVectorWideningLegalization &&
-             VT.getSizeInBits() < ResVT.getSizeInBits()) {
+  } else if (VT.getSizeInBits() < ResVT.getSizeInBits()) {
     Sad = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Sad,
                       DAG.getIntPtrConstant(0, DL));
   }
@@ -44314,6 +43596,30 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
                             HADDBuilder);
   }
 
+  // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
+  // (sub Y, (sext (vXi1 X))).
+  // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
+  // generic DAG combine without a legal type check, but adding this there
+  // caused regressions.
+  if (VT.isVector()) {
+    const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+    if (Op0.getOpcode() == ISD::ZERO_EXTEND &&
+        Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
+        TLI.isTypeLegal(Op0.getOperand(0).getValueType())) {
+      SDLoc DL(N);
+      SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
+      return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
+    }
+
+    if (Op1.getOpcode() == ISD::ZERO_EXTEND &&
+        Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
+        TLI.isTypeLegal(Op1.getOperand(0).getValueType())) {
+      SDLoc DL(N);
+      SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
+      return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
+    }
+  }
+
   return combineAddOrSubToADCOrSBB(N, DAG);
 }
 
@@ -44701,6 +44007,21 @@ static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
   if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
     return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
 
+  // If this is a broadcast load inserted into an upper undef, use a larger
+  // broadcast load.
+  if (Vec.isUndef() && IdxVal != 0 && SubVec.hasOneUse() &&
+      SubVec.getOpcode() == X86ISD::VBROADCAST_LOAD) {
+    auto *MemIntr = cast<MemIntrinsicSDNode>(SubVec);
+    SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
+    SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
+    SDValue BcastLd =
+        DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
+                                MemIntr->getMemoryVT(),
+                                MemIntr->getMemOperand());
+    DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
+    return BcastLd;
+  }
+
   return SDValue();
 }
 
@@ -44873,6 +44194,20 @@ static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
       InVec.getOperand(0).getValueSizeInBits() <= VT.getSizeInBits())
     return DAG.getNode(X86ISD::VBROADCAST, SDLoc(N), VT, InVec.getOperand(0));
 
+  if (InVec.getOpcode() == X86ISD::VBROADCAST_LOAD && InVec.hasOneUse()) {
+    auto *MemIntr = cast<MemIntrinsicSDNode>(InVec);
+    if (MemIntr->getMemoryVT().getSizeInBits() <= VT.getSizeInBits()) {
+      SDVTList Tys = DAG.getVTList(VT, MVT::Other);
+      SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
+      SDValue BcastLd =
+          DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
+                                  MemIntr->getMemoryVT(),
+                                  MemIntr->getMemOperand());
+      DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
+      return BcastLd;
+    }
+  }
+
   // If we're extracting the lowest subvector and we're the only user,
   // we may be able to perform this with a smaller vector width.
   if (IdxVal == 0 && InVec.hasOneUse()) {
@@ -45036,15 +44371,6 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
     }
   }
 
-  // Combine (ext_invec (ext_invec X)) -> (ext_invec X)
-  // Disabling for widening legalization for now. We can enable if we find a
-  // case that needs it. Otherwise it can be deleted when we switch to
-  // widening legalization.
-  if (!ExperimentalVectorWideningLegalization &&
-      In.getOpcode() == N->getOpcode() &&
-      TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getOperand(0).getValueType()))
-    return DAG.getNode(N->getOpcode(), SDLoc(N), VT, In.getOperand(0));
-
   // Attempt to combine as a shuffle.
   // TODO: SSE41 support
   if (Subtarget.hasAVX() && N->getOpcode() != ISD::SIGN_EXTEND_VECTOR_INREG) {
@@ -45057,6 +44383,20 @@ static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
+static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
+                             TargetLowering::DAGCombinerInfo &DCI) {
+  EVT VT = N->getValueType(0);
+
+  APInt KnownUndef, KnownZero;
+  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+  APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
+  if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
+                                     KnownZero, DCI))
+    return SDValue(N, 0);
+
+  return SDValue();
+}
+
 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -45190,13 +44530,15 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case X86ISD::FMSUBADD:    return combineFMADDSUB(N, DAG, Subtarget);
   case X86ISD::MOVMSK:      return combineMOVMSK(N, DAG, DCI, Subtarget);
   case X86ISD::MGATHER:
-  case X86ISD::MSCATTER:
+  case X86ISD::MSCATTER:    return combineX86GatherScatter(N, DAG, DCI);
   case ISD::MGATHER:
-  case ISD::MSCATTER:       return combineGatherScatter(N, DAG, DCI, Subtarget);
+  case ISD::MSCATTER:       return combineGatherScatter(N, DAG, DCI);
   case X86ISD::PCMPEQ:
   case X86ISD::PCMPGT:      return combineVectorCompare(N, DAG, Subtarget);
   case X86ISD::PMULDQ:
   case X86ISD::PMULUDQ:     return combinePMULDQ(N, DAG, DCI, Subtarget);
+  case X86ISD::KSHIFTL:
+  case X86ISD::KSHIFTR:     return combineKSHIFT(N, DAG, DCI);
   }
 
   return SDValue();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 61a47442a6753..5c967ca1eca39 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -425,7 +425,8 @@ namespace llvm {
       // Tests Types Of a FP Values for scalar types.
       VFPCLASSS,
 
-      // Broadcast scalar to vector.
+      // Broadcast (splat) scalar or element 0 of a vector. If the operand is
+      // a vector, this node may change the vector length as part of the splat.
       VBROADCAST,
       // Broadcast mask to vector.
       VBROADCASTM,
@@ -614,6 +615,9 @@ namespace llvm {
       // extract_vector_elt, store.
       VEXTRACT_STORE,
 
+      // scalar broadcast from memory
+      VBROADCAST_LOAD,
+
       // Store FP control world into i16 memory.
       FNSTCW16m,
 
@@ -1150,8 +1154,8 @@ namespace llvm {
       return nullptr; // nothing to do, move along.
     }
 
-    unsigned getRegisterByName(const char* RegName, EVT VT,
-                               SelectionDAG &DAG) const override;
+    Register getRegisterByName(const char* RegName, EVT VT,
+                               const MachineFunction &MF) const override;
 
     /// If a physical register, this returns the register that receives the
     /// exception address on entry to an EH pad.
@@ -1203,6 +1207,10 @@ namespace llvm {
                                            CallingConv::ID CC,
                                            EVT VT) const override;
 
+    unsigned getVectorTypeBreakdownForCallingConv(
+        LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
+        unsigned &NumIntermediates, MVT &RegisterVT) const override;
+
     bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
 
     bool supportSwiftError() const override;
diff --git a/llvm/lib/Target/X86/X86InsertPrefetch.cpp b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
index 70225f0f4b27b..2b1e3f23efd74 100644
--- a/llvm/lib/Target/X86/X86InsertPrefetch.cpp
+++ b/llvm/lib/Target/X86/X86InsertPrefetch.cpp
@@ -108,7 +108,7 @@ bool X86InsertPrefetch::findPrefetchInfo(const FunctionSamples *TopSamples,
                                          Prefetches &Prefetches) const {
   assert(Prefetches.empty() &&
          "Expected caller passed empty PrefetchInfo vector.");
-  static const std::pair<const StringRef, unsigned> HintTypes[] = {
+  static constexpr std::pair<StringLiteral, unsigned> HintTypes[] = {
       {"_nta_", X86::PREFETCHNTA},
       {"_t0_", X86::PREFETCHT0},
       {"_t1_", X86::PREFETCHT1},
@@ -173,7 +173,7 @@ bool X86InsertPrefetch::doInitialization(Module &M) {
 
 void X86InsertPrefetch::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.setPreservesAll();
-  AU.addRequired<MachineModuleInfo>();
+  AU.addRequired<MachineModuleInfoWrapperPass>();
 }
 
 bool X86InsertPrefetch::runOnMachineFunction(MachineFunction &MF) {
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index b8c936feac125..2d3b8a556816a 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -74,6 +74,7 @@ class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc,
   PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" # VTName);
 
   PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT);
+  PatFrag BroadcastLdFrag = !cast<PatFrag>("X86VBroadcastld" # EltSizeName);
 
   ComplexPattern ScalarIntMemCPat = !if (!eq (EltTypeName, "f32"),
                                           !cast<ComplexPattern>("sse_load_f32"),
@@ -753,14 +754,14 @@ let isCommutable = 1 in
 def VINSERTPSZrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
       (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),
       "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-      [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>,
+      [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, timm:$src3))]>,
       EVEX_4V, Sched<[SchedWriteFShuffle.XMM]>;
 def VINSERTPSZrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),
       (ins VR128X:$src1, f32mem:$src2, u8imm:$src3),
       "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
       [(set VR128X:$dst, (X86insertps VR128X:$src1,
                           (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
-                          imm:$src3))]>,
+                          timm:$src3))]>,
       EVEX_4V, EVEX_CD8<32, CD8VT1>,
       Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;
 }
@@ -1123,50 +1124,104 @@ multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
                                      X86VectorVTInfo MaskInfo,
                                      X86VectorVTInfo DestInfo,
                                      X86VectorVTInfo SrcInfo,
-                                     SDPatternOperator UnmaskedOp = X86VBroadcast> {
-  let ExeDomain = DestInfo.ExeDomain, hasSideEffects = 0 in {
-  defm r : AVX512_maskable_split<opc, MRMSrcReg, MaskInfo,
-                   (outs MaskInfo.RC:$dst),
-                   (ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src",
-                   (MaskInfo.VT
-                    (bitconvert
-                     (DestInfo.VT
-                      (UnmaskedOp (SrcInfo.VT SrcInfo.RC:$src))))),
-                   (MaskInfo.VT
-                    (bitconvert
-                     (DestInfo.VT
-                      (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))))>,
-                   T8PD, EVEX, Sched<[SchedRR]>;
-  let mayLoad = 1 in
-  defm m : AVX512_maskable_split<opc, MRMSrcMem, MaskInfo,
-                   (outs MaskInfo.RC:$dst),
-                   (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
-                   (MaskInfo.VT
-                    (bitconvert
-                     (DestInfo.VT (UnmaskedOp
-                                   (SrcInfo.ScalarLdFrag addr:$src))))),
-                   (MaskInfo.VT
-                    (bitconvert
-                     (DestInfo.VT (X86VBroadcast
-                                   (SrcInfo.ScalarLdFrag addr:$src)))))>,
-                   T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>,
-                   Sched<[SchedRM]>;
-  }
+                                     bit IsConvertibleToThreeAddress,
+                                     SDPatternOperator UnmaskedOp = X86VBroadcast,
+                                     SDPatternOperator UnmaskedBcastOp = SrcInfo.BroadcastLdFrag> {
+  let hasSideEffects = 0 in
+  def r : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst), (ins SrcInfo.RC:$src),
+                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
+                   [(set MaskInfo.RC:$dst,
+                     (MaskInfo.VT
+                      (bitconvert
+                       (DestInfo.VT
+                        (UnmaskedOp (SrcInfo.VT SrcInfo.RC:$src))))))],
+                   DestInfo.ExeDomain>, T8PD, EVEX, Sched<[SchedRR]>;
+  def rkz : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst),
+                     (ins MaskInfo.KRCWM:$mask, SrcInfo.RC:$src),
+                     !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
+                      "${dst} {${mask}} {z}, $src}"),
+                      [(set MaskInfo.RC:$dst,
+                        (vselect MaskInfo.KRCWM:$mask,
+                         (MaskInfo.VT
+                          (bitconvert
+                           (DestInfo.VT
+                            (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))))),
+                         MaskInfo.ImmAllZerosV))],
+                      DestInfo.ExeDomain>, T8PD, EVEX, EVEX_KZ, Sched<[SchedRR]>;
+  let Constraints = "$src0 = $dst" in
+  def rk : AVX512PI<opc, MRMSrcReg, (outs MaskInfo.RC:$dst),
+                    (ins MaskInfo.RC:$src0, MaskInfo.KRCWM:$mask,
+                         SrcInfo.RC:$src),
+                    !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",
+                    "${dst} {${mask}}, $src}"),
+                    [(set MaskInfo.RC:$dst,
+                      (vselect MaskInfo.KRCWM:$mask,
+                       (MaskInfo.VT
+                        (bitconvert
+                         (DestInfo.VT
+                          (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))))),
+                       MaskInfo.RC:$src0))],
+                     DestInfo.ExeDomain>, T8PD, EVEX, EVEX_K, Sched<[SchedRR]>;
+
+  let hasSideEffects = 0, mayLoad = 1 in
+  def m : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),
+                   (ins SrcInfo.ScalarMemOp:$src),
+                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
+                   [(set MaskInfo.RC:$dst,
+                     (MaskInfo.VT
+                      (bitconvert
+                       (DestInfo.VT
+                        (UnmaskedBcastOp addr:$src)))))],
+                   DestInfo.ExeDomain>, T8PD, EVEX,
+                   EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;
+
+  def mkz : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),
+                     (ins MaskInfo.KRCWM:$mask, SrcInfo.ScalarMemOp:$src),
+                     !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
+                      "${dst} {${mask}} {z}, $src}"),
+                      [(set MaskInfo.RC:$dst,
+                        (vselect MaskInfo.KRCWM:$mask,
+                         (MaskInfo.VT
+                          (bitconvert
+                           (DestInfo.VT
+                            (SrcInfo.BroadcastLdFrag addr:$src)))),
+                         MaskInfo.ImmAllZerosV))],
+                      DestInfo.ExeDomain>, T8PD, EVEX, EVEX_KZ,
+                      EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;
+
+  let Constraints = "$src0 = $dst",
+      isConvertibleToThreeAddress = IsConvertibleToThreeAddress in
+  def mk : AVX512PI<opc, MRMSrcMem, (outs MaskInfo.RC:$dst),
+                    (ins MaskInfo.RC:$src0, MaskInfo.KRCWM:$mask,
+                         SrcInfo.ScalarMemOp:$src),
+                    !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}}|",
+                    "${dst} {${mask}}, $src}"),
+                    [(set MaskInfo.RC:$dst,
+                      (vselect MaskInfo.KRCWM:$mask,
+                       (MaskInfo.VT
+                        (bitconvert
+                         (DestInfo.VT
+                          (SrcInfo.BroadcastLdFrag addr:$src)))),
+                       MaskInfo.RC:$src0))],
+                     DestInfo.ExeDomain>, T8PD, EVEX, EVEX_K,
+                     EVEX_CD8<SrcInfo.EltSize, CD8VT1>, Sched<[SchedRM]>;
 }
 
 // Helper class to force mask and broadcast result to same type.
 multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr, string Name,
                                SchedWrite SchedRR, SchedWrite SchedRM,
                                X86VectorVTInfo DestInfo,
-                               X86VectorVTInfo SrcInfo> :
+                               X86VectorVTInfo SrcInfo,
+                               bit IsConvertibleToThreeAddress> :
   avx512_broadcast_rm_split<opc, OpcodeStr, Name, SchedRR, SchedRM,
-                            DestInfo, DestInfo, SrcInfo>;
+                            DestInfo, DestInfo, SrcInfo,
+                            IsConvertibleToThreeAddress>;
 
 multiclass avx512_fp_broadcast_sd<bits<8> opc, string OpcodeStr,
                                                        AVX512VLVectorVTInfo _> {
   let Predicates = [HasAVX512] in {
     defm Z  : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
-                                  WriteFShuffle256Ld, _.info512, _.info128>,
+                                  WriteFShuffle256Ld, _.info512, _.info128, 1>,
               avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info512,
                                       _.info128>,
               EVEX_V512;
@@ -1174,7 +1229,7 @@ multiclass avx512_fp_broadcast_sd<bits<8> opc, string OpcodeStr,
 
   let Predicates = [HasVLX] in {
     defm Z256  : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
-                                     WriteFShuffle256Ld, _.info256, _.info128>,
+                                     WriteFShuffle256Ld, _.info256, _.info128, 1>,
                  avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info256,
                                          _.info128>,
                  EVEX_V256;
@@ -1185,7 +1240,7 @@ multiclass avx512_fp_broadcast_ss<bits<8> opc, string OpcodeStr,
                                                        AVX512VLVectorVTInfo _> {
   let Predicates = [HasAVX512] in {
     defm Z  : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
-                                  WriteFShuffle256Ld, _.info512, _.info128>,
+                                  WriteFShuffle256Ld, _.info512, _.info128, 1>,
               avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info512,
                                       _.info128>,
               EVEX_V512;
@@ -1193,12 +1248,12 @@ multiclass avx512_fp_broadcast_ss<bits<8> opc, string OpcodeStr,
 
   let Predicates = [HasVLX] in {
     defm Z256  : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
-                                     WriteFShuffle256Ld, _.info256, _.info128>,
+                                     WriteFShuffle256Ld, _.info256, _.info128, 1>,
                  avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info256,
                                          _.info128>,
                  EVEX_V256;
     defm Z128  : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
-                                     WriteFShuffle256Ld, _.info128, _.info128>,
+                                     WriteFShuffle256Ld, _.info128, _.info128, 1>,
                  avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info128,
                                          _.info128>,
                  EVEX_V128;
@@ -1283,30 +1338,34 @@ defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info,
                                                  X86VBroadcast, GR64, HasAVX512>, VEX_W;
 
 multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,
-                                        AVX512VLVectorVTInfo _, Predicate prd> {
+                                        AVX512VLVectorVTInfo _, Predicate prd,
+                                        bit IsConvertibleToThreeAddress> {
   let Predicates = [prd] in {
     defm Z :   avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle256,
-                                   WriteShuffle256Ld, _.info512, _.info128>,
+                                   WriteShuffle256Ld, _.info512, _.info128,
+                                   IsConvertibleToThreeAddress>,
                                   EVEX_V512;
   }
   let Predicates = [prd, HasVLX] in {
     defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle256,
-                                    WriteShuffle256Ld, _.info256, _.info128>,
+                                    WriteShuffle256Ld, _.info256, _.info128,
+                                    IsConvertibleToThreeAddress>,
                                  EVEX_V256;
     defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle,
-                                    WriteShuffleXLd, _.info128, _.info128>,
+                                    WriteShuffleXLd, _.info128, _.info128,
+                                    IsConvertibleToThreeAddress>,
                                  EVEX_V128;
   }
 }
 
 defm VPBROADCASTB  : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb",
-                                           avx512vl_i8_info, HasBWI>;
+                                           avx512vl_i8_info, HasBWI, 0>;
 defm VPBROADCASTW  : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw",
-                                           avx512vl_i16_info, HasBWI>;
+                                           avx512vl_i16_info, HasBWI, 0>;
 defm VPBROADCASTD  : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd",
-                                           avx512vl_i32_info, HasAVX512>;
+                                           avx512vl_i32_info, HasAVX512, 1>;
 defm VPBROADCASTQ  : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq",
-                                           avx512vl_i64_info, HasAVX512>, VEX_W1X;
+                                           avx512vl_i64_info, HasAVX512, 1>, VEX_W1X;
 
 multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
                           X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
@@ -1337,6 +1396,10 @@ let Predicates = [HasAVX512] in {
   // 32-bit targets will fail to load a i64 directly but can use ZEXT_LOAD.
   def : Pat<(v8i64 (X86VBroadcast (v2i64 (X86vzload64 addr:$src)))),
             (VPBROADCASTQZm addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8.
+  def : Pat<(v16i32 (X86VBroadcast (loadi32 addr:$src))),
+            (VPBROADCASTDZm addr:$src)>;
 }
 
 let Predicates = [HasVLX] in {
@@ -1345,6 +1408,12 @@ let Predicates = [HasVLX] in {
             (VPBROADCASTQZ128m addr:$src)>;
   def : Pat<(v4i64 (X86VBroadcast (v2i64 (X86vzload64 addr:$src)))),
             (VPBROADCASTQZ256m addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8.
+  def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
+            (VPBROADCASTDZ128m addr:$src)>;
+  def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))),
+            (VPBROADCASTDZ256m addr:$src)>;
 }
 let Predicates = [HasVLX, HasBWI] in {
   // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably.
@@ -1365,6 +1434,12 @@ let Predicates = [HasVLX, HasBWI] in {
   def : Pat<(v16i16 (X86VBroadcast
               (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
             (VPBROADCASTWZ256m addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8.
+  def : Pat<(v8i16 (X86VBroadcast (loadi16 addr:$src))),
+            (VPBROADCASTWZ128m addr:$src)>;
+  def : Pat<(v16i16 (X86VBroadcast (loadi16 addr:$src))),
+            (VPBROADCASTWZ256m addr:$src)>;
 }
 let Predicates = [HasBWI] in {
   // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably.
@@ -1377,6 +1452,10 @@ let Predicates = [HasBWI] in {
   def : Pat<(v32i16 (X86VBroadcast
               (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
             (VPBROADCASTWZm addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8.
+  def : Pat<(v32i16 (X86VBroadcast (loadi16 addr:$src))),
+            (VPBROADCASTWZm addr:$src)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1612,12 +1691,12 @@ multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,
   let Predicates = [HasDQI] in
     defm Z :    avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle256,
                                           WriteShuffle256Ld, _Dst.info512,
-                                          _Src.info512, _Src.info128, null_frag>,
+                                          _Src.info512, _Src.info128, 0, null_frag, null_frag>,
                                           EVEX_V512;
   let Predicates = [HasDQI, HasVLX] in
     defm Z256 : avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle256,
                                           WriteShuffle256Ld, _Dst.info256,
-                                          _Src.info256, _Src.info128, null_frag>,
+                                          _Src.info256, _Src.info128, 0, null_frag, null_frag>,
                                           EVEX_V256;
 }
 
@@ -1628,7 +1707,7 @@ multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,
   let Predicates = [HasDQI, HasVLX] in
     defm Z128 : avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle,
                                           WriteShuffleXLd, _Dst.info128,
-                                          _Src.info128, _Src.info128, null_frag>,
+                                          _Src.info128, _Src.info128, 0, null_frag, null_frag>,
                                           EVEX_V128;
 }
 
@@ -1696,7 +1775,7 @@ multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,
               OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
               !strconcat("$src2, ${src3}", _.BroadcastStr ),
               (_.VT (X86VPermt2 _.RC:$src2,
-               IdxVT.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))), 1>,
+               IdxVT.RC:$src1,(_.VT (_.BroadcastLdFrag addr:$src3)))), 1>,
               AVX5128IBase, EVEX_4V, EVEX_B,
               Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -1773,7 +1852,7 @@ multiclass avx512_perm_i_lowering<string InstrStr, X86VectorVTInfo _,
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                              (X86VPermt2 _.RC:$src2,
                                          (IdxVT.VT (bitconvert  (CastVT.VT _.RC:$src1))),
-                                         (X86VBroadcast (_.ScalarLdFrag addr:$src3))),
+                                         (_.BroadcastLdFrag addr:$src3)),
                              (_.VT (bitconvert  (CastVT.VT _.RC:$src1))))),
             (!cast<Instruction>(InstrStr#"rmbk") _.RC:$src1, _.KRCWM:$mask,
                                                  _.RC:$src2, addr:$src3)>;
@@ -1812,7 +1891,7 @@ multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,
               OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
               !strconcat("$src2, ${src3}", _.BroadcastStr ),
               (_.VT (X86VPermt2 _.RC:$src1,
-               IdxVT.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))), 1>,
+               IdxVT.RC:$src2,(_.VT (_.BroadcastLdFrag addr:$src3)))), 1>,
               AVX5128IBase, EVEX_4V, EVEX_B,
               Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -1913,7 +1992,7 @@ multiclass WriteFVarBlendask<bits<8> opc, string OpcodeStr,
 }
 multiclass WriteFVarBlendask_rmb<bits<8> opc, string OpcodeStr,
                                  X86FoldableSchedWrite sched, X86VectorVTInfo _> {
-  let mayLoad = 1, hasSideEffects = 0 in {
+  let ExeDomain = _.ExeDomain, mayLoad = 1, hasSideEffects = 0 in {
   def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
       (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),
        !strconcat(OpcodeStr,
@@ -1997,9 +2076,9 @@ multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
                       (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
                       "vcmp"#_.Suffix,
                       "$cc, $src2, $src1", "$src1, $src2, $cc",
-                      (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc),
+                      (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),
                       (OpNode_su (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                                 imm:$cc)>, EVEX_4V, VEX_LIG, Sched<[sched]>;
+                                 timm:$cc)>, EVEX_4V, VEX_LIG, Sched<[sched]>;
   let mayLoad = 1 in
   defm  rm_Int  : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
                     (outs _.KRC:$dst),
@@ -2007,9 +2086,9 @@ multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
                     "vcmp"#_.Suffix,
                     "$cc, $src2, $src1", "$src1, $src2, $cc",
                     (OpNode (_.VT _.RC:$src1), _.ScalarIntMemCPat:$src2,
-                        imm:$cc),
+                        timm:$cc),
                     (OpNode_su (_.VT _.RC:$src1), _.ScalarIntMemCPat:$src2,
-                        imm:$cc)>, EVEX_4V, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>,
+                        timm:$cc)>, EVEX_4V, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
 
   defm  rrb_Int  : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
@@ -2018,9 +2097,9 @@ multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
                      "vcmp"#_.Suffix,
                      "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc",
                      (OpNodeSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                                imm:$cc),
+                                timm:$cc),
                      (OpNodeSAE_su (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                                   imm:$cc)>,
+                                   timm:$cc)>,
                      EVEX_4V, VEX_LIG, EVEX_B, Sched<[sched]>;
 
   let isCodeGenOnly = 1 in {
@@ -2031,7 +2110,7 @@ multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
                            "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
                 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
                                           _.FRC:$src2,
-                                          imm:$cc))]>,
+                                          timm:$cc))]>,
                 EVEX_4V, VEX_LIG, Sched<[sched]>;
     def rm : AVX512Ii8<0xC2, MRMSrcMem,
               (outs _.KRC:$dst),
@@ -2040,7 +2119,7 @@ multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeSAE,
                          "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
               [(set _.KRC:$dst, (OpNode _.FRC:$src1,
                                         (_.ScalarLdFrag addr:$src2),
-                                        imm:$cc))]>,
+                                        timm:$cc))]>,
               EVEX_4V, VEX_LIG, EVEX_CD8<_.EltSize, CD8VT1>,
               Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
@@ -2109,7 +2188,7 @@ multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, PatFrag OpNode,
               !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst",
                                     "|$dst, $src1, ${src2}", _.BroadcastStr, "}"),
               [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
-                              (X86VBroadcast (_.ScalarLdFrag addr:$src2))))]>,
+                              (_.BroadcastLdFrag addr:$src2)))]>,
               EVEX_4V, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   def rmbk : AVX512BI<opc, MRMSrcMem,
                (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
@@ -2119,8 +2198,7 @@ multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, PatFrag OpNode,
                           "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
                [(set _.KRC:$dst, (and _.KRCWM:$mask,
                                       (OpNode_su (_.VT _.RC:$src1),
-                                        (X86VBroadcast
-                                          (_.ScalarLdFrag addr:$src2)))))]>,
+                                        (_.BroadcastLdFrag addr:$src2))))]>,
                EVEX_4V, EVEX_K, EVEX_B,
                Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -2288,8 +2366,7 @@ multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, PatFrag Frag,
                         "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"),
              [(set _.KRC:$dst, (_.KVT (Frag:$cc
                                        (_.VT _.RC:$src1),
-                                       (X86VBroadcast
-                                        (_.ScalarLdFrag addr:$src2)),
+                                       (_.BroadcastLdFrag addr:$src2),
                                        cond)))]>,
              EVEX_4V, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   def rmibk : AVX512AIi8<opc, MRMSrcMem,
@@ -2301,23 +2378,21 @@ multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, PatFrag Frag,
               [(set _.KRC:$dst, (and _.KRCWM:$mask,
                                      (_.KVT (Frag_su:$cc
                                              (_.VT _.RC:$src1),
-                                             (X86VBroadcast
-                                              (_.ScalarLdFrag addr:$src2)),
+                                             (_.BroadcastLdFrag addr:$src2),
                                              cond))))]>,
               EVEX_4V, EVEX_K, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
 
-  def : Pat<(_.KVT (CommFrag:$cc (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
+  def : Pat<(_.KVT (CommFrag:$cc (_.BroadcastLdFrag addr:$src2),
                     (_.VT _.RC:$src1), cond)),
             (!cast<Instruction>(Name#_.ZSuffix#"rmib")
              _.RC:$src1, addr:$src2, (CommFrag.OperandTransform $cc))>;
 
   def : Pat<(and _.KRCWM:$mask,
-                 (_.KVT (CommFrag_su:$cc (X86VBroadcast
-                                       (_.ScalarLdFrag addr:$src2)),
+                 (_.KVT (CommFrag_su:$cc (_.BroadcastLdFrag addr:$src2),
                                       (_.VT _.RC:$src1), cond))),
             (!cast<Instruction>(Name#_.ZSuffix#"rmibk")
              _.KRCWM:$mask, _.RC:$src1, addr:$src2,
-             (CommFrag.OperandTransform $cc))>;
+             (CommFrag_su.OperandTransform $cc))>;
 }
 
 multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, PatFrag Frag,
@@ -2462,14 +2537,19 @@ def X86cmpmSAE_su : PatFrag<(ops node:$src1, node:$src2, node:$cc),
   return N->hasOneUse();
 }]>;
 
+def X86cmpm_imm_commute : SDNodeXForm<timm, [{
+  uint8_t Imm = X86::getSwappedVCMPImm(N->getZExtValue() & 0x1f);
+  return getI8Imm(Imm, SDLoc(N));
+}]>;
+
 multiclass avx512_vcmp_common<X86FoldableSchedWrite sched, X86VectorVTInfo _,
                               string Name> {
   defm  rri  : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
                    (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,u8imm:$cc),
                    "vcmp"#_.Suffix,
                    "$cc, $src2, $src1", "$src1, $src2, $cc",
-                   (X86cmpm (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc),
-                   (X86cmpm_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc),
+                   (X86cmpm (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),
+                   (X86cmpm_su (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),
                    1>, Sched<[sched]>;
 
   defm  rmi  : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
@@ -2477,9 +2557,9 @@ multiclass avx512_vcmp_common<X86FoldableSchedWrite sched, X86VectorVTInfo _,
                 "vcmp"#_.Suffix,
                 "$cc, $src2, $src1", "$src1, $src2, $cc",
                 (X86cmpm (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)),
-                         imm:$cc),
+                         timm:$cc),
                 (X86cmpm_su (_.VT _.RC:$src1), (_.VT (_.LdFrag addr:$src2)),
-                            imm:$cc)>,
+                            timm:$cc)>,
                 Sched<[sched.Folded, sched.ReadAfterFold]>;
 
   defm  rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
@@ -2489,38 +2569,37 @@ multiclass avx512_vcmp_common<X86FoldableSchedWrite sched, X86VectorVTInfo _,
                 "$cc, ${src2}"#_.BroadcastStr#", $src1",
                 "$src1, ${src2}"#_.BroadcastStr#", $cc",
                 (X86cmpm (_.VT _.RC:$src1),
-                        (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
-                        imm:$cc),
+                        (_.VT (_.BroadcastLdFrag addr:$src2)),
+                        timm:$cc),
                 (X86cmpm_su (_.VT _.RC:$src1),
-                            (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
-                            imm:$cc)>,
+                            (_.VT (_.BroadcastLdFrag addr:$src2)),
+                            timm:$cc)>,
                 EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
 
   // Patterns for selecting with loads in other operand.
   def : Pat<(X86cmpm (_.LdFrag addr:$src2), (_.VT _.RC:$src1),
-                     CommutableCMPCC:$cc),
+                     timm:$cc),
             (!cast<Instruction>(Name#_.ZSuffix#"rmi") _.RC:$src1, addr:$src2,
-                                                      imm:$cc)>;
+                                                      (X86cmpm_imm_commute timm:$cc))>;
 
   def : Pat<(and _.KRCWM:$mask, (X86cmpm_su (_.LdFrag addr:$src2),
                                             (_.VT _.RC:$src1),
-                                            CommutableCMPCC:$cc)),
+                                            timm:$cc)),
             (!cast<Instruction>(Name#_.ZSuffix#"rmik") _.KRCWM:$mask,
                                                        _.RC:$src1, addr:$src2,
-                                                       imm:$cc)>;
+                                                       (X86cmpm_imm_commute timm:$cc))>;
 
-  def : Pat<(X86cmpm (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
-                     (_.VT _.RC:$src1), CommutableCMPCC:$cc),
+  def : Pat<(X86cmpm (_.BroadcastLdFrag addr:$src2),
+                     (_.VT _.RC:$src1), timm:$cc),
             (!cast<Instruction>(Name#_.ZSuffix#"rmbi") _.RC:$src1, addr:$src2,
-                                                       imm:$cc)>;
+                                                       (X86cmpm_imm_commute timm:$cc))>;
 
-  def : Pat<(and _.KRCWM:$mask, (X86cmpm_su (X86VBroadcast
-                                             (_.ScalarLdFrag addr:$src2)),
+  def : Pat<(and _.KRCWM:$mask, (X86cmpm_su (_.BroadcastLdFrag addr:$src2),
                                             (_.VT _.RC:$src1),
-                                            CommutableCMPCC:$cc)),
+                                            timm:$cc)),
             (!cast<Instruction>(Name#_.ZSuffix#"rmbik") _.KRCWM:$mask,
                                                         _.RC:$src1, addr:$src2,
-                                                        imm:$cc)>;
+                                                        (X86cmpm_imm_commute timm:$cc))>;
 }
 
 multiclass avx512_vcmp_sae<X86FoldableSchedWrite sched, X86VectorVTInfo _> {
@@ -2530,9 +2609,9 @@ multiclass avx512_vcmp_sae<X86FoldableSchedWrite sched, X86VectorVTInfo _> {
                      "vcmp"#_.Suffix,
                      "$cc, {sae}, $src2, $src1",
                      "$src1, $src2, {sae}, $cc",
-                     (X86cmpmSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), imm:$cc),
+                     (X86cmpmSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2), timm:$cc),
                      (X86cmpmSAE_su (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                                    imm:$cc)>,
+                                    timm:$cc)>,
                      EVEX_B, Sched<[sched]>;
 }
 
@@ -2556,12 +2635,12 @@ defm VCMPPS : avx512_vcmp<SchedWriteFCmp, avx512vl_f32_info>,
 // Patterns to select fp compares with load as first operand.
 let Predicates = [HasAVX512] in {
   def : Pat<(v1i1 (X86cmpms (loadf64 addr:$src2), FR64X:$src1,
-                            CommutableCMPCC:$cc)),
-            (VCMPSDZrm FR64X:$src1, addr:$src2, imm:$cc)>;
+                            timm:$cc)),
+            (VCMPSDZrm FR64X:$src1, addr:$src2, (X86cmpm_imm_commute timm:$cc))>;
 
   def : Pat<(v1i1 (X86cmpms (loadf32 addr:$src2), FR32X:$src1,
-                            CommutableCMPCC:$cc)),
-            (VCMPSSZrm FR32X:$src1, addr:$src2, imm:$cc)>;
+                            timm:$cc)),
+            (VCMPSSZrm FR32X:$src1, addr:$src2, (X86cmpm_imm_commute timm:$cc))>;
 }
 
 // ----------------------------------------------------------------
@@ -2587,7 +2666,7 @@ multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr,
                       (ins _.RC:$src1, i32u8imm:$src2),
                       OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                       [(set _.KRC:$dst,(X86Vfpclasss (_.VT _.RC:$src1),
-                              (i32 imm:$src2)))]>,
+                              (i32 timm:$src2)))]>,
                       Sched<[sched]>;
       def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
                       (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
@@ -2595,7 +2674,7 @@ multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr,
                       "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
                       [(set _.KRC:$dst,(and _.KRCWM:$mask,
                                       (X86Vfpclasss_su (_.VT _.RC:$src1),
-                                      (i32 imm:$src2))))]>,
+                                      (i32 timm:$src2))))]>,
                       EVEX_K, Sched<[sched]>;
     def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.IntScalarMemOp:$src1, i32u8imm:$src2),
@@ -2603,7 +2682,7 @@ multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr,
                               "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set _.KRC:$dst,
                           (X86Vfpclasss _.ScalarIntMemCPat:$src1,
-                                       (i32 imm:$src2)))]>,
+                                       (i32 timm:$src2)))]>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
     def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.KRCWM:$mask, _.IntScalarMemOp:$src1, i32u8imm:$src2),
@@ -2611,7 +2690,7 @@ multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr,
                     "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
                     [(set _.KRC:$dst,(and _.KRCWM:$mask,
                         (X86Vfpclasss_su _.ScalarIntMemCPat:$src1,
-                            (i32 imm:$src2))))]>,
+                            (i32 timm:$src2))))]>,
                     EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -2627,7 +2706,7 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                       (ins _.RC:$src1, i32u8imm:$src2),
                       OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                       [(set _.KRC:$dst,(X86Vfpclass (_.VT _.RC:$src1),
-                                       (i32 imm:$src2)))]>,
+                                       (i32 timm:$src2)))]>,
                       Sched<[sched]>;
   def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
                       (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
@@ -2635,7 +2714,7 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                       "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
                       [(set _.KRC:$dst,(and _.KRCWM:$mask,
                                        (X86Vfpclass_su (_.VT _.RC:$src1),
-                                       (i32 imm:$src2))))]>,
+                                       (i32 timm:$src2))))]>,
                       EVEX_K, Sched<[sched]>;
   def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.MemOp:$src1, i32u8imm:$src2),
@@ -2643,7 +2722,7 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                     "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set _.KRC:$dst,(X86Vfpclass
                                      (_.VT (_.LdFrag addr:$src1)),
-                                     (i32 imm:$src2)))]>,
+                                     (i32 timm:$src2)))]>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
@@ -2651,7 +2730,7 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                     "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
                     [(set _.KRC:$dst, (and _.KRCWM:$mask, (X86Vfpclass_su
                                   (_.VT (_.LdFrag addr:$src1)),
-                                  (i32 imm:$src2))))]>,
+                                  (i32 timm:$src2))))]>,
                     EVEX_K, Sched<[sched.Folded, sched.ReadAfterFold]>;
   def rmb : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
@@ -2659,9 +2738,8 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                                       _.BroadcastStr##", $dst|$dst, ${src1}"
                                                   ##_.BroadcastStr##", $src2}",
                     [(set _.KRC:$dst,(X86Vfpclass
-                                     (_.VT (X86VBroadcast
-                                           (_.ScalarLdFrag addr:$src1))),
-                                     (i32 imm:$src2)))]>,
+                                     (_.VT (_.BroadcastLdFrag addr:$src1)),
+                                     (i32 timm:$src2)))]>,
                     EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   def rmbk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
                     (ins _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2),
@@ -2669,9 +2747,8 @@ multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr,
                           _.BroadcastStr##", $dst {${mask}}|$dst {${mask}}, ${src1}"##
                                                    _.BroadcastStr##", $src2}",
                     [(set _.KRC:$dst,(and _.KRCWM:$mask, (X86Vfpclass_su
-                                     (_.VT (X86VBroadcast
-                                           (_.ScalarLdFrag addr:$src1))),
-                                     (i32 imm:$src2))))]>,
+                                     (_.VT (_.BroadcastLdFrag addr:$src1)),
+                                     (i32 timm:$src2))))]>,
                     EVEX_B, EVEX_K,  Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 
@@ -3049,7 +3126,7 @@ multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
     def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
                  !strconcat(OpcodeStr,
                             "\t{$imm, $src, $dst|$dst, $src, $imm}"),
-                            [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>,
+                            [(set KRC:$dst, (OpNode KRC:$src, (i8 timm:$imm)))]>,
                  Sched<[sched]>;
 }
 
@@ -3095,6 +3172,30 @@ multiclass axv512_icmp_packed_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,
            Narrow.KRC)>;
 }
 
+multiclass axv512_icmp_packed_rmb_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,
+                                                  string InstStr,
+                                                  X86VectorVTInfo Narrow,
+                                                  X86VectorVTInfo Wide> {
+  // Broadcast load.
+  def : Pat<(Narrow.KVT (Frag (Narrow.VT Narrow.RC:$src1),
+                              (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)))),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmb")
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2),
+           Narrow.KRC)>;
+
+  def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
+                             (Frag_su (Narrow.VT Narrow.RC:$src1),
+                                      (Narrow.BroadcastLdFrag addr:$src2)))),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmbk")
+            (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2),
+           Narrow.KRC)>;
+}
+
 // Patterns for comparing 128/256-bit integer vectors using 512-bit instruction.
 multiclass axv512_icmp_packed_cc_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,
                                                  string InstStr,
@@ -3103,7 +3204,7 @@ multiclass axv512_icmp_packed_cc_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,
 def : Pat<(Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),
                                 (Narrow.VT Narrow.RC:$src2), cond)),
           (COPY_TO_REGCLASS
-           (!cast<Instruction>(InstStr##Zrri)
+           (!cast<Instruction>(InstStr#"Zrri")
             (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
             (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
             (Frag.OperandTransform $cc)), Narrow.KRC)>;
@@ -3112,34 +3213,108 @@ def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
                            (Narrow.KVT (Frag_su:$cc (Narrow.VT Narrow.RC:$src1),
                                                     (Narrow.VT Narrow.RC:$src2),
                                                     cond)))),
-          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrrik)
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrrik")
            (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
-           (Frag.OperandTransform $cc)), Narrow.KRC)>;
+           (Frag_su.OperandTransform $cc)), Narrow.KRC)>;
+}
+
+multiclass axv512_icmp_packed_cc_rmb_no_vlx_lowering<PatFrag Frag, PatFrag Frag_su,
+                                                     PatFrag CommFrag, PatFrag CommFrag_su,
+                                                     string InstStr,
+                                                     X86VectorVTInfo Narrow,
+                                                     X86VectorVTInfo Wide> {
+// Broadcast load.
+def : Pat<(Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),
+                                (Narrow.BroadcastLdFrag addr:$src2), cond)),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmib")
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2, (Frag.OperandTransform $cc)), Narrow.KRC)>;
+
+def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
+                           (Narrow.KVT
+                            (Frag_su:$cc (Narrow.VT Narrow.RC:$src1),
+                                         (Narrow.BroadcastLdFrag addr:$src2),
+                                         cond)))),
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmibk")
+           (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
+           (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+           addr:$src2, (Frag_su.OperandTransform $cc)), Narrow.KRC)>;
+
+// Commuted with broadcast load.
+def : Pat<(Narrow.KVT (CommFrag:$cc (Narrow.BroadcastLdFrag addr:$src2),
+                                    (Narrow.VT Narrow.RC:$src1),
+                                    cond)),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmib")
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2, (CommFrag.OperandTransform $cc)), Narrow.KRC)>;
+
+def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
+                           (Narrow.KVT
+                            (CommFrag_su:$cc (Narrow.BroadcastLdFrag addr:$src2),
+                                             (Narrow.VT Narrow.RC:$src1), 
+                                             cond)))),
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmibk")
+           (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
+           (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+           addr:$src2, (CommFrag_su.OperandTransform $cc)), Narrow.KRC)>;
 }
 
 // Same as above, but for fp types which don't use PatFrags.
-multiclass axv512_cmp_packed_cc_no_vlx_lowering<SDNode OpNode, PatFrag OpNode_su,
-                                                string InstStr,
+multiclass axv512_cmp_packed_cc_no_vlx_lowering<string InstStr,
                                                 X86VectorVTInfo Narrow,
                                                 X86VectorVTInfo Wide> {
-def : Pat<(Narrow.KVT (OpNode (Narrow.VT Narrow.RC:$src1),
-                              (Narrow.VT Narrow.RC:$src2), imm:$cc)),
+def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT Narrow.RC:$src1),
+                               (Narrow.VT Narrow.RC:$src2), timm:$cc)),
           (COPY_TO_REGCLASS
-           (!cast<Instruction>(InstStr##Zrri)
+           (!cast<Instruction>(InstStr#"Zrri")
             (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
             (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
-            imm:$cc), Narrow.KRC)>;
+            timm:$cc), Narrow.KRC)>;
 
 def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
-                           (OpNode_su (Narrow.VT Narrow.RC:$src1),
-                                      (Narrow.VT Narrow.RC:$src2), imm:$cc))),
-          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrrik)
+                           (X86cmpm_su (Narrow.VT Narrow.RC:$src1),
+                                       (Narrow.VT Narrow.RC:$src2), timm:$cc))),
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrrik")
            (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
-           imm:$cc), Narrow.KRC)>;
+           timm:$cc), Narrow.KRC)>;
+
+// Broadcast load.
+def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT Narrow.RC:$src1),
+                               (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)), timm:$cc)),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmbi")
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2, timm:$cc), Narrow.KRC)>;
+
+def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
+                           (X86cmpm_su (Narrow.VT Narrow.RC:$src1),
+                                       (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)), timm:$cc))),
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")
+           (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
+           (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+           addr:$src2, timm:$cc), Narrow.KRC)>;
+
+// Commuted with broadcast load.
+def : Pat<(Narrow.KVT (X86cmpm (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)),
+                               (Narrow.VT Narrow.RC:$src1), timm:$cc)),
+          (COPY_TO_REGCLASS
+           (!cast<Instruction>(InstStr#"Zrmbi")
+            (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+            addr:$src2, (X86cmpm_imm_commute timm:$cc)), Narrow.KRC)>;
+
+def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
+                           (X86cmpm_su (Narrow.VT (Narrow.BroadcastLdFrag addr:$src2)),
+                                       (Narrow.VT Narrow.RC:$src1), timm:$cc))),
+          (COPY_TO_REGCLASS (!cast<Instruction>(InstStr#"Zrmbik")
+           (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
+           (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
+           addr:$src2, (X86cmpm_imm_commute timm:$cc)), Narrow.KRC)>;
 }
 
 let Predicates = [HasAVX512, NoVLX] in {
@@ -3157,6 +3332,18 @@ let Predicates = [HasAVX512, NoVLX] in {
 
   defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, X86pcmpgtm_su, "VPCMPGTQ", v2i64x_info, v8i64_info>;
   defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, X86pcmpeqm_c_su, "VPCMPEQQ", v2i64x_info, v8i64_info>;
+
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpgtm, X86pcmpgtm_su, "VPCMPGTD", v8i32x_info, v16i32_info>;
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpeqm_c, X86pcmpeqm_c_su, "VPCMPEQD", v8i32x_info, v16i32_info>;
+
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpgtm, X86pcmpgtm_su, "VPCMPGTD", v4i32x_info, v16i32_info>;
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpeqm_c, X86pcmpeqm_c_su, "VPCMPEQD", v4i32x_info, v16i32_info>;
+
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpgtm, X86pcmpgtm_su, "VPCMPGTQ", v4i64x_info, v8i64_info>;
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpeqm_c, X86pcmpeqm_c_su, "VPCMPEQQ", v4i64x_info, v8i64_info>;
+
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpgtm, X86pcmpgtm_su, "VPCMPGTQ", v2i64x_info, v8i64_info>;
+  defm : axv512_icmp_packed_rmb_no_vlx_lowering<X86pcmpeqm_c, X86pcmpeqm_c_su, "VPCMPEQQ", v2i64x_info, v8i64_info>;
   }
 
   defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPD", v8i32x_info, v16i32_info>;
@@ -3171,10 +3358,22 @@ let Predicates = [HasAVX512, NoVLX] in {
   defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, X86pcmpm_su, "VPCMPQ", v2i64x_info, v8i64_info>;
   defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, X86pcmpum_su, "VPCMPUQ", v2i64x_info, v8i64_info>;
 
-  defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, X86cmpm_su, "VCMPPS", v8f32x_info, v16f32_info>;
-  defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, X86cmpm_su, "VCMPPS", v4f32x_info, v16f32_info>;
-  defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, X86cmpm_su, "VCMPPD", v4f64x_info, v8f64_info>;
-  defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, X86cmpm_su, "VCMPPD", v2f64x_info, v8f64_info>;
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, X86pcmpm_commute, X86pcmpm_commute_su, "VPCMPD", v8i32x_info, v16i32_info>;
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, X86pcmpum_commute, X86pcmpum_commute_su, "VPCMPUD", v8i32x_info, v16i32_info>;
+
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, X86pcmpm_commute, X86pcmpm_commute_su, "VPCMPD", v4i32x_info, v16i32_info>;
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, X86pcmpum_commute, X86pcmpum_commute_su, "VPCMPUD", v4i32x_info, v16i32_info>;
+
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, X86pcmpm_commute, X86pcmpm_commute_su, "VPCMPQ", v4i64x_info, v8i64_info>;
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, X86pcmpum_commute, X86pcmpum_commute_su, "VPCMPUQ", v4i64x_info, v8i64_info>;
+
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpm, X86pcmpm_su, X86pcmpm_commute, X86pcmpm_commute_su, "VPCMPQ", v2i64x_info, v8i64_info>;
+  defm : axv512_icmp_packed_cc_rmb_no_vlx_lowering<X86pcmpum, X86pcmpum_su, X86pcmpum_commute, X86pcmpum_commute_su, "VPCMPUQ", v2i64x_info, v8i64_info>;
+
+  defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPS", v8f32x_info, v16f32_info>;
+  defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPS", v4f32x_info, v16f32_info>;
+  defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPD", v4f64x_info, v8f64_info>;
+  defm : axv512_cmp_packed_cc_no_vlx_lowering<"VCMPPD", v2f64x_info, v8f64_info>;
 }
 
 let Predicates = [HasBWI, NoVLX] in {
@@ -4160,16 +4359,32 @@ def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), fp32imm0)),
           (COPY_TO_REGCLASS (v4f32 (VMOVSSZrrkz VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),
            (v4f32 (COPY_TO_REGCLASS FR32X:$src1, VR128X)))), FR32X)>;
 
+def : Pat<(f32 (X86selects VK1WM:$mask, (loadf32 addr:$src), (f32 FR32X:$src0))),
+          (COPY_TO_REGCLASS
+           (v4f32 (VMOVSSZrmk (v4f32 (COPY_TO_REGCLASS FR32X:$src0, VR128X)),
+                                                       VK1WM:$mask, addr:$src)),
+           FR32X)>;
+def : Pat<(f32 (X86selects VK1WM:$mask, (loadf32 addr:$src), fp32imm0)),
+          (COPY_TO_REGCLASS (v4f32 (VMOVSSZrmkz VK1WM:$mask, addr:$src)), FR32X)>;
+
 def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
           (COPY_TO_REGCLASS (v2f64 (VMOVSDZrrk
            (v2f64 (COPY_TO_REGCLASS FR64X:$src2, VR128X)),
            VK1WM:$mask, (v2f64 (IMPLICIT_DEF)),
            (v2f64 (COPY_TO_REGCLASS FR64X:$src1, VR128X)))), FR64X)>;
 
-def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), fpimm0)),
+def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), fp64imm0)),
           (COPY_TO_REGCLASS (v2f64 (VMOVSDZrrkz VK1WM:$mask, (v2f64 (IMPLICIT_DEF)),
            (v2f64 (COPY_TO_REGCLASS FR64X:$src1, VR128X)))), FR64X)>;
 
+def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), (f64 FR64X:$src0))),
+          (COPY_TO_REGCLASS
+           (v2f64 (VMOVSDZrmk (v2f64 (COPY_TO_REGCLASS FR64X:$src0, VR128X)),
+                                                       VK1WM:$mask, addr:$src)),
+           FR64X)>;
+def : Pat<(f64 (X86selects VK1WM:$mask, (loadf64 addr:$src), fp64imm0)),
+          (COPY_TO_REGCLASS (v2f64 (VMOVSDZrmkz VK1WM:$mask, addr:$src)), FR64X)>;
+
 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
   def VMOVSSZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
                            (ins VR128X:$src1, VR128X:$src2),
@@ -4511,8 +4726,7 @@ multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                   "${src2}"##_.BroadcastStr##", $src1",
                   "$src1, ${src2}"##_.BroadcastStr,
                   (_.VT (OpNode _.RC:$src1,
-                                (X86VBroadcast
-                                    (_.ScalarLdFrag addr:$src2))))>,
+                                (_.BroadcastLdFrag addr:$src2)))>,
                   AVX512BIBase, EVEX_4V, EVEX_B,
                   Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -4638,8 +4852,7 @@ multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr,
                     "${src2}"##_Brdct.BroadcastStr##", $src1",
                      "$src1, ${src2}"##_Brdct.BroadcastStr,
                     (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
-                                 (_Brdct.VT (X86VBroadcast
-                                          (_Brdct.ScalarLdFrag addr:$src2))))))>,
+                                 (_Brdct.VT (_Brdct.BroadcastLdFrag addr:$src2)))))>,
                     AVX512BIBase, EVEX_4V, EVEX_B,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -4711,8 +4924,7 @@ multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     "${src2}"##_Src.BroadcastStr##", $src1",
                      "$src1, ${src2}"##_Src.BroadcastStr,
                     (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
-                                 (_Src.VT (X86VBroadcast
-                                          (_Src.ScalarLdFrag addr:$src2))))))>,
+                                 (_Src.VT (_Src.BroadcastLdFrag addr:$src2)))))>,
                     EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -4951,32 +5163,6 @@ let Predicates = [HasVLX] in {
   def : Pat<(X86andnp VR128X:$src1, (loadv8i16 addr:$src2)),
             (VPANDNQZ128rm VR128X:$src1, addr:$src2)>;
 
-  def : Pat<(and VR128X:$src1,
-                 (bc_v4i32 (v4f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDDZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(or VR128X:$src1,
-                (bc_v4i32 (v4f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPORDZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(xor VR128X:$src1,
-                 (bc_v4i32 (v4f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPXORDZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR128X:$src1,
-                      (bc_v4i32 (v4f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDNDZ128rmb VR128X:$src1, addr:$src2)>;
-
-  def : Pat<(and VR128X:$src1,
-                 (bc_v2i64 (v2f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDQZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(or VR128X:$src1,
-                (bc_v2i64 (v2f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPORQZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(xor VR128X:$src1,
-                 (bc_v2i64 (v2f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPXORQZ128rmb VR128X:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR128X:$src1,
-                      (bc_v2i64 (v2f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDNQZ128rmb VR128X:$src1, addr:$src2)>;
-
   def : Pat<(v32i8 (and VR256X:$src1, VR256X:$src2)),
             (VPANDQZ256rr VR256X:$src1, VR256X:$src2)>;
   def : Pat<(v16i16 (and VR256X:$src1, VR256X:$src2)),
@@ -5016,32 +5202,6 @@ let Predicates = [HasVLX] in {
             (VPANDNQZ256rm VR256X:$src1, addr:$src2)>;
   def : Pat<(X86andnp VR256X:$src1, (loadv16i16 addr:$src2)),
             (VPANDNQZ256rm VR256X:$src1, addr:$src2)>;
-
-  def : Pat<(and VR256X:$src1,
-                 (bc_v8i32 (v8f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDDZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(or VR256X:$src1,
-                (bc_v8i32 (v8f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPORDZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(xor VR256X:$src1,
-                 (bc_v8i32 (v8f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPXORDZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR256X:$src1,
-                      (bc_v8i32 (v8f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDNDZ256rmb VR256X:$src1, addr:$src2)>;
-
-  def : Pat<(and VR256X:$src1,
-                 (bc_v4i64 (v4f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDQZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(or VR256X:$src1,
-                (bc_v4i64 (v4f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPORQZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(xor VR256X:$src1,
-                 (bc_v4i64 (v4f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPXORQZ256rmb VR256X:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR256X:$src1,
-                      (bc_v4i64 (v4f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDNQZ256rmb VR256X:$src1, addr:$src2)>;
 }
 
 let Predicates = [HasAVX512] in {
@@ -5084,32 +5244,6 @@ let Predicates = [HasAVX512] in {
             (VPANDNQZrm VR512:$src1, addr:$src2)>;
   def : Pat<(X86andnp VR512:$src1, (loadv32i16 addr:$src2)),
             (VPANDNQZrm VR512:$src1, addr:$src2)>;
-
-  def : Pat<(and VR512:$src1,
-                 (bc_v16i32 (v16f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDDZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(or VR512:$src1,
-                (bc_v16i32 (v16f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPORDZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(xor VR512:$src1,
-                 (bc_v16i32 (v16f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPXORDZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR512:$src1,
-                      (bc_v16i32 (v16f32 (X86VBroadcast (loadf32 addr:$src2))))),
-            (VPANDNDZrmb VR512:$src1, addr:$src2)>;
-
-  def : Pat<(and VR512:$src1,
-                 (bc_v8i64 (v8f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDQZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(or VR512:$src1,
-                (bc_v8i64 (v8f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPORQZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(xor VR512:$src1,
-                 (bc_v8i64 (v8f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPXORQZrmb VR512:$src1, addr:$src2)>;
-  def : Pat<(X86andnp VR512:$src1,
-                      (bc_v8i64 (v8f64 (X86VBroadcast (loadf64 addr:$src2))))),
-            (VPANDNQZrmb VR512:$src1, addr:$src2)>;
 }
 
 // Patterns to catch vselect with different type than logic op.
@@ -5149,15 +5283,13 @@ multiclass avx512_logical_lowering_bcast<string InstrStr, SDNode OpNode,
                                          X86VectorVTInfo IntInfo> {
   // Register-broadcast logical operations.
   def : Pat<(IntInfo.VT (OpNode _.RC:$src1,
-                         (bitconvert (_.VT (X86VBroadcast
-                                            (_.ScalarLdFrag addr:$src2)))))),
+                         (bitconvert (_.VT (_.BroadcastLdFrag addr:$src2))))),
             (!cast<Instruction>(InstrStr#rmb) _.RC:$src1, addr:$src2)>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (bitconvert
                     (IntInfo.VT (OpNode _.RC:$src1,
                                  (bitconvert (_.VT
-                                              (X86VBroadcast
-                                               (_.ScalarLdFrag addr:$src2))))))),
+                                              (_.BroadcastLdFrag addr:$src2)))))),
                    _.RC:$src0)),
             (!cast<Instruction>(InstrStr#rmbk) _.RC:$src0, _.KRCWM:$mask,
              _.RC:$src1, addr:$src2)>;
@@ -5165,8 +5297,7 @@ multiclass avx512_logical_lowering_bcast<string InstrStr, SDNode OpNode,
                    (bitconvert
                     (IntInfo.VT (OpNode _.RC:$src1,
                                  (bitconvert (_.VT
-                                              (X86VBroadcast
-                                               (_.ScalarLdFrag addr:$src2))))))),
+                                              (_.BroadcastLdFrag addr:$src2)))))),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(InstrStr#rmbkz)  _.KRCWM:$mask,
              _.RC:$src1, addr:$src2)>;
@@ -5303,7 +5434,8 @@ multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo
 }
 multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
                                 SDNode OpNode, SDNode VecNode, SDNode SaeNode,
-                                X86FoldableSchedWrite sched, bit IsCommutable> {
+                                X86FoldableSchedWrite sched, bit IsCommutable,
+                                string EVEX2VexOvrd> {
   let ExeDomain = _.ExeDomain in {
   defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                            (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
@@ -5323,7 +5455,8 @@ multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
                          (ins _.FRC:$src1, _.FRC:$src2),
                           OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                           [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
-                          Sched<[sched]> {
+                          Sched<[sched]>,
+                          EVEX2VEXOverride<EVEX2VexOvrd#"rr"> {
     let isCommutable = IsCommutable;
   }
   def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
@@ -5331,7 +5464,8 @@ multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
                          OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                          [(set _.FRC:$dst, (OpNode _.FRC:$src1,
                          (_.ScalarLdFrag addr:$src2)))]>,
-                         Sched<[sched.Folded, sched.ReadAfterFold]>;
+                         Sched<[sched.Folded, sched.ReadAfterFold]>,
+                         EVEX2VEXOverride<EVEX2VexOvrd#"rm">;
   }
 
   defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
@@ -5361,10 +5495,12 @@ multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
                               SDNode VecNode, SDNode SaeNode,
                               X86SchedWriteSizes sched, bit IsCommutable> {
   defm SSZ : avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, OpNode,
-                              VecNode, SaeNode, sched.PS.Scl, IsCommutable>,
+                              VecNode, SaeNode, sched.PS.Scl, IsCommutable,
+                              NAME#"SS">,
                               XS, EVEX_4V, VEX_LIG,  EVEX_CD8<32, CD8VT1>;
   defm SDZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, OpNode,
-                              VecNode, SaeNode, sched.PD.Scl, IsCommutable>,
+                              VecNode, SaeNode, sched.PD.Scl, IsCommutable,
+                              NAME#"SD">,
                               XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
 }
 defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86fadds, X86faddRnds,
@@ -5384,13 +5520,14 @@ defm VMAX : avx512_binop_s_sae<0x5F, "vmax", X86fmax, X86fmaxs, X86fmaxSAEs,
 // X86fminc and X86fmaxc instead of X86fmin and X86fmax
 multiclass avx512_comutable_binop_s<bits<8> opc, string OpcodeStr,
                                     X86VectorVTInfo _, SDNode OpNode,
-                                    X86FoldableSchedWrite sched> {
+                                    X86FoldableSchedWrite sched,
+                                    string EVEX2VEXOvrd> {
   let isCodeGenOnly = 1, Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {
   def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
                          (ins _.FRC:$src1, _.FRC:$src2),
                           OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                           [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
-                          Sched<[sched]> {
+                          Sched<[sched]>, EVEX2VEXOverride<EVEX2VEXOvrd#"rr"> {
     let isCommutable = 1;
   }
   def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
@@ -5398,24 +5535,27 @@ multiclass avx512_comutable_binop_s<bits<8> opc, string OpcodeStr,
                          OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                          [(set _.FRC:$dst, (OpNode _.FRC:$src1,
                          (_.ScalarLdFrag addr:$src2)))]>,
-                         Sched<[sched.Folded, sched.ReadAfterFold]>;
+                         Sched<[sched.Folded, sched.ReadAfterFold]>,
+                         EVEX2VEXOverride<EVEX2VEXOvrd#"rm">;
   }
 }
 defm VMINCSSZ : avx512_comutable_binop_s<0x5D, "vminss", f32x_info, X86fminc,
-                                         SchedWriteFCmp.Scl>, XS, EVEX_4V,
-                                         VEX_LIG, EVEX_CD8<32, CD8VT1>;
+                                         SchedWriteFCmp.Scl, "VMINCSS">, XS,
+                                         EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
 
 defm VMINCSDZ : avx512_comutable_binop_s<0x5D, "vminsd", f64x_info, X86fminc,
-                                         SchedWriteFCmp.Scl>, XD, VEX_W, EVEX_4V,
-                                         VEX_LIG, EVEX_CD8<64, CD8VT1>;
+                                         SchedWriteFCmp.Scl, "VMINCSD">, XD,
+                                         VEX_W, EVEX_4V, VEX_LIG,
+                                         EVEX_CD8<64, CD8VT1>;
 
 defm VMAXCSSZ : avx512_comutable_binop_s<0x5F, "vmaxss", f32x_info, X86fmaxc,
-                                         SchedWriteFCmp.Scl>, XS, EVEX_4V,
-                                         VEX_LIG, EVEX_CD8<32, CD8VT1>;
+                                         SchedWriteFCmp.Scl, "VMAXCSS">, XS,
+                                         EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
 
 defm VMAXCSDZ : avx512_comutable_binop_s<0x5F, "vmaxsd", f64x_info, X86fmaxc,
-                                         SchedWriteFCmp.Scl>, XD, VEX_W, EVEX_4V,
-                                         VEX_LIG, EVEX_CD8<64, CD8VT1>;
+                                         SchedWriteFCmp.Scl, "VMAXCSD">, XD,
+                                         VEX_W, EVEX_4V, VEX_LIG,
+                                         EVEX_CD8<64, CD8VT1>;
 
 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
                             X86VectorVTInfo _, X86FoldableSchedWrite sched,
@@ -5438,8 +5578,7 @@ multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpN
                      (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
                      "${src2}"##_.BroadcastStr##", $src1",
                      "$src1, ${src2}"##_.BroadcastStr,
-                     (OpNode  _.RC:$src1, (_.VT (X86VBroadcast
-                                                (_.ScalarLdFrag addr:$src2))))>,
+                     (OpNode  _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2)))>,
                      EVEX_4V, EVEX_B,
                      Sched<[sched.Folded, sched.ReadAfterFold]>;
     }
@@ -5569,8 +5708,7 @@ multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
                    (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
                    "${src2}"##_.BroadcastStr##", $src1",
                    "$src1, ${src2}"##_.BroadcastStr,
-                   (OpNode  _.RC:$src1, (_.VT (X86VBroadcast
-                                              (_.ScalarLdFrag addr:$src2))))>,
+                   (OpNode  _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2)))>,
                    EVEX_4V, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -5725,13 +5863,13 @@ multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
   defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),
                    (ins _.RC:$src1, u8imm:$src2), OpcodeStr,
                       "$src2, $src1", "$src1, $src2",
-                   (_.VT (OpNode _.RC:$src1, (i8 imm:$src2)))>,
+                   (_.VT (OpNode _.RC:$src1, (i8 timm:$src2)))>,
                    Sched<[sched]>;
   defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
                    (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,
                        "$src2, $src1", "$src1, $src2",
                    (_.VT (OpNode (_.VT (_.LdFrag addr:$src1)),
-                          (i8 imm:$src2)))>,
+                          (i8 timm:$src2)))>,
                    Sched<[sched.Folded]>;
   }
 }
@@ -5743,7 +5881,7 @@ multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
   defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
                    (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,
       "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2",
-     (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2)))>,
+     (_.VT (OpNode (_.BroadcastLdFrag addr:$src1), (i8 timm:$src2)))>,
      EVEX_B, Sched<[sched.Folded]>;
 }
 
@@ -5885,17 +6023,17 @@ let Predicates = [HasAVX512, NoVLX] in {
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
                  VR128X:$src2)), sub_xmm)>;
 
-  def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 imm:$src2))),
+  def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPSRAQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
-                 imm:$src2)), sub_ymm)>;
+                 timm:$src2)), sub_ymm)>;
 
-  def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 imm:$src2))),
+  def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPSRAQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
-                 imm:$src2)), sub_xmm)>;
+                 timm:$src2)), sub_xmm)>;
 }
 
 //===-------------------------------------------------------------------===//
@@ -5927,8 +6065,7 @@ multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
                     "${src2}"##_.BroadcastStr##", $src1",
                     "$src1, ${src2}"##_.BroadcastStr,
-                    (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
-                                                (_.ScalarLdFrag addr:$src2)))))>,
+                    (_.VT (OpNode _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))))>,
                     AVX5128IBase, EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -6036,27 +6173,27 @@ let Predicates = [HasAVX512, NoVLX] in {
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
                         sub_ymm)>;
 
-  def : Pat<(v2i64 (X86vrotli (v2i64 VR128X:$src1), (i8 imm:$src2))),
+  def : Pat<(v2i64 (X86vrotli (v2i64 VR128X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPROLQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
-                        imm:$src2)), sub_xmm)>;
-  def : Pat<(v4i64 (X86vrotli (v4i64 VR256X:$src1), (i8 imm:$src2))),
+                        timm:$src2)), sub_xmm)>;
+  def : Pat<(v4i64 (X86vrotli (v4i64 VR256X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPROLQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
-                       imm:$src2)), sub_ymm)>;
+                       timm:$src2)), sub_ymm)>;
 
-  def : Pat<(v4i32 (X86vrotli (v4i32 VR128X:$src1), (i8 imm:$src2))),
+  def : Pat<(v4i32 (X86vrotli (v4i32 VR128X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v16i32
               (VPROLDZri
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
-                        imm:$src2)), sub_xmm)>;
-  def : Pat<(v8i32 (X86vrotli (v8i32 VR256X:$src1), (i8 imm:$src2))),
+                        timm:$src2)), sub_xmm)>;
+  def : Pat<(v8i32 (X86vrotli (v8i32 VR256X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v16i32
               (VPROLDZri
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
-                        imm:$src2)), sub_ymm)>;
+                        timm:$src2)), sub_ymm)>;
 }
 
 // Use 512bit VPROR/VPRORI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
@@ -6087,27 +6224,27 @@ let Predicates = [HasAVX512, NoVLX] in {
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
                         sub_ymm)>;
 
-  def : Pat<(v2i64 (X86vrotri (v2i64 VR128X:$src1), (i8 imm:$src2))),
+  def : Pat<(v2i64 (X86vrotri (v2i64 VR128X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPRORQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
-                        imm:$src2)), sub_xmm)>;
-  def : Pat<(v4i64 (X86vrotri (v4i64 VR256X:$src1), (i8 imm:$src2))),
+                        timm:$src2)), sub_xmm)>;
+  def : Pat<(v4i64 (X86vrotri (v4i64 VR256X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v8i64
               (VPRORQZri
                 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
-                       imm:$src2)), sub_ymm)>;
+                       timm:$src2)), sub_ymm)>;
 
-  def : Pat<(v4i32 (X86vrotri (v4i32 VR128X:$src1), (i8 imm:$src2))),
+  def : Pat<(v4i32 (X86vrotri (v4i32 VR128X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v16i32
               (VPRORDZri
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
-                        imm:$src2)), sub_xmm)>;
-  def : Pat<(v8i32 (X86vrotri (v8i32 VR256X:$src1), (i8 imm:$src2))),
+                        timm:$src2)), sub_xmm)>;
+  def : Pat<(v8i32 (X86vrotri (v8i32 VR256X:$src1), (i8 timm:$src2))),
             (EXTRACT_SUBREG (v16i32
               (VPRORDZri
                 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
-                        imm:$src2)), sub_ymm)>;
+                        timm:$src2)), sub_ymm)>;
 }
 
 //===-------------------------------------------------------------------===//
@@ -6202,8 +6339,7 @@ multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
                    "$src1, ${src2}"##_.BroadcastStr,
                    (_.VT (OpNode
                             _.RC:$src1,
-                            (Ctrl.VT (X86VBroadcast
-                                       (Ctrl.ScalarLdFrag addr:$src2)))))>,
+                            (Ctrl.VT (Ctrl.BroadcastLdFrag addr:$src2))))>,
                    T8PD, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
                    Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -6393,7 +6529,7 @@ multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
             OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
             !strconcat("$src2, ${src3}", _.BroadcastStr ),
             (OpNode _.RC:$src2,
-             _.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))), 1, 0>,
+             _.RC:$src1,(_.VT (_.BroadcastLdFrag addr:$src3))), 1, 0>,
              AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -6467,7 +6603,7 @@ multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
          OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
          "$src2, ${src3}"##_.BroadcastStr,
          (_.VT (OpNode _.RC:$src2,
-                      (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
+                      (_.VT (_.BroadcastLdFrag addr:$src3)),
                       _.RC:$src1)), 1, 0>, AVX512FMA3Base, EVEX_B,
          Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
@@ -6545,7 +6681,7 @@ multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
          (ins _.RC:$src2, _.ScalarMemOp:$src3),
          OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
          "$src2, ${src3}"##_.BroadcastStr,
-         (_.VT (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
+         (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src3)),
                        _.RC:$src1, _.RC:$src2)), 1, 0>,
          AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
@@ -6938,7 +7074,7 @@ multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
             OpcodeStr,   !strconcat("${src3}", _.BroadcastStr,", $src2"),
             !strconcat("$src2, ${src3}", _.BroadcastStr ),
             (OpNode _.RC:$src2,
-                    (_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))),
+                    (_.VT (_.BroadcastLdFrag addr:$src3)),
                     _.RC:$src1)>,
             AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
@@ -7478,14 +7614,13 @@ multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                          OpcodeStr,
                          "${src}"##Broadcast, "${src}"##Broadcast,
                          (_.VT (OpNode (_Src.VT
-                                  (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
+                                  (_Src.BroadcastLdFrag addr:$src))
                             )),
                          (vselect MaskRC:$mask,
                                   (_.VT
                                    (OpNode
                                     (_Src.VT
-                                     (X86VBroadcast
-                                      (_Src.ScalarLdFrag addr:$src))))),
+                                     (_Src.BroadcastLdFrag addr:$src)))),
                                   _.RC:$src0),
                          vselect, "$src0 = $dst">,
                          EVEX, EVEX_B, Sched<[sched.Folded]>;
@@ -7620,14 +7755,14 @@ let Predicates = [HasAVX512] in {
                      v8f32x_info.ImmAllZerosV),
             (VCVTPD2PSZrmkz VK8WM:$mask, addr:$src)>;
 
-  def : Pat<(v8f32 (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v8f32 (fpround (v8f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTPD2PSZrmb addr:$src)>;
   def : Pat<(vselect VK8WM:$mask,
-                     (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src)))),
+                     (fpround (v8f64 (X86VBroadcastld64 addr:$src))),
                      (v8f32 VR256X:$src0)),
             (VCVTPD2PSZrmbk VR256X:$src0, VK8WM:$mask, addr:$src)>;
   def : Pat<(vselect VK8WM:$mask,
-                     (fpround (v8f64 (X86VBroadcast (loadf64 addr:$src)))),
+                     (fpround (v8f64 (X86VBroadcastld64 addr:$src))),
                      v8f32x_info.ImmAllZerosV),
             (VCVTPD2PSZrmbkz VK8WM:$mask, addr:$src)>;
 }
@@ -7651,14 +7786,14 @@ let Predicates = [HasVLX] in {
                      v4f32x_info.ImmAllZerosV),
             (VCVTPD2PSZ256rmkz VK4WM:$mask, addr:$src)>;
 
-  def : Pat<(v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v4f32 (fpround (v4f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTPD2PSZ256rmb addr:$src)>;
   def : Pat<(vselect VK4WM:$mask,
-                     (v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+                     (v4f32 (fpround (v4f64 (X86VBroadcastld64 addr:$src)))),
                      VR128X:$src0),
             (VCVTPD2PSZ256rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;
   def : Pat<(vselect VK4WM:$mask,
-                     (v4f32 (fpround (v4f64 (X86VBroadcast (loadf64 addr:$src))))),
+                     (v4f32 (fpround (v4f64 (X86VBroadcastld64 addr:$src)))),
                      v4f32x_info.ImmAllZerosV),
             (VCVTPD2PSZ256rmbkz VK4WM:$mask, addr:$src)>;
 
@@ -7682,12 +7817,12 @@ let Predicates = [HasVLX] in {
                           VK2WM:$mask),
             (VCVTPD2PSZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(X86vfpround (v2f64 (X86VBroadcast (loadf64 addr:$src)))),
+  def : Pat<(X86vfpround (v2f64 (X86VBroadcastld64 addr:$src))),
             (VCVTPD2PSZ128rmb addr:$src)>;
-  def : Pat<(X86vmfpround (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86vmfpround (v2f64 (X86VBroadcastld64 addr:$src)),
                           (v4f32 VR128X:$src0), VK2WM:$mask),
             (VCVTPD2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86vmfpround (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86vmfpround (v2f64 (X86VBroadcastld64 addr:$src)),
                           v4f32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTPD2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
@@ -8168,12 +8303,12 @@ let Predicates = [HasVLX] in {
                           VK2WM:$mask),
             (VCVTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4i32 (X86cvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v4i32 (X86cvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTPD2DQZ128rmb addr:$src)>;
-  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)),
                           (v4i32 VR128X:$src0), VK2WM:$mask),
             (VCVTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvtp2Int (v2f64 (X86VBroadcastld64 addr:$src)),
                           v4i32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;
 
@@ -8197,12 +8332,12 @@ let Predicates = [HasVLX] in {
                           VK2WM:$mask),
             (VCVTTPD2DQZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4i32 (X86cvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v4i32 (X86cvttp2si (v2f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTTPD2DQZ128rmb addr:$src)>;
-  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcastld64 addr:$src)),
                           (v4i32 VR128X:$src0), VK2WM:$mask),
             (VCVTTPD2DQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvttp2si (v2f64 (X86VBroadcastld64 addr:$src)),
                           v4i32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTTPD2DQZ128rmbkz VK2WM:$mask, addr:$src)>;
 
@@ -8226,12 +8361,12 @@ let Predicates = [HasVLX] in {
                            VK2WM:$mask),
             (VCVTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4i32 (X86cvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v4i32 (X86cvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTPD2UDQZ128rmb addr:$src)>;
-  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)),
                            (v4i32 VR128X:$src0), VK2WM:$mask),
             (VCVTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvtp2UInt (v2f64 (X86VBroadcastld64 addr:$src)),
                            v4i32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;
 
@@ -8255,12 +8390,12 @@ let Predicates = [HasVLX] in {
                           VK2WM:$mask),
             (VCVTTPD2UDQZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4i32 (X86cvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))))),
+  def : Pat<(v4i32 (X86cvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)))),
             (VCVTTPD2UDQZ128rmb addr:$src)>;
-  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)),
                           (v4i32 VR128X:$src0), VK2WM:$mask),
             (VCVTTPD2UDQZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(X86mcvttp2ui (v2f64 (X86VBroadcastld64 addr:$src)),
                           v4i32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTTPD2UDQZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
@@ -8393,12 +8528,12 @@ let Predicates = [HasDQI, HasVLX] in {
                            VK2WM:$mask),
             (VCVTQQ2PSZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4f32 (X86VSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))))),
+  def : Pat<(v4f32 (X86VSintToFP (v2i64 (X86VBroadcastld64 addr:$src)))),
             (VCVTQQ2PSZ128rmb addr:$src)>;
-  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcastld64 addr:$src)),
                            (v4f32 VR128X:$src0), VK2WM:$mask),
             (VCVTQQ2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+  def : Pat<(X86VMSintToFP (v2i64 (X86VBroadcastld64 addr:$src)),
                            v4f32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTQQ2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
 
@@ -8422,12 +8557,12 @@ let Predicates = [HasDQI, HasVLX] in {
                            VK2WM:$mask),
             (VCVTUQQ2PSZ128rmkz VK2WM:$mask, addr:$src)>;
 
-  def : Pat<(v4f32 (X86VUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))))),
+  def : Pat<(v4f32 (X86VUintToFP (v2i64 (X86VBroadcastld64 addr:$src)))),
             (VCVTUQQ2PSZ128rmb addr:$src)>;
-  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcastld64 addr:$src)),
                            (v4f32 VR128X:$src0), VK2WM:$mask),
             (VCVTUQQ2PSZ128rmbk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcast (loadi64 addr:$src))),
+  def : Pat<(X86VMUintToFP (v2i64 (X86VBroadcastld64 addr:$src)),
                            v4f32x_info.ImmAllZerosV, VK2WM:$mask),
             (VCVTUQQ2PSZ128rmbkz VK2WM:$mask, addr:$src)>;
 }
@@ -8550,21 +8685,21 @@ let ExeDomain = GenericDomain in {
              (ins _src.RC:$src1, i32u8imm:$src2),
              "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
              [(set _dest.RC:$dst,
-                   (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2)))]>,
+                   (X86cvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2)))]>,
              Sched<[RR]>;
   let Constraints = "$src0 = $dst" in
   def rrk : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
              (ins _dest.RC:$src0, _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
              "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
              [(set _dest.RC:$dst,
-                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2),
+                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2),
                                  _dest.RC:$src0, _src.KRCWM:$mask))]>,
              Sched<[RR]>, EVEX_K;
   def rrkz : AVX512AIi8<0x1D, MRMDestReg, (outs _dest.RC:$dst),
              (ins _src.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
              "vcvtps2ph\t{$src2, $src1, $dst {${mask}} {z}|$dst {${mask}} {z}, $src1, $src2}",
              [(set _dest.RC:$dst,
-                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 imm:$src2),
+                   (X86mcvtps2ph (_src.VT _src.RC:$src1), (i32 timm:$src2),
                                  _dest.ImmAllZerosV, _src.KRCWM:$mask))]>,
              Sched<[RR]>, EVEX_KZ;
   let hasSideEffects = 0, mayStore = 1 in {
@@ -8605,17 +8740,17 @@ let Predicates = [HasAVX512] in {
   }
 
   def : Pat<(store (f64 (extractelt
-                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128X:$src1, i32:$src2))),
+                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128X:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
-            (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, imm:$src2)>;
+            (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;
   def : Pat<(store (i64 (extractelt
-                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128X:$src1, i32:$src2))),
+                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128X:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
-            (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, imm:$src2)>;
-  def : Pat<(store (v8i16 (X86cvtps2ph VR256X:$src1, i32:$src2)), addr:$dst),
-            (VCVTPS2PHZ256mr addr:$dst, VR256X:$src1, imm:$src2)>;
-  def : Pat<(store (v16i16 (X86cvtps2ph VR512:$src1, i32:$src2)), addr:$dst),
-            (VCVTPS2PHZmr addr:$dst, VR512:$src1, imm:$src2)>;
+            (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, timm:$src2)>;
+  def : Pat<(store (v8i16 (X86cvtps2ph VR256X:$src1, timm:$src2)), addr:$dst),
+            (VCVTPS2PHZ256mr addr:$dst, VR256X:$src1, timm:$src2)>;
+  def : Pat<(store (v16i16 (X86cvtps2ph VR512:$src1, timm:$src2)), addr:$dst),
+            (VCVTPS2PHZmr addr:$dst, VR512:$src1, timm:$src2)>;
 }
 
 // Patterns for matching conversions from float to half-float and vice versa.
@@ -8739,7 +8874,7 @@ multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
                           (ins _.ScalarMemOp:$src), OpcodeStr,
                           "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
                           (OpNode (_.VT
-                            (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
+                            (_.BroadcastLdFrag addr:$src)))>,
                           EVEX, T8PD, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -8833,7 +8968,7 @@ multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                          (ins _.ScalarMemOp:$src), OpcodeStr,
                          "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
                          (OpNode (_.VT
-                                  (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
+                                  (_.BroadcastLdFrag addr:$src)))>,
                          EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -8914,7 +9049,7 @@ multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
                           (ins _.ScalarMemOp:$src), OpcodeStr,
                           "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
                           (fsqrt (_.VT
-                            (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
+                            (_.BroadcastLdFrag addr:$src)))>,
                           EVEX, EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -9023,14 +9158,14 @@ multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,
                            (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
                            "$src3, $src2, $src1", "$src1, $src2, $src3",
                            (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                           (i32 imm:$src3)))>,
+                           (i32 timm:$src3)))>,
                            Sched<[sched]>;
 
   defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                          (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
                          "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",
                          (_.VT (X86RndScalesSAE (_.VT _.RC:$src1), (_.VT _.RC:$src2),
-                         (i32 imm:$src3)))>, EVEX_B,
+                         (i32 timm:$src3)))>, EVEX_B,
                          Sched<[sched]>;
 
   defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
@@ -9038,7 +9173,7 @@ multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,
                          OpcodeStr,
                          "$src3, $src2, $src1", "$src1, $src2, $src3",
                          (_.VT (X86RndScales _.RC:$src1,
-                                _.ScalarIntMemCPat:$src2, (i32 imm:$src3)))>,
+                                _.ScalarIntMemCPat:$src2, (i32 timm:$src3)))>,
                          Sched<[sched.Folded, sched.ReadAfterFold]>;
 
   let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [HasAVX512] in {
@@ -9056,15 +9191,15 @@ multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,
   }
 
   let Predicates = [HasAVX512] in {
-    def : Pat<(X86VRndScale _.FRC:$src1, imm:$src2),
+    def : Pat<(X86VRndScale _.FRC:$src1, timm:$src2),
               (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
-               _.FRC:$src1, imm:$src2))>;
+               _.FRC:$src1, timm:$src2))>;
   }
 
   let Predicates = [HasAVX512, OptForSize] in {
-    def : Pat<(X86VRndScale (_.ScalarLdFrag addr:$src1), imm:$src2),
+    def : Pat<(X86VRndScale (_.ScalarLdFrag addr:$src1), timm:$src2),
               (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
-               addr:$src1, imm:$src2))>;
+               addr:$src1, timm:$src2))>;
   }
 }
 
@@ -10083,19 +10218,19 @@ multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNo
                       (ins _.RC:$src1, i32u8imm:$src2),
                       OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
                       (OpNode (_.VT _.RC:$src1),
-                              (i32 imm:$src2))>, Sched<[sched]>;
+                              (i32 timm:$src2))>, Sched<[sched]>;
   defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                     (ins _.MemOp:$src1, i32u8imm:$src2),
                     OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
                     (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
-                            (i32 imm:$src2))>,
+                            (i32 timm:$src2))>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                     (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
                     OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr,
                     "${src1}"##_.BroadcastStr##", $src2",
-                    (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
-                            (i32 imm:$src2))>, EVEX_B,
+                    (OpNode (_.VT (_.BroadcastLdFrag addr:$src1)),
+                            (i32 timm:$src2))>, EVEX_B,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10110,7 +10245,7 @@ multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
                       OpcodeStr##_.Suffix, "$src2, {sae}, $src1",
                       "$src1, {sae}, $src2",
                       (OpNode (_.VT _.RC:$src1),
-                              (i32 imm:$src2))>,
+                              (i32 timm:$src2))>,
                       EVEX_B, Sched<[sched]>;
 }
 
@@ -10143,22 +10278,22 @@ multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
                       OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                       (OpNode (_.VT _.RC:$src1),
                               (_.VT _.RC:$src2),
-                              (i32 imm:$src3))>,
+                              (i32 timm:$src3))>,
                       Sched<[sched]>;
   defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                     (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
                     OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                     (OpNode (_.VT _.RC:$src1),
                             (_.VT (bitconvert (_.LdFrag addr:$src2))),
-                            (i32 imm:$src3))>,
+                            (i32 timm:$src3))>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                     (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
                     OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
                     "$src1, ${src2}"##_.BroadcastStr##", $src3",
                     (OpNode (_.VT _.RC:$src1),
-                            (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
-                            (i32 imm:$src3))>, EVEX_B,
+                            (_.VT (_.BroadcastLdFrag addr:$src2)),
+                            (i32 timm:$src3))>, EVEX_B,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10174,7 +10309,7 @@ multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
                   OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                   (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
                                (SrcInfo.VT SrcInfo.RC:$src2),
-                               (i8 imm:$src3)))>,
+                               (i8 timm:$src3)))>,
                   Sched<[sched]>;
   defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
                 (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3),
@@ -10182,7 +10317,7 @@ multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
                 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
                              (SrcInfo.VT (bitconvert
                                                 (SrcInfo.LdFrag addr:$src2))),
-                             (i8 imm:$src3)))>,
+                             (i8 timm:$src3)))>,
                 Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10200,8 +10335,8 @@ multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
                     "$src1, ${src2}"##_.BroadcastStr##", $src3",
                     (OpNode (_.VT _.RC:$src1),
-                            (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
-                            (i8 imm:$src3))>, EVEX_B,
+                            (_.VT (_.BroadcastLdFrag addr:$src2)),
+                            (i8 timm:$src3))>, EVEX_B,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -10215,15 +10350,14 @@ multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
                       OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                       (OpNode (_.VT _.RC:$src1),
                               (_.VT _.RC:$src2),
-                              (i32 imm:$src3))>,
+                              (i32 timm:$src3))>,
                       Sched<[sched]>;
   defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
-                    (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
+                    (ins _.RC:$src1, _.IntScalarMemOp:$src2, i32u8imm:$src3),
                     OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                     (OpNode (_.VT _.RC:$src1),
-                            (_.VT (scalar_to_vector
-                                      (_.ScalarLdFrag addr:$src2))),
-                            (i32 imm:$src3))>,
+                            (_.VT _.ScalarIntMemCPat:$src2),
+                            (i32 timm:$src3))>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10239,7 +10373,7 @@ multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
                       "$src1, $src2, {sae}, $src3",
                       (OpNode (_.VT _.RC:$src1),
                               (_.VT _.RC:$src2),
-                              (i32 imm:$src3))>,
+                              (i32 timm:$src3))>,
                       EVEX_B, Sched<[sched]>;
 }
 
@@ -10253,7 +10387,7 @@ multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode
                       "$src1, $src2, {sae}, $src3",
                       (OpNode (_.VT _.RC:$src1),
                               (_.VT _.RC:$src2),
-                              (i32 imm:$src3))>,
+                              (i32 timm:$src3))>,
                       EVEX_B, Sched<[sched]>;
 }
 
@@ -10375,7 +10509,7 @@ multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,
                   OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                   (_.VT (bitconvert
                          (CastInfo.VT (X86Shuf128 _.RC:$src1, _.RC:$src2,
-                                                  (i8 imm:$src3)))))>,
+                                                  (i8 timm:$src3)))))>,
                   Sched<[sched]>, EVEX2VEXOverride<EVEX2VEXOvrd#"rr">;
   defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
@@ -10384,7 +10518,7 @@ multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,
                  (bitconvert
                   (CastInfo.VT (X86Shuf128 _.RC:$src1,
                                            (CastInfo.LdFrag addr:$src2),
-                                           (i8 imm:$src3)))))>,
+                                           (i8 timm:$src3)))))>,
                 Sched<[sched.Folded, sched.ReadAfterFold]>,
                 EVEX2VEXOverride<EVEX2VEXOvrd#"rm">;
   defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
@@ -10395,8 +10529,8 @@ multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,
                      (bitconvert
                       (CastInfo.VT
                        (X86Shuf128 _.RC:$src1,
-                                   (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
-                                   (i8 imm:$src3)))))>, EVEX_B,
+                                   (_.BroadcastLdFrag addr:$src2),
+                                   (i8 timm:$src3)))))>, EVEX_B,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10465,14 +10599,14 @@ multiclass avx512_valign<bits<8> opc, string OpcodeStr,
   defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
                   (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),
                   OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
-                  (_.VT (X86VAlign _.RC:$src1, _.RC:$src2, (i8 imm:$src3)))>,
+                  (_.VT (X86VAlign _.RC:$src1, _.RC:$src2, (i8 timm:$src3)))>,
                   Sched<[sched]>, EVEX2VEXOverride<"VPALIGNRrri">;
   defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
                 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
                 (_.VT (X86VAlign _.RC:$src1,
                                  (bitconvert (_.LdFrag addr:$src2)),
-                                 (i8 imm:$src3)))>,
+                                 (i8 timm:$src3)))>,
                 Sched<[sched.Folded, sched.ReadAfterFold]>,
                 EVEX2VEXOverride<"VPALIGNRrmi">;
 
@@ -10481,8 +10615,8 @@ multiclass avx512_valign<bits<8> opc, string OpcodeStr,
                    OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
                    "$src1, ${src2}"##_.BroadcastStr##", $src3",
                    (X86VAlign _.RC:$src1,
-                              (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
-                              (i8 imm:$src3))>, EVEX_B,
+                              (_.VT (_.BroadcastLdFrag addr:$src2)),
+                              (i8 timm:$src3))>, EVEX_B,
                    Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -10515,13 +10649,13 @@ defm VPALIGNR: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr",
 
 // Fragments to help convert valignq into masked valignd. Or valignq/valignd
 // into vpalignr.
-def ValignqImm32XForm : SDNodeXForm<imm, [{
+def ValignqImm32XForm : SDNodeXForm<timm, [{
   return getI8Imm(N->getZExtValue() * 2, SDLoc(N));
 }]>;
-def ValignqImm8XForm : SDNodeXForm<imm, [{
+def ValignqImm8XForm : SDNodeXForm<timm, [{
   return getI8Imm(N->getZExtValue() * 8, SDLoc(N));
 }]>;
-def ValigndImm8XForm : SDNodeXForm<imm, [{
+def ValigndImm8XForm : SDNodeXForm<timm, [{
   return getI8Imm(N->getZExtValue() * 4, SDLoc(N));
 }]>;
 
@@ -10531,40 +10665,40 @@ multiclass avx512_vpalign_mask_lowering<string OpcodeStr, SDNode OpNode,
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1, From.RC:$src2,
-                                              imm:$src3))),
+                                              timm:$src3))),
                             To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rrik") To.RC:$src0, To.KRCWM:$mask,
                                                   To.RC:$src1, To.RC:$src2,
-                                                  (ImmXForm imm:$src3))>;
+                                                  (ImmXForm timm:$src3))>;
 
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1, From.RC:$src2,
-                                              imm:$src3))),
+                                              timm:$src3))),
                             To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rrikz") To.KRCWM:$mask,
                                                    To.RC:$src1, To.RC:$src2,
-                                                   (ImmXForm imm:$src3))>;
+                                                   (ImmXForm timm:$src3))>;
 
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1,
                                               (From.LdFrag addr:$src2),
-                                      imm:$src3))),
+                                      timm:$src3))),
                             To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rmik") To.RC:$src0, To.KRCWM:$mask,
                                                   To.RC:$src1, addr:$src2,
-                                                  (ImmXForm imm:$src3))>;
+                                                  (ImmXForm timm:$src3))>;
 
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1,
                                               (From.LdFrag addr:$src2),
-                                      imm:$src3))),
+                                      timm:$src3))),
                             To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rmikz") To.KRCWM:$mask,
                                                    To.RC:$src1, addr:$src2,
-                                                   (ImmXForm imm:$src3))>;
+                                                   (ImmXForm timm:$src3))>;
 }
 
 multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,
@@ -10573,35 +10707,32 @@ multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,
                                            SDNodeXForm ImmXForm> :
       avx512_vpalign_mask_lowering<OpcodeStr, OpNode, From, To, ImmXForm> {
   def : Pat<(From.VT (OpNode From.RC:$src1,
-                             (bitconvert (To.VT (X86VBroadcast
-                                                (To.ScalarLdFrag addr:$src2)))),
-                             imm:$src3)),
+                             (bitconvert (To.VT (To.BroadcastLdFrag addr:$src2))),
+                             timm:$src3)),
             (!cast<Instruction>(OpcodeStr#"rmbi") To.RC:$src1, addr:$src2,
-                                                  (ImmXForm imm:$src3))>;
+                                                  (ImmXForm timm:$src3))>;
 
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1,
                                       (bitconvert
-                                       (To.VT (X86VBroadcast
-                                               (To.ScalarLdFrag addr:$src2)))),
-                                      imm:$src3))),
+                                       (To.VT (To.BroadcastLdFrag addr:$src2))),
+                                      timm:$src3))),
                             To.RC:$src0)),
             (!cast<Instruction>(OpcodeStr#"rmbik") To.RC:$src0, To.KRCWM:$mask,
                                                    To.RC:$src1, addr:$src2,
-                                                   (ImmXForm imm:$src3))>;
+                                                   (ImmXForm timm:$src3))>;
 
   def : Pat<(To.VT (vselect To.KRCWM:$mask,
                             (bitconvert
                              (From.VT (OpNode From.RC:$src1,
                                       (bitconvert
-                                       (To.VT (X86VBroadcast
-                                               (To.ScalarLdFrag addr:$src2)))),
-                                      imm:$src3))),
+                                       (To.VT (To.BroadcastLdFrag addr:$src2))),
+                                      timm:$src3))),
                             To.ImmAllZerosV)),
             (!cast<Instruction>(OpcodeStr#"rmbikz") To.KRCWM:$mask,
                                                     To.RC:$src1, addr:$src2,
-                                                    (ImmXForm imm:$src3))>;
+                                                    (ImmXForm timm:$src3))>;
 }
 
 let Predicates = [HasAVX512] in {
@@ -10659,8 +10790,7 @@ multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
                   (ins _.ScalarMemOp:$src1), OpcodeStr,
                   "${src1}"##_.BroadcastStr,
                   "${src1}"##_.BroadcastStr,
-                  (_.VT (OpNode (_.VT (X86VBroadcast
-                                       (_.ScalarLdFrag addr:$src1)))))>,
+                  (_.VT (OpNode (_.VT (_.BroadcastLdFrag addr:$src1))))>,
              EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
              Sched<[sched.Folded]>;
 }
@@ -10803,16 +10933,16 @@ defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup,
 // AVX-512 - MOVDDUP
 //===----------------------------------------------------------------------===//
 
-multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
+multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr,
                               X86FoldableSchedWrite sched, X86VectorVTInfo _> {
   let ExeDomain = _.ExeDomain in {
   defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
                    (ins _.RC:$src), OpcodeStr, "$src", "$src",
-                   (_.VT (OpNode (_.VT _.RC:$src)))>, EVEX,
+                   (_.VT (X86VBroadcast (_.VT _.RC:$src)))>, EVEX,
                    Sched<[sched]>;
   defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
                  (ins _.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
-                 (_.VT (OpNode (_.ScalarLdFrag addr:$src)))>,
+                 (_.VT (_.BroadcastLdFrag addr:$src))>,
                  EVEX, EVEX_CD8<_.EltSize, CD8VH>,
                  Sched<[sched.Folded]>;
   }
@@ -10826,7 +10956,7 @@ multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
   let Predicates = [HasAVX512, HasVLX] in {
     defm Z256 : avx512_unary_rm<opc, OpcodeStr, X86Movddup, sched.YMM,
                                 VTInfo.info256>, EVEX_V256;
-    defm Z128 : avx512_movddup_128<opc, OpcodeStr, X86VBroadcast, sched.XMM,
+    defm Z128 : avx512_movddup_128<opc, OpcodeStr, sched.XMM,
                                    VTInfo.info128>, EVEX_V128;
   }
 }
@@ -10855,10 +10985,10 @@ def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
                    immAllZerosV),
           (VMOVDDUPZ128rrkz VK2WM:$mask, (v2f64 (COPY_TO_REGCLASS FR64X:$src, VR128X)))>;
 
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
                    (v2f64 VR128X:$src0)),
           (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
-def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
+def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcastld64 addr:$src)),
                    immAllZerosV),
           (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
 
@@ -11041,14 +11171,14 @@ multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
   def rr : AVX512<opc, MRMr,
              (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>,
+             [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 timm:$src2))))]>,
              Sched<[sched]>;
   def rm : AVX512<opc, MRMm,
            (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
            [(set _.RC:$dst,(_.VT (OpNode
                                  (_.VT (bitconvert (_.LdFrag addr:$src1))),
-                                 (i8 imm:$src2))))]>,
+                                 (i8 timm:$src2))))]>,
            Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -11075,6 +11205,7 @@ defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq",
 multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
                                 string OpcodeStr, X86FoldableSchedWrite sched,
                                 X86VectorVTInfo _dst, X86VectorVTInfo _src> {
+  let isCommutable = 1 in
   def rr : AVX512BI<opc, MRMSrcReg,
              (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
@@ -11111,7 +11242,7 @@ defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw",
 
 // Transforms to swizzle an immediate to enable better matching when
 // memory operand isn't in the right place.
-def VPTERNLOG321_imm8 : SDNodeXForm<imm, [{
+def VPTERNLOG321_imm8 : SDNodeXForm<timm, [{
   // Convert a VPTERNLOG immediate by swapping operand 0 and operand 2.
   uint8_t Imm = N->getZExtValue();
   // Swap bits 1/4 and 3/6.
@@ -11122,7 +11253,7 @@ def VPTERNLOG321_imm8 : SDNodeXForm<imm, [{
   if (Imm & 0x40) NewImm |= 0x08;
   return getI8Imm(NewImm, SDLoc(N));
 }]>;
-def VPTERNLOG213_imm8 : SDNodeXForm<imm, [{
+def VPTERNLOG213_imm8 : SDNodeXForm<timm, [{
   // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
   uint8_t Imm = N->getZExtValue();
   // Swap bits 2/4 and 3/5.
@@ -11133,7 +11264,7 @@ def VPTERNLOG213_imm8 : SDNodeXForm<imm, [{
   if (Imm & 0x20) NewImm |= 0x08;
   return getI8Imm(NewImm, SDLoc(N));
 }]>;
-def VPTERNLOG132_imm8 : SDNodeXForm<imm, [{
+def VPTERNLOG132_imm8 : SDNodeXForm<timm, [{
   // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
   uint8_t Imm = N->getZExtValue();
   // Swap bits 1/2 and 5/6.
@@ -11144,7 +11275,7 @@ def VPTERNLOG132_imm8 : SDNodeXForm<imm, [{
   if (Imm & 0x40) NewImm |= 0x20;
   return getI8Imm(NewImm, SDLoc(N));
 }]>;
-def VPTERNLOG231_imm8 : SDNodeXForm<imm, [{
+def VPTERNLOG231_imm8 : SDNodeXForm<timm, [{
   // Convert a VPTERNLOG immediate by moving operand 1 to the end.
   uint8_t Imm = N->getZExtValue();
   // Move bits 1->2, 2->4, 3->6, 4->1, 5->3, 6->5
@@ -11157,7 +11288,7 @@ def VPTERNLOG231_imm8 : SDNodeXForm<imm, [{
   if (Imm & 0x40) NewImm |= 0x20;
   return getI8Imm(NewImm, SDLoc(N));
 }]>;
-def VPTERNLOG312_imm8 : SDNodeXForm<imm, [{
+def VPTERNLOG312_imm8 : SDNodeXForm<timm, [{
   // Convert a VPTERNLOG immediate by moving operand 2 to the beginning.
   uint8_t Imm = N->getZExtValue();
   // Move bits 1->4, 2->1, 3->5, 4->2, 5->6, 6->3
@@ -11181,7 +11312,7 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
                       (OpNode (_.VT _.RC:$src1),
                               (_.VT _.RC:$src2),
                               (_.VT _.RC:$src3),
-                              (i8 imm:$src4)), 1, 1>,
+                              (i8 timm:$src4)), 1, 1>,
                       AVX512AIi8Base, EVEX_4V, Sched<[sched]>;
   defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
                     (ins _.RC:$src2, _.MemOp:$src3, u8imm:$src4),
@@ -11189,7 +11320,7 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     (OpNode (_.VT _.RC:$src1),
                             (_.VT _.RC:$src2),
                             (_.VT (bitconvert (_.LdFrag addr:$src3))),
-                            (i8 imm:$src4)), 1, 0>,
+                            (i8 timm:$src4)), 1, 0>,
                     AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
@@ -11198,146 +11329,145 @@ multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
                     "$src2, ${src3}"##_.BroadcastStr##", $src4",
                     (OpNode (_.VT _.RC:$src1),
                             (_.VT _.RC:$src2),
-                            (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
-                            (i8 imm:$src4)), 1, 0>, EVEX_B,
+                            (_.VT (_.BroadcastLdFrag addr:$src3)),
+                            (i8 timm:$src4)), 1, 0>, EVEX_B,
                     AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
                     Sched<[sched.Folded, sched.ReadAfterFold]>;
   }// Constraints = "$src1 = $dst"
 
   // Additional patterns for matching passthru operand in other positions.
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
+                   (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+             _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 imm:$src4)),
+                   (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, _.RC:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
+             _.RC:$src2, _.RC:$src3, (VPTERNLOG213_imm8 timm:$src4))>;
 
   // Additional patterns for matching loads in other positions.
   def : Pat<(_.VT (OpNode (bitconvert (_.LdFrag addr:$src3)),
-                          _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
+                          _.RC:$src2, _.RC:$src1, (i8 timm:$src4))),
             (!cast<Instruction>(Name#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
-                                   addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+                                   addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (OpNode _.RC:$src1,
                           (bitconvert (_.LdFrag addr:$src3)),
-                          _.RC:$src2, (i8 imm:$src4))),
+                          _.RC:$src2, (i8 timm:$src4))),
             (!cast<Instruction>(Name#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
-                                   addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
+                                   addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
 
   // Additional patterns for matching zero masking with loads in other
   // positions.
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
+                    _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src2, (i8 imm:$src4)),
+                    _.RC:$src2, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
 
   // Additional patterns for matching masked loads with different
   // operand orders.
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src2, (i8 imm:$src4)),
+                    _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
+                    _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src2, _.RC:$src1,
-                    (bitconvert (_.LdFrag addr:$src3)), (i8 imm:$src4)),
+                    (bitconvert (_.LdFrag addr:$src3)), (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src1, (i8 imm:$src4)),
+                    _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode (bitconvert (_.LdFrag addr:$src3)),
-                    _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
+                    _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 timm:$src4))>;
 
   // Additional patterns for matching broadcasts in other positions.
-  def : Pat<(_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                          _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
+  def : Pat<(_.VT (OpNode (_.BroadcastLdFrag addr:$src3),
+                          _.RC:$src2, _.RC:$src1, (i8 timm:$src4))),
             (!cast<Instruction>(Name#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
-                                   addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+                                   addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (OpNode _.RC:$src1,
-                          (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                          _.RC:$src2, (i8 imm:$src4))),
+                          (_.BroadcastLdFrag addr:$src3),
+                          _.RC:$src2, (i8 timm:$src4))),
             (!cast<Instruction>(Name#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
-                                   addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
+                                   addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
 
   // Additional patterns for matching zero masking with broadcasts in other
   // positions.
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
+                   (OpNode (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,
              _.KRCWM:$mask, _.RC:$src2, addr:$src3,
-             (VPTERNLOG321_imm8 imm:$src4))>;
+             (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src1,
-                    (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src2, (i8 imm:$src4)),
+                    (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src2, (i8 timm:$src4)),
                    _.ImmAllZerosV)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,
              _.KRCWM:$mask, _.RC:$src2, addr:$src3,
-             (VPTERNLOG132_imm8 imm:$src4))>;
+             (VPTERNLOG132_imm8 timm:$src4))>;
 
   // Additional patterns for matching masked broadcasts with different
   // operand orders.
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode _.RC:$src1,
-                    (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src2, (i8 imm:$src4)),
+                   (OpNode _.RC:$src1, (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
+                   (OpNode (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src2, _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src2, _.RC:$src1,
-                    (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    (i8 imm:$src4)), _.RC:$src1)),
+                    (_.BroadcastLdFrag addr:$src3),
+                    (i8 timm:$src4)), _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
                    (OpNode _.RC:$src2,
-                    (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src1, (i8 imm:$src4)),
+                    (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src1, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 timm:$src4))>;
   def : Pat<(_.VT (vselect _.KRCWM:$mask,
-                   (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
-                    _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
+                   (OpNode (_.BroadcastLdFrag addr:$src3),
+                    _.RC:$src1, _.RC:$src2, (i8 timm:$src4)),
                    _.RC:$src1)),
             (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
-             _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
+             _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 timm:$src4))>;
 }
 
 multiclass avx512_common_ternlog<string OpcodeStr, X86SchedWriteWidths sched,
@@ -11358,6 +11488,209 @@ defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", SchedWriteVecALU,
 defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", SchedWriteVecALU,
                                         avx512vl_i64_info>, VEX_W;
 
+// Patterns to fold bitcasted FP broadcasts.
+// FIXME: Need better DAG canonicalization.
+let Predicates = [HasVLX] in {
+  def : Pat<(X86vpternlog VR128X:$src1, VR128X:$src2,
+                          (bc_v4i32 (v4f32 (X86VBroadcastld32 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v4i32 (v4f32 (X86VBroadcastld32 addr:$src3))),
+                          VR128X:$src2, VR128X:$src1, (i8 timm:$src4)),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR128X:$src1,
+                          (bc_v4i32 (v4f32 (X86VBroadcastld32 addr:$src3))),
+                          VR128X:$src2, (i8 timm:$src4)),
+            (VPTERNLOGDZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(X86vpternlog VR128X:$src1, VR128X:$src2,
+                          (bc_v2i64 (v2f64 (X86VBroadcastld64 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v2i64 (v2f64 (X86VBroadcastld64 addr:$src3))),
+                          VR128X:$src2, VR128X:$src1, (i8 timm:$src4)),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR128X:$src1,
+                          (bc_v2i64 (v2f64 (X86VBroadcastld64 addr:$src3))),
+                          VR128X:$src2, (i8 timm:$src4)),
+            (VPTERNLOGQZ128rmbi VR128X:$src1, VR128X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(X86vpternlog VR256X:$src1, VR256X:$src2,
+                          (bc_v8i32 (v8f32 (X86VBroadcastld32 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v8i32 (v8f32 (X86VBroadcastld32 addr:$src3))),
+                          VR256X:$src2, VR256X:$src1, (i8 timm:$src4)),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR256X:$src1,
+                          (bc_v8i32 (v8f32 (X86VBroadcastld32 addr:$src3))),
+                          VR256X:$src2, (i8 timm:$src4)),
+            (VPTERNLOGDZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(X86vpternlog VR256X:$src1, VR256X:$src2,
+                          (bc_v4i64 (v4f64 (X86VBroadcastld64 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v4i64 (v4f64 (X86VBroadcastld64 addr:$src3))),
+                          VR256X:$src2, VR256X:$src1, (i8 timm:$src4)),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR256X:$src1,
+                          (bc_v4i64 (v4f64 (X86VBroadcastld64 addr:$src3))),
+                          VR256X:$src2, (i8 timm:$src4)),
+            (VPTERNLOGQZ256rmbi VR256X:$src1, VR256X:$src2, addr:$src3,
+                                (VPTERNLOG132_imm8 timm:$src4))>;
+}
+
+let Predicates = [HasAVX512] in {
+  def : Pat<(X86vpternlog VR512:$src1, VR512:$src2,
+                          (bc_v16i32 (v16f32 (X86VBroadcastld32 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v16i32 (v16f32 (X86VBroadcastld32 addr:$src3))),
+                          VR512:$src2, VR512:$src1, (i8 timm:$src4)),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR512:$src1,
+                          (bc_v16i32 (v16f32 (X86VBroadcastld32 addr:$src3))),
+                          VR512:$src2, (i8 timm:$src4)),
+            (VPTERNLOGDZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(X86vpternlog VR512:$src1, VR512:$src2,
+                          (bc_v8i64 (v8f64 (X86VBroadcastld64 addr:$src3))),
+                          (i8 timm:$src4)),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             timm:$src4)>;
+  def : Pat<(X86vpternlog (bc_v8i64 (v8f64 (X86VBroadcastld64 addr:$src3))),
+                          VR512:$src2, VR512:$src1, (i8 timm:$src4)),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(X86vpternlog VR512:$src1,
+                          (bc_v8i64 (v8f64 (X86VBroadcastld64 addr:$src3))),
+                          VR512:$src2, (i8 timm:$src4)),
+            (VPTERNLOGQZrmbi VR512:$src1, VR512:$src2, addr:$src3,
+                             (VPTERNLOG132_imm8 timm:$src4))>;
+}
+
+// Patterns to use VPTERNLOG for vXi16/vXi8 vectors.
+let Predicates = [HasVLX] in {
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1, VR128X:$src2, VR128X:$src3,
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ128rri VR128X:$src1, VR128X:$src2, VR128X:$src3,
+                               timm:$src4)>;
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (loadv16i8 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               timm:$src4)>;
+  def : Pat<(v16i8 (X86vpternlog (loadv16i8 addr:$src3), VR128X:$src2,
+                                 VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i8 (X86vpternlog VR128X:$src1, (loadv16i8 addr:$src3),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1, VR128X:$src2, VR128X:$src3,
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ128rri VR128X:$src1, VR128X:$src2, VR128X:$src3,
+                               timm:$src4)>;
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1, VR128X:$src2,
+                                 (loadv8i16 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               timm:$src4)>;
+  def : Pat<(v8i16 (X86vpternlog (loadv8i16 addr:$src3), VR128X:$src2,
+                                 VR128X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v8i16 (X86vpternlog VR128X:$src1, (loadv8i16 addr:$src3),
+                                 VR128X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ128rmi VR128X:$src1, VR128X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1, VR256X:$src2, VR256X:$src3,
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZ256rri VR256X:$src1, VR256X:$src2, VR256X:$src3,
+                               timm:$src4)>;
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                 (loadv32i8 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               timm:$src4)>;
+  def : Pat<(v32i8 (X86vpternlog (loadv32i8 addr:$src3), VR256X:$src2,
+                                 VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i8 (X86vpternlog VR256X:$src1, (loadv32i8 addr:$src3),
+                                 VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1, VR256X:$src2, VR256X:$src3,
+                                  (i8 timm:$src4))),
+            (VPTERNLOGQZ256rri VR256X:$src1, VR256X:$src2, VR256X:$src3,
+                               timm:$src4)>;
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1, VR256X:$src2,
+                                  (loadv16i16 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               timm:$src4)>;
+  def : Pat<(v16i16 (X86vpternlog (loadv16i16 addr:$src3), VR256X:$src2,
+                                  VR256X:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v16i16 (X86vpternlog VR256X:$src1, (loadv16i16 addr:$src3),
+                                  VR256X:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZ256rmi VR256X:$src1, VR256X:$src2, addr:$src3,
+                               (VPTERNLOG132_imm8 timm:$src4))>;
+}
+
+let Predicates = [HasAVX512] in {
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1, VR512:$src2, VR512:$src3,
+                                 (i8 timm:$src4))),
+            (VPTERNLOGQZrri VR512:$src1, VR512:$src2, VR512:$src3,
+                            timm:$src4)>;
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1, VR512:$src2,
+                                 (loadv64i8 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            timm:$src4)>;
+  def : Pat<(v64i8 (X86vpternlog (loadv64i8 addr:$src3), VR512:$src2,
+                                  VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v64i8 (X86vpternlog VR512:$src1, (loadv64i8 addr:$src3),
+                                 VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            (VPTERNLOG132_imm8 timm:$src4))>;
+
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2, VR512:$src3,
+                                  (i8 timm:$src4))),
+            (VPTERNLOGQZrri VR512:$src1, VR512:$src2, VR512:$src3,
+                            timm:$src4)>;
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, VR512:$src2,
+                                  (loadv32i16 addr:$src3), (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            timm:$src4)>;
+  def : Pat<(v32i16 (X86vpternlog (loadv32i16 addr:$src3), VR512:$src2,
+                                  VR512:$src1, (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            (VPTERNLOG321_imm8 timm:$src4))>;
+  def : Pat<(v32i16 (X86vpternlog VR512:$src1, (loadv32i16 addr:$src3),
+                                 VR512:$src2, (i8 timm:$src4))),
+            (VPTERNLOGQZrmi VR512:$src1, VR512:$src2, addr:$src3,
+                            (VPTERNLOG132_imm8 timm:$src4))>;
+}
+
 // Patterns to implement vnot using vpternlog instead of creating all ones
 // using pcmpeq or vpternlog and then xoring with that. The value 15 is chosen
 // so that the result is only dependent on src0. But we use the same source
@@ -11469,14 +11802,14 @@ multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr,
                         (X86VFixupimm (_.VT _.RC:$src1),
                                       (_.VT _.RC:$src2),
                                       (TblVT.VT _.RC:$src3),
-                                      (i32 imm:$src4))>, Sched<[sched]>;
+                                      (i32 timm:$src4))>, Sched<[sched]>;
     defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
                       (ins _.RC:$src2, _.MemOp:$src3, i32u8imm:$src4),
                       OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
                       (X86VFixupimm (_.VT _.RC:$src1),
                                     (_.VT _.RC:$src2),
                                     (TblVT.VT (bitconvert (TblVT.LdFrag addr:$src3))),
-                                    (i32 imm:$src4))>,
+                                    (i32 timm:$src4))>,
                       Sched<[sched.Folded, sched.ReadAfterFold]>;
     defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
                       (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
@@ -11484,8 +11817,8 @@ multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr,
                     "$src2, ${src3}"##_.BroadcastStr##", $src4",
                       (X86VFixupimm (_.VT _.RC:$src1),
                                     (_.VT _.RC:$src2),
-                                    (TblVT.VT (X86VBroadcast(TblVT.ScalarLdFrag addr:$src3))),
-                                    (i32 imm:$src4))>,
+                                    (TblVT.VT (TblVT.BroadcastLdFrag addr:$src3)),
+                                    (i32 timm:$src4))>,
                     EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
   } // Constraints = "$src1 = $dst"
 }
@@ -11502,7 +11835,7 @@ let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
                       (X86VFixupimmSAE (_.VT _.RC:$src1),
                                        (_.VT _.RC:$src2),
                                        (TblVT.VT _.RC:$src3),
-                                       (i32 imm:$src4))>,
+                                       (i32 timm:$src4))>,
                       EVEX_B, Sched<[sched]>;
   }
 }
@@ -11518,7 +11851,7 @@ multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr,
                       (X86VFixupimms (_.VT _.RC:$src1),
                                      (_.VT _.RC:$src2),
                                      (_src3VT.VT _src3VT.RC:$src3),
-                                     (i32 imm:$src4))>, Sched<[sched]>;
+                                     (i32 timm:$src4))>, Sched<[sched]>;
     defm rrib : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
                       (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
                       OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
@@ -11526,7 +11859,7 @@ multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr,
                       (X86VFixupimmSAEs (_.VT _.RC:$src1),
                                         (_.VT _.RC:$src2),
                                         (_src3VT.VT _src3VT.RC:$src3),
-                                        (i32 imm:$src4))>,
+                                        (i32 timm:$src4))>,
                       EVEX_B, Sched<[sched.Folded, sched.ReadAfterFold]>;
     defm rmi : AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
                      (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
@@ -11535,13 +11868,13 @@ multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr,
                                     (_.VT _.RC:$src2),
                                     (_src3VT.VT (scalar_to_vector
                                               (_src3VT.ScalarLdFrag addr:$src3))),
-                                    (i32 imm:$src4))>,
+                                    (i32 timm:$src4))>,
                      Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
 
 multiclass avx512_fixupimm_packed_all<X86SchedWriteWidths sched,
-                                      AVX512VLVectorVTInfo _Vec, 
+                                      AVX512VLVectorVTInfo _Vec,
                                       AVX512VLVectorVTInfo _Tbl> {
   let Predicates = [HasAVX512] in
     defm Z    : avx512_fixupimm_packed_sae<0x54, "vfixupimm", sched.ZMM,
@@ -11775,7 +12108,7 @@ multiclass VBMI2_shift_var_rmb<bits<8> Op, string OpStr, SDNode OpNode,
               "${src3}"##VTI.BroadcastStr##", $src2",
               "$src2, ${src3}"##VTI.BroadcastStr,
               (OpNode VTI.RC:$src1, VTI.RC:$src2,
-               (VTI.VT (X86VBroadcast (VTI.ScalarLdFrag addr:$src3))))>,
+               (VTI.VT (VTI.BroadcastLdFrag addr:$src3)))>,
               AVX512FMA3Base, EVEX_B,
               Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -11872,8 +12205,7 @@ multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode,
                                    OpStr, "${src3}"##VTI.BroadcastStr##", $src2",
                                    "$src2, ${src3}"##VTI.BroadcastStr,
                                    (OpNode VTI.RC:$src1, VTI.RC:$src2,
-                                    (VTI.VT (X86VBroadcast
-                                             (VTI.ScalarLdFrag addr:$src3))))>,
+                                    (VTI.VT (VTI.BroadcastLdFrag addr:$src3)))>,
                                    EVEX_4V, EVEX_CD8<32, CD8VF>, EVEX_B,
                                    T8PD, Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
@@ -12009,8 +12341,8 @@ multiclass GF2P8AFFINE_avx512_rmb_imm<bits<8> Op, string OpStr, SDNode OpNode,
                 OpStr, "$src3, ${src2}"##BcstVTI.BroadcastStr##", $src1",
                 "$src1, ${src2}"##BcstVTI.BroadcastStr##", $src3",
                 (OpNode (VTI.VT VTI.RC:$src1),
-                 (bitconvert (BcstVTI.VT (X86VBroadcast (loadi64 addr:$src2)))),
-                 (i8 imm:$src3))>, EVEX_B,
+                 (bitconvert (BcstVTI.VT (X86VBroadcastld64 addr:$src2))),
+                 (i8 timm:$src3))>, EVEX_B,
                  Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -12121,7 +12453,7 @@ multiclass avx512_vp2intersect_modes<X86VectorVTInfo _> {
                   !strconcat("vp2intersect", _.Suffix, "\t{${src2}", _.BroadcastStr,
                              ", $src1, $dst|$dst, $src1, ${src2}", _.BroadcastStr ,"}"),
                   [(set _.KRPC:$dst, (X86vp2intersect
-                             _.RC:$src1, (_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src2)))))]>,
+                             _.RC:$src1, (_.VT (_.BroadcastLdFrag addr:$src2))))]>,
                   EVEX_4V, T8XD, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>;
 }
 
@@ -12222,12 +12554,12 @@ let Predicates = [HasBF16, HasVLX] in {
             (VCVTNEPS2BF16Z128rmkz VK4WM:$mask, addr:$src)>;
 
   def : Pat<(v8i16 (X86cvtneps2bf16 (v4f32
-                                     (X86VBroadcast (loadf32 addr:$src))))),
+                                     (X86VBroadcastld32 addr:$src)))),
             (VCVTNEPS2BF16Z128rmb addr:$src)>;
-  def : Pat<(X86mcvtneps2bf16 (v4f32 (X86VBroadcast (loadf32 addr:$src))),
+  def : Pat<(X86mcvtneps2bf16 (v4f32 (X86VBroadcastld32 addr:$src)),
                               (v8i16 VR128X:$src0), VK4WM:$mask),
             (VCVTNEPS2BF16Z128rmbk VR128X:$src0, VK4WM:$mask, addr:$src)>;
-  def : Pat<(X86mcvtneps2bf16 (v4f32 (X86VBroadcast (loadf32 addr:$src))),
+  def : Pat<(X86mcvtneps2bf16 (v4f32 (X86VBroadcastld32 addr:$src)),
                               v8i16x_info.ImmAllZerosV, VK4WM:$mask),
             (VCVTNEPS2BF16Z128rmbkz VK4WM:$mask, addr:$src)>;
 }
@@ -12254,7 +12586,7 @@ multiclass avx512_dpbf16ps_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
                   !strconcat("${src3}", _.BroadcastStr,", $src2"),
                   !strconcat("$src2, ${src3}", _.BroadcastStr),
                   (_.VT (OpNode _.RC:$src1, _.RC:$src2,
-                  (src_v.VT (X86VBroadcast(src_v.ScalarLdFrag addr:$src3)))))>,
+                  (src_v.VT (src_v.BroadcastLdFrag addr:$src3))))>,
                   EVEX_B, EVEX_4V;
 
 }
diff --git a/llvm/lib/Target/X86/X86InstrCMovSetCC.td b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
index 099f6aa8d8bb9..330b8c7a8a43a 100644
--- a/llvm/lib/Target/X86/X86InstrCMovSetCC.td
+++ b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
@@ -20,19 +20,19 @@ let Uses = [EFLAGS], Predicates = [HasCMov], Constraints = "$src1 = $dst",
     : I<0x40, MRMSrcRegCC, (outs GR16:$dst), (ins GR16:$src1, GR16:$src2, ccode:$cond),
         "cmov${cond}{w}\t{$src2, $dst|$dst, $src2}",
         [(set GR16:$dst,
-              (X86cmov GR16:$src1, GR16:$src2, imm:$cond, EFLAGS))]>,
+              (X86cmov GR16:$src1, GR16:$src2, timm:$cond, EFLAGS))]>,
               TB, OpSize16;
   def CMOV32rr
     : I<0x40, MRMSrcRegCC, (outs GR32:$dst), (ins GR32:$src1, GR32:$src2, ccode:$cond),
         "cmov${cond}{l}\t{$src2, $dst|$dst, $src2}",
         [(set GR32:$dst,
-              (X86cmov GR32:$src1, GR32:$src2, imm:$cond, EFLAGS))]>,
+              (X86cmov GR32:$src1, GR32:$src2, timm:$cond, EFLAGS))]>,
               TB, OpSize32;
   def CMOV64rr
     :RI<0x40, MRMSrcRegCC, (outs GR64:$dst), (ins GR64:$src1, GR64:$src2, ccode:$cond),
         "cmov${cond}{q}\t{$src2, $dst|$dst, $src2}",
         [(set GR64:$dst,
-              (X86cmov GR64:$src1, GR64:$src2, imm:$cond, EFLAGS))]>, TB;
+              (X86cmov GR64:$src1, GR64:$src2, timm:$cond, EFLAGS))]>, TB;
 }
 
 let Uses = [EFLAGS], Predicates = [HasCMov], Constraints = "$src1 = $dst",
@@ -41,29 +41,46 @@ let Uses = [EFLAGS], Predicates = [HasCMov], Constraints = "$src1 = $dst",
     : I<0x40, MRMSrcMemCC, (outs GR16:$dst), (ins GR16:$src1, i16mem:$src2, ccode:$cond),
         "cmov${cond}{w}\t{$src2, $dst|$dst, $src2}",
         [(set GR16:$dst, (X86cmov GR16:$src1, (loadi16 addr:$src2),
-                                  imm:$cond, EFLAGS))]>, TB, OpSize16;
+                                  timm:$cond, EFLAGS))]>, TB, OpSize16;
   def CMOV32rm
     : I<0x40, MRMSrcMemCC, (outs GR32:$dst), (ins GR32:$src1, i32mem:$src2, ccode:$cond),
         "cmov${cond}{l}\t{$src2, $dst|$dst, $src2}",
         [(set GR32:$dst, (X86cmov GR32:$src1, (loadi32 addr:$src2),
-                                  imm:$cond, EFLAGS))]>, TB, OpSize32;
+                                  timm:$cond, EFLAGS))]>, TB, OpSize32;
   def CMOV64rm
     :RI<0x40, MRMSrcMemCC, (outs GR64:$dst), (ins GR64:$src1, i64mem:$src2, ccode:$cond),
         "cmov${cond}{q}\t{$src2, $dst|$dst, $src2}",
         [(set GR64:$dst, (X86cmov GR64:$src1, (loadi64 addr:$src2),
-                                  imm:$cond, EFLAGS))]>, TB;
+                                  timm:$cond, EFLAGS))]>, TB;
 } // Uses = [EFLAGS], Predicates = [HasCMov], Constraints = "$src1 = $dst"
 } // isCodeGenOnly = 1, ForceDisassemble = 1
 
+def inv_cond_XFORM : SDNodeXForm<imm, [{
+  X86::CondCode CC = static_cast<X86::CondCode>(N->getZExtValue());
+  return CurDAG->getTargetConstant(X86::GetOppositeBranchCondition(CC),
+                                   SDLoc(N), MVT::i8);
+}]>;
+
+// Conditional moves with folded loads with operands swapped and conditions
+// inverted.
+let Predicates = [HasCMov] in {
+  def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, timm:$cond, EFLAGS),
+            (CMOV16rm GR16:$src2, addr:$src1, (inv_cond_XFORM timm:$cond))>;
+  def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, timm:$cond, EFLAGS),
+            (CMOV32rm GR32:$src2, addr:$src1, (inv_cond_XFORM timm:$cond))>;
+  def : Pat<(X86cmov (loadi64 addr:$src1), GR64:$src2, timm:$cond, EFLAGS),
+            (CMOV64rm GR64:$src2, addr:$src1, (inv_cond_XFORM timm:$cond))>;
+}
+
 // SetCC instructions.
 let Uses = [EFLAGS], isCodeGenOnly = 1, ForceDisassemble = 1 in {
   def SETCCr : I<0x90, MRMXrCC, (outs GR8:$dst), (ins ccode:$cond),
                 "set${cond}\t$dst",
-                [(set GR8:$dst, (X86setcc imm:$cond, EFLAGS))]>,
+                [(set GR8:$dst, (X86setcc timm:$cond, EFLAGS))]>,
                 TB, Sched<[WriteSETCC]>;
   def SETCCm : I<0x90, MRMXmCC, (outs), (ins i8mem:$dst, ccode:$cond),
                 "set${cond}\t$dst",
-                [(store (X86setcc imm:$cond, EFLAGS), addr:$dst)]>,
+                [(store (X86setcc timm:$cond, EFLAGS), addr:$dst)]>,
                 TB, Sched<[WriteSETCCStore]>;
 } // Uses = [EFLAGS]
 
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 07d1762f6d007..78d8dd3c0d031 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -542,7 +542,7 @@ multiclass CMOVrr_PSEUDO<RegisterClass RC, ValueType VT> {
   def CMOV#NAME  : I<0, Pseudo,
                     (outs RC:$dst), (ins RC:$t, RC:$f, i8imm:$cond),
                     "#CMOV_"#NAME#" PSEUDO!",
-                    [(set RC:$dst, (VT (X86cmov RC:$t, RC:$f, imm:$cond,
+                    [(set RC:$dst, (VT (X86cmov RC:$t, RC:$f, timm:$cond,
                                                 EFLAGS)))]>;
 }
 
@@ -593,66 +593,66 @@ let usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS] in {
   defm _VK64   : CMOVrr_PSEUDO<VK64, v64i1>;
 } // usesCustomInserter = 1, hasNoSchedulingInfo = 1, Uses = [EFLAGS]
 
-def : Pat<(f128 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
+def : Pat<(f128 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
 
 let Predicates = [NoVLX] in {
-  def : Pat<(v16i8 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
-  def : Pat<(v8i16 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
-  def : Pat<(v4i32 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
-  def : Pat<(v4f32 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
-  def : Pat<(v2f64 (X86cmov VR128:$t, VR128:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128 VR128:$t, VR128:$f, imm:$cond)>;
-
-  def : Pat<(v32i8 (X86cmov VR256:$t, VR256:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256 VR256:$t, VR256:$f, imm:$cond)>;
-  def : Pat<(v16i16 (X86cmov VR256:$t, VR256:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256 VR256:$t, VR256:$f, imm:$cond)>;
-  def : Pat<(v8i32 (X86cmov VR256:$t, VR256:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256 VR256:$t, VR256:$f, imm:$cond)>;
-  def : Pat<(v8f32 (X86cmov VR256:$t, VR256:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256 VR256:$t, VR256:$f, imm:$cond)>;
-  def : Pat<(v4f64 (X86cmov VR256:$t, VR256:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256 VR256:$t, VR256:$f, imm:$cond)>;
+  def : Pat<(v16i8 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
+  def : Pat<(v8i16 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
+  def : Pat<(v4i32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
+  def : Pat<(v4f32 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
+  def : Pat<(v2f64 (X86cmov VR128:$t, VR128:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128 VR128:$t, VR128:$f, timm:$cond)>;
+
+  def : Pat<(v32i8 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>;
+  def : Pat<(v16i16 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>;
+  def : Pat<(v8i32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>;
+  def : Pat<(v8f32 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>;
+  def : Pat<(v4f64 (X86cmov VR256:$t, VR256:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256 VR256:$t, VR256:$f, timm:$cond)>;
 }
 let Predicates = [HasVLX] in {
-  def : Pat<(v16i8 (X86cmov VR128X:$t, VR128X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128X VR128X:$t, VR128X:$f, imm:$cond)>;
-  def : Pat<(v8i16 (X86cmov VR128X:$t, VR128X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128X VR128X:$t, VR128X:$f, imm:$cond)>;
-  def : Pat<(v4i32 (X86cmov VR128X:$t, VR128X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128X VR128X:$t, VR128X:$f, imm:$cond)>;
-  def : Pat<(v4f32 (X86cmov VR128X:$t, VR128X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128X VR128X:$t, VR128X:$f, imm:$cond)>;
-  def : Pat<(v2f64 (X86cmov VR128X:$t, VR128X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR128X VR128X:$t, VR128X:$f, imm:$cond)>;
-
-  def : Pat<(v32i8 (X86cmov VR256X:$t, VR256X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256X VR256X:$t, VR256X:$f, imm:$cond)>;
-  def : Pat<(v16i16 (X86cmov VR256X:$t, VR256X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256X VR256X:$t, VR256X:$f, imm:$cond)>;
-  def : Pat<(v8i32 (X86cmov VR256X:$t, VR256X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256X VR256X:$t, VR256X:$f, imm:$cond)>;
-  def : Pat<(v8f32 (X86cmov VR256X:$t, VR256X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256X VR256X:$t, VR256X:$f, imm:$cond)>;
-  def : Pat<(v4f64 (X86cmov VR256X:$t, VR256X:$f, imm:$cond, EFLAGS)),
-            (CMOV_VR256X VR256X:$t, VR256X:$f, imm:$cond)>;
+  def : Pat<(v16i8 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>;
+  def : Pat<(v8i16 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>;
+  def : Pat<(v4i32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>;
+  def : Pat<(v4f32 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>;
+  def : Pat<(v2f64 (X86cmov VR128X:$t, VR128X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR128X VR128X:$t, VR128X:$f, timm:$cond)>;
+
+  def : Pat<(v32i8 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>;
+  def : Pat<(v16i16 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>;
+  def : Pat<(v8i32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>;
+  def : Pat<(v8f32 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>;
+  def : Pat<(v4f64 (X86cmov VR256X:$t, VR256X:$f, timm:$cond, EFLAGS)),
+            (CMOV_VR256X VR256X:$t, VR256X:$f, timm:$cond)>;
 }
 
-def : Pat<(v64i8 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR512 VR512:$t, VR512:$f, imm:$cond)>;
-def : Pat<(v32i16 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR512 VR512:$t, VR512:$f, imm:$cond)>;
-def : Pat<(v16i32 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR512 VR512:$t, VR512:$f, imm:$cond)>;
-def : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR512 VR512:$t, VR512:$f, imm:$cond)>;
-def : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, imm:$cond, EFLAGS)),
-          (CMOV_VR512 VR512:$t, VR512:$f, imm:$cond)>;
+def : Pat<(v64i8 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+def : Pat<(v32i16 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+def : Pat<(v16i32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+def : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+def : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
+          (CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
 
 //===----------------------------------------------------------------------===//
 // Normal-Instructions-With-Lock-Prefix Pseudo Instructions
@@ -1276,23 +1276,6 @@ def : Pat<(X86cmp GR32:$src1, 0),
 def : Pat<(X86cmp GR64:$src1, 0),
           (TEST64rr GR64:$src1, GR64:$src1)>;
 
-def inv_cond_XFORM : SDNodeXForm<imm, [{
-  X86::CondCode CC = static_cast<X86::CondCode>(N->getZExtValue());
-  return CurDAG->getTargetConstant(X86::GetOppositeBranchCondition(CC),
-                                   SDLoc(N), MVT::i8);
-}]>;
-
-// Conditional moves with folded loads with operands swapped and conditions
-// inverted.
-let Predicates = [HasCMov] in {
-  def : Pat<(X86cmov (loadi16 addr:$src1), GR16:$src2, imm:$cond, EFLAGS),
-            (CMOV16rm GR16:$src2, addr:$src1, (inv_cond_XFORM imm:$cond))>;
-  def : Pat<(X86cmov (loadi32 addr:$src1), GR32:$src2, imm:$cond, EFLAGS),
-            (CMOV32rm GR32:$src2, addr:$src1, (inv_cond_XFORM imm:$cond))>;
-  def : Pat<(X86cmov (loadi64 addr:$src1), GR64:$src2, imm:$cond, EFLAGS),
-            (CMOV64rm GR64:$src2, addr:$src1, (inv_cond_XFORM imm:$cond))>;
-}
-
 // zextload bool -> zextload byte
 // i1 stored in one byte in zero-extended form.
 // Upper bits cleanup should be executed before Store.
diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td
index ce961827d0519..e1e6eea59884a 100644
--- a/llvm/lib/Target/X86/X86InstrControl.td
+++ b/llvm/lib/Target/X86/X86InstrControl.td
@@ -75,7 +75,7 @@ let isBranch = 1, isTerminator = 1, Uses = [EFLAGS], SchedRW = [WriteJump],
   def JCC_1 : Ii8PCRel <0x70, AddCCFrm, (outs),
                         (ins brtarget8:$dst, ccode:$cond),
                         "j${cond}\t$dst",
-                        [(X86brcond bb:$dst, imm:$cond, EFLAGS)]>;
+                        [(X86brcond bb:$dst, timm:$cond, EFLAGS)]>;
   let hasSideEffects = 0 in {
     def JCC_2 : Ii16PCRel<0x80, AddCCFrm, (outs),
                           (ins brtarget16:$dst, ccode:$cond),
diff --git a/llvm/lib/Target/X86/X86InstrFoldTables.cpp b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
index 0f89a8053c74d..f3b286e0375c6 100644
--- a/llvm/lib/Target/X86/X86InstrFoldTables.cpp
+++ b/llvm/lib/Target/X86/X86InstrFoldTables.cpp
@@ -292,6 +292,8 @@ static const X86MemoryFoldTableEntry MemoryFoldTable0[] = {
   { X86::JMP32r_NT,           X86::JMP32m_NT,           TB_FOLDED_LOAD },
   { X86::JMP64r,              X86::JMP64m,              TB_FOLDED_LOAD },
   { X86::JMP64r_NT,           X86::JMP64m_NT,           TB_FOLDED_LOAD },
+  { X86::MMX_MOVD64from64rr,  X86::MMX_MOVD64from64rm,  TB_FOLDED_STORE | TB_NO_REVERSE },
+  { X86::MMX_MOVD64grr,       X86::MMX_MOVD64mr,        TB_FOLDED_STORE | TB_NO_REVERSE },
   { X86::MOV16ri,             X86::MOV16mi,             TB_FOLDED_STORE },
   { X86::MOV16rr,             X86::MOV16mr,             TB_FOLDED_STORE },
   { X86::MOV32ri,             X86::MOV32mi,             TB_FOLDED_STORE },
@@ -5372,6 +5374,18 @@ static const X86MemoryFoldTableEntry BroadcastFoldTable2[] = {
   { X86::VPORQZ128rr,    X86::VPORQZ128rmb,    TB_BCAST_Q },
   { X86::VPORQZ256rr,    X86::VPORQZ256rmb,    TB_BCAST_Q },
   { X86::VPORQZrr,       X86::VPORQZrmb,       TB_BCAST_Q },
+  { X86::VPTESTMDZ128rr, X86::VPTESTMDZ128rmb, TB_BCAST_D },
+  { X86::VPTESTMDZ256rr, X86::VPTESTMDZ256rmb, TB_BCAST_D },
+  { X86::VPTESTMDZrr,    X86::VPTESTMDZrmb,    TB_BCAST_D },
+  { X86::VPTESTMQZ128rr, X86::VPTESTMQZ128rmb, TB_BCAST_Q },
+  { X86::VPTESTMQZ256rr, X86::VPTESTMQZ256rmb, TB_BCAST_Q },
+  { X86::VPTESTMQZrr,    X86::VPTESTMQZrmb,    TB_BCAST_Q },
+  { X86::VPTESTNMDZ128rr,X86::VPTESTNMDZ128rmb,TB_BCAST_D },
+  { X86::VPTESTNMDZ256rr,X86::VPTESTNMDZ256rmb,TB_BCAST_D },
+  { X86::VPTESTNMDZrr,   X86::VPTESTNMDZrmb,   TB_BCAST_D },
+  { X86::VPTESTNMQZ128rr,X86::VPTESTNMQZ128rmb,TB_BCAST_Q },
+  { X86::VPTESTNMQZ256rr,X86::VPTESTNMQZ256rmb,TB_BCAST_Q },
+  { X86::VPTESTNMQZrr,   X86::VPTESTNMQZrmb,   TB_BCAST_Q },
   { X86::VPXORDZ128rr,   X86::VPXORDZ128rmb,   TB_BCAST_D },
   { X86::VPXORDZ256rr,   X86::VPXORDZ256rmb,   TB_BCAST_D },
   { X86::VPXORDZrr,      X86::VPXORDZrmb,      TB_BCAST_D },
diff --git a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
index d75b492594bf5..de6f8a81dff65 100644
--- a/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/llvm/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -103,6 +103,8 @@ def X86vzld  : SDNode<"X86ISD::VZEXT_LOAD", SDTLoad,
                       [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 def X86vextractst  : SDNode<"X86ISD::VEXTRACT_STORE", SDTStore,
                      [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def X86VBroadcastld  : SDNode<"X86ISD::VBROADCAST_LOAD", SDTLoad,
+                      [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
 def SDTVtrunc    : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
                                         SDTCisInt<0>, SDTCisInt<1>,
@@ -954,6 +956,26 @@ def X86vextractstore64 : PatFrag<(ops node:$val, node:$ptr),
   return cast<MemIntrinsicSDNode>(N)->getMemoryVT().getStoreSize() == 8;
 }]>;
 
+def X86VBroadcastld8 : PatFrag<(ops node:$src),
+                               (X86VBroadcastld node:$src), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT().getStoreSize() == 1;
+}]>;
+
+def X86VBroadcastld16 : PatFrag<(ops node:$src),
+                                (X86VBroadcastld node:$src), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT().getStoreSize() == 2;
+}]>;
+
+def X86VBroadcastld32 : PatFrag<(ops node:$src),
+                                (X86VBroadcastld node:$src), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT().getStoreSize() == 4;
+}]>;
+
+def X86VBroadcastld64 : PatFrag<(ops node:$src),
+                                (X86VBroadcastld node:$src), [{
+  return cast<MemIntrinsicSDNode>(N)->getMemoryVT().getStoreSize() == 8;
+}]>;
+
 
 def fp32imm0 : PatLeaf<(f32 fpimm), [{
   return N->isExactlyValue(+0.0);
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 1831986e2ab8e..3c68ac2291f20 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -1122,6 +1122,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     return nullptr;
   case X86::SUB32ri8:
   case X86::SUB32ri: {
+    if (!MI.getOperand(2).isImm())
+      return nullptr;
     int64_t Imm = MI.getOperand(2).getImm();
     if (!isInt<32>(-Imm))
       return nullptr;
@@ -1148,6 +1150,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
 
   case X86::SUB64ri8:
   case X86::SUB64ri32: {
+    if (!MI.getOperand(2).isImm())
+      return nullptr;
     int64_t Imm = MI.getOperand(2).getImm();
     if (!isInt<32>(-Imm))
       return nullptr;
@@ -1177,40 +1181,62 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
   case X86::VMOVUPDZrmk:      case X86::VMOVAPDZrmk:
   case X86::VMOVUPSZ128rmk:   case X86::VMOVAPSZ128rmk:
   case X86::VMOVUPSZ256rmk:   case X86::VMOVAPSZ256rmk:
-  case X86::VMOVUPSZrmk:      case X86::VMOVAPSZrmk: {
+  case X86::VMOVUPSZrmk:      case X86::VMOVAPSZrmk:
+  case X86::VBROADCASTSDZ256mk:
+  case X86::VBROADCASTSDZmk:
+  case X86::VBROADCASTSSZ128mk:
+  case X86::VBROADCASTSSZ256mk:
+  case X86::VBROADCASTSSZmk:
+  case X86::VPBROADCASTDZ128mk:
+  case X86::VPBROADCASTDZ256mk:
+  case X86::VPBROADCASTDZmk:
+  case X86::VPBROADCASTQZ128mk:
+  case X86::VPBROADCASTQZ256mk:
+  case X86::VPBROADCASTQZmk: {
     unsigned Opc;
     switch (MIOpc) {
     default: llvm_unreachable("Unreachable!");
-    case X86::VMOVDQU8Z128rmk:  Opc = X86::VPBLENDMBZ128rmk; break;
-    case X86::VMOVDQU8Z256rmk:  Opc = X86::VPBLENDMBZ256rmk; break;
-    case X86::VMOVDQU8Zrmk:     Opc = X86::VPBLENDMBZrmk;    break;
-    case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk; break;
-    case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk; break;
-    case X86::VMOVDQU16Zrmk:    Opc = X86::VPBLENDMWZrmk;    break;
-    case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break;
-    case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break;
-    case X86::VMOVDQU32Zrmk:    Opc = X86::VPBLENDMDZrmk;    break;
-    case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break;
-    case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break;
-    case X86::VMOVDQU64Zrmk:    Opc = X86::VPBLENDMQZrmk;    break;
-    case X86::VMOVUPDZ128rmk:   Opc = X86::VBLENDMPDZ128rmk; break;
-    case X86::VMOVUPDZ256rmk:   Opc = X86::VBLENDMPDZ256rmk; break;
-    case X86::VMOVUPDZrmk:      Opc = X86::VBLENDMPDZrmk;    break;
-    case X86::VMOVUPSZ128rmk:   Opc = X86::VBLENDMPSZ128rmk; break;
-    case X86::VMOVUPSZ256rmk:   Opc = X86::VBLENDMPSZ256rmk; break;
-    case X86::VMOVUPSZrmk:      Opc = X86::VBLENDMPSZrmk;    break;
-    case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk; break;
-    case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk; break;
-    case X86::VMOVDQA32Zrmk:    Opc = X86::VPBLENDMDZrmk;    break;
-    case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk; break;
-    case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk; break;
-    case X86::VMOVDQA64Zrmk:    Opc = X86::VPBLENDMQZrmk;    break;
-    case X86::VMOVAPDZ128rmk:   Opc = X86::VBLENDMPDZ128rmk; break;
-    case X86::VMOVAPDZ256rmk:   Opc = X86::VBLENDMPDZ256rmk; break;
-    case X86::VMOVAPDZrmk:      Opc = X86::VBLENDMPDZrmk;    break;
-    case X86::VMOVAPSZ128rmk:   Opc = X86::VBLENDMPSZ128rmk; break;
-    case X86::VMOVAPSZ256rmk:   Opc = X86::VBLENDMPSZ256rmk; break;
-    case X86::VMOVAPSZrmk:      Opc = X86::VBLENDMPSZrmk;    break;
+    case X86::VMOVDQU8Z128rmk:    Opc = X86::VPBLENDMBZ128rmk; break;
+    case X86::VMOVDQU8Z256rmk:    Opc = X86::VPBLENDMBZ256rmk; break;
+    case X86::VMOVDQU8Zrmk:       Opc = X86::VPBLENDMBZrmk;    break;
+    case X86::VMOVDQU16Z128rmk:   Opc = X86::VPBLENDMWZ128rmk; break;
+    case X86::VMOVDQU16Z256rmk:   Opc = X86::VPBLENDMWZ256rmk; break;
+    case X86::VMOVDQU16Zrmk:      Opc = X86::VPBLENDMWZrmk;    break;
+    case X86::VMOVDQU32Z128rmk:   Opc = X86::VPBLENDMDZ128rmk; break;
+    case X86::VMOVDQU32Z256rmk:   Opc = X86::VPBLENDMDZ256rmk; break;
+    case X86::VMOVDQU32Zrmk:      Opc = X86::VPBLENDMDZrmk;    break;
+    case X86::VMOVDQU64Z128rmk:   Opc = X86::VPBLENDMQZ128rmk; break;
+    case X86::VMOVDQU64Z256rmk:   Opc = X86::VPBLENDMQZ256rmk; break;
+    case X86::VMOVDQU64Zrmk:      Opc = X86::VPBLENDMQZrmk;    break;
+    case X86::VMOVUPDZ128rmk:     Opc = X86::VBLENDMPDZ128rmk; break;
+    case X86::VMOVUPDZ256rmk:     Opc = X86::VBLENDMPDZ256rmk; break;
+    case X86::VMOVUPDZrmk:        Opc = X86::VBLENDMPDZrmk;    break;
+    case X86::VMOVUPSZ128rmk:     Opc = X86::VBLENDMPSZ128rmk; break;
+    case X86::VMOVUPSZ256rmk:     Opc = X86::VBLENDMPSZ256rmk; break;
+    case X86::VMOVUPSZrmk:        Opc = X86::VBLENDMPSZrmk;    break;
+    case X86::VMOVDQA32Z128rmk:   Opc = X86::VPBLENDMDZ128rmk; break;
+    case X86::VMOVDQA32Z256rmk:   Opc = X86::VPBLENDMDZ256rmk; break;
+    case X86::VMOVDQA32Zrmk:      Opc = X86::VPBLENDMDZrmk;    break;
+    case X86::VMOVDQA64Z128rmk:   Opc = X86::VPBLENDMQZ128rmk; break;
+    case X86::VMOVDQA64Z256rmk:   Opc = X86::VPBLENDMQZ256rmk; break;
+    case X86::VMOVDQA64Zrmk:      Opc = X86::VPBLENDMQZrmk;    break;
+    case X86::VMOVAPDZ128rmk:     Opc = X86::VBLENDMPDZ128rmk; break;
+    case X86::VMOVAPDZ256rmk:     Opc = X86::VBLENDMPDZ256rmk; break;
+    case X86::VMOVAPDZrmk:        Opc = X86::VBLENDMPDZrmk;    break;
+    case X86::VMOVAPSZ128rmk:     Opc = X86::VBLENDMPSZ128rmk; break;
+    case X86::VMOVAPSZ256rmk:     Opc = X86::VBLENDMPSZ256rmk; break;
+    case X86::VMOVAPSZrmk:        Opc = X86::VBLENDMPSZrmk;    break;
+    case X86::VBROADCASTSDZ256mk: Opc = X86::VBLENDMPDZ256rmbk; break;
+    case X86::VBROADCASTSDZmk:    Opc = X86::VBLENDMPDZrmbk;    break;
+    case X86::VBROADCASTSSZ128mk: Opc = X86::VBLENDMPSZ128rmbk; break;
+    case X86::VBROADCASTSSZ256mk: Opc = X86::VBLENDMPSZ256rmbk; break;
+    case X86::VBROADCASTSSZmk:    Opc = X86::VBLENDMPSZrmbk;    break;
+    case X86::VPBROADCASTDZ128mk: Opc = X86::VPBLENDMDZ128rmbk; break;
+    case X86::VPBROADCASTDZ256mk: Opc = X86::VPBLENDMDZ256rmbk; break;
+    case X86::VPBROADCASTDZmk:    Opc = X86::VPBLENDMDZrmbk;    break;
+    case X86::VPBROADCASTQZ128mk: Opc = X86::VPBLENDMQZ128rmbk; break;
+    case X86::VPBROADCASTQZ256mk: Opc = X86::VPBLENDMQZ256rmbk; break;
+    case X86::VPBROADCASTQZmk:    Opc = X86::VPBLENDMQZrmbk;    break;
     }
 
     NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
@@ -1224,6 +1250,7 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
               .add(MI.getOperand(7));
     break;
   }
+
   case X86::VMOVDQU8Z128rrk:
   case X86::VMOVDQU8Z256rrk:
   case X86::VMOVDQU8Zrrk:
@@ -1720,6 +1747,27 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
     return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
                                                    OpIdx1, OpIdx2);
   }
+  case X86::VCMPSDZrr:
+  case X86::VCMPSSZrr:
+  case X86::VCMPPDZrri:
+  case X86::VCMPPSZrri:
+  case X86::VCMPPDZ128rri:
+  case X86::VCMPPSZ128rri:
+  case X86::VCMPPDZ256rri:
+  case X86::VCMPPSZ256rri:
+  case X86::VCMPPDZrrik:
+  case X86::VCMPPSZrrik:
+  case X86::VCMPPDZ128rrik:
+  case X86::VCMPPSZ128rrik:
+  case X86::VCMPPDZ256rrik:
+  case X86::VCMPPSZ256rrik: {
+    unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x1f;
+    Imm = X86::getSwappedVCMPImm(Imm);
+    auto &WorkingMI = cloneIfNew(MI);
+    WorkingMI.getOperand(MI.getNumOperands() - 1).setImm(Imm);
+    return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
+                                                   OpIdx1, OpIdx2);
+  }
   case X86::VPERM2F128rr:
   case X86::VPERM2I128rr: {
     // Flip permute source immediate.
@@ -1926,7 +1974,8 @@ X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
   return true;
 }
 
-bool X86InstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+bool X86InstrInfo::findCommutedOpIndices(const MachineInstr &MI,
+                                         unsigned &SrcOpIdx1,
                                          unsigned &SrcOpIdx2) const {
   const MCInstrDesc &Desc = MI.getDesc();
   if (!Desc.isCommutable())
@@ -1963,17 +2012,23 @@ bool X86InstrInfo::findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
     // Ordered/Unordered/Equal/NotEqual tests
     unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
     switch (Imm) {
+    default:
+      // EVEX versions can be commuted.
+      if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
+        break;
+      return false;
     case 0x00: // EQUAL
     case 0x03: // UNORDERED
     case 0x04: // NOT EQUAL
     case 0x07: // ORDERED
-      // The indices of the commutable operands are 1 and 2 (or 2 and 3
-      // when masked).
-      // Assign them to the returned operand indices here.
-      return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
-                                  2 + OpOffset);
+      break;
     }
-    return false;
+
+    // The indices of the commutable operands are 1 and 2 (or 2 and 3
+    // when masked).
+    // Assign them to the returned operand indices here.
+    return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
+                                2 + OpOffset);
   }
   case X86::MOVSSrr:
     // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
@@ -2270,7 +2325,7 @@ unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) {
   }
 }
 
-/// Get the VPCMP immediate if the opcodes are swapped.
+/// Get the VPCMP immediate if the operands are swapped.
 unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
   switch (Imm) {
   default: llvm_unreachable("Unreachable!");
@@ -2288,7 +2343,7 @@ unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
   return Imm;
 }
 
-/// Get the VPCOM immediate if the opcodes are swapped.
+/// Get the VPCOM immediate if the operands are swapped.
 unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
   switch (Imm) {
   default: llvm_unreachable("Unreachable!");
@@ -2306,6 +2361,23 @@ unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
   return Imm;
 }
 
+/// Get the VCMP immediate if the operands are swapped.
+unsigned X86::getSwappedVCMPImm(unsigned Imm) {
+  // Only need the lower 2 bits to distinquish.
+  switch (Imm & 0x3) {
+  default: llvm_unreachable("Unreachable!");
+  case 0x00: case 0x03:
+    // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
+    break;
+  case 0x01: case 0x02:
+    // Need to toggle bits 3:0. Bit 4 stays the same.
+    Imm ^= 0xf;
+    break;
+  }
+
+  return Imm;
+}
+
 bool X86InstrInfo::isUnpredicatedTerminator(const MachineInstr &MI) const {
   if (!MI.isTerminator()) return false;
 
@@ -4290,7 +4362,10 @@ unsigned X86InstrInfo::getPartialRegUpdateClearance(
 
 // Return true for any instruction the copies the high bits of the first source
 // operand into the unused high bits of the destination operand.
-static bool hasUndefRegUpdate(unsigned Opcode, bool ForLoadFold = false) {
+static bool hasUndefRegUpdate(unsigned Opcode, unsigned &OpNum,
+                              bool ForLoadFold = false) {
+  // Set the OpNum parameter to the first source operand.
+  OpNum = 1;
   switch (Opcode) {
   case X86::VCVTSI2SSrr:
   case X86::VCVTSI2SSrm:
@@ -4449,6 +4524,14 @@ static bool hasUndefRegUpdate(unsigned Opcode, bool ForLoadFold = false) {
   case X86::VSQRTSDZm:
   case X86::VSQRTSDZm_Int:
     return true;
+  case X86::VMOVSSZrrk:
+  case X86::VMOVSDZrrk:
+    OpNum = 3;
+    return true;
+  case X86::VMOVSSZrrkz:
+  case X86::VMOVSDZrrkz:
+    OpNum = 2;
+    return true;
   }
 
   return false;
@@ -4471,12 +4554,9 @@ static bool hasUndefRegUpdate(unsigned Opcode, bool ForLoadFold = false) {
 unsigned
 X86InstrInfo::getUndefRegClearance(const MachineInstr &MI, unsigned &OpNum,
                                    const TargetRegisterInfo *TRI) const {
-  if (!hasUndefRegUpdate(MI.getOpcode()))
+  if (!hasUndefRegUpdate(MI.getOpcode(), OpNum))
     return 0;
 
-  // Set the OpNum parameter to the first source operand.
-  OpNum = 1;
-
   const MachineOperand &MO = MI.getOperand(OpNum);
   if (MO.isUndef() && Register::isPhysicalRegister(MO.getReg())) {
     return UndefRegClearance;
@@ -4720,7 +4800,8 @@ MachineInstr *X86InstrInfo::foldMemoryOperandCustom(
 
 static bool shouldPreventUndefRegUpdateMemFold(MachineFunction &MF,
                                                MachineInstr &MI) {
-  if (!hasUndefRegUpdate(MI.getOpcode(), /*ForLoadFold*/true) ||
+  unsigned Ignored;
+  if (!hasUndefRegUpdate(MI.getOpcode(), Ignored, /*ForLoadFold*/true) ||
       !MI.getOperand(1).isReg())
     return false;
 
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 3dd516fc19a2f..0502bbd7a013a 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -67,6 +67,9 @@ unsigned getSwappedVPCMPImm(unsigned Imm);
 /// Get the VPCOM immediate if the opcodes are swapped.
 unsigned getSwappedVPCOMImm(unsigned Imm);
 
+/// Get the VCMP immediate if the opcodes are swapped.
+unsigned getSwappedVCMPImm(unsigned Imm);
+
 } // namespace X86
 
 /// isGlobalStubReference - Return true if the specified TargetFlag operand is
@@ -251,7 +254,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
   ///     findCommutedOpIndices(MI, Op1, Op2);
   /// can be interpreted as a query asking to find an operand that would be
   /// commutable with the operand#1.
-  bool findCommutedOpIndices(MachineInstr &MI, unsigned &SrcOpIdx1,
+  bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1,
                              unsigned &SrcOpIdx2) const override;
 
   /// Returns an adjusted FMA opcode that must be used in FMA instruction that
diff --git a/llvm/lib/Target/X86/X86InstrInfo.td b/llvm/lib/Target/X86/X86InstrInfo.td
index a492d57fb8ccc..83216852b15a2 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.td
+++ b/llvm/lib/Target/X86/X86InstrInfo.td
@@ -2697,12 +2697,12 @@ def SLWPCB64 : I<0x12, MRM1r, (outs GR64:$dst), (ins), "slwpcb\t$dst",
 multiclass lwpins_intr<RegisterClass RC> {
   def rri : Ii32<0x12, MRM0r, (outs), (ins RC:$src0, GR32:$src1, i32imm:$cntl),
                  "lwpins\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
-                 [(set EFLAGS, (X86lwpins RC:$src0, GR32:$src1, imm:$cntl))]>,
+                 [(set EFLAGS, (X86lwpins RC:$src0, GR32:$src1, timm:$cntl))]>,
                  XOP_4V, XOPA;
   let mayLoad = 1 in
   def rmi : Ii32<0x12, MRM0m, (outs), (ins RC:$src0, i32mem:$src1, i32imm:$cntl),
                  "lwpins\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
-                 [(set EFLAGS, (X86lwpins RC:$src0, (loadi32 addr:$src1), imm:$cntl))]>,
+                 [(set EFLAGS, (X86lwpins RC:$src0, (loadi32 addr:$src1), timm:$cntl))]>,
                  XOP_4V, XOPA;
 }
 
@@ -2714,11 +2714,11 @@ let Defs = [EFLAGS] in {
 multiclass lwpval_intr<RegisterClass RC, Intrinsic Int> {
   def rri : Ii32<0x12, MRM1r, (outs), (ins RC:$src0, GR32:$src1, i32imm:$cntl),
                  "lwpval\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
-                 [(Int RC:$src0, GR32:$src1, imm:$cntl)]>, XOP_4V, XOPA;
+                 [(Int RC:$src0, GR32:$src1, timm:$cntl)]>, XOP_4V, XOPA;
   let mayLoad = 1 in
   def rmi : Ii32<0x12, MRM1m, (outs), (ins RC:$src0, i32mem:$src1, i32imm:$cntl),
                  "lwpval\t{$cntl, $src1, $src0|$src0, $src1, $cntl}",
-                 [(Int RC:$src0, (loadi32 addr:$src1), imm:$cntl)]>,
+                 [(Int RC:$src0, (loadi32 addr:$src1), timm:$cntl)]>,
                  XOP_4V, XOPA;
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrMMX.td b/llvm/lib/Target/X86/X86InstrMMX.td
index f60378597fa77..cd9a866c91cbb 100644
--- a/llvm/lib/Target/X86/X86InstrMMX.td
+++ b/llvm/lib/Target/X86/X86InstrMMX.td
@@ -30,7 +30,6 @@ def MMX_SET0 : I<0, Pseudo, (outs VR64:$dst), (ins), "", []>;
 
 let Constraints = "$src1 = $dst" in {
   // MMXI_binop_rm_int - Simple MMX binary operator based on intrinsic.
-  // When this is cleaned up, remove the FIXME from X86RecognizableInstr.cpp.
   multiclass MMXI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId,
                                X86FoldableSchedWrite sched, bit Commutable = 0,
                                X86MemOperand OType = i64mem> {
@@ -67,7 +66,7 @@ let Constraints = "$src1 = $dst" in {
     def ri : MMXIi8<opc2, ImmForm, (outs VR64:$dst),
                                    (ins VR64:$src1, i32u8imm:$src2),
                     !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
-           [(set VR64:$dst, (IntId2 VR64:$src1, imm:$src2))]>,
+           [(set VR64:$dst, (IntId2 VR64:$src1, timm:$src2))]>,
            Sched<[schedImm]>;
   }
 }
@@ -114,13 +113,13 @@ multiclass ssse3_palign_mm<string asm, Intrinsic IntId,
   def rri  : MMXSS3AI<0x0F, MRMSrcReg, (outs VR64:$dst),
       (ins VR64:$src1, VR64:$src2, u8imm:$src3),
       !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-      [(set VR64:$dst, (IntId VR64:$src1, VR64:$src2, (i8 imm:$src3)))]>,
+      [(set VR64:$dst, (IntId VR64:$src1, VR64:$src2, (i8 timm:$src3)))]>,
       Sched<[sched]>;
   def rmi  : MMXSS3AI<0x0F, MRMSrcMem, (outs VR64:$dst),
       (ins VR64:$src1, i64mem:$src2, u8imm:$src3),
       !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
       [(set VR64:$dst, (IntId VR64:$src1,
-                       (bitconvert (load_mmx addr:$src2)), (i8 imm:$src3)))]>,
+                       (bitconvert (load_mmx addr:$src2)), (i8 timm:$src3)))]>,
       Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -496,14 +495,14 @@ def MMX_PSHUFWri : MMXIi8<0x70, MRMSrcReg,
                           (outs VR64:$dst), (ins VR64:$src1, u8imm:$src2),
                           "pshufw\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                           [(set VR64:$dst,
-                             (int_x86_sse_pshuf_w VR64:$src1, imm:$src2))]>,
+                             (int_x86_sse_pshuf_w VR64:$src1, timm:$src2))]>,
                           Sched<[SchedWriteShuffle.MMX]>;
 def MMX_PSHUFWmi : MMXIi8<0x70, MRMSrcMem,
                           (outs VR64:$dst), (ins i64mem:$src1, u8imm:$src2),
                           "pshufw\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                           [(set VR64:$dst,
                              (int_x86_sse_pshuf_w (load_mmx addr:$src1),
-                                                   imm:$src2))]>,
+                                                   timm:$src2))]>,
                           Sched<[SchedWriteShuffle.MMX.Folded]>;
 
 // -- Conversion Instructions
@@ -535,7 +534,7 @@ def MMX_PEXTRWrr: MMXIi8<0xC5, MRMSrcReg,
                      (outs GR32orGR64:$dst), (ins VR64:$src1, i32u8imm:$src2),
                      "pextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                      [(set GR32orGR64:$dst, (int_x86_mmx_pextr_w VR64:$src1,
-                                             imm:$src2))]>,
+                                             timm:$src2))]>,
                      Sched<[WriteVecExtract]>;
 let Constraints = "$src1 = $dst" in {
 let Predicates = [HasMMX, HasSSE1] in {
@@ -544,7 +543,7 @@ let Predicates = [HasMMX, HasSSE1] in {
                     (ins VR64:$src1, GR32orGR64:$src2, i32u8imm:$src3),
                     "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}",
                     [(set VR64:$dst, (int_x86_mmx_pinsr_w VR64:$src1,
-                                      GR32orGR64:$src2, imm:$src3))]>,
+                                      GR32orGR64:$src2, timm:$src3))]>,
                     Sched<[WriteVecInsert, ReadDefault, ReadInt2Fpu]>;
 
   def MMX_PINSRWrm : MMXIi8<0xC4, MRMSrcMem,
@@ -553,7 +552,7 @@ let Predicates = [HasMMX, HasSSE1] in {
                    "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}",
                    [(set VR64:$dst, (int_x86_mmx_pinsr_w VR64:$src1,
                                        (i32 (anyext (loadi16 addr:$src2))),
-                                     imm:$src3))]>,
+                                     timm:$src3))]>,
                    Sched<[WriteVecInsert.Folded, WriteVecInsert.ReadAfterFold]>;
 }
 }
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index 8d3199e3cfa9d..09a04c0338b42 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -370,7 +370,7 @@ defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64, "movapd
 defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32, "movups",
                                  SSEPackedSingle, SchedWriteFMoveLS.YMM>,
                                  PS, VEX, VEX_L, VEX_WIG;
-defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, "movupd", 
+defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, "movupd",
                                  SSEPackedDouble, SchedWriteFMoveLS.YMM>,
                                  PD, VEX, VEX_L, VEX_WIG;
 }
@@ -1728,12 +1728,12 @@ multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop,
   let isCommutable = 1 in
   def rr : SIi8<0xC2, MRMSrcReg,
                 (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), asm,
-                [(set RC:$dst, (OpNode (VT RC:$src1), RC:$src2, imm:$cc))]>,
+                [(set RC:$dst, (OpNode (VT RC:$src1), RC:$src2, timm:$cc))]>,
                 Sched<[sched]>;
   def rm : SIi8<0xC2, MRMSrcMem,
                 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm,
                 [(set RC:$dst, (OpNode (VT RC:$src1),
-                                         (ld_frag addr:$src2), imm:$cc))]>,
+                                         (ld_frag addr:$src2), timm:$cc))]>,
                 Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -1766,13 +1766,13 @@ multiclass sse12_cmp_scalar_int<Operand memop,
   def rr_Int : SIi8<0xC2, MRMSrcReg, (outs VR128:$dst),
                       (ins VR128:$src1, VR128:$src, u8imm:$cc), asm,
                         [(set VR128:$dst, (Int VR128:$src1,
-                                               VR128:$src, imm:$cc))]>,
+                                               VR128:$src, timm:$cc))]>,
            Sched<[sched]>;
 let mayLoad = 1 in
   def rm_Int : SIi8<0xC2, MRMSrcMem, (outs VR128:$dst),
                       (ins VR128:$src1, memop:$src, u8imm:$cc), asm,
                         [(set VR128:$dst, (Int VR128:$src1,
-                                               mem_cpat:$src, imm:$cc))]>,
+                                               mem_cpat:$src, timm:$cc))]>,
            Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -1891,12 +1891,12 @@ multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop,
   let isCommutable = 1 in
   def rri : PIi8<0xC2, MRMSrcReg,
              (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), asm,
-             [(set RC:$dst, (VT (X86cmpp RC:$src1, RC:$src2, imm:$cc)))], d>,
+             [(set RC:$dst, (VT (X86cmpp RC:$src1, RC:$src2, timm:$cc)))], d>,
             Sched<[sched]>;
   def rmi : PIi8<0xC2, MRMSrcMem,
              (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm,
              [(set RC:$dst,
-               (VT (X86cmpp RC:$src1, (ld_frag addr:$src2), imm:$cc)))], d>,
+               (VT (X86cmpp RC:$src1, (ld_frag addr:$src2), timm:$cc)))], d>,
             Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -1921,7 +1921,7 @@ let Constraints = "$src1 = $dst" in {
                  SchedWriteFCmpSizes.PD.XMM, SSEPackedDouble, memopv2f64>, PD;
 }
 
-def CommutableCMPCC : PatLeaf<(imm), [{
+def CommutableCMPCC : PatLeaf<(timm), [{
   uint64_t Imm = N->getZExtValue() & 0x7;
   return (Imm == 0x00 || Imm == 0x03 || Imm == 0x04 || Imm == 0x07);
 }]>;
@@ -1930,47 +1930,47 @@ def CommutableCMPCC : PatLeaf<(imm), [{
 let Predicates = [HasAVX] in {
   def : Pat<(v4f64 (X86cmpp (loadv4f64 addr:$src2), VR256:$src1,
                             CommutableCMPCC:$cc)),
-            (VCMPPDYrmi VR256:$src1, addr:$src2, imm:$cc)>;
+            (VCMPPDYrmi VR256:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(v8f32 (X86cmpp (loadv8f32 addr:$src2), VR256:$src1,
                             CommutableCMPCC:$cc)),
-            (VCMPPSYrmi VR256:$src1, addr:$src2, imm:$cc)>;
+            (VCMPPSYrmi VR256:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(v2f64 (X86cmpp (loadv2f64 addr:$src2), VR128:$src1,
                             CommutableCMPCC:$cc)),
-            (VCMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>;
+            (VCMPPDrmi VR128:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(v4f32 (X86cmpp (loadv4f32 addr:$src2), VR128:$src1,
                             CommutableCMPCC:$cc)),
-            (VCMPPSrmi VR128:$src1, addr:$src2, imm:$cc)>;
+            (VCMPPSrmi VR128:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(f64 (X86cmps (loadf64 addr:$src2), FR64:$src1,
                           CommutableCMPCC:$cc)),
-            (VCMPSDrm FR64:$src1, addr:$src2, imm:$cc)>;
+            (VCMPSDrm FR64:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(f32 (X86cmps (loadf32 addr:$src2), FR32:$src1,
                           CommutableCMPCC:$cc)),
-            (VCMPSSrm FR32:$src1, addr:$src2, imm:$cc)>;
+            (VCMPSSrm FR32:$src1, addr:$src2, timm:$cc)>;
 }
 
 let Predicates = [UseSSE2] in {
   def : Pat<(v2f64 (X86cmpp (memopv2f64 addr:$src2), VR128:$src1,
                             CommutableCMPCC:$cc)),
-            (CMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>;
+            (CMPPDrmi VR128:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(f64 (X86cmps (loadf64 addr:$src2), FR64:$src1,
                           CommutableCMPCC:$cc)),
-            (CMPSDrm FR64:$src1, addr:$src2, imm:$cc)>;
+            (CMPSDrm FR64:$src1, addr:$src2, timm:$cc)>;
 }
 
 let Predicates = [UseSSE1] in {
   def : Pat<(v4f32 (X86cmpp (memopv4f32 addr:$src2), VR128:$src1,
                             CommutableCMPCC:$cc)),
-            (CMPPSrmi VR128:$src1, addr:$src2, imm:$cc)>;
+            (CMPPSrmi VR128:$src1, addr:$src2, timm:$cc)>;
 
   def : Pat<(f32 (X86cmps (loadf32 addr:$src2), FR32:$src1,
                           CommutableCMPCC:$cc)),
-            (CMPSSrm FR32:$src1, addr:$src2, imm:$cc)>;
+            (CMPSSrm FR32:$src1, addr:$src2, timm:$cc)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -1985,13 +1985,13 @@ multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop,
   def rmi : PIi8<0xC6, MRMSrcMem, (outs RC:$dst),
                    (ins RC:$src1, x86memop:$src2, u8imm:$src3), asm,
                    [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2),
-                                       (i8 imm:$src3))))], d>,
+                                       (i8 timm:$src3))))], d>,
             Sched<[sched.Folded, sched.ReadAfterFold]>;
   let isCommutable = IsCommutable in
   def rri : PIi8<0xC6, MRMSrcReg, (outs RC:$dst),
                  (ins RC:$src1, RC:$src2, u8imm:$src3), asm,
                  [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2,
-                                     (i8 imm:$src3))))], d>,
+                                     (i8 timm:$src3))))], d>,
             Sched<[sched]>;
 }
 
@@ -2736,7 +2736,7 @@ defm : scalar_math_patterns<fadd, "ADDSD", X86Movsd, v2f64, f64, FR64, loadf64,
 defm : scalar_math_patterns<fsub, "SUBSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;
 defm : scalar_math_patterns<fmul, "MULSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;
 defm : scalar_math_patterns<fdiv, "DIVSD", X86Movsd, v2f64, f64, FR64, loadf64, UseSSE2>;
- 
+
 /// Unop Arithmetic
 /// In addition, we also have a special variant of the scalar form here to
 /// represent the associated intrinsic operation.  This form is unlike the
@@ -3497,7 +3497,7 @@ multiclass PDI_binop_rmi<bits<8> opc, bits<8> opc2, Format ImmForm,
        !if(Is2Addr,
            !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-       [(set RC:$dst, (DstVT (OpNode2 RC:$src1, (i8 imm:$src2))))]>,
+       [(set RC:$dst, (DstVT (OpNode2 RC:$src1, (i8 timm:$src2))))]>,
        Sched<[schedImm]>;
 }
 
@@ -3529,7 +3529,7 @@ multiclass PDI_binop_ri<bits<8> opc, Format ImmForm, string OpcodeStr,
        !if(Is2Addr,
            !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")),
-       [(set RC:$dst, (VT (OpNode RC:$src1, (i8 imm:$src2))))]>,
+       [(set RC:$dst, (VT (OpNode RC:$src1, (i8 timm:$src2))))]>,
        Sched<[sched]>;
 }
 
@@ -3612,7 +3612,7 @@ let Predicates = [HasAVX, prd] in {
                       !strconcat("v", OpcodeStr,
                                  "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                       [(set VR128:$dst,
-                        (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))]>,
+                        (vt128 (OpNode VR128:$src1, (i8 timm:$src2))))]>,
                       VEX, Sched<[sched.XMM]>, VEX_WIG;
   def V#NAME#mi : Ii8<0x70, MRMSrcMem, (outs VR128:$dst),
                       (ins i128mem:$src1, u8imm:$src2),
@@ -3620,7 +3620,7 @@ let Predicates = [HasAVX, prd] in {
                                  "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                      [(set VR128:$dst,
                        (vt128 (OpNode (load addr:$src1),
-                        (i8 imm:$src2))))]>, VEX,
+                        (i8 timm:$src2))))]>, VEX,
                   Sched<[sched.XMM.Folded]>, VEX_WIG;
 }
 
@@ -3630,7 +3630,7 @@ let Predicates = [HasAVX2, prd] in {
                        !strconcat("v", OpcodeStr,
                                   "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                        [(set VR256:$dst,
-                         (vt256 (OpNode VR256:$src1, (i8 imm:$src2))))]>,
+                         (vt256 (OpNode VR256:$src1, (i8 timm:$src2))))]>,
                        VEX, VEX_L, Sched<[sched.YMM]>, VEX_WIG;
   def V#NAME#Ymi : Ii8<0x70, MRMSrcMem, (outs VR256:$dst),
                        (ins i256mem:$src1, u8imm:$src2),
@@ -3638,7 +3638,7 @@ let Predicates = [HasAVX2, prd] in {
                                   "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                       [(set VR256:$dst,
                         (vt256 (OpNode (load addr:$src1),
-                         (i8 imm:$src2))))]>, VEX, VEX_L,
+                         (i8 timm:$src2))))]>, VEX, VEX_L,
                    Sched<[sched.YMM.Folded]>, VEX_WIG;
 }
 
@@ -3648,7 +3648,7 @@ let Predicates = [UseSSE2] in {
                !strconcat(OpcodeStr,
                           "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                [(set VR128:$dst,
-                 (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))]>,
+                 (vt128 (OpNode VR128:$src1, (i8 timm:$src2))))]>,
                Sched<[sched.XMM]>;
   def mi : Ii8<0x70, MRMSrcMem,
                (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2),
@@ -3656,7 +3656,7 @@ let Predicates = [UseSSE2] in {
                           "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                [(set VR128:$dst,
                  (vt128 (OpNode (memop addr:$src1),
-                        (i8 imm:$src2))))]>,
+                        (i8 timm:$src2))))]>,
                Sched<[sched.XMM.Folded]>;
 }
 }
@@ -4827,7 +4827,7 @@ multiclass ssse3_palignr<string asm, ValueType VT, RegisterClass RC,
         !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
         !strconcat(asm,
                   "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-      [(set RC:$dst, (VT (X86PAlignr RC:$src1, RC:$src2, (i8 imm:$src3))))]>,
+      [(set RC:$dst, (VT (X86PAlignr RC:$src1, RC:$src2, (i8 timm:$src3))))]>,
       Sched<[sched]>;
   let mayLoad = 1 in
   def rmi : SS3AI<0x0F, MRMSrcMem, (outs RC:$dst),
@@ -4838,7 +4838,7 @@ multiclass ssse3_palignr<string asm, ValueType VT, RegisterClass RC,
                   "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
       [(set RC:$dst, (VT (X86PAlignr RC:$src1,
                                      (memop_frag addr:$src2),
-                                     (i8 imm:$src3))))]>,
+                                     (i8 timm:$src3))))]>,
       Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 }
@@ -5315,7 +5315,7 @@ multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {
         !strconcat(asm,
                    "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
       [(set VR128:$dst,
-        (X86insertps VR128:$src1, VR128:$src2, imm:$src3))]>,
+        (X86insertps VR128:$src1, VR128:$src2, timm:$src3))]>,
       Sched<[SchedWriteFShuffle.XMM]>;
   def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst),
       (ins VR128:$src1, f32mem:$src2, u8imm:$src3),
@@ -5326,7 +5326,7 @@ multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1> {
       [(set VR128:$dst,
         (X86insertps VR128:$src1,
                    (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
-                    imm:$src3))]>,
+                    timm:$src3))]>,
       Sched<[SchedWriteFShuffle.XMM.Folded, SchedWriteFShuffle.XMM.ReadAfterFold]>;
 }
 
@@ -5352,7 +5352,7 @@ multiclass sse41_fp_unop_p<bits<8> opc, string OpcodeStr,
                   (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2),
                   !strconcat(OpcodeStr,
                   "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-                  [(set RC:$dst, (VT (OpNode RC:$src1, imm:$src2)))]>,
+                  [(set RC:$dst, (VT (OpNode RC:$src1, timm:$src2)))]>,
                   Sched<[sched]>;
 
   // Vector intrinsic operation, mem
@@ -5361,13 +5361,13 @@ multiclass sse41_fp_unop_p<bits<8> opc, string OpcodeStr,
                   !strconcat(OpcodeStr,
                   "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                   [(set RC:$dst,
-                        (VT (OpNode (mem_frag addr:$src1),imm:$src2)))]>,
+                        (VT (OpNode (mem_frag addr:$src1), timm:$src2)))]>,
                   Sched<[sched.Folded]>;
 }
 
 multiclass avx_fp_unop_rm<bits<8> opcss, bits<8> opcsd,
                           string OpcodeStr, X86FoldableSchedWrite sched> {
-let ExeDomain = SSEPackedSingle, hasSideEffects = 0 in {
+let ExeDomain = SSEPackedSingle, hasSideEffects = 0, isCodeGenOnly = 1 in {
   def SSr : SS4AIi8<opcss, MRMSrcReg,
         (outs FR32:$dst), (ins FR32:$src1, FR32:$src2, i32u8imm:$src3),
         !strconcat(OpcodeStr,
@@ -5382,7 +5382,7 @@ let ExeDomain = SSEPackedSingle, hasSideEffects = 0 in {
         []>, Sched<[sched.Folded, sched.ReadAfterFold]>;
 } // ExeDomain = SSEPackedSingle, hasSideEffects = 0
 
-let ExeDomain = SSEPackedDouble, hasSideEffects = 0 in {
+let ExeDomain = SSEPackedDouble, hasSideEffects = 0, isCodeGenOnly = 1 in {
   def SDr : SS4AIi8<opcsd, MRMSrcReg,
         (outs FR64:$dst), (ins FR64:$src1, FR64:$src2, i32u8imm:$src3),
         !strconcat(OpcodeStr,
@@ -5400,7 +5400,7 @@ let ExeDomain = SSEPackedDouble, hasSideEffects = 0 in {
 
 multiclass sse41_fp_unop_s<bits<8> opcss, bits<8> opcsd,
                            string OpcodeStr, X86FoldableSchedWrite sched> {
-let ExeDomain = SSEPackedSingle, hasSideEffects = 0 in {
+let ExeDomain = SSEPackedSingle, hasSideEffects = 0, isCodeGenOnly = 1 in {
   def SSr : SS4AIi8<opcss, MRMSrcReg,
                     (outs FR32:$dst), (ins FR32:$src1, i32u8imm:$src2),
                     !strconcat(OpcodeStr,
@@ -5415,7 +5415,7 @@ let ExeDomain = SSEPackedSingle, hasSideEffects = 0 in {
                     []>, Sched<[sched.Folded, sched.ReadAfterFold]>;
 } // ExeDomain = SSEPackedSingle, hasSideEffects = 0
 
-let ExeDomain = SSEPackedDouble, hasSideEffects = 0 in {
+let ExeDomain = SSEPackedDouble, hasSideEffects = 0, isCodeGenOnly = 1 in {
   def SDr : SS4AIi8<opcsd, MRMSrcReg,
                     (outs FR64:$dst), (ins FR64:$src1, i32u8imm:$src2),
                     !strconcat(OpcodeStr,
@@ -5435,7 +5435,7 @@ multiclass sse41_fp_binop_s<bits<8> opcss, bits<8> opcsd,
                             string OpcodeStr, X86FoldableSchedWrite sched,
                             ValueType VT32, ValueType VT64,
                             SDNode OpNode, bit Is2Addr = 1> {
-let ExeDomain = SSEPackedSingle, isCodeGenOnly = 1 in {
+let ExeDomain = SSEPackedSingle in {
   def SSr_Int : SS4AIi8<opcss, MRMSrcReg,
         (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3),
         !if(Is2Addr,
@@ -5443,7 +5443,7 @@ let ExeDomain = SSEPackedSingle, isCodeGenOnly = 1 in {
                 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
             !strconcat(OpcodeStr,
                 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-        [(set VR128:$dst, (VT32 (OpNode VR128:$src1, VR128:$src2, imm:$src3)))]>,
+        [(set VR128:$dst, (VT32 (OpNode VR128:$src1, VR128:$src2, timm:$src3)))]>,
         Sched<[sched]>;
 
   def SSm_Int : SS4AIi8<opcss, MRMSrcMem,
@@ -5454,11 +5454,11 @@ let ExeDomain = SSEPackedSingle, isCodeGenOnly = 1 in {
             !strconcat(OpcodeStr,
                 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
         [(set VR128:$dst,
-             (OpNode VR128:$src1, sse_load_f32:$src2, imm:$src3))]>,
+             (OpNode VR128:$src1, sse_load_f32:$src2, timm:$src3))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>;
 } // ExeDomain = SSEPackedSingle, isCodeGenOnly = 1
 
-let ExeDomain = SSEPackedDouble, isCodeGenOnly = 1 in {
+let ExeDomain = SSEPackedDouble in {
   def SDr_Int : SS4AIi8<opcsd, MRMSrcReg,
         (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3),
         !if(Is2Addr,
@@ -5466,7 +5466,7 @@ let ExeDomain = SSEPackedDouble, isCodeGenOnly = 1 in {
                 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
             !strconcat(OpcodeStr,
                 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-        [(set VR128:$dst, (VT64 (OpNode VR128:$src1, VR128:$src2, imm:$src3)))]>,
+        [(set VR128:$dst, (VT64 (OpNode VR128:$src1, VR128:$src2, timm:$src3)))]>,
         Sched<[sched]>;
 
   def SDm_Int : SS4AIi8<opcsd, MRMSrcMem,
@@ -5477,7 +5477,7 @@ let ExeDomain = SSEPackedDouble, isCodeGenOnly = 1 in {
             !strconcat(OpcodeStr,
                 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
         [(set VR128:$dst,
-              (OpNode VR128:$src1, sse_load_f64:$src2, imm:$src3))]>,
+              (OpNode VR128:$src1, sse_load_f64:$src2, timm:$src3))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>;
 } // ExeDomain = SSEPackedDouble, isCodeGenOnly = 1
 }
@@ -5512,17 +5512,17 @@ let Predicates = [UseAVX] in {
 }
 
 let Predicates = [UseAVX] in {
-  def : Pat<(X86VRndScale FR32:$src1, imm:$src2),
-            (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src1, imm:$src2)>;
-  def : Pat<(X86VRndScale FR64:$src1, imm:$src2),
-            (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src1, imm:$src2)>;
+  def : Pat<(X86VRndScale FR32:$src1, timm:$src2),
+            (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src1, timm:$src2)>;
+  def : Pat<(X86VRndScale FR64:$src1, timm:$src2),
+            (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src1, timm:$src2)>;
 }
 
 let Predicates = [UseAVX, OptForSize] in {
-  def : Pat<(X86VRndScale (loadf32 addr:$src1), imm:$src2),
-            (VROUNDSSm (f32 (IMPLICIT_DEF)), addr:$src1, imm:$src2)>;
-  def : Pat<(X86VRndScale (loadf64 addr:$src1), imm:$src2),
-            (VROUNDSDm (f64 (IMPLICIT_DEF)), addr:$src1, imm:$src2)>;
+  def : Pat<(X86VRndScale (loadf32 addr:$src1), timm:$src2),
+            (VROUNDSSm (f32 (IMPLICIT_DEF)), addr:$src1, timm:$src2)>;
+  def : Pat<(X86VRndScale (loadf64 addr:$src1), timm:$src2),
+            (VROUNDSDm (f64 (IMPLICIT_DEF)), addr:$src1, timm:$src2)>;
 }
 
 let ExeDomain = SSEPackedSingle in
@@ -5539,17 +5539,17 @@ defm ROUND  : sse41_fp_binop_s<0x0A, 0x0B, "round", SchedWriteFRnd.Scl,
                                v4f32, v2f64, X86RndScales>;
 
 let Predicates = [UseSSE41] in {
-  def : Pat<(X86VRndScale FR32:$src1, imm:$src2),
-            (ROUNDSSr FR32:$src1, imm:$src2)>;
-  def : Pat<(X86VRndScale FR64:$src1, imm:$src2),
-            (ROUNDSDr FR64:$src1, imm:$src2)>;
+  def : Pat<(X86VRndScale FR32:$src1, timm:$src2),
+            (ROUNDSSr FR32:$src1, timm:$src2)>;
+  def : Pat<(X86VRndScale FR64:$src1, timm:$src2),
+            (ROUNDSDr FR64:$src1, timm:$src2)>;
 }
 
 let Predicates = [UseSSE41, OptForSize] in {
-  def : Pat<(X86VRndScale (loadf32 addr:$src1), imm:$src2),
-            (ROUNDSSm addr:$src1, imm:$src2)>;
-  def : Pat<(X86VRndScale (loadf64 addr:$src1), imm:$src2),
-            (ROUNDSDm addr:$src1, imm:$src2)>;
+  def : Pat<(X86VRndScale (loadf32 addr:$src1), timm:$src2),
+            (ROUNDSSm addr:$src1, timm:$src2)>;
+  def : Pat<(X86VRndScale (loadf64 addr:$src1), timm:$src2),
+            (ROUNDSDm addr:$src1, timm:$src2)>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -5830,7 +5830,7 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr,
                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-        [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))]>,
+        [(set RC:$dst, (IntId RC:$src1, RC:$src2, timm:$src3))]>,
         Sched<[sched]>;
   def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, x86memop:$src2, u8imm:$src3),
@@ -5840,7 +5840,7 @@ multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr,
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
         [(set RC:$dst,
-          (IntId RC:$src1, (memop_frag addr:$src2), imm:$src3))]>,
+          (IntId RC:$src1, (memop_frag addr:$src2), timm:$src3))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -5857,7 +5857,7 @@ multiclass SS41I_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,
                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))]>,
+        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,
         Sched<[sched]>;
   def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, x86memop:$src2, u8imm:$src3),
@@ -5867,27 +5867,27 @@ multiclass SS41I_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
         [(set RC:$dst,
-          (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), imm:$src3)))]>,
+          (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), timm:$src3)))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
-def BlendCommuteImm2 : SDNodeXForm<imm, [{
+def BlendCommuteImm2 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue() & 0x03;
   return getI8Imm(Imm ^ 0x03, SDLoc(N));
 }]>;
 
-def BlendCommuteImm4 : SDNodeXForm<imm, [{
+def BlendCommuteImm4 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue() & 0x0f;
   return getI8Imm(Imm ^ 0x0f, SDLoc(N));
 }]>;
 
-def BlendCommuteImm8 : SDNodeXForm<imm, [{
+def BlendCommuteImm8 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue() & 0xff;
   return getI8Imm(Imm ^ 0xff, SDLoc(N));
 }]>;
 
 // Turn a 4-bit blendi immediate to 8-bit for use with pblendw.
-def BlendScaleImm4 : SDNodeXForm<imm, [{
+def BlendScaleImm4 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 4; ++i) {
@@ -5898,7 +5898,7 @@ def BlendScaleImm4 : SDNodeXForm<imm, [{
 }]>;
 
 // Turn a 2-bit blendi immediate to 8-bit for use with pblendw.
-def BlendScaleImm2 : SDNodeXForm<imm, [{
+def BlendScaleImm2 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 2; ++i) {
@@ -5909,7 +5909,7 @@ def BlendScaleImm2 : SDNodeXForm<imm, [{
 }]>;
 
 // Turn a 2-bit blendi immediate to 4-bit for use with pblendd.
-def BlendScaleImm2to4 : SDNodeXForm<imm, [{
+def BlendScaleImm2to4 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 2; ++i) {
@@ -5920,7 +5920,7 @@ def BlendScaleImm2to4 : SDNodeXForm<imm, [{
 }]>;
 
 // Turn a 4-bit blendi immediate to 8-bit for use with pblendw and invert it.
-def BlendScaleCommuteImm4 : SDNodeXForm<imm, [{
+def BlendScaleCommuteImm4 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 4; ++i) {
@@ -5931,7 +5931,7 @@ def BlendScaleCommuteImm4 : SDNodeXForm<imm, [{
 }]>;
 
 // Turn a 2-bit blendi immediate to 8-bit for use with pblendw and invert it.
-def BlendScaleCommuteImm2 : SDNodeXForm<imm, [{
+def BlendScaleCommuteImm2 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 2; ++i) {
@@ -5942,7 +5942,7 @@ def BlendScaleCommuteImm2 : SDNodeXForm<imm, [{
 }]>;
 
 // Turn a 2-bit blendi immediate to 4-bit for use with pblendd and invert it.
-def BlendScaleCommuteImm2to4 : SDNodeXForm<imm, [{
+def BlendScaleCommuteImm2to4 : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   uint8_t NewImm = 0;
   for (unsigned i = 0; i != 2; ++i) {
@@ -6012,7 +6012,7 @@ let ExeDomain = d, Constraints = !if(Is2Addr, "$src1 = $dst", "") in {
                 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
-        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))]>,
+        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,
         Sched<[sched]>;
   def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, x86memop:$src2, u8imm:$src3),
@@ -6022,14 +6022,14 @@ let ExeDomain = d, Constraints = !if(Is2Addr, "$src1 = $dst", "") in {
             !strconcat(OpcodeStr,
                 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")),
         [(set RC:$dst,
-          (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), imm:$src3)))]>,
+          (OpVT (OpNode RC:$src1, (memop_frag addr:$src2), timm:$src3)))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
   // Pattern to commute if load is in first source.
-  def : Pat<(OpVT (OpNode (memop_frag addr:$src2), RC:$src1, imm:$src3)),
+  def : Pat<(OpVT (OpNode (memop_frag addr:$src2), RC:$src1, timm:$src3)),
             (!cast<Instruction>(NAME#"rmi") RC:$src1, addr:$src2,
-                                            (commuteXForm imm:$src3))>;
+                                            (commuteXForm timm:$src3))>;
 }
 
 let Predicates = [HasAVX] in {
@@ -6065,37 +6065,37 @@ let Predicates = [HasAVX2] in {
 // Emulate vXi32/vXi64 blends with vXf32/vXf64 or pblendw.
 // ExecutionDomainFixPass will cleanup domains later on.
 let Predicates = [HasAVX1Only] in {
-def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), imm:$src3),
-          (VBLENDPDYrri VR256:$src1, VR256:$src2, imm:$src3)>;
-def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), imm:$src3),
-          (VBLENDPDYrmi VR256:$src1, addr:$src2, imm:$src3)>;
-def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, imm:$src3),
-          (VBLENDPDYrmi VR256:$src1, addr:$src2, (BlendCommuteImm4 imm:$src3))>;
+def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), timm:$src3),
+          (VBLENDPDYrri VR256:$src1, VR256:$src2, timm:$src3)>;
+def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), timm:$src3),
+          (VBLENDPDYrmi VR256:$src1, addr:$src2, timm:$src3)>;
+def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, timm:$src3),
+          (VBLENDPDYrmi VR256:$src1, addr:$src2, (BlendCommuteImm4 timm:$src3))>;
 
 // Use pblendw for 128-bit integer to keep it in the integer domain and prevent
 // it from becoming movsd via commuting under optsize.
-def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), imm:$src3),
-          (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 imm:$src3))>;
-def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), imm:$src3),
-          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 imm:$src3))>;
-def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, imm:$src3),
-          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 imm:$src3))>;
-
-def : Pat<(X86Blendi (v8i32 VR256:$src1), (v8i32 VR256:$src2), imm:$src3),
-          (VBLENDPSYrri VR256:$src1, VR256:$src2, imm:$src3)>;
-def : Pat<(X86Blendi VR256:$src1, (loadv8i32 addr:$src2), imm:$src3),
-          (VBLENDPSYrmi VR256:$src1, addr:$src2, imm:$src3)>;
-def : Pat<(X86Blendi (loadv8i32 addr:$src2), VR256:$src1, imm:$src3),
-          (VBLENDPSYrmi VR256:$src1, addr:$src2, (BlendCommuteImm8 imm:$src3))>;
+def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),
+          (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 timm:$src3))>;
+def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), timm:$src3),
+          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 timm:$src3))>;
+def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, timm:$src3),
+          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 timm:$src3))>;
+
+def : Pat<(X86Blendi (v8i32 VR256:$src1), (v8i32 VR256:$src2), timm:$src3),
+          (VBLENDPSYrri VR256:$src1, VR256:$src2, timm:$src3)>;
+def : Pat<(X86Blendi VR256:$src1, (loadv8i32 addr:$src2), timm:$src3),
+          (VBLENDPSYrmi VR256:$src1, addr:$src2, timm:$src3)>;
+def : Pat<(X86Blendi (loadv8i32 addr:$src2), VR256:$src1, timm:$src3),
+          (VBLENDPSYrmi VR256:$src1, addr:$src2, (BlendCommuteImm8 timm:$src3))>;
 
 // Use pblendw for 128-bit integer to keep it in the integer domain and prevent
 // it from becoming movss via commuting under optsize.
-def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), imm:$src3),
-          (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi VR128:$src1, (loadv4i32 addr:$src2), imm:$src3),
-          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi (loadv4i32 addr:$src2), VR128:$src1, imm:$src3),
-          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 imm:$src3))>;
+def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), timm:$src3),
+          (VPBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi VR128:$src1, (loadv4i32 addr:$src2), timm:$src3),
+          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi (loadv4i32 addr:$src2), VR128:$src1, timm:$src3),
+          (VPBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;
 }
 
 defm BLENDPS : SS41I_blend_rmi<0x0C, "blendps", X86Blendi, v4f32,
@@ -6111,19 +6111,19 @@ defm PBLENDW : SS41I_blend_rmi<0x0E, "pblendw", X86Blendi, v8i16,
 let Predicates = [UseSSE41] in {
 // Use pblendw for 128-bit integer to keep it in the integer domain and prevent
 // it from becoming movss via commuting under optsize.
-def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), imm:$src3),
-          (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 imm:$src3))>;
-def : Pat<(X86Blendi VR128:$src1, (memopv2i64 addr:$src2), imm:$src3),
-          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 imm:$src3))>;
-def : Pat<(X86Blendi (memopv2i64 addr:$src2), VR128:$src1, imm:$src3),
-          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 imm:$src3))>;
+def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),
+          (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm2 timm:$src3))>;
+def : Pat<(X86Blendi VR128:$src1, (memopv2i64 addr:$src2), timm:$src3),
+          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm2 timm:$src3))>;
+def : Pat<(X86Blendi (memopv2i64 addr:$src2), VR128:$src1, timm:$src3),
+          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2 timm:$src3))>;
 
-def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), imm:$src3),
-          (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi VR128:$src1, (memopv4i32 addr:$src2), imm:$src3),
-          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi (memopv4i32 addr:$src2), VR128:$src1, imm:$src3),
-          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 imm:$src3))>;
+def : Pat<(X86Blendi (v4i32 VR128:$src1), (v4i32 VR128:$src2), timm:$src3),
+          (PBLENDWrri VR128:$src1, VR128:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi VR128:$src1, (memopv4i32 addr:$src2), timm:$src3),
+          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi (memopv4i32 addr:$src2), VR128:$src1, timm:$src3),
+          (PBLENDWrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;
 }
 
 // For insertion into the zero index (low half) of a 256-bit vector, it is
@@ -6596,7 +6596,7 @@ let Constraints = "$src1 = $dst", Predicates = [HasSHA] in {
                          "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}",
                          [(set VR128:$dst,
                            (int_x86_sha1rnds4 VR128:$src1, VR128:$src2,
-                            (i8 imm:$src3)))]>, TA,
+                            (i8 timm:$src3)))]>, TA,
                          Sched<[SchedWriteVecIMul.XMM]>;
   def SHA1RNDS4rmi : Ii8<0xCC, MRMSrcMem, (outs VR128:$dst),
                          (ins VR128:$src1, i128mem:$src2, u8imm:$src3),
@@ -6604,7 +6604,7 @@ let Constraints = "$src1 = $dst", Predicates = [HasSHA] in {
                          [(set VR128:$dst,
                            (int_x86_sha1rnds4 VR128:$src1,
                             (memop addr:$src2),
-                            (i8 imm:$src3)))]>, TA,
+                            (i8 timm:$src3)))]>, TA,
                          Sched<[SchedWriteVecIMul.XMM.Folded,
                                 SchedWriteVecIMul.XMM.ReadAfterFold]>;
 
@@ -6722,26 +6722,26 @@ let Predicates = [HasAVX, HasAES] in {
       (ins VR128:$src1, u8imm:$src2),
       "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
       [(set VR128:$dst,
-        (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>,
+        (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>,
       Sched<[WriteAESKeyGen]>, VEX, VEX_WIG;
   def VAESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),
       (ins i128mem:$src1, u8imm:$src2),
       "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
       [(set VR128:$dst,
-        (int_x86_aesni_aeskeygenassist (load addr:$src1), imm:$src2))]>,
+        (int_x86_aesni_aeskeygenassist (load addr:$src1), timm:$src2))]>,
       Sched<[WriteAESKeyGen.Folded]>, VEX, VEX_WIG;
 }
 def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst),
   (ins VR128:$src1, u8imm:$src2),
   "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   [(set VR128:$dst,
-    (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>,
+    (int_x86_aesni_aeskeygenassist VR128:$src1, timm:$src2))]>,
   Sched<[WriteAESKeyGen]>;
 def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),
   (ins i128mem:$src1, u8imm:$src2),
   "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}",
   [(set VR128:$dst,
-    (int_x86_aesni_aeskeygenassist (memop addr:$src1), imm:$src2))]>,
+    (int_x86_aesni_aeskeygenassist (memop addr:$src1), timm:$src2))]>,
   Sched<[WriteAESKeyGen.Folded]>;
 
 //===----------------------------------------------------------------------===//
@@ -6749,7 +6749,7 @@ def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst),
 //===----------------------------------------------------------------------===//
 
 // Immediate transform to help with commuting.
-def PCLMULCommuteImm : SDNodeXForm<imm, [{
+def PCLMULCommuteImm : SDNodeXForm<timm, [{
   uint8_t Imm = N->getZExtValue();
   return getI8Imm((uint8_t)((Imm >> 4) | (Imm << 4)), SDLoc(N));
 }]>;
@@ -6762,7 +6762,7 @@ let Predicates = [NoAVX, HasPCLMUL] in {
               (ins VR128:$src1, VR128:$src2, u8imm:$src3),
               "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}",
               [(set VR128:$dst,
-                (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))]>,
+                (int_x86_pclmulqdq VR128:$src1, VR128:$src2, timm:$src3))]>,
                 Sched<[WriteCLMul]>;
 
     def PCLMULQDQrm : PCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst),
@@ -6770,14 +6770,14 @@ let Predicates = [NoAVX, HasPCLMUL] in {
               "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}",
               [(set VR128:$dst,
                  (int_x86_pclmulqdq VR128:$src1, (memop addr:$src2),
-                  imm:$src3))]>,
+                  timm:$src3))]>,
               Sched<[WriteCLMul.Folded, WriteCLMul.ReadAfterFold]>;
   } // Constraints = "$src1 = $dst"
 
   def : Pat<(int_x86_pclmulqdq (memop addr:$src2), VR128:$src1,
-                                (i8 imm:$src3)),
+                                (i8 timm:$src3)),
             (PCLMULQDQrm VR128:$src1, addr:$src2,
-                          (PCLMULCommuteImm imm:$src3))>;
+                          (PCLMULCommuteImm timm:$src3))>;
 } // Predicates = [NoAVX, HasPCLMUL]
 
 // SSE aliases
@@ -6799,21 +6799,21 @@ multiclass vpclmulqdq<RegisterClass RC, X86MemOperand MemOp,
             (ins RC:$src1, RC:$src2, u8imm:$src3),
             "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
             [(set RC:$dst,
-              (IntId RC:$src1, RC:$src2, imm:$src3))]>,
+              (IntId RC:$src1, RC:$src2, timm:$src3))]>,
             Sched<[WriteCLMul]>;
 
   def rm : PCLMULIi8<0x44, MRMSrcMem, (outs RC:$dst),
             (ins RC:$src1, MemOp:$src2, u8imm:$src3),
             "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
             [(set RC:$dst,
-               (IntId RC:$src1, (LdFrag addr:$src2), imm:$src3))]>,
+               (IntId RC:$src1, (LdFrag addr:$src2), timm:$src3))]>,
             Sched<[WriteCLMul.Folded, WriteCLMul.ReadAfterFold]>;
 
   // We can commute a load in the first operand by swapping the sources and
   // rotating the immediate.
-  def : Pat<(IntId (LdFrag addr:$src2), RC:$src1, (i8 imm:$src3)),
+  def : Pat<(IntId (LdFrag addr:$src2), RC:$src1, (i8 timm:$src3)),
             (!cast<Instruction>(NAME#"rm") RC:$src1, addr:$src2,
-                                           (PCLMULCommuteImm imm:$src3))>;
+                                           (PCLMULCommuteImm timm:$src3))>;
 }
 
 let Predicates = [HasAVX, NoVLX_Or_NoVPCLMULQDQ, HasPCLMUL] in
@@ -6857,8 +6857,8 @@ let Constraints = "$src = $dst" in {
 def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst),
                  (ins VR128:$src, u8imm:$len, u8imm:$idx),
                  "extrq\t{$idx, $len, $src|$src, $len, $idx}",
-                 [(set VR128:$dst, (X86extrqi VR128:$src, imm:$len,
-                                    imm:$idx))]>,
+                 [(set VR128:$dst, (X86extrqi VR128:$src, timm:$len,
+                                    timm:$idx))]>,
                  PD, Sched<[SchedWriteVecALU.XMM]>;
 def EXTRQ  : I<0x79, MRMSrcReg, (outs VR128:$dst),
               (ins VR128:$src, VR128:$mask),
@@ -6871,7 +6871,7 @@ def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst),
                    (ins VR128:$src, VR128:$src2, u8imm:$len, u8imm:$idx),
                    "insertq\t{$idx, $len, $src2, $src|$src, $src2, $len, $idx}",
                    [(set VR128:$dst, (X86insertqi VR128:$src, VR128:$src2,
-                                      imm:$len, imm:$idx))]>,
+                                      timm:$len, timm:$idx))]>,
                    XD, Sched<[SchedWriteVecALU.XMM]>;
 def INSERTQ  : I<0x79, MRMSrcReg, (outs VR128:$dst),
                  (ins VR128:$src, VR128:$mask),
@@ -6911,10 +6911,10 @@ def : Pat<(nontemporalstore FR64:$src, addr:$dst),
 //
 class avx_broadcast_rm<bits<8> opc, string OpcodeStr, RegisterClass RC,
                            X86MemOperand x86memop, ValueType VT,
-                           PatFrag ld_frag, SchedWrite Sched> :
+                           PatFrag bcast_frag, SchedWrite Sched> :
   AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src),
         !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-        [(set RC:$dst, (VT (X86VBroadcast (ld_frag addr:$src))))]>,
+        [(set RC:$dst, (VT (bcast_frag addr:$src)))]>,
         Sched<[Sched]>, VEX;
 
 // AVX2 adds register forms
@@ -6927,15 +6927,15 @@ class avx2_broadcast_rr<bits<8> opc, string OpcodeStr, RegisterClass RC,
 
 let ExeDomain = SSEPackedSingle, Predicates = [HasAVX, NoVLX] in {
   def VBROADCASTSSrm  : avx_broadcast_rm<0x18, "vbroadcastss", VR128,
-                                         f32mem, v4f32, loadf32,
+                                         f32mem, v4f32, X86VBroadcastld32,
                                          SchedWriteFShuffle.XMM.Folded>;
   def VBROADCASTSSYrm : avx_broadcast_rm<0x18, "vbroadcastss", VR256,
-                                         f32mem, v8f32, loadf32,
+                                         f32mem, v8f32, X86VBroadcastld32,
                                          SchedWriteFShuffle.XMM.Folded>, VEX_L;
 }
 let ExeDomain = SSEPackedDouble, Predicates = [HasAVX, NoVLX] in
 def VBROADCASTSDYrm  : avx_broadcast_rm<0x19, "vbroadcastsd", VR256, f64mem,
-                                        v4f64, loadf64,
+                                        v4f64, X86VBroadcastld64,
                                         SchedWriteFShuffle.XMM.Folded>, VEX_L;
 
 let ExeDomain = SSEPackedSingle, Predicates = [HasAVX2, NoVLX] in {
@@ -7142,13 +7142,13 @@ multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr,
     def ri  : AVXAIi8<opc_rmi, MRMSrcReg, (outs RC:$dst),
              (ins RC:$src1, u8imm:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set RC:$dst, (f_vt (X86VPermilpi RC:$src1, (i8 imm:$src2))))]>, VEX,
+             [(set RC:$dst, (f_vt (X86VPermilpi RC:$src1, (i8 timm:$src2))))]>, VEX,
              Sched<[sched]>;
     def mi  : AVXAIi8<opc_rmi, MRMSrcMem, (outs RC:$dst),
              (ins x86memop_f:$src1, u8imm:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set RC:$dst,
-               (f_vt (X86VPermilpi (load addr:$src1), (i8 imm:$src2))))]>, VEX,
+               (f_vt (X86VPermilpi (load addr:$src1), (i8 timm:$src2))))]>, VEX,
              Sched<[sched.Folded]>;
   }// Predicates = [HasAVX, NoVLX]
 }
@@ -7180,38 +7180,38 @@ def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, u8imm:$src3),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                              (i8 imm:$src3))))]>, VEX_4V, VEX_L,
+                              (i8 timm:$src3))))]>, VEX_4V, VEX_L,
           Sched<[WriteFShuffle256]>;
 def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, u8imm:$src3),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4f64 addr:$src2),
-                             (i8 imm:$src3)))]>, VEX_4V, VEX_L,
+                             (i8 timm:$src3)))]>, VEX_4V, VEX_L,
           Sched<[WriteFShuffle256.Folded, WriteFShuffle256.ReadAfterFold]>;
 }
 
 // Immediate transform to help with commuting.
-def Perm2XCommuteImm : SDNodeXForm<imm, [{
+def Perm2XCommuteImm : SDNodeXForm<timm, [{
   return getI8Imm(N->getZExtValue() ^ 0x22, SDLoc(N));
 }]>;
 
 let Predicates = [HasAVX] in {
 // Pattern with load in other operand.
 def : Pat<(v4f64 (X86VPerm2x128 (loadv4f64 addr:$src2),
-                                VR256:$src1, (i8 imm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm imm:$imm))>;
+                                VR256:$src1, (i8 timm:$imm))),
+          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
 }
 
 let Predicates = [HasAVX1Only] in {
-def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
-          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 timm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, timm:$imm)>;
 def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1,
-                  (loadv4i64 addr:$src2), (i8 imm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
+                  (loadv4i64 addr:$src2), (i8 timm:$imm))),
+          (VPERM2F128rm VR256:$src1, addr:$src2, timm:$imm)>;
 // Pattern with load in other operand.
 def : Pat<(v4i64 (X86VPerm2x128 (loadv4i64 addr:$src2),
-                                VR256:$src1, (i8 imm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm imm:$imm))>;
+                                VR256:$src1, (i8 timm:$imm))),
+          (VPERM2F128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7256,7 +7256,7 @@ multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop,
   def rr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst),
                (ins RC:$src1, i32u8imm:$src2),
                "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-               [(set VR128:$dst, (X86cvtps2ph RC:$src1, imm:$src2))]>,
+               [(set VR128:$dst, (X86cvtps2ph RC:$src1, timm:$src2))]>,
                TAPD, VEX, Sched<[RR]>;
   let hasSideEffects = 0, mayStore = 1 in
   def mr : Ii8<0x1D, MRMDestMem, (outs),
@@ -7281,15 +7281,15 @@ let Predicates = [HasF16C, NoVLX] in {
             (VCVTPH2PSrm addr:$src)>;
 
   def : Pat<(store (f64 (extractelt
-                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128:$src1, i32:$src2))),
+                         (bc_v2f64 (v8i16 (X86cvtps2ph VR128:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
-            (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>;
+            (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;
   def : Pat<(store (i64 (extractelt
-                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128:$src1, i32:$src2))),
+                         (bc_v2i64 (v8i16 (X86cvtps2ph VR128:$src1, timm:$src2))),
                          (iPTR 0))), addr:$dst),
-            (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>;
-  def : Pat<(store (v8i16 (X86cvtps2ph VR256:$src1, i32:$src2)), addr:$dst),
-            (VCVTPS2PHYmr addr:$dst, VR256:$src1, imm:$src2)>;
+            (VCVTPS2PHmr addr:$dst, VR128:$src1, timm:$src2)>;
+  def : Pat<(store (v8i16 (X86cvtps2ph VR256:$src1, timm:$src2)), addr:$dst),
+            (VCVTPS2PHYmr addr:$dst, VR256:$src1, timm:$src2)>;
 }
 
 // Patterns for  matching conversions from float to half-float and vice versa.
@@ -7326,20 +7326,20 @@ multiclass AVX2_blend_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode,
         (ins RC:$src1, RC:$src2, u8imm:$src3),
         !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))]>,
+        [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))]>,
         Sched<[sched]>, VEX_4V;
   def rmi : AVX2AIi8<opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, x86memop:$src2, u8imm:$src3),
         !strconcat(OpcodeStr,
             "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
         [(set RC:$dst,
-          (OpVT (OpNode RC:$src1, (load addr:$src2), imm:$src3)))]>,
+          (OpVT (OpNode RC:$src1, (load addr:$src2), timm:$src3)))]>,
         Sched<[sched.Folded, sched.ReadAfterFold]>, VEX_4V;
 
   // Pattern to commute if load is in first source.
-  def : Pat<(OpVT (OpNode (load addr:$src2), RC:$src1, imm:$src3)),
+  def : Pat<(OpVT (OpNode (load addr:$src2), RC:$src1, timm:$src3)),
             (!cast<Instruction>(NAME#"rmi") RC:$src1, addr:$src2,
-                                            (commuteXForm imm:$src3))>;
+                                            (commuteXForm timm:$src3))>;
 }
 
 let Predicates = [HasAVX2] in {
@@ -7350,19 +7350,19 @@ defm VPBLENDDY : AVX2_blend_rmi<0x02, "vpblendd", X86Blendi, v8i32,
                                 SchedWriteBlend.YMM, VR256, i256mem,
                                 BlendCommuteImm8>, VEX_L;
 
-def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), imm:$src3),
-          (VPBLENDDYrri VR256:$src1, VR256:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), imm:$src3),
-          (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleImm4 imm:$src3))>;
-def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, imm:$src3),
-          (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleCommuteImm4 imm:$src3))>;
+def : Pat<(X86Blendi (v4i64 VR256:$src1), (v4i64 VR256:$src2), timm:$src3),
+          (VPBLENDDYrri VR256:$src1, VR256:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi VR256:$src1, (loadv4i64 addr:$src2), timm:$src3),
+          (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleImm4 timm:$src3))>;
+def : Pat<(X86Blendi (loadv4i64 addr:$src2), VR256:$src1, timm:$src3),
+          (VPBLENDDYrmi VR256:$src1, addr:$src2, (BlendScaleCommuteImm4 timm:$src3))>;
 
-def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), imm:$src3),
-          (VPBLENDDrri VR128:$src1, VR128:$src2, (BlendScaleImm2to4 imm:$src3))>;
-def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), imm:$src3),
-          (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleImm2to4 imm:$src3))>;
-def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, imm:$src3),
-          (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2to4 imm:$src3))>;
+def : Pat<(X86Blendi (v2i64 VR128:$src1), (v2i64 VR128:$src2), timm:$src3),
+          (VPBLENDDrri VR128:$src1, VR128:$src2, (BlendScaleImm2to4 timm:$src3))>;
+def : Pat<(X86Blendi VR128:$src1, (loadv2i64 addr:$src2), timm:$src3),
+          (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleImm2to4 timm:$src3))>;
+def : Pat<(X86Blendi (loadv2i64 addr:$src2), VR128:$src1, timm:$src3),
+          (VPBLENDDrmi VR128:$src1, addr:$src2, (BlendScaleCommuteImm2to4 timm:$src3))>;
 }
 
 // For insertion into the zero index (low half) of a 256-bit vector, it is
@@ -7406,7 +7406,7 @@ def : Pat<(insert_subvector (loadv32i8 addr:$src2), (v16i8 VR128:$src1), (iPTR 0
 //               destination operand
 //
 multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,
-                          X86MemOperand x86memop, PatFrag ld_frag,
+                          X86MemOperand x86memop, PatFrag bcast_frag,
                           ValueType OpVT128, ValueType OpVT256, Predicate prd> {
   let Predicates = [HasAVX2, prd] in {
     def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
@@ -7417,7 +7417,7 @@ multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,
     def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src),
                   !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                   [(set VR128:$dst,
-                   (OpVT128 (X86VBroadcast (ld_frag addr:$src))))]>,
+                   (OpVT128 (bcast_frag addr:$src)))]>,
                   Sched<[SchedWriteShuffle.XMM.Folded]>, VEX;
     def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
@@ -7427,7 +7427,7 @@ multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,
     def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                    [(set VR256:$dst,
-                    (OpVT256 (X86VBroadcast (ld_frag addr:$src))))]>,
+                    (OpVT256 (bcast_frag addr:$src)))]>,
                    Sched<[SchedWriteShuffle.XMM.Folded]>, VEX, VEX_L;
 
     // Provide aliases for broadcast from the same register class that
@@ -7438,13 +7438,13 @@ multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,
   }
 }
 
-defm VPBROADCASTB  : avx2_broadcast<0x78, "vpbroadcastb", i8mem, loadi8,
+defm VPBROADCASTB  : avx2_broadcast<0x78, "vpbroadcastb", i8mem, X86VBroadcastld8,
                                     v16i8, v32i8, NoVLX_Or_NoBWI>;
-defm VPBROADCASTW  : avx2_broadcast<0x79, "vpbroadcastw", i16mem, loadi16,
+defm VPBROADCASTW  : avx2_broadcast<0x79, "vpbroadcastw", i16mem, X86VBroadcastld16,
                                     v8i16, v16i16, NoVLX_Or_NoBWI>;
-defm VPBROADCASTD  : avx2_broadcast<0x58, "vpbroadcastd", i32mem, loadi32,
+defm VPBROADCASTD  : avx2_broadcast<0x58, "vpbroadcastd", i32mem, X86VBroadcastld32,
                                     v4i32, v8i32, NoVLX>;
-defm VPBROADCASTQ  : avx2_broadcast<0x59, "vpbroadcastq", i64mem, loadi64,
+defm VPBROADCASTQ  : avx2_broadcast<0x59, "vpbroadcastq", i64mem, X86VBroadcastld64,
                                     v2i64, v4i64, NoVLX>;
 
 let Predicates = [HasAVX2, NoVLX] in {
@@ -7453,6 +7453,12 @@ let Predicates = [HasAVX2, NoVLX] in {
             (VPBROADCASTQrm addr:$src)>;
   def : Pat<(v4i64 (X86VBroadcast (v2i64 (X86vzload64 addr:$src)))),
             (VPBROADCASTQYrm addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8/i16.
+  def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
+            (VPBROADCASTDrm addr:$src)>;
+  def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))),
+            (VPBROADCASTDYrm addr:$src)>;
 }
 let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {
   // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably.
@@ -7473,6 +7479,12 @@ let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in {
   def : Pat<(v16i16 (X86VBroadcast
               (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
             (VPBROADCASTWYrm addr:$src)>;
+
+  // FIXME this is to handle aligned extloads from i8.
+  def : Pat<(v8i16 (X86VBroadcast (loadi16 addr:$src))),
+            (VPBROADCASTWrm addr:$src)>;
+  def : Pat<(v16i16 (X86VBroadcast (loadi16 addr:$src))),
+            (VPBROADCASTWYrm addr:$src)>;
 }
 
 let Predicates = [HasAVX2, NoVLX] in {
@@ -7518,11 +7530,11 @@ let Predicates = [HasAVX2, NoVLX] in {
 
 // AVX1 broadcast patterns
 let Predicates = [HasAVX1Only] in {
-def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))),
+def : Pat<(v8i32 (X86VBroadcastld32 addr:$src)),
           (VBROADCASTSSYrm addr:$src)>;
-def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))),
+def : Pat<(v4i64 (X86VBroadcastld64 addr:$src)),
           (VBROADCASTSDYrm addr:$src)>;
-def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
+def : Pat<(v4i32 (X86VBroadcastld32 addr:$src)),
           (VBROADCASTSSrm addr:$src)>;
 }
 
@@ -7532,7 +7544,7 @@ let Predicates = [HasAVX, NoVLX] in {
   // 128bit broadcasts:
   def : Pat<(v2f64 (X86VBroadcast f64:$src)),
             (VMOVDDUPrr (v2f64 (COPY_TO_REGCLASS FR64:$src, VR128)))>;
-  def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),
+  def : Pat<(v2f64 (X86VBroadcastld64 addr:$src)),
             (VMOVDDUPrm addr:$src)>;
 
   def : Pat<(v2f64 (X86VBroadcast v2f64:$src)),
@@ -7568,7 +7580,7 @@ let Predicates = [HasAVX1Only] in {
 
   def : Pat<(v2i64 (X86VBroadcast i64:$src)),
             (VPSHUFDri (VMOV64toPQIrr GR64:$src), 0x44)>;
-  def : Pat<(v2i64 (X86VBroadcast (loadi64 addr:$src))),
+  def : Pat<(v2i64 (X86VBroadcastld64 addr:$src)),
             (VMOVDDUPrm addr:$src)>;
 }
 
@@ -7611,7 +7623,7 @@ multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
                        !strconcat(OpcodeStr,
                            "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                        [(set VR256:$dst,
-                         (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>,
+                         (OpVT (X86VPermi VR256:$src1, (i8 timm:$src2))))]>,
                        Sched<[Sched]>, VEX, VEX_L;
     def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst),
                        (ins memOp:$src1, u8imm:$src2),
@@ -7619,7 +7631,7 @@ multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
                            "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                        [(set VR256:$dst,
                          (OpVT (X86VPermi (mem_frag addr:$src1),
-                                (i8 imm:$src2))))]>,
+                                (i8 timm:$src2))))]>,
                        Sched<[Sched.Folded, Sched.ReadAfterFold]>, VEX, VEX_L;
   }
 }
@@ -7638,19 +7650,19 @@ def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, u8imm:$src3),
           "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                            (i8 imm:$src3))))]>, Sched<[WriteShuffle256]>,
+                            (i8 timm:$src3))))]>, Sched<[WriteShuffle256]>,
           VEX_4V, VEX_L;
 def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, u8imm:$src3),
           "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4i64 addr:$src2),
-                             (i8 imm:$src3)))]>,
+                             (i8 timm:$src3)))]>,
           Sched<[WriteShuffle256.Folded, WriteShuffle256.ReadAfterFold]>, VEX_4V, VEX_L;
 
 let Predicates = [HasAVX2] in
 def : Pat<(v4i64 (X86VPerm2x128 (loadv4i64 addr:$src2),
-                                VR256:$src1, (i8 imm:$imm))),
-          (VPERM2I128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm imm:$imm))>;
+                                VR256:$src1, (i8 timm:$imm))),
+          (VPERM2I128rm VR256:$src1, addr:$src2, (Perm2XCommuteImm timm:$imm))>;
 
 
 //===----------------------------------------------------------------------===//
@@ -7735,7 +7747,7 @@ defm VPMASKMOVQ : avx2_pmovmask<"vpmaskmovq",
                                 int_x86_avx2_maskstore_q_256>, VEX_W;
 
 multiclass maskmov_lowering<string InstrStr, RegisterClass RC, ValueType VT,
-                          ValueType MaskVT, string BlendStr, ValueType ZeroVT> {
+                          ValueType MaskVT> {
     // masked store
     def: Pat<(masked_store (VT RC:$src), addr:$ptr, (MaskVT RC:$mask)),
              (!cast<Instruction>(InstrStr#"mr") addr:$ptr, RC:$mask, RC:$src)>;
@@ -7747,23 +7759,23 @@ multiclass maskmov_lowering<string InstrStr, RegisterClass RC, ValueType VT,
              (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr)>;
 }
 let Predicates = [HasAVX] in {
-  defm : maskmov_lowering<"VMASKMOVPS", VR128, v4f32, v4i32, "VBLENDVPS", v4i32>;
-  defm : maskmov_lowering<"VMASKMOVPD", VR128, v2f64, v2i64, "VBLENDVPD", v4i32>;
-  defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8f32, v8i32, "VBLENDVPSY", v8i32>;
-  defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4f64, v4i64, "VBLENDVPDY", v8i32>;
+  defm : maskmov_lowering<"VMASKMOVPS", VR128, v4f32, v4i32>;
+  defm : maskmov_lowering<"VMASKMOVPD", VR128, v2f64, v2i64>;
+  defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8f32, v8i32>;
+  defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4f64, v4i64>;
 }
 let Predicates = [HasAVX1Only] in {
   // load/store i32/i64 not supported use ps/pd version
-  defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8i32, v8i32, "VBLENDVPSY", v8i32>;
-  defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4i64, v4i64, "VBLENDVPDY", v8i32>;
-  defm : maskmov_lowering<"VMASKMOVPS", VR128, v4i32, v4i32, "VBLENDVPS", v4i32>;
-  defm : maskmov_lowering<"VMASKMOVPD", VR128, v2i64, v2i64, "VBLENDVPD", v4i32>;
+  defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8i32, v8i32>;
+  defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4i64, v4i64>;
+  defm : maskmov_lowering<"VMASKMOVPS", VR128, v4i32, v4i32>;
+  defm : maskmov_lowering<"VMASKMOVPD", VR128, v2i64, v2i64>;
 }
 let Predicates = [HasAVX2] in {
-  defm : maskmov_lowering<"VPMASKMOVDY", VR256, v8i32, v8i32, "VBLENDVPSY", v8i32>;
-  defm : maskmov_lowering<"VPMASKMOVQY", VR256, v4i64, v4i64, "VBLENDVPDY", v8i32>;
-  defm : maskmov_lowering<"VPMASKMOVD", VR128, v4i32, v4i32, "VBLENDVPS", v4i32>;
-  defm : maskmov_lowering<"VPMASKMOVQ", VR128, v2i64, v2i64, "VBLENDVPD", v4i32>;
+  defm : maskmov_lowering<"VPMASKMOVDY", VR256, v8i32, v8i32>;
+  defm : maskmov_lowering<"VPMASKMOVQY", VR256, v4i64, v4i64>;
+  defm : maskmov_lowering<"VPMASKMOVD", VR128, v4i32, v4i32>;
+  defm : maskmov_lowering<"VPMASKMOVQ", VR128, v2i64, v2i64>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7931,13 +7943,13 @@ multiclass GF2P8AFFINE_rmi<bits<8> Op, string OpStr, ValueType OpVT,
       OpStr##"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}") in {
   def rri : Ii8<Op, MRMSrcReg, (outs RC:$dst),
               (ins RC:$src1, RC:$src2, u8imm:$src3), "",
-              [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))],
+              [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, timm:$src3)))],
               SSEPackedInt>, Sched<[SchedWriteVecALU.XMM]>;
   def rmi : Ii8<Op, MRMSrcMem, (outs RC:$dst),
               (ins RC:$src1, X86MemOp:$src2, u8imm:$src3), "",
               [(set RC:$dst, (OpVT (OpNode RC:$src1,
                                     (MemOpFrag addr:$src2),
-                              imm:$src3)))], SSEPackedInt>,
+                              timm:$src3)))], SSEPackedInt>,
               Sched<[SchedWriteVecALU.XMM.Folded, SchedWriteVecALU.XMM.ReadAfterFold]>;
   }
 }
diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td
index 7050e19174946..7f41feb6c0d90 100644
--- a/llvm/lib/Target/X86/X86InstrSystem.td
+++ b/llvm/lib/Target/X86/X86InstrSystem.td
@@ -43,7 +43,7 @@ def INT3 : I<0xcc, RawFrm, (outs), (ins), "int3", [(int_x86_int (i8 3))]>;
 let SchedRW = [WriteSystem] in {
 
 def INT : Ii8<0xcd, RawFrm, (outs), (ins u8imm:$trap), "int\t$trap",
-              [(int_x86_int imm:$trap)]>;
+              [(int_x86_int timm:$trap)]>;
 
 
 def SYSCALL  : I<0x05, RawFrm, (outs), (ins), "syscall", []>, TB;
diff --git a/llvm/lib/Target/X86/X86InstrTSX.td b/llvm/lib/Target/X86/X86InstrTSX.td
index fc0da845299f5..3a1212342a134 100644
--- a/llvm/lib/Target/X86/X86InstrTSX.td
+++ b/llvm/lib/Target/X86/X86InstrTSX.td
@@ -45,7 +45,7 @@ def XTEST : I<0x01, MRM_D6, (outs), (ins),
 
 def XABORT : Ii8<0xc6, MRM_F8, (outs), (ins i8imm:$imm),
                  "xabort\t$imm",
-                 [(int_x86_xabort imm:$imm)]>, Requires<[HasRTM]>;
+                 [(int_x86_xabort timm:$imm)]>, Requires<[HasRTM]>;
 } // SchedRW
 
 // HLE prefixes
diff --git a/llvm/lib/Target/X86/X86InstrXOP.td b/llvm/lib/Target/X86/X86InstrXOP.td
index 9a972a9adf9a0..229af366d940a 100644
--- a/llvm/lib/Target/X86/X86InstrXOP.td
+++ b/llvm/lib/Target/X86/X86InstrXOP.td
@@ -143,13 +143,13 @@ multiclass xop3opimm<bits<8> opc, string OpcodeStr, SDNode OpNode,
            (ins VR128:$src1, u8imm:$src2),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
            [(set VR128:$dst,
-              (vt128 (OpNode (vt128 VR128:$src1), imm:$src2)))]>,
+              (vt128 (OpNode (vt128 VR128:$src1), timm:$src2)))]>,
            XOP, Sched<[sched]>;
   def mi : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst),
            (ins i128mem:$src1, u8imm:$src2),
            !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
            [(set VR128:$dst,
-              (vt128 (OpNode (vt128 (load addr:$src1)), imm:$src2)))]>,
+              (vt128 (OpNode (vt128 (load addr:$src1)), timm:$src2)))]>,
            XOP, Sched<[sched.Folded, sched.ReadAfterFold]>;
 }
 
@@ -251,7 +251,7 @@ multiclass xopvpcom<bits<8> opc, string Suffix, SDNode OpNode, ValueType vt128,
              "\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}"),
              [(set VR128:$dst,
                 (vt128 (OpNode (vt128 VR128:$src1), (vt128 VR128:$src2),
-                               imm:$cc)))]>,
+                               timm:$cc)))]>,
              XOP_4V, Sched<[sched]>;
     def mi : IXOPi8<opc, MRMSrcMem, (outs VR128:$dst),
              (ins VR128:$src1, i128mem:$src2, u8imm:$cc),
@@ -260,14 +260,14 @@ multiclass xopvpcom<bits<8> opc, string Suffix, SDNode OpNode, ValueType vt128,
              [(set VR128:$dst,
                 (vt128 (OpNode (vt128 VR128:$src1),
                                (vt128 (load addr:$src2)),
-                                imm:$cc)))]>,
+                                timm:$cc)))]>,
              XOP_4V, Sched<[sched.Folded, sched.ReadAfterFold]>;
   }
 
   def : Pat<(OpNode (load addr:$src2),
-                    (vt128 VR128:$src1), imm:$cc),
+                    (vt128 VR128:$src1), timm:$cc),
             (!cast<Instruction>(NAME#"mi") VR128:$src1, addr:$src2,
-                                           (CommuteVPCOMCC imm:$cc))>;
+                                           (CommuteVPCOMCC timm:$cc))>;
 }
 
 defm VPCOMB  : xopvpcom<0xCC, "b", X86vpcom, v16i8, SchedWriteVecALU.XMM>;
@@ -422,7 +422,7 @@ multiclass xop_vpermil2<bits<8> Opc, string OpcodeStr, RegisterClass RC,
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set RC:$dst,
-           (VT (X86vpermil2 RC:$src1, RC:$src2, RC:$src3, (i8 imm:$src4))))]>,
+           (VT (X86vpermil2 RC:$src1, RC:$src2, RC:$src3, (i8 timm:$src4))))]>,
         Sched<[sched]>;
   def rm : IXOP5<Opc, MRMSrcMemOp4, (outs RC:$dst),
         (ins RC:$src1, RC:$src2, intmemop:$src3, u4imm:$src4),
@@ -430,7 +430,7 @@ multiclass xop_vpermil2<bits<8> Opc, string OpcodeStr, RegisterClass RC,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set RC:$dst,
           (VT (X86vpermil2 RC:$src1, RC:$src2, (IntLdFrag addr:$src3),
-                           (i8 imm:$src4))))]>, VEX_W,
+                           (i8 timm:$src4))))]>, VEX_W,
         Sched<[sched.Folded, sched.ReadAfterFold, sched.ReadAfterFold]>;
   def mr : IXOP5<Opc, MRMSrcMem, (outs RC:$dst),
         (ins RC:$src1, fpmemop:$src2, RC:$src3, u4imm:$src4),
@@ -438,7 +438,7 @@ multiclass xop_vpermil2<bits<8> Opc, string OpcodeStr, RegisterClass RC,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set RC:$dst,
           (VT (X86vpermil2 RC:$src1, (FPLdFrag addr:$src2),
-                           RC:$src3, (i8 imm:$src4))))]>,
+                           RC:$src3, (i8 timm:$src4))))]>,
         Sched<[sched.Folded, sched.ReadAfterFold,
                // fpmemop:$src2
                ReadDefault, ReadDefault, ReadDefault, ReadDefault, ReadDefault,
diff --git a/llvm/lib/Target/X86/X86IntrinsicsInfo.h b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
index 40141d8946294..1d7adbaa9e99c 100644
--- a/llvm/lib/Target/X86/X86IntrinsicsInfo.h
+++ b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
@@ -23,7 +23,7 @@ enum IntrinsicType : uint16_t {
   GATHER, SCATTER, PREFETCH, RDSEED, RDRAND, RDPMC, RDTSC, XTEST, XGETBV, ADX, FPCLASSS,
   INTR_TYPE_1OP, INTR_TYPE_2OP, INTR_TYPE_3OP, INTR_TYPE_4OP,
   INTR_TYPE_3OP_IMM8,
-  CMP_MASK_CC,CMP_MASK_SCALAR_CC, VSHIFT, COMI, COMI_RM, BLENDV,
+  CMP_MASK_CC,CMP_MASK_SCALAR_CC, VSHIFT, COMI, COMI_RM, BLENDV, BEXTRI,
   CVTPD2PS_MASK,
   INTR_TYPE_1OP_SAE, INTR_TYPE_2OP_SAE,
   INTR_TYPE_1OP_MASK_SAE, INTR_TYPE_2OP_MASK_SAE, INTR_TYPE_3OP_MASK_SAE,
@@ -1101,8 +1101,8 @@ static const IntrinsicData  IntrinsicsWithoutChain[] = {
   X86_INTRINSIC_DATA(ssse3_pshuf_b_128, INTR_TYPE_2OP, X86ISD::PSHUFB, 0),
   X86_INTRINSIC_DATA(subborrow_32,      ADX, X86ISD::SBB, X86ISD::SUB),
   X86_INTRINSIC_DATA(subborrow_64,      ADX, X86ISD::SBB, X86ISD::SUB),
-  X86_INTRINSIC_DATA(tbm_bextri_u32,    INTR_TYPE_2OP, X86ISD::BEXTR, 0),
-  X86_INTRINSIC_DATA(tbm_bextri_u64,    INTR_TYPE_2OP, X86ISD::BEXTR, 0),
+  X86_INTRINSIC_DATA(tbm_bextri_u32,    BEXTRI, X86ISD::BEXTR, 0),
+  X86_INTRINSIC_DATA(tbm_bextri_u64,    BEXTRI, X86ISD::BEXTR, 0),
   X86_INTRINSIC_DATA(vcvtph2ps_128,     INTR_TYPE_1OP, X86ISD::CVTPH2PS, 0),
   X86_INTRINSIC_DATA(vcvtph2ps_256,     INTR_TYPE_1OP, X86ISD::CVTPH2PS, 0),
   X86_INTRINSIC_DATA(vcvtps2ph_128,     INTR_TYPE_2OP, X86ISD::CVTPS2PH, 0),
diff --git a/llvm/lib/Target/X86/X86RetpolineThunks.cpp b/llvm/lib/Target/X86/X86RetpolineThunks.cpp
index 1a0f6ecb01e61..f8464c7e82981 100644
--- a/llvm/lib/Target/X86/X86RetpolineThunks.cpp
+++ b/llvm/lib/Target/X86/X86RetpolineThunks.cpp
@@ -58,8 +58,8 @@ class X86RetpolineThunks : public MachineFunctionPass {
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     MachineFunctionPass::getAnalysisUsage(AU);
-    AU.addRequired<MachineModuleInfo>();
-    AU.addPreserved<MachineModuleInfo>();
+    AU.addRequired<MachineModuleInfoWrapperPass>();
+    AU.addPreserved<MachineModuleInfoWrapperPass>();
   }
 
 private:
@@ -97,7 +97,7 @@ bool X86RetpolineThunks::runOnMachineFunction(MachineFunction &MF) {
   TII = STI->getInstrInfo();
   Is64Bit = TM->getTargetTriple().getArch() == Triple::x86_64;
 
-  MMI = &getAnalysis<MachineModuleInfo>();
+  MMI = &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
   Module &M = const_cast<Module &>(*MMI->getModule());
 
   // If this function is not a thunk, check to see if we need to insert
@@ -279,7 +279,7 @@ void X86RetpolineThunks::populateThunk(MachineFunction &MF,
 
   CallTarget->addLiveIn(Reg);
   CallTarget->setHasAddressTaken();
-  CallTarget->setLogAlignment(4);
+  CallTarget->setAlignment(Align(16));
   insertRegReturnAddrClobber(*CallTarget, Reg);
   CallTarget->back().setPreInstrSymbol(MF, TargetSym);
   BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index ce926b40ac376..4d7495641d923 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -427,6 +427,9 @@ class X86Subtarget final : public X86GenSubtargetInfo {
   /// Use software floating point for code generation.
   bool UseSoftFloat = false;
 
+  /// Use alias analysis during code generation.
+  bool UseAA = false;
+
   /// The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned stackAlignment = 4;
@@ -741,6 +744,7 @@ class X86Subtarget final : public X86GenSubtargetInfo {
            X86ProcFamily == IntelTRM;
   }
   bool useSoftFloat() const { return UseSoftFloat; }
+  bool useAA() const override { return UseAA; }
 
   /// Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
   /// no-sse2). There isn't any reason to disable it if the target processor
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 8f3d18707619c..9b74a593b182d 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -106,8 +106,7 @@ static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
   llvm_unreachable("unknown subtarget type");
 }
 
-static std::string computeDataLayout(const Triple &TT,
-                                     bool AddressSpaces = true) {
+static std::string computeDataLayout(const Triple &TT) {
   // X86 is little endian
   std::string Ret = "e";
 
@@ -119,8 +118,7 @@ static std::string computeDataLayout(const Triple &TT,
     Ret += "-p:32:32";
 
   // Address spaces for 32 bit signed, 32 bit unsigned, and 64 bit pointers.
-  if (AddressSpaces)
-    Ret += "-p270:32:32-p271:32:32-p272:64:64";
+  Ret += "-p270:32:32-p271:32:32-p272:64:64";
 
   // Some ABIs align 64 bit integers and doubles to 64 bits, others to 32.
   if (TT.isArch64Bit() || TT.isOSWindows() || TT.isOSNaCl())
@@ -223,8 +221,7 @@ X86TargetMachine::X86TargetMachine(const Target &T, const Triple &TT,
           getEffectiveRelocModel(TT, JIT, RM),
           getEffectiveX86CodeModel(CM, JIT, TT.getArch() == Triple::x86_64),
           OL),
-      TLOF(createTLOF(getTargetTriple())),
-      DLNoAddrSpaces(computeDataLayout(TT, /*AddressSpaces=*/false)) {
+      TLOF(createTLOF(getTargetTriple())) {
   // On PS4, the "return address" of a 'noreturn' call must still be within
   // the calling function, and TrapUnreachable is an easy way to get that.
   if (TT.isPS4() || TT.isOSBinFormatMachO()) {
@@ -318,13 +315,6 @@ X86TargetMachine::getSubtargetImpl(const Function &F) const {
   return I.get();
 }
 
-bool X86TargetMachine::isCompatibleDataLayout(
-    const DataLayout &Candidate) const {
-  // Maintain compatibility with datalayouts that don't have address space
-  // pointer sizes.
-  return DL == Candidate || DLNoAddrSpaces == Candidate;
-}
-
 //===----------------------------------------------------------------------===//
 // Command line options for x86
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h
index aa970376b2ab2..b999e2e86af68 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.h
+++ b/llvm/lib/Target/X86/X86TargetMachine.h
@@ -30,7 +30,6 @@ class X86RegisterBankInfo;
 class X86TargetMachine final : public LLVMTargetMachine {
   std::unique_ptr<TargetLoweringObjectFile> TLOF;
   mutable StringMap<std::unique_ptr<X86Subtarget>> SubtargetMap;
-  const DataLayout DLNoAddrSpaces;
 
 public:
   X86TargetMachine(const Target &T, const Triple &TT, StringRef CPU,
@@ -53,8 +52,6 @@ class X86TargetMachine final : public LLVMTargetMachine {
   TargetLoweringObjectFile *getObjFileLowering() const override {
     return TLOF.get();
   }
-
-  bool isCompatibleDataLayout(const DataLayout &Candidate) const override;
 };
 
 } // end namespace llvm
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 7c5d54af34f33..83c78451253bf 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -50,8 +50,6 @@ using namespace llvm;
 
 #define DEBUG_TYPE "x86tti"
 
-extern cl::opt<bool> ExperimentalVectorWideningLegalization;
-
 //===----------------------------------------------------------------------===//
 //
 // X86 cost model.
@@ -920,8 +918,7 @@ int X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
       // FIXME: We can use permq for 64-bit or larger extracts from 256-bit
       // vectors.
       int OrigSubElts = SubTp->getVectorNumElements();
-      if (ExperimentalVectorWideningLegalization &&
-          NumSubElts > OrigSubElts &&
+      if (NumSubElts > OrigSubElts &&
           (Index % OrigSubElts) == 0 && (NumSubElts % OrigSubElts) == 0 &&
           LT.second.getVectorElementType() ==
             SubLT.second.getVectorElementType() &&
@@ -1333,12 +1330,6 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
   // TODO: For AVX512DQ + AVX512VL, we also have cheap casts for 128-bit and
   // 256-bit wide vectors.
 
-  // Used with widening legalization
-  static const TypeConversionCostTblEntry AVX512FConversionTblWide[] = {
-    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
-  };
-
   static const TypeConversionCostTblEntry AVX512FConversionTbl[] = {
     { ISD::FP_EXTEND, MVT::v8f64,   MVT::v8f32,  1 },
     { ISD::FP_EXTEND, MVT::v8f64,   MVT::v16f32, 3 },
@@ -1356,6 +1347,8 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i8,  1 },
     { ISD::SIGN_EXTEND, MVT::v16i32, MVT::v16i16, 1 },
     { ISD::ZERO_EXTEND, MVT::v16i32, MVT::v16i16, 1 },
+    { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
+    { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i8,   1 },
     { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i16,  1 },
     { ISD::ZERO_EXTEND, MVT::v8i64,  MVT::v8i16,  1 },
     { ISD::SIGN_EXTEND, MVT::v8i64,  MVT::v8i32,  1 },
@@ -1410,28 +1403,19 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::FP_TO_UINT,  MVT::v16i8,  MVT::v16f32, 2 },
   };
 
-  static const TypeConversionCostTblEntry AVX2ConversionTblWide[] = {
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,   1 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i8,   1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16,  1 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i16,  1 },
-  };
-
   static const TypeConversionCostTblEntry AVX2ConversionTbl[] = {
     { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,   3 },
     { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,   3 },
     { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,   3 },
     { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,   3 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,   3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i8,   3 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,   3 },
-    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i8,   3 },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,   1 },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i8,   1 },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,   1 },
+    { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i8,   1 },
     { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8,  1 },
     { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8,  1 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16,  3 },
-    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i16,  3 },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16,  1 },
+    { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i16,  1 },
     { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16,  1 },
     { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16,  1 },
     { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  1 },
@@ -1450,24 +1434,18 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::UINT_TO_FP,  MVT::v8f32,  MVT::v8i32,  8 },
   };
 
-  static const TypeConversionCostTblEntry AVXConversionTblWide[] = {
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,  4 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,  4 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16, 4 },
-  };
-
   static const TypeConversionCostTblEntry AVXConversionTbl[] = {
     { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i1,  6 },
     { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i1,  4 },
     { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i1,  7 },
     { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i1,  4 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,  6 },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i8,  4 },
     { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i8,  4 },
-    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,  7 },
+    { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i8,  4 },
     { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i8,  4 },
     { ISD::SIGN_EXTEND, MVT::v16i16, MVT::v16i8, 4 },
     { ISD::ZERO_EXTEND, MVT::v16i16, MVT::v16i8, 4 },
-    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16, 6 },
+    { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i16, 4 },
     { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i16, 3 },
     { ISD::SIGN_EXTEND, MVT::v8i32,  MVT::v8i16, 4 },
     { ISD::ZERO_EXTEND, MVT::v8i32,  MVT::v8i16, 4 },
@@ -1480,7 +1458,10 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::TRUNCATE,    MVT::v4i8,  MVT::v4i64,  4 },
     { ISD::TRUNCATE,    MVT::v4i16, MVT::v4i64,  4 },
     { ISD::TRUNCATE,    MVT::v4i32, MVT::v4i64,  4 },
+    { ISD::TRUNCATE,    MVT::v8i8,  MVT::v8i64, 11 },
+    { ISD::TRUNCATE,    MVT::v8i16, MVT::v8i64,  9 },
     { ISD::TRUNCATE,    MVT::v8i32, MVT::v8i64,  9 },
+    { ISD::TRUNCATE,    MVT::v16i8, MVT::v16i64, 11 },
 
     { ISD::SINT_TO_FP,  MVT::v4f32, MVT::v4i1,  3 },
     { ISD::SINT_TO_FP,  MVT::v4f64, MVT::v4i1,  3 },
@@ -1566,15 +1547,11 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::TRUNCATE,    MVT::v8i8,   MVT::v8i32,  3 },
     { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  3 },
     { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, 6 },
+    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i64,  1 }, // PSHUFB
 
     { ISD::UINT_TO_FP,  MVT::f64,    MVT::i64,    4 },
   };
 
-  static const TypeConversionCostTblEntry SSE2ConversionTblWide[] = {
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, 2*10 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2*10 },
-  };
-
   static const TypeConversionCostTblEntry SSE2ConversionTbl[] = {
     // These are somewhat magic numbers justified by looking at the output of
     // Intel's IACA, running some kernels and making sure when we take
@@ -1584,7 +1561,8 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::SINT_TO_FP, MVT::v4f32, MVT::v8i16, 15 },
     { ISD::SINT_TO_FP, MVT::v2f64, MVT::v8i16, 8*10 },
     { ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i32, 5 },
-    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, 4*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v4i32, 2*10 },
+    { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i32, 2*10 },
     { ISD::SINT_TO_FP, MVT::v4f32, MVT::v2i64, 15 },
     { ISD::SINT_TO_FP, MVT::v2f64, MVT::v2i64, 2*10 },
 
@@ -1628,27 +1606,26 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
     { ISD::ZERO_EXTEND, MVT::v4i64,  MVT::v4i32,  3 },
     { ISD::SIGN_EXTEND, MVT::v4i64,  MVT::v4i32,  5 },
 
+    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i16,  2 }, // PAND+PACKUSWB
     { ISD::TRUNCATE,    MVT::v4i8,   MVT::v4i16,  4 },
     { ISD::TRUNCATE,    MVT::v8i8,   MVT::v8i16,  2 },
     { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i16, 3 },
+    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i32,  3 }, // PAND+3*PACKUSWB
+    { ISD::TRUNCATE,    MVT::v2i16,  MVT::v2i32,  1 },
     { ISD::TRUNCATE,    MVT::v4i8,   MVT::v4i32,  3 },
     { ISD::TRUNCATE,    MVT::v4i16,  MVT::v4i32,  3 },
     { ISD::TRUNCATE,    MVT::v8i8,   MVT::v8i32,  4 },
     { ISD::TRUNCATE,    MVT::v16i8,  MVT::v16i32, 7 },
     { ISD::TRUNCATE,    MVT::v8i16,  MVT::v8i32,  5 },
     { ISD::TRUNCATE,    MVT::v16i16, MVT::v16i32, 10 },
+    { ISD::TRUNCATE,    MVT::v2i8,   MVT::v2i64,  4 }, // PAND+3*PACKUSWB
+    { ISD::TRUNCATE,    MVT::v2i16,  MVT::v2i64,  2 }, // PSHUFD+PSHUFLW
+    { ISD::TRUNCATE,    MVT::v2i32,  MVT::v2i64,  1 }, // PSHUFD
   };
 
   std::pair<int, MVT> LTSrc = TLI->getTypeLegalizationCost(DL, Src);
   std::pair<int, MVT> LTDest = TLI->getTypeLegalizationCost(DL, Dst);
 
-  if (ST->hasSSE2() && !ST->hasAVX() &&
-      ExperimentalVectorWideningLegalization) {
-    if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTblWide, ISD,
-                                                   LTDest.second, LTSrc.second))
-      return LTSrc.first * Entry->Cost;
-  }
-
   if (ST->hasSSE2() && !ST->hasAVX()) {
     if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
                                                    LTDest.second, LTSrc.second))
@@ -1680,35 +1657,18 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                                                      SimpleDstTy, SimpleSrcTy))
         return Entry->Cost;
 
-    if (ST->hasAVX512() && ExperimentalVectorWideningLegalization)
-      if (const auto *Entry = ConvertCostTableLookup(AVX512FConversionTblWide, ISD,
-                                                     SimpleDstTy, SimpleSrcTy))
-        return Entry->Cost;
-
     if (ST->hasAVX512())
       if (const auto *Entry = ConvertCostTableLookup(AVX512FConversionTbl, ISD,
                                                      SimpleDstTy, SimpleSrcTy))
         return Entry->Cost;
   }
 
-  if (ST->hasAVX2() && ExperimentalVectorWideningLegalization) {
-    if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTblWide, ISD,
-                                                   SimpleDstTy, SimpleSrcTy))
-      return Entry->Cost;
-  }
-
   if (ST->hasAVX2()) {
     if (const auto *Entry = ConvertCostTableLookup(AVX2ConversionTbl, ISD,
                                                    SimpleDstTy, SimpleSrcTy))
       return Entry->Cost;
   }
 
-  if (ST->hasAVX() && ExperimentalVectorWideningLegalization) {
-    if (const auto *Entry = ConvertCostTableLookup(AVXConversionTblWide, ISD,
-                                                   SimpleDstTy, SimpleSrcTy))
-      return Entry->Cost;
-  }
-
   if (ST->hasAVX()) {
     if (const auto *Entry = ConvertCostTableLookup(AVXConversionTbl, ISD,
                                                    SimpleDstTy, SimpleSrcTy))
@@ -1721,12 +1681,6 @@ int X86TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
       return Entry->Cost;
   }
 
-  if (ST->hasSSE2() && ExperimentalVectorWideningLegalization) {
-    if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTblWide, ISD,
-                                                   SimpleDstTy, SimpleSrcTy))
-      return Entry->Cost;
-  }
-
   if (ST->hasSSE2()) {
     if (const auto *Entry = ConvertCostTableLookup(SSE2ConversionTbl, ISD,
                                                    SimpleDstTy, SimpleSrcTy))
@@ -1787,6 +1741,11 @@ int X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
     }
   }
 
+  static const CostTblEntry SLMCostTbl[] = {
+    // slm pcmpeq/pcmpgt throughput is 2
+    { ISD::SETCC,   MVT::v2i64,   2 },
+  };
+
   static const CostTblEntry AVX512BWCostTbl[] = {
     { ISD::SETCC,   MVT::v32i16,  1 },
     { ISD::SETCC,   MVT::v64i8,   1 },
@@ -1873,6 +1832,10 @@ int X86TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
     { ISD::SELECT,  MVT::v4f32,   3 }, // andps + andnps + orps
   };
 
+  if (ST->isSLM())
+    if (const auto *Entry = CostTableLookup(SLMCostTbl, ISD, MTy))
+      return LT.first * (ExtraCost + Entry->Cost);
+
   if (ST->hasBWI())
     if (const auto *Entry = CostTableLookup(AVX512BWCostTbl, ISD, MTy))
       return LT.first * (ExtraCost + Entry->Cost);
@@ -2581,7 +2544,7 @@ int X86TTIImpl::getArithmeticReductionCost(unsigned Opcode, Type *ValTy,
   // in the table.
   // FIXME: Is there a better way to do this?
   EVT VT = TLI->getValueType(DL, ValTy);
-  if (VT.isSimple() && ExperimentalVectorWideningLegalization) {
+  if (VT.isSimple()) {
     MVT MTy = VT.getSimpleVT();
     if (IsPairwise) {
       if (ST->hasAVX())
@@ -3275,7 +3238,7 @@ bool X86TTIImpl::isLegalMaskedStore(Type *DataType) {
   return isLegalMaskedLoad(DataType);
 }
 
-bool X86TTIImpl::isLegalNTLoad(Type *DataType, llvm::Align Alignment) {
+bool X86TTIImpl::isLegalNTLoad(Type *DataType, Align Alignment) {
   unsigned DataSize = DL.getTypeStoreSize(DataType);
   // The only supported nontemporal loads are for aligned vectors of 16 or 32
   // bytes.  Note that 32-byte nontemporal vector loads are supported by AVX2
@@ -3286,7 +3249,7 @@ bool X86TTIImpl::isLegalNTLoad(Type *DataType, llvm::Align Alignment) {
   return false;
 }
 
-bool X86TTIImpl::isLegalNTStore(Type *DataType, llvm::Align Alignment) {
+bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) {
   unsigned DataSize = DL.getTypeStoreSize(DataType);
 
   // SSE4A supports nontemporal stores of float and double at arbitrary
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index 27d3b65c25bf4..9b948dbbb4cb9 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -187,8 +187,8 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
   bool canMacroFuseCmp();
   bool isLegalMaskedLoad(Type *DataType);
   bool isLegalMaskedStore(Type *DataType);
-  bool isLegalNTLoad(Type *DataType, llvm::Align Alignment);
-  bool isLegalNTStore(Type *DataType, llvm::Align Alignment);
+  bool isLegalNTLoad(Type *DataType, Align Alignment);
+  bool isLegalNTStore(Type *DataType, Align Alignment);
   bool isLegalMaskedGather(Type *DataType);
   bool isLegalMaskedScatter(Type *DataType);
   bool isLegalMaskedExpandLoad(Type *DataType);
diff --git a/llvm/lib/Target/X86/X86VZeroUpper.cpp b/llvm/lib/Target/X86/X86VZeroUpper.cpp
index a07d2f20acab8..9280d030b5d5f 100644
--- a/llvm/lib/Target/X86/X86VZeroUpper.cpp
+++ b/llvm/lib/Target/X86/X86VZeroUpper.cpp
@@ -292,8 +292,7 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
   // need to insert any VZEROUPPER instructions.  This is constant-time, so it
   // is cheap in the common case of no ymm/zmm use.
   bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
-  const TargetRegisterClass *RCs[2] = {&X86::VR256RegClass, &X86::VR512RegClass};
-  for (auto *RC : RCs) {
+  for (auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
     if (!YmmOrZmmUsed) {
       for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); i != e;
            i++) {
@@ -304,9 +303,8 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
       }
     }
   }
-  if (!YmmOrZmmUsed) {
+  if (!YmmOrZmmUsed)
     return false;
-  }
 
   assert(BlockStates.empty() && DirtySuccessors.empty() &&
          "X86VZeroUpper state should be clear");
diff --git a/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp b/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
index 46d1faa186681..6b3dc27cb8869 100644
--- a/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
+++ b/llvm/lib/Target/XCore/XCoreAsmPrinter.cpp
@@ -115,7 +115,7 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
 
   MCSymbol *GVSym = getSymbol(GV);
   const Constant *C = GV->getInitializer();
-  const llvm::Align Align(DL.getPrefTypeAlignment(C->getType()));
+  const Align Alignment(DL.getPrefTypeAlignment(C->getType()));
 
   // Mark the start of the global
   getTargetStreamer().emitCCTopData(GVSym->getName());
@@ -143,7 +143,7 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
     llvm_unreachable("Unknown linkage type!");
   }
 
-  EmitAlignment(std::max(Align, llvm::Align(4)), GV);
+  EmitAlignment(std::max(Alignment, Align(4)), GV);
 
   if (GV->isThreadLocal()) {
     report_fatal_error("TLS is not supported by this target!");
diff --git a/llvm/lib/Target/XCore/XCoreISelLowering.cpp b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
index fc6fd40c82e1e..bf006fd673f18 100644
--- a/llvm/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/llvm/lib/Target/XCore/XCoreISelLowering.cpp
@@ -171,8 +171,8 @@ XCoreTargetLowering::XCoreTargetLowering(const TargetMachine &TM,
   setTargetDAGCombine(ISD::INTRINSIC_VOID);
   setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
 
-  setMinFunctionAlignment(llvm::Align(2));
-  setPrefFunctionAlignment(llvm::Align(4));
+  setMinFunctionAlignment(Align(2));
+  setPrefFunctionAlignment(Align(4));
 }
 
 bool XCoreTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
@@ -414,8 +414,8 @@ SDValue XCoreTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
          "Unexpected extension type");
   assert(LD->getMemoryVT() == MVT::i32 && "Unexpected load EVT");
 
-  if (allowsMemoryAccess(Context, DAG.getDataLayout(), LD->getMemoryVT(),
-                         *LD->getMemOperand()))
+  if (allowsMemoryAccessForAlignment(Context, DAG.getDataLayout(),
+                                     LD->getMemoryVT(), *LD->getMemOperand()))
     return SDValue();
 
   SDValue Chain = LD->getChain();
@@ -488,8 +488,8 @@ SDValue XCoreTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   assert(!ST->isTruncatingStore() && "Unexpected store type");
   assert(ST->getMemoryVT() == MVT::i32 && "Unexpected store EVT");
 
-  if (allowsMemoryAccess(Context, DAG.getDataLayout(), ST->getMemoryVT(),
-                         *ST->getMemOperand()))
+  if (allowsMemoryAccessForAlignment(Context, DAG.getDataLayout(),
+                                     ST->getMemoryVT(), *ST->getMemOperand()))
     return SDValue();
 
   SDValue Chain = ST->getChain();
@@ -1780,8 +1780,9 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     // Replace unaligned store of unaligned load with memmove.
     StoreSDNode *ST = cast<StoreSDNode>(N);
     if (!DCI.isBeforeLegalize() ||
-        allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(),
-                           ST->getMemoryVT(), *ST->getMemOperand()) ||
+        allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
+                                       ST->getMemoryVT(),
+                                       *ST->getMemOperand()) ||
         ST->isVolatile() || ST->isIndexed()) {
       break;
     }
diff --git a/llvm/lib/TextAPI/CMakeLists.txt b/llvm/lib/TextAPI/CMakeLists.txt
index 91b776b2caa6f..d959fb972a802 100644
--- a/llvm/lib/TextAPI/CMakeLists.txt
+++ b/llvm/lib/TextAPI/CMakeLists.txt
@@ -5,7 +5,9 @@ add_llvm_library(LLVMTextAPI
   MachO/ArchitectureSet.cpp
   MachO/InterfaceFile.cpp
   MachO/PackedVersion.cpp
+  MachO/Platform.cpp
   MachO/Symbol.cpp
+  MachO/Target.cpp
   MachO/TextStub.cpp
   MachO/TextStubCommon.cpp
 
diff --git a/llvm/lib/TextAPI/MachO/Architecture.cpp b/llvm/lib/TextAPI/MachO/Architecture.cpp
index a66a982fa1532..699fb5f4587ad 100644
--- a/llvm/lib/TextAPI/MachO/Architecture.cpp
+++ b/llvm/lib/TextAPI/MachO/Architecture.cpp
@@ -68,6 +68,10 @@ std::pair<uint32_t, uint32_t> getCPUTypeFromArchitecture(Architecture Arch) {
   return std::make_pair(0, 0);
 }
 
+Architecture mapToArchitecture(const Triple &Target) {
+  return getArchitectureFromName(Target.getArchName());
+}
+
 raw_ostream &operator<<(raw_ostream &OS, Architecture Arch) {
   OS << getArchitectureName(Arch);
   return OS;
diff --git a/llvm/lib/TextAPI/MachO/InterfaceFile.cpp b/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
index 54ba8cc312673..c40a952a6a8bd 100644
--- a/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
+++ b/llvm/lib/TextAPI/MachO/InterfaceFile.cpp
@@ -27,36 +27,65 @@ typename C::iterator addEntry(C &Container, StringRef InstallName) {
 
   return Container.emplace(I, InstallName);
 }
+
+template <typename C>
+typename C::iterator addEntry(C &Container, const Target &Target_) {
+  auto Iter =
+      lower_bound(Container, Target_, [](const Target &LHS, const Target &RHS) {
+        return LHS < RHS;
+      });
+  if ((Iter != std::end(Container)) && !(Target_ < *Iter))
+    return Iter;
+
+  return Container.insert(Iter, Target_);
+}
 } // end namespace detail.
 
-void InterfaceFile::addAllowableClient(StringRef Name,
-                                       ArchitectureSet Architectures) {
-  auto Client = detail::addEntry(AllowableClients, Name);
-  Client->addArchitectures(Architectures);
+void InterfaceFileRef::addTarget(const Target &Target) {
+  detail::addEntry(Targets, Target);
+}
+
+void InterfaceFile::addAllowableClient(StringRef InstallName,
+                                       const Target &Target) {
+  auto Client = detail::addEntry(AllowableClients, InstallName);
+  Client->addTarget(Target);
 }
 
 void InterfaceFile::addReexportedLibrary(StringRef InstallName,
-                                         ArchitectureSet Architectures) {
+                                         const Target &Target) {
   auto Lib = detail::addEntry(ReexportedLibraries, InstallName);
-  Lib->addArchitectures(Architectures);
+  Lib->addTarget(Target);
 }
 
-void InterfaceFile::addUUID(Architecture Arch, StringRef UUID) {
-  auto I = partition_point(UUIDs,
-                           [=](const std::pair<Architecture, std::string> &O) {
-                             return O.first < Arch;
-                           });
+void InterfaceFile::addParentUmbrella(const Target &Target_, StringRef Parent) {
+  auto Iter = lower_bound(ParentUmbrellas, Target_,
+                          [](const std::pair<Target, std::string> &LHS,
+                             Target RHS) { return LHS.first < RHS; });
 
-  if (I != UUIDs.end() && Arch == I->first) {
-    I->second = UUID;
+  if ((Iter != ParentUmbrellas.end()) && !(Target_ < Iter->first)) {
+    Iter->second = Parent;
     return;
   }
 
-  UUIDs.emplace(I, Arch, UUID);
+  ParentUmbrellas.emplace(Iter, Target_, Parent);
   return;
 }
 
-void InterfaceFile::addUUID(Architecture Arch, uint8_t UUID[16]) {
+void InterfaceFile::addUUID(const Target &Target_, StringRef UUID) {
+  auto Iter = lower_bound(UUIDs, Target_,
+                          [](const std::pair<Target, std::string> &LHS,
+                             Target RHS) { return LHS.first < RHS; });
+
+  if ((Iter != UUIDs.end()) && !(Target_ < Iter->first)) {
+    Iter->second = UUID;
+    return;
+  }
+
+  UUIDs.emplace(Iter, Target_, UUID);
+  return;
+}
+
+void InterfaceFile::addUUID(const Target &Target, uint8_t UUID[16]) {
   std::stringstream Stream;
   for (unsigned i = 0; i < 16; ++i) {
     if (i == 4 || i == 6 || i == 8 || i == 10)
@@ -64,17 +93,30 @@ void InterfaceFile::addUUID(Architecture Arch, uint8_t UUID[16]) {
     Stream << std::setfill('0') << std::setw(2) << std::uppercase << std::hex
            << static_cast<int>(UUID[i]);
   }
-  addUUID(Arch, Stream.str());
+  addUUID(Target, Stream.str());
+}
+
+void InterfaceFile::addTarget(const Target &Target) {
+  detail::addEntry(Targets, Target);
+}
+
+InterfaceFile::const_filtered_target_range
+InterfaceFile::targets(ArchitectureSet Archs) const {
+  std::function<bool(const Target &)> fn = [Archs](const Target &Target_) {
+    return Archs.has(Target_.Arch);
+  };
+  return make_filter_range(Targets, fn);
 }
 
 void InterfaceFile::addSymbol(SymbolKind Kind, StringRef Name,
-                              ArchitectureSet Archs, SymbolFlags Flags) {
+                              const TargetList &Targets, SymbolFlags Flags) {
   Name = copyString(Name);
   auto result = Symbols.try_emplace(SymbolsMapKey{Kind, Name}, nullptr);
   if (result.second)
-    result.first->second = new (Allocator) Symbol{Kind, Name, Archs, Flags};
+    result.first->second = new (Allocator) Symbol{Kind, Name, Targets, Flags};
   else
-    result.first->second->addArchitectures(Archs);
+    for (const auto &Target : Targets)
+      result.first->second->addTarget(Target);
 }
 
 } // end namespace MachO.
diff --git a/llvm/lib/TextAPI/MachO/Platform.cpp b/llvm/lib/TextAPI/MachO/Platform.cpp
new file mode 100644
index 0000000000000..588ec9a4d83be
--- /dev/null
+++ b/llvm/lib/TextAPI/MachO/Platform.cpp
@@ -0,0 +1,91 @@
+//===- llvm/TextAPI/MachO/Platform.cpp - Platform ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementations of Platform Helper functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/Triple.h"
+#include "llvm/TextAPI/MachO/Platform.h"
+
+namespace llvm {
+namespace MachO {
+
+PlatformKind mapToPlatformKind(PlatformKind Platform, bool WantSim) {
+  switch (Platform) {
+  default:
+    return Platform;
+  case PlatformKind::iOS:
+    return WantSim ? PlatformKind::iOSSimulator : PlatformKind::iOS;
+  case PlatformKind::tvOS:
+    return WantSim ? PlatformKind::tvOSSimulator : PlatformKind::tvOS;
+  case PlatformKind::watchOS:
+    return WantSim ? PlatformKind::watchOSSimulator : PlatformKind::watchOS;
+  }
+  llvm_unreachable("Unknown llvm.MachO.PlatformKind enum");
+}
+
+PlatformKind mapToPlatformKind(const Triple &Target) {
+  switch (Target.getOS()) {
+  default:
+    return PlatformKind::unknown;
+  case Triple::MacOSX:
+    return PlatformKind::macOS;
+  case Triple::IOS:
+    if (Target.isSimulatorEnvironment())
+      return PlatformKind::iOSSimulator;
+    if (Target.getEnvironment() == Triple::MacABI)
+      return PlatformKind::macCatalyst;
+    return PlatformKind::iOS;
+  case Triple::TvOS:
+    return Target.isSimulatorEnvironment() ? PlatformKind::tvOSSimulator
+                                           : PlatformKind::tvOS;
+  case Triple::WatchOS:
+    return Target.isSimulatorEnvironment() ? PlatformKind::watchOSSimulator
+                                           : PlatformKind::watchOS;
+    // TODO: add bridgeOS once in llvm::Triple
+  }
+  llvm_unreachable("Unknown Target Triple");
+}
+
+PlatformSet mapToPlatformSet(ArrayRef<Triple> Targets) {
+  PlatformSet Result;
+  for (const auto &Target : Targets)
+    Result.insert(mapToPlatformKind(Target));
+  return Result;
+}
+
+StringRef getPlatformName(PlatformKind Platform) {
+  switch (Platform) {
+  case PlatformKind::unknown:
+    return "unknown";
+  case PlatformKind::macOS:
+    return "macOS";
+  case PlatformKind::iOS:
+    return "iOS";
+  case PlatformKind::tvOS:
+    return "tvOS";
+  case PlatformKind::watchOS:
+    return "watchOS";
+  case PlatformKind::bridgeOS:
+    return "bridgeOS";
+  case PlatformKind::macCatalyst:
+    return "macCatalyst";
+  case PlatformKind::iOSSimulator:
+    return "iOS Simulator";
+  case PlatformKind::tvOSSimulator:
+    return "tvOS Simulator";
+  case PlatformKind::watchOSSimulator:
+    return "watchOS Simulator";
+  }
+  llvm_unreachable("Unknown llvm.MachO.PlatformKind enum");
+}
+
+} // end namespace MachO.
+} // end namespace llvm.
diff --git a/llvm/lib/TextAPI/MachO/Symbol.cpp b/llvm/lib/TextAPI/MachO/Symbol.cpp
index 731b264f6082d..9f2d8172beed4 100644
--- a/llvm/lib/TextAPI/MachO/Symbol.cpp
+++ b/llvm/lib/TextAPI/MachO/Symbol.cpp
@@ -45,5 +45,14 @@ LLVM_DUMP_METHOD void Symbol::dump(raw_ostream &OS) const {
 }
 #endif
 
+Symbol::const_filtered_target_range
+Symbol::targets(ArchitectureSet Architectures) const {
+  std::function<bool(const Target &)> FN =
+      [Architectures](const Target &Target) {
+        return Architectures.has(Target.Arch);
+      };
+  return make_filter_range(Targets, FN);
+}
+
 } // end namespace MachO.
 } // end namespace llvm.
diff --git a/llvm/lib/TextAPI/MachO/Target.cpp b/llvm/lib/TextAPI/MachO/Target.cpp
new file mode 100644
index 0000000000000..3052aa53ac233
--- /dev/null
+++ b/llvm/lib/TextAPI/MachO/Target.cpp
@@ -0,0 +1,45 @@
+//===- tapi/Core/Target.cpp - Target ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TextAPI/MachO/Target.h"
+
+namespace llvm {
+namespace MachO {
+
+Target::operator std::string() const {
+  return (getArchitectureName(Arch) + " (" + getPlatformName(Platform) + ")")
+      .str();
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const Target &Target) {
+  OS << std::string(Target);
+  return OS;
+}
+
+PlatformSet mapToPlatformSet(ArrayRef<Target> Targets) {
+  PlatformSet Result;
+  for (const auto &Target : Targets)
+    Result.insert(Target.Platform);
+  return Result;
+}
+
+ArchitectureSet mapToArchitectureSet(ArrayRef<Target> Targets) {
+  ArchitectureSet Result;
+  for (const auto &Target : Targets)
+    Result.set(Target.Arch);
+  return Result;
+}
+
+} // end namespace MachO.
+} // end namespace llvm.
\ No newline at end of file
diff --git a/llvm/lib/TextAPI/MachO/TextStub.cpp b/llvm/lib/TextAPI/MachO/TextStub.cpp
index 371866736c024..78c9f54ba22dd 100644
--- a/llvm/lib/TextAPI/MachO/TextStub.cpp
+++ b/llvm/lib/TextAPI/MachO/TextStub.cpp
@@ -246,7 +246,7 @@ template <> struct MappingTraits<const InterfaceFile *> {
     NormalizedTBD(IO &IO, const InterfaceFile *&File) {
       Architectures = File->getArchitectures();
       UUIDs = File->uuids();
-      Platform = File->getPlatform();
+      Platforms = File->getPlatforms();
       InstallName = File->getInstallName();
       CurrentVersion = PackedVersion(File->getCurrentVersion());
       CompatibilityVersion = PackedVersion(File->getCompatibilityVersion());
@@ -263,7 +263,10 @@ template <> struct MappingTraits<const InterfaceFile *> {
       if (File->isInstallAPI())
         Flags |= TBDFlags::InstallAPI;
 
-      ParentUmbrella = File->getParentUmbrella();
+      for (const auto &Iter : File->umbrellas()) {
+        ParentUmbrella = Iter.second;
+        break;
+      }
 
       std::set<ArchitectureSet> ArchSet;
       for (const auto &Library : File->allowableClients())
@@ -396,6 +399,29 @@ template <> struct MappingTraits<const InterfaceFile *> {
       }
     }
 
+    // TBD v1 - TBD v3 files only support one platform and several
+    // architectures. It is possible to have more than one platform for TBD v3
+    // files, but the architectures don't apply to all
+    // platforms, specifically to filter out the i386 slice from
+    // platform macCatalyst.
+    TargetList synthesizeTargets(ArchitectureSet Architectures,
+                                          const PlatformSet &Platforms) {
+      TargetList Targets;
+
+      for (auto Platform : Platforms) {
+        Platform = mapToPlatformKind(Platform, Architectures.hasX86());
+
+        for (const auto &&Architecture : Architectures) {
+          if ((Architecture == AK_i386) &&
+              (Platform == PlatformKind::macCatalyst))
+            continue;
+
+          Targets.emplace_back(Architecture, Platform);
+        }
+      }
+      return Targets;
+    }
+
     const InterfaceFile *denormalize(IO &IO) {
       auto Ctx = reinterpret_cast<TextAPIContext *>(IO.getContext());
       assert(Ctx);
@@ -403,16 +429,16 @@ template <> struct MappingTraits<const InterfaceFile *> {
       auto *File = new InterfaceFile;
       File->setPath(Ctx->Path);
       File->setFileType(Ctx->FileKind);
+      File->addTargets(synthesizeTargets(Architectures, Platforms));
       for (auto &ID : UUIDs)
         File->addUUID(ID.first, ID.second);
-      File->setPlatform(Platform);
-      File->setArchitectures(Architectures);
       File->setInstallName(InstallName);
       File->setCurrentVersion(CurrentVersion);
       File->setCompatibilityVersion(CompatibilityVersion);
       File->setSwiftABIVersion(SwiftABIVersion);
       File->setObjCConstraint(ObjCConstraint);
-      File->setParentUmbrella(ParentUmbrella);
+      for (const auto &Target : File->targets())
+        File->addParentUmbrella(Target, ParentUmbrella);
 
       if (Ctx->FileKind == FileType::TBD_V1) {
         File->setTwoLevelNamespace();
@@ -425,76 +451,80 @@ template <> struct MappingTraits<const InterfaceFile *> {
       }
 
       for (const auto &Section : Exports) {
-        for (const auto &Library : Section.AllowableClients)
-          File->addAllowableClient(Library, Section.Architectures);
-        for (const auto &Library : Section.ReexportedLibraries)
-          File->addReexportedLibrary(Library, Section.Architectures);
+        const auto Targets =
+            synthesizeTargets(Section.Architectures, Platforms);
+
+        for (const auto &Lib : Section.AllowableClients)
+          for (const auto &Target : Targets)
+            File->addAllowableClient(Lib, Target);
+
+        for (const auto &Lib : Section.ReexportedLibraries)
+          for (const auto &Target : Targets)
+            File->addReexportedLibrary(Lib, Target);
 
         for (const auto &Symbol : Section.Symbols) {
           if (Ctx->FileKind != FileType::TBD_V3 &&
               Symbol.value.startswith("_OBJC_EHTYPE_$_"))
             File->addSymbol(SymbolKind::ObjectiveCClassEHType,
-                            Symbol.value.drop_front(15), Section.Architectures);
+                            Symbol.value.drop_front(15), Targets);
           else
-            File->addSymbol(SymbolKind::GlobalSymbol, Symbol,
-                            Section.Architectures);
+            File->addSymbol(SymbolKind::GlobalSymbol, Symbol, Targets);
         }
         for (auto &Symbol : Section.Classes) {
           auto Name = Symbol.value;
           if (Ctx->FileKind != FileType::TBD_V3)
             Name = Name.drop_front();
-          File->addSymbol(SymbolKind::ObjectiveCClass, Name,
-                          Section.Architectures);
+          File->addSymbol(SymbolKind::ObjectiveCClass, Name, Targets);
         }
         for (auto &Symbol : Section.ClassEHs)
-          File->addSymbol(SymbolKind::ObjectiveCClassEHType, Symbol,
-                          Section.Architectures);
+          File->addSymbol(SymbolKind::ObjectiveCClassEHType, Symbol, Targets);
         for (auto &Symbol : Section.IVars) {
           auto Name = Symbol.value;
           if (Ctx->FileKind != FileType::TBD_V3)
             Name = Name.drop_front();
           File->addSymbol(SymbolKind::ObjectiveCInstanceVariable, Name,
-                          Section.Architectures);
+                          Targets);
         }
         for (auto &Symbol : Section.WeakDefSymbols)
-          File->addSymbol(SymbolKind::GlobalSymbol, Symbol,
-                          Section.Architectures, SymbolFlags::WeakDefined);
+          File->addSymbol(SymbolKind::GlobalSymbol, Symbol, Targets,
+                          SymbolFlags::WeakDefined);
         for (auto &Symbol : Section.TLVSymbols)
-          File->addSymbol(SymbolKind::GlobalSymbol, Symbol,
-                          Section.Architectures, SymbolFlags::ThreadLocalValue);
+          File->addSymbol(SymbolKind::GlobalSymbol, Symbol, Targets,
+                          SymbolFlags::ThreadLocalValue);
       }
 
       for (const auto &Section : Undefineds) {
+        const auto Targets =
+            synthesizeTargets(Section.Architectures, Platforms);
         for (auto &Symbol : Section.Symbols) {
           if (Ctx->FileKind != FileType::TBD_V3 &&
               Symbol.value.startswith("_OBJC_EHTYPE_$_"))
             File->addSymbol(SymbolKind::ObjectiveCClassEHType,
-                            Symbol.value.drop_front(15), Section.Architectures,
+                            Symbol.value.drop_front(15), Targets,
                             SymbolFlags::Undefined);
           else
-            File->addSymbol(SymbolKind::GlobalSymbol, Symbol,
-                            Section.Architectures, SymbolFlags::Undefined);
+            File->addSymbol(SymbolKind::GlobalSymbol, Symbol, Targets,
+                            SymbolFlags::Undefined);
         }
         for (auto &Symbol : Section.Classes) {
           auto Name = Symbol.value;
           if (Ctx->FileKind != FileType::TBD_V3)
             Name = Name.drop_front();
-          File->addSymbol(SymbolKind::ObjectiveCClass, Name,
-                          Section.Architectures, SymbolFlags::Undefined);
+          File->addSymbol(SymbolKind::ObjectiveCClass, Name, Targets,
+                          SymbolFlags::Undefined);
         }
         for (auto &Symbol : Section.ClassEHs)
-          File->addSymbol(SymbolKind::ObjectiveCClassEHType, Symbol,
-                          Section.Architectures, SymbolFlags::Undefined);
+          File->addSymbol(SymbolKind::ObjectiveCClassEHType, Symbol, Targets,
+                          SymbolFlags::Undefined);
         for (auto &Symbol : Section.IVars) {
           auto Name = Symbol.value;
           if (Ctx->FileKind != FileType::TBD_V3)
             Name = Name.drop_front();
-          File->addSymbol(SymbolKind::ObjectiveCInstanceVariable, Name,
-                          Section.Architectures, SymbolFlags::Undefined);
+          File->addSymbol(SymbolKind::ObjectiveCInstanceVariable, Name, Targets,
+                          SymbolFlags::Undefined);
         }
         for (auto &Symbol : Section.WeakRefSymbols)
-          File->addSymbol(SymbolKind::GlobalSymbol, Symbol,
-                          Section.Architectures,
+          File->addSymbol(SymbolKind::GlobalSymbol, Symbol, Targets,
                           SymbolFlags::Undefined | SymbolFlags::WeakReferenced);
       }
 
@@ -513,7 +543,7 @@ template <> struct MappingTraits<const InterfaceFile *> {
 
     std::vector<Architecture> Architectures;
     std::vector<UUID> UUIDs;
-    PlatformKind Platform{PlatformKind::unknown};
+    PlatformSet Platforms;
     StringRef InstallName;
     PackedVersion CurrentVersion;
     PackedVersion CompatibilityVersion;
@@ -567,7 +597,7 @@ template <> struct MappingTraits<const InterfaceFile *> {
     IO.mapRequired("archs", Keys->Architectures);
     if (Ctx->FileKind != FileType::TBD_V1)
       IO.mapOptional("uuids", Keys->UUIDs);
-    IO.mapRequired("platform", Keys->Platform);
+    IO.mapRequired("platform", Keys->Platforms);
     if (Ctx->FileKind != FileType::TBD_V1)
       IO.mapOptional("flags", Keys->Flags, TBDFlags::None);
     IO.mapRequired("install-name", Keys->InstallName);
diff --git a/llvm/lib/TextAPI/MachO/TextStubCommon.cpp b/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
index 00382cd24573a..cfd9ac8d0cf6f 100644
--- a/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
+++ b/llvm/lib/TextAPI/MachO/TextStubCommon.cpp
@@ -41,9 +41,21 @@ void ScalarEnumerationTraits<ObjCConstraintType>::enumeration(
   IO.enumCase(Constraint, "gc", ObjCConstraintType::GC);
 }
 
-void ScalarTraits<PlatformKind>::output(const PlatformKind &Value, void *,
-                                        raw_ostream &OS) {
-  switch (Value) {
+void ScalarTraits<PlatformSet>::output(const PlatformSet &Values, void *IO,
+                                       raw_ostream &OS) {
+
+  const auto *Ctx = reinterpret_cast<TextAPIContext *>(IO);
+  assert((!Ctx || Ctx->FileKind != FileType::Invalid) &&
+         "File type is not set in context");
+
+  if (Ctx && Ctx->FileKind == TBD_V3 && Values.count(PlatformKind::macOS) &&
+      Values.count(PlatformKind::macCatalyst)) {
+    OS << "zippered";
+    return;
+  }
+
+  assert(Values.size() == 1U);
+  switch (*Values.begin()) {
   default:
     llvm_unreachable("unexpected platform");
     break;
@@ -64,21 +76,44 @@ void ScalarTraits<PlatformKind>::output(const PlatformKind &Value, void *,
     break;
   }
 }
-StringRef ScalarTraits<PlatformKind>::input(StringRef Scalar, void *,
-                                            PlatformKind &Value) {
-  Value = StringSwitch<PlatformKind>(Scalar)
-              .Case("macosx", PlatformKind::macOS)
-              .Case("ios", PlatformKind::iOS)
-              .Case("watchos", PlatformKind::watchOS)
-              .Case("tvos", PlatformKind::tvOS)
-              .Case("bridgeos", PlatformKind::bridgeOS)
-              .Default(PlatformKind::unknown);
-
-  if (Value == PlatformKind::unknown)
+
+StringRef ScalarTraits<PlatformSet>::input(StringRef Scalar, void *IO,
+                                           PlatformSet &Values) {
+  const auto *Ctx = reinterpret_cast<TextAPIContext *>(IO);
+  assert((!Ctx || Ctx->FileKind != FileType::Invalid) &&
+         "File type is not set in context");
+
+  if (Scalar == "zippered") {
+    if (Ctx && Ctx->FileKind == FileType::TBD_V3) {
+      Values.insert(PlatformKind::macOS);
+      Values.insert(PlatformKind::macCatalyst);
+      return {};
+    }
+    return "invalid platform";
+  }
+
+  auto Platform = StringSwitch<PlatformKind>(Scalar)
+                      .Case("unknown", PlatformKind::unknown)
+                      .Case("macosx", PlatformKind::macOS)
+                      .Case("ios", PlatformKind::iOS)
+                      .Case("watchos", PlatformKind::watchOS)
+                      .Case("tvos", PlatformKind::tvOS)
+                      .Case("bridgeos", PlatformKind::bridgeOS)
+                      .Case("iosmac", PlatformKind::macCatalyst)
+                      .Default(PlatformKind::unknown);
+
+  if (Platform == PlatformKind::macCatalyst)
+    if (Ctx && Ctx->FileKind != FileType::TBD_V3)
+      return "invalid platform";
+
+  if (Platform == PlatformKind::unknown)
     return "unknown platform";
+
+  Values.insert(Platform);
   return {};
 }
-QuotingType ScalarTraits<PlatformKind>::mustQuote(StringRef) {
+
+QuotingType ScalarTraits<PlatformSet>::mustQuote(StringRef) {
   return QuotingType::None;
 }
 
@@ -166,10 +201,11 @@ StringRef ScalarTraits<UUID>::input(StringRef Scalar, void *, UUID &Value) {
   auto UUID = Split.second.trim();
   if (UUID.empty())
     return "invalid uuid string pair";
-  Value.first = getArchitectureFromName(Arch);
   Value.second = UUID;
+  Value.first = Target{getArchitectureFromName(Arch), PlatformKind::unknown};
   return {};
 }
+
 QuotingType ScalarTraits<UUID>::mustQuote(StringRef) {
   return QuotingType::Single;
 }
diff --git a/llvm/lib/TextAPI/MachO/TextStubCommon.h b/llvm/lib/TextAPI/MachO/TextStubCommon.h
index c4dd1075b1c83..a558cbcec9fb6 100644
--- a/llvm/lib/TextAPI/MachO/TextStubCommon.h
+++ b/llvm/lib/TextAPI/MachO/TextStubCommon.h
@@ -21,7 +21,7 @@
 #include "llvm/TextAPI/MachO/InterfaceFile.h"
 #include "llvm/TextAPI/MachO/PackedVersion.h"
 
-using UUID = std::pair<llvm::MachO::Architecture, std::string>;
+using UUID = std::pair<llvm::MachO::Target, std::string>;
 
 LLVM_YAML_STRONG_TYPEDEF(llvm::StringRef, FlowStringRef)
 LLVM_YAML_STRONG_TYPEDEF(uint8_t, SwiftVersion)
@@ -41,9 +41,9 @@ template <> struct ScalarEnumerationTraits<MachO::ObjCConstraintType> {
   static void enumeration(IO &, MachO::ObjCConstraintType &);
 };
 
-template <> struct ScalarTraits<MachO::PlatformKind> {
-  static void output(const MachO::PlatformKind &, void *, raw_ostream &);
-  static StringRef input(StringRef, void *, MachO::PlatformKind &);
+template <> struct ScalarTraits<MachO::PlatformSet> {
+  static void output(const MachO::PlatformSet &, void *, raw_ostream &);
+  static StringRef input(StringRef, void *, MachO::PlatformSet &);
   static QuotingType mustQuote(StringRef);
 };
 
diff --git a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
index 18ab6637305ef..286191abff202 100644
--- a/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
+++ b/llvm/lib/ToolDrivers/llvm-lib/LibDriver.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/ToolDrivers/llvm-lib/LibDriver.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/BinaryFormat/COFF.h"
 #include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Bitcode/BitcodeReader.h"
@@ -141,6 +142,125 @@ static void doList(opt::InputArgList& Args) {
   fatalOpenError(std::move(Err), B->getBufferIdentifier());
 }
 
+static COFF::MachineTypes getCOFFFileMachine(MemoryBufferRef MB) {
+  std::error_code EC;
+  object::COFFObjectFile Obj(MB, EC);
+  if (EC) {
+    llvm::errs() << MB.getBufferIdentifier()
+                 << ": failed to open: " << EC.message() << '\n';
+    exit(1);
+  }
+
+  uint16_t Machine = Obj.getMachine();
+  if (Machine != COFF::IMAGE_FILE_MACHINE_I386 &&
+      Machine != COFF::IMAGE_FILE_MACHINE_AMD64 &&
+      Machine != COFF::IMAGE_FILE_MACHINE_ARMNT &&
+      Machine != COFF::IMAGE_FILE_MACHINE_ARM64) {
+    llvm::errs() << MB.getBufferIdentifier() << ": unknown machine: " << Machine
+                 << '\n';
+    exit(1);
+  }
+
+  return static_cast<COFF::MachineTypes>(Machine);
+}
+
+static COFF::MachineTypes getBitcodeFileMachine(MemoryBufferRef MB) {
+  Expected<std::string> TripleStr = getBitcodeTargetTriple(MB);
+  if (!TripleStr) {
+    llvm::errs() << MB.getBufferIdentifier()
+                 << ": failed to get target triple from bitcode\n";
+    exit(1);
+  }
+
+  switch (Triple(*TripleStr).getArch()) {
+  case Triple::x86:
+    return COFF::IMAGE_FILE_MACHINE_I386;
+  case Triple::x86_64:
+    return COFF::IMAGE_FILE_MACHINE_AMD64;
+  case Triple::arm:
+    return COFF::IMAGE_FILE_MACHINE_ARMNT;
+  case Triple::aarch64:
+    return COFF::IMAGE_FILE_MACHINE_ARM64;
+  default:
+    llvm::errs() << MB.getBufferIdentifier()
+                 << ": unknown arch in target triple " << *TripleStr << '\n';
+    exit(1);
+  }
+}
+
+static void appendFile(std::vector<NewArchiveMember> &Members,
+                       COFF::MachineTypes &LibMachine,
+                       std::string &LibMachineSource, MemoryBufferRef MB) {
+  file_magic Magic = identify_magic(MB.getBuffer());
+
+  if (Magic != file_magic::coff_object && Magic != file_magic::bitcode &&
+      Magic != file_magic::archive && Magic != file_magic::windows_resource) {
+    llvm::errs() << MB.getBufferIdentifier()
+                 << ": not a COFF object, bitcode, archive or resource file\n";
+    exit(1);
+  }
+
+  // If a user attempts to add an archive to another archive, llvm-lib doesn't
+  // handle the first archive file as a single file. Instead, it extracts all
+  // members from the archive and add them to the second archive. This beahvior
+  // is for compatibility with Microsoft's lib command.
+  if (Magic == file_magic::archive) {
+    Error Err = Error::success();
+    object::Archive Archive(MB, Err);
+    fatalOpenError(std::move(Err), MB.getBufferIdentifier());
+
+    for (auto &C : Archive.children(Err)) {
+      Expected<MemoryBufferRef> ChildMB = C.getMemoryBufferRef();
+      if (!ChildMB) {
+        handleAllErrors(ChildMB.takeError(), [&](const ErrorInfoBase &EIB) {
+          llvm::errs() << MB.getBufferIdentifier() << ": " << EIB.message()
+                       << "\n";
+        });
+        exit(1);
+      }
+
+      appendFile(Members, LibMachine, LibMachineSource, *ChildMB);
+    }
+
+    fatalOpenError(std::move(Err), MB.getBufferIdentifier());
+    return;
+  }
+
+  // Check that all input files have the same machine type.
+  // Mixing normal objects and LTO bitcode files is fine as long as they
+  // have the same machine type.
+  // Doing this here duplicates the header parsing work that writeArchive()
+  // below does, but it's not a lot of work and it's a bit awkward to do
+  // in writeArchive() which needs to support many tools, can't assume the
+  // input is COFF, and doesn't have a good way to report errors.
+  if (Magic == file_magic::coff_object || Magic == file_magic::bitcode) {
+    COFF::MachineTypes FileMachine = (Magic == file_magic::coff_object)
+                                         ? getCOFFFileMachine(MB)
+                                         : getBitcodeFileMachine(MB);
+
+    // FIXME: Once lld-link rejects multiple resource .obj files:
+    // Call convertResToCOFF() on .res files and add the resulting
+    // COFF file to the .lib output instead of adding the .res file, and remove
+    // this check. See PR42180.
+    if (FileMachine != COFF::IMAGE_FILE_MACHINE_UNKNOWN) {
+      if (LibMachine == COFF::IMAGE_FILE_MACHINE_UNKNOWN) {
+        LibMachine = FileMachine;
+        LibMachineSource =
+            (" (inferred from earlier file '" + MB.getBufferIdentifier() + "')")
+                .str();
+      } else if (LibMachine != FileMachine) {
+        llvm::errs() << MB.getBufferIdentifier() << ": file machine type "
+                     << machineToStr(FileMachine)
+                     << " conflicts with library machine type "
+                     << machineToStr(LibMachine) << LibMachineSource << '\n';
+        exit(1);
+      }
+    }
+  }
+
+  Members.emplace_back(MB);
+}
+
 int llvm::libDriverMain(ArrayRef<const char *> ArgsArr) {
   BumpPtrAllocator Alloc;
   StringSaver Saver(Alloc);
@@ -195,104 +315,40 @@ int llvm::libDriverMain(ArrayRef<const char *> ArgsArr) {
         std::string(" (from '/machine:") + Arg->getValue() + "' flag)";
   }
 
-  // Create a NewArchiveMember for each input file.
+  std::vector<std::unique_ptr<MemoryBuffer>> MBs;
+  StringSet<> Seen;
   std::vector<NewArchiveMember> Members;
+
+  // Create a NewArchiveMember for each input file.
   for (auto *Arg : Args.filtered(OPT_INPUT)) {
+    // Find a file
     std::string Path = findInputFile(Arg->getValue(), SearchPaths);
     if (Path.empty()) {
       llvm::errs() << Arg->getValue() << ": no such file or directory\n";
       return 1;
     }
 
-    Expected<NewArchiveMember> MOrErr =
-        NewArchiveMember::getFile(Saver.save(Path), /*Deterministic=*/true);
-    if (!MOrErr) {
-      handleAllErrors(MOrErr.takeError(), [&](const ErrorInfoBase &EIB) {
-        llvm::errs() << Arg->getValue() << ": " << EIB.message() << "\n";
-      });
-      return 1;
-    }
-
-    file_magic Magic = identify_magic(MOrErr->Buf->getBuffer());
-    if (Magic != file_magic::coff_object && Magic != file_magic::bitcode &&
-        Magic != file_magic::windows_resource) {
-      llvm::errs() << Arg->getValue()
-                   << ": not a COFF object, bitcode or resource file\n";
-      return 1;
-    }
-
-    // Check that all input files have the same machine type.
-    // Mixing normal objects and LTO bitcode files is fine as long as they
-    // have the same machine type.
-    // Doing this here duplicates the header parsing work that writeArchive()
-    // below does, but it's not a lot of work and it's a bit awkward to do
-    // in writeArchive() which needs to support many tools, can't assume the
-    // input is COFF, and doesn't have a good way to report errors.
-    COFF::MachineTypes FileMachine = COFF::IMAGE_FILE_MACHINE_UNKNOWN;
-    if (Magic == file_magic::coff_object) {
-      std::error_code EC;
-      object::COFFObjectFile Obj(*MOrErr->Buf, EC);
-      if (EC) {
-        llvm::errs() << Arg->getValue() << ": failed to open: " << EC.message()
-                     << '\n';
-        return 1;
-      }
-      uint16_t Machine = Obj.getMachine();
-      if (Machine != COFF::IMAGE_FILE_MACHINE_I386 &&
-          Machine != COFF::IMAGE_FILE_MACHINE_AMD64 &&
-          Machine != COFF::IMAGE_FILE_MACHINE_ARMNT &&
-          Machine != COFF::IMAGE_FILE_MACHINE_ARM64) {
-        llvm::errs() << Arg->getValue() << ": unknown machine: " << Machine
-                     << '\n';
-        return 1;
-      }
-      FileMachine = static_cast<COFF::MachineTypes>(Machine);
-    } else if (Magic == file_magic::bitcode) {
-      Expected<std::string> TripleStr = getBitcodeTargetTriple(*MOrErr->Buf);
-      if (!TripleStr) {
-        llvm::errs() << Arg->getValue()
-                     << ": failed to get target triple from bitcode\n";
-        return 1;
-      }
-      switch (Triple(*TripleStr).getArch()) {
-      case Triple::x86:
-        FileMachine = COFF::IMAGE_FILE_MACHINE_I386;
-        break;
-      case Triple::x86_64:
-        FileMachine = COFF::IMAGE_FILE_MACHINE_AMD64;
-        break;
-      case Triple::arm:
-        FileMachine = COFF::IMAGE_FILE_MACHINE_ARMNT;
-        break;
-      case Triple::aarch64:
-        FileMachine = COFF::IMAGE_FILE_MACHINE_ARM64;
-        break;
-      default:
-        llvm::errs() << Arg->getValue() << ": unknown arch in target triple "
-                     << *TripleStr << '\n';
-        return 1;
-      }
-    }
-
-    // FIXME: Once lld-link rejects multiple resource .obj files:
-    // Call convertResToCOFF() on .res files and add the resulting
-    // COFF file to the .lib output instead of adding the .res file, and remove
-    // this check. See PR42180.
-    if (FileMachine != COFF::IMAGE_FILE_MACHINE_UNKNOWN) {
-      if (LibMachine == COFF::IMAGE_FILE_MACHINE_UNKNOWN) {
-        LibMachine = FileMachine;
-        LibMachineSource = std::string(" (inferred from earlier file '") +
-                           Arg->getValue() + "')";
-      } else if (LibMachine != FileMachine) {
-        llvm::errs() << Arg->getValue() << ": file machine type "
-                     << machineToStr(FileMachine)
-                     << " conflicts with library machine type "
-                     << machineToStr(LibMachine) << LibMachineSource << '\n';
-        return 1;
-      }
-    }
-
-    Members.emplace_back(std::move(*MOrErr));
+    // Input files are uniquified by pathname. If you specify the exact same
+    // path more than once, all but the first one are ignored.
+    //
+    // Note that there's a loophole in the rule; you can prepend `.\` or
+    // something like that to a path to make it look different, and they are
+    // handled as if they were different files. This behavior is compatible with
+    // Microsoft lib.exe.
+    if (!Seen.insert(Path).second)
+      continue;
+
+    // Open a file.
+    ErrorOr<std::unique_ptr<MemoryBuffer>> MOrErr =
+        MemoryBuffer::getFile(Path, -1, false);
+    fatalOpenError(errorCodeToError(MOrErr.getError()), Path);
+    MemoryBufferRef MBRef = (*MOrErr)->getMemBufferRef();
+
+    // Append a file.
+    appendFile(Members, LibMachine, LibMachineSource, MBRef);
+
+    // Take the ownership of the file buffer to keep the file open.
+    MBs.push_back(std::move(*MOrErr));
   }
 
   // Create an archive file.
diff --git a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
index 5e7587b226428..6e2867f5708ce 100644
--- a/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
+++ b/llvm/lib/Transforms/AggressiveInstCombine/AggressiveInstCombine.cpp
@@ -121,14 +121,13 @@ static bool foldGuardedRotateToFunnelShift(Instruction &I) {
   BasicBlock *GuardBB = Phi.getIncomingBlock(RotSrc == P1);
   BasicBlock *RotBB = Phi.getIncomingBlock(RotSrc != P1);
   Instruction *TermI = GuardBB->getTerminator();
-  BasicBlock *TrueBB, *FalseBB;
   ICmpInst::Predicate Pred;
-  if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(RotAmt), m_ZeroInt()), TrueBB,
-                         FalseBB)))
+  BasicBlock *PhiBB = Phi.getParent();
+  if (!match(TermI, m_Br(m_ICmp(Pred, m_Specific(RotAmt), m_ZeroInt()),
+                         m_SpecificBB(PhiBB), m_SpecificBB(RotBB))))
     return false;
 
-  BasicBlock *PhiBB = Phi.getParent();
-  if (Pred != CmpInst::ICMP_EQ || TrueBB != PhiBB || FalseBB != RotBB)
+  if (Pred != CmpInst::ICMP_EQ)
     return false;
 
   // We matched a variation of this IR pattern:
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 418814107e524..2c42cf8a6d259 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -1106,7 +1106,7 @@ static void lowerLocalAllocas(ArrayRef<CoroAllocaAllocInst*> LocalAllocas,
 
     // Allocate memory.
     auto Alloca = Builder.CreateAlloca(Builder.getInt8Ty(), AI->getSize());
-    Alloca->setAlignment(AI->getAlignment());
+    Alloca->setAlignment(MaybeAlign(AI->getAlignment()));
 
     for (auto U : AI->users()) {
       // Replace gets with the allocation.
diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
index 95a9f31cced3d..dd9f74a881ee3 100644
--- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -304,7 +304,7 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote,
           // of the previous load.
           LoadInst *newLoad =
               IRB.CreateLoad(OrigLoad->getType(), V, V->getName() + ".val");
-          newLoad->setAlignment(OrigLoad->getAlignment());
+          newLoad->setAlignment(MaybeAlign(OrigLoad->getAlignment()));
           // Transfer the AA info too.
           AAMDNodes AAInfo;
           OrigLoad->getAAMetadata(AAInfo);
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index 94ed7003c965c..1455a906103aa 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Analysis/EHPersonalities.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Argument.h"
 #include "llvm/IR/Attributes.h"
@@ -135,6 +136,12 @@ static cl::opt<unsigned> DepRecInterval(
     cl::desc("Number of iterations until dependences are recomputed."),
     cl::init(4));
 
+static cl::opt<bool> EnableHeapToStack("enable-heap-to-stack-conversion",
+                                       cl::init(true), cl::Hidden);
+
+static cl::opt<int> MaxHeapToStackSize("max-heap-to-stack-size",
+                                       cl::init(128), cl::Hidden);
+
 /// Logic operators for the change status enum class.
 ///
 ///{
@@ -154,7 +161,7 @@ ChangeStatus llvm::operator&(ChangeStatus l, ChangeStatus r) {
 /// limit how much effort is invested, we will never visit more values than
 /// specified by \p MaxValues.
 template <typename AAType, typename StateTy>
-bool genericValueTraversal(
+static bool genericValueTraversal(
     Attributor &A, IRPosition IRP, const AAType &QueryingAA, StateTy &State,
     const function_ref<bool(Value &, StateTy &, bool)> &VisitValueCB,
     int MaxValues = 8) {
@@ -479,6 +486,7 @@ void IRPosition::verify() {
   }
 }
 
+namespace {
 /// Helper functions to clamp a state \p S of type \p StateType with the
 /// information in \p R and indicate/return if \p S did change (as-in update is
 /// required to be run again).
@@ -874,12 +882,17 @@ ChangeStatus AAReturnedValuesImpl::manifest(Attributor &A) {
     if (Function *F = dyn_cast<Function>(&AnchorValue)) {
       for (const Use &U : F->uses())
         if (CallBase *CB = dyn_cast<CallBase>(U.getUser()))
-          if (CB->isCallee(&U))
-            Changed = ReplaceCallSiteUsersWith(*CB, *RVC) | Changed;
+          if (CB->isCallee(&U)) {
+            Constant *RVCCast =
+                ConstantExpr::getTruncOrBitCast(RVC, CB->getType());
+            Changed = ReplaceCallSiteUsersWith(*CB, *RVCCast) | Changed;
+          }
     } else {
       assert(isa<CallBase>(AnchorValue) &&
              "Expcected a function or call base anchor!");
-      Changed = ReplaceCallSiteUsersWith(cast<CallBase>(AnchorValue), *RVC);
+      Constant *RVCCast =
+          ConstantExpr::getTruncOrBitCast(RVC, AnchorValue.getType());
+      Changed = ReplaceCallSiteUsersWith(cast<CallBase>(AnchorValue), *RVCCast);
     }
     if (Changed == ChangeStatus::CHANGED)
       STATS_DECLTRACK(UniqueConstantReturnValue, FunctionReturn,
@@ -1517,10 +1530,38 @@ struct AANoRecurseImpl : public AANoRecurse {
 struct AANoRecurseFunction final : AANoRecurseImpl {
   AANoRecurseFunction(const IRPosition &IRP) : AANoRecurseImpl(IRP) {}
 
+  /// See AbstractAttribute::initialize(...).
+  void initialize(Attributor &A) override {
+    AANoRecurseImpl::initialize(A);
+    if (const Function *F = getAnchorScope())
+      if (A.getInfoCache().getSccSize(*F) == 1)
+        return;
+    indicatePessimisticFixpoint();
+  }
+
   /// See AbstractAttribute::updateImpl(...).
   ChangeStatus updateImpl(Attributor &A) override {
-    // TODO: Implement this.
-    return indicatePessimisticFixpoint();
+
+    auto CheckForNoRecurse = [&](Instruction &I) {
+      ImmutableCallSite ICS(&I);
+      if (ICS.hasFnAttr(Attribute::NoRecurse))
+        return true;
+
+      const auto &NoRecurseAA =
+          A.getAAFor<AANoRecurse>(*this, IRPosition::callsite_function(ICS));
+      if (!NoRecurseAA.isAssumedNoRecurse())
+        return false;
+
+      // Recursion to the same function
+      if (ICS.getCalledFunction() == getAnchorScope())
+        return false;
+
+      return true;
+    };
+
+    if (!A.checkForAllCallLikeInstructions(CheckForNoRecurse, *this))
+      return indicatePessimisticFixpoint();
+    return ChangeStatus::UNCHANGED;
   }
 
   void trackStatistics() const override { STATS_DECLTRACK_FN_ATTR(norecurse) }
@@ -1750,11 +1791,18 @@ struct AANoAliasCallSiteArgument final : AANoAliasImpl {
       if (!ArgOp->getType()->isPointerTy())
         continue;
 
-      // TODO: Use AliasAnalysis
-      //       AAResults& AAR = ..;
-      //       if(AAR.isNoAlias(&getAssociatedValue(), ArgOp))
-      //          return indicatePessimitisicFixpoint();
+      if (const Function *F = getAnchorScope()) {
+        if (AAResults *AAR = A.getInfoCache().getAAResultsForFunction(*F)) {
+          LLVM_DEBUG(dbgs()
+                     << "[Attributor][NoAliasCSArg] Check alias between "
+                        "callsite arguments "
+                     << AAR->isNoAlias(&getAssociatedValue(), ArgOp) << " "
+                     << getAssociatedValue() << " " << *ArgOp << "\n");
 
+          if (AAR->isNoAlias(&getAssociatedValue(), ArgOp))
+            continue;
+        }
+      }
       return indicatePessimisticFixpoint();
     }
 
@@ -1844,12 +1892,14 @@ struct AAIsDeadImpl : public AAIsDead {
 
   void exploreFromEntry(Attributor &A, const Function *F) {
     ToBeExploredPaths.insert(&(F->getEntryBlock().front()));
-    assumeLive(A, F->getEntryBlock());
 
     for (size_t i = 0; i < ToBeExploredPaths.size(); ++i)
       if (const Instruction *NextNoReturnI =
               findNextNoReturn(A, ToBeExploredPaths[i]))
         NoReturnCalls.insert(NextNoReturnI);
+
+    // Mark the block live after we looked for no-return instructions.
+    assumeLive(A, F->getEntryBlock());
   }
 
   /// Find the next assumed noreturn instruction in the block of \p I starting
@@ -2423,7 +2473,7 @@ struct AAAlignImpl : AAAlign {
       } else if (auto *LI = dyn_cast<LoadInst>(U.getUser())) {
         if (LI->getPointerOperand() == &AnchorVal)
           if (LI->getAlignment() < getAssumedAlign()) {
-            LI->setAlignment(getAssumedAlign());
+            LI->setAlignment(Align(getAssumedAlign()));
             STATS_DECLTRACK(AAAlign, Load,
                             "Number of times alignemnt added to a load");
             Changed = ChangeStatus::CHANGED;
@@ -3180,6 +3230,213 @@ struct AAValueSimplifyCallSiteArgument : AAValueSimplifyFloating {
   }
 };
 
+/// ----------------------- Heap-To-Stack Conversion ---------------------------
+struct AAHeapToStackImpl : public AAHeapToStack {
+  AAHeapToStackImpl(const IRPosition &IRP) : AAHeapToStack(IRP) {}
+
+  const std::string getAsStr() const override {
+    return "[H2S] Mallocs: " + std::to_string(MallocCalls.size());
+  }
+
+  ChangeStatus manifest(Attributor &A) override {
+    assert(getState().isValidState() &&
+           "Attempted to manifest an invalid state!");
+
+    ChangeStatus HasChanged = ChangeStatus::UNCHANGED;
+    Function *F = getAssociatedFunction();
+    const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+
+    for (Instruction *MallocCall : MallocCalls) {
+      // This malloc cannot be replaced.
+      if (BadMallocCalls.count(MallocCall))
+        continue;
+
+      for (Instruction *FreeCall : FreesForMalloc[MallocCall]) {
+        LLVM_DEBUG(dbgs() << "H2S: Removing free call: " << *FreeCall << "\n");
+        A.deleteAfterManifest(*FreeCall);
+        HasChanged = ChangeStatus::CHANGED;
+      }
+
+      LLVM_DEBUG(dbgs() << "H2S: Removing malloc call: " << *MallocCall
+                        << "\n");
+
+      Constant *Size;
+      if (isCallocLikeFn(MallocCall, TLI)) {
+        auto *Num = cast<ConstantInt>(MallocCall->getOperand(0));
+        auto *SizeT = dyn_cast<ConstantInt>(MallocCall->getOperand(1));
+        APInt TotalSize = SizeT->getValue() * Num->getValue();
+        Size =
+            ConstantInt::get(MallocCall->getOperand(0)->getType(), TotalSize);
+      } else {
+        Size = cast<ConstantInt>(MallocCall->getOperand(0));
+      }
+
+      unsigned AS = cast<PointerType>(MallocCall->getType())->getAddressSpace();
+      Instruction *AI = new AllocaInst(Type::getInt8Ty(F->getContext()), AS,
+                                       Size, "", MallocCall->getNextNode());
+
+      if (AI->getType() != MallocCall->getType())
+        AI = new BitCastInst(AI, MallocCall->getType(), "malloc_bc",
+                             AI->getNextNode());
+
+      MallocCall->replaceAllUsesWith(AI);
+
+      if (auto *II = dyn_cast<InvokeInst>(MallocCall)) {
+        auto *NBB = II->getNormalDest();
+        BranchInst::Create(NBB, MallocCall->getParent());
+        A.deleteAfterManifest(*MallocCall);
+      } else {
+        A.deleteAfterManifest(*MallocCall);
+      }
+
+      if (isCallocLikeFn(MallocCall, TLI)) {
+        auto *BI = new BitCastInst(AI, MallocCall->getType(), "calloc_bc",
+                                   AI->getNextNode());
+        Value *Ops[] = {
+            BI, ConstantInt::get(F->getContext(), APInt(8, 0, false)), Size,
+            ConstantInt::get(Type::getInt1Ty(F->getContext()), false)};
+
+        Type *Tys[] = {BI->getType(), MallocCall->getOperand(0)->getType()};
+        Module *M = F->getParent();
+        Function *Fn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
+        CallInst::Create(Fn, Ops, "", BI->getNextNode());
+      }
+      HasChanged = ChangeStatus::CHANGED;
+    }
+
+    return HasChanged;
+  }
+
+  /// Collection of all malloc calls in a function.
+  SmallSetVector<Instruction *, 4> MallocCalls;
+
+  /// Collection of malloc calls that cannot be converted.
+  DenseSet<const Instruction *> BadMallocCalls;
+
+  /// A map for each malloc call to the set of associated free calls.
+  DenseMap<Instruction *, SmallPtrSet<Instruction *, 4>> FreesForMalloc;
+
+  ChangeStatus updateImpl(Attributor &A) override;
+};
+
+ChangeStatus AAHeapToStackImpl::updateImpl(Attributor &A) {
+  const Function *F = getAssociatedFunction();
+  const auto *TLI = A.getInfoCache().getTargetLibraryInfoForFunction(*F);
+
+  auto UsesCheck = [&](Instruction &I) {
+    SmallPtrSet<const Use *, 8> Visited;
+    SmallVector<const Use *, 8> Worklist;
+
+    for (Use &U : I.uses())
+      Worklist.push_back(&U);
+
+    while (!Worklist.empty()) {
+      const Use *U = Worklist.pop_back_val();
+      if (!Visited.insert(U).second)
+        continue;
+
+      auto *UserI = U->getUser();
+
+      if (isa<LoadInst>(UserI) || isa<StoreInst>(UserI))
+        continue;
+
+      // NOTE: Right now, if a function that has malloc pointer as an argument
+      // frees memory, we assume that the malloc pointer is freed.
+
+      // TODO: Add nofree callsite argument attribute to indicate that pointer
+      // argument is not freed.
+      if (auto *CB = dyn_cast<CallBase>(UserI)) {
+        if (!CB->isArgOperand(U))
+          continue;
+
+        if (CB->isLifetimeStartOrEnd())
+          continue;
+
+        // Record malloc.
+        if (isFreeCall(UserI, TLI)) {
+          FreesForMalloc[&I].insert(
+              cast<Instruction>(const_cast<User *>(UserI)));
+          continue;
+        }
+
+        // If a function does not free memory we are fine
+        const auto &NoFreeAA =
+            A.getAAFor<AANoFree>(*this, IRPosition::callsite_function(*CB));
+
+        unsigned ArgNo = U - CB->arg_begin();
+        const auto &NoCaptureAA = A.getAAFor<AANoCapture>(
+            *this, IRPosition::callsite_argument(*CB, ArgNo));
+
+        if (!NoCaptureAA.isAssumedNoCapture() || !NoFreeAA.isAssumedNoFree()) {
+          LLVM_DEBUG(dbgs() << "[H2S] Bad user: " << *UserI << "\n");
+          return false;
+        }
+        continue;
+      }
+
+      if (isa<GetElementPtrInst>(UserI) || isa<BitCastInst>(UserI)) {
+        for (Use &U : UserI->uses())
+          Worklist.push_back(&U);
+        continue;
+      }
+
+      // Unknown user.
+      LLVM_DEBUG(dbgs() << "[H2S] Unknown user: " << *UserI << "\n");
+      return false;
+    }
+    return true;
+  };
+
+  auto MallocCallocCheck = [&](Instruction &I) {
+    if (isMallocLikeFn(&I, TLI)) {
+      if (auto *Size = dyn_cast<ConstantInt>(I.getOperand(0)))
+        if (!Size->getValue().sle(MaxHeapToStackSize))
+          return true;
+    } else if (isCallocLikeFn(&I, TLI)) {
+      bool Overflow = false;
+      if (auto *Num = dyn_cast<ConstantInt>(I.getOperand(0)))
+        if (auto *Size = dyn_cast<ConstantInt>(I.getOperand(1)))
+          if (!(Size->getValue().umul_ov(Num->getValue(), Overflow))
+                   .sle(MaxHeapToStackSize))
+            if (!Overflow)
+              return true;
+    } else {
+      BadMallocCalls.insert(&I);
+      return true;
+    }
+
+    if (BadMallocCalls.count(&I))
+      return true;
+
+    if (UsesCheck(I))
+      MallocCalls.insert(&I);
+    else
+      BadMallocCalls.insert(&I);
+    return true;
+  };
+
+  size_t NumBadMallocs = BadMallocCalls.size();
+
+  A.checkForAllCallLikeInstructions(MallocCallocCheck, *this);
+
+  if (NumBadMallocs != BadMallocCalls.size())
+    return ChangeStatus::CHANGED;
+
+  return ChangeStatus::UNCHANGED;
+}
+
+struct AAHeapToStackFunction final : public AAHeapToStackImpl {
+  AAHeapToStackFunction(const IRPosition &IRP) : AAHeapToStackImpl(IRP) {}
+
+  /// See AbstractAttribute::trackStatistics()
+  void trackStatistics() const override {
+    STATS_DECL(MallocCalls, Function,
+               "Number of MallocCalls converted to allocas");
+    BUILD_STAT_NAME(MallocCalls, Function) += MallocCalls.size();
+  }
+};
+} // namespace
+
 /// ----------------------------------------------------------------------------
 ///                               Attributor
 /// ----------------------------------------------------------------------------
@@ -3312,6 +3569,26 @@ bool Attributor::checkForAllReturnedValues(
       });
 }
 
+static bool
+checkForAllInstructionsImpl(InformationCache::OpcodeInstMapTy &OpcodeInstMap,
+                            const function_ref<bool(Instruction &)> &Pred,
+                            const AAIsDead *LivenessAA, bool &AnyDead,
+                            const ArrayRef<unsigned> &Opcodes) {
+  for (unsigned Opcode : Opcodes) {
+    for (Instruction *I : OpcodeInstMap[Opcode]) {
+      // Skip dead instructions.
+      if (LivenessAA && LivenessAA->isAssumedDead(I)) {
+        AnyDead = true;
+        continue;
+      }
+
+      if (!Pred(*I))
+        return false;
+    }
+  }
+  return true;
+}
+
 bool Attributor::checkForAllInstructions(
     const llvm::function_ref<bool(Instruction &)> &Pred,
     const AbstractAttribute &QueryingAA, const ArrayRef<unsigned> &Opcodes) {
@@ -3330,18 +3607,8 @@ bool Attributor::checkForAllInstructions(
 
   auto &OpcodeInstMap =
       InfoCache.getOpcodeInstMapForFunction(*AssociatedFunction);
-  for (unsigned Opcode : Opcodes) {
-    for (Instruction *I : OpcodeInstMap[Opcode]) {
-      // Skip dead instructions.
-      if (LivenessAA.isAssumedDead(I)) {
-        AnyDead = true;
-        continue;
-      }
-
-      if (!Pred(*I))
-        return false;
-    }
-  }
+  if (!checkForAllInstructionsImpl(OpcodeInstMap, Pred, &LivenessAA, AnyDead, Opcodes))
+    return false;
 
   // If we actually used liveness information so we have to record a dependence.
   if (AnyDead)
@@ -3627,6 +3894,48 @@ ChangeStatus Attributor::run(Module &M) {
   return ManifestChange;
 }
 
+void Attributor::initializeInformationCache(Function &F) {
+
+  // Walk all instructions to find interesting instructions that might be
+  // queried by abstract attributes during their initialization or update.
+  // This has to happen before we create attributes.
+  auto &ReadOrWriteInsts = InfoCache.FuncRWInstsMap[&F];
+  auto &InstOpcodeMap = InfoCache.FuncInstOpcodeMap[&F];
+
+  for (Instruction &I : instructions(&F)) {
+    bool IsInterestingOpcode = false;
+
+    // To allow easy access to all instructions in a function with a given
+    // opcode we store them in the InfoCache. As not all opcodes are interesting
+    // to concrete attributes we only cache the ones that are as identified in
+    // the following switch.
+    // Note: There are no concrete attributes now so this is initially empty.
+    switch (I.getOpcode()) {
+    default:
+      assert((!ImmutableCallSite(&I)) && (!isa<CallBase>(&I)) &&
+             "New call site/base instruction type needs to be known int the "
+             "Attributor.");
+      break;
+    case Instruction::Load:
+      // The alignment of a pointer is interesting for loads.
+    case Instruction::Store:
+      // The alignment of a pointer is interesting for stores.
+    case Instruction::Call:
+    case Instruction::CallBr:
+    case Instruction::Invoke:
+    case Instruction::CleanupRet:
+    case Instruction::CatchSwitch:
+    case Instruction::Resume:
+    case Instruction::Ret:
+      IsInterestingOpcode = true;
+    }
+    if (IsInterestingOpcode)
+      InstOpcodeMap[I.getOpcode()].push_back(&I);
+    if (I.mayReadOrWriteMemory())
+      ReadOrWriteInsts.push_back(&I);
+  }
+}
+
 void Attributor::identifyDefaultAbstractAttributes(Function &F) {
   if (!VisitedFunctions.insert(&F).second)
     return;
@@ -3653,6 +3962,13 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
   // Every function might be "no-return".
   getOrCreateAAFor<AANoReturn>(FPos);
 
+  // Every function might be "no-recurse".
+  getOrCreateAAFor<AANoRecurse>(FPos);
+
+  // Every function might be applicable for Heap-To-Stack conversion.
+  if (EnableHeapToStack)
+    getOrCreateAAFor<AAHeapToStack>(FPos);
+
   // Return attributes are only appropriate if the return type is non void.
   Type *ReturnType = F.getReturnType();
   if (!ReturnType->isVoidTy()) {
@@ -3706,52 +4022,9 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
     }
   }
 
-  // Walk all instructions to find more attribute opportunities and also
-  // interesting instructions that might be queried by abstract attributes
-  // during their initialization or update.
-  auto &ReadOrWriteInsts = InfoCache.FuncRWInstsMap[&F];
-  auto &InstOpcodeMap = InfoCache.FuncInstOpcodeMap[&F];
-
-  for (Instruction &I : instructions(&F)) {
-    bool IsInterestingOpcode = false;
-
-    // To allow easy access to all instructions in a function with a given
-    // opcode we store them in the InfoCache. As not all opcodes are interesting
-    // to concrete attributes we only cache the ones that are as identified in
-    // the following switch.
-    // Note: There are no concrete attributes now so this is initially empty.
-    switch (I.getOpcode()) {
-    default:
-      assert((!ImmutableCallSite(&I)) && (!isa<CallBase>(&I)) &&
-             "New call site/base instruction type needs to be known int the "
-             "attributor.");
-      break;
-    case Instruction::Load:
-      // The alignment of a pointer is interesting for loads.
-      getOrCreateAAFor<AAAlign>(
-          IRPosition::value(*cast<LoadInst>(I).getPointerOperand()));
-      break;
-    case Instruction::Store:
-      // The alignment of a pointer is interesting for stores.
-      getOrCreateAAFor<AAAlign>(
-          IRPosition::value(*cast<StoreInst>(I).getPointerOperand()));
-      break;
-    case Instruction::Call:
-    case Instruction::CallBr:
-    case Instruction::Invoke:
-    case Instruction::CleanupRet:
-    case Instruction::CatchSwitch:
-    case Instruction::Resume:
-    case Instruction::Ret:
-      IsInterestingOpcode = true;
-    }
-    if (IsInterestingOpcode)
-      InstOpcodeMap[I.getOpcode()].push_back(&I);
-    if (I.mayReadOrWriteMemory())
-      ReadOrWriteInsts.push_back(&I);
-
+  auto CallSitePred = [&](Instruction &I) -> bool {
     CallSite CS(&I);
-    if (CS && CS.getCalledFunction()) {
+    if (CS.getCalledFunction()) {
       for (int i = 0, e = CS.getCalledFunction()->arg_size(); i < e; i++) {
 
         IRPosition CSArgPos = IRPosition::callsite_argument(CS, i);
@@ -3775,7 +4048,32 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
         getOrCreateAAFor<AAAlign>(CSArgPos);
       }
     }
-  }
+    return true;
+  };
+
+  auto &OpcodeInstMap = InfoCache.getOpcodeInstMapForFunction(F);
+  bool Success, AnyDead = false;
+  Success = checkForAllInstructionsImpl(
+      OpcodeInstMap, CallSitePred, nullptr, AnyDead,
+      {(unsigned)Instruction::Invoke, (unsigned)Instruction::CallBr,
+       (unsigned)Instruction::Call});
+  (void)Success;
+  assert(Success && !AnyDead && "Expected the check call to be successful!");
+
+  auto LoadStorePred = [&](Instruction &I) -> bool {
+    if (isa<LoadInst>(I))
+      getOrCreateAAFor<AAAlign>(
+          IRPosition::value(*cast<LoadInst>(I).getPointerOperand()));
+    else
+      getOrCreateAAFor<AAAlign>(
+          IRPosition::value(*cast<StoreInst>(I).getPointerOperand()));
+    return true;
+  };
+  Success = checkForAllInstructionsImpl(
+      OpcodeInstMap, LoadStorePred, nullptr, AnyDead,
+      {(unsigned)Instruction::Load, (unsigned)Instruction::Store});
+  (void)Success;
+  assert(Success && !AnyDead && "Expected the check call to be successful!");
 }
 
 /// Helpers to ease debugging through output streams and print calls.
@@ -3837,7 +4135,7 @@ void AbstractAttribute::print(raw_ostream &OS) const {
 ///                       Pass (Manager) Boilerplate
 /// ----------------------------------------------------------------------------
 
-static bool runAttributorOnModule(Module &M) {
+static bool runAttributorOnModule(Module &M, AnalysisGetter &AG) {
   if (DisableAttributor)
     return false;
 
@@ -3846,9 +4144,12 @@ static bool runAttributorOnModule(Module &M) {
 
   // Create an Attributor and initially empty information cache that is filled
   // while we identify default attribute opportunities.
-  InformationCache InfoCache(M.getDataLayout());
+  InformationCache InfoCache(M, AG);
   Attributor A(InfoCache, DepRecInterval);
 
+  for (Function &F : M)
+    A.initializeInformationCache(F);
+
   for (Function &F : M) {
     if (F.hasExactDefinition())
       NumFnWithExactDefinition++;
@@ -3879,7 +4180,8 @@ static bool runAttributorOnModule(Module &M) {
 }
 
 PreservedAnalyses AttributorPass::run(Module &M, ModuleAnalysisManager &AM) {
-  if (runAttributorOnModule(M)) {
+  AnalysisGetter AG(AM);
+  if (runAttributorOnModule(M, AG)) {
     // FIXME: Think about passes we will preserve and add them here.
     return PreservedAnalyses::none();
   }
@@ -3898,11 +4200,14 @@ struct AttributorLegacyPass : public ModulePass {
   bool runOnModule(Module &M) override {
     if (skipModule(M))
       return false;
-    return runAttributorOnModule(M);
+
+    AnalysisGetter AG;
+    return runAttributorOnModule(M, AG);
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     // FIXME: Think about passes we will preserve and add them here.
+    AU.addRequired<TargetLibraryInfoWrapperPass>();
   }
 };
 
@@ -3926,6 +4231,7 @@ const char AADereferenceable::ID = 0;
 const char AAAlign::ID = 0;
 const char AANoCapture::ID = 0;
 const char AAValueSimplify::ID = 0;
+const char AAHeapToStack::ID = 0;
 
 // Macro magic to create the static generator function for attributes that
 // follow the naming scheme.
@@ -3987,6 +4293,23 @@ const char AAValueSimplify::ID = 0;
     return *AA;                                                                \
   }
 
+#define CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(CLASS)            \
+  CLASS &CLASS::createForPosition(const IRPosition &IRP, Attributor &A) {      \
+    CLASS *AA = nullptr;                                                       \
+    switch (IRP.getPositionKind()) {                                           \
+      SWITCH_PK_INV(CLASS, IRP_INVALID, "invalid")                             \
+      SWITCH_PK_INV(CLASS, IRP_ARGUMENT, "argument")                           \
+      SWITCH_PK_INV(CLASS, IRP_FLOAT, "floating")                              \
+      SWITCH_PK_INV(CLASS, IRP_RETURNED, "returned")                           \
+      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_RETURNED, "call site returned")       \
+      SWITCH_PK_INV(CLASS, IRP_CALL_SITE_ARGUMENT, "call site argument")       \
+      SWITCH_PK_INV(CLASS, IRP_CALL_SITE, "call site")                         \
+      SWITCH_PK_CREATE(CLASS, IRP, IRP_FUNCTION, Function)                     \
+    }                                                                          \
+    AA->initialize(A);                                                         \
+    return *AA;                                                                \
+  }
+
 CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUnwind)
 CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoSync)
 CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFree)
@@ -4004,6 +4327,8 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoCapture)
 
 CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
 
+CREATE_FUNCTION_ONLY_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAHeapToStack)
+
 #undef CREATE_FUNCTION_ABSTRACT_ATTRIBUTE_FOR_POSITION
 #undef CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION
 #undef CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION
@@ -4012,5 +4337,6 @@ CREATE_ALL_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAValueSimplify)
 
 INITIALIZE_PASS_BEGIN(AttributorLegacyPass, "attributor",
                       "Deduce and propagate attributes", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(AttributorLegacyPass, "attributor",
                     "Deduce and propagate attributes", false, false)
diff --git a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
index 0e923a717e6ea..b174c63a577b1 100644
--- a/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -78,11 +78,8 @@ STATISTIC(NumNoRecurse, "Number of functions marked as norecurse");
 STATISTIC(NumNoUnwind, "Number of functions marked as nounwind");
 STATISTIC(NumNoFree, "Number of functions marked as nofree");
 
-// FIXME: This is disabled by default to avoid exposing security vulnerabilities
-// in C/C++ code compiled by clang:
-// http://lists.llvm.org/pipermail/cfe-dev/2017-January/052066.html
 static cl::opt<bool> EnableNonnullArgPropagation(
-    "enable-nonnull-arg-prop", cl::Hidden,
+    "enable-nonnull-arg-prop", cl::init(true), cl::Hidden,
     cl::desc("Try to propagate nonnull argument attributes from callsites to "
              "caller functions."));
 
diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 9c7fd5e1a813f..feac1b6088489 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -2285,14 +2285,10 @@ OptimizeFunctions(Module &M,
     // So, remove unreachable blocks from the function, because a) there's
     // no point in analyzing them and b) GlobalOpt should otherwise grow
     // some more complicated logic to break these cycles.
-    // Removing unreachable blocks might invalidate the dominator so we
-    // recalculate it.
     if (!F->isDeclaration()) {
-      if (removeUnreachableBlocks(*F)) {
-        auto &DT = LookupDomTree(*F);
-        DT.recalculate(*F);
-        Changed = true;
-      }
+      auto &DT = LookupDomTree(*F);
+      DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+      Changed |= removeUnreachableBlocks(*F, &DTU);
     }
 
     Changed |= processGlobal(*F, GetTLI, LookupDomTree);
diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index 79a5a9b80a9d1..31571c4a20a87 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -85,12 +85,6 @@ static cl::opt<int>
                                 "multiple of TCC_Basic)"));
 
 namespace {
-
-/// A sequence of basic blocks.
-///
-/// A 0-sized SmallVector is slightly cheaper to move than a std::vector.
-using BlockSequence = SmallVector<BasicBlock *, 0>;
-
 // Same as blockEndsInUnreachable in CodeGen/BranchFolding.cpp. Do not modify
 // this function unless you modify the MBB version as well.
 //
@@ -169,31 +163,6 @@ static bool markFunctionCold(Function &F, bool UpdateEntryCount = false) {
   return Changed;
 }
 
-class HotColdSplitting {
-public:
-  HotColdSplitting(ProfileSummaryInfo *ProfSI,
-                   function_ref<BlockFrequencyInfo *(Function &)> GBFI,
-                   function_ref<TargetTransformInfo &(Function &)> GTTI,
-                   std::function<OptimizationRemarkEmitter &(Function &)> *GORE,
-                   function_ref<AssumptionCache *(Function &)> LAC)
-      : PSI(ProfSI), GetBFI(GBFI), GetTTI(GTTI), GetORE(GORE), LookupAC(LAC) {}
-  bool run(Module &M);
-
-private:
-  bool isFunctionCold(const Function &F) const;
-  bool shouldOutlineFrom(const Function &F) const;
-  bool outlineColdRegions(Function &F, bool HasProfileSummary);
-  Function *extractColdRegion(const BlockSequence &Region, DominatorTree &DT,
-                              BlockFrequencyInfo *BFI, TargetTransformInfo &TTI,
-                              OptimizationRemarkEmitter &ORE,
-                              AssumptionCache *AC, unsigned Count);
-  ProfileSummaryInfo *PSI;
-  function_ref<BlockFrequencyInfo *(Function &)> GetBFI;
-  function_ref<TargetTransformInfo &(Function &)> GetTTI;
-  std::function<OptimizationRemarkEmitter &(Function &)> *GetORE;
-  function_ref<AssumptionCache *(Function &)> LookupAC;
-};
-
 class HotColdSplittingLegacyPass : public ModulePass {
 public:
   static char ID;
@@ -659,6 +628,11 @@ bool HotColdSplitting::outlineColdRegions(Function &F, bool HasProfileSummary) {
     } while (!Region.empty());
   }
 
+  // We need to explicitly clear the assumption cache since the value tracking
+  // may now be invalid as part of the function has changed.
+  if (Changed)
+    if (AssumptionCache *AC = LookupAC(F))
+      AC->clear();
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/IPO/IPO.cpp b/llvm/lib/Transforms/IPO/IPO.cpp
index 465ffbd905525..bddf752115999 100644
--- a/llvm/lib/Transforms/IPO/IPO.cpp
+++ b/llvm/lib/Transforms/IPO/IPO.cpp
@@ -114,6 +114,10 @@ void LLVMAddIPSCCPPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createIPSCCPPass());
 }
 
+void LLVMAddMergeFunctionsPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createMergeFunctionsPass());
+}
+
 void LLVMAddInternalizePass(LLVMPassManagerRef PM, unsigned AllButMain) {
   auto PreserveMain = [=](const GlobalValue &GV) {
     return AllButMain && GV.getName() == "main";
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 1bda405da7f5f..4b72261131c16 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -239,7 +239,7 @@ static void mergeInlinedArrayAllocas(
         }
 
         if (Align1 > Align2)
-          AvailableAlloca->setAlignment(AI->getAlignment());
+          AvailableAlloca->setAlignment(MaybeAlign(AI->getAlignment()));
       }
 
       AI->eraseFromParent();
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index d0cf63b35f4b9..c9e7a19c380a2 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -130,6 +130,12 @@ static cl::opt<bool> ProfileSampleAccurate(
              "callsite and function as having 0 samples. Otherwise, treat "
              "un-sampled callsites and functions conservatively as unknown. "));
 
+static cl::opt<bool> ProfileAccurateForSymsInList(
+    "profile-accurate-for-symsinlist", cl::Hidden, cl::ZeroOrMore,
+    cl::init(true),
+    cl::desc("For symbols in profile symbol list, regard their profiles to "
+             "be accurate. It may be overriden by profile-sample-accurate. "));
+
 namespace {
 
 using BlockWeightMap = DenseMap<const BasicBlock *, uint64_t>;
@@ -139,9 +145,11 @@ using EdgeWeightMap = DenseMap<Edge, uint64_t>;
 using BlockEdgeMap =
     DenseMap<const BasicBlock *, SmallVector<const BasicBlock *, 8>>;
 
+class SampleProfileLoader;
+
 class SampleCoverageTracker {
 public:
-  SampleCoverageTracker() = default;
+  SampleCoverageTracker(SampleProfileLoader &SPL) : SPLoader(SPL){};
 
   bool markSamplesUsed(const FunctionSamples *FS, uint32_t LineOffset,
                        uint32_t Discriminator, uint64_t Samples);
@@ -187,6 +195,8 @@ class SampleCoverageTracker {
   /// keyed by FunctionSamples pointers, but these stats are cleared after
   /// every function, so we just need to keep a single counter.
   uint64_t TotalUsedSamples = 0;
+
+  SampleProfileLoader &SPLoader;
 };
 
 class GUIDToFuncNameMapper {
@@ -269,8 +279,9 @@ class SampleProfileLoader {
       std::function<AssumptionCache &(Function &)> GetAssumptionCache,
       std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo)
       : GetAC(std::move(GetAssumptionCache)),
-        GetTTI(std::move(GetTargetTransformInfo)), Filename(Name),
-        RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {}
+        GetTTI(std::move(GetTargetTransformInfo)), CoverageTracker(*this),
+        Filename(Name), RemappingFilename(RemapName),
+        IsThinLTOPreLink(IsThinLTOPreLink) {}
 
   bool doInitialization(Module &M);
   bool runOnModule(Module &M, ModuleAnalysisManager *AM,
@@ -279,6 +290,8 @@ class SampleProfileLoader {
   void dump() { Reader->dump(); }
 
 protected:
+  friend class SampleCoverageTracker;
+
   bool runOnFunction(Function &F, ModuleAnalysisManager *AM);
   unsigned getFunctionLoc(Function &F);
   bool emitAnnotations(Function &F);
@@ -307,6 +320,8 @@ class SampleProfileLoader {
   bool propagateThroughEdges(Function &F, bool UpdateBlockCount);
   void computeDominanceAndLoopInfo(Function &F);
   void clearFunctionData();
+  bool callsiteIsHot(const FunctionSamples *CallsiteFS,
+                     ProfileSummaryInfo *PSI);
 
   /// Map basic blocks to their computed weights.
   ///
@@ -404,6 +419,17 @@ class SampleProfileLoader {
   // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
   // all the function symbols defined or declared in current module.
   DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
+
+  // All the Names used in FunctionSamples including outline function
+  // names, inline instance names and call target names.
+  StringSet<> NamesInProfile;
+
+  // For symbol in profile symbol list, whether to regard their profiles
+  // to be accurate. It is mainly decided by existance of profile symbol
+  // list and -profile-accurate-for-symsinlist flag, but it can be
+  // overriden by -profile-sample-accurate or profile-sample-accurate
+  // attribute.
+  bool ProfAccForSymsInList;
 };
 
 class SampleProfileLoaderLegacyPass : public ModulePass {
@@ -459,14 +485,23 @@ class SampleProfileLoaderLegacyPass : public ModulePass {
 /// To decide whether an inlined callsite is hot, we compare the callsite
 /// sample count with the hot cutoff computed by ProfileSummaryInfo, it is
 /// regarded as hot if the count is above the cutoff value.
-static bool callsiteIsHot(const FunctionSamples *CallsiteFS,
-                          ProfileSummaryInfo *PSI) {
+///
+/// When ProfileAccurateForSymsInList is enabled and profile symbol list
+/// is present, functions in the profile symbol list but without profile will
+/// be regarded as cold and much less inlining will happen in CGSCC inlining
+/// pass, so we tend to lower the hot criteria here to allow more early
+/// inlining to happen for warm callsites and it is helpful for performance.
+bool SampleProfileLoader::callsiteIsHot(const FunctionSamples *CallsiteFS,
+                                        ProfileSummaryInfo *PSI) {
   if (!CallsiteFS)
     return false; // The callsite was not inlined in the original binary.
 
   assert(PSI && "PSI is expected to be non null");
   uint64_t CallsiteTotalSamples = CallsiteFS->getTotalSamples();
-  return PSI->isHotCount(CallsiteTotalSamples);
+  if (ProfAccForSymsInList)
+    return !PSI->isColdCount(CallsiteTotalSamples);
+  else
+    return PSI->isHotCount(CallsiteTotalSamples);
 }
 
 /// Mark as used the sample record for the given function samples at
@@ -503,7 +538,7 @@ SampleCoverageTracker::countUsedRecords(const FunctionSamples *FS,
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
-      if (callsiteIsHot(CalleeSamples, PSI))
+      if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
         Count += countUsedRecords(CalleeSamples, PSI);
     }
 
@@ -522,7 +557,7 @@ SampleCoverageTracker::countBodyRecords(const FunctionSamples *FS,
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
-      if (callsiteIsHot(CalleeSamples, PSI))
+      if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
         Count += countBodyRecords(CalleeSamples, PSI);
     }
 
@@ -543,7 +578,7 @@ SampleCoverageTracker::countBodySamples(const FunctionSamples *FS,
   for (const auto &I : FS->getCallsiteSamples())
     for (const auto &J : I.second) {
       const FunctionSamples *CalleeSamples = &J.second;
-      if (callsiteIsHot(CalleeSamples, PSI))
+      if (SPLoader.callsiteIsHot(CalleeSamples, PSI))
         Total += countBodySamples(CalleeSamples, PSI);
     }
 
@@ -866,6 +901,14 @@ bool SampleProfileLoader::inlineHotFunctions(
     Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
   DenseSet<Instruction *> PromotedInsns;
 
+  // ProfAccForSymsInList is used in callsiteIsHot. The assertion makes sure
+  // Profile symbol list is ignored when profile-sample-accurate is on.
+  assert((!ProfAccForSymsInList ||
+          (!ProfileSampleAccurate &&
+           !F.hasFnAttribute("profile-sample-accurate"))) &&
+         "ProfAccForSymsInList should be false when profile-sample-accurate "
+         "is enabled");
+
   DenseMap<Instruction *, const FunctionSamples *> localNotInlinedCallSites;
   bool Changed = false;
   while (true) {
@@ -1643,6 +1686,15 @@ bool SampleProfileLoader::doInitialization(Module &M) {
   ProfileIsValid = (Reader->read() == sampleprof_error::success);
   PSL = Reader->getProfileSymbolList();
 
+  // While profile-sample-accurate is on, ignore symbol list.
+  ProfAccForSymsInList =
+      ProfileAccurateForSymsInList && PSL && !ProfileSampleAccurate;
+  if (ProfAccForSymsInList) {
+    NamesInProfile.clear();
+    if (auto NameTable = Reader->getNameTable())
+      NamesInProfile.insert(NameTable->begin(), NameTable->end());
+  }
+
   if (!RemappingFilename.empty()) {
     // Apply profile remappings to the loaded profile data if requested.
     // For now, we only support remapping symbols encoded using the Itanium
@@ -1733,17 +1785,42 @@ bool SampleProfileLoader::runOnFunction(Function &F, ModuleAnalysisManager *AM)
   // conservatively by getEntryCount as the same as unknown (None). This is
   // to avoid newly added code to be treated as cold. If we have samples
   // this will be overwritten in emitAnnotations.
-  //
+  uint64_t initialEntryCount = -1;
+
+  ProfAccForSymsInList = ProfileAccurateForSymsInList && PSL;
+  if (ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) {
+    // initialize all the function entry counts to 0. It means all the
+    // functions without profile will be regarded as cold.
+    initialEntryCount = 0;
+    // profile-sample-accurate is a user assertion which has a higher precedence
+    // than symbol list. When profile-sample-accurate is on, ignore symbol list.
+    ProfAccForSymsInList = false;
+  }
+
   // PSL -- profile symbol list include all the symbols in sampled binary.
-  // If ProfileSampleAccurate is true or F has profile-sample-accurate
-  // attribute, and if there is no profile symbol list read in, initialize
-  // all the function entry counts to 0; if there is profile symbol list, only
-  // initialize the entry count to 0 when current function is in the list.
-  uint64_t initialEntryCount =
-      ((ProfileSampleAccurate || F.hasFnAttribute("profile-sample-accurate")) &&
-       (!PSL || PSL->contains(F.getName())))
-          ? 0
-          : -1;
+  // If ProfileAccurateForSymsInList is enabled, PSL is used to treat
+  // old functions without samples being cold, without having to worry
+  // about new and hot functions being mistakenly treated as cold.
+  if (ProfAccForSymsInList) {
+    // Initialize the entry count to 0 for functions in the list.
+    if (PSL->contains(F.getName()))
+      initialEntryCount = 0;
+
+    // Function in the symbol list but without sample will be regarded as
+    // cold. To minimize the potential negative performance impact it could
+    // have, we want to be a little conservative here saying if a function
+    // shows up in the profile, no matter as outline function, inline instance
+    // or call targets, treat the function as not being cold. This will handle
+    // the cases such as most callsites of a function are inlined in sampled
+    // binary but not inlined in current build (because of source code drift,
+    // imprecise debug information, or the callsites are all cold individually
+    // but not cold accumulatively...), so the outline function showing up as
+    // cold in sampled binary will actually not be cold after current build.
+    StringRef CanonName = FunctionSamples::getCanonicalFnName(F);
+    if (NamesInProfile.count(CanonName))
+      initialEntryCount = -1;
+  }
+
   F.setEntryCount(ProfileCount(initialEntryCount, Function::PCT_Real));
   std::unique_ptr<OptimizationRemarkEmitter> OwnedORE;
   if (AM) {
diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 4490a7ef62adf..4055fe049999b 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -713,7 +713,7 @@ void runWholeProgramDevirtOnIndex(
 
 void updateIndexWPDForExports(
     ModuleSummaryIndex &Summary,
-    StringMap<FunctionImporter::ExportSetTy> &ExportLists,
+    function_ref<bool(StringRef, GlobalValue::GUID)> isExported,
     std::map<ValueInfo, std::vector<VTableSlotSummary>> &LocalWPDTargetsMap) {
   for (auto &T : LocalWPDTargetsMap) {
     auto &VI = T.first;
@@ -721,9 +721,7 @@ void updateIndexWPDForExports(
     assert(VI.getSummaryList().size() == 1 &&
            "Devirt of local target has more than one copy");
     auto &S = VI.getSummaryList()[0];
-    const auto &ExportList = ExportLists.find(S->modulePath());
-    if (ExportList == ExportLists.end() ||
-        !ExportList->second.count(VI.getGUID()))
+    if (!isExported(S->modulePath(), VI.getGUID()))
       continue;
 
     // It's been exported by a cross module import.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 9d640e849dc1d..40cc188fbb60c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1666,25 +1666,42 @@ Instruction *InstCombiner::visitSub(BinaryOperator &I) {
 
   const APInt *Op0C;
   if (match(Op0, m_APInt(Op0C))) {
-    unsigned BitWidth = I.getType()->getScalarSizeInBits();
 
-    // -(X >>u 31) -> (X >>s 31)
-    // -(X >>s 31) -> (X >>u 31)
     if (Op0C->isNullValue()) {
+      Value *Op1Wide;
+      match(Op1, m_TruncOrSelf(m_Value(Op1Wide)));
+      bool HadTrunc = Op1Wide != Op1;
+      bool NoTruncOrTruncIsOneUse = !HadTrunc || Op1->hasOneUse();
+      unsigned BitWidth = Op1Wide->getType()->getScalarSizeInBits();
+
       Value *X;
       const APInt *ShAmt;
-      if (match(Op1, m_LShr(m_Value(X), m_APInt(ShAmt))) &&
+      // -(X >>u 31) -> (X >>s 31)
+      if (NoTruncOrTruncIsOneUse &&
+          match(Op1Wide, m_LShr(m_Value(X), m_APInt(ShAmt))) &&
           *ShAmt == BitWidth - 1) {
-        Value *ShAmtOp = cast<Instruction>(Op1)->getOperand(1);
-        return BinaryOperator::CreateAShr(X, ShAmtOp);
+        Value *ShAmtOp = cast<Instruction>(Op1Wide)->getOperand(1);
+        Instruction *NewShift = BinaryOperator::CreateAShr(X, ShAmtOp);
+        NewShift->copyIRFlags(Op1Wide);
+        if (!HadTrunc)
+          return NewShift;
+        Builder.Insert(NewShift);
+        return TruncInst::CreateTruncOrBitCast(NewShift, Op1->getType());
       }
-      if (match(Op1, m_AShr(m_Value(X), m_APInt(ShAmt))) &&
+      // -(X >>s 31) -> (X >>u 31)
+      if (NoTruncOrTruncIsOneUse &&
+          match(Op1Wide, m_AShr(m_Value(X), m_APInt(ShAmt))) &&
           *ShAmt == BitWidth - 1) {
-        Value *ShAmtOp = cast<Instruction>(Op1)->getOperand(1);
-        return BinaryOperator::CreateLShr(X, ShAmtOp);
+        Value *ShAmtOp = cast<Instruction>(Op1Wide)->getOperand(1);
+        Instruction *NewShift = BinaryOperator::CreateLShr(X, ShAmtOp);
+        NewShift->copyIRFlags(Op1Wide);
+        if (!HadTrunc)
+          return NewShift;
+        Builder.Insert(NewShift);
+        return TruncInst::CreateTruncOrBitCast(NewShift, Op1->getType());
       }
 
-      if (Op1->hasOneUse()) {
+      if (!HadTrunc && Op1->hasOneUse()) {
         Value *LHS, *RHS;
         SelectPatternFlavor SPF = matchSelectPattern(Op1, LHS, RHS).Flavor;
         if (SPF == SPF_ABS || SPF == SPF_NABS) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index b9e8df8c29acf..db5095b6fea5e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1049,9 +1049,103 @@ static Value *foldIsPowerOf2(ICmpInst *Cmp0, ICmpInst *Cmp1, bool JoinedByAnd,
   return nullptr;
 }
 
+/// Commuted variants are assumed to be handled by calling this function again
+/// with the parameters swapped.
+static Value *foldUnsignedUnderflowCheck(ICmpInst *ZeroICmp,
+                                         ICmpInst *UnsignedICmp, bool IsAnd,
+                                         const SimplifyQuery &Q,
+                                         InstCombiner::BuilderTy &Builder) {
+  Value *ZeroCmpOp;
+  ICmpInst::Predicate EqPred;
+  if (!match(ZeroICmp, m_ICmp(EqPred, m_Value(ZeroCmpOp), m_Zero())) ||
+      !ICmpInst::isEquality(EqPred))
+    return nullptr;
+
+  auto IsKnownNonZero = [&](Value *V) {
+    return isKnownNonZero(V, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT);
+  };
+
+  ICmpInst::Predicate UnsignedPred;
+
+  Value *A, *B;
+  if (match(UnsignedICmp,
+            m_c_ICmp(UnsignedPred, m_Specific(ZeroCmpOp), m_Value(A))) &&
+      match(ZeroCmpOp, m_c_Add(m_Specific(A), m_Value(B))) &&
+      (ZeroICmp->hasOneUse() || UnsignedICmp->hasOneUse())) {
+    if (UnsignedICmp->getOperand(0) != ZeroCmpOp)
+      UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);
+
+    auto GetKnownNonZeroAndOther = [&](Value *&NonZero, Value *&Other) {
+      if (!IsKnownNonZero(NonZero))
+        std::swap(NonZero, Other);
+      return IsKnownNonZero(NonZero);
+    };
+
+    // Given  ZeroCmpOp = (A + B)
+    //   ZeroCmpOp <= A && ZeroCmpOp != 0  -->  (0-B) <  A
+    //   ZeroCmpOp >  A || ZeroCmpOp == 0  -->  (0-B) >= A
+    //
+    //   ZeroCmpOp <  A && ZeroCmpOp != 0  -->  (0-X) <  Y  iff
+    //   ZeroCmpOp >= A || ZeroCmpOp == 0  -->  (0-X) >= Y  iff
+    //     with X being the value (A/B) that is known to be non-zero,
+    //     and Y being remaining value.
+    if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
+        IsAnd)
+      return Builder.CreateICmpULT(Builder.CreateNeg(B), A);
+    if (UnsignedPred == ICmpInst::ICMP_ULT && EqPred == ICmpInst::ICMP_NE &&
+        IsAnd && GetKnownNonZeroAndOther(B, A))
+      return Builder.CreateICmpULT(Builder.CreateNeg(B), A);
+    if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
+        !IsAnd)
+      return Builder.CreateICmpUGE(Builder.CreateNeg(B), A);
+    if (UnsignedPred == ICmpInst::ICMP_UGE && EqPred == ICmpInst::ICMP_EQ &&
+        !IsAnd && GetKnownNonZeroAndOther(B, A))
+      return Builder.CreateICmpUGE(Builder.CreateNeg(B), A);
+  }
+
+  Value *Base, *Offset;
+  if (!match(ZeroCmpOp, m_Sub(m_Value(Base), m_Value(Offset))))
+    return nullptr;
+
+  if (!match(UnsignedICmp,
+             m_c_ICmp(UnsignedPred, m_Specific(Base), m_Specific(Offset))) ||
+      !ICmpInst::isUnsigned(UnsignedPred))
+    return nullptr;
+  if (UnsignedICmp->getOperand(0) != Base)
+    UnsignedPred = ICmpInst::getSwappedPredicate(UnsignedPred);
+
+  // Base >=/> Offset && (Base - Offset) != 0  <-->  Base > Offset
+  // (no overflow and not null)
+  if ((UnsignedPred == ICmpInst::ICMP_UGE ||
+       UnsignedPred == ICmpInst::ICMP_UGT) &&
+      EqPred == ICmpInst::ICMP_NE && IsAnd)
+    return Builder.CreateICmpUGT(Base, Offset);
+
+  // Base <=/< Offset || (Base - Offset) == 0  <-->  Base <= Offset
+  // (overflow or null)
+  if ((UnsignedPred == ICmpInst::ICMP_ULE ||
+       UnsignedPred == ICmpInst::ICMP_ULT) &&
+      EqPred == ICmpInst::ICMP_EQ && !IsAnd)
+    return Builder.CreateICmpULE(Base, Offset);
+
+  // Base <= Offset && (Base - Offset) != 0  -->  Base < Offset
+  if (UnsignedPred == ICmpInst::ICMP_ULE && EqPred == ICmpInst::ICMP_NE &&
+      IsAnd)
+    return Builder.CreateICmpULT(Base, Offset);
+
+  // Base > Offset || (Base - Offset) == 0  -->  Base >= Offset
+  if (UnsignedPred == ICmpInst::ICMP_UGT && EqPred == ICmpInst::ICMP_EQ &&
+      !IsAnd)
+    return Builder.CreateICmpUGE(Base, Offset);
+
+  return nullptr;
+}
+
 /// Fold (icmp)&(icmp) if possible.
 Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                     Instruction &CxtI) {
+  const SimplifyQuery Q = SQ.getWithInstruction(&CxtI);
+
   // Fold (!iszero(A & K1) & !iszero(A & K2)) ->  (A & (K1 | K2)) == (K1 | K2)
   // if K1 and K2 are a one-bit mask.
   if (Value *V = foldAndOrOfICmpsOfAndWithPow2(LHS, RHS, true, CxtI))
@@ -1094,6 +1188,13 @@ Value *InstCombiner::foldAndOfICmps(ICmpInst *LHS, ICmpInst *RHS,
   if (Value *V = foldIsPowerOf2(LHS, RHS, true /* JoinedByAnd */, Builder))
     return V;
 
+  if (Value *X =
+          foldUnsignedUnderflowCheck(LHS, RHS, /*IsAnd=*/true, Q, Builder))
+    return X;
+  if (Value *X =
+          foldUnsignedUnderflowCheck(RHS, LHS, /*IsAnd=*/true, Q, Builder))
+    return X;
+
   // This only handles icmp of constants: (icmp1 A, C1) & (icmp2 B, C2).
   Value *LHS0 = LHS->getOperand(0), *RHS0 = RHS->getOperand(0);
   ConstantInt *LHSC = dyn_cast<ConstantInt>(LHS->getOperand(1));
@@ -1858,6 +1959,20 @@ Instruction *InstCombiner::visitAnd(BinaryOperator &I) {
       A->getType()->isIntOrIntVectorTy(1))
     return SelectInst::Create(A, Op0, Constant::getNullValue(I.getType()));
 
+  // and(ashr(subNSW(Y, X), ScalarSizeInBits(Y)-1), X) --> X s> Y ? X : 0.
+  {
+    Value *X, *Y;
+    const APInt *ShAmt;
+    Type *Ty = I.getType();
+    if (match(&I, m_c_And(m_OneUse(m_AShr(m_NSWSub(m_Value(Y), m_Value(X)),
+                                          m_APInt(ShAmt))),
+                          m_Deferred(X))) &&
+        *ShAmt == Ty->getScalarSizeInBits() - 1) {
+      Value *NewICmpInst = Builder.CreateICmpSGT(X, Y);
+      return SelectInst::Create(NewICmpInst, X, ConstantInt::getNullValue(Ty));
+    }
+  }
+
   return nullptr;
 }
 
@@ -2071,6 +2186,8 @@ Value *InstCombiner::matchSelectFromAndOr(Value *A, Value *C, Value *B,
 /// Fold (icmp)|(icmp) if possible.
 Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
                                    Instruction &CxtI) {
+  const SimplifyQuery Q = SQ.getWithInstruction(&CxtI);
+
   // Fold (iszero(A & K1) | iszero(A & K2)) ->  (A & (K1 | K2)) != (K1 | K2)
   // if K1 and K2 are a one-bit mask.
   if (Value *V = foldAndOrOfICmpsOfAndWithPow2(LHS, RHS, false, CxtI))
@@ -2196,6 +2313,13 @@ Value *InstCombiner::foldOrOfICmps(ICmpInst *LHS, ICmpInst *RHS,
   if (Value *V = foldIsPowerOf2(LHS, RHS, false /* JoinedByAnd */, Builder))
     return V;
 
+  if (Value *X =
+          foldUnsignedUnderflowCheck(LHS, RHS, /*IsAnd=*/false, Q, Builder))
+    return X;
+  if (Value *X =
+          foldUnsignedUnderflowCheck(RHS, LHS, /*IsAnd=*/false, Q, Builder))
+    return X;
+
   // This only handles icmp of constants: (icmp1 A, C1) | (icmp2 B, C2).
   if (!LHSC || !RHSC)
     return nullptr;
@@ -2586,6 +2710,21 @@ Instruction *InstCombiner::visitOr(BinaryOperator &I) {
     }
   }
 
+  // or(ashr(subNSW(Y, X), ScalarSizeInBits(Y)-1), X)  --> X s> Y ? -1 : X.
+  {
+    Value *X, *Y;
+    const APInt *ShAmt;
+    Type *Ty = I.getType();
+    if (match(&I, m_c_Or(m_OneUse(m_AShr(m_NSWSub(m_Value(Y), m_Value(X)),
+                                         m_APInt(ShAmt))),
+                         m_Deferred(X))) &&
+        *ShAmt == Ty->getScalarSizeInBits() - 1) {
+      Value *NewICmpInst = Builder.CreateICmpSGT(X, Y);
+      return SelectInst::Create(NewICmpInst, ConstantInt::getAllOnesValue(Ty),
+                                X);
+    }
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
index 5f37a00f56cf5..2cfd3f5bb17f6 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp
@@ -154,6 +154,6 @@ Instruction *InstCombiner::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
   
   LoadInst *Load = new LoadInst(RMWI.getType(), RMWI.getPointerOperand());
   Load->setAtomic(Ordering, RMWI.getSyncScopeID());
-  Load->setAlignment(DL.getABITypeAlignment(RMWI.getType()));
+  Load->setAlignment(MaybeAlign(DL.getABITypeAlignment(RMWI.getType())));
   return Load;
 }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index e96e45e802ac1..bc458ebf65212 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -185,7 +185,7 @@ Instruction *InstCombiner::SimplifyAnyMemTransfer(AnyMemTransferInst *MI) {
   Value *Dest = Builder.CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
   LoadInst *L = Builder.CreateLoad(IntType, Src);
   // Alignment from the mem intrinsic will be better, so use it.
-  L->setAlignment(CopySrcAlign);
+  L->setAlignment(MaybeAlign(CopySrcAlign));
   if (CopyMD)
     L->setMetadata(LLVMContext::MD_tbaa, CopyMD);
   MDNode *LoopMemParallelMD =
@@ -2234,6 +2234,15 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       return replaceInstUsesWith(*II, Add);
     }
 
+    // Try to simplify the underlying FMul.
+    if (Value *V = SimplifyFMulInst(II->getArgOperand(0), II->getArgOperand(1),
+                                    II->getFastMathFlags(),
+                                    SQ.getWithInstruction(II))) {
+      auto *FAdd = BinaryOperator::CreateFAdd(V, II->getArgOperand(2));
+      FAdd->copyFastMathFlags(II);
+      return FAdd;
+    }
+
     LLVM_FALLTHROUGH;
   }
   case Intrinsic::fma: {
@@ -2258,9 +2267,12 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
       return II;
     }
 
-    // fma x, 1, z -> fadd x, z
-    if (match(Src1, m_FPOne())) {
-      auto *FAdd = BinaryOperator::CreateFAdd(Src0, II->getArgOperand(2));
+    // Try to simplify the underlying FMul. We can only apply simplifications
+    // that do not require rounding.
+    if (Value *V = SimplifyFMAFMul(II->getArgOperand(0), II->getArgOperand(1),
+                                   II->getFastMathFlags(),
+                                   SQ.getWithInstruction(II))) {
+      auto *FAdd = BinaryOperator::CreateFAdd(V, II->getArgOperand(2));
       FAdd->copyFastMathFlags(II);
       return FAdd;
     }
@@ -3951,10 +3963,21 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
     break;
   }
   case Intrinsic::experimental_gc_relocate: {
+    auto &GCR = *cast<GCRelocateInst>(II);
+
+    // If we have two copies of the same pointer in the statepoint argument
+    // list, canonicalize to one.  This may let us common gc.relocates.
+    if (GCR.getBasePtr() == GCR.getDerivedPtr() &&
+        GCR.getBasePtrIndex() != GCR.getDerivedPtrIndex()) {
+      auto *OpIntTy = GCR.getOperand(2)->getType();
+      II->setOperand(2, ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));
+      return II;
+    }
+    
     // Translate facts known about a pointer before relocating into
     // facts about the relocate value, while being careful to
     // preserve relocation semantics.
-    Value *DerivedPtr = cast<GCRelocateInst>(II)->getDerivedPtr();
+    Value *DerivedPtr = GCR.getDerivedPtr();
 
     // Remove the relocation if unused, note that this check is required
     // to prevent the cases below from looping forever.
@@ -4179,10 +4202,10 @@ static IntrinsicInst *findInitTrampoline(Value *Callee) {
 }
 
 static void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
+  unsigned NumArgs = Call.getNumArgOperands();
   ConstantInt *Op0C = dyn_cast<ConstantInt>(Call.getOperand(0));
-  ConstantInt *Op1C = (Call.getNumArgOperands() == 1)
-                          ? nullptr
-                          : dyn_cast<ConstantInt>(Call.getOperand(1));
+  ConstantInt *Op1C =
+      (NumArgs == 1) ? nullptr : dyn_cast<ConstantInt>(Call.getOperand(1));
   // Bail out if the allocation size is zero.
   if ((Op0C && Op0C->isNullValue()) || (Op1C && Op1C->isNullValue()))
     return;
@@ -4208,6 +4231,21 @@ static void annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
       Call.addAttribute(AttributeList::ReturnIndex,
                         Attribute::getWithDereferenceableOrNullBytes(
                             Call.getContext(), Size.getZExtValue()));
+  } else if (isStrdupLikeFn(&Call, TLI)) {
+    uint64_t Len = GetStringLength(Call.getOperand(0));
+    if (Len) {
+      // strdup
+      if (NumArgs == 1)
+        Call.addAttribute(AttributeList::ReturnIndex,
+                          Attribute::getWithDereferenceableOrNullBytes(
+                              Call.getContext(), Len));
+      // strndup
+      else if (NumArgs == 2 && Op1C)
+        Call.addAttribute(
+            AttributeList::ReturnIndex,
+            Attribute::getWithDereferenceableOrNullBytes(
+                Call.getContext(), std::min(Len, Op1C->getZExtValue() + 1)));
+    }
   }
 }
 
@@ -4216,9 +4254,6 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) {
   if (isAllocationFn(&Call, &TLI))
     annotateAnyAllocSite(Call, &TLI);
 
-  if (isAllocLikeFn(&Call, &TLI))
-    return visitAllocSite(Call);
-
   bool Changed = false;
 
   // Mark any parameters that are known to be non-null with the nonnull
@@ -4349,6 +4384,9 @@ Instruction *InstCombiner::visitCallBase(CallBase &Call) {
     if (I) return eraseInstFromFunction(*I);
   }
 
+  if (isAllocLikeFn(&Call, &TLI))
+    return visitAllocSite(Call);
+
   return Changed ? &Call : nullptr;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
index c9823fa28f4c2..74c69808f1585 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -140,7 +140,7 @@ Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
   }
 
   AllocaInst *New = AllocaBuilder.CreateAlloca(CastElTy, Amt);
-  New->setAlignment(AI.getAlignment());
+  New->setAlignment(MaybeAlign(AI.getAlignment()));
   New->takeName(&AI);
   New->setUsedWithInAlloca(AI.isUsedWithInAlloca());
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index e23e85bb16ad8..ddc7de39d8d2a 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -3638,7 +3638,8 @@ Value *InstCombiner::foldUnsignedMultiplicationOverflowCheck(ICmpInst &I) {
 /// TODO: A large part of this logic is duplicated in InstSimplify's
 /// simplifyICmpWithBinOp(). We should be able to share that and avoid the code
 /// duplication.
-Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
+Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I, const SimplifyQuery &SQ) {
+  const SimplifyQuery Q = SQ.getWithInstruction(&I);
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
 
   // Special logic for binary operators.
@@ -3684,21 +3685,21 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
     D = BO1->getOperand(1);
   }
 
-  // icmp (X+Y), X -> icmp Y, 0 for equalities or if there is no overflow.
+  // icmp (A+B), A -> icmp B, 0 for equalities or if there is no overflow.
+  // icmp (A+B), B -> icmp A, 0 for equalities or if there is no overflow.
   if ((A == Op1 || B == Op1) && NoOp0WrapProblem)
     return new ICmpInst(Pred, A == Op1 ? B : A,
                         Constant::getNullValue(Op1->getType()));
 
-  // icmp X, (X+Y) -> icmp 0, Y for equalities or if there is no overflow.
+  // icmp C, (C+D) -> icmp 0, D for equalities or if there is no overflow.
+  // icmp D, (C+D) -> icmp 0, C for equalities or if there is no overflow.
   if ((C == Op0 || D == Op0) && NoOp1WrapProblem)
     return new ICmpInst(Pred, Constant::getNullValue(Op0->getType()),
                         C == Op0 ? D : C);
 
-  // icmp (X+Y), (X+Z) -> icmp Y, Z for equalities or if there is no overflow.
+  // icmp (A+B), (A+D) -> icmp B, D for equalities or if there is no overflow.
   if (A && C && (A == C || A == D || B == C || B == D) && NoOp0WrapProblem &&
-      NoOp1WrapProblem &&
-      // Try not to increase register pressure.
-      BO0->hasOneUse() && BO1->hasOneUse()) {
+      NoOp1WrapProblem) {
     // Determine Y and Z in the form icmp (X+Y), (X+Z).
     Value *Y, *Z;
     if (A == C) {
@@ -3722,39 +3723,39 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
     return new ICmpInst(Pred, Y, Z);
   }
 
-  // icmp slt (X + -1), Y -> icmp sle X, Y
+  // icmp slt (A + -1), Op1 -> icmp sle A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLT &&
       match(B, m_AllOnes()))
     return new ICmpInst(CmpInst::ICMP_SLE, A, Op1);
 
-  // icmp sge (X + -1), Y -> icmp sgt X, Y
+  // icmp sge (A + -1), Op1 -> icmp sgt A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGE &&
       match(B, m_AllOnes()))
     return new ICmpInst(CmpInst::ICMP_SGT, A, Op1);
 
-  // icmp sle (X + 1), Y -> icmp slt X, Y
+  // icmp sle (A + 1), Op1 -> icmp slt A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SLE && match(B, m_One()))
     return new ICmpInst(CmpInst::ICMP_SLT, A, Op1);
 
-  // icmp sgt (X + 1), Y -> icmp sge X, Y
+  // icmp sgt (A + 1), Op1 -> icmp sge A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_SGT && match(B, m_One()))
     return new ICmpInst(CmpInst::ICMP_SGE, A, Op1);
 
-  // icmp sgt X, (Y + -1) -> icmp sge X, Y
+  // icmp sgt Op0, (C + -1) -> icmp sge Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGT &&
       match(D, m_AllOnes()))
     return new ICmpInst(CmpInst::ICMP_SGE, Op0, C);
 
-  // icmp sle X, (Y + -1) -> icmp slt X, Y
+  // icmp sle Op0, (C + -1) -> icmp slt Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLE &&
       match(D, m_AllOnes()))
     return new ICmpInst(CmpInst::ICMP_SLT, Op0, C);
 
-  // icmp sge X, (Y + 1) -> icmp sgt X, Y
+  // icmp sge Op0, (C + 1) -> icmp sgt Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SGE && match(D, m_One()))
     return new ICmpInst(CmpInst::ICMP_SGT, Op0, C);
 
-  // icmp slt X, (Y + 1) -> icmp sle X, Y
+  // icmp slt Op0, (C + 1) -> icmp sle Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_SLT && match(D, m_One()))
     return new ICmpInst(CmpInst::ICMP_SLE, Op0, C);
 
@@ -3762,33 +3763,33 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
   // canonicalization from (X -nuw 1) to (X + -1) means that the combinations
   // wouldn't happen even if they were implemented.
   //
-  // icmp ult (X - 1), Y -> icmp ule X, Y
-  // icmp uge (X - 1), Y -> icmp ugt X, Y
-  // icmp ugt X, (Y - 1) -> icmp uge X, Y
-  // icmp ule X, (Y - 1) -> icmp ult X, Y
+  // icmp ult (A - 1), Op1 -> icmp ule A, Op1
+  // icmp uge (A - 1), Op1 -> icmp ugt A, Op1
+  // icmp ugt Op0, (C - 1) -> icmp uge Op0, C
+  // icmp ule Op0, (C - 1) -> icmp ult Op0, C
 
-  // icmp ule (X + 1), Y -> icmp ult X, Y
+  // icmp ule (A + 1), Op0 -> icmp ult A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_ULE && match(B, m_One()))
     return new ICmpInst(CmpInst::ICMP_ULT, A, Op1);
 
-  // icmp ugt (X + 1), Y -> icmp uge X, Y
+  // icmp ugt (A + 1), Op0 -> icmp uge A, Op1
   if (A && NoOp0WrapProblem && Pred == CmpInst::ICMP_UGT && match(B, m_One()))
     return new ICmpInst(CmpInst::ICMP_UGE, A, Op1);
 
-  // icmp uge X, (Y + 1) -> icmp ugt X, Y
+  // icmp uge Op0, (C + 1) -> icmp ugt Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_UGE && match(D, m_One()))
     return new ICmpInst(CmpInst::ICMP_UGT, Op0, C);
 
-  // icmp ult X, (Y + 1) -> icmp ule X, Y
+  // icmp ult Op0, (C + 1) -> icmp ule Op0, C
   if (C && NoOp1WrapProblem && Pred == CmpInst::ICMP_ULT && match(D, m_One()))
     return new ICmpInst(CmpInst::ICMP_ULE, Op0, C);
 
   // if C1 has greater magnitude than C2:
-  //  icmp (X + C1), (Y + C2) -> icmp (X + C3), Y
+  //  icmp (A + C1), (C + C2) -> icmp (A + C3), C
   //  s.t. C3 = C1 - C2
   //
   // if C2 has greater magnitude than C1:
-  //  icmp (X + C1), (Y + C2) -> icmp X, (Y + C3)
+  //  icmp (A + C1), (C + C2) -> icmp A, (C + C3)
   //  s.t. C3 = C2 - C1
   if (A && C && NoOp0WrapProblem && NoOp1WrapProblem &&
       (BO0->hasOneUse() || BO1->hasOneUse()) && !I.isUnsigned())
@@ -3826,10 +3827,10 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
     D = BO1->getOperand(1);
   }
 
-  // icmp (X-Y), X -> icmp 0, Y for equalities or if there is no overflow.
+  // icmp (A-B), A -> icmp 0, B for equalities or if there is no overflow.
   if (A == Op1 && NoOp0WrapProblem)
     return new ICmpInst(Pred, Constant::getNullValue(Op1->getType()), B);
-  // icmp X, (X-Y) -> icmp Y, 0 for equalities or if there is no overflow.
+  // icmp C, (C-D) -> icmp D, 0 for equalities or if there is no overflow.
   if (C == Op0 && NoOp1WrapProblem)
     return new ICmpInst(Pred, D, Constant::getNullValue(Op0->getType()));
 
@@ -3840,16 +3841,21 @@ Instruction *InstCombiner::foldICmpBinOp(ICmpInst &I) {
   // C u</u>= (C - D) --> C u</u>= D
   if (C == Op0 && (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))
     return new ICmpInst(Pred, C, D);
-
-  // icmp (Y-X), (Z-X) -> icmp Y, Z for equalities or if there is no overflow.
-  if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem &&
-      // Try not to increase register pressure.
-      BO0->hasOneUse() && BO1->hasOneUse())
+  // (A - B) u>=/u< A --> B u>/u<= A  iff B != 0
+  if (A == Op1 && (Pred == ICmpInst::ICMP_UGE || Pred == ICmpInst::ICMP_ULT) &&
+      isKnownNonZero(B, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+    return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), B, A);
+  // C u<=/u> (C - D) --> C u</u>= D  iff B != 0
+  if (C == Op0 && (Pred == ICmpInst::ICMP_ULE || Pred == ICmpInst::ICMP_UGT) &&
+      isKnownNonZero(D, Q.DL, /*Depth=*/0, Q.AC, Q.CxtI, Q.DT))
+    return new ICmpInst(CmpInst::getFlippedStrictnessPredicate(Pred), C, D);
+
+  // icmp (A-B), (C-B) -> icmp A, C for equalities or if there is no overflow.
+  if (B && D && B == D && NoOp0WrapProblem && NoOp1WrapProblem)
     return new ICmpInst(Pred, A, C);
-  // icmp (X-Y), (X-Z) -> icmp Z, Y for equalities or if there is no overflow.
-  if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem &&
-      // Try not to increase register pressure.
-      BO0->hasOneUse() && BO1->hasOneUse())
+
+  // icmp (A-B), (A-D) -> icmp D, B for equalities or if there is no overflow.
+  if (A && C && A == C && NoOp0WrapProblem && NoOp1WrapProblem)
     return new ICmpInst(Pred, D, B);
 
   // icmp (0-X) < cst --> x > -cst
@@ -5135,16 +5141,11 @@ llvm::getFlippedStrictnessPredicateAndConstant(CmpInst::Predicate Pred,
     return WillIncrement ? !C->isMaxValue(IsSigned) : !C->isMinValue(IsSigned);
   };
 
-  // For scalars, SimplifyICmpInst should have already handled
-  // the edge cases for us, so we just assert on them.
-  // For vectors, we must handle the edge cases.
-  if (isa<ConstantInt>(C)) {
-    // A <= MAX -> TRUE ; A >= MIN -> TRUE
-    assert(ConstantIsOk(cast<ConstantInt>(C)));
+  if (auto *CI = dyn_cast<ConstantInt>(C)) {
+    // Bail out if the constant can't be safely incremented/decremented.
+    if (!ConstantIsOk(CI))
+      return llvm::None;
   } else if (Type->isVectorTy()) {
-    // TODO? If the edge cases for vectors were guaranteed to be handled as they
-    // are for scalar, we could remove the min/max checks. However, to do that,
-    // we would have to use insertelement/shufflevector to replace edge values.
     unsigned NumElts = Type->getVectorNumElements();
     for (unsigned i = 0; i != NumElts; ++i) {
       Constant *Elt = C->getAggregateElement(i);
@@ -5349,6 +5350,7 @@ static Instruction *foldVectorCmp(CmpInst &Cmp,
 
 Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   bool Changed = false;
+  const SimplifyQuery Q = SQ.getWithInstruction(&I);
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   unsigned Op0Cplxity = getComplexity(Op0);
   unsigned Op1Cplxity = getComplexity(Op1);
@@ -5363,8 +5365,7 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
     Changed = true;
   }
 
-  if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1,
-                                  SQ.getWithInstruction(&I)))
+  if (Value *V = SimplifyICmpInst(I.getPredicate(), Op0, Op1, Q))
     return replaceInstUsesWith(I, V);
 
   // Comparing -val or val with non-zero is the same as just comparing val
@@ -5397,6 +5398,9 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   if (Instruction *Res = foldICmpWithDominatingICmp(I))
     return Res;
 
+  if (Instruction *Res = foldICmpBinOp(I, Q))
+    return Res;
+
   if (Instruction *Res = foldICmpUsingKnownBits(I))
     return Res;
 
@@ -5474,9 +5478,6 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
   if (Instruction *R = foldICmpWithCastOp(I))
     return R;
 
-  if (Instruction *Res = foldICmpBinOp(I))
-    return Res;
-
   if (Instruction *Res = foldICmpWithMinMax(I))
     return Res;
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index 5e4a56ba25782..dcdbee15fe56b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -351,6 +351,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Instruction *visitOr(BinaryOperator &I);
   Instruction *visitXor(BinaryOperator &I);
   Instruction *visitShl(BinaryOperator &I);
+  Instruction *foldVariableSignZeroExtensionOfVariableHighBitExtract(
+      BinaryOperator &OldAShr);
   Instruction *visitAShr(BinaryOperator &I);
   Instruction *visitLShr(BinaryOperator &I);
   Instruction *commonShiftTransforms(BinaryOperator &I);
@@ -868,7 +870,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner
   Instruction *foldICmpWithConstant(ICmpInst &Cmp);
   Instruction *foldICmpInstWithConstant(ICmpInst &Cmp);
   Instruction *foldICmpInstWithConstantNotInt(ICmpInst &Cmp);
-  Instruction *foldICmpBinOp(ICmpInst &Cmp);
+  Instruction *foldICmpBinOp(ICmpInst &Cmp, const SimplifyQuery &SQ);
   Instruction *foldICmpEquality(ICmpInst &Cmp);
   Instruction *foldIRemByPowerOfTwoToBitTest(ICmpInst &I);
   Instruction *foldICmpWithZero(ICmpInst &Cmp);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 79c2453e99f8e..eb01b4b7d7d15 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -197,7 +197,7 @@ static Instruction *simplifyAllocaArraySize(InstCombiner &IC, AllocaInst &AI) {
     if (C->getValue().getActiveBits() <= 64) {
       Type *NewTy = ArrayType::get(AI.getAllocatedType(), C->getZExtValue());
       AllocaInst *New = IC.Builder.CreateAlloca(NewTy, nullptr, AI.getName());
-      New->setAlignment(AI.getAlignment());
+      New->setAlignment(MaybeAlign(AI.getAlignment()));
 
       // Scan to the end of the allocation instructions, to skip over a block of
       // allocas if possible...also skip interleaved debug info
@@ -345,7 +345,8 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
   if (AI.getAllocatedType()->isSized()) {
     // If the alignment is 0 (unspecified), assign it the preferred alignment.
     if (AI.getAlignment() == 0)
-      AI.setAlignment(DL.getPrefTypeAlignment(AI.getAllocatedType()));
+      AI.setAlignment(
+          MaybeAlign(DL.getPrefTypeAlignment(AI.getAllocatedType())));
 
     // Move all alloca's of zero byte objects to the entry block and merge them
     // together.  Note that we only do this for alloca's, because malloc should
@@ -377,12 +378,12 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
         // assign it the preferred alignment.
         if (EntryAI->getAlignment() == 0)
           EntryAI->setAlignment(
-              DL.getPrefTypeAlignment(EntryAI->getAllocatedType()));
+              MaybeAlign(DL.getPrefTypeAlignment(EntryAI->getAllocatedType())));
         // Replace this zero-sized alloca with the one at the start of the entry
         // block after ensuring that the address will be aligned enough for both
         // types.
-        unsigned MaxAlign = std::max(EntryAI->getAlignment(),
-                                     AI.getAlignment());
+        const MaybeAlign MaxAlign(
+            std::max(EntryAI->getAlignment(), AI.getAlignment()));
         EntryAI->setAlignment(MaxAlign);
         if (AI.getType() != EntryAI->getType())
           return new BitCastInst(EntryAI, AI.getType());
@@ -960,9 +961,9 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
       LoadAlign != 0 ? LoadAlign : DL.getABITypeAlignment(LI.getType());
 
   if (KnownAlign > EffectiveLoadAlign)
-    LI.setAlignment(KnownAlign);
+    LI.setAlignment(MaybeAlign(KnownAlign));
   else if (LoadAlign == 0)
-    LI.setAlignment(EffectiveLoadAlign);
+    LI.setAlignment(MaybeAlign(EffectiveLoadAlign));
 
   // Replace GEP indices if possible.
   if (Instruction *NewGEPI = replaceGEPIdxWithZero(*this, Op, LI)) {
@@ -1031,9 +1032,9 @@ Instruction *InstCombiner::visitLoadInst(LoadInst &LI) {
             Builder.CreateLoad(LI.getType(), SI->getOperand(2),
                                SI->getOperand(2)->getName() + ".val");
         assert(LI.isUnordered() && "implied by above");
-        V1->setAlignment(Align);
+        V1->setAlignment(MaybeAlign(Align));
         V1->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
-        V2->setAlignment(Align);
+        V2->setAlignment(MaybeAlign(Align));
         V2->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
         return SelectInst::Create(SI->getCondition(), V1, V2);
       }
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 7f152aac52db5..a301d9eef60e8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -213,6 +213,25 @@ Instruction *InstCombiner::visitMul(BinaryOperator &I) {
   if (Instruction *FoldedMul = foldBinOpIntoSelectOrPhi(I))
     return FoldedMul;
 
+  // TODO: This is a specific form of a much more general pattern.
+  //       We could detect a select with any binop identity constant, or we
+  //       could use SimplifyBinOp to see if either arm of the select reduces.
+  //       But that needs to be done carefully and/or while removing potential
+  //       reverse canonicalizations as in InstCombiner::foldSelectIntoOp().
+  // mul (select Cond, 1, -1), Op1 --> select Cond, Op1, -Op1
+  // mul (select Cond, -1, 1), Op1 --> select Cond, -Op1, Op1
+  // mul Op0, (select Cond, 1, -1) --> select Cond, Op0, -Op0
+  // mul Op0, (select Cond, -1, 1) --> select Cond, -Op0, Op0
+  Value *Cond;
+  if (match(Op0, m_OneUse(m_Select(m_Value(Cond), m_One(), m_AllOnes()))))
+    return SelectInst::Create(Cond, Op1, Builder.CreateNeg(Op1));
+  if (match(Op0, m_OneUse(m_Select(m_Value(Cond), m_AllOnes(), m_One()))))
+    return SelectInst::Create(Cond, Builder.CreateNeg(Op1), Op1);
+  if (match(Op1, m_OneUse(m_Select(m_Value(Cond), m_One(), m_AllOnes()))))
+    return SelectInst::Create(Cond, Op0, Builder.CreateNeg(Op0));
+  if (match(Op1, m_OneUse(m_Select(m_Value(Cond), m_AllOnes(), m_One()))))
+    return SelectInst::Create(Cond, Builder.CreateNeg(Op0), Op0);
+
   // Simplify mul instructions with a constant RHS.
   if (isa<Constant>(Op1)) {
     // Canonicalize (X+C1)*CI -> X*CI+C1*CI.
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 6ccfac28d0523..55ac820213076 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -1126,6 +1126,7 @@ static Value *simplifyWithOpReplaced(Value *V, Value *Op, Value *ReplaceOp,
 ///   %sel = select i1 %cmp, i32 -2147483648, i32 %add
 ///
 /// We can't replace %sel with %add unless we strip away the flags.
+/// TODO: Wrapping flags could be preserved in some cases with better analysis.
 static Value *foldSelectValueEquivalence(SelectInst &Sel, ICmpInst &Cmp,
                                          const SimplifyQuery &Q) {
   if (!Cmp.isEquality())
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
index b451beebd0f35..9d96ddc4040d8 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -131,7 +131,8 @@ reassociateShiftAmtsOfTwoSameDirectionShifts(BinaryOperator *Sh0,
 //   c,d,e,f) (ShiftShAmt-MaskShAmt) s>= 0 (i.e. ShiftShAmt u>= MaskShAmt)
 static Instruction *
 dropRedundantMaskingOfLeftShiftInput(BinaryOperator *OuterShift,
-                                     const SimplifyQuery &SQ) {
+                                     const SimplifyQuery &Q,
+                                     InstCombiner::BuilderTy &Builder) {
   assert(OuterShift->getOpcode() == Instruction::BinaryOps::Shl &&
          "The input must be 'shl'!");
 
@@ -151,46 +152,100 @@ dropRedundantMaskingOfLeftShiftInput(BinaryOperator *OuterShift,
       m_Shr(m_Shl(m_AllOnes(), m_Value(MaskShAmt)), m_Deferred(MaskShAmt));
 
   Value *X;
+  Constant *NewMask;
   if (match(Masked, m_c_And(m_CombineOr(MaskA, MaskB), m_Value(X)))) {
     // Can we simplify (MaskShAmt+ShiftShAmt) ?
-    Value *SumOfShAmts =
-        SimplifyAddInst(MaskShAmt, ShiftShAmt, /*IsNSW=*/false, /*IsNUW=*/false,
-                        SQ.getWithInstruction(OuterShift));
+    auto *SumOfShAmts = dyn_cast_or_null<Constant>(SimplifyAddInst(
+        MaskShAmt, ShiftShAmt, /*IsNSW=*/false, /*IsNUW=*/false, Q));
     if (!SumOfShAmts)
       return nullptr; // Did not simplify.
-    // Is the total shift amount *not* smaller than the bit width?
-    // FIXME: could also rely on ConstantRange.
-    unsigned BitWidth = X->getType()->getScalarSizeInBits();
+    Type *Ty = X->getType();
+    unsigned BitWidth = Ty->getScalarSizeInBits();
+    // In this pattern SumOfShAmts correlates with the number of low bits that
+    // shall remain in the root value (OuterShift). If SumOfShAmts is less than
+    // bitwidth, we'll need to also produce a mask to keep SumOfShAmts low bits.
+    // So, does *any* channel need a mask?
     if (!match(SumOfShAmts, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_UGE,
-                                               APInt(BitWidth, BitWidth))))
-      return nullptr;
+                                               APInt(BitWidth, BitWidth)))) {
+      // But for a mask we need to get rid of old masking instruction.
+      if (!Masked->hasOneUse())
+        return nullptr; // Else we can't perform the fold.
+      // The mask must be computed in a type twice as wide to ensure
+      // that no bits are lost if the sum-of-shifts is wider than the base type.
+      Type *ExtendedTy = Ty->getExtendedType();
+      auto *ExtendedSumOfShAmts =
+          ConstantExpr::getZExt(SumOfShAmts, ExtendedTy);
+      // And compute the mask as usual: ~(-1 << (SumOfShAmts))
+      auto *ExtendedAllOnes = ConstantExpr::getAllOnesValue(ExtendedTy);
+      auto *ExtendedInvertedMask =
+          ConstantExpr::getShl(ExtendedAllOnes, ExtendedSumOfShAmts);
+      auto *ExtendedMask = ConstantExpr::getNot(ExtendedInvertedMask);
+      NewMask = ConstantExpr::getTrunc(ExtendedMask, Ty);
+    } else
+      NewMask = nullptr; // No mask needed.
     // All good, we can do this fold.
   } else if (match(Masked, m_c_And(m_CombineOr(MaskC, MaskD), m_Value(X))) ||
              match(Masked, m_Shr(m_Shl(m_Value(X), m_Value(MaskShAmt)),
                                  m_Deferred(MaskShAmt)))) {
     // Can we simplify (ShiftShAmt-MaskShAmt) ?
-    Value *ShAmtsDiff =
-        SimplifySubInst(ShiftShAmt, MaskShAmt, /*IsNSW=*/false, /*IsNUW=*/false,
-                        SQ.getWithInstruction(OuterShift));
+    auto *ShAmtsDiff = dyn_cast_or_null<Constant>(SimplifySubInst(
+        ShiftShAmt, MaskShAmt, /*IsNSW=*/false, /*IsNUW=*/false, Q));
     if (!ShAmtsDiff)
       return nullptr; // Did not simplify.
-    // Is the difference non-negative? (is ShiftShAmt u>= MaskShAmt ?)
-    // FIXME: could also rely on ConstantRange.
-    if (!match(ShAmtsDiff, m_NonNegative()))
-      return nullptr;
+    // In this pattern ShAmtsDiff correlates with the number of high bits that
+    // shall be unset in the root value (OuterShift). If ShAmtsDiff is negative,
+    // we'll need to also produce a mask to unset ShAmtsDiff high bits.
+    // So, does *any* channel need a mask? (is ShiftShAmt u>= MaskShAmt ?)
+    if (!match(ShAmtsDiff, m_NonNegative())) {
+      // This sub-fold (with mask) is invalid for 'ashr' "masking" instruction.
+      if (match(Masked, m_AShr(m_Value(), m_Value())))
+        return nullptr;
+      // For a mask we need to get rid of old masking instruction.
+      if (!Masked->hasOneUse())
+        return nullptr; // Else we can't perform the fold.
+      Type *Ty = X->getType();
+      unsigned BitWidth = Ty->getScalarSizeInBits();
+      // The mask must be computed in a type twice as wide to ensure
+      // that no bits are lost if the sum-of-shifts is wider than the base type.
+      Type *ExtendedTy = Ty->getExtendedType();
+      auto *ExtendedNumHighBitsToClear = ConstantExpr::getZExt(
+          ConstantExpr::getAdd(
+              ConstantExpr::getNeg(ShAmtsDiff),
+              ConstantInt::get(Ty, BitWidth, /*isSigned=*/false)),
+          ExtendedTy);
+      // And compute the mask as usual: (-1 l>> (ShAmtsDiff))
+      auto *ExtendedAllOnes = ConstantExpr::getAllOnesValue(ExtendedTy);
+      auto *ExtendedMask =
+          ConstantExpr::getLShr(ExtendedAllOnes, ExtendedNumHighBitsToClear);
+      NewMask = ConstantExpr::getTrunc(ExtendedMask, Ty);
+    } else
+      NewMask = nullptr; // No mask needed.
     // All good, we can do this fold.
   } else
     return nullptr; // Don't know anything about this pattern.
 
   // No 'NUW'/'NSW'!
   // We no longer know that we won't shift-out non-0 bits.
-  return BinaryOperator::Create(OuterShift->getOpcode(), X, ShiftShAmt);
+  auto *NewShift =
+      BinaryOperator::Create(OuterShift->getOpcode(), X, ShiftShAmt);
+  if (!NewMask)
+    return NewShift;
+
+  Builder.Insert(NewShift);
+  return BinaryOperator::Create(Instruction::And, NewShift, NewMask);
 }
 
 Instruction *InstCombiner::commonShiftTransforms(BinaryOperator &I) {
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
   assert(Op0->getType() == Op1->getType());
 
+  // If the shift amount is a one-use `sext`, we can demote it to `zext`.
+  Value *Y;
+  if (match(Op1, m_OneUse(m_SExt(m_Value(Y))))) {
+    Value *NewExt = Builder.CreateZExt(Y, I.getType(), Op1->getName());
+    return BinaryOperator::Create(I.getOpcode(), Op0, NewExt);
+  }
+
   // See if we can fold away this shift.
   if (SimplifyDemandedInstructionBits(I))
     return &I;
@@ -740,9 +795,10 @@ Instruction *InstCombiner::FoldShiftByConstant(Value *Op0, Constant *Op1,
 }
 
 Instruction *InstCombiner::visitShl(BinaryOperator &I) {
+  const SimplifyQuery Q = SQ.getWithInstruction(&I);
+
   if (Value *V = SimplifyShlInst(I.getOperand(0), I.getOperand(1),
-                                 I.hasNoSignedWrap(), I.hasNoUnsignedWrap(),
-                                 SQ.getWithInstruction(&I)))
+                                 I.hasNoSignedWrap(), I.hasNoUnsignedWrap(), Q))
     return replaceInstUsesWith(I, V);
 
   if (Instruction *X = foldVectorBinop(I))
@@ -751,7 +807,7 @@ Instruction *InstCombiner::visitShl(BinaryOperator &I) {
   if (Instruction *V = commonShiftTransforms(I))
     return V;
 
-  if (Instruction *V = dropRedundantMaskingOfLeftShiftInput(&I, SQ))
+  if (Instruction *V = dropRedundantMaskingOfLeftShiftInput(&I, Q, Builder))
     return V;
 
   Value *Op0 = I.getOperand(0), *Op1 = I.getOperand(1);
@@ -983,6 +1039,75 @@ Instruction *InstCombiner::visitLShr(BinaryOperator &I) {
   return nullptr;
 }
 
+Instruction *
+InstCombiner::foldVariableSignZeroExtensionOfVariableHighBitExtract(
+    BinaryOperator &OldAShr) {
+  assert(OldAShr.getOpcode() == Instruction::AShr &&
+         "Must be called with arithmetic right-shift instruction only.");
+
+  // Check that constant C is a splat of the element-wise bitwidth of V.
+  auto BitWidthSplat = [](Constant *C, Value *V) {
+    return match(
+        C, m_SpecificInt_ICMP(ICmpInst::Predicate::ICMP_EQ,
+                              APInt(C->getType()->getScalarSizeInBits(),
+                                    V->getType()->getScalarSizeInBits())));
+  };
+
+  // It should look like variable-length sign-extension on the outside:
+  //   (Val << (bitwidth(Val)-Nbits)) a>> (bitwidth(Val)-Nbits)
+  Value *NBits;
+  Instruction *MaybeTrunc;
+  Constant *C1, *C2;
+  if (!match(&OldAShr,
+             m_AShr(m_Shl(m_Instruction(MaybeTrunc),
+                          m_ZExtOrSelf(m_Sub(m_Constant(C1),
+                                             m_ZExtOrSelf(m_Value(NBits))))),
+                    m_ZExtOrSelf(m_Sub(m_Constant(C2),
+                                       m_ZExtOrSelf(m_Deferred(NBits)))))) ||
+      !BitWidthSplat(C1, &OldAShr) || !BitWidthSplat(C2, &OldAShr))
+    return nullptr;
+
+  // There may or may not be a truncation after outer two shifts.
+  Instruction *HighBitExtract;
+  match(MaybeTrunc, m_TruncOrSelf(m_Instruction(HighBitExtract)));
+  bool HadTrunc = MaybeTrunc != HighBitExtract;
+
+  // And finally, the innermost part of the pattern must be a right-shift.
+  Value *X, *NumLowBitsToSkip;
+  if (!match(HighBitExtract, m_Shr(m_Value(X), m_Value(NumLowBitsToSkip))))
+    return nullptr;
+
+  // Said right-shift must extract high NBits bits - C0 must be it's bitwidth.
+  Constant *C0;
+  if (!match(NumLowBitsToSkip,
+             m_ZExtOrSelf(
+                 m_Sub(m_Constant(C0), m_ZExtOrSelf(m_Specific(NBits))))) ||
+      !BitWidthSplat(C0, HighBitExtract))
+    return nullptr;
+
+  // Since the NBits is identical for all shifts, if the outermost and
+  // innermost shifts are identical, then outermost shifts are redundant.
+  // If we had truncation, do keep it though.
+  if (HighBitExtract->getOpcode() == OldAShr.getOpcode())
+    return replaceInstUsesWith(OldAShr, MaybeTrunc);
+
+  // Else, if there was a truncation, then we need to ensure that one
+  // instruction will go away.
+  if (HadTrunc && !match(&OldAShr, m_c_BinOp(m_OneUse(m_Value()), m_Value())))
+    return nullptr;
+
+  // Finally, bypass two innermost shifts, and perform the outermost shift on
+  // the operands of the innermost shift.
+  Instruction *NewAShr =
+      BinaryOperator::Create(OldAShr.getOpcode(), X, NumLowBitsToSkip);
+  NewAShr->copyIRFlags(HighBitExtract); // We can preserve 'exact'-ness.
+  if (!HadTrunc)
+    return NewAShr;
+
+  Builder.Insert(NewAShr);
+  return TruncInst::CreateTruncOrBitCast(NewAShr, OldAShr.getType());
+}
+
 Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
   if (Value *V = SimplifyAShrInst(I.getOperand(0), I.getOperand(1), I.isExact(),
                                   SQ.getWithInstruction(&I)))
@@ -1057,6 +1182,9 @@ Instruction *InstCombiner::visitAShr(BinaryOperator &I) {
     }
   }
 
+  if (Instruction *R = foldVariableSignZeroExtensionOfVariableHighBitExtract(I))
+    return R;
+
   // See if we can turn a signed shr into an unsigned shr.
   if (MaskedValueIsZero(Op0, APInt::getSignMask(BitWidth), 0, &I))
     return BinaryOperator::CreateLShr(Op0, Op1);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index 41cadb387e5c6..ecb486c544e03 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1667,7 +1667,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     // to an index of zero, so replace it with zero if it is not zero already.
     Type *EltTy = GTI.getIndexedType();
     if (EltTy->isSized() && DL.getTypeAllocSize(EltTy) == 0)
-      if (!isa<Constant>(*I) || !cast<Constant>(*I)->isNullValue()) {
+      if (!isa<Constant>(*I) || !match(I->get(), m_Zero())) {
         *I = Constant::getNullValue(NewIndexType);
         MadeChange = true;
       }
@@ -2557,9 +2557,7 @@ Instruction *InstCombiner::visitReturnInst(ReturnInst &RI) {
 Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
   // Change br (not X), label True, label False to: br X, label False, True
   Value *X = nullptr;
-  BasicBlock *TrueDest;
-  BasicBlock *FalseDest;
-  if (match(&BI, m_Br(m_Not(m_Value(X)), TrueDest, FalseDest)) &&
+  if (match(&BI, m_Br(m_Not(m_Value(X)), m_BasicBlock(), m_BasicBlock())) &&
       !isa<Constant>(X)) {
     // Swap Destinations and condition...
     BI.setCondition(X);
@@ -2577,8 +2575,8 @@ Instruction *InstCombiner::visitBranchInst(BranchInst &BI) {
 
   // Canonicalize, for example, icmp_ne -> icmp_eq or fcmp_one -> fcmp_oeq.
   CmpInst::Predicate Pred;
-  if (match(&BI, m_Br(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())), TrueDest,
-                      FalseDest)) &&
+  if (match(&BI, m_Br(m_OneUse(m_Cmp(Pred, m_Value(), m_Value())),
+                      m_BasicBlock(), m_BasicBlock())) &&
       !isCanonicalPredicate(Pred)) {
     // Swap destinations and condition.
     CmpInst *Cond = cast<CmpInst>(BI.getCondition());
diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 21280f2d6970f..91b6a91ab8191 100644
--- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -580,10 +580,10 @@ char ASanGlobalsMetadataWrapperPass::ID = 0;
 
 /// AddressSanitizer: instrument the code in module to find memory bugs.
 struct AddressSanitizer {
-  AddressSanitizer(Module &M, GlobalsMetadata &GlobalsMD,
+  AddressSanitizer(Module &M, const GlobalsMetadata *GlobalsMD,
                    bool CompileKernel = false, bool Recover = false,
                    bool UseAfterScope = false)
-      : UseAfterScope(UseAfterScope || ClUseAfterScope), GlobalsMD(GlobalsMD) {
+      : UseAfterScope(UseAfterScope || ClUseAfterScope), GlobalsMD(*GlobalsMD) {
     this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover;
     this->CompileKernel =
         ClEnableKasan.getNumOccurrences() > 0 ? ClEnableKasan : CompileKernel;
@@ -692,7 +692,7 @@ struct AddressSanitizer {
   FunctionCallee AsanMemmove, AsanMemcpy, AsanMemset;
   InlineAsm *EmptyAsm;
   Value *LocalDynamicShadow = nullptr;
-  GlobalsMetadata GlobalsMD;
+  const GlobalsMetadata &GlobalsMD;
   DenseMap<const AllocaInst *, bool> ProcessedAllocas;
 };
 
@@ -722,7 +722,7 @@ class AddressSanitizerLegacyPass : public FunctionPass {
         getAnalysis<ASanGlobalsMetadataWrapperPass>().getGlobalsMD();
     const TargetLibraryInfo *TLI =
         &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
-    AddressSanitizer ASan(*F.getParent(), GlobalsMD, CompileKernel, Recover,
+    AddressSanitizer ASan(*F.getParent(), &GlobalsMD, CompileKernel, Recover,
                           UseAfterScope);
     return ASan.instrumentFunction(F, TLI);
   }
@@ -735,10 +735,10 @@ class AddressSanitizerLegacyPass : public FunctionPass {
 
 class ModuleAddressSanitizer {
 public:
-  ModuleAddressSanitizer(Module &M, GlobalsMetadata &GlobalsMD,
+  ModuleAddressSanitizer(Module &M, const GlobalsMetadata *GlobalsMD,
                          bool CompileKernel = false, bool Recover = false,
                          bool UseGlobalsGC = true, bool UseOdrIndicator = false)
-      : GlobalsMD(GlobalsMD), UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC),
+      : GlobalsMD(*GlobalsMD), UseGlobalsGC(UseGlobalsGC && ClUseGlobalsGC),
         // Enable aliases as they should have no downside with ODR indicators.
         UsePrivateAlias(UseOdrIndicator || ClUsePrivateAlias),
         UseOdrIndicator(UseOdrIndicator || ClUseOdrIndicator),
@@ -798,7 +798,7 @@ class ModuleAddressSanitizer {
   }
   int GetAsanVersion(const Module &M) const;
 
-  GlobalsMetadata GlobalsMD;
+  const GlobalsMetadata &GlobalsMD;
   bool CompileKernel;
   bool Recover;
   bool UseGlobalsGC;
@@ -845,7 +845,7 @@ class ModuleAddressSanitizerLegacyPass : public ModulePass {
   bool runOnModule(Module &M) override {
     GlobalsMetadata &GlobalsMD =
         getAnalysis<ASanGlobalsMetadataWrapperPass>().getGlobalsMD();
-    ModuleAddressSanitizer ASanModule(M, GlobalsMD, CompileKernel, Recover,
+    ModuleAddressSanitizer ASanModule(M, &GlobalsMD, CompileKernel, Recover,
                                       UseGlobalGC, UseOdrIndicator);
     return ASanModule.instrumentModule(M);
   }
@@ -1048,7 +1048,7 @@ struct FunctionStackPoisoner : public InstVisitor<FunctionStackPoisoner> {
     if (!II.isLifetimeStartOrEnd())
       return;
     // Found lifetime intrinsic, add ASan instrumentation if necessary.
-    ConstantInt *Size = dyn_cast<ConstantInt>(II.getArgOperand(0));
+    auto *Size = cast<ConstantInt>(II.getArgOperand(0));
     // If size argument is undefined, don't do anything.
     if (Size->isMinusOne()) return;
     // Check that size doesn't saturate uint64_t and can
@@ -1171,7 +1171,7 @@ PreservedAnalyses AddressSanitizerPass::run(Function &F,
   Module &M = *F.getParent();
   if (auto *R = MAM.getCachedResult<ASanGlobalsMetadataAnalysis>(M)) {
     const TargetLibraryInfo *TLI = &AM.getResult<TargetLibraryAnalysis>(F);
-    AddressSanitizer Sanitizer(M, *R, CompileKernel, Recover, UseAfterScope);
+    AddressSanitizer Sanitizer(M, R, CompileKernel, Recover, UseAfterScope);
     if (Sanitizer.instrumentFunction(F, TLI))
       return PreservedAnalyses::none();
     return PreservedAnalyses::all();
@@ -1193,7 +1193,7 @@ ModuleAddressSanitizerPass::ModuleAddressSanitizerPass(bool CompileKernel,
 PreservedAnalyses ModuleAddressSanitizerPass::run(Module &M,
                                                   AnalysisManager<Module> &AM) {
   GlobalsMetadata &GlobalsMD = AM.getResult<ASanGlobalsMetadataAnalysis>(M);
-  ModuleAddressSanitizer Sanitizer(M, GlobalsMD, CompileKernel, Recover,
+  ModuleAddressSanitizer Sanitizer(M, &GlobalsMD, CompileKernel, Recover,
                                    UseGlobalGC, UseOdrIndicator);
   if (Sanitizer.instrumentModule(M))
     return PreservedAnalyses::none();
@@ -1790,7 +1790,7 @@ void ModuleAddressSanitizer::createInitializerPoisonCalls(
     // Must have a function or null ptr.
     if (Function *F = dyn_cast<Function>(CS->getOperand(1))) {
       if (F->getName() == kAsanModuleCtorName) continue;
-      ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+      auto *Priority = cast<ConstantInt>(CS->getOperand(0));
       // Don't instrument CTORs that will run before asan.module_ctor.
       if (Priority->getLimitedValue() <= GetCtorAndDtorPriority(TargetTriple))
         continue;
@@ -2899,18 +2899,19 @@ void FunctionStackPoisoner::copyArgsPassedByValToAllocas() {
   for (Argument &Arg : F.args()) {
     if (Arg.hasByValAttr()) {
       Type *Ty = Arg.getType()->getPointerElementType();
-      unsigned Align = Arg.getParamAlignment();
-      if (Align == 0) Align = DL.getABITypeAlignment(Ty);
+      unsigned Alignment = Arg.getParamAlignment();
+      if (Alignment == 0)
+        Alignment = DL.getABITypeAlignment(Ty);
 
       AllocaInst *AI = IRB.CreateAlloca(
           Ty, nullptr,
           (Arg.hasName() ? Arg.getName() : "Arg" + Twine(Arg.getArgNo())) +
               ".byval");
-      AI->setAlignment(Align);
+      AI->setAlignment(Align(Alignment));
       Arg.replaceAllUsesWith(AI);
 
       uint64_t AllocSize = DL.getTypeAllocSize(Ty);
-      IRB.CreateMemCpy(AI, Align, &Arg, Align, AllocSize);
+      IRB.CreateMemCpy(AI, Alignment, &Arg, Alignment, AllocSize);
     }
   }
 }
@@ -2941,7 +2942,7 @@ Value *FunctionStackPoisoner::createAllocaForLayout(
   }
   assert((ClRealignStack & (ClRealignStack - 1)) == 0);
   size_t FrameAlignment = std::max(L.FrameAlignment, (size_t)ClRealignStack);
-  Alloca->setAlignment(FrameAlignment);
+  Alloca->setAlignment(MaybeAlign(FrameAlignment));
   return IRB.CreatePointerCast(Alloca, IntptrTy);
 }
 
@@ -2950,7 +2951,7 @@ void FunctionStackPoisoner::createDynamicAllocasInitStorage() {
   IRBuilder<> IRB(dyn_cast<Instruction>(FirstBB.begin()));
   DynamicAllocaLayout = IRB.CreateAlloca(IntptrTy, nullptr);
   IRB.CreateStore(Constant::getNullValue(IntptrTy), DynamicAllocaLayout);
-  DynamicAllocaLayout->setAlignment(32);
+  DynamicAllocaLayout->setAlignment(Align(32));
 }
 
 void FunctionStackPoisoner::processDynamicAllocas() {
@@ -3297,7 +3298,7 @@ void FunctionStackPoisoner::handleDynamicAllocaCall(AllocaInst *AI) {
 
   // Insert new alloca with new NewSize and Align params.
   AllocaInst *NewAlloca = IRB.CreateAlloca(IRB.getInt8Ty(), NewSize);
-  NewAlloca->setAlignment(Align);
+  NewAlloca->setAlignment(MaybeAlign(Align));
 
   // NewAddress = Address + Align
   Value *NewAddress = IRB.CreateAdd(IRB.CreatePtrToInt(NewAlloca, IntptrTy),
diff --git a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
index 2279c1bcb6a89..c0353cba0b2fd 100644
--- a/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/DataFlowSanitizer.cpp
@@ -1212,7 +1212,7 @@ Value *DFSanFunction::loadShadow(Value *Addr, uint64_t Size, uint64_t Align,
     return DFS.ZeroShadow;
   case 1: {
     LoadInst *LI = new LoadInst(DFS.ShadowTy, ShadowAddr, "", Pos);
-    LI->setAlignment(ShadowAlign);
+    LI->setAlignment(MaybeAlign(ShadowAlign));
     return LI;
   }
   case 2: {
diff --git a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
index df7606dfa247d..1416dd2468365 100644
--- a/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/HWAddressSanitizer.cpp
@@ -166,6 +166,11 @@ static cl::opt<bool>
                             cl::desc("instrument landing pads"), cl::Hidden,
                             cl::init(false), cl::ZeroOrMore);
 
+static cl::opt<bool> ClUseShortGranules(
+    "hwasan-use-short-granules",
+    cl::desc("use short granules in allocas and outlined checks"), cl::Hidden,
+    cl::init(false), cl::ZeroOrMore);
+
 static cl::opt<bool> ClInstrumentPersonalityFunctions(
     "hwasan-instrument-personality-functions",
     cl::desc("instrument personality functions"), cl::Hidden, cl::init(false),
@@ -269,6 +274,7 @@ class HWAddressSanitizer {
 
   bool CompileKernel;
   bool Recover;
+  bool UseShortGranules;
   bool InstrumentLandingPads;
 
   Function *HwasanCtorFunction;
@@ -372,11 +378,14 @@ void HWAddressSanitizer::initializeModule() {
   HwasanCtorFunction = nullptr;
 
   // Older versions of Android do not have the required runtime support for
-  // global or personality function instrumentation. On other platforms we
-  // currently require using the latest version of the runtime.
+  // short granules, global or personality function instrumentation. On other
+  // platforms we currently require using the latest version of the runtime.
   bool NewRuntime =
       !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30);
 
+  UseShortGranules =
+      ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime;
+
   // If we don't have personality function support, fall back to landing pads.
   InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences()
                               ? ClInstrumentLandingPads
@@ -608,9 +617,11 @@ void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite,
       TargetTriple.isOSBinFormatELF() && !Recover) {
     Module *M = IRB.GetInsertBlock()->getParent()->getParent();
     Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy);
-    IRB.CreateCall(
-        Intrinsic::getDeclaration(M, Intrinsic::hwasan_check_memaccess),
-        {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
+    IRB.CreateCall(Intrinsic::getDeclaration(
+                       M, UseShortGranules
+                              ? Intrinsic::hwasan_check_memaccess_shortgranules
+                              : Intrinsic::hwasan_check_memaccess),
+                   {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)});
     return;
   }
 
@@ -763,6 +774,8 @@ static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) {
 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI,
                                    Value *Tag, size_t Size) {
   size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
+  if (!UseShortGranules)
+    Size = AlignedSize;
 
   Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty());
   if (ClInstrumentWithCalls) {
@@ -1171,7 +1184,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
     uint64_t Size = getAllocaSizeInBytes(*AI);
     uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment());
     AI->setAlignment(
-        std::max(AI->getAlignment(), Mapping.getObjectAlignment()));
+        MaybeAlign(std::max(AI->getAlignment(), Mapping.getObjectAlignment())));
     if (Size != AlignedSize) {
       Type *AllocatedType = AI->getAllocatedType();
       if (AI->isArrayAllocation()) {
@@ -1184,7 +1197,7 @@ bool HWAddressSanitizer::sanitizeFunction(Function &F) {
       auto *NewAI = new AllocaInst(
           TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI);
       NewAI->takeName(AI);
-      NewAI->setAlignment(AI->getAlignment());
+      NewAI->setAlignment(MaybeAlign(AI->getAlignment()));
       NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca());
       NewAI->setSwiftError(AI->isSwiftError());
       NewAI->copyMetadata(*AI);
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index a21604ad7dfda..0b83abe5bc219 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -740,9 +740,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
     PD = It->second;
   }
 
-  // Match the linkage and visibility of the name global, except on COFF, where
-  // the linkage must be local and consequentially the visibility must be
-  // default.
+  // Match the linkage and visibility of the name global. COFF supports using
+  // comdats with internal symbols, so do that if we can.
   Function *Fn = Inc->getParent()->getParent();
   GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
   GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
@@ -758,19 +757,21 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   // new comdat group for the counters and profiling data. If we use the comdat
   // of the parent function, that will result in relocations against discarded
   // sections.
-  Comdat *Cmdt = nullptr;
-  GlobalValue::LinkageTypes CounterLinkage = Linkage;
-  if (needsComdatForCounter(*Fn, *M)) {
-    StringRef CmdtPrefix = getInstrProfComdatPrefix();
+  bool NeedComdat = needsComdatForCounter(*Fn, *M);
+  if (NeedComdat) {
     if (TT.isOSBinFormatCOFF()) {
-      // For COFF, the comdat group name must be the name of a symbol in the
-      // group. Use the counter variable name, and upgrade its linkage to
-      // something externally visible, like linkonce_odr.
-      CmdtPrefix = getInstrProfCountersVarPrefix();
-      CounterLinkage = GlobalValue::LinkOnceODRLinkage;
+      // For COFF, put the counters, data, and values each into their own
+      // comdats. We can't use a group because the Visual C++ linker will
+      // report duplicate symbol errors if there are multiple external symbols
+      // with the same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
+      Linkage = GlobalValue::LinkOnceODRLinkage;
+      Visibility = GlobalValue::HiddenVisibility;
     }
-    Cmdt = M->getOrInsertComdat(getVarName(Inc, CmdtPrefix));
   }
+  auto MaybeSetComdat = [=](GlobalVariable *GV) {
+    if (NeedComdat)
+      GV->setComdat(M->getOrInsertComdat(GV->getName()));
+  };
 
   uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
   LLVMContext &Ctx = M->getContext();
@@ -785,8 +786,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   CounterPtr->setSection(
       getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
   CounterPtr->setAlignment(8);
-  CounterPtr->setComdat(Cmdt);
-  CounterPtr->setLinkage(CounterLinkage);
+  MaybeSetComdat(CounterPtr);
+  CounterPtr->setLinkage(Linkage);
 
   auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
   // Allocate statically the array of pointers to value profile nodes for
@@ -807,7 +808,7 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
       ValuesVar->setSection(
           getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
       ValuesVar->setAlignment(8);
-      ValuesVar->setComdat(Cmdt);
+      MaybeSetComdat(ValuesVar);
       ValuesPtrExpr =
           ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
     }
@@ -840,7 +841,8 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
   Data->setVisibility(Visibility);
   Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
   Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
-  Data->setComdat(Cmdt);
+  MaybeSetComdat(Data);
+  Data->setLinkage(Linkage);
 
   PD.RegionCounters = CounterPtr;
   PD.DataVar = Data;
diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index dd7b4fcbd0fba..f9354069da326 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -2562,6 +2562,11 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     return false;
   }
 
+  void handleInvariantGroup(IntrinsicInst &I) {
+    setShadow(&I, getShadow(&I, 0));
+    setOrigin(&I, getOrigin(&I, 0));
+  }
+
   void handleLifetimeStart(IntrinsicInst &I) {
     if (!PoisonStack)
       return;
@@ -2993,6 +2998,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     case Intrinsic::lifetime_start:
       handleLifetimeStart(I);
       break;
+    case Intrinsic::launder_invariant_group:
+    case Intrinsic::strip_invariant_group:
+      handleInvariantGroup(I);
+      break;
     case Intrinsic::bswap:
       handleBswap(I);
       break;
@@ -3627,10 +3636,10 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   int getNumOutputArgs(InlineAsm *IA, CallBase *CB) {
     int NumRetOutputs = 0;
     int NumOutputs = 0;
-    Type *RetTy = dyn_cast<Value>(CB)->getType();
+    Type *RetTy = cast<Value>(CB)->getType();
     if (!RetTy->isVoidTy()) {
       // Register outputs are returned via the CallInst return value.
-      StructType *ST = dyn_cast_or_null<StructType>(RetTy);
+      auto *ST = dyn_cast<StructType>(RetTy);
       if (ST)
         NumRetOutputs = ST->getNumElements();
       else
@@ -3667,7 +3676,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     // corresponding CallInst has nO+nI+1 operands (the last operand is the
     // function to be called).
     const DataLayout &DL = F.getParent()->getDataLayout();
-    CallBase *CB = dyn_cast<CallBase>(&I);
+    CallBase *CB = cast<CallBase>(&I);
     IRBuilder<> IRB(&I);
     InlineAsm *IA = cast<InlineAsm>(CB->getCalledValue());
     int OutputArgs = getNumOutputArgs(IA, CB);
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index c8cf1805c66d4..e776d59cccb5b 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -985,9 +985,9 @@ class PGOUseFunc {
 public:
   PGOUseFunc(Function &Func, Module *Modu,
              std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
-             BranchProbabilityInfo *BPI = nullptr,
-             BlockFrequencyInfo *BFIin = nullptr, bool IsCS = false)
-      : F(Func), M(Modu), BFI(BFIin),
+             BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFIin,
+             ProfileSummaryInfo *PSI, bool IsCS)
+      : F(Func), M(Modu), BFI(BFIin), PSI(PSI),
         FuncInfo(Func, ComdatMembers, false, BPI, BFIin, IsCS),
         FreqAttr(FFA_Normal), IsCS(IsCS) {}
 
@@ -1042,6 +1042,7 @@ class PGOUseFunc {
   Function &F;
   Module *M;
   BlockFrequencyInfo *BFI;
+  ProfileSummaryInfo *PSI;
 
   // This member stores the shared information with class PGOGenFunc.
   FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
@@ -1079,15 +1080,9 @@ class PGOUseFunc {
   // FIXME: This function should be removed once the functionality in
   // the inliner is implemented.
   void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
-    if (ProgramMaxCount == 0)
-      return;
-    // Threshold of the hot functions.
-    const BranchProbability HotFunctionThreshold(1, 100);
-    // Threshold of the cold functions.
-    const BranchProbability ColdFunctionThreshold(2, 10000);
-    if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
+    if (PSI->isHotCount(EntryCount))
       FreqAttr = FFA_Hot;
-    else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
+    else if (PSI->isColdCount(MaxCount))
       FreqAttr = FFA_Cold;
   }
 };
@@ -1596,7 +1591,8 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
 static bool annotateAllFunctions(
     Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
     function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
-    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI, bool IsCS) {
+    function_ref<BlockFrequencyInfo *(Function &)> LookupBFI,
+    ProfileSummaryInfo *PSI, bool IsCS) {
   LLVM_DEBUG(dbgs() << "Read in profile counters: ");
   auto &Ctx = M.getContext();
   // Read the counter array from file.
@@ -1627,6 +1623,13 @@ static bool annotateAllFunctions(
     return false;
   }
 
+  // Add the profile summary (read from the header of the indexed summary) here
+  // so that we can use it below when reading counters (which checks if the
+  // function should be marked with a cold or inlinehint attribute).
+  M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
+                      IsCS ? ProfileSummary::PSK_CSInstr
+                           : ProfileSummary::PSK_Instr);
+
   std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
   collectComdatMembers(M, ComdatMembers);
   std::vector<Function *> HotFunctions;
@@ -1639,7 +1642,7 @@ static bool annotateAllFunctions(
     // Split indirectbr critical edges here before computing the MST rather than
     // later in getInstrBB() to avoid invalidating it.
     SplitIndirectBrCriticalEdges(F, BPI, BFI);
-    PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, IsCS);
+    PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI, PSI, IsCS);
     bool AllZeros = false;
     if (!Func.readCounters(PGOReader.get(), AllZeros))
       continue;
@@ -1687,9 +1690,6 @@ static bool annotateAllFunctions(
       }
     }
   }
-  M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
-                      IsCS ? ProfileSummary::PSK_CSInstr
-                           : ProfileSummary::PSK_Instr);
 
   // Set function hotness attribute from the profile.
   // We have to apply these attributes at the end because their presence
@@ -1731,8 +1731,10 @@ PreservedAnalyses PGOInstrumentationUse::run(Module &M,
     return &FAM.getResult<BlockFrequencyAnalysis>(F);
   };
 
+  auto *PSI = &AM.getResult<ProfileSummaryAnalysis>(M);
+
   if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
-                            LookupBPI, LookupBFI, IsCS))
+                            LookupBPI, LookupBFI, PSI, IsCS))
     return PreservedAnalyses::all();
 
   return PreservedAnalyses::none();
@@ -1749,7 +1751,8 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
     return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
   };
 
-  return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI,
+  auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
+  return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI, PSI,
                               IsCS);
 }
 
diff --git a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
index ac78b4d993ed2..9f81bb16d0a7b 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp
@@ -374,8 +374,8 @@ bool MemOPSizeOpt::perform(MemIntrinsic *MI) {
         Ctx, Twine("MemOP.Case.") + Twine(SizeId), &Func, DefaultBB);
     Instruction *NewInst = MI->clone();
     // Fix the argument.
-    MemIntrinsic * MemI = dyn_cast<MemIntrinsic>(NewInst);
-    IntegerType *SizeType = dyn_cast<IntegerType>(MemI->getLength()->getType());
+    auto *MemI = cast<MemIntrinsic>(NewInst);
+    auto *SizeType = dyn_cast<IntegerType>(MemI->getLength()->getType());
     assert(SizeType && "Expected integer type size argument.");
     ConstantInt *CaseSizeId = ConstantInt::get(SizeType, SizeId);
     MemI->setLength(CaseSizeId);
diff --git a/llvm/lib/Transforms/ObjCARC/PtrState.cpp b/llvm/lib/Transforms/ObjCARC/PtrState.cpp
index 3243481dee0d9..26dd416d61845 100644
--- a/llvm/lib/Transforms/ObjCARC/PtrState.cpp
+++ b/llvm/lib/Transforms/ObjCARC/PtrState.cpp
@@ -275,6 +275,10 @@ void BottomUpPtrState::HandlePotentialUse(BasicBlock *BB, Instruction *Inst,
     } else {
       InsertAfter = std::next(Inst->getIterator());
     }
+
+    if (InsertAfter != BB->end())
+      InsertAfter = skipDebugIntrinsics(InsertAfter);
+
     InsertReverseInsertPt(&*InsertAfter);
   };
 
diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
index a301542a2066e..2d135b41279f6 100644
--- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
+++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
@@ -321,7 +321,7 @@ bool AlignmentFromAssumptionsPass::processAssumption(CallInst *ACall) {
         LI->getPointerOperand(), SE);
 
       if (NewAlignment > LI->getAlignment()) {
-        LI->setAlignment(NewAlignment);
+        LI->setAlignment(MaybeAlign(NewAlignment));
         ++NumLoadAlignChanged;
       }
     } else if (StoreInst *SI = dyn_cast<StoreInst>(J)) {
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index ff9e6d94d8597..9f340afbf7c28 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -401,7 +401,7 @@ void ConstantHoistingPass::collectConstantCandidates(
     return;
 
   // Get offset from the base GV.
-  PointerType *GVPtrTy = dyn_cast<PointerType>(BaseGV->getType());
+  PointerType *GVPtrTy = cast<PointerType>(BaseGV->getType());
   IntegerType *PtrIntTy = DL->getIntPtrType(*Ctx, GVPtrTy->getAddressSpace());
   APInt Offset(DL->getTypeSizeInBits(PtrIntTy), /*val*/0, /*isSigned*/true);
   auto *GEPO = cast<GEPOperator>(ConstExpr);
diff --git a/llvm/lib/Transforms/Scalar/DCE.cpp b/llvm/lib/Transforms/Scalar/DCE.cpp
index 83c1d7c7347b8..a79d775aa7f3a 100644
--- a/llvm/lib/Transforms/Scalar/DCE.cpp
+++ b/llvm/lib/Transforms/Scalar/DCE.cpp
@@ -38,17 +38,19 @@ namespace {
   //===--------------------------------------------------------------------===//
   // DeadInstElimination pass implementation
   //
-  struct DeadInstElimination : public BasicBlockPass {
-    static char ID; // Pass identification, replacement for typeid
-    DeadInstElimination() : BasicBlockPass(ID) {
-      initializeDeadInstEliminationPass(*PassRegistry::getPassRegistry());
-    }
-    bool runOnBasicBlock(BasicBlock &BB) override {
-      if (skipBasicBlock(BB))
-        return false;
-      auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
-      TargetLibraryInfo *TLI = TLIP ? &TLIP->getTLI(*BB.getParent()) : nullptr;
-      bool Changed = false;
+struct DeadInstElimination : public FunctionPass {
+  static char ID; // Pass identification, replacement for typeid
+  DeadInstElimination() : FunctionPass(ID) {
+    initializeDeadInstEliminationPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnFunction(Function &F) override {
+    if (skipFunction(F))
+      return false;
+    auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
+    TargetLibraryInfo *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
+
+    bool Changed = false;
+    for (auto &BB : F) {
       for (BasicBlock::iterator DI = BB.begin(); DI != BB.end(); ) {
         Instruction *Inst = &*DI++;
         if (isInstructionTriviallyDead(Inst, TLI)) {
@@ -60,13 +62,14 @@ namespace {
           ++DIEEliminated;
         }
       }
-      return Changed;
     }
+    return Changed;
+  }
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.setPreservesCFG();
     }
-  };
+};
 }
 
 char DeadInstElimination::ID = 0;
diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
index 4f53874c096e0..934853507478b 100644
--- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
+++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
@@ -233,8 +233,6 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI,
     if (!DivDominates && !DT.dominates(RemInst, DivInst)) {
       // We have matching div-rem pair, but they are in two different blocks,
       // neither of which dominates one another.
-      assert(!RemOriginallyWasInExpandedForm &&
-             "Won't happen for expanded-form rem.");
       // FIXME: We could hoist both ops to the common predecessor block?
       continue;
     }
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 64da5e081d46b..ce540683dae26 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -1382,6 +1382,7 @@ class EarlyCSELegacyCommonPass : public FunctionPass {
       AU.addPreserved<MemorySSAWrapperPass>();
     }
     AU.addPreserved<GlobalsAAWrapperPass>();
+    AU.addPreserved<AAResultsWrapperPass>();
     AU.setPreservesCFG();
   }
 };
diff --git a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
index 31670b1464e41..e6abf1ceb026d 100644
--- a/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/FlattenCFGPass.cpp
@@ -11,10 +11,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Transforms/Utils/Local.h"
 #include "llvm/IR/CFG.h"
+#include "llvm/IR/ValueHandle.h"
 #include "llvm/Pass.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
+
 using namespace llvm;
 
 #define DEBUG_TYPE "flattencfg"
@@ -52,15 +54,23 @@ FunctionPass *llvm::createFlattenCFGPass() { return new FlattenCFGPass(); }
 static bool iterativelyFlattenCFG(Function &F, AliasAnalysis *AA) {
   bool Changed = false;
   bool LocalChange = true;
+
+  // Use block handles instead of iterating over function blocks directly
+  // to avoid using iterators invalidated by erasing blocks.
+  std::vector<WeakVH> Blocks;
+  Blocks.reserve(F.size());
+  for (auto &BB : F)
+    Blocks.push_back(&BB);
+
   while (LocalChange) {
     LocalChange = false;
 
-    // Loop over all of the basic blocks and remove them if they are unneeded...
-    //
-    for (Function::iterator BBIt = F.begin(); BBIt != F.end();) {
-      if (FlattenCFG(&*BBIt++, AA)) {
-        LocalChange = true;
-      }
+    // Loop over all of the basic blocks and try to flatten them.
+    for (WeakVH &BlockHandle : Blocks) {
+      // Skip blocks erased by FlattenCFG.
+      if (auto *BB = cast_or_null<BasicBlock>(BlockHandle))
+        if (FlattenCFG(BB, AA))
+          LocalChange = true;
     }
     Changed |= LocalChange;
   }
diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp
index 4f83e869b3032..4d2eac0451dfc 100644
--- a/llvm/lib/Transforms/Scalar/Float2Int.cpp
+++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp
@@ -60,11 +60,13 @@ namespace {
       if (skipFunction(F))
         return false;
 
-      return Impl.runImpl(F);
+      const DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+      return Impl.runImpl(F, DT);
     }
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.setPreservesCFG();
+      AU.addRequired<DominatorTreeWrapperPass>();
       AU.addPreserved<GlobalsAAWrapperPass>();
     }
 
@@ -116,21 +118,29 @@ static Instruction::BinaryOps mapBinOpcode(unsigned Opcode) {
 
 // Find the roots - instructions that convert from the FP domain to
 // integer domain.
-void Float2IntPass::findRoots(Function &F, SmallPtrSet<Instruction*,8> &Roots) {
-  for (auto &I : instructions(F)) {
-    if (isa<VectorType>(I.getType()))
+void Float2IntPass::findRoots(Function &F, const DominatorTree &DT,
+                              SmallPtrSet<Instruction*,8> &Roots) {
+  for (BasicBlock &BB : F) {
+    // Unreachable code can take on strange forms that we are not prepared to
+    // handle. For example, an instruction may have itself as an operand.
+    if (!DT.isReachableFromEntry(&BB))
       continue;
-    switch (I.getOpcode()) {
-    default: break;
-    case Instruction::FPToUI:
-    case Instruction::FPToSI:
-      Roots.insert(&I);
-      break;
-    case Instruction::FCmp:
-      if (mapFCmpPred(cast<CmpInst>(&I)->getPredicate()) !=
-          CmpInst::BAD_ICMP_PREDICATE)
+
+    for (Instruction &I : BB) {
+      if (isa<VectorType>(I.getType()))
+        continue;
+      switch (I.getOpcode()) {
+      default: break;
+      case Instruction::FPToUI:
+      case Instruction::FPToSI:
         Roots.insert(&I);
-      break;
+        break;
+      case Instruction::FCmp:
+        if (mapFCmpPred(cast<CmpInst>(&I)->getPredicate()) !=
+            CmpInst::BAD_ICMP_PREDICATE)
+          Roots.insert(&I);
+        break;
+      }
     }
   }
 }
@@ -503,7 +513,7 @@ void Float2IntPass::cleanup() {
     I.first->eraseFromParent();
 }
 
-bool Float2IntPass::runImpl(Function &F) {
+bool Float2IntPass::runImpl(Function &F, const DominatorTree &DT) {
   LLVM_DEBUG(dbgs() << "F2I: Looking at function " << F.getName() << "\n");
   // Clear out all state.
   ECs = EquivalenceClasses<Instruction*>();
@@ -513,7 +523,7 @@ bool Float2IntPass::runImpl(Function &F) {
 
   Ctx = &F.getParent()->getContext();
 
-  findRoots(F, Roots);
+  findRoots(F, DT, Roots);
 
   walkBackwards(Roots);
   walkForwards();
@@ -527,8 +537,9 @@ bool Float2IntPass::runImpl(Function &F) {
 namespace llvm {
 FunctionPass *createFloat2IntPass() { return new Float2IntLegacyPass(); }
 
-PreservedAnalyses Float2IntPass::run(Function &F, FunctionAnalysisManager &) {
-  if (!runImpl(F))
+PreservedAnalyses Float2IntPass::run(Function &F, FunctionAnalysisManager &AM) {
+  const DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  if (!runImpl(F, DT))
     return PreservedAnalyses::all();
 
   PreservedAnalyses PA;
diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index 46f4ee488225e..79b901ac0db8f 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -889,9 +889,8 @@ class GVNHoist {
 
   void updateAlignment(Instruction *I, Instruction *Repl) {
     if (auto *ReplacementLoad = dyn_cast<LoadInst>(Repl)) {
-      ReplacementLoad->setAlignment(
-          std::min(ReplacementLoad->getAlignment(),
-                   cast<LoadInst>(I)->getAlignment()));
+      ReplacementLoad->setAlignment(MaybeAlign(std::min(
+          ReplacementLoad->getAlignment(), cast<LoadInst>(I)->getAlignment())));
       ++NumLoadsRemoved;
     } else if (auto *ReplacementStore = dyn_cast<StoreInst>(Repl)) {
       ReplacementStore->setAlignment(
@@ -900,8 +899,8 @@ class GVNHoist {
       ++NumStoresRemoved;
     } else if (auto *ReplacementAlloca = dyn_cast<AllocaInst>(Repl)) {
       ReplacementAlloca->setAlignment(
-          std::max(ReplacementAlloca->getAlignment(),
-                   cast<AllocaInst>(I)->getAlignment()));
+          MaybeAlign(std::max(ReplacementAlloca->getAlignment(),
+                              cast<AllocaInst>(I)->getAlignment())));
     } else if (isa<CallInst>(Repl)) {
       ++NumCallsRemoved;
     }
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index 104a8a13e4d88..a9fdfbaef3f4c 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -124,6 +124,11 @@ static cl::opt<bool>
 DisableLFTR("disable-lftr", cl::Hidden, cl::init(false),
             cl::desc("Disable Linear Function Test Replace optimization"));
 
+static cl::opt<bool>
+LoopPredication("indvars-predicate-loops", cl::Hidden, cl::init(false),
+                cl::desc("Predicate conditions in read only loops"));
+
+
 namespace {
 
 struct RewritePhi;
@@ -144,7 +149,7 @@ class IndVarSimplify {
   bool rewriteNonIntegerIVs(Loop *L);
 
   bool simplifyAndExtend(Loop *L, SCEVExpander &Rewriter, LoopInfo *LI);
-  bool optimizeLoopExits(Loop *L);
+  bool optimizeLoopExits(Loop *L, SCEVExpander &Rewriter);
 
   bool canLoopBeDeleted(Loop *L, SmallVector<RewritePhi, 8> &RewritePhiSet);
   bool rewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter);
@@ -1672,6 +1677,10 @@ Instruction *WidenIV::widenIVUse(NarrowIVDefUse DU, SCEVExpander &Rewriter) {
     return nullptr;
   }
 
+  // if we reached this point then we are going to replace
+  // DU.NarrowUse with WideUse. Reattach DbgValue then.
+  replaceAllDbgUsesWith(*DU.NarrowUse, *WideUse, *WideUse, *DT);
+
   ExtendKindMap[DU.NarrowUse] = WideAddRec.second;
   // Returning WideUse pushes it on the worklist.
   return WideUse;
@@ -1797,14 +1806,9 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) {
       DeadInsts.emplace_back(DU.NarrowDef);
   }
 
-  // Attach any debug information to the new PHI. Since OrigPhi and WidePHI
-  // evaluate the same recurrence, we can just copy the debug info over.
-  SmallVector<DbgValueInst *, 1> DbgValues;
-  llvm::findDbgValues(DbgValues, OrigPhi);
-  auto *MDPhi = MetadataAsValue::get(WidePhi->getContext(),
-                                     ValueAsMetadata::get(WidePhi));
-  for (auto &DbgValue : DbgValues)
-    DbgValue->setOperand(0, MDPhi);
+  // Attach any debug information to the new PHI.
+  replaceAllDbgUsesWith(*OrigPhi, *WidePhi, *WidePhi, *DT);
+
   return WidePhi;
 }
 
@@ -2642,7 +2646,7 @@ bool IndVarSimplify::sinkUnusedInvariants(Loop *L) {
   return MadeAnyChanges;
 }
 
-bool IndVarSimplify::optimizeLoopExits(Loop *L) {
+bool IndVarSimplify::optimizeLoopExits(Loop *L, SCEVExpander &Rewriter) {
   SmallVector<BasicBlock*, 16> ExitingBlocks;
   L->getExitingBlocks(ExitingBlocks);
 
@@ -2720,8 +2724,7 @@ bool IndVarSimplify::optimizeLoopExits(Loop *L) {
     assert(MaxExitCount->getType() == ExitCount->getType());
     
     // Can we prove that some other exit must be taken strictly before this
-    // one?  TODO: handle cases where ule is known, and equality is covered
-    // by a dominating exit
+    // one?
     if (SE->isLoopEntryGuardedByCond(L, CmpInst::ICMP_ULT,
                                      MaxExitCount, ExitCount)) {
       bool ExitIfTrue = !L->contains(*succ_begin(ExitingBB));
@@ -2738,14 +2741,136 @@ bool IndVarSimplify::optimizeLoopExits(Loop *L) {
     // TODO: If we can prove that the exiting iteration is equal to the exit
     // count for this exit and that no previous exit oppurtunities exist within
     // the loop, then we can discharge all other exits.  (May fall out of
-    // previous TODO.) 
-    
-    // TODO: If we can't prove any relation between our exit count and the
-    // loops exit count, but taking this exit doesn't require actually running
-    // the loop (i.e. no side effects, no computed values used in exit), then
-    // we can replace the exit test with a loop invariant test which exits on
-    // the first iteration.  
+    // previous TODO.)
   }
+
+  // Finally, see if we can rewrite our exit conditions into a loop invariant
+  // form.  If we have a read-only loop, and we can tell that we must exit down
+  // a path which does not need any of the values computed within the loop, we
+  // can rewrite the loop to exit on the first iteration.  Note that this
+  // doesn't either a) tell us the loop exits on the first iteration (unless
+  // *all* exits are predicateable) or b) tell us *which* exit might be taken.
+  // This transformation looks a lot like a restricted form of dead loop
+  // elimination, but restricted to read-only loops and without neccesssarily
+  // needing to kill the loop entirely. 
+  if (!LoopPredication)
+    return Changed;
+
+  if (!SE->hasLoopInvariantBackedgeTakenCount(L))
+    return Changed;
+
+  // Note: ExactBTC is the exact backedge taken count *iff* the loop exits
+  // through *explicit* control flow.  We have to eliminate the possibility of
+  // implicit exits (see below) before we know it's truly exact.
+  const SCEV *ExactBTC = SE->getBackedgeTakenCount(L);
+  if (isa<SCEVCouldNotCompute>(ExactBTC) ||
+      !SE->isLoopInvariant(ExactBTC, L) ||
+      !isSafeToExpand(ExactBTC, *SE))
+    return Changed;
+
+  auto Filter = [&](BasicBlock *ExitingBB) {
+    // If our exiting block exits multiple loops, we can only rewrite the
+    // innermost one.  Otherwise, we're changing how many times the innermost
+    // loop runs before it exits. 
+    if (LI->getLoopFor(ExitingBB) != L)
+      return true;
+
+    // Can't rewrite non-branch yet.
+    BranchInst *BI = dyn_cast<BranchInst>(ExitingBB->getTerminator());
+    if (!BI)
+      return true;
+
+    // If already constant, nothing to do.
+    if (isa<Constant>(BI->getCondition()))
+      return true;
+
+    // If the exit block has phis, we need to be able to compute the values
+    // within the loop which contains them.  This assumes trivially lcssa phis
+    // have already been removed; TODO: generalize
+    BasicBlock *ExitBlock =
+    BI->getSuccessor(L->contains(BI->getSuccessor(0)) ? 1 : 0);
+    if (!empty(ExitBlock->phis()))
+      return true;
+
+    const SCEV *ExitCount = SE->getExitCount(L, ExitingBB);
+    assert(!isa<SCEVCouldNotCompute>(ExactBTC) && "implied by having exact trip count");
+    if (!SE->isLoopInvariant(ExitCount, L) ||
+        !isSafeToExpand(ExitCount, *SE))
+      return true;
+
+    return false;
+  };
+  auto Erased = std::remove_if(ExitingBlocks.begin(), ExitingBlocks.end(),
+                               Filter);
+  ExitingBlocks.erase(Erased, ExitingBlocks.end());
+
+  if (ExitingBlocks.empty())
+    return Changed;
+
+  // We rely on not being able to reach an exiting block on a later iteration
+  // than it's statically compute exit count.  The implementaton of
+  // getExitCount currently has this invariant, but assert it here so that
+  // breakage is obvious if this ever changes..
+  assert(llvm::all_of(ExitingBlocks, [&](BasicBlock *ExitingBB) {
+        return DT->dominates(ExitingBB, L->getLoopLatch());
+      }));
+
+  // At this point, ExitingBlocks consists of only those blocks which are
+  // predicatable.  Given that, we know we have at least one exit we can
+  // predicate if the loop is doesn't have side effects and doesn't have any
+  // implicit exits (because then our exact BTC isn't actually exact).
+  // @Reviewers - As structured, this is O(I^2) for loop nests.  Any
+  // suggestions on how to improve this?  I can obviously bail out for outer
+  // loops, but that seems less than ideal.  MemorySSA can find memory writes,
+  // is that enough for *all* side effects?
+  for (BasicBlock *BB : L->blocks())
+    for (auto &I : *BB)
+      // TODO:isGuaranteedToTransfer
+      if (I.mayHaveSideEffects() || I.mayThrow())
+        return Changed;
+
+  // Finally, do the actual predication for all predicatable blocks.  A couple
+  // of notes here:
+  // 1) We don't bother to constant fold dominated exits with identical exit
+  //    counts; that's simply a form of CSE/equality propagation and we leave
+  //    it for dedicated passes.
+  // 2) We insert the comparison at the branch.  Hoisting introduces additional
+  //    legality constraints and we leave that to dedicated logic.  We want to
+  //    predicate even if we can't insert a loop invariant expression as
+  //    peeling or unrolling will likely reduce the cost of the otherwise loop
+  //    varying check.
+  Rewriter.setInsertPoint(L->getLoopPreheader()->getTerminator());
+  IRBuilder<> B(L->getLoopPreheader()->getTerminator());
+  Value *ExactBTCV = nullptr; //lazy generated if needed
+  for (BasicBlock *ExitingBB : ExitingBlocks) {
+    const SCEV *ExitCount = SE->getExitCount(L, ExitingBB);
+
+    auto *BI = cast<BranchInst>(ExitingBB->getTerminator());
+    Value *NewCond;
+    if (ExitCount == ExactBTC) {
+      NewCond = L->contains(BI->getSuccessor(0)) ?
+        B.getFalse() : B.getTrue();
+    } else {
+      Value *ECV = Rewriter.expandCodeFor(ExitCount);
+      if (!ExactBTCV)
+        ExactBTCV = Rewriter.expandCodeFor(ExactBTC);
+      Value *RHS = ExactBTCV;
+      if (ECV->getType() != RHS->getType()) {
+        Type *WiderTy = SE->getWiderType(ECV->getType(), RHS->getType());
+        ECV = B.CreateZExt(ECV, WiderTy);
+        RHS = B.CreateZExt(RHS, WiderTy);
+      }
+      auto Pred = L->contains(BI->getSuccessor(0)) ?
+        ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
+      NewCond = B.CreateICmp(Pred, ECV, RHS);
+    }
+    Value *OldCond = BI->getCondition();
+    BI->setCondition(NewCond);
+    if (OldCond->use_empty())
+      DeadInsts.push_back(OldCond);
+    Changed = true;
+  }
+
   return Changed;
 }
 
@@ -2805,7 +2930,7 @@ bool IndVarSimplify::run(Loop *L) {
   // Eliminate redundant IV cycles.
   NumElimIV += Rewriter.replaceCongruentIVs(L, DT, DeadInsts);
 
-  Changed |= optimizeLoopExits(L);
+  Changed |= optimizeLoopExits(L, Rewriter);
 
   // If we have a trip count expression, rewrite the loop's exit condition
   // using it.  
@@ -2845,7 +2970,7 @@ bool IndVarSimplify::run(Loop *L) {
       // that our definition of "high cost" is not exactly principled.  
       if (Rewriter.isHighCostExpansion(ExitCount, L))
         continue;
-      
+
       // Check preconditions for proper SCEVExpander operation. SCEV does not
       // express SCEVExpander's dependencies, such as LoopSimplify. Instead
       // any pass that uses the SCEVExpander must do it. This does not work
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index 52ff7cbcaef2a..5c2176f873bc0 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -963,7 +963,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AliasAnalysis *AA, LoopInfo *LI,
 
     // Now that we've finished hoisting make sure that LI and DT are still
     // valid.
-#ifndef NDEBUG
+#ifdef EXPENSIVE_CHECKS
   if (Changed) {
     assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
            "Dominator tree verification failed");
@@ -1383,8 +1383,7 @@ static Instruction *CloneInstructionInExitBlock(
   if (!I.getName().empty())
     New->setName(I.getName() + ".le");
 
-  MemoryAccess *OldMemAcc;
-  if (MSSAU && (OldMemAcc = MSSAU->getMemorySSA()->getMemoryAccess(&I))) {
+  if (MSSAU && MSSAU->getMemorySSA()->getMemoryAccess(&I)) {
     // Create a new MemoryAccess and let MemorySSA set its defining access.
     MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
         New, nullptr, New->getParent(), MemorySSA::Beginning);
@@ -2109,15 +2108,14 @@ bool llvm::promoteLoopAccessesToScalars(
       SomePtr->getName() + ".promoted", Preheader->getTerminator());
   if (SawUnorderedAtomic)
     PreheaderLoad->setOrdering(AtomicOrdering::Unordered);
-  PreheaderLoad->setAlignment(Alignment);
+  PreheaderLoad->setAlignment(MaybeAlign(Alignment));
   PreheaderLoad->setDebugLoc(DL);
   if (AATags)
     PreheaderLoad->setAAMetadata(AATags);
   SSA.AddAvailableValue(Preheader, PreheaderLoad);
 
-  MemoryAccess *PreheaderLoadMemoryAccess;
   if (MSSAU) {
-    PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
+    MemoryAccess *PreheaderLoadMemoryAccess = MSSAU->createMemoryAccessInBB(
         PreheaderLoad, nullptr, PreheaderLoad->getParent(), MemorySSA::End);
     MemoryUse *NewMemUse = cast<MemoryUse>(PreheaderLoadMemoryAccess);
     MSSAU->insertUse(NewMemUse, /*RenameUses=*/true);
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index a323d80dbe683..9f93c68e6128d 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -85,6 +85,9 @@ STATISTIC(NonEqualTripCount, "Loop trip counts are not the same");
 STATISTIC(NonAdjacent, "Loops are not adjacent");
 STATISTIC(NonEmptyPreheader, "Loop has a non-empty preheader");
 STATISTIC(FusionNotBeneficial, "Fusion is not beneficial");
+STATISTIC(NonIdenticalGuards, "Candidates have different guards");
+STATISTIC(NonEmptyExitBlock, "Candidate has a non-empty exit block");
+STATISTIC(NonEmptyGuardBlock, "Candidate has a non-empty guard block");
 
 enum FusionDependenceAnalysisChoice {
   FUSION_DEPENDENCE_ANALYSIS_SCEV,
@@ -144,6 +147,8 @@ struct FusionCandidate {
   SmallVector<Instruction *, 16> MemWrites;
   /// Are all of the members of this fusion candidate still valid
   bool Valid;
+  /// Guard branch of the loop, if it exists
+  BranchInst *GuardBranch;
 
   /// Dominator and PostDominator trees are needed for the
   /// FusionCandidateCompare function, required by FusionCandidateSet to
@@ -158,8 +163,14 @@ struct FusionCandidate {
                   const PostDominatorTree *PDT, OptimizationRemarkEmitter &ORE)
       : Preheader(L->getLoopPreheader()), Header(L->getHeader()),
         ExitingBlock(L->getExitingBlock()), ExitBlock(L->getExitBlock()),
-        Latch(L->getLoopLatch()), L(L), Valid(true), DT(DT), PDT(PDT),
-        ORE(ORE) {
+        Latch(L->getLoopLatch()), L(L), Valid(true), GuardBranch(nullptr),
+        DT(DT), PDT(PDT), ORE(ORE) {
+
+    // TODO: This is temporary while we fuse both rotated and non-rotated
+    // loops. Once we switch to only fusing rotated loops, the initialization of
+    // GuardBranch can be moved into the initialization list above.
+    if (isRotated())
+      GuardBranch = L->getLoopGuardBranch();
 
     // Walk over all blocks in the loop and check for conditions that may
     // prevent fusion. For each block, walk over all instructions and collect
@@ -218,16 +229,55 @@ struct FusionCandidate {
     assert(Latch == L->getLoopLatch() && "Latch is out of sync");
   }
 
+  /// Get the entry block for this fusion candidate.
+  ///
+  /// If this fusion candidate represents a guarded loop, the entry block is the
+  /// loop guard block. If it represents an unguarded loop, the entry block is
+  /// the preheader of the loop.
+  BasicBlock *getEntryBlock() const {
+    if (GuardBranch)
+      return GuardBranch->getParent();
+    else
+      return Preheader;
+  }
+
+  /// Given a guarded loop, get the successor of the guard that is not in the
+  /// loop.
+  ///
+  /// This method returns the successor of the loop guard that is not located
+  /// within the loop (i.e., the successor of the guard that is not the
+  /// preheader).
+  /// This method is only valid for guarded loops.
+  BasicBlock *getNonLoopBlock() const {
+    assert(GuardBranch && "Only valid on guarded loops.");
+    assert(GuardBranch->isConditional() &&
+           "Expecting guard to be a conditional branch.");
+    return (GuardBranch->getSuccessor(0) == Preheader)
+               ? GuardBranch->getSuccessor(1)
+               : GuardBranch->getSuccessor(0);
+  }
+
+  bool isRotated() const {
+    assert(L && "Expecting loop to be valid.");
+    assert(Latch && "Expecting latch to be valid.");
+    return L->isLoopExiting(Latch);
+  }
+
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   LLVM_DUMP_METHOD void dump() const {
-    dbgs() << "\tPreheader: " << (Preheader ? Preheader->getName() : "nullptr")
+    dbgs() << "\tGuardBranch: "
+           << (GuardBranch ? GuardBranch->getName() : "nullptr") << "\n"
+           << "\tPreheader: " << (Preheader ? Preheader->getName() : "nullptr")
            << "\n"
            << "\tHeader: " << (Header ? Header->getName() : "nullptr") << "\n"
            << "\tExitingBB: "
            << (ExitingBlock ? ExitingBlock->getName() : "nullptr") << "\n"
            << "\tExitBB: " << (ExitBlock ? ExitBlock->getName() : "nullptr")
            << "\n"
-           << "\tLatch: " << (Latch ? Latch->getName() : "nullptr") << "\n";
+           << "\tLatch: " << (Latch ? Latch->getName() : "nullptr") << "\n"
+           << "\tEntryBlock: "
+           << (getEntryBlock() ? getEntryBlock()->getName() : "nullptr")
+           << "\n";
   }
 #endif
 
@@ -303,21 +353,24 @@ struct FusionCandidateCompare {
                   const FusionCandidate &RHS) const {
     const DominatorTree *DT = LHS.DT;
 
+    BasicBlock *LHSEntryBlock = LHS.getEntryBlock();
+    BasicBlock *RHSEntryBlock = RHS.getEntryBlock();
+
     // Do not save PDT to local variable as it is only used in asserts and thus
     // will trigger an unused variable warning if building without asserts.
     assert(DT && LHS.PDT && "Expecting valid dominator tree");
 
     // Do this compare first so if LHS == RHS, function returns false.
-    if (DT->dominates(RHS.Preheader, LHS.Preheader)) {
+    if (DT->dominates(RHSEntryBlock, LHSEntryBlock)) {
       // RHS dominates LHS
       // Verify LHS post-dominates RHS
-      assert(LHS.PDT->dominates(LHS.Preheader, RHS.Preheader));
+      assert(LHS.PDT->dominates(LHSEntryBlock, RHSEntryBlock));
       return false;
     }
 
-    if (DT->dominates(LHS.Preheader, RHS.Preheader)) {
+    if (DT->dominates(LHSEntryBlock, RHSEntryBlock)) {
       // Verify RHS Postdominates LHS
-      assert(LHS.PDT->dominates(RHS.Preheader, LHS.Preheader));
+      assert(LHS.PDT->dominates(RHSEntryBlock, LHSEntryBlock));
       return true;
     }
 
@@ -538,11 +591,14 @@ struct LoopFuser {
                                const FusionCandidate &FC1) const {
     assert(FC0.Preheader && FC1.Preheader && "Expecting valid preheaders");
 
-    if (DT.dominates(FC0.Preheader, FC1.Preheader))
-      return PDT.dominates(FC1.Preheader, FC0.Preheader);
+    BasicBlock *FC0EntryBlock = FC0.getEntryBlock();
+    BasicBlock *FC1EntryBlock = FC1.getEntryBlock();
+
+    if (DT.dominates(FC0EntryBlock, FC1EntryBlock))
+      return PDT.dominates(FC1EntryBlock, FC0EntryBlock);
 
-    if (DT.dominates(FC1.Preheader, FC0.Preheader))
-      return PDT.dominates(FC0.Preheader, FC1.Preheader);
+    if (DT.dominates(FC1EntryBlock, FC0EntryBlock))
+      return PDT.dominates(FC0EntryBlock, FC1EntryBlock);
 
     return false;
   }
@@ -677,11 +733,22 @@ struct LoopFuser {
             continue;
           }
 
-          // For now we skip fusing if the second candidate has any instructions
-          // in the preheader. This is done because we currently do not have the
-          // safety checks to determine if it is save to move the preheader of
-          // the second candidate past the body of the first candidate. Once
-          // these checks are added, this condition can be removed.
+          // Ensure that FC0 and FC1 have identical guards.
+          // If one (or both) are not guarded, this check is not necessary.
+          if (FC0->GuardBranch && FC1->GuardBranch &&
+              !haveIdenticalGuards(*FC0, *FC1)) {
+            LLVM_DEBUG(dbgs() << "Fusion candidates do not have identical "
+                                 "guards. Not Fusing.\n");
+            reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
+                                                       NonIdenticalGuards);
+            continue;
+          }
+
+          // The following three checks look for empty blocks in FC0 and FC1. If
+          // any of these blocks are non-empty, we do not fuse. This is done
+          // because we currently do not have the safety checks to determine if
+          // it is safe to move the blocks past other blocks in the loop. Once
+          // these checks are added, these conditions can be relaxed.
           if (!isEmptyPreheader(*FC1)) {
             LLVM_DEBUG(dbgs() << "Fusion candidate does not have empty "
                                  "preheader. Not fusing.\n");
@@ -690,6 +757,24 @@ struct LoopFuser {
             continue;
           }
 
+          if (FC0->GuardBranch && !isEmptyExitBlock(*FC0)) {
+            LLVM_DEBUG(dbgs() << "Fusion candidate does not have empty exit "
+                                 "block. Not fusing.\n");
+            reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
+                                                       NonEmptyExitBlock);
+            continue;
+          }
+
+          if (FC1->GuardBranch && !isEmptyGuardBlock(*FC1)) {
+            LLVM_DEBUG(dbgs() << "Fusion candidate does not have empty guard "
+                                 "block. Not fusing.\n");
+            reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
+                                                       NonEmptyGuardBlock);
+            continue;
+          }
+
+          // Check the dependencies across the loops and do not fuse if it would
+          // violate them.
           if (!dependencesAllowFusion(*FC0, *FC1)) {
             LLVM_DEBUG(dbgs() << "Memory dependencies do not allow fusion!\n");
             reportLoopFusion<OptimizationRemarkMissed>(*FC0, *FC1,
@@ -895,7 +980,7 @@ struct LoopFuser {
     LLVM_DEBUG(dbgs() << "Check if " << FC0 << " can be fused with " << FC1
                       << "\n");
     assert(FC0.L->getLoopDepth() == FC1.L->getLoopDepth());
-    assert(DT.dominates(FC0.Preheader, FC1.Preheader));
+    assert(DT.dominates(FC0.getEntryBlock(), FC1.getEntryBlock()));
 
     for (Instruction *WriteL0 : FC0.MemWrites) {
       for (Instruction *WriteL1 : FC1.MemWrites)
@@ -945,18 +1030,89 @@ struct LoopFuser {
     return true;
   }
 
-  /// Determine if the exit block of \p FC0 is the preheader of \p FC1. In this
-  /// case, there is no code in between the two fusion candidates, thus making
-  /// them adjacent.
+  /// Determine if two fusion candidates are adjacent in the CFG.
+  ///
+  /// This method will determine if there are additional basic blocks in the CFG
+  /// between the exit of \p FC0 and the entry of \p FC1.
+  /// If the two candidates are guarded loops, then it checks whether the
+  /// non-loop successor of the \p FC0 guard branch is the entry block of \p
+  /// FC1. If not, then the loops are not adjacent. If the two candidates are
+  /// not guarded loops, then it checks whether the exit block of \p FC0 is the
+  /// preheader of \p FC1.
   bool isAdjacent(const FusionCandidate &FC0,
                   const FusionCandidate &FC1) const {
-    return FC0.ExitBlock == FC1.Preheader;
+    // If the successor of the guard branch is FC1, then the loops are adjacent
+    if (FC0.GuardBranch)
+      return FC0.getNonLoopBlock() == FC1.getEntryBlock();
+    else
+      return FC0.ExitBlock == FC1.getEntryBlock();
+  }
+
+  /// Determine if two fusion candidates have identical guards
+  ///
+  /// This method will determine if two fusion candidates have the same guards.
+  /// The guards are considered the same if:
+  ///   1. The instructions to compute the condition used in the compare are
+  ///      identical.
+  ///   2. The successors of the guard have the same flow into/around the loop.
+  /// If the compare instructions are identical, then the first successor of the
+  /// guard must go to the same place (either the preheader of the loop or the
+  /// NonLoopBlock). In other words, the the first successor of both loops must
+  /// both go into the loop (i.e., the preheader) or go around the loop (i.e.,
+  /// the NonLoopBlock). The same must be true for the second successor.
+  bool haveIdenticalGuards(const FusionCandidate &FC0,
+                           const FusionCandidate &FC1) const {
+    assert(FC0.GuardBranch && FC1.GuardBranch &&
+           "Expecting FC0 and FC1 to be guarded loops.");
+
+    if (auto FC0CmpInst =
+            dyn_cast<Instruction>(FC0.GuardBranch->getCondition()))
+      if (auto FC1CmpInst =
+              dyn_cast<Instruction>(FC1.GuardBranch->getCondition()))
+        if (!FC0CmpInst->isIdenticalTo(FC1CmpInst))
+          return false;
+
+    // The compare instructions are identical.
+    // Now make sure the successor of the guards have the same flow into/around
+    // the loop
+    if (FC0.GuardBranch->getSuccessor(0) == FC0.Preheader)
+      return (FC1.GuardBranch->getSuccessor(0) == FC1.Preheader);
+    else
+      return (FC1.GuardBranch->getSuccessor(1) == FC1.Preheader);
+  }
+
+  /// Check that the guard for \p FC *only* contains the cmp/branch for the
+  /// guard.
+  /// Once we are able to handle intervening code, any code in the guard block
+  /// for FC1 will need to be treated as intervening code and checked whether
+  /// it can safely move around the loops.
+  bool isEmptyGuardBlock(const FusionCandidate &FC) const {
+    assert(FC.GuardBranch && "Expecting a fusion candidate with guard branch.");
+    if (auto *CmpInst = dyn_cast<Instruction>(FC.GuardBranch->getCondition())) {
+      auto *GuardBlock = FC.GuardBranch->getParent();
+      // If the generation of the cmp value is in GuardBlock, then the size of
+      // the guard block should be 2 (cmp + branch). If the generation of the
+      // cmp value is in a different block, then the size of the guard block
+      // should only be 1.
+      if (CmpInst->getParent() == GuardBlock)
+        return GuardBlock->size() == 2;
+      else
+        return GuardBlock->size() == 1;
+    }
+
+    return false;
   }
 
   bool isEmptyPreheader(const FusionCandidate &FC) const {
+    assert(FC.Preheader && "Expecting a valid preheader");
     return FC.Preheader->size() == 1;
   }
 
+  bool isEmptyExitBlock(const FusionCandidate &FC) const {
+    assert(FC.ExitBlock && "Expecting a valid exit block");
+    return FC.ExitBlock->size() == 1;
+  }
+
   /// Fuse two fusion candidates, creating a new fused loop.
   ///
   /// This method contains the mechanics of fusing two loops, represented by \p
@@ -993,6 +1149,12 @@ struct LoopFuser {
     LLVM_DEBUG(dbgs() << "Fusion Candidate 0: \n"; FC0.dump();
                dbgs() << "Fusion Candidate 1: \n"; FC1.dump(););
 
+    // Fusing guarded loops is handled slightly differently than non-guarded
+    // loops and has been broken out into a separate method instead of trying to
+    // intersperse the logic within a single method.
+    if (FC0.GuardBranch)
+      return fuseGuardedLoops(FC0, FC1);
+
     assert(FC1.Preheader == FC0.ExitBlock);
     assert(FC1.Preheader->size() == 1 &&
            FC1.Preheader->getSingleSuccessor() == FC1.Header);
@@ -1137,8 +1299,6 @@ struct LoopFuser {
     SE.verify();
 #endif
 
-    FuseCounter++;
-
     LLVM_DEBUG(dbgs() << "Fusion done:\n");
 
     return FC0.L;
@@ -1170,6 +1330,232 @@ struct LoopFuser {
              << " and " << NV("Cand2", StringRef(FC1.Preheader->getName()))
              << ": " << Stat.getDesc());
   }
+
+  /// Fuse two guarded fusion candidates, creating a new fused loop.
+  ///
+  /// Fusing guarded loops is handled much the same way as fusing non-guarded
+  /// loops. The rewiring of the CFG is slightly different though, because of
+  /// the presence of the guards around the loops and the exit blocks after the
+  /// loop body. As such, the new loop is rewired as follows:
+  ///    1. Keep the guard branch from FC0 and use the non-loop block target
+  /// from the FC1 guard branch.
+  ///    2. Remove the exit block from FC0 (this exit block should be empty
+  /// right now).
+  ///    3. Remove the guard branch for FC1
+  ///    4. Remove the preheader for FC1.
+  /// The exit block successor for the latch of FC0 is updated to be the header
+  /// of FC1 and the non-exit block successor of the latch of FC1 is updated to
+  /// be the header of FC0, thus creating the fused loop.
+  Loop *fuseGuardedLoops(const FusionCandidate &FC0,
+                         const FusionCandidate &FC1) {
+    assert(FC0.GuardBranch && FC1.GuardBranch && "Expecting guarded loops");
+
+    BasicBlock *FC0GuardBlock = FC0.GuardBranch->getParent();
+    BasicBlock *FC1GuardBlock = FC1.GuardBranch->getParent();
+    BasicBlock *FC0NonLoopBlock = FC0.getNonLoopBlock();
+    BasicBlock *FC1NonLoopBlock = FC1.getNonLoopBlock();
+
+    assert(FC0NonLoopBlock == FC1GuardBlock && "Loops are not adjacent");
+
+    SmallVector<DominatorTree::UpdateType, 8> TreeUpdates;
+
+    ////////////////////////////////////////////////////////////////////////////
+    // Update the Loop Guard
+    ////////////////////////////////////////////////////////////////////////////
+    // The guard for FC0 is updated to guard both FC0 and FC1. This is done by
+    // changing the NonLoopGuardBlock for FC0 to the NonLoopGuardBlock for FC1.
+    // Thus, one path from the guard goes to the preheader for FC0 (and thus
+    // executes the new fused loop) and the other path goes to the NonLoopBlock
+    // for FC1 (where FC1 guard would have gone if FC1 was not executed).
+    FC0.GuardBranch->replaceUsesOfWith(FC0NonLoopBlock, FC1NonLoopBlock);
+    FC0.ExitBlock->getTerminator()->replaceUsesOfWith(FC1GuardBlock,
+                                                      FC1.Header);
+
+    // The guard of FC1 is not necessary anymore.
+    FC1.GuardBranch->eraseFromParent();
+    new UnreachableInst(FC1GuardBlock->getContext(), FC1GuardBlock);
+
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC1GuardBlock, FC1.Preheader));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC1GuardBlock, FC1NonLoopBlock));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC0GuardBlock, FC1GuardBlock));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Insert, FC0GuardBlock, FC1NonLoopBlock));
+
+    assert(pred_begin(FC1GuardBlock) == pred_end(FC1GuardBlock) &&
+           "Expecting guard block to have no predecessors");
+    assert(succ_begin(FC1GuardBlock) == succ_end(FC1GuardBlock) &&
+           "Expecting guard block to have no successors");
+
+    // Remember the phi nodes originally in the header of FC0 in order to rewire
+    // them later. However, this is only necessary if the new loop carried
+    // values might not dominate the exiting branch. While we do not generally
+    // test if this is the case but simply insert intermediate phi nodes, we
+    // need to make sure these intermediate phi nodes have different
+    // predecessors. To this end, we filter the special case where the exiting
+    // block is the latch block of the first loop. Nothing needs to be done
+    // anyway as all loop carried values dominate the latch and thereby also the
+    // exiting branch.
+    // KB: This is no longer necessary because FC0.ExitingBlock == FC0.Latch
+    // (because the loops are rotated. Thus, nothing will ever be added to
+    // OriginalFC0PHIs.
+    SmallVector<PHINode *, 8> OriginalFC0PHIs;
+    if (FC0.ExitingBlock != FC0.Latch)
+      for (PHINode &PHI : FC0.Header->phis())
+        OriginalFC0PHIs.push_back(&PHI);
+
+    assert(OriginalFC0PHIs.empty() && "Expecting OriginalFC0PHIs to be empty!");
+
+    // Replace incoming blocks for header PHIs first.
+    FC1.Preheader->replaceSuccessorsPhiUsesWith(FC0.Preheader);
+    FC0.Latch->replaceSuccessorsPhiUsesWith(FC1.Latch);
+
+    // The old exiting block of the first loop (FC0) has to jump to the header
+    // of the second as we need to execute the code in the second header block
+    // regardless of the trip count. That is, if the trip count is 0, so the
+    // back edge is never taken, we still have to execute both loop headers,
+    // especially (but not only!) if the second is a do-while style loop.
+    // However, doing so might invalidate the phi nodes of the first loop as
+    // the new values do only need to dominate their latch and not the exiting
+    // predicate. To remedy this potential problem we always introduce phi
+    // nodes in the header of the second loop later that select the loop carried
+    // value, if the second header was reached through an old latch of the
+    // first, or undef otherwise. This is sound as exiting the first implies the
+    // second will exit too, __without__ taking the back-edge (their
+    // trip-counts are equal after all).
+    FC0.ExitingBlock->getTerminator()->replaceUsesOfWith(FC0.ExitBlock,
+                                                         FC1.Header);
+
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC0.ExitingBlock, FC0.ExitBlock));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Insert, FC0.ExitingBlock, FC1.Header));
+
+    // Remove FC0 Exit Block
+    // The exit block for FC0 is no longer needed since control will flow
+    // directly to the header of FC1. Since it is an empty block, it can be
+    // removed at this point.
+    // TODO: In the future, we can handle non-empty exit blocks my merging any
+    // instructions from FC0 exit block into FC1 exit block prior to removing
+    // the block.
+    assert(pred_begin(FC0.ExitBlock) == pred_end(FC0.ExitBlock) &&
+           "Expecting exit block to be empty");
+    FC0.ExitBlock->getTerminator()->eraseFromParent();
+    new UnreachableInst(FC0.ExitBlock->getContext(), FC0.ExitBlock);
+
+    // Remove FC1 Preheader
+    // The pre-header of L1 is not necessary anymore.
+    assert(pred_begin(FC1.Preheader) == pred_end(FC1.Preheader));
+    FC1.Preheader->getTerminator()->eraseFromParent();
+    new UnreachableInst(FC1.Preheader->getContext(), FC1.Preheader);
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(
+        DominatorTree::Delete, FC1.Preheader, FC1.Header));
+
+    // Moves the phi nodes from the second to the first loops header block.
+    while (PHINode *PHI = dyn_cast<PHINode>(&FC1.Header->front())) {
+      if (SE.isSCEVable(PHI->getType()))
+        SE.forgetValue(PHI);
+      if (PHI->hasNUsesOrMore(1))
+        PHI->moveBefore(&*FC0.Header->getFirstInsertionPt());
+      else
+        PHI->eraseFromParent();
+    }
+
+    // Introduce new phi nodes in the second loop header to ensure
+    // exiting the first and jumping to the header of the second does not break
+    // the SSA property of the phis originally in the first loop. See also the
+    // comment above.
+    Instruction *L1HeaderIP = &FC1.Header->front();
+    for (PHINode *LCPHI : OriginalFC0PHIs) {
+      int L1LatchBBIdx = LCPHI->getBasicBlockIndex(FC1.Latch);
+      assert(L1LatchBBIdx >= 0 &&
+             "Expected loop carried value to be rewired at this point!");
+
+      Value *LCV = LCPHI->getIncomingValue(L1LatchBBIdx);
+
+      PHINode *L1HeaderPHI = PHINode::Create(
+          LCV->getType(), 2, LCPHI->getName() + ".afterFC0", L1HeaderIP);
+      L1HeaderPHI->addIncoming(LCV, FC0.Latch);
+      L1HeaderPHI->addIncoming(UndefValue::get(LCV->getType()),
+                               FC0.ExitingBlock);
+
+      LCPHI->setIncomingValue(L1LatchBBIdx, L1HeaderPHI);
+    }
+
+    // Update the latches
+
+    // Replace latch terminator destinations.
+    FC0.Latch->getTerminator()->replaceUsesOfWith(FC0.Header, FC1.Header);
+    FC1.Latch->getTerminator()->replaceUsesOfWith(FC1.Header, FC0.Header);
+
+    // If FC0.Latch and FC0.ExitingBlock are the same then we have already
+    // performed the updates above.
+    if (FC0.Latch != FC0.ExitingBlock)
+      TreeUpdates.emplace_back(DominatorTree::UpdateType(
+          DominatorTree::Insert, FC0.Latch, FC1.Header));
+
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC0.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Insert,
+                                                       FC1.Latch, FC0.Header));
+    TreeUpdates.emplace_back(DominatorTree::UpdateType(DominatorTree::Delete,
+                                                       FC1.Latch, FC1.Header));
+
+    // All done
+    // Apply the updates to the Dominator Tree and cleanup.
+
+    assert(succ_begin(FC1GuardBlock) == succ_end(FC1GuardBlock) &&
+           "FC1GuardBlock has successors!!");
+    assert(pred_begin(FC1GuardBlock) == pred_end(FC1GuardBlock) &&
+           "FC1GuardBlock has predecessors!!");
+
+    // Update DT/PDT
+    DTU.applyUpdates(TreeUpdates);
+
+    LI.removeBlock(FC1.Preheader);
+    DTU.deleteBB(FC1.Preheader);
+    DTU.deleteBB(FC0.ExitBlock);
+    DTU.flush();
+
+    // Is there a way to keep SE up-to-date so we don't need to forget the loops
+    // and rebuild the information in subsequent passes of fusion?
+    SE.forgetLoop(FC1.L);
+    SE.forgetLoop(FC0.L);
+
+    // Merge the loops.
+    SmallVector<BasicBlock *, 8> Blocks(FC1.L->block_begin(),
+                                        FC1.L->block_end());
+    for (BasicBlock *BB : Blocks) {
+      FC0.L->addBlockEntry(BB);
+      FC1.L->removeBlockFromLoop(BB);
+      if (LI.getLoopFor(BB) != FC1.L)
+        continue;
+      LI.changeLoopFor(BB, FC0.L);
+    }
+    while (!FC1.L->empty()) {
+      const auto &ChildLoopIt = FC1.L->begin();
+      Loop *ChildLoop = *ChildLoopIt;
+      FC1.L->removeChildLoop(ChildLoopIt);
+      FC0.L->addChildLoop(ChildLoop);
+    }
+
+    // Delete the now empty loop L1.
+    LI.erase(FC1.L);
+
+#ifndef NDEBUG
+    assert(!verifyFunction(*FC0.Header->getParent(), &errs()));
+    assert(DT.verify(DominatorTree::VerificationLevel::Fast));
+    assert(PDT.verify());
+    LI.verify(DT);
+    SE.verify();
+#endif
+
+    LLVM_DEBUG(dbgs() << "Fusion done:\n");
+
+    return FC0.L;
+  }
 };
 
 struct LoopFuseLegacy : public FunctionPass {
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 62ceba6414a59..852bbefaf20b6 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -3165,6 +3165,7 @@ void LSRInstance::GenerateIVChain(const IVChain &Chain, SCEVExpander &Rewriter,
     LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
     return;
   }
+  assert(IVSrc && "Failed to find IV chain source");
 
   LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
   Type *IVTy = IVSrc->getType();
@@ -4835,6 +4836,7 @@ void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
         }
       }
     }
+    assert(Best && "Failed to find best LSRUse candidate");
 
     LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
                       << " will yield profitable reuse.\n");
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index 1b2dd7dacdf39..8d88be4203141 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -166,7 +166,7 @@ static bool computeUnrollAndJamCount(
   bool UseUpperBound = false;
   bool ExplicitUnroll = computeUnrollCount(
       L, TTI, DT, LI, SE, EphValues, ORE, OuterTripCount, MaxTripCount,
-      OuterTripMultiple, OuterLoopSize, UP, UseUpperBound);
+      /*MaxOrZero*/ false, OuterTripMultiple, OuterLoopSize, UP, UseUpperBound);
   if (ExplicitUnroll || UseUpperBound) {
     // If the user explicitly set the loop as unrolled, dont UnJ it. Leave it
     // for the unroller instead.
@@ -295,7 +295,7 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
 
   TargetTransformInfo::UnrollingPreferences UP =
       gatherUnrollingPreferences(L, SE, TTI, nullptr, nullptr, OptLevel, None,
-                                 None, None, None, None, None, None);
+                                 None, None, None, None, None, None, None);
   if (AllowUnrollAndJam.getNumOccurrences() > 0)
     UP.UnrollAndJam = AllowUnrollAndJam;
   if (UnrollAndJamThreshold.getNumOccurrences() > 0)
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index d52ce9e815b66..a6d4164c36455 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -179,7 +179,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     Optional<unsigned> UserThreshold, Optional<unsigned> UserCount,
     Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
     Optional<bool> UserUpperBound, Optional<bool> UserAllowPeeling,
-    Optional<bool> UserAllowProfileBasedPeeling) {
+    Optional<bool> UserAllowProfileBasedPeeling,
+    Optional<unsigned> UserFullUnrollMaxCount) {
   TargetTransformInfo::UnrollingPreferences UP;
 
   // Set up the defaults
@@ -261,6 +262,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
     UP.AllowPeeling = *UserAllowPeeling;
   if (UserAllowProfileBasedPeeling.hasValue())
     UP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
+  if (UserFullUnrollMaxCount.hasValue())
+    UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;
 
   return UP;
 }
@@ -734,7 +737,7 @@ bool llvm::computeUnrollCount(
     Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
     ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues,
     OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount,
-    unsigned &TripMultiple, unsigned LoopSize,
+    bool MaxOrZero, unsigned &TripMultiple, unsigned LoopSize,
     TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound) {
 
   // Check for explicit Count.
@@ -785,18 +788,34 @@ bool llvm::computeUnrollCount(
   // Also we need to check if we exceed FullUnrollMaxCount.
   // If using the upper bound to unroll, TripMultiple should be set to 1 because
   // we do not know when loop may exit.
-  // MaxTripCount and ExactTripCount cannot both be non zero since we only
+
+  // We can unroll by the upper bound amount if it's generally allowed or if
+  // we know that the loop is executed either the upper bound or zero times.
+  // (MaxOrZero unrolling keeps only the first loop test, so the number of
+  // loop tests remains the same compared to the non-unrolled version, whereas
+  // the generic upper bound unrolling keeps all but the last loop test so the
+  // number of loop tests goes up which may end up being worse on targets with
+  // constrained branch predictor resources so is controlled by an option.)
+  // In addition we only unroll small upper bounds.
+  unsigned FullUnrollMaxTripCount = MaxTripCount;
+  if (!(UP.UpperBound || MaxOrZero) ||
+      FullUnrollMaxTripCount > UnrollMaxUpperBound)
+    FullUnrollMaxTripCount = 0;
+
+  // UnrollByMaxCount and ExactTripCount cannot both be non zero since we only
   // compute the former when the latter is zero.
   unsigned ExactTripCount = TripCount;
-  assert((ExactTripCount == 0 || MaxTripCount == 0) &&
-         "ExtractTripCount and MaxTripCount cannot both be non zero.");
-  unsigned FullUnrollTripCount = ExactTripCount ? ExactTripCount : MaxTripCount;
+  assert((ExactTripCount == 0 || FullUnrollMaxTripCount == 0) &&
+         "ExtractTripCount and UnrollByMaxCount cannot both be non zero.");
+
+  unsigned FullUnrollTripCount =
+      ExactTripCount ? ExactTripCount : FullUnrollMaxTripCount;
   UP.Count = FullUnrollTripCount;
   if (FullUnrollTripCount && FullUnrollTripCount <= UP.FullUnrollMaxCount) {
     // When computing the unrolled size, note that BEInsns are not replicated
     // like the rest of the loop body.
     if (getUnrolledLoopSize(LoopSize, UP) < UP.Threshold) {
-      UseUpperBound = (MaxTripCount == FullUnrollTripCount);
+      UseUpperBound = (FullUnrollMaxTripCount == FullUnrollTripCount);
       TripCount = FullUnrollTripCount;
       TripMultiple = UP.UpperBound ? 1 : TripMultiple;
       return ExplicitUnroll;
@@ -810,7 +829,7 @@ bool llvm::computeUnrollCount(
         unsigned Boost =
             getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);
         if (Cost->UnrolledCost < UP.Threshold * Boost / 100) {
-          UseUpperBound = (MaxTripCount == FullUnrollTripCount);
+          UseUpperBound = (FullUnrollMaxTripCount == FullUnrollTripCount);
           TripCount = FullUnrollTripCount;
           TripMultiple = UP.UpperBound ? 1 : TripMultiple;
           return ExplicitUnroll;
@@ -886,6 +905,8 @@ bool llvm::computeUnrollCount(
                   "because "
                   "unrolled size is too large.";
       });
+    LLVM_DEBUG(dbgs() << "  partially unrolling with count: " << UP.Count
+                      << "\n");
     return ExplicitUnroll;
   }
   assert(TripCount == 0 &&
@@ -907,6 +928,12 @@ bool llvm::computeUnrollCount(
     return false;
   }
 
+  // Don't unroll a small upper bound loop unless user or TTI asked to do so.
+  if (MaxTripCount && !UP.Force && MaxTripCount < UnrollMaxUpperBound) {
+    UP.Count = 0;
+    return false;
+  }
+
   // Check if the runtime trip count is too small when profile is available.
   if (L->getHeader()->getParent()->hasProfileData()) {
     if (auto ProfileTripCount = getLoopEstimatedTripCount(L)) {
@@ -970,7 +997,11 @@ bool llvm::computeUnrollCount(
 
   if (UP.Count > UP.MaxCount)
     UP.Count = UP.MaxCount;
-  LLVM_DEBUG(dbgs() << "  partially unrolling with count: " << UP.Count
+
+  if (MaxTripCount && UP.Count > MaxTripCount)
+    UP.Count = MaxTripCount;
+
+  LLVM_DEBUG(dbgs() << "  runtime unrolling with count: " << UP.Count
                     << "\n");
   if (UP.Count < 2)
     UP.Count = 0;
@@ -986,7 +1017,8 @@ static LoopUnrollResult tryToUnrollLoop(
     Optional<unsigned> ProvidedThreshold, Optional<bool> ProvidedAllowPartial,
     Optional<bool> ProvidedRuntime, Optional<bool> ProvidedUpperBound,
     Optional<bool> ProvidedAllowPeeling,
-    Optional<bool> ProvidedAllowProfileBasedPeeling) {
+    Optional<bool> ProvidedAllowProfileBasedPeeling,
+    Optional<unsigned> ProvidedFullUnrollMaxCount) {
   LLVM_DEBUG(dbgs() << "Loop Unroll: F["
                     << L->getHeader()->getParent()->getName() << "] Loop %"
                     << L->getHeader()->getName() << "\n");
@@ -1011,7 +1043,8 @@ static LoopUnrollResult tryToUnrollLoop(
   TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
       L, SE, TTI, BFI, PSI, OptLevel, ProvidedThreshold, ProvidedCount,
       ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
-      ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
+      ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling,
+      ProvidedFullUnrollMaxCount);
 
   // Exit early if unrolling is disabled. For OptForSize, we pick the loop size
   // as threshold later on.
@@ -1032,10 +1065,10 @@ static LoopUnrollResult tryToUnrollLoop(
     return LoopUnrollResult::Unmodified;
   }
 
-  // When optimizing for size, use LoopSize as threshold, to (fully) unroll
-  // loops, if it does not increase code size.
+  // When optimizing for size, use LoopSize + 1 as threshold (we use < Threshold
+  // later), to (fully) unroll loops, if it does not increase code size.
   if (OptForSize)
-    UP.Threshold = std::max(UP.Threshold, LoopSize);
+    UP.Threshold = std::max(UP.Threshold, LoopSize + 1);
 
   if (NumInlineCandidates != 0) {
     LLVM_DEBUG(dbgs() << "  Not unrolling loop with inlinable calls.\n");
@@ -1044,7 +1077,6 @@ static LoopUnrollResult tryToUnrollLoop(
 
   // Find trip count and trip multiple if count is not available
   unsigned TripCount = 0;
-  unsigned MaxTripCount = 0;
   unsigned TripMultiple = 1;
   // If there are multiple exiting blocks but one of them is the latch, use the
   // latch for the trip count estimation. Otherwise insist on a single exiting
@@ -1074,28 +1106,18 @@ static LoopUnrollResult tryToUnrollLoop(
 
   // Try to find the trip count upper bound if we cannot find the exact trip
   // count.
+  unsigned MaxTripCount = 0;
   bool MaxOrZero = false;
   if (!TripCount) {
     MaxTripCount = SE.getSmallConstantMaxTripCount(L);
     MaxOrZero = SE.isBackedgeTakenCountMaxOrZero(L);
-    // We can unroll by the upper bound amount if it's generally allowed or if
-    // we know that the loop is executed either the upper bound or zero times.
-    // (MaxOrZero unrolling keeps only the first loop test, so the number of
-    // loop tests remains the same compared to the non-unrolled version, whereas
-    // the generic upper bound unrolling keeps all but the last loop test so the
-    // number of loop tests goes up which may end up being worse on targets with
-    // constrained branch predictor resources so is controlled by an option.)
-    // In addition we only unroll small upper bounds.
-    if (!(UP.UpperBound || MaxOrZero) || MaxTripCount > UnrollMaxUpperBound) {
-      MaxTripCount = 0;
-    }
   }
 
   // computeUnrollCount() decides whether it is beneficial to use upper bound to
   // fully unroll the loop.
   bool UseUpperBound = false;
   bool IsCountSetExplicitly = computeUnrollCount(
-      L, TTI, DT, LI, SE, EphValues, &ORE, TripCount, MaxTripCount,
+      L, TTI, DT, LI, SE, EphValues, &ORE, TripCount, MaxTripCount, MaxOrZero,
       TripMultiple, LoopSize, UP, UseUpperBound);
   if (!UP.Count)
     return LoopUnrollResult::Unmodified;
@@ -1174,6 +1196,7 @@ class LoopUnroll : public LoopPass {
   Optional<bool> ProvidedUpperBound;
   Optional<bool> ProvidedAllowPeeling;
   Optional<bool> ProvidedAllowProfileBasedPeeling;
+  Optional<unsigned> ProvidedFullUnrollMaxCount;
 
   LoopUnroll(int OptLevel = 2, bool OnlyWhenForced = false,
              bool ForgetAllSCEV = false, Optional<unsigned> Threshold = None,
@@ -1181,13 +1204,15 @@ class LoopUnroll : public LoopPass {
              Optional<bool> AllowPartial = None, Optional<bool> Runtime = None,
              Optional<bool> UpperBound = None,
              Optional<bool> AllowPeeling = None,
-             Optional<bool> AllowProfileBasedPeeling = None)
+             Optional<bool> AllowProfileBasedPeeling = None,
+             Optional<unsigned> ProvidedFullUnrollMaxCount = None)
       : LoopPass(ID), OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced),
         ForgetAllSCEV(ForgetAllSCEV), ProvidedCount(std::move(Count)),
         ProvidedThreshold(Threshold), ProvidedAllowPartial(AllowPartial),
         ProvidedRuntime(Runtime), ProvidedUpperBound(UpperBound),
         ProvidedAllowPeeling(AllowPeeling),
-        ProvidedAllowProfileBasedPeeling(AllowProfileBasedPeeling) {
+        ProvidedAllowProfileBasedPeeling(AllowProfileBasedPeeling),
+        ProvidedFullUnrollMaxCount(ProvidedFullUnrollMaxCount) {
     initializeLoopUnrollPass(*PassRegistry::getPassRegistry());
   }
 
@@ -1213,7 +1238,8 @@ class LoopUnroll : public LoopPass {
         L, DT, LI, SE, TTI, AC, ORE, nullptr, nullptr, PreserveLCSSA, OptLevel,
         OnlyWhenForced, ForgetAllSCEV, ProvidedCount, ProvidedThreshold,
         ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
-        ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
+        ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling,
+        ProvidedFullUnrollMaxCount);
 
     if (Result == LoopUnrollResult::FullyUnrolled)
       LPM.markLoopAsDeleted(*L);
@@ -1297,7 +1323,8 @@ PreservedAnalyses LoopFullUnrollPass::run(Loop &L, LoopAnalysisManager &AM,
                                  /*Threshold*/ None, /*AllowPartial*/ false,
                                  /*Runtime*/ false, /*UpperBound*/ false,
                                  /*AllowPeeling*/ false,
-                                 /*AllowProfileBasedPeeling*/ false) !=
+                                 /*AllowProfileBasedPeeling*/ false,
+                                 /*FullUnrollMaxCount*/ None) !=
                  LoopUnrollResult::Unmodified;
   if (!Changed)
     return PreservedAnalyses::all();
@@ -1439,7 +1466,7 @@ PreservedAnalyses LoopUnrollPass::run(Function &F,
         UnrollOpts.ForgetSCEV, /*Count*/ None,
         /*Threshold*/ None, UnrollOpts.AllowPartial, UnrollOpts.AllowRuntime,
         UnrollOpts.AllowUpperBound, LocalAllowPeeling,
-        UnrollOpts.AllowProfileBasedPeeling);
+        UnrollOpts.AllowProfileBasedPeeling, UnrollOpts.FullUnrollMaxCount);
     Changed |= Result != LoopUnrollResult::Unmodified;
 
     // The parent must not be damaged by unrolling!
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index cdb1d79066773..d85f20b3f80c1 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -161,7 +161,7 @@ static void handlePhiDef(CallInst *Expect) {
     return Result;
   };
 
-  auto *PhiDef = dyn_cast<PHINode>(V);
+  auto *PhiDef = cast<PHINode>(V);
 
   // Get the first dominating conditional branch of the operand
   // i's incoming block.
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 16078566e9438..37d6d6085e430 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -895,7 +895,7 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpy, Value *cpyDest,
   // If the destination wasn't sufficiently aligned then increase its alignment.
   if (!isDestSufficientlyAligned) {
     assert(isa<AllocaInst>(cpyDest) && "Can only increase alloca alignment!");
-    cast<AllocaInst>(cpyDest)->setAlignment(srcAlign);
+    cast<AllocaInst>(cpyDest)->setAlignment(MaybeAlign(srcAlign));
   }
 
   // Drop any cached information about the call, because we may have changed
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 902e459b8a90d..91c879097afc2 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -1332,7 +1332,7 @@ LoadExpression *NewGVN::createLoadExpression(Type *LoadType, Value *PointerOp,
   E->setOpcode(0);
   E->op_push_back(PointerOp);
   if (LI)
-    E->setAlignment(LI->getAlignment());
+    E->setAlignment(MaybeAlign(LI->getAlignment()));
 
   // TODO: Value number heap versions. We may be able to discover
   // things alias analysis can't on it's own (IE that a store and a
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index 47d226f83a426..48bbdd8d1b33f 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -2530,7 +2530,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
   // statepoints surviving this pass.  This makes testing easier and the
   // resulting IR less confusing to human readers.
   DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
-  bool MadeChange = removeUnreachableBlocks(F, nullptr, &DTU);
+  bool MadeChange = removeUnreachableBlocks(F, &DTU);
   // Flush the Dominator Tree.
   DTU.getDomTree();
 
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index 0ce967c8d0b7d..10fbdc8aacd2f 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -1757,7 +1757,7 @@ static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
                      [](const LatticeVal &LV) { return LV.isOverdefined(); }))
       return false;
     std::vector<Constant *> ConstVals;
-    auto *ST = dyn_cast<StructType>(V->getType());
+    auto *ST = cast<StructType>(V->getType());
     for (unsigned i = 0, e = ST->getNumElements(); i != e; ++i) {
       LatticeVal V = IVs[i];
       ConstVals.push_back(V.isConstant()
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 2aa8ae239bf56..cec65ba76edaf 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -1266,11 +1266,11 @@ static void speculatePHINodeLoads(PHINode &PN) {
   PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(),
                                         PN.getName() + ".sroa.speculated");
 
-  // Get the AA tags and alignment to use from one of the loads.  It doesn't
+  // Get the AA tags and alignment to use from one of the loads. It does not
   // matter which one we get and if any differ.
   AAMDNodes AATags;
   SomeLoad->getAAMetadata(AATags);
-  unsigned Align = SomeLoad->getAlignment();
+  const MaybeAlign Align = MaybeAlign(SomeLoad->getAlignment());
 
   // Rewrite all loads of the PN to use the new PHI.
   while (!PN.use_empty()) {
@@ -1368,8 +1368,8 @@ static void speculateSelectInstLoads(SelectInst &SI) {
     NumLoadsSpeculated += 2;
 
     // Transfer alignment and AA info if present.
-    TL->setAlignment(LI->getAlignment());
-    FL->setAlignment(LI->getAlignment());
+    TL->setAlignment(MaybeAlign(LI->getAlignment()));
+    FL->setAlignment(MaybeAlign(LI->getAlignment()));
 
     AAMDNodes Tags;
     LI->getAAMetadata(Tags);
@@ -1888,6 +1888,14 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
   bool HaveCommonEltTy = true;
   auto CheckCandidateType = [&](Type *Ty) {
     if (auto *VTy = dyn_cast<VectorType>(Ty)) {
+      // Return if bitcast to vectors is different for total size in bits.
+      if (!CandidateTys.empty()) {
+        VectorType *V = CandidateTys[0];
+        if (DL.getTypeSizeInBits(VTy) != DL.getTypeSizeInBits(V)) {
+          CandidateTys.clear();
+          return;
+        }
+      }
       CandidateTys.push_back(VTy);
       if (!CommonEltTy)
         CommonEltTy = VTy->getElementType();
@@ -3110,7 +3118,7 @@ class llvm::sroa::AllocaSliceRewriter
         unsigned LoadAlign = LI->getAlignment();
         if (!LoadAlign)
           LoadAlign = DL.getABITypeAlignment(LI->getType());
-        LI->setAlignment(std::min(LoadAlign, getSliceAlign()));
+        LI->setAlignment(MaybeAlign(std::min(LoadAlign, getSliceAlign())));
         continue;
       }
       if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 869cf00e0a89c..688b8b9079e05 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -123,6 +123,10 @@ void LLVMAddAggressiveDCEPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createAggressiveDCEPass());
 }
 
+void LLVMAddDCEPass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createDeadCodeEliminationPass());
+}
+
 void LLVMAddBitTrackingDCEPass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createBitTrackingDCEPass());
 }
diff --git a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
index ea1fbc9e73bbf..71316ce8f7583 100644
--- a/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -306,12 +306,21 @@ bool llvm::inferLibFuncAttributes(Function &F, const TargetLibraryInfo &TLI) {
   case LibFunc_memcpy:
     Changed |= setDoesNotAlias(F, 0);
     Changed |= setDoesNotAlias(F, 1);
-    LLVM_FALLTHROUGH;
+    Changed |= setReturnedArg(F, 0);
+    Changed |= setDoesNotThrow(F);
+    Changed |= setDoesNotCapture(F, 1);
+    Changed |= setOnlyReadsMemory(F, 1);
+    return Changed;
   case LibFunc_memmove:
     Changed |= setReturnedArg(F, 0);
-    LLVM_FALLTHROUGH;
+    Changed |= setDoesNotThrow(F);
+    Changed |= setDoesNotCapture(F, 1);
+    Changed |= setOnlyReadsMemory(F, 1);
+    return Changed;
   case LibFunc_mempcpy:
   case LibFunc_memccpy:
+    Changed |= setDoesNotAlias(F, 0);
+    Changed |= setDoesNotAlias(F, 1);
     Changed |= setDoesNotThrow(F);
     Changed |= setDoesNotCapture(F, 1);
     Changed |= setOnlyReadsMemory(F, 1);
@@ -842,6 +851,12 @@ Value *llvm::emitStrLen(Value *Ptr, IRBuilder<> &B, const DataLayout &DL,
                      B.getInt8PtrTy(), castToCStr(Ptr, B), B, TLI);
 }
 
+Value *llvm::emitStrDup(Value *Ptr, IRBuilder<> &B,
+                        const TargetLibraryInfo *TLI) {
+  return emitLibCall(LibFunc_strdup, B.getInt8PtrTy(), B.getInt8PtrTy(),
+                     castToCStr(Ptr, B), B, TLI);
+}
+
 Value *llvm::emitStrChr(Value *Ptr, char C, IRBuilder<> &B,
                         const TargetLibraryInfo *TLI) {
   Type *I8Ptr = B.getInt8PtrTy();
diff --git a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
index df299f673f651..9a6761040bd89 100644
--- a/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
+++ b/llvm/lib/Transforms/Utils/BypassSlowDivision.cpp
@@ -448,13 +448,17 @@ bool llvm::bypassSlowDivision(BasicBlock *BB,
   DivCacheTy PerBBDivCache;
 
   bool MadeChange = false;
-  Instruction* Next = &*BB->begin();
+  Instruction *Next = &*BB->begin();
   while (Next != nullptr) {
     // We may add instructions immediately after I, but we want to skip over
     // them.
-    Instruction* I = Next;
+    Instruction *I = Next;
     Next = Next->getNextNode();
 
+    // Ignore dead code to save time and avoid bugs.
+    if (I->hasNUses(0))
+      continue;
+
     FastDivInsertionTask Task(I, BypassWidths);
     if (Value *Replacement = Task.getReplacement(PerBBDivCache)) {
       I->replaceAllUsesWith(Replacement);
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index fa6d3f8ae8738..fcacc3a68002c 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -293,10 +293,8 @@ static BasicBlock *getCommonExitBlock(const SetVector<BasicBlock *> &Blocks) {
         CommonExitBlock = Succ;
         continue;
       }
-      if (CommonExitBlock == Succ)
-        continue;
-
-      return true;
+      if (CommonExitBlock != Succ)
+        return true;
     }
     return false;
   };
@@ -537,15 +535,41 @@ void CodeExtractor::findAllocas(ValueSet &SinkCands, ValueSet &HoistCands,
   }
 }
 
+bool CodeExtractor::isEligible() const {
+  if (Blocks.empty())
+    return false;
+  BasicBlock *Header = *Blocks.begin();
+  Function *F = Header->getParent();
+
+  // For functions with varargs, check that varargs handling is only done in the
+  // outlined function, i.e vastart and vaend are only used in outlined blocks.
+  if (AllowVarArgs && F->getFunctionType()->isVarArg()) {
+    auto containsVarArgIntrinsic = [](const Instruction &I) {
+      if (const CallInst *CI = dyn_cast<CallInst>(&I))
+        if (const Function *Callee = CI->getCalledFunction())
+          return Callee->getIntrinsicID() == Intrinsic::vastart ||
+                 Callee->getIntrinsicID() == Intrinsic::vaend;
+      return false;
+    };
+
+    for (auto &BB : *F) {
+      if (Blocks.count(&BB))
+        continue;
+      if (llvm::any_of(BB, containsVarArgIntrinsic))
+        return false;
+    }
+  }
+  return true;
+}
+
 void CodeExtractor::findInputsOutputs(ValueSet &Inputs, ValueSet &Outputs,
                                       const ValueSet &SinkCands) const {
   for (BasicBlock *BB : Blocks) {
     // If a used value is defined outside the region, it's an input.  If an
     // instruction is used outside the region, it's an output.
     for (Instruction &II : *BB) {
-      for (User::op_iterator OI = II.op_begin(), OE = II.op_end(); OI != OE;
-           ++OI) {
-        Value *V = *OI;
+      for (auto &OI : II.operands()) {
+        Value *V = OI;
         if (!SinkCands.count(V) && definedInCaller(Blocks, V))
           Inputs.insert(V);
       }
@@ -1341,27 +1365,6 @@ Function *CodeExtractor::extractCodeRegion() {
   BasicBlock *header = *Blocks.begin();
   Function *oldFunction = header->getParent();
 
-  // For functions with varargs, check that varargs handling is only done in the
-  // outlined function, i.e vastart and vaend are only used in outlined blocks.
-  if (AllowVarArgs && oldFunction->getFunctionType()->isVarArg()) {
-    auto containsVarArgIntrinsic = [](Instruction &I) {
-      if (const CallInst *CI = dyn_cast<CallInst>(&I))
-        if (const Function *F = CI->getCalledFunction())
-          return F->getIntrinsicID() == Intrinsic::vastart ||
-                 F->getIntrinsicID() == Intrinsic::vaend;
-      return false;
-    };
-
-    for (auto &BB : *oldFunction) {
-      if (Blocks.count(&BB))
-        continue;
-      if (llvm::any_of(BB, containsVarArgIntrinsic))
-        return nullptr;
-    }
-  }
-  ValueSet inputs, outputs, SinkingCands, HoistingCands;
-  BasicBlock *CommonExit = nullptr;
-
   // Calculate the entry frequency of the new function before we change the root
   //   block.
   BlockFrequency EntryFreq;
@@ -1428,6 +1431,8 @@ Function *CodeExtractor::extractCodeRegion() {
   }
   newFuncRoot->getInstList().push_back(BranchI);
 
+  ValueSet inputs, outputs, SinkingCands, HoistingCands;
+  BasicBlock *CommonExit = nullptr;
   findAllocas(SinkingCands, HoistingCands, CommonExit);
   assert(HoistingCands.empty() || CommonExit);
 
diff --git a/llvm/lib/Transforms/Utils/FlattenCFG.cpp b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
index 0c52e6f3703b8..893f23eb60482 100644
--- a/llvm/lib/Transforms/Utils/FlattenCFG.cpp
+++ b/llvm/lib/Transforms/Utils/FlattenCFG.cpp
@@ -67,7 +67,7 @@ class FlattenCFGOpt {
 /// Before:
 ///   ......
 ///   %cmp10 = fcmp une float %tmp1, %tmp2
-///   br i1 %cmp1, label %if.then, label %lor.rhs
+///   br i1 %cmp10, label %if.then, label %lor.rhs
 ///
 /// lor.rhs:
 ///   ......
@@ -251,8 +251,8 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
     bool EverChanged = false;
     for (; CurrBlock != FirstCondBlock;
          CurrBlock = CurrBlock->getSinglePredecessor()) {
-      BranchInst *BI = dyn_cast<BranchInst>(CurrBlock->getTerminator());
-      CmpInst *CI = dyn_cast<CmpInst>(BI->getCondition());
+      auto *BI = cast<BranchInst>(CurrBlock->getTerminator());
+      auto *CI = dyn_cast<CmpInst>(BI->getCondition());
       if (!CI)
         continue;
 
@@ -278,7 +278,7 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
 
   // Do the transformation.
   BasicBlock *CB;
-  BranchInst *PBI = dyn_cast<BranchInst>(FirstCondBlock->getTerminator());
+  BranchInst *PBI = cast<BranchInst>(FirstCondBlock->getTerminator());
   bool Iteration = true;
   IRBuilder<>::InsertPointGuard Guard(Builder);
   Value *PC = PBI->getCondition();
@@ -444,7 +444,7 @@ bool FlattenCFGOpt::MergeIfRegion(BasicBlock *BB, IRBuilder<> &Builder) {
   FirstEntryBlock->getInstList().pop_back();
   FirstEntryBlock->getInstList()
       .splice(FirstEntryBlock->end(), SecondEntryBlock->getInstList());
-  BranchInst *PBI = dyn_cast<BranchInst>(FirstEntryBlock->getTerminator());
+  BranchInst *PBI = cast<BranchInst>(FirstEntryBlock->getTerminator());
   Value *CC = PBI->getCondition();
   BasicBlock *SaveInsertBB = Builder.GetInsertBlock();
   BasicBlock::iterator SaveInsertPt = Builder.GetInsertPoint();
@@ -453,6 +453,16 @@ bool FlattenCFGOpt::MergeIfRegion(BasicBlock *BB, IRBuilder<> &Builder) {
   PBI->replaceUsesOfWith(CC, NC);
   Builder.SetInsertPoint(SaveInsertBB, SaveInsertPt);
 
+  // Handle PHI node to replace its predecessors to FirstEntryBlock.
+  for (BasicBlock *Succ : successors(PBI)) {
+    for (PHINode &Phi : Succ->phis()) {
+      for (unsigned i = 0, e = Phi.getNumIncomingValues(); i != e; ++i) {
+        if (Phi.getIncomingBlock(i) == SecondEntryBlock)
+          Phi.setIncomingBlock(i, FirstEntryBlock);
+      }
+    }
+  }
+
   // Remove IfTrue1
   if (IfTrue1 != FirstEntryBlock) {
     IfTrue1->dropAllReferences();
diff --git a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
index c9cc0990f237c..76b4635ad5016 100644
--- a/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
+++ b/llvm/lib/Transforms/Utils/FunctionImportUtils.cpp
@@ -210,7 +210,7 @@ void FunctionImportGlobalProcessing::processGlobalForThinLTO(GlobalValue &GV) {
       if (Function *F = dyn_cast<Function>(&GV)) {
         if (!F->isDeclaration()) {
           for (auto &S : VI.getSummaryList()) {
-            FunctionSummary *FS = dyn_cast<FunctionSummary>(S->getBaseObject());
+            auto *FS = cast<FunctionSummary>(S->getBaseObject());
             if (FS->modulePath() == M.getModuleIdentifier()) {
               F->setEntryCount(Function::ProfileCount(FS->entryCount(),
                                                       Function::PCT_Synthetic));
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 7242eb1d3de24..94339c2ba00f9 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -1132,10 +1132,10 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
 /// often possible though. If alignment is important, a more reliable approach
 /// is to simply align all global variables and allocation instructions to
 /// their preferred alignment from the beginning.
-static unsigned enforceKnownAlignment(Value *V, unsigned Align,
+static unsigned enforceKnownAlignment(Value *V, unsigned Alignment,
                                       unsigned PrefAlign,
                                       const DataLayout &DL) {
-  assert(PrefAlign > Align);
+  assert(PrefAlign > Alignment);
 
   V = V->stripPointerCasts();
 
@@ -1146,36 +1146,36 @@ static unsigned enforceKnownAlignment(Value *V, unsigned Align,
     // stripPointerCasts recurses through infinite layers of bitcasts,
     // while computeKnownBits is not allowed to traverse more than 6
     // levels.
-    Align = std::max(AI->getAlignment(), Align);
-    if (PrefAlign <= Align)
-      return Align;
+    Alignment = std::max(AI->getAlignment(), Alignment);
+    if (PrefAlign <= Alignment)
+      return Alignment;
 
     // If the preferred alignment is greater than the natural stack alignment
     // then don't round up. This avoids dynamic stack realignment.
-    if (DL.exceedsNaturalStackAlignment(llvm::Align(PrefAlign)))
-      return Align;
-    AI->setAlignment(PrefAlign);
+    if (DL.exceedsNaturalStackAlignment(Align(PrefAlign)))
+      return Alignment;
+    AI->setAlignment(MaybeAlign(PrefAlign));
     return PrefAlign;
   }
 
   if (auto *GO = dyn_cast<GlobalObject>(V)) {
     // TODO: as above, this shouldn't be necessary.
-    Align = std::max(GO->getAlignment(), Align);
-    if (PrefAlign <= Align)
-      return Align;
+    Alignment = std::max(GO->getAlignment(), Alignment);
+    if (PrefAlign <= Alignment)
+      return Alignment;
 
     // If there is a large requested alignment and we can, bump up the alignment
     // of the global.  If the memory we set aside for the global may not be the
     // memory used by the final program then it is impossible for us to reliably
     // enforce the preferred alignment.
     if (!GO->canIncreaseAlignment())
-      return Align;
+      return Alignment;
 
     GO->setAlignment(PrefAlign);
     return PrefAlign;
   }
 
-  return Align;
+  return Alignment;
 }
 
 unsigned llvm::getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign,
@@ -2210,12 +2210,10 @@ void llvm::removeUnwindEdge(BasicBlock *BB, DomTreeUpdater *DTU) {
 
 /// removeUnreachableBlocks - Remove blocks that are not reachable, even
 /// if they are in a dead cycle.  Return true if a change was made, false
-/// otherwise. If `LVI` is passed, this function preserves LazyValueInfo
-/// after modifying the CFG.
-bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI,
-                                   DomTreeUpdater *DTU,
+/// otherwise.
+bool llvm::removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
                                    MemorySSAUpdater *MSSAU) {
-  SmallPtrSet<BasicBlock*, 16> Reachable;
+  SmallPtrSet<BasicBlock *, 16> Reachable;
   bool Changed = markAliveBlocks(F, Reachable, DTU);
 
   // If there are unreachable blocks in the CFG...
@@ -2223,21 +2221,21 @@ bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI,
     return Changed;
 
   assert(Reachable.size() < F.size());
-  NumRemoved += F.size()-Reachable.size();
+  NumRemoved += F.size() - Reachable.size();
 
   SmallSetVector<BasicBlock *, 8> DeadBlockSet;
-  for (Function::iterator I = ++F.begin(), E = F.end(); I != E; ++I) {
-    auto *BB = &*I;
-    if (Reachable.count(BB))
+  for (BasicBlock &BB : F) {
+    // Skip reachable basic blocks
+    if (Reachable.find(&BB) != Reachable.end())
       continue;
-    DeadBlockSet.insert(BB);
+    DeadBlockSet.insert(&BB);
   }
 
   if (MSSAU)
     MSSAU->removeBlocks(DeadBlockSet);
 
   // Loop over all of the basic blocks that are not reachable, dropping all of
-  // their internal references. Update DTU and LVI if available.
+  // their internal references. Update DTU if available.
   std::vector<DominatorTree::UpdateType> Updates;
   for (auto *BB : DeadBlockSet) {
     for (BasicBlock *Successor : successors(BB)) {
@@ -2246,26 +2244,18 @@ bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI,
       if (DTU)
         Updates.push_back({DominatorTree::Delete, BB, Successor});
     }
-    if (LVI)
-      LVI->eraseBlock(BB);
     BB->dropAllReferences();
-  }
-  for (Function::iterator I = ++F.begin(); I != F.end();) {
-    auto *BB = &*I;
-    if (Reachable.count(BB)) {
-      ++I;
-      continue;
-    }
     if (DTU) {
-      // Remove the terminator of BB to clear the successor list of BB.
-      if (BB->getTerminator())
-        BB->getInstList().pop_back();
+      Instruction *TI = BB->getTerminator();
+      assert(TI && "Basic block should have a terminator");
+      // Terminators like invoke can have users. We have to replace their users,
+      // before removing them.
+      if (!TI->use_empty())
+        TI->replaceAllUsesWith(UndefValue::get(TI->getType()));
+      TI->eraseFromParent();
       new UnreachableInst(BB->getContext(), BB);
       assert(succ_empty(BB) && "The successor list of BB isn't empty before "
                                "applying corresponding DTU updates.");
-      ++I;
-    } else {
-      I = F.getBasicBlockList().erase(I);
     }
   }
 
@@ -2281,7 +2271,11 @@ bool llvm::removeUnreachableBlocks(Function &F, LazyValueInfo *LVI,
     }
     if (!Deleted)
       return false;
+  } else {
+    for (auto *BB : DeadBlockSet)
+      BB->eraseFromParent();
   }
+
   return true;
 }
 
diff --git a/llvm/lib/Transforms/Utils/ModuleUtils.cpp b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
index c84beceee1914..1ef3757017a81 100644
--- a/llvm/lib/Transforms/Utils/ModuleUtils.cpp
+++ b/llvm/lib/Transforms/Utils/ModuleUtils.cpp
@@ -73,7 +73,7 @@ static void appendToUsedList(Module &M, StringRef Name, ArrayRef<GlobalValue *>
   SmallPtrSet<Constant *, 16> InitAsSet;
   SmallVector<Constant *, 16> Init;
   if (GV) {
-    ConstantArray *CA = dyn_cast<ConstantArray>(GV->getInitializer());
+    auto *CA = cast<ConstantArray>(GV->getInitializer());
     for (auto &Op : CA->operands()) {
       Constant *C = cast_or_null<Constant>(Op);
       if (InitAsSet.insert(C).second)
diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
index 2e0a07f105de5..7352ce83adb43 100644
--- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -94,6 +94,12 @@ static cl::opt<unsigned> PHINodeFoldingThreshold(
     cl::desc(
         "Control the amount of phi node folding to perform (default = 2)"));
 
+static cl::opt<unsigned> TwoEntryPHINodeFoldingThreshold(
+    "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
+    cl::desc("Control the maximal total instruction cost that we are willing "
+             "to speculatively execute to fold a 2-entry PHI node into a "
+             "select (default = 4)"));
+
 static cl::opt<bool> DupRet(
     "simplifycfg-dup-ret", cl::Hidden, cl::init(false),
     cl::desc("Duplicate return instructions into unconditional branches"));
@@ -332,7 +338,7 @@ static unsigned ComputeSpeculationCost(const User *I,
 /// CostRemaining, false is returned and CostRemaining is undefined.
 static bool DominatesMergePoint(Value *V, BasicBlock *BB,
                                 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
-                                unsigned &CostRemaining,
+                                int &BudgetRemaining,
                                 const TargetTransformInfo &TTI,
                                 unsigned Depth = 0) {
   // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
@@ -375,7 +381,7 @@ static bool DominatesMergePoint(Value *V, BasicBlock *BB,
   if (!isSafeToSpeculativelyExecute(I))
     return false;
 
-  unsigned Cost = ComputeSpeculationCost(I, TTI);
+  BudgetRemaining -= ComputeSpeculationCost(I, TTI);
 
   // Allow exactly one instruction to be speculated regardless of its cost
   // (as long as it is safe to do so).
@@ -383,17 +389,14 @@ static bool DominatesMergePoint(Value *V, BasicBlock *BB,
   // or other expensive operation. The speculation of an expensive instruction
   // is expected to be undone in CodeGenPrepare if the speculation has not
   // enabled further IR optimizations.
-  if (Cost > CostRemaining &&
+  if (BudgetRemaining < 0 &&
       (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0))
     return false;
 
-  // Avoid unsigned wrap.
-  CostRemaining = (Cost > CostRemaining) ? 0 : CostRemaining - Cost;
-
   // Okay, we can only really hoist these out if their operands do
   // not take us over the cost threshold.
   for (User::op_iterator i = I->op_begin(), e = I->op_end(); i != e; ++i)
-    if (!DominatesMergePoint(*i, BB, AggressiveInsts, CostRemaining, TTI,
+    if (!DominatesMergePoint(*i, BB, AggressiveInsts, BudgetRemaining, TTI,
                              Depth + 1))
       return false;
   // Okay, it's safe to do this!  Remember this instruction.
@@ -629,8 +632,7 @@ struct ConstantComparesGatherer {
   /// vector.
   /// One "Extra" case is allowed to differ from the other.
   void gather(Value *V) {
-    Instruction *I = dyn_cast<Instruction>(V);
-    bool isEQ = (I->getOpcode() == Instruction::Or);
+    bool isEQ = (cast<Instruction>(V)->getOpcode() == Instruction::Or);
 
     // Keep a stack (SmallVector for efficiency) for depth-first traversal
     SmallVector<Value *, 8> DFT;
@@ -2322,10 +2324,8 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
   // instructions.  While we are at it, keep track of the instructions
   // that need to be moved to the dominating block.
   SmallPtrSet<Instruction *, 4> AggressiveInsts;
-  unsigned MaxCostVal0 = PHINodeFoldingThreshold,
-           MaxCostVal1 = PHINodeFoldingThreshold;
-  MaxCostVal0 *= TargetTransformInfo::TCC_Basic;
-  MaxCostVal1 *= TargetTransformInfo::TCC_Basic;
+  int BudgetRemaining =
+      TwoEntryPHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
 
   for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
     PHINode *PN = cast<PHINode>(II++);
@@ -2336,9 +2336,9 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
     }
 
     if (!DominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
-                             MaxCostVal0, TTI) ||
+                             BudgetRemaining, TTI) ||
         !DominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
-                             MaxCostVal1, TTI))
+                             BudgetRemaining, TTI))
       return false;
   }
 
@@ -2400,6 +2400,7 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI,
         return false;
       }
   }
+  assert(DomBlock && "Failed to find root DomBlock");
 
   LLVM_DEBUG(dbgs() << "FOUND IF CONDITION!  " << *IfCond
                     << "  T: " << IfTrue->getName()
@@ -2945,42 +2946,8 @@ static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB,
                                            BasicBlock *QTB, BasicBlock *QFB,
                                            BasicBlock *PostBB, Value *Address,
                                            bool InvertPCond, bool InvertQCond,
-                                           const DataLayout &DL) {
-  auto IsaBitcastOfPointerType = [](const Instruction &I) {
-    return Operator::getOpcode(&I) == Instruction::BitCast &&
-           I.getType()->isPointerTy();
-  };
-
-  // If we're not in aggressive mode, we only optimize if we have some
-  // confidence that by optimizing we'll allow P and/or Q to be if-converted.
-  auto IsWorthwhile = [&](BasicBlock *BB) {
-    if (!BB)
-      return true;
-    // Heuristic: if the block can be if-converted/phi-folded and the
-    // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
-    // thread this store.
-    unsigned N = 0;
-    for (auto &I : BB->instructionsWithoutDebug()) {
-      // Cheap instructions viable for folding.
-      if (isa<BinaryOperator>(I) || isa<GetElementPtrInst>(I) ||
-          isa<StoreInst>(I))
-        ++N;
-      // Free instructions.
-      else if (I.isTerminator() || IsaBitcastOfPointerType(I))
-        continue;
-      else
-        return false;
-    }
-    // The store we want to merge is counted in N, so add 1 to make sure
-    // we're counting the instructions that would be left.
-    return N <= (PHINodeFoldingThreshold + 1);
-  };
-
-  if (!MergeCondStoresAggressively &&
-      (!IsWorthwhile(PTB) || !IsWorthwhile(PFB) || !IsWorthwhile(QTB) ||
-       !IsWorthwhile(QFB)))
-    return false;
-
+                                           const DataLayout &DL,
+                                           const TargetTransformInfo &TTI) {
   // For every pointer, there must be exactly two stores, one coming from
   // PTB or PFB, and the other from QTB or QFB. We don't support more than one
   // store (to any address) in PTB,PFB or QTB,QFB.
@@ -3021,6 +2988,46 @@ static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB,
     if (&*I != PStore && I->mayReadOrWriteMemory())
       return false;
 
+  // If we're not in aggressive mode, we only optimize if we have some
+  // confidence that by optimizing we'll allow P and/or Q to be if-converted.
+  auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
+    if (!BB)
+      return true;
+    // Heuristic: if the block can be if-converted/phi-folded and the
+    // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
+    // thread this store.
+    int BudgetRemaining =
+        PHINodeFoldingThreshold * TargetTransformInfo::TCC_Basic;
+    for (auto &I : BB->instructionsWithoutDebug()) {
+      // Consider terminator instruction to be free.
+      if (I.isTerminator())
+        continue;
+      // If this is one the stores that we want to speculate out of this BB,
+      // then don't count it's cost, consider it to be free.
+      if (auto *S = dyn_cast<StoreInst>(&I))
+        if (llvm::find(FreeStores, S))
+          continue;
+      // Else, we have a white-list of instructions that we are ak speculating.
+      if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
+        return false; // Not in white-list - not worthwhile folding.
+      // And finally, if this is a non-free instruction that we are okay
+      // speculating, ensure that we consider the speculation budget.
+      BudgetRemaining -= TTI.getUserCost(&I);
+      if (BudgetRemaining < 0)
+        return false; // Eagerly refuse to fold as soon as we're out of budget.
+    }
+    assert(BudgetRemaining >= 0 &&
+           "When we run out of budget we will eagerly return from within the "
+           "per-instruction loop.");
+    return true;
+  };
+
+  const SmallVector<StoreInst *, 2> FreeStores = {PStore, QStore};
+  if (!MergeCondStoresAggressively &&
+      (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
+       !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
+    return false;
+
   // If PostBB has more than two predecessors, we need to split it so we can
   // sink the store.
   if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
@@ -3098,7 +3105,8 @@ static bool mergeConditionalStoreToAddress(BasicBlock *PTB, BasicBlock *PFB,
 }
 
 static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
-                                   const DataLayout &DL) {
+                                   const DataLayout &DL,
+                                   const TargetTransformInfo &TTI) {
   // The intention here is to find diamonds or triangles (see below) where each
   // conditional block contains a store to the same address. Both of these
   // stores are conditional, so they can't be unconditionally sunk. But it may
@@ -3200,7 +3208,7 @@ static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
   bool Changed = false;
   for (auto *Address : CommonAddresses)
     Changed |= mergeConditionalStoreToAddress(
-        PTB, PFB, QTB, QFB, PostBB, Address, InvertPCond, InvertQCond, DL);
+        PTB, PFB, QTB, QFB, PostBB, Address, InvertPCond, InvertQCond, DL, TTI);
   return Changed;
 }
 
@@ -3209,7 +3217,8 @@ static bool mergeConditionalStores(BranchInst *PBI, BranchInst *QBI,
 /// that PBI and BI are both conditional branches, and BI is in one of the
 /// successor blocks of PBI - PBI branches to BI.
 static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
-                                           const DataLayout &DL) {
+                                           const DataLayout &DL,
+                                           const TargetTransformInfo &TTI) {
   assert(PBI->isConditional() && BI->isConditional());
   BasicBlock *BB = BI->getParent();
 
@@ -3265,7 +3274,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI,
   // If both branches are conditional and both contain stores to the same
   // address, remove the stores from the conditionals and create a conditional
   // merged store at the end.
-  if (MergeCondStores && mergeConditionalStores(PBI, BI, DL))
+  if (MergeCondStores && mergeConditionalStores(PBI, BI, DL, TTI))
     return true;
 
   // If this is a conditional branch in an empty block, and if any
@@ -3888,7 +3897,7 @@ bool SimplifyCFGOpt::SimplifyCommonResume(ResumeInst *RI) {
 // Simplify resume that is only used by a single (non-phi) landing pad.
 bool SimplifyCFGOpt::SimplifySingleResume(ResumeInst *RI) {
   BasicBlock *BB = RI->getParent();
-  LandingPadInst *LPInst = dyn_cast<LandingPadInst>(BB->getFirstNonPHI());
+  auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
   assert(RI->getValue() == LPInst &&
          "Resume must unwind the exception that caused control to here");
 
@@ -5937,7 +5946,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
     if (BranchInst *PBI = dyn_cast<BranchInst>((*PI)->getTerminator()))
       if (PBI != BI && PBI->isConditional())
-        if (SimplifyCondBranchToCondBranch(PBI, BI, DL))
+        if (SimplifyCondBranchToCondBranch(PBI, BI, DL, TTI))
           return requestResimplify();
 
   // Look for diamond patterns.
@@ -5945,7 +5954,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
     if (BasicBlock *PrevBB = allPredecessorsComeFromSameSource(BB))
       if (BranchInst *PBI = dyn_cast<BranchInst>(PrevBB->getTerminator()))
         if (PBI != BI && PBI->isConditional())
-          if (mergeConditionalStores(PBI, BI, DL))
+          if (mergeConditionalStores(PBI, BI, DL, TTI))
             return requestResimplify();
 
   return false;
diff --git a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
index b1c4a5b7c2793..1fb4f28f33641 100644
--- a/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
+++ b/llvm/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -35,6 +35,7 @@
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/KnownBits.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Transforms/Utils/BuildLibCalls.h"
 #include "llvm/Transforms/Utils/SizeOpts.h"
 
@@ -189,21 +190,23 @@ static bool canTransformToMemCmp(CallInst *CI, Value *Str, uint64_t Len,
 static void annotateDereferenceableBytes(CallInst *CI,
                                          ArrayRef<unsigned> ArgNos,
                                          uint64_t DereferenceableBytes) {
-  const Function *F = CI->getFunction();
+  const Function *F = CI->getCaller();
   if (!F)
     return;
   for (unsigned ArgNo : ArgNos) {
     uint64_t DerefBytes = DereferenceableBytes;
     unsigned AS = CI->getArgOperand(ArgNo)->getType()->getPointerAddressSpace();
-    if (!llvm::NullPointerIsDefined(F, AS))
+    if (!llvm::NullPointerIsDefined(F, AS) ||
+        CI->paramHasAttr(ArgNo, Attribute::NonNull))
       DerefBytes = std::max(CI->getDereferenceableOrNullBytes(
                                 ArgNo + AttributeList::FirstArgIndex),
                             DereferenceableBytes);
-
+  
     if (CI->getDereferenceableBytes(ArgNo + AttributeList::FirstArgIndex) <
         DerefBytes) {
       CI->removeParamAttr(ArgNo, Attribute::Dereferenceable);
-      if (!llvm::NullPointerIsDefined(F, AS))
+      if (!llvm::NullPointerIsDefined(F, AS) ||
+          CI->paramHasAttr(ArgNo, Attribute::NonNull))
         CI->removeParamAttr(ArgNo, Attribute::DereferenceableOrNull);
       CI->addParamAttr(ArgNo, Attribute::getWithDereferenceableBytes(
                                   CI->getContext(), DerefBytes));
@@ -211,6 +214,40 @@ static void annotateDereferenceableBytes(CallInst *CI,
   }
 }
 
+static void annotateNonNullBasedOnAccess(CallInst *CI,
+                                         ArrayRef<unsigned> ArgNos) {
+  Function *F = CI->getCaller();
+  if (!F)
+    return;
+
+  for (unsigned ArgNo : ArgNos) {
+    if (CI->paramHasAttr(ArgNo, Attribute::NonNull))
+      continue;
+    unsigned AS = CI->getArgOperand(ArgNo)->getType()->getPointerAddressSpace();
+    if (llvm::NullPointerIsDefined(F, AS))
+      continue;
+
+    CI->addParamAttr(ArgNo, Attribute::NonNull);
+    annotateDereferenceableBytes(CI, ArgNo, 1);
+  }
+}
+
+static void annotateNonNullAndDereferenceable(CallInst *CI, ArrayRef<unsigned> ArgNos,
+                               Value *Size, const DataLayout &DL) {
+  if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
+    annotateNonNullBasedOnAccess(CI, ArgNos);
+    annotateDereferenceableBytes(CI, ArgNos, LenC->getZExtValue());
+  } else if (isKnownNonZero(Size, DL)) {
+    annotateNonNullBasedOnAccess(CI, ArgNos);
+    const APInt *X, *Y;
+    uint64_t DerefMin = 1;
+    if (match(Size, m_Select(m_Value(), m_APInt(X), m_APInt(Y)))) {
+      DerefMin = std::min(X->getZExtValue(), Y->getZExtValue());
+      annotateDereferenceableBytes(CI, ArgNos, DerefMin);
+    }
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // String and Memory Library Call Optimizations
 //===----------------------------------------------------------------------===//
@@ -219,10 +256,13 @@ Value *LibCallSimplifier::optimizeStrCat(CallInst *CI, IRBuilder<> &B) {
   // Extract some information from the instruction
   Value *Dst = CI->getArgOperand(0);
   Value *Src = CI->getArgOperand(1);
+  annotateNonNullBasedOnAccess(CI, {0, 1});
 
   // See if we can get the length of the input string.
   uint64_t Len = GetStringLength(Src);
-  if (Len == 0)
+  if (Len)
+    annotateDereferenceableBytes(CI, 1, Len);
+  else
     return nullptr;
   --Len; // Unbias length.
 
@@ -257,24 +297,34 @@ Value *LibCallSimplifier::optimizeStrNCat(CallInst *CI, IRBuilder<> &B) {
   // Extract some information from the instruction.
   Value *Dst = CI->getArgOperand(0);
   Value *Src = CI->getArgOperand(1);
+  Value *Size = CI->getArgOperand(2);
   uint64_t Len;
+  annotateNonNullBasedOnAccess(CI, 0);
+  if (isKnownNonZero(Size, DL))
+    annotateNonNullBasedOnAccess(CI, 1);
 
   // We don't do anything if length is not constant.
-  if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
+  ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size);
+  if (LengthArg) {
     Len = LengthArg->getZExtValue();
-  else
+    // strncat(x, c, 0) -> x
+    if (!Len)
+      return Dst;
+  } else {
     return nullptr;
+  }
 
   // See if we can get the length of the input string.
   uint64_t SrcLen = GetStringLength(Src);
-  if (SrcLen == 0)
+  if (SrcLen) {
+    annotateDereferenceableBytes(CI, 1, SrcLen);
+    --SrcLen; // Unbias length.
+  } else {
     return nullptr;
-  --SrcLen; // Unbias length.
+  }
 
-  // Handle the simple, do-nothing cases:
   // strncat(x, "", c) -> x
-  // strncat(x,  c, 0) -> x
-  if (SrcLen == 0 || Len == 0)
+  if (SrcLen == 0)
     return Dst;
 
   // We don't optimize this case.
@@ -290,13 +340,18 @@ Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilder<> &B) {
   Function *Callee = CI->getCalledFunction();
   FunctionType *FT = Callee->getFunctionType();
   Value *SrcStr = CI->getArgOperand(0);
+  annotateNonNullBasedOnAccess(CI, 0);
 
   // If the second operand is non-constant, see if we can compute the length
   // of the input string and turn this into memchr.
   ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
   if (!CharC) {
     uint64_t Len = GetStringLength(SrcStr);
-    if (Len == 0 || !FT->getParamType(1)->isIntegerTy(32)) // memchr needs i32.
+    if (Len)
+      annotateDereferenceableBytes(CI, 0, Len);
+    else
+      return nullptr;
+    if (!FT->getParamType(1)->isIntegerTy(32)) // memchr needs i32.
       return nullptr;
 
     return emitMemChr(SrcStr, CI->getArgOperand(1), // include nul.
@@ -329,6 +384,7 @@ Value *LibCallSimplifier::optimizeStrChr(CallInst *CI, IRBuilder<> &B) {
 Value *LibCallSimplifier::optimizeStrRChr(CallInst *CI, IRBuilder<> &B) {
   Value *SrcStr = CI->getArgOperand(0);
   ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
+  annotateNonNullBasedOnAccess(CI, 0);
 
   // Cannot fold anything if we're not looking for a constant.
   if (!CharC)
@@ -376,7 +432,12 @@ Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilder<> &B) {
 
   // strcmp(P, "x") -> memcmp(P, "x", 2)
   uint64_t Len1 = GetStringLength(Str1P);
+  if (Len1)
+    annotateDereferenceableBytes(CI, 0, Len1);
   uint64_t Len2 = GetStringLength(Str2P);
+  if (Len2)
+    annotateDereferenceableBytes(CI, 1, Len2);
+
   if (Len1 && Len2) {
     return emitMemCmp(Str1P, Str2P,
                       ConstantInt::get(DL.getIntPtrType(CI->getContext()),
@@ -399,17 +460,22 @@ Value *LibCallSimplifier::optimizeStrCmp(CallInst *CI, IRBuilder<> &B) {
           TLI);
   }
 
+  annotateNonNullBasedOnAccess(CI, {0, 1});
   return nullptr;
 }
 
 Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilder<> &B) {
-  Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
+  Value *Str1P = CI->getArgOperand(0);
+  Value *Str2P = CI->getArgOperand(1);
+  Value *Size = CI->getArgOperand(2);
   if (Str1P == Str2P) // strncmp(x,x,n)  -> 0
     return ConstantInt::get(CI->getType(), 0);
 
+  if (isKnownNonZero(Size, DL))
+    annotateNonNullBasedOnAccess(CI, {0, 1});
   // Get the length argument if it is constant.
   uint64_t Length;
-  if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
+  if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size))
     Length = LengthArg->getZExtValue();
   else
     return nullptr;
@@ -418,7 +484,7 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilder<> &B) {
     return ConstantInt::get(CI->getType(), 0);
 
   if (Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
-    return emitMemCmp(Str1P, Str2P, CI->getArgOperand(2), B, DL, TLI);
+    return emitMemCmp(Str1P, Str2P, Size, B, DL, TLI);
 
   StringRef Str1, Str2;
   bool HasStr1 = getConstantStringInfo(Str1P, Str1);
@@ -440,7 +506,11 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilder<> &B) {
                         CI->getType());
 
   uint64_t Len1 = GetStringLength(Str1P);
+  if (Len1)
+    annotateDereferenceableBytes(CI, 0, Len1);
   uint64_t Len2 = GetStringLength(Str2P);
+  if (Len2)
+    annotateDereferenceableBytes(CI, 1, Len2);
 
   // strncmp to memcmp
   if (!HasStr1 && HasStr2) {
@@ -462,20 +532,38 @@ Value *LibCallSimplifier::optimizeStrNCmp(CallInst *CI, IRBuilder<> &B) {
   return nullptr;
 }
 
+Value *LibCallSimplifier::optimizeStrNDup(CallInst *CI, IRBuilder<> &B) {
+  Value *Src = CI->getArgOperand(0);
+  ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
+  uint64_t SrcLen = GetStringLength(Src);
+  if (SrcLen && Size) {
+    annotateDereferenceableBytes(CI, 0, SrcLen);
+    if (SrcLen <= Size->getZExtValue() + 1)
+      return emitStrDup(Src, B, TLI);
+  }
+
+  return nullptr;
+}
+
 Value *LibCallSimplifier::optimizeStrCpy(CallInst *CI, IRBuilder<> &B) {
   Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
   if (Dst == Src) // strcpy(x,x)  -> x
     return Src;
-
+  
+  annotateNonNullBasedOnAccess(CI, {0, 1});
   // See if we can get the length of the input string.
   uint64_t Len = GetStringLength(Src);
-  if (Len == 0)
+  if (Len)
+    annotateDereferenceableBytes(CI, 1, Len);
+  else
     return nullptr;
 
   // We have enough information to now generate the memcpy call to do the
   // copy for us.  Make a memcpy to copy the nul byte with align = 1.
-  B.CreateMemCpy(Dst, 1, Src, 1,
-                 ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len));
+  CallInst *NewCI =
+      B.CreateMemCpy(Dst, 1, Src, 1,
+                     ConstantInt::get(DL.getIntPtrType(CI->getContext()), Len));
+  NewCI->setAttributes(CI->getAttributes());
   return Dst;
 }
 
@@ -489,7 +577,9 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilder<> &B) {
 
   // See if we can get the length of the input string.
   uint64_t Len = GetStringLength(Src);
-  if (Len == 0)
+  if (Len)
+    annotateDereferenceableBytes(CI, 1, Len);
+  else
     return nullptr;
 
   Type *PT = Callee->getFunctionType()->getParamType(0);
@@ -499,7 +589,8 @@ Value *LibCallSimplifier::optimizeStpCpy(CallInst *CI, IRBuilder<> &B) {
 
   // We have enough information to now generate the memcpy call to do the
   // copy for us.  Make a memcpy to copy the nul byte with align = 1.
-  B.CreateMemCpy(Dst, 1, Src, 1, LenV);
+  CallInst *NewCI = B.CreateMemCpy(Dst, 1, Src, 1, LenV);
+  NewCI->setAttributes(CI->getAttributes());
   return DstEnd;
 }
 
@@ -507,37 +598,47 @@ Value *LibCallSimplifier::optimizeStrNCpy(CallInst *CI, IRBuilder<> &B) {
   Function *Callee = CI->getCalledFunction();
   Value *Dst = CI->getArgOperand(0);
   Value *Src = CI->getArgOperand(1);
-  Value *LenOp = CI->getArgOperand(2);
+  Value *Size = CI->getArgOperand(2);
+  annotateNonNullBasedOnAccess(CI, 0);
+  if (isKnownNonZero(Size, DL))
+    annotateNonNullBasedOnAccess(CI, 1);
+
+  uint64_t Len;
+  if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(Size))
+    Len = LengthArg->getZExtValue();
+  else
+    return nullptr;
+
+  // strncpy(x, y, 0) -> x
+  if (Len == 0)
+    return Dst;
 
   // See if we can get the length of the input string.
   uint64_t SrcLen = GetStringLength(Src);
-  if (SrcLen == 0)
+  if (SrcLen) {
+    annotateDereferenceableBytes(CI, 1, SrcLen);
+    --SrcLen; // Unbias length.
+  } else {
     return nullptr;
-  --SrcLen;
+  }
 
   if (SrcLen == 0) {
     // strncpy(x, "", y) -> memset(align 1 x, '\0', y)
-    B.CreateMemSet(Dst, B.getInt8('\0'), LenOp, 1);
+    CallInst *NewCI = B.CreateMemSet(Dst, B.getInt8('\0'), Size, 1);
+    AttrBuilder ArgAttrs(CI->getAttributes().getParamAttributes(0));
+    NewCI->setAttributes(NewCI->getAttributes().addParamAttributes(
+        CI->getContext(), 0, ArgAttrs));
     return Dst;
   }
 
-  uint64_t Len;
-  if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(LenOp))
-    Len = LengthArg->getZExtValue();
-  else
-    return nullptr;
-
-  if (Len == 0)
-    return Dst; // strncpy(x, y, 0) -> x
-
   // Let strncpy handle the zero padding
   if (Len > SrcLen + 1)
     return nullptr;
 
   Type *PT = Callee->getFunctionType()->getParamType(0);
   // strncpy(x, s, c) -> memcpy(align 1 x, align 1 s, c) [s and c are constant]
-  B.CreateMemCpy(Dst, 1, Src, 1, ConstantInt::get(DL.getIntPtrType(PT), Len));
-
+  CallInst *NewCI = B.CreateMemCpy(Dst, 1, Src, 1, ConstantInt::get(DL.getIntPtrType(PT), Len));
+  NewCI->setAttributes(CI->getAttributes());
   return Dst;
 }
 
@@ -633,7 +734,10 @@ Value *LibCallSimplifier::optimizeStringLength(CallInst *CI, IRBuilder<> &B,
 }
 
 Value *LibCallSimplifier::optimizeStrLen(CallInst *CI, IRBuilder<> &B) {
-  return optimizeStringLength(CI, B, 8);
+  if (Value *V = optimizeStringLength(CI, B, 8))
+    return V;
+  annotateNonNullBasedOnAccess(CI, 0);
+  return nullptr;
 }
 
 Value *LibCallSimplifier::optimizeWcslen(CallInst *CI, IRBuilder<> &B) {
@@ -781,23 +885,35 @@ Value *LibCallSimplifier::optimizeStrStr(CallInst *CI, IRBuilder<> &B) {
     Value *StrChr = emitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TLI);
     return StrChr ? B.CreateBitCast(StrChr, CI->getType()) : nullptr;
   }
+
+  annotateNonNullBasedOnAccess(CI, {0, 1});
+  return nullptr;
+}
+
+Value *LibCallSimplifier::optimizeMemRChr(CallInst *CI, IRBuilder<> &B) {
+  if (isKnownNonZero(CI->getOperand(2), DL))
+    annotateNonNullBasedOnAccess(CI, 0);
   return nullptr;
 }
 
 Value *LibCallSimplifier::optimizeMemChr(CallInst *CI, IRBuilder<> &B) {
   Value *SrcStr = CI->getArgOperand(0);
+  Value *Size = CI->getArgOperand(2);
+  annotateNonNullAndDereferenceable(CI, 0, Size, DL);
   ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
-  ConstantInt *LenC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
+  ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
 
   // memchr(x, y, 0) -> null
   if (LenC) {
     if (LenC->isZero())
       return Constant::getNullValue(CI->getType());
-    annotateDereferenceableBytes(CI, {0}, LenC->getZExtValue());
+  } else {
+    // From now on we need at least constant length and string.
+    return nullptr;
   }
-  // From now on we need at least constant length and string.
+
   StringRef Str;
-  if (!LenC || !getConstantStringInfo(SrcStr, Str, 0, /*TrimAtNul=*/false))
+  if (!getConstantStringInfo(SrcStr, Str, 0, /*TrimAtNul=*/false))
     return nullptr;
 
   // Truncate the string to LenC. If Str is smaller than LenC we will still only
@@ -940,6 +1056,7 @@ static Value *optimizeMemCmpConstantSize(CallInst *CI, Value *LHS, Value *RHS,
       Ret = 1;
     return ConstantInt::get(CI->getType(), Ret);
   }
+
   return nullptr;
 }
 
@@ -952,14 +1069,19 @@ Value *LibCallSimplifier::optimizeMemCmpBCmpCommon(CallInst *CI,
   if (LHS == RHS) // memcmp(s,s,x) -> 0
     return Constant::getNullValue(CI->getType());
 
+  annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
   // Handle constant lengths.
-  if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size)) {
-    if (Value *Res = optimizeMemCmpConstantSize(CI, LHS, RHS,
-                                                LenC->getZExtValue(), B, DL))
-      return Res;
-    annotateDereferenceableBytes(CI, {0, 1}, LenC->getZExtValue());
-  }
+  ConstantInt *LenC = dyn_cast<ConstantInt>(Size);
+  if (!LenC)
+    return nullptr;
+
+  // memcmp(d,s,0) -> 0
+  if (LenC->getZExtValue() == 0)
+    return Constant::getNullValue(CI->getType());
 
+  if (Value *Res =
+          optimizeMemCmpConstantSize(CI, LHS, RHS, LenC->getZExtValue(), B, DL))
+    return Res;
   return nullptr;
 }
 
@@ -984,17 +1106,16 @@ Value *LibCallSimplifier::optimizeBCmp(CallInst *CI, IRBuilder<> &B) {
   return optimizeMemCmpBCmpCommon(CI, B);
 }
 
-Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilder<> &B,
-                                         bool isIntrinsic) {
+Value *LibCallSimplifier::optimizeMemCpy(CallInst *CI, IRBuilder<> &B) {
   Value *Size = CI->getArgOperand(2);
-  if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size))
-    annotateDereferenceableBytes(CI, {0, 1}, LenC->getZExtValue());
-
-  if (isIntrinsic)
+  annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
+  if (isa<IntrinsicInst>(CI))
     return nullptr;
 
   // memcpy(x, y, n) -> llvm.memcpy(align 1 x, align 1 y, n)
-  B.CreateMemCpy(CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, Size);
+  CallInst *NewCI =
+      B.CreateMemCpy(CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, Size);
+  NewCI->setAttributes(CI->getAttributes());
   return CI->getArgOperand(0);
 }
 
@@ -1007,17 +1128,17 @@ Value *LibCallSimplifier::optimizeMemPCpy(CallInst *CI, IRBuilder<> &B) {
   return B.CreateInBoundsGEP(B.getInt8Ty(), Dst, N);
 }
 
-Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilder<> &B, bool isIntrinsic) {
+Value *LibCallSimplifier::optimizeMemMove(CallInst *CI, IRBuilder<> &B) {
   Value *Size = CI->getArgOperand(2);
-  if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size))
-    annotateDereferenceableBytes(CI, {0, 1}, LenC->getZExtValue());
-
-  if (isIntrinsic)
+  annotateNonNullAndDereferenceable(CI, {0, 1}, Size, DL);
+  if (isa<IntrinsicInst>(CI))
     return nullptr;
 
   // memmove(x, y, n) -> llvm.memmove(align 1 x, align 1 y, n)
-  B.CreateMemMove( CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, Size);
-  return  CI->getArgOperand(0);
+  CallInst *NewCI =
+      B.CreateMemMove(CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, Size);
+  NewCI->setAttributes(CI->getAttributes());
+  return CI->getArgOperand(0);
 }
 
 /// Fold memset[_chk](malloc(n), 0, n) --> calloc(1, n).
@@ -1064,13 +1185,10 @@ Value *LibCallSimplifier::foldMallocMemset(CallInst *Memset, IRBuilder<> &B) {
   return nullptr;
 }
 
-Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilder<> &B,
-                                         bool isIntrinsic) {
+Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilder<> &B) {
   Value *Size = CI->getArgOperand(2);
-  if (ConstantInt *LenC = dyn_cast<ConstantInt>(Size))
-    annotateDereferenceableBytes(CI, {0}, LenC->getZExtValue());
-
-  if (isIntrinsic)
+  annotateNonNullAndDereferenceable(CI, 0, Size, DL);
+  if (isa<IntrinsicInst>(CI))
     return nullptr;
 
   if (auto *Calloc = foldMallocMemset(CI, B))
@@ -1078,7 +1196,8 @@ Value *LibCallSimplifier::optimizeMemSet(CallInst *CI, IRBuilder<> &B,
 
   // memset(p, v, n) -> llvm.memset(align 1 p, v, n)
   Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
-  B.CreateMemSet(CI->getArgOperand(0), Val, Size, 1);
+  CallInst *NewCI = B.CreateMemSet(CI->getArgOperand(0), Val, Size, 1);
+  NewCI->setAttributes(CI->getAttributes());
   return CI->getArgOperand(0);
 }
 
@@ -1153,21 +1272,18 @@ static Value *optimizeDoubleFP(CallInst *CI, IRBuilder<> &B,
   if (!V[0] || (isBinary && !V[1]))
     return nullptr;
 
-  StringRef CalleeNm = CalleeFn->getName();
-  AttributeList CalleeAt = CalleeFn->getAttributes();
-  bool CalleeIn = CalleeFn->isIntrinsic();
-
   // If call isn't an intrinsic, check that it isn't within a function with the
   // same name as the float version of this call, otherwise the result is an
   // infinite loop.  For example, from MinGW-w64:
   //
   // float expf(float val) { return (float) exp((double) val); }
-  if (!CalleeIn) {
-    const Function *Fn = CI->getFunction();
-    StringRef FnName = Fn->getName();
-    if (FnName.back() == 'f' &&
-        FnName.size() == (CalleeNm.size() + 1) &&
-        FnName.startswith(CalleeNm))
+  StringRef CalleeName = CalleeFn->getName();
+  bool IsIntrinsic = CalleeFn->isIntrinsic();
+  if (!IsIntrinsic) {
+    StringRef CallerName = CI->getFunction()->getName();
+    if (!CallerName.empty() && CallerName.back() == 'f' &&
+        CallerName.size() == (CalleeName.size() + 1) &&
+        CallerName.startswith(CalleeName))
       return nullptr;
   }
 
@@ -1177,16 +1293,16 @@ static Value *optimizeDoubleFP(CallInst *CI, IRBuilder<> &B,
 
   // g((double) float) -> (double) gf(float)
   Value *R;
-  if (CalleeIn) {
+  if (IsIntrinsic) {
     Module *M = CI->getModule();
     Intrinsic::ID IID = CalleeFn->getIntrinsicID();
     Function *Fn = Intrinsic::getDeclaration(M, IID, B.getFloatTy());
     R = isBinary ? B.CreateCall(Fn, V) : B.CreateCall(Fn, V[0]);
+  } else {
+    AttributeList CalleeAttrs = CalleeFn->getAttributes();
+    R = isBinary ? emitBinaryFloatFnCall(V[0], V[1], CalleeName, B, CalleeAttrs)
+                 : emitUnaryFloatFnCall(V[0], CalleeName, B, CalleeAttrs);
   }
-  else
-    R = isBinary ? emitBinaryFloatFnCall(V[0], V[1], CalleeNm, B, CalleeAt)
-                 : emitUnaryFloatFnCall(V[0], CalleeNm, B, CalleeAt);
-
   return B.CreateFPExt(R, B.getDoubleTy());
 }
 
@@ -1342,9 +1458,7 @@ Value *LibCallSimplifier::replacePowWithExp(CallInst *Pow, IRBuilder<> &B) {
       StringRef ExpName;
       Intrinsic::ID ID;
       Value *ExpFn;
-      LibFunc LibFnFloat;
-      LibFunc LibFnDouble;
-      LibFunc LibFnLongDouble;
+      LibFunc LibFnFloat, LibFnDouble, LibFnLongDouble;
 
       switch (LibFn) {
       default:
@@ -1694,48 +1808,157 @@ Value *LibCallSimplifier::optimizeFMinFMax(CallInst *CI, IRBuilder<> &B) {
   return B.CreateCall(F, { CI->getArgOperand(0), CI->getArgOperand(1) });
 }
 
-Value *LibCallSimplifier::optimizeLog(CallInst *CI, IRBuilder<> &B) {
-  Function *Callee = CI->getCalledFunction();
+Value *LibCallSimplifier::optimizeLog(CallInst *Log, IRBuilder<> &B) {
+  Function *LogFn = Log->getCalledFunction();
+  AttributeList Attrs = LogFn->getAttributes();
+  StringRef LogNm = LogFn->getName();
+  Intrinsic::ID LogID = LogFn->getIntrinsicID();
+  Module *Mod = Log->getModule();
+  Type *Ty = Log->getType();
   Value *Ret = nullptr;
-  StringRef Name = Callee->getName();
-  if (UnsafeFPShrink && hasFloatVersion(Name))
-    Ret = optimizeUnaryDoubleFP(CI, B, true);
 
-  if (!CI->isFast())
-    return Ret;
-  Value *Op1 = CI->getArgOperand(0);
-  auto *OpC = dyn_cast<CallInst>(Op1);
+  if (UnsafeFPShrink && hasFloatVersion(LogNm))
+    Ret = optimizeUnaryDoubleFP(Log, B, true);
 
   // The earlier call must also be 'fast' in order to do these transforms.
-  if (!OpC || !OpC->isFast())
+  CallInst *Arg = dyn_cast<CallInst>(Log->getArgOperand(0));
+  if (!Log->isFast() || !Arg || !Arg->isFast() || !Arg->hasOneUse())
     return Ret;
 
-  // log(pow(x,y)) -> y*log(x)
-  // This is only applicable to log, log2, log10.
-  if (Name != "log" && Name != "log2" && Name != "log10")
+  LibFunc LogLb, ExpLb, Exp2Lb, Exp10Lb, PowLb;
+
+  // This is only applicable to log(), log2(), log10().
+  if (TLI->getLibFunc(LogNm, LogLb))
+    switch (LogLb) {
+    case LibFunc_logf:
+      LogID = Intrinsic::log;
+      ExpLb = LibFunc_expf;
+      Exp2Lb = LibFunc_exp2f;
+      Exp10Lb = LibFunc_exp10f;
+      PowLb = LibFunc_powf;
+      break;
+    case LibFunc_log:
+      LogID = Intrinsic::log;
+      ExpLb = LibFunc_exp;
+      Exp2Lb = LibFunc_exp2;
+      Exp10Lb = LibFunc_exp10;
+      PowLb = LibFunc_pow;
+      break;
+    case LibFunc_logl:
+      LogID = Intrinsic::log;
+      ExpLb = LibFunc_expl;
+      Exp2Lb = LibFunc_exp2l;
+      Exp10Lb = LibFunc_exp10l;
+      PowLb = LibFunc_powl;
+      break;
+    case LibFunc_log2f:
+      LogID = Intrinsic::log2;
+      ExpLb = LibFunc_expf;
+      Exp2Lb = LibFunc_exp2f;
+      Exp10Lb = LibFunc_exp10f;
+      PowLb = LibFunc_powf;
+      break;
+    case LibFunc_log2:
+      LogID = Intrinsic::log2;
+      ExpLb = LibFunc_exp;
+      Exp2Lb = LibFunc_exp2;
+      Exp10Lb = LibFunc_exp10;
+      PowLb = LibFunc_pow;
+      break;
+    case LibFunc_log2l:
+      LogID = Intrinsic::log2;
+      ExpLb = LibFunc_expl;
+      Exp2Lb = LibFunc_exp2l;
+      Exp10Lb = LibFunc_exp10l;
+      PowLb = LibFunc_powl;
+      break;
+    case LibFunc_log10f:
+      LogID = Intrinsic::log10;
+      ExpLb = LibFunc_expf;
+      Exp2Lb = LibFunc_exp2f;
+      Exp10Lb = LibFunc_exp10f;
+      PowLb = LibFunc_powf;
+      break;
+    case LibFunc_log10:
+      LogID = Intrinsic::log10;
+      ExpLb = LibFunc_exp;
+      Exp2Lb = LibFunc_exp2;
+      Exp10Lb = LibFunc_exp10;
+      PowLb = LibFunc_pow;
+      break;
+    case LibFunc_log10l:
+      LogID = Intrinsic::log10;
+      ExpLb = LibFunc_expl;
+      Exp2Lb = LibFunc_exp2l;
+      Exp10Lb = LibFunc_exp10l;
+      PowLb = LibFunc_powl;
+      break;
+    default:
+      return Ret;
+    }
+  else if (LogID == Intrinsic::log || LogID == Intrinsic::log2 ||
+           LogID == Intrinsic::log10) {
+    if (Ty->getScalarType()->isFloatTy()) {
+      ExpLb = LibFunc_expf;
+      Exp2Lb = LibFunc_exp2f;
+      Exp10Lb = LibFunc_exp10f;
+      PowLb = LibFunc_powf;
+    } else if (Ty->getScalarType()->isDoubleTy()) {
+      ExpLb = LibFunc_exp;
+      Exp2Lb = LibFunc_exp2;
+      Exp10Lb = LibFunc_exp10;
+      PowLb = LibFunc_pow;
+    } else
+      return Ret;
+  } else
     return Ret;
 
   IRBuilder<>::FastMathFlagGuard Guard(B);
-  FastMathFlags FMF;
-  FMF.setFast();
-  B.setFastMathFlags(FMF);
+  B.setFastMathFlags(FastMathFlags::getFast());
+
+  Function *ArgFn = Arg->getCalledFunction();
+  StringRef ArgNm = ArgFn->getName();
+  Intrinsic::ID ArgID = ArgFn->getIntrinsicID();
+  LibFunc ArgLb = NotLibFunc;
+  TLI->getLibFunc(ArgNm, ArgLb);
+
+  // log(pow(x,y)) -> y*log(x)
+  if (ArgLb == PowLb || ArgID == Intrinsic::pow) {
+    Value *LogX =
+        Log->doesNotAccessMemory()
+            ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
+                           Arg->getOperand(0), "log")
+            : emitUnaryFloatFnCall(Arg->getOperand(0), LogNm, B, Attrs);
+    Value *MulY = B.CreateFMul(Arg->getArgOperand(1), LogX, "mul");
+    // Since pow() may have side effects, e.g. errno,
+    // dead code elimination may not be trusted to remove it.
+    substituteInParent(Arg, MulY);
+    return MulY;
+  }
+  // log(exp{,2,10}(y)) -> y*log({e,2,10})
+  // TODO: There is no exp10() intrinsic yet.
+  else if (ArgLb == ExpLb || ArgLb == Exp2Lb || ArgLb == Exp10Lb ||
+           ArgID == Intrinsic::exp || ArgID == Intrinsic::exp2) {
+    Constant *Eul;
+    if (ArgLb == ExpLb || ArgID == Intrinsic::exp)
+      // FIXME: The Euler number should be M_E, but it's place of definition
+      // is not quite standard.
+      Eul = ConstantFP::get(Log->getType(), 2.7182818284590452354);
+    else if (ArgLb == Exp2Lb || ArgID == Intrinsic::exp2)
+      Eul = ConstantFP::get(Log->getType(), 2.0);
+    else
+      Eul = ConstantFP::get(Log->getType(), 10.0);
+    Value *LogE = Log->doesNotAccessMemory()
+                      ? B.CreateCall(Intrinsic::getDeclaration(Mod, LogID, Ty),
+                                     Eul, "log")
+                      : emitUnaryFloatFnCall(Eul, LogNm, B, Attrs);
+    Value *MulY = B.CreateFMul(Arg->getArgOperand(0), LogE, "mul");
+    // Since exp() may have side effects, e.g. errno,
+    // dead code elimination may not be trusted to remove it.
+    substituteInParent(Arg, MulY);
+    return MulY;
+  }
 
-  LibFunc Func;
-  Function *F = OpC->getCalledFunction();
-  if (F && ((TLI->getLibFunc(F->getName(), Func) && TLI->has(Func) &&
-      Func == LibFunc_pow) || F->getIntrinsicID() == Intrinsic::pow))
-    return B.CreateFMul(OpC->getArgOperand(1),
-      emitUnaryFloatFnCall(OpC->getOperand(0), Callee->getName(), B,
-                           Callee->getAttributes()), "mul");
-
-  // log(exp2(y)) -> y*log(2)
-  if (F && Name == "log" && TLI->getLibFunc(F->getName(), Func) &&
-      TLI->has(Func) && Func == LibFunc_exp2)
-    return B.CreateFMul(
-        OpC->getArgOperand(0),
-        emitUnaryFloatFnCall(ConstantFP::get(CI->getType(), 2.0),
-                             Callee->getName(), B, Callee->getAttributes()),
-        "logmul");
   return Ret;
 }
 
@@ -2187,6 +2410,7 @@ Value *LibCallSimplifier::optimizePrintF(CallInst *CI, IRBuilder<> &B) {
     return New;
   }
 
+  annotateNonNullBasedOnAccess(CI, 0);
   return nullptr;
 }
 
@@ -2281,21 +2505,21 @@ Value *LibCallSimplifier::optimizeSPrintF(CallInst *CI, IRBuilder<> &B) {
     return New;
   }
 
+  annotateNonNullBasedOnAccess(CI, {0, 1});
   return nullptr;
 }
 
 Value *LibCallSimplifier::optimizeSnPrintFString(CallInst *CI, IRBuilder<> &B) {
-  // Check for a fixed format string.
-  StringRef FormatStr;
-  if (!getConstantStringInfo(CI->getArgOperand(2), FormatStr))
-    return nullptr;
-
   // Check for size
   ConstantInt *Size = dyn_cast<ConstantInt>(CI->getArgOperand(1));
   if (!Size)
     return nullptr;
 
   uint64_t N = Size->getZExtValue();
+  // Check for a fixed format string.
+  StringRef FormatStr;
+  if (!getConstantStringInfo(CI->getArgOperand(2), FormatStr))
+    return nullptr;
 
   // If we just have a format string (nothing else crazy) transform it.
   if (CI->getNumArgOperands() == 3) {
@@ -2368,6 +2592,8 @@ Value *LibCallSimplifier::optimizeSnPrintF(CallInst *CI, IRBuilder<> &B) {
     return V;
   }
 
+  if (isKnownNonZero(CI->getOperand(1), DL))
+    annotateNonNullBasedOnAccess(CI, 0);
   return nullptr;
 }
 
@@ -2553,6 +2779,7 @@ Value *LibCallSimplifier::optimizeFRead(CallInst *CI, IRBuilder<> &B) {
 }
 
 Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilder<> &B) {
+  annotateNonNullBasedOnAccess(CI, 0);
   if (!CI->use_empty())
     return nullptr;
 
@@ -2565,6 +2792,12 @@ Value *LibCallSimplifier::optimizePuts(CallInst *CI, IRBuilder<> &B) {
   return nullptr;
 }
 
+Value *LibCallSimplifier::optimizeBCopy(CallInst *CI, IRBuilder<> &B) {
+  // bcopy(src, dst, n) -> llvm.memmove(dst, src, n)
+  return B.CreateMemMove(CI->getArgOperand(1), 1, CI->getArgOperand(0), 1,
+                         CI->getArgOperand(2));
+}
+
 bool LibCallSimplifier::hasFloatVersion(StringRef FuncName) {
   LibFunc Func;
   SmallString<20> FloatFuncName = FuncName;
@@ -2607,6 +2840,8 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
       return optimizeStrLen(CI, Builder);
     case LibFunc_strpbrk:
       return optimizeStrPBrk(CI, Builder);
+    case LibFunc_strndup:
+      return optimizeStrNDup(CI, Builder);
     case LibFunc_strtol:
     case LibFunc_strtod:
     case LibFunc_strtof:
@@ -2623,6 +2858,8 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
       return optimizeStrStr(CI, Builder);
     case LibFunc_memchr:
       return optimizeMemChr(CI, Builder);
+    case LibFunc_memrchr:
+      return optimizeMemRChr(CI, Builder);
     case LibFunc_bcmp:
       return optimizeBCmp(CI, Builder);
     case LibFunc_memcmp:
@@ -2639,6 +2876,8 @@ Value *LibCallSimplifier::optimizeStringMemoryLibCall(CallInst *CI,
       return optimizeRealloc(CI, Builder);
     case LibFunc_wcslen:
       return optimizeWcslen(CI, Builder);
+    case LibFunc_bcopy:
+      return optimizeBCopy(CI, Builder);
     default:
       break;
     }
@@ -2678,11 +2917,21 @@ Value *LibCallSimplifier::optimizeFloatingPointLibCall(CallInst *CI,
   case LibFunc_sqrt:
   case LibFunc_sqrtl:
     return optimizeSqrt(CI, Builder);
+  case LibFunc_logf:
   case LibFunc_log:
+  case LibFunc_logl:
+  case LibFunc_log10f:
   case LibFunc_log10:
+  case LibFunc_log10l:
+  case LibFunc_log1pf:
   case LibFunc_log1p:
+  case LibFunc_log1pl:
+  case LibFunc_log2f:
   case LibFunc_log2:
+  case LibFunc_log2l:
+  case LibFunc_logbf:
   case LibFunc_logb:
+  case LibFunc_logbl:
     return optimizeLog(CI, Builder);
   case LibFunc_tan:
   case LibFunc_tanf:
@@ -2773,16 +3022,18 @@ Value *LibCallSimplifier::optimizeCall(CallInst *CI) {
     case Intrinsic::exp2:
       return optimizeExp2(CI, Builder);
     case Intrinsic::log:
+    case Intrinsic::log2:
+    case Intrinsic::log10:
       return optimizeLog(CI, Builder);
     case Intrinsic::sqrt:
       return optimizeSqrt(CI, Builder);
     // TODO: Use foldMallocMemset() with memset intrinsic.
     case Intrinsic::memset:
-      return optimizeMemSet(CI, Builder, true);
+      return optimizeMemSet(CI, Builder);
     case Intrinsic::memcpy:
-      return optimizeMemCpy(CI, Builder, true);
+      return optimizeMemCpy(CI, Builder);
     case Intrinsic::memmove:
-      return optimizeMemMove(CI, Builder, true);
+      return optimizeMemMove(CI, Builder);
     default:
       return nullptr;
     }
@@ -2955,7 +3206,9 @@ FortifiedLibCallSimplifier::isFortifiedCallFoldable(CallInst *CI,
       uint64_t Len = GetStringLength(CI->getArgOperand(*StrOp));
       // If the length is 0 we don't know how long it is and so we can't
       // remove the check.
-      if (Len == 0)
+      if (Len)
+        annotateDereferenceableBytes(CI, *StrOp, Len);
+      else
         return false;
       return ObjSizeCI->getZExtValue() >= Len;
     }
@@ -2972,8 +3225,9 @@ FortifiedLibCallSimplifier::isFortifiedCallFoldable(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
                                                      IRBuilder<> &B) {
   if (isFortifiedCallFoldable(CI, 3, 2)) {
-    B.CreateMemCpy(CI->getArgOperand(0), 1, CI->getArgOperand(1), 1,
-                   CI->getArgOperand(2));
+    CallInst *NewCI = B.CreateMemCpy(
+        CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, CI->getArgOperand(2));
+    NewCI->setAttributes(CI->getAttributes());
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -2982,8 +3236,9 @@ Value *FortifiedLibCallSimplifier::optimizeMemCpyChk(CallInst *CI,
 Value *FortifiedLibCallSimplifier::optimizeMemMoveChk(CallInst *CI,
                                                       IRBuilder<> &B) {
   if (isFortifiedCallFoldable(CI, 3, 2)) {
-    B.CreateMemMove(CI->getArgOperand(0), 1, CI->getArgOperand(1), 1,
-                    CI->getArgOperand(2));
+    CallInst *NewCI = B.CreateMemMove(
+        CI->getArgOperand(0), 1, CI->getArgOperand(1), 1, CI->getArgOperand(2));
+    NewCI->setAttributes(CI->getAttributes());
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -2995,7 +3250,9 @@ Value *FortifiedLibCallSimplifier::optimizeMemSetChk(CallInst *CI,
 
   if (isFortifiedCallFoldable(CI, 3, 2)) {
     Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
-    B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1);
+    CallInst *NewCI =
+        B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1);
+    NewCI->setAttributes(CI->getAttributes());
     return CI->getArgOperand(0);
   }
   return nullptr;
@@ -3031,7 +3288,9 @@ Value *FortifiedLibCallSimplifier::optimizeStrpCpyChk(CallInst *CI,
 
   // Maybe we can stil fold __st[rp]cpy_chk to __memcpy_chk.
   uint64_t Len = GetStringLength(Src);
-  if (Len == 0)
+  if (Len)
+    annotateDereferenceableBytes(CI, 1, Len);
+  else
     return nullptr;
 
   Type *SizeTTy = DL.getIntPtrType(CI->getContext());
diff --git a/llvm/lib/Transforms/Utils/VNCoercion.cpp b/llvm/lib/Transforms/Utils/VNCoercion.cpp
index a77bf50fe10b8..591e1fd2dbee1 100644
--- a/llvm/lib/Transforms/Utils/VNCoercion.cpp
+++ b/llvm/lib/Transforms/Utils/VNCoercion.cpp
@@ -431,7 +431,7 @@ Value *getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy,
     PtrVal = Builder.CreateBitCast(PtrVal, DestPTy);
     LoadInst *NewLoad = Builder.CreateLoad(DestTy, PtrVal);
     NewLoad->takeName(SrcVal);
-    NewLoad->setAlignment(SrcVal->getAlignment());
+    NewLoad->setAlignment(MaybeAlign(SrcVal->getAlignment()));
 
     LLVM_DEBUG(dbgs() << "GVN WIDENED LOAD: " << *SrcVal << "\n");
     LLVM_DEBUG(dbgs() << "TO: " << *NewLoad << "\n");
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 5f97740a9c6f5..f44976c723ec0 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -1092,7 +1092,7 @@ bool Vectorizer::vectorizeLoadChain(
   LoadInst *L0 = cast<LoadInst>(Chain[0]);
 
   // If the vector has an int element, default to int for the whole load.
-  Type *LoadTy;
+  Type *LoadTy = nullptr;
   for (const auto &V : Chain) {
     LoadTy = cast<LoadInst>(V)->getType();
     if (LoadTy->isIntOrIntVectorTy())
@@ -1104,6 +1104,7 @@ bool Vectorizer::vectorizeLoadChain(
       break;
     }
   }
+  assert(LoadTy && "Can't determine LoadInst type from chain");
 
   unsigned Sz = DL.getTypeSizeInBits(LoadTy);
   unsigned AS = L0->getPointerAddressSpace();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 61406c97fd0b3..9731abbb50aaf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -742,7 +742,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
           assert(VecTy && "did not find vectorized version of stored type");
           unsigned Alignment = getLoadStoreAlignment(ST);
           assert(Alignment && "Alignment should be set");
-          if (!TTI->isLegalNTStore(VecTy, llvm::Align(Alignment))) {
+          if (!TTI->isLegalNTStore(VecTy, Align(Alignment))) {
             reportVectorizationFailure(
                 "nontemporal store instruction cannot be vectorized",
                 "nontemporal store instruction cannot be vectorized",
@@ -759,7 +759,7 @@ bool LoopVectorizationLegality::canVectorizeInstrs() {
           assert(VecTy && "did not find vectorized version of load type");
           unsigned Alignment = getLoadStoreAlignment(LD);
           assert(Alignment && "Alignment should be set");
-          if (!TTI->isLegalNTLoad(VecTy, llvm::Align(Alignment))) {
+          if (!TTI->isLegalNTLoad(VecTy, Align(Alignment))) {
             reportVectorizationFailure(
                 "nontemporal load instruction cannot be vectorized",
                 "nontemporal load instruction cannot be vectorized",
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 549a24cbdc3a7..7e95038a5ebcf 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2730,8 +2730,19 @@ void InnerLoopVectorizer::emitMemRuntimeChecks(Loop *L, BasicBlock *Bypass) {
   if (!MemRuntimeCheck)
     return;
 
-  assert(!BB->getParent()->hasOptSize() &&
-         "Cannot emit memory checks when optimizing for size");
+  if (BB->getParent()->hasOptSize()) {
+    assert(Cost->Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
+           "Cannot emit memory checks when optimizing for size, unless forced "
+           "to vectorize.");
+    ORE->emit([&]() {
+      return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
+                                        L->getStartLoc(), L->getHeader())
+             << "Code-size may be reduced by not forcing "
+                "vectorization, or by source-code modifications "
+                "eliminating the need for runtime checks "
+                "(e.g., adding 'restrict').";
+    });
+  }
 
   // Create a new block containing the memory check.
   BB->setName("vector.memcheck");
@@ -4167,7 +4178,7 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) {
   case Instruction::FCmp: {
     // Widen compares. Generate vector compares.
     bool FCmp = (I.getOpcode() == Instruction::FCmp);
-    auto *Cmp = dyn_cast<CmpInst>(&I);
+    auto *Cmp = cast<CmpInst>(&I);
     setDebugLocFromInst(Builder, Cmp);
     for (unsigned Part = 0; Part < UF; ++Part) {
       Value *A = getOrCreateVectorValue(Cmp->getOperand(0), Part);
@@ -4200,7 +4211,7 @@ void InnerLoopVectorizer::widenInstruction(Instruction &I) {
   case Instruction::Trunc:
   case Instruction::FPTrunc:
   case Instruction::BitCast: {
-    auto *CI = dyn_cast<CastInst>(&I);
+    auto *CI = cast<CastInst>(&I);
     setDebugLocFromInst(Builder, CI);
 
     /// Vectorize casts.
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 7fbcb23a349e8..196dbe12b876d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -194,10 +194,13 @@ static bool allSameBlock(ArrayRef<Value *> VL) {
   return true;
 }
 
-/// \returns True if all of the values in \p VL are constants.
+/// \returns True if all of the values in \p VL are constants (but not
+/// globals/constant expressions).
 static bool allConstant(ArrayRef<Value *> VL) {
+  // Constant expressions and globals can't be vectorized like normal integer/FP
+  // constants.
   for (Value *i : VL)
-    if (!isa<Constant>(i))
+    if (!isa<Constant>(i) || isa<ConstantExpr>(i) || isa<GlobalValue>(i))
       return false;
   return true;
 }
@@ -1118,6 +1121,14 @@ class BoUpSLP {
 #endif
   };
 
+  /// Checks if the instruction is marked for deletion.
+  bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
+
+  /// Marks values operands for later deletion by replacing them with Undefs.
+  void eraseInstructions(ArrayRef<Value *> AV);
+
+  ~BoUpSLP();
+
 private:
   /// Checks if all users of \p I are the part of the vectorization tree.
   bool areAllUsersVectorized(Instruction *I) const;
@@ -1487,15 +1498,14 @@ class BoUpSLP {
   /// This is required to ensure that there are no incorrect collisions in the
   /// AliasCache, which can happen if a new instruction is allocated at the
   /// same address as a previously deleted instruction.
-  void eraseInstruction(Instruction *I) {
-    I->removeFromParent();
-    I->dropAllReferences();
-    DeletedInstructions.emplace_back(I);
+  void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) {
+    auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first;
+    It->getSecond() = It->getSecond() && ReplaceOpsWithUndef;
   }
 
   /// Temporary store for deleted instructions. Instructions will be deleted
   /// eventually when the BoUpSLP is destructed.
-  SmallVector<unique_value, 8> DeletedInstructions;
+  DenseMap<Instruction *, bool> DeletedInstructions;
 
   /// A list of values that need to extracted out of the tree.
   /// This list holds pairs of (Internal Scalar : External User). External User
@@ -2052,6 +2062,30 @@ template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
 
 } // end namespace llvm
 
+BoUpSLP::~BoUpSLP() {
+  for (const auto &Pair : DeletedInstructions) {
+    // Replace operands of ignored instructions with Undefs in case if they were
+    // marked for deletion.
+    if (Pair.getSecond()) {
+      Value *Undef = UndefValue::get(Pair.getFirst()->getType());
+      Pair.getFirst()->replaceAllUsesWith(Undef);
+    }
+    Pair.getFirst()->dropAllReferences();
+  }
+  for (const auto &Pair : DeletedInstructions) {
+    assert(Pair.getFirst()->use_empty() &&
+           "trying to erase instruction with users.");
+    Pair.getFirst()->eraseFromParent();
+  }
+}
+
+void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) {
+  for (auto *V : AV) {
+    if (auto *I = dyn_cast<Instruction>(V))
+      eraseInstruction(I, /*ReplaceWithUndef=*/true);
+  };
+}
+
 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
                         ArrayRef<Value *> UserIgnoreLst) {
   ExtraValueToDebugLocsMap ExternallyUsedValues;
@@ -2270,7 +2304,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
                 (unsigned) Instruction::ShuffleVector : S.getOpcode();
   switch (ShuffleOrOp) {
     case Instruction::PHI: {
-      PHINode *PH = dyn_cast<PHINode>(VL0);
+      auto *PH = cast<PHINode>(VL0);
 
       // Check for terminator values (e.g. invoke).
       for (unsigned j = 0; j < VL.size(); ++j)
@@ -2710,7 +2744,7 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
         ValueList Operands;
         // Prepare the operand vector.
         for (Value *V : VL) {
-          CallInst *CI2 = dyn_cast<CallInst>(V);
+          auto *CI2 = cast<CallInst>(V);
           Operands.push_back(CI2->getArgOperand(i));
         }
         buildTree_rec(Operands, Depth + 1, {TE, i});
@@ -3525,6 +3559,7 @@ void BoUpSLP::setInsertPointAfterBundle(TreeEntry *E) {
         break;
     }
   }
+  assert(LastInst && "Failed to find last instruction in bundle");
 
   // Set the insertion point after the last instruction in the bundle. Set the
   // debug location to Front.
@@ -3537,7 +3572,7 @@ Value *BoUpSLP::Gather(ArrayRef<Value *> VL, VectorType *Ty) {
   // Generate the 'InsertElement' instruction.
   for (unsigned i = 0; i < Ty->getNumElements(); ++i) {
     Vec = Builder.CreateInsertElement(Vec, VL[i], Builder.getInt32(i));
-    if (Instruction *Insrt = dyn_cast<Instruction>(Vec)) {
+    if (auto *Insrt = dyn_cast<InsertElementInst>(Vec)) {
       GatherSeq.insert(Insrt);
       CSEBlocks.insert(Insrt->getParent());
 
@@ -3673,7 +3708,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
   switch (ShuffleOrOp) {
     case Instruction::PHI: {
-      PHINode *PH = dyn_cast<PHINode>(VL0);
+      auto *PH = cast<PHINode>(VL0);
       Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
       Builder.SetCurrentDebugLocation(PH->getDebugLoc());
       PHINode *NewPhi = Builder.CreatePHI(VecTy, PH->getNumIncomingValues());
@@ -3797,7 +3832,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
         return E->VectorizedValue;
       }
 
-      CastInst *CI = dyn_cast<CastInst>(VL0);
+      auto *CI = cast<CastInst>(VL0);
       Value *V = Builder.CreateCast(CI->getOpcode(), InVec, VecTy);
       if (NeedToShuffleReuses) {
         V = Builder.CreateShuffleVector(V, UndefValue::get(VecTy),
@@ -3947,11 +3982,10 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
       if (getTreeEntry(PO))
         ExternalUses.push_back(ExternalUser(PO, cast<User>(VecPtr), 0));
 
-      unsigned Alignment = LI->getAlignment();
+      MaybeAlign Alignment = MaybeAlign(LI->getAlignment());
       LI = Builder.CreateLoad(VecTy, VecPtr);
-      if (!Alignment) {
-        Alignment = DL->getABITypeAlignment(ScalarLoadTy);
-      }
+      if (!Alignment)
+        Alignment = MaybeAlign(DL->getABITypeAlignment(ScalarLoadTy));
       LI->setAlignment(Alignment);
       Value *V = propagateMetadata(LI, E->Scalars);
       if (IsReorder) {
@@ -4286,20 +4320,18 @@ BoUpSLP::vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues) {
     for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
       Value *Scalar = Entry->Scalars[Lane];
 
+#ifndef NDEBUG
       Type *Ty = Scalar->getType();
       if (!Ty->isVoidTy()) {
-#ifndef NDEBUG
         for (User *U : Scalar->users()) {
           LLVM_DEBUG(dbgs() << "SLP: \tvalidating user:" << *U << ".\n");
 
-          // It is legal to replace users in the ignorelist by undef.
+          // It is legal to delete users in the ignorelist.
           assert((getTreeEntry(U) || is_contained(UserIgnoreList, U)) &&
-                 "Replacing out-of-tree value with undef");
+                 "Deleting out-of-tree value");
         }
-#endif
-        Value *Undef = UndefValue::get(Ty);
-        Scalar->replaceAllUsesWith(Undef);
       }
+#endif
       LLVM_DEBUG(dbgs() << "SLP: \tErasing scalar:" << *Scalar << ".\n");
       eraseInstruction(cast<Instruction>(Scalar));
     }
@@ -4315,7 +4347,7 @@ void BoUpSLP::optimizeGatherSequence() {
                     << " gather sequences instructions.\n");
   // LICM InsertElementInst sequences.
   for (Instruction *I : GatherSeq) {
-    if (!isa<InsertElementInst>(I) && !isa<ShuffleVectorInst>(I))
+    if (isDeleted(I))
       continue;
 
     // Check if this block is inside a loop.
@@ -4369,6 +4401,8 @@ void BoUpSLP::optimizeGatherSequence() {
     // For all instructions in blocks containing gather sequences:
     for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e;) {
       Instruction *In = &*it++;
+      if (isDeleted(In))
+        continue;
       if (!isa<InsertElementInst>(In) && !isa<ExtractElementInst>(In))
         continue;
 
@@ -5251,19 +5285,6 @@ bool SLPVectorizerPass::runImpl(Function &F, ScalarEvolution *SE_,
   return Changed;
 }
 
-/// Check that the Values in the slice in VL array are still existent in
-/// the WeakTrackingVH array.
-/// Vectorization of part of the VL array may cause later values in the VL array
-/// to become invalid. We track when this has happened in the WeakTrackingVH
-/// array.
-static bool hasValueBeenRAUWed(ArrayRef<Value *> VL,
-                               ArrayRef<WeakTrackingVH> VH, unsigned SliceBegin,
-                               unsigned SliceSize) {
-  VL = VL.slice(SliceBegin, SliceSize);
-  VH = VH.slice(SliceBegin, SliceSize);
-  return !std::equal(VL.begin(), VL.end(), VH.begin());
-}
-
 bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
                                             unsigned VecRegSize) {
   const unsigned ChainLen = Chain.size();
@@ -5275,20 +5296,20 @@ bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef<Value *> Chain, BoUpSLP &R,
   if (!isPowerOf2_32(Sz) || VF < 2)
     return false;
 
-  // Keep track of values that were deleted by vectorizing in the loop below.
-  const SmallVector<WeakTrackingVH, 8> TrackValues(Chain.begin(), Chain.end());
-
   bool Changed = false;
   // Look for profitable vectorizable trees at all offsets, starting at zero.
   for (unsigned i = 0, e = ChainLen; i + VF <= e; ++i) {
 
+    ArrayRef<Value *> Operands = Chain.slice(i, VF);
     // Check that a previous iteration of this loop did not delete the Value.
-    if (hasValueBeenRAUWed(Chain, TrackValues, i, VF))
+    if (llvm::any_of(Operands, [&R](Value *V) {
+          auto *I = dyn_cast<Instruction>(V);
+          return I && R.isDeleted(I);
+        }))
       continue;
 
     LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << i
                       << "\n");
-    ArrayRef<Value *> Operands = Chain.slice(i, VF);
 
     R.buildTree(Operands);
     if (R.isTreeTinyAndNotFullyVectorizable())
@@ -5480,9 +5501,6 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
   bool CandidateFound = false;
   int MinCost = SLPCostThreshold;
 
-  // Keep track of values that were deleted by vectorizing in the loop below.
-  SmallVector<WeakTrackingVH, 8> TrackValues(VL.begin(), VL.end());
-
   unsigned NextInst = 0, MaxInst = VL.size();
   for (unsigned VF = MaxVF; NextInst + 1 < MaxInst && VF >= MinVF; VF /= 2) {
     // No actual vectorization should happen, if number of parts is the same as
@@ -5502,13 +5520,16 @@ bool SLPVectorizerPass::tryToVectorizeList(ArrayRef<Value *> VL, BoUpSLP &R,
       if (!isPowerOf2_32(OpsWidth) || OpsWidth < 2)
         break;
 
+      ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
       // Check that a previous iteration of this loop did not delete the Value.
-      if (hasValueBeenRAUWed(VL, TrackValues, I, OpsWidth))
+      if (llvm::any_of(Ops, [&R](Value *V) {
+            auto *I = dyn_cast<Instruction>(V);
+            return I && R.isDeleted(I);
+          }))
         continue;
 
       LLVM_DEBUG(dbgs() << "SLP: Analyzing " << OpsWidth << " operations "
                         << "\n");
-      ArrayRef<Value *> Ops = VL.slice(I, OpsWidth);
 
       R.buildTree(Ops);
       Optional<ArrayRef<unsigned>> Order = R.bestOrder();
@@ -5729,23 +5750,23 @@ class HorizontalReduction {
       case RK_Min:
         Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSLT(LHS, RHS)
                                           : Builder.CreateFCmpOLT(LHS, RHS);
-        break;
+        return Builder.CreateSelect(Cmp, LHS, RHS, Name);
       case RK_Max:
         Cmp = Opcode == Instruction::ICmp ? Builder.CreateICmpSGT(LHS, RHS)
                                           : Builder.CreateFCmpOGT(LHS, RHS);
-        break;
+        return Builder.CreateSelect(Cmp, LHS, RHS, Name);
       case RK_UMin:
         assert(Opcode == Instruction::ICmp && "Expected integer types.");
         Cmp = Builder.CreateICmpULT(LHS, RHS);
-        break;
+        return Builder.CreateSelect(Cmp, LHS, RHS, Name);
       case RK_UMax:
         assert(Opcode == Instruction::ICmp && "Expected integer types.");
         Cmp = Builder.CreateICmpUGT(LHS, RHS);
-        break;
+        return Builder.CreateSelect(Cmp, LHS, RHS, Name);
       case RK_None:
-        llvm_unreachable("Unknown reduction operation.");
+        break;
       }
-      return Builder.CreateSelect(Cmp, LHS, RHS, Name);
+      llvm_unreachable("Unknown reduction operation.");
     }
 
   public:
@@ -6425,6 +6446,9 @@ class HorizontalReduction {
       }
       // Update users.
       ReductionRoot->replaceAllUsesWith(VectorizedTree);
+      // Mark all scalar reduction ops for deletion, they are replaced by the
+      // vector reductions.
+      V.eraseInstructions(IgnoreList);
     }
     return VectorizedTree != nullptr;
   }
@@ -6679,18 +6703,13 @@ static bool tryToVectorizeHorReductionOrInstOperands(
   // horizontal reduction.
   // Interrupt the process if the Root instruction itself was vectorized or all
   // sub-trees not higher that RecursionMaxDepth were analyzed/vectorized.
-  SmallVector<std::pair<WeakTrackingVH, unsigned>, 8> Stack(1, {Root, 0});
+  SmallVector<std::pair<Instruction *, unsigned>, 8> Stack(1, {Root, 0});
   SmallPtrSet<Value *, 8> VisitedInstrs;
   bool Res = false;
   while (!Stack.empty()) {
-    Value *V;
+    Instruction *Inst;
     unsigned Level;
-    std::tie(V, Level) = Stack.pop_back_val();
-    if (!V)
-      continue;
-    auto *Inst = dyn_cast<Instruction>(V);
-    if (!Inst)
-      continue;
+    std::tie(Inst, Level) = Stack.pop_back_val();
     auto *BI = dyn_cast<BinaryOperator>(Inst);
     auto *SI = dyn_cast<SelectInst>(Inst);
     if (BI || SI) {
@@ -6731,8 +6750,8 @@ static bool tryToVectorizeHorReductionOrInstOperands(
       for (auto *Op : Inst->operand_values())
         if (VisitedInstrs.insert(Op).second)
           if (auto *I = dyn_cast<Instruction>(Op))
-            if (!isa<PHINode>(I) && I->getParent() == BB)
-              Stack.emplace_back(Op, Level);
+            if (!isa<PHINode>(I) && !R.isDeleted(I) && I->getParent() == BB)
+              Stack.emplace_back(I, Level);
   }
   return Res;
 }
@@ -6801,11 +6820,10 @@ bool SLPVectorizerPass::vectorizeCmpInst(CmpInst *CI, BasicBlock *BB,
 }
 
 bool SLPVectorizerPass::vectorizeSimpleInstructions(
-    SmallVectorImpl<WeakVH> &Instructions, BasicBlock *BB, BoUpSLP &R) {
+    SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, BoUpSLP &R) {
   bool OpsChanged = false;
-  for (auto &VH : reverse(Instructions)) {
-    auto *I = dyn_cast_or_null<Instruction>(VH);
-    if (!I)
+  for (auto *I : reverse(Instructions)) {
+    if (R.isDeleted(I))
       continue;
     if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I))
       OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
@@ -6834,7 +6852,7 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
       if (!P)
         break;
 
-      if (!VisitedInstrs.count(P))
+      if (!VisitedInstrs.count(P) && !R.isDeleted(P))
         Incoming.push_back(P);
     }
 
@@ -6878,9 +6896,12 @@ bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
 
   VisitedInstrs.clear();
 
-  SmallVector<WeakVH, 8> PostProcessInstructions;
+  SmallVector<Instruction *, 8> PostProcessInstructions;
   SmallDenseSet<Instruction *, 4> KeyNodes;
   for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
+    // Skip instructions marked for the deletion.
+    if (R.isDeleted(&*it))
+      continue;
     // We may go through BB multiple times so skip the one we have checked.
     if (!VisitedInstrs.insert(&*it).second) {
       if (it->use_empty() && KeyNodes.count(&*it) > 0 &&
@@ -6973,10 +6994,10 @@ bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
       SetVector<Value *> Candidates(GEPList.begin(), GEPList.end());
 
       // Some of the candidates may have already been vectorized after we
-      // initially collected them. If so, the WeakTrackingVHs will have
-      // nullified the
-      // values, so remove them from the set of candidates.
-      Candidates.remove(nullptr);
+      // initially collected them. If so, they are marked as deleted, so remove
+      // them from the set of candidates.
+      Candidates.remove_if(
+          [&R](Value *I) { return R.isDeleted(cast<Instruction>(I)); });
 
       // Remove from the set of candidates all pairs of getelementptrs with
       // constant differences. Such getelementptrs are likely not good
@@ -6984,18 +7005,18 @@ bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
       // computed from the other. We also ensure all candidate getelementptr
       // indices are unique.
       for (int I = 0, E = GEPList.size(); I < E && Candidates.size() > 1; ++I) {
-        auto *GEPI = cast<GetElementPtrInst>(GEPList[I]);
+        auto *GEPI = GEPList[I];
         if (!Candidates.count(GEPI))
           continue;
         auto *SCEVI = SE->getSCEV(GEPList[I]);
         for (int J = I + 1; J < E && Candidates.size() > 1; ++J) {
-          auto *GEPJ = cast<GetElementPtrInst>(GEPList[J]);
+          auto *GEPJ = GEPList[J];
           auto *SCEVJ = SE->getSCEV(GEPList[J]);
           if (isa<SCEVConstant>(SE->getMinusSCEV(SCEVI, SCEVJ))) {
-            Candidates.remove(GEPList[I]);
-            Candidates.remove(GEPList[J]);
+            Candidates.remove(GEPI);
+            Candidates.remove(GEPJ);
           } else if (GEPI->idx_begin()->get() == GEPJ->idx_begin()->get()) {
-            Candidates.remove(GEPList[J]);
+            Candidates.remove(GEPJ);
           }
         }
       }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
index 7ed7d21b6caa8..b22d3190d654c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp
@@ -21,7 +21,7 @@ void VPlanHCFGTransforms::VPInstructionsToVPRecipes(
     LoopVectorizationLegality::InductionList *Inductions,
     SmallPtrSetImpl<Instruction *> &DeadInstructions) {
 
-  VPRegionBlock *TopRegion = dyn_cast<VPRegionBlock>(Plan->getEntry());
+  auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
   ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
 
   // Condition bit VPValues get deleted during transformation to VPRecipes.
diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
index e5ab24e52df6b..9019ed15ec5ff 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp
@@ -346,11 +346,14 @@ SmallVector<VPlanSlp::MultiNodeOpTy, 4> VPlanSlp::reorderMultiNodeOps() {
 
 void VPlanSlp::dumpBundle(ArrayRef<VPValue *> Values) {
   dbgs() << " Ops: ";
-  for (auto Op : Values)
-    if (auto *Instr = cast_or_null<VPInstruction>(Op)->getUnderlyingInstr())
-      dbgs() << *Instr << " | ";
-    else
-      dbgs() << " nullptr | ";
+  for (auto Op : Values) {
+    if (auto *VPInstr = cast_or_null<VPInstruction>(Op))
+      if (auto *Instr = VPInstr->getUnderlyingInstr()) {
+        dbgs() << *Instr << " | ";
+        continue;
+      }
+    dbgs() << " nullptr | ";
+  }
   dbgs() << "\n";
 }
 
diff --git a/llvm/lib/XRay/FDRRecordProducer.cpp b/llvm/lib/XRay/FDRRecordProducer.cpp
index 02132b283e920..479b710444be8 100644
--- a/llvm/lib/XRay/FDRRecordProducer.cpp
+++ b/llvm/lib/XRay/FDRRecordProducer.cpp
@@ -130,7 +130,7 @@ Expected<std::unique_ptr<Record>> FileBasedRecordProducer::produce() {
     R = std::move(BufferExtentsOrError.get());
     assert(R != nullptr);
     assert(isa<BufferExtents>(R.get()));
-    auto BE = dyn_cast<BufferExtents>(R.get());
+    auto BE = cast<BufferExtents>(R.get());
     CurrentBufferBytes = BE->size();
     return std::move(R);
   }
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 552ab7b2a88ff..e212c941b6a2a 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -460,6 +460,11 @@ else() # if this is included from LLVM's CMake
       endif()
     endforeach()
 
+    if(NOT RUNTIMES_${name}_LLVM_ENABLE_RUNTIMES)
+      string(REPLACE ";" "|" LLVM_ENABLE_RUNTIMES_PASSTHROUGH "${LLVM_ENABLE_RUNTIMES}")
+      list(APPEND ${name}_extra_args -DLLVM_ENABLE_RUNTIMES=${LLVM_ENABLE_RUNTIMES_PASSTHROUGH})
+    endif()
+
     if(target IN_LIST LLVM_RUNTIME_BUILD_ID_LINK_TARGETS)
       list(APPEND EXTRA_ARGS STRIP_TOOL ${CMAKE_CURRENT_BINARY_DIR}/llvm-strip-link)
     endif()
@@ -482,7 +487,6 @@ else() # if this is included from LLVM's CMake
                                         -DCOMPILER_RT_DEFAULT_TARGET_ONLY=ON
                                         -DLLVM_RUNTIMES_TARGET=${name}
                                         ${${name}_extra_args}
-                             PASSTHROUGH_PREFIXES LLVM_ENABLE_RUNTIMES
                              TOOLCHAIN_TOOLS clang lld llvm-ar llvm-ranlib llvm-nm llvm-objcopy llvm-objdump llvm-strip
                              EXTRA_TARGETS ${${name}_extra_targets}
                                            ${${name}_test_targets}
diff --git a/llvm/test/Analysis/BasicAA/gep-alias.ll b/llvm/test/Analysis/BasicAA/gep-alias.ll
index 1e435af2f12f9..5fd77e19e637d 100644
--- a/llvm/test/Analysis/BasicAA/gep-alias.ll
+++ b/llvm/test/Analysis/BasicAA/gep-alias.ll
@@ -247,7 +247,7 @@ define i32 @test12(i32 %x, i32 %y, i8* %p) nounwind {
 ; CHECK: [[U0ADDR:%[a-zA-Z0-9_]+]] = getelementptr inbounds [3 x i8], [3 x i8]* %u, i32 0, i32 0
 ; CHECK: [[U0:%[a-zA-Z0-9_]+]] = load i8, i8* [[U0ADDR]], align 1
 ; CHECK: [[U0ARG:%[a-zA-Z0-9_]+]] = zext i8 [[U0]] to i32
-; CHECK: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0), i32 [[T0ARG]], i32 [[U0ARG]])
+; CHECK: call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0), i32 [[T0ARG]], i32 [[U0ARG]])
 ; CHECK: ret
 define void @test13() {
 entry:
diff --git a/llvm/test/Analysis/ConstantFolding/math-1.ll b/llvm/test/Analysis/ConstantFolding/math-1.ll
index 173320789ae15..5950950171023 100644
--- a/llvm/test/Analysis/ConstantFolding/math-1.ll
+++ b/llvm/test/Analysis/ConstantFolding/math-1.ll
@@ -92,14 +92,14 @@ define double @i_exp2() {
   ret double %res
 }
 
-; FIXME
+; FIXME: exp10() is not widely supported.
 declare float @exp10f(float)
 define float @f_exp10f() {
 ; CHECK-LABEL: @f_exp10f(
-; CHECK-NEXT:    [[RES:%.*]] = tail call fast float @exp10f(float 1.000000e+00)
+; CHECK-NEXT:    [[RES:%.*]] = tail call float @exp10f(float 1.000000e+00)
 ; CHECK-NEXT:    ret float [[RES]]
 ;
-  %res = tail call fast float @exp10f(float 1.0)
+  %res = tail call float @exp10f(float 1.0)
   ret float %res
 }
 
@@ -121,12 +121,10 @@ define double @i_log() {
   ret double %res
 }
 
-; FIXME
 declare float @log2f(float)
 define float @f_log2f() {
 ; CHECK-LABEL: @f_log2f(
-; CHECK-NEXT:    [[RES:%.*]] = tail call fast float @log2f(float 1.000000e+00)
-; CHECK-NEXT:    ret float [[RES]]
+; CHECK-NEXT:    ret float 0.000000e+00
 ;
   %res = tail call fast float @log2f(float 1.0)
   ret float %res
diff --git a/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll b/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
index 9a43ca318384b..0c95b537340ce 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/fp-cast.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 ;
 ; Note: The scalarized vector instructions costs are not including any
 ; extracts, due to the undef operands.
@@ -118,7 +118,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v19 = fptosi <2 x double> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v20 = fptosi <2 x float> undef to <2 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptosi <2 x float> undef to <2 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v22 = fptosi <2 x float> undef to <2 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v23 = fptosi <2 x float> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v24 = fptosi <4 x fp128> undef to <4 x i64>
@@ -131,7 +131,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v31 = fptosi <4 x double> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 10 for instruction:   %v32 = fptosi <4 x float> undef to <4 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptosi <4 x float> undef to <4 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v34 = fptosi <4 x float> undef to <4 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v35 = fptosi <4 x float> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v36 = fptosi <8 x fp128> undef to <8 x i64>
@@ -144,7 +144,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v43 = fptosi <8 x double> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 20 for instruction:   %v44 = fptosi <8 x float> undef to <8 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 24 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptosi <8 x float> undef to <8 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v46 = fptosi <8 x float> undef to <8 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v47 = fptosi <8 x float> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 8 for instruction:   %v48 = fptosi <16 x double> undef to <16 x i64>
@@ -153,7 +153,7 @@ define void @fptosi() {
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v51 = fptosi <16 x double> undef to <16 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 40 for instruction:   %v52 = fptosi <16 x float> undef to <16 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 48 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptosi <16 x float> undef to <16 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v54 = fptosi <16 x float> undef to <16 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v55 = fptosi <16 x float> undef to <16 x i8>
 
@@ -241,7 +241,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v19 = fptoui <2 x double> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v20 = fptoui <2 x float> undef to <2 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v21 = fptoui <2 x float> undef to <2 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v22 = fptoui <2 x float> undef to <2 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v23 = fptoui <2 x float> undef to <2 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 6 for instruction:   %v24 = fptoui <4 x fp128> undef to <4 x i64>
@@ -254,7 +254,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v31 = fptoui <4 x double> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 10 for instruction:   %v32 = fptoui <4 x float> undef to <4 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 12 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v33 = fptoui <4 x float> undef to <4 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v34 = fptoui <4 x float> undef to <4 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v35 = fptoui <4 x float> undef to <4 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 12 for instruction:   %v36 = fptoui <8 x fp128> undef to <8 x i64>
@@ -267,7 +267,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v43 = fptoui <8 x double> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 20 for instruction:   %v44 = fptoui <8 x float> undef to <8 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 24 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v45 = fptoui <8 x float> undef to <8 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v46 = fptoui <8 x float> undef to <8 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 24 for instruction:   %v47 = fptoui <8 x float> undef to <8 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 8 for instruction:   %v48 = fptoui <16 x double> undef to <16 x i64>
@@ -276,7 +276,7 @@ define void @fptoui() {
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v51 = fptoui <16 x double> undef to <16 x i8>
 ; CHECK: Cost Model: Found an estimated cost of 40 for instruction:   %v52 = fptoui <16 x float> undef to <16 x i64>
 ; Z13:   Cost Model: Found an estimated cost of 48 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v53 = fptoui <16 x float> undef to <16 x i32>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v54 = fptoui <16 x float> undef to <16 x i16>
 ; CHECK: Cost Model: Found an estimated cost of 48 for instruction:   %v55 = fptoui <16 x float> undef to <16 x i8>
 
@@ -391,7 +391,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v15 = sitofp <2 x i32> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v16 = sitofp <2 x i32> undef to <2 x double>
 ; Z13:   Cost Model: Found an estimated cost of 14 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v17 = sitofp <2 x i32> undef to <2 x float>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v18 = sitofp <2 x i16> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v19 = sitofp <2 x i16> undef to <2 x double>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v20 = sitofp <2 x i16> undef to <2 x float>
@@ -404,7 +404,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v27 = sitofp <4 x i32> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v28 = sitofp <4 x i32> undef to <4 x double>
 ; Z13:   Cost Model: Found an estimated cost of 13 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v29 = sitofp <4 x i32> undef to <4 x float>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v30 = sitofp <4 x i16> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v31 = sitofp <4 x i16> undef to <4 x double>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v32 = sitofp <4 x i16> undef to <4 x float>
@@ -417,7 +417,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v39 = sitofp <8 x i32> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v40 = sitofp <8 x i32> undef to <8 x double>
 ; Z13:   Cost Model: Found an estimated cost of 25 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v41 = sitofp <8 x i32> undef to <8 x float>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v42 = sitofp <8 x i16> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v43 = sitofp <8 x i16> undef to <8 x double>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v44 = sitofp <8 x i16> undef to <8 x float>
@@ -428,7 +428,7 @@ define void @sitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v49 = sitofp <16 x i64> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v50 = sitofp <16 x i32> undef to <16 x double>
 ; Z13:   Cost Model: Found an estimated cost of 49 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v51 = sitofp <16 x i32> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v52 = sitofp <16 x i16> undef to <16 x double>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v53 = sitofp <16 x i16> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v54 = sitofp <16 x i8> undef to <16 x double>
@@ -513,7 +513,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 5 for instruction:   %v15 = uitofp <2 x i32> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v16 = uitofp <2 x i32> undef to <2 x double>
 ; Z13:   Cost Model: Found an estimated cost of 14 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v17 = uitofp <2 x i32> undef to <2 x float>
 ; CHECK: Cost Model: Found an estimated cost of 7 for instruction:   %v18 = uitofp <2 x i16> undef to <2 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v19 = uitofp <2 x i16> undef to <2 x double>
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v20 = uitofp <2 x i16> undef to <2 x float>
@@ -526,7 +526,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 9 for instruction:   %v27 = uitofp <4 x i32> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v28 = uitofp <4 x i32> undef to <4 x double>
 ; Z13:   Cost Model: Found an estimated cost of 13 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
-; AR13:  Cost Model: Found an estimated cost of 1 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
+; Z15:   Cost Model: Found an estimated cost of 1 for instruction:   %v29 = uitofp <4 x i32> undef to <4 x float>
 ; CHECK: Cost Model: Found an estimated cost of 13 for instruction:   %v30 = uitofp <4 x i16> undef to <4 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v31 = uitofp <4 x i16> undef to <4 x double>
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v32 = uitofp <4 x i16> undef to <4 x float>
@@ -539,7 +539,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 17 for instruction:   %v39 = uitofp <8 x i32> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v40 = uitofp <8 x i32> undef to <8 x double>
 ; Z13:   Cost Model: Found an estimated cost of 25 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
-; AR13:  Cost Model: Found an estimated cost of 2 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
+; Z15:   Cost Model: Found an estimated cost of 2 for instruction:   %v41 = uitofp <8 x i32> undef to <8 x float>
 ; CHECK: Cost Model: Found an estimated cost of 25 for instruction:   %v42 = uitofp <8 x i16> undef to <8 x fp128>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v43 = uitofp <8 x i16> undef to <8 x double>
 ; CHECK: Cost Model: Found an estimated cost of 33 for instruction:   %v44 = uitofp <8 x i16> undef to <8 x float>
@@ -550,7 +550,7 @@ define void @uitofp() {
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v49 = uitofp <16 x i64> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 49 for instruction:   %v50 = uitofp <16 x i32> undef to <16 x double>
 ; Z13:   Cost Model: Found an estimated cost of 49 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
-; AR13:  Cost Model: Found an estimated cost of 4 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
+; Z15:   Cost Model: Found an estimated cost of 4 for instruction:   %v51 = uitofp <16 x i32> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v52 = uitofp <16 x i16> undef to <16 x double>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v53 = uitofp <16 x i16> undef to <16 x float>
 ; CHECK: Cost Model: Found an estimated cost of 65 for instruction:   %v54 = uitofp <16 x i8> undef to <16 x double>
diff --git a/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll b/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
index bbde627838bc4..4bc0508304f9f 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/intrinsics.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 
 define void @bswap_i64(i64 %arg, <2 x i64> %arg2) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_i64':
@@ -69,15 +69,15 @@ define void @bswap_i64_mem(i64* %src, i64 %arg, i64* %dst) {
 define void @bswap_v2i64_mem(<2 x i64>* %src, <2 x i64> %arg, <2 x i64>* %dst) {
 ; CHECK:Printing analysis 'Cost Model Analysis' for function 'bswap_v2i64_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <2 x i64>, <2 x i64>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp2, <2 x i64>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <2 x i64>, <2 x i64>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <2 x i64> %swp3, <2 x i64>* %dst
 
   %Ld1  = load <2 x i64>, <2 x i64>* %src
   %swp1 = tail call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %Ld1)
@@ -117,15 +117,15 @@ define void @bswap_i32_mem(i32* %src, i32 %arg, i32* %dst) {
 define void @bswap_v4i32_mem(<4 x i32>* %src, <4 x i32> %arg, <4 x i32>* %dst) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_v4i32_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <4 x i32>, <4 x i32>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp2, <4 x i32>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <4 x i32>, <4 x i32>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <4 x i32> %swp3, <4 x i32>* %dst
 %Ld1  = load <4 x i32>, <4 x i32>* %src
   %swp1 = tail call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %Ld1)
 
@@ -164,15 +164,15 @@ define void @bswap_i16_mem(i16* %src, i16 %arg, i16* %dst) {
 define void @bswap_v8i16_mem(<8 x i16>* %src, <8 x i16> %arg, <8 x i16>* %dst) {
 ; CHECK: Printing analysis 'Cost Model Analysis' for function 'bswap_v8i16_mem':
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %Ld1 = load <8 x i16>, <8 x i16>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp1 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld1)
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp2 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %arg)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp2, <8 x i16>* %dst
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %Ld2 = load <8 x i16>, <8 x i16>* %src
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %swp3 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld2)
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   store <8 x i16> %swp3, <8 x i16>* %dst
 %Ld1  = load <8 x i16>, <8 x i16>* %src
   %swp1 = tail call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %Ld1)
 
diff --git a/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll b/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
index 86706309e6d14..f3b915d0a64d4 100644
--- a/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
+++ b/llvm/test/Analysis/CostModel/SystemZ/logic-miscext3.ll
@@ -1,25 +1,25 @@
 ; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
 ; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=arch13 \
-; RUN:  | FileCheck %s -check-prefixes=CHECK,AR13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z15 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z15
 
 define void @fun0(i32 %a)  {
 ; CHECK-LABEL: Printing analysis 'Cost Model Analysis' for function 'fun0':
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c0 = xor i32 %l0, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res0 = or i32 %a, %c0
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i32 %a, %c0
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i32 %a, %c0
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c1 = xor i32 %l1, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res1 = and i32 %a, %c1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i32 %a, %c1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i32 %a, %c1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c2 = and i32 %l2, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res2 = xor i32 %c2, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i32 %c2, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i32 %c2, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c3 = or i32 %l3, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res3 = xor i32 %c3, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i32 %c3, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i32 %c3, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c4 = xor i32 %l4, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res4 = xor i32 %c4, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i32 %c4, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i32 %c4, -1
 
 entry:
   %l0 = load i32, i32* undef
@@ -54,19 +54,19 @@ define void @fun1(i64 %a)  {
 ; CHECK-LABEL: Printing analysis 'Cost Model Analysis' for function 'fun1':
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c0 = xor i64 %l0, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res0 = or i64 %a, %c0
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i64 %a, %c0
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res0 = or i64 %a, %c0
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c1 = xor i64 %l1, -1
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res1 = and i64 %a, %c1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i64 %a, %c1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res1 = and i64 %a, %c1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c2 = and i64 %l2, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res2 = xor i64 %c2, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i64 %c2, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res2 = xor i64 %c2, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c3 = or i64 %l3, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res3 = xor i64 %c3, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i64 %c3, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res3 = xor i64 %c3, -1
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %c4 = xor i64 %l4, %a
 ; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %res4 = xor i64 %c4, -1
-; AR13:  Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i64 %c4, -1
+; Z15:   Cost Model: Found an estimated cost of 0 for instruction:   %res4 = xor i64 %c4, -1
 entry:
   %l0 = load i64, i64* undef
   %c0 = xor i64 %l0, -1
diff --git a/llvm/test/Analysis/CostModel/X86/arith-fix.ll b/llvm/test/Analysis/CostModel/X86/arith-fix.ll
index b65ddb6308e68..b72ca1c1eaea7 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-fix.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-fix.ll
@@ -38,9 +38,9 @@ define i32 @smul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 74 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 152 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 33 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -57,9 +57,9 @@ define i32 @smul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -171,9 +171,9 @@ define i32 @smul(i32 %arg) {
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -190,9 +190,9 @@ define i32 @smul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.smul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.smul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I32 = call i32 @llvm.smul.fix.i32(i32 undef, i32 undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.smul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.smul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.smul.fix.i16(i16 undef, i16 undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.smul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.smul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -272,9 +272,9 @@ define i32 @umul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -291,9 +291,9 @@ define i32 @umul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -405,9 +405,9 @@ define i32 @umul(i32 %arg) {
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
@@ -424,9 +424,9 @@ define i32 @umul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call <4 x i64> @llvm.umul.fix.v4i64(<4 x i64> undef, <4 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call <8 x i64> @llvm.umul.fix.v8i64(<8 x i64> undef, <8 x i64> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.umul.fix.i32(i32 undef, i32 undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call <4 x i32> @llvm.umul.fix.v4i32(<4 x i32> undef, <4 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call <8 x i32> @llvm.umul.fix.v8i32(<8 x i32> undef, <8 x i32> undef, i32 3)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call <16 x i32> @llvm.umul.fix.v16i32(<16 x i32> undef, <16 x i32> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call i16 @llvm.umul.fix.i16(i16 undef, i16 undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call <8 x i16> @llvm.umul.fix.v8i16(<8 x i16> undef, <8 x i16> undef, i32 3)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call <16 x i16> @llvm.umul.fix.v16i16(<16 x i16> undef, <16 x i16> undef, i32 3)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-overflow.ll b/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
index b705ed901fac0..5ab9b845ea282 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-overflow.ll
@@ -171,9 +171,9 @@ define i32 @sadd(i32 %arg) {
 ;
 ; SLM-LABEL: 'sadd'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.sadd.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.sadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.sadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.sadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.sadd.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.sadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.sadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -409,9 +409,9 @@ define i32 @uadd(i32 %arg) {
 ;
 ; SLM-LABEL: 'uadd'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.uadd.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.uadd.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.uadd.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.uadd.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.uadd.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.uadd.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -647,9 +647,9 @@ define i32 @ssub(i32 %arg) {
 ;
 ; SLM-LABEL: 'ssub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.ssub.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.ssub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.ssub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.ssub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.ssub.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.ssub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.ssub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -885,9 +885,9 @@ define i32 @usub(i32 %arg) {
 ;
 ; SLM-LABEL: 'usub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = call { i64, i1 } @llvm.usub.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.usub.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.usub.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.usub.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.usub.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.usub.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -994,9 +994,9 @@ define i32 @smul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 27 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 120 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 66 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 136 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1013,9 +1013,9 @@ define i32 @smul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1123,13 +1123,13 @@ define i32 @smul(i32 %arg) {
 ;
 ; SLM-LABEL: 'smul'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %I64 = call { i64, i1 } @llvm.smul.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.smul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 50 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 104 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 90 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 184 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 98 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 200 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 35 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 70 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1146,9 +1146,9 @@ define i32 @smul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 48 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.smul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 100 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.smul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call { i32, i1 } @llvm.smul.with.overflow.i32(i32 undef, i32 undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 25 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 62 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.smul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 128 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.smul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %I16 = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 undef, i16 undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.smul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.smul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1232,9 +1232,9 @@ define i32 @umul(i32 %arg) {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 54 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 112 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 56 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 116 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 132 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1251,9 +1251,9 @@ define i32 @umul(i32 %arg) {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1361,13 +1361,13 @@ define i32 @umul(i32 %arg) {
 ;
 ; SLM-LABEL: 'umul'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %I64 = call { i64, i1 } @llvm.umul.with.overflow.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call { <2 x i64>, <2 x i1> } @llvm.umul.with.overflow.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 42 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 88 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 180 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 46 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 96 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 196 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
@@ -1384,9 +1384,9 @@ define i32 @umul(i32 %arg) {
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4I64 = call { <4 x i64>, <4 x i1> } @llvm.umul.with.overflow.v4i64(<4 x i64> undef, <4 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 84 for instruction: %V8I64 = call { <8 x i64>, <8 x i1> } @llvm.umul.with.overflow.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %I32 = call { i32, i1 } @llvm.umul.with.overflow.i32(i32 undef, i32 undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 52 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
-; GLM-NEXT:  Cost Model: Found an estimated cost of 108 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V4I32 = call { <4 x i32>, <4 x i1> } @llvm.umul.with.overflow.v4i32(<4 x i32> undef, <4 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I32 = call { <8 x i32>, <8 x i1> } @llvm.umul.with.overflow.v8i32(<8 x i32> undef, <8 x i32> undef)
+; GLM-NEXT:  Cost Model: Found an estimated cost of 124 for instruction: %V16I32 = call { <16 x i32>, <16 x i1> } @llvm.umul.with.overflow.v16i32(<16 x i32> undef, <16 x i32> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %I16 = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 undef, i16 undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I16 = call { <8 x i16>, <8 x i1> } @llvm.umul.with.overflow.v8i16(<8 x i16> undef, <8 x i16> undef)
 ; GLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I16 = call { <16 x i16>, <16 x i1> } @llvm.umul.with.overflow.v16i16(<16 x i16> undef, <16 x i16> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-ssat.ll b/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
index d99bba3268d93..c951330bc92e0 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-ssat.ll
@@ -170,9 +170,9 @@ define i32 @add(i32 %arg) {
 ;
 ; SLM-LABEL: 'add'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.sadd.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I64 = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V8I64 = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.sadd.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -404,9 +404,9 @@ define i32 @sub(i32 %arg) {
 ;
 ; SLM-LABEL: 'sub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.ssub.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 43 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V2I64 = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V4I64 = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 59 for instruction: %V8I64 = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I32 = call i32 @llvm.ssub.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I32 = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V8I32 = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/arith-usat.ll b/llvm/test/Analysis/CostModel/X86/arith-usat.ll
index e7aaba8c02458..a83c16e840700 100644
--- a/llvm/test/Analysis/CostModel/X86/arith-usat.ll
+++ b/llvm/test/Analysis/CostModel/X86/arith-usat.ll
@@ -170,9 +170,9 @@ define i32 @add(i32 %arg) {
 ;
 ; SLM-LABEL: 'add'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.uadd.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.uadd.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.uadd.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.uadd.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = call <8 x i32> @llvm.uadd.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
@@ -404,9 +404,9 @@ define i32 @sub(i32 %arg) {
 ;
 ; SLM-LABEL: 'sub'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I64 = call i64 @llvm.usub.sat.i64(i64 undef, i64 undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V2I64 = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> undef, <2 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 14 for instruction: %V4I64 = call <4 x i64> @llvm.usub.sat.v4i64(<4 x i64> undef, <4 x i64> undef)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 28 for instruction: %V8I64 = call <8 x i64> @llvm.usub.sat.v8i64(<8 x i64> undef, <8 x i64> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %I32 = call i32 @llvm.usub.sat.i32(i32 undef, i32 undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> undef, <4 x i32> undef)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = call <8 x i32> @llvm.usub.sat.v8i32(<8 x i32> undef, <8 x i32> undef)
diff --git a/llvm/test/Analysis/CostModel/X86/cast.ll b/llvm/test/Analysis/CostModel/X86/cast.ll
index c3e1f624cdc99..b43e871ddf6ee 100644
--- a/llvm/test/Analysis/CostModel/X86/cast.ll
+++ b/llvm/test/Analysis/CostModel/X86/cast.ll
@@ -90,12 +90,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %D3 = zext <16 x i16> undef to <16 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %D4 = zext <16 x i8> undef to <16 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %D5 = zext <16 x i1> undef to <16 x i32>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %F = trunc <8 x i32> undef to <8 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F1 = trunc <16 x i16> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %F2 = trunc <8 x i32> undef to <8 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %G1 = trunc <16 x i32> undef to <16 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G2 = trunc <16 x i32> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
@@ -120,12 +120,12 @@ define i32 @zext_sext(<8 x i1> %in) {
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D3 = zext <16 x i16> undef to <16 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %D4 = zext <16 x i8> undef to <16 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %D5 = zext <16 x i1> undef to <16 x i32>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %E = trunc <4 x i64> undef to <4 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F = trunc <8 x i32> undef to <8 x i16>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F1 = trunc <16 x i16> undef to <16 x i8>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F2 = trunc <8 x i32> undef to <8 x i8>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
-; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %F3 = trunc <4 x i64> undef to <4 x i8>
+; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G = trunc <8 x i64> undef to <8 x i32>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %G1 = trunc <16 x i32> undef to <16 x i16>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %G2 = trunc <16 x i32> undef to <16 x i8>
 ; SSE41-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
diff --git a/llvm/test/Analysis/CostModel/X86/fshl.ll b/llvm/test/Analysis/CostModel/X86/fshl.ll
index 38621c35a8868..976fa750ff9c0 100644
--- a/llvm/test/Analysis/CostModel/X86/fshl.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshl.ll
@@ -57,9 +57,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 ;
 ; SLM-LABEL: 'var_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 %c64)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'var_funnel_i64'
@@ -385,9 +385,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatvar_funnel_i64'
@@ -777,9 +777,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ;
 ; SLM-LABEL: 'constant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'constant_funnel_i64'
@@ -1093,9 +1093,9 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 ;
 ; SLM-LABEL: 'splatconstant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshl.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshl.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshl.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatconstant_funnel_i64'
diff --git a/llvm/test/Analysis/CostModel/X86/fshr.ll b/llvm/test/Analysis/CostModel/X86/fshr.ll
index 31e409cc9a7b8..ef74f9d711786 100644
--- a/llvm/test/Analysis/CostModel/X86/fshr.ll
+++ b/llvm/test/Analysis/CostModel/X86/fshr.ll
@@ -57,9 +57,9 @@ define void @var_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8 x i64
 ;
 ; SLM-LABEL: 'var_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 %c64)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 17 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %c128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 34 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %c256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 68 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %c512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'var_funnel_i64'
@@ -385,9 +385,9 @@ define void @splatvar_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u128 = shufflevector <2 x i64> %c128, <2 x i64> undef, <2 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u256 = shufflevector <4 x i64> %c256, <4 x i64> undef, <4 x i32> zeroinitializer
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %u512 = shufflevector <8 x i64> %c512, <8 x i64> undef, <8 x i32> zeroinitializer
-; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> %u128)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 22 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> %u256)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 44 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> %u512)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatvar_funnel_i64'
@@ -777,9 +777,9 @@ define void @constant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256, <8
 ;
 ; SLM-LABEL: 'constant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 30 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 60 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 1, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 1, i64 7, i64 15, i64 31>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 64 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 1, i64 7, i64 15, i64 31, i64 1, i64 7, i64 15, i64 31>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'constant_funnel_i64'
@@ -1093,9 +1093,9 @@ define void @splatconstant_funnel_i64(i64 %a64, <2 x i64> %a128, <4 x i64> %a256
 ;
 ; SLM-LABEL: 'splatconstant_funnel_i64'
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %I64 = call i64 @llvm.fshr.i64(i64 %a64, i64 %b64, i64 7)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
-; SLM-NEXT:  Cost Model: Found an estimated cost of 36 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V2I64 = call <2 x i64> @llvm.fshr.v2i64(<2 x i64> %a128, <2 x i64> %b128, <2 x i64> <i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V4I64 = call <4 x i64> @llvm.fshr.v4i64(<4 x i64> %a256, <4 x i64> %b256, <4 x i64> <i64 7, i64 7, i64 7, i64 7>)
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V8I64 = call <8 x i64> @llvm.fshr.v8i64(<8 x i64> %a512, <8 x i64> %b512, <8 x i64> <i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7, i64 7>)
 ; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; GLM-LABEL: 'splatconstant_funnel_i64'
diff --git a/llvm/test/Analysis/CostModel/X86/icmp.ll b/llvm/test/Analysis/CostModel/X86/icmp.ll
index caf20d1b9cb5c..af84cf9fc9cb0 100644
--- a/llvm/test/Analysis/CostModel/X86/icmp.ll
+++ b/llvm/test/Analysis/CostModel/X86/icmp.ll
@@ -11,7 +11,7 @@
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX1
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mattr=+xop,+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,XOPAVX2
 ;
-; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
+; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=slm | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=goldmont | FileCheck %s --check-prefixes=CHECK,SSE,SSE42
 ; RUN: opt < %s -mtriple=x86_64-apple-macosx10.8.0 -cost-model -analyze -mcpu=btver2 | FileCheck %s --check-prefixes=BTVER2
 
@@ -269,6 +269,29 @@ define i32 @cmp_int_eq(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_eq'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp eq <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp eq <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp eq <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp eq i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp eq <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp eq <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp eq <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp eq <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp eq i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp eq <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp eq <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp eq <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp eq <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp eq i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp eq <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp eq <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp eq <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp eq <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_eq'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp eq i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp eq <16 x i8> undef, undef
@@ -573,6 +596,29 @@ define i32 @cmp_int_ne(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ne'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ne <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ne <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ne <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ne i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ne <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ne <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ne <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ne <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ne i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ne <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ne <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ne <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ne <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ne i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp ne <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp ne <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp ne <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp ne <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ne'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ne i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ne <16 x i8> undef, undef
@@ -877,6 +923,29 @@ define i32 @cmp_int_sge(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sge'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sge <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sge <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sge <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sge i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sge <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sge <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sge <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sge <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sge i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sge <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sge <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sge <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sge <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sge i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp sge <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp sge <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp sge <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp sge <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sge'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sge i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sge <16 x i8> undef, undef
@@ -1181,6 +1250,29 @@ define i32 @cmp_int_uge(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_uge'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp uge <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp uge <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp uge <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp uge i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp uge <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp uge <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp uge <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp uge <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp uge i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp uge <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp uge <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp uge <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp uge <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp uge i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = icmp uge <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = icmp uge <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = icmp uge <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V16I64 = icmp uge <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_uge'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp uge i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp uge <16 x i8> undef, undef
@@ -1485,6 +1577,29 @@ define i32 @cmp_int_sgt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sgt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp sgt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp sgt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp sgt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sgt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp sgt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp sgt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp sgt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp sgt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sgt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp sgt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp sgt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp sgt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp sgt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sgt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp sgt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp sgt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp sgt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp sgt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sgt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sgt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp sgt <16 x i8> undef, undef
@@ -1789,6 +1904,29 @@ define i32 @cmp_int_ugt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ugt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ugt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ugt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ugt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ugt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ugt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ugt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ugt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ugt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ugt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ugt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ugt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ugt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ugt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ugt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ugt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp ugt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp ugt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp ugt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ugt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ugt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ugt <16 x i8> undef, undef
@@ -2093,6 +2231,29 @@ define i32 @cmp_int_sle(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_sle'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp sle <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp sle <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp sle <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp sle i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp sle <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp sle <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp sle <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp sle <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp sle i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp sle <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp sle <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp sle <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp sle <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp sle i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2I64 = icmp sle <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V4I64 = icmp sle <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V8I64 = icmp sle <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V16I64 = icmp sle <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_sle'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp sle i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp sle <16 x i8> undef, undef
@@ -2397,6 +2558,29 @@ define i32 @cmp_int_ule(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ule'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I8 = icmp ule <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I8 = icmp ule <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V128I8 = icmp ule <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ule i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I16 = icmp ule <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I16 = icmp ule <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I16 = icmp ule <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V64I16 = icmp ule <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ule i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4I32 = icmp ule <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8I32 = icmp ule <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V16I32 = icmp ule <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V32I32 = icmp ule <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ule i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2I64 = icmp ule <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4I64 = icmp ule <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 20 for instruction: %V8I64 = icmp ule <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V16I64 = icmp ule <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ule'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ule i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I8 = icmp ule <16 x i8> undef, undef
@@ -2701,6 +2885,29 @@ define i32 @cmp_int_slt(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_slt'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V32I8 = icmp slt <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V64I8 = icmp slt <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V128I8 = icmp slt <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp slt i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8I16 = icmp slt <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V16I16 = icmp slt <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V32I16 = icmp slt <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V64I16 = icmp slt <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp slt i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4I32 = icmp slt <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8I32 = icmp slt <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16I32 = icmp slt <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V32I32 = icmp slt <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp slt i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2I64 = icmp slt <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4I64 = icmp slt <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V8I64 = icmp slt <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V16I64 = icmp slt <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_slt'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp slt i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16I8 = icmp slt <16 x i8> undef, undef
@@ -3005,6 +3212,29 @@ define i32 @cmp_int_ult(i32 %arg) {
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
 ; XOPAVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
+; SLM-LABEL: 'cmp_int_ult'
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V32I8 = icmp ult <32 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V64I8 = icmp ult <64 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V128I8 = icmp ult <128 x i8> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I16 = icmp ult i16 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8I16 = icmp ult <8 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16I16 = icmp ult <16 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V32I16 = icmp ult <32 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V64I16 = icmp ult <64 x i16> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I32 = icmp ult i32 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4I32 = icmp ult <4 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V8I32 = icmp ult <8 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V16I32 = icmp ult <16 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 24 for instruction: %V32I32 = icmp ult <32 x i32> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I64 = icmp ult i64 undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2I64 = icmp ult <2 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 8 for instruction: %V4I64 = icmp ult <4 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 16 for instruction: %V8I64 = icmp ult <8 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 32 for instruction: %V16I64 = icmp ult <16 x i64> undef, undef
+; SLM-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
+;
 ; BTVER2-LABEL: 'cmp_int_ult'
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %I8 = icmp ult i8 undef, undef
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16I8 = icmp ult <16 x i8> undef, undef
diff --git a/llvm/test/Analysis/CostModel/X86/trunc.ll b/llvm/test/Analysis/CostModel/X86/trunc.ll
index 9668da564133e..37d18c3e3518a 100644
--- a/llvm/test/Analysis/CostModel/X86/trunc.ll
+++ b/llvm/test/Analysis/CostModel/X86/trunc.ll
@@ -13,35 +13,35 @@
 
 define i32 @trunc_vXi32() {
 ; SSE-LABEL: 'trunc_vXi32'
-; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
-; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
+; SSE-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; SSE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi32'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi32'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512-LABEL: 'trunc_vXi32'
-; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
 ; AVX512-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi32'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i32>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i32>
@@ -56,12 +56,12 @@ define i32 @trunc_vXi32() {
 
 define i32 @trunc_vXi16() {
 ; SSE2-LABEL: 'trunc_vXi16'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -69,12 +69,12 @@ define i32 @trunc_vXi16() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi16'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -82,12 +82,12 @@ define i32 @trunc_vXi16() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi16'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -95,12 +95,12 @@ define i32 @trunc_vXi16() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi16'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -108,12 +108,12 @@ define i32 @trunc_vXi16() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi16'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -121,12 +121,12 @@ define i32 @trunc_vXi16() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'trunc_vXi16'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -134,12 +134,12 @@ define i32 @trunc_vXi16() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'trunc_vXi16'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -147,12 +147,12 @@ define i32 @trunc_vXi16() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi16'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i16>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i16>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 6 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i16>
@@ -175,19 +175,19 @@ define i32 @trunc_vXi16() {
 
 define i32 @trunc_vXi8() {
 ; SSE2-LABEL: 'trunc_vXi8'
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -196,19 +196,19 @@ define i32 @trunc_vXi8() {
 ; SSE2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSSE3-LABEL: 'trunc_vXi8'
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 9 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 19 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 39 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 79 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 159 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -217,19 +217,19 @@ define i32 @trunc_vXi8() {
 ; SSSE3-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; SSE42-LABEL: 'trunc_vXi8'
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 63 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -238,19 +238,19 @@ define i32 @trunc_vXi8() {
 ; SSE42-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX1-LABEL: 'trunc_vXi8'
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -259,19 +259,19 @@ define i32 @trunc_vXi8() {
 ; AVX1-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX2-LABEL: 'trunc_vXi8'
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -280,19 +280,19 @@ define i32 @trunc_vXi8() {
 ; AVX2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512F-LABEL: 'trunc_vXi8'
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -301,19 +301,19 @@ define i32 @trunc_vXi8() {
 ; AVX512F-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; AVX512BW-LABEL: 'trunc_vXi8'
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
@@ -322,19 +322,19 @@ define i32 @trunc_vXi8() {
 ; AVX512BW-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret i32 undef
 ;
 ; BTVER2-LABEL: 'trunc_vXi8'
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V2i64 = trunc <2 x i64> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V4i64 = trunc <4 x i64> undef to <4 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V8i64 = trunc <8 x i64> undef to <8 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 11 for instruction: %V16i64 = trunc <16 x i64> undef to <16 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 23 for instruction: %V32i64 = trunc <32 x i64> undef to <32 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 47 for instruction: %V64i64 = trunc <64 x i64> undef to <64 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V2i32 = trunc <2 x i32> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V4i32 = trunc <4 x i32> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V8i32 = trunc <8 x i32> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 7 for instruction: %V16i32 = trunc <16 x i32> undef to <16 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 15 for instruction: %V32i32 = trunc <32 x i32> undef to <32 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 31 for instruction: %V64i32 = trunc <64 x i32> undef to <64 x i8>
-; BTVER2-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
+; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2i16 = trunc <2 x i16> undef to <2 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4i16 = trunc <4 x i16> undef to <4 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %V8i16 = trunc <8 x i16> undef to <8 x i8>
 ; BTVER2-NEXT:  Cost Model: Found an estimated cost of 4 for instruction: %V16i16 = trunc <16 x i16> undef to <16 x i8>
diff --git a/llvm/test/Analysis/DDG/basic-a.ll b/llvm/test/Analysis/DDG/basic-a.ll
new file mode 100644
index 0000000000000..4c05259a88604
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-a.ll
@@ -0,0 +1,189 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test1.for.body ], [ 0, %test1.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %conv = uitofp i64 %n to float
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %conv
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx1, align 4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test1.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+;; No memory dependencies.
+;; void test1(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    a[i] = b[i] + n;
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond1 = icmp ne i64 0, %n
+  br i1 %exitcond1, label %test1.for.body, label %for.end
+
+test1.for.body:                                         ; preds = %entry, %test1.for.body
+  %i.02 = phi i64 [ %inc, %test1.for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %conv = uitofp i64 %n to float
+  %add = fadd float %0, %conv
+  %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx1, align 4
+  %inc = add i64 %i.02, 1
+  %exitcond = icmp ne i64 %inc, %n
+  br i1 %exitcond, label %test1.for.body, label %for.end
+
+for.end:                                          ; preds = %test1.for.body, %entry
+  ret void
+}
+
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.body':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test2.for.body ], [ 0, %test2.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx1, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+; CHECK-NEXT:  [memory] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test2.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+;; Loop-independent memory dependencies.
+;; void test2(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    a[i] = b[i] + a[i];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond1 = icmp ne i64 0, %n
+  br i1 %exitcond1, label %test2.for.body, label %for.end
+
+test2.for.body:                                         ; preds = %entry, %test2.for.body
+  %i.02 = phi i64 [ %inc, %test2.for.body ], [ 0, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %a, i64 %i.02
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd float %0, %1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx2, align 4
+  %inc = add i64 %i.02, 1
+  %exitcond = icmp ne i64 %inc, %n
+  br i1 %exitcond, label %test2.for.body, label %for.end
+
+for.end:                                          ; preds = %test2.for.body, %entry
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Analysis/DDG/basic-b.ll b/llvm/test/Analysis/DDG/basic-b.ll
new file mode 100644
index 0000000000000..2a90d028a6fd4
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-b.ll
@@ -0,0 +1,216 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test1.for.body ], [ 1, %test1.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub1 = add i64 %i.02, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %sub1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx3, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [memory] to [[N9]]
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp, label %test1.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+;; Loop-carried dependence requiring edge-reversal to expose a cycle
+;; in the graph.
+;; void test(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 1; i < n-1; i++)
+;;    a[i] = b[i] + a[i-1];
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %sub = add i64 %n, -1
+  %cmp1 = icmp ult i64 1, %sub
+  br i1 %cmp1, label %test1.for.body, label %for.end
+
+test1.for.body:                                         ; preds = %entry, %test1.for.body
+  %i.02 = phi i64 [ %inc, %test1.for.body ], [ 1, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %sub1 = add i64 %i.02, -1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %sub1
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx3, align 4
+  %inc = add i64 %i.02, 1
+  %cmp = icmp ult i64 %inc, %sub
+  br i1 %cmp, label %test1.for.body, label %for.end
+
+for.end:                                          ; preds = %test1.for.body, %entry
+  ret void
+}
+
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.body':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %i.02 = phi i64 [ %inc, %test2.for.body ], [ 1, %test2.for.body.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N6:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = load float, float* %arrayidx, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add1 = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx2 = getelementptr inbounds float, float* %a, i64 %add1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx2, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+; CHECK-NEXT:  [memory] to [[N10:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %0, %1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N10]]
+
+; CHECK: Node Address:[[N10]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx3, align 4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %i.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp, label %test2.for.body, label %for.end.loopexit
+; CHECK-NEXT: Edges:none!
+
+
+;; Forward loop-carried dependence *not* causing a cycle.
+;; void test2(unsigned long n, float * restrict a, float * restrict b) {
+;;  for (unsigned long i = 1; i < n-1; i++)
+;;    a[i] = b[i] + a[i+1];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %sub = add i64 %n, -1
+  %cmp1 = icmp ult i64 1, %sub
+  br i1 %cmp1, label %test2.for.body, label %for.end
+
+test2.for.body:                                         ; preds = %entry, %test2.for.body
+  %i.02 = phi i64 [ %inc, %test2.for.body ], [ 1, %entry ]
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %i.02
+  %0 = load float, float* %arrayidx, align 4
+  %add1 = add i64 %i.02, 1
+  %arrayidx2 = getelementptr inbounds float, float* %a, i64 %add1
+  %1 = load float, float* %arrayidx2, align 4
+  %add = fadd float %0, %1
+  %arrayidx3 = getelementptr inbounds float, float* %a, i64 %i.02
+  store float %add, float* %arrayidx3, align 4
+  %inc = add i64 %i.02, 1
+  %cmp = icmp ult i64 %inc, %sub
+  br i1 %cmp, label %test2.for.body, label %for.end
+
+for.end:                                          ; preds = %test2.for.body, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/DDG/basic-loopnest.ll b/llvm/test/Analysis/DDG/basic-loopnest.ll
new file mode 100644
index 0000000000000..a2abbf8c09bde
--- /dev/null
+++ b/llvm/test/Analysis/DDG/basic-loopnest.ll
@@ -0,0 +1,434 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.cond1.preheader':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %test1.for.cond1.preheader.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %sub = add i64 %n, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %cmp21 = icmp ult i64 1, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   br i1 %cmp21, label %for.body4.preheader, label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N10:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %for.body4.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N14:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N15:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N15]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N14]]
+
+; CHECK: Node Address:[[N14]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N16:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N16]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx5, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %2 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N18]]:single-instruction
+; CHECK-NEXT: Instructions: 
+; CHECK-NEXT:   %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %sub7 = add i64 %j.02, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19]]
+
+; CHECK: Node Address:[[N19]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %sub7
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N20]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %3 = load float, float* %arrayidx8, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17]]
+
+; CHECK: Node Address:[[N17]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %1, %3
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N26:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %4 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N27:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N27]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N26]]
+
+; CHECK: Node Address:[[N26]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx11, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [memory] to [[N20]]
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+; CHECK-NEXT:  [def-use] to [[N10]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp2 = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N21:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N21]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp2, label %for.body4, label %for.inc12.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc13 = add i64 %i.04, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N22:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N22]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc13, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N23:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N23]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test1.for.cond1.preheader, label %for.end14.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N24:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.body4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N25:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+
+
+;; This test has a cycle.
+;; void test1(unsigned long n, float a[][n], float b[][n]) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    for (unsigned long j = 1; j < n-1; j++)
+;;      a[i][j] = b[i][j] + a[i][j-1];
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond3 = icmp ne i64 0, %n
+  br i1 %exitcond3, label %test1.for.cond1.preheader, label %for.end14
+
+test1.for.cond1.preheader:                              ; preds = %entry, %for.inc12
+  %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %entry ]
+  %sub = add i64 %n, -1
+  %cmp21 = icmp ult i64 1, %sub
+  br i1 %cmp21, label %for.body4, label %for.inc12
+
+for.body4:                                        ; preds = %test1.for.cond1.preheader, %for.body4
+  %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %test1.for.cond1.preheader ]
+  %0 = mul nsw i64 %i.04, %n
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+  %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+  %1 = load float, float* %arrayidx5, align 4
+  %2 = mul nsw i64 %i.04, %n
+  %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+  %sub7 = add i64 %j.02, -1
+  %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %sub7
+  %3 = load float, float* %arrayidx8, align 4
+  %add = fadd float %1, %3
+  %4 = mul nsw i64 %i.04, %n
+  %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+  %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+  store float %add, float* %arrayidx11, align 4
+  %inc = add i64 %j.02, 1
+  %cmp2 = icmp ult i64 %inc, %sub
+  br i1 %cmp2, label %for.body4, label %for.inc12
+
+for.inc12:                                        ; preds = %for.body4, %test1.for.cond1.preheader
+  %inc13 = add i64 %i.04, 1
+  %exitcond = icmp ne i64 %inc13, %n
+  br i1 %exitcond, label %test1.for.cond1.preheader, label %for.end14
+
+for.end14:                                        ; preds = %for.inc12, %entry
+  ret void
+}
+
+
+
+; CHECK-LABEL: 'DDG' for loop 'test2.for.cond1.preheader':
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %test2.for.cond1.preheader.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N2:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N3:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N4:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N5:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N6:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %sub = add i64 %n, -1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N8:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N8]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %cmp21 = icmp ult i64 1, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N9:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N9]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   br i1 %cmp21, label %for.body4.preheader, label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N10:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %for.body4.preheader ]
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N11:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N12:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N13:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N14:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N5]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %0 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N15:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N15]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N14]]
+
+; CHECK: Node Address:[[N14]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N16:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N16]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %1 = load float, float* %arrayidx5, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N4]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %2 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N18:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N18]]:single-instruction
+; CHECK-NEXT: Instructions: 
+; CHECK-NEXT:   %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N13]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add7 = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N19]]
+
+; CHECK: Node Address:[[N19]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %add7
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N20:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N20]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %3 = load float, float* %arrayidx8, align 4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N17]]
+; CHECK-NEXT:  [memory] to [[N26:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N17]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %add = fadd float %1, %3
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N26]]
+
+; CHECK: Node Address:[[N3]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %4 = mul nsw i64 %i.04, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N27:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N27]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N12]]
+
+; CHECK: Node Address:[[N12]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N26]]
+
+; CHECK: Node Address:[[N26]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    store float %add, float* %arrayidx11, align 4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N11]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc = add i64 %j.02, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N7]]
+; CHECK-NEXT:  [def-use] to [[N10]]
+
+; CHECK: Node Address:[[N7]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %cmp2 = icmp ult i64 %inc, %sub
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N21:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N21]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %cmp2, label %for.body4, label %for.inc12.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N2]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %inc13 = add i64 %i.04, 1
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N22:0x[0-9a-f]*]]
+; CHECK-NEXT:  [def-use] to [[N1]]
+
+; CHECK: Node Address:[[N22]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    %exitcond = icmp ne i64 %inc13, %n
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [def-use] to [[N23:0x[0-9a-f]*]]
+
+; CHECK: Node Address:[[N23]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br i1 %exitcond, label %test2.for.cond1.preheader, label %for.end14.loopexit
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N24:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.body4
+; CHECK-NEXT: Edges:none!
+
+; CHECK: Node Address:[[N25:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:    br label %for.inc12
+; CHECK-NEXT: Edges:none!
+
+;; This test has no cycles.
+;; void test2(unsigned long n, float a[][n], float b[][n]) {
+;;  for (unsigned long i = 0; i < n; i++)
+;;    for (unsigned long j = 1; j < n-1; j++)
+;;      a[i][j] = b[i][j] + a[i][j+1];
+;; }
+
+define void @test2(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %exitcond3 = icmp ne i64 0, %n
+  br i1 %exitcond3, label %test2.for.cond1.preheader, label %for.end14
+
+test2.for.cond1.preheader:                              ; preds = %entry, %for.inc12
+  %i.04 = phi i64 [ %inc13, %for.inc12 ], [ 0, %entry ]
+  %sub = add i64 %n, -1
+  %cmp21 = icmp ult i64 1, %sub
+  br i1 %cmp21, label %for.body4, label %for.inc12
+
+for.body4:                                        ; preds = %test2.for.cond1.preheader, %for.body4
+  %j.02 = phi i64 [ %inc, %for.body4 ], [ 1, %test2.for.cond1.preheader ]
+  %0 = mul nsw i64 %i.04, %n
+  %arrayidx = getelementptr inbounds float, float* %b, i64 %0
+  %arrayidx5 = getelementptr inbounds float, float* %arrayidx, i64 %j.02
+  %1 = load float, float* %arrayidx5, align 4
+  %2 = mul nsw i64 %i.04, %n
+  %arrayidx6 = getelementptr inbounds float, float* %a, i64 %2
+  %add7 = add i64 %j.02, 1
+  %arrayidx8 = getelementptr inbounds float, float* %arrayidx6, i64 %add7
+  %3 = load float, float* %arrayidx8, align 4
+  %add = fadd float %1, %3
+  %4 = mul nsw i64 %i.04, %n
+  %arrayidx10 = getelementptr inbounds float, float* %a, i64 %4
+  %arrayidx11 = getelementptr inbounds float, float* %arrayidx10, i64 %j.02
+  store float %add, float* %arrayidx11, align 4
+  %inc = add i64 %j.02, 1
+  %cmp2 = icmp ult i64 %inc, %sub
+  br i1 %cmp2, label %for.body4, label %for.inc12
+
+for.inc12:                                        ; preds = %for.body4, %test2.for.cond1.preheader
+  %inc13 = add i64 %i.04, 1
+  %exitcond = icmp ne i64 %inc13, %n
+  br i1 %exitcond, label %test2.for.cond1.preheader, label %for.end14
+
+for.end14:                                        ; preds = %for.inc12, %entry
+  ret void
+}
\ No newline at end of file
diff --git a/llvm/test/Analysis/DDG/root-node.ll b/llvm/test/Analysis/DDG/root-node.ll
new file mode 100644
index 0000000000000..1175796e3c2c6
--- /dev/null
+++ b/llvm/test/Analysis/DDG/root-node.ll
@@ -0,0 +1,52 @@
+; RUN: opt < %s -disable-output "-passes=print<ddg>" 2>&1 | FileCheck %s
+
+; CHECK-LABEL: 'DDG' for loop 'test1.for.body':
+
+; CHECK: Node Address:[[N1:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %i2.03 = phi i64 [ 0, %for.body.lr.ph ], [ %inc2, %test1.for.body ]
+
+; CHECK: Node Address:[[N2:0x[0-9a-f]*]]:single-instruction
+; CHECK-NEXT: Instructions:
+; CHECK-NEXT:   %i1.02 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %test1.for.body ]
+
+; CHECK: Node Address:[[ROOT:0x[0-9a-f]*]]:root
+; CHECK-NEXT: Edges:
+; CHECK-NEXT:  [rooted] to [[N1]]
+; CHECK-NEXT:  [rooted] to [[N2]]
+
+
+;; // Two separate components in the graph. Root node must link to both.
+;; void test1(unsigned long n, float * restrict a, float * restrict b) {
+;;   for (unsigned long i1 = 0, i2 = 0; i1 < n; i1++, i2++) {
+;;     a[i1] = 1;
+;;     b[i2] = -1;
+;;   }
+;; }
+
+define void @test1(i64 %n, float* noalias %a, float* noalias %b) {
+entry:
+  %cmp1 = icmp ult i64 0, %n
+  br i1 %cmp1, label %for.body.lr.ph, label %for.end
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %test1.for.body
+
+test1.for.body:                                         ; preds = %for.body.lr.ph, %test1.for.body
+  %i2.03 = phi i64 [ 0, %for.body.lr.ph ], [ %inc2, %test1.for.body ]
+  %i1.02 = phi i64 [ 0, %for.body.lr.ph ], [ %inc, %test1.for.body ]
+  %arrayidx = getelementptr inbounds float, float* %a, i64 %i1.02
+  store float 1.000000e+00, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i64 %i2.03
+  store float -1.000000e+00, float* %arrayidx1, align 4
+  %inc = add i64 %i1.02, 1
+  %inc2 = add i64 %i2.03, 1
+  %cmp = icmp ult i64 %inc, %n
+  br i1 %cmp, label %test1.for.body, label %for.cond.for.end_crit_edge
+
+for.cond.for.end_crit_edge:                       ; preds = %test1.for.body
+  br label %for.end
+
+for.end:                                          ; preds = %for.cond.for.end_crit_edge, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
index 57ccf85b8408b..7063f20cd0c30 100644
--- a/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
+++ b/llvm/test/Analysis/DependenceAnalysis/SimpleSIVNoValidityCheck.ll
@@ -235,3 +235,4 @@ for.end11.loopexit:                               ; preds = %for.inc9
 for.end11:                                        ; preds = %for.end11.loopexit, %entry
   ret void
 }
+
diff --git a/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll
new file mode 100644
index 0000000000000..cb3e42de363a4
--- /dev/null
+++ b/llvm/test/Analysis/DivergenceAnalysis/AMDGPU/b42473-r1-crash.ll
@@ -0,0 +1,111 @@
+; RUN: opt -mtriple amdgcn-unknown-amdhsa -analyze -divergence -use-gpu-divergence-analysis %s | FileCheck %s
+
+declare i32 @gf2(i32)
+declare i32 @gf1(i32)
+
+define  void @tw1(i32 addrspace(4)* noalias nocapture readonly %A, i32 addrspace(4)* noalias nocapture %B) local_unnamed_addr #2 {
+; CHECK: Printing analysis 'Legacy Divergence Analysis' for function 'tw1':
+; CHECK: DIVERGENT: i32 addrspace(4)* %A
+; CHECK: DIVERGENT: i32 addrspace(4)* %B
+entry:
+; CHECK: DIVERGENT:       %call = tail call i32 @gf2(i32 0) #0
+; CHECK: DIVERGENT:       %cmp = icmp ult i32 %call, 16
+; CHECK: DIVERGENT:       br i1 %cmp, label %if.then, label %new_exit
+  %call = tail call  i32 @gf2(i32 0) #3
+  %cmp = icmp ult i32 %call, 16
+  br i1 %cmp, label %if.then, label %new_exit
+
+if.then:
+; CHECK: DIVERGENT:       %call1 = tail call i32 @gf1(i32 0) #0
+; CHECK: DIVERGENT:       %arrayidx = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %call1
+; CHECK: DIVERGENT:       %0 = load i32, i32 addrspace(4)* %arrayidx, align 4
+; CHECK: DIVERGENT:       %cmp225 = icmp sgt i32 %0, 0
+; CHECK: DIVERGENT:       %arrayidx10 = getelementptr inbounds i32, i32 addrspace(4)* %B, i32 %call1
+; CHECK: DIVERGENT:       br i1 %cmp225, label %while.body.preheader, label %if.then.while.end_crit_edge
+  %call1 = tail call  i32 @gf1(i32 0) #4
+  %arrayidx = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %call1
+  %0 = load i32, i32 addrspace(4)* %arrayidx, align 4
+  %cmp225 = icmp sgt i32 %0, 0
+  %arrayidx10 = getelementptr inbounds i32, i32 addrspace(4)* %B, i32 %call1
+  br i1 %cmp225, label %while.body.preheader, label %if.then.while.end_crit_edge
+
+while.body.preheader:
+  br label %while.body
+
+if.then.while.end_crit_edge:
+; CHECK: DIVERGENT:       %.pre = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %.pre = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  br label %while.end
+
+while.body:
+; CHECK-NOT: DIVERGENT:                  %i.026 = phi i32 [ %inc, %if.end.while.body_crit_edge ], [ 0, %while.body.preheader ]
+; CHECK: DIVERGENT:       %call3 = tail call i32 @gf1(i32 0) #0
+; CHECK: DIVERGENT:       %cmp4 = icmp ult i32 %call3, 10
+; CHECK: DIVERGENT:       %arrayidx6 = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %i.026
+; CHECK: DIVERGENT:       %1 = load i32, i32 addrspace(4)* %arrayidx6, align 4
+; CHECK: DIVERGENT:       br i1 %cmp4, label %if.then5, label %if.else
+  %i.026 = phi i32 [ %inc, %if.end.while.body_crit_edge ], [ 0, %while.body.preheader ]
+  %call3 = tail call  i32 @gf1(i32 0) #4
+  %cmp4 = icmp ult i32 %call3, 10
+  %arrayidx6 = getelementptr inbounds i32, i32 addrspace(4)* %A, i32 %i.026
+  %1 = load i32, i32 addrspace(4)* %arrayidx6, align 4
+  br i1 %cmp4, label %if.then5, label %if.else
+
+if.then5:
+; CHECK: DIVERGENT:       %mul = shl i32 %1, 1
+; CHECK: DIVERGENT:       %2 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK: DIVERGENT:       %add = add nsw i32 %2, %mul
+  %mul = shl i32 %1, 1
+  %2 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %add = add nsw i32 %2, %mul
+  br label %if.end
+
+if.else:
+; CHECK: DIVERGENT:       %mul9 = shl i32 %1, 2
+; CHECK: DIVERGENT:       %3 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK: DIVERGENT:       %add11 = add nsw i32 %3, %mul9
+  %mul9 = shl i32 %1, 2
+  %3 = load i32, i32 addrspace(4)* %arrayidx10, align 4
+  %add11 = add nsw i32 %3, %mul9
+  br label %if.end
+
+if.end:
+; CHECK: DIVERGENT:       %storemerge = phi i32 [ %add11, %if.else ], [ %add, %if.then5 ]
+; CHECK: DIVERGENT:       store i32 %storemerge, i32 addrspace(4)* %arrayidx10, align 4
+; CHECK-NOT: DIVERGENT:                  %inc = add nuw nsw i32 %i.026, 1
+; CHECK: DIVERGENT:       %exitcond = icmp ne i32 %inc, %0
+; CHECK: DIVERGENT:       br i1 %exitcond, label %if.end.while.body_crit_edge, label %while.end.loopexit
+  %storemerge = phi i32 [ %add11, %if.else ], [ %add, %if.then5 ]
+  store i32 %storemerge, i32 addrspace(4)* %arrayidx10, align 4
+  %inc = add nuw nsw i32 %i.026, 1
+  %exitcond = icmp ne i32 %inc, %0
+  br i1 %exitcond, label %if.end.while.body_crit_edge, label %while.end.loopexit
+
+if.end.while.body_crit_edge:
+  br label %while.body
+
+while.end.loopexit:
+; CHECK: DIVERGENT:       %storemerge.lcssa = phi i32 [ %storemerge, %if.end ]
+  %storemerge.lcssa = phi i32 [ %storemerge, %if.end ]
+  br label %while.end
+
+while.end:
+; CHECK: DIVERGENT:       %4 = phi i32 [ %.pre, %if.then.while.end_crit_edge ], [ %storemerge.lcssa, %while.end.loopexit ]
+; CHECK: DIVERGENT:       %i.0.lcssa = phi i32 [ 0, %if.then.while.end_crit_edge ], [ %0, %while.end.loopexit ]
+; CHECK: DIVERGENT:       %sub = sub nsw i32 %4, %i.0.lcssa
+; CHECK: DIVERGENT:       store i32 %sub, i32 addrspace(4)* %arrayidx10, align 4
+  %4 = phi i32 [ %.pre, %if.then.while.end_crit_edge ], [ %storemerge.lcssa, %while.end.loopexit ]
+  %i.0.lcssa = phi i32 [ 0, %if.then.while.end_crit_edge ], [ %0, %while.end.loopexit ]
+  %sub = sub nsw i32 %4, %i.0.lcssa
+  store i32 %sub, i32 addrspace(4)* %arrayidx10, align 4
+  br label %new_exit
+
+new_exit:
+  ret void
+}
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone }
+attributes #3 = { nounwind readnone }
+attributes #4 = { nounwind readnone }
diff --git a/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
new file mode 100644
index 0000000000000..f601d48f4b9ef
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/loop-rotate-disablebasicaa.ll
@@ -0,0 +1,26 @@
+; RUN: opt -disable-basicaa -print-memoryssa -disable-output %s 2>&1 | FileCheck %s
+
+; Note: if @foo is modelled as a MemoryDef, this test will assert with -loop-rotate, due to MemorySSA not
+; being preserved when moving instructions that may not read from or write to memory.
+
+; CHECK-LABEL: @main
+; CHECK-NOT: MemoryDef
+define void @main() {
+entry:
+  br label %for.cond120
+
+for.cond120:                                      ; preds = %for.body127, %entry
+  call void @foo()
+  br i1 undef, label %for.body127, label %for.cond.cleanup126
+
+for.cond.cleanup126:                              ; preds = %for.cond120
+  unreachable
+
+for.body127:                                      ; preds = %for.cond120
+  %0 = load i16**, i16*** undef, align 1
+  br label %for.cond120
+}
+
+declare void @foo() readnone
+
+
diff --git a/llvm/test/Analysis/MemorySSA/pr40754.ll b/llvm/test/Analysis/MemorySSA/pr40754.ll
index 8db320d80239f..3262a0cdd46e8 100644
--- a/llvm/test/Analysis/MemorySSA/pr40754.ll
+++ b/llvm/test/Analysis/MemorySSA/pr40754.ll
@@ -11,44 +11,45 @@ target triple = "systemz-unknown"
 ; Function Attrs: norecurse noreturn nounwind
 define dso_local void @func_65() local_unnamed_addr {
 ; CHECK-LABEL: @func_65()
-  br label %1
+label0:
+  br label %label1
 
-; <label>:1:                                      ; preds = %.thread, %0
-  br label %2
+label1:                                      ; preds = %.thread, %label0
+  br label %label2
 
-; <label>:2:                                      ; preds = %.critedge, %1
-  br label %3
+label2:                                      ; preds = %.critedge, %label1
+  br label %label3
 
-; <label>:3:                                      ; preds = %5, %2
-  %storemerge = phi i32 [ 0, %2 ], [ %6, %5 ]
+label3:                                      ; preds = %label5, %label2
+  %storemerge = phi i32 [ 0, %label2 ], [ %tmp6, %label5 ]
   store i32 %storemerge, i32* @g_185, align 4
-  %4 = icmp ult i32 %storemerge, 2
-  br i1 %4, label %5, label %.thread.loopexit
-
-; <label>:5:                                      ; preds = %3
-  %6 = add i32 %storemerge, 1
-  %7 = zext i32 %6 to i64
-  %8 = getelementptr [8 x [4 x [6 x i32]]], [8 x [4 x [6 x i32]]]* @g_120, i64 0, i64 undef, i64 %7, i64 undef
-  %9 = load i32, i32* %8, align 4
-  %10 = icmp eq i32 %9, 0
-  br i1 %10, label %3, label %11
-
-; <label>:11:                                     ; preds = %5
-  %storemerge.lcssa4 = phi i32 [ %storemerge, %5 ]
-  %12 = icmp eq i32 %storemerge.lcssa4, 0
-  br i1 %12, label %.critedge, label %.thread.loopexit3
-
-.critedge:                                        ; preds = %11
+  %tmp4 = icmp ult i32 %storemerge, 2
+  br i1 %tmp4, label %label5, label %.thread.loopexit
+
+label5:                                      ; preds = %label3
+  %tmp6 = add i32 %storemerge, 1
+  %tmp7 = zext i32 %tmp6 to i64
+  %tmp8 = getelementptr [8 x [4 x [6 x i32]]], [8 x [4 x [6 x i32]]]* @g_120, i64 0, i64 undef, i64 %tmp7, i64 undef
+  %tmp9 = load i32, i32* %tmp8, align 4
+  %tmp10 = icmp eq i32 %tmp9, 0
+  br i1 %tmp10, label %label3, label %label11
+
+label11:                                     ; preds = %label5
+  %storemerge.lcssa4 = phi i32 [ %storemerge, %label5 ]
+  %tmp12 = icmp eq i32 %storemerge.lcssa4, 0
+  br i1 %tmp12, label %.critedge, label %.thread.loopexit3
+
+.critedge:                                        ; preds = %label11
   store i16 0, i16* @g_329, align 2
-  br label %2
+  br label %label2
 
-.thread.loopexit:                                 ; preds = %3
+.thread.loopexit:                                 ; preds = %label3
   br label %.thread
 
-.thread.loopexit3:                                ; preds = %11
+.thread.loopexit3:                                ; preds = %label11
   br label %.thread
 
 .thread:                                          ; preds = %.thread.loopexit3, %.thread.loopexit
-  br label %1
+  br label %label1
 }
 
diff --git a/llvm/test/Analysis/MemorySSA/pr41640.ll b/llvm/test/Analysis/MemorySSA/pr41640.ll
index 5a76544242516..5d2d523202b15 100644
--- a/llvm/test/Analysis/MemorySSA/pr41640.ll
+++ b/llvm/test/Analysis/MemorySSA/pr41640.ll
@@ -1,5 +1,4 @@
-; RUN: opt -S -licm -enable-mssa-loop-dependency %s | FileCheck %s
-; REQUIRES: asserts
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
 target datalayout = "E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64"
 target triple = "s390x-ibm-linux"
 
@@ -8,6 +7,8 @@ target triple = "s390x-ibm-linux"
 @g_1087 = external dso_local global i32**, align 8
 
 ; CHECK-LABEL: @f1()
+; CHECK: 5 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
 define dso_local fastcc void @f1() unnamed_addr #0 {
 label0:
   br i1 undef, label %thread-pre-split.i.preheader, label %label5
diff --git a/llvm/test/Analysis/MemorySSA/PR42940.ll b/llvm/test/Analysis/MemorySSA/pr42940.ll
similarity index 100%
rename from llvm/test/Analysis/MemorySSA/PR42940.ll
rename to llvm/test/Analysis/MemorySSA/pr42940.ll
diff --git a/llvm/test/Analysis/MemorySSA/PR43044.ll b/llvm/test/Analysis/MemorySSA/pr43044.ll
similarity index 100%
rename from llvm/test/Analysis/MemorySSA/PR43044.ll
rename to llvm/test/Analysis/MemorySSA/pr43044.ll
diff --git a/llvm/test/Analysis/MemorySSA/pr43317.ll b/llvm/test/Analysis/MemorySSA/pr43317.ll
new file mode 100644
index 0000000000000..b1fbfdbaad895
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43317.ll
@@ -0,0 +1,35 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+@v_274 = external dso_local global i64, align 1
+@v_295 = external dso_local global i16, align 1
+@v_335 = external dso_local global i32, align 1
+
+; CHECK-LABEL: @main()
+; CHECK-NOT: 5 = MemoryPhi(
+; CHECK-NOT: 6 = MemoryPhi(
+; CHECK: 4 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
+define dso_local void @main() {
+entry:
+  store i32 undef, i32* @v_335, align 1
+  br i1 undef, label %gate, label %exit
+
+nopredentry1:                                     ; No predecessors!
+  br label %preinfiniteloop
+
+nopredentry2:                                     ; No predecessors!
+  br label %gate
+
+gate:                                             ; preds = %nopredentry2, %entry
+  br i1 undef, label %preinfiniteloop, label %exit
+
+preinfiniteloop:                                  ; preds = %gate, %nopredentry1
+  br label %infiniteloop
+
+infiniteloop:                                     ; preds = %infiniteloop, %preinfiniteloop
+  store i16 undef, i16* @v_295, align 1
+  br label %infiniteloop
+
+exit:                                             ; preds = %gate, %entry
+  store i64 undef, i64* @v_274, align 1
+  ret void
+}
diff --git a/llvm/test/Analysis/MemorySSA/pr43320.ll b/llvm/test/Analysis/MemorySSA/pr43320.ll
new file mode 100644
index 0000000000000..6aca3f9eeb148
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43320.ll
@@ -0,0 +1,33 @@
+; RUN: opt -licm -enable-mssa-loop-dependency -verify-memoryssa -S < %s | FileCheck %s
+; REQUIRES: asserts
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64-unknown-none-eabi"
+
+; CHECK-LABEL: @e()
+define void @e() {
+entry:
+  br label %g
+
+g:                                                ; preds = %cleanup, %entry
+  %0 = load i32, i32* null, align 4
+  %and = and i32 %0, undef
+  store i32 %and, i32* null, align 4
+  br i1 undef, label %if.end8, label %if.then
+
+if.then:                                          ; preds = %g
+  br i1 undef, label %k, label %cleanup
+
+k:                                                ; preds = %if.end8, %if.then
+  br i1 undef, label %if.end8, label %cleanup
+
+if.end8:                                          ; preds = %k, %g
+  br i1 undef, label %for.cond.preheader, label %k
+
+for.cond.preheader:                               ; preds = %if.end8
+  unreachable
+
+cleanup:                                          ; preds = %k, %if.then
+  br label %g
+}
+
diff --git a/llvm/test/Analysis/MemorySSA/pr43427.ll b/llvm/test/Analysis/MemorySSA/pr43427.ll
new file mode 100644
index 0000000000000..f708878221717
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43427.ll
@@ -0,0 +1,42 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: @f()
+; CHECK: 8 = MemoryPhi(
+; CHECK: 7 = MemoryPhi(
+; CHECK: 9 = MemoryPhi(
+define void @f() {
+entry:
+  %e = alloca i16, align 1
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %cleanup, %entry
+  store i16 undef, i16* %e, align 1
+  call void @g()
+  br i1 undef, label %for.end, label %if.else
+
+for.end:                                          ; preds = %lbl1
+  br i1 undef, label %lbl3, label %lbl2
+
+lbl2:                                             ; preds = %lbl3, %for.end
+  br label %lbl3
+
+lbl3:                                             ; preds = %lbl2, %for.end
+  br i1 undef, label %lbl2, label %cleanup
+
+cleanup:                                          ; preds = %lbl3
+  %cleanup.dest = load i32, i32* undef, align 1
+  %switch = icmp ult i32 %cleanup.dest, 1
+  br i1 %switch, label %cleanup.cont, label %lbl1
+
+cleanup.cont:                                     ; preds = %cleanup
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* null)
+  ret void
+
+if.else:                                          ; preds = %lbl1
+  br label %lbl1
+}
+
+declare void @g()
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
diff --git a/llvm/test/Analysis/MemorySSA/pr43438.ll b/llvm/test/Analysis/MemorySSA/pr43438.ll
new file mode 100644
index 0000000000000..69bbcedde8623
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43438.ll
@@ -0,0 +1,100 @@
+; RUN: opt -disable-output -licm -print-memoryssa -enable-mssa-loop-dependency=true < %s 2>&1 | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK-LABEL: @main()
+; CHECK: 5 = MemoryPhi(
+; CHECK-NOT: 7 = MemoryPhi(
+@v_67 = external dso_local global i32, align 1
+@v_76 = external dso_local global i16, align 1
+@v_86 = external dso_local global i16 *, align 1
+
+define dso_local void @main() {
+entry:
+  %v_59 = alloca i16, align 2
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.body, %entry
+  br i1 undef, label %for.body, label %for.end
+
+for.body:                                         ; preds = %for.cond
+  store i16 undef, i16* %v_59, align 2
+  br label %for.cond
+
+for.end:                                          ; preds = %for.cond
+  br i1 undef, label %if.else568, label %cond.end82
+
+cond.false69:                                     ; No predecessors!
+  br label %cond.end82
+
+cond.end82:                                       ; preds = %cond.false69, %cond.true55
+  br i1 undef, label %if.else568, label %land.lhs.true87
+
+land.lhs.true87:                                  ; preds = %cond.end82
+  br i1 undef, label %if.then88, label %if.else568
+
+if.then88:                                        ; preds = %land.lhs.true87
+  store i16 * @v_76, i16 ** @v_86, align 1
+  br label %if.end569
+
+if.else568:                                       ; preds = %land.lhs.true87, %cond.end82, %for.end
+  store volatile i32 undef, i32 * @v_67, align 1
+  br label %if.end569
+
+if.end569:                                        ; preds = %if.else568, %if.then88
+  ret void
+}
+
+
+; CHECK-LABEL: @f()
+; CHECK: 8 = MemoryPhi(
+; CHECK: 7 = MemoryPhi(
+; CHECK: 11 = MemoryPhi(
+; CHECK: 10 = MemoryPhi(
+; CHECK: 9 = MemoryPhi(
+define void @f() {
+entry:
+  %e = alloca i16, align 1
+  br label %lbl1
+
+lbl1:                                             ; preds = %if.else, %for.end5, %entry
+  store i16 undef, i16* %e, align 1
+  %0 = load i16, i16* %e, align 1
+  %call = call i16 @g(i16 %0)
+  br i1 undef, label %for.end, label %if.else
+
+for.end:                                          ; preds = %if.then
+  br i1 true, label %for.cond2, label %lbl2
+
+lbl2:                                             ; preds = %for.body4, %if.end
+  br label %for.cond2
+
+for.cond2:                                        ; preds = %lbl3
+  br i1 undef, label %for.body4, label %for.end5
+
+for.body4:                                        ; preds = %for.cond2
+  br label %lbl2
+
+for.end5:                                         ; preds = %for.cond2
+  switch i32 undef, label %unreachable [
+    i32 0, label %if.end12
+    i32 2, label %lbl1
+  ]
+
+if.else:                                          ; preds = %lbl1
+  switch i32 undef, label %unreachable [
+    i32 0, label %if.end12
+    i32 2, label %lbl1
+  ]
+
+if.end12:                                         ; preds = %cleanup.cont11s, %cleanup.cont
+  call void @llvm.lifetime.end.p0i8(i64 1, i8* undef)
+  ret void
+
+unreachable:                                      ; preds = %if.else, %for.end5
+  unreachable
+}
+
+declare i16 @g(i16)
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
diff --git a/llvm/test/Analysis/MemorySSA/pr43493.ll b/llvm/test/Analysis/MemorySSA/pr43493.ll
new file mode 100644
index 0000000000000..69506e8c0434b
--- /dev/null
+++ b/llvm/test/Analysis/MemorySSA/pr43493.ll
@@ -0,0 +1,27 @@
+; RUN: opt -enable-mssa-loop-dependency=true -loop-rotate -verify-memoryssa -S %s | FileCheck %s
+; REQUIRES: asserts
+
+; CHECK-LABEL: @func_35()
+define void @func_35() {
+entry:
+  br i1 undef, label %for.cond1704.preheader, label %return
+
+for.cond1704.preheader:                           ; preds = %entry
+  br label %for.cond1704
+
+for.cond1704:                                     ; preds = %for.cond1704.preheader, %for.body1707
+  br i1 false, label %for.body1707, label %return.loopexit
+
+for.body1707:                                     ; preds = %for.cond1704
+  store i32 1712, i32* undef, align 1
+  br label %for.cond1704
+
+for.body1102:                                     ; preds = %for.body1102
+  br i1 undef, label %for.body1102, label %return
+
+return.loopexit:                                  ; preds = %for.cond1704
+  br label %return
+
+return:                                           ; preds = %return.loopexit, %for.body1102, %entry
+  ret void
+}
diff --git a/llvm/test/Analysis/ScalarEvolution/limit-depth.ll b/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
index db68a4f84c912..6fdf8c5df9742 100644
--- a/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
+++ b/llvm/test/Analysis/ScalarEvolution/limit-depth.ll
@@ -46,7 +46,7 @@ define void @test_mul(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
 define void @test_sext(i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f) {
 ; CHECK-LABEL: @test_sext
 ; CHECK:        %se2 = sext i64 %iv2.inc to i128
-; CHECK-NEXT:   -->  {(1 + (sext i64 {(sext i32 (1 + %a)<nsw> to i64),+,1}<nsw><%loop> to i128))<nsw>,+,1}<nsw><%loop2>
+; CHECK-NEXT:   -->  {(1 + (sext i64 {(sext i32 (1 + %a) to i64),+,1}<nsw><%loop> to i128))<nsw>,+,1}<nsw><%loop2>
 entry:
   br label %loop
 
diff --git a/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll b/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll
new file mode 100644
index 0000000000000..4686a2024e33e
--- /dev/null
+++ b/llvm/test/Analysis/ScalarEvolution/multiple-max-iterations.ll
@@ -0,0 +1,2 @@
+; Ensure we can pass -scalar-evolution-max-iterations multiple times
+; RUN: opt -S -scalar-evolution -scalar-evolution-max-iterations=42 -scalar-evolution-max-iterations=42 < %s
diff --git a/llvm/test/Analysis/ScalarEvolution/nsw.ll b/llvm/test/Analysis/ScalarEvolution/nsw.ll
index 69427368625d4..ca24f9d4a04bb 100644
--- a/llvm/test/Analysis/ScalarEvolution/nsw.ll
+++ b/llvm/test/Analysis/ScalarEvolution/nsw.ll
@@ -163,7 +163,7 @@ bb5:                                              ; preds = %bb2
 declare void @f(i32)
 
 ; CHECK-LABEL: nswnowrap
-; CHECK: --> {(1 + %v)<nsw>,+,1}<nsw><%for.body>{{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: (2 + %v)
+; CHECK: --> {(1 + %v)<nsw>,+,1}<nsw><%for.body>{{ U: [^ ]+ S: [^ ]+}}{{ *}}Exits: (1 + ((1 + %v)<nsw> smax %v))
 define void @nswnowrap(i32 %v, i32* %buf) {
 entry:
   %add = add nsw i32 %v, 1
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
index 5e7d72d5e4f36..d0086ee2e6acc 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count12.ll
@@ -1,7 +1,7 @@
 ; RUN: opt < %s -analyze -scalar-evolution | FileCheck %s
 
 ; CHECK: Determining loop execution counts for: @test
-; CHECK: Loop %for.body: backedge-taken count is ((-2 + %len)<nsw> /u 2)
+; CHECK: Loop %for.body: backedge-taken count is ((-2 + %len) /u 2)
 ; CHECK: Loop %for.body: max backedge-taken count is 1073741823
 
 define zeroext i16 @test(i16* nocapture %p, i32 %len) nounwind readonly {
diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
index c0a1d12fa00e8..9a080b34743f9 100644
--- a/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
+++ b/llvm/test/Analysis/ScalarEvolution/trip-count9.ll
@@ -179,7 +179,7 @@ exit:
 }
 
 ; CHECK: Determining loop execution counts for: @nsw_startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x)<nsw> smax %n))
+; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax %n))
 ; CHECK: Loop %loop: max backedge-taken count is -1
 define void @nsw_startx(i4 %n, i4 %x) {
 entry:
@@ -195,7 +195,7 @@ exit:
 }
 
 ; CHECK: Determining loop execution counts for: @nsw_startx_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x)<nsw> smax %n)) /u 2)
+; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax %n)) /u 2)
 ; CHECK: Loop %loop: max backedge-taken count is 7
 define void @nsw_startx_step2(i4 %n, i4 %x) {
 entry:
@@ -381,7 +381,7 @@ exit:
 }
 
 ; CHECK: Determining loop execution counts for: @even_nsw_startx
-; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x)<nsw> smax (2 * %n)))
+; CHECK: Loop %loop: backedge-taken count is (-1 + (-1 * %x) + ((1 + %x) smax (2 * %n)))
 ; CHECK: Loop %loop: max backedge-taken count is -2
 define void @even_nsw_startx(i4 %n, i4 %x) {
 entry:
@@ -398,7 +398,7 @@ exit:
 }
 
 ; CHECK: Determining loop execution counts for: @even_nsw_startx_step2
-; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x)<nsw> smax (2 * %n))) /u 2)
+; CHECK: Loop %loop: backedge-taken count is ((-1 + (-1 * %x) + ((2 + %x) smax (2 * %n))) /u 2)
 ; CHECK: Loop %loop: max backedge-taken count is 7
 define void @even_nsw_startx_step2(i4 %n, i4 %x) {
 entry:
diff --git a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
index 2cbd5a65e0c98..c1f6f18f629bc 100644
--- a/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
+++ b/llvm/test/Analysis/TypeBasedAliasAnalysis/memcpyopt.ll
@@ -8,7 +8,7 @@ target datalayout = "e-p:64:64:64"
 
 define void @foo(i8* nocapture %p, i8* nocapture %q, i8* nocapture %s) nounwind {
 ; CHECK: @foo
-; CHECK-NEXT: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(16) %p, i8* align 1 dereferenceable(16) %q, i64 16, i1 false), !tbaa !0
+; CHECK-NEXT: tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) %p, i8* nonnull align 1 dereferenceable(16) %q, i64 16, i1 false), !tbaa !0
 ; CHECK-NEXT: store i8 2, i8* %s, align 1, !tbaa [[TAGA:!.*]]
 ; CHECK-NEXT: ret void
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %p, i8* %q, i64 16, i1 false), !tbaa !2
diff --git a/llvm/test/Bindings/llvm-c/echo.ll b/llvm/test/Bindings/llvm-c/echo.ll
index bb6556e763e58..c29991a122137 100644
--- a/llvm/test/Bindings/llvm-c/echo.ll
+++ b/llvm/test/Bindings/llvm-c/echo.ll
@@ -138,6 +138,23 @@ done:
   ret i32 %p
 }
 
+define void @memops(i8* %ptr) {
+  %a = load i8, i8* %ptr
+  %b = load volatile i8, i8* %ptr
+  %c = load i8, i8* %ptr, align 8
+  %d = load atomic i8, i8* %ptr acquire, align 32
+  store i8 0, i8* %ptr
+  store volatile i8 0, i8* %ptr
+  store i8 0, i8* %ptr, align 8
+  store atomic i8 0, i8* %ptr release, align 32
+  %e = atomicrmw add i8* %ptr, i8 0 monotonic
+  %f = atomicrmw volatile xchg i8* %ptr, i8 0 acq_rel
+  %g = cmpxchg i8* %ptr, i8 1, i8 2 seq_cst acquire
+  %h = cmpxchg weak i8* %ptr, i8 1, i8 2 seq_cst acquire
+  %i = cmpxchg volatile i8* %ptr, i8 1, i8 2 monotonic monotonic
+  ret void
+}
+
 declare void @personalityFn()
 
 define void @exn() personality void ()* @personalityFn {
diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll
index b17aeda8ebbb2..9e7f6ac991754 100644
--- a/llvm/test/Bitcode/compatibility.ll
+++ b/llvm/test/Bitcode/compatibility.ll
@@ -861,6 +861,27 @@ define void @fastmathflags_vector_select(<2 x i1> %cond, <2 x double> %op1, <2 x
   ret void
 }
 
+define void @fastmathflags_phi(i1 %cond, float %f1, float %f2, double %d1, double %d2, half %h1, half %h2) {
+entry:
+  br i1 %cond, label %L1, label %L2
+L1:
+  br label %exit
+L2:
+  br label %exit
+exit:
+  %p.nnan = phi nnan float [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nnan = phi nnan float [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.ninf = phi ninf double [ %d1, %L1 ], [ %d2, %L2 ]
+  ; CHECK: %p.ninf = phi ninf double [ %d1, %L1 ], [ %d2, %L2 ]
+  %p.contract = phi contract half [ %h1, %L1 ], [ %h2, %L2 ]
+  ; CHECK: %p.contract = phi contract half [ %h1, %L1 ], [ %h2, %L2 ]
+  %p.nsz.reassoc = phi reassoc nsz float [ %f1, %L1 ], [ %f2, %L2 ]
+  ; CHECK: %p.nsz.reassoc = phi reassoc nsz float [ %f1, %L1 ], [ %f2, %L2 ]
+  %p.fast = phi fast half [ %h2, %L1 ], [ %h1, %L2 ]
+  ; CHECK: %p.fast = phi fast half [ %h2, %L1 ], [ %h1, %L2 ]
+  ret void
+}
+
 ; Check various fast math flags and floating-point types on calls.
 
 declare float @fmf1()
diff --git a/llvm/test/Bitcode/upgrade-datalayout.ll b/llvm/test/Bitcode/upgrade-datalayout.ll
new file mode 100644
index 0000000000000..8c00294f2ba5b
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout.ll
@@ -0,0 +1,9 @@
+; Test to make sure datalayout is automatically upgraded.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: target datalayout = "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
diff --git a/llvm/test/Bitcode/upgrade-datalayout2.ll b/llvm/test/Bitcode/upgrade-datalayout2.ll
new file mode 100644
index 0000000000000..21de5b8a67d1f
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout2.ll
@@ -0,0 +1,10 @@
+; Test to make sure datalayout is not automatically upgraded if it does not
+; match a possible x86 datalayout.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; CHECK: target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
diff --git a/llvm/test/Bitcode/upgrade-datalayout3.ll b/llvm/test/Bitcode/upgrade-datalayout3.ll
new file mode 100644
index 0000000000000..526ba6069dc6f
--- /dev/null
+++ b/llvm/test/Bitcode/upgrade-datalayout3.ll
@@ -0,0 +1,8 @@
+; Test to make sure datalayout is automatically upgraded.
+;
+; RUN: llvm-as %s -o - | llvm-dis - | FileCheck %s
+
+target datalayout = "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32"
+target triple = "i686-pc-windows-msvc"
+
+; CHECK: target datalayout = "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-S32"
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
index f5075216352fb..969cacb89d10e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-switch.ll
@@ -1412,3 +1412,45 @@ bb4:                                              ; preds = %bb1
 
 declare i64* @ham(i32)
 
+define internal void @bar() unnamed_addr #1 {
+  ; CHECK-LABEL: name: bar
+  ; CHECK: bb.1 (%ir-block.0):
+    unreachable
+}
+
+define i1 @i1_value_cmp_is_signed(i1) {
+  ; CHECK-LABEL: name: i1_value_cmp_is_signed
+  ; CHECK: bb.1.Entry:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $w0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[COPY]](s32)
+  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; CHECK:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[TRUNC]](s1), [[C1]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.2.BadValue:
+  ; CHECK:   successors:
+  ; CHECK:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK:   BL @bar, csr_aarch64_aapcs, implicit-def $lr, implicit $sp
+  ; CHECK:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; CHECK: bb.3.OkValue:
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s8) = G_ZEXT [[TRUNC]](s1)
+  ; CHECK:   [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ZEXT]](s8)
+  ; CHECK:   $w0 = COPY [[ANYEXT]](s32)
+  ; CHECK:   RET_ReallyLR implicit $w0
+Entry:
+  switch i1 %0, label %BadValue [
+    i1 false, label %OkValue
+    i1 true, label %OkValue
+  ]
+
+BadValue:
+  call fastcc void @bar()
+  unreachable
+
+OkValue:
+  ret i1 %0
+}
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
index dd5339a615ce3..4f9dcd1686afb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll
@@ -389,8 +389,7 @@ define void @store(i64* %addr, i64 addrspace(42)* %addr42, i64 %val1, i64 %val2)
 ; CHECK-LABEL: name: intrinsics
 ; CHECK: [[CUR:%[0-9]+]]:_(s32) = COPY $w0
 ; CHECK: [[BITS:%[0-9]+]]:_(s32) = COPY $w1
-; CHECK: [[CREG:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTRINSIC intrinsic(@llvm.returnaddress), [[CREG]]
+; CHECK: [[PTR:%[0-9]+]]:_(p0) = G_INTRINSIC intrinsic(@llvm.returnaddress), 0
 ; CHECK: [[PTR_VEC:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.ptr.vec
 ; CHECK: [[VEC:%[0-9]+]]:_(<8 x s8>) = G_LOAD [[PTR_VEC]]
 ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.aarch64.neon.st2), [[VEC]](<8 x s8>), [[VEC]](<8 x s8>), [[PTR]](p0)
@@ -1135,18 +1134,28 @@ define void @test_memcpy(i8* %dst, i8* %src, i64 %size) {
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) :: (store 1 into %ir.dst), (load 1 from %ir.src)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
 
+define void @test_memcpy_tail(i8* %dst, i8* %src, i64 %size) {
+; CHECK-LABEL: name: test_memcpy_tail
+; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
+; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
+; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 1 :: (store 1 into %ir.dst), (load 1 from %ir.src)
+  tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
+  ret void
+}
+
 declare void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)*, i8 addrspace(1)*, i64, i1)
 define void @test_memcpy_nonzero_as(i8 addrspace(1)* %dst, i8 addrspace(1) * %src, i64 %size) {
 ; CHECK-LABEL: name: test_memcpy_nonzero_as
 ; CHECK: [[DST:%[0-9]+]]:_(p1) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p1) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p1), [[SRC]](p1), [[SIZE]](s64) :: (store 1 into %ir.dst, addrspace 1), (load 1 from %ir.src, addrspace 1)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p1), [[SRC]](p1), [[SIZE]](s64), 0 :: (store 1 into %ir.dst, addrspace 1), (load 1 from %ir.src, addrspace 1)
   call void @llvm.memcpy.p1i8.p1i8.i64(i8 addrspace(1)* %dst, i8 addrspace(1)* %src, i64 %size, i1 0)
   ret void
 }
@@ -1157,7 +1166,7 @@ define void @test_memmove(i8* %dst, i8* %src, i64 %size) {
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) :: (store 1 into %ir.dst), (load 1 from %ir.src)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64), 0 :: (store 1 into %ir.dst), (load 1 from %ir.src)
   call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
@@ -1169,7 +1178,7 @@ define void @test_memset(i8* %dst, i8 %val, i64 %size) {
 ; CHECK: [[SRC_C:%[0-9]+]]:_(s32) = COPY $w1
 ; CHECK: [[SRC:%[0-9]+]]:_(s8) = G_TRUNC [[SRC_C]]
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64) :: (store 1 into %ir.dst)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64), 0 :: (store 1 into %ir.dst)
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %val, i64 %size, i1 0)
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
index 7d6ba8489b70e..72af098a89469 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-musttail.ll
@@ -1,10 +1,13 @@
-; RUN: not llc %s -mtriple aarch64-unknown-unknown -debug-only=aarch64-call-lowering -global-isel -o - 2>&1 | FileCheck %s
+; RUN: not llc %s -mtriple aarch64-apple-darwin -debug-only=aarch64-call-lowering -global-isel -global-isel-abort=2 -o - 2>&1 | FileCheck %s
 ; REQUIRES: asserts
 
-; CHECK: Cannot lower musttail calls yet.
-; CHECK-NEXT: LLVM ERROR: unable to translate instruction: call (in function: foo)
-declare void @must_callee(i8*)
-define void @foo(i32*) {
-  musttail call void @must_callee(i8* null)
+; Verify that we fall back to SelectionDAG, and error out when we can't tail call musttail functions
+; CHECK: ... Cannot tail call externally-defined function with weak linkage for this OS.
+; CHECK-NEXT: Failed to lower musttail call as tail call
+; CHECK-NEXT: warning: Instruction selection used fallback path for caller_weak
+; CHECK-NEXT: LLVM ERROR: failed to perform tail call elimination on a call site marked musttail
+declare extern_weak void @callee_weak()
+define void @caller_weak() {
+  musttail call void @callee_weak()
   ret void
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
index 5be29e04512fb..115fa4fb2a37a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-tail-call.ll
@@ -77,32 +77,77 @@ define i32 @test_nonvoid_ret() {
 
 declare void @varargs(i32, double, i64, ...)
 define void @test_varargs() {
-  ; On Darwin, everything is passed on the stack. Since the caller has no stack,
-  ; we don't tail call.
-  ; DARWIN-LABEL: name: test_varargs
-  ; DARWIN: bb.1 (%ir-block.0):
-  ; DARWIN:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
-  ; DARWIN:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
-  ; DARWIN:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
-  ; DARWIN:   ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
-  ; DARWIN:   $w0 = COPY [[C]](s32)
-  ; DARWIN:   $d0 = COPY [[C1]](s64)
-  ; DARWIN:   $x1 = COPY [[C2]](s64)
+  ; COMMON-LABEL: name: test_varargs
+  ; COMMON: bb.1 (%ir-block.0):
+  ; COMMON:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; COMMON:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+  ; COMMON:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; COMMON:   $w0 = COPY [[C]](s32)
+  ; COMMON:   $d0 = COPY [[C1]](s64)
+  ; COMMON:   $x1 = COPY [[C2]](s64)
+  ; COMMON:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12)
+  ret void
+}
+
+; Darwin should not tail call here, because the last parameter to @varargs is
+; not fixed. So, it's passed on the stack, which will make us not fit. On
+; Windows, it's passed in a register, so it's safe to tail call.
+define void @test_varargs_2() {
+  ; DARWIN-LABEL: name: test_varargs_2
+  ; DARWIN-NOT: TCRETURNdi @varargs
   ; DARWIN:   BL @varargs, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $d0, implicit $x1
-  ; DARWIN:   ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   ADJCALLSTACKUP 8, 0, implicit-def $sp, implicit $sp
   ; DARWIN:   RET_ReallyLR
 
-  ; Windows uses registers, so we don't need to worry about using the stack.
-  ; WINDOWS-LABEL: name: test_varargs
+  ; WINDOWS-LABEL: name: test_varargs_2
   ; WINDOWS: bb.1 (%ir-block.0):
   ; WINDOWS:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
   ; WINDOWS:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
   ; WINDOWS:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; WINDOWS:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 314
   ; WINDOWS:   $w0 = COPY [[C]](s32)
   ; WINDOWS:   $d0 = COPY [[C1]](s64)
   ; WINDOWS:   $x1 = COPY [[C2]](s64)
-  ; WINDOWS:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1
-  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12)
+  ; WINDOWS:   $x2 = COPY [[C3]](s64)
+  ; WINDOWS:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1, implicit $x2
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12, i64 314)
+  ret void
+}
+
+; Same deal here, even though we have enough room to fit. On Darwin, we'll pass
+; the last argument to @varargs on the stack. We don't allow tail calling
+; varargs arguments that are on the stack.
+define void @test_varargs_3([8 x <2 x double>], <4 x half> %arg) {
+  ; DARWIN-LABEL: name: test_varargs_3
+  ; DARWIN-NOT: TCRETURNdi @varargs
+  ; DARWIN:   BL @varargs, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $w0, implicit $d0, implicit $x1
+  ; DARWIN:   ADJCALLSTACKUP 8, 0, implicit-def $sp, implicit $sp
+  ; DARWIN:   RET_ReallyLR
+
+  ; WINDOWS-LABEL: name: test_varargs_3
+  ; WINDOWS: bb.1 (%ir-block.1):
+  ; WINDOWS:   liveins: $q0, $q1, $q2, $q3, $q4, $q5, $q6, $q7
+  ; WINDOWS:   [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+  ; WINDOWS:   [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+  ; WINDOWS:   [[COPY2:%[0-9]+]]:_(<2 x s64>) = COPY $q2
+  ; WINDOWS:   [[COPY3:%[0-9]+]]:_(<2 x s64>) = COPY $q3
+  ; WINDOWS:   [[COPY4:%[0-9]+]]:_(<2 x s64>) = COPY $q4
+  ; WINDOWS:   [[COPY5:%[0-9]+]]:_(<2 x s64>) = COPY $q5
+  ; WINDOWS:   [[COPY6:%[0-9]+]]:_(<2 x s64>) = COPY $q6
+  ; WINDOWS:   [[COPY7:%[0-9]+]]:_(<2 x s64>) = COPY $q7
+  ; WINDOWS:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; WINDOWS:   [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[FRAME_INDEX]](p0) :: (invariant load 8 from %fixed-stack.0, align 1)
+  ; WINDOWS:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 42
+  ; WINDOWS:   [[C1:%[0-9]+]]:_(s64) = G_FCONSTANT double 1.000000e+00
+  ; WINDOWS:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; WINDOWS:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 314
+  ; WINDOWS:   $w0 = COPY [[C]](s32)
+  ; WINDOWS:   $d0 = COPY [[C1]](s64)
+  ; WINDOWS:   $x1 = COPY [[C2]](s64)
+  ; WINDOWS:   $x2 = COPY [[C3]](s64)
+  ; WINDOWS:   TCRETURNdi @varargs, 0, csr_aarch64_aapcs, implicit $sp, implicit $w0, implicit $d0, implicit $x1, implicit $x2
+  tail call void(i32, double, i64, ...) @varargs(i32 42, double 1.0, i64 12, i64 314)
   ret void
 }
 
@@ -224,3 +269,18 @@ define hidden swiftcc i64 @swiftself_indirect_tail(i64* swiftself %arg) {
   %tmp2 = tail call swiftcc i64 %tmp1(i64* swiftself %arg)
   ret i64 %tmp2
 }
+
+; Verify that we can tail call musttail callees.
+declare void @must_callee(i8*)
+define void @foo(i32*) {
+  ; COMMON-LABEL: name: foo
+  ; COMMON: bb.1 (%ir-block.1):
+  ; COMMON:   liveins: $x0
+  ; COMMON:   [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+  ; COMMON:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; COMMON:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
+  ; COMMON:   $x0 = COPY [[INTTOPTR]](p0)
+  ; COMMON:   TCRETURNdi @must_callee, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0
+  musttail call void @must_callee(i8* null)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll
new file mode 100644
index 0000000000000..0a0f94aa0a348
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator-variadic-musttail.ll
@@ -0,0 +1,223 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -o - 2>&1 | FileCheck %s
+
+; There are two things we want to test here:
+;  (1) We can tail call musttail calls.
+;  (2) We spill and reload all of the arguments around a normal call.
+
+declare i32 @musttail_variadic_callee(i32, ...)
+define i32 @test_musttail_variadic(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    b _musttail_variadic_callee
+  %r = musttail call i32 (i32, ...) @musttail_variadic_callee(i32 %arg0, ...)
+  ret i32 %r
+}
+
+declare [2 x i64] @musttail_variadic_aggret_callee(i32 %arg0, ...)
+define [2 x i64] @test_musttail_variadic_aggret(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic_aggret:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    b _musttail_variadic_aggret_callee
+  %r = musttail call [2 x i64] (i32, ...) @musttail_variadic_aggret_callee(i32 %arg0, ...)
+  ret [2 x i64] %r
+}
+
+; Test musttailing with a normal call in the block. Test that we spill and
+; restore, as a normal call will clobber all argument registers.
+@asdf = internal constant [4 x i8] c"asdf"
+declare void @puts(i8*)
+define i32 @test_musttail_variadic_spill(i32 %arg0, ...) {
+; CHECK-LABEL: test_musttail_variadic_spill:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #224 ; =224
+; CHECK-NEXT:    stp x28, x27, [sp, #128] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #144] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #160] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #176] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #192] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #208] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 224
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    .cfi_offset w21, -40
+; CHECK-NEXT:    .cfi_offset w22, -48
+; CHECK-NEXT:    .cfi_offset w23, -56
+; CHECK-NEXT:    .cfi_offset w24, -64
+; CHECK-NEXT:    .cfi_offset w25, -72
+; CHECK-NEXT:    .cfi_offset w26, -80
+; CHECK-NEXT:    .cfi_offset w27, -88
+; CHECK-NEXT:    .cfi_offset w28, -96
+; CHECK-NEXT:    mov w19, w0
+; CHECK-NEXT:  Lloh0:
+; CHECK-NEXT:    adrp x0, _asdf@PAGE
+; CHECK-NEXT:  Lloh1:
+; CHECK-NEXT:    add x0, x0, _asdf@PAGEOFF
+; CHECK-NEXT:    mov x20, x1
+; CHECK-NEXT:    mov x21, x2
+; CHECK-NEXT:    mov x22, x3
+; CHECK-NEXT:    mov x23, x4
+; CHECK-NEXT:    mov x24, x5
+; CHECK-NEXT:    mov x25, x6
+; CHECK-NEXT:    mov x26, x7
+; CHECK-NEXT:    stp q1, q0, [sp, #96] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #64] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q5, q4, [sp, #32] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp] ; 32-byte Folded Spill
+; CHECK-NEXT:    mov x27, x8
+; CHECK-NEXT:    bl _puts
+; CHECK-NEXT:    ldp q1, q0, [sp, #96] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q3, q2, [sp, #64] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q5, q4, [sp, #32] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q7, q6, [sp] ; 32-byte Folded Reload
+; CHECK-NEXT:    mov w0, w19
+; CHECK-NEXT:    mov x1, x20
+; CHECK-NEXT:    mov x2, x21
+; CHECK-NEXT:    mov x3, x22
+; CHECK-NEXT:    mov x4, x23
+; CHECK-NEXT:    mov x5, x24
+; CHECK-NEXT:    mov x6, x25
+; CHECK-NEXT:    mov x7, x26
+; CHECK-NEXT:    mov x8, x27
+; CHECK-NEXT:    ldp x29, x30, [sp, #208] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #192] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #176] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #160] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #144] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #128] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #224 ; =224
+; CHECK-NEXT:    b _musttail_variadic_callee
+; CHECK-NEXT:    .loh AdrpAdd Lloh0, Lloh1
+  call void @puts(i8* getelementptr ([4 x i8], [4 x i8]* @asdf, i32 0, i32 0))
+  %r = musttail call i32 (i32, ...) @musttail_variadic_callee(i32 %arg0, ...)
+  ret i32 %r
+}
+
+; Test musttailing with a varargs call in the block. Test that we spill and
+; reload all arguments in the variadic argument pack.
+declare void @llvm.va_start(i8*) nounwind
+declare void(i8*, ...)* @get_f(i8* %this)
+define void @f_thunk(i8* %this, ...) {
+; CHECK-LABEL: f_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    sub sp, sp, #256 ; =256
+; CHECK-NEXT:    stp x28, x27, [sp, #160] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x26, x25, [sp, #176] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x24, x23, [sp, #192] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x22, x21, [sp, #208] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x20, x19, [sp, #224] ; 16-byte Folded Spill
+; CHECK-NEXT:    stp x29, x30, [sp, #240] ; 16-byte Folded Spill
+; CHECK-NEXT:    .cfi_def_cfa_offset 256
+; CHECK-NEXT:    .cfi_offset w30, -8
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    .cfi_offset w19, -24
+; CHECK-NEXT:    .cfi_offset w20, -32
+; CHECK-NEXT:    .cfi_offset w21, -40
+; CHECK-NEXT:    .cfi_offset w22, -48
+; CHECK-NEXT:    .cfi_offset w23, -56
+; CHECK-NEXT:    .cfi_offset w24, -64
+; CHECK-NEXT:    .cfi_offset w25, -72
+; CHECK-NEXT:    .cfi_offset w26, -80
+; CHECK-NEXT:    .cfi_offset w27, -88
+; CHECK-NEXT:    .cfi_offset w28, -96
+; CHECK-NEXT:    mov x27, x8
+; CHECK-NEXT:    add x8, sp, #128 ; =128
+; CHECK-NEXT:    add x9, sp, #256 ; =256
+; CHECK-NEXT:    mov x19, x0
+; CHECK-NEXT:    mov x20, x1
+; CHECK-NEXT:    mov x21, x2
+; CHECK-NEXT:    mov x22, x3
+; CHECK-NEXT:    mov x23, x4
+; CHECK-NEXT:    mov x24, x5
+; CHECK-NEXT:    mov x25, x6
+; CHECK-NEXT:    mov x26, x7
+; CHECK-NEXT:    stp q1, q0, [sp, #96] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q3, q2, [sp, #64] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q5, q4, [sp, #32] ; 32-byte Folded Spill
+; CHECK-NEXT:    stp q7, q6, [sp] ; 32-byte Folded Spill
+; CHECK-NEXT:    str x9, [x8]
+; CHECK-NEXT:    bl _get_f
+; CHECK-NEXT:    mov x9, x0
+; CHECK-NEXT:    ldp q1, q0, [sp, #96] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q3, q2, [sp, #64] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q5, q4, [sp, #32] ; 32-byte Folded Reload
+; CHECK-NEXT:    ldp q7, q6, [sp] ; 32-byte Folded Reload
+; CHECK-NEXT:    mov x0, x19
+; CHECK-NEXT:    mov x1, x20
+; CHECK-NEXT:    mov x2, x21
+; CHECK-NEXT:    mov x3, x22
+; CHECK-NEXT:    mov x4, x23
+; CHECK-NEXT:    mov x5, x24
+; CHECK-NEXT:    mov x6, x25
+; CHECK-NEXT:    mov x7, x26
+; CHECK-NEXT:    mov x8, x27
+; CHECK-NEXT:    ldp x29, x30, [sp, #240] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x20, x19, [sp, #224] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x22, x21, [sp, #208] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x24, x23, [sp, #192] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x26, x25, [sp, #176] ; 16-byte Folded Reload
+; CHECK-NEXT:    ldp x28, x27, [sp, #160] ; 16-byte Folded Reload
+; CHECK-NEXT:    add sp, sp, #256 ; =256
+; CHECK-NEXT:    br x9
+  %ap = alloca [4 x i8*], align 16
+  %ap_i8 = bitcast [4 x i8*]* %ap to i8*
+  call void @llvm.va_start(i8* %ap_i8)
+  %fptr = call void(i8*, ...)*(i8*) @get_f(i8* %this)
+  musttail call void (i8*, ...) %fptr(i8* %this, ...)
+  ret void
+}
+
+; We don't need any spills and reloads here, but we should still emit the
+; copies in call lowering.
+define void @g_thunk(i8* %fptr_i8, ...) {
+; CHECK-LABEL: g_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    br x0
+  %fptr = bitcast i8* %fptr_i8 to void (i8*, ...)*
+  musttail call void (i8*, ...) %fptr(i8* %fptr_i8, ...)
+  ret void
+}
+
+; Test that this works with multiple exits and basic blocks.
+%struct.Foo = type { i1, i8*, i8* }
+@g = external global i32
+define void @h_thunk(%struct.Foo* %this, ...) {
+; CHECK-LABEL: h_thunk:
+; CHECK:       ; %bb.0:
+; CHECK-NEXT:    ldrb w9, [x0]
+; CHECK-NEXT:    tbz w9, #0, LBB5_2
+; CHECK-NEXT:  ; %bb.1: ; %then
+; CHECK-NEXT:    ldr x9, [x0, #8]
+; CHECK-NEXT:    br x9
+; CHECK-NEXT:  LBB5_2: ; %else
+; CHECK-NEXT:  Lloh2:
+; CHECK-NEXT:    adrp x10, _g@GOTPAGE
+; CHECK-NEXT:    ldr x9, [x0, #16]
+; CHECK-NEXT:  Lloh3:
+; CHECK-NEXT:    ldr x10, [x10, _g@GOTPAGEOFF]
+; CHECK-NEXT:    mov w11, #42
+; CHECK-NEXT:  Lloh4:
+; CHECK-NEXT:    str w11, [x10]
+; CHECK-NEXT:    br x9
+; CHECK-NEXT:    .loh AdrpLdrGotStr Lloh2, Lloh3, Lloh4
+  %cond_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 0
+  %cond = load i1, i1* %cond_p
+  br i1 %cond, label %then, label %else
+
+then:
+  %a_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 1
+  %a_i8 = load i8*, i8** %a_p
+  %a = bitcast i8* %a_i8 to void (%struct.Foo*, ...)*
+  musttail call void (%struct.Foo*, ...) %a(%struct.Foo* %this, ...)
+  ret void
+
+else:
+  %b_p = getelementptr %struct.Foo, %struct.Foo* %this, i32 0, i32 2
+  %b_i8 = load i8*, i8** %b_p
+  %b = bitcast i8* %b_i8 to void (%struct.Foo*, ...)*
+  store i32 42, i32* @g
+  musttail call void (%struct.Foo*, ...) %b(%struct.Foo* %this, ...)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
index b5e66d69ca07b..d6d39281f9bea 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/call-translator.ll
@@ -100,7 +100,6 @@ define {double, i64, i32} @test_struct_return({double, i64, i32}* %addr) {
 }
 
 ; CHECK-LABEL: name: test_arr_call
-; CHECK: hasCalls: true
 ; CHECK: %0:_(p0) = COPY $x0
 ; CHECK: [[LD1:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.addr)
 ; CHECK: [[CST1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
index e333f742e04df..03794caf808bb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/gisel-fail-intermediate-legalizer.ll
@@ -1,8 +1,8 @@
 ;RUN: llc -mtriple=aarch64-unknown-unknown -o - -global-isel -global-isel-abort=2 %s 2>&1 | FileCheck %s
 ; CHECK: fallback
 ; CHECK-LABEL: foo
-define i16 @foo(half* %p) {
-  %tmp0 = load half, half* %p
-  %tmp1 = fptoui half %tmp0 to i16
+define i16 @foo(fp128* %p) {
+  %tmp0 = load fp128, fp128* %p
+  %tmp1 = fptoui fp128 %tmp0 to i16
   ret i16 %tmp1
 }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
index 064c74a434f06..ddb9d3507d902 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memcpy.mir
@@ -22,6 +22,22 @@
     ret void
   }
 
+  define void @test_memcpy2_const_optsize(i32* nocapture %dst, i32* nocapture readonly %src) local_unnamed_addr #2 {
+  entry:
+    %0 = bitcast i32* %dst to i8*
+    %1 = bitcast i32* %src to i8*
+    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 72, i1 false)
+    ret void
+  }
+
+  define void @test_memcpy2_const_minsize(i32* nocapture %dst, i32* nocapture readonly %src) local_unnamed_addr #3 {
+  entry:
+    %0 = bitcast i32* %dst to i8*
+    %1 = bitcast i32* %src to i8*
+    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 72, i1 false)
+    ret void
+  }
+
   define void @test_memcpy3_const_arrays_unaligned(i32* nocapture %dst, i32* nocapture readonly %src) local_unnamed_addr #0 {
   entry:
     %0 = bitcast i32* %dst to i8*
@@ -32,6 +48,8 @@
 
   attributes #0 = { nounwind ssp uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="cyclone" "target-features"="+aes,+crypto,+fp-armv8,+neon,+sha2,+zcm,+zcz" "unsafe-fp-math"="false" "use-soft-float"="false" }
   attributes #1 = { argmemonly nounwind }
+  attributes #2 = { optsize }
+  attributes #3 = { minsize }
 
 ...
 ---
@@ -52,12 +70,12 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
     ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     ; CHECK: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = COPY $x2
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -104,7 +122,81 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 72
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    RET_ReallyLR
+
+...
+---
+name:            test_memcpy2_const_optsize
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+machineFunctionInfo: {}
+body:             |
+  bb.1.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy2_const_optsize
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[LOAD:%[0-9]+]]:_(s128) = G_LOAD [[COPY1]](p0) :: (load 16 from %ir.1, align 4)
+    ; CHECK: G_STORE [[LOAD]](s128), [[COPY]](p0) :: (store 16 into %ir.0, align 4)
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C]](s64)
+    ; CHECK: [[LOAD1:%[0-9]+]]:_(s128) = G_LOAD [[GEP]](p0) :: (load 16 from %ir.1 + 16, align 4)
+    ; CHECK: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s64)
+    ; CHECK: G_STORE [[LOAD1]](s128), [[GEP1]](p0) :: (store 16 into %ir.0 + 16, align 4)
+    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 32
+    ; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C1]](s64)
+    ; CHECK: [[LOAD2:%[0-9]+]]:_(s128) = G_LOAD [[GEP2]](p0) :: (load 16 from %ir.1 + 32, align 4)
+    ; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C1]](s64)
+    ; CHECK: G_STORE [[LOAD2]](s128), [[GEP3]](p0) :: (store 16 into %ir.0 + 32, align 4)
+    ; CHECK: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 48
+    ; CHECK: [[GEP4:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C2]](s64)
+    ; CHECK: [[LOAD3:%[0-9]+]]:_(s128) = G_LOAD [[GEP4]](p0) :: (load 16 from %ir.1 + 48, align 4)
+    ; CHECK: [[GEP5:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C2]](s64)
+    ; CHECK: G_STORE [[LOAD3]](s128), [[GEP5]](p0) :: (store 16 into %ir.0 + 48, align 4)
+    ; CHECK: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 64
+    ; CHECK: [[GEP6:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C3]](s64)
+    ; CHECK: [[LOAD4:%[0-9]+]]:_(s64) = G_LOAD [[GEP6]](p0) :: (load 8 from %ir.1 + 64, align 4)
+    ; CHECK: [[GEP7:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C3]](s64)
+    ; CHECK: G_STORE [[LOAD4]](s64), [[GEP7]](p0) :: (store 8 into %ir.0 + 64, align 4)
+    ; CHECK: RET_ReallyLR
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 72
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    RET_ReallyLR
+
+...
+---
+name:            test_memcpy2_const_minsize
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+machineFunctionInfo: {}
+body:             |
+  bb.1.entry:
+    liveins: $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy2_const_minsize
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 72
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    ; CHECK: RET_ReallyLR
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s64) = G_CONSTANT i64 72
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -171,7 +263,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 143
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
index 9dc50c979fa94..aebca6778c94e 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memmove.mir
@@ -55,12 +55,12 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
     ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $x2
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[COPY2]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     ; CHECK: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = COPY $x2
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -94,7 +94,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 48
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -111,12 +111,12 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 96
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     ; CHECK: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 96
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -156,7 +156,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 52
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
index 209dae31e79ba..5029551bf1a65 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-memset.mir
@@ -54,14 +54,14 @@ body:             |
     ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
     ; CHECK: [[TRUNC:%[0-9]+]]:_(s8) = G_TRUNC [[COPY1]](s32)
     ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[COPY]](p0), [[TRUNC]](s8), [[ZEXT]](s64) :: (store 1 into %ir.dst)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[COPY]](p0), [[TRUNC]](s8), [[ZEXT]](s64), 1 :: (store 1 into %ir.dst)
     ; CHECK: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(s32) = COPY $w1
     %2:_(s32) = COPY $w2
     %3:_(s8) = G_TRUNC %1(s32)
     %4:_(s64) = G_ZEXT %2(s32)
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64) :: (store 1 into %ir.dst)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), 1 :: (store 1 into %ir.dst)
     RET_ReallyLR
 
 ...
@@ -90,7 +90,7 @@ body:             |
     %1:_(s32) = COPY $w1
     %3:_(s64) = G_CONSTANT i64 16
     %2:_(s8) = G_TRUNC %1(s32)
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %2(s8), %3(s64) :: (store 1 into %ir.dst)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %2(s8), %3(s64), 1 :: (store 1 into %ir.dst)
     RET_ReallyLR
 
 ...
@@ -114,7 +114,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(s8) = G_CONSTANT i8 64
     %2:_(s64) = G_CONSTANT i64 16
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64) :: (store 1 into %ir.dst)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64), 1 :: (store 1 into %ir.dst)
     RET_ReallyLR
 
 ...
@@ -142,7 +142,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(s8) = G_CONSTANT i8 64
     %2:_(s64) = G_CONSTANT i64 18
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64) :: (store 1 into %ir.dst)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %1(s8), %2(s64), 1 :: (store 1 into %ir.dst)
     RET_ReallyLR
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memcpy.mir b/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memcpy.mir
index 873ac6d672f18..00ce27415d48a 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memcpy.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/inline-small-memcpy.mir
@@ -53,7 +53,7 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 32
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
@@ -75,12 +75,12 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
     ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
     ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 36
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY]](p0), [[COPY1]](p0), [[C]](s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     ; CHECK: RET_ReallyLR
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s64) = G_CONSTANT i64 36
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %2(s64), 1 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
     RET_ReallyLR
 
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bswap.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bswap.mir
new file mode 100644
index 0000000000000..5e5cbf63bb073
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-bswap.mir
@@ -0,0 +1,44 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - -verify-machineinstrs | FileCheck %s
+--- |
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64"
+
+  declare i16 @llvm.bswap.i16(i16) #0
+
+  define i16 @bswap_s16(i16 %a) { ret i16 0 }
+
+  attributes #0 = { nounwind readnone speculatable willreturn }
+
+...
+---
+name:            bswap_s16
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$w0' }
+frameInfo:
+  maxAlignment:    1
+machineFunctionInfo: {}
+body:             |
+  bb.1:
+    liveins: $w0
+
+    ; CHECK-LABEL: name: bswap_s16
+    ; CHECK: liveins: $w0
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[BSWAP:%[0-9]+]]:_(s32) = G_BSWAP [[COPY1]]
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+    ; CHECK: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BSWAP]], [[C]](s64)
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY [[LSHR]](s32)
+    ; CHECK: $w0 = COPY [[COPY2]](s32)
+    ; CHECK: RET_ReallyLR implicit $w0
+    %1:_(s32) = COPY $w0
+    %0:_(s16) = G_TRUNC %1(s32)
+    %2:_(s16) = G_BSWAP %0
+    %3:_(s32) = G_ANYEXT %2(s16)
+    $w0 = COPY %3(s32)
+    RET_ReallyLR implicit $w0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir
index 46836a3f2914f..5175f3388dacb 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir
@@ -23,9 +23,33 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s32) = COPY $w2
-    %4:_(s1) = G_CONSTANT i1 false
     %3:_(s64) = G_ZEXT %2(s32)
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), %4(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), 0
+    RET_ReallyLR
+
+...
+---
+name:            test_memcpy_tail
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w2, $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy_tail
+    ; CHECK: liveins: $w2, $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: $x1 = COPY [[COPY1]](p0)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: TCRETURNdi &memcpy, 0, csr_aarch64_aapcs, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s32) = COPY $w2
+    %3:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), 1
     RET_ReallyLR
 
 ...
@@ -52,9 +76,8 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s32) = COPY $w2
-    %4:_(s1) = G_CONSTANT i1 false
     %3:_(s64) = G_ZEXT %2(s32)
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %3(s64), %4(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %3(s64), 0
     RET_ReallyLR
 
 ...
@@ -82,10 +105,65 @@ body:             |
     %0:_(p0) = COPY $x0
     %1:_(s32) = COPY $w1
     %2:_(s32) = COPY $w2
-    %5:_(s1) = G_CONSTANT i1 false
     %3:_(s8) = G_TRUNC %1(s32)
     %4:_(s64) = G_ZEXT %2(s32)
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), %5(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), 0
     RET_ReallyLR
 
 ...
+---
+name:            no_tail_call
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w2, $x0, $x1
+
+    ; CHECK-LABEL: name: no_tail_call
+    ; CHECK: liveins: $w2, $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: $x1 = COPY [[COPY1]](p0)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[ZEXT]](s64)
+    ; CHECK: RET_ReallyLR implicit $x0
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s32) = COPY $w2
+    %3:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), 1
+    $x0 = COPY %3
+    RET_ReallyLR implicit $x0
+
+...
+---
+name:            dont_tc_twice
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w2, $x0, $x1
+    ; CHECK-LABEL: name: dont_tc_twice
+    ; CHECK: liveins: $w2, $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: $x1 = COPY [[COPY1]](p0)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: TCRETURNdi &memset, 0, csr_aarch64_aapcs, implicit $sp
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s32) = COPY $w2
+    %4:_(s1) = G_CONSTANT i1 false
+    %3:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), 1
+    TCRETURNdi &memset, 0, csr_aarch64_aapcs, implicit $sp
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi-insertpt-decrement.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi-insertpt-decrement.mir
new file mode 100644
index 0000000000000..87db6070214ca
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi-insertpt-decrement.mir
@@ -0,0 +1,99 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=aarch64-unknown-unknown -verify-machineinstrs -run-pass=legalizer %s -o - | FileCheck %s
+--- |
+  target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+  define void @snork() {
+  bb:
+    br label %bb1
+
+  bb1:                                              ; preds = %bb3, %bb
+    %lsr.iv = phi i16* [ %scevgep, %bb3 ], [ undef, %bb ]
+    %tmp = phi i1 [ %tmp9, %bb3 ], [ undef, %bb ]
+    br i1 %tmp, label %bb10, label %bb3
+
+  bb3:                                              ; preds = %bb1
+    %tmp5 = getelementptr i16, i16* null, i64 2
+    %tmp6 = load i16, i16* %lsr.iv, align 2, !tbaa !0
+    %tmp7 = icmp eq i16 %tmp6, -1
+    %tmp8 = load i16, i16* %tmp5, align 2, !tbaa !0
+    %tmp9 = icmp eq i16 %tmp8, -1
+    %scevgep = getelementptr i16, i16* %lsr.iv, i64 2
+    br i1 %tmp7, label %bb10, label %bb1
+
+  bb10:                                             ; preds = %bb3, %bb1
+    ret void
+  }
+
+  !0 = !{!1, !1, i64 0}
+  !1 = !{!"short", !2, i64 0}
+  !2 = !{!"omnipotent char", !3, i64 0}
+  !3 = !{!"Simple C/C++ TBAA"}
+
+...
+---
+name:            snork
+alignment:       16
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    1
+body:             |
+  ; CHECK-LABEL: name: snork
+  ; CHECK: bb.0.bb:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+  ; CHECK:   [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[C]](s64)
+  ; CHECK:   [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
+  ; CHECK: bb.1.bb1:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:_(p0) = G_PHI %6(p0), %bb.2, [[DEF]](p0), %bb.0
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(s16) = G_PHI %20(s16), %bb.2, [[DEF1]](s16), %bb.0
+  ; CHECK:   [[TRUNC:%[0-9]+]]:_(s1) = G_TRUNC [[PHI1]](s16)
+  ; CHECK:   G_BRCOND [[TRUNC]](s1), %bb.3
+  ; CHECK: bb.2.bb3:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.1(0x40000000)
+  ; CHECK:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; CHECK:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[INTTOPTR]], [[C1]](s64)
+  ; CHECK:   [[LOAD:%[0-9]+]]:_(s16) = G_LOAD [[PHI]](p0) :: (load 2 from %ir.lsr.iv)
+  ; CHECK:   [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD]](s16)
+  ; CHECK:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ZEXT]](s32), [[C2]]
+  ; CHECK:   [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[ICMP]](s32)
+  ; CHECK:   [[LOAD1:%[0-9]+]]:_(s16) = G_LOAD [[GEP]](p0) :: (load 2 from %ir.tmp5)
+  ; CHECK:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LOAD1]](s16)
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
+  ; CHECK:   [[ICMP1:%[0-9]+]]:_(s32) = G_ICMP intpred(eq), [[ZEXT1]](s32), [[COPY]]
+  ; CHECK:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[PHI]], [[C1]](s64)
+  ; CHECK:   [[TRUNC2:%[0-9]+]]:_(s16) = G_TRUNC [[ICMP1]](s32)
+  ; CHECK:   G_BRCOND [[TRUNC1]](s1), %bb.3
+  ; CHECK:   G_BR %bb.1
+  ; CHECK: bb.3.bb10:
+  ; CHECK:   RET 0
+  bb.1.bb:
+    %3:_(s64) = G_CONSTANT i64 0
+    %2:_(p0) = G_INTTOPTR %3(s64)
+    %7:_(s16) = G_CONSTANT i16 -1
+    %12:_(p0) = G_IMPLICIT_DEF
+    %13:_(s1) = G_IMPLICIT_DEF
+
+  bb.2.bb1:
+    %0:_(p0) = G_PHI %11(p0), %bb.3, %12(p0), %bb.1
+    %1:_(s1) = G_PHI %10(s1), %bb.3, %13(s1), %bb.1
+    G_BRCOND %1(s1), %bb.4
+
+  bb.3.bb3:
+    %4:_(s64) = G_CONSTANT i64 4
+    %5:_(p0) = G_GEP %2, %4(s64)
+    %6:_(s16) = G_LOAD %0(p0) :: (load 2 from %ir.lsr.iv)
+    %8:_(s1) = G_ICMP intpred(eq), %6(s16), %7
+    %9:_(s16) = G_LOAD %5(p0) :: (load 2 from %ir.tmp5)
+    %10:_(s1) = G_ICMP intpred(eq), %9(s16), %7
+    %11:_(p0) = G_GEP %0, %4(s64)
+    G_BRCOND %8(s1), %bb.4
+    G_BR %bb.2
+
+  bb.4.bb10:
+    RET 0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
index a92b06446cc64..1c210a41d6a03 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-phi.mir
@@ -40,6 +40,11 @@
     ret i32 0
   }
 
+  define i32 @legalize_phi_check_insertpt(i64 %a) {
+  entry:
+    ret i32 0
+  }
+
 ...
 ---
 name:            legalize_phi
@@ -596,3 +601,47 @@ body:             |
     RET_ReallyLR implicit $w0
 
 ...
+---
+name:            legalize_phi_check_insertpt
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+body:             |
+  ; Check that the G_MERGE here gets inserted after all the PHIs.
+  ; CHECK-LABEL: name: legalize_phi_check_insertpt
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $x0, $x1
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+  ; CHECK:   [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+  ; CHECK:   [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; CHECK:   G_BR %bb.1
+  ; CHECK: bb.1:
+  ; CHECK:   [[PHI:%[0-9]+]]:_(s64) = G_PHI [[DEF]](s64), %bb.0
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(s64) = G_PHI [[DEF1]](s64), %bb.0
+  ; CHECK:   [[PHI2:%[0-9]+]]:_(s64) = G_PHI [[COPY]](s64), %bb.0
+  ; CHECK:   [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[PHI]](s64), [[PHI1]](s64)
+  ; CHECK:   G_STORE [[MV]](s128), [[COPY1]](p0) :: (store 16)
+  ; CHECK:   G_STORE [[PHI2]](s64), [[COPY1]](p0) :: (store 8)
+  ; CHECK:   RET_ReallyLR
+  bb.0:
+    successors: %bb.1(0x40000000)
+    liveins: $x0, $x1
+
+    %0:_(s64) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s128) = G_IMPLICIT_DEF
+    G_BR %bb.1
+
+  bb.1:
+    %3:_(s128) = G_PHI %2(s128), %bb.0
+    %4:_(s64) = G_PHI %0(s64), %bb.0
+    G_STORE %3(s128), %1(p0) :: (store 16)
+    G_STORE %4(s64), %1(p0) :: (store 8)
+    RET_ReallyLR
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-shift.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-shift.mir
new file mode 100644
index 0000000000000..24c551c80fe5e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-vector-shift.mir
@@ -0,0 +1,78 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -march=aarch64 -run-pass=legalizer -global-isel-abort=1 %s -o - | FileCheck %s
+---
+name:            lshr_v4s32
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: lshr_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[LSHR:%[0-9]+]]:_(<4 x s32>) = G_LSHR [[COPY]], [[COPY1]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[LSHR]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<4 x s32>) = G_LSHR %0, %1(<4 x s32>)
+    $q0 = COPY %2(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            lshr_v2s64
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: lshr_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[LSHR:%[0-9]+]]:_(<2 x s64>) = G_LSHR [[COPY]], [[COPY1]](<2 x s64>)
+    ; CHECK: $q0 = COPY [[LSHR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = COPY $q1
+    %2:_(<2 x s64>) = G_LSHR %0, %1(<2 x s64>)
+    $q0 = COPY %2(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ashr_v4s32
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: ashr_v4s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s32>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $q1
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<4 x s32>) = G_ASHR [[COPY]], [[COPY1]](<4 x s32>)
+    ; CHECK: $q0 = COPY [[ASHR]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s32>) = COPY $q0
+    %1:_(<4 x s32>) = COPY $q1
+    %2:_(<4 x s32>) = G_ASHR %0, %1(<4 x s32>)
+    $q0 = COPY %2(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            ashr_v2s64
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: ashr_v2s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<2 x s64>) = COPY $q1
+    ; CHECK: [[ASHR:%[0-9]+]]:_(<2 x s64>) = G_ASHR [[COPY]], [[COPY1]](<2 x s64>)
+    ; CHECK: $q0 = COPY [[ASHR]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s64>) = COPY $q0
+    %1:_(<2 x s64>) = COPY $q1
+    %2:_(<2 x s64>) = G_ASHR %0, %1(<2 x s64>)
+    $q0 = COPY %2(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll b/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll
new file mode 100644
index 0000000000000..68d833338ad67
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/memcpy_chk_no_tail.ll
@@ -0,0 +1,30 @@
+; RUN: llc -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-ios13.0.0"
+
+%struct.int_sqrt = type { i32, i32 }
+
+; Function Attrs: nounwind optsize ssp uwtable
+; CHECK-LABEL: @usqrt
+; CHECK-NOT: b memcpy
+; CHECK: bl _memcpy
+define void @usqrt(i32 %x, %struct.int_sqrt* %q) local_unnamed_addr #0 {
+  %a = alloca i32, align 4
+  %bc = bitcast i32* %a to i8*
+  %bc2 = bitcast %struct.int_sqrt* %q to i8*
+  %obj = tail call i64 @llvm.objectsize.i64.p0i8(i8* %bc2, i1 false, i1 true, i1 false)
+  %call = call i8* @__memcpy_chk(i8* %bc2, i8* nonnull %bc, i64 1000, i64 %obj) #4
+  ret void
+}
+
+; Function Attrs: nofree nounwind optsize
+declare i8* @__memcpy_chk(i8*, i8*, i64, i64) local_unnamed_addr #2
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare i64 @llvm.objectsize.i64.p0i8(i8*, i1 immarg, i1 immarg, i1 immarg) #3
+attributes #0 = { optsize "disable-tail-calls"="false" "frame-pointer"="all" }
+attributes #2 = { nofree nounwind "disable-tail-calls"="false" "frame-pointer"="all" }
+attributes #3 = { nounwind readnone speculatable willreturn }
+attributes #4 = { nounwind optsize }
+
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
index 95da841e71d23..cf9925a9a8a51 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/select-vector-shift.mir
@@ -57,6 +57,35 @@ body:             |
     $q0 = COPY %2(<4 x s32>)
     RET_ReallyLR implicit $q0
 
+...
+---
+name:            shl_v2i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: shl_v2i64
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+    ; CHECK: [[USHLv2i64_:%[0-9]+]]:fpr128 = USHLv2i64 [[COPY]], [[COPY1]]
+    ; CHECK: $q0 = COPY [[USHLv2i64_]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = COPY $q1
+    %2:fpr(<2 x s64>) = G_SHL %0, %1(<2 x s64>)
+    $q0 = COPY %2(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
 ...
 ---
 name:            ashr_v2i32
@@ -118,3 +147,33 @@ body:             |
     RET_ReallyLR implicit $q0
 
 ...
+---
+name:            ashr_v4i64
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+  - { id: 2, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.1:
+    liveins: $q0, $q1
+
+    ; CHECK-LABEL: name: ashr_v4i64
+    ; CHECK: liveins: $q0, $q1
+    ; CHECK: [[COPY:%[0-9]+]]:fpr128 = COPY $q0
+    ; CHECK: [[COPY1:%[0-9]+]]:fpr128 = COPY $q1
+    ; CHECK: [[NEGv2i64_:%[0-9]+]]:fpr128 = NEGv2i64 [[COPY1]]
+    ; CHECK: [[SSHLv2i64_:%[0-9]+]]:fpr128 = SSHLv2i64 [[COPY]], [[NEGv2i64_]]
+    ; CHECK: $q0 = COPY [[SSHLv2i64_]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s64>) = COPY $q0
+    %1:fpr(<2 x s64>) = COPY $q1
+    %2:fpr(<2 x s64>) = G_ASHR %0, %1(<2 x s64>)
+    $q0 = COPY %2(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll b/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll
new file mode 100644
index 0000000000000..533e126292ae1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/tail-call-no-save-fp-lr.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc %s -verify-machineinstrs -mtriple aarch64-apple-darwin -global-isel -global-isel-abort=1 -o - 2>&1 | FileCheck %s
+
+; Check that we get a tail call to foo without saving fp/lr.
+define void @bar(i32 %a) #1 {
+; CHECK-LABEL: bar:
+; CHECK:       ; %bb.0: ; %entry
+; CHECK-NEXT:    b _zoo
+entry:
+  tail call void @zoo(i32 undef)
+  ret void
+}
+
+define void @zoo(i32 %a) {
+entry:
+  ret void
+}
+
+attributes #1 = { "frame-pointer"="all" }
+
diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
index 2d5f667510058..35417deec1acd 100644
--- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll
+++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll
@@ -101,6 +101,7 @@
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Common Subexpression Elimination
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
@@ -114,6 +115,7 @@
 ; CHECK-NEXT:       Live Variable Analysis
 ; CHECK-NEXT:       Eliminate PHI nodes for register allocation
 ; CHECK-NEXT:       Two-Address instruction pass
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Slot index numbering
 ; CHECK-NEXT:       Live Interval Analysis
 ; CHECK-NEXT:       Simple Register Coalescing
diff --git a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
index 2ebb083081075..ad1093028c1a6 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-sve-asm.ll
@@ -8,6 +8,7 @@ target triple = "aarch64-none-linux-gnu"
 ; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
 ; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
 ; CHECK: [[ARG4:%[0-9]+]]:zpr_3b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
 define <vscale x 16 x i8> @test_svadd_i8(<vscale x 16 x i8> %Zn, <vscale x 16 x i8> %Zm) {
   %1 = tail call <vscale x 16 x i8> asm "add $0.b, $1.b, $2.b", "=w,w,y"(<vscale x 16 x i8> %Zn, <vscale x 16 x i8> %Zm)
   ret <vscale x 16 x i8> %1
@@ -18,6 +19,7 @@ define <vscale x 16 x i8> @test_svadd_i8(<vscale x 16 x i8> %Zn, <vscale x 16 x
 ; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
 ; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
 ; CHECK: [[ARG4:%[0-9]+]]:zpr_4b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
 define <vscale x 2 x i64> @test_svsub_i64(<vscale x 2 x i64> %Zn, <vscale x 2 x i64> %Zm) {
   %1 = tail call <vscale x 2 x i64> asm "sub $0.d, $1.d, $2.d", "=w,w,x"(<vscale x 2 x i64> %Zn, <vscale x 2 x i64> %Zm)
   ret <vscale x 2 x i64> %1
@@ -28,6 +30,7 @@ define <vscale x 2 x i64> @test_svsub_i64(<vscale x 2 x i64> %Zn, <vscale x 2 x
 ; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
 ; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
 ; CHECK: [[ARG4:%[0-9]+]]:zpr_3b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
 define <vscale x 8 x half> @test_svfmul_f16(<vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm) {
   %1 = tail call <vscale x 8 x half> asm "fmul $0.h, $1.h, $2.h", "=w,w,y"(<vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
   ret <vscale x 8 x half> %1
@@ -38,7 +41,30 @@ define <vscale x 8 x half> @test_svfmul_f16(<vscale x 8 x half> %Zn, <vscale x 8
 ; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
 ; CHECK: [[ARG3:%[0-9]+]]:zpr = COPY [[ARG2]]
 ; CHECK: [[ARG4:%[0-9]+]]:zpr_4b = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
 define <vscale x 4 x float> @test_svfmul_f(<vscale x 4 x float> %Zn, <vscale x 4 x float> %Zm) {
   %1 = tail call <vscale x 4 x float> asm "fmul $0.s, $1.s, $2.s", "=w,w,x"(<vscale x 4 x float> %Zn, <vscale x 4 x float> %Zm)
   ret <vscale x 4 x float> %1
 }
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z1
+; CHECK: [[ARG2:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG3:%[0-9]+]]:ppr = COPY $p0
+; CHECK: [[ARG4:%[0-9]+]]:ppr_3b = COPY [[ARG3]]
+; CHECK: INLINEASM {{.*}} [[ARG4]]
+define <vscale x 8 x half> @test_svfadd_f16(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm) {
+  %1 = tail call <vscale x 8 x half> asm "fadd $0.h, $1/m, $2.h, $3.h", "=w,@3Upl,w,w"(<vscale x 16 x i1> %Pg, <vscale x 8 x half> %Zn, <vscale x 8 x half> %Zm)
+  ret <vscale x 8 x half> %1
+}
+
+; Function Attrs: nounwind readnone
+; CHECK: [[ARG1:%[0-9]+]]:zpr = COPY $z0
+; CHECK: [[ARG2:%[0-9]+]]:ppr = COPY $p0
+; CHECK: [[ARG3:%[0-9]+]]:ppr = COPY [[ARG2]]
+; CHECK: [[ARG4:%[0-9]+]]:zpr = COPY [[ARG1]]
+; CHECK: INLINEASM {{.*}} [[ARG3]]
+define <vscale x 4 x i32> @test_incp(<vscale x 16 x i1> %Pg, <vscale x 4 x i32> %Zn) {
+  %1 = tail call <vscale x 4 x i32> asm "incp $0.s, $1", "=w,@3Upa,0"(<vscale x 16 x i1> %Pg, <vscale x 4 x i32> %Zn)
+  ret <vscale x 4 x i32> %1
+}
diff --git a/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll b/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll
new file mode 100644
index 0000000000000..8ad93af8e72c2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-code-model-large-darwin.ll
@@ -0,0 +1,16 @@
+; Check that AArch64 is honoring code-model=large at -O0 and -O2.
+;
+; RUN: llc -mtriple=arm64-apple-darwin19 -code-model=large -O0 -o - %s | FileCheck %s
+; RUN: llc -mtriple=arm64-apple-darwin19 -code-model=large -O2 -o - %s | FileCheck %s
+
+; CHECK: adrp    [[REG1:x[0-9]+]], _bar@GOTPAGE
+; CHECK: ldr     [[REG1]], {{\[}}[[REG1]], _bar@GOTPAGEOFF]
+; CHECK: blr     [[REG1]]
+
+declare void @bar()
+
+define void @foo() {
+  call void @bar()
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AArch64/arm64-vshift.ll b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
index 6b0fe40b5a090..8c78bf673830d 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vshift.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vshift.ll
@@ -1192,6 +1192,96 @@ define <2 x i64> @ushll2_2d(<4 x i32>* %A) nounwind {
         ret <2 x i64> %tmp3
 }
 
+declare <16 x i8> @llvm.aarch64.neon.ushl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.ushl.v2i64(<2 x i64>, <2 x i64>)
+
+define <8 x i16> @neon.ushll8h_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.ushll8h_constant_shift
+;CHECK: ushll.8h v0, {{v[0-9]+}}, #1
+  %tmp1 = load <8 x i8>, <8 x i8>* %A
+  %tmp2 = zext <8 x i8> %tmp1 to <8 x i16>
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp2, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %tmp3
+}
+
+define <8 x i16> @neon.ushl8h_no_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.ushl8h_no_constant_shift
+;CHECK: ushl.8h v0, v0, v0
+  %tmp1 = load <8 x i8>, <8 x i8>* %A
+  %tmp2 = zext <8 x i8> %tmp1 to <8 x i16>
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp2, <8 x i16> %tmp2)
+  ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.ushl8h_constant_shift_extend_not_2x(<4 x i8>* %A) nounwind {
+;CHECK-LABEL: @neon.ushl8h_constant_shift_extend_not_2x
+;CHECK-NOT: ushll.8h v0,
+;CHECK: ldrb    w8, [x0]
+;CHECK: fmov    s0, w8
+;CHECK: ldrb    w8, [x0, #1]
+;CHECK: mov.s   v0[1], w8
+;CHECK: ldrb    w8, [x0, #2]
+;CHECK: mov.s   v0[2], w8
+;CHECK: ldrb    w8, [x0, #3]
+;CHECK: mov.s   v0[3], w8
+;CHECK: shl.4s v0, v0, #1
+  %tmp1 = load <4 x i8>, <4 x i8>* %A
+  %tmp2 = zext <4 x i8> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+define <8 x i16> @neon.ushl8_noext_constant_shift(<8 x i16>* %A) nounwind {
+; CHECK-LABEL: neon.ushl8_noext_constant_shift
+; CHECK:      ldr       q0, [x0]
+; CHECK-NEXT: shl.8h   v0, v0, #1
+; CHECK-NEXT: ret
+  %tmp1 = load <8 x i16>, <8 x i16>* %A
+  %tmp3 = call <8 x i16> @llvm.aarch64.neon.ushl.v8i16(<8 x i16> %tmp1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+  ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.ushll4s_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.ushll4s_constant_shift
+;CHECK: ushll.4s v0, {{v[0-9]+}}, #1
+  %tmp1 = load <4 x i16>, <4 x i16>* %A
+  %tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+; FIXME: unnecessary ushll.4s v0, v0, #0?
+define <4 x i32> @neon.ushll4s_neg_constant_shift(<4 x i16>* %A) nounwind {
+; CHECK-LABEL: neon.ushll4s_neg_constant_shift
+; CHECK: movi.2d v1, #0xffffffffffffffff
+; CHECK: ushll.4s v0, v0, #0
+; CHECK: ushl.4s v0, v0, v1
+  %tmp1 = load <4 x i16>, <4 x i16>* %A
+  %tmp2 = zext <4 x i16> %tmp1 to <4 x i32>
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>)
+  ret <4 x i32> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <4 x i32> @neon.ushll4s_constant_fold() nounwind {
+; CHECK-LABEL: neon.ushll4s_constant_fold
+; CHECK: shl.4s v0, v0, #1
+;
+  %tmp3 = call <4 x i32> @llvm.aarch64.neon.ushl.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+  ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @neon.ushll2d_constant_shift(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.ushll2d_constant_shift
+;CHECK: ushll.2d v0, {{v[0-9]+}}, #1
+  %tmp1 = load <2 x i32>, <2 x i32>* %A
+  %tmp2 = zext <2 x i32> %tmp1 to <2 x i64>
+  %tmp3 = call <2 x i64> @llvm.aarch64.neon.ushl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 1, i64 1>)
+  ret <2 x i64> %tmp3
+}
+
 define <8 x i16> @sshll8h(<8 x i8>* %A) nounwind {
 ;CHECK-LABEL: sshll8h:
 ;CHECK: sshll.8h v0, {{v[0-9]+}}, #1
@@ -1201,21 +1291,128 @@ define <8 x i16> @sshll8h(<8 x i8>* %A) nounwind {
         ret <8 x i16> %tmp3
 }
 
-define <4 x i32> @sshll4s(<4 x i16>* %A) nounwind {
-;CHECK-LABEL: sshll4s:
+define <2 x i64> @sshll2d(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: sshll2d:
+;CHECK: sshll.2d v0, {{v[0-9]+}}, #1
+        %tmp1 = load <2 x i32>, <2 x i32>* %A
+        %tmp2 = sext <2 x i32> %tmp1 to <2 x i64>
+        %tmp3 = shl <2 x i64> %tmp2, <i64 1, i64 1>
+        ret <2 x i64> %tmp3
+}
+
+declare <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8>, <16 x i8>)
+declare <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16>, <8 x i16>)
+declare <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64>, <2 x i64>)
+
+define <16 x i8> @neon.sshl16b_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_constant_shift
+;CHECK: shl.16b {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+        ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @neon.sshl16b_non_splat_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_non_splat_constant_shift
+;CHECK: sshl.16b {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 6, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
+        ret <16 x i8> %tmp2
+}
+
+define <16 x i8> @neon.sshl16b_neg_constant_shift(<16 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl16b_neg_constant_shift
+;CHECK: sshl.16b {{v[0-9]+}}, {{v[0-9]+}}, {{v[0-9]+}}
+        %tmp1 = load <16 x i8>, <16 x i8>* %A
+        %tmp2 = call <16 x i8> @llvm.aarch64.neon.sshl.v16i8(<16 x i8> %tmp1, <16 x i8> <i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2, i8 -2>)
+        ret <16 x i8> %tmp2
+}
+
+define <8 x i16> @neon.sshll8h_constant_shift(<8 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshll8h_constant_shift
+;CHECK: sshll.8h v0, {{v[0-9]+}}, #1
+        %tmp1 = load <8 x i8>, <8 x i8>* %A
+        %tmp2 = sext <8 x i8> %tmp1 to <8 x i16>
+        %tmp3 = call <8 x i16> @llvm.aarch64.neon.sshl.v8i16(<8 x i16> %tmp2, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
+        ret <8 x i16> %tmp3
+}
+
+define <4 x i32> @neon.sshl4s_wrong_ext_constant_shift(<4 x i8>* %A) nounwind {
+;CHECK-LABEL: neon.sshl4s_wrong_ext_constant_shift
+;CHECK:       ldrsb   w8, [x0]
+;CHECK-NEXT:  fmov    s0, w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #1]
+;CHECK-NEXT:  mov.s   v0[1], w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #2]
+;CHECK-NEXT:  mov.s   v0[2], w8
+;CHECK-NEXT:  ldrsb   w8, [x0, #3]
+;CHECK-NEXT:  mov.s   v0[3], w8
+;CHECK-NEXT:  shl.4s  v0, v0, #1
+        %tmp1 = load <4 x i8>, <4 x i8>* %A
+        %tmp2 = sext <4 x i8> %tmp1 to <4 x i32>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+        ret <4 x i32> %tmp3
+}
+
+define <4 x i32> @neon.sshll4s_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.sshll4s_constant_shift
 ;CHECK: sshll.4s v0, {{v[0-9]+}}, #1
         %tmp1 = load <4 x i16>, <4 x i16>* %A
         %tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
-        %tmp3 = shl <4 x i32> %tmp2, <i32 1, i32 1, i32 1, i32 1>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
         ret <4 x i32> %tmp3
 }
 
-define <2 x i64> @sshll2d(<2 x i32>* %A) nounwind {
-;CHECK-LABEL: sshll2d:
+define <4 x i32> @neon.sshll4s_neg_constant_shift(<4 x i16>* %A) nounwind {
+;CHECK-LABEL: neon.sshll4s_neg_constant_shift
+;CHECK: movi.2d v1, #0xffffffffffffffff
+;CHECK: sshll.4s v0, v0, #0
+;CHECK: sshl.4s v0, v0, v1
+        %tmp1 = load <4 x i16>, <4 x i16>* %A
+        %tmp2 = sext <4 x i16> %tmp1 to <4 x i32>
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp2, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>)
+        ret <4 x i32> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <4 x i32> @neon.sshl4s_constant_fold() nounwind {
+;CHECK-LABEL: neon.sshl4s_constant_fold
+;CHECK: shl.4s {{v[0-9]+}}, {{v[0-9]+}}, #2
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> <i32 0, i32 1, i32 2, i32 3>, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
+        ret <4 x i32> %tmp3
+}
+
+define <4 x i32> @neon.sshl4s_no_fold(<4 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.sshl4s_no_fold
+;CHECK: shl.4s {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp1 = load <4 x i32>, <4 x i32>* %A
+        %tmp3 = call <4 x i32> @llvm.aarch64.neon.sshl.v4i32(<4 x i32> %tmp1, <4 x i32> <i32 1, i32 1, i32 1, i32 1>)
+        ret <4 x i32> %tmp3
+}
+
+define <2 x i64> @neon.sshll2d_constant_shift(<2 x i32>* %A) nounwind {
+;CHECK-LABEL: neon.sshll2d_constant_shift
 ;CHECK: sshll.2d v0, {{v[0-9]+}}, #1
         %tmp1 = load <2 x i32>, <2 x i32>* %A
         %tmp2 = sext <2 x i32> %tmp1 to <2 x i64>
-        %tmp3 = shl <2 x i64> %tmp2, <i64 1, i64 1>
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 1, i64 1>)
+        ret <2 x i64> %tmp3
+}
+
+; FIXME: should be constant folded.
+define <2 x i64> @neon.sshl2d_constant_fold() nounwind {
+;CHECK-LABEL: neon.sshl2d_constant_fold
+;CHECK: shl.2d {{v[0-9]+}}, {{v[0-9]+}}, #1
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> <i64 99, i64 1000>, <2 x i64> <i64 1, i64 1>)
+        ret <2 x i64> %tmp3
+}
+
+define <2 x i64> @neon.sshl2d_no_fold(<2 x i64>* %A) nounwind {
+;CHECK-LABEL: neon.sshl2d_no_fold
+;CHECK: shl.2d {{v[0-9]+}}, {{v[0-9]+}}, #2
+        %tmp2 = load <2 x i64>, <2 x i64>* %A
+        %tmp3 = call <2 x i64> @llvm.aarch64.neon.sshl.v2i64(<2 x i64> %tmp2, <2 x i64> <i64 2, i64 2>)
         ret <2 x i64> %tmp3
 }
 
diff --git a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
index 579158568b6fd..5806794661403 100644
--- a/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
+++ b/llvm/test/CodeGen/AArch64/callbr-asm-obj-file.ll
@@ -6,11 +6,11 @@
 
 @l = common hidden local_unnamed_addr global i32 0, align 4
 
-; CHECK-LABEL: 0000000000000000 test1:
-; CHECK-LABEL: 0000000000000018 $d.1:
-; CHECK-LABEL: 0000000000000020 $x.2:
+; CHECK-LABEL: test1:
+; CHECK-LABEL: $d.1:
+; CHECK-LABEL: $x.2:
 ; CHECK-NEXT:    b #16 <$x.4+0x4>
-; CHECK-LABEL: 000000000000002c $x.4:
+; CHECK-LABEL: $x.4:
 ; CHECK-NEXT:    b #4 <$x.4+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
@@ -40,11 +40,11 @@ declare dso_local i32 @g(...) local_unnamed_addr
 
 declare dso_local i32 @i(...) local_unnamed_addr
 
-; CHECK-LABEL: 000000000000003c test2:
-; CHECK:         bl #0 <test2+0x18>
-; CHECK-LABEL: 0000000000000064 $d.5:
-; CHECK-LABEL: 000000000000006c $x.6:
-; CHECK-NEXT:    b #-24 <test2+0x18>
+; CHECK-LABEL: test2:
+; CHECK:         bl #0 <test2+0x10>
+; CHECK-LABEL: $d.5:
+; CHECK-LABEL: $x.6:
+; CHECK-NEXT:    b #16 <$x.8+0x4>
 define hidden i32 @test2() local_unnamed_addr {
   %1 = load i32, i32* @l, align 4
   %2 = icmp eq i32 %1, 0
@@ -57,7 +57,7 @@ define hidden i32 @test2() local_unnamed_addr {
 
 6:                                                ; preds = %3
   callbr void asm sideeffect "1: nop\0A\09.quad b\0A\09b ${1:l}\0A\09.quad ${0:c}", "i,X"(i32* null, i8* blockaddress(@test2, %7))
-          to label %10 [label %9]
+          to label %10 [label %7]
 
 7:                                                ; preds = %3
   %8 = tail call i32 bitcast (i32 (...)* @i to i32 ()*)()
@@ -70,11 +70,11 @@ define hidden i32 @test2() local_unnamed_addr {
   ret i32 undef
 }
 
-; CHECK-LABEL: 0000000000000084 test3:
-; CHECK-LABEL: 00000000000000a8 $d.9:
-; CHECK-LABEL: 00000000000000b0 $x.10:
+; CHECK-LABEL: test3:
+; CHECK-LABEL: $d.9:
+; CHECK-LABEL: $x.10:
 ; CHECK-NEXT:    b #20 <$x.12+0x8>
-; CHECK-LABEL: 00000000000000bc $x.12:
+; CHECK-LABEL: $x.12:
 ; CHECK-NEXT:    b #4 <$x.12+0x4>
 ; CHECK-NEXT:    mov w0, wzr
 ; CHECK-NEXT:    ldr x30, [sp], #16
diff --git a/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll
new file mode 100644
index 0000000000000..902ec7a9c095c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/fast-isel-branch-uncond-debug.ll
@@ -0,0 +1,44 @@
+; RUN: llc -mtriple=aarch64-arm-none-eabi -O1 -opt-bisect-limit=2 -o - %s  2> /dev/null | FileCheck %s
+
+define dso_local i32 @a() #0 !dbg !7 {
+entry:
+; CHECK:    b       .LBB0_1
+; CHECK:  .LBB0_1:
+  call void @llvm.dbg.value(metadata i32 0, metadata !12, metadata !DIExpression()), !dbg !13
+  br label %for.cond, !dbg !14
+
+; CHECK:    b       .LBB0_1
+; CHECK:  .Lfunc_end0:
+for.cond:
+  br label %for.cond, !dbg !15, !llvm.loop !18
+}
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1
+
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "fast-isel-branch-uncond-debug.ll", directory: "/tmp")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!""}
+!7 = distinct !DISubprogram(name: "a", scope: !1, file: !1, line: 1, type: !8, scopeLine: 1, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !{!12}
+!12 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 2, type: !10)
+!13 = !DILocation(line: 0, scope: !7)
+!14 = !DILocation(line: 3, column: 3, scope: !7)
+!15 = !DILocation(line: 3, column: 3, scope: !16)
+!16 = distinct !DILexicalBlock(scope: !17, file: !1, line: 3, column: 3)
+!17 = distinct !DILexicalBlock(scope: !7, file: !1, line: 3, column: 3)
+!18 = distinct !{!18, !19, !20}
+!19 = !DILocation(line: 3, column: 3, scope: !17)
+!20 = !DILocation(line: 4, column: 5, scope: !17)
+ 
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/fastcc-reserved.ll b/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
index a463e62217943..a527354ea1866 100644
--- a/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
+++ b/llvm/test/CodeGen/AArch64/fastcc-reserved.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s
 
 ; This test is designed to be run in the situation where the
 ; call-frame is not reserved (hence disable-fp-elim), but where
diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
index 9e276cdfbf161..856b00a84b238 100644
--- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
+++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_lane.ll
@@ -175,8 +175,7 @@ entry:
 
 define dso_local half @t_vfmsh_lane_f16(half %a, half %b, <4 x half> %c, i32 %lane) {
 ; CHECK-LABEL: t_vfmsh_lane_f16:
-; CHECK:         fneg h1, h1
-; CHECK:         fmadd h0, h1, h2, h0
+; CHECK:         fmsub h0, h1, h2, h0
 ; CHECK-NEXT:    ret
 entry:
   %0 = fsub half 0xH8000, %b
@@ -187,9 +186,8 @@ entry:
 
 define dso_local half @t_vfmsh_laneq_f16(half %a, half %b, <8 x half> %c, i32 %lane) {
 ; CHECK-LABEL: t_vfmsh_laneq_f16:
-; CHECK:         fneg h1, h1
-; CHECK-NEXT:    fmadd h0, h1, h2, h0
-; CHECK-NEXT:    ret
+; CHECK:       fmsub h0, h1, h2, h0
+; CHECK-NEXT:  ret
 entry:
   %0 = fsub half 0xH8000, %b
   %extract = extractelement <8 x half> %c, i32 0
diff --git a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
index 9a5a0a5b1a9b3..f3ad71b9c4455 100644
--- a/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
+++ b/llvm/test/CodeGen/AArch64/fp16_intrinsic_scalar_3op.ll
@@ -10,44 +10,83 @@ entry:
 }
 
 define half @fnma16(half %a, half %b, half %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fnma16:
 ; CHECK: fnmadd h0, h0, h1, h2
+entry:
   %0 = tail call half @llvm.fma.f16(half %a, half %b, half %c)
   %mul = fmul half %0, -1.000000e+00
   ret half %mul
 }
 
 define half @fms16(half %a, half %b, half %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fms16:
 ; CHECK: fmsub h0, h0, h1, h2
+entry:
   %mul = fmul half %b, -1.000000e+00
   %0 = tail call half @llvm.fma.f16(half %a, half %mul, half %c)
   ret half %0
 }
 
 define half @fms16_com(half %a, half %b, half %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fms16_com:
-
-; FIXME:       This should be a fmsub.
-
-; CHECK:       fneg  h1, h1
-; CHECK-NEXT:  fmadd h0, h1, h0, h2
+; CHECK:       fmsub h0, h1, h0, h2
+; CHECK-NEXT:  ret
+entry:
   %mul = fmul half %b, -1.000000e+00
   %0 = tail call half @llvm.fma.f16(half %mul, half %a, half %c)
   ret half %0
 }
 
 define half @fnms16(half %a, half %b, half %c) nounwind readnone ssp {
-entry:
 ; CHECK-LABEL: fnms16:
-; CHECK: fnmsub h0, h0, h1, h2
+; CHECK:       fnmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
   %mul = fmul half %c, -1.000000e+00
   %0 = tail call half @llvm.fma.f16(half %a, half %b, half %mul)
   ret half %0
 }
 
-declare half @llvm.fma.f16(half, half, half)
+define half @test_fmsub(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fmsub:
+; CHECK:       fmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %val = call half @llvm.fma.f16(half %nega, half %b, half %c)
+  ret half %val
+}
+
+define half @test_fnmadd(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fnmadd:
+; CHECK:       fnmadd h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %negc = fsub half -0.0, %c
+  %val = call half @llvm.fma.f16(half %nega, half %b, half %negc)
+  ret half %val
+}
 
+define half @test_fmadd(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fmadd:
+; CHECK:       fmadd h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %nega = fsub half -0.0, %a
+  %negb = fsub half -0.0, %b
+  %val = call half @llvm.fma.f16(half %nega, half %negb, half %c)
+  ret half %val
+}
+
+define half @test_fnmsub(half %a, half %b, half %c) {
+; CHECK-LABEL: test_fnmsub:
+; CHECK:       fnmsub h0, h0, h1, h2
+; CHECK-NEXT:  ret
+entry:
+  %negc = fsub half -0.0, %c
+  %val = call half @llvm.fma.f16(half %a, half %b, half %negc)
+  ret half %val
+}
+
+declare half @llvm.fma.f16(half, half, half)
diff --git a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
index d853b148d57f4..48c5bf68efab9 100644
--- a/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
+++ b/llvm/test/CodeGen/AArch64/hwasan-check-memaccess.ll
@@ -8,11 +8,11 @@ define i8* @f1(i8* %x0, i8* %x1) {
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x9, x0
-  ; CHECK-NEXT: bl __hwasan_check_x1_123
+  ; CHECK-NEXT: bl __hwasan_check_x1_1
   ; CHECK-NEXT: mov x0, x1
   ; CHECK-NEXT: ldr x30, [sp], #16
   ; CHECK-NEXT: ret
-  call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 123)
+  call void @llvm.hwasan.check.memaccess(i8* %x0, i8* %x1, i32 1)
   ret i8* %x1
 }
 
@@ -22,20 +22,21 @@ define i8* @f2(i8* %x0, i8* %x1) {
   ; CHECK-NEXT: .cfi_def_cfa_offset 16
   ; CHECK-NEXT: .cfi_offset w30, -16
   ; CHECK-NEXT: mov x9, x1
-  ; CHECK-NEXT: bl __hwasan_check_x0_456
+  ; CHECK-NEXT: bl __hwasan_check_x0_2_short
   ; CHECK-NEXT: ldr x30, [sp], #16
   ; CHECK-NEXT: ret
-  call void @llvm.hwasan.check.memaccess(i8* %x1, i8* %x0, i32 456)
+  call void @llvm.hwasan.check.memaccess.shortgranules(i8* %x1, i8* %x0, i32 2)
   ret i8* %x0
 }
 
 declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
+declare void @llvm.hwasan.check.memaccess.shortgranules(i8*, i8*, i32)
 
-; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x0_456,comdat
-; CHECK-NEXT: .type __hwasan_check_x0_456,@function
-; CHECK-NEXT: .weak __hwasan_check_x0_456
-; CHECK-NEXT: .hidden __hwasan_check_x0_456
-; CHECK-NEXT: __hwasan_check_x0_456:
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x0_2_short,comdat
+; CHECK-NEXT: .type __hwasan_check_x0_2_short,@function
+; CHECK-NEXT: .weak __hwasan_check_x0_2_short
+; CHECK-NEXT: .hidden __hwasan_check_x0_2_short
+; CHECK-NEXT: __hwasan_check_x0_2_short:
 ; CHECK-NEXT: ubfx x16, x0, #4, #52
 ; CHECK-NEXT: ldrb w16, [x9, x16]
 ; CHECK-NEXT: cmp x16, x0, lsr #56
@@ -46,7 +47,7 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: cmp w16, #15
 ; CHECK-NEXT: b.hi .Ltmp2
 ; CHECK-NEXT: and x17, x0, #0xf
-; CHECK-NEXT: add x17, x17, #255
+; CHECK-NEXT: add x17, x17, #3
 ; CHECK-NEXT: cmp w16, w17
 ; CHECK-NEXT: b.ls .Ltmp2
 ; CHECK-NEXT: orr x16, x0, #0xf
@@ -56,17 +57,17 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: .Ltmp2:
 ; CHECK-NEXT: stp x0, x1, [sp, #-256]!
 ; CHECK-NEXT: stp x29, x30, [sp, #232]
-; CHECK-NEXT: mov x1, #456
-; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch
-; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch]
+; CHECK-NEXT: mov x1, #2
+; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch_v2
+; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch_v2]
 ; CHECK-NEXT: br  x16
 
 
-; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x1_123,comdat
-; CHECK-NEXT: .type __hwasan_check_x1_123,@function
-; CHECK-NEXT: .weak __hwasan_check_x1_123
-; CHECK-NEXT: .hidden __hwasan_check_x1_123
-; CHECK-NEXT: __hwasan_check_x1_123:
+; CHECK:      .section .text.hot,"axG",@progbits,__hwasan_check_x1_1,comdat
+; CHECK-NEXT: .type __hwasan_check_x1_1,@function
+; CHECK-NEXT: .weak __hwasan_check_x1_1
+; CHECK-NEXT: .hidden __hwasan_check_x1_1
+; CHECK-NEXT: __hwasan_check_x1_1:
 ; CHECK-NEXT: ubfx x16, x1, #4, #52
 ; CHECK-NEXT: ldrb w16, [x9, x16]
 ; CHECK-NEXT: cmp x16, x1, lsr #56
@@ -74,21 +75,10 @@ declare void @llvm.hwasan.check.memaccess(i8*, i8*, i32)
 ; CHECK-NEXT: .Ltmp4:
 ; CHECK-NEXT: ret
 ; CHECK-NEXT: .Ltmp3:
-; CHECK-NEXT: cmp w16, #15
-; CHECK-NEXT: b.hi .Ltmp5
-; CHECK-NEXT: and x17, x1, #0xf
-; CHECK-NEXT: add x17, x17, #2047
-; CHECK-NEXT: cmp w16, w17
-; CHECK-NEXT: b.ls .Ltmp5
-; CHECK-NEXT: orr x16, x1, #0xf
-; CHECK-NEXT: ldrb w16, [x16]
-; CHECK-NEXT: cmp x16, x1, lsr #56
-; CHECK-NEXT: b.eq .Ltmp4
-; CHECK-NEXT: .Ltmp5:
 ; CHECK-NEXT: stp x0, x1, [sp, #-256]!
 ; CHECK-NEXT: stp x29, x30, [sp, #232]
 ; CHECK-NEXT: mov x0, x1
-; CHECK-NEXT: mov x1, #123
+; CHECK-NEXT: mov x1, #1
 ; CHECK-NEXT: adrp  x16, :got:__hwasan_tag_mismatch
 ; CHECK-NEXT: ldr x16, [x16, :got_lo12:__hwasan_tag_mismatch]
 ; CHECK-NEXT: br  x16
diff --git a/llvm/test/CodeGen/AArch64/ldst-opt-mte.mir b/llvm/test/CodeGen/AArch64/ldst-opt-mte.mir
new file mode 100644
index 0000000000000..b44258abd8871
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ldst-opt-mte.mir
@@ -0,0 +1,285 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -run-pass aarch64-ldst-opt  -verify-machineinstrs  -o - %s | FileCheck %s
+---
+
+### STG and its offset limits
+
+# CHECK-LABEL: name: test_STG_post
+# CHECK: STGPostIndex $x0, $x0, 7
+name: test_STG_post
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post_same_reg
+# CHECK: STGPostIndex $x1, $x0, 7
+name: test_STG_post_same_reg
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1
+
+    STGOffset $x1, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post_unaligned
+# CHECK:      STGOffset $x0, $x0, 0
+# CHECK-NEXT: ADDXri $x0, 8, 0
+name: test_STG_post_unaligned
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 8, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post2
+# CHECK: STGPostIndex $x0, $x0, -256
+name: test_STG_post2
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = SUBXri $x0, 4096, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post3
+# CHECK:      STGOffset $x0, $x0, 0
+# CHECK-NEXT: SUBXri $x0, 4112, 0
+name: test_STG_post3
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = SUBXri $x0, 4112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post4
+# CHECK: STGPostIndex $x0, $x0, 255
+name: test_STG_post4
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 4080, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STG_post5
+# CHECK:      STGOffset $x0, $x0, 0
+# CHECK-NEXT: ADDXri $x0, 4096, 0
+name: test_STG_post5
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 4096, 0
+    RET_ReallyLR implicit $x0
+...
+
+### The rest of ST*G variants.
+
+# CHECK-LABEL: name: test_STZG_post
+# CHECK: STZGPostIndex $x0, $x0, 7
+name: test_STZG_post
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STZGOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_ST2G_post
+# CHECK: ST2GPostIndex $x0, $x0, 7
+name: test_ST2G_post
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    ST2GOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STZ2G_post
+# CHECK: STZ2GPostIndex $x0, $x0, 7
+name: test_STZ2G_post
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STZ2GOffset $x0, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+### STGP and its offset limits
+
+# CHECK-LABEL: name: test_STGP_post
+# CHECK: STGPpost $x1, $x2, $x0, 7
+name: test_STGP_post
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_post2
+# CHECK: STGPpost $x1, $x2, $x0, -64
+name: test_STGP_post2
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 0
+    $x0 = SUBXri $x0, 1024, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_post3
+# CHECK:      STGPi $x1, $x2, $x0, 0
+# CHECK-NEXT: SUBXri $x0, 1040, 0
+name: test_STGP_post3
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 0
+    $x0 = SUBXri $x0, 1040, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_post4
+# CHECK: STGPpost $x1, $x2, $x0, 63
+name: test_STGP_post4
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 0
+    $x0 = ADDXri $x0, 1008, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_post5
+# CHECK:      STGPi $x1, $x2, $x0, 0
+# CHECK-NEXT: ADDXri $x0, 1024, 0
+name: test_STGP_post5
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 0
+    $x0 = ADDXri $x0, 1024, 0
+    RET_ReallyLR implicit $x0
+...
+
+### Pre-indexed forms
+
+# CHECK-LABEL: name: test_STG_pre
+# CHECK: STGPreIndex $x0, $x0, 10
+name: test_STG_pre
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGOffset $x0, $x0, 10
+    $x0 = ADDXri $x0, 160, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_pre
+# CHECK: STGPpre $x1, $x2, $x0, 10
+name: test_STGP_pre
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    STGPi $x1, $x2, $x0, 10
+    $x0 = ADDXri $x0, 160, 0
+    RET_ReallyLR implicit $x0
+...
+
+### Pre-indexed forms with add/sub coming before the store.
+
+# CHECK-LABEL: name: test_STG_pre_back
+# CHECK: STGPreIndex $x0, $x0, 2
+name: test_STG_pre_back
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    $x0 = ADDXri $x0, 32, 0
+    STGOffset $x0, $x0, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_pre_back
+# CHECK: STGPpre $x1, $x2, $x0, -3
+name: test_STGP_pre_back
+body: |
+  bb.0.entry:
+    liveins: $x0, $x1, $x2
+
+    $x0 = SUBXri $x0, 48, 0
+    STGPi $x1, $x2, $x0, 0
+    RET_ReallyLR implicit $x0
+...
+
+### STGP with source register == address register
+
+# CHECK-LABEL: name: test_STGP_post_same_reg
+# CHECK: STGPpost $x0, $x0, $x0, 7
+name: test_STGP_post_same_reg
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGPi $x0, $x0, $x0, 0
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# CHECK-LABEL: name: test_STGP_pre_same_reg
+# CHECK: STGPpre $x0, $x0, $x0, 7
+name: test_STGP_pre_same_reg
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    STGPi $x0, $x0, $x0, 7
+    $x0 = ADDXri $x0, 112, 0
+    RET_ReallyLR implicit $x0
+...
+
+# This case can not be merged because the source register is always read before writeback.
+# CHECK-LABEL: name: test_STGP_pre_back_same_reg
+# CHECK:      SUBXri $x0, 48, 0
+# CHECK-NEXT: STGPi $x0, $x0, $x0, 0
+name: test_STGP_pre_back_same_reg
+body: |
+  bb.0.entry:
+    liveins: $x0
+
+    $x0 = SUBXri $x0, 48, 0
+    STGPi $x0, $x0, $x0, 0
+    RET_ReallyLR implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/neon-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-dot-product.ll
index 4b3c7cf4a6574..b6131e1d045ed 100644
--- a/llvm/test/CodeGen/AArch64/neon-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-dot-product.ll
@@ -128,3 +128,145 @@ entry:
   %vdot1.i = call <4 x i32> @llvm.aarch64.neon.sdot.v4i32.v16i8(<4 x i32> %a, <16 x i8> %b, <16 x i8> %.cast3) #2
   ret <4 x i32> %vdot1.i
 }
+
+define fastcc void @test_sdot_v4i8(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i8* noalias nocapture readonly %2) {
+entry:
+; CHECK-LABEL: test_sdot_v4i8:
+; CHECK:  sdot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %3 = bitcast i8* %0 to i32*
+  %4 = load i8, i8* %1, align 1
+  %5 = sext i8 %4 to i32
+  %6 = load i8, i8* %2, align 1
+  %7 = sext i8 %6 to i32
+  %8 = mul nsw i32 %7, %5
+  %9 = getelementptr inbounds i8, i8* %1, i64 1
+  %10 = load i8, i8* %9, align 1
+  %11 = sext i8 %10 to i32
+  %12 = getelementptr inbounds i8, i8* %2, i64 1
+  %13 = load i8, i8* %12, align 1
+  %14 = sext i8 %13 to i32
+  %15 = mul nsw i32 %14, %11
+  %16 = add nsw i32 %15, %8
+  %17 = getelementptr inbounds i8, i8* %1, i64 2
+  %18 = load i8, i8* %17, align 1
+  %19 = sext i8 %18 to i32
+  %20 = getelementptr inbounds i8, i8* %2, i64 2
+  %21 = load i8, i8* %20, align 1
+  %22 = sext i8 %21 to i32
+  %23 = mul nsw i32 %22, %19
+  %24 = add nsw i32 %23, %16
+  %25 = getelementptr inbounds i8, i8* %1, i64 3
+  %26 = load i8, i8* %25, align 1
+  %27 = sext i8 %26 to i32
+  %28 = getelementptr inbounds i8, i8* %2, i64 3
+  %29 = load i8, i8* %28, align 1
+  %30 = sext i8 %29 to i32
+  %31 = mul nsw i32 %30, %27
+  %32 = add nsw i32 %31, %24
+  store i32 %32, i32* %3, align 64
+  ret void
+}
+
+define fastcc void @test_udot_v4i8(i8* noalias nocapture %0, i8* noalias nocapture readonly %1, i8* noalias nocapture readonly %2) {
+entry:
+; CHECK-LABEL: test_udot_v4i8:
+; CHECK:  udot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %3 = bitcast i8* %0 to i32*
+  %4 = load i8, i8* %1, align 1
+  %5 = zext i8 %4 to i32
+  %6 = load i8, i8* %2, align 1
+  %7 = zext i8 %6 to i32
+  %8 = mul nsw i32 %7, %5
+  %9 = getelementptr inbounds i8, i8* %1, i64 1
+  %10 = load i8, i8* %9, align 1
+  %11 = zext i8 %10 to i32
+  %12 = getelementptr inbounds i8, i8* %2, i64 1
+  %13 = load i8, i8* %12, align 1
+  %14 = zext i8 %13 to i32
+  %15 = mul nsw i32 %14, %11
+  %16 = add nsw i32 %15, %8
+  %17 = getelementptr inbounds i8, i8* %1, i64 2
+  %18 = load i8, i8* %17, align 1
+  %19 = zext i8 %18 to i32
+  %20 = getelementptr inbounds i8, i8* %2, i64 2
+  %21 = load i8, i8* %20, align 1
+  %22 = zext i8 %21 to i32
+  %23 = mul nsw i32 %22, %19
+  %24 = add nsw i32 %23, %16
+  %25 = getelementptr inbounds i8, i8* %1, i64 3
+  %26 = load i8, i8* %25, align 1
+  %27 = zext i8 %26 to i32
+  %28 = getelementptr inbounds i8, i8* %2, i64 3
+  %29 = load i8, i8* %28, align 1
+  %30 = zext i8 %29 to i32
+  %31 = mul nsw i32 %30, %27
+  %32 = add nsw i32 %31, %24
+  store i32 %32, i32* %3, align 64
+  ret void
+}
+
+declare i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32>)
+
+define i32 @test_udot_v8i8(i8* nocapture readonly %a, i8* nocapture readonly %b) {
+entry:
+; CHECK-LABEL: test_udot_v8i8:
+; CHECK:  udot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %0 = bitcast i8* %a to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0
+  %2 = zext <8 x i8> %1 to <8 x i32>
+  %3 = bitcast i8* %b to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3
+  %5 = zext <8 x i8> %4 to <8 x i32>
+  %6 = mul nuw nsw <8 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %6)
+  ret i32 %7
+}
+
+define i32 @test_sdot_v8i8(i8* nocapture readonly %a, i8* nocapture readonly %b) {
+entry:
+; CHECK-LABEL: test_sdot_v8i8:
+; CHECK:  sdot {{v[0-9]+}}.2s, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
+  %0 = bitcast i8* %a to <8 x i8>*
+  %1 = load <8 x i8>, <8 x i8>* %0
+  %2 = sext <8 x i8> %1 to <8 x i32>
+  %3 = bitcast i8* %b to <8 x i8>*
+  %4 = load <8 x i8>, <8 x i8>* %3
+  %5 = sext <8 x i8> %4 to <8 x i32>
+  %6 = mul nsw <8 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> %6)
+  ret i32 %7
+}
+
+declare i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32>)
+
+define i32 @test_udot_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %sum) {
+entry:
+; CHECK-LABEL: test_udot_v16i8:
+; CHECK:  udot {{v[0-9]+}}.4s, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+  %0 = bitcast i8* %a to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0
+  %2 = zext <16 x i8> %1 to <16 x i32>
+  %3 = bitcast i8* %b to <16 x i8>*
+  %4 = load <16 x i8>, <16 x i8>* %3
+  %5 = zext <16 x i8> %4 to <16 x i32>
+  %6 = mul nuw nsw <16 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %6)
+  %op.extra = add i32 %7, %sum
+  ret i32 %op.extra
+}
+
+define i32 @test_sdot_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %sum) {
+entry:
+; CHECK-LABEL: test_sdot_v16i8:
+; CHECK:  sdot {{v[0-9]+}}.4s, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b
+  %0 = bitcast i8* %a to <16 x i8>*
+  %1 = load <16 x i8>, <16 x i8>* %0
+  %2 = sext <16 x i8> %1 to <16 x i32>
+  %3 = bitcast i8* %b to <16 x i8>*
+  %4 = load <16 x i8>, <16 x i8>* %3
+  %5 = sext <16 x i8> %4 to <16 x i32>
+  %6 = mul nsw <16 x i32> %5, %2
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v16i32(<16 x i32> %6)
+  %op.extra = add nsw i32 %7, %sum
+  ret i32 %op.extra
+}
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat.ll b/llvm/test/CodeGen/AArch64/sadd_sat.ll
index 943458496f539..4af00a5b79884 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat.ll
@@ -54,17 +54,13 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
 ; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
diff --git a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
index 40b945885f5bf..5a2a24ee9c8fe 100644
--- a/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/sadd_sat_vec.ll
@@ -36,17 +36,13 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v3.16b, #127
+; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.sadd.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
@@ -57,29 +53,21 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmlt v16.16b, v4.16b, #0
+; CHECK-NEXT:    cmlt v7.16b, v4.16b, #0
 ; CHECK-NEXT:    movi v6.16b, #127
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    add v7.16b, v1.16b, v3.16b
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v2.16b, v2.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v4.16b
 ; CHECK-NEXT:    cmlt v16.16b, v7.16b, #0
 ; CHECK-NEXT:    movi v5.16b, #127
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v16.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v16.16b
-; CHECK-NEXT:    cmge v16.16b, v7.16b, #0
-; CHECK-NEXT:    cmeq v3.16b, v1.16b, v3.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v16.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmlt v3.16b, v3.16b, #0
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, v7.16b
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -102,42 +90,26 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.16b, v21.16b, #0
+; CHECK-NEXT:    cmlt v4.16b, v4.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v16.16b
 ; CHECK-NEXT:    movi v22.16b, #127
 ; CHECK-NEXT:    add v23.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmlt v4.16b, v5.16b, #0
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, v19.16b
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.16b, v23.16b, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmlt v4.16b, v6.16b, #0
+; CHECK-NEXT:    cmgt v2.16b, v2.16b, v21.16b
 ; CHECK-NEXT:    movi v17.16b, #127
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmlt v4.16b, v7.16b, #0
+; CHECK-NEXT:    cmgt v3.16b, v3.16b, v23.16b
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v24.16b, v16.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v5.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v19.16b, #0
-; CHECK-NEXT:    cmge v6.16b, v6.16b, #0
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v5.16b, v1.16b, v5.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v21.16b, #0
-; CHECK-NEXT:    cmge v7.16b, v7.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v6.16b, v2.16b, v6.16b
-; CHECK-NEXT:    cmeq v2.16b, v2.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v23.16b, #0
-; CHECK-NEXT:    cmeq v7.16b, v3.16b, v7.16b
-; CHECK-NEXT:    cmeq v3.16b, v3.16b, v24.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -151,17 +123,13 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
 ; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
 ; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
+; CHECK-NEXT:    cmlt v1.8h, v1.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.sadd.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
@@ -172,29 +140,21 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
+; CHECK-NEXT:    cmlt v7.8h, v4.8h, #0
 ; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    add v7.8h, v1.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v2.8h, v2.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v4.8h
 ; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
 ; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmlt v3.8h, v3.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v7.8h
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -217,42 +177,26 @@ define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
+; CHECK-NEXT:    cmlt v4.8h, v4.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v16.8h
 ; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
 ; CHECK-NEXT:    add v23.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmlt v4.8h, v5.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v19.8h
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmlt v4.8h, v6.8h, #0
+; CHECK-NEXT:    cmgt v2.8h, v2.8h, v21.8h
 ; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmlt v4.8h, v7.8h, #0
+; CHECK-NEXT:    cmgt v3.8h, v3.8h, v23.8h
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
-; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
-; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
-; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -269,16 +213,12 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldr d1, [x1]
 ; CHECK-NEXT:    movi v2.8b, #127
 ; CHECK-NEXT:    add v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
 ; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
+; CHECK-NEXT:    cmlt v1.8b, v1.8b, #0
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, v3.8b
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
@@ -311,17 +251,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
@@ -348,17 +284,13 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    cmlt v2.2s, v2.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
@@ -380,16 +312,12 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldr d1, [x1]
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
@@ -414,17 +342,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
 ; CHECK-NEXT:    add v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    cmlt v2.2s, v2.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
@@ -443,17 +367,13 @@ define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v3.16b, #127
+; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.sadd.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
@@ -468,27 +388,19 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 ; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
 ; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
 ; CHECK-NEXT:    add v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
+; CHECK-NEXT:    cmlt v7.8h, v6.8h, #0
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v5.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    add v7.8h, v0.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v2.8h, v2.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v6.8h
 ; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    cmlt v3.8h, v3.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v7.8h
+; CHECK-NEXT:    eor v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bsl v4.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
 ; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
 ; CHECK-NEXT:    str q0, [x2]
@@ -508,16 +420,12 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldr b1, [x1]
 ; CHECK-NEXT:    movi v2.8b, #127
 ; CHECK-NEXT:    add v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
 ; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
+; CHECK-NEXT:    cmlt v1.8b, v1.8b, #0
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, v3.8b
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
@@ -535,16 +443,12 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldr h1, [x1]
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    add v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    cmlt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
@@ -561,17 +465,13 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
 ; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v2.16b, #127
+; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
@@ -585,17 +485,13 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
 ; CHECK-NEXT:    add v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v2.16b, #127
+; CHECK-NEXT:    cmlt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
@@ -607,17 +503,13 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
+; CHECK-NEXT:    cmlt v1.2s, v1.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
@@ -628,17 +520,13 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
 ; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    cmlt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
@@ -649,29 +537,21 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
+; CHECK-NEXT:    cmlt v7.4s, v4.4s, #0
 ; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    add v7.4s, v1.4s, v3.4s
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmlt v2.4s, v2.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v4.4s
 ; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
 ; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
-; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmlt v3.4s, v3.4s, #0
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, v7.4s
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -694,42 +574,26 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
+; CHECK-NEXT:    cmlt v4.4s, v4.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v16.4s
 ; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
 ; CHECK-NEXT:    add v23.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmlt v4.4s, v5.4s, #0
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, v19.4s
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmlt v4.4s, v6.4s, #0
+; CHECK-NEXT:    cmgt v2.4s, v2.4s, v21.4s
 ; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmlt v4.4s, v7.4s, #0
+; CHECK-NEXT:    cmgt v3.4s, v3.4s, v23.4s
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
-; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
-; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
-; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -743,18 +607,14 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v2.2d, #0
 ; CHECK-NEXT:    mov x8, #9223372036854775807
 ; CHECK-NEXT:    cmlt v3.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v5.2d
+; CHECK-NEXT:    cmlt v1.2d, v1.2d, #0
 ; CHECK-NEXT:    dup v4.2d, x8
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v4.16b, v3.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
@@ -766,31 +626,23 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    add v4.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v6.2d, v4.2d, #0
-; CHECK-NEXT:    dup v7.2d, x8
+; CHECK-NEXT:    cmlt v5.2d, v4.2d, #0
+; CHECK-NEXT:    dup v6.2d, x8
+; CHECK-NEXT:    mvn v7.16b, v5.16b
+; CHECK-NEXT:    mov v16.16b, v6.16b
+; CHECK-NEXT:    bsl v16.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    add v5.2d, v1.2d, v3.2d
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    mov v17.16b, v7.16b
-; CHECK-NEXT:    bsl v17.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmlt v6.2d, v5.2d, #0
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    bsl v7.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v6.2d
-; CHECK-NEXT:    cmge v6.2d, v5.2d, #0
-; CHECK-NEXT:    cmeq v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v6.2d
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v17.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v7.16b, v5.16b
+; CHECK-NEXT:    cmlt v2.2d, v2.2d, #0
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    cmlt v7.2d, v5.2d, #0
+; CHECK-NEXT:    cmlt v3.2d, v3.2d, #0
+; CHECK-NEXT:    cmgt v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v7.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v2.16b
+; CHECK-NEXT:    bsl v0.16b, v16.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v6.16b, v5.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -812,42 +664,26 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-NEXT:    bsl v25.16b, v20.16b, v24.16b
 ; CHECK-NEXT:    mvn v20.16b, v22.16b
 ; CHECK-NEXT:    mov v24.16b, v21.16b
+; CHECK-NEXT:    cmlt v4.2d, v4.2d, #0
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v16.2d
 ; CHECK-NEXT:    add v19.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    bsl v24.16b, v22.16b, v20.16b
 ; CHECK-NEXT:    mvn v20.16b, v23.16b
 ; CHECK-NEXT:    mov v22.16b, v21.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmlt v4.2d, v5.2d, #0
+; CHECK-NEXT:    cmgt v1.2d, v1.2d, v17.2d
 ; CHECK-NEXT:    bsl v22.16b, v23.16b, v20.16b
 ; CHECK-NEXT:    cmlt v20.2d, v19.2d, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmlt v4.2d, v6.2d, #0
+; CHECK-NEXT:    cmgt v2.2d, v2.2d, v18.2d
 ; CHECK-NEXT:    mvn v23.16b, v20.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmlt v4.2d, v7.2d, #0
+; CHECK-NEXT:    cmgt v3.2d, v3.2d, v19.2d
 ; CHECK-NEXT:    bsl v21.16b, v20.16b, v23.16b
-; CHECK-NEXT:    cmge v4.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v20.2d, v16.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v5.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v17.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v6.2d, #0
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v18.2d, #0
-; CHECK-NEXT:    cmge v7.2d, v7.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmeq v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmeq v2.2d, v2.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v19.2d, #0
-; CHECK-NEXT:    cmeq v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmeq v3.2d, v3.2d, v20.2d
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v5.16b, v1.16b
-; CHECK-NEXT:    and v2.16b, v6.16b, v2.16b
-; CHECK-NEXT:    and v3.16b, v7.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v25.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v18.16b
diff --git a/llvm/test/CodeGen/AArch64/shift-by-signext.ll b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
new file mode 100644
index 0000000000000..8e6a4d2500c50
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/shift-by-signext.ll
@@ -0,0 +1,122 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t0_shl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsl w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t1_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsr w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) nounwind {
+; CHECK-LABEL: t2_ashr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    asr w0, w0, w8
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <4 x i32> @t3_vec_shl(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t3_vec_shl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = shl <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t4_vec_lshr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t4_vec_lshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    ushl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = lshr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t5_vec_ashr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t5_vec_ashr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
+; CHECK-NEXT:    shl v1.4s, v1.4s, #24
+; CHECK-NEXT:    sshr v1.4s, v1.4s, #24
+; CHECK-NEXT:    neg v1.4s, v1.4s
+; CHECK-NEXT:    sshl v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ret
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = ashr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+
+; This is not valid for funnel shifts
+declare i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+declare i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+define i32 @n6_fshl(i32 %x, i32 %y, i8 %shamt) nounwind {
+; CHECK-LABEL: n6_fshl:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w9, w2, #0x1f
+; CHECK-NEXT:    neg w9, w9
+; CHECK-NEXT:    lsl w8, w0, w2
+; CHECK-NEXT:    lsr w9, w1, w9
+; CHECK-NEXT:    orr w8, w8, w9
+; CHECK-NEXT:    tst w2, #0x1f
+; CHECK-NEXT:    csel w0, w0, w8, eq
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+define i32 @n7_fshr(i32 %x, i32 %y, i8 %shamt) nounwind {
+; CHECK-LABEL: n7_fshr:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    and w9, w2, #0x1f
+; CHECK-NEXT:    neg w9, w9
+; CHECK-NEXT:    lsr w8, w1, w2
+; CHECK-NEXT:    lsl w9, w0, w9
+; CHECK-NEXT:    orr w8, w9, w8
+; CHECK-NEXT:    tst w2, #0x1f
+; CHECK-NEXT:    csel w0, w1, w8, eq
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+
+define i32 @n8_extrause(i32 %x, i8 %shamt, i32* %shamt_wide_store) nounwind {
+; CHECK-LABEL: n8_extrause:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sxtb w8, w1
+; CHECK-NEXT:    lsl w0, w0, w8
+; CHECK-NEXT:    str w8, [x2]
+; CHECK-NEXT:    ret
+  %shamt_wide = sext i8 %shamt to i32
+  store i32 %shamt_wide, i32* %shamt_wide_store, align 4
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat.ll b/llvm/test/CodeGen/AArch64/ssub_sat.ll
index 9dc200523d2d3..593bc090c626b 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat.ll
@@ -54,18 +54,13 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: vec:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
 ; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
diff --git a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
index 3718b6bbc9de8..664793bba2776 100644
--- a/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/ssub_sat_vec.ll
@@ -37,18 +37,13 @@ define <16 x i8> @v16i8(<16 x i8> %x, <16 x i8> %y) nounwind {
 ; CHECK-LABEL: v16i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v3.16b, #127
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <16 x i8> @llvm.ssub.sat.v16i8(<16 x i8> %x, <16 x i8> %y)
@@ -59,31 +54,21 @@ define <32 x i8> @v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; CHECK-LABEL: v32i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmlt v16.16b, v4.16b, #0
+; CHECK-NEXT:    cmlt v7.16b, v4.16b, #0
 ; CHECK-NEXT:    movi v6.16b, #127
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    sub v7.16b, v1.16b, v3.16b
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmgt v2.16b, v2.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v4.16b
 ; CHECK-NEXT:    cmlt v16.16b, v7.16b, #0
 ; CHECK-NEXT:    movi v5.16b, #127
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v16.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v2.16b, v0.16b, v2.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v16.16b
-; CHECK-NEXT:    cmge v16.16b, v7.16b, #0
-; CHECK-NEXT:    cmeq v3.16b, v1.16b, v3.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v16.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmgt v3.16b, v3.16b, #0
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, v7.16b
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -106,46 +91,26 @@ define <64 x i8> @v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.16b, v21.16b, #0
+; CHECK-NEXT:    cmgt v4.16b, v4.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v16.16b
 ; CHECK-NEXT:    movi v22.16b, #127
 ; CHECK-NEXT:    sub v23.16b, v3.16b, v7.16b
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmgt v4.16b, v5.16b, #0
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, v19.16b
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.16b, v23.16b, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmgt v4.16b, v6.16b, #0
+; CHECK-NEXT:    cmgt v2.16b, v2.16b, v21.16b
 ; CHECK-NEXT:    movi v17.16b, #127
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmgt v4.16b, v7.16b, #0
+; CHECK-NEXT:    cmgt v3.16b, v3.16b, v23.16b
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.16b, v4.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v24.16b, v16.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v5.16b, #0
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmeq v4.16b, v0.16b, v4.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v19.16b, #0
-; CHECK-NEXT:    cmge v6.16b, v6.16b, #0
-; CHECK-NEXT:    cmge v2.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v5.16b, v1.16b, v5.16b
-; CHECK-NEXT:    cmeq v1.16b, v1.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v21.16b, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.16b, v7.16b, #0
-; CHECK-NEXT:    cmge v3.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v6.16b, v2.16b, v6.16b
-; CHECK-NEXT:    cmeq v2.16b, v2.16b, v24.16b
-; CHECK-NEXT:    cmge v24.16b, v23.16b, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.16b, v3.16b, v7.16b
-; CHECK-NEXT:    cmeq v3.16b, v3.16b, v24.16b
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -159,18 +124,13 @@ define <8 x i16> @v8i16(<8 x i16> %x, <8 x i16> %y) nounwind {
 ; CHECK-LABEL: v8i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v2.8h, #0
 ; CHECK-NEXT:    cmlt v4.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v5.8h
 ; CHECK-NEXT:    mvni v3.8h, #128, lsl #8
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v2.8h
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <8 x i16> @llvm.ssub.sat.v8i16(<8 x i16> %x, <8 x i16> %y)
@@ -181,31 +141,21 @@ define <16 x i16> @v16i16(<16 x i16> %x, <16 x i16> %y) nounwind {
 ; CHECK-LABEL: v16i16:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v4.8h, #0
+; CHECK-NEXT:    cmlt v7.8h, v4.8h, #0
 ; CHECK-NEXT:    mvni v6.8h, #128, lsl #8
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    sub v7.8h, v1.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmgt v2.8h, v2.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v4.8h
 ; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
 ; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v0.8h, v2.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v1.8h, v3.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmgt v3.8h, v3.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v7.8h
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -228,46 +178,26 @@ define <32 x i16> @v32i16(<32 x i16> %x, <32 x i16> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.8h, v21.8h, #0
+; CHECK-NEXT:    cmgt v4.8h, v4.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v16.8h
 ; CHECK-NEXT:    mvni v22.8h, #128, lsl #8
 ; CHECK-NEXT:    sub v23.8h, v3.8h, v7.8h
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmgt v4.8h, v5.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v19.8h
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.8h, v23.8h, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmgt v4.8h, v6.8h, #0
+; CHECK-NEXT:    cmgt v2.8h, v2.8h, v21.8h
 ; CHECK-NEXT:    mvni v17.8h, #128, lsl #8
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmgt v4.8h, v7.8h, #0
+; CHECK-NEXT:    cmgt v3.8h, v3.8h, v23.8h
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.8h, v4.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmge v24.8h, v16.8h, #0
-; CHECK-NEXT:    cmge v5.8h, v5.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmeq v4.8h, v0.8h, v4.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v19.8h, #0
-; CHECK-NEXT:    cmge v6.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmeq v5.8h, v1.8h, v5.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v21.8h, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.8h, v7.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmeq v6.8h, v2.8h, v6.8h
-; CHECK-NEXT:    cmeq v2.8h, v2.8h, v24.8h
-; CHECK-NEXT:    cmge v24.8h, v23.8h, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.8h, v3.8h, v7.8h
-; CHECK-NEXT:    cmeq v3.8h, v3.8h, v24.8h
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -284,17 +214,12 @@ define void @v8i8(<8 x i8>* %px, <8 x i8>* %py, <8 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldr d1, [x1]
 ; CHECK-NEXT:    movi v2.8b, #127
 ; CHECK-NEXT:    sub v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
 ; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
+; CHECK-NEXT:    cmgt v1.8b, v1.8b, #0
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, v3.8b
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
@@ -327,18 +252,13 @@ define void @v4i8(<4 x i8>* %px, <4 x i8>* %py, <4 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    shl v1.4h, v1.4h, #8
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
+; CHECK-NEXT:    cmgt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
@@ -365,18 +285,13 @@ define void @v2i8(<2 x i8>* %px, <2 x i8>* %py, <2 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #24
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #24
 ; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    cmgt v2.2s, v2.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v2.8b, v2.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #24
 ; CHECK-NEXT:    mov w8, v0.s[1]
@@ -398,17 +313,12 @@ define void @v4i16(<4 x i16>* %px, <4 x i16>* %py, <4 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldr d1, [x1]
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    cmgt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str d0, [x2]
 ; CHECK-NEXT:    ret
@@ -433,18 +343,13 @@ define void @v2i16(<2 x i16>* %px, <2 x i16>* %py, <2 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    shl v2.2s, v2.2s, #16
 ; CHECK-NEXT:    shl v0.2s, v0.2s, #16
 ; CHECK-NEXT:    sub v3.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmge v2.2s, v2.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v3.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v3.2s, #0
-; CHECK-NEXT:    cmeq v2.2s, v0.2s, v2.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v1.2s, #128, lsl #24
+; CHECK-NEXT:    cmgt v2.2s, v2.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v3.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v2.8b, v2.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v1.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v2.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v1.8b, v3.8b
 ; CHECK-NEXT:    ushr v0.2s, v0.2s, #16
 ; CHECK-NEXT:    mov w8, v0.s[1]
@@ -463,18 +368,13 @@ define <12 x i8> @v12i8(<12 x i8> %x, <12 x i8> %y) nounwind {
 ; CHECK-LABEL: v12i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v2.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v2.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v3.16b, #127
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v2.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <12 x i8> @llvm.ssub.sat.v12i8(<12 x i8> %x, <12 x i8> %y)
@@ -489,29 +389,19 @@ define void @v12i16(<12 x i16>* %px, <12 x i16>* %py, <12 x i16>* %pz) nounwind
 ; CHECK-NEXT:    mvni v5.8h, #128, lsl #8
 ; CHECK-NEXT:    mvni v4.8h, #128, lsl #8
 ; CHECK-NEXT:    sub v6.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmlt v16.8h, v6.8h, #0
+; CHECK-NEXT:    cmlt v7.8h, v6.8h, #0
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v5.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    sub v7.8h, v0.8h, v3.8h
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmgt v2.8h, v2.8h, #0
+; CHECK-NEXT:    cmgt v1.8h, v1.8h, v6.8h
 ; CHECK-NEXT:    cmlt v16.8h, v7.8h, #0
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v4.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.8h, v2.8h, #0
-; CHECK-NEXT:    cmge v1.8h, v1.8h, #0
-; CHECK-NEXT:    cmge v16.8h, v6.8h, #0
-; CHECK-NEXT:    cmge v3.8h, v3.8h, #0
-; CHECK-NEXT:    cmge v0.8h, v0.8h, #0
-; CHECK-NEXT:    cmeq v2.8h, v1.8h, v2.8h
-; CHECK-NEXT:    cmeq v1.8h, v1.8h, v16.8h
-; CHECK-NEXT:    cmge v16.8h, v7.8h, #0
-; CHECK-NEXT:    cmeq v3.8h, v0.8h, v3.8h
-; CHECK-NEXT:    cmeq v0.8h, v0.8h, v16.8h
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v2.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    cmgt v3.8h, v3.8h, #0
+; CHECK-NEXT:    cmgt v0.8h, v0.8h, v7.8h
+; CHECK-NEXT:    eor v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bsl v4.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v6.16b
 ; CHECK-NEXT:    bsl v0.16b, v4.16b, v7.16b
 ; CHECK-NEXT:    str q0, [x2]
@@ -531,17 +421,12 @@ define void @v1i8(<1 x i8>* %px, <1 x i8>* %py, <1 x i8>* %pz) nounwind {
 ; CHECK-NEXT:    ldr b1, [x1]
 ; CHECK-NEXT:    movi v2.8b, #127
 ; CHECK-NEXT:    sub v3.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmge v1.8b, v1.8b, #0
-; CHECK-NEXT:    cmge v0.8b, v0.8b, #0
-; CHECK-NEXT:    cmge v5.8b, v3.8b, #0
 ; CHECK-NEXT:    cmlt v4.8b, v3.8b, #0
-; CHECK-NEXT:    cmeq v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    cmeq v0.8b, v0.8b, v5.8b
+; CHECK-NEXT:    cmgt v1.8b, v1.8b, #0
+; CHECK-NEXT:    cmgt v0.8b, v0.8b, v3.8b
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    st1 { v0.b }[0], [x2]
 ; CHECK-NEXT:    ret
@@ -559,17 +444,12 @@ define void @v1i16(<1 x i16>* %px, <1 x i16>* %py, <1 x i16>* %pz) nounwind {
 ; CHECK-NEXT:    ldr h1, [x1]
 ; CHECK-NEXT:    mvni v2.4h, #128, lsl #8
 ; CHECK-NEXT:    sub v3.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmge v1.4h, v1.4h, #0
-; CHECK-NEXT:    cmge v0.4h, v0.4h, #0
-; CHECK-NEXT:    cmge v5.4h, v3.4h, #0
 ; CHECK-NEXT:    cmlt v4.4h, v3.4h, #0
-; CHECK-NEXT:    cmeq v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    cmeq v0.4h, v0.4h, v5.4h
+; CHECK-NEXT:    cmgt v1.4h, v1.4h, #0
+; CHECK-NEXT:    cmgt v0.4h, v0.4h, v3.4h
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v2.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v2.8b, v3.8b
 ; CHECK-NEXT:    str h0, [x2]
 ; CHECK-NEXT:    ret
@@ -586,18 +466,13 @@ define <16 x i4> @v16i4(<16 x i4> %x, <16 x i4> %y) nounwind {
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #4
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #4
 ; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v2.16b, #127
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #4
 ; CHECK-NEXT:    ret
@@ -611,18 +486,13 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 ; CHECK-NEXT:    shl v1.16b, v1.16b, #7
 ; CHECK-NEXT:    shl v0.16b, v0.16b, #7
 ; CHECK-NEXT:    sub v3.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmge v1.16b, v1.16b, #0
-; CHECK-NEXT:    cmge v0.16b, v0.16b, #0
-; CHECK-NEXT:    cmge v5.16b, v3.16b, #0
 ; CHECK-NEXT:    cmlt v4.16b, v3.16b, #0
-; CHECK-NEXT:    cmeq v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    cmeq v0.16b, v0.16b, v5.16b
 ; CHECK-NEXT:    movi v2.16b, #127
+; CHECK-NEXT:    cmgt v1.16b, v1.16b, #0
+; CHECK-NEXT:    cmgt v0.16b, v0.16b, v3.16b
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v2.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v2.16b, v3.16b
 ; CHECK-NEXT:    sshr v0.16b, v0.16b, #7
 ; CHECK-NEXT:    ret
@@ -634,18 +504,13 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; CHECK-LABEL: v2i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmge v1.2s, v1.2s, #0
-; CHECK-NEXT:    cmge v0.2s, v0.2s, #0
-; CHECK-NEXT:    cmge v5.2s, v2.2s, #0
 ; CHECK-NEXT:    cmlt v4.2s, v2.2s, #0
-; CHECK-NEXT:    cmeq v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    cmeq v0.2s, v0.2s, v5.2s
 ; CHECK-NEXT:    mvni v3.2s, #128, lsl #24
+; CHECK-NEXT:    cmgt v1.2s, v1.2s, #0
+; CHECK-NEXT:    cmgt v0.2s, v0.2s, v2.2s
 ; CHECK-NEXT:    mvn v5.8b, v4.8b
-; CHECK-NEXT:    mvn v1.8b, v1.8b
-; CHECK-NEXT:    mvn v0.8b, v0.8b
 ; CHECK-NEXT:    bsl v3.8b, v4.8b, v5.8b
-; CHECK-NEXT:    and v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    eor v0.8b, v1.8b, v0.8b
 ; CHECK-NEXT:    bsl v0.8b, v3.8b, v2.8b
 ; CHECK-NEXT:    ret
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
@@ -656,18 +521,13 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; CHECK-LABEL: v4i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v2.4s, #0
 ; CHECK-NEXT:    cmlt v4.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v5.4s
 ; CHECK-NEXT:    mvni v3.4s, #128, lsl #24
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v2.4s
 ; CHECK-NEXT:    mvn v5.16b, v4.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v3.16b, v4.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v3.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
@@ -678,31 +538,21 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; CHECK-LABEL: v8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmlt v16.4s, v4.4s, #0
+; CHECK-NEXT:    cmlt v7.4s, v4.4s, #0
 ; CHECK-NEXT:    mvni v6.4s, #128, lsl #24
+; CHECK-NEXT:    mvn v16.16b, v7.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v16.16b
 ; CHECK-NEXT:    sub v7.4s, v1.4s, v3.4s
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v6.16b, v16.16b, v17.16b
+; CHECK-NEXT:    cmgt v2.4s, v2.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v4.4s
 ; CHECK-NEXT:    cmlt v16.4s, v7.4s, #0
 ; CHECK-NEXT:    mvni v5.4s, #128, lsl #24
-; CHECK-NEXT:    mvn v17.16b, v16.16b
-; CHECK-NEXT:    bsl v5.16b, v16.16b, v17.16b
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v16.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v2.4s, v0.4s, v2.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    cmge v16.4s, v7.4s, #0
-; CHECK-NEXT:    cmeq v3.4s, v1.4s, v3.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v16.4s
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    cmgt v3.4s, v3.4s, #0
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, v7.4s
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v16.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v5.16b, v16.16b, v2.16b
 ; CHECK-NEXT:    bsl v0.16b, v6.16b, v4.16b
 ; CHECK-NEXT:    bsl v1.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    ret
@@ -725,46 +575,26 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
 ; CHECK-NEXT:    bsl v20.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.4s, v21.4s, #0
+; CHECK-NEXT:    cmgt v4.4s, v4.4s, #0
+; CHECK-NEXT:    cmgt v0.4s, v0.4s, v16.4s
 ; CHECK-NEXT:    mvni v22.4s, #128, lsl #24
 ; CHECK-NEXT:    sub v23.4s, v3.4s, v7.4s
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmgt v4.4s, v5.4s, #0
+; CHECK-NEXT:    cmgt v1.4s, v1.4s, v19.4s
 ; CHECK-NEXT:    bsl v22.16b, v24.16b, v25.16b
 ; CHECK-NEXT:    cmlt v24.4s, v23.4s, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmgt v4.4s, v6.4s, #0
+; CHECK-NEXT:    cmgt v2.4s, v2.4s, v21.4s
 ; CHECK-NEXT:    mvni v17.4s, #128, lsl #24
 ; CHECK-NEXT:    mvn v25.16b, v24.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmgt v4.4s, v7.4s, #0
+; CHECK-NEXT:    cmgt v3.4s, v3.4s, v23.4s
 ; CHECK-NEXT:    bsl v17.16b, v24.16b, v25.16b
-; CHECK-NEXT:    cmge v4.4s, v4.4s, #0
-; CHECK-NEXT:    cmge v0.4s, v0.4s, #0
-; CHECK-NEXT:    cmge v24.4s, v16.4s, #0
-; CHECK-NEXT:    cmge v5.4s, v5.4s, #0
-; CHECK-NEXT:    cmge v1.4s, v1.4s, #0
-; CHECK-NEXT:    cmeq v4.4s, v0.4s, v4.4s
-; CHECK-NEXT:    cmeq v0.4s, v0.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v19.4s, #0
-; CHECK-NEXT:    cmge v6.4s, v6.4s, #0
-; CHECK-NEXT:    cmge v2.4s, v2.4s, #0
-; CHECK-NEXT:    cmeq v5.4s, v1.4s, v5.4s
-; CHECK-NEXT:    cmeq v1.4s, v1.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v21.4s, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.4s, v7.4s, #0
-; CHECK-NEXT:    cmge v3.4s, v3.4s, #0
-; CHECK-NEXT:    cmeq v6.4s, v2.4s, v6.4s
-; CHECK-NEXT:    cmeq v2.4s, v2.4s, v24.4s
-; CHECK-NEXT:    cmge v24.4s, v23.4s, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.4s, v3.4s, v7.4s
-; CHECK-NEXT:    cmeq v3.4s, v3.4s, v24.4s
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v18.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v20.16b, v19.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v21.16b
@@ -778,19 +608,14 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; CHECK-LABEL: v2i64:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v2.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v2.2d, #0
 ; CHECK-NEXT:    mov x8, #9223372036854775807
 ; CHECK-NEXT:    cmlt v3.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v5.2d
+; CHECK-NEXT:    cmgt v1.2d, v1.2d, #0
 ; CHECK-NEXT:    dup v4.2d, x8
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    mvn v5.16b, v3.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
 ; CHECK-NEXT:    bsl v4.16b, v3.16b, v5.16b
-; CHECK-NEXT:    and v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    eor v0.16b, v1.16b, v0.16b
 ; CHECK-NEXT:    bsl v0.16b, v4.16b, v2.16b
 ; CHECK-NEXT:    ret
   %z = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
@@ -802,33 +627,23 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub v4.2d, v0.2d, v2.2d
 ; CHECK-NEXT:    mov x8, #9223372036854775807
-; CHECK-NEXT:    cmlt v6.2d, v4.2d, #0
-; CHECK-NEXT:    dup v7.2d, x8
+; CHECK-NEXT:    cmlt v5.2d, v4.2d, #0
+; CHECK-NEXT:    dup v6.2d, x8
+; CHECK-NEXT:    mvn v7.16b, v5.16b
+; CHECK-NEXT:    mov v16.16b, v6.16b
+; CHECK-NEXT:    bsl v16.16b, v5.16b, v7.16b
 ; CHECK-NEXT:    sub v5.2d, v1.2d, v3.2d
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    mov v17.16b, v7.16b
-; CHECK-NEXT:    bsl v17.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmlt v6.2d, v5.2d, #0
-; CHECK-NEXT:    mvn v16.16b, v6.16b
-; CHECK-NEXT:    bsl v7.16b, v6.16b, v16.16b
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v2.2d, v0.2d, v2.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v6.2d
-; CHECK-NEXT:    cmge v6.2d, v5.2d, #0
-; CHECK-NEXT:    cmeq v3.2d, v1.2d, v3.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v6.2d
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    and v0.16b, v2.16b, v0.16b
-; CHECK-NEXT:    and v1.16b, v3.16b, v1.16b
-; CHECK-NEXT:    bsl v0.16b, v17.16b, v4.16b
-; CHECK-NEXT:    bsl v1.16b, v7.16b, v5.16b
+; CHECK-NEXT:    cmgt v2.2d, v2.2d, #0
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v4.2d
+; CHECK-NEXT:    cmlt v7.2d, v5.2d, #0
+; CHECK-NEXT:    cmgt v3.2d, v3.2d, #0
+; CHECK-NEXT:    cmgt v1.2d, v1.2d, v5.2d
+; CHECK-NEXT:    eor v0.16b, v2.16b, v0.16b
+; CHECK-NEXT:    mvn v2.16b, v7.16b
+; CHECK-NEXT:    eor v1.16b, v3.16b, v1.16b
+; CHECK-NEXT:    bsl v6.16b, v7.16b, v2.16b
+; CHECK-NEXT:    bsl v0.16b, v16.16b, v4.16b
+; CHECK-NEXT:    bsl v1.16b, v6.16b, v5.16b
 ; CHECK-NEXT:    ret
   %z = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -850,46 +665,26 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; CHECK-NEXT:    bsl v25.16b, v20.16b, v24.16b
 ; CHECK-NEXT:    mvn v20.16b, v22.16b
 ; CHECK-NEXT:    mov v24.16b, v21.16b
+; CHECK-NEXT:    cmgt v4.2d, v4.2d, #0
+; CHECK-NEXT:    cmgt v0.2d, v0.2d, v16.2d
 ; CHECK-NEXT:    sub v19.2d, v3.2d, v7.2d
 ; CHECK-NEXT:    bsl v24.16b, v22.16b, v20.16b
 ; CHECK-NEXT:    mvn v20.16b, v23.16b
 ; CHECK-NEXT:    mov v22.16b, v21.16b
+; CHECK-NEXT:    eor v0.16b, v4.16b, v0.16b
+; CHECK-NEXT:    cmgt v4.2d, v5.2d, #0
+; CHECK-NEXT:    cmgt v1.2d, v1.2d, v17.2d
 ; CHECK-NEXT:    bsl v22.16b, v23.16b, v20.16b
 ; CHECK-NEXT:    cmlt v20.2d, v19.2d, #0
+; CHECK-NEXT:    eor v1.16b, v4.16b, v1.16b
+; CHECK-NEXT:    cmgt v4.2d, v6.2d, #0
+; CHECK-NEXT:    cmgt v2.2d, v2.2d, v18.2d
 ; CHECK-NEXT:    mvn v23.16b, v20.16b
+; CHECK-NEXT:    eor v2.16b, v4.16b, v2.16b
+; CHECK-NEXT:    cmgt v4.2d, v7.2d, #0
+; CHECK-NEXT:    cmgt v3.2d, v3.2d, v19.2d
 ; CHECK-NEXT:    bsl v21.16b, v20.16b, v23.16b
-; CHECK-NEXT:    cmge v4.2d, v4.2d, #0
-; CHECK-NEXT:    cmge v0.2d, v0.2d, #0
-; CHECK-NEXT:    cmge v20.2d, v16.2d, #0
-; CHECK-NEXT:    cmge v5.2d, v5.2d, #0
-; CHECK-NEXT:    cmge v1.2d, v1.2d, #0
-; CHECK-NEXT:    cmeq v4.2d, v0.2d, v4.2d
-; CHECK-NEXT:    cmeq v0.2d, v0.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v17.2d, #0
-; CHECK-NEXT:    cmge v6.2d, v6.2d, #0
-; CHECK-NEXT:    cmge v2.2d, v2.2d, #0
-; CHECK-NEXT:    cmeq v5.2d, v1.2d, v5.2d
-; CHECK-NEXT:    cmeq v1.2d, v1.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v18.2d, #0
-; CHECK-NEXT:    mvn v4.16b, v4.16b
-; CHECK-NEXT:    mvn v0.16b, v0.16b
-; CHECK-NEXT:    cmge v7.2d, v7.2d, #0
-; CHECK-NEXT:    cmge v3.2d, v3.2d, #0
-; CHECK-NEXT:    cmeq v6.2d, v2.2d, v6.2d
-; CHECK-NEXT:    cmeq v2.2d, v2.2d, v20.2d
-; CHECK-NEXT:    cmge v20.2d, v19.2d, #0
-; CHECK-NEXT:    and v0.16b, v4.16b, v0.16b
-; CHECK-NEXT:    mvn v4.16b, v5.16b
-; CHECK-NEXT:    mvn v1.16b, v1.16b
-; CHECK-NEXT:    cmeq v7.2d, v3.2d, v7.2d
-; CHECK-NEXT:    cmeq v3.2d, v3.2d, v20.2d
-; CHECK-NEXT:    and v1.16b, v4.16b, v1.16b
-; CHECK-NEXT:    mvn v4.16b, v6.16b
-; CHECK-NEXT:    mvn v2.16b, v2.16b
-; CHECK-NEXT:    and v2.16b, v4.16b, v2.16b
-; CHECK-NEXT:    mvn v4.16b, v7.16b
-; CHECK-NEXT:    mvn v3.16b, v3.16b
-; CHECK-NEXT:    and v3.16b, v4.16b, v3.16b
+; CHECK-NEXT:    eor v3.16b, v4.16b, v3.16b
 ; CHECK-NEXT:    bsl v0.16b, v25.16b, v16.16b
 ; CHECK-NEXT:    bsl v1.16b, v24.16b, v17.16b
 ; CHECK-NEXT:    bsl v2.16b, v22.16b, v18.16b
diff --git a/llvm/test/CodeGen/AArch64/stgp.ll b/llvm/test/CodeGen/AArch64/stgp.ll
index b4af16fd9ff79..d82b45134f552 100644
--- a/llvm/test/CodeGen/AArch64/stgp.ll
+++ b/llvm/test/CodeGen/AArch64/stgp.ll
@@ -65,7 +65,7 @@ entry:
 define void @stgp_alloca(i64 %a, i64 %b) {
 entry:
 ; CHECK-LABEL: stgp_alloca:
-; CHECK: stgp x0, x1, [sp]
+; CHECK: stgp x0, x1, [sp, #-32]!
 ; CHECK: stgp x1, x0, [sp, #16]
 ; CHECK: ret
   %x = alloca i8, i32 32, align 16
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
new file mode 100644
index 0000000000000..2350353a27424
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-counting-bits.ll
@@ -0,0 +1,83 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; CNT
+;
+
+define <vscale x 16 x i8> @cnt_i8(<vscale x 16 x i8> %a, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) {
+; CHECK-LABEL: cnt_i8:
+; CHECK: cnt z0.b, p0/m, z1.b
+; CHECK-NEXT: ret
+  %out = call <vscale x 16 x i8> @llvm.aarch64.sve.cnt.nxv16i8(<vscale x 16 x i8> %a,
+                                                               <vscale x 16 x i1> %pg,
+                                                               <vscale x 16 x i8> %b)
+  ret <vscale x 16 x i8> %out
+}
+
+define <vscale x 8 x i16> @cnt_i16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) {
+; CHECK-LABEL: cnt_i16:
+; CHECK: cnt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8i16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x i16> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cnt_i32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) {
+; CHECK-LABEL: cnt_i32:
+; CHECK: cnt z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4i32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x i32> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cnt_i64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) {
+; CHECK-LABEL: cnt_i64:
+; CHECK: cnt z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2i64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x i64> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+define <vscale x 8 x i16> @cnt_f16(<vscale x 8 x i16> %a, <vscale x 8 x i1> %pg, <vscale x 8 x half> %b) {
+; CHECK-LABEL: cnt_f16:
+; CHECK: cnt z0.h, p0/m, z1.h
+; CHECK-NEXT: ret
+  %out = call <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8f16(<vscale x 8 x i16> %a,
+                                                               <vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x half> %b)
+  ret <vscale x 8 x i16> %out
+}
+
+define <vscale x 4 x i32> @cnt_f32(<vscale x 4 x i32> %a, <vscale x 4 x i1> %pg, <vscale x 4 x float> %b) {
+; CHECK-LABEL: cnt_f32:
+; CHECK: cnt z0.s, p0/m, z1.s
+; CHECK-NEXT: ret
+  %out = call <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4f32(<vscale x 4 x i32> %a,
+                                                               <vscale x 4 x i1> %pg,
+                                                               <vscale x 4 x float> %b)
+  ret <vscale x 4 x i32> %out
+}
+
+define <vscale x 2 x i64> @cnt_f64(<vscale x 2 x i64> %a, <vscale x 2 x i1> %pg, <vscale x 2 x double> %b) {
+; CHECK-LABEL: cnt_f64:
+; CHECK: cnt z0.d, p0/m, z1.d
+; CHECK-NEXT: ret
+  %out = call <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2f64(<vscale x 2 x i64> %a,
+                                                               <vscale x 2 x i1> %pg,
+                                                               <vscale x 2 x double> %b)
+  ret <vscale x 2 x i64> %out
+}
+
+declare <vscale x 16 x i8> @llvm.aarch64.sve.cnt.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8i16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4i32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2i64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <vscale x 8 x i16> @llvm.aarch64.sve.cnt.nxv8f16(<vscale x 8 x i16>, <vscale x 8 x i1>, <vscale x 8 x half>)
+declare <vscale x 4 x i32> @llvm.aarch64.sve.cnt.nxv4f32(<vscale x 4 x i32>, <vscale x 4 x i1>, <vscale x 4 x float>)
+declare <vscale x 2 x i64> @llvm.aarch64.sve.cnt.nxv2f64(<vscale x 2 x i64>, <vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll
new file mode 100644
index 0000000000000..d918d432ae6a8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-pred-operations.ll
@@ -0,0 +1,65 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+;
+; PUNPKHI
+;
+
+define <vscale x 8 x i1> @punpkhi_b16(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: punpkhi_b16
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.punpkhi.nxv8i1(<vscale x 16 x i1> %a)
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 4 x i1> @punpkhi_b8(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: punpkhi_b8
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.punpkhi.nxv4i1(<vscale x 8 x i1> %a)
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 2 x i1> @punpkhi_b4(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: punpkhi_b4
+; CHECK: punpkhi p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.punpkhi.nxv2i1(<vscale x 4 x i1> %a)
+  ret <vscale x 2 x i1> %res
+}
+
+;
+; PUNPKLO
+;
+
+define <vscale x 8 x i1> @punpklo_b16(<vscale x 16 x i1> %a) {
+; CHECK-LABEL: punpklo_b16
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 8 x i1> @llvm.aarch64.sve.punpklo.nxv8i1(<vscale x 16 x i1> %a)
+  ret <vscale x 8 x i1> %res
+}
+
+define <vscale x 4 x i1> @punpklo_b8(<vscale x 8 x i1> %a) {
+; CHECK-LABEL: punpklo_b8
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 4 x i1> @llvm.aarch64.sve.punpklo.nxv4i1(<vscale x 8 x i1> %a)
+  ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 2 x i1> @punpklo_b4(<vscale x 4 x i1> %a) {
+; CHECK-LABEL: punpklo_b4
+; CHECK: punpklo p0.h, p0.b
+; CHECK-NEXT: ret
+  %res = call <vscale x 2 x i1> @llvm.aarch64.sve.punpklo.nxv2i1(<vscale x 4 x i1> %a)
+  ret <vscale x 2 x i1> %res
+}
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.punpkhi.nxv8i1(<vscale x 16 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.punpkhi.nxv4i1(<vscale x 8 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.punpkhi.nxv2i1(<vscale x 4 x i1>)
+
+declare <vscale x 8 x i1> @llvm.aarch64.sve.punpklo.nxv8i1(<vscale x 16 x i1>)
+declare <vscale x 4 x i1> @llvm.aarch64.sve.punpklo.nxv4i1(<vscale x 8 x i1>)
+declare <vscale x 2 x i1> @llvm.aarch64.sve.punpklo.nxv2i1(<vscale x 4 x i1>)
diff --git a/llvm/test/CodeGen/AArch64/tail-call.ll b/llvm/test/CodeGen/AArch64/tail-call.ll
index fafad4b0c580f..f1c5daf86c9ef 100644
--- a/llvm/test/CodeGen/AArch64/tail-call.ll
+++ b/llvm/test/CodeGen/AArch64/tail-call.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=SDAG,COMMON
-; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=GISEL,COMMON
+; RUN: llc -global-isel -global-isel-abort=1 -verify-machineinstrs < %s -mtriple=aarch64-none-linux-gnu -tailcallopt | FileCheck %s --check-prefixes=GISEL,COMMON
 
 declare fastcc void @callee_stack0()
 declare fastcc void @callee_stack8([8 x i64], i64)
diff --git a/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll b/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
index c780d15b58db0..8fd302fe553c1 100644
--- a/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/AArch64/tailcall-mem-intrinsics.ll
@@ -1,4 +1,5 @@
 ; RUN: llc -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+; RUN: llc -global-isel-abort=1 -verify-machineinstrs -mtriple=aarch64-unknown-unknown -global-isel < %s | FileCheck %s
 
 ; CHECK-LABEL: tail_memcpy:
 ; CHECK: b memcpy
diff --git a/llvm/test/CodeGen/AArch64/vararg-tallcall.ll b/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
index 56c56213af354..aa239a8d57ec9 100644
--- a/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
+++ b/llvm/test/CodeGen/AArch64/vararg-tallcall.ll
@@ -1,5 +1,7 @@
 ; RUN: llc -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
 ; RUN: llc -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-windows-msvc %s -o - | FileCheck %s
+; RUN: llc -global-isel -global-isel-abort=2 -verify-machineinstrs -mtriple=aarch64-linux-gnu %s -o - | FileCheck %s
 
 target datalayout = "e-m:w-p:64:64-i32:32-i64:64-i128:128-n32:64-S128"
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
new file mode 100644
index 0000000000000..fac8511c0f620
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/global-value.ll
@@ -0,0 +1,156 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -stop-after=legalizer < %s | FileCheck -check-prefix=HSA %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -mcpu=hawaii -stop-after=legalizer < %s | FileCheck -check-prefix=PAL %s
+
+@external_constant = external addrspace(4) constant i32, align 4
+@external_constant32 = external addrspace(6) constant i32, align 4
+@external_global = external addrspace(1) global i32, align 4
+
+@internal_constant = internal addrspace(4) constant i32 9, align 4
+@internal_constant32 = internal addrspace(6) constant i32 9, align 4
+@internal_global = internal addrspace(1) global i32 9, align 4
+
+
+define i32 addrspace(4)* @external_constant_got() {
+  ; HSA-LABEL: name: external_constant_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant + 4, target-flags(amdgpu-gotprel32-hi) @external_constant + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p4)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: external_constant_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @external_constant + 4, 0, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(4)* @external_constant
+}
+
+define i32 addrspace(1)* @external_global_got() {
+  ; HSA-LABEL: name: external_global_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_global + 4, target-flags(amdgpu-gotprel32-hi) @external_global + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p1)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: external_global_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_global + 4, target-flags(amdgpu-gotprel32-hi) @external_global + 4, implicit-def $scc
+  ; PAL:   [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p1)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(1)* @external_global
+}
+
+define i32 addrspace(4)* @internal_constant_pcrel() {
+  ; HSA-LABEL: name: internal_constant_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_constant + 4, target-flags(amdgpu-rel32-hi) @internal_constant + 4, implicit-def $scc
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: internal_constant_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @internal_constant + 4, 0, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p4)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(4)* @internal_constant
+}
+
+define i32 addrspace(1)* @internal_global_pcrel() {
+  ; HSA-LABEL: name: internal_global_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p1) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_global + 4, target-flags(amdgpu-rel32-hi) @internal_global + 4, implicit-def $scc
+  ; HSA:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p1)
+  ; HSA:   $vgpr0 = COPY [[UV]](s32)
+  ; HSA:   $vgpr1 = COPY [[UV1]](s32)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ; PAL-LABEL: name: internal_global_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p1) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_global + 4, target-flags(amdgpu-rel32-hi) @internal_global + 4, implicit-def $scc
+  ; PAL:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[SI_PC_ADD_REL_OFFSET]](p1)
+  ; PAL:   $vgpr0 = COPY [[UV]](s32)
+  ; PAL:   $vgpr1 = COPY [[UV1]](s32)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0, implicit $vgpr1
+  ret i32 addrspace(1)* @internal_global
+}
+
+define i32 addrspace(6)* @external_constant32_got() {
+  ; HSA-LABEL: name: external_constant32_got
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @external_constant32 + 4, target-flags(amdgpu-gotprel32-hi) @external_constant32 + 4, implicit-def $scc
+  ; HSA:   [[LOAD:%[0-9]+]]:_(p4) = G_LOAD [[SI_PC_ADD_REL_OFFSET]](p4) :: (dereferenceable invariant load 8 from got, addrspace 4)
+  ; HSA:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[LOAD]](p4), 0
+  ; HSA:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ; PAL-LABEL: name: external_constant32_got
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @external_constant32 + 4, 0, implicit-def $scc
+  ; PAL:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; PAL:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ret i32 addrspace(6)* @external_constant32
+}
+
+define i32 addrspace(6)* @internal_constant32_pcrel() {
+  ; HSA-LABEL: name: internal_constant32_pcrel
+  ; HSA: bb.1 (%ir-block.0):
+  ; HSA:   liveins: $sgpr30_sgpr31
+  ; HSA:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; HSA:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-rel32-lo) @internal_constant32 + 4, target-flags(amdgpu-rel32-hi) @internal_constant32 + 4, implicit-def $scc
+  ; HSA:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; HSA:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; HSA:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; HSA:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ; PAL-LABEL: name: internal_constant32_pcrel
+  ; PAL: bb.1 (%ir-block.0):
+  ; PAL:   liveins: $sgpr30_sgpr31
+  ; PAL:   [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr30_sgpr31
+  ; PAL:   [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64(p4) = SI_PC_ADD_REL_OFFSET @internal_constant32 + 4, 0, implicit-def $scc
+  ; PAL:   [[EXTRACT:%[0-9]+]]:_(p6) = G_EXTRACT [[SI_PC_ADD_REL_OFFSET]](p4), 0
+  ; PAL:   $vgpr0 = COPY [[EXTRACT]](p6)
+  ; PAL:   [[COPY1:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
+  ; PAL:   S_SETPC_B64_return [[COPY1]], implicit $vgpr0
+  ret i32 addrspace(6)* @internal_constant32
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.exp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.exp.mir
index 071625f98c0ff..99b28f382c07a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.exp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.exp.mir
@@ -10,24 +10,20 @@ body: |
   bb.0:
     liveins: $vgpr0
     %0:vgpr(s32) = COPY $vgpr0
-    %1:sgpr(s32) = G_CONSTANT i32 1
-    %2:sgpr(s32) = G_CONSTANT i32 15
-    %3:sgpr(s1)  = G_CONSTANT i1 0
-    %4:sgpr(s1)  = G_CONSTANT i1 1
 
     ; CHECK: EXP 1, %0, %0, %0, %0, 0, 0, 15, implicit $exec
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %1:sgpr(s32), %2:sgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %3:sgpr(s1), %3:sgpr(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp),1, 15, %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), 0, 0
 
     ; CHECK: EXP_DONE 1, %0, %0, %0, %0, 0, 0, 15, implicit $exec
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %1:sgpr(s32), %2:sgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %4:sgpr(s1), %3:sgpr(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 1, 15, %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), %0:vgpr(s32), 1, 0
 
     %5:vgpr(<2 x s16>) = G_BITCAST %0(s32)
 
     ; CHECK: [[UNDEF0:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; CHECK: EXP 1, %0, %0, [[UNDEF0]], [[UNDEF0]], 0, 1, 15, implicit $exec
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.compr), %1:sgpr(s32), %2:sgpr(s32), %5:vgpr(<2 x s16>), %5:vgpr(<2 x s16>), %3:sgpr(s1), %3:sgpr(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.compr), 1, 15, %5:vgpr(<2 x s16>), %5:vgpr(<2 x s16>), 0, 0
 
     ; CHECK: [[UNDEF1:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
     ; CHECK: EXP_DONE 1, %0, %0, [[UNDEF1]], [[UNDEF1]], 0, 1, 15, implicit $exec
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.compr), %1:sgpr(s32), %2:sgpr(s32), %5:vgpr(<2 x s16>), %5:vgpr(<2 x s16>), %4:sgpr(s1), %3:sgpr(s1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.compr), 1, 15, %5:vgpr(<2 x s16>), %5:vgpr(<2 x s16>), 1, 0
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ldexp.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ldexp.s16.mir
index 55600aca5d86d..cd9df36f27563 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ldexp.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.ldexp.s16.mir
@@ -1,10 +1,10 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
-# XUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=2 -pass-remarks-missed='gisel*'  %s -o /dev/null 2>&1 | FileCheck -check-prefix=SI-ERR %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=2 -pass-remarks-missed='gisel*'  %s -o /dev/null 2>&1 | FileCheck -check-prefix=SI-ERR %s
 
 # SI-ERR: remark: <unknown>:0:0: cannot select: %3:vgpr(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.ldexp), %2:sgpr(s16), %1:vgpr(s32) (in function: ldexp_s16_vsv)
 # SI-ERR-NEXT: remark: <unknown>:0:0: cannot select: %3:vgpr(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.ldexp), %2:vgpr(s16), %1:sgpr(s32) (in function: ldexp_s16_vvs)
-# SI-ERR-NEXT: remark: <unknown>:0:0: cannot select: %3:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ldexp), %2:vgpr(s16), %1:vgpr(s32) (in function: ldexp_s16_vvv)
+# SI-ERR-NEXT: remark: <unknown>:0:0: cannot select: %3:vgpr(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.ldexp), %2:vgpr(s16), %1:vgpr(s32) (in function: ldexp_s16_vvv)
 
 ---
 name: ldexp_s16_vsv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.s.sendmsg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.s.sendmsg.mir
index 1339d3dd56cae..4129e7be49f6e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.s.sendmsg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-amdgcn.s.sendmsg.mir
@@ -18,8 +18,7 @@ body:             |
     ; GCN: S_SENDMSG 1, implicit $exec, implicit $m0
     ; GCN: S_ENDPGM 0
     %0:sgpr(s32) = COPY $sgpr0
-    %2:sgpr(s32) = G_CONSTANT i32 1
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.s.sendmsg), %2(s32), %0(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.s.sendmsg), 1, %0(s32)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
index 361486c1051ce..e449830e8fe48 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-and.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr="+wavefrontsize32" -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s  | FileCheck -check-prefix=WAVE32 %s
 
 ---
@@ -148,12 +149,10 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: and_s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; WAVE64: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; WAVE64: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
-    ; WAVE64: [[AND:%[0-9]+]]:vgpr(s16) = G_AND [[TRUNC]], [[TRUNC1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[AND]](s16)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     ; WAVE32-LABEL: name: and_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
@@ -355,16 +354,17 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: and_v2s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE64: [[AND:%[0-9]+]]:vgpr(<2 x s16>) = G_AND [[COPY]], [[COPY1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[AND]](<2 x s16>)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     ; WAVE32-LABEL: name: and_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE32: [[AND:%[0-9]+]]:vgpr(<2 x s16>) = G_AND [[COPY]], [[COPY1]]
-    ; WAVE32: S_ENDPGM 0, implicit [[AND]](<2 x s16>)
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE32: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE32: S_ENDPGM 0, implicit [[V_AND_B32_e64_]]
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(<2 x s16>) = COPY $vgpr1
     %2:vgpr(<2 x s16>) = G_AND %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
index d1862a18b4ce8..73fbe56009299 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-atomicrmw-fadd-local.mir
@@ -1,9 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
 # RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
-# XUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
 # RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
-# XUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX9 %s
 
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
index 5f4a1b441e5ee..13e5c94d892d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-extract.mir
@@ -1,46 +1,47 @@
-# RUN: llc -march=amdgcn -run-pass=instruction-select -verify-machineinstrs -global-isel %s -o - | FileCheck %s
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck %s
 ---
 name:            extract512
 legalized:       true
 regBankSelected: true
 
-# CHECK-LABEL: extract512
-# CHECK: [[BASE:%[0-9]+]]:sreg_512 = IMPLICIT_DEF
-# CHECK: [[SGPR0:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub0
-# CHECK: [[SGPR1:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub1
-# CHECK: [[SGPR2:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub2
-# CHECK: [[SGPR3:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub3
-# CHECK: [[SGPR4:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub4
-# CHECK: [[SGPR5:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub5
-# CHECK: [[SGPR6:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub6
-# CHECK: [[SGPR7:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub7
-# CHECK: [[SGPR8:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub8
-# CHECK: [[SGPR9:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub9
-# CHECK: [[SGPR10:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub10
-# CHECK: [[SGPR11:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub11
-# CHECK: [[SGPR12:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub12
-# CHECK: [[SGPR13:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub13
-# CHECK: [[SGPR14:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub14
-# CHECK: [[SGPR15:%[0-9]+]]:sreg_32_xm0 = COPY [[BASE]].sub15
-# CHECK: $sgpr0 = COPY [[SGPR0]]
-# CHECK: $sgpr1 = COPY [[SGPR1]]
-# CHECK: $sgpr2 = COPY [[SGPR2]]
-# CHECK: $sgpr3 = COPY [[SGPR3]]
-# CHECK: $sgpr4 = COPY [[SGPR4]]
-# CHECK: $sgpr5 = COPY [[SGPR5]]
-# CHECK: $sgpr6 = COPY [[SGPR6]]
-# CHECK: $sgpr7 = COPY [[SGPR7]]
-# CHECK: $sgpr8 = COPY [[SGPR8]]
-# CHECK: $sgpr9 = COPY [[SGPR9]]
-# CHECK: $sgpr10 = COPY [[SGPR10]]
-# CHECK: $sgpr11 = COPY [[SGPR11]]
-# CHECK: $sgpr12 = COPY [[SGPR12]]
-# CHECK: $sgpr13 = COPY [[SGPR13]]
-# CHECK: $sgpr14 = COPY [[SGPR14]]
-# CHECK: $sgpr15 = COPY [[SGPR15]]
-
 body: |
   bb.0:
+    ; CHECK-LABEL: name: extract512
+    ; CHECK: [[DEF:%[0-9]+]]:sreg_512 = IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub0
+    ; CHECK: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub1
+    ; CHECK: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub2
+    ; CHECK: [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub3
+    ; CHECK: [[COPY4:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub4
+    ; CHECK: [[COPY5:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub5
+    ; CHECK: [[COPY6:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub6
+    ; CHECK: [[COPY7:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub7
+    ; CHECK: [[COPY8:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub8
+    ; CHECK: [[COPY9:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub9
+    ; CHECK: [[COPY10:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub10
+    ; CHECK: [[COPY11:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub11
+    ; CHECK: [[COPY12:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub12
+    ; CHECK: [[COPY13:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub13
+    ; CHECK: [[COPY14:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub14
+    ; CHECK: [[COPY15:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub15
+    ; CHECK: $sgpr0 = COPY [[COPY]]
+    ; CHECK: $sgpr1 = COPY [[COPY1]]
+    ; CHECK: $sgpr2 = COPY [[COPY2]]
+    ; CHECK: $sgpr3 = COPY [[COPY3]]
+    ; CHECK: $sgpr4 = COPY [[COPY4]]
+    ; CHECK: $sgpr5 = COPY [[COPY5]]
+    ; CHECK: $sgpr6 = COPY [[COPY6]]
+    ; CHECK: $sgpr7 = COPY [[COPY7]]
+    ; CHECK: $sgpr8 = COPY [[COPY8]]
+    ; CHECK: $sgpr9 = COPY [[COPY9]]
+    ; CHECK: $sgpr10 = COPY [[COPY10]]
+    ; CHECK: $sgpr11 = COPY [[COPY11]]
+    ; CHECK: $sgpr12 = COPY [[COPY12]]
+    ; CHECK: $sgpr13 = COPY [[COPY13]]
+    ; CHECK: $sgpr14 = COPY [[COPY14]]
+    ; CHECK: $sgpr15 = COPY [[COPY15]]
+    ; CHECK: SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15
     %0:sgpr(s512) = G_IMPLICIT_DEF
     %1:sgpr(s32) = G_EXTRACT %0:sgpr(s512), 0
     %2:sgpr(s32) = G_EXTRACT %0:sgpr(s512), 32
@@ -75,3 +76,84 @@ body: |
     $sgpr14 = COPY %15:sgpr(s32)
     $sgpr15 = COPY %16:sgpr(s32)
     SI_RETURN_TO_EPILOG $sgpr0, $sgpr1, $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14, $sgpr15
+...
+
+---
+name:            extract_s_s32_s1024
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: extract_s_s32_s1024
+    ; CHECK: [[DEF:%[0-9]+]]:sreg_1024 = IMPLICIT_DEF
+    ; CHECK: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub0
+    ; CHECK: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub1
+    ; CHECK: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub2
+    ; CHECK: [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub3
+    ; CHECK: [[COPY4:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub4
+    ; CHECK: [[COPY5:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub5
+    ; CHECK: [[COPY6:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub6
+    ; CHECK: [[COPY7:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub7
+    ; CHECK: [[COPY8:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub8
+    ; CHECK: [[COPY9:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub9
+    ; CHECK: [[COPY10:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub10
+    ; CHECK: [[COPY11:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub11
+    ; CHECK: [[COPY12:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub12
+    ; CHECK: [[COPY13:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub13
+    ; CHECK: [[COPY14:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub14
+    ; CHECK: [[COPY15:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub15
+    ; CHECK: [[COPY16:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub16
+    ; CHECK: [[COPY17:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub17
+    ; CHECK: [[COPY18:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub18
+    ; CHECK: [[COPY19:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub19
+    ; CHECK: [[COPY20:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub20
+    ; CHECK: [[COPY21:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub21
+    ; CHECK: [[COPY22:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub22
+    ; CHECK: [[COPY23:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub23
+    ; CHECK: [[COPY24:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub24
+    ; CHECK: [[COPY25:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub25
+    ; CHECK: [[COPY26:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub26
+    ; CHECK: [[COPY27:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub27
+    ; CHECK: [[COPY28:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub28
+    ; CHECK: [[COPY29:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub29
+    ; CHECK: [[COPY30:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub30
+    ; CHECK: [[COPY31:%[0-9]+]]:sreg_32_xm0 = COPY [[DEF]].sub31
+    ; CHECK: S_ENDPGM 0, implicit [[DEF]], implicit [[COPY]], implicit [[COPY1]], implicit [[COPY2]], implicit [[COPY3]], implicit [[COPY4]], implicit [[COPY5]], implicit [[COPY6]], implicit [[COPY7]], implicit [[COPY8]], implicit [[COPY9]], implicit [[COPY10]], implicit [[COPY11]], implicit [[COPY12]], implicit [[COPY13]], implicit [[COPY14]], implicit [[COPY15]], implicit [[COPY16]], implicit [[COPY17]], implicit [[COPY18]], implicit [[COPY19]], implicit [[COPY20]], implicit [[COPY21]], implicit [[COPY22]], implicit [[COPY23]], implicit [[COPY24]], implicit [[COPY25]], implicit [[COPY26]], implicit [[COPY27]], implicit [[COPY28]], implicit [[COPY29]], implicit [[COPY30]], implicit [[COPY31]]
+    %0:sgpr(s1024) = G_IMPLICIT_DEF
+    %1:sgpr(s32) = G_EXTRACT %0:sgpr, 0
+    %2:sgpr(s32) = G_EXTRACT %0:sgpr, 32
+    %3:sgpr(s32) = G_EXTRACT %0:sgpr, 64
+    %4:sgpr(s32) = G_EXTRACT %0:sgpr, 96
+    %5:sgpr(s32) = G_EXTRACT %0:sgpr, 128
+    %6:sgpr(s32) = G_EXTRACT %0:sgpr, 160
+    %7:sgpr(s32) = G_EXTRACT %0:sgpr, 192
+    %8:sgpr(s32) = G_EXTRACT %0:sgpr, 224
+    %9:sgpr(s32) = G_EXTRACT %0:sgpr, 256
+    %10:sgpr(s32) = G_EXTRACT %0:sgpr, 288
+    %11:sgpr(s32) = G_EXTRACT %0:sgpr, 320
+    %12:sgpr(s32) = G_EXTRACT %0:sgpr, 352
+    %13:sgpr(s32) = G_EXTRACT %0:sgpr, 384
+    %14:sgpr(s32) = G_EXTRACT %0:sgpr, 416
+    %15:sgpr(s32) = G_EXTRACT %0:sgpr, 448
+    %16:sgpr(s32) = G_EXTRACT %0:sgpr, 480
+
+    %17:sgpr(s32) = G_EXTRACT %0:sgpr, 512
+    %18:sgpr(s32) = G_EXTRACT %0:sgpr, 544
+    %19:sgpr(s32) = G_EXTRACT %0:sgpr, 576
+    %20:sgpr(s32) = G_EXTRACT %0:sgpr, 608
+    %21:sgpr(s32) = G_EXTRACT %0:sgpr, 640
+    %22:sgpr(s32) = G_EXTRACT %0:sgpr, 672
+    %23:sgpr(s32) = G_EXTRACT %0:sgpr, 704
+    %24:sgpr(s32) = G_EXTRACT %0:sgpr, 736
+    %25:sgpr(s32) = G_EXTRACT %0:sgpr, 768
+    %26:sgpr(s32) = G_EXTRACT %0:sgpr, 800
+    %27:sgpr(s32) = G_EXTRACT %0:sgpr, 832
+    %28:sgpr(s32) = G_EXTRACT %0:sgpr, 864
+    %29:sgpr(s32) = G_EXTRACT %0:sgpr, 896
+    %30:sgpr(s32) = G_EXTRACT %0:sgpr, 928
+    %31:sgpr(s32) = G_EXTRACT %0:sgpr, 960
+    %32:sgpr(s32) = G_EXTRACT %0:sgpr, 992
+
+    S_ENDPGM 0, implicit %0, implicit %1, implicit %2, implicit %3, implicit %4, implicit %5, implicit %6, implicit %7, implicit %8, implicit %9, implicit %10, implicit %11, implicit %12, implicit %13, implicit %14, implicit %15, implicit %16, implicit %17, implicit %18, implicit %19, implicit %20, implicit %21, implicit %22, implicit %23, implicit %24, implicit %25, implicit %26, implicit %27, implicit %28, implicit %29, implicit %30, implicit %31, implicit %32
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fabs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fabs.mir
index 037b6bf2ad4bb..56e9e98b1dec5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fabs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fabs.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# XUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# XUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=0-verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name: fabs_s32_ss
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fneg.mir
index ed0109e1f5ba8..3618e59a93c0f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fneg.mir
@@ -1,7 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# XUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
-# XUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=0-verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s | FileCheck -check-prefix=GCN %s
 
 ---
 name: fneg_s32_ss
@@ -95,11 +95,10 @@ body: |
     liveins: $vgpr0
     ; GCN-LABEL: name: fneg_s16_vv
     ; GCN: liveins: $vgpr0
-    ; GCN: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; GCN: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; GCN: [[FNEG:%[0-9]+]]:vgpr_32(s16) = G_FNEG [[TRUNC]]
-    ; GCN: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY [[FNEG]](s16)
-    ; GCN: $vgpr0 = COPY [[COPY1]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32768
+    ; GCN: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[S_MOV_B32_]], [[COPY]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_XOR_B32_e32_]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s16) = G_TRUNC %0
     %2:vgpr(s16) = G_FNEG %1
@@ -349,12 +348,11 @@ body: |
     liveins: $vgpr0
     ; GCN-LABEL: name: fneg_fabs_s16_vv
     ; GCN: liveins: $vgpr0
-    ; GCN: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; GCN: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; GCN: [[FABS:%[0-9]+]]:vgpr(s16) = G_FABS [[TRUNC]]
-    ; GCN: [[FNEG:%[0-9]+]]:vgpr_32(s16) = G_FNEG [[FABS]]
-    ; GCN: [[COPY1:%[0-9]+]]:sreg_32_xm0(s32) = COPY [[FNEG]](s16)
-    ; GCN: $vgpr0 = COPY [[COPY1]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32768
+    ; GCN: [[V_OR_B32_e32_:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[S_MOV_B32_]], [[COPY]], implicit $exec
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY [[V_OR_B32_e32_]]
+    ; GCN: $vgpr0 = COPY [[COPY1]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s16) = G_TRUNC %0
     %2:vgpr(s16) = G_FABS %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptosi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptosi.mir
new file mode 100644
index 0000000000000..f4c961a3bfb51
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptosi.mir
@@ -0,0 +1,132 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck %s -check-prefix=GCN
+
+---
+name: fptosi_s32_to_s32_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptosi_s32_to_s32_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[V_CVT_I32_F32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e64 0, [[COPY]], 0, 0, implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_FPTOSI %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: fptosi_s32_to_s32_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GCN-LABEL: name: fptosi_s32_to_s32_vs
+    ; GCN: liveins: $sgpr0
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GCN: [[V_CVT_I32_F32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e64 0, [[COPY]], 0, 0, implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s32) = G_FPTOSI %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: fptosi_s32_to_s32_fneg_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptosi_s32_to_s32_fneg_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[V_CVT_I32_F32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e64 1, [[COPY]], 0, 0, implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_FNEG %0
+    %2:vgpr(s32) = G_FPTOSI %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: fptosi_s16_to_s32_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptosi_s16_to_s32_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_TRUNC %0
+    %2:vgpr(s32) = G_FPTOSI %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: fptosi_s16_to_s32_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GCN-LABEL: name: fptosi_s16_to_s32_vs
+    ; GCN: liveins: $sgpr0
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s16) = G_TRUNC %0
+    %2:vgpr(s32) = G_FPTOSI %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: fptosi_s16_to_s32_fneg_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptosi_s16_to_s32_fneg_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32768
+    ; GCN: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[S_MOV_B32_]], [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[V_XOR_B32_e32_]], implicit $exec
+    ; GCN: [[V_CVT_I32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_I32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_I32_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_TRUNC %0
+    %2:vgpr(s16) = G_FNEG %1
+    %3:vgpr(s32) = G_FPTOSI %2
+    $vgpr0 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
index 81e5ad01c0afb..8b78ea6435ae9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-fptoui.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs -o - %s | FileCheck %s -check-prefix=GCN
 
 ---
@@ -6,27 +7,99 @@ name:            fptoui
 legalized:       true
 regBankSelected: true
 
-# GCN-LABEL: name: fptoui
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0, $vgpr3_vgpr4
 
-    ; GCN: [[SGPR:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GCN-LABEL: name: fptoui
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; GCN: [[V_CVT_U32_F32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e64 0, [[COPY]], 0, 0, implicit $exec
+    ; GCN: [[V_CVT_U32_F32_e64_1:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e64 0, [[COPY1]], 0, 0, implicit $exec
+    ; GCN: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_U32_F32_e64_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
+    ; GCN: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_U32_F32_e64_1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     %0:sgpr(s32) = COPY $sgpr0
 
-    ; GCN: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr0
 
     %2:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; fptoui s
-    ; GCN: V_CVT_U32_F32_e64 0, [[SGPR]], 0, 0
     %3:vgpr(s32) = G_FPTOUI %0
 
     ; fptoui v
-    ; GCN: V_CVT_U32_F32_e64 0, [[VGPR]], 0, 0
     %4:vgpr(s32) = G_FPTOUI %1
 
     G_STORE %3, %2 :: (store 4, addrspace 1)
     G_STORE %4, %2 :: (store 4, addrspace 1)
 ...
+
+---
+name: fptoui_s16_to_s32_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptoui_s16_to_s32_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_U32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_U32_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_TRUNC %0
+    %2:vgpr(s32) = G_FPTOUI %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: fptoui_s16_to_s32_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; GCN-LABEL: name: fptoui_s16_to_s32_vs
+    ; GCN: liveins: $sgpr0
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_U32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_U32_F32_e32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s16) = G_TRUNC %0
+    %2:vgpr(s32) = G_FPTOUI %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: fptoui_s16_to_s32_fneg_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; GCN-LABEL: name: fptoui_s16_to_s32_fneg_vv
+    ; GCN: liveins: $vgpr0
+    ; GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GCN: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 32768
+    ; GCN: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[S_MOV_B32_]], [[COPY]], implicit $exec
+    ; GCN: [[V_CVT_F32_F16_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_F16_e32 [[V_XOR_B32_e32_]], implicit $exec
+    ; GCN: [[V_CVT_U32_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_U32_F32_e32 [[V_CVT_F32_F16_e32_]], implicit $exec
+    ; GCN: $vgpr0 = COPY [[V_CVT_U32_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_TRUNC %0
+    %2:vgpr(s16) = G_FNEG %1
+    %3:vgpr(s32) = G_FPTOUI %2
+    $vgpr0 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
index 5e14f7e50083f..fec8155a075cb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-implicit-def.mir
@@ -121,6 +121,7 @@ body: |
     ; GCN-LABEL: name: implicit_def_p3_vgpr
     ; GCN: [[DEF:%[0-9]+]]:vreg_64 = IMPLICIT_DEF
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4, implicit $exec
+    ; GCN: $m0 = S_MOV_B32 -1
     ; GCN: FLAT_STORE_DWORD [[DEF]], [[V_MOV_B32_e32_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
     %0:vgpr(p3) = G_IMPLICIT_DEF
     %1:vgpr(s32) = G_CONSTANT i32 4
@@ -203,3 +204,32 @@ body: |
     %0:vcc(s1) = G_IMPLICIT_DEF
     S_ENDPGM 0, implicit %0
 ...
+
+---
+
+name:            implicit_def_s1024_sgpr
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: implicit_def_s1024_sgpr
+    ; GCN: [[DEF:%[0-9]+]]:sreg_1024 = IMPLICIT_DEF
+    ; GCN: S_ENDPGM 0, implicit [[DEF]]
+    %0:sgpr(s1024) = G_IMPLICIT_DEF
+    S_ENDPGM 0, implicit %0
+...
+---
+
+name:            implicit_def_s1024_vgpr
+legalized:       true
+regBankSelected: true
+
+body: |
+  bb.0:
+    ; GCN-LABEL: name: implicit_def_s1024_vgpr
+    ; GCN: [[DEF:%[0-9]+]]:vreg_1024 = IMPLICIT_DEF
+    ; GCN: S_ENDPGM 0, implicit [[DEF]]
+    %0:vgpr(s1024) = G_IMPLICIT_DEF
+    S_ENDPGM 0, implicit %0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
new file mode 100644
index 0000000000000..2c53482178d42
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-constant.mir
@@ -0,0 +1,896 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs  -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX7 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s | FileCheck -check-prefix=GFX10 %s
+
+---
+
+name: load_constant_s32_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %1
+
+...
+
+---
+
+name: load_constant_v2s16_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2s16_from_4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_v2s16_from_4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_v2s16_from_4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_v2s16_from_4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x s16>) = G_LOAD %0 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %1
+
+...
+
+
+---
+name: load_constant_v2s32
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2s32
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_v2s32
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_v2s32
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_v2s32
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x s32>) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+name: load_constant_v2s32_align4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2s32_align4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_v2s32_align4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_v2s32_align4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_v2s32_align4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x s32>) = G_LOAD %0 :: (load 8, align 4, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+name: load_constant_v4s16_align4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v4s16_align4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_v4s16_align4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_v4s16_align4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_v4s16_align4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<4 x s16>) = G_LOAD %0 :: (load 8, align 4, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_v4s32_align4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v4s32_align4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX7-LABEL: name: load_constant_v4s32_align4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX8-LABEL: name: load_constant_v4s32_align4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX10-LABEL: name: load_constant_v4s32_align4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<4 x  s32>) = G_LOAD %0 :: (load 16, align 4, addrspace 4)
+    $sgpr0_sgpr1_sgpr2_sgpr3 = COPY %1
+
+...
+
+---
+
+name: load_constant_s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s64
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_s64
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_s64
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_s64
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_s64_align4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s64_align4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_s64_align4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_s64_align4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_s64_align4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_LOAD %0 :: (load 8, align 4, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_v2s64
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2s64
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX7-LABEL: name: load_constant_v2s64
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX8-LABEL: name: load_constant_v2s64
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    ; GFX10-LABEL: name: load_constant_v2s64
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sreg_128 = S_LOAD_DWORDX4_IMM [[COPY]], 0, 0, 0 :: (load 16, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[S_LOAD_DWORDX4_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x s64>) = G_LOAD %0 :: (load 16, align 4, addrspace 4)
+    $sgpr0_sgpr1_sgpr2_sgpr3 = COPY %1
+
+...
+
+---
+
+name: load_constant_v2p1
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2p1
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX6: [[LOAD:%[0-9]+]]:sreg_128(<2 x p1>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<2 x p1>)
+    ; GFX7-LABEL: name: load_constant_v2p1
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX7: [[LOAD:%[0-9]+]]:sreg_128(<2 x p1>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<2 x p1>)
+    ; GFX8-LABEL: name: load_constant_v2p1
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX8: [[LOAD:%[0-9]+]]:sreg_128(<2 x p1>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<2 x p1>)
+    ; GFX10-LABEL: name: load_constant_v2p1
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX10: [[LOAD:%[0-9]+]]:sreg_128(<2 x p1>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<2 x p1>)
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x p1>) = G_LOAD %0 :: (load 16, align 4, addrspace 4)
+    $sgpr0_sgpr1_sgpr2_sgpr3 = COPY %1
+
+...
+
+---
+
+name: load_constant_s128_align4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s128_align4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX6: [[LOAD:%[0-9]+]]:sreg_128(s128) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](s128)
+    ; GFX7-LABEL: name: load_constant_s128_align4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX7: [[LOAD:%[0-9]+]]:sreg_128(s128) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](s128)
+    ; GFX8-LABEL: name: load_constant_s128_align4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX8: [[LOAD:%[0-9]+]]:sreg_128(s128) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](s128)
+    ; GFX10-LABEL: name: load_constant_s128_align4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX10: [[LOAD:%[0-9]+]]:sreg_128(s128) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](s128)
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s128) = G_LOAD %0 :: (load 16, align 4, addrspace 4)
+    $sgpr0_sgpr1_sgpr2_sgpr3 = COPY %1
+
+...
+
+---
+
+name: load_constant_p3_from_4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_p3_from_4
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_p3_from_4
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_p3_from_4
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_p3_from_4
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(p3) = G_LOAD %0 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %1
+
+...
+
+---
+
+name: load_constant_p1_from_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_p1_from_8
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_p1_from_8
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_p1_from_8
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_p1_from_8
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(p4) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_p999_from_8
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_p999_from_8
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX6: [[LOAD:%[0-9]+]]:sreg_64_xexec(p999) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[LOAD]](p999)
+    ; GFX7-LABEL: name: load_constant_p999_from_8
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX7: [[LOAD:%[0-9]+]]:sreg_64_xexec(p999) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[LOAD]](p999)
+    ; GFX8-LABEL: name: load_constant_p999_from_8
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX8: [[LOAD:%[0-9]+]]:sreg_64_xexec(p999) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[LOAD]](p999)
+    ; GFX10-LABEL: name: load_constant_p999_from_8
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX10: [[LOAD:%[0-9]+]]:sreg_64_xexec(p999) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[LOAD]](p999)
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(p999) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_v2p3
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2p3
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX6: [[LOAD:%[0-9]+]]:sreg_64_xexec(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX7-LABEL: name: load_constant_v2p3
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX7: [[LOAD:%[0-9]+]]:sreg_64_xexec(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX8-LABEL: name: load_constant_v2p3
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX8: [[LOAD:%[0-9]+]]:sreg_64_xexec(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[LOAD]](<2 x p3>)
+    ; GFX10-LABEL: name: load_constant_v2p3
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX10: [[LOAD:%[0-9]+]]:sreg_64_xexec(<2 x p3>) = G_LOAD [[COPY]](p4) :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[LOAD]](<2 x p3>)
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x p3>) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_v2s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v2s16
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_v2s16
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_v2s16
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_v2s16
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 0, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<2 x s16>) = G_LOAD %0 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %1
+
+...
+
+---
+
+name: load_constant_v4s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v4s16
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX7-LABEL: name: load_constant_v4s16
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX8-LABEL: name: load_constant_v4s16
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; GFX10-LABEL: name: load_constant_v4s16
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]], 0, 0, 0 :: (load 8, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1 = COPY [[S_LOAD_DWORDX2_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<4 x s16>) = G_LOAD %0 :: (load 8, align 8, addrspace 4)
+    $sgpr0_sgpr1 = COPY %1
+
+...
+
+---
+
+name: load_constant_v8s16
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_v8s16
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX6: [[LOAD:%[0-9]+]]:sreg_128(<8 x s16>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX6: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<8 x s16>)
+    ; GFX7-LABEL: name: load_constant_v8s16
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX7: [[LOAD:%[0-9]+]]:sreg_128(<8 x s16>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX7: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<8 x s16>)
+    ; GFX8-LABEL: name: load_constant_v8s16
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX8: [[LOAD:%[0-9]+]]:sreg_128(<8 x s16>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX8: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<8 x s16>)
+    ; GFX10-LABEL: name: load_constant_v8s16
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; GFX10: [[LOAD:%[0-9]+]]:sreg_128(<8 x s16>) = G_LOAD [[COPY]](p4) :: (load 16, align 4, addrspace 4)
+    ; GFX10: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[LOAD]](<8 x s16>)
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(<8 x  s16>) = G_LOAD %0 :: (load 16, align 4, addrspace 4)
+    $sgpr0_sgpr1_sgpr2_sgpr3 = COPY %1
+
+...
+
+################################################################################
+### Stress addressing modes
+################################################################################
+
+---
+
+name: load_constant_s32_from_4_gep_1020
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_1020
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 255, 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_1020
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 255, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1020
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1020, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1020
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1020, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 1020
+    %2:sgpr(p4) = G_GEP %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
+
+---
+
+name: load_constant_s32_from_4_gep_1024
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_1024
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1024
+    ; GFX6: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_1024
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM_ci:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci [[COPY]], 256, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM_ci]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1024
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1024, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1024
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1024, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 1024
+    %2:sgpr(p4) = G_GEP %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
+
+---
+
+name: load_constant_s32_from_4_gep_1048575
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_1048575
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048575
+    ; GFX6: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_1048575
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM_ci:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci [[COPY]], 262143, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM_ci]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1048575
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1048575, 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1048575
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]], 1048575, 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_IMM]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 1048575
+    %2:sgpr(p4) = G_GEP %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
+
+---
+
+name: load_constant_s32_from_4_gep_1048576
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_1048576
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048576
+    ; GFX6: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_1048576
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM_ci:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci [[COPY]], 262144, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM_ci]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1048576
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048576
+    ; GFX8: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1048576
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1048576
+    ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 1048576
+    %2:sgpr(p4) = G_GEP %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
+
+---
+
+name: load_constant_s32_from_4_gep_1073741823
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins:  $sgpr0_sgpr1
+
+    ; GFX6-LABEL: name: load_constant_s32_from_4_gep_1073741823
+    ; GFX6: liveins: $sgpr0_sgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX6: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1073741823
+    ; GFX6: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX6: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX7-LABEL: name: load_constant_s32_from_4_gep_1073741823
+    ; GFX7: liveins: $sgpr0_sgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX7: [[S_LOAD_DWORD_IMM_ci:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM_ci [[COPY]], 268435455, 0, 0 :: (load 4, addrspace 4)
+    ; GFX7: $sgpr0 = COPY [[S_LOAD_DWORD_IMM_ci]]
+    ; GFX8-LABEL: name: load_constant_s32_from_4_gep_1073741823
+    ; GFX8: liveins: $sgpr0_sgpr1
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX8: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1073741823
+    ; GFX8: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX8: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    ; GFX10-LABEL: name: load_constant_s32_from_4_gep_1073741823
+    ; GFX10: liveins: $sgpr0_sgpr1
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
+    ; GFX10: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 1073741823
+    ; GFX10: [[S_LOAD_DWORD_SGPR:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_SGPR [[COPY]], [[S_MOV_B32_]], 0, 0 :: (load 4, addrspace 4)
+    ; GFX10: $sgpr0 = COPY [[S_LOAD_DWORD_SGPR]]
+    %0:sgpr(p4) = COPY $sgpr0_sgpr1
+    %1:sgpr(s64) = G_CONSTANT i64 1073741823
+    %2:sgpr(p4) = G_GEP %0, %1
+    %3:sgpr(s32) = G_LOAD %2 :: (load 4, align 4, addrspace 4)
+    $sgpr0 = COPY %3
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
index 513f5b08c6a23..65a7fc7f4aa70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-private.mir
@@ -20,12 +20,12 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_4
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_4
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
@@ -51,12 +51,12 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_2
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_2
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_USHORT_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 2, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_LOAD %0 :: (load 2, align 2, addrspace 5)
@@ -82,12 +82,12 @@ body: |
     ; GFX6-LABEL: name: load_private_s32_from_1
     ; GFX6: liveins: $vgpr0
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
@@ -208,12 +208,12 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2047, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_2047
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 2047
@@ -243,14 +243,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
     ; GFX6: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_AND_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_AND_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_2047_known_bits
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2147483647, implicit $exec
     ; GFX9: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_AND_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_AND_B32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2047, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 2147483647
@@ -283,12 +283,12 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 2048, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_2048
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2048, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 2048, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 2048
@@ -318,14 +318,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2047
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965249, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -2047
@@ -355,14 +355,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m2048
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294965248, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -2048
@@ -392,12 +392,12 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_4095
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[COPY]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 4095
@@ -427,14 +427,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_4096
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 4096
@@ -464,14 +464,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4095
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963201, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -4095
@@ -501,14 +501,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m4096
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294963200, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -4096
@@ -538,14 +538,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_8191
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8191, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 8191
@@ -575,14 +575,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_8192
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 8192, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 8192
@@ -612,14 +612,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8191
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959105, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -8191
@@ -649,14 +649,14 @@ body: |
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_gep_m8192
     ; GFX9: liveins: $vgpr0
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4294959104, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(s32) = G_CONSTANT i32 -8192
@@ -681,10 +681,10 @@ body: |
   bb.0:
 
     ; GFX6-LABEL: name: load_private_s32_from_4_constant_0
-    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
     ; GFX9-LABEL: name: load_private_s32_from_4_constant_0
-    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
     %0:vgpr(p5) = G_CONSTANT i32 0
     %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
@@ -707,10 +707,10 @@ body: |
   bb.0:
 
     ; GFX6-LABEL: name: load_private_s32_from_4_constant_sgpr_16
-    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
     ; GFX9-LABEL: name: load_private_s32_from_4_constant_sgpr_16
-    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 16, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFSET]]
     %0:sgpr(p5) = G_CONSTANT i32 16
     %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
@@ -733,10 +733,10 @@ body: |
   bb.0:
 
     ; GFX6-LABEL: name: load_private_s32_from_1_constant_4095
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFSET]]
     ; GFX9-LABEL: name: load_private_s32_from_1_constant_4095
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFSET:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFSET]]
     %0:vgpr(p5) = G_CONSTANT i32 4095
     %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
@@ -760,11 +760,11 @@ body: |
 
     ; GFX6-LABEL: name: load_private_s32_from_1_constant_4096
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_constant_4096
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = G_CONSTANT i32 4096
     %1:vgpr(s32) = G_LOAD %0 :: (load 1, align 1, addrspace 5)
@@ -789,10 +789,10 @@ body: |
   bb.0:
 
     ; GFX6-LABEL: name: load_private_s32_from_fi
-    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_fi
-    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     %0:vgpr(p5) = G_FRAME_INDEX %stack.0
     %1:vgpr(s32) = G_LOAD %0 :: (load 4, align 4, addrspace 5)
@@ -820,10 +820,10 @@ body: |
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_fi_offset_4095
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4095, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4095, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = G_FRAME_INDEX %stack.0
     %1:vgpr(s32) = G_CONSTANT i32 4095
@@ -853,13 +853,13 @@ body: |
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
-    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX6: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX6: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     ; GFX9-LABEL: name: load_private_s32_from_1_fi_offset_4096
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
     ; GFX9: [[V_ADD_U32_e64_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
-    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
+    ; GFX9: [[BUFFER_LOAD_UBYTE_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_UBYTE_OFFEN [[V_ADD_U32_e64_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 1, addrspace 5)
     ; GFX9: $vgpr0 = COPY [[BUFFER_LOAD_UBYTE_OFFEN]]
     %0:vgpr(p5) = G_FRAME_INDEX %stack.0
     %1:vgpr(s32) = G_CONSTANT i32 4096
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
index 613601841d93b..8332de8992bd0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-merge-values.mir
@@ -563,3 +563,29 @@ body: |
     %2:vreg_64(s64) = G_MERGE_VALUES %0, %1
     S_ENDPGM 0, implicit %2
 ...
+
+---
+name: test_merge_values_s_s1024_s_s256_s_s256_s_s256_s_s256
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+
+    ; GCN-LABEL: name: test_merge_values_s_s1024_s_s256_s_s256_s_s256_s_s256
+    ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+    ; GCN: [[COPY:%[0-9]+]]:sreg_256 = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7
+    ; GCN: [[DEF:%[0-9]+]]:sreg_256 = IMPLICIT_DEF
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_256 = COPY $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+    ; GCN: [[DEF1:%[0-9]+]]:sreg_256 = IMPLICIT_DEF
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_1024 = REG_SEQUENCE [[COPY]], %subreg.sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7, [[DEF]], %subreg.sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15, [[COPY1]], %subreg.sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23, [[DEF1]], %subreg.sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31
+    ; GCN: S_ENDPGM 0, implicit [[REG_SEQUENCE]]
+    %0:sgpr(s256) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7,
+    %1:sgpr(s256) = G_IMPLICIT_DEF
+    %2:sgpr(s256) = COPY $sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15
+    %3:sgpr(s256) = G_IMPLICIT_DEF
+    %4:sgpr(s1024) = G_MERGE_VALUES %0, %1, %2, %3
+    S_ENDPGM 0, implicit %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
index d4df9f9a40316..1423b0a3471b0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-or.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr="+wavefrontsize32" -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s  | FileCheck -check-prefix=WAVE32 %s
 
 ---
@@ -148,12 +149,10 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: or_s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; WAVE64: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; WAVE64: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
-    ; WAVE64: [[OR:%[0-9]+]]:vgpr(s16) = G_OR [[TRUNC]], [[TRUNC1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[OR]](s16)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; WAVE32-LABEL: name: or_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
@@ -355,16 +354,17 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: or_v2s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE64: [[OR:%[0-9]+]]:vgpr(<2 x s16>) = G_OR [[COPY]], [[COPY1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[OR]](<2 x s16>)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     ; WAVE32-LABEL: name: or_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE32: [[OR:%[0-9]+]]:vgpr(<2 x s16>) = G_OR [[COPY]], [[COPY1]]
-    ; WAVE32: S_ENDPGM 0, implicit [[OR]](<2 x s16>)
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE32: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE32: S_ENDPGM 0, implicit [[V_OR_B32_e64_]]
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(<2 x s16>) = COPY $vgpr1
     %2:vgpr(<2 x s16>) = G_OR %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
index 8c8c40ab9e5c7..63d6427052c9d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sitofp.mir
@@ -1,4 +1,6 @@
-# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=GCN
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=WAVE32 %s
 
 ---
 
@@ -6,27 +8,307 @@ name:            sitofp
 legalized:       true
 regBankSelected: true
 
-# GCN-LABEL: name: sitofp
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0, $vgpr3_vgpr4
 
-    ; GCN: [[SGPR:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE64-LABEL: name: sitofp
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; WAVE64: [[V_CVT_F32_I32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE64: [[V_CVT_F32_I32_e64_1:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e64 [[COPY1]], 0, 0, implicit $exec
+    ; WAVE64: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
+    ; WAVE64: FLAT_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_1]], 0, 0, 0, 0, implicit $exec, implicit $flat_scr :: (store 4, addrspace 1)
+    ; WAVE32-LABEL: name: sitofp
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[COPY2:%[0-9]+]]:vreg_64 = COPY $vgpr3_vgpr4
+    ; WAVE32: [[V_CVT_F32_I32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE32: [[V_CVT_F32_I32_e64_1:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e64 [[COPY1]], 0, 0, implicit $exec
+    ; WAVE32: GLOBAL_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    ; WAVE32: GLOBAL_STORE_DWORD [[COPY2]], [[V_CVT_F32_I32_e64_1]], 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %0:sgpr(s32) = COPY $sgpr0
 
-    ; GCN: [[VGPR:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     %1:vgpr(s32) = COPY $vgpr0
 
     %2:vgpr(p1) = COPY $vgpr3_vgpr4
 
     ; sitofp s
-    ; GCN: V_CVT_F32_I32_e64 [[SGPR]], 0, 0
     %3:vgpr(s32) = G_SITOFP %0
 
     ; sitofp v
-    ; GCN: V_CVT_F32_I32_e64 [[VGPR]], 0, 0
     %4:vgpr(s32) = G_SITOFP %1
 
     G_STORE %3, %2 :: (store 4, addrspace 1)
     G_STORE %4, %2 :: (store 4, addrspace 1)
 ...
+
+---
+name: sitofp_s32_to_s16_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s32_to_s16_vv
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: sitofp_s32_to_s16_vv
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_SITOFP %0
+    %2:vgpr(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: sitofp_s32_to_s16_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s32_to_s16_vs
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE64: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: sitofp_s32_to_s16_vs
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE32: [[V_CVT_F32_I32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_I32_e32 [[COPY]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_I32_e32_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s16) = G_SITOFP %0
+    %2:vgpr(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: sitofp_s1_to_s32_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s32_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 3212836864, implicit $scc
+    ; WAVE64: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s32_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 3212836864, implicit $scc
+    ; WAVE32: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_SITOFP %2
+    $sgpr0 = COPY %3
+...
+
+---
+name: sitofp_s1_to_s16_to_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s16_to_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 48128, implicit $scc
+    ; WAVE64: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s16_to_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 48128, implicit $scc
+    ; WAVE32: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s16) = G_SITOFP %2
+    %4:sgpr(s32) = G_ANYEXT %3
+    $sgpr0 = COPY %4
+...
+
+---
+name: sitofp_s1_to_s64_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s64_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 0, -4616189618054758400, implicit $scc
+    ; WAVE64: $sgpr0_sgpr1 = COPY [[S_CSELECT_B64_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s64_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 0, -4616189618054758400, implicit $scc
+    ; WAVE32: $sgpr0_sgpr1 = COPY [[S_CSELECT_B64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s64) = G_SITOFP %2
+    $sgpr0_sgpr1 = COPY %3
+...
+
+---
+name: sitofp_s1_to_s32_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s32_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3212836864, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s32_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3212836864, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_SITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: sitofp_s1_to_s16_to_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s16_to_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3212836864, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s16_to_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 3212836864, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s16) = G_SITOFP %2
+    %4:vgpr(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: sitofp_s1_to_s64_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: sitofp_s1_to_s64_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: [[V_CVT_F64_I32_e32_:%[0-9]+]]:vreg_64 = V_CVT_F64_I32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0_vgpr1 = COPY [[V_CVT_F64_I32_e32_]]
+    ; WAVE32-LABEL: name: sitofp_s1_to_s64_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: [[V_CVT_F64_I32_e32_:%[0-9]+]]:vreg_64 = V_CVT_F64_I32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0_vgpr1 = COPY [[V_CVT_F64_I32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s64) = G_SITOFP %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax.mir
index 50811a47cd2c5..1ffe7d3d679d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smax.mir
@@ -1,10 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s
-# RUN: FileCheck -check-prefix=ERR  %s < %t
-
-# ERR-NOT: remark:
-# ERR: remark: <unknown>:0:0: cannot select: %2:sgpr(s32) = G_SMAX %0:sgpr, %1:sgpr (in function: smax_s32_ss)
-# ERR-NOT: remark:
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
 name: smax_s32_ss
@@ -15,10 +10,10 @@ body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
     ; GCN-LABEL: name: smax_s32_ss
-    ; GCN: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; GCN: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; GCN: [[SMAX:%[0-9]+]]:sgpr(s32) = G_SMAX [[COPY]], [[COPY1]]
-    ; GCN: S_ENDPGM 0, implicit [[SMAX]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GCN: [[S_MAX_I32_:%[0-9]+]]:sreg_32 = S_MAX_I32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GCN: S_ENDPGM 0, implicit [[S_MAX_I32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:sgpr(s32) = G_SMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin.mir
index 39fd79660f76d..3c37a9425d47a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-smin.mir
@@ -1,10 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s
-# RUN: FileCheck -check-prefix=ERR  %s < %t
-
-# ERR-NOT: remark:
-# ERR: remark: <unknown>:0:0: cannot select: %2:sgpr(s32) = G_SMIN %0:sgpr, %1:sgpr (in function: smin_s32_ss)
-# ERR-NOT: remark:
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o -  | FileCheck -check-prefix=GCN %s
 
 ---
 name: smin_s32_ss
@@ -15,10 +10,10 @@ body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
     ; GCN-LABEL: name: smin_s32_ss
-    ; GCN: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; GCN: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; GCN: [[SMIN:%[0-9]+]]:sgpr(s32) = G_SMIN [[COPY]], [[COPY1]]
-    ; GCN: S_ENDPGM 0, implicit [[SMIN]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GCN: [[S_MIN_I32_:%[0-9]+]]:sreg_32 = S_MIN_I32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GCN: S_ENDPGM 0, implicit [[S_MIN_I32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:sgpr(s32) = G_SMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
index 2d8634025c96e..ee5ff53ca6769 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
@@ -21,12 +21,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_4
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -52,12 +52,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_SHORT_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 5)
+    ; GFX6: BUFFER_STORE_SHORT_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_2
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_SHORT_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 5)
+    ; GFX9: BUFFER_STORE_SHORT_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 2, addrspace 5)
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 2, align 2, addrspace 5)
@@ -83,12 +83,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_1
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     %0:vgpr(s32) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 1, align 1, addrspace 5)
@@ -114,12 +114,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_v2s16
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -145,12 +145,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_p3
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -176,12 +176,12 @@ body: |
     ; GFX6: liveins: $vgpr0, $vgpr1
     ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_p5
     ; GFX9: liveins: $vgpr0, $vgpr1
     ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
-    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -209,10 +209,10 @@ body: |
     ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4095, implicit $exec
     ; GFX6: %2:vgpr_32, dead %4:sreg_64_xexec = V_ADD_I32_e64 [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], 0, implicit $exec
     ; GFX6: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_2]], %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_2]], %2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_1_fi_offset_4095
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4095, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 4095, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     %0:vgpr(p5) = G_FRAME_INDEX %stack.0
     %1:vgpr(s32) = G_CONSTANT i32 4095
     %2:vgpr(p5) = G_GEP %0, %1
@@ -239,10 +239,10 @@ body: |
 
     ; GFX6-LABEL: name: store_private_s32_to_1_constant_4095
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; GFX6: BUFFER_STORE_BYTE_OFFSET [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX6: BUFFER_STORE_BYTE_OFFSET [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_1_constant_4095
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
-    ; GFX9: BUFFER_STORE_BYTE_OFFSET [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX9: BUFFER_STORE_BYTE_OFFSET [[V_MOV_B32_e32_]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4095, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     %0:vgpr(p5) = G_CONSTANT i32 4095
     %1:vgpr(s32) = G_CONSTANT i32 0
     G_STORE %1, %0 :: (store 1, align 1, addrspace 5)
@@ -268,11 +268,11 @@ body: |
     ; GFX6-LABEL: name: store_private_s32_to_1_constant_4096
     ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
-    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX6: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     ; GFX9-LABEL: name: store_private_s32_to_1_constant_4096
     ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
-    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
+    ; GFX9: BUFFER_STORE_BYTE_OFFEN [[V_MOV_B32_e32_]], [[V_MOV_B32_e32_1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 1, addrspace 5)
     %0:vgpr(p5) = G_CONSTANT i32 4096
     %1:vgpr(s32) = G_CONSTANT i32 0
     G_STORE %1, %0 :: (store 1, align 1, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir
new file mode 100644
index 0000000000000..6cd1a1a157ecf
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uaddo.mir
@@ -0,0 +1,197 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX10 %s
+
+---
+name: uaddo_s32_s1_sss
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; GFX6-LABEL: name: uaddo_s32_s1_sss
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX6: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX8-LABEL: name: uaddo_s32_s1_sss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX8: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX9-LABEL: name: uaddo_s32_s1_sss
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX9: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX9: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX9: $scc = COPY [[COPY2]]
+    ; GFX9: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX9: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX10-LABEL: name: uaddo_s32_s1_sss
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX10: [[S_ADD_U32_:%[0-9]+]]:sreg_32 = S_ADD_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX10: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX10: $scc = COPY [[COPY2]]
+    ; GFX10: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX10: S_ENDPGM 0, implicit [[S_ADD_U32_]], implicit [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = COPY $sgpr1
+    %2:sgpr(s32), %3:scc(s1) = G_UADDO %0, %1
+    %4:sgpr(s32) = G_SELECT %3, %0, %1
+    S_ENDPGM 0, implicit %2, implicit %4
+...
+
+---
+name: uaddo_s32_s1_vvv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: uaddo_s32_s1_vvv
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: uaddo_s32_s1_vvv
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX8: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: uaddo_s32_s1_vvv
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: uaddo_s32_s1_vvv
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32), %3:vcc(s1) = G_UADDO %0, %1
+    %4:vgpr(s32) = G_SELECT %3, %0, %1
+    S_ENDPGM 0, implicit %2, implicit %4
+...
+
+---
+name: uaddo_s32_s1_vsv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+
+    ; GFX6-LABEL: name: uaddo_s32_s1_vsv
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: uaddo_s32_s1_vsv
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: uaddo_s32_s1_vsv
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: uaddo_s32_s1_vsv
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s32) = COPY $vgpr0
+    %2:vgpr(s32), %3:vcc(s1) = G_UADDO %0, %1
+    %4:vgpr(s32) = G_CONSTANT i32 0
+    %5:vgpr(s32) = G_CONSTANT i32 1
+    %6:vgpr(s32) = G_SELECT %3, %4, %5
+    S_ENDPGM 0, implicit %2, implicit %6
+...
+
+---
+name: uaddo_s32_s1_vvs
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+
+    ; GFX6-LABEL: name: uaddo_s32_s1_vvs
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX6: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: uaddo_s32_s1_vvs
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX8: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: uaddo_s32_s1_vvs
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX9: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: uaddo_s32_s1_vvs
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX10: [[V_ADD_I32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_ADD_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_ADD_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_ADD_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:sgpr(s32) = COPY $sgpr0
+    %2:vgpr(s32), %3:vcc(s1) = G_UADDO %0, %1
+    %4:vgpr(s32) = G_CONSTANT i32 0
+    %5:vgpr(s32) = G_CONSTANT i32 1
+    %6:vgpr(s32) = G_SELECT %3, %4, %5
+    S_ENDPGM 0, implicit %2, implicit %6
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir
new file mode 100644
index 0000000000000..ee2377a66a20a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-uitofp.mir
@@ -0,0 +1,323 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=hawaii -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=WAVE32 %s
+
+---
+name: uitofp_s32_to_s32_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s32_to_s32_vv
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
+    ; WAVE32-LABEL: name: uitofp_s32_to_s32_vv
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_UITOFP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: uitofp_s32_to_s32_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s32_to_s32_vs
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE64: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
+    ; WAVE32-LABEL: name: uitofp_s32_to_s32_vs
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE32: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[COPY]], 0, 0, implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F32_U32_e64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s32) = G_UITOFP %0
+    $vgpr0 = COPY %1
+...
+
+---
+name: uitofp_s32_to_s16_vv
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s32_to_s16_vv
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: uitofp_s32_to_s16_vv
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s16) = G_UITOFP %0
+    %2:vgpr(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: uitofp_s32_to_s16_vs
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s32_to_s16_vs
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE64: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: uitofp_s32_to_s16_vs
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; WAVE32: [[V_CVT_F32_U32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e32 [[COPY]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CVT_F32_U32_e32_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s16) = G_UITOFP %0
+    %2:vgpr(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: uitofp_s1_to_s32_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s32_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 1065353216, implicit $scc
+    ; WAVE64: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s32_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 1065353216, implicit $scc
+    ; WAVE32: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s32) = G_UITOFP %2
+    $sgpr0 = COPY %3
+...
+
+---
+name: uitofp_s1_to_s16_to_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s16_to_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 15360, implicit $scc
+    ; WAVE64: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s16_to_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32_xm0 = S_CSELECT_B32 0, 15360, implicit $scc
+    ; WAVE32: $sgpr0 = COPY [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s16) = G_UITOFP %2
+    %4:sgpr(s32) = G_ANYEXT %3
+    $sgpr0 = COPY %4
+...
+
+---
+name: uitofp_s1_to_s64_s_scc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s64_s_scc
+    ; WAVE64: liveins: $sgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE64: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE64: $scc = COPY [[COPY1]]
+    ; WAVE64: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 0, 4607182418800017408, implicit $scc
+    ; WAVE64: $sgpr0_sgpr1 = COPY [[S_CSELECT_B64_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s64_s_scc
+    ; WAVE32: liveins: $sgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32: S_CMP_EQ_U32 [[COPY]], [[S_MOV_B32_]], implicit-def $scc
+    ; WAVE32: [[COPY1:%[0-9]+]]:sreg_32 = COPY $scc
+    ; WAVE32: $scc = COPY [[COPY1]]
+    ; WAVE32: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 0, 4607182418800017408, implicit $scc
+    ; WAVE32: $sgpr0_sgpr1 = COPY [[S_CSELECT_B64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = G_CONSTANT i32 0
+    %2:scc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:sgpr(s64) = G_UITOFP %2
+    $sgpr0_sgpr1 = COPY %3
+...
+
+---
+name: uitofp_s1_to_s32_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s32_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1065353216, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s32_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1065353216, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s32) = G_UITOFP %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: uitofp_s1_to_s16_to_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s16_to_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1065353216, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s16_to_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1065353216, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: [[V_CVT_F16_F32_e32_:%[0-9]+]]:vgpr_32 = V_CVT_F16_F32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0 = COPY [[V_CVT_F16_F32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s16) = G_UITOFP %2
+    %4:vgpr(s32) = G_ANYEXT %3
+    $vgpr0 = COPY %4
+...
+
+---
+name: uitofp_s1_to_s64_v_vcc
+legalized: true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; WAVE64-LABEL: name: uitofp_s1_to_s64_v_vcc
+    ; WAVE64: liveins: $vgpr0
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE64: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE64: [[V_CVT_F64_U32_e32_:%[0-9]+]]:vreg_64 = V_CVT_F64_U32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE64: $vgpr0_vgpr1 = COPY [[V_CVT_F64_U32_e32_]]
+    ; WAVE32-LABEL: name: uitofp_s1_to_s64_v_vcc
+    ; WAVE32: liveins: $vgpr0
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[COPY]], [[V_MOV_B32_e32_]], implicit $exec
+    ; WAVE32: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[V_CMP_EQ_U32_e64_]], implicit $exec
+    ; WAVE32: [[V_CVT_F64_U32_e32_:%[0-9]+]]:vreg_64 = V_CVT_F64_U32_e32 [[V_CNDMASK_B32_e64_]], implicit $exec
+    ; WAVE32: $vgpr0_vgpr1 = COPY [[V_CVT_F64_U32_e32_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = G_CONSTANT i32 0
+    %2:vcc(s1) = G_ICMP intpred(eq), %0, %1
+    %3:vgpr(s64) = G_UITOFP %2
+    $vgpr0_vgpr1 = COPY %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax.mir
index d4537a07c4dad..a5ae6ab6ca8d8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umax.mir
@@ -1,10 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s
-# RUN: FileCheck -check-prefix=ERR  %s < %t
-
-# ERR-NOT: remark:
-# ERR: remark: <unknown>:0:0: cannot select: %2:sgpr(s32) = G_UMAX %0:sgpr, %1:sgpr (in function: umax_s32_ss)
-# ERR-NOT: remark:
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck -check-prefix=GCN %s
 
 ---
 name: umax_s32_ss
@@ -15,10 +10,10 @@ body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
     ; GCN-LABEL: name: umax_s32_ss
-    ; GCN: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; GCN: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; GCN: [[UMAX:%[0-9]+]]:sgpr(s32) = G_UMAX [[COPY]], [[COPY1]]
-    ; GCN: S_ENDPGM 0, implicit [[UMAX]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GCN: [[S_MAX_U32_:%[0-9]+]]:sreg_32 = S_MAX_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GCN: S_ENDPGM 0, implicit [[S_MAX_U32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:sgpr(s32) = G_UMAX %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin.mir
index cfa573c9fca99..be097fc126d1f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-umin.mir
@@ -1,10 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -global-isel-abort=2 -pass-remarks-missed='gisel*' -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s
-# RUN: FileCheck -check-prefix=ERR  %s < %t
-
-# ERR-NOT: remark:
-# ERR: remark: <unknown>:0:0: cannot select: %2:sgpr(s32) = G_UMIN %0:sgpr, %1:sgpr (in function: umin_s32_ss)
-# ERR-NOT: remark:
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs %s -o - 2>%t | FileCheck -check-prefix=GCN %s
 
 ---
 name: umin_s32_ss
@@ -15,10 +10,10 @@ body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
     ; GCN-LABEL: name: umin_s32_ss
-    ; GCN: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; GCN: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; GCN: [[UMIN:%[0-9]+]]:sgpr(s32) = G_UMIN [[COPY]], [[COPY1]]
-    ; GCN: S_ENDPGM 0, implicit [[UMIN]](s32)
+    ; GCN: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GCN: [[S_MIN_U32_:%[0-9]+]]:sreg_32 = S_MIN_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GCN: S_ENDPGM 0, implicit [[S_MIN_U32_]]
     %0:sgpr(s32) = COPY $sgpr0
     %1:sgpr(s32) = COPY $sgpr1
     %2:sgpr(s32) = G_UMIN %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
index e048827337f5f..8bc4e08c59a70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-unmerge-values.mir
@@ -183,6 +183,11 @@ body: |
   bb.0:
     liveins: $sgpr0_sgpr1_sgpr2_sgpr3
 
+    ; GCN-LABEL: name: test_unmerge_values_s_s64_s_s64_s64_s_s192
+    ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: [[DEF:%[0-9]+]]:sgpr(s192) = G_IMPLICIT_DEF
+    ; GCN: [[UV:%[0-9]+]]:sgpr(s64), [[UV1:%[0-9]+]]:sgpr(s64), [[UV2:%[0-9]+]]:sgpr(s64) = G_UNMERGE_VALUES [[DEF]](s192)
+    ; GCN: S_ENDPGM 0, implicit [[UV]](s64), implicit [[UV1]](s64), implicit [[UV2]](s64)
     %0:sgpr(s192) = G_IMPLICIT_DEF
     %1:sgpr(s64), %2:sgpr(s64), %3:sgpr(s64) = G_UNMERGE_VALUES %0
     S_ENDPGM 0, implicit %1, implicit %2, implicit %3
@@ -229,3 +234,26 @@ body: |
     %1:vgpr(s32), %2:vgpr(s32) = G_UNMERGE_VALUES %0
     S_ENDPGM 0, implicit %1, implicit %2
 ...
+
+---
+name: test_unmerge_values_s_s256_s_s1024
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+
+    ; GCN-LABEL: name: test_unmerge_values_s_s256_s_s1024
+    ; GCN: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; GCN: [[DEF:%[0-9]+]]:sreg_1024 = IMPLICIT_DEF
+    ; GCN: [[COPY:%[0-9]+]]:sreg_256 = COPY [[DEF]].sub0_sub1_sub2_sub3_sub4_sub5_sub6_sub7
+    ; GCN: [[COPY1:%[0-9]+]]:sreg_256 = COPY [[DEF]].sub8_sub9_sub10_sub11_sub12_sub13_sub14_sub15
+    ; GCN: [[COPY2:%[0-9]+]]:sreg_256 = COPY [[DEF]].sub16_sub17_sub18_sub19_sub20_sub21_sub22_sub23
+    ; GCN: [[COPY3:%[0-9]+]]:sreg_256 = COPY [[DEF]].sub24_sub25_sub26_sub27_sub28_sub29_sub30_sub31
+    ; GCN: S_ENDPGM 0, implicit [[COPY]], implicit [[COPY1]], implicit [[COPY2]], implicit [[COPY3]]
+    %0:sgpr(s1024) = G_IMPLICIT_DEF
+    %1:sgpr(s256), %2:sgpr(s256), %3:sgpr(s256), %4:sgpr(s256) = G_UNMERGE_VALUES %0
+    S_ENDPGM 0, implicit %1, implicit %2, implicit %3, implicit %4
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir
new file mode 100644
index 0000000000000..171a2d5659dd3
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-usubo.mir
@@ -0,0 +1,197 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX8 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx1010 -run-pass=instruction-select -verify-machineinstrs -o - %s  | FileCheck -check-prefix=GFX10 %s
+
+---
+name: usubo_s32_s1_sss
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+
+    ; GFX6-LABEL: name: usubo_s32_s1_sss
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX6: [[S_SUB_U32_:%[0-9]+]]:sreg_32 = S_SUB_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX6: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX6: $scc = COPY [[COPY2]]
+    ; GFX6: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX6: S_ENDPGM 0, implicit [[S_SUB_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX8-LABEL: name: usubo_s32_s1_sss
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX8: [[S_SUB_U32_:%[0-9]+]]:sreg_32 = S_SUB_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX8: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX8: $scc = COPY [[COPY2]]
+    ; GFX8: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX8: S_ENDPGM 0, implicit [[S_SUB_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX9-LABEL: name: usubo_s32_s1_sss
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX9: [[S_SUB_U32_:%[0-9]+]]:sreg_32 = S_SUB_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX9: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX9: $scc = COPY [[COPY2]]
+    ; GFX9: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX9: S_ENDPGM 0, implicit [[S_SUB_U32_]], implicit [[S_CSELECT_B32_]]
+    ; GFX10-LABEL: name: usubo_s32_s1_sss
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr1
+    ; GFX10: [[S_SUB_U32_:%[0-9]+]]:sreg_32 = S_SUB_U32 [[COPY]], [[COPY1]], implicit-def $scc
+    ; GFX10: [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $scc
+    ; GFX10: $scc = COPY [[COPY2]]
+    ; GFX10: [[S_CSELECT_B32_:%[0-9]+]]:sreg_32 = S_CSELECT_B32 [[COPY]], [[COPY1]], implicit $scc
+    ; GFX10: S_ENDPGM 0, implicit [[S_SUB_U32_]], implicit [[S_CSELECT_B32_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:sgpr(s32) = COPY $sgpr1
+    %2:sgpr(s32), %3:scc(s1) = G_USUBO %0, %1
+    %4:sgpr(s32) = G_SELECT %3, %0, %1
+    S_ENDPGM 0, implicit %2, implicit %4
+...
+
+---
+name: usubo_s32_s1_vvv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+
+    ; GFX6-LABEL: name: usubo_s32_s1_vvv
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: usubo_s32_s1_vvv
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX8: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: usubo_s32_s1_vvv
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: usubo_s32_s1_vvv
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX10: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[COPY1]], 0, [[COPY]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:vgpr(s32) = COPY $vgpr1
+    %2:vgpr(s32), %3:vcc(s1) = G_USUBO %0, %1
+    %4:vgpr(s32) = G_SELECT %3, %0, %1
+    S_ENDPGM 0, implicit %2, implicit %4
+...
+
+---
+name: usubo_s32_s1_vsv
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+
+    ; GFX6-LABEL: name: usubo_s32_s1_vsv
+    ; GFX6: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: usubo_s32_s1_vsv
+    ; GFX8: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: usubo_s32_s1_vsv
+    ; GFX9: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: usubo_s32_s1_vsv
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:sgpr(s32) = COPY $sgpr0
+    %1:vgpr(s32) = COPY $vgpr0
+    %2:vgpr(s32), %3:vcc(s1) = G_USUBO %0, %1
+    %4:vgpr(s32) = G_CONSTANT i32 0
+    %5:vgpr(s32) = G_CONSTANT i32 1
+    %6:vgpr(s32) = G_SELECT %3, %4, %5
+    S_ENDPGM 0, implicit %2, implicit %6
+...
+
+---
+name: usubo_s32_s1_vvs
+legalized: true
+regBankSelected: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $vgpr0
+
+    ; GFX6-LABEL: name: usubo_s32_s1_vvs
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX6: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX6: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX6: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX6: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX8-LABEL: name: usubo_s32_s1_vvs
+    ; GFX8: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX8: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX8: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX8: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX8: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX8: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX9-LABEL: name: usubo_s32_s1_vvs
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX9: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX9: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX9: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX9: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    ; GFX10-LABEL: name: usubo_s32_s1_vvs
+    ; GFX10: $vcc_hi = IMPLICIT_DEF
+    ; GFX10: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX10: [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr0
+    ; GFX10: [[V_SUB_I32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_I32_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_SUB_I32_e64 [[COPY]], [[COPY1]], 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; GFX10: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; GFX10: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, [[V_MOV_B32_e32_1]], 0, [[V_MOV_B32_e32_]], [[V_SUB_I32_e64_1]], implicit $exec
+    ; GFX10: S_ENDPGM 0, implicit [[V_SUB_I32_e64_]], implicit [[V_CNDMASK_B32_e64_]]
+    %0:vgpr(s32) = COPY $vgpr0
+    %1:sgpr(s32) = COPY $sgpr0
+    %2:vgpr(s32), %3:vcc(s1) = G_USUBO %0, %1
+    %4:vgpr(s32) = G_CONSTANT i32 0
+    %5:vgpr(s32) = G_CONSTANT i32 1
+    %6:vgpr(s32) = G_SELECT %3, %4, %5
+    S_ENDPGM 0, implicit %2, implicit %6
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
index d6f38009f5730..f915b3b0a61a6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-xor.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=tahiti -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -run-pass=instruction-select -verify-machineinstrs -global-isel-abort=0 -o - %s  | FileCheck -check-prefix=WAVE64 %s
 # RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1010 -mattr="+wavefrontsize32" -run-pass=instruction-select -global-isel-abort=0 -verify-machineinstrs -o - %s  | FileCheck -check-prefix=WAVE32 %s
 
 ---
@@ -148,12 +149,10 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: xor_s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; WAVE64: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
-    ; WAVE64: [[TRUNC1:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY1]](s32)
-    ; WAVE64: [[XOR:%[0-9]+]]:vgpr(s16) = G_XOR [[TRUNC]], [[TRUNC1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[XOR]](s16)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; WAVE32-LABEL: name: xor_s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
     ; WAVE32: $vcc_hi = IMPLICIT_DEF
@@ -355,16 +354,17 @@ body: |
     liveins: $vgpr0, $vgpr1
     ; WAVE64-LABEL: name: xor_v2s16_vgpr_vgpr_vgpr
     ; WAVE64: liveins: $vgpr0, $vgpr1
-    ; WAVE64: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE64: [[XOR:%[0-9]+]]:vgpr(<2 x s16>) = G_XOR [[COPY]], [[COPY1]]
-    ; WAVE64: S_ENDPGM 0, implicit [[XOR]](<2 x s16>)
+    ; WAVE64: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE64: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE64: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE64: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     ; WAVE32-LABEL: name: xor_v2s16_vgpr_vgpr_vgpr
     ; WAVE32: liveins: $vgpr0, $vgpr1
-    ; WAVE32: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr1
-    ; WAVE32: [[XOR:%[0-9]+]]:vgpr(<2 x s16>) = G_XOR [[COPY]], [[COPY1]]
-    ; WAVE32: S_ENDPGM 0, implicit [[XOR]](<2 x s16>)
+    ; WAVE32: $vcc_hi = IMPLICIT_DEF
+    ; WAVE32: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; WAVE32: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; WAVE32: [[V_XOR_B32_e64_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e64 [[COPY]], [[COPY1]], implicit $exec
+    ; WAVE32: S_ENDPGM 0, implicit [[V_XOR_B32_e64_]]
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(<2 x s16>) = COPY $vgpr1
     %2:vgpr(<2 x s16>) = G_XOR %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
new file mode 100644
index 0000000000000..0934375dc8034
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgcn-sendmsg.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -march=amdgcn -O0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - | FileCheck %s
+
+declare void @llvm.amdgcn.s.sendmsg(i32 immarg, i32)
+
+define amdgpu_ps void @test_sendmsg(i32 inreg %m0) {
+  ; CHECK-LABEL: name: test_sendmsg
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.s.sendmsg), 12, [[COPY]](s32)
+  ; CHECK:   S_ENDPGM
+  call void @llvm.amdgcn.s.sendmsg(i32 12, i32 %m0)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
index e5e8b6840d2f8..2957b1833ed9e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_ps.ll
@@ -9,10 +9,7 @@ define amdgpu_ps void @disabled_input(float inreg %arg0, float %psinput0, float
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr2
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-  ; CHECK:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), [[C]](s32), [[C1]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), [[C2]](s1), [[C2]](s1)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 15, [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), 0, 0
   ; CHECK:   S_ENDPGM 0
 main_body:
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg0, float %arg0, float %arg0, float %psinput1, i1 false, i1 false) #0
@@ -27,17 +24,14 @@ define amdgpu_ps void @disabled_input_struct(float inreg %arg0, { float, float }
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
   ; CHECK:   [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
-  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-  ; CHECK:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
-  ; CHECK:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
-  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), [[C]](s32), [[C1]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), [[C2]](s1), [[C2]](s1)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 15, [[COPY]](s32), [[COPY]](s32), [[COPY]](s32), [[COPY1]](s32), 0, 0
   ; CHECK:   S_ENDPGM 0
 main_body:
   call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %arg0, float %arg0, float %arg0, float %psinput1, i1 false, i1 false) #0
   ret void
 }
 
-declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1)  #0
+declare void @llvm.amdgcn.exp.f32(i32 immarg, i32 immarg, float, float, float, float, i1 immarg, i1 immarg)  #0
 
 attributes #0 = { nounwind }
 attributes #1 = { "InitialPSInputAddr"="0x00002" }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
index 5f90340fbd602..e93d39289a447 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-amdgpu_vs.ll
@@ -1,9 +1,8 @@
 ; RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=irtranslator -global-isel %s -o - | FileCheck %s
 
-
 ; CHECK-LABEL: name: test_f32_inreg
 ; CHECK: [[S0:%[0-9]+]]:_(s32) = COPY $sgpr2
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S0]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[S0]]
 define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
   ret void
@@ -11,7 +10,7 @@ define amdgpu_vs void @test_f32_inreg(float inreg %arg0) {
 
 ; CHECK-LABEL: name: test_f32
 ; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY $vgpr0
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[V0]]
 define amdgpu_vs void @test_f32(float %arg0) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
   ret void
@@ -33,7 +32,7 @@ define amdgpu_vs void @test_ptr2_inreg(i32 addrspace(4)* inreg %arg0) {
 ; CHECK: [[S4:%[0-9]+]]:_(s32) = COPY $sgpr4
 ; CHECK: [[S34:%[0-9]+]]:_(p4) = G_MERGE_VALUES [[S3]](s32), [[S4]](s32)
 ; CHECK: G_LOAD [[S34]]
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[S2]]
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[S2]](s32)
 define amdgpu_vs void @test_sgpr_alignment0(float inreg %arg0, i32 addrspace(4)* inreg %arg1) {
   %tmp0 = load volatile i32, i32 addrspace(4)* %arg1
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg0, float undef, float undef, float undef, i1 false, i1 false) #0
@@ -45,7 +44,7 @@ define amdgpu_vs void @test_sgpr_alignment0(float inreg %arg0, i32 addrspace(4)*
 ; CHECK: [[S1:%[0-9]+]]:_(s32) = COPY $sgpr3
 ; CHECK: [[V0:%[0-9]+]]:_(s32) = COPY $vgpr0
 ; CHECK: [[V1:%[0-9]+]]:_(s32) = COPY $vgpr1
-; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), %{{[0-9]+}}(s32), %{{[0-9]+}}(s32), [[V0]](s32), [[S0]](s32), [[V1]](s32), [[S1]](s32)
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 32, 15, [[V0]](s32), [[S0]](s32), [[V1]](s32), [[S1]](s32)
 define amdgpu_vs void @test_order(float inreg %arg0, float inreg %arg1, float %arg2, float %arg3) {
   call void @llvm.amdgcn.exp.f32(i32 32, i32 15, float %arg2, float %arg0, float %arg3, float %arg1, i1 false, i1 false) #0
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
index f45037027147f..66a04b45c424d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-struct-return-intrinsics.ll
@@ -9,14 +9,13 @@ define amdgpu_ps void @test_div_scale(float %arg0, float %arg1) {
   ; CHECK:   liveins: $vgpr0, $vgpr1
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
   ; CHECK:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
-  ; CHECK:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
   ; CHECK:   [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
   ; CHECK:   [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
-  ; CHECK:   [[INT:%[0-9]+]]:_(s32), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[COPY]](s32), [[COPY1]](s32), [[C]](s1)
+  ; CHECK:   [[INT:%[0-9]+]]:_(s32), [[INT1:%[0-9]+]]:_(s1) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.scale), [[COPY]](s32), [[COPY1]](s32), -1
   ; CHECK:   [[SEXT:%[0-9]+]]:_(s32) = G_SEXT [[INT1]](s1)
   ; CHECK:   G_STORE [[INT]](s32), [[DEF]](p1) :: (store 4 into `float addrspace(1)* undef`, addrspace 1)
   ; CHECK:   G_STORE [[SEXT]](s32), [[DEF1]](p1) :: (store 4 into `i32 addrspace(1)* undef`, addrspace 1)
-  ; CHECK:   S_ENDPGM
+  ; CHECK:   S_ENDPGM 0
   %call = call { float, i1 } @llvm.amdgcn.div.scale.f32(float %arg0, float %arg1, i1 true)
   %extract0 = extractvalue { float, i1 } %call, 0
   %extract1 = extractvalue { float, i1 } %call, 1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir
index 598432b1c3287..5b53a8c342644 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.mir
@@ -437,6 +437,81 @@ body: |
     S_NOP 0, implicit %16
 ...
 ---
+name: legal_v32s32
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr14, $vgpr15, $vgpr16, $vgpr17, $vgpr18, $vgpr19, $vgpr20, $vgpr21, $vgpr22, $vgpr23, $vgpr24, $vgpr25, $vgpr26, $vgpr27, $vgpr28, $vgpr29, $vgpr30, $vgpr31
+    ; CHECK-LABEL: name: legal_v32s32
+    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s32) = COPY $vgpr8
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s32) = COPY $vgpr9
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s32) = COPY $vgpr10
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s32) = COPY $vgpr11
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s32) = COPY $vgpr12
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
+    ; CHECK: [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
+    ; CHECK: [[COPY16:%[0-9]+]]:_(s32) = COPY $vgpr16
+    ; CHECK: [[COPY17:%[0-9]+]]:_(s32) = COPY $vgpr17
+    ; CHECK: [[COPY18:%[0-9]+]]:_(s32) = COPY $vgpr18
+    ; CHECK: [[COPY19:%[0-9]+]]:_(s32) = COPY $vgpr19
+    ; CHECK: [[COPY20:%[0-9]+]]:_(s32) = COPY $vgpr20
+    ; CHECK: [[COPY21:%[0-9]+]]:_(s32) = COPY $vgpr21
+    ; CHECK: [[COPY22:%[0-9]+]]:_(s32) = COPY $vgpr22
+    ; CHECK: [[COPY23:%[0-9]+]]:_(s32) = COPY $vgpr23
+    ; CHECK: [[COPY24:%[0-9]+]]:_(s32) = COPY $vgpr24
+    ; CHECK: [[COPY25:%[0-9]+]]:_(s32) = COPY $vgpr25
+    ; CHECK: [[COPY26:%[0-9]+]]:_(s32) = COPY $vgpr26
+    ; CHECK: [[COPY27:%[0-9]+]]:_(s32) = COPY $vgpr27
+    ; CHECK: [[COPY28:%[0-9]+]]:_(s32) = COPY $vgpr28
+    ; CHECK: [[COPY29:%[0-9]+]]:_(s32) = COPY $vgpr29
+    ; CHECK: [[COPY30:%[0-9]+]]:_(s32) = COPY $vgpr30
+    ; CHECK: [[COPY31:%[0-9]+]]:_(s32) = COPY $vgpr31
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32), [[COPY12]](s32), [[COPY13]](s32), [[COPY14]](s32), [[COPY15]](s32), [[COPY16]](s32), [[COPY17]](s32), [[COPY18]](s32), [[COPY19]](s32), [[COPY20]](s32), [[COPY21]](s32), [[COPY22]](s32), [[COPY23]](s32), [[COPY24]](s32), [[COPY25]](s32), [[COPY26]](s32), [[COPY27]](s32), [[COPY28]](s32), [[COPY29]](s32), [[COPY30]](s32), [[COPY31]](s32)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<32 x s32>)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = COPY $vgpr6
+    %7:_(s32) = COPY $vgpr7
+    %8:_(s32) = COPY $vgpr8
+    %9:_(s32) = COPY $vgpr9
+    %10:_(s32) = COPY $vgpr10
+    %11:_(s32) = COPY $vgpr11
+    %12:_(s32) = COPY $vgpr12
+    %13:_(s32) = COPY $vgpr13
+    %14:_(s32) = COPY $vgpr14
+    %15:_(s32) = COPY $vgpr15
+    %16:_(s32) = COPY $vgpr16
+    %17:_(s32) = COPY $vgpr17
+    %18:_(s32) = COPY $vgpr18
+    %19:_(s32) = COPY $vgpr19
+    %20:_(s32) = COPY $vgpr20
+    %21:_(s32) = COPY $vgpr21
+    %22:_(s32) = COPY $vgpr22
+    %23:_(s32) = COPY $vgpr23
+    %24:_(s32) = COPY $vgpr24
+    %25:_(s32) = COPY $vgpr25
+    %26:_(s32) = COPY $vgpr26
+    %27:_(s32) = COPY $vgpr27
+    %28:_(s32) = COPY $vgpr28
+    %29:_(s32) = COPY $vgpr29
+    %30:_(s32) = COPY $vgpr30
+    %31:_(s32) = COPY $vgpr31
+    %32:_(<32 x s32>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15, %16, %17, %18, %19, %20, %21, %22, %23, %24, %25, %26, %27, %28, %29, %30, %31
+    S_NOP 0, implicit %32
+...
+---
 name: legal_v2s64
 body: |
   bb.0:
@@ -584,6 +659,50 @@ body: |
     S_NOP 0, implicit %8
 ...
 
+---
+name: legal_v16s64
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5, $vgpr6_vgpr7, $vgpr8_vgpr9, $vgpr10_vgpr11, $vgpr12_vgpr13, $vgpr14_vgpr15, $vgpr16_vgpr17, $vgpr18_vgpr19, $vgpr20_vgpr21, $vgpr22_vgpr23, $vgpr24_vgpr25, $vgpr26_vgpr27, $vgpr28_vgpr29, $vgpr30_vgpr31
+    ; CHECK-LABEL: name: legal_v16s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s64) = COPY $vgpr2_vgpr3
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s64) = COPY $vgpr4_vgpr5
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s64) = COPY $vgpr6_vgpr7
+    ; CHECK: [[COPY4:%[0-9]+]]:_(s64) = COPY $vgpr8_vgpr9
+    ; CHECK: [[COPY5:%[0-9]+]]:_(s64) = COPY $vgpr10_vgpr11
+    ; CHECK: [[COPY6:%[0-9]+]]:_(s64) = COPY $vgpr12_vgpr13
+    ; CHECK: [[COPY7:%[0-9]+]]:_(s64) = COPY $vgpr14_vgpr15
+    ; CHECK: [[COPY8:%[0-9]+]]:_(s64) = COPY $vgpr16_vgpr17
+    ; CHECK: [[COPY9:%[0-9]+]]:_(s64) = COPY $vgpr18_vgpr19
+    ; CHECK: [[COPY10:%[0-9]+]]:_(s64) = COPY $vgpr20_vgpr21
+    ; CHECK: [[COPY11:%[0-9]+]]:_(s64) = COPY $vgpr22_vgpr23
+    ; CHECK: [[COPY12:%[0-9]+]]:_(s64) = COPY $vgpr24_vgpr25
+    ; CHECK: [[COPY13:%[0-9]+]]:_(s64) = COPY $vgpr26_vgpr27
+    ; CHECK: [[COPY14:%[0-9]+]]:_(s64) = COPY $vgpr28_vgpr29
+    ; CHECK: [[COPY15:%[0-9]+]]:_(s64) = COPY $vgpr30_vgpr31
+    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<16 x s64>) = G_BUILD_VECTOR [[COPY]](s64), [[COPY1]](s64), [[COPY2]](s64), [[COPY3]](s64), [[COPY4]](s64), [[COPY5]](s64), [[COPY6]](s64), [[COPY7]](s64), [[COPY8]](s64), [[COPY9]](s64), [[COPY10]](s64), [[COPY11]](s64), [[COPY12]](s64), [[COPY13]](s64), [[COPY14]](s64), [[COPY15]](s64)
+    ; CHECK: S_NOP 0, implicit [[BUILD_VECTOR]](<16 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s64) = COPY $vgpr2_vgpr3
+    %2:_(s64) = COPY $vgpr4_vgpr5
+    %3:_(s64) = COPY $vgpr6_vgpr7
+    %4:_(s64) = COPY $vgpr8_vgpr9
+    %5:_(s64) = COPY $vgpr10_vgpr11
+    %6:_(s64) = COPY $vgpr12_vgpr13
+    %7:_(s64) = COPY $vgpr14_vgpr15
+    %8:_(s64) = COPY $vgpr16_vgpr17
+    %9:_(s64) = COPY $vgpr18_vgpr19
+    %10:_(s64) = COPY $vgpr20_vgpr21
+    %11:_(s64) = COPY $vgpr22_vgpr23
+    %12:_(s64) = COPY $vgpr24_vgpr25
+    %13:_(s64) = COPY $vgpr26_vgpr27
+    %14:_(s64) = COPY $vgpr28_vgpr29
+    %15:_(s64) = COPY $vgpr30_vgpr31
+    %16:_(<16 x s64>) = G_BUILD_VECTOR %0, %1, %2, %3, %4, %5, %6, %7, %8, %9, %10, %11, %12, %13, %14, %15
+    S_NOP 0, implicit %16
+...
+
 ---
 name: legal_v2s128
 body: |
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
index 38f6e02bd25d5..c71010238750c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
@@ -661,3 +661,53 @@ body: |
     %2:_(s32) = G_EXTRACT_VECTOR_ELT %0, %1
     $vgpr0 = COPY %2
 ...
+---
+name: extract_vector_elt_0_v2i64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v2i64
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[COPY]](<2 x s64>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_EXTRACT_VECTOR_ELT %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: extract_vector_elt_0_v8i64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v8i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<8 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[DEF]](<8 x s64>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<8 x s64>) = G_IMPLICIT_DEF
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_EXTRACT_VECTOR_ELT %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
+
+---
+name: extract_vector_elt_0_v16i64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+
+    ; CHECK-LABEL: name: extract_vector_elt_0_v16i64
+    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s64) = G_EXTRACT [[DEF]](<16 x s64>), 0
+    ; CHECK: $vgpr0_vgpr1 = COPY [[EXTRACT]](s64)
+    %0:_(<16 x s64>) = G_IMPLICIT_DEF
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s64) = G_EXTRACT_VECTOR_ELT %0, %1
+    $vgpr0_vgpr1 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
index 2da5ffd1d034e..3412052f7734a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptosi.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
 
 ---
 name: test_fptosi_s32_to_s32
@@ -7,10 +8,14 @@ body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fptosi_s32_to_s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
-    ; CHECK: $vgpr0 = COPY [[FPTOSI]](s32)
+    ; SI-LABEL: name: test_fptosi_s32_to_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
+    ; SI: $vgpr0 = COPY [[FPTOSI]](s32)
+    ; VI-LABEL: name: test_fptosi_s32_to_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
+    ; VI: $vgpr0 = COPY [[FPTOSI]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FPTOSI %0
     $vgpr0 = COPY %1
@@ -22,10 +27,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_fptosi_s64_to_s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
-    ; CHECK: $vgpr0 = COPY [[FPTOSI]](s32)
+    ; SI-LABEL: name: test_fptosi_s64_to_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
+    ; SI: $vgpr0 = COPY [[FPTOSI]](s32)
+    ; VI-LABEL: name: test_fptosi_s64_to_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
+    ; VI: $vgpr0 = COPY [[FPTOSI]](s32)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_FPTOSI %0
     $vgpr0 = COPY %1
@@ -37,13 +46,20 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_fptosi_v2s32_to_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
-    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s32)
-    ; CHECK: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s32)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-LABEL: name: test_fptosi_v2s32_to_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s32)
+    ; SI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fptosi_v2s32_to_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s32)
+    ; VI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = G_FPTOSI %0
     $vgpr0_vgpr1 = COPY %1
@@ -55,14 +71,91 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_fptosi_v2s64_to_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
-    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
-    ; CHECK: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s64)
-    ; CHECK: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s64)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-LABEL: name: test_fptosi_v2s64_to_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s64)
+    ; SI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s64)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fptosi_v2s64_to_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[UV]](s64)
+    ; VI: [[FPTOSI1:%[0-9]+]]:_(s32) = G_FPTOSI [[UV1]](s64)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOSI]](s32), [[FPTOSI1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s32>) = G_FPTOSI %0
     $vgpr0_vgpr1 = COPY %1
 ...
+
+---
+name: test_fptosi_s16_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptosi_s16_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[FPEXT]](s32)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptosi_s16_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s16) = G_FPTOSI [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTOSI]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FPTOSI %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fptosi_s32_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptosi_s32_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptosi_s32_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s32)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; VI: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_FPTOSI %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fptosi_s64_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fptosi_s64_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptosi_s64_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FPTOSI:%[0-9]+]]:_(s32) = G_FPTOSI [[COPY]](s64)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOSI]](s32)
+    ; VI: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s16) = G_FPTOSI %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
index f82c72bd2332e..5b1b200003313 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptoui.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-mesa-mesa3d -run-pass=legalizer %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=tahiti -run-pass=legalizer %s -o - | FileCheck -check-prefix=SI %s
+# RUN: llc -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -run-pass=legalizer %s -o - | FileCheck -check-prefix=VI %s
 
 ---
 name: test_fptoui_s32_s32
@@ -7,10 +8,14 @@ body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_fptoui_s32_s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
-    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
-    ; CHECK: $vgpr0 = COPY [[FPTOUI]](s32)
+    ; SI-LABEL: name: test_fptoui_s32_s32
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; SI: $vgpr0 = COPY [[FPTOUI]](s32)
+    ; VI-LABEL: name: test_fptoui_s32_s32
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; VI: $vgpr0 = COPY [[FPTOUI]](s32)
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_FPTOUI %0
     $vgpr0 = COPY %1
@@ -22,10 +27,14 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_fptoui_s32_s64
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
-    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
-    ; CHECK: $vgpr0 = COPY [[FPTOUI]](s32)
+    ; SI-LABEL: name: test_fptoui_s32_s64
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
+    ; SI: $vgpr0 = COPY [[FPTOUI]](s32)
+    ; VI-LABEL: name: test_fptoui_s32_s64
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
+    ; VI: $vgpr0 = COPY [[FPTOUI]](s32)
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_FPTOUI %0
     $vgpr0 = COPY %1
@@ -37,13 +46,20 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1
 
-    ; CHECK-LABEL: name: test_fptoui_v2s32_to_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
-    ; CHECK: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
-    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s32)
-    ; CHECK: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s32)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-LABEL: name: test_fptoui_v2s32_to_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; SI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s32)
+    ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s32)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fptoui_v2s32_to_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; VI: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s32)
+    ; VI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s32)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = G_FPTOUI %0
     $vgpr0_vgpr1 = COPY %1
@@ -55,14 +71,91 @@ body: |
   bb.0:
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3
 
-    ; CHECK-LABEL: name: test_fptoui_v2s64_to_v2s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
-    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
-    ; CHECK: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s64)
-    ; CHECK: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s64)
-    ; CHECK: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
-    ; CHECK: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; SI-LABEL: name: test_fptoui_v2s64_to_v2s32
+    ; SI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; SI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s64)
+    ; SI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s64)
+    ; SI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; SI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
+    ; VI-LABEL: name: test_fptoui_v2s64_to_v2s32
+    ; VI: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; VI: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[UV]](s64)
+    ; VI: [[FPTOUI1:%[0-9]+]]:_(s32) = G_FPTOUI [[UV1]](s64)
+    ; VI: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[FPTOUI]](s32), [[FPTOUI1]](s32)
+    ; VI: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
     %0:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
     %1:_(<2 x s32>) = G_FPTOUI %0
     $vgpr0_vgpr1 = COPY %1
 ...
+
+---
+name: test_fptoui_s16_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptoui_s16_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; SI: [[FPEXT:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC]](s16)
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[FPEXT]](s32)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOUI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptoui_s16_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[COPY]](s32)
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s16) = G_FPTOUI [[TRUNC]](s16)
+    ; VI: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[FPTOUI]](s16)
+    ; VI: $vgpr0 = COPY [[ANYEXT]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_TRUNC %0
+    %2:_(s16) = G_FPTOUI %1
+    %3:_(s32) = G_ANYEXT %2
+    $vgpr0 = COPY %3
+...
+
+---
+name: test_fptoui_s32_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0
+
+    ; SI-LABEL: name: test_fptoui_s32_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOUI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptoui_s32_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s32)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOUI]](s32)
+    ; VI: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s16) = G_FPTOUI %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
+
+---
+name: test_fptoui_s64_to_s16
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; SI-LABEL: name: test_fptoui_s64_to_s16
+    ; SI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; SI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
+    ; SI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOUI]](s32)
+    ; SI: $vgpr0 = COPY [[COPY1]](s32)
+    ; VI-LABEL: name: test_fptoui_s64_to_s16
+    ; VI: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; VI: [[FPTOUI:%[0-9]+]]:_(s32) = G_FPTOUI [[COPY]](s64)
+    ; VI: [[COPY1:%[0-9]+]]:_(s32) = COPY [[FPTOUI]](s32)
+    ; VI: $vgpr0 = COPY [[COPY1]](s32)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(s16) = G_FPTOUI %0
+    %2:_(s32) = G_ANYEXT %1
+    $vgpr0 = COPY %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
index e9effef375ce6..900852497185a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
@@ -166,9 +166,8 @@ body: |
   bb.0:
 
     ; CHECK-LABEL: name: test_implicit_def_s1024
-    ; CHECK: [[DEF:%[0-9]+]]:_(s512) = G_IMPLICIT_DEF
-    ; CHECK: [[DEF1:%[0-9]+]]:_(s512) = G_IMPLICIT_DEF
-    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[DEF]](s512), 0
+    ; CHECK: [[DEF:%[0-9]+]]:_(s1024) = G_IMPLICIT_DEF
+    ; CHECK: [[EXTRACT:%[0-9]+]]:_(s32) = G_EXTRACT [[DEF]](s1024), 0
     ; CHECK: $vgpr0 = COPY [[EXTRACT]](s32)
     %0:_(s1024) = G_IMPLICIT_DEF
     %1:_(s32) = G_EXTRACT %0, 0
@@ -289,10 +288,8 @@ body: |
   bb.0:
 
     ; CHECK-LABEL: name: test_implicit_def_v32s32
-    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[DEF1:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-    ; CHECK: [[CONCAT_VECTORS:%[0-9]+]]:_(<32 x s32>) = G_CONCAT_VECTORS [[DEF]](<16 x s32>), [[DEF1]](<16 x s32>)
-    ; CHECK: S_NOP 0, implicit [[CONCAT_VECTORS]](<32 x s32>)
+    ; CHECK: [[DEF:%[0-9]+]]:_(<32 x s32>) = G_IMPLICIT_DEF
+    ; CHECK: S_NOP 0, implicit [[DEF]](<32 x s32>)
     %0:_(<32 x s32>) = G_IMPLICIT_DEF
     S_NOP 0, implicit %0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
index 75759b3e7548a..df1cef761d271 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
@@ -96,3 +96,22 @@ body: |
     %3:_(<16 x s32>) = G_INSERT_VECTOR_ELT %0, %1, %2
     S_ENDPGM 0, implicit %3
 ...
+
+---
+name: insert_vector_elt_0_v16s64
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+
+    ; CHECK-LABEL: name: insert_vector_elt_0_v16s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
+    ; CHECK: [[DEF:%[0-9]+]]:_(<16 x s64>) = G_IMPLICIT_DEF
+    ; CHECK: [[INSERT:%[0-9]+]]:_(<16 x s64>) = G_INSERT [[DEF]], [[COPY]](s64), 0
+    ; CHECK: S_ENDPGM 0, implicit [[INSERT]](<16 x s64>)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(<16 x s64>) = G_IMPLICIT_DEF
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(<16 x s64>) = G_INSERT_VECTOR_ELT %1, %0, %2
+    S_ENDPGM 0, implicit %3
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index f17323cdd3f5a..b8c330a08f6c3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -546,8 +546,7 @@ body: |
   ; CHECK: bb.0:
   ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
   ; CHECK:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
-  ; CHECK:   [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
-  ; CHECK:   [[DEF1:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF:%[0-9]+]]:_(<32 x s32>) = G_IMPLICIT_DEF
   ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr4
   ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
   ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
@@ -555,10 +554,8 @@ body: |
   ; CHECK:   G_BR %bb.2
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.2(0x80000000)
-  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
-  ; CHECK:   [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](<16 x s32>)
-  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
-  ; CHECK:   [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](<16 x s32>)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32), [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<32 x s32>)
+  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32), [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<32 x s32>)
   ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UV32]]
   ; CHECK:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UV33]]
   ; CHECK:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[UV2]], [[UV34]]
@@ -592,13 +589,10 @@ body: |
   ; CHECK:   [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[UV30]], [[UV62]]
   ; CHECK:   [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UV31]], [[UV63]]
   ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<32 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32), [[ADD2]](s32), [[ADD3]](s32), [[ADD4]](s32), [[ADD5]](s32), [[ADD6]](s32), [[ADD7]](s32), [[ADD8]](s32), [[ADD9]](s32), [[ADD10]](s32), [[ADD11]](s32), [[ADD12]](s32), [[ADD13]](s32), [[ADD14]](s32), [[ADD15]](s32), [[ADD16]](s32), [[ADD17]](s32), [[ADD18]](s32), [[ADD19]](s32), [[ADD20]](s32), [[ADD21]](s32), [[ADD22]](s32), [[ADD23]](s32), [[ADD24]](s32), [[ADD25]](s32), [[ADD26]](s32), [[ADD27]](s32), [[ADD28]](s32), [[ADD29]](s32), [[ADD30]](s32), [[ADD31]](s32)
-  ; CHECK:   [[UV64:%[0-9]+]]:_(<16 x s32>), [[UV65:%[0-9]+]]:_(<16 x s32>) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<32 x s32>)
   ; CHECK:   G_BR %bb.2
   ; CHECK: bb.2:
-  ; CHECK:   [[PHI:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[UV64]](<16 x s32>), %bb.1
-  ; CHECK:   [[PHI1:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF1]](<16 x s32>), %bb.0, [[UV65]](<16 x s32>), %bb.1
-  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<32 x s32>) = G_CONCAT_VECTORS [[PHI]](<16 x s32>), [[PHI1]](<16 x s32>)
-  ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31, implicit [[CONCAT_VECTORS]](<32 x s32>)
+  ; CHECK:   [[PHI:%[0-9]+]]:_(<32 x s32>) = G_PHI [[DEF]](<32 x s32>), %bb.0, [[BUILD_VECTOR]](<32 x s32>), %bb.1
+  ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31, implicit [[PHI]](<32 x s32>)
   bb.0:
     successors: %bb.1, %bb.2
     liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
@@ -621,6 +615,132 @@ body: |
     S_SETPC_B64 undef $sgpr30_sgpr31, implicit %5
 
 ...
+
+---
+name: test_phi_v64s32
+tracksRegLiveness: true
+
+body: |
+  ; CHECK-LABEL: name: test_phi_v64s32
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+  ; CHECK:   [[DEF:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF3:%[0-9]+]]:_(<16 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; CHECK:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; CHECK:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+  ; CHECK:   G_BRCOND [[ICMP]](s1), %bb.1
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32), [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32), [[UV12:%[0-9]+]]:_(s32), [[UV13:%[0-9]+]]:_(s32), [[UV14:%[0-9]+]]:_(s32), [[UV15:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV16:%[0-9]+]]:_(s32), [[UV17:%[0-9]+]]:_(s32), [[UV18:%[0-9]+]]:_(s32), [[UV19:%[0-9]+]]:_(s32), [[UV20:%[0-9]+]]:_(s32), [[UV21:%[0-9]+]]:_(s32), [[UV22:%[0-9]+]]:_(s32), [[UV23:%[0-9]+]]:_(s32), [[UV24:%[0-9]+]]:_(s32), [[UV25:%[0-9]+]]:_(s32), [[UV26:%[0-9]+]]:_(s32), [[UV27:%[0-9]+]]:_(s32), [[UV28:%[0-9]+]]:_(s32), [[UV29:%[0-9]+]]:_(s32), [[UV30:%[0-9]+]]:_(s32), [[UV31:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](<16 x s32>)
+  ; CHECK:   [[UV32:%[0-9]+]]:_(s32), [[UV33:%[0-9]+]]:_(s32), [[UV34:%[0-9]+]]:_(s32), [[UV35:%[0-9]+]]:_(s32), [[UV36:%[0-9]+]]:_(s32), [[UV37:%[0-9]+]]:_(s32), [[UV38:%[0-9]+]]:_(s32), [[UV39:%[0-9]+]]:_(s32), [[UV40:%[0-9]+]]:_(s32), [[UV41:%[0-9]+]]:_(s32), [[UV42:%[0-9]+]]:_(s32), [[UV43:%[0-9]+]]:_(s32), [[UV44:%[0-9]+]]:_(s32), [[UV45:%[0-9]+]]:_(s32), [[UV46:%[0-9]+]]:_(s32), [[UV47:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF2]](<16 x s32>)
+  ; CHECK:   [[UV48:%[0-9]+]]:_(s32), [[UV49:%[0-9]+]]:_(s32), [[UV50:%[0-9]+]]:_(s32), [[UV51:%[0-9]+]]:_(s32), [[UV52:%[0-9]+]]:_(s32), [[UV53:%[0-9]+]]:_(s32), [[UV54:%[0-9]+]]:_(s32), [[UV55:%[0-9]+]]:_(s32), [[UV56:%[0-9]+]]:_(s32), [[UV57:%[0-9]+]]:_(s32), [[UV58:%[0-9]+]]:_(s32), [[UV59:%[0-9]+]]:_(s32), [[UV60:%[0-9]+]]:_(s32), [[UV61:%[0-9]+]]:_(s32), [[UV62:%[0-9]+]]:_(s32), [[UV63:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF3]](<16 x s32>)
+  ; CHECK:   [[UV64:%[0-9]+]]:_(s32), [[UV65:%[0-9]+]]:_(s32), [[UV66:%[0-9]+]]:_(s32), [[UV67:%[0-9]+]]:_(s32), [[UV68:%[0-9]+]]:_(s32), [[UV69:%[0-9]+]]:_(s32), [[UV70:%[0-9]+]]:_(s32), [[UV71:%[0-9]+]]:_(s32), [[UV72:%[0-9]+]]:_(s32), [[UV73:%[0-9]+]]:_(s32), [[UV74:%[0-9]+]]:_(s32), [[UV75:%[0-9]+]]:_(s32), [[UV76:%[0-9]+]]:_(s32), [[UV77:%[0-9]+]]:_(s32), [[UV78:%[0-9]+]]:_(s32), [[UV79:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF]](<16 x s32>)
+  ; CHECK:   [[UV80:%[0-9]+]]:_(s32), [[UV81:%[0-9]+]]:_(s32), [[UV82:%[0-9]+]]:_(s32), [[UV83:%[0-9]+]]:_(s32), [[UV84:%[0-9]+]]:_(s32), [[UV85:%[0-9]+]]:_(s32), [[UV86:%[0-9]+]]:_(s32), [[UV87:%[0-9]+]]:_(s32), [[UV88:%[0-9]+]]:_(s32), [[UV89:%[0-9]+]]:_(s32), [[UV90:%[0-9]+]]:_(s32), [[UV91:%[0-9]+]]:_(s32), [[UV92:%[0-9]+]]:_(s32), [[UV93:%[0-9]+]]:_(s32), [[UV94:%[0-9]+]]:_(s32), [[UV95:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF1]](<16 x s32>)
+  ; CHECK:   [[UV96:%[0-9]+]]:_(s32), [[UV97:%[0-9]+]]:_(s32), [[UV98:%[0-9]+]]:_(s32), [[UV99:%[0-9]+]]:_(s32), [[UV100:%[0-9]+]]:_(s32), [[UV101:%[0-9]+]]:_(s32), [[UV102:%[0-9]+]]:_(s32), [[UV103:%[0-9]+]]:_(s32), [[UV104:%[0-9]+]]:_(s32), [[UV105:%[0-9]+]]:_(s32), [[UV106:%[0-9]+]]:_(s32), [[UV107:%[0-9]+]]:_(s32), [[UV108:%[0-9]+]]:_(s32), [[UV109:%[0-9]+]]:_(s32), [[UV110:%[0-9]+]]:_(s32), [[UV111:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF2]](<16 x s32>)
+  ; CHECK:   [[UV112:%[0-9]+]]:_(s32), [[UV113:%[0-9]+]]:_(s32), [[UV114:%[0-9]+]]:_(s32), [[UV115:%[0-9]+]]:_(s32), [[UV116:%[0-9]+]]:_(s32), [[UV117:%[0-9]+]]:_(s32), [[UV118:%[0-9]+]]:_(s32), [[UV119:%[0-9]+]]:_(s32), [[UV120:%[0-9]+]]:_(s32), [[UV121:%[0-9]+]]:_(s32), [[UV122:%[0-9]+]]:_(s32), [[UV123:%[0-9]+]]:_(s32), [[UV124:%[0-9]+]]:_(s32), [[UV125:%[0-9]+]]:_(s32), [[UV126:%[0-9]+]]:_(s32), [[UV127:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[DEF3]](<16 x s32>)
+  ; CHECK:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[UV]], [[UV64]]
+  ; CHECK:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[UV1]], [[UV65]]
+  ; CHECK:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[UV2]], [[UV66]]
+  ; CHECK:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[UV3]], [[UV67]]
+  ; CHECK:   [[ADD4:%[0-9]+]]:_(s32) = G_ADD [[UV4]], [[UV68]]
+  ; CHECK:   [[ADD5:%[0-9]+]]:_(s32) = G_ADD [[UV5]], [[UV69]]
+  ; CHECK:   [[ADD6:%[0-9]+]]:_(s32) = G_ADD [[UV6]], [[UV70]]
+  ; CHECK:   [[ADD7:%[0-9]+]]:_(s32) = G_ADD [[UV7]], [[UV71]]
+  ; CHECK:   [[ADD8:%[0-9]+]]:_(s32) = G_ADD [[UV8]], [[UV72]]
+  ; CHECK:   [[ADD9:%[0-9]+]]:_(s32) = G_ADD [[UV9]], [[UV73]]
+  ; CHECK:   [[ADD10:%[0-9]+]]:_(s32) = G_ADD [[UV10]], [[UV74]]
+  ; CHECK:   [[ADD11:%[0-9]+]]:_(s32) = G_ADD [[UV11]], [[UV75]]
+  ; CHECK:   [[ADD12:%[0-9]+]]:_(s32) = G_ADD [[UV12]], [[UV76]]
+  ; CHECK:   [[ADD13:%[0-9]+]]:_(s32) = G_ADD [[UV13]], [[UV77]]
+  ; CHECK:   [[ADD14:%[0-9]+]]:_(s32) = G_ADD [[UV14]], [[UV78]]
+  ; CHECK:   [[ADD15:%[0-9]+]]:_(s32) = G_ADD [[UV15]], [[UV79]]
+  ; CHECK:   [[ADD16:%[0-9]+]]:_(s32) = G_ADD [[UV16]], [[UV80]]
+  ; CHECK:   [[ADD17:%[0-9]+]]:_(s32) = G_ADD [[UV17]], [[UV81]]
+  ; CHECK:   [[ADD18:%[0-9]+]]:_(s32) = G_ADD [[UV18]], [[UV82]]
+  ; CHECK:   [[ADD19:%[0-9]+]]:_(s32) = G_ADD [[UV19]], [[UV83]]
+  ; CHECK:   [[ADD20:%[0-9]+]]:_(s32) = G_ADD [[UV20]], [[UV84]]
+  ; CHECK:   [[ADD21:%[0-9]+]]:_(s32) = G_ADD [[UV21]], [[UV85]]
+  ; CHECK:   [[ADD22:%[0-9]+]]:_(s32) = G_ADD [[UV22]], [[UV86]]
+  ; CHECK:   [[ADD23:%[0-9]+]]:_(s32) = G_ADD [[UV23]], [[UV87]]
+  ; CHECK:   [[ADD24:%[0-9]+]]:_(s32) = G_ADD [[UV24]], [[UV88]]
+  ; CHECK:   [[ADD25:%[0-9]+]]:_(s32) = G_ADD [[UV25]], [[UV89]]
+  ; CHECK:   [[ADD26:%[0-9]+]]:_(s32) = G_ADD [[UV26]], [[UV90]]
+  ; CHECK:   [[ADD27:%[0-9]+]]:_(s32) = G_ADD [[UV27]], [[UV91]]
+  ; CHECK:   [[ADD28:%[0-9]+]]:_(s32) = G_ADD [[UV28]], [[UV92]]
+  ; CHECK:   [[ADD29:%[0-9]+]]:_(s32) = G_ADD [[UV29]], [[UV93]]
+  ; CHECK:   [[ADD30:%[0-9]+]]:_(s32) = G_ADD [[UV30]], [[UV94]]
+  ; CHECK:   [[ADD31:%[0-9]+]]:_(s32) = G_ADD [[UV31]], [[UV95]]
+  ; CHECK:   [[ADD32:%[0-9]+]]:_(s32) = G_ADD [[UV32]], [[UV96]]
+  ; CHECK:   [[ADD33:%[0-9]+]]:_(s32) = G_ADD [[UV33]], [[UV97]]
+  ; CHECK:   [[ADD34:%[0-9]+]]:_(s32) = G_ADD [[UV34]], [[UV98]]
+  ; CHECK:   [[ADD35:%[0-9]+]]:_(s32) = G_ADD [[UV35]], [[UV99]]
+  ; CHECK:   [[ADD36:%[0-9]+]]:_(s32) = G_ADD [[UV36]], [[UV100]]
+  ; CHECK:   [[ADD37:%[0-9]+]]:_(s32) = G_ADD [[UV37]], [[UV101]]
+  ; CHECK:   [[ADD38:%[0-9]+]]:_(s32) = G_ADD [[UV38]], [[UV102]]
+  ; CHECK:   [[ADD39:%[0-9]+]]:_(s32) = G_ADD [[UV39]], [[UV103]]
+  ; CHECK:   [[ADD40:%[0-9]+]]:_(s32) = G_ADD [[UV40]], [[UV104]]
+  ; CHECK:   [[ADD41:%[0-9]+]]:_(s32) = G_ADD [[UV41]], [[UV105]]
+  ; CHECK:   [[ADD42:%[0-9]+]]:_(s32) = G_ADD [[UV42]], [[UV106]]
+  ; CHECK:   [[ADD43:%[0-9]+]]:_(s32) = G_ADD [[UV43]], [[UV107]]
+  ; CHECK:   [[ADD44:%[0-9]+]]:_(s32) = G_ADD [[UV44]], [[UV108]]
+  ; CHECK:   [[ADD45:%[0-9]+]]:_(s32) = G_ADD [[UV45]], [[UV109]]
+  ; CHECK:   [[ADD46:%[0-9]+]]:_(s32) = G_ADD [[UV46]], [[UV110]]
+  ; CHECK:   [[ADD47:%[0-9]+]]:_(s32) = G_ADD [[UV47]], [[UV111]]
+  ; CHECK:   [[ADD48:%[0-9]+]]:_(s32) = G_ADD [[UV48]], [[UV112]]
+  ; CHECK:   [[ADD49:%[0-9]+]]:_(s32) = G_ADD [[UV49]], [[UV113]]
+  ; CHECK:   [[ADD50:%[0-9]+]]:_(s32) = G_ADD [[UV50]], [[UV114]]
+  ; CHECK:   [[ADD51:%[0-9]+]]:_(s32) = G_ADD [[UV51]], [[UV115]]
+  ; CHECK:   [[ADD52:%[0-9]+]]:_(s32) = G_ADD [[UV52]], [[UV116]]
+  ; CHECK:   [[ADD53:%[0-9]+]]:_(s32) = G_ADD [[UV53]], [[UV117]]
+  ; CHECK:   [[ADD54:%[0-9]+]]:_(s32) = G_ADD [[UV54]], [[UV118]]
+  ; CHECK:   [[ADD55:%[0-9]+]]:_(s32) = G_ADD [[UV55]], [[UV119]]
+  ; CHECK:   [[ADD56:%[0-9]+]]:_(s32) = G_ADD [[UV56]], [[UV120]]
+  ; CHECK:   [[ADD57:%[0-9]+]]:_(s32) = G_ADD [[UV57]], [[UV121]]
+  ; CHECK:   [[ADD58:%[0-9]+]]:_(s32) = G_ADD [[UV58]], [[UV122]]
+  ; CHECK:   [[ADD59:%[0-9]+]]:_(s32) = G_ADD [[UV59]], [[UV123]]
+  ; CHECK:   [[ADD60:%[0-9]+]]:_(s32) = G_ADD [[UV60]], [[UV124]]
+  ; CHECK:   [[ADD61:%[0-9]+]]:_(s32) = G_ADD [[UV61]], [[UV125]]
+  ; CHECK:   [[ADD62:%[0-9]+]]:_(s32) = G_ADD [[UV62]], [[UV126]]
+  ; CHECK:   [[ADD63:%[0-9]+]]:_(s32) = G_ADD [[UV63]], [[UV127]]
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:_(<64 x s32>) = G_BUILD_VECTOR [[ADD]](s32), [[ADD1]](s32), [[ADD2]](s32), [[ADD3]](s32), [[ADD4]](s32), [[ADD5]](s32), [[ADD6]](s32), [[ADD7]](s32), [[ADD8]](s32), [[ADD9]](s32), [[ADD10]](s32), [[ADD11]](s32), [[ADD12]](s32), [[ADD13]](s32), [[ADD14]](s32), [[ADD15]](s32), [[ADD16]](s32), [[ADD17]](s32), [[ADD18]](s32), [[ADD19]](s32), [[ADD20]](s32), [[ADD21]](s32), [[ADD22]](s32), [[ADD23]](s32), [[ADD24]](s32), [[ADD25]](s32), [[ADD26]](s32), [[ADD27]](s32), [[ADD28]](s32), [[ADD29]](s32), [[ADD30]](s32), [[ADD31]](s32), [[ADD32]](s32), [[ADD33]](s32), [[ADD34]](s32), [[ADD35]](s32), [[ADD36]](s32), [[ADD37]](s32), [[ADD38]](s32), [[ADD39]](s32), [[ADD40]](s32), [[ADD41]](s32), [[ADD42]](s32), [[ADD43]](s32), [[ADD44]](s32), [[ADD45]](s32), [[ADD46]](s32), [[ADD47]](s32), [[ADD48]](s32), [[ADD49]](s32), [[ADD50]](s32), [[ADD51]](s32), [[ADD52]](s32), [[ADD53]](s32), [[ADD54]](s32), [[ADD55]](s32), [[ADD56]](s32), [[ADD57]](s32), [[ADD58]](s32), [[ADD59]](s32), [[ADD60]](s32), [[ADD61]](s32), [[ADD62]](s32), [[ADD63]](s32)
+  ; CHECK:   [[UV128:%[0-9]+]]:_(<16 x s32>), [[UV129:%[0-9]+]]:_(<16 x s32>), [[UV130:%[0-9]+]]:_(<16 x s32>), [[UV131:%[0-9]+]]:_(<16 x s32>) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<64 x s32>)
+  ; CHECK:   G_BR %bb.2
+  ; CHECK: bb.2:
+  ; CHECK:   [[PHI:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF]](<16 x s32>), %bb.0, [[UV128]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI1:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF1]](<16 x s32>), %bb.0, [[UV129]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI2:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF2]](<16 x s32>), %bb.0, [[UV130]](<16 x s32>), %bb.1
+  ; CHECK:   [[PHI3:%[0-9]+]]:_(<16 x s32>) = G_PHI [[DEF3]](<16 x s32>), %bb.0, [[UV131]](<16 x s32>), %bb.1
+  ; CHECK:   [[CONCAT_VECTORS:%[0-9]+]]:_(<64 x s32>) = G_CONCAT_VECTORS [[PHI]](<16 x s32>), [[PHI1]](<16 x s32>), [[PHI2]](<16 x s32>), [[PHI3]](<16 x s32>)
+  ; CHECK:   S_SETPC_B64 undef $sgpr30_sgpr31, implicit [[CONCAT_VECTORS]](<64 x s32>)
+  bb.0:
+    successors: %bb.1, %bb.2
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3, $vgpr4
+
+    %0:_(<64 x s32>) = G_IMPLICIT_DEF
+    %1:_(s32) = COPY $vgpr4
+    %2:_(s32) = G_CONSTANT i32 0
+    %3:_(s1) = G_ICMP intpred(eq), %1, %2
+    G_BRCOND %3, %bb.1
+    G_BR %bb.2
+
+  bb.1:
+    successors: %bb.2
+
+    %4:_(<64 x s32>) = G_ADD %0, %0
+    G_BR %bb.2
+
+  bb.2:
+    %5:_(<64 x s32>) = G_PHI %0, %bb.0, %4, %bb.1
+    S_SETPC_B64 undef $sgpr30_sgpr31, implicit %5
+
+...
+
 ---
 name: test_phi_s64
 tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
index 8c4d9d0a69aa6..5b9aa93cf9e3a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
@@ -143,12 +143,12 @@ body: |
 ...
 
 ---
-name: test_sitofp_s32_to_s16
+name: test_sitofp_s16_to_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_sitofp_s32_to_s16
+    ; CHECK-LABEL: name: test_sitofp_s16_to_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[SITOFP:%[0-9]+]]:_(s16) = G_SITOFP [[COPY]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[SITOFP]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
index 2ac828763893d..6fa7d4418a11e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
@@ -112,12 +112,12 @@ body: |
 ...
 
 ---
-name: test_uitofp_s32_to_s16
+name: test_uitofp_s16_to_s32
 body: |
   bb.0:
     liveins: $vgpr0
 
-    ; CHECK-LABEL: name: test_uitofp_s32_to_s16
+    ; CHECK-LABEL: name: test_uitofp_s16_to_s32
     ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
     ; CHECK: [[UITOFP:%[0-9]+]]:_(s16) = G_UITOFP [[COPY]](s32)
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[UITOFP]](s16)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
new file mode 100644
index 0000000000000..77c2f80ac02b6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.ll
@@ -0,0 +1,2 @@
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.ll | FileCheck -check-prefix=GCN %S/../llvm.amdgcn.init.exec.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll
new file mode 100644
index 0000000000000..1e8fce0e34f2c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.init.exec.wave32.ll
@@ -0,0 +1,2 @@
+; Runs original SDAG test with -global-isel
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs < %S/../llvm.amdgcn.init.exec.wave32.ll | FileCheck -check-prefixes=GCN,GFX1032  %S/../llvm.amdgcn.init.exec.wave32.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
new file mode 100644
index 0000000000000..0643505bf9995
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -0,0 +1,519 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7 + 4095, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7 + 4095, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY7]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> inreg %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Make sure unpack code is emitted outside of loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY9]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY6]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   %11:vgpr_32, dead %21:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_16
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; PACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; PACKED:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4095
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   %11:vgpr_32, dead %22:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4095
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4095
+  ; PACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; PACKED:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; UNPACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   %11:vgpr_32, dead %22:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY8]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XY_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_voffset_add_4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; PACKED:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; PACKED:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; PACKED:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; PACKED:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; PACKED:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; PACKED:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XY_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; UNPACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096
+  ; UNPACKED: bb.1 (%ir-block.0):
+  ; UNPACKED:   successors: %bb.2(0x80000000)
+  ; UNPACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; UNPACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; UNPACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; UNPACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; UNPACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; UNPACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; UNPACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; UNPACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; UNPACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; UNPACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; UNPACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; UNPACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; UNPACKED:   %13:vgpr_32, dead %47:sreg_64_xexec = V_ADD_I32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; UNPACKED:   [[S_MOV_B32_1:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; UNPACKED:   [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY9]], [[COPY4]], implicit $exec
+  ; UNPACKED:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+  ; UNPACKED:   [[V_LSHRREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e64 [[COPY10]], [[COPY5]], implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[V_LSHRREV_B32_e64_]], %subreg.sub1, [[COPY5]], %subreg.sub2, [[V_LSHRREV_B32_e64_1]], %subreg.sub3
+  ; UNPACKED:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; UNPACKED:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; UNPACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; UNPACKED: bb.2:
+  ; UNPACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY11]], implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]].sub0, implicit $exec
+  ; UNPACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]].sub1, implicit $exec
+  ; UNPACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; UNPACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY12]], implicit $exec
+  ; UNPACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; UNPACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; UNPACKED:   BUFFER_STORE_FORMAT_D16_XYZW_gfx80_OFFEN_exact [[REG_SEQUENCE1]], %13, [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; UNPACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; UNPACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; UNPACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; UNPACKED: bb.3:
+  ; UNPACKED:   successors: %bb.4(0x80000000)
+  ; UNPACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; UNPACKED: bb.4:
+  ; UNPACKED:   S_ENDPGM 0
+  ; PACKED-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16_add_4096
+  ; PACKED: bb.1 (%ir-block.0):
+  ; PACKED:   successors: %bb.2(0x80000000)
+  ; PACKED:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; PACKED:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; PACKED:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; PACKED:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; PACKED:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; PACKED:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; PACKED:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; PACKED:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; PACKED:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; PACKED:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; PACKED:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; PACKED:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; PACKED:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; PACKED:   %13:vgpr_32, dead %31:sreg_64_xexec = V_ADD_I32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; PACKED:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; PACKED:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; PACKED:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; PACKED: bb.2:
+  ; PACKED:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; PACKED:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; PACKED:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; PACKED:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; PACKED:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY10]], implicit $exec
+  ; PACKED:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; PACKED:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; PACKED:   BUFFER_STORE_FORMAT_D16_XYZW_OFFEN_exact [[REG_SEQUENCE1]], %13, [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; PACKED:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; PACKED:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; PACKED:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; PACKED: bb.3:
+  ; PACKED:   successors: %bb.4(0x80000000)
+  ; PACKED:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; PACKED: bb.4:
+  ; PACKED:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
new file mode 100644
index 0000000000000..5fa523f7b8da2
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f32.ll
@@ -0,0 +1,314 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=tonga -stop-after=instruction-select -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx810 -stop-after=instruction-select -o - %s | FileCheck %s
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_X_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7 + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> inreg %rsrc, <3 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZ_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> inreg %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY10]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY11]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE4]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_soffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %13:vgpr_32, dead %15:sreg_64_xexec = V_ADD_I32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], %13, [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4095
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %13:vgpr_32, dead %15:sreg_64_xexec = V_ADD_I32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], %13, [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32_voffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %13:vgpr_32, dead %15:sreg_64_xexec = V_ADD_I32_e64 [[COPY6]], [[COPY8]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_FORMAT_XY_OFFEN_exact [[REG_SEQUENCE1]], %13, [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+
+; Check what happens with offset add inside a waterfall loop
+define amdgpu_ps void @raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096(<4 x i32> %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store_format__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %15:vgpr_32, dead %33:sreg_64_xexec = V_ADD_I32_e64 [[COPY8]], [[COPY10]], 0, implicit $exec
+  ; CHECK:   [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY12:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY11]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY11]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY12]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY12]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_FORMAT_XYZW_OFFEN_exact [[REG_SEQUENCE1]], %15, [[REG_SEQUENCE4]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.format.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.format.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
new file mode 100644
index 0000000000000..eea1750d63eee
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -0,0 +1,791 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -stop-after=instruction-select -verify-machineinstrs -o - %s | FileCheck %s
+; FIXME: Test with SI when argument lowering not broken for f16
+
+; Natural mapping
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float inreg %val, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr8
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE3]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for soffset
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]], [[COPY6]], implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[V_READFIRSTLANE_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY7]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE3]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 1)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 2)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 1, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 3)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 4)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_slc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 1, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 6)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 0, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 5)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_glc_slc_dlc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 1, 1, 0, 1, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 7)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32(<4 x i32> inreg %rsrc, <2 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32(<4 x i32> inreg %rsrc, <3 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v3f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2
+  ; CHECK:   BUFFER_STORE_DWORDX3_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32(<4 x i32> inreg %rsrc, <4 x float> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f32
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY9:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1, [[COPY6]], %subreg.sub2, [[COPY7]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORDX4_OFFEN_exact [[REG_SEQUENCE1]], [[COPY8]], [[REG_SEQUENCE]], [[COPY9]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 16 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8(<4 x i32> inreg %rsrc, i32 %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i8
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_BYTE_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 1 into custom TargetCustom7, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %val.trunc = trunc i32 %val to i8
+  call void @llvm.amdgcn.raw.buffer.store.i8(i8 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16(<4 x i32> inreg %rsrc, i32 %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_i16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %val.trunc = trunc i32 %val to i16
+  call void @llvm.amdgcn.raw.buffer.store.i16(i16 %val.trunc, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16(<4 x i32> inreg %rsrc, half %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_SHORT_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 2 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> inreg %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16(<4 x i32> %rsrc, <4 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v4f16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY8]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE3]], [[COPY9]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE4:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORDX2_OFFEN_exact [[REG_SEQUENCE1]], [[COPY6]], [[REG_SEQUENCE4]], [[COPY7]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], $noreg, [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7 + 4095, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096(<4 x i32> inreg %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__sgpr_soffset_f32_voffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7 + 4096, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 4096, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4095
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096(<4 x i32> inreg %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f32_voffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4095
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4095, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 4096
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[COPY5]], [[REG_SEQUENCE]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset, i32 4096, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_16
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 16
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 16
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4095
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4095
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4095
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096(<4 x i32> inreg %rsrc, <2 x half> %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_v2f16_soffset_add_4096
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sreg_32_xm0 = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 4096
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %13:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 4096
+  call void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half> %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000(<4 x i32> %rsrc, float %val, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_offset_add_5000
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0 = S_MOV_B32 5000
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+  ; CHECK:   %11:vgpr_32, dead %29:sreg_64_xexec = V_ADD_I32_e64 [[COPY5]], [[COPY7]], 0, implicit $exec
+  ; CHECK:   [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY8]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY9]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], %11, [[REG_SEQUENCE3]], [[COPY6]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  %voffset.add = add i32 %voffset, 5000
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %voffset.add, i32 %soffset, i32 0)
+  ret void
+}
+
+; An add of the offset is necessary, with a waterfall loop. Make sure the add is done outside of the waterfall loop.
+define amdgpu_ps void @raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset(<4 x i32> %rsrc, float %val, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_store__vgpr_rsrc__vgpr_val__5000_voffset__sgpr_soffset_offset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+  ; CHECK:   [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 4096, implicit $exec
+  ; CHECK:   [[COPY6:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+  ; CHECK:   [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE1]], [[COPY6]], implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub0, implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]].sub1, implicit $exec
+  ; CHECK:   [[REG_SEQUENCE2:%[0-9]+]]:sreg_64_xexec = REG_SEQUENCE [[V_READFIRSTLANE_B32_2]], %subreg.sub0, [[V_READFIRSTLANE_B32_3]], %subreg.sub1
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[REG_SEQUENCE2]], [[COPY7]], implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[REG_SEQUENCE3:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN_exact [[COPY4]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE3]], [[COPY5]], 904, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into custom TargetCustom7 + 5000, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 5000, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.raw.buffer.store.i8(i8, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.i16(i16, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f16(half, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f16(<2 x half>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f16(<4 x half>, <4 x i32>, i32, i32, i32 immarg)
+
+declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v3f32(<3 x float>, <4 x i32>, i32, i32, i32 immarg)
+declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll
new file mode 100644
index 0000000000000..55d142cc3c1a6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.sleep.ll
@@ -0,0 +1,45 @@
+; RUN: llc -global-isel -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+declare void @llvm.amdgcn.s.sleep(i32) #0
+
+; GCN-LABEL: {{^}}test_s_sleep:
+; GCN: s_sleep 0{{$}}
+; GCN: s_sleep 1{{$}}
+; GCN: s_sleep 2{{$}}
+; GCN: s_sleep 3{{$}}
+; GCN: s_sleep 4{{$}}
+; GCN: s_sleep 5{{$}}
+; GCN: s_sleep 6{{$}}
+; GCN: s_sleep 7{{$}}
+; GCN: s_sleep 8{{$}}
+; GCN: s_sleep 9{{$}}
+; GCN: s_sleep 10{{$}}
+; GCN: s_sleep 11{{$}}
+; GCN: s_sleep 12{{$}}
+; GCN: s_sleep 13{{$}}
+; GCN: s_sleep 14{{$}}
+; GCN: s_sleep 15{{$}}
+define amdgpu_kernel void @test_s_sleep(i32 %x) #0 {
+  call void @llvm.amdgcn.s.sleep(i32 0)
+  call void @llvm.amdgcn.s.sleep(i32 1)
+  call void @llvm.amdgcn.s.sleep(i32 2)
+  call void @llvm.amdgcn.s.sleep(i32 3)
+  call void @llvm.amdgcn.s.sleep(i32 4)
+  call void @llvm.amdgcn.s.sleep(i32 5)
+  call void @llvm.amdgcn.s.sleep(i32 6)
+  call void @llvm.amdgcn.s.sleep(i32 7)
+
+  ; Values that might only work on VI
+  call void @llvm.amdgcn.s.sleep(i32 8)
+  call void @llvm.amdgcn.s.sleep(i32 9)
+  call void @llvm.amdgcn.s.sleep(i32 10)
+  call void @llvm.amdgcn.s.sleep(i32 11)
+  call void @llvm.amdgcn.s.sleep(i32 12)
+  call void @llvm.amdgcn.s.sleep(i32 13)
+  call void @llvm.amdgcn.s.sleep(i32 14)
+  call void @llvm.amdgcn.s.sleep(i32 15)
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
index 5d59c58afd153..946a138bee4f2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.workitem.id.ll
@@ -88,5 +88,32 @@ bb2:
   ret void
 }
 
+; ALL-LABEL: {{^}}test_workitem_id_x_func:
+; ALL: s_waitcnt
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_x_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: {{^}}test_workitem_id_y_func:
+; ALL: v_lshrrev_b32_e32 v2, 10, v2
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_y_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.y()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
+; ALL-LABEL: {{^}}test_workitem_id_z_func:
+; ALL: v_lshrrev_b32_e32 v2, 20, v2
+; ALL-NEXT: v_and_b32_e32 v2, 0x3ff, v2
+define void @test_workitem_id_z_func(i32 addrspace(1)* %out) #1 {
+  %id = call i32 @llvm.amdgcn.workitem.id.z()
+  store i32 %id, i32 addrspace(1)* %out
+  ret void
+}
+
 attributes #0 = { nounwind readnone }
 attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
index 32839ffafc44b..65afcc6b7c70e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn-exp.mir
@@ -23,28 +23,20 @@ body: |
   bb.0:
     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3
     ; CHECK-LABEL: name: exp_s
-    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
-    ; CHECK: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
     ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
     ; CHECK: [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
     ; CHECK: [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
-    ; CHECK: [[C2:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 false
-    ; CHECK: [[C3:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 false
     ; CHECK: [[COPY4:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
     ; CHECK: [[COPY5:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
     ; CHECK: [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY2]](s32)
     ; CHECK: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY3]](s32)
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), [[C]](s32), [[C1]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), [[C2]](s1), [[C3]](s1)
-    %0:_(s32) = G_CONSTANT i32 0
-    %1:_(s32) = G_CONSTANT i32 0
-    %2:_(s32) = COPY $sgpr0
-    %3:_(s32) = COPY $sgpr1
-    %4:_(s32) = COPY $sgpr2
-    %5:_(s32) = COPY $sgpr3
-    %6:_(s1) = G_CONSTANT i1 0
-    %7:_(s1) = G_CONSTANT i1 0
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), %0, %1, %2, %3, %4, %5, %6, %7
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 0, [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), 0, 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $sgpr2
+    %3:_(s32) = COPY $sgpr3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), 0, 0, %0, %1, %2, %3, 0, 0
 ...
 ---
 name: exp_v
@@ -54,22 +46,14 @@ body: |
   bb.0:
     liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
     ; CHECK-LABEL: name: exp_v
-    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
-    ; CHECK: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
     ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
     ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
     ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
     ; CHECK: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
-    ; CHECK: [[C2:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 false
-    ; CHECK: [[C3:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 false
-    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), [[C]](s32), [[C1]](s32), [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[C2]](s1), [[C3]](s1)
-    %0:_(s32) = G_CONSTANT i32 0
-    %1:_(s32) = G_CONSTANT i32 0
-    %2:_(s32) = COPY $vgpr0
-    %3:_(s32) = COPY $vgpr1
-    %4:_(s32) = COPY $vgpr2
-    %5:_(s32) = COPY $vgpr3
-    %6:_(s1) = G_CONSTANT i1 0
-    %7:_(s1) = G_CONSTANT i1 0
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), %0, %1, %2, %3, %4, %5, %6, %7
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp), 0, 0, [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), 0, 0
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.exp.f32), 0, 0, %0, %1, %2, %3, 0, 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir
new file mode 100644
index 0000000000000..54c3a5009ca8c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.ds.swizzle.mir
@@ -0,0 +1,21 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast  -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy  -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name: ds_swizzle_s
+legalized: true
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; CHECK-LABEL: name: ds_swizzle_s
+    ; CHECK: liveins: $sgpr0
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[COPY]](s32)
+    ; CHECK: [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ds.swizzle), [[COPY1]](s32), 0
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.ds.swizzle), %0, 0
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
new file mode 100644
index 0000000000000..5618c1f910a5d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.load.1d.ll
@@ -0,0 +1,181 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @load_1d_vgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32 %s) {
+  ; CHECK-LABEL: name: load_1d_vgpr_vaddr__sgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY9]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Copy needed for VGPR argument
+define amdgpu_ps void @load_1d_sgpr_vaddr__sgpr_srsrc(<8 x i32> inreg %rsrc, i32 inreg %s) {
+  ; CHECK-LABEL: name: load_1d_sgpr_vaddr__sgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY10]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop needed for rsrc
+define amdgpu_ps void @load_1d_vgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 %s) {
+  ; CHECK-LABEL: name: load_1d_vgpr_vaddr__vgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF2]], %bb.1, %19, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %12(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY8]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY9]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop needed for rsrc, copy needed for vaddr
+define amdgpu_ps void @load_1d_sgpr_vaddr__vgpr_srsrc(<8 x i32> %rsrc, i32 inreg %s) {
+  ; CHECK-LABEL: name: load_1d_sgpr_vaddr__vgpr_srsrc
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY8]](s32)
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF2]], %bb.1, %20, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %12(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.load.1d), 15, [[COPY9]](s32), [[BUILD_VECTOR1]](<8 x s32>), 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY10]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32 immarg, i32, <8 x i32>, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
new file mode 100644
index 0000000000000..0ea100dc20299
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.image.sample.1d.ll
@@ -0,0 +1,268 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__sgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr11
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr12
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr13
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Copy required for VGPR input
+define amdgpu_ps void @sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float inreg %s) {
+  ; CHECK-LABEL: name: sample_1d_sgpr_vaddr__sgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $sgpr14
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr10
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr11
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr12
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr13
+  ; CHECK:   [[COPY12:%[0-9]+]]:sgpr(s32) = COPY $sgpr14
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[COPY12]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY13]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[COPY14:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY14]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for rsrc
+define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp(<8 x i32> %rsrc, <4 x i32> inreg %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__vgpr_rsrc__sgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY9:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY10:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY11:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR1]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_2]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for sampler
+define amdgpu_ps void @sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp(<8 x i32> inreg %rsrc, <4 x i32> %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__sgpr_rsrc__vgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY11:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR]](<8 x s32>), [[BUILD_VECTOR2]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+; Waterfall loop for rsrc and sampler
+define amdgpu_ps void @sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp(<8 x i32> %rsrc, <4 x i32> %samp, float %s) {
+  ; CHECK-LABEL: name: sample_1d_vgpr_vaddr__vgpr_rsrc__vgpr_samp
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY $vgpr7
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY $vgpr8
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY $vgpr9
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY $vgpr10
+  ; CHECK:   [[COPY11:%[0-9]+]]:vgpr(s32) = COPY $vgpr11
+  ; CHECK:   [[COPY12:%[0-9]+]]:vgpr(s32) = COPY $vgpr12
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<8 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32)
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY8]](s32), [[COPY9]](s32), [[COPY10]](s32), [[COPY11]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sgpr(p1) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:vgpr(<4 x s32>) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF2:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64), [[UV2:%[0-9]+]]:vreg_64(s64), [[UV3:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<8 x s32>)
+  ; CHECK:   [[UV4:%[0-9]+]]:vreg_64(s64), [[UV5:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF2]], %bb.1, %24, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(<4 x s32>) = G_PHI [[DEF1]](<4 x s32>), %bb.1, %17(<4 x s32>), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_5:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV2]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV2:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_2:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV2]](s64), [[UV2]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_2]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_6:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_7:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV3]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV3:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_3:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV3]](s64), [[UV3]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_2:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_3]], [[S_AND_B64_1]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR2:%[0-9]+]]:sgpr(<8 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32), [[V_READFIRSTLANE_B32_4]](s32), [[V_READFIRSTLANE_B32_5]](s32), [[V_READFIRSTLANE_B32_6]](s32), [[V_READFIRSTLANE_B32_7]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_8:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV4]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_9:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV4]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV4:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_8]](s32), [[V_READFIRSTLANE_B32_9]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_4:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV4]](s64), [[UV4]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_3:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_4]], [[S_AND_B64_2]], implicit-def $scc
+  ; CHECK:   [[V_READFIRSTLANE_B32_10:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV5]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_11:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV5]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV5:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_10]](s32), [[V_READFIRSTLANE_B32_11]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_5:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV5]](s64), [[UV5]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_4:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_5]], [[S_AND_B64_3]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR3:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_8]](s32), [[V_READFIRSTLANE_B32_9]](s32), [[V_READFIRSTLANE_B32_10]](s32), [[V_READFIRSTLANE_B32_11]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(<4 x s32>) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[COPY12]](s32), [[BUILD_VECTOR2]](<8 x s32>), [[BUILD_VECTOR3]](<4 x s32>), 0, 0, 0 :: (dereferenceable load 16 from custom TargetCustom8)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_4]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   [[COPY13:%[0-9]+]]:vgpr(p1) = COPY [[DEF]](p1)
+  ; CHECK:   G_STORE [[INT]](<4 x s32>), [[COPY13]](p1) :: (store 16 into `<4 x float> addrspace(1)* undef`, addrspace 1)
+  ; CHECK:   S_ENDPGM 0
+  %v = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 0, i32 0)
+  store <4 x float> %v, <4 x float> addrspace(1)* undef
+  ret void
+}
+
+declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 immarg, float, <8 x i32>, <4 x i32>, i1 immarg, i32 immarg, i32 immarg) #0
+
+attributes #0 = { nounwind readonly }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
new file mode 100644
index 0000000000000..52da31214d201
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.kill.mir
@@ -0,0 +1,68 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o -  %s| FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o -  %s| FileCheck %s
+
+---
+name: kill_scc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1
+    ; CHECK-LABEL: name: kill_scc
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+    ; CHECK: [[COPY2:%[0-9]+]]:vcc(s1) = COPY [[ICMP]](s1)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[COPY2]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %2
+...
+
+---
+name: kill_vcc
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: kill_vcc
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[COPY1]]
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[ICMP]](s1)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %2
+...
+
+---
+name: kill_constant_true
+legalized: true
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: kill_constant_true
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 true
+    ; CHECK: [[COPY:%[0-9]+]]:vcc(s1) = COPY [[C]](s1)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[COPY]](s1)
+    %0:_(s1) = G_CONSTANT i1 true
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %0
+...
+
+---
+name: kill_constant_false
+legalized: true
+
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: kill_constant_false
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s1) = G_CONSTANT i1 false
+    ; CHECK: [[COPY:%[0-9]+]]:vcc(s1) = COPY [[C]](s1)
+    ; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), [[COPY]](s1)
+    %0:_(s1) = G_CONSTANT i1 false
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.kill), %0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
new file mode 100644
index 0000000000000..71014f1ab8114
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.raw.buffer.load.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__sgpr_val__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY6]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc
+define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for soffset
+define amdgpu_ps float @raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__sgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr1
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps float @raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: raw_buffer_load__vgpr_rsrc__vgpr_val__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr5
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %15, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %9(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY5]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.raw.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
new file mode 100644
index 0000000000000..5991e3c891712
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.load.ll
@@ -0,0 +1,179 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 inreg %vindex, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY7]](s32), [[COPY8]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc
+define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[COPY6]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for soffset
+define amdgpu_ps float @struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__sgpr_rsrc__vgpr_val__vgpr_vindex_vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps float @struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_load__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:vgpr(s32) = G_IMPLICIT_DEF
+  ; CHECK:   [[DEF1:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF1]], %bb.1, %16, %bb.2
+  ; CHECK:   [[PHI1:%[0-9]+]]:vgpr(s32) = G_PHI [[DEF]](s32), %bb.1, %10(s32), %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY6]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   [[INT:%[0-9]+]]:vgpr(s32) = G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.load), [[BUILD_VECTOR1]](<4 x s32>), [[COPY4]](s32), [[COPY5]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable load 4 from custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   $vgpr0 = COPY [[INT]](s32)
+  ; CHECK:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret float %val
+}
+
+declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
new file mode 100644
index 0000000000000..6b8dd4cfd9941
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-amdgcn.struct.buffer.store.ll
@@ -0,0 +1,174 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-fast -o - %s | FileCheck %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -stop-after=regbankselect -regbankselect-greedy -o - %s | FileCheck %s
+
+; Natural mapping
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Copies for VGPR arguments
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, float inreg %val, i32 inreg %vindex, i32 inreg %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__sgpr_val__sgpr_vindex__sgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:sgpr(s32) = COPY $sgpr6
+  ; CHECK:   [[COPY5:%[0-9]+]]:sgpr(s32) = COPY $sgpr7
+  ; CHECK:   [[COPY6:%[0-9]+]]:sgpr(s32) = COPY $sgpr8
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr9
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[COPY4]](s32)
+  ; CHECK:   [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[COPY5]](s32)
+  ; CHECK:   [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[COPY6]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY8]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY9]](s32), [[COPY10]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc
+define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> %rsrc, float %val, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__sgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[COPY7]](s32), 0 :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for soffset
+define amdgpu_ps void @struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> inreg %rsrc, float %val, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__sgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; CHECK:   [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr2
+  ; CHECK:   [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr3
+  ; CHECK:   [[COPY2:%[0-9]+]]:sgpr(s32) = COPY $sgpr4
+  ; CHECK:   [[COPY3:%[0-9]+]]:sgpr(s32) = COPY $sgpr5
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr3
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_]](s32), [[COPY7]](s32), implicit $exec
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[V_READFIRSTLANE_B32_]](s32), 0 :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[V_CMP_EQ_U32_e64_]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+; Waterfall for rsrc and soffset
+define amdgpu_ps void @struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset(<4 x i32> %rsrc, float %val, i32 %vindex, i32 %voffset, i32 %soffset) {
+  ; CHECK-LABEL: name: struct_buffer_store__vgpr_rsrc__vgpr_val__vgpr_vindex__vgpr_voffset__vgpr_soffset
+  ; CHECK: bb.1 (%ir-block.0):
+  ; CHECK:   successors: %bb.2(0x80000000)
+  ; CHECK:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; CHECK:   [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+  ; CHECK:   [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr2
+  ; CHECK:   [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr3
+  ; CHECK:   [[COPY4:%[0-9]+]]:vgpr(s32) = COPY $vgpr4
+  ; CHECK:   [[COPY5:%[0-9]+]]:vgpr(s32) = COPY $vgpr5
+  ; CHECK:   [[COPY6:%[0-9]+]]:vgpr(s32) = COPY $vgpr6
+  ; CHECK:   [[COPY7:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr7
+  ; CHECK:   [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x s32>) = G_BUILD_VECTOR [[COPY]](s32), [[COPY1]](s32), [[COPY2]](s32), [[COPY3]](s32)
+  ; CHECK:   [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+  ; CHECK:   [[UV:%[0-9]+]]:vreg_64(s64), [[UV1:%[0-9]+]]:vreg_64(s64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x s32>)
+  ; CHECK:   [[S_MOV_B64_term:%[0-9]+]]:sreg_64_xexec = S_MOV_B64_term $exec
+  ; CHECK: bb.2:
+  ; CHECK:   successors: %bb.3(0x40000000), %bb.2(0x40000000)
+  ; CHECK:   [[PHI:%[0-9]+]]:sreg_64 = PHI [[DEF]], %bb.1, %14, %bb.2
+  ; CHECK:   [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV]](s64), [[UV]](s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub0(s64), implicit $exec
+  ; CHECK:   [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[UV1]].sub1(s64), implicit $exec
+  ; CHECK:   [[MV1:%[0-9]+]]:sreg_64_xexec(s64) = G_MERGE_VALUES [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[MV1]](s64), [[UV1]](s64), implicit $exec
+  ; CHECK:   [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_1]], [[V_CMP_EQ_U64_e64_]], implicit-def $scc
+  ; CHECK:   [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x s32>) = G_BUILD_VECTOR [[V_READFIRSTLANE_B32_]](s32), [[V_READFIRSTLANE_B32_1]](s32), [[V_READFIRSTLANE_B32_2]](s32), [[V_READFIRSTLANE_B32_3]](s32)
+  ; CHECK:   [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0(s32) = V_READFIRSTLANE_B32 [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]](s32), [[COPY7]](s32), implicit $exec
+  ; CHECK:   [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U32_e64_]], [[S_AND_B64_]], implicit-def $scc
+  ; CHECK:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.struct.buffer.store), [[COPY4]](s32), [[BUILD_VECTOR1]](<4 x s32>), [[COPY5]](s32), [[COPY6]](s32), [[V_READFIRSTLANE_B32_4]](s32), 0 :: (dereferenceable store 4 into custom TargetCustom7, align 1, addrspace 4)
+  ; CHECK:   [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64 = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
+  ; CHECK:   $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
+  ; CHECK:   S_CBRANCH_EXECNZ %bb.2, implicit $exec
+  ; CHECK: bb.3:
+  ; CHECK:   successors: %bb.4(0x80000000)
+  ; CHECK:   $exec = S_MOV_B64_term [[S_MOV_B64_term]]
+  ; CHECK: bb.4:
+  ; CHECK:   S_ENDPGM 0
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
+  ret void
+}
+
+declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
index 81a9863362bfa..206c9fd2d223d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-extract.mir
@@ -29,3 +29,31 @@ body: |
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(s32) = G_EXTRACT %0, 0
 ...
+
+---
+name: extract_s32_0_s1024_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK-LABEL: name: extract_s32_0_s1024_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s1024) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    ; CHECK: [[EXTRACT:%[0-9]+]]:vgpr(s32) = G_EXTRACT [[COPY]](s1024), 0
+    %0:_(s1024) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31
+    %1:_(s32) = G_EXTRACT %0, 0
+...
+
+---
+name: extract_s32_0_s1024_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
+    ; CHECK-LABEL: name: extract_s32_0_s1024_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s1024) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
+    ; CHECK: [[EXTRACT:%[0-9]+]]:sgpr(s32) = G_EXTRACT [[COPY]](s1024), 0
+    %0:_(s1024) = COPY $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7_sgpr8_sgpr9_sgpr10_sgpr11_sgpr12_sgpr13_sgpr14_sgpr15_sgpr16_sgpr17_sgpr18_sgpr19_sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27_sgpr28_sgpr29_sgpr30_sgpr31
+    %1:_(s32) = G_EXTRACT %0, 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir
new file mode 100644
index 0000000000000..953db08a70399
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-fceil.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: fceil_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: fceil_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FCEIL_:%[0-9]+]]:vgpr(s32) = G_FCEIL [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_FCEIL %0
+...
+
+---
+name: fceil_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: fceil_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FCEIL_:%[0-9]+]]:vgpr(s32) = G_FCEIL [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FCEIL %0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
index 77a444d6d14e9..585fabebf9f9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frame-index.mir
@@ -17,7 +17,7 @@ stack:
 body: |
   bb.0:
     ; CHECK-LABEL: name: test_frame_index_p5
-    ; CHECK: [[FRAME_INDEX:%[0-9]+]]:sgpr(p5) = G_FRAME_INDEX %stack.0.ptr0
+    ; CHECK: [[FRAME_INDEX:%[0-9]+]]:vgpr(p5) = G_FRAME_INDEX %stack.0.ptr0
     %0:_(p5) = G_FRAME_INDEX %stack.0.ptr0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frint.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frint.mir
new file mode 100644
index 0000000000000..edf201967fe6d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-frint.mir
@@ -0,0 +1,31 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+
+---
+name: frint_s
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: frint_s
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[FRINT_:%[0-9]+]]:vgpr(s32) = G_FRINT [[COPY]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_FRINT %0
+...
+
+---
+name: frint_v
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: frint_v
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[FRINT_:%[0-9]+]]:vgpr(s32) = G_FRINT [[COPY]]
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_FRINT %0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
index d129383817b3d..49ac13f666687 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-load.mir
@@ -68,6 +68,7 @@
   define amdgpu_kernel void @load_constant_i32_uniform_align4() {ret void}
   define amdgpu_kernel void @load_constant_i32_uniform_align2() {ret void}
   define amdgpu_kernel void @load_constant_i32_uniform_align1() {ret void}
+  define amdgpu_kernel void @load_private_uniform_sgpr_i32() {ret void}
 
   declare i32 @llvm.amdgcn.workitem.id.x() #0
   attributes #0 = { nounwind readnone }
@@ -635,3 +636,19 @@ body: |
     %0:_(p4) = COPY $sgpr0_sgpr1
     %1:_(s32) = G_LOAD %0 :: (load 4, addrspace 4, align 1)
 ...
+
+---
+name: load_private_uniform_sgpr_i32
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+
+    ; CHECK-LABEL: name: load_private_uniform_sgpr_i32
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sgpr0
+    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY [[COPY]](p5)
+    ; CHECK: [[LOAD:%[0-9]+]]:vgpr(s32) = G_LOAD [[COPY1]](p5) :: (load 4, addrspace 5)
+    %0:_(p5) = COPY $sgpr0
+    %1:_(s32) = G_LOAD %0 :: (load 4, addrspace 5, align 4)
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
index e48292127d547..aedfc8d1c4d2f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-sitofp.mir
@@ -29,3 +29,39 @@ body: |
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_SITOFP %0
 ...
+
+---
+name: sitofp_s_s1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: sitofp_s_s1
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK: [[SITOFP:%[0-9]+]]:vgpr(s32) = G_SITOFP [[ICMP]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    %3:_(s32) = G_SITOFP %2
+...
+
+---
+name: sitofp_v_s1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: sitofp_v_s1
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK: [[SITOFP:%[0-9]+]]:vgpr(s32) = G_SITOFP [[ICMP]](s1)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    %3:_(s32) = G_SITOFP %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
index 4eb3cae9ef90b..4401a3c8bd9b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-smulh.mir
@@ -1,5 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: smulh_s32_ss
@@ -8,10 +12,16 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    ; CHECK-LABEL: name: smulh_s32_ss
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; CHECK: [[SMULH:%[0-9]+]]:sgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: smulh_s32_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY2]]
+    ; GFX9-LABEL: name: smulh_s32_ss
+    ; GFX9: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GFX9: [[SMULH:%[0-9]+]]:sgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = G_SMULH %0, %1
@@ -24,10 +34,15 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: smulh_s32_sv
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: smulh_s32_sv
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; GFX9-LABEL: name: smulh_s32_sv
+    ; GFX9: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s32) = G_SMULH %0, %1
@@ -40,11 +55,17 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: smulh_s32_vs
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY2]]
+
+    ; GFX6-LABEL: name: smulh_s32_vs
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY2]]
+    ; GFX9-LABEL: name: smulh_s32_vs
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY2]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $sgpr0
     %2:_(s32) = G_SMULH %0, %1
@@ -57,10 +78,15 @@ legalized: true
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: smulh_s32_vv
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; CHECK: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: smulh_s32_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX6: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
+    ; GFX9-LABEL: name: smulh_s32_vv
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX9: [[SMULH:%[0-9]+]]:vgpr(s32) = G_SMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_SMULH %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
index 07f47c559a77c..bb0054bb25eca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-uitofp.mir
@@ -29,3 +29,39 @@ body: |
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = G_UITOFP %0
 ...
+
+---
+name: uitofp_s_s1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: uitofp_s_s1
+    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:scc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[ICMP]](s1)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    %3:_(s32) = G_UITOFP %2
+...
+
+---
+name: uitofp_v_s1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: uitofp_v_s1
+    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 0
+    ; CHECK: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+    ; CHECK: [[UITOFP:%[0-9]+]]:vgpr(s32) = G_UITOFP [[ICMP]](s1)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = G_CONSTANT i32 0
+    %2:_(s1) = G_ICMP intpred(eq), %0, %1
+    %3:_(s32) = G_UITOFP %2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
index d801149e7a16a..f62cd4cd4695c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-umulh.mir
@@ -1,5 +1,9 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -march=amdgcn -mcpu=fiji -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
+# RUN: llc -march=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+# XUN: llc -march=amdgcn -mcpu=tahiti -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX6 %s
+
+# RUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-fast -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
+# XUN: llc -march=amdgcn -mcpu=gfx900 -run-pass=regbankselect -regbankselect-greedy -verify-machineinstrs -o - %s | FileCheck -check-prefix=GFX9 %s
 
 ---
 name: umulh_s32_ss
@@ -8,10 +12,16 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $sgpr1
-    ; CHECK-LABEL: name: umulh_s32_ss
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
-    ; CHECK: [[UMULH:%[0-9]+]]:sgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: umulh_s32_ss
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY2]]
+    ; GFX9-LABEL: name: umulh_s32_ss
+    ; GFX9: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; GFX9: [[UMULH:%[0-9]+]]:sgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $sgpr1
     %2:_(s32) = G_UMULH %0, %1
@@ -24,10 +34,15 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: umulh_s32_sv
-    ; CHECK: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: umulh_s32_sv
+    ; GFX6: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; GFX9-LABEL: name: umulh_s32_sv
+    ; GFX9: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $sgpr0
     %1:_(s32) = COPY $vgpr0
     %2:_(s32) = G_UMULH %0, %1
@@ -40,11 +55,17 @@ legalized: true
 body: |
   bb.0:
     liveins: $sgpr0, $vgpr0
-    ; CHECK-LABEL: name: umulh_s32_vs
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
-    ; CHECK: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
-    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY2]]
+
+    ; GFX6-LABEL: name: umulh_s32_vs
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX6: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX6: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY2]]
+    ; GFX9-LABEL: name: umulh_s32_vs
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; GFX9: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY [[COPY1]](s32)
+    ; GFX9: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY2]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $sgpr0
     %2:_(s32) = G_UMULH %0, %1
@@ -57,10 +78,15 @@ legalized: true
 body: |
   bb.0:
     liveins: $vgpr0, $vgpr1
-    ; CHECK-LABEL: name: umulh_s32_vv
-    ; CHECK: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
-    ; CHECK: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
-    ; CHECK: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+
+    ; GFX6-LABEL: name: umulh_s32_vv
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX6: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
+    ; GFX9-LABEL: name: umulh_s32_vv
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; GFX9: [[UMULH:%[0-9]+]]:vgpr(s32) = G_UMULH [[COPY]], [[COPY1]]
     %0:_(s32) = COPY $vgpr0
     %1:_(s32) = COPY $vgpr1
     %2:_(s32) = G_UMULH %0, %1
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
index c2dfbe2789042..8ad231c4c9109 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-register-count.ll
@@ -1,15 +1,134 @@
-; RUN: llc -march=amdgcn -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=gfx908 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
 
-declare <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float, float, <32 x float>, i32, i32, i32)
+; GCN-LABEL: {{^}}kernel_32_agprs:
+; GCN:    .amdhsa_next_free_vgpr 32
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+; GCN:    VGPRBlocks: 7
+; GCN:    NumVGPRsForWavesPerEU: 32
+; GCN:    Occupancy: 8
+define amdgpu_kernel void @kernel_32_agprs() {
+bb:
+  call void asm sideeffect "", "~{v8}" ()
+  call void asm sideeffect "", "~{a31}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_0_agprs:
+; GCN:    .amdhsa_next_free_vgpr 1
+; GCN:    NumVgprs: 1
+; GCN:    NumAgprs: 0
+; GCN:    TotalNumVgprs: 1
+; GCN:    VGPRBlocks: 0
+; GCN:    NumVGPRsForWavesPerEU: 1
+; GCN:    Occupancy: 10
+define amdgpu_kernel void @kernel_0_agprs() {
+bb:
+  call void asm sideeffect "", "~{v0}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_40_vgprs:
+; GCN:    .amdhsa_next_free_vgpr 40
+; GCN:    NumVgprs: 40
+; GCN:    NumAgprs: 16
+; GCN:    TotalNumVgprs: 40
+; GCN:    VGPRBlocks: 9
+; GCN:    NumVGPRsForWavesPerEU: 40
+; GCN:    Occupancy: 6
+define amdgpu_kernel void @kernel_40_vgprs() {
+bb:
+  call void asm sideeffect "", "~{v39}" ()
+  call void asm sideeffect "", "~{a15}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_32_agprs:
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+define void @func_32_agprs() #0 {
+bb:
+  call void asm sideeffect "", "~{v8}" ()
+  call void asm sideeffect "", "~{a31}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_32_vgprs:
+; GCN:    NumVgprs: 32
+; GCN:    NumAgprs: 9
+; GCN:    TotalNumVgprs: 32
+define void @func_32_vgprs() {
+bb:
+  call void asm sideeffect "", "~{v31}" ()
+  call void asm sideeffect "", "~{a8}" ()
+  ret void
+}
 
-; GCN-LABEL: {{^}}test_32_agprs:
-; GCN: v_mfma_f32_32x32x1f32 a[0:31], {{v[0-9]+}}, {{v[0-9]+}},
-; GCN-NOT: v28
-; GCN: NumVgprs: 32
-; GCN: VGPRBlocks: 7
-define amdgpu_kernel void @test_32_agprs(<32 x float> addrspace(1)* %arg) {
+; GCN-LABEL: {{^}}func_0_agprs:
+; GCN:    NumVgprs: 1
+; GCN:    NumAgprs: 0
+; GCN:    TotalNumVgprs: 1
+define amdgpu_kernel void @func_0_agprs() {
 bb:
-  %mai.1 = tail call <32 x float> @llvm.amdgcn.mfma.f32.32x32x1f32(float 1.0, float 2.0, <32 x float> <float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0, float 0.0>, i32 0, i32 0, i32 0)
-  store <32 x float> %mai.1, <32 x float> addrspace(1)* %arg
+  call void asm sideeffect "", "~{v0}" ()
   ret void
 }
+
+; GCN-LABEL: {{^}}kernel_max_gprs:
+; GCN:    .amdhsa_next_free_vgpr 256
+; GCN:    NumVgprs: 256
+; GCN:    NumAgprs: 256
+; GCN:    TotalNumVgprs: 256
+; GCN:    VGPRBlocks: 63
+; GCN:    NumVGPRsForWavesPerEU: 256
+; GCN:    Occupancy: 1
+define amdgpu_kernel void @kernel_max_gprs() {
+bb:
+  call void asm sideeffect "", "~{v255}" ()
+  call void asm sideeffect "", "~{a255}" ()
+  ret void
+}
+
+; GCN-LABEL: {{^}}kernel_call_func_32_agprs:
+; GCN:    .amdhsa_next_free_vgpr 32
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+; GCN:    VGPRBlocks: 7
+; GCN:    NumVGPRsForWavesPerEU: 32
+; GCN:    Occupancy: 8
+define amdgpu_kernel void @kernel_call_func_32_agprs() {
+bb:
+  call void @func_32_agprs() #0
+  ret void
+}
+
+; GCN-LABEL: {{^}}func_call_func_32_agprs:
+; GCN:    NumVgprs: 9
+; GCN:    NumAgprs: 32
+; GCN:    TotalNumVgprs: 32
+define void @func_call_func_32_agprs() {
+bb:
+  call void @func_32_agprs() #0
+  ret void
+}
+
+declare void @undef_func()
+
+; GCN-LABEL: {{^}}kernel_call_undef_func:
+; GCN:    .amdhsa_next_free_vgpr 24
+; GCN:    NumVgprs: 24
+; GCN:    NumAgprs: 24
+; GCN:    TotalNumVgprs: 24
+; GCN:    VGPRBlocks: 5
+; GCN:    NumVGPRsForWavesPerEU: 24
+; GCN:    Occupancy: 10
+define amdgpu_kernel void @kernel_call_undef_func() {
+bb:
+  call void @undef_func()
+  ret void
+}
+
+attributes #0 = { nounwind noinline }
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse.ll b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
index 9c537d5bc5f8e..40bb9be033f8e 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
@@ -391,14 +391,13 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_mov_b32 s0, 0xff0000
-; SI-NEXT:    s_mov_b32 s1, 0xff00
-; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f
-; SI-NEXT:    s_mov_b32 s3, 0xf0f0f0f0
-; SI-NEXT:    s_mov_b32 s6, 0x33333333
-; SI-NEXT:    s_mov_b32 s8, 0xcccccccc
-; SI-NEXT:    s_mov_b32 s9, 0x55555555
-; SI-NEXT:    s_mov_b32 s10, 0xaaaaaaaa
+; SI-NEXT:    s_mov_b32 s0, 0xff00
+; SI-NEXT:    s_mov_b32 s1, 0xf0f0f0f
+; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f0
+; SI-NEXT:    s_mov_b32 s3, 0x33333333
+; SI-NEXT:    s_mov_b32 s6, 0xcccccccc
+; SI-NEXT:    s_mov_b32 s8, 0x55555555
+; SI-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshl_b64 v[2:3], v[0:1], 8
 ; SI-NEXT:    v_alignbit_b32 v4, v1, v0, 24
@@ -408,39 +407,39 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; SI-NEXT:    v_lshl_b64 v[1:2], v[0:1], 24
 ; SI-NEXT:    v_lshlrev_b32_e32 v1, 24, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; SI-NEXT:    v_and_b32_e32 v0, s0, v0
-; SI-NEXT:    v_and_b32_e32 v4, s0, v4
+; SI-NEXT:    v_and_b32_e32 v0, 0xff0000, v0
+; SI-NEXT:    v_and_b32_e32 v4, 0xff0000, v4
 ; SI-NEXT:    v_and_b32_e32 v5, 0xff000000, v5
-; SI-NEXT:    v_and_b32_e32 v7, s1, v7
+; SI-NEXT:    v_and_b32_e32 v7, s0, v7
 ; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; SI-NEXT:    v_and_b32_e32 v2, s1, v2
+; SI-NEXT:    v_and_b32_e32 v2, s0, v2
 ; SI-NEXT:    v_or_b32_e32 v4, v5, v4
 ; SI-NEXT:    v_or_b32_e32 v5, v7, v6
 ; SI-NEXT:    v_or_b32_e32 v0, v1, v0
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v3
 ; SI-NEXT:    v_or_b32_e32 v1, v4, v5
 ; SI-NEXT:    v_or_b32_e32 v3, v0, v2
-; SI-NEXT:    v_and_b32_e32 v0, s2, v1
-; SI-NEXT:    v_and_b32_e32 v2, s3, v1
-; SI-NEXT:    v_and_b32_e32 v1, s2, v3
-; SI-NEXT:    v_and_b32_e32 v3, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s1, v1
+; SI-NEXT:    v_and_b32_e32 v2, s2, v1
+; SI-NEXT:    v_and_b32_e32 v1, s1, v3
+; SI-NEXT:    v_and_b32_e32 v3, s2, v3
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_and_b32_e32 v1, s6, v3
-; SI-NEXT:    v_and_b32_e32 v0, s6, v2
-; SI-NEXT:    v_and_b32_e32 v3, s8, v3
-; SI-NEXT:    v_and_b32_e32 v2, s8, v2
+; SI-NEXT:    v_and_b32_e32 v1, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s3, v2
+; SI-NEXT:    v_and_b32_e32 v3, s6, v3
+; SI-NEXT:    v_and_b32_e32 v2, s6, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_and_b32_e32 v1, s9, v3
-; SI-NEXT:    v_and_b32_e32 v0, s9, v2
-; SI-NEXT:    v_and_b32_e32 v3, s10, v3
-; SI-NEXT:    v_and_b32_e32 v2, s10, v2
+; SI-NEXT:    v_and_b32_e32 v1, s8, v3
+; SI-NEXT:    v_and_b32_e32 v0, s8, v2
+; SI-NEXT:    v_and_b32_e32 v3, s9, v3
+; SI-NEXT:    v_and_b32_e32 v2, s9, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 1
 ; SI-NEXT:    v_or_b32_e32 v1, v3, v1
@@ -454,18 +453,17 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
 ; FLAT-NEXT:    v_mov_b32_e32 v4, 8
-; FLAT-NEXT:    s_mov_b32 s2, 0xff0000
-; FLAT-NEXT:    s_mov_b32 s3, 0xf0f0f0f
+; FLAT-NEXT:    s_mov_b32 s2, 0xf0f0f0f
+; FLAT-NEXT:    s_mov_b32 s3, 0xf0f0f0f0
 ; FLAT-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLAT-NEXT:    v_mov_b32_e32 v1, s1
 ; FLAT-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; FLAT-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; FLAT-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
-; FLAT-NEXT:    s_mov_b32 s0, 0xf0f0f0f0
-; FLAT-NEXT:    s_mov_b32 s1, 0x33333333
-; FLAT-NEXT:    s_mov_b32 s6, 0xcccccccc
-; FLAT-NEXT:    s_mov_b32 s8, 0x55555555
-; FLAT-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
+; FLAT-NEXT:    s_mov_b32 s0, 0x33333333
+; FLAT-NEXT:    s_mov_b32 s1, 0xcccccccc
+; FLAT-NEXT:    s_mov_b32 s6, 0x55555555
+; FLAT-NEXT:    s_mov_b32 s8, 0xaaaaaaaa
 ; FLAT-NEXT:    s_mov_b32 s7, 0xf000
 ; FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; FLAT-NEXT:    v_lshlrev_b64 v[2:3], 24, v[0:1]
@@ -475,9 +473,9 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 8, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v4, 24, v0
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 8, v0
-; FLAT-NEXT:    v_and_b32_e32 v2, s2, v2
+; FLAT-NEXT:    v_and_b32_e32 v2, 0xff0000, v2
 ; FLAT-NEXT:    v_and_b32_e32 v6, 0xff000000, v6
-; FLAT-NEXT:    v_and_b32_e32 v0, s2, v0
+; FLAT-NEXT:    v_and_b32_e32 v0, 0xff0000, v0
 ; FLAT-NEXT:    v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; FLAT-NEXT:    v_or_b32_e32 v2, v6, v2
 ; FLAT-NEXT:    v_and_b32_e32 v3, 0xff00, v3
@@ -485,29 +483,29 @@ define amdgpu_kernel void @v_brev_i64(i64 addrspace(1)* noalias %out, i64 addrsp
 ; FLAT-NEXT:    v_or_b32_e32 v0, v4, v0
 ; FLAT-NEXT:    v_or_b32_sdwa v2, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; FLAT-NEXT:    v_or_b32_e32 v3, v0, v2
-; FLAT-NEXT:    v_and_b32_e32 v0, s3, v1
-; FLAT-NEXT:    v_and_b32_e32 v2, s0, v1
-; FLAT-NEXT:    v_and_b32_e32 v1, s3, v3
-; FLAT-NEXT:    v_and_b32_e32 v3, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s2, v1
+; FLAT-NEXT:    v_and_b32_e32 v2, s3, v1
+; FLAT-NEXT:    v_and_b32_e32 v1, s2, v3
+; FLAT-NEXT:    v_and_b32_e32 v3, s3, v3
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 4, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 4, v[2:3]
 ; FLAT-NEXT:    v_or_b32_e32 v3, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s1, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s1, v2
-; FLAT-NEXT:    v_and_b32_e32 v3, s6, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s6, v2
+; FLAT-NEXT:    v_and_b32_e32 v1, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s0, v2
+; FLAT-NEXT:    v_and_b32_e32 v3, s1, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s1, v2
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 2, v[2:3]
-; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    v_or_b32_e32 v3, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s8, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s8, v2
-; FLAT-NEXT:    v_and_b32_e32 v3, s9, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s9, v2
+; FLAT-NEXT:    v_and_b32_e32 v1, s6, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s6, v2
+; FLAT-NEXT:    v_and_b32_e32 v3, s8, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s8, v2
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
+; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    v_or_b32_e32 v1, v3, v1
 ; FLAT-NEXT:    v_or_b32_e32 v0, v2, v0
 ; FLAT-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -789,23 +787,20 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; SI-NEXT:    v_mov_b32_e32 v1, 0
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[0:3], 0 addr64
-; SI-NEXT:    s_mov_b32 s0, 0xff0000
-; SI-NEXT:    s_mov_b32 s1, 0xff000000
-; SI-NEXT:    s_mov_b32 s2, 0xff00
-; SI-NEXT:    s_movk_i32 s3, 0xff
-; SI-NEXT:    s_mov_b32 s8, 0xf0f0f0f
-; SI-NEXT:    s_mov_b32 s9, 0xf0f0f0f0
-; SI-NEXT:    s_mov_b32 s10, 0x33333333
-; SI-NEXT:    s_mov_b32 s11, 0xcccccccc
-; SI-NEXT:    s_mov_b32 s12, 0x55555555
-; SI-NEXT:    s_mov_b32 s13, 0xaaaaaaaa
+; SI-NEXT:    s_mov_b32 s0, 0xff00
+; SI-NEXT:    s_mov_b32 s1, 0xf0f0f0f
+; SI-NEXT:    s_mov_b32 s2, 0xf0f0f0f0
+; SI-NEXT:    s_mov_b32 s3, 0x33333333
+; SI-NEXT:    s_mov_b32 s8, 0xcccccccc
+; SI-NEXT:    s_mov_b32 s9, 0x55555555
+; SI-NEXT:    s_mov_b32 s10, 0xaaaaaaaa
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[2:3], 8
 ; SI-NEXT:    v_alignbit_b32 v6, v3, v2, 24
 ; SI-NEXT:    v_alignbit_b32 v7, v3, v2, 8
-; SI-NEXT:    v_lshrrev_b32_e32 v8, 24, v3
 ; SI-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
+; SI-NEXT:    v_lshrrev_b32_e32 v8, 24, v3
 ; SI-NEXT:    v_lshl_b64 v[3:4], v[2:3], 24
 ; SI-NEXT:    v_lshlrev_b32_e32 v10, 24, v2
 ; SI-NEXT:    v_lshlrev_b32_e32 v11, 8, v2
@@ -817,78 +812,80 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; SI-NEXT:    v_lshlrev_b32_e32 v16, 24, v0
 ; SI-NEXT:    v_lshlrev_b32_e32 v17, 8, v0
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 24
-; SI-NEXT:    v_and_b32_e32 v0, s0, v6
-; SI-NEXT:    v_and_b32_e32 v2, s1, v7
-; SI-NEXT:    v_and_b32_e32 v6, s2, v9
-; SI-NEXT:    v_and_b32_e32 v7, s0, v11
-; SI-NEXT:    v_and_b32_e32 v9, s0, v12
-; SI-NEXT:    v_and_b32_e32 v11, s1, v13
-; SI-NEXT:    v_or_b32_e32 v0, v2, v0
-; SI-NEXT:    v_or_b32_e32 v2, v6, v8
-; SI-NEXT:    v_and_b32_e32 v12, s2, v15
-; SI-NEXT:    v_and_b32_e32 v13, s0, v17
-; SI-NEXT:    v_and_b32_e32 v5, s3, v5
-; SI-NEXT:    v_and_b32_e32 v4, s2, v4
-; SI-NEXT:    v_and_b32_e32 v3, s3, v3
-; SI-NEXT:    v_and_b32_e32 v1, s2, v1
-; SI-NEXT:    v_or_b32_e32 v6, v10, v7
-; SI-NEXT:    v_or_b32_e32 v7, v11, v9
-; SI-NEXT:    v_or_b32_e32 v2, v0, v2
-; SI-NEXT:    v_or_b32_e32 v8, v12, v14
-; SI-NEXT:    v_or_b32_e32 v0, v4, v5
+; SI-NEXT:    v_and_b32_e32 v6, 0xff0000, v6
+; SI-NEXT:    v_and_b32_e32 v7, 0xff000000, v7
+; SI-NEXT:    v_mov_b32_e32 v0, 0xff0000
+; SI-NEXT:    v_or_b32_e32 v6, v7, v6
+; SI-NEXT:    v_mov_b32_e32 v7, 0xff00
+; SI-NEXT:    v_and_b32_e32 v2, v0, v11
+; SI-NEXT:    v_and_b32_e32 v11, v0, v12
+; SI-NEXT:    v_and_b32_e32 v9, s0, v9
+; SI-NEXT:    v_and_b32_e32 v12, 0xff000000, v13
+; SI-NEXT:    v_and_b32_e32 v0, v0, v17
+; SI-NEXT:    v_and_b32_e32 v13, v7, v15
+; SI-NEXT:    v_and_b32_e32 v1, v7, v1
+; SI-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; SI-NEXT:    v_or_b32_e32 v8, v9, v8
+; SI-NEXT:    v_or_b32_e32 v2, v10, v2
+; SI-NEXT:    v_and_b32_e32 v5, 0xff, v5
+; SI-NEXT:    v_and_b32_e32 v4, s0, v4
+; SI-NEXT:    v_or_b32_e32 v7, v16, v0
 ; SI-NEXT:    v_or_b32_e32 v1, v1, v3
-; SI-NEXT:    v_or_b32_e32 v9, v16, v13
-; SI-NEXT:    v_or_b32_e32 v5, v7, v8
-; SI-NEXT:    v_or_b32_e32 v3, v6, v0
-; SI-NEXT:    v_or_b32_e32 v7, v9, v1
-; SI-NEXT:    v_and_b32_e32 v0, s8, v2
-; SI-NEXT:    v_and_b32_e32 v1, s8, v3
-; SI-NEXT:    v_and_b32_e32 v2, s9, v2
-; SI-NEXT:    v_and_b32_e32 v3, s9, v3
-; SI-NEXT:    v_and_b32_e32 v4, s8, v5
-; SI-NEXT:    v_and_b32_e32 v6, s9, v5
-; SI-NEXT:    v_and_b32_e32 v5, s8, v7
-; SI-NEXT:    v_and_b32_e32 v7, s9, v7
-; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
-; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
+; SI-NEXT:    v_or_b32_e32 v9, v12, v11
+; SI-NEXT:    v_or_b32_e32 v10, v13, v14
+; SI-NEXT:    v_or_b32_e32 v0, v4, v5
+; SI-NEXT:    v_or_b32_e32 v5, v9, v10
+; SI-NEXT:    v_or_b32_e32 v6, v6, v8
+; SI-NEXT:    v_or_b32_e32 v7, v7, v1
+; SI-NEXT:    v_or_b32_e32 v3, v2, v0
+; SI-NEXT:    v_and_b32_e32 v0, s1, v6
+; SI-NEXT:    v_and_b32_e32 v2, s2, v6
+; SI-NEXT:    v_and_b32_e32 v4, s1, v5
+; SI-NEXT:    v_and_b32_e32 v6, s2, v5
+; SI-NEXT:    v_and_b32_e32 v5, s1, v7
+; SI-NEXT:    v_and_b32_e32 v7, s2, v7
+; SI-NEXT:    v_and_b32_e32 v1, s1, v3
+; SI-NEXT:    v_and_b32_e32 v3, s2, v3
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 4
 ; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 4
-; SI-NEXT:    v_or_b32_e32 v3, v3, v1
-; SI-NEXT:    v_or_b32_e32 v2, v2, v0
+; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 4
+; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 4
 ; SI-NEXT:    v_or_b32_e32 v7, v7, v5
 ; SI-NEXT:    v_or_b32_e32 v6, v6, v4
-; SI-NEXT:    v_and_b32_e32 v1, s10, v3
-; SI-NEXT:    v_and_b32_e32 v0, s10, v2
-; SI-NEXT:    v_and_b32_e32 v5, s10, v7
-; SI-NEXT:    v_and_b32_e32 v4, s10, v6
-; SI-NEXT:    v_and_b32_e32 v3, s11, v3
-; SI-NEXT:    v_and_b32_e32 v2, s11, v2
-; SI-NEXT:    v_and_b32_e32 v7, s11, v7
-; SI-NEXT:    v_and_b32_e32 v6, s11, v6
-; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
-; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
+; SI-NEXT:    v_or_b32_e32 v3, v3, v1
+; SI-NEXT:    v_or_b32_e32 v2, v2, v0
+; SI-NEXT:    v_and_b32_e32 v5, s3, v7
+; SI-NEXT:    v_and_b32_e32 v4, s3, v6
+; SI-NEXT:    v_and_b32_e32 v7, s8, v7
+; SI-NEXT:    v_and_b32_e32 v6, s8, v6
+; SI-NEXT:    v_and_b32_e32 v1, s3, v3
+; SI-NEXT:    v_and_b32_e32 v0, s3, v2
+; SI-NEXT:    v_and_b32_e32 v3, s8, v3
+; SI-NEXT:    v_and_b32_e32 v2, s8, v2
 ; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 2
 ; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 2
+; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 2
+; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 2
+; SI-NEXT:    v_or_b32_e32 v7, v7, v5
+; SI-NEXT:    v_or_b32_e32 v9, v6, v4
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_or_b32_e32 v7, v7, v5
-; SI-NEXT:    v_or_b32_e32 v6, v6, v4
-; SI-NEXT:    v_and_b32_e32 v1, s12, v3
-; SI-NEXT:    v_and_b32_e32 v0, s12, v2
-; SI-NEXT:    v_and_b32_e32 v5, s12, v7
-; SI-NEXT:    v_and_b32_e32 v4, s12, v6
-; SI-NEXT:    v_and_b32_e32 v3, s13, v3
-; SI-NEXT:    v_and_b32_e32 v2, s13, v2
-; SI-NEXT:    v_and_b32_e32 v7, s13, v7
-; SI-NEXT:    v_and_b32_e32 v6, s13, v6
+; SI-NEXT:    v_and_b32_e32 v5, s9, v7
+; SI-NEXT:    v_and_b32_e32 v4, s9, v9
+; SI-NEXT:    v_and_b32_e32 v1, s9, v3
+; SI-NEXT:    v_and_b32_e32 v0, s9, v2
+; SI-NEXT:    v_and_b32_e32 v6, s10, v7
+; SI-NEXT:    v_lshl_b64 v[7:8], v[4:5], 1
+; SI-NEXT:    v_and_b32_e32 v5, s10, v9
+; SI-NEXT:    v_and_b32_e32 v3, s10, v3
+; SI-NEXT:    v_and_b32_e32 v2, s10, v2
 ; SI-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
 ; SI-NEXT:    v_lshr_b64 v[2:3], v[2:3], 1
-; SI-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; SI-NEXT:    v_lshr_b64 v[6:7], v[6:7], 1
+; SI-NEXT:    v_lshr_b64 v[4:5], v[5:6], 1
 ; SI-NEXT:    v_or_b32_e32 v3, v3, v1
 ; SI-NEXT:    v_or_b32_e32 v2, v2, v0
-; SI-NEXT:    v_or_b32_e32 v1, v7, v5
-; SI-NEXT:    v_or_b32_e32 v0, v6, v4
+; SI-NEXT:    v_or_b32_e32 v1, v5, v8
+; SI-NEXT:    v_or_b32_e32 v0, v4, v7
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -898,65 +895,63 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x2c
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v0, 4, v0
 ; FLAT-NEXT:    v_mov_b32_e32 v8, 8
-; FLAT-NEXT:    s_mov_b32 s2, 0xff0000
-; FLAT-NEXT:    s_mov_b32 s3, 0xff000000
+; FLAT-NEXT:    v_mov_b32_e32 v10, 0xff0000
+; FLAT-NEXT:    s_mov_b32 s2, 0xf0f0f0f
 ; FLAT-NEXT:    s_waitcnt lgkmcnt(0)
 ; FLAT-NEXT:    v_mov_b32_e32 v1, s1
 ; FLAT-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
 ; FLAT-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; FLAT-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
-; FLAT-NEXT:    s_mov_b32 s0, 0xff00
-; FLAT-NEXT:    s_mov_b32 s1, 0xf0f0f0f
-; FLAT-NEXT:    s_mov_b32 s8, 0xf0f0f0f0
-; FLAT-NEXT:    s_mov_b32 s9, 0x33333333
-; FLAT-NEXT:    s_mov_b32 s10, 0xcccccccc
-; FLAT-NEXT:    s_mov_b32 s11, 0x55555555
-; FLAT-NEXT:    s_mov_b32 s12, 0xaaaaaaaa
+; FLAT-NEXT:    s_mov_b32 s0, 0xf0f0f0f0
+; FLAT-NEXT:    s_mov_b32 s1, 0x33333333
+; FLAT-NEXT:    s_mov_b32 s3, 0xcccccccc
+; FLAT-NEXT:    s_mov_b32 s8, 0x55555555
+; FLAT-NEXT:    s_mov_b32 s9, 0xaaaaaaaa
 ; FLAT-NEXT:    s_mov_b32 s7, 0xf000
 ; FLAT-NEXT:    s_mov_b32 s6, -1
 ; FLAT-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 24, v[2:3]
-; FLAT-NEXT:    v_lshlrev_b32_sdwa v11, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
-; FLAT-NEXT:    v_lshlrev_b32_sdwa v14, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; FLAT-NEXT:    v_lshlrev_b32_sdwa v12, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; FLAT-NEXT:    v_lshlrev_b32_sdwa v15, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
 ; FLAT-NEXT:    v_lshlrev_b64 v[8:9], 8, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b64 v[6:7], 8, v[2:3]
 ; FLAT-NEXT:    v_alignbit_b32 v4, v3, v2, 24
-; FLAT-NEXT:    v_alignbit_b32 v10, v3, v2, 8
-; FLAT-NEXT:    v_or_b32_sdwa v3, v11, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
-; FLAT-NEXT:    v_alignbit_b32 v12, v1, v0, 24
-; FLAT-NEXT:    v_alignbit_b32 v13, v1, v0, 8
+; FLAT-NEXT:    v_alignbit_b32 v11, v3, v2, 8
+; FLAT-NEXT:    v_or_b32_sdwa v3, v12, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; FLAT-NEXT:    v_or_b32_sdwa v12, v15, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; FLAT-NEXT:    v_alignbit_b32 v13, v1, v0, 24
+; FLAT-NEXT:    v_alignbit_b32 v14, v1, v0, 8
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v8, 24, v0
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v15, 8, v0
-; FLAT-NEXT:    v_or_b32_sdwa v11, v14, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 24, v[0:1]
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v6, 24, v2
 ; FLAT-NEXT:    v_lshlrev_b32_e32 v2, 8, v2
-; FLAT-NEXT:    v_and_b32_e32 v0, s2, v4
-; FLAT-NEXT:    v_and_b32_e32 v4, s3, v10
-; FLAT-NEXT:    v_and_b32_e32 v2, s2, v2
+; FLAT-NEXT:    v_and_b32_e32 v0, 0xff0000, v4
+; FLAT-NEXT:    v_and_b32_e32 v4, 0xff000000, v11
+; FLAT-NEXT:    v_and_b32_e32 v2, v10, v2
+; FLAT-NEXT:    v_and_b32_e32 v11, v10, v13
 ; FLAT-NEXT:    v_or_b32_e32 v0, v4, v0
-; FLAT-NEXT:    v_and_b32_e32 v1, s0, v1
-; FLAT-NEXT:    v_and_b32_e32 v10, s2, v12
-; FLAT-NEXT:    v_and_b32_e32 v12, s3, v13
-; FLAT-NEXT:    v_and_b32_e32 v4, s0, v5
-; FLAT-NEXT:    v_and_b32_e32 v13, s2, v15
-; FLAT-NEXT:    v_or_b32_e32 v5, v12, v10
+; FLAT-NEXT:    v_and_b32_e32 v1, 0xff00, v1
+; FLAT-NEXT:    v_and_b32_e32 v13, 0xff000000, v14
+; FLAT-NEXT:    v_and_b32_e32 v4, 0xff00, v5
+; FLAT-NEXT:    v_and_b32_e32 v10, v10, v15
+; FLAT-NEXT:    v_or_b32_e32 v5, v13, v11
 ; FLAT-NEXT:    v_or_b32_e32 v2, v6, v2
 ; FLAT-NEXT:    v_or_b32_e32 v3, v0, v3
 ; FLAT-NEXT:    v_or_b32_sdwa v0, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; FLAT-NEXT:    v_or_b32_e32 v6, v8, v13
+; FLAT-NEXT:    v_or_b32_e32 v6, v8, v10
 ; FLAT-NEXT:    v_or_b32_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v2, v0
-; FLAT-NEXT:    v_or_b32_e32 v5, v5, v11
+; FLAT-NEXT:    v_or_b32_e32 v5, v5, v12
 ; FLAT-NEXT:    v_or_b32_e32 v8, v6, v1
-; FLAT-NEXT:    v_and_b32_e32 v0, s1, v3
-; FLAT-NEXT:    v_and_b32_e32 v1, s1, v7
-; FLAT-NEXT:    v_and_b32_e32 v2, s8, v3
-; FLAT-NEXT:    v_and_b32_e32 v3, s8, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s1, v5
-; FLAT-NEXT:    v_and_b32_e32 v6, s8, v5
-; FLAT-NEXT:    v_and_b32_e32 v5, s1, v8
-; FLAT-NEXT:    v_and_b32_e32 v7, s8, v8
+; FLAT-NEXT:    v_and_b32_e32 v0, s2, v3
+; FLAT-NEXT:    v_and_b32_e32 v1, s2, v7
+; FLAT-NEXT:    v_and_b32_e32 v2, s0, v3
+; FLAT-NEXT:    v_and_b32_e32 v3, s0, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s2, v5
+; FLAT-NEXT:    v_and_b32_e32 v6, s0, v5
+; FLAT-NEXT:    v_and_b32_e32 v5, s2, v8
+; FLAT-NEXT:    v_and_b32_e32 v7, s0, v8
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 4, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 4, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 4, v[4:5]
@@ -965,14 +960,14 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v7, v5
 ; FLAT-NEXT:    v_or_b32_e32 v6, v6, v4
-; FLAT-NEXT:    v_and_b32_e32 v1, s9, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s9, v2
-; FLAT-NEXT:    v_and_b32_e32 v5, s9, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s9, v6
-; FLAT-NEXT:    v_and_b32_e32 v3, s10, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s10, v2
-; FLAT-NEXT:    v_and_b32_e32 v7, s10, v7
-; FLAT-NEXT:    v_and_b32_e32 v6, s10, v6
+; FLAT-NEXT:    v_and_b32_e32 v1, s1, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s1, v2
+; FLAT-NEXT:    v_and_b32_e32 v5, s1, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s1, v6
+; FLAT-NEXT:    v_and_b32_e32 v3, s3, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s3, v2
+; FLAT-NEXT:    v_and_b32_e32 v7, s3, v7
+; FLAT-NEXT:    v_and_b32_e32 v6, s3, v6
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 2, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 2, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 2, v[4:5]
@@ -981,14 +976,14 @@ define amdgpu_kernel void @v_brev_v2i64(<2 x i64> addrspace(1)* noalias %out, <2
 ; FLAT-NEXT:    v_or_b32_e32 v2, v2, v0
 ; FLAT-NEXT:    v_or_b32_e32 v7, v7, v5
 ; FLAT-NEXT:    v_or_b32_e32 v6, v6, v4
-; FLAT-NEXT:    v_and_b32_e32 v1, s11, v3
-; FLAT-NEXT:    v_and_b32_e32 v0, s11, v2
-; FLAT-NEXT:    v_and_b32_e32 v5, s11, v7
-; FLAT-NEXT:    v_and_b32_e32 v4, s11, v6
-; FLAT-NEXT:    v_and_b32_e32 v3, s12, v3
-; FLAT-NEXT:    v_and_b32_e32 v2, s12, v2
-; FLAT-NEXT:    v_and_b32_e32 v7, s12, v7
-; FLAT-NEXT:    v_and_b32_e32 v6, s12, v6
+; FLAT-NEXT:    v_and_b32_e32 v1, s8, v3
+; FLAT-NEXT:    v_and_b32_e32 v0, s8, v2
+; FLAT-NEXT:    v_and_b32_e32 v5, s8, v7
+; FLAT-NEXT:    v_and_b32_e32 v4, s8, v6
+; FLAT-NEXT:    v_and_b32_e32 v3, s9, v3
+; FLAT-NEXT:    v_and_b32_e32 v2, s9, v2
+; FLAT-NEXT:    v_and_b32_e32 v7, s9, v7
+; FLAT-NEXT:    v_and_b32_e32 v6, s9, v6
 ; FLAT-NEXT:    v_lshlrev_b64 v[0:1], 1, v[0:1]
 ; FLAT-NEXT:    v_lshrrev_b64 v[2:3], 1, v[2:3]
 ; FLAT-NEXT:    v_lshlrev_b64 v[4:5], 1, v[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir b/llvm/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir
index 323396795dd46..4ac48b1133aad 100644
--- a/llvm/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir
+++ b/llvm/test/CodeGen/AMDGPU/break-vmem-soft-clauses.mir
@@ -393,12 +393,12 @@ name: trivial_clause_load_mubuf4_x2
 body: |
   bb.0:
     ; GCN-LABEL: name: trivial_clause_load_mubuf4_x2
-    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
 
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFEN $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -407,13 +407,13 @@ name: break_clause_simple_load_mubuf_offen_ptr
 body: |
   bb.0:
     ; GCN-LABEL: name: break_clause_simple_load_mubuf_offen_ptr
-    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; XNACK-NEXT: S_NOP 0
-    ; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
 
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -424,11 +424,11 @@ name: mubuf_load4_overwrite_ptr
 body: |
   bb.0:
     ; GCN-LABEL: name: mubuf_load4_overwrite_ptr
-    ; GCN: $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     ; GCN-NEXT: $vgpr2 = V_MOV_B32_e32 $vgpr0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr1 = V_MOV_B32_e32 0, implicit $exec
     $vgpr2 = V_MOV_B32_e32 $vgpr0, implicit $exec
     S_ENDPGM 0
@@ -443,11 +443,11 @@ body: |
     ; GCN-LABEL: name: break_clause_flat_load_mubuf_load
     ; GCN: $vgpr0 = FLAT_LOAD_DWORD $vgpr2_vgpr3, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
     ; XNACK-NEXT: S_NOP 0
-    ; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
 
     $vgpr0 = FLAT_LOAD_DWORD $vgpr2_vgpr3, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
-    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 # Break a clause from interference between mubuf and flat instructions
@@ -462,7 +462,7 @@ name: break_clause_mubuf_load_flat_load
 
 body: |
   bb.0:
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr1 = FLAT_LOAD_DWORD $vgpr2_vgpr3, 0, 0, 0, 0, implicit $exec, implicit $flat_scr
 
     S_ENDPGM 0
@@ -504,12 +504,12 @@ name: break_clause_atomic_rtn_into_ptr_mubuf4
 body: |
   bb.0:
     ; GCN-LABEL: name: break_clause_atomic_rtn_into_ptr_mubuf4
-    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; XNACK-NEXT: S_NOP 0
     ; GCN-NEXT: $vgpr2 = BUFFER_ATOMIC_ADD_OFFEN_RTN $vgpr2, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
 
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr2 = BUFFER_ATOMIC_ADD_OFFEN_RTN $vgpr2, $vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
@@ -521,11 +521,11 @@ body: |
   bb.0:
     ; GCN-LABEL: name: break_clause_atomic_nortn_ptr_load_mubuf4
     ; GCN: BUFFER_ATOMIC_ADD_OFFEN $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, implicit $exec
-    ; GCN-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
 
     BUFFER_ATOMIC_ADD_OFFEN $vgpr0, $vgpr1, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -536,11 +536,11 @@ name: no_break_clause_mubuf_load_novaddr
 body: |
   bb.0:
     ; GCN-LABEL: name: no_break_clause_mubuf_load_novaddr
-    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN-NEXT: $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN-NEXT: S_ENDPGM 0
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/clamp-omod-special-case.mir b/llvm/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
index 16021126d1ea0..f631bcd258115 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
+++ b/llvm/test/CodeGen/AMDGPU/clamp-omod-special-case.mir
@@ -55,10 +55,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -117,10 +117,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_MAX_F32_e64 0, killed %20, 0, killed %20, 1, 3, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -180,10 +180,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 0, 3, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -245,10 +245,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_MUL_F32_e64 0, killed %20, 0, 1056964608, 1, 0, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -322,10 +322,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 0, 3, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -387,10 +387,10 @@ body:             |
     %26 = V_LSHL_B64 killed %25, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
     %18 = COPY %26
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %26, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %20 = V_ADD_F32_e64 0, killed %17, 0, 1065353216, 0, 0, implicit $exec
     %21 = V_ADD_F32_e64 0, killed %20, 0, killed %20, 1, 0, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %21, %26, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-extend-pruned-subrange.mir b/llvm/test/CodeGen/AMDGPU/coalescer-extend-pruned-subrange.mir
index 7839d514a1443..599cacb826155 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer-extend-pruned-subrange.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-extend-pruned-subrange.mir
@@ -30,7 +30,7 @@ body: |
     %14:vgpr_32 = V_AND_B32_e32 1, %13, implicit $exec
     %15:sreg_64_xexec = V_CMP_EQ_U32_e64 0, %14, implicit $exec
     %16:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, %15, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN_exact %16, undef %17:vgpr_32, undef %18:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into constant-pool, align 1, addrspace 4)
+    BUFFER_STORE_DWORD_OFFEN_exact %16, undef %17:vgpr_32, undef %18:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into constant-pool, align 1, addrspace 4)
     S_ENDPGM 0
 
   bb.2:
@@ -78,7 +78,7 @@ body: |
 
   bb.8:
     successors: %bb.10
-    %31:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN undef %32:vgpr_32, undef %33:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from constant-pool, align 1, addrspace 4)
+    %31:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN undef %32:vgpr_32, undef %33:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from constant-pool, align 1, addrspace 4)
     %34:sreg_64_xexec = V_CMP_NE_U32_e64 0, %31, implicit $exec
     %35:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, -1, %34, implicit $exec
     %28:vgpr_32 = COPY %35
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-copymi-not-live.mir b/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-copymi-not-live.mir
index d7d8b41f68330..bc549f7bb87b4 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-copymi-not-live.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-copymi-not-live.mir
@@ -83,7 +83,7 @@ body:             |
 
   bb.9:
     successors: %bb.10(0x80000000)
-    %19:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %18, undef %20:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
+    %19:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %18, undef %20:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
     %21:sreg_64 = V_CMP_NE_U32_e64 target-flags(amdgpu-gotprel) 0, killed %19.sub0, implicit $exec
     %22:sreg_64 = COPY $exec, implicit-def $exec
     %23:sreg_64 = S_AND_B64 %22, %21, implicit-def dead $scc
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-prune-error.mir b/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-prune-error.mir
index 4dffe32f9b180..67399883ae07d 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-prune-error.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-subranges-another-prune-error.mir
@@ -68,7 +68,7 @@ body:             |
     %23:vreg_128 = COPY killed %17
     %24:sreg_64 = COPY killed %16
     %25:vgpr_32 = V_OR_B32_e32 %22, %11, implicit $exec
-    %26:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %25, undef %27:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
+    %26:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %25, undef %27:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
     %28:vgpr_32 = V_LSHRREV_B32_e32 30, killed %26.sub0, implicit $exec
     %29:vreg_128 = COPY killed %21
     %29.sub0:vreg_128 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-subregjoin-fullcopy.mir b/llvm/test/CodeGen/AMDGPU/coalescer-subregjoin-fullcopy.mir
index eb3d6169e9762..773466af7adb2 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer-subregjoin-fullcopy.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-subregjoin-fullcopy.mir
@@ -11,7 +11,7 @@
 #
 # GCN-LABEL: bb.6:
 # GCN: successors: %bb.7(0x{{[0-9]+}}), %bb.18(0x{{[0-9]+}})
-# GCN: %{{[0-9]+}}:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %{{[0-9]+}}, 0, 0, 0, 0, 0, 0, implicit $exec
+# GCN: %{{[0-9]+}}:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET %{{[0-9]+}}, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 #
 
 --- |
@@ -69,7 +69,7 @@ body: |
     %10:sreg_64 = COPY killed %5
     undef %11.sub2:sreg_128 = COPY %4
     %11.sub3:sreg_128 = COPY %3
-    %12:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET killed %11, 0, 0, 0, 0, 0, 0, implicit $exec
+    %12:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET killed %11, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     undef %13.sub1:vreg_128 = COPY %9.sub1
     %13.sub2:vreg_128 = COPY %9.sub2
     %14:sreg_64 = V_CMP_GT_F32_e64 0, target-flags(amdgpu-rel32-lo) 0, 0, killed %12.sub3, 0, implicit $exec
@@ -161,7 +161,7 @@ body: |
   bb.18:
     successors: %bb.7(0x80000000)
     dead %59:vgpr_32 = V_FMA_F32 0, killed %9.sub2, 0, undef %60:vgpr_32, 0, undef %61:vgpr_32, 0, 0, implicit $exec
-    dead %62:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN undef %63:vgpr_32, undef %64:sreg_128, undef %65:sreg_32, 0, 0, 0, 0, 0, implicit $exec
+    dead %62:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN undef %63:vgpr_32, undef %64:sreg_128, undef %65:sreg_32, 0, 0, 0, 0, 0, 0, implicit $exec
     undef %66.sub1:vreg_128 = COPY %13.sub1
     %66.sub2:vreg_128 = COPY %13.sub2
     %67:sreg_64 = V_CMP_NGT_F32_e64 0, 0, 0, undef %68:vgpr_32, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/coalescer-with-subregs-bad-identical.mir b/llvm/test/CodeGen/AMDGPU/coalescer-with-subregs-bad-identical.mir
index a01f1e71dacef..4c532e89398e1 100644
--- a/llvm/test/CodeGen/AMDGPU/coalescer-with-subregs-bad-identical.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalescer-with-subregs-bad-identical.mir
@@ -148,7 +148,7 @@ body: |
     %43:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %44:sreg_128, 12, 0, 0 :: (dereferenceable invariant load 4)
     %45:vgpr_32 = V_MUL_LO_I32 killed %42, killed %43, implicit $exec
     %46:vgpr_32 = V_LSHLREV_B32_e32 2, killed %45, implicit $exec
-    %47:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN killed %46, undef %48:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from constant-pool, align 1, addrspace 4)
+    %47:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN killed %46, undef %48:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from constant-pool, align 1, addrspace 4)
     %49:sreg_64 = V_CMP_NE_U32_e64 0, killed %47, implicit $exec
     %50:sreg_64 = COPY $exec, implicit-def $exec
     %51:sreg_64 = S_AND_B64 %50, %49, implicit-def dead $scc
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir b/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir
index 708814e3df458..1a26a507cd9bf 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir
@@ -33,7 +33,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
@@ -44,7 +44,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   DBG_VALUE
@@ -80,7 +80,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -92,7 +92,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12, implicit-def $scc
@@ -141,7 +141,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
@@ -152,7 +152,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN: bb.4:
@@ -188,7 +188,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -200,7 +200,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12, implicit-def $scc
@@ -249,7 +249,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
@@ -260,7 +260,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN: bb.4:
@@ -297,7 +297,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -309,7 +309,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12, implicit-def $scc
@@ -358,7 +358,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
@@ -370,7 +370,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
@@ -408,7 +408,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -420,7 +420,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     %15:sgpr_32 = IMPLICIT_DEF
@@ -471,7 +471,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
   ; GCN:   $exec = S_MOV_B64_term [[S_AND_B64_1]]
@@ -482,7 +482,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF
@@ -520,7 +520,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -532,7 +532,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12, implicit-def $scc
@@ -583,7 +583,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
@@ -595,7 +595,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
@@ -631,7 +631,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -643,7 +643,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12,  implicit-def $scc
@@ -691,7 +691,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
@@ -703,7 +703,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
@@ -739,7 +739,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -751,7 +751,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12,  implicit-def $scc
@@ -799,7 +799,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
@@ -811,7 +811,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.5(0x80000000)
   ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
@@ -850,7 +850,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -862,7 +862,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12, implicit-def $scc
diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf2.mir b/llvm/test/CodeGen/AMDGPU/collapse-endcf2.mir
index 44a8e38a5655b..9219083bb64ce 100644
--- a/llvm/test/CodeGen/AMDGPU/collapse-endcf2.mir
+++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf2.mir
@@ -42,7 +42,7 @@ body:             |
   ; GCN:   %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec
   ; GCN:   %5.sub3:sgpr_128 = S_MOV_B32 61440
   ; GCN:   %5.sub2:sgpr_128 = S_MOV_B32 0
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN:   [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec
   ; GCN:   [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec
   ; GCN:   [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
@@ -54,7 +54,7 @@ body:             |
   ; GCN:   %5.sub0:sgpr_128 = COPY %5.sub2
   ; GCN:   %5.sub1:sgpr_128 = COPY %5.sub2
   ; GCN:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
   ; GCN: bb.3:
   ; GCN:   successors: %bb.4(0x80000000)
   ; GCN:   $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc
@@ -91,7 +91,7 @@ body:             |
     %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec
     %5.sub3:sgpr_128 = S_MOV_B32 61440
     %5.sub2:sgpr_128 = S_MOV_B32 0
-    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
     %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec
     %12:sreg_64 = COPY $exec, implicit-def $exec
     %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc
@@ -103,7 +103,7 @@ body:             |
     %5.sub0:sgpr_128 = COPY %5.sub2
     %5.sub1:sgpr_128 = COPY %5.sub2
     %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
+    BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1)
 
   bb.3:
     $exec = S_OR_B64 $exec, %12,  implicit-def $scc
diff --git a/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir b/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
index e5ff97a7be3de..92e29f3a52909 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/constant-fold-imm-immreg.mir
@@ -54,7 +54,7 @@ body:             |
     %8 = S_MOV_B32 9999
     %9 = S_AND_B32 killed %7, killed %8, implicit-def dead $scc
     %10 = COPY %9
-    BUFFER_STORE_DWORD_OFFSET killed %10, killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed %10, killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -219,7 +219,7 @@ body:             |
     %10 = REG_SEQUENCE killed %7, 1, killed %6, 2, killed %9, 3, killed %8, 4
     %12 = S_LSHL_B32 killed %5, 12, implicit-def dead $scc
     %13 = COPY %12
-    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -419,7 +419,7 @@ body:             |
     %10 = REG_SEQUENCE killed %7, 1, killed %6, 2, killed %9, 3, killed %8, 4
     %12 = S_ASHR_I32 killed %5, 12, implicit-def dead $scc
     %13 = COPY %12
-    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -627,7 +627,7 @@ body:             |
     %10 = REG_SEQUENCE killed %7, 1, killed %6, 2, killed %9, 3, killed %8, 4
     %12 = S_LSHR_B32 killed %5, 12, implicit-def dead $scc
     %13 = COPY %12
-    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed %13, killed %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/couldnt-join-subrange-3.mir b/llvm/test/CodeGen/AMDGPU/couldnt-join-subrange-3.mir
index 4679831c786d1..bba41584bc97f 100644
--- a/llvm/test/CodeGen/AMDGPU/couldnt-join-subrange-3.mir
+++ b/llvm/test/CodeGen/AMDGPU/couldnt-join-subrange-3.mir
@@ -291,7 +291,7 @@ body:             |
   bb.3..lr.ph3410.preheader:
     successors: %bb.4(0x80000000)
 
-    dead %22:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %53.sub3, undef %24:sreg_128, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
+    dead %22:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN killed %53.sub3, undef %24:sreg_128, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from constant-pool, align 1, addrspace 4)
     dead %60:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
     %36:sreg_64 = S_AND_B64 $exec, -1, implicit-def dead $scc
     dead %67:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
index a39833455a153..70e5df5788aee 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_subvector_vec4_vec3.ll
@@ -12,7 +12,7 @@ define amdgpu_hs void @main([0 x i8] addrspace(6)* inreg %arg) {
   ; GCN:   [[DEF:%[0-9]+]]:sreg_32_xm0 = IMPLICIT_DEF
   ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
   ; GCN:   [[DEF1:%[0-9]+]]:sreg_128 = IMPLICIT_DEF
-  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom TargetCustom7, align 1, addrspace 4)
+  ; GCN:   [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[DEF1]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 16 from custom TargetCustom7, align 1, addrspace 4)
   ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2
   ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub1
   ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0
@@ -21,7 +21,7 @@ define amdgpu_hs void @main([0 x i8] addrspace(6)* inreg %arg) {
   ; GCN:   [[DEF2:%[0-9]+]]:sreg_32_xm0 = IMPLICIT_DEF
   ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[DEF2]]
   ; GCN:   [[DEF3:%[0-9]+]]:sreg_128 = IMPLICIT_DEF
-  ; GCN:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom TargetCustom7, align 1, addrspace 4)
+  ; GCN:   BUFFER_STORE_DWORDX3_OFFEN_exact killed [[COPY4]], [[COPY5]], [[DEF3]], [[S_MOV_B32_]], 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 12 into custom TargetCustom7, align 1, addrspace 4)
   ; GCN:   S_ENDPGM 0
 main_body:
   %tmp25 = call <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32> undef, i32 undef, i32 0, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f64.ll b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
index 907121f1cd46b..4bc8f5ec4636b 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
@@ -4,7 +4,7 @@
 declare double @llvm.fma.f64(double, double, double) nounwind readnone
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
 declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>) nounwind readnone
-
+declare double @llvm.fabs.f64(double) nounwind readnone
 
 ; FUNC-LABEL: {{^}}fma_f64:
 ; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
@@ -45,3 +45,156 @@ define amdgpu_kernel void @fma_v4f64(<4 x double> addrspace(1)* %out, <4 x doubl
    store <4 x double> %r3, <4 x double> addrspace(1)* %out
    ret void
 }
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], |v\[[0-9]+:[0-9]+\]|, v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r0)
+   %r3 = tail call double @llvm.fma.f64(double %fabs, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r1)
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fabs, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|}}
+define amdgpu_kernel void @fma_f64_abs_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r2)
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fabs)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r0
+   %r3 = tail call double @llvm.fma.f64(double %fsub, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r1
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fsub, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_neg_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_neg_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fsub = fsub double -0.000000e+00, %r2
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fsub)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_neg_src0(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r0)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %fsub, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_abs_neg_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r1)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %fsub, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_abs_neg_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|}}
+define amdgpu_kernel void @fma_f64_abs_neg_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %fabs = call double @llvm.fabs.f64(double %r2)
+   %fsub = fsub double -0.000000e+00, %fabs
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double %fsub)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src0:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_lit_src0(double addrspace(1)* %out,
+                     double addrspace(1)* %in2, double addrspace(1)* %in3) {
+   %r1 = load double, double addrspace(1)* %in2
+   %r2 = load double, double addrspace(1)* %in3
+   %r3 = tail call double @llvm.fma.f64(double +2.0, double %r1, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src1:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+define amdgpu_kernel void @fma_f64_lit_src1(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in3) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r2 = load double, double addrspace(1)* %in3
+   %r3 = tail call double @llvm.fma.f64(double %r0, double +2.0, double %r2)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
+
+; FUNC-LABEL: {{^}}fma_f64_lit_src2:
+; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0}}
+define amdgpu_kernel void @fma_f64_lit_src2(double addrspace(1)* %out, double addrspace(1)* %in1,
+                     double addrspace(1)* %in2) {
+   %r0 = load double, double addrspace(1)* %in1
+   %r1 = load double, double addrspace(1)* %in2
+   %r3 = tail call double @llvm.fma.f64(double %r0, double %r1, double +2.0)
+   store double %r3, double addrspace(1)* %out
+   ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index d18705f1dd410..d8a4a0b841d01 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -126,7 +126,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f32(float addrspace(1)* %out, float %x, f
 ; GFX8_10:      v_mul_f16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; VI-FLUSH:     v_mad_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
 ; VI-DENORM:    v_fma_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
-; GFX10-DENORM: v_fmac_f16_e64 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}
+; GFX10-DENORM: v_fma_f16 v{{[0-9]+}}, -v{{[0-9]+}}, v{{[0-9]+}}, 1.0
 ; GFX10-FLUSH:  v_sub_f16_e32 v{{[0-9]+}}, 1.0, v{{[0-9]+}}
 define amdgpu_kernel void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %x = bitcast i16 %x.arg to half
@@ -152,7 +152,7 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(half addrspace(1)* %out, i
 ; VI-FLUSH-DAG:     v_mac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0
 ; VI-DENORM-DAG:    v_fma_f16 [[MAD:v[0-9]+]], [[X]], 2.0, v{{[0-9]+}}
 ; GFX10-FLUSH-DAG:  v_add_f16_e32 [[MAD:v[0-9]+]], s{{[0-9]+}}, [[MUL2]]
-; GFX10-DENORM-DAG: v_fmac_f16_e64 [[MAD:v[0-9]+]], [[X]], 2.0
+; GFX10-DENORM-DAG: v_fma_f16 [[MAD:v[0-9]+]], [[X]], 2.0, s{{[0-9]+}}
 
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
@@ -174,7 +174,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(half addrspace(1)* %out, i
 ; VI-FLUSH-DAG:     v_mad_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; VI-DENORM-DAG:    v_fma_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, v{{[0-9]+}}
 ; GFX10-FLUSH-DAG:  v_add_f16_e32 [[MAD:v[0-9]+]], s{{[0-9]+}}, [[MUL2]]
-; GFX10-DENORM-DAG: v_fmac_f16_e64 [[MAD:v[0-9]+]], |[[X]]|, 2.0
+; GFX10-DENORM-DAG: v_fma_f16 [[MAD:v[0-9]+]], |[[X]]|, 2.0, s{{[0-9]+}}
 
 ; GCN-DAG: buffer_store_short [[MUL2]]
 ; GCN-DAG: buffer_store_short [[MAD]]
@@ -201,8 +201,8 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(half addrspace(1)* %out, i
 ; GFX10-FLUSH:  v_add_f16_e64 [[MUL2:v[0-9]+]], |[[X:s[0-9]+]]|, |{{s[0-9]+}}|
 ; GFX10-FLUSH:  v_add_f16_e32 {{v[0-9]+}}, {{s[0-9]+}}, [[MUL2]]
 ; GFX10-FLUSH:  v_add_f16_e32 {{v[0-9]+}}, {{s[0-9]+}}, [[MUL2]]
-; GFX10-DENORM: v_fmac_f16_e64 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0
-; GFX10-DENORM: v_fmac_f16_e64 {{v[0-9]+}}, |[[X]]|, 2.0
+; GFX10-DENORM: v_fma_f16 {{v[0-9]+}}, |[[X:s[0-9]+]]|, 2.0, s{{[0-9]+}}
+; GFX10-DENORM: v_fma_f16 {{v[0-9]+}}, |[[X]]|, 2.0, s{{[0-9]+}}
 
 define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(half addrspace(1)* %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
   %x = bitcast i16 %x.arg to half
diff --git a/llvm/test/CodeGen/AMDGPU/fold-fi-mubuf.mir b/llvm/test/CodeGen/AMDGPU/fold-fi-mubuf.mir
index a015a1ef4d113..f80176508befe 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-fi-mubuf.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-fi-mubuf.mir
@@ -23,13 +23,13 @@ body:             |
     ; GCN: liveins: $sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
-    ; GCN: [[BUFFER_LOAD_DWORD_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN [[V_MOV_B32_e32_]], [[COPY]], 0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: [[BUFFER_LOAD_DWORD_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN [[V_MOV_B32_e32_]], [[COPY]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_IDXEN]]
     ; GCN: SI_RETURN_TO_EPILOG $vgpr0
     %0:sgpr_128 = COPY $sgpr12_sgpr13_sgpr14_sgpr15
     %1:sreg_32_xm0 = S_MOV_B32 0
     %2:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
-    %3:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %2, %0, %1, 0, 0, 0, 0, 0, implicit $exec
+    %3:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %2, %0, %1, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr0 = COPY %3
     SI_RETURN_TO_EPILOG $vgpr0
 
@@ -57,12 +57,12 @@ body:             |
     ; GCN: liveins: $sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: [[COPY:%[0-9]+]]:sgpr_128 = COPY $sgpr12_sgpr13_sgpr14_sgpr15
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
-    ; GCN: [[BUFFER_LOAD_DWORD_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN [[V_MOV_B32_e32_]], [[COPY]], $sgpr32, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: [[BUFFER_LOAD_DWORD_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN [[V_MOV_B32_e32_]], [[COPY]], $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_IDXEN]]
     ; GCN: SI_RETURN_TO_EPILOG $vgpr0
     %0:sgpr_128 = COPY $sgpr12_sgpr13_sgpr14_sgpr15
     %2:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
-    %3:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %2, %0, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
+    %3:vgpr_32 = BUFFER_LOAD_DWORD_IDXEN %2, %0, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr0 = COPY %3
     SI_RETURN_TO_EPILOG $vgpr0
 
@@ -87,15 +87,15 @@ body:             |
 
     ; GCN-LABEL: name: fold_fi_mubuf_scratch_scratch_wave_offset
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 7, implicit $exec
-    ; GCN: BUFFER_STORE_DWORD_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
-    ; GCN: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: BUFFER_STORE_DWORD_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     ; GCN: S_ENDPGM 0, implicit $vgpr0
     %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     %1:vgpr_32 = V_MOV_B32_e32 7, implicit $exec
 
-    BUFFER_STORE_DWORD_OFFEN %1:vgpr_32, %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr33, 0, 0, 0, 0, 0, implicit $exec
-    %2:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr33, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %1:vgpr_32, %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr33, 0, 0, 0, 0, 0, 0, implicit $exec
+    %2:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr33, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr0 = COPY %2
     S_ENDPGM 0, implicit $vgpr0
 
@@ -119,15 +119,15 @@ body:             |
 
     ; GCN-LABEL: name: no_fold_fi_mubuf_scratch_sp_offset
     ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 7, implicit $exec
-    ; GCN: BUFFER_STORE_DWORD_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
-    ; GCN: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: BUFFER_STORE_DWORD_OFFEN [[V_MOV_B32_e32_]], %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
+    ; GCN: [[BUFFER_LOAD_DWORD_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
     ; GCN: $vgpr0 = COPY [[BUFFER_LOAD_DWORD_OFFEN]]
     ; GCN: S_ENDPGM 0, implicit $vgpr0
     %0:vgpr_32 = V_MOV_B32_e32 %stack.0, implicit $exec
     %1:vgpr_32 = V_MOV_B32_e32 7, implicit $exec
 
-    BUFFER_STORE_DWORD_OFFEN %1:vgpr_32, %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
-    %2:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %1:vgpr_32, %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
+    %2:vgpr_32 = BUFFER_LOAD_DWORD_OFFEN %0:vgpr_32, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr0 = COPY %2
     S_ENDPGM 0, implicit $vgpr0
 
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
index 7fe6ce845ab9e..f2d423a707851 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-copy.mir
@@ -17,7 +17,7 @@ body:             |
     %4:vgpr_32 = V_LSHLREV_B32_e64 killed %3, %0, implicit $exec
     %5:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
     %6:vreg_64 = REG_SEQUENCE killed %4, %subreg.sub0, killed %5, %subreg.sub1
-    %7:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %6, %2, 0, 4, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %6, %2, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %8:sreg_32_xm0 = S_MOV_B32 65535
     %9:vgpr_32 = COPY %8
     %10:vgpr_32 = V_AND_B32_e32 %7, %9, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/fold-imm-f16-f32.mir b/llvm/test/CodeGen/AMDGPU/fold-imm-f16-f32.mir
index 3ab99551012fd..1e596b79016ab 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-imm-f16-f32.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-imm-f16-f32.mir
@@ -158,10 +158,10 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
     %12 = V_MOV_B32_e32 1065353216, implicit $exec
     %13 = V_ADD_F16_e64 0, killed %11, 0, %12, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %13, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %13, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -222,13 +222,13 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %13 = V_MOV_B32_e32 1065353216, implicit $exec
     %14 = V_ADD_F16_e64 0, killed %11, 0, %13, 0, 0, implicit $exec
     %15 = V_ADD_F16_e64 0, killed %12, 0, killed %13, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -289,14 +289,14 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %14 = V_MOV_B32_e32 1065353216, implicit $exec
     %15 = V_ADD_F16_e64 0, %11, 0, %14, 0, 0, implicit $exec
     %16 = V_ADD_F32_e64 0, killed %13, 0, killed %14, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_DWORD_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -360,16 +360,16 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %14 = V_MOV_B32_e32 1065353216, implicit $exec
     %15 = V_ADD_F16_e64 0, %11, 0, %14, 0, 0, implicit $exec
     %16 = V_ADD_F16_e64 0, %12, 0, %14, 0, 0, implicit $exec
     %17 = V_ADD_F32_e64 0, killed %13, 0, killed %14, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_DWORD_OFFSET killed %17, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %17, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -427,13 +427,13 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %13 = V_MOV_B32_e32 1, implicit $exec
     %14 = V_ADD_F16_e64 0, killed %11, 0, %13, 0, 0, implicit $exec
     %15 = V_ADD_F16_e64 0, killed %12, 0, killed %13, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -494,16 +494,16 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %13 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %14 = V_MOV_B32_e32 -2, implicit $exec
     %15 = V_ADD_F16_e64 0, %11, 0, %14, 0, 0, implicit $exec
     %16 = V_ADD_F16_e64 0, %12, 0, %14, 0, 0, implicit $exec
     %17 = V_ADD_F32_e64 0, killed %13, 0, killed %14, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_DWORD_OFFSET killed %17, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %16, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %17, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -564,13 +564,13 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     %13 = V_MOV_B32_e32 15360, implicit $exec
     %14 = V_ADD_F32_e64 0, %11, 0, %13, 0, 0, implicit $exec
     %15 = V_ADD_F32_e64 0, %12, 0, %13, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
-    BUFFER_STORE_DWORD_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -631,13 +631,13 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
     %13 = V_MOV_B32_e32 80886784, implicit $exec
     %14 = V_ADD_F16_e64 0, %11, 0, %13, 0, 0, implicit $exec
     %15 = V_ADD_F16_e64 0, %12, 0, %13, 0, 0, implicit $exec
-    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
@@ -697,13 +697,13 @@ body:             |
     %8 = S_MOV_B32 61440
     %9 = S_MOV_B32 -1
     %10 = REG_SEQUENCE killed %7, 1, killed %5, 2, killed %9, 3, killed %8, 4
-    %11 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
-    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
+    %11 = BUFFER_LOAD_DWORD_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    %12 = BUFFER_LOAD_USHORT_OFFSET %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 2 from `half addrspace(1)* undef`)
     %13 = V_MOV_B32_e32 305413120, implicit $exec
     %14 = V_ADD_F32_e64 0, %11, 0, %13, 0, 0, implicit $exec
     %15 = V_ADD_F16_e64 0, %12, 0, %13, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
-    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed %14, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `float addrspace(1)* undef`)
+    BUFFER_STORE_SHORT_OFFSET killed %15, %10, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 2 into `half addrspace(1)* undef`)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir b/llvm/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
index 65b254e7616ad..e26f0c934fce4 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-immediate-output-mods.mir
@@ -60,13 +60,13 @@ body:             |
     %17 = REG_SEQUENCE killed %6, 17, %13, 18
     %18 = REG_SEQUENCE killed %4, 17, %13, 18
     %20 = COPY %29
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %22 = COPY %29
-    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, implicit $exec
+    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %23 = V_MOV_B32_e32 1090519040, implicit $exec
     %24 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit $exec
     %26 = COPY %29
-    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -131,13 +131,13 @@ body:             |
     %17 = REG_SEQUENCE killed %6, 17, %13, 18
     %18 = REG_SEQUENCE killed %4, 17, %13, 18
     %20 = COPY %29
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %22 = COPY %29
-    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, implicit $exec
+    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %23 = V_MOV_B32_e32 1090519040, implicit $exec
     %24 = V_MAC_F32_e64 0, killed %19, 0, killed %21, 0, %23, 0, 2, implicit $exec
     %26 = COPY %29
-    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -202,13 +202,13 @@ body:             |
     %17 = REG_SEQUENCE killed %6, 17, %13, 18
     %18 = REG_SEQUENCE killed %4, 17, %13, 18
     %20 = COPY %29
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %22 = COPY %29
-    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, implicit $exec
+    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %23 = V_MOV_B32_e32 1090519040, implicit $exec
     %24 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 1, 0, implicit $exec
     %26 = COPY %29
-    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -273,13 +273,13 @@ body:             |
     %17 = REG_SEQUENCE killed %6, 17, %13, 18
     %18 = REG_SEQUENCE killed %4, 17, %13, 18
     %20 = COPY %29
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %20, killed %14, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %22 = COPY %29
-    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, implicit $exec
+    %21 = BUFFER_LOAD_DWORD_ADDR64 %22, killed %17, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %23 = V_MOV_B32_e32 1090519040, implicit $exec
     %24 = V_MAD_F32 0, killed %19, 0, killed %21, 0, %23, 0, 1, implicit $exec
     %26 = COPY %29
-    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %24, %26, killed %18, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-multiple.mir b/llvm/test/CodeGen/AMDGPU/fold-multiple.mir
index ef35b26345795..d8c396c9d4a4f 100644
--- a/llvm/test/CodeGen/AMDGPU/fold-multiple.mir
+++ b/llvm/test/CodeGen/AMDGPU/fold-multiple.mir
@@ -34,7 +34,7 @@ body:             |
     %3 = S_LSHL_B32 %1, killed %1, implicit-def dead $scc
     %4 = V_AND_B32_e64 killed %2, killed %3, implicit $exec
     %5 = IMPLICIT_DEF
-    BUFFER_STORE_DWORD_OFFSET killed %4, killed %5, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed %4, killed %5, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/fold-over-exec.mir b/llvm/test/CodeGen/AMDGPU/fold-over-exec.mir
new file mode 100644
index 0000000000000..9cc6c257b029d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold-over-exec.mir
@@ -0,0 +1,38 @@
+# RUN: llc -mtriple=amdgcn -verify-machineinstrs -run-pass si-fold-operands %s -o - | FileCheck %s
+
+# CHECK-LABEL: bb.2:
+# CHECK: %I_out
+# CHECK-NOT: %I
+---
+name:            main
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $sgpr0, $vgpr0, $vgpr1
+    %0:sreg_32 = S_MOV_B32 0
+    %1:sreg_64 = S_MOV_B64 0
+    %tid_x:vgpr_32 = COPY $vgpr0
+    %tid_y:vgpr_32 = COPY $vgpr1
+    %arg:sgpr_32 = COPY $sgpr0
+
+  bb.1: ; loop
+    successors: %bb.2(0x40000000), %bb.1(0x40000000)
+    %I:sreg_32 = PHI %0, %bb.0, %I_inc, %bb.1
+    %break_inc:sreg_64 = PHI %1, %bb.0, %break, %bb.1
+    %cond:sreg_64 = V_CMP_LE_U32_e64 %arg, %tid_x, implicit $exec
+    %break:sreg_64 = SI_IF_BREAK %cond, %break_inc, implicit-def $scc
+    %I_out:vgpr_32 = COPY %I, implicit $exec
+    %I_inc:sreg_32 = S_ADD_I32 %I, 1, implicit-def $scc
+    SI_LOOP %break, %bb.1, implicit-def dead $exec, implicit $exec, implicit-def $scc
+    S_BRANCH %bb.2
+
+  bb.2: ; end loop
+    successors: %bb.3(0x40000000)
+    %2:sreg_64_xexec = V_CMP_EQ_U32_e64 %I_out, %tid_y, implicit $exec
+    %3:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %2, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.3: ; exit
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/fold_acc_copy_into_valu.mir b/llvm/test/CodeGen/AMDGPU/fold_acc_copy_into_valu.mir
new file mode 100644
index 0000000000000..11af6e19ecb24
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fold_acc_copy_into_valu.mir
@@ -0,0 +1,15 @@
+# RUN: llc -march=amdgcn -mcpu=gfx908 -o - -run-pass si-fix-sgpr-copies -verify-machineinstrs %s | FileCheck -check-prefix=GCN %s
+
+# GCN-LABEL: fold_acc_copy_into_valu
+# GCN: [[COPY:%[0-9]+]]:vgpr_32 = COPY %0.sub0
+# GCN: %2:vgpr_32 = V_AND_B32_e32 [[COPY]], undef %3:vgpr_32, implicit $exec
+---
+name:            fold_acc_copy_into_valu
+body:             |
+  bb.0.entry:
+
+    %0:areg_1024 = IMPLICIT_DEF
+    %1:sreg_32_xm0 = COPY %0.sub0
+    %3:vgpr_32 = V_AND_B32_e32 %1, undef %2:vgpr_32, implicit $exec
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-buffer-store-v-interp.mir b/llvm/test/CodeGen/AMDGPU/hazard-buffer-store-v-interp.mir
index 8cc294f57b268..bd6244127e6f9 100644
--- a/llvm/test/CodeGen/AMDGPU/hazard-buffer-store-v-interp.mir
+++ b/llvm/test/CodeGen/AMDGPU/hazard-buffer-store-v-interp.mir
@@ -12,7 +12,7 @@ body:             |
   bb.0.entry:
     liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr7, $vgpr8, $vgpr9, $vgpr10
   
-    BUFFER_STORE_DWORDX4_OFFSET_exact killed $vgpr7_vgpr8_vgpr9_vgpr10, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 96, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORDX4_OFFSET_exact killed $vgpr7_vgpr8_vgpr9_vgpr10, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 96, 0, 0, 0, 0, 0, implicit $exec
     $vgpr7 = V_INTERP_P1_F32 $vgpr0, 0, 0, implicit $m0, implicit $exec
     S_ENDPGM 0
 
diff --git a/llvm/test/CodeGen/AMDGPU/hazard-hidden-bundle.mir b/llvm/test/CodeGen/AMDGPU/hazard-hidden-bundle.mir
index 9ef2431df6ee7..d0f32f287473c 100644
--- a/llvm/test/CodeGen/AMDGPU/hazard-hidden-bundle.mir
+++ b/llvm/test/CodeGen/AMDGPU/hazard-hidden-bundle.mir
@@ -57,7 +57,7 @@ body: |
     BUNDLE implicit-def $sgpr0_sgpr1, implicit $sgpr10_sgpr11 {
       $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM $sgpr10_sgpr11, 0, 0, 0
     }
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -91,5 +91,5 @@ body: |
     }
 
   bb.2:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll b/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
index abcf3342fcf45..e7ad0bd0122e6 100644
--- a/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
+++ b/llvm/test/CodeGen/AMDGPU/illegal-sgpr-to-vgpr-copy.ll
@@ -1,5 +1,5 @@
 ; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck -check-prefix=ERR %s
-; RUN: not llc -march=amdgcn < %s | FileCheck -check-prefix=GCN %s
+; RUN: not llc -march=amdgcn < %s 2>&1 | FileCheck -check-prefix=GCN %s
 
 ; ERR: error: <unknown>:0:0: in function illegal_vgpr_to_sgpr_copy_i32 void (): illegal SGPR to VGPR copy
 ; GCN: ; illegal copy v1 to s9
@@ -43,7 +43,8 @@ define amdgpu_kernel void @illegal_vgpr_to_sgpr_copy_v16i32() #0 {
 }
 
 ; ERR: error: <unknown>:0:0: in function illegal_agpr_to_sgpr_copy_i32 void (): illegal SGPR to VGPR copy
-; GCN: ; illegal copy a1 to s9
+; GCN: v_accvgpr_read_b32 [[COPY1:v[0-9]+]], a1
+; GCN: ; illegal copy [[COPY1]] to s9
 define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_i32() #1 {
   %agpr = call i32 asm sideeffect "; def $0", "=${a1}"()
   call void asm sideeffect "; use $0", "${s9}"(i32 %agpr)
@@ -51,7 +52,9 @@ define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_i32() #1 {
 }
 
 ; ERR: error: <unknown>:0:0: in function illegal_agpr_to_sgpr_copy_v2i32 void (): illegal SGPR to VGPR copy
-; GCN: ; illegal copy a[0:1] to s[10:11]
+; GCN-DAG: v_accvgpr_read_b32 v[[COPY1L:[0-9]+]], a0
+; GCN-DAG: v_accvgpr_read_b32 v[[COPY1H:[0-9]+]], a1
+; GCN: ; illegal copy v{{\[}}[[COPY1L]]:[[COPY1H]]] to s[10:11]
 define amdgpu_kernel void @illegal_agpr_to_sgpr_copy_v2i32() #1 {
   %vgpr = call <2 x i32> asm sideeffect "; def $0", "=${a[0:1]}"()
   call void asm sideeffect "; use $0", "${s[10:11]}"(<2 x i32> %vgpr)
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
index 053a684407640..40722ce43741f 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -104,7 +104,7 @@ define amdgpu_kernel void @extract_w_offset_vgpr(i32 addrspace(1)* %out) {
   ; GCN:   $exec = S_MOV_B64 renamable $sgpr0_sgpr1
   ; GCN:   $vgpr0 = SI_SPILL_V32_RESTORE %stack.8, $sgpr96_sgpr97_sgpr98_sgpr99, $sgpr3, 0, implicit $exec :: (load 4 from %stack.8, addrspace 5)
   ; GCN:   $sgpr4_sgpr5_sgpr6_sgpr7 = SI_SPILL_S128_RESTORE %stack.1, implicit $exec, implicit $sgpr96_sgpr97_sgpr98_sgpr99, implicit $sgpr3, implicit-def dead $m0 :: (load 16 from %stack.1, align 4, addrspace 5)
-  ; GCN:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr0, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.load, addrspace 1)
+  ; GCN:   BUFFER_STORE_DWORD_OFFSET renamable $vgpr0, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.load, addrspace 1)
   ; GCN:   S_ENDPGM 0
 entry:
   %id = call i32 @llvm.amdgcn.workitem.id.x() #1
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem.mir
index 09f1ba9010608..b305cfddb5a5d 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-flat-vmem.mir
@@ -41,7 +41,7 @@ body: |
   ; CHECK: bb.1:
   ; CHECK:   successors: %bb.2(0x80000000)
   ; CHECK:   $vgpr0 = V_MOV_B32_e32 0, implicit $exec
-  ; CHECK:   BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+  ; CHECK:   BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
   ; CHECK: bb.2:
   ; CHECK:   S_ENDPGM 0
   bb.0:
@@ -51,7 +51,7 @@ body: |
   bb.1:
     successors: %bb.2
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2:
     S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-exp.mir b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-exp.mir
index 9f39dc3415099..5797bb5cfa29a 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-exp.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-exp.mir
@@ -49,10 +49,10 @@ body:             |
   bb.0 (%ir-block.2):
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
-    $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
-    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    $vgpr2 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
+    $vgpr3 = BUFFER_LOAD_DWORD_OFFSET killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 4 from `float addrspace(1)* undef`)
     EXP_DONE 0, killed $vgpr0, killed $vgpr1, killed $vgpr2, killed $vgpr3, -1, -1, 15, implicit $exec
     $vgpr0 = V_MOV_B32_e32 1056964608, implicit $exec
     $vgpr1 = V_MOV_B32_e32 1065353216, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/inserted-wait-states.mir b/llvm/test/CodeGen/AMDGPU/inserted-wait-states.mir
index 6e67f7df30a7b..1ab10fa92f7b8 100644
--- a/llvm/test/CodeGen/AMDGPU/inserted-wait-states.mir
+++ b/llvm/test/CodeGen/AMDGPU/inserted-wait-states.mir
@@ -230,17 +230,17 @@ name: vmem_gt_8dw_store
 
 body: |
   bb.0:
-    BUFFER_STORE_DWORD_OFFSET $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET $vgpr3, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_DWORDX3_OFFSET $vgpr2_vgpr3_vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORDX3_OFFSET $vgpr2_vgpr3_vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_DWORDX4_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORDX4_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_DWORDX4_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORDX4_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_FORMAT_XYZ_OFFSET $vgpr2_vgpr3_vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_FORMAT_XYZ_OFFSET $vgpr2_vgpr3_vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
-    BUFFER_STORE_FORMAT_XYZW_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_FORMAT_XYZW_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
     BUFFER_ATOMIC_CMPSWAP_X2_OFFSET $vgpr2_vgpr3_vgpr4_vgpr5, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, implicit $exec
     $vgpr3 = V_MOV_B32_e32 0, implicit $exec
@@ -553,10 +553,10 @@ body:             |
     dead $sgpr6_sgpr7 = KILL $sgpr4_sgpr5
     $sgpr8 = S_MOV_B32 $sgpr5
     $vgpr0 = V_MOV_B32_e32 killed $sgpr8, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.A.addr + 4)
+    BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.A.addr + 4)
     $sgpr8 = S_MOV_B32 $sgpr4, implicit killed $sgpr4_sgpr5
     $vgpr0 = V_MOV_B32_e32 killed $sgpr8, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.A.addr)
+    BUFFER_STORE_DWORD_OFFSET $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr9, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.A.addr)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir b/llvm/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
index 48f0be4ff8fd7..0a60eaf7c03f9 100644
--- a/llvm/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
+++ b/llvm/test/CodeGen/AMDGPU/invert-br-undef-vcc.mir
@@ -64,7 +64,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 100, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
     S_BRANCH %bb.3
 
@@ -72,7 +72,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 9, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
 
   bb.3.done:
@@ -80,7 +80,7 @@ body:             |
 
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/lds-branch-vmem-hazard.mir b/llvm/test/CodeGen/AMDGPU/lds-branch-vmem-hazard.mir
index 69c038b976b8a..566b1c06fb125 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-branch-vmem-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/lds-branch-vmem-hazard.mir
@@ -12,7 +12,7 @@ body:            |
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -24,7 +24,7 @@ name:            hazard_buf_branch_lds
 body:            |
   bb.0:
     successors: %bb.1
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:
@@ -56,11 +56,11 @@ name:            no_hazard_buf_branch_buf
 body:            |
   bb.0:
     successors: %bb.1
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -75,7 +75,7 @@ body:            |
     $vgpr1 = DS_READ_B32 undef $vgpr0, 0, 0, implicit $m0, implicit $exec
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -87,7 +87,7 @@ name:            no_hazard_lds_branch_buf_samebb
 body:            |
   bb.0:
     $vgpr1 = DS_READ_B32 undef $vgpr0, 0, 0, implicit $m0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -101,7 +101,7 @@ body:            |
   bb.0:
     successors: %bb.0
     $vgpr1 = DS_READ_B32 undef $vgpr0, 0, 0, implicit $m0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.0
 ...
 
@@ -118,8 +118,8 @@ body:            |
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -137,7 +137,7 @@ body:            |
 
   bb.1:
     $vgpr1 = DS_READ_B32 undef $vgpr0, 0, 0, implicit $m0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -150,11 +150,11 @@ body:            |
   bb.0:
     successors: %bb.1
     $vgpr1 = DS_READ_B32 undef $vgpr0, 0, 0, implicit $m0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -171,7 +171,7 @@ body:            |
 
   bb.1:
     S_WAITCNT_VSCNT undef $sgpr_null, 1
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -189,7 +189,7 @@ body:            |
 
   bb.1:
     S_WAITCNT_VSCNT undef $sgpr_null, 0
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -206,7 +206,7 @@ body:            |
 
   bb.1:
     S_WAITCNT_VSCNT undef $sgpr0, 0
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
@@ -223,7 +223,7 @@ body:            |
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
index fb79ba8ae4a4f..f3998d9a899d0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.ll
@@ -1,4 +1,5 @@
-;RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s --check-prefix=GCN
+; RUN: llc < %s -march=amdgcn -mcpu=gfx900 -verify-machineinstrs | FileCheck %s --check-prefix=GCN
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=-wavefrontsize32,+wavefrontsize64 -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}full_mask:
 ; GCN: s_mov_b64 exec, -1
@@ -51,7 +52,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_u32_e32 v0, s0, v0
+; GCN: v_add{{(_nc)?}}_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input(i32 inreg %count, i32 %a) {
 main_body:
   call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 19)
@@ -65,7 +66,7 @@ main_body:
 ; GCN: s_bfm_b64 exec, s1, 0
 ; GCN: s_cmp_eq_u32 s1, 64
 ; GCN: s_cmov_b64 exec, -1
-; GCN: v_add_u32_e32 v0, s0, v0
+; GCN: v_add{{(_nc)?}}_u32_e32 v0, s0, v0
 define amdgpu_ps float @reuse_input2(i32 inreg %count, i32 %a) {
 main_body:
   %s = add i32 %a, %count
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll
new file mode 100644
index 0000000000000..3c66f6e01b4b8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.init.exec.wave32.ll
@@ -0,0 +1,31 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -mattr=+wavefrontsize32,-wavefrontsize64 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX1032 %s
+
+; GCN-LABEL: {{^}}test_init_exec:
+; GFX1032: s_mov_b32 exec_lo, 0x12345
+; GFX1064: s_mov_b64 exec, 0x12345
+; GCN: v_add_f32_e32 v0,
+define amdgpu_ps float @test_init_exec(float %a, float %b) {
+main_body:
+  %s = fadd float %a, %b
+  call void @llvm.amdgcn.init.exec(i64 74565)
+  ret float %s
+}
+
+; GCN-LABEL: {{^}}test_init_exec_from_input:
+; GCN: s_bfe_u32 s0, s3, 0x70008
+; GFX1032: s_bfm_b32 exec_lo, s0, 0
+; GFX1032: s_cmp_eq_u32 s0, 32
+; GFX1032: s_cmov_b32 exec_lo, -1
+; GFX1064: s_bfm_b64 exec, s0, 0
+; GFX1064: s_cmp_eq_u32 s0, 64
+; GFX1064: s_cmov_b64 exec, -1
+; GCN: v_add_f32_e32 v0,
+define amdgpu_ps float @test_init_exec_from_input(i32 inreg, i32 inreg, i32 inreg, i32 inreg %count, float %a, float %b) {
+main_body:
+  %s = fadd float %a, %b
+  call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 8)
+  ret float %s
+}
+
+declare void @llvm.amdgcn.init.exec(i64)
+declare void @llvm.amdgcn.init.exec.from.input(i32, i32)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
index 958a72566b5f7..9f18f4df40bf6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.load.ll
@@ -400,6 +400,46 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}raw_buffer_load_x1_offset_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-NEXT: buffer_load_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+;CHECK: s_waitcnt
+define amdgpu_ps void @raw_buffer_load_x1_offset_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %r1 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4, i32 0, i32 0)
+  %r2 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 8, i32 0, i32 0)
+  %r3 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 12, i32 0, i32 0)
+  %r4 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 16, i32 0, i32 0)
+  %r5 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 28, i32 0, i32 0)
+  %r6 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 32, i32 0, i32 0)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_load_x1_offset_swizzled_not_merged:
+;CHECK-NEXT: %bb.
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:4
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:12
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:16
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:28
+;CHECK-NEXT: buffer_load_dword v{{[0-9]}}, off, s[0:3], 0 offset:32
+;CHECK: s_waitcnt
+define amdgpu_ps void @raw_buffer_load_x1_offset_swizzled_not_merged(<4 x i32> inreg %rsrc) {
+main_body:
+  %r1 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 4, i32 0, i32 8)
+  %r2 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 8, i32 0, i32 8)
+  %r3 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 12, i32 0, i32 8)
+  %r4 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 16, i32 0, i32 8)
+  %r5 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 28, i32 0, i32 8)
+  %r6 = call float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32> %rsrc, i32 32, i32 0, i32 8)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r1, float %r2, float %r3, float %r4, i1 true, i1 true)
+  call void @llvm.amdgcn.exp.f32(i32 0, i32 15, float %r5, float %r6, float undef, float undef, i1 true, i1 true)
+  ret void
+}
+
 declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #0
 declare <2 x float> @llvm.amdgcn.raw.buffer.load.v2f32(<4 x i32>, i32, i32, i32) #0
 declare <4 x float> @llvm.amdgcn.raw.buffer.load.v4f32(<4 x i32>, i32, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
index 7d1a5a3b99a0f..1bfe0aa4086e7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.ll
@@ -276,6 +276,37 @@ main_body:
   ret void
 }
 
+;CHECK-LABEL: {{^}}raw_buffer_store_x1_offset_merged:
+;CHECK-NOT: s_waitcnt
+;CHECK-DAG: buffer_store_dwordx4 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:4
+;CHECK-DAG: buffer_store_dwordx2 v[{{[0-9]}}:{{[0-9]}}], off, s[0:3], 0 offset:28
+define amdgpu_ps void @raw_buffer_store_x1_offset_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 4, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 8, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 12, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 16, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 28, i32 0, i32 0)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 32, i32 0, i32 0)
+  ret void
+}
+
+;CHECK-LABEL: {{^}}raw_buffer_store_x1_offset_swizzled_not_merged:
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:4
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:8
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:12
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:16
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:28
+;CHECK-DAG: buffer_store_dword v{{[0-9]}}, off, s[0:3], 0 offset:32
+define amdgpu_ps void @raw_buffer_store_x1_offset_swizzled_not_merged(<4 x i32> inreg %rsrc, float %v1, float %v2, float %v3, float %v4, float %v5, float %v6) {
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v1, <4 x i32> %rsrc, i32 4, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v2, <4 x i32> %rsrc, i32 8, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v3, <4 x i32> %rsrc, i32 12, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v4, <4 x i32> %rsrc, i32 16, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v5, <4 x i32> %rsrc, i32 28, i32 0, i32 8)
+  call void @llvm.amdgcn.raw.buffer.store.f32(float %v6, <4 x i32> %rsrc, i32 32, i32 0, i32 8)
+  ret void
+}
+
 declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.v2f32(<2 x float>, <4 x i32>, i32, i32, i32) #0
 declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32) #0
diff --git a/llvm/test/CodeGen/AMDGPU/loop_header_nopred.mir b/llvm/test/CodeGen/AMDGPU/loop_header_nopred.mir
new file mode 100644
index 0000000000000..a1f23777e30e7
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/loop_header_nopred.mir
@@ -0,0 +1,92 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=amdgcn -o - -run-pass=block-placement -mcpu=gfx1010 -mattr=-inst-fwd-prefetch-bug -verify-machineinstrs %s | FileCheck -check-prefix=GCN %s
+
+# Used to fail with
+# Assertion `Out && "Header of loop has no predecessors from outside loop?"
+
+---
+name:            loop_header_nopred
+body:             |
+  ; GCN-LABEL: name: loop_header_nopred
+  ; GCN: bb.0:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   S_CBRANCH_VCCZ %bb.3, implicit $vcc
+  ; GCN:   S_BRANCH %bb.2
+  ; GCN: bb.6 (align 64):
+  ; GCN:   successors: %bb.7(0x04000000), %bb.1(0x7c000000)
+  ; GCN:   S_CBRANCH_VCCNZ %bb.7, implicit $vcc
+  ; GCN: bb.1:
+  ; GCN:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GCN:   S_CBRANCH_VCCNZ %bb.2, implicit $vcc
+  ; GCN: bb.3:
+  ; GCN:   successors: %bb.4(0x40000000), %bb.6(0x40000000)
+  ; GCN:   SI_MASK_BRANCH %bb.6, implicit $exec
+  ; GCN:   S_BRANCH %bb.4
+  ; GCN: bb.2 (align 64):
+  ; GCN:   successors: %bb.4(0x40000000), %bb.6(0x40000000)
+  ; GCN:   SI_MASK_BRANCH %bb.6, implicit $exec
+  ; GCN:   S_BRANCH %bb.4
+  ; GCN: bb.4:
+  ; GCN:   successors: %bb.5(0x04000000), %bb.4(0x7c000000)
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_NOP 0
+  ; GCN:   S_CBRANCH_EXECZ %bb.4, implicit $exec
+  ; GCN: bb.5:
+  ; GCN:   successors: %bb.6(0x80000000)
+  ; GCN:   S_BRANCH %bb.6
+  ; GCN: bb.7:
+  ; GCN:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    S_CBRANCH_VCCZ %bb.3, implicit $vcc
+    S_BRANCH %bb.2
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+  bb.3:
+    successors: %bb.4(0x40000000), %bb.6(0x40000000)
+
+    SI_MASK_BRANCH %bb.6, implicit $exec
+    S_BRANCH %bb.4
+
+  bb.4:
+    successors: %bb.5(0x04000000), %bb.4(0x7c000000)
+
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_NOP 0
+    S_CBRANCH_EXECZ %bb.4, implicit $exec
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+  bb.6:
+    successors: %bb.7(0x04000000), %bb.1(0x7c000000)
+
+    S_CBRANCH_VCCZ %bb.1, implicit $vcc
+
+  bb.7:
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 716caf25247ca..b53d65e1bc300 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -1,5 +1,5 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=pitcairn < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
+; RUN: llc -march=amdgcn -mcpu=pitcairn < %s | FileCheck -enable-var-scope -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG -check-prefix=FUNC %s
 
 
 ; FUNC-LABEL: {{^}}v_test_imax_sge_i32:
@@ -7,8 +7,10 @@
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp sge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -27,8 +29,10 @@ define amdgpu_kernel void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrs
 ; EG: MAX_INT
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sge_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %aptr, <4 x i32> addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %bptr, i32 %tid
   %a = load <4 x i32>, <4 x i32> addrspace(1)* %aptr, align 4
-  %b = load <4 x i32>, <4 x i32> addrspace(1)* %bptr, align 4
+  %b = load <4 x i32>, <4 x i32> addrspace(1)* %gep.in, align 4
   %cmp = icmp sge <4 x i32> %a, %b
   %val = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
   store <4 x i32> %val, <4 x i32> addrspace(1)* %out, align 4
@@ -101,8 +105,10 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_v2i32(<2 x i32> addrspace(1)* %ou
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp sgt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -125,8 +131,10 @@ define amdgpu_kernel void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i
 
 ; EG: MAX_UINT
 define amdgpu_kernel void @v_test_umax_uge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
   %a = load i32, i32 addrspace(1)* %aptr, align 4
-  %b = load i32, i32 addrspace(1)* %bptr, align 4
+  %b = load i32, i32 addrspace(1)* %gep.in, align 4
   %cmp = icmp uge i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
   store i32 %val, i32 addrspace(1)* %out, align 4
@@ -182,7 +190,9 @@ define amdgpu_kernel void @v_test_umax_uge_i8(i8 addrspace(1)* %out, i8 addrspac
 
 ; EG: MAX_UINT
 define amdgpu_kernel void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
-  %a = load i32, i32 addrspace(1)* %aptr, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %bptr, i32 %tid
+  %a = load i32, i32 addrspace(1)* %gep.in, align 4
   %b = load i32, i32 addrspace(1)* %bptr, align 4
   %cmp = icmp ugt i32 %a, %b
   %val = select i1 %cmp, i32 %a, i32 %b
@@ -320,3 +330,9 @@ define amdgpu_kernel void @test_imax_sge_i64(i64 addrspace(1)* %out, i64 %a, i64
   store i64 %val, i64 addrspace(1)* %out, align 8
   ret void
 }
+
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
index f33c2115dcb25..673fff50b39c4 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-atomic-insert-end.mir
@@ -86,7 +86,7 @@ body:             |
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 0
     S_WAITCNT 127
-    $vgpr1_vgpr2 = BUFFER_LOAD_DWORDX2_ADDR64 killed $vgpr1_vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 8 from %ir.tid.gep)
+    $vgpr1_vgpr2 = BUFFER_LOAD_DWORDX2_ADDR64 killed $vgpr1_vgpr2, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile load 8 from %ir.tid.gep)
     $vgpr0 = V_XOR_B32_e32 1, killed $vgpr0, implicit $exec
     V_CMP_NE_U32_e32 0, killed $vgpr0, implicit-def $vcc, implicit $exec
     $sgpr2_sgpr3 = S_AND_SAVEEXEC_B64 killed $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-atomics.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-atomics.mir
index 1046b9729df46..99348a57b9f6a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-atomics.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-atomics.mir
@@ -23,13 +23,13 @@ body:             |
     $sgpr10 = S_MOV_B32 4294967295, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $sgpr11 = S_MOV_B32 15204352, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into `i32 addrspace(5)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into `i32 addrspace(5)* undef`)
     S_WAITCNT 127
     S_CMP_LG_U32 killed $sgpr2, 0, implicit-def $scc
     S_WAITCNT 3855
     $vgpr0 = V_MOV_B32_e32 2, implicit $exec
     $vgpr1 = V_MOV_B32_e32 32772, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into `i32 addrspace(5)* undef`)
+    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into `i32 addrspace(5)* undef`)
     S_CBRANCH_SCC0 %bb.1, implicit killed $scc
 
   bb.2:
@@ -55,7 +55,7 @@ body:             |
     S_WAITCNT 127
     $sgpr0 = S_LSHL_B32 killed $sgpr0, 2, implicit-def dead $scc
     $vgpr0 = V_ADD_I32_e32 killed $sgpr0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (load syncscope("agent-one-as") unordered 4 from `i32 addrspace(1)* undef`), (load syncscope("workgroup-one-as") seq_cst 4 from `[8192 x i32] addrspace(5)* undef`)
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (load syncscope("agent-one-as") unordered 4 from `i32 addrspace(1)* undef`), (load syncscope("workgroup-one-as") seq_cst 4 from `[8192 x i32] addrspace(5)* undef`)
     $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit-def $vgpr1_vgpr2, implicit $sgpr4_sgpr5
     $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit $sgpr4_sgpr5, implicit $exec
     S_WAITCNT 3952
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
index bf24ce15acb62..f52275af48c9e 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
@@ -117,13 +117,13 @@ body:             |
     $sgpr10 = S_MOV_B32 4294967295, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $sgpr11 = S_MOV_B32 15204352, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr01)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr01)
     S_WAITCNT 127
     S_CMP_LG_U32 killed $sgpr2, 0, implicit-def $scc
     S_WAITCNT 3855
     $vgpr0 = V_MOV_B32_e32 2, implicit $exec
     $vgpr1 = V_MOV_B32_e32 32772, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr12)
+    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr12)
     S_CBRANCH_SCC0 %bb.1.if, implicit killed $scc
 
   bb.2.else:
@@ -149,7 +149,7 @@ body:             |
     S_WAITCNT 127
     $sgpr0 = S_LSHL_B32 killed $sgpr0, 2, implicit-def dead $scc
     $vgpr0 = V_ADD_I32_e32 killed $sgpr0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (non-temporal load 4 from %ir.else_ptr), (non-temporal load 4 from %ir.if_ptr)
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (non-temporal load 4 from %ir.else_ptr), (non-temporal load 4 from %ir.if_ptr)
     $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit-def $vgpr1_vgpr2, implicit $sgpr4_sgpr5
     $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit $sgpr4_sgpr5, implicit $exec
     S_WAITCNT 3952
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
index a6088b0677a06..c543b80454b62 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-2.mir
@@ -97,13 +97,13 @@ body:             |
     $sgpr10 = S_MOV_B32 4294967295, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $sgpr11 = S_MOV_B32 15204352, implicit-def $sgpr8_sgpr9_sgpr10_sgpr11
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr01)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr01)
     S_WAITCNT 127
     S_CMP_LG_U32 killed $sgpr2, 0, implicit-def $scc
     S_WAITCNT 3855
     $vgpr0 = V_MOV_B32_e32 2, implicit $exec
     $vgpr1 = V_MOV_B32_e32 32772, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr12)
+    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, killed $vgpr1, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.scratchptr12)
     S_CBRANCH_SCC0 %bb.1.if, implicit killed $scc
 
   bb.2.else:
@@ -129,7 +129,7 @@ body:             |
     S_WAITCNT 127
     $sgpr0 = S_LSHL_B32 killed $sgpr0, 2, implicit-def dead $scc
     $vgpr0 = V_ADD_I32_e32 killed $sgpr0, killed $vgpr0, implicit-def dead $vcc, implicit $exec
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %ir.else_ptr), (non-temporal load 4 from %ir.if_ptr)
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFEN killed $vgpr0, killed $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %ir.else_ptr), (non-temporal load 4 from %ir.if_ptr)
     $vgpr1 = V_MOV_B32_e32 $sgpr4, implicit $exec, implicit-def $vgpr1_vgpr2, implicit $sgpr4_sgpr5
     $vgpr2 = V_MOV_B32_e32 killed $sgpr5, implicit $exec, implicit $sgpr4_sgpr5, implicit $exec
     S_WAITCNT 3952
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.mir b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
index ac412a8fc29b6..b46cfb16b7ba5 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.mir
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.mir
@@ -337,7 +337,7 @@ body:             |
 # GCN:      dead early-clobber %4:vreg_128, dead early-clobber %3:vreg_128, dead early-clobber %5:vgpr_32 = BUNDLE %0, %2, %1, implicit $exec {
 # GCN-NEXT:   dead %3:vreg_128 = IMAGE_SAMPLE_LZ_V4_V2 %0, %1, %2, 15, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # GCN-NEXT:   dead %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, 0, 0, implicit $exec
-# GCN-NEXT:   dead %5:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %0, %2, 0, 0, 0, 0, 0, 0, implicit $exec
+# GCN-NEXT:   dead %5:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %0, %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # GCN-NEXT: }
 
 ---
@@ -357,7 +357,7 @@ body:             |
     %2 = IMPLICIT_DEF
     %3:vreg_128 = IMAGE_SAMPLE_LZ_V4_V2 %0, %1, %2, 15, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %4:vreg_128 = GLOBAL_LOAD_DWORDX4 %0, 0, 0, 0, 0, implicit $exec
-    %5:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %0, %2, 0, 0, 0, 0, 0, 0, implicit $exec
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_ADDR64 %0, %2, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 
 # GCN-LABEL: {{^}}name: atomic{{$}}
diff --git a/llvm/test/CodeGen/AMDGPU/merge-load-store.mir b/llvm/test/CodeGen/AMDGPU/merge-load-store.mir
index becd2e1b9c1ec..6bff48467b594 100644
--- a/llvm/test/CodeGen/AMDGPU/merge-load-store.mir
+++ b/llvm/test/CodeGen/AMDGPU/merge-load-store.mir
@@ -169,10 +169,10 @@ body:             |
 ---
 # CHECK-LABEL: merge_mmos
 # CHECK: S_BUFFER_LOAD_DWORDX2_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 8, align 4)
-# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 4)
-# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8, align 4)
-# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 64, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8 from %ir.ptr_addr1 + 64, align 4
-# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 64, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into %ir.ptr_addr1 + 64, align 4
+# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8, align 4)
+# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8, align 4)
+# CHECK: BUFFER_LOAD_DWORDX2_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 8 from %ir.ptr_addr1 + 64, align 4
+# CHECK: BUFFER_STORE_DWORDX2_OFFSET_exact killed %{{[0-9]+}}, %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 8 into %ir.ptr_addr1 + 64, align 4
 name: merge_mmos
 tracksRegLiveness: true
 body: |
@@ -182,14 +182,14 @@ body: |
     %0:sreg_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3
     %1:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 0, 0, 0 :: (dereferenceable invariant load 4)
     %2:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM %0, 1, 0, 0 :: (dereferenceable invariant load 4)
-    %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
-    %4:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 4, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
-    BUFFER_STORE_DWORD_OFFSET_exact %3, %0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
-    BUFFER_STORE_DWORD_OFFSET_exact %4, %0, 0, 4, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
-    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 64, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 64)
-    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 68, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 68)
-    BUFFER_STORE_DWORD_OFFSET_exact %5, %0, 0, 64, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 64)
-    BUFFER_STORE_DWORD_OFFSET_exact %6, %0, 0, 68, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 68)
+    %3:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
+    %4:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %3, %0, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
+    BUFFER_STORE_DWORD_OFFSET_exact %4, %0, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4)
+    %5:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 64)
+    %6:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %0, 0, 68, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable load 4 from %ir.ptr_addr1 + 68)
+    BUFFER_STORE_DWORD_OFFSET_exact %5, %0, 0, 64, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 64)
+    BUFFER_STORE_DWORD_OFFSET_exact %6, %0, 0, 68, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable store 4 into %ir.ptr_addr1 + 68)
 
     S_ENDPGM 0
 
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
index 9cfd92f86c4fb..ccff6bb51275e 100644
--- a/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-legalize-operands.mir
@@ -25,7 +25,7 @@
 # W64: [[CMP1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W64: [[CMP:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[CMP0]], [[CMP1]], implicit-def $scc
 # W64: [[TMPEXEC:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # W64: $exec = S_XOR_B64_term $exec, [[TMPEXEC]], implicit-def $scc
 # W64: S_CBRANCH_EXECNZ %bb.1, implicit $exec
 # W64-LABEL  bb.2:
@@ -47,7 +47,7 @@
 # W32: [[CMP1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W32: [[CMP:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[CMP0]], [[CMP1]], implicit-def $scc
 # W32: [[TMPEXEC:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # TODO: S_XOR_B32_term should be `implicit-def $scc`
 # W32: $exec_lo = S_XOR_B32_term $exec_lo, [[TMPEXEC]]
 # W32: S_CBRANCH_EXECNZ %bb.1, implicit $exec
@@ -72,7 +72,7 @@ body:             |
     %1:vgpr_32 = COPY $vgpr1
     %0:vgpr_32 = COPY $vgpr0
     %6:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
-    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN %4, killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     $vgpr0 = COPY %7
     S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
@@ -94,7 +94,7 @@ body:             |
 # W64: [[CMP1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W64: [[CMP:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[CMP0]], [[CMP1]], implicit-def $scc
 # W64: [[TMPEXEC:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # W64: $exec = S_XOR_B64_term $exec, [[TMPEXEC]], implicit-def $scc
 # W64: S_CBRANCH_EXECNZ %bb.1, implicit $exec
 # W64-LABEL  bb.2:
@@ -116,7 +116,7 @@ body:             |
 # W32: [[CMP1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W32: [[CMP:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[CMP0]], [[CMP1]], implicit-def $scc
 # W32: [[TMPEXEC:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # TODO: S_XOR_B32_term should be `implicit-def $scc`
 # W32: $exec_lo = S_XOR_B32_term $exec_lo, [[TMPEXEC]]
 # W32: S_CBRANCH_EXECNZ %bb.1, implicit $exec
@@ -141,7 +141,7 @@ body:             |
     %1:vgpr_32 = COPY $vgpr1
     %0:vgpr_32 = COPY $vgpr0
     %6:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
-    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFEN %4, killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     $vgpr0 = COPY %7
     S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
@@ -163,7 +163,7 @@ body:             |
 # W64: [[CMP1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W64: [[CMP:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[CMP0]], [[CMP1]], implicit-def $scc
 # W64: [[TMPEXEC:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # W64: $exec = S_XOR_B64_term $exec, [[TMPEXEC]], implicit-def $scc
 # W64: S_CBRANCH_EXECNZ %bb.1, implicit $exec
 # W64-LABEL  bb.2:
@@ -185,7 +185,7 @@ body:             |
 # W32: [[CMP1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W32: [[CMP:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[CMP0]], [[CMP1]], implicit-def $scc
 # W32: [[TMPEXEC:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # TODO: S_XOR_B32_term should be `implicit-def $scc`
 # W32: $exec_lo = S_XOR_B32_term $exec_lo, [[TMPEXEC]]
 # W32: S_CBRANCH_EXECNZ %bb.1, implicit $exec
@@ -210,7 +210,7 @@ body:             |
     %1:vgpr_32 = COPY $vgpr1
     %0:vgpr_32 = COPY $vgpr0
     %6:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
-    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_BOTHEN %4, killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     $vgpr0 = COPY %7
     S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
@@ -226,7 +226,7 @@ body:             |
 # ADDR64: %9:vgpr_32, %12:sreg_64_xexec = V_ADD_I32_e64 %14.sub0, %4.sub0, 0, implicit $exec
 # ADDR64: %10:vgpr_32, dead %13:sreg_64_xexec = V_ADDC_U32_e64 %14.sub1, %4.sub1, killed %12, 0, implicit $exec
 # ADDR64: %11:vreg_64 = REG_SEQUENCE %9, %subreg.sub0, %10, %subreg.sub1
-# ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 %11, killed %18, 0, 0, 0, 0, 0, 0, implicit $exec
+# ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 %11, killed %18, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 ---
 name:            addr64
 liveins:
@@ -246,7 +246,7 @@ body:             |
     %1:vgpr_32 = COPY $vgpr1
     %0:vgpr_32 = COPY $vgpr0
     %6:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
-    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 %4, killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 %4, killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     $vgpr0 = COPY %7
     S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
@@ -269,7 +269,7 @@ body:             |
 # W64-NO-ADDR64: [[CMP1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W64-NO-ADDR64: [[CMP:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[CMP0]], [[CMP1]], implicit-def $scc
 # W64-NO-ADDR64: [[TMPEXEC:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W64-NO-ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W64-NO-ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # W64-NO-ADDR64: $exec = S_XOR_B64_term $exec, [[TMPEXEC]], implicit-def $scc
 # W64-NO-ADDR64: S_CBRANCH_EXECNZ %bb.1, implicit $exec
 # W64-NO-ADDR64-LABEL  bb.2:
@@ -289,7 +289,7 @@ body:             |
 # W32: [[CMP1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[SRSRC]].sub2_sub3, [[VRSRC]].sub2_sub3, implicit $exec
 # W32: [[CMP:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[CMP0]], [[CMP1]], implicit-def $scc
 # W32: [[TMPEXEC:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[CMP]], implicit-def $exec, implicit-def $scc, implicit $exec
-# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[SRSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# W32: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed [[SRSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # TODO: S_XOR_B32_term should be `implicit-def $scc`
 # W32: $exec_lo = S_XOR_B32_term $exec_lo, [[TMPEXEC]]
 # W32: S_CBRANCH_EXECNZ %bb.1, implicit $exec
@@ -303,7 +303,7 @@ body:             |
 # ADDR64: [[RSRCFMTHI:%[0-9]+]]:sgpr_32 = S_MOV_B32 61440
 # ADDR64: [[ZERORSRC:%[0-9]+]]:sreg_128 = REG_SEQUENCE [[ZERO64]], %subreg.sub0_sub1, [[RSRCFMTLO]], %subreg.sub2, [[RSRCFMTHI]], %subreg.sub3
 # ADDR64: [[VADDR64:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[RSRCPTR]].sub0, %subreg.sub0, [[RSRCPTR]].sub1, %subreg.sub1
-# ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[VADDR64]], [[ZERORSRC]], 0, 0, 0, 0, 0, 0, implicit $exec
+# ADDR64: {{[0-9]+}}:vgpr_32 = BUFFER_LOAD_FORMAT_X_ADDR64 [[VADDR64]], [[ZERORSRC]], 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
 ---
 name:            offset
@@ -324,7 +324,7 @@ body:             |
     %1:vgpr_32 = COPY $vgpr1
     %0:vgpr_32 = COPY $vgpr0
     %6:sreg_128 = REG_SEQUENCE %0, %subreg.sub0, %1, %subreg.sub1, %2, %subreg.sub2, %3, %subreg.sub3
-    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed %6, 0, 0, 0, 0, 0, 0, implicit $exec
+    %7:vgpr_32 = BUFFER_LOAD_FORMAT_X_OFFSET killed %6, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     $vgpr0 = COPY %7
     S_SETPC_B64_return $sgpr30_sgpr31, implicit $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/nsa-vmem-hazard.mir b/llvm/test/CodeGen/AMDGPU/nsa-vmem-hazard.mir
index 0dd135723d8cd..39d3efe2a1de3 100644
--- a/llvm/test/CodeGen/AMDGPU/nsa-vmem-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/nsa-vmem-hazard.mir
@@ -9,7 +9,7 @@ name:            hazard_image_sample_d_buf_off6
 body:            |
   bb.0:
     $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_SAMPLE_D_V4_V9_nsa_gfx10 undef $vgpr3, undef $vgpr8, undef $vgpr7, undef $vgpr5, undef $vgpr4, undef $vgpr6, undef $vgpr0, undef $vgpr2, undef $vgpr2, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, undef $sgpr8_sgpr9_sgpr10_sgpr11, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, 0, implicit $exec
 ...
 
 # GCN-LABEL: name: no_hazard_image_sample_d_buf_off1
@@ -20,7 +20,7 @@ name:            no_hazard_image_sample_d_buf_off1
 body:            |
   bb.0:
     $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_SAMPLE_D_V4_V9_nsa_gfx10 undef $vgpr3, undef $vgpr8, undef $vgpr7, undef $vgpr5, undef $vgpr4, undef $vgpr6, undef $vgpr0, undef $vgpr2, undef $vgpr2, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, undef $sgpr8_sgpr9_sgpr10_sgpr11, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 1, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 1, 0, 0, 0, 0, 0, implicit $exec
 ...
 
 # GCN-LABEL: name: no_hazard_image_sample_d_buf_far
@@ -33,7 +33,7 @@ body:            |
   bb.0:
     $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_SAMPLE_D_V4_V9_nsa_gfx10 undef $vgpr3, undef $vgpr8, undef $vgpr7, undef $vgpr5, undef $vgpr4, undef $vgpr6, undef $vgpr0, undef $vgpr2, undef $vgpr2, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, undef $sgpr8_sgpr9_sgpr10_sgpr11, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     V_NOP_e32 implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, 0, implicit $exec
 ...
 
 # Non-NSA
@@ -45,7 +45,7 @@ name:            no_hazard_image_sample_v4_v2_buf_off6
 body:            |
   bb.0:
     $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_SAMPLE_V4_V2_gfx10 undef $vgpr1_vgpr2, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, undef $sgpr8_sgpr9_sgpr10_sgpr11, 15, 1, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, 0, implicit $exec
 ...
 
 # Less than 4 dwords
@@ -57,5 +57,5 @@ name:            no_hazard_image_sample_v4_v3_buf_off6
 body:            |
   bb.0:
     $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_SAMPLE_V4_V3_nsa_gfx10 undef $vgpr1, undef $vgpr2, undef $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, undef $sgpr8_sgpr9_sgpr10_sgpr11, 15, 2, 0, 0, 0, 0, 0, 0, 0, 0, implicit $exec
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFSET undef $sgpr0_sgpr1_sgpr2_sgpr3, undef $sgpr4, 6, 0, 0, 0, 0, 0, implicit $exec
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir b/llvm/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
index 3af2f0457fbba..0e7708210a90e 100644
--- a/llvm/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
+++ b/llvm/test/CodeGen/AMDGPU/opt-sgpr-to-vgpr-copy.mir
@@ -137,7 +137,7 @@ body:             |
     %28 = REG_SEQUENCE %6, 17, killed %27, 18
     %29 = V_MOV_B32_e32 0, implicit $exec
     %30 = COPY %24
-    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.bb2:
     SI_END_CF %1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
@@ -243,7 +243,7 @@ body:             |
     %37 = REG_SEQUENCE %6, 17, killed %36, 18
     %38 = V_MOV_B32_e32 0, implicit $exec
     %39 = COPY %33
-    BUFFER_STORE_DWORD_ADDR64 killed %38, killed %39, killed %37, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %38, killed %39, killed %37, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.bb2:
     SI_END_CF %1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
@@ -332,7 +332,7 @@ body:             |
     %28 = REG_SEQUENCE %6, 17, killed %27, 18
     %29 = V_MOV_B32_e32 0, implicit $exec
     %30 = COPY %24
-    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 killed %29, killed %30, killed %28, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.bb2:
     SI_END_CF %1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir b/llvm/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
index 7da7dc8d31992..53ca546969b9c 100644
--- a/llvm/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
+++ b/llvm/test/CodeGen/AMDGPU/optimize-if-exec-masking.mir
@@ -151,7 +151,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -159,7 +159,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -188,7 +188,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -196,7 +196,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -225,7 +225,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -233,14 +233,14 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
 ---
 # CHECK-LABEL: name: optimize_if_and_saveexec_xor_valu_middle
 # CHECK: $sgpr2_sgpr3 = S_AND_B64 $sgpr0_sgpr1, killed $vcc, implicit-def $scc
-# CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+# CHECK-NEXT: BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 # CHECK-NEXT: $sgpr0_sgpr1 = S_XOR_B64 $sgpr2_sgpr3, killed $sgpr0_sgpr1, implicit-def $scc
 # CHECK-NEXT: $exec = COPY killed $sgpr2_sgpr3
 # CHECK-NEXT: SI_MASK_BRANCH
@@ -255,7 +255,7 @@ body:             |
     $vcc = V_CMP_EQ_I32_e64 0, killed $vgpr0, implicit $exec
     $vgpr0 = V_MOV_B32_e32 4, implicit $exec
     $sgpr2_sgpr3 = S_AND_B64 $sgpr0_sgpr1, killed $vcc, implicit-def $scc
-    BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET $vgpr0, undef $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr0_sgpr1 = S_XOR_B64 $sgpr2_sgpr3, killed $sgpr0_sgpr1, implicit-def $scc
     $exec = S_MOV_B64_term killed $sgpr2_sgpr3
     SI_MASK_BRANCH %bb.2, implicit $exec
@@ -266,7 +266,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -274,7 +274,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -304,7 +304,7 @@ body:             |
 
   bb.1.if:
     liveins: $sgpr0_sgpr1 , $sgpr4_sgpr5_sgpr6_sgpr7
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1, $sgpr4_sgpr5_sgpr6_sgpr7
@@ -312,7 +312,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -346,7 +346,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -356,7 +356,7 @@ body:             |
     $sgpr1 = S_MOV_B32 1
     $sgpr2 = S_MOV_B32 -1
     $sgpr3 = S_MOV_B32 61440
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -387,7 +387,7 @@ body:             |
     S_SLEEP 0, implicit $sgpr2_sgpr3
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -395,7 +395,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -426,7 +426,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -434,7 +434,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -463,7 +463,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -471,7 +471,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -500,7 +500,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -508,7 +508,7 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -539,7 +539,7 @@ body:             |
 
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
-    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2.end:
     liveins: $vgpr0, $sgpr0_sgpr1
@@ -547,6 +547,6 @@ body:             |
     $exec = S_OR_B64 $exec, killed $sgpr0_sgpr1, implicit-def $scc
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/pei-reg-scavenger-position.mir b/llvm/test/CodeGen/AMDGPU/pei-reg-scavenger-position.mir
index 7a1cfa32a60c3..39915f2755ce0 100644
--- a/llvm/test/CodeGen/AMDGPU/pei-reg-scavenger-position.mir
+++ b/llvm/test/CodeGen/AMDGPU/pei-reg-scavenger-position.mir
@@ -27,12 +27,12 @@ body:             |
   ; CHECK:   successors: %bb.1(0x80000000)
   ; CHECK:   liveins: $sgpr33, $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK:   $sgpr4 = S_ADD_U32 $sgpr32, 524288, implicit-def $scc
-  ; CHECK:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, align 8192, addrspace 5)
+  ; CHECK:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, align 8192, addrspace 5)
   ; CHECK:   S_BRANCH %bb.1
   ; CHECK: bb.1:
   ; CHECK:   liveins: $sgpr0_sgpr1_sgpr2_sgpr3
   ; CHECK:   $sgpr4 = S_ADD_U32 $sgpr32, 524288, implicit-def $scc
-  ; CHECK:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, align 8192, addrspace 5)
+  ; CHECK:   $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr0_sgpr1_sgpr2_sgpr3, killed $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec :: (load 4 from %stack.0, align 8192, addrspace 5)
   ; CHECK:   S_ENDPGM 0, implicit $vgpr0
   bb.0:
     $vgpr0 = SI_SPILL_V32_RESTORE %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr32, 0, implicit $exec :: (load 4 from %stack.0, addrspace 5)
diff --git a/llvm/test/CodeGen/AMDGPU/phi-elimination-assertion.mir b/llvm/test/CodeGen/AMDGPU/phi-elimination-assertion.mir
index 9417927074beb..e2e6ea76103c7 100644
--- a/llvm/test/CodeGen/AMDGPU/phi-elimination-assertion.mir
+++ b/llvm/test/CodeGen/AMDGPU/phi-elimination-assertion.mir
@@ -26,8 +26,8 @@ body:             |
 
 # CHECK-LABEL: name:            foo
 # CHECK:   bb.3:
-# CHECK-NEXT:     %3:sreg_32_xm0 = COPY killed %4
 # CHECK-NEXT:     dead %2:sreg_32_xm0 = IMPLICIT_DEF
+# CHECK-NEXT:     %3:sreg_32_xm0 = COPY killed %4
 # CHECK-NEXT:     S_NOP 0, implicit killed %3
 
 
diff --git a/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir b/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir
new file mode 100644
index 0000000000000..807029a92f348
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/phi-elimination-end-cf.mir
@@ -0,0 +1,54 @@
+# RUN: llc -mtriple amdgcn -run-pass livevars -run-pass phi-node-elimination -verify-machineinstrs -o - %s | FileCheck %s
+
+# CHECK-LABEL:  phi-cf-test
+# CHECK: bb.0:
+# CHECK:     [[COND:%[0-9]+]]:sreg_64 = V_CMP_EQ_U32_e64
+# CHECK:     [[IF_SOURCE0:%[0-9]+]]:sreg_64 = SI_IF [[COND]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+# CHECK:     [[IF_INPUT_REG:%[0-9]+]]:sreg_64 =  S_MOV_B64_term killed [[IF_SOURCE0]], implicit $exec
+
+# CHECK: bb.1:
+# CHECK:     [[END_CF_ARG:%[0-9]+]]:sreg_64 = COPY killed [[IF_INPUT_REG]]
+# CHECK:     SI_END_CF killed [[END_CF_ARG]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+
+# CHECK: bb.2:
+# CHECK:     [[IF_SOURCE1:%[0-9]+]]:sreg_64 = SI_IF [[COND]], %bb.1, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+# CHECK:     [[IF_INPUT_REG]]:sreg_64 =  S_MOV_B64_term killed [[IF_SOURCE1]], implicit $exec
+
+
+...
+---
+name:            phi-cf-test
+tracksRegLiveness: true
+body:             |
+
+  bb.0:
+    successors: %bb.3(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0
+
+    %5:vgpr_32(s32) = COPY $vgpr0
+    %0:sreg_64 = V_CMP_EQ_U32_e64 0, %5(s32), implicit $exec
+    %18:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %22:sreg_64 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.3
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+    %24:sreg_64 = PHI %20, %bb.3, %22, %bb.0
+    %23:vgpr_32 = PHI %19, %bb.3, %18, %bb.0
+    SI_END_CF %24, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    %3:vgpr_32, dead %10:sreg_64 = nsw V_ADD_I32_e64 1, %23, 0, implicit $exec
+
+  bb.3:
+    successors: %bb.3(0x40000000), %bb.2(0x40000000)
+
+    %4:vgpr_32 = PHI %19, %bb.3, %3, %bb.2, %18, %bb.0
+    %15:sreg_32_xm0 = S_MOV_B32 61440
+    %16:sreg_32_xm0 = S_MOV_B32 -1
+    %17:sreg_128 = REG_SEQUENCE undef %14:sreg_32_xm0, %subreg.sub0, undef %12:sreg_32_xm0, %subreg.sub1, %16, %subreg.sub2, %15, %subreg.sub3
+    BUFFER_STORE_DWORD_OFFSET %4, %17, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`, addrspace 1)
+    %19:vgpr_32 = COPY %4
+    %20:sreg_64 = SI_IF %0, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.3
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/power-sched-no-instr-sunit.mir b/llvm/test/CodeGen/AMDGPU/power-sched-no-instr-sunit.mir
index ed648ece0c71a..66bd4c163c669 100644
--- a/llvm/test/CodeGen/AMDGPU/power-sched-no-instr-sunit.mir
+++ b/llvm/test/CodeGen/AMDGPU/power-sched-no-instr-sunit.mir
@@ -17,6 +17,6 @@ body:             |
     S_BARRIER
     $agpr0_agpr1_agpr2_agpr3_agpr4_agpr5_agpr6_agpr7_agpr8_agpr9_agpr10_agpr11_agpr12_agpr13_agpr14_agpr15_agpr16_agpr17_agpr18_agpr19_agpr20_agpr21_agpr22_agpr23_agpr24_agpr25_agpr26_agpr27_agpr28_agpr29_agpr30_agpr31 = V_MFMA_F32_32X32X1F32 undef $vgpr0, undef $vgpr0, 0, 0, 0, 2, implicit $exec
     $vgpr0 = V_ACCVGPR_READ_B32 $agpr31, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, undef $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr6, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN killed $vgpr0, undef $vgpr0, $sgpr8_sgpr9_sgpr10_sgpr11, $sgpr6, 0, 0, 0, 0, 0, 0, implicit $exec
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/regcoal-subrange-join-seg.mir b/llvm/test/CodeGen/AMDGPU/regcoal-subrange-join-seg.mir
index 196301f4cb07b..1d9ab685c5320 100644
--- a/llvm/test/CodeGen/AMDGPU/regcoal-subrange-join-seg.mir
+++ b/llvm/test/CodeGen/AMDGPU/regcoal-subrange-join-seg.mir
@@ -185,7 +185,7 @@ body:             |
   bb.28:
     %9 = S_FF1_I32_B32 undef %10
     %13 = V_MAD_U32_U24 killed %9, 48, 32, 0, implicit $exec
-    %45 = BUFFER_LOAD_DWORD_OFFEN killed %13, undef %15, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 4)
+    %45 = BUFFER_LOAD_DWORD_OFFEN killed %13, undef %15, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (dereferenceable invariant load 4)
     %46 = V_AND_B32_e32 1, killed %45, implicit $exec
     %21 = S_BUFFER_LOAD_DWORD_SGPR undef %22, undef %23, 0, 0 :: (dereferenceable invariant load 4)
     %25 = V_CMP_GE_F32_e64 0, 0, 0, killed %21, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/regcoalesce-dbg.mir b/llvm/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
index 016c5ad320239..a92fe49e1b737 100644
--- a/llvm/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
+++ b/llvm/test/CodeGen/AMDGPU/regcoalesce-dbg.mir
@@ -70,7 +70,7 @@ body:             |
     %13.sub2_sub3 = COPY killed %12
     %20 = V_LSHL_B64 killed %19, 2, implicit $exec
     %16 = COPY killed %5
-    BUFFER_STORE_DWORD_ADDR64 killed %16, killed %20, killed %13, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
+    BUFFER_STORE_DWORD_ADDR64 killed %16, killed %20, killed %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
index 0413075dd86c9..331cccd853c24 100644
--- a/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
+++ b/llvm/test/CodeGen/AMDGPU/rename-independent-subregs-mac-operands.mir
@@ -134,10 +134,10 @@ body:             |
     %6.sub2 = COPY %6.sub0
 
   bb.2:
-    BUFFER_STORE_DWORD_OFFEN %6.sub3, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 12, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %6.sub2, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 8, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %6.sub1, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %6.sub0, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %6.sub3, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 12, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %6.sub2, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 8, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %6.sub1, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 4, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %6.sub0, %0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr30_sgpr31 = COPY %5
     S_SETPC_B64_return $sgpr30_sgpr31
 
diff --git a/llvm/test/CodeGen/AMDGPU/saddo.ll b/llvm/test/CodeGen/AMDGPU/saddo.ll
index 355b14b8ea1cc..73b9248d7e73f 100644
--- a/llvm/test/CodeGen/AMDGPU/saddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/saddo.ll
@@ -13,29 +13,25 @@ declare { <2 x i32>, <2 x i1> } @llvm.sadd.with.overflow.v2i32(<2 x i32>, <2 x i
 define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
 ; SI-LABEL: saddo_i64_zext:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s8
-; SI-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], -1
-; SI-NEXT:    s_mov_b32 s5, s9
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; SI-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[10:11], -1
-; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; SI-NEXT:    s_add_u32 s2, s10, s0
-; SI-NEXT:    s_addc_u32 s3, s11, s1
-; SI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v1, v0
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    v_add_i32_e32 v0, vcc, s2, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s6
+; SI-NEXT:    s_add_u32 s10, s6, s8
+; SI-NEXT:    s_addc_u32 s11, s7, s9
+; SI-NEXT:    v_mov_b32_e32 v1, s7
+; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[10:11], v[0:1]
+; SI-NEXT:    v_cmp_lt_i64_e64 s[6:7], s[8:9], 0
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    s_xor_b64 s[4:5], s[6:7], vcc
+; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT:    v_mov_b32_e32 v1, s11
+; SI-NEXT:    v_add_i32_e32 v0, vcc, s10, v0
 ; SI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: saddo_i64_zext:
@@ -43,22 +39,18 @@ define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b
 ; VI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s6
+; VI-NEXT:    s_add_u32 s8, s6, s0
+; VI-NEXT:    s_addc_u32 s9, s7, s1
+; VI-NEXT:    v_mov_b32_e32 v2, s7
+; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[1:2]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; VI-NEXT:    v_mov_b32_e32 v3, s9
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s8, v2
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
-; VI-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], -1
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; VI-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], -1
-; VI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[2:3]
-; VI-NEXT:    s_add_u32 s2, s6, s0
-; VI-NEXT:    s_addc_u32 s3, s7, s1
-; VI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v2
-; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v3, v2
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; VI-NEXT:    v_mov_b32_e32 v3, s3
-; VI-NEXT:    v_add_u32_e32 v2, vcc, s2, v2
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -68,22 +60,18 @@ define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b
 ; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_mov_b32_e32 v1, s6
+; GFX9-NEXT:    s_add_u32 s8, s6, s0
+; GFX9-NEXT:    s_addc_u32 s9, s7, s1
+; GFX9-NEXT:    v_mov_b32_e32 v2, s7
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[8:9], v[1:2]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[0:1], 0
+; GFX9-NEXT:    v_mov_b32_e32 v3, s9
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s8, v2
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[0:1], -1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[2:3], s[6:7], -1
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[2:3]
-; GFX9-NEXT:    s_add_u32 s2, s6, s0
-; GFX9-NEXT:    s_addc_u32 s3, s7, s1
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], v3, v2
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; GFX9-NEXT:    v_mov_b32_e32 v3, s3
-; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, s2, v2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX9-NEXT:    s_endpgm
@@ -99,32 +87,27 @@ define amdgpu_kernel void @saddo_i64_zext(i64 addrspace(1)* %out, i64 %a, i64 %b
 define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)* %carryout, i32 %a, i32 %b) nounwind {
 ; SI-LABEL: s_saddo_i32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx4 s[8:11], s[0:1], 0x9
-; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0xd
-; SI-NEXT:    s_mov_b32 s7, 0xf000
-; SI-NEXT:    s_mov_b32 s6, -1
+; SI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s4, s8
-; SI-NEXT:    v_cmp_gt_i32_e64 s[2:3], s1, -1
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; SI-NEXT:    v_cmp_gt_i32_e64 s[2:3], s0, -1
-; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; SI-NEXT:    s_add_i32 s2, s0, s1
-; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s2, -1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v1, v0
-; SI-NEXT:    s_mov_b32 s5, s9
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    s_mov_b32 s8, s10
-; SI-NEXT:    s_mov_b32 s9, s11
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_mov_b32 s10, s6
-; SI-NEXT:    s_mov_b32 s11, s7
+; SI-NEXT:    s_mov_b32 s0, s4
+; SI-NEXT:    v_cmp_lt_i32_e64 s[10:11], s9, 0
+; SI-NEXT:    s_add_i32 s9, s8, s9
+; SI-NEXT:    v_mov_b32_e32 v0, s8
+; SI-NEXT:    s_mov_b32 s1, s5
+; SI-NEXT:    v_cmp_lt_i32_e32 vcc, s9, v0
+; SI-NEXT:    v_mov_b32_e32 v0, s9
+; SI-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT:    s_xor_b64 s[0:1], s[10:11], vcc
+; SI-NEXT:    s_mov_b32 s4, s6
+; SI-NEXT:    s_mov_b32 s5, s7
+; SI-NEXT:    s_mov_b32 s6, s2
+; SI-NEXT:    s_mov_b32 s7, s3
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    buffer_store_byte v0, off, s[8:11], 0
+; SI-NEXT:    buffer_store_byte v0, off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_saddo_i32:
@@ -133,18 +116,13 @@ define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v0, s4
-; VI-NEXT:    v_cmp_gt_i32_e64 s[2:3], s1, -1
-; VI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[2:3]
-; VI-NEXT:    v_cmp_gt_i32_e64 s[2:3], s0, -1
-; VI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[2:3]
-; VI-NEXT:    s_add_i32 s2, s0, s1
-; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], s2, -1
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; VI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
+; VI-NEXT:    s_add_i32 s1, s0, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
+; VI-NEXT:    v_mov_b32_e32 v4, s1
 ; VI-NEXT:    v_mov_b32_e32 v1, s5
-; VI-NEXT:    v_mov_b32_e32 v4, s2
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
 ; VI-NEXT:    flat_store_dword v[0:1], v4
 ; VI-NEXT:    v_mov_b32_e32 v2, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s7
@@ -158,18 +136,13 @@ define amdgpu_kernel void @s_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x34
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GFX9-NEXT:    v_cmp_gt_i32_e64 s[2:3], s1, -1
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[2:3]
-; GFX9-NEXT:    v_cmp_gt_i32_e64 s[2:3], s0, -1
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[2:3]
-; GFX9-NEXT:    s_add_i32 s2, s0, s1
-; GFX9-NEXT:    v_cmp_gt_i32_e64 s[0:1], s2, -1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], s1, 0
+; GFX9-NEXT:    s_add_i32 s1, s0, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, s1, v4
+; GFX9-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GFX9-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
 ; GFX9-NEXT:    global_store_dword v[0:1], v4, off
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s7
@@ -204,19 +177,12 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; SI-NEXT:    s_mov_b32 s6, s14
 ; SI-NEXT:    s_mov_b32 s7, s15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v1
-; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, vcc
-; SI-NEXT:    v_add_i32_e32 v0, vcc, v1, v0
-; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v3, v2
-; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v3, v1
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    buffer_store_dword v0, off, s[4:7], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v2, vcc, v1, v0
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v1
+; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v2, v0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT:    buffer_store_dword v2, off, s[4:7], 0
 ; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
 ; SI-NEXT:    s_endpgm
 ;
@@ -235,17 +201,11 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; VI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v6
-; VI-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v4, v6
-; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v4
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v7, v5
-; VI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v7, v5
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT:    flat_store_dword v[2:3], v4
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v4, v6
+; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    flat_store_dword v[2:3], v5
 ; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; VI-NEXT:    flat_store_byte v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -265,17 +225,11 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; GFX9-NEXT:    v_add_u32_e32 v4, v6, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v4
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v7, v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], v7, v5
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT:    global_store_dword v[2:3], v4, off
+; GFX9-NEXT:    v_add_u32_e32 v5, v6, v4
+; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], v5, v6
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT:    global_store_dword v[2:3], v5, off
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -292,31 +246,27 @@ define amdgpu_kernel void @v_saddo_i32(i32 addrspace(1)* %out, i1 addrspace(1)*
 define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)* %carryout, i64 %a, i64 %b) nounwind {
 ; SI-LABEL: s_saddo_i64:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s15, 0xf000
-; SI-NEXT:    s_mov_b32 s14, -1
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    s_mov_b32 s10, -1
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[10:11], -1
-; SI-NEXT:    s_add_u32 s2, s8, s10
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[8:9], -1
-; SI-NEXT:    s_addc_u32 s3, s9, s11
-; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v1, v0
-; SI-NEXT:    v_mov_b32_e32 v0, s2
-; SI-NEXT:    s_mov_b32 s12, s6
-; SI-NEXT:    s_mov_b32 s13, s7
-; SI-NEXT:    s_mov_b32 s6, s14
-; SI-NEXT:    s_mov_b32 s7, s15
-; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT:    s_add_u32 s12, s4, s6
+; SI-NEXT:    v_mov_b32_e32 v0, s4
+; SI-NEXT:    s_addc_u32 s13, s5, s7
+; SI-NEXT:    v_mov_b32_e32 v1, s5
+; SI-NEXT:    v_cmp_lt_i64_e32 vcc, s[12:13], v[0:1]
+; SI-NEXT:    v_mov_b32_e32 v0, s12
+; SI-NEXT:    v_cmp_lt_i64_e64 s[4:5], s[6:7], 0
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    s_mov_b32 s2, s10
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    v_mov_b32_e32 v1, s13
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT:    s_xor_b64 s[0:1], s[4:5], vcc
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
+; SI-NEXT:    buffer_store_byte v0, off, s[8:11], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: s_saddo_i64:
@@ -324,22 +274,18 @@ define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
+; VI-NEXT:    v_mov_b32_e32 v4, s4
+; VI-NEXT:    s_add_u32 s0, s4, s6
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
-; VI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[6:7], -1
+; VI-NEXT:    s_addc_u32 s1, s5, s7
+; VI-NEXT:    v_mov_b32_e32 v5, s5
+; VI-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
-; VI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[4:5], -1
-; VI-NEXT:    s_add_u32 s2, s4, s6
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
-; VI-NEXT:    s_addc_u32 s3, s5, s7
-; VI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; VI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v4
-; VI-NEXT:    v_mov_b32_e32 v5, s3
-; VI-NEXT:    v_mov_b32_e32 v4, s2
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; VI-NEXT:    v_mov_b32_e32 v5, s1
+; VI-NEXT:    v_mov_b32_e32 v4, s0
+; VI-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[4:5]
 ; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; VI-NEXT:    flat_store_byte v[0:1], v2
@@ -350,22 +296,18 @@ define amdgpu_kernel void @s_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
+; GFX9-NEXT:    v_mov_b32_e32 v4, s4
+; GFX9-NEXT:    s_add_u32 s0, s4, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[6:7], -1
+; GFX9-NEXT:    s_addc_u32 s1, s5, s7
+; GFX9-NEXT:    v_mov_b32_e32 v5, s5
+; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, s[0:1], v[4:5]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[4:5], -1
-; GFX9-NEXT:    s_add_u32 s2, s4, s6
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-NEXT:    s_addc_u32 s3, s5, s7
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_gt_i64_e64 s[0:1], s[2:3], -1
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v4
-; GFX9-NEXT:    v_mov_b32_e32 v5, s3
-; GFX9-NEXT:    v_mov_b32_e32 v4, s2
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[2:3], s[6:7], 0
+; GFX9-NEXT:    v_mov_b32_e32 v5, s1
+; GFX9-NEXT:    v_mov_b32_e32 v4, s0
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[2:3], vcc
 ; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[4:5], off
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
@@ -398,19 +340,12 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; SI-NEXT:    s_mov_b32 s6, s14
 ; SI-NEXT:    s_mov_b32 s7, s15
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[2:3]
-; SI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; SI-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[0:1]
-; SI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; SI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; SI-NEXT:    v_cmp_lt_i64_e64 s[0:1], -1, v[0:1]
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v5, v2
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; SI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v3, vcc
+; SI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[2:3]
+; SI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[4:5], v[0:1]
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[4:7], 0
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
 ; SI-NEXT:    buffer_store_byte v0, off, s[12:15], 0
 ; SI-NEXT:    s_endpgm
@@ -430,18 +365,12 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; VI-NEXT:    v_mov_b32_e32 v0, s2
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
-; VI-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; VI-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[6:7]
-; VI-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v6, v4
-; VI-NEXT:    v_addc_u32_e32 v5, vcc, v7, v5, vcc
-; VI-NEXT:    v_cmp_lt_i64_e64 s[0:1], -1, v[4:5]
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v8
-; VI-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[0:1], v9, v6
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
+; VI-NEXT:    v_addc_u32_e32 v9, vcc, v7, v5, vcc
+; VI-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; VI-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
 ; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; VI-NEXT:    flat_store_byte v[0:1], v2
 ; VI-NEXT:    s_endpgm
@@ -461,18 +390,12 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i64_e32 vcc, -1, v[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX9-NEXT:    v_add_co_u32_e32 v4, vcc, v6, v4
-; GFX9-NEXT:    v_addc_co_u32_e32 v5, vcc, v7, v5, vcc
-; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], -1, v[4:5]
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v8
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[0:1], v9, v6
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[0:1]
-; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[4:5], off
+; GFX9-NEXT:    v_add_co_u32_e32 v8, vcc, v6, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v9, vcc, v7, v5, vcc
+; GFX9-NEXT:    v_cmp_gt_i64_e32 vcc, 0, v[4:5]
+; GFX9-NEXT:    v_cmp_lt_i64_e64 s[0:1], v[8:9], v[6:7]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[0:1]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; GFX9-NEXT:    global_store_byte v[0:1], v2, off
 ; GFX9-NEXT:    s_endpgm
@@ -489,48 +412,35 @@ define amdgpu_kernel void @v_saddo_i64(i64 addrspace(1)* %out, i1 addrspace(1)*
 define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> addrspace(1)* %carryout, <2 x i32> addrspace(1)* %aptr, <2 x i32> addrspace(1)* %bptr) nounwind {
 ; SI-LABEL: v_saddo_v2i32:
 ; SI:       ; %bb.0:
-; SI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x9
-; SI-NEXT:    s_mov_b32 s15, 0xf000
-; SI-NEXT:    s_mov_b32 s14, -1
-; SI-NEXT:    s_mov_b32 s2, s14
-; SI-NEXT:    s_mov_b32 s3, s15
+; SI-NEXT:    s_load_dwordx8 s[8:15], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s19, 0xf000
+; SI-NEXT:    s_mov_b32 s18, -1
+; SI-NEXT:    s_mov_b32 s2, s18
+; SI-NEXT:    s_mov_b32 s3, s19
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    s_mov_b32 s0, s10
-; SI-NEXT:    s_mov_b32 s1, s11
-; SI-NEXT:    s_mov_b32 s10, s14
-; SI-NEXT:    s_mov_b32 s11, s15
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[8:11], 0
+; SI-NEXT:    s_mov_b32 s0, s14
+; SI-NEXT:    s_mov_b32 s1, s15
+; SI-NEXT:    s_mov_b32 s14, s18
+; SI-NEXT:    s_mov_b32 s15, s19
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], off, s[12:15], 0
 ; SI-NEXT:    buffer_load_dwordx2 v[2:3], off, s[0:3], 0
-; SI-NEXT:    s_mov_b32 s12, s6
-; SI-NEXT:    s_mov_b32 s13, s7
-; SI-NEXT:    s_mov_b32 s6, s14
-; SI-NEXT:    s_mov_b32 s7, s15
+; SI-NEXT:    s_mov_b32 s16, s10
+; SI-NEXT:    s_mov_b32 s17, s11
+; SI-NEXT:    s_mov_b32 s10, s18
+; SI-NEXT:    s_mov_b32 s11, s19
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v2
-; SI-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v0
-; SI-NEXT:    v_cndmask_b32_e64 v5, 0, 1, vcc
-; SI-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v3
-; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, vcc
-; SI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v1
-; SI-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; SI-NEXT:    v_add_i32_e32 v1, vcc, v1, v3
-; SI-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v0
-; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], -1, v1
-; SI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
-; SI-NEXT:    v_cmp_eq_u32_e64 s[0:1], v6, v2
-; SI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[2:3]
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
-; SI-NEXT:    v_cmp_ne_u32_e64 s[4:5], v6, v2
-; SI-NEXT:    s_and_b64 s[0:1], s[0:1], s[4:5]
-; SI-NEXT:    v_cmp_eq_u32_e32 vcc, v5, v4
-; SI-NEXT:    v_cmp_ne_u32_e64 s[2:3], v5, v3
-; SI-NEXT:    s_waitcnt expcnt(0)
+; SI-NEXT:    v_add_i32_e32 v5, vcc, v1, v3
+; SI-NEXT:    v_add_i32_e32 v4, vcc, v0, v2
+; SI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v3
+; SI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v5, v1
+; SI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v2
+; SI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v4, v0
 ; SI-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; SI-NEXT:    s_and_b64 s[0:1], vcc, s[2:3]
+; SI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
 ; SI-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[12:15], 0
+; SI-NEXT:    buffer_store_dwordx2 v[4:5], off, s[8:11], 0
+; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[16:19], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; VI-LABEL: v_saddo_v2i32:
@@ -543,33 +453,21 @@ define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32>
 ; VI-NEXT:    v_mov_b32_e32 v7, s5
 ; VI-NEXT:    flat_load_dwordx2 v[6:7], v[6:7]
 ; VI-NEXT:    flat_load_dwordx2 v[4:5], v[4:5]
-; VI-NEXT:    v_mov_b32_e32 v0, s2
-; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_mov_b32_e32 v2, s0
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
+; VI-NEXT:    v_mov_b32_e32 v0, s2
+; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; VI-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v6
-; VI-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; VI-NEXT:    v_add_u32_e32 v4, vcc, v6, v4
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v5
-; VI-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; VI-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v7
-; VI-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v7, v5
-; VI-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v4
-; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], -1, v5
-; VI-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
-; VI-NEXT:    v_cmp_eq_u32_e64 s[0:1], v10, v6
-; VI-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[2:3]
-; VI-NEXT:    v_cmp_ne_u32_e64 s[4:5], v10, v6
-; VI-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v8
-; VI-NEXT:    v_cmp_ne_u32_e64 s[2:3], v9, v7
-; VI-NEXT:    s_and_b64 s[0:1], s[0:1], s[4:5]
-; VI-NEXT:    flat_store_dwordx2 v[2:3], v[4:5]
+; VI-NEXT:    v_add_u32_e32 v9, vcc, v7, v5
+; VI-NEXT:    v_add_u32_e32 v8, vcc, v6, v4
+; VI-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
+; VI-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
+; VI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; VI-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
+; VI-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; VI-NEXT:    flat_store_dwordx2 v[2:3], v[8:9]
 ; VI-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
-; VI-NEXT:    s_and_b64 s[0:1], vcc, s[2:3]
+; VI-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
 ; VI-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; VI-NEXT:    s_endpgm
@@ -584,33 +482,21 @@ define amdgpu_kernel void @v_saddo_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32>
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s5
 ; GFX9-NEXT:    global_load_dwordx2 v[6:7], v[6:7], off
 ; GFX9-NEXT:    global_load_dwordx2 v[4:5], v[4:5], off
-; GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s1
+; GFX9-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v4
-; GFX9-NEXT:    v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v5
-; GFX9-NEXT:    v_add_u32_e32 v4, v6, v4
-; GFX9-NEXT:    v_add_u32_e32 v5, v7, v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e32 vcc, -1, v7
-; GFX9-NEXT:    v_cndmask_b32_e64 v10, 0, 1, vcc
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[0:1], -1, v4
-; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], -1, v5
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, 0, 1, s[0:1]
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], v10, v6
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s[2:3]
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[4:5], v10, v6
-; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, v9, v8
-; GFX9-NEXT:    v_cmp_ne_u32_e64 s[2:3], v9, v7
-; GFX9-NEXT:    s_and_b64 s[0:1], s[0:1], s[4:5]
-; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[4:5], off
+; GFX9-NEXT:    v_add_u32_e32 v9, v7, v5
+; GFX9-NEXT:    v_add_u32_e32 v8, v6, v4
+; GFX9-NEXT:    v_cmp_gt_i32_e64 s[0:1], 0, v5
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[4:5], v9, v7
+; GFX9-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v4
+; GFX9-NEXT:    v_cmp_lt_i32_e64 s[2:3], v8, v6
+; GFX9-NEXT:    s_xor_b64 s[0:1], s[0:1], s[4:5]
+; GFX9-NEXT:    global_store_dwordx2 v[2:3], v[8:9], off
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s[0:1]
-; GFX9-NEXT:    s_and_b64 s[0:1], vcc, s[2:3]
+; GFX9-NEXT:    s_xor_b64 s[0:1], vcc, s[2:3]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, 0, 1, s[0:1]
 ; GFX9-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX9-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir b/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir
new file mode 100644
index 0000000000000..0d2f90793fc39
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sched-assert-dead-def-subreg-use-other-subreg.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-machineinstrs  -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+
+# This would assert that a dead def should have no uses, but the dead
+# def and use have different subreg indices.
+
+---
+name:            multi_def_dead_reg_subreg_check
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+  scratchRSrcReg:  '$sgpr24_sgpr25_sgpr26_sgpr27'
+  scratchWaveOffsetReg: '$sgpr32'
+  frameOffsetReg:  '$sgpr32'
+  stackPtrOffsetReg: '$sgpr32'
+  argumentInfo:
+    privateSegmentBuffer: { reg: '$sgpr0_sgpr1_sgpr2_sgpr3' }
+    privateSegmentWaveByteOffset: { reg: '$sgpr33' }
+body:             |
+  ; CHECK-LABEL: name: multi_def_dead_reg_subreg_check
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   liveins: $sgpr6_sgpr7
+  ; CHECK:   undef %0.sub3:vreg_512 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 0, [[V_MOV_B32_e32_]], implicit $exec
+  ; CHECK:   [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   [[COPY:%[0-9]+]]:vreg_512 = COPY %0
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   BUFFER_STORE_DWORD_OFFEN %0.sub3, undef %5:vgpr_32, $sgpr24_sgpr25_sgpr26_sgpr27, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, align 8, addrspace 5)
+  ; CHECK:   dead %6:vgpr_32 = DS_READ_B32_gfx9 undef %7:vgpr_32, 0, 0, implicit $exec
+  ; CHECK:   dead %8:vreg_64 = DS_READ_B64_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec
+  ; CHECK:   dead %9:vreg_128 = DS_READ_B128_gfx9 [[V_ADD_U32_e32_]], 0, 0, implicit $exec
+  ; CHECK:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[COPY]].sub0
+  ; CHECK:   undef %11.sub1:vreg_512 = COPY [[COPY]].sub1
+  ; CHECK:   INLINEASM &"", 1, 851978, def dead [[COPY1]], 851978, def dead [[COPY]].sub1, 2147483657, [[COPY1]], 2147549193, [[COPY]].sub1
+  ; CHECK:   %11.sub0:vreg_512 = COPY [[COPY]].sub0
+  ; CHECK:   %11.sub3:vreg_512 = COPY [[COPY]].sub3
+  ; CHECK:   dead %10:vgpr_32 = V_ADD_I32_e32 4, [[V_MOV_B32_e32_1]], implicit-def dead $vcc, implicit $exec
+  ; CHECK:   %11.sub2:vreg_512 = COPY undef [[V_MOV_B32_e32_]]
+  ; CHECK:   %11.sub5:vreg_512 = COPY undef [[V_MOV_B32_e32_]]
+  ; CHECK:   [[COPY2:%[0-9]+]]:vreg_512 = COPY %11
+  ; CHECK:   S_BRANCH %bb.1
+  bb.0:
+    liveins: $sgpr6_sgpr7
+
+    undef %0.sub3:vreg_512 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:vgpr_32 = V_ADD_U32_e32 0, %1, implicit $exec
+    %3:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %4:vreg_512 = COPY %0
+
+  bb.1:
+    BUFFER_STORE_DWORD_OFFEN %0.sub3, undef %5:vgpr_32, $sgpr24_sgpr25_sgpr26_sgpr27, $sgpr32, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4, align 8, addrspace 5)
+    %6:vgpr_32 = DS_READ_B32_gfx9 undef %7:vgpr_32, 0, 0, implicit $exec
+    %8:vreg_64 = DS_READ_B64_gfx9 %1, 0, 0, implicit $exec
+    %9:vreg_128 = DS_READ_B128_gfx9 %2, 0, 0, implicit $exec
+    %10:vgpr_32 = V_ADD_I32_e32 4, %3, implicit-def dead $vcc, implicit $exec
+    undef %11.sub0:vreg_512 = COPY %4.sub0
+    %12:vgpr_32 = COPY %4.sub0
+    %11.sub1:vreg_512 = COPY %4.sub1
+    INLINEASM &"", 1, 851978, def dead %12, 851978, def dead %4.sub1, 2147483657, %12, 2147549193, %4.sub1
+    %11.sub2:vreg_512 = COPY undef %1
+    %11.sub3:vreg_512 = COPY %4.sub3
+    %11.sub5:vreg_512 = COPY undef %1
+    %4:vreg_512 = COPY %11
+    S_BRANCH %bb.1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
index eee471cb073bf..b954b778dc653 100644
--- a/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
+++ b/llvm/test/CodeGen/AMDGPU/sched-crash-dbg-value.mir
@@ -279,10 +279,10 @@ body:             |
     %80:vgpr_32 = IMPLICIT_DEF
     %81:vgpr_32 = IMPLICIT_DEF
     %84:vgpr_32 = IMPLICIT_DEF
-    BUFFER_STORE_DWORD_OFFEN %84, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 108, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %81, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 104, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %80, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 100, 0, 0, 0, 0, implicit $exec
-    BUFFER_STORE_DWORD_OFFEN %78, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 96, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %84, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 108, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %81, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 104, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %80, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 100, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN %78, %stack.0.tmp5, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr101, 96, 0, 0, 0, 0, 0, implicit $exec
     %85:vgpr_32 = IMPLICIT_DEF
     %86:vgpr_32 = IMPLICIT_DEF
     %87:vgpr_32 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-barrier.mir b/llvm/test/CodeGen/AMDGPU/schedule-barrier.mir
index a72a406ff094f..e52b955ffaf71 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-barrier.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-barrier.mir
@@ -30,14 +30,14 @@ body: |
     %33.sub1:sgpr_128 = V_READFIRSTLANE_B32 %44.sub1, implicit $exec
     %33.sub2:sgpr_128 = V_READFIRSTLANE_B32 %45.sub2, implicit $exec
     %33.sub3:sgpr_128 = V_READFIRSTLANE_B32 %46.sub3, implicit $exec
-    %15:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %33, 0, 0, 0, 0, 0, 0, implicit $exec
+    %15:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %33, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %39:vgpr_32 = V_MUL_LO_U32 %15, %15, implicit $exec
 
     undef %27.sub0:sgpr_128 = V_READFIRSTLANE_B32 %26.sub0, implicit $exec
     %27.sub1:sgpr_128 = V_READFIRSTLANE_B32 %41.sub1, implicit $exec
     %27.sub2:sgpr_128 = V_READFIRSTLANE_B32 %42.sub2, implicit $exec
     %27.sub3:sgpr_128 = V_READFIRSTLANE_B32 %43.sub3, implicit $exec
-    %19:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %27, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %27, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     %40:vgpr_32 = V_MUL_LO_U32 %19, %19, implicit $exec
 
     %23:vgpr_32 = V_ADD_U32_e32 %39, %40, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir b/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
index 2aaf7f10b69a7..42c42a48a6169 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
+++ b/llvm/test/CodeGen/AMDGPU/shrink-vop3-carry-out.mir
@@ -81,11 +81,11 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %29, %9 = V_ADD_I32_e64 %19, %17, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -165,11 +165,11 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %29, %9 = V_SUB_I32_e64 %19, %17, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -249,11 +249,11 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %29, %9 = V_SUBREV_I32_e64 %19, %17, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed %9, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %29, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %29, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -332,12 +332,12 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %9 = S_MOV_B64 0
     %29, $vcc = V_ADDC_U32_e64 %19, %17, %9, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -417,12 +417,12 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     $vcc = S_MOV_B64 0
     %29, $vcc = V_ADDC_U32_e64 %19, %17, $vcc, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
@@ -502,11 +502,11 @@ body:             |
     %13 = REG_SEQUENCE killed %5, 17, %12, 18
     %28 = V_LSHL_B64 killed %27, 2, implicit $exec
     %16 = REG_SEQUENCE killed %4, 17, %12, 18
-    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, implicit $exec
-    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, implicit $exec
+    %17 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 0, 0, 0, 0, 0, 0, implicit $exec
+    %19 = BUFFER_LOAD_DWORD_ADDR64 %28, %13, 0, 4, 0, 0, 0, 0, 0, implicit $exec
     %29, $vcc = V_ADDC_U32_e64 %19, %17, undef $vcc, 0, implicit $exec
     %24 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $vcc, implicit $exec
-    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %24, %28, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.ll b/llvm/test/CodeGen/AMDGPU/sminmax.ll
index ea4b70012570c..e56db7edb7749 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.ll
@@ -1,7 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SIVI,FUNC %s
+; RUN:  llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,FUNC %s
+; RUN:  llc -march=r600 -mcpu=cypress -verify-machineinstrs < %s | FileCheck -check-prefixes=EG,FUNC %s
 
 ; FUNC-LABEL: {{^}}s_abs_i32:
 ; GCN: s_abs_i32
@@ -28,7 +28,9 @@ define amdgpu_kernel void @s_abs_i32(i32 addrspace(1)* %out, i32 %val) nounwind
 
 ; EG: MAX_INT
 define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
-  %val = load i32, i32 addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %src, i32 %tid
+  %val = load i32, i32 addrspace(1)* %gep.in, align 4
   %neg = sub i32 0, %val
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
@@ -43,7 +45,9 @@ define amdgpu_kernel void @v_abs_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %
 ; GCN: v_max_i32_e32 [[MAX:v[0-9]+]], [[SRC]], [[NEG]]
 ; GCN: v_mul_lo_u32 v{{[0-9]+}}, [[MAX]], [[MAX]]
 define amdgpu_kernel void @v_abs_i32_repeat_user(i32 addrspace(1)* %out, i32 addrspace(1)* %src) nounwind {
-  %val = load i32, i32 addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds i32, i32 addrspace(1)* %src, i32 %tid
+  %val = load i32, i32 addrspace(1)* %gep.in, align 4
   %neg = sub i32 0, %val
   %cond = icmp sgt i32 %val, %neg
   %res = select i1 %cond, i32 %val, i32 %neg
@@ -96,7 +100,9 @@ define amdgpu_kernel void @v_abs_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> a
   %z1 = insertelement <2 x i32> %z0, i32 0, i32 1
   %t0 = insertelement <2 x i32> undef, i32 2, i32 0
   %t1 = insertelement <2 x i32> %t0, i32 2, i32 1
-  %val = load <2 x i32>, <2 x i32> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <2 x i32>, <2 x i32> addrspace(1)* %src, i32 %tid
+  %val = load <2 x i32>, <2 x i32> addrspace(1)* %gep.in, align 4
   %neg = sub <2 x i32> %z1, %val
   %cond = icmp sgt <2 x i32> %val, %neg
   %res = select <2 x i1> %cond, <2 x i32> %val, <2 x i32> %neg
@@ -178,7 +184,9 @@ define amdgpu_kernel void @v_abs_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> a
   %t1 = insertelement <4 x i32> %t0, i32 2, i32 1
   %t2 = insertelement <4 x i32> %t1, i32 2, i32 2
   %t3 = insertelement <4 x i32> %t2, i32 2, i32 3
-  %val = load <4 x i32>, <4 x i32> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.r600.read.tidig.x()
+  %gep.in = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(1)* %src, i32 %tid
+  %val = load <4 x i32>, <4 x i32> addrspace(1)* %gep.in, align 4
   %neg = sub <4 x i32> %z3, %val
   %cond = icmp sgt <4 x i32> %val, %neg
   %res = select <4 x i1> %cond, <4 x i32> %val, <4 x i32> %neg
@@ -259,3 +267,8 @@ define amdgpu_kernel void @v_min_max_i32_user(i32 addrspace(1)* %out0, i32 addrs
   store volatile i1 %cond0, i1 addrspace(1)* undef
   ret void
 }
+
+declare i32 @llvm.r600.read.tidig.x() #0
+
+attributes #0 = { nounwind readnone }
+attributes #1 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 8efc1ce223973..184082dd36250 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -1,6 +1,6 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,CIVI,GCN %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CI,CIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GCN %s
+; RUN: llc -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=VI,CIVI,GCN %s
+; RUN: llc -march=amdgcn -mcpu=bonaire -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=CI,CIVI,GCN %s
 
 ; GCN-LABEL: {{^}}s_abs_v2i16:
 ; GFX9: s_load_dword [[VAL:s[0-9]+]]
@@ -85,7 +85,7 @@ define amdgpu_kernel void @s_abs_v2i16_2(<2 x i16> addrspace(1)* %out, <2 x i16>
 }
 
 ; GCN-LABEL: {{^}}v_abs_v2i16_2:
-; GFX9: buffer_load_dword [[VAL:v[0-9]+]]
+; GFX9: global_load_dword [[VAL:v[0-9]+]]
 ; GFX9: v_pk_sub_i16 [[SUB:v[0-9]+]], 0, [[VAL]]
 ; GFX9: v_pk_max_i16 [[MAX:v[0-9]+]], [[VAL]], [[SUB]]
 ; GFX9: v_pk_sub_u16 [[ADD:v[0-9]+]], [[MAX]], -2 op_sel_hi:[1,0]
@@ -94,7 +94,9 @@ define amdgpu_kernel void @v_abs_v2i16_2(<2 x i16> addrspace(1)* %out, <2 x i16>
   %z1 = insertelement <2 x i16> %z0, i16 0, i16 1
   %t0 = insertelement <2 x i16> undef, i16 2, i16 0
   %t1 = insertelement <2 x i16> %t0, i16 2, i16 1
-  %val = load <2 x i16>, <2 x i16> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <2 x i16>, <2 x i16> addrspace(1)* %src, i32 %tid
+  %val = load <2 x i16>, <2 x i16> addrspace(1)* %gep.in, align 4
   %neg = sub <2 x i16> %z1, %val
   %cond = icmp sgt <2 x i16> %val, %neg
   %res = select <2 x i1> %cond, <2 x i16> %val, <2 x i16> %neg
@@ -129,7 +131,7 @@ define amdgpu_kernel void @s_abs_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> %
 }
 
 ; GCN-LABEL: {{^}}v_abs_v4i16:
-; GFX9: buffer_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
+; GFX9: global_load_dwordx2 v{{\[}}[[VAL0:[0-9]+]]:[[VAL1:[0-9]+]]{{\]}}
 
 ; GFX9-DAG: v_pk_sub_i16 [[SUB0:v[0-9]+]], 0, v[[VAL0]]
 ; GFX9-DAG: v_pk_max_i16 [[MAX0:v[0-9]+]], v[[VAL0]], [[SUB0]]
@@ -147,7 +149,9 @@ define amdgpu_kernel void @v_abs_v4i16(<4 x i16> addrspace(1)* %out, <4 x i16> a
   %t1 = insertelement <4 x i16> %t0, i16 2, i16 1
   %t2 = insertelement <4 x i16> %t1, i16 2, i16 2
   %t3 = insertelement <4 x i16> %t2, i16 2, i16 3
-  %val = load <4 x i16>, <4 x i16> addrspace(1)* %src, align 4
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep.in = getelementptr inbounds <4 x i16>, <4 x i16> addrspace(1)* %src, i32 %tid
+  %val = load <4 x i16>, <4 x i16> addrspace(1)* %gep.in, align 4
   %neg = sub <4 x i16> %z3, %val
   %cond = icmp sgt <4 x i16> %val, %neg
   %res = select <4 x i1> %cond, <4 x i16> %val, <4 x i16> %neg
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
new file mode 100644
index 0000000000000..7a2085fa36153
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs | FileCheck -check-prefix=GCN %s
+
+; An assert was hit when frame offset register was used to address FrameIndex.
+define amdgpu_kernel void @kernel_background_evaluate(float addrspace(5)* %kg, <4 x i32> addrspace(1)* %input, <4 x float> addrspace(1)* %output, i32 %i) {
+; GCN-LABEL: kernel_background_evaluate:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_dword s6, s[0:1], 0x24
+; GCN-NEXT:    s_mov_b32 s36, SCRATCH_RSRC_DWORD0
+; GCN-NEXT:    s_mov_b32 s37, SCRATCH_RSRC_DWORD1
+; GCN-NEXT:    s_mov_b32 s38, -1
+; GCN-NEXT:    s_mov_b32 s39, 0x31c16000
+; GCN-NEXT:    s_mov_b32 s33, s3
+; GCN-NEXT:    s_mov_b64 s[0:1], s[36:37]
+; GCN-NEXT:    v_mov_b32_e32 v1, 0x2000
+; GCN-NEXT:    v_mov_b32_e32 v2, 0x4000
+; GCN-NEXT:    s_mov_b64 s[2:3], s[38:39]
+; GCN-NEXT:    v_mov_b32_e32 v3, 0
+; GCN-NEXT:    v_mov_b32_e32 v4, 0x400000
+; GCN-NEXT:    s_add_u32 s32, s33, 0xc0000
+; GCN-NEXT:    v_add_nc_u32_e64 v32, 4, 0x4000
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_getpc_b64 s[4:5]
+; GCN-NEXT:    s_add_u32 s4, s4, svm_eval_nodes@rel32@lo+4
+; GCN-NEXT:    s_addc_u32 s5, s5, svm_eval_nodes@rel32@hi+4
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s6
+; GCN-NEXT:    s_swappc_b64 s[30:31], s[4:5]
+; GCN-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GCN-NEXT:    s_and_saveexec_b32 s0, vcc_lo
+; GCN-NEXT:    ; mask branch BB0_2
+; GCN-NEXT:    s_cbranch_execz BB0_2
+; GCN-NEXT:  BB0_1: ; %if.then4.i
+; GCN-NEXT:    buffer_load_dword v0, v32, s[36:39], s32 offen
+; GCN-NEXT:    buffer_load_dword v1, v32, s[36:39], s32 offen offset:4
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    v_mul_lo_u32 v0, 0x41c64e6d, v0
+; GCN-NEXT:    v_add_nc_u32_e32 v0, 0x3039, v0
+; GCN-NEXT:    buffer_store_dword v0, v0, s[36:39], s33 offen
+; GCN-NEXT:  BB0_2: ; %shader_eval_surface.exit
+; GCN-NEXT:    s_endpgm
+entry:
+  %sd = alloca < 1339 x i32>, align 16, addrspace(5)
+  %state = alloca <4 x i32>, align 4, addrspace(5)
+  %rslt = call i32 @svm_eval_nodes(float addrspace(5)* %kg, <1339 x i32> addrspace(5)* %sd, <4 x i32> addrspace(5)* %state, i32 0, i32 4194304)
+  %cmp = icmp eq i32 %rslt, 0
+  br i1 %cmp, label %shader_eval_surface.exit, label %if.then4.i
+
+if.then4.i:                                       ; preds = %entry
+  %rng_hash.i.i = getelementptr inbounds < 4 x i32>, <4 x i32> addrspace(5)* %state, i32 0, i32 1
+  %tmp0 = load i32, i32 addrspace(5)* %rng_hash.i.i, align 4
+  %rng_offset.i.i = getelementptr inbounds <4 x i32>, <4 x i32> addrspace(5)* %state, i32 0, i32 2
+  %tmp1 = load i32, i32 addrspace(5)* %rng_offset.i.i, align 4
+  %add.i.i = add i32 %tmp1, %tmp0
+  %add1.i.i = add i32 %add.i.i, 0
+  %mul.i.i.i.i = mul i32 %add1.i.i, 1103515245
+  %add.i.i.i.i = add i32 %mul.i.i.i.i, 12345
+  store i32 %add.i.i.i.i, i32 addrspace(5)* undef, align 16
+  br label %shader_eval_surface.exit
+
+shader_eval_surface.exit:                         ; preds = %entry
+  ret void
+}
+
+declare hidden i32 @svm_eval_nodes(float addrspace(5)*, <1339 x i32> addrspace(5)*, <4 x i32> addrspace(5)*, i32, i32) local_unnamed_addr
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-undef-def-with-other-subreg-defs.mir b/llvm/test/CodeGen/AMDGPU/subreg-undef-def-with-other-subreg-defs.mir
new file mode 100644
index 0000000000000..3a574977e4d8a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/subreg-undef-def-with-other-subreg-defs.mir
@@ -0,0 +1,86 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-machineinstrs -run-pass=machine-scheduler -verify-misched -o - %s | FileCheck %s
+
+# Deciding which lanes are killed needs to account for other defs in the
+# instruction.
+#
+# addVRegDefDeps would encounter the %0.sub0 def and erase %0 from
+# current vreg uses because it shared no lanes with %0.sub1 use on the
+# nop. It then didn't see the lanemask when it reached the second
+# subreg def, and failed to add the necessary dependency between the
+# asm and S_NOP
+
+---
+name:            no_live_subrange_at_use
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; CHECK-LABEL: name: no_live_subrange_at_use
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   %0.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load 4, addrspace 3)
+  ; CHECK:   INLINEASM &"", 1, 851978, def %0, 2147549193, %0(tied-def 3)
+  ; CHECK:   INLINEASM &"", 1, 851977, [[DS_READ_B32_gfx9_]]
+  ; CHECK:   INLINEASM &"", 1, 851978, def undef %0.sub0, 851978, def undef %0.sub1
+  ; CHECK:   S_NOP 0, implicit %0.sub1
+  ; CHECK:   $sgpr10 = S_MOV_B32 -1
+  ; CHECK:   S_BRANCH %bb.1
+  bb.0:
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %0.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+  bb.1:
+    %2:vgpr_32 = DS_READ_B32_gfx9 %1, 0, 0, implicit $exec :: (load 4, addrspace 3)
+    INLINEASM &"", 1, 851978, def %0, 2147549193, %0(tied-def 3)
+    INLINEASM &"", 1, 851977, %2
+    INLINEASM &"", 1, 851978, def undef %0.sub0, 851978, def %0.sub1
+    S_NOP 0, implicit %0.sub1
+    $sgpr10 = S_MOV_B32 -1
+    S_BRANCH %bb.1
+
+...
+
+# Different operand order
+---
+name:            no_live_subrange_at_use_swap
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body:             |
+  ; CHECK-LABEL: name: no_live_subrange_at_use_swap
+  ; CHECK: bb.0:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   %0.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; CHECK: bb.1:
+  ; CHECK:   successors: %bb.1(0x80000000)
+  ; CHECK:   [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load 4, addrspace 3)
+  ; CHECK:   INLINEASM &"", 1, 851978, def %0, 2147549193, %0(tied-def 3)
+  ; CHECK:   INLINEASM &"", 1, 851977, [[DS_READ_B32_gfx9_]]
+  ; CHECK:   INLINEASM &"", 1, 851978, def undef %0.sub1, 851978, def undef %0.sub0
+  ; CHECK:   S_NOP 0, implicit %0.sub1
+  ; CHECK:   $sgpr10 = S_MOV_B32 -1
+  ; CHECK:   S_BRANCH %bb.1
+  bb.0:
+    undef %0.sub0:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %0.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+
+  bb.1:
+    %2:vgpr_32 = DS_READ_B32_gfx9 %1, 0, 0, implicit $exec :: (load 4, addrspace 3)
+    INLINEASM &"", 1, 851978, def %0, 2147549193, %0(tied-def 3)
+    INLINEASM &"", 1, 851977, %2
+    INLINEASM &"", 1, 851978, def %0.sub1, 851978, def undef %0.sub0
+    S_NOP 0, implicit %0.sub1
+    $sgpr10 = S_MOV_B32 -1
+    S_BRANCH %bb.1
+
+...
diff --git a/llvm/test/CodeGen/AMDGPU/twoaddr-fma.mir b/llvm/test/CodeGen/AMDGPU/twoaddr-fma.mir
index 52966402e8a18..b0aeb91787cd2 100644
--- a/llvm/test/CodeGen/AMDGPU/twoaddr-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/twoaddr-fma.mir
@@ -181,3 +181,23 @@ body:             |
     %1 = V_FMAC_F32_e32 %stack.0, %0, %2, implicit $exec
 
 ...
+
+# GCN-LABEL: name: test_fmaak_inline_literal_f16
+# GCN: %2:vgpr_32 = V_FMAAK_F16 16384, killed %0, 49664, implicit $exec
+
+---
+name:            test_fmaak_inline_literal_f16
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%3' }
+body:             |
+  bb.0:
+    liveins: $vgpr0
+  
+    %3:vgpr_32 = COPY killed $vgpr0
+
+    %26:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
+    %28:vgpr_32 = V_FMAC_F16_e32 16384, killed %3, %26, implicit $exec
+    S_ENDPGM 0
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/twoaddr-mad.mir b/llvm/test/CodeGen/AMDGPU/twoaddr-mad.mir
index 886785af78e08..e75194bf4b8f0 100644
--- a/llvm/test/CodeGen/AMDGPU/twoaddr-mad.mir
+++ b/llvm/test/CodeGen/AMDGPU/twoaddr-mad.mir
@@ -188,3 +188,23 @@ body:             |
     %1 = V_MAC_F32_e32 %stack.0, %0, %2, implicit $exec
 
 ...
+
+# GCN-LABEL: name: test_madak_inline_literal_f16
+# GCN: %2:vgpr_32 = V_MADAK_F16 16384, killed %0, 49664, implicit $exec
+
+---
+name:            test_madak_inline_literal_f16
+liveins:
+  - { reg: '$vgpr0', virtual-reg: '%3' }
+body:             |
+  bb.0:
+    liveins: $vgpr0
+  
+    %3:vgpr_32 = COPY killed $vgpr0
+
+    %26:vgpr_32 = V_MOV_B32_e32 49664, implicit $exec
+    %28:vgpr_32 = V_MAC_F16_e32 16384, killed %3, %26, implicit $exec
+    S_ENDPGM 0
+
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir b/llvm/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
index 41444b0ef0cde..330426fb0ad13 100644
--- a/llvm/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
+++ b/llvm/test/CodeGen/AMDGPU/vccz-corrupt-bug-workaround.mir
@@ -88,7 +88,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 9, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     S_BRANCH %bb.3
 
@@ -96,7 +96,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 100, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
 
   bb.3.done:
@@ -104,7 +104,7 @@ body:             |
 
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
     S_ENDPGM 0
 
 ...
@@ -149,7 +149,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 9, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 0, implicit $exec
     S_BRANCH %bb.3
 
@@ -157,7 +157,7 @@ body:             |
     liveins: $sgpr6, $sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x00000003
 
     $vgpr0 = V_MOV_B32_e32 100, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (volatile store 4 into `i32 addrspace(1)* undef`)
     $vgpr0 = V_MOV_B32_e32 1, implicit $exec
 
   bb.3.done:
@@ -165,7 +165,7 @@ body:             |
 
     $sgpr3 = S_MOV_B32 61440
     $sgpr2 = S_MOV_B32 -1
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, killed $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out)
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/vmem-to-salu-hazard.mir b/llvm/test/CodeGen/AMDGPU/vmem-to-salu-hazard.mir
index 9d45c5b19e656..10ed241acb585 100644
--- a/llvm/test/CodeGen/AMDGPU/vmem-to-salu-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/vmem-to-salu-hazard.mir
@@ -11,7 +11,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr0 = S_MOV_B32 0
 ...
 # GCN-LABEL: name: vmem_smem_write_sgpr
@@ -25,7 +25,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $sgpr0 = S_LOAD_DWORD_IMM $sgpr0_sgpr1, 0, 0, 0
 ...
 # GCN-LABEL: name: vmem_snop_write_sgpr
@@ -40,7 +40,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_NOP 0
     $sgpr0 = S_MOV_B32 0
 ...
@@ -55,7 +55,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr2 = V_ADD_F32_e32 $vgpr1, $vgpr1, implicit $exec
     $sgpr0 = S_MOV_B32 0
 ...
@@ -70,7 +70,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_WAITCNT 0
     $sgpr0 = S_MOV_B32 0
 ...
@@ -86,7 +86,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_WAITCNT 1
     $sgpr0 = S_MOV_B32 0
 ...
@@ -101,7 +101,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     $exec = S_MOV_B64 7
 ...
 # GCN-LABEL: name: vmem_write_exec_expread
@@ -114,7 +114,7 @@ body:             |
   bb.0:
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $exec_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $exec_lo, 0, 0, 0, 0, 0, 0, implicit $exec
     $exec = S_MOV_B64 7
 ...
 # GCN-LABEL: name: ds_write_m0
@@ -143,7 +143,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.1:
     $sgpr0 = S_MOV_B32 0
@@ -161,7 +161,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.1
 
   bb.1:
@@ -181,7 +181,7 @@ body:             |
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
     $sgpr4 = IMPLICIT_DEF
     $vgpr0 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.2
 
   bb.1:
@@ -206,7 +206,7 @@ body:             |
     $sgpr0 = S_MOV_B32 0
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, 0, implicit $exec
     S_BRANCH %bb.0
 ...
 # GCN-LABEL: name: ds_write_exec
diff --git a/llvm/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir b/llvm/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir
index 644651ded335c..5dbe5d58d9bc3 100644
--- a/llvm/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir
+++ b/llvm/test/CodeGen/AMDGPU/vmem-vcc-hazard.mir
@@ -19,7 +19,7 @@ body:             |
     $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_branch_to_next
 # GCN:      bb.1:
@@ -40,7 +40,7 @@ body:             |
     S_BRANCH %bb.1
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_fallthrough_no_hazard_too_far
 # GCN:      bb.1:
@@ -61,7 +61,7 @@ body:             |
     $sgpr0 = S_MOV_B32 0
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_fallthrough_no_hazard_nops
 # GCN:      bb.1:
@@ -78,7 +78,7 @@ body:             |
     S_NOP 4
 
   bb.1:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_branch_around
 # GCN:      bb.2:
@@ -107,7 +107,7 @@ body:             |
     S_NOP 0
 
   bb.2:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_branch_backedge
 # GCN:      S_NOP
@@ -123,7 +123,7 @@ body:             |
 
     $vgpr0 = IMPLICIT_DEF
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.1:
     $vgpr0 = IMPLICIT_DEF
@@ -156,7 +156,7 @@ body:             |
     $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
 
   bb.2:
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 # GCN-LABEL: name: vmem_vcc_self_loop
 # GCN:      S_NOP
@@ -172,7 +172,7 @@ body:             |
 
     $vgpr0 = IMPLICIT_DEF
     $sgpr0_sgpr1_sgpr2_sgpr3 = IMPLICIT_DEF
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr1 = V_ADDC_U32_e32 $vgpr0, $vgpr0, implicit-def $vcc, implicit $vcc, implicit $exec
     S_BRANCH %bb.0
 ...
@@ -198,7 +198,7 @@ body:             |
     successors: %bb.1
 
     $sgpr0 = S_MOV_B32 0
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr1 = V_ADDC_U32_e32 $vgpr1, $vgpr1, implicit-def $vcc, implicit $vcc, implicit $exec
     S_BRANCH %bb.1
 ...
@@ -224,7 +224,7 @@ body:             |
     successors: %bb.1
 
     $sgpr0 = S_MOV_B32 0
-    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, implicit $exec
+    $vgpr1 = BUFFER_LOAD_DWORD_OFFEN $vgpr0, $sgpr0_sgpr1_sgpr2_sgpr3, $vcc_lo, 0, 0, 0, 0, 0, 0, implicit $exec
     $vgpr1 = V_ADDC_U32_e32 $vgpr1, $vgpr1, implicit-def $vcc, implicit $vcc, implicit $exec
     S_BRANCH %bb.1
 ...
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
index d6e983ae5904c..bfd92347d92b6 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-irreducible.mir
@@ -78,7 +78,7 @@ body: |
 
   bb.1:
     successors: %bb.2
-    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, renamable $vgpr2, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, renamable $vgpr2, renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 
   bb.2:
     successors: %bb.3, %bb.6
@@ -86,7 +86,7 @@ body: |
 
   bb.3:
     successors: %bb.4, %bb.5
-    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, killed renamable $vgpr2, killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFEN_exact killed renamable $vgpr3, killed renamable $vgpr2, killed renamable $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_CBRANCH_VCCNZ %bb.5, implicit $vcc
 
   bb.4:
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index a71ca5db7658d..85c79144d0e52 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -871,33 +871,6 @@ endif:
   ret void
 }
 
-; GCN-LABEL: {{^}}test_init_exec:
-; GFX1032: s_mov_b32 exec_lo, 0x12345
-; GFX1064: s_mov_b64 exec, 0x12345
-; GCN: v_add_f32_e32 v0,
-define amdgpu_ps float @test_init_exec(float %a, float %b) {
-main_body:
-  %s = fadd float %a, %b
-  call void @llvm.amdgcn.init.exec(i64 74565)
-  ret float %s
-}
-
-; GCN-LABEL: {{^}}test_init_exec_from_input:
-; GCN: s_bfe_u32 s0, s3, 0x70008
-; GFX1032: s_bfm_b32 exec_lo, s0, 0
-; GFX1032: s_cmp_eq_u32 s0, 32
-; GFX1032: s_cmov_b32 exec_lo, -1
-; GFX1064: s_bfm_b64 exec, s0, 0
-; GFX1064: s_cmp_eq_u32 s0, 64
-; GFX1064: s_cmov_b64 exec, -1
-; GCN: v_add_f32_e32 v0,
-define amdgpu_ps float @test_init_exec_from_input(i32 inreg, i32 inreg, i32 inreg, i32 inreg %count, float %a, float %b) {
-main_body:
-  %s = fadd float %a, %b
-  call void @llvm.amdgcn.init.exec.from.input(i32 %count, i32 8)
-  ret float %s
-}
-
 ; GCN-LABEL: {{^}}test_vgprblocks_w32_attr:
 ; Test that the wave size can be overridden in function attributes and that the block size is correct as a result
 ; GFX10DEFWAVE: ; VGPRBlocks: 1
@@ -1132,8 +1105,6 @@ declare i32 @llvm.amdgcn.icmp.i32.i32(i32, i32, i32)
 declare void @llvm.amdgcn.kill(i1)
 declare i1 @llvm.amdgcn.wqm.vote(i1)
 declare i1 @llvm.amdgcn.ps.live()
-declare void @llvm.amdgcn.init.exec(i64)
-declare void @llvm.amdgcn.init.exec.from.input(i32, i32)
 declare i64 @llvm.cttz.i64(i64, i1)
 declare i32 @llvm.cttz.i32(i32, i1)
 
diff --git a/llvm/test/CodeGen/ARM/CGP/arm-cgp-casts.ll b/llvm/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
index e269aacad28c8..538f110ffd767 100644
--- a/llvm/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
+++ b/llvm/test/CodeGen/ARM/CGP/arm-cgp-casts.ll
@@ -1,24 +1,68 @@
-; RUN: llc -mtriple=thumbv8.main -mcpu=cortex-m33 %s -arm-disable-cgp=false -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NODSP
-; RUN: llc -mtriple=thumbv7-linux-android %s -arm-disable-cgp=false -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NODSP
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.main -mcpu=cortex-m33 %s -arm-disable-cgp=false -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NODSP --check-prefix=CHECK-NODSP-V8
+; RUN: llc -mtriple=thumbv7-linux-android %s -arm-disable-cgp=false -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NODSP --check-prefix=CHECK-NODSP-V7
 ; RUN: llc -mtriple=thumbv7em -mcpu=cortex-m7 %s -arm-disable-cgp=false -arm-enable-scalar-dsp=true -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DSP
 ; RUN: llc -mtriple=thumbv8 %s -arm-disable-cgp=false -arm-enable-scalar-dsp=true -arm-enable-scalar-dsp-imms=true -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-DSP-IMM
 
 ; Transform will fail because the trunc is not a sink.
-; CHECK-LABEL: dsp_trunc
-; CHECK:   add   [[ADD:[^ ]+]],
-; CHECK-DSP-NEXT: ldrh  r1, [r3]
-; CHECK-DSP-NEXT: ldrh  r2, [r2]
-; CHECK-DSP-NEXT: subs  r1, r1, [[ADD]]
-; CHECK-DSP-NEXT: add   r0, r2
-; CHECK-DSP-NEXT: uxth  r3, r1
-; CHECK-DSP-NEXT: uxth  r2, r0
-; CHECK-DSP-NEXT: cmp   r2, r3
-
-; CHECK-DSP-IMM: usub16
-; CHECK-DSP-IMM: usub16
-; CHECK-DSP-IMM: uadd16
-; CHECK-DSP-IMM: cmp
+
 define i16 @dsp_trunc(i32 %arg0, i32 %arg1, i16* %gep0, i16* %gep1) {
+; CHECK-NODSP-V8-LABEL: dsp_trunc:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    add r1, r0
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r2]
+; CHECK-NODSP-V8-NEXT:    ldrh r2, [r3]
+; CHECK-NODSP-V8-NEXT:    add r0, r1
+; CHECK-NODSP-V8-NEXT:    subs r1, r2, r1
+; CHECK-NODSP-V8-NEXT:    uxth r3, r0
+; CHECK-NODSP-V8-NEXT:    uxth r2, r1
+; CHECK-NODSP-V8-NEXT:    cmp r3, r2
+; CHECK-NODSP-V8-NEXT:    it lo
+; CHECK-NODSP-V8-NEXT:    movlo r0, r1
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: dsp_trunc:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrh r2, [r2]
+; CHECK-NODSP-V7-NEXT:    add r1, r0
+; CHECK-NODSP-V7-NEXT:    ldrh r3, [r3]
+; CHECK-NODSP-V7-NEXT:    adds r0, r2, r1
+; CHECK-NODSP-V7-NEXT:    subs r1, r3, r1
+; CHECK-NODSP-V7-NEXT:    uxth r3, r0
+; CHECK-NODSP-V7-NEXT:    uxth r2, r1
+; CHECK-NODSP-V7-NEXT:    cmp r3, r2
+; CHECK-NODSP-V7-NEXT:    it lo
+; CHECK-NODSP-V7-NEXT:    movlo r0, r1
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: dsp_trunc:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    add r0, r1
+; CHECK-DSP-NEXT:    ldrh r1, [r3]
+; CHECK-DSP-NEXT:    ldrh r2, [r2]
+; CHECK-DSP-NEXT:    subs r1, r1, r0
+; CHECK-DSP-NEXT:    add r0, r2
+; CHECK-DSP-NEXT:    uxth r3, r1
+; CHECK-DSP-NEXT:    uxth r2, r0
+; CHECK-DSP-NEXT:    cmp r2, r3
+; CHECK-DSP-NEXT:    it lo
+; CHECK-DSP-NEXT:    movlo r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: dsp_trunc:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    add r0, r1
+; CHECK-DSP-IMM-NEXT:    movs r1, #0
+; CHECK-DSP-IMM-NEXT:    uxth r0, r0
+; CHECK-DSP-IMM-NEXT:    usub16 r1, r1, r0
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r2]
+; CHECK-DSP-IMM-NEXT:    ldrh r3, [r3]
+; CHECK-DSP-IMM-NEXT:    usub16 r0, r0, r1
+; CHECK-DSP-IMM-NEXT:    uadd16 r1, r3, r1
+; CHECK-DSP-IMM-NEXT:    cmp r0, r1
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %add0 = add i32 %arg0, %arg1
   %conv0 = trunc i32 %add0 to i16
@@ -32,11 +76,16 @@ entry:
   ret i16 %res
 }
 
-; CHECK-LABEL: trunc_i16_i8
-; CHECK: ldrh
-; CHECK: uxtb
-; CHECK: cmp
 define i8 @trunc_i16_i8(i16* %ptr, i16 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: trunc_i16_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ldrh r0, [r0]
+; CHECK-NEXT:    add r0, r1
+; CHECK-NEXT:    uxtb r0, r0
+; CHECK-NEXT:    cmp r0, r2
+; CHECK-NEXT:    it ls
+; CHECK-NEXT:    movls r0, r2
+; CHECK-NEXT:    bx lr
 entry:
   %0 = load i16, i16* %ptr
   %1 = add i16 %0, %arg0
@@ -48,11 +97,70 @@ entry:
 
 ; The pass perform the transform, but a uxtb will still be inserted to handle
 ; the zext to the icmp.
-; CHECK-LABEL: icmp_i32_zext:
-; CHECK: sub
-; CHECK: uxtb
-; CHECK: cmp
 define i8 @icmp_i32_zext(i8* %ptr) {
+; CHECK-NODSP-V8-LABEL: icmp_i32_zext:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrb r2, [r0], #1
+; CHECK-NODSP-V8-NEXT:    subs r1, r2, #1
+; CHECK-NODSP-V8-NEXT:    .p2align 2
+; CHECK-NODSP-V8-NEXT:  .LBB2_1: @ %body
+; CHECK-NODSP-V8-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V8-NEXT:    uxtb r3, r1
+; CHECK-NODSP-V8-NEXT:    cmp r2, r3
+; CHECK-NODSP-V8-NEXT:    itt ne
+; CHECK-NODSP-V8-NEXT:    movne r0, r1
+; CHECK-NODSP-V8-NEXT:    bxne lr
+; CHECK-NODSP-V8-NEXT:    ldrb r1, [r0, r2]
+; CHECK-NODSP-V8-NEXT:    adds r2, #1
+; CHECK-NODSP-V8-NEXT:    b .LBB2_1
+;
+; CHECK-NODSP-V7-LABEL: icmp_i32_zext:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrb r2, [r0], #1
+; CHECK-NODSP-V7-NEXT:    subs r1, r2, #1
+; CHECK-NODSP-V7-NEXT:  .LBB2_1: @ %body
+; CHECK-NODSP-V7-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V7-NEXT:    uxtb r3, r1
+; CHECK-NODSP-V7-NEXT:    cmp r2, r3
+; CHECK-NODSP-V7-NEXT:    itt ne
+; CHECK-NODSP-V7-NEXT:    movne r0, r1
+; CHECK-NODSP-V7-NEXT:    bxne lr
+; CHECK-NODSP-V7-NEXT:    ldrb r1, [r0, r2]
+; CHECK-NODSP-V7-NEXT:    adds r2, #1
+; CHECK-NODSP-V7-NEXT:    b .LBB2_1
+;
+; CHECK-DSP-LABEL: icmp_i32_zext:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrb r2, [r0], #1
+; CHECK-DSP-NEXT:    subs r1, r2, #1
+; CHECK-DSP-NEXT:  .LBB2_1: @ %body
+; CHECK-DSP-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-NEXT:    uxtb r3, r1
+; CHECK-DSP-NEXT:    cmp r2, r3
+; CHECK-DSP-NEXT:    itt ne
+; CHECK-DSP-NEXT:    movne r0, r1
+; CHECK-DSP-NEXT:    bxne lr
+; CHECK-DSP-NEXT:    ldrb r1, [r0, r2]
+; CHECK-DSP-NEXT:    adds r2, #1
+; CHECK-DSP-NEXT:    b .LBB2_1
+;
+; CHECK-DSP-IMM-LABEL: icmp_i32_zext:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrb r2, [r0], #1
+; CHECK-DSP-IMM-NEXT:    subs r1, r2, #1
+; CHECK-DSP-IMM-NEXT:  .LBB2_1: @ %body
+; CHECK-DSP-IMM-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-IMM-NEXT:    uxtb r3, r1
+; CHECK-DSP-IMM-NEXT:    cmp r2, r3
+; CHECK-DSP-IMM-NEXT:    bne .LBB2_3
+; CHECK-DSP-IMM-NEXT:  @ %bb.2: @ %if.end
+; CHECK-DSP-IMM-NEXT:    @ in Loop: Header=BB2_1 Depth=1
+; CHECK-DSP-IMM-NEXT:    ldrb r1, [r0, r2]
+; CHECK-DSP-IMM-NEXT:    adds r2, #1
+; CHECK-DSP-IMM-NEXT:    b .LBB2_1
+; CHECK-DSP-IMM-NEXT:  .LBB2_3: @ %exit
+; CHECK-DSP-IMM-NEXT:    mov r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %gep = getelementptr inbounds i8, i8* %ptr, i32 0
   %0 = load i8, i8* %gep, align 1
@@ -81,10 +189,70 @@ exit:
 }
 
 ; Won't don't handle sext
-; CHECK-LABEL: icmp_sext_zext_store_i8_i16
-; CHECK: ldrb
-; CHECK: ldrsh
 define i32 @icmp_sext_zext_store_i8_i16() {
+; CHECK-NODSP-V8-LABEL: icmp_sext_zext_store_i8_i16:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    movw r0, :lower16:d_uch
+; CHECK-NODSP-V8-NEXT:    movt r0, :upper16:d_uch
+; CHECK-NODSP-V8-NEXT:    ldrb r1, [r0, #2]
+; CHECK-NODSP-V8-NEXT:    movw r0, :lower16:d_sh
+; CHECK-NODSP-V8-NEXT:    movt r0, :upper16:d_sh
+; CHECK-NODSP-V8-NEXT:    ldrsh.w r0, [r0, #4]
+; CHECK-NODSP-V8-NEXT:    movw r2, :lower16:sh1
+; CHECK-NODSP-V8-NEXT:    subs r0, r1, r0
+; CHECK-NODSP-V8-NEXT:    clz r0, r0
+; CHECK-NODSP-V8-NEXT:    movt r2, :upper16:sh1
+; CHECK-NODSP-V8-NEXT:    lsrs r0, r0, #5
+; CHECK-NODSP-V8-NEXT:    strh r1, [r2]
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: icmp_sext_zext_store_i8_i16:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    movw r0, :lower16:d_sh
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:d_uch
+; CHECK-NODSP-V7-NEXT:    movt r0, :upper16:d_sh
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:d_uch
+; CHECK-NODSP-V7-NEXT:    ldrb r1, [r1, #2]
+; CHECK-NODSP-V7-NEXT:    movw r2, :lower16:sh1
+; CHECK-NODSP-V7-NEXT:    ldrsh.w r0, [r0, #4]
+; CHECK-NODSP-V7-NEXT:    movt r2, :upper16:sh1
+; CHECK-NODSP-V7-NEXT:    strh r1, [r2]
+; CHECK-NODSP-V7-NEXT:    subs r0, r1, r0
+; CHECK-NODSP-V7-NEXT:    clz r0, r0
+; CHECK-NODSP-V7-NEXT:    lsrs r0, r0, #5
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: icmp_sext_zext_store_i8_i16:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    movw r0, :lower16:d_uch
+; CHECK-DSP-NEXT:    movw r1, :lower16:sh1
+; CHECK-DSP-NEXT:    movt r0, :upper16:d_uch
+; CHECK-DSP-NEXT:    movt r1, :upper16:sh1
+; CHECK-DSP-NEXT:    ldrb r0, [r0, #2]
+; CHECK-DSP-NEXT:    strh r0, [r1]
+; CHECK-DSP-NEXT:    movw r1, :lower16:d_sh
+; CHECK-DSP-NEXT:    movt r1, :upper16:d_sh
+; CHECK-DSP-NEXT:    ldrsh.w r1, [r1, #4]
+; CHECK-DSP-NEXT:    subs r0, r0, r1
+; CHECK-DSP-NEXT:    clz r0, r0
+; CHECK-DSP-NEXT:    lsrs r0, r0, #5
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: icmp_sext_zext_store_i8_i16:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    movw r0, :lower16:d_sh
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:d_uch
+; CHECK-DSP-IMM-NEXT:    movt r0, :upper16:d_sh
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:d_uch
+; CHECK-DSP-IMM-NEXT:    ldrb r1, [r1, #2]
+; CHECK-DSP-IMM-NEXT:    movw r2, :lower16:sh1
+; CHECK-DSP-IMM-NEXT:    ldrsh.w r0, [r0, #4]
+; CHECK-DSP-IMM-NEXT:    movt r2, :upper16:sh1
+; CHECK-DSP-IMM-NEXT:    strh r1, [r2]
+; CHECK-DSP-IMM-NEXT:    subs r0, r1, r0
+; CHECK-DSP-IMM-NEXT:    clz r0, r0
+; CHECK-DSP-IMM-NEXT:    lsrs r0, r0, #5
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i8, i8* getelementptr inbounds ([16 x i8], [16 x i8]* @d_uch, i32 0, i32 2), align 1
   %conv = zext i8 %0 to i16
@@ -97,12 +265,66 @@ entry:
   ret i32 %conv3
 }
 
-; CHECK-LABEL: or_icmp_ugt:
-; CHECK:     ldrb
-; CHECK:     subs.w
-; CHECK-NOT: uxt
-; CHECK:     cmp
 define i1 @or_icmp_ugt(i32 %arg, i8* %ptr) {
+; CHECK-NODSP-V8-LABEL: or_icmp_ugt:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrb r1, [r1]
+; CHECK-NODSP-V8-NEXT:    adds r2, r1, #3
+; CHECK-NODSP-V8-NEXT:    subs.w r0, r0, r2, lsl #1
+; CHECK-NODSP-V8-NEXT:    it ne
+; CHECK-NODSP-V8-NEXT:    movne r0, #1
+; CHECK-NODSP-V8-NEXT:    subs r1, #1
+; CHECK-NODSP-V8-NEXT:    movs r2, #0
+; CHECK-NODSP-V8-NEXT:    cmp r1, #3
+; CHECK-NODSP-V8-NEXT:    it hi
+; CHECK-NODSP-V8-NEXT:    movhi r2, #1
+; CHECK-NODSP-V8-NEXT:    orrs r0, r2
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: or_icmp_ugt:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrb r1, [r1]
+; CHECK-NODSP-V7-NEXT:    adds r2, r1, #3
+; CHECK-NODSP-V7-NEXT:    subs r1, #1
+; CHECK-NODSP-V7-NEXT:    subs.w r0, r0, r2, lsl #1
+; CHECK-NODSP-V7-NEXT:    mov.w r2, #0
+; CHECK-NODSP-V7-NEXT:    it ne
+; CHECK-NODSP-V7-NEXT:    movne r0, #1
+; CHECK-NODSP-V7-NEXT:    cmp r1, #3
+; CHECK-NODSP-V7-NEXT:    it hi
+; CHECK-NODSP-V7-NEXT:    movhi r2, #1
+; CHECK-NODSP-V7-NEXT:    orrs r0, r2
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: or_icmp_ugt:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrb r1, [r1]
+; CHECK-DSP-NEXT:    adds r2, r1, #3
+; CHECK-DSP-NEXT:    subs r1, #1
+; CHECK-DSP-NEXT:    subs.w r0, r0, r2, lsl #1
+; CHECK-DSP-NEXT:    mov.w r2, #0
+; CHECK-DSP-NEXT:    it ne
+; CHECK-DSP-NEXT:    movne r0, #1
+; CHECK-DSP-NEXT:    cmp r1, #3
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movhi r2, #1
+; CHECK-DSP-NEXT:    orrs r0, r2
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: or_icmp_ugt:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrb r1, [r1]
+; CHECK-DSP-IMM-NEXT:    adds r2, r1, #3
+; CHECK-DSP-IMM-NEXT:    subs.w r0, r0, r2, lsl #1
+; CHECK-DSP-IMM-NEXT:    it ne
+; CHECK-DSP-IMM-NEXT:    movne r0, #1
+; CHECK-DSP-IMM-NEXT:    subs r1, #1
+; CHECK-DSP-IMM-NEXT:    movs r2, #0
+; CHECK-DSP-IMM-NEXT:    cmp r1, #3
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r2, #1
+; CHECK-DSP-IMM-NEXT:    orrs r0, r2
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i8, i8* %ptr
   %1 = zext i8 %0 to i32
@@ -117,11 +339,117 @@ entry:
 
 ; We currently only handle truncs as sinks, so a uxt will still be needed for
 ; the icmp ugt instruction.
-; CHECK-LABEL: urem_trunc_icmps
-; CHECK: cmp
-; CHECK: uxt
-; CHECK: cmp
 define void @urem_trunc_icmps(i16** %in, i32* %g, i32* %k) {
+; CHECK-NODSP-V8-LABEL: urem_trunc_icmps:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldr r0, [r0]
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    cbz r0, .LBB5_3
+; CHECK-NODSP-V8-NEXT:  @ %bb.1: @ %cond.false.i
+; CHECK-NODSP-V8-NEXT:    movs r3, #5
+; CHECK-NODSP-V8-NEXT:    udiv r3, r3, r0
+; CHECK-NODSP-V8-NEXT:    muls r0, r3, r0
+; CHECK-NODSP-V8-NEXT:    rsb.w r0, r0, #5
+; CHECK-NODSP-V8-NEXT:    .p2align 2
+; CHECK-NODSP-V8-NEXT:  .LBB5_2: @ %body
+; CHECK-NODSP-V8-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V8-NEXT:    uxtb r3, r0
+; CHECK-NODSP-V8-NEXT:    cmp r3, #7
+; CHECK-NODSP-V8-NEXT:    mov.w r3, #0
+; CHECK-NODSP-V8-NEXT:    it hi
+; CHECK-NODSP-V8-NEXT:    movhi r3, #1
+; CHECK-NODSP-V8-NEXT:    str r3, [r1]
+; CHECK-NODSP-V8-NEXT:    ldr r3, [r2]
+; CHECK-NODSP-V8-NEXT:    cmp r3, #0
+; CHECK-NODSP-V8-NEXT:    it ne
+; CHECK-NODSP-V8-NEXT:    bxne lr
+; CHECK-NODSP-V8-NEXT:    adds r0, #1
+; CHECK-NODSP-V8-NEXT:    b .LBB5_2
+; CHECK-NODSP-V8-NEXT:  .LBB5_3: @ %exit
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: urem_trunc_icmps:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r4, r5, r7, lr}
+; CHECK-NODSP-V7-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NODSP-V7-NEXT:    ldr r0, [r0]
+; CHECK-NODSP-V7-NEXT:    mov r5, r1
+; CHECK-NODSP-V7-NEXT:    ldrh r1, [r0]
+; CHECK-NODSP-V7-NEXT:    cbz r1, .LBB5_4
+; CHECK-NODSP-V7-NEXT:  @ %bb.1: @ %cond.false.i
+; CHECK-NODSP-V7-NEXT:    movs r0, #5
+; CHECK-NODSP-V7-NEXT:    mov r4, r2
+; CHECK-NODSP-V7-NEXT:    bl __aeabi_uidivmod
+; CHECK-NODSP-V7-NEXT:  .LBB5_2: @ %body
+; CHECK-NODSP-V7-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r1
+; CHECK-NODSP-V7-NEXT:    cmp r0, #7
+; CHECK-NODSP-V7-NEXT:    mov.w r0, #0
+; CHECK-NODSP-V7-NEXT:    it hi
+; CHECK-NODSP-V7-NEXT:    movhi r0, #1
+; CHECK-NODSP-V7-NEXT:    str r0, [r5]
+; CHECK-NODSP-V7-NEXT:    ldr r0, [r4]
+; CHECK-NODSP-V7-NEXT:    cbnz r0, .LBB5_4
+; CHECK-NODSP-V7-NEXT:  @ %bb.3: @ %for.inc
+; CHECK-NODSP-V7-NEXT:    @ in Loop: Header=BB5_2 Depth=1
+; CHECK-NODSP-V7-NEXT:    adds r1, #1
+; CHECK-NODSP-V7-NEXT:    b .LBB5_2
+; CHECK-NODSP-V7-NEXT:  .LBB5_4: @ %exit
+; CHECK-NODSP-V7-NEXT:    pop {r4, r5, r7, pc}
+;
+; CHECK-DSP-LABEL: urem_trunc_icmps:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldr r0, [r0]
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    cbz r0, .LBB5_3
+; CHECK-DSP-NEXT:  @ %bb.1: @ %cond.false.i
+; CHECK-DSP-NEXT:    movs r3, #5
+; CHECK-DSP-NEXT:    udiv r3, r3, r0
+; CHECK-DSP-NEXT:    muls r0, r3, r0
+; CHECK-DSP-NEXT:    rsb.w r0, r0, #5
+; CHECK-DSP-NEXT:  .LBB5_2: @ %body
+; CHECK-DSP-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-NEXT:    uxtb r3, r0
+; CHECK-DSP-NEXT:    cmp r3, #7
+; CHECK-DSP-NEXT:    mov.w r3, #0
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movhi r3, #1
+; CHECK-DSP-NEXT:    str r3, [r1]
+; CHECK-DSP-NEXT:    ldr r3, [r2]
+; CHECK-DSP-NEXT:    cmp r3, #0
+; CHECK-DSP-NEXT:    it ne
+; CHECK-DSP-NEXT:    bxne lr
+; CHECK-DSP-NEXT:    adds r0, #1
+; CHECK-DSP-NEXT:    b .LBB5_2
+; CHECK-DSP-NEXT:  .LBB5_3: @ %exit
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: urem_trunc_icmps:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldr r0, [r0]
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    cbz r0, .LBB5_4
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %cond.false.i
+; CHECK-DSP-IMM-NEXT:    movs r3, #5
+; CHECK-DSP-IMM-NEXT:    udiv r3, r3, r0
+; CHECK-DSP-IMM-NEXT:    muls r0, r3, r0
+; CHECK-DSP-IMM-NEXT:    rsb.w r0, r0, #5
+; CHECK-DSP-IMM-NEXT:  .LBB5_2: @ %body
+; CHECK-DSP-IMM-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-IMM-NEXT:    uxtb r3, r0
+; CHECK-DSP-IMM-NEXT:    cmp r3, #7
+; CHECK-DSP-IMM-NEXT:    mov.w r3, #0
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r3, #1
+; CHECK-DSP-IMM-NEXT:    str r3, [r1]
+; CHECK-DSP-IMM-NEXT:    ldr r3, [r2]
+; CHECK-DSP-IMM-NEXT:    cbnz r3, .LBB5_4
+; CHECK-DSP-IMM-NEXT:  @ %bb.3: @ %for.inc
+; CHECK-DSP-IMM-NEXT:    @ in Loop: Header=BB5_2 Depth=1
+; CHECK-DSP-IMM-NEXT:    adds r0, #1
+; CHECK-DSP-IMM-NEXT:    b .LBB5_2
+; CHECK-DSP-IMM-NEXT:  .LBB5_4: @ %exit
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %ptr = load i16*, i16** %in, align 4
   %ld = load i16, i16* %ptr, align 2
@@ -152,13 +480,44 @@ exit:
 
 ; Check that %exp requires uxth in all cases, and will also be required to
 ; promote %1 for the call - unless we can generate a uadd16.
-; CHECK-LABEL: zext_load_sink_call:
-; CHECK: uxt
-; CHECK-DSP-IMM: uadd16
-; CHECK: cmp
-; CHECK-NODSP: uxt
-; CHECK-DSP-IMM-NOT: uxt
 define i32 @zext_load_sink_call(i16* %ptr, i16 %exp) {
+; CHECK-NODSP-LABEL: zext_load_sink_call:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-NEXT:    uxth r2, r1
+; CHECK-NODSP-NEXT:    cmp r0, r2
+; CHECK-NODSP-NEXT:    itt eq
+; CHECK-NODSP-NEXT:    moveq r0, #0
+; CHECK-NODSP-NEXT:    bxeq lr
+; CHECK-NODSP-NEXT:    adds r1, #3
+; CHECK-NODSP-NEXT:    uxth r1, r1
+; CHECK-NODSP-NEXT:    b dummy
+;
+; CHECK-DSP-LABEL: zext_load_sink_call:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    uxth r2, r1
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    itt eq
+; CHECK-DSP-NEXT:    moveq r0, #0
+; CHECK-DSP-NEXT:    bxeq lr
+; CHECK-DSP-NEXT:    adds r1, #3
+; CHECK-DSP-NEXT:    uxth r1, r1
+; CHECK-DSP-NEXT:    b dummy
+;
+; CHECK-DSP-IMM-LABEL: zext_load_sink_call:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    uxth r2, r1
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    movs r1, #3
+; CHECK-DSP-IMM-NEXT:    uadd16 r1, r2, r1
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    bne .LBB6_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %exit
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB6_2: @ %if.then
+; CHECK-DSP-IMM-NEXT:    b dummy
 entry:
   %0 = load i16, i16* %ptr, align 4
   %1 = add i16 %exp, 3
@@ -176,9 +535,36 @@ exit:
   ret i32 %exitval
 }
 
-; CHECK-LABEL: bitcast_i16
-; CHECK-NOT: uxt
 define i16 @bitcast_i16(i16 zeroext %arg0, i16 zeroext %arg1) {
+; CHECK-NODSP-LABEL: bitcast_i16:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    adds r0, #1
+; CHECK-NODSP-NEXT:    movw r2, #12345
+; CHECK-NODSP-NEXT:    cmp r0, r2
+; CHECK-NODSP-NEXT:    it hi
+; CHECK-NODSP-NEXT:    movwhi r1, #32657
+; CHECK-NODSP-NEXT:    mov r0, r1
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: bitcast_i16:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    adds r0, #1
+; CHECK-DSP-NEXT:    movw r2, #12345
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movwhi r1, #32657
+; CHECK-DSP-NEXT:    mov r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: bitcast_i16:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    adds r2, r0, #1
+; CHECK-DSP-IMM-NEXT:    movw r0, #32657
+; CHECK-DSP-IMM-NEXT:    movw r3, #12345
+; CHECK-DSP-IMM-NEXT:    cmp r2, r3
+; CHECK-DSP-IMM-NEXT:    it ls
+; CHECK-DSP-IMM-NEXT:    movls r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %cast = bitcast i16 12345 to i16
   %add = add nuw i16 %arg0, 1
@@ -187,9 +573,15 @@ entry:
   ret i16 %res
 }
 
-; CHECK-LABEL: bitcast_i8
-; CHECK-NOT: uxt
 define i8 @bitcast_i8(i8 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: bitcast_i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mvn r2, #127
+; CHECK-NEXT:    cmp.w r1, r0, lsl #1
+; CHECK-NEXT:    it ls
+; CHECK-NEXT:    movls r2, #127
+; CHECK-NEXT:    mov r0, r2
+; CHECK-NEXT:    bx lr
 entry:
   %cast = bitcast i8 127 to i8
   %mul = shl nuw i8 %arg0, 1
@@ -198,9 +590,37 @@ entry:
   ret i8 %res
 }
 
-; CHECK-LABEL: bitcast_i16_minus
-; CHECK-NOT: uxt
 define i16 @bitcast_i16_minus(i16 zeroext %arg0, i16 zeroext %arg1) {
+; CHECK-NODSP-LABEL: bitcast_i16_minus:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    eor r2, r0, #7
+; CHECK-NODSP-NEXT:    movw r0, #32657
+; CHECK-NODSP-NEXT:    cmp r2, r1
+; CHECK-NODSP-NEXT:    itt eq
+; CHECK-NODSP-NEXT:    movweq r0, #53191
+; CHECK-NODSP-NEXT:    movteq r0, #65535
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: bitcast_i16_minus:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    eor r2, r0, #7
+; CHECK-DSP-NEXT:    movw r0, #32657
+; CHECK-DSP-NEXT:    cmp r2, r1
+; CHECK-DSP-NEXT:    itt eq
+; CHECK-DSP-NEXT:    movweq r0, #53191
+; CHECK-DSP-NEXT:    movteq r0, #65535
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: bitcast_i16_minus:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    eor r2, r0, #7
+; CHECK-DSP-IMM-NEXT:    movw r0, #32657
+; CHECK-DSP-IMM-NEXT:    cmp r2, r1
+; CHECK-DSP-IMM-NEXT:    it eq
+; CHECK-DSP-IMM-NEXT:    movweq r0, #53191
+; CHECK-DSP-IMM-NEXT:    it eq
+; CHECK-DSP-IMM-NEXT:    movteq r0, #65535
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %cast = bitcast i16 -12345 to i16
   %xor = xor i16 %arg0, 7
@@ -209,9 +629,15 @@ entry:
   ret i16 %res
 }
 
-; CHECK-LABEL: bitcast_i8_minus
-; CHECK-NOT: uxt
 define i8 @bitcast_i8_minus(i8 zeroext %arg0, i8 zeroext %arg1) {
+; CHECK-LABEL: bitcast_i8_minus:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    and r2, r0, #3
+; CHECK-NEXT:    mvn r0, #127
+; CHECK-NEXT:    cmp r2, r1
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    mvnne r0, #126
+; CHECK-NEXT:    bx lr
 entry:
   %cast = bitcast i8 -127 to i8
   %and = and i8 %arg0, 3
@@ -226,9 +652,48 @@ declare i32 @dummy(i32, i32)
 @sh1 = hidden local_unnamed_addr global i16 0, align 2
 @d_sh = hidden local_unnamed_addr global [16 x i16] zeroinitializer, align 2
 
-; CHECK-LABEL: two_stage_zext_trunc_mix
-; CHECK-NOT: uxt
 define i8* @two_stage_zext_trunc_mix(i32* %this, i32 %__pos1, i32 %__n1, i32** %__str, i32 %__pos2, i32 %__n2) {
+; CHECK-NODSP-V8-LABEL: two_stage_zext_trunc_mix:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldr.w r12, [sp]
+; CHECK-NODSP-V8-NEXT:    ldrb r2, [r0]
+; CHECK-NODSP-V8-NEXT:    add.w r0, r3, r12
+; CHECK-NODSP-V8-NEXT:    lsls r2, r2, #31
+; CHECK-NODSP-V8-NEXT:    it eq
+; CHECK-NODSP-V8-NEXT:    addeq r0, r3, r1
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: two_stage_zext_trunc_mix:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrb r2, [r0]
+; CHECK-NODSP-V7-NEXT:    ldr.w r12, [sp]
+; CHECK-NODSP-V7-NEXT:    add.w r0, r3, r12
+; CHECK-NODSP-V7-NEXT:    lsls r2, r2, #31
+; CHECK-NODSP-V7-NEXT:    it eq
+; CHECK-NODSP-V7-NEXT:    addeq r0, r3, r1
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: two_stage_zext_trunc_mix:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldr r2, [sp]
+; CHECK-DSP-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-NEXT:    add r2, r3
+; CHECK-DSP-NEXT:    lsls r0, r0, #31
+; CHECK-DSP-NEXT:    it eq
+; CHECK-DSP-NEXT:    addeq r2, r3, r1
+; CHECK-DSP-NEXT:    mov r0, r2
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: two_stage_zext_trunc_mix:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldr.w r12, [sp]
+; CHECK-DSP-IMM-NEXT:    ldrb r2, [r0]
+; CHECK-DSP-IMM-NEXT:    adds r0, r3, r1
+; CHECK-DSP-IMM-NEXT:    add r12, r3
+; CHECK-DSP-IMM-NEXT:    lsls r1, r2, #31
+; CHECK-DSP-IMM-NEXT:    it ne
+; CHECK-DSP-IMM-NEXT:    movne r0, r12
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %__size_.i.i.i.i = bitcast i32** %__str to i8*
   %0 = load i8, i8* %__size_.i.i.i.i, align 4
@@ -250,9 +715,60 @@ entry:
   ret i8* %res
 }
 
-; CHECK-LABEL: search_through_zext_1
-; CHECK-NOT: uxt
 define i8 @search_through_zext_1(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-NODSP-V8-LABEL: search_through_zext_1:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    subs r3, r1, r0
+; CHECK-NODSP-V8-NEXT:    add.w r12, r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp r3, r2
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp r12, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: search_through_zext_1:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    subs r3, r1, r0
+; CHECK-NODSP-V7-NEXT:    cmp r3, r2
+; CHECK-NODSP-V7-NEXT:    mov r3, r1
+; CHECK-NODSP-V7-NEXT:    it hi
+; CHECK-NODSP-V7-NEXT:    movhi r3, r0
+; CHECK-NODSP-V7-NEXT:    add r0, r1
+; CHECK-NODSP-V7-NEXT:    cmp r0, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r3, #0
+; CHECK-NODSP-V7-NEXT:    mov r0, r3
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: search_through_zext_1:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    subs r3, r1, r0
+; CHECK-DSP-NEXT:    cmp r3, r2
+; CHECK-DSP-NEXT:    mov r3, r1
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movhi r3, r0
+; CHECK-DSP-NEXT:    add r0, r1
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r3, #0
+; CHECK-DSP-NEXT:    mov r0, r3
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: search_through_zext_1:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    subs r3, r1, r0
+; CHECK-DSP-IMM-NEXT:    cmp r3, r2
+; CHECK-DSP-IMM-NEXT:    mov r3, r1
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r3, r0
+; CHECK-DSP-IMM-NEXT:    add r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    cmp r1, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r3
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %add = add nuw i8 %a, %b
   %conv = zext i8 %add to i16
@@ -274,10 +790,69 @@ if.end:
 ; TODO: We should be able to remove the uxtb here. The transform fails because
 ; the icmp ugt uses an i32, which is too large... but this doesn't matter
 ; because it won't be writing a large value to a register as a result.
-; CHECK-LABEL: search_through_zext_2
-; CHECK: uxtb
-; CHECK: uxtb
 define i8 @search_through_zext_2(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c, i32 %d) {
+; CHECK-NODSP-V8-LABEL: search_through_zext_2:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    push {r7, lr}
+; CHECK-NODSP-V8-NEXT:    sub.w lr, r1, r0
+; CHECK-NODSP-V8-NEXT:    add.w r12, r0, r1
+; CHECK-NODSP-V8-NEXT:    uxtb.w lr, lr
+; CHECK-NODSP-V8-NEXT:    uxtb.w r12, r12
+; CHECK-NODSP-V8-NEXT:    cmp lr, r3
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp r12, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    pop {r7, pc}
+;
+; CHECK-NODSP-V7-LABEL: search_through_zext_2:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    sub.w r12, r1, r0
+; CHECK-NODSP-V7-NEXT:    uxtb.w r12, r12
+; CHECK-NODSP-V7-NEXT:    cmp r12, r3
+; CHECK-NODSP-V7-NEXT:    mov r3, r1
+; CHECK-NODSP-V7-NEXT:    it hi
+; CHECK-NODSP-V7-NEXT:    movhi r3, r0
+; CHECK-NODSP-V7-NEXT:    add r0, r1
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r0
+; CHECK-NODSP-V7-NEXT:    cmp r0, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r3, #0
+; CHECK-NODSP-V7-NEXT:    mov r0, r3
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: search_through_zext_2:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    sub.w r12, r1, r0
+; CHECK-DSP-NEXT:    uxtb.w r12, r12
+; CHECK-DSP-NEXT:    cmp r12, r3
+; CHECK-DSP-NEXT:    mov r3, r1
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movhi r3, r0
+; CHECK-DSP-NEXT:    add r0, r1
+; CHECK-DSP-NEXT:    uxtb r0, r0
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r3, #0
+; CHECK-DSP-NEXT:    mov r0, r3
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: search_through_zext_2:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    sub.w r12, r1, r0
+; CHECK-DSP-IMM-NEXT:    uxtb.w r12, r12
+; CHECK-DSP-IMM-NEXT:    cmp r12, r3
+; CHECK-DSP-IMM-NEXT:    mov r3, r1
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r3, r0
+; CHECK-DSP-IMM-NEXT:    add r0, r1
+; CHECK-DSP-IMM-NEXT:    uxtb r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    cmp r1, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r3
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %add = add nuw i8 %a, %b
   %conv = zext i8 %add to i16
@@ -299,10 +874,53 @@ if.end:
 ; TODO: We should be able to remove the uxtb here as all the calculations are
 ; performed on i8s. The promotion of i8 to i16 and then the later truncation
 ; results in the uxtb.
-; CHECK-LABEL: search_through_zext_3
-; CHECK: uxtb
-; CHECK: uxtb
 define i8 @search_through_zext_3(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c, i32 %d) {
+; CHECK-NODSP-LABEL: search_through_zext_3:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    add.w r12, r0, r1
+; CHECK-NODSP-NEXT:    uxtb.w r12, r12
+; CHECK-NODSP-NEXT:    cmp r12, r2
+; CHECK-NODSP-NEXT:    itt hs
+; CHECK-NODSP-NEXT:    movhs r0, #0
+; CHECK-NODSP-NEXT:    bxhs lr
+; CHECK-NODSP-NEXT:    sub.w r2, r1, r12
+; CHECK-NODSP-NEXT:    uxtb r2, r2
+; CHECK-NODSP-NEXT:    cmp r2, r3
+; CHECK-NODSP-NEXT:    it ls
+; CHECK-NODSP-NEXT:    movls r0, r1
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: search_through_zext_3:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    add.w r12, r0, r1
+; CHECK-DSP-NEXT:    uxtb.w r12, r12
+; CHECK-DSP-NEXT:    cmp r12, r2
+; CHECK-DSP-NEXT:    itt hs
+; CHECK-DSP-NEXT:    movhs r0, #0
+; CHECK-DSP-NEXT:    bxhs lr
+; CHECK-DSP-NEXT:    sub.w r2, r1, r12
+; CHECK-DSP-NEXT:    uxtb r2, r2
+; CHECK-DSP-NEXT:    cmp r2, r3
+; CHECK-DSP-NEXT:    it ls
+; CHECK-DSP-NEXT:    movls r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: search_through_zext_3:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    add.w r12, r0, r1
+; CHECK-DSP-IMM-NEXT:    uxtb.w r12, r12
+; CHECK-DSP-IMM-NEXT:    cmp r12, r2
+; CHECK-DSP-IMM-NEXT:    bhs .LBB14_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %if.then
+; CHECK-DSP-IMM-NEXT:    sub.w r2, r1, r12
+; CHECK-DSP-IMM-NEXT:    uxtb r2, r2
+; CHECK-DSP-IMM-NEXT:    cmp r2, r3
+; CHECK-DSP-IMM-NEXT:    it ls
+; CHECK-DSP-IMM-NEXT:    movls r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB14_2:
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %add = add nuw i8 %a, %b
   %conv = zext i8 %add to i16
@@ -323,9 +941,67 @@ if.end:
 }
 
 ; TODO: We should be able to remove the uxt that gets introduced for %conv2
-; CHECK-LABEL: search_through_zext_cmp
-; CHECK: uxt
 define i8 @search_through_zext_cmp(i8 zeroext %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-NODSP-V8-LABEL: search_through_zext_cmp:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    subs r3, r1, r0
+; CHECK-NODSP-V8-NEXT:    subs.w r12, r1, r0
+; CHECK-NODSP-V8-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V8-NEXT:    it ne
+; CHECK-NODSP-V8-NEXT:    movne.w r12, #1
+; CHECK-NODSP-V8-NEXT:    cmp r3, r2
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp r12, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: search_through_zext_cmp:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    subs r3, r1, r0
+; CHECK-NODSP-V7-NEXT:    subs.w r12, r1, r0
+; CHECK-NODSP-V7-NEXT:    it ne
+; CHECK-NODSP-V7-NEXT:    movne.w r12, #1
+; CHECK-NODSP-V7-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V7-NEXT:    cmp r3, r2
+; CHECK-NODSP-V7-NEXT:    it ls
+; CHECK-NODSP-V7-NEXT:    movls r0, r1
+; CHECK-NODSP-V7-NEXT:    cmp r12, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r0, #0
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: search_through_zext_cmp:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    subs r3, r1, r0
+; CHECK-DSP-NEXT:    subs.w r12, r1, r0
+; CHECK-DSP-NEXT:    uxtb r3, r3
+; CHECK-DSP-NEXT:    it ne
+; CHECK-DSP-NEXT:    movne.w r12, #1
+; CHECK-DSP-NEXT:    cmp r3, r2
+; CHECK-DSP-NEXT:    it ls
+; CHECK-DSP-NEXT:    movls r0, r1
+; CHECK-DSP-NEXT:    cmp r12, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r0, #0
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: search_through_zext_cmp:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    subs.w r12, r1, r0
+; CHECK-DSP-IMM-NEXT:    it ne
+; CHECK-DSP-IMM-NEXT:    movne.w r12, #1
+; CHECK-DSP-IMM-NEXT:    subs r3, r1, r0
+; CHECK-DSP-IMM-NEXT:    uxtb r3, r3
+; CHECK-DSP-IMM-NEXT:    cmp r3, r2
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    cmp r12, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %cmp = icmp ne i8 %a, %b
   %conv = zext i1 %cmp to i16
@@ -344,9 +1020,58 @@ if.end:
   ret i8 %retval
 }
 
-; CHECK-LABEL: search_through_zext_load
-; CHECK-NOT: uxt
 define i8 @search_through_zext_load(i8* %a, i8 zeroext %b, i16 zeroext %c) {
+; CHECK-NODSP-V8-LABEL: search_through_zext_load:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrb r3, [r0]
+; CHECK-NODSP-V8-NEXT:    mov r0, r1
+; CHECK-NODSP-V8-NEXT:    subs r1, r1, r3
+; CHECK-NODSP-V8-NEXT:    cmp r1, r2
+; CHECK-NODSP-V8-NEXT:    it hi
+; CHECK-NODSP-V8-NEXT:    movhi r0, r3
+; CHECK-NODSP-V8-NEXT:    cmp r3, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: search_through_zext_load:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V7-NEXT:    subs r3, r1, r0
+; CHECK-NODSP-V7-NEXT:    cmp r3, r2
+; CHECK-NODSP-V7-NEXT:    it hi
+; CHECK-NODSP-V7-NEXT:    movhi r1, r0
+; CHECK-NODSP-V7-NEXT:    cmp r0, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r1, #0
+; CHECK-NODSP-V7-NEXT:    mov r0, r1
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: search_through_zext_load:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-NEXT:    subs r3, r1, r0
+; CHECK-DSP-NEXT:    cmp r3, r2
+; CHECK-DSP-NEXT:    it hi
+; CHECK-DSP-NEXT:    movhi r1, r0
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r1, #0
+; CHECK-DSP-NEXT:    mov r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: search_through_zext_load:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrb r3, [r0]
+; CHECK-DSP-IMM-NEXT:    subs r0, r1, r3
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r1, r3
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    cmp r3, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r1
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %load = load i8, i8* %a
   %conv = zext i8 %load to i16
@@ -365,11 +1090,72 @@ if.end:
   ret i8 %retval
 }
 
-; CHECK-LABEL: trunc_sink_less_than
-; CHECK-NOT: uxth
-; CHECK: cmp
-; CHECK: uxtb
 define i16 @trunc_sink_less_than_cmp(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d) {
+; CHECK-NODSP-V8-LABEL: trunc_sink_less_than_cmp:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    push {r7, lr}
+; CHECK-NODSP-V8-NEXT:    sub.w r12, r1, r0
+; CHECK-NODSP-V8-NEXT:    adds r3, #1
+; CHECK-NODSP-V8-NEXT:    uxth.w lr, r12
+; CHECK-NODSP-V8-NEXT:    uxtb.w r12, r12
+; CHECK-NODSP-V8-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V8-NEXT:    cmp r12, r3
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp lr, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    pop {r7, pc}
+;
+; CHECK-NODSP-V7-LABEL: trunc_sink_less_than_cmp:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r7, lr}
+; CHECK-NODSP-V7-NEXT:    push {r7, lr}
+; CHECK-NODSP-V7-NEXT:    adds r3, #1
+; CHECK-NODSP-V7-NEXT:    sub.w r12, r1, r0
+; CHECK-NODSP-V7-NEXT:    uxtb.w lr, r12
+; CHECK-NODSP-V7-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V7-NEXT:    cmp lr, r3
+; CHECK-NODSP-V7-NEXT:    it ls
+; CHECK-NODSP-V7-NEXT:    movls r0, r1
+; CHECK-NODSP-V7-NEXT:    uxth.w r1, r12
+; CHECK-NODSP-V7-NEXT:    cmp r1, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r0, #0
+; CHECK-NODSP-V7-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-LABEL: trunc_sink_less_than_cmp:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    push {r7, lr}
+; CHECK-DSP-NEXT:    adds r3, #1
+; CHECK-DSP-NEXT:    sub.w r12, r1, r0
+; CHECK-DSP-NEXT:    uxtb.w lr, r12
+; CHECK-DSP-NEXT:    uxtb r3, r3
+; CHECK-DSP-NEXT:    cmp lr, r3
+; CHECK-DSP-NEXT:    it ls
+; CHECK-DSP-NEXT:    movls r0, r1
+; CHECK-DSP-NEXT:    uxth.w r1, r12
+; CHECK-DSP-NEXT:    cmp r1, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r0, #0
+; CHECK-DSP-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-IMM-LABEL: trunc_sink_less_than_cmp:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    push {r7, lr}
+; CHECK-DSP-IMM-NEXT:    adds r3, #1
+; CHECK-DSP-IMM-NEXT:    sub.w r12, r1, r0
+; CHECK-DSP-IMM-NEXT:    uxtb r3, r3
+; CHECK-DSP-IMM-NEXT:    uxtb.w lr, r12
+; CHECK-DSP-IMM-NEXT:    cmp lr, r3
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    uxth.w r3, r12
+; CHECK-DSP-IMM-NEXT:    cmp r3, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r1
+; CHECK-DSP-IMM-NEXT:    pop {r7, pc}
 entry:
   %sub = sub nuw i16 %b, %a
   %cmp = icmp ult i16 %sub, %c
@@ -388,11 +1174,72 @@ if.end:
 }
 
 ; TODO: We should be able to remove the uxth introduced to handle %sub
-; CHECK-LABEL: trunc_sink_less_than_arith
-; CHECK: uxth
-; CHECK: cmp
-; CHECK: uxtb
 define i16 @trunc_sink_less_than_arith(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-NODSP-V8-LABEL: trunc_sink_less_than_arith:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    push {r4, lr}
+; CHECK-NODSP-V8-NEXT:    sub.w lr, r1, r0
+; CHECK-NODSP-V8-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-NODSP-V8-NEXT:    add r3, lr
+; CHECK-NODSP-V8-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V8-NEXT:    uxth.w r4, lr
+; CHECK-NODSP-V8-NEXT:    cmp r12, r3
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, r1
+; CHECK-NODSP-V8-NEXT:    cmp r4, r2
+; CHECK-NODSP-V8-NEXT:    it hs
+; CHECK-NODSP-V8-NEXT:    movhs r0, #0
+; CHECK-NODSP-V8-NEXT:    pop {r4, pc}
+;
+; CHECK-NODSP-V7-LABEL: trunc_sink_less_than_arith:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r7, lr}
+; CHECK-NODSP-V7-NEXT:    push {r7, lr}
+; CHECK-NODSP-V7-NEXT:    sub.w lr, r1, r0
+; CHECK-NODSP-V7-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-NODSP-V7-NEXT:    add r3, lr
+; CHECK-NODSP-V7-NEXT:    uxtb r3, r3
+; CHECK-NODSP-V7-NEXT:    cmp r12, r3
+; CHECK-NODSP-V7-NEXT:    it ls
+; CHECK-NODSP-V7-NEXT:    movls r0, r1
+; CHECK-NODSP-V7-NEXT:    uxth.w r1, lr
+; CHECK-NODSP-V7-NEXT:    cmp r1, r2
+; CHECK-NODSP-V7-NEXT:    it hs
+; CHECK-NODSP-V7-NEXT:    movhs r0, #0
+; CHECK-NODSP-V7-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-LABEL: trunc_sink_less_than_arith:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    push {r7, lr}
+; CHECK-DSP-NEXT:    sub.w r12, r1, r0
+; CHECK-DSP-NEXT:    add r3, r12
+; CHECK-DSP-NEXT:    uxtb.w lr, r3
+; CHECK-DSP-NEXT:    ldr r3, [sp, #8]
+; CHECK-DSP-NEXT:    cmp r3, lr
+; CHECK-DSP-NEXT:    it ls
+; CHECK-DSP-NEXT:    movls r0, r1
+; CHECK-DSP-NEXT:    uxth.w r1, r12
+; CHECK-DSP-NEXT:    cmp r1, r2
+; CHECK-DSP-NEXT:    it hs
+; CHECK-DSP-NEXT:    movhs r0, #0
+; CHECK-DSP-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-IMM-LABEL: trunc_sink_less_than_arith:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    push {r7, lr}
+; CHECK-DSP-IMM-NEXT:    sub.w lr, r1, r0
+; CHECK-DSP-IMM-NEXT:    ldr.w r12, [sp, #8]
+; CHECK-DSP-IMM-NEXT:    add r3, lr
+; CHECK-DSP-IMM-NEXT:    uxtb r3, r3
+; CHECK-DSP-IMM-NEXT:    cmp r12, r3
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    uxth.w r3, lr
+; CHECK-DSP-IMM-NEXT:    cmp r3, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r1
+; CHECK-DSP-IMM-NEXT:    pop {r7, pc}
 entry:
   %sub = sub nuw i16 %b, %a
   %cmp = icmp ult i16 %sub, %c
@@ -410,11 +1257,42 @@ if.end:
   ret i16 %retval
 }
 
-; CHECK-LABEL: trunc_sink_less_than_store
-; CHECK-NOT: uxt
-; CHECK: cmp
-; CHECK-NOT: uxt
 define i16 @trunc_sink_less_than_store(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8* %e) {
+; CHECK-NODSP-LABEL: trunc_sink_less_than_store:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    subs r0, r1, r0
+; CHECK-NODSP-NEXT:    cmp r0, r2
+; CHECK-NODSP-NEXT:    iteee hs
+; CHECK-NODSP-NEXT:    movhs r0, #0
+; CHECK-NODSP-NEXT:    ldrlo r1, [sp]
+; CHECK-NODSP-NEXT:    addlo r2, r3, r0
+; CHECK-NODSP-NEXT:    strblo r2, [r1]
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: trunc_sink_less_than_store:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    subs r0, r1, r0
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    iteee hs
+; CHECK-DSP-NEXT:    movhs r0, #0
+; CHECK-DSP-NEXT:    ldrlo r1, [sp]
+; CHECK-DSP-NEXT:    addlo r2, r3, r0
+; CHECK-DSP-NEXT:    strblo r2, [r1]
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: trunc_sink_less_than_store:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    subs r0, r1, r0
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    bhs .LBB19_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %if.then
+; CHECK-DSP-IMM-NEXT:    ldr r1, [sp]
+; CHECK-DSP-IMM-NEXT:    adds r2, r3, r0
+; CHECK-DSP-IMM-NEXT:    strb r2, [r1]
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB19_2:
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %sub = sub nuw i16 %b, %a
   %cmp = icmp ult i16 %sub, %c
@@ -431,9 +1309,34 @@ if.end:
   ret i16 %retval
 }
 
-; CHECK-LABEL: trunc_sink_less_than_ret
-; CHECK: uxt{{.*}}b
 define i8 @trunc_sink_less_than_ret(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-NODSP-LABEL: trunc_sink_less_than_ret:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    subs r1, r1, r0
+; CHECK-NODSP-NEXT:    movs r0, #0
+; CHECK-NODSP-NEXT:    cmp r1, r2
+; CHECK-NODSP-NEXT:    it lo
+; CHECK-NODSP-NEXT:    uxtablo r0, r3, r1
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: trunc_sink_less_than_ret:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    subs r1, r1, r0
+; CHECK-DSP-NEXT:    movs r0, #0
+; CHECK-DSP-NEXT:    cmp r1, r2
+; CHECK-DSP-NEXT:    it lo
+; CHECK-DSP-NEXT:    uxtablo r0, r3, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: trunc_sink_less_than_ret:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    subs r1, r1, r0
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    cmp r1, r2
+; CHECK-DSP-IMM-NEXT:    uxtab r3, r3, r1
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r0, r3
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %sub = sub nuw i16 %b, %a
   %cmp = icmp ult i16 %sub, %c
@@ -449,11 +1352,37 @@ if.end:
   ret i8 %retval
 }
 
-; CHECK-LABEL: trunc_sink_less_than_zext_ret
-; CHECK-NOT: uxth
-; CHECK: sub
-; CHECK: uxtb
 define zeroext i8 @trunc_sink_less_than_zext_ret(i16 zeroext %a, i16 zeroext %b, i16 zeroext %c, i8 zeroext %d, i8 zeroext %e) {
+; CHECK-NODSP-LABEL: trunc_sink_less_than_zext_ret:
+; CHECK-NODSP:       @ %bb.0: @ %entry
+; CHECK-NODSP-NEXT:    subs r0, r1, r0
+; CHECK-NODSP-NEXT:    movs r1, #0
+; CHECK-NODSP-NEXT:    cmp r0, r2
+; CHECK-NODSP-NEXT:    it lo
+; CHECK-NODSP-NEXT:    addlo r1, r3, r0
+; CHECK-NODSP-NEXT:    uxtb r0, r1
+; CHECK-NODSP-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: trunc_sink_less_than_zext_ret:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    subs r0, r1, r0
+; CHECK-DSP-NEXT:    movs r1, #0
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    it lo
+; CHECK-DSP-NEXT:    addlo r1, r3, r0
+; CHECK-DSP-NEXT:    uxtb r0, r1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: trunc_sink_less_than_zext_ret:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    subs r0, r1, r0
+; CHECK-DSP-IMM-NEXT:    adds r1, r3, r0
+; CHECK-DSP-IMM-NEXT:    movs r3, #0
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    it lo
+; CHECK-DSP-IMM-NEXT:    movlo r3, r1
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r3
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %sub = sub nuw i16 %b, %a
   %cmp = icmp ult i16 %sub, %c
@@ -469,9 +1398,13 @@ if.end:
   ret i8 %retval
 }
 
-; CHECK-LABEL: bitcast_i1
-; CHECK-NOT: uxt
 define i32 @bitcast_i1(i16 zeroext %a, i32 %b, i32 %c) {
+; CHECK-LABEL: bitcast_i1:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    ands r0, r0, #1
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne r0, r1
+; CHECK-NEXT:    bx lr
 entry:
   %0 = bitcast i1 1 to i1
   %1 = trunc i16 %a to i1
@@ -490,12 +1423,87 @@ exit:
   ret i32 %retval
 }
 
-; CHECK-LABEL: search_back_through_trunc
-; CHECK-NOT: uxt
-; CHECK: cmp
-; CHECK: strb
-; CHECK: strb
 define void @search_back_through_trunc(i8* %a, i8* %b, i8* %c, i8* %d, i16* %e) {
+; CHECK-NODSP-V8-LABEL: search_back_through_trunc:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    push {r7, lr}
+; CHECK-NODSP-V8-NEXT:    ldrb.w r12, [r0]
+; CHECK-NODSP-V8-NEXT:    ldrb.w lr, [r1]
+; CHECK-NODSP-V8-NEXT:    ldrb r1, [r2]
+; CHECK-NODSP-V8-NEXT:    ldrb r0, [r3]
+; CHECK-NODSP-V8-NEXT:    orr.w r12, lr, r12, lsl #8
+; CHECK-NODSP-V8-NEXT:    orr.w r0, r0, r1, lsl #8
+; CHECK-NODSP-V8-NEXT:    cmp r12, r0
+; CHECK-NODSP-V8-NEXT:    beq .LBB23_2
+; CHECK-NODSP-V8-NEXT:  @ %bb.1: @ %if.else136
+; CHECK-NODSP-V8-NEXT:    ldr r0, [sp, #8]
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    uxtb.w lr, r0
+; CHECK-NODSP-V8-NEXT:    lsrs r1, r0, #8
+; CHECK-NODSP-V8-NEXT:  .LBB23_2: @ %if.end183
+; CHECK-NODSP-V8-NEXT:    strb r1, [r2]
+; CHECK-NODSP-V8-NEXT:    strb.w lr, [r3]
+; CHECK-NODSP-V8-NEXT:    pop {r7, pc}
+;
+; CHECK-NODSP-V7-LABEL: search_back_through_trunc:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r4, lr}
+; CHECK-NODSP-V7-NEXT:    push {r4, lr}
+; CHECK-NODSP-V7-NEXT:    ldrb r4, [r0]
+; CHECK-NODSP-V7-NEXT:    ldrb.w r12, [r2]
+; CHECK-NODSP-V7-NEXT:    ldrb r0, [r1]
+; CHECK-NODSP-V7-NEXT:    ldrb.w lr, [r3]
+; CHECK-NODSP-V7-NEXT:    orr.w r4, r0, r4, lsl #8
+; CHECK-NODSP-V7-NEXT:    orr.w r1, lr, r12, lsl #8
+; CHECK-NODSP-V7-NEXT:    cmp r4, r1
+; CHECK-NODSP-V7-NEXT:    itttt ne
+; CHECK-NODSP-V7-NEXT:    ldrne r0, [sp, #8]
+; CHECK-NODSP-V7-NEXT:    ldrhne r0, [r0]
+; CHECK-NODSP-V7-NEXT:    lsrne.w r12, r0, #8
+; CHECK-NODSP-V7-NEXT:    uxtbne r0, r0
+; CHECK-NODSP-V7-NEXT:    strb.w r12, [r2]
+; CHECK-NODSP-V7-NEXT:    strb r0, [r3]
+; CHECK-NODSP-V7-NEXT:    pop {r4, pc}
+;
+; CHECK-DSP-LABEL: search_back_through_trunc:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    push {r4, lr}
+; CHECK-DSP-NEXT:    ldrb r4, [r0]
+; CHECK-DSP-NEXT:    ldrb r0, [r1]
+; CHECK-DSP-NEXT:    ldrb.w r12, [r2]
+; CHECK-DSP-NEXT:    ldrb.w lr, [r3]
+; CHECK-DSP-NEXT:    orr.w lr, lr, r12, lsl #8
+; CHECK-DSP-NEXT:    orr.w r1, r0, r4, lsl #8
+; CHECK-DSP-NEXT:    cmp r1, lr
+; CHECK-DSP-NEXT:    itttt ne
+; CHECK-DSP-NEXT:    ldrne r0, [sp, #8]
+; CHECK-DSP-NEXT:    ldrhne r0, [r0]
+; CHECK-DSP-NEXT:    lsrne.w r12, r0, #8
+; CHECK-DSP-NEXT:    uxtbne r0, r0
+; CHECK-DSP-NEXT:    strb.w r12, [r2]
+; CHECK-DSP-NEXT:    strb r0, [r3]
+; CHECK-DSP-NEXT:    pop {r4, pc}
+;
+; CHECK-DSP-IMM-LABEL: search_back_through_trunc:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    push {r4, lr}
+; CHECK-DSP-IMM-NEXT:    ldrb r4, [r0]
+; CHECK-DSP-IMM-NEXT:    ldrb.w r12, [r2]
+; CHECK-DSP-IMM-NEXT:    ldrb r0, [r1]
+; CHECK-DSP-IMM-NEXT:    ldrb.w lr, [r3]
+; CHECK-DSP-IMM-NEXT:    orr.w r4, r0, r4, lsl #8
+; CHECK-DSP-IMM-NEXT:    orr.w r1, lr, r12, lsl #8
+; CHECK-DSP-IMM-NEXT:    cmp r4, r1
+; CHECK-DSP-IMM-NEXT:    beq .LBB23_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %if.else136
+; CHECK-DSP-IMM-NEXT:    ldr r0, [sp, #8]
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    lsr.w r12, r0, #8
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r0
+; CHECK-DSP-IMM-NEXT:  .LBB23_2: @ %if.end183
+; CHECK-DSP-IMM-NEXT:    strb.w r12, [r2]
+; CHECK-DSP-IMM-NEXT:    strb r0, [r3]
+; CHECK-DSP-IMM-NEXT:    pop {r4, pc}
 entry:
   %0 = load i8, i8* %a, align 1
   %conv106 = zext i8 %0 to i16
@@ -534,11 +1542,136 @@ if.end183:
 @a = common dso_local local_unnamed_addr global i8 0, align 1
 @d = common dso_local local_unnamed_addr global i32 0, align 4
 
-; CHECK-LABEL: and_trunc
-; CHECK: ldrh
-; CHECK: sxth
-; CHECK: uxtb
 define void @and_trunc_two_zext() {
+; CHECK-NODSP-V8-LABEL: and_trunc_two_zext:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    movw r1, :lower16:b
+; CHECK-NODSP-V8-NEXT:    movt r1, :upper16:b
+; CHECK-NODSP-V8-NEXT:    ldrh r1, [r1]
+; CHECK-NODSP-V8-NEXT:    movw r3, :lower16:f
+; CHECK-NODSP-V8-NEXT:    sxth r2, r1
+; CHECK-NODSP-V8-NEXT:    movt r3, :upper16:f
+; CHECK-NODSP-V8-NEXT:    str r2, [r3]
+; CHECK-NODSP-V8-NEXT:    movw r3, :lower16:a
+; CHECK-NODSP-V8-NEXT:    movt r3, :upper16:a
+; CHECK-NODSP-V8-NEXT:    movw r0, :lower16:c
+; CHECK-NODSP-V8-NEXT:    movw r2, :lower16:e
+; CHECK-NODSP-V8-NEXT:    ldrb r3, [r3]
+; CHECK-NODSP-V8-NEXT:    movt r0, :upper16:c
+; CHECK-NODSP-V8-NEXT:    and r1, r1, #1
+; CHECK-NODSP-V8-NEXT:    movt r2, :upper16:e
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    strb r1, [r2]
+; CHECK-NODSP-V8-NEXT:    muls r1, r3, r1
+; CHECK-NODSP-V8-NEXT:    uxtb r1, r1
+; CHECK-NODSP-V8-NEXT:    movw r2, :lower16:d
+; CHECK-NODSP-V8-NEXT:    orrs r0, r1
+; CHECK-NODSP-V8-NEXT:    movt r2, :upper16:d
+; CHECK-NODSP-V8-NEXT:    lsls r0, r0, #16
+; CHECK-NODSP-V8-NEXT:    str r1, [r2]
+; CHECK-NODSP-V8-NEXT:    it eq
+; CHECK-NODSP-V8-NEXT:    bxeq lr
+; CHECK-NODSP-V8-NEXT:    .p2align 2
+; CHECK-NODSP-V8-NEXT:  .LBB24_1: @ %for.cond
+; CHECK-NODSP-V8-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V8-NEXT:    b .LBB24_1
+;
+; CHECK-NODSP-V7-LABEL: and_trunc_two_zext:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:b
+; CHECK-NODSP-V7-NEXT:    movw r2, :lower16:a
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:b
+; CHECK-NODSP-V7-NEXT:    movt r2, :upper16:a
+; CHECK-NODSP-V7-NEXT:    ldrh r1, [r1]
+; CHECK-NODSP-V7-NEXT:    movw r0, :lower16:c
+; CHECK-NODSP-V7-NEXT:    ldrb r2, [r2]
+; CHECK-NODSP-V7-NEXT:    movt r0, :upper16:c
+; CHECK-NODSP-V7-NEXT:    and r3, r1, #1
+; CHECK-NODSP-V7-NEXT:    ldrh.w r12, [r0]
+; CHECK-NODSP-V7-NEXT:    movw r0, :lower16:e
+; CHECK-NODSP-V7-NEXT:    muls r2, r3, r2
+; CHECK-NODSP-V7-NEXT:    movt r0, :upper16:e
+; CHECK-NODSP-V7-NEXT:    strb r3, [r0]
+; CHECK-NODSP-V7-NEXT:    sxth r0, r1
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:f
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:f
+; CHECK-NODSP-V7-NEXT:    str r0, [r1]
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:d
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:d
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r2
+; CHECK-NODSP-V7-NEXT:    str r0, [r1]
+; CHECK-NODSP-V7-NEXT:    orr.w r0, r0, r12
+; CHECK-NODSP-V7-NEXT:    lsls r0, r0, #16
+; CHECK-NODSP-V7-NEXT:    it eq
+; CHECK-NODSP-V7-NEXT:    bxeq lr
+; CHECK-NODSP-V7-NEXT:  .LBB24_1: @ %for.cond
+; CHECK-NODSP-V7-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NODSP-V7-NEXT:    b .LBB24_1
+;
+; CHECK-DSP-LABEL: and_trunc_two_zext:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    movw r0, :lower16:b
+; CHECK-DSP-NEXT:    movw r2, :lower16:f
+; CHECK-DSP-NEXT:    movt r0, :upper16:b
+; CHECK-DSP-NEXT:    movt r2, :upper16:f
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    sxth r1, r0
+; CHECK-DSP-NEXT:    and r0, r0, #1
+; CHECK-DSP-NEXT:    str r1, [r2]
+; CHECK-DSP-NEXT:    movw r1, :lower16:e
+; CHECK-DSP-NEXT:    movt r1, :upper16:e
+; CHECK-DSP-NEXT:    strb r0, [r1]
+; CHECK-DSP-NEXT:    movw r1, :lower16:a
+; CHECK-DSP-NEXT:    movt r1, :upper16:a
+; CHECK-DSP-NEXT:    ldrb r1, [r1]
+; CHECK-DSP-NEXT:    muls r0, r1, r0
+; CHECK-DSP-NEXT:    movw r1, :lower16:d
+; CHECK-DSP-NEXT:    uxtb r0, r0
+; CHECK-DSP-NEXT:    movt r1, :upper16:d
+; CHECK-DSP-NEXT:    str r0, [r1]
+; CHECK-DSP-NEXT:    movw r1, :lower16:c
+; CHECK-DSP-NEXT:    movt r1, :upper16:c
+; CHECK-DSP-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-NEXT:    orrs r0, r1
+; CHECK-DSP-NEXT:    lsls r0, r0, #16
+; CHECK-DSP-NEXT:    it eq
+; CHECK-DSP-NEXT:    bxeq lr
+; CHECK-DSP-NEXT:  .LBB24_1: @ %for.cond
+; CHECK-DSP-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-NEXT:    b .LBB24_1
+;
+; CHECK-DSP-IMM-LABEL: and_trunc_two_zext:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:b
+; CHECK-DSP-IMM-NEXT:    movw r2, :lower16:a
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:b
+; CHECK-DSP-IMM-NEXT:    movt r2, :upper16:a
+; CHECK-DSP-IMM-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-IMM-NEXT:    movw r0, :lower16:c
+; CHECK-DSP-IMM-NEXT:    ldrb r2, [r2]
+; CHECK-DSP-IMM-NEXT:    movt r0, :upper16:c
+; CHECK-DSP-IMM-NEXT:    and r3, r1, #1
+; CHECK-DSP-IMM-NEXT:    ldrh.w r12, [r0]
+; CHECK-DSP-IMM-NEXT:    movw r0, :lower16:e
+; CHECK-DSP-IMM-NEXT:    muls r2, r3, r2
+; CHECK-DSP-IMM-NEXT:    movt r0, :upper16:e
+; CHECK-DSP-IMM-NEXT:    strb r3, [r0]
+; CHECK-DSP-IMM-NEXT:    sxth r0, r1
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:f
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:f
+; CHECK-DSP-IMM-NEXT:    str r0, [r1]
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:d
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r2
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:d
+; CHECK-DSP-IMM-NEXT:    str r0, [r1]
+; CHECK-DSP-IMM-NEXT:    orr.w r0, r0, r12
+; CHECK-DSP-IMM-NEXT:    lsls r0, r0, #16
+; CHECK-DSP-IMM-NEXT:    beq .LBB24_2
+; CHECK-DSP-IMM-NEXT:  .LBB24_1: @ %for.cond
+; CHECK-DSP-IMM-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-DSP-IMM-NEXT:    b .LBB24_1
+; CHECK-DSP-IMM-NEXT:  .LBB24_2: @ %if.end
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i16, i16* @c, align 2
   %1 = load i16, i16* @b, align 2
@@ -563,9 +1696,79 @@ if.end:
   ret void
 }
 
-; CHECK-LABEL: zext_urem_trunc
-; CHECK-NOT: uxt
 define void @zext_urem_trunc() {
+; CHECK-NODSP-V8-LABEL: zext_urem_trunc:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    movw r0, :lower16:c
+; CHECK-NODSP-V8-NEXT:    movt r0, :upper16:c
+; CHECK-NODSP-V8-NEXT:    ldrh r1, [r0]
+; CHECK-NODSP-V8-NEXT:    movw r0, :lower16:e
+; CHECK-NODSP-V8-NEXT:    movt r0, :upper16:e
+; CHECK-NODSP-V8-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V8-NEXT:    cbz r1, .LBB25_2
+; CHECK-NODSP-V8-NEXT:  @ %bb.1: @ %cond.false
+; CHECK-NODSP-V8-NEXT:    udiv r2, r0, r1
+; CHECK-NODSP-V8-NEXT:    mls r0, r2, r1, r0
+; CHECK-NODSP-V8-NEXT:  .LBB25_2: @ %cond.end
+; CHECK-NODSP-V8-NEXT:    movw r1, :lower16:a
+; CHECK-NODSP-V8-NEXT:    movt r1, :upper16:a
+; CHECK-NODSP-V8-NEXT:    strb r0, [r1]
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: zext_urem_trunc:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r7, lr}
+; CHECK-NODSP-V7-NEXT:    push {r7, lr}
+; CHECK-NODSP-V7-NEXT:    movw r0, :lower16:e
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:c
+; CHECK-NODSP-V7-NEXT:    movt r0, :upper16:e
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:c
+; CHECK-NODSP-V7-NEXT:    ldrh r1, [r1]
+; CHECK-NODSP-V7-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V7-NEXT:    cbz r1, .LBB25_2
+; CHECK-NODSP-V7-NEXT:  @ %bb.1: @ %cond.false
+; CHECK-NODSP-V7-NEXT:    bl __aeabi_uidivmod
+; CHECK-NODSP-V7-NEXT:    mov r0, r1
+; CHECK-NODSP-V7-NEXT:  .LBB25_2: @ %cond.end
+; CHECK-NODSP-V7-NEXT:    movw r1, :lower16:a
+; CHECK-NODSP-V7-NEXT:    movt r1, :upper16:a
+; CHECK-NODSP-V7-NEXT:    strb r0, [r1]
+; CHECK-NODSP-V7-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-LABEL: zext_urem_trunc:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    movw r1, :lower16:c
+; CHECK-DSP-NEXT:    movw r0, :lower16:e
+; CHECK-DSP-NEXT:    movt r1, :upper16:c
+; CHECK-DSP-NEXT:    movt r0, :upper16:e
+; CHECK-DSP-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-NEXT:    cmp r1, #0
+; CHECK-DSP-NEXT:    itt ne
+; CHECK-DSP-NEXT:    udivne r2, r0, r1
+; CHECK-DSP-NEXT:    mlsne r0, r2, r1, r0
+; CHECK-DSP-NEXT:    movw r1, :lower16:a
+; CHECK-DSP-NEXT:    movt r1, :upper16:a
+; CHECK-DSP-NEXT:    strb r0, [r1]
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: zext_urem_trunc:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    movw r0, :lower16:e
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:c
+; CHECK-DSP-IMM-NEXT:    movt r0, :upper16:e
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:c
+; CHECK-DSP-IMM-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-IMM-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-IMM-NEXT:    cbz r1, .LBB25_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %cond.false
+; CHECK-DSP-IMM-NEXT:    udiv r2, r0, r1
+; CHECK-DSP-IMM-NEXT:    mls r0, r2, r1, r0
+; CHECK-DSP-IMM-NEXT:  .LBB25_2: @ %cond.end
+; CHECK-DSP-IMM-NEXT:    movw r1, :lower16:a
+; CHECK-DSP-IMM-NEXT:    movt r1, :upper16:a
+; CHECK-DSP-IMM-NEXT:    strb r0, [r1]
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i16, i16* @c, align 2
   %cmp = icmp eq i16 %0, 0
@@ -584,11 +1787,88 @@ cond.end:
   ret void
 }
 
-; CHECK-LABEL: dont_replace_trunc_1
-; CHECK: sxth
-; CHECK: uxtb
-; CHECK: uxth
 define i1 @dont_replace_trunc_1(i8* %a, i16* %b, i16* %c, i32* %d, i8* %e, i32* %f) {
+; CHECK-NODSP-V8-LABEL: dont_replace_trunc_1:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    push {r4, lr}
+; CHECK-NODSP-V8-NEXT:    ldrh r1, [r1]
+; CHECK-NODSP-V8-NEXT:    ldrd r12, lr, [sp, #8]
+; CHECK-NODSP-V8-NEXT:    sxth r4, r1
+; CHECK-NODSP-V8-NEXT:    and r1, r1, #1
+; CHECK-NODSP-V8-NEXT:    ldrh r2, [r2]
+; CHECK-NODSP-V8-NEXT:    str.w r4, [lr]
+; CHECK-NODSP-V8-NEXT:    strb.w r1, [r12]
+; CHECK-NODSP-V8-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V8-NEXT:    muls r0, r1, r0
+; CHECK-NODSP-V8-NEXT:    uxtb r1, r0
+; CHECK-NODSP-V8-NEXT:    orr.w r0, r2, r1
+; CHECK-NODSP-V8-NEXT:    uxth r0, r0
+; CHECK-NODSP-V8-NEXT:    clz r0, r0
+; CHECK-NODSP-V8-NEXT:    lsrs r0, r0, #5
+; CHECK-NODSP-V8-NEXT:    str r1, [r3]
+; CHECK-NODSP-V8-NEXT:    pop {r4, pc}
+;
+; CHECK-NODSP-V7-LABEL: dont_replace_trunc_1:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r4, lr}
+; CHECK-NODSP-V7-NEXT:    push {r4, lr}
+; CHECK-NODSP-V7-NEXT:    ldrh r1, [r1]
+; CHECK-NODSP-V7-NEXT:    ldrd lr, r12, [sp, #8]
+; CHECK-NODSP-V7-NEXT:    ldrh r2, [r2]
+; CHECK-NODSP-V7-NEXT:    sxth r4, r1
+; CHECK-NODSP-V7-NEXT:    and r1, r1, #1
+; CHECK-NODSP-V7-NEXT:    str.w r4, [r12]
+; CHECK-NODSP-V7-NEXT:    strb.w r1, [lr]
+; CHECK-NODSP-V7-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V7-NEXT:    muls r0, r1, r0
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r0
+; CHECK-NODSP-V7-NEXT:    str r0, [r3]
+; CHECK-NODSP-V7-NEXT:    orrs r0, r2
+; CHECK-NODSP-V7-NEXT:    uxth r0, r0
+; CHECK-NODSP-V7-NEXT:    clz r0, r0
+; CHECK-NODSP-V7-NEXT:    lsrs r0, r0, #5
+; CHECK-NODSP-V7-NEXT:    pop {r4, pc}
+;
+; CHECK-DSP-LABEL: dont_replace_trunc_1:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    push {r7, lr}
+; CHECK-DSP-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-NEXT:    ldrh.w r12, [r2]
+; CHECK-DSP-NEXT:    ldr r2, [sp, #12]
+; CHECK-DSP-NEXT:    sxth.w lr, r1
+; CHECK-DSP-NEXT:    and r1, r1, #1
+; CHECK-DSP-NEXT:    str.w lr, [r2]
+; CHECK-DSP-NEXT:    ldr r2, [sp, #8]
+; CHECK-DSP-NEXT:    strb r1, [r2]
+; CHECK-DSP-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-NEXT:    muls r0, r1, r0
+; CHECK-DSP-NEXT:    uxtb r0, r0
+; CHECK-DSP-NEXT:    str r0, [r3]
+; CHECK-DSP-NEXT:    orr.w r0, r0, r12
+; CHECK-DSP-NEXT:    uxth r0, r0
+; CHECK-DSP-NEXT:    clz r0, r0
+; CHECK-DSP-NEXT:    lsrs r0, r0, #5
+; CHECK-DSP-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-IMM-LABEL: dont_replace_trunc_1:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    push {r4, lr}
+; CHECK-DSP-IMM-NEXT:    ldrd lr, r12, [sp, #8]
+; CHECK-DSP-IMM-NEXT:    ldrh r1, [r1]
+; CHECK-DSP-IMM-NEXT:    ldrh r2, [r2]
+; CHECK-DSP-IMM-NEXT:    sxth r4, r1
+; CHECK-DSP-IMM-NEXT:    str.w r4, [r12]
+; CHECK-DSP-IMM-NEXT:    and r1, r1, #1
+; CHECK-DSP-IMM-NEXT:    strb.w r1, [lr]
+; CHECK-DSP-IMM-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-IMM-NEXT:    muls r0, r1, r0
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r0
+; CHECK-DSP-IMM-NEXT:    str r0, [r3]
+; CHECK-DSP-IMM-NEXT:    orrs r0, r2
+; CHECK-DSP-IMM-NEXT:    uxth r0, r0
+; CHECK-DSP-IMM-NEXT:    clz r0, r0
+; CHECK-DSP-IMM-NEXT:    lsrs r0, r0, #5
+; CHECK-DSP-IMM-NEXT:    pop {r4, pc}
 entry:
   %0 = load i16, i16* %c, align 2
   %1 = load i16, i16* %b, align 2
@@ -607,10 +1887,55 @@ entry:
   ret i1 %tobool
 }
 
-; CHECK-LABEL: dont_replace_trunc_2
-; CHECK: cmp
-; CHECK: uxtb
 define i32 @dont_replace_trunc_2(i16* %a, i8* %b) {
+; CHECK-NODSP-V8-LABEL: dont_replace_trunc_2:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    cmp r0, #8
+; CHECK-NODSP-V8-NEXT:    it ls
+; CHECK-NODSP-V8-NEXT:    movls r0, #0
+; CHECK-NODSP-V8-NEXT:    ldrb r2, [r1]
+; CHECK-NODSP-V8-NEXT:    uxtb r0, r0
+; CHECK-NODSP-V8-NEXT:    orrs r0, r2
+; CHECK-NODSP-V8-NEXT:    strb r0, [r1]
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: dont_replace_trunc_2:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V7-NEXT:    ldrb r2, [r1]
+; CHECK-NODSP-V7-NEXT:    cmp r0, #8
+; CHECK-NODSP-V7-NEXT:    it ls
+; CHECK-NODSP-V7-NEXT:    movls r0, #0
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r0
+; CHECK-NODSP-V7-NEXT:    orrs r0, r2
+; CHECK-NODSP-V7-NEXT:    strb r0, [r1]
+; CHECK-NODSP-V7-NEXT:    bx lr
+;
+; CHECK-DSP-LABEL: dont_replace_trunc_2:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    cmp r0, #8
+; CHECK-DSP-NEXT:    it ls
+; CHECK-DSP-NEXT:    movls r0, #0
+; CHECK-DSP-NEXT:    ldrb r2, [r1]
+; CHECK-DSP-NEXT:    uxtb r0, r0
+; CHECK-DSP-NEXT:    orrs r0, r2
+; CHECK-DSP-NEXT:    strb r0, [r1]
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: dont_replace_trunc_2:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    movs r2, #0
+; CHECK-DSP-IMM-NEXT:    ldrb r3, [r1]
+; CHECK-DSP-IMM-NEXT:    cmp r0, #8
+; CHECK-DSP-IMM-NEXT:    it hi
+; CHECK-DSP-IMM-NEXT:    movhi r2, r0
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r2
+; CHECK-DSP-IMM-NEXT:    orrs r0, r3
+; CHECK-DSP-IMM-NEXT:    strb r0, [r1]
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i16, i16* %a, align 2
   %cmp = icmp ugt i16 %0, 8
@@ -623,10 +1948,77 @@ entry:
   ret i32 %conv5
 }
 
-; CHECK-LABEL: replace_trunk_with_mask
-; CHECK: div
-; CHECK: uxtb
 define i32 @replace_trunk_with_mask(i16* %a) {
+; CHECK-NODSP-V8-LABEL: replace_trunk_with_mask:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    cmp r0, #0
+; CHECK-NODSP-V8-NEXT:    itt eq
+; CHECK-NODSP-V8-NEXT:    moveq r0, #0
+; CHECK-NODSP-V8-NEXT:    bxeq lr
+; CHECK-NODSP-V8-NEXT:    movw r1, #535
+; CHECK-NODSP-V8-NEXT:    udiv r2, r1, r0
+; CHECK-NODSP-V8-NEXT:    mls r0, r2, r0, r1
+; CHECK-NODSP-V8-NEXT:    movw r1, #43691
+; CHECK-NODSP-V8-NEXT:    uxtb r0, r0
+; CHECK-NODSP-V8-NEXT:    movt r1, #43690
+; CHECK-NODSP-V8-NEXT:    umull r0, r1, r0, r1
+; CHECK-NODSP-V8-NEXT:    lsrs r0, r1, #1
+; CHECK-NODSP-V8-NEXT:    bx lr
+;
+; CHECK-NODSP-V7-LABEL: replace_trunk_with_mask:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    .save {r7, lr}
+; CHECK-NODSP-V7-NEXT:    push {r7, lr}
+; CHECK-NODSP-V7-NEXT:    ldrh r1, [r0]
+; CHECK-NODSP-V7-NEXT:    cbz r1, .LBB28_2
+; CHECK-NODSP-V7-NEXT:  @ %bb.1: @ %cond.false
+; CHECK-NODSP-V7-NEXT:    movw r0, #535
+; CHECK-NODSP-V7-NEXT:    bl __aeabi_uidivmod
+; CHECK-NODSP-V7-NEXT:    uxtb r0, r1
+; CHECK-NODSP-V7-NEXT:    movw r1, #43691
+; CHECK-NODSP-V7-NEXT:    movt r1, #43690
+; CHECK-NODSP-V7-NEXT:    umull r0, r1, r0, r1
+; CHECK-NODSP-V7-NEXT:    lsrs r0, r1, #1
+; CHECK-NODSP-V7-NEXT:    pop {r7, pc}
+; CHECK-NODSP-V7-NEXT:  .LBB28_2:
+; CHECK-NODSP-V7-NEXT:    movs r0, #0
+; CHECK-NODSP-V7-NEXT:    pop {r7, pc}
+;
+; CHECK-DSP-LABEL: replace_trunk_with_mask:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    cmp r0, #0
+; CHECK-DSP-NEXT:    itt eq
+; CHECK-DSP-NEXT:    moveq r0, #0
+; CHECK-DSP-NEXT:    bxeq lr
+; CHECK-DSP-NEXT:    movw r1, #535
+; CHECK-DSP-NEXT:    udiv r2, r1, r0
+; CHECK-DSP-NEXT:    mls r0, r2, r0, r1
+; CHECK-DSP-NEXT:    movw r1, #43691
+; CHECK-DSP-NEXT:    uxtb r0, r0
+; CHECK-DSP-NEXT:    movt r1, #43690
+; CHECK-DSP-NEXT:    umull r0, r1, r0, r1
+; CHECK-DSP-NEXT:    lsrs r0, r1, #1
+; CHECK-DSP-NEXT:    bx lr
+;
+; CHECK-DSP-IMM-LABEL: replace_trunk_with_mask:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    cbz r0, .LBB28_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1: @ %cond.false
+; CHECK-DSP-IMM-NEXT:    movw r1, #535
+; CHECK-DSP-IMM-NEXT:    udiv r2, r1, r0
+; CHECK-DSP-IMM-NEXT:    mls r0, r2, r0, r1
+; CHECK-DSP-IMM-NEXT:    movw r1, #43691
+; CHECK-DSP-IMM-NEXT:    movt r1, #43690
+; CHECK-DSP-IMM-NEXT:    uxtb r0, r0
+; CHECK-DSP-IMM-NEXT:    umull r0, r1, r0, r1
+; CHECK-DSP-IMM-NEXT:    lsrs r0, r1, #1
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB28_2:
+; CHECK-DSP-IMM-NEXT:    movs r0, #0
+; CHECK-DSP-IMM-NEXT:    bx lr
 entry:
   %0 = load i16, i16* %a
   %cmp = icmp eq i16 %0, 0
@@ -644,12 +2036,94 @@ cond.end:
   ret i32 %cond
 }
 
-; CHECK-LABEL: test_i8_sitofp
-; CHECK: uxtb [[UXT:r[0-9]+]], r1
-; CHECK: sxtb [[SXT:r[0-9]+]], r1
-; CHECK: vmov [[VMOV:s[0-9]+]], [[SXT]]
-; CHECK: vcvt.f32.s32 [[CVT:s[0-9]+]], [[VMOV]]
 define float @test_i8_sitofp(i8* %ptr, i8 %arg) {
+; CHECK-NODSP-V8-LABEL: test_i8_sitofp:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V8-NEXT:    uxtb r2, r1
+; CHECK-NODSP-V8-NEXT:    cmp r0, r2
+; CHECK-NODSP-V8-NEXT:    bne .LBB29_2
+; CHECK-NODSP-V8-NEXT:  @ %bb.1:
+; CHECK-NODSP-V8-NEXT:    vldr s0, .LCPI29_0
+; CHECK-NODSP-V8-NEXT:    vmov r0, s0
+; CHECK-NODSP-V8-NEXT:    bx lr
+; CHECK-NODSP-V8-NEXT:  .LBB29_2: @ %if.end
+; CHECK-NODSP-V8-NEXT:    sxtb r0, r1
+; CHECK-NODSP-V8-NEXT:    vmov s0, r0
+; CHECK-NODSP-V8-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NODSP-V8-NEXT:    vmov.f32 s2, #2.000000e+01
+; CHECK-NODSP-V8-NEXT:    vdiv.f32 s0, s0, s2
+; CHECK-NODSP-V8-NEXT:    vmov r0, s0
+; CHECK-NODSP-V8-NEXT:    bx lr
+; CHECK-NODSP-V8-NEXT:    .p2align 2
+; CHECK-NODSP-V8-NEXT:  @ %bb.3:
+; CHECK-NODSP-V8-NEXT:  .LCPI29_0:
+; CHECK-NODSP-V8-NEXT:    .long 0 @ float 0
+;
+; CHECK-NODSP-V7-LABEL: test_i8_sitofp:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrb r0, [r0]
+; CHECK-NODSP-V7-NEXT:    uxtb r2, r1
+; CHECK-NODSP-V7-NEXT:    cmp r0, r2
+; CHECK-NODSP-V7-NEXT:    ittt eq
+; CHECK-NODSP-V7-NEXT:    vldreq s0, .LCPI29_0
+; CHECK-NODSP-V7-NEXT:    vmoveq r0, s0
+; CHECK-NODSP-V7-NEXT:    bxeq lr
+; CHECK-NODSP-V7-NEXT:    sxtb r0, r1
+; CHECK-NODSP-V7-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-NODSP-V7-NEXT:    vmov s2, r0
+; CHECK-NODSP-V7-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-NODSP-V7-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-NODSP-V7-NEXT:    vmov r0, s0
+; CHECK-NODSP-V7-NEXT:    bx lr
+; CHECK-NODSP-V7-NEXT:    .p2align 2
+; CHECK-NODSP-V7-NEXT:  @ %bb.1:
+; CHECK-NODSP-V7-NEXT:  .LCPI29_0:
+; CHECK-NODSP-V7-NEXT:    .long 0 @ float 0
+;
+; CHECK-DSP-LABEL: test_i8_sitofp:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-NEXT:    uxtb r2, r1
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    ittt eq
+; CHECK-DSP-NEXT:    vldreq s0, .LCPI29_0
+; CHECK-DSP-NEXT:    vmoveq r0, s0
+; CHECK-DSP-NEXT:    bxeq lr
+; CHECK-DSP-NEXT:    sxtb r0, r1
+; CHECK-DSP-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-DSP-NEXT:    vmov s2, r0
+; CHECK-DSP-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-DSP-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-DSP-NEXT:    vmov r0, s0
+; CHECK-DSP-NEXT:    bx lr
+; CHECK-DSP-NEXT:    .p2align 2
+; CHECK-DSP-NEXT:  @ %bb.1:
+; CHECK-DSP-NEXT:  .LCPI29_0:
+; CHECK-DSP-NEXT:    .long 0 @ float 0
+;
+; CHECK-DSP-IMM-LABEL: test_i8_sitofp:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrb r0, [r0]
+; CHECK-DSP-IMM-NEXT:    uxtb r2, r1
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    bne .LBB29_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1:
+; CHECK-DSP-IMM-NEXT:    vldr s0, .LCPI29_0
+; CHECK-DSP-IMM-NEXT:    vmov r0, s0
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB29_2: @ %if.end
+; CHECK-DSP-IMM-NEXT:    sxtb r0, r1
+; CHECK-DSP-IMM-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-DSP-IMM-NEXT:    vmov s2, r0
+; CHECK-DSP-IMM-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-DSP-IMM-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-DSP-IMM-NEXT:    vmov r0, s0
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:    .p2align 2
+; CHECK-DSP-IMM-NEXT:  @ %bb.3:
+; CHECK-DSP-IMM-NEXT:  .LCPI29_0:
+; CHECK-DSP-IMM-NEXT:    .long 0 @ float 0
 entry:
   %0 = load i8, i8* %ptr, align 1
    %cmp = icmp eq i8 %0, %arg
@@ -665,12 +2139,94 @@ exit:
   ret float %res
 }
 
-; CHECK-LABEL: test_i16_sitofp
-; CHECK: uxth [[UXT:r[0-9]+]], r1
-; CHECK: sxth [[SXT:r[0-9]+]], r1
-; CHECK: vmov [[VMOV:s[0-9]+]], [[SXT]]
-; CHECK: vcvt.f32.s32 [[CVT:s[0-9]+]], [[VMOV]]
 define float @test_i16_sitofp(i16* %ptr, i16 %arg) {
+; CHECK-NODSP-V8-LABEL: test_i16_sitofp:
+; CHECK-NODSP-V8:       @ %bb.0: @ %entry
+; CHECK-NODSP-V8-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V8-NEXT:    uxth r2, r1
+; CHECK-NODSP-V8-NEXT:    cmp r0, r2
+; CHECK-NODSP-V8-NEXT:    bne .LBB30_2
+; CHECK-NODSP-V8-NEXT:  @ %bb.1:
+; CHECK-NODSP-V8-NEXT:    vldr s0, .LCPI30_0
+; CHECK-NODSP-V8-NEXT:    vmov r0, s0
+; CHECK-NODSP-V8-NEXT:    bx lr
+; CHECK-NODSP-V8-NEXT:  .LBB30_2: @ %if.end
+; CHECK-NODSP-V8-NEXT:    sxth r0, r1
+; CHECK-NODSP-V8-NEXT:    vmov s0, r0
+; CHECK-NODSP-V8-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NODSP-V8-NEXT:    vmov.f32 s2, #2.000000e+01
+; CHECK-NODSP-V8-NEXT:    vdiv.f32 s0, s0, s2
+; CHECK-NODSP-V8-NEXT:    vmov r0, s0
+; CHECK-NODSP-V8-NEXT:    bx lr
+; CHECK-NODSP-V8-NEXT:    .p2align 2
+; CHECK-NODSP-V8-NEXT:  @ %bb.3:
+; CHECK-NODSP-V8-NEXT:  .LCPI30_0:
+; CHECK-NODSP-V8-NEXT:    .long 0 @ float 0
+;
+; CHECK-NODSP-V7-LABEL: test_i16_sitofp:
+; CHECK-NODSP-V7:       @ %bb.0: @ %entry
+; CHECK-NODSP-V7-NEXT:    ldrh r0, [r0]
+; CHECK-NODSP-V7-NEXT:    uxth r2, r1
+; CHECK-NODSP-V7-NEXT:    cmp r0, r2
+; CHECK-NODSP-V7-NEXT:    ittt eq
+; CHECK-NODSP-V7-NEXT:    vldreq s0, .LCPI30_0
+; CHECK-NODSP-V7-NEXT:    vmoveq r0, s0
+; CHECK-NODSP-V7-NEXT:    bxeq lr
+; CHECK-NODSP-V7-NEXT:    sxth r0, r1
+; CHECK-NODSP-V7-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-NODSP-V7-NEXT:    vmov s2, r0
+; CHECK-NODSP-V7-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-NODSP-V7-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-NODSP-V7-NEXT:    vmov r0, s0
+; CHECK-NODSP-V7-NEXT:    bx lr
+; CHECK-NODSP-V7-NEXT:    .p2align 2
+; CHECK-NODSP-V7-NEXT:  @ %bb.1:
+; CHECK-NODSP-V7-NEXT:  .LCPI30_0:
+; CHECK-NODSP-V7-NEXT:    .long 0 @ float 0
+;
+; CHECK-DSP-LABEL: test_i16_sitofp:
+; CHECK-DSP:       @ %bb.0: @ %entry
+; CHECK-DSP-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-NEXT:    uxth r2, r1
+; CHECK-DSP-NEXT:    cmp r0, r2
+; CHECK-DSP-NEXT:    ittt eq
+; CHECK-DSP-NEXT:    vldreq s0, .LCPI30_0
+; CHECK-DSP-NEXT:    vmoveq r0, s0
+; CHECK-DSP-NEXT:    bxeq lr
+; CHECK-DSP-NEXT:    sxth r0, r1
+; CHECK-DSP-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-DSP-NEXT:    vmov s2, r0
+; CHECK-DSP-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-DSP-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-DSP-NEXT:    vmov r0, s0
+; CHECK-DSP-NEXT:    bx lr
+; CHECK-DSP-NEXT:    .p2align 2
+; CHECK-DSP-NEXT:  @ %bb.1:
+; CHECK-DSP-NEXT:  .LCPI30_0:
+; CHECK-DSP-NEXT:    .long 0 @ float 0
+;
+; CHECK-DSP-IMM-LABEL: test_i16_sitofp:
+; CHECK-DSP-IMM:       @ %bb.0: @ %entry
+; CHECK-DSP-IMM-NEXT:    ldrh r0, [r0]
+; CHECK-DSP-IMM-NEXT:    uxth r2, r1
+; CHECK-DSP-IMM-NEXT:    cmp r0, r2
+; CHECK-DSP-IMM-NEXT:    bne .LBB30_2
+; CHECK-DSP-IMM-NEXT:  @ %bb.1:
+; CHECK-DSP-IMM-NEXT:    vldr s0, .LCPI30_0
+; CHECK-DSP-IMM-NEXT:    vmov r0, s0
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:  .LBB30_2: @ %if.end
+; CHECK-DSP-IMM-NEXT:    sxth r0, r1
+; CHECK-DSP-IMM-NEXT:    vmov.f32 s0, #2.000000e+01
+; CHECK-DSP-IMM-NEXT:    vmov s2, r0
+; CHECK-DSP-IMM-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-DSP-IMM-NEXT:    vdiv.f32 s0, s2, s0
+; CHECK-DSP-IMM-NEXT:    vmov r0, s0
+; CHECK-DSP-IMM-NEXT:    bx lr
+; CHECK-DSP-IMM-NEXT:    .p2align 2
+; CHECK-DSP-IMM-NEXT:  @ %bb.3:
+; CHECK-DSP-IMM-NEXT:  .LCPI30_0:
+; CHECK-DSP-IMM-NEXT:    .long 0 @ float 0
 entry:
   %0 = load i16, i16* %ptr, align 1
    %cmp = icmp eq i16 %0, %arg
diff --git a/llvm/test/CodeGen/ARM/CGP/arm-cgp-overflow.ll b/llvm/test/CodeGen/ARM/CGP/arm-cgp-overflow.ll
index 4873b8836502a..c446ddbdd07a7 100644
--- a/llvm/test/CodeGen/ARM/CGP/arm-cgp-overflow.ll
+++ b/llvm/test/CodeGen/ARM/CGP/arm-cgp-overflow.ll
@@ -264,10 +264,9 @@ define i8 @underflow_if_sub(i32 %arg, i8 zeroext %arg1) {
 }
 
 ; CHECK-LABEL: underflow_if_sub_signext
-; CHECK: uxtb [[UXT1:r[0-9]+]], r1
-; CHECK: sub{{.*}} [[SUB:r[0-9]+]], #11
-; CHECK: uxtb [[UXT_SUB:r[0-9]+]], [[SUB]]
-; CHECK: cmp{{.*}}[[UXT_SUB]]
+; CHECK:      cmp r0, #0
+; CHECK-NEXT: uxtb  r1, r1
+; CHECK-NOT:  xtb
 define i8 @underflow_if_sub_signext(i32 %arg, i8 signext %arg1) {
   %cmp = icmp sgt i32 %arg, 0
   %conv = zext i1 %cmp to i32
diff --git a/llvm/test/CodeGen/ARM/CGP/arm-cgp-phis-ret.ll b/llvm/test/CodeGen/ARM/CGP/arm-cgp-phis-ret.ll
index e7adc5ae2491b..9b07a80e9a1c1 100644
--- a/llvm/test/CodeGen/ARM/CGP/arm-cgp-phis-ret.ll
+++ b/llvm/test/CodeGen/ARM/CGP/arm-cgp-phis-ret.ll
@@ -184,3 +184,35 @@ define i16 @promote_arg_return(i16 zeroext %arg1, i16 zeroext %arg2, i8* %res) {
   store i8 %conv, i8* %res
   ret i16 %arg1
 }
+
+; CHECK-COMMON-LABEL: signext_bitcast_phi_select
+; CHECK: uxth [[UXT:r[0-9]+]], r0
+; CHECK: sxth [[SXT:r[0-9]+]], [[UXT]]
+; CHECK: cmp [[SXT]],
+; CHECK-NOT: xth
+define i16 @signext_bitcast_phi_select(i16 signext %start, i16* %in) {
+entry:
+  %const = bitcast i16 -1 to i16
+  br label %for.body
+
+for.body:
+  %idx = phi i16 [ %select, %if.else ], [ %start, %entry ]
+  %cmp.i = icmp sgt i16 %idx, %const
+  br i1 %cmp.i, label %exit, label %if.then
+
+if.then:
+  %idx.next = getelementptr i16, i16* %in, i16 %idx
+  %ld = load i16, i16* %idx.next, align 2
+  %cmp1.i = icmp eq i16 %ld, %idx
+  br i1 %cmp1.i, label %exit, label %if.else
+
+if.else:
+  %lobit = lshr i16 %idx, 15
+  %lobit.not = xor i16 %lobit, 1
+  %select = add nuw i16 %lobit.not, %idx
+  br label %for.body
+
+exit:
+  %res = phi i16 [ %ld, %if.then ], [ 0, %for.body ]
+  ret i16 %res
+}
diff --git a/llvm/test/CodeGen/ARM/CGP/arm-cgp-signed.ll b/llvm/test/CodeGen/ARM/CGP/arm-cgp-signed.ll
index 44f3829c6b442..596893724d203 100644
--- a/llvm/test/CodeGen/ARM/CGP/arm-cgp-signed.ll
+++ b/llvm/test/CodeGen/ARM/CGP/arm-cgp-signed.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=thumbv7m -arm-disable-cgp=false %s -o - | FileCheck %s
-; RUN: llc -mtriple=thumbv8m.main -arm-disable-cgp=false %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv7em -arm-disable-cgp=false %s -o - | FileCheck %s
+; RUN: llc -mtriple=thumbv8m.main -mattr=+dsp -arm-disable-cgp=false %s -o - | FileCheck %s
 ; RUN: llc -mtriple=thumbv7 %s -arm-disable-cgp=false -o - | FileCheck %s
 ; RUN: llc -mtriple=armv8 %s -arm-disable-cgp=false -o - | FileCheck %s
 
@@ -45,8 +45,8 @@ define i16 @test_srem(i16 zeroext %arg) {
 
 ; CHECK-LABEL: test_signext_b
 ; CHECK: ldrb [[LDR:r[0-9]+]], [r0]
-; CHECK: sxtb [[SXT:r[0-9]+]], [[LDR]]
-; CHECK: cm{{.*}} [[SXT]]
+; CHECK: uxtab [[UXT:r[0-9]+]], [[LDR]], r1
+; CHECK: cm{{.*}} [[UXT]], #128
 define i32 @test_signext_b(i8* %ptr, i8 signext %arg) {
 entry:
   %0 = load i8, i8* %ptr, align 1
@@ -56,10 +56,28 @@ entry:
   ret i32 %res
 }
 
+; CHECK-LABEL: test_signext_b_ult_slt
+; CHECK: ldrb [[LDR:r[0-9]+]], [r0]
+; CHECK: uxtab [[ADD:r[0-9]+]], [[LDR]], r1
+; CHECK: uxtb [[UXT:r[0-9]+]], r1
+; CHECK: cmp [[ADD]], [[UXT]]
+; CHECK: uxtb [[TRUNC:r[0-9]+]], [[ADD]]
+; CHECK: cmp [[TRUNC]], #127
+define i32 @test_signext_b_ult_slt(i8* %ptr, i8 signext %arg) {
+entry:
+  %0 = load i8, i8* %ptr, align 1
+  %1 = add nuw nsw i8 %0, %arg
+  %cmp = icmp sle i8 %1, 126
+  %cmp.1 = icmp ule i8 %1, %arg
+  %or = and i1 %cmp, %cmp.1
+  %res = select i1 %or, i32 42, i32 57
+  ret i32 %res
+}
+
 ; CHECK-LABEL: test_signext_h
 ; CHECK: ldrh [[LDR:r[0-9]+]], [r0]
-; CHECK: sxth [[SXT:r[0-9]+]], [[LDR]]
-; CHECK: cm{{.*}} [[SXT]]
+; CHECK: uxtah [[ADD:r[0-9]+]], [[LDR]], r1
+; CHECK: cm{{.*}} [[ADD]],
 define i32 @test_signext_h(i16* %ptr, i16 signext %arg) {
 entry:
   %0 = load i16, i16* %ptr, align 1
@@ -68,3 +86,4 @@ entry:
   %res = select i1 %cmp, i32 42, i32 20894
   ret i32 %res
 }
+
diff --git a/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
index 5b78cd5b6fbaa..ca8ab95cfa0d3 100644
--- a/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
+++ b/llvm/test/CodeGen/ARM/GlobalISel/arm-legalize-load-store.mir
@@ -11,7 +11,7 @@
   define void @test_gep_s16() { ret void }
 
   attributes #0 = { "target-features"="+vfp2" }
-  attributes #1 = { "target-features"="-vfp2d16sp" }
+  attributes #1 = { "target-features"="-vfp2sp" }
 ...
 ---
 name:            test_legal_loads_stores
diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll
index 7650d682736a7..cbfc1e86a362e 100644
--- a/llvm/test/CodeGen/ARM/O3-pipeline.ll
+++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll
@@ -74,6 +74,7 @@
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
 ; CHECK-NEXT:      Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Common Subexpression Elimination
 ; CHECK-NEXT:      MachinePostDominator Tree Construction
@@ -112,6 +113,7 @@
 ; CHECK-NEXT:      Machine Loop Invariant Code Motion
 ; CHECK-NEXT:      PostRA Machine Sink
 ; CHECK-NEXT:      Machine Block Frequency Analysis
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      MachinePostDominator Tree Construction
 ; CHECK-NEXT:      Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:      Machine Optimization Remark Emitter
@@ -143,6 +145,7 @@
 ; CHECK-NEXT:      Thumb2 instruction size reduce pass
 ; CHECK-NEXT:      Unpack machine instruction bundles
 ; CHECK-NEXT:      optimise barriers pass
+; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      ARM constant island placement and branch shortening pass
 ; CHECK-NEXT:      MachineDominator Tree Construction
 ; CHECK-NEXT:      Machine Natural Loop Construction
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll b/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
index 669972cb99dca..68702b722c759 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/complex_dot_prod.ll
@@ -8,13 +8,13 @@
 ; CHECK: smultt
 ; CHECK: smlalbb
 ; CHECK: smultt
-; CHECK: smlaldx
 ; CHECK: smlalbb
-; CHECK: smlaldx
 ; CHECK: smultt
 ; CHECK: smlalbb
-; CHECK: smlaldx
 ; CHECK: smultt
+; CHECK: smlaldx
+; CHECK: smlaldx
+; CHECK: smlaldx
 ; CHECK: pop.w	{r4, r5, r6, r7, r8, r9, r10, r11, pc}
 define dso_local arm_aapcscc void @complex_dot_prod(i16* nocapture readonly %pSrcA, i16* nocapture readonly %pSrcB, i32* nocapture %realResult, i32* nocapture %imagResult) {
 entry:
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll b/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
index 66170aa40c09f..37e39a0241716 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/multi-use-loads.ll
@@ -19,8 +19,8 @@ define i32 @add_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture rea
 ; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
 ; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
-; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    sxtah r1, r1, lr
+; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
 ; CHECK-LE-NEXT:    bne .LBB0_2
 ; CHECK-LE-NEXT:  @ %bb.3: @ %for.cond.cleanup
@@ -47,13 +47,13 @@ define i32 @add_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture rea
 ; CHECK-BE-NEXT:  .LBB0_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
-; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
-; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
-; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
-; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
 ; CHECK-BE-NEXT:    add r1, lr
+; CHECK-BE-NEXT:    subs r0, #1
+; CHECK-BE-NEXT:    smlabb r12, r6, r4, r5
 ; CHECK-BE-NEXT:    bne .LBB0_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
@@ -154,8 +154,8 @@ define i32 @mul_bottom_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapt
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    mul r1, lr, r1
 ; CHECK-BE-NEXT:    bne .LBB1_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
@@ -215,17 +215,17 @@ define i32 @mul_top_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture
 ; CHECK-LE-NEXT:    cmp r0, #1
 ; CHECK-LE-NEXT:    blt .LBB2_4
 ; CHECK-LE-NEXT:  @ %bb.1: @ %for.body.preheader
-; CHECK-LE-NEXT:    subs r2, #2
+; CHECK-LE-NEXT:    sub.w lr, r2, #2
 ; CHECK-LE-NEXT:    subs r3, #2
 ; CHECK-LE-NEXT:    mov.w r12, #0
 ; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    .p2align 2
 ; CHECK-LE-NEXT:  .LBB2_2: @ %for.body
 ; CHECK-LE-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-LE-NEXT:    ldr r4, [r2, #2]!
-; CHECK-LE-NEXT:    ldr lr, [r3, #2]!
-; CHECK-LE-NEXT:    asrs r5, r4, #16
-; CHECK-LE-NEXT:    smlad r12, r4, lr, r12
+; CHECK-LE-NEXT:    ldr r2, [lr, #2]!
+; CHECK-LE-NEXT:    ldr r4, [r3, #2]!
+; CHECK-LE-NEXT:    asrs r5, r2, #16
+; CHECK-LE-NEXT:    smlad r12, r2, r4, r12
 ; CHECK-LE-NEXT:    subs r0, #1
 ; CHECK-LE-NEXT:    mul r1, r5, r1
 ; CHECK-LE-NEXT:    bne .LBB2_2
@@ -257,8 +257,8 @@ define i32 @mul_top_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    mul r1, r6, r1
 ; CHECK-BE-NEXT:    bne .LBB2_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
@@ -343,8 +343,8 @@ define i32 @and_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture rea
 ;
 ; CHECK-BE-LABEL: and_user:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r5, r6, lr}
-; CHECK-BE-NEXT:    push {r4, r5, r6, lr}
+; CHECK-BE-NEXT:    .save {r4, r5, r6, r7, lr}
+; CHECK-BE-NEXT:    push {r4, r5, r6, r7, lr}
 ; CHECK-BE-NEXT:    cmp r0, #1
 ; CHECK-BE-NEXT:    blt .LBB3_4
 ; CHECK-BE-NEXT:  @ %bb.1: @ %for.body.preheader
@@ -356,23 +356,23 @@ define i32 @and_user(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture rea
 ; CHECK-BE-NEXT:  .LBB3_2: @ %for.body
 ; CHECK-BE-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-BE-NEXT:    ldrsh lr, [r3, #2]!
-; CHECK-BE-NEXT:    ldrsh r4, [r2, #2]!
-; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
-; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
-; CHECK-BE-NEXT:    smlabb r4, r4, lr, r12
-; CHECK-BE-NEXT:    uxth.w lr, lr
-; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
+; CHECK-BE-NEXT:    ldrsh r5, [r2, #2]!
+; CHECK-BE-NEXT:    ldrsh.w r4, [r3, #2]
+; CHECK-BE-NEXT:    ldrsh.w r7, [r2, #2]
+; CHECK-BE-NEXT:    uxth.w r6, lr
+; CHECK-BE-NEXT:    smlabb r5, r5, lr, r12
+; CHECK-BE-NEXT:    smlabb r12, r7, r4, r5
 ; CHECK-BE-NEXT:    subs r0, #1
-; CHECK-BE-NEXT:    mul r1, lr, r1
+; CHECK-BE-NEXT:    mul r1, r6, r1
 ; CHECK-BE-NEXT:    bne .LBB3_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
-; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
 ; CHECK-BE-NEXT:  .LBB3_4:
 ; CHECK-BE-NEXT:    mov.w r12, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    add.w r0, r12, r1
-; CHECK-BE-NEXT:    pop {r4, r5, r6, pc}
+; CHECK-BE-NEXT:    pop {r4, r5, r6, r7, pc}
 entry:
   %cmp24 = icmp sgt i32 %arg, 0
   br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
@@ -466,10 +466,10 @@ define i32 @multi_uses(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture r
 ; CHECK-BE-NEXT:    ldrsh.w r5, [r3, #2]
 ; CHECK-BE-NEXT:    ldrsh.w r6, [r2, #2]
 ; CHECK-BE-NEXT:    smlabb r4, r4, r1, r12
-; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    smlabb r12, r6, r5, r4
 ; CHECK-BE-NEXT:    eor.w r6, r1, lr
-; CHECK-BE-NEXT:    mul r1, r6, r1
+; CHECK-BE-NEXT:    muls r1, r6, r1
+; CHECK-BE-NEXT:    subs r0, #1
 ; CHECK-BE-NEXT:    lsl.w lr, r1, #16
 ; CHECK-BE-NEXT:    bne .LBB4_2
 ; CHECK-BE-NEXT:  @ %bb.3: @ %for.cond.cleanup
diff --git a/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll b/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
index 83418dcb15a06..c72d458453798 100644
--- a/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
+++ b/llvm/test/CodeGen/ARM/ParallelDSP/unroll-n-jam-smlad.ll
@@ -46,7 +46,6 @@ entry:
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
-; CHECK-REG-PRESSURE: ldr{{.*}}, [sp
 ; CHECK-REG-PRESSURE: bne .LBB0_1
 
 for.body:
diff --git a/llvm/test/CodeGen/ARM/a15-partial-update.ll b/llvm/test/CodeGen/ARM/a15-partial-update.ll
index c20aaa2fa5dd8..a3971d6a034bf 100644
--- a/llvm/test/CodeGen/ARM/a15-partial-update.ll
+++ b/llvm/test/CodeGen/ARM/a15-partial-update.ll
@@ -56,6 +56,11 @@ ret:
   ret void
 }
 
+; If minimizing size, that overrides perf, so no extra vmov.f64 here.
+
+; TODO: This (and above) could use a splat load to remove the false
+;       dependence with no extra instruction.
+
 define void @t2_minsize(<4 x i8> *%in, <4 x i8> *%out, i32 %n) minsize {
 ; CHECK-LABEL: t2_minsize:
 ; CHECK:       @ %bb.0: @ %entry
@@ -63,7 +68,6 @@ define void @t2_minsize(<4 x i8> *%in, <4 x i8> *%out, i32 %n) minsize {
 ; CHECK-NEXT:    add r1, r1, #4
 ; CHECK-NEXT:  .LBB2_1: @ %loop
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vmov.f64 d16, #5.000000e-01
 ; CHECK-NEXT:    vld1.32 {d16[0]}, [r0:32]
 ; CHECK-NEXT:    vmovl.u8 q8, d16
 ; CHECK-NEXT:    vuzp.8 d16, d18
diff --git a/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll b/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
index 2dfd217057356..aaf7189a6bbeb 100644
--- a/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
+++ b/llvm/test/CodeGen/ARM/addsubcarry-promotion.ll
@@ -10,11 +10,11 @@
 define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ; ARM-LABEL: fn1:
 ; ARM:       @ %bb.0: @ %entry
+; ARM-NEXT:    rsb r2, r2, #0
 ; ARM-NEXT:    adds r0, r1, r0
-; ARM-NEXT:    mov r3, #0
-; ARM-NEXT:    adc r0, r3, #0
 ; ARM-NEXT:    movw r1, #65535
-; ARM-NEXT:    sub r0, r0, r2
+; ARM-NEXT:    sxth r2, r2
+; ARM-NEXT:    adc r0, r2, #0
 ; ARM-NEXT:    uxth r0, r0
 ; ARM-NEXT:    cmp r0, r1
 ; ARM-NEXT:    bxeq lr
@@ -24,11 +24,12 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMBV6M-LABEL: fn1:
 ; THUMBV6M:       @ %bb.0: @ %entry
+; THUMBV6M-NEXT:    rsbs r2, r2, #0
+; THUMBV6M-NEXT:    sxth r2, r2
 ; THUMBV6M-NEXT:    movs r3, #0
 ; THUMBV6M-NEXT:    adds r0, r1, r0
-; THUMBV6M-NEXT:    adcs r3, r3
-; THUMBV6M-NEXT:    subs r0, r3, r2
-; THUMBV6M-NEXT:    uxth r0, r0
+; THUMBV6M-NEXT:    adcs r3, r2
+; THUMBV6M-NEXT:    uxth r0, r3
 ; THUMBV6M-NEXT:    ldr r1, .LCPI0_0
 ; THUMBV6M-NEXT:    cmp r0, r1
 ; THUMBV6M-NEXT:    beq .LBB0_2
@@ -44,11 +45,12 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMBV8M-BASE-LABEL: fn1:
 ; THUMBV8M-BASE:       @ %bb.0: @ %entry
+; THUMBV8M-BASE-NEXT:    rsbs r2, r2, #0
+; THUMBV8M-BASE-NEXT:    sxth r2, r2
 ; THUMBV8M-BASE-NEXT:    movs r3, #0
 ; THUMBV8M-BASE-NEXT:    adds r0, r1, r0
-; THUMBV8M-BASE-NEXT:    adcs r3, r3
-; THUMBV8M-BASE-NEXT:    subs r0, r3, r2
-; THUMBV8M-BASE-NEXT:    uxth r0, r0
+; THUMBV8M-BASE-NEXT:    adcs r3, r2
+; THUMBV8M-BASE-NEXT:    uxth r0, r3
 ; THUMBV8M-BASE-NEXT:    movw r1, #65535
 ; THUMBV8M-BASE-NEXT:    cmp r0, r1
 ; THUMBV8M-BASE-NEXT:    beq .LBB0_2
@@ -60,11 +62,11 @@ define void @fn1(i32 %a, i32 %b, i32 %c) local_unnamed_addr #0 {
 ;
 ; THUMB-LABEL: fn1:
 ; THUMB:       @ %bb.0: @ %entry
+; THUMB-NEXT:    rsbs r2, r2, #0
 ; THUMB-NEXT:    adds r0, r0, r1
-; THUMB-NEXT:    mov.w r3, #0
-; THUMB-NEXT:    adc r0, r3, #0
 ; THUMB-NEXT:    movw r1, #65535
-; THUMB-NEXT:    subs r0, r0, r2
+; THUMB-NEXT:    sxth r2, r2
+; THUMB-NEXT:    adc r0, r2, #0
 ; THUMB-NEXT:    uxth r0, r0
 ; THUMB-NEXT:    cmp r0, r1
 ; THUMB-NEXT:    it eq
diff --git a/llvm/test/CodeGen/ARM/addsubo-legalization.ll b/llvm/test/CodeGen/ARM/addsubo-legalization.ll
index c899e5f73ead3..e9143d814d3de 100644
--- a/llvm/test/CodeGen/ARM/addsubo-legalization.ll
+++ b/llvm/test/CodeGen/ARM/addsubo-legalization.ll
@@ -95,76 +95,48 @@ define <2 x i1> @usubo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 define <2 x i1> @saddo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 ; CHECK-LABEL: saddo:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
-; CHECK-NEXT:    vld1.64 {d16, d17}, [r1]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    movs r6, #0
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    vld1.64 {d20, d21}, [r0]
 ; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vmov.32 r1, d16[1]
-; CHECK-NEXT:    vld1.64 {d18, d19}, [r0]
-; CHECK-NEXT:    vmov.32 r2, d17[1]
-; CHECK-NEXT:    vadd.i64 q8, q9, q8
-; CHECK-NEXT:    vmov.32 r12, d18[1]
-; CHECK-NEXT:    vmov.32 r4, d19[1]
-; CHECK-NEXT:    vmov.32 lr, d16[1]
-; CHECK-NEXT:    vmov.32 r7, d17[1]
-; CHECK-NEXT:    cmp.w r1, #-1
+; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
+; CHECK-NEXT:    vadd.i64 q8, q10, q9
+; CHECK-NEXT:    vmov.32 r2, d20[0]
+; CHECK-NEXT:    vmov.32 r1, d20[1]
+; CHECK-NEXT:    vmov.32 r12, d16[0]
+; CHECK-NEXT:    vmov.32 r8, d16[1]
+; CHECK-NEXT:    vmov.32 lr, d17[0]
+; CHECK-NEXT:    vmov.32 r4, d21[0]
+; CHECK-NEXT:    vmov.32 r5, d17[1]
+; CHECK-NEXT:    vmov.32 r6, d18[1]
+; CHECK-NEXT:    vmov.32 r7, d21[1]
+; CHECK-NEXT:    subs.w r2, r12, r2
+; CHECK-NEXT:    vmov.32 r2, d19[1]
+; CHECK-NEXT:    sbcs.w r1, r8, r1
 ; CHECK-NEXT:    mov.w r1, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    cmp.w r2, #-1
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    cmp.w r12, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    cmp.w r4, #-1
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r4, #1
-; CHECK-NEXT:    cmp.w lr, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r6, #1
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    cmp.w r7, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #1
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    subs.w r4, lr, r4
+; CHECK-NEXT:    sbcs.w r7, r5, r7
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
 ; CHECK-NEXT:    cmp r3, #0
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    cmp r4, #0
-; CHECK-NEXT:    vdup.32 d19, r3
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    asrs r7, r6, #31
+; CHECK-NEXT:    vdup.32 d21, r3
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vdup.32 d23, r2
-; CHECK-NEXT:    vdup.32 d21, r4
-; CHECK-NEXT:    vdup.32 d18, r6
-; CHECK-NEXT:    vdup.32 d22, r1
-; CHECK-NEXT:    vdup.32 d20, r5
-; CHECK-NEXT:    vceq.i32 q9, q10, q9
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    vdup.32 d20, r1
 ; CHECK-NEXT:    vst1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    vceq.i32 q10, q10, q11
-; CHECK-NEXT:    vrev64.32 q11, q9
-; CHECK-NEXT:    vrev64.32 q12, q10
-; CHECK-NEXT:    vand q9, q9, q11
-; CHECK-NEXT:    vand q10, q10, q12
-; CHECK-NEXT:    vbic q9, q10, q9
+; CHECK-NEXT:    asrs r2, r2, #31
+; CHECK-NEXT:    vdup.32 d19, r2
+; CHECK-NEXT:    vdup.32 d18, r7
+; CHECK-NEXT:    veor q9, q9, q10
 ; CHECK-NEXT:    vmovn.i64 d18, q9
 ; CHECK-NEXT:    vmov r2, r1, d18
 ; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
   %x = load <2 x i64>, <2 x i64>* %ptr, align 8
   %y = load <2 x i64>, <2 x i64>* %ptr2, align 8
   %s = call {<2 x i64>, <2 x i1>} @llvm.sadd.with.overflow.v2i64(<2 x i64> %x, <2 x i64> %y)
@@ -177,77 +149,64 @@ define <2 x i1> @saddo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 define <2 x i1> @ssubo(<2 x i64> *%ptr, <2 x i64> *%ptr2) {
 ; CHECK-LABEL: ssubo:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
 ; CHECK-NEXT:    vld1.64 {d18, d19}, [r1]
-; CHECK-NEXT:    movs r5, #0
-; CHECK-NEXT:    movs r6, #0
-; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    vld1.64 {d20, d21}, [r0]
 ; CHECK-NEXT:    vsub.i64 q8, q10, q9
+; CHECK-NEXT:    vmov.32 r1, d20[0]
 ; CHECK-NEXT:    vmov.32 r12, d20[1]
-; CHECK-NEXT:    vmov.32 lr, d21[1]
-; CHECK-NEXT:    vmov.32 r1, d16[1]
-; CHECK-NEXT:    vmov.32 r2, d17[1]
-; CHECK-NEXT:    vmov.32 r4, d18[1]
-; CHECK-NEXT:    vmov.32 r7, d19[1]
-; CHECK-NEXT:    cmp.w r1, #-1
+; CHECK-NEXT:    vmov.32 r3, d16[0]
+; CHECK-NEXT:    vmov.32 lr, d16[1]
+; CHECK-NEXT:    vmov.32 r4, d21[0]
+; CHECK-NEXT:    vmov.32 r5, d17[0]
+; CHECK-NEXT:    vmov.32 r6, d21[1]
+; CHECK-NEXT:    vmov.32 r7, d17[1]
+; CHECK-NEXT:    vmov.32 r8, d18[1]
+; CHECK-NEXT:    subs r1, r3, r1
+; CHECK-NEXT:    vmov.32 r3, d18[0]
+; CHECK-NEXT:    sbcs.w r1, lr, r12
+; CHECK-NEXT:    vmov.32 r12, d19[0]
 ; CHECK-NEXT:    mov.w r1, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r1, #1
-; CHECK-NEXT:    cmp r1, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r1, #-1
-; CHECK-NEXT:    cmp.w r2, #-1
-; CHECK-NEXT:    mov.w r2, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r2, #1
-; CHECK-NEXT:    cmp.w r12, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r5, #1
-; CHECK-NEXT:    cmp r5, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r5, #-1
-; CHECK-NEXT:    cmp.w lr, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r6, #1
-; CHECK-NEXT:    cmp.w r4, #-1
-; CHECK-NEXT:    mov.w r4, #0
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r4, #1
-; CHECK-NEXT:    cmp r4, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r1, #1
+; CHECK-NEXT:    subs r5, r5, r4
+; CHECK-NEXT:    vmov.32 r5, d19[1]
+; CHECK-NEXT:    sbcs r7, r6
+; CHECK-NEXT:    mov.w r7, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r7, #1
+; CHECK-NEXT:    cmp r7, #0
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    movne.w r7, #-1
+; CHECK-NEXT:    vdup.32 d21, r7
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    sbcs.w r3, r2, r8
+; CHECK-NEXT:    mov.w r3, #0
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r3, #1
+; CHECK-NEXT:    rsbs.w r6, r12, #0
+; CHECK-NEXT:    sbcs.w r6, r2, r5
+; CHECK-NEXT:    it lt
+; CHECK-NEXT:    movlt r2, #1
+; CHECK-NEXT:    cmp r2, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r4, #-1
-; CHECK-NEXT:    cmp.w r7, #-1
-; CHECK-NEXT:    it gt
-; CHECK-NEXT:    movgt r3, #1
+; CHECK-NEXT:    movne.w r2, #-1
 ; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    vdup.32 d19, r2
 ; CHECK-NEXT:    it ne
 ; CHECK-NEXT:    movne.w r3, #-1
-; CHECK-NEXT:    vdup.32 d19, r3
-; CHECK-NEXT:    cmp r6, #0
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r6, #-1
-; CHECK-NEXT:    vdup.32 d21, r6
-; CHECK-NEXT:    cmp r2, #0
-; CHECK-NEXT:    vdup.32 d18, r4
+; CHECK-NEXT:    cmp r1, #0
 ; CHECK-NEXT:    it ne
-; CHECK-NEXT:    movne.w r2, #-1
-; CHECK-NEXT:    vdup.32 d23, r2
-; CHECK-NEXT:    vdup.32 d20, r5
-; CHECK-NEXT:    vdup.32 d22, r1
-; CHECK-NEXT:    vceq.i32 q9, q10, q9
+; CHECK-NEXT:    movne.w r1, #-1
+; CHECK-NEXT:    vdup.32 d18, r3
+; CHECK-NEXT:    vdup.32 d20, r1
+; CHECK-NEXT:    veor q9, q9, q10
 ; CHECK-NEXT:    vst1.64 {d16, d17}, [r0]
-; CHECK-NEXT:    vceq.i32 q10, q10, q11
-; CHECK-NEXT:    vrev64.32 q11, q9
-; CHECK-NEXT:    vrev64.32 q12, q10
-; CHECK-NEXT:    vand q9, q9, q11
-; CHECK-NEXT:    vand q10, q10, q12
-; CHECK-NEXT:    vmvn q9, q9
-; CHECK-NEXT:    vbic q9, q9, q10
 ; CHECK-NEXT:    vmovn.i64 d18, q9
 ; CHECK-NEXT:    vmov r2, r1, d18
 ; CHECK-NEXT:    mov r0, r2
-; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
   %x = load <2 x i64>, <2 x i64>* %ptr, align 8
   %y = load <2 x i64>, <2 x i64>* %ptr2, align 8
   %s = call {<2 x i64>, <2 x i1>} @llvm.ssub.with.overflow.v2i64(<2 x i64> %x, <2 x i64> %y)
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
index c008222a95f2d..3e2124e4be87d 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-minsize.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 4	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
index c144e1846cc07..734223488b9cd 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-mixed.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s
 
 ; CHECK-NOT: .eabi_attribute 30
 ; CHECK-NOT: Tag_ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
index f779402bee01a..7f52b869ccb87 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-optnone.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 6	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
index 3c7df7cd76580..89b1973a889a6 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization-optsize.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 
 ; CHECK: .eabi_attribute 30, 3	@ Tag_ABI_optimization_goals
 ; CHECK-OBJ:          TagName: ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes-optimization.ll b/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
index 6c1abae6b14ce..85a123f7b0b0f 100644
--- a/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes-optimization.ll
@@ -2,9 +2,9 @@
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 | FileCheck %s --check-prefix=SPEED
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 | FileCheck %s --check-prefix=MAXSPEED
 
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=NONE-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=SPEED-OBJ
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=MAXSPEED-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O0 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=NONE-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O1 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=SPEED-OBJ
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-a7 -O3 -filetype obj -o - | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=MAXSPEED-OBJ
 
 ; NONE:     .eabi_attribute 30, 5	@ Tag_ABI_optimization_goals
 ; SPEED:    .eabi_attribute 30, 1	@ Tag_ABI_optimization_goals
diff --git a/llvm/test/CodeGen/ARM/build-attributes.ll b/llvm/test/CodeGen/ARM/build-attributes.ll
index 39d173c4f9f5c..5eb6bafd5c689 100644
--- a/llvm/test/CodeGen/ARM/build-attributes.ll
+++ b/llvm/test/CodeGen/ARM/build-attributes.ll
@@ -34,8 +34,8 @@
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-DEFAULT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-neon,-d32 | FileCheck %s --check-prefix=CORTEX-A5-NONEON
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A5-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A5-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a5 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A5-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A8-SOFT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=soft  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A8-SOFT-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a8 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-A8-HARD
@@ -50,16 +50,16 @@
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a9 -float-abi=soft | FileCheck %s --check-prefix=CORTEX-A9-SOFT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-DEFAULT-FAST
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A12-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A12-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A12-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a12 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 | FileCheck %s --check-prefix=CORTEX-A15
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A15-FAST
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 | FileCheck %s --check-prefix=CORTEX-A17-DEFAULT
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-FAST
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
-; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-A17-NOFPU
+; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a17 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A17-NOFPU-FAST
 
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -enable-no-trapping-fp-math | FileCheck %s --check-prefix=NO-TRAPPING-MATH
 ; RUN: llc < %s -mtriple=armv7-linux-gnueabi -mcpu=cortex-a15 -denormal-fp-math=ieee | FileCheck %s --check-prefix=DENORMAL-IEEE
@@ -96,8 +96,8 @@
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard | FileCheck %s --check-prefix=CORTEX-M4-HARD
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -float-abi=hard  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M4-HARD-FAST
 ; RUN: llc < %s -mtriple=thumbv7m-linux-gnueabi -mcpu=cortex-m4 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2d16sp | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SOFT
-; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2d16sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2sp | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SOFT
+; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-vfp2sp  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-fp64 | FileCheck %s --check-prefix=CORTEX-M7 --check-prefix=CORTEX-M7-SINGLE
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 -mattr=-fp64  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-M7-FAST
 ; RUN: llc < %s -mtriple=thumbv7em-linux-gnueabi -mcpu=cortex-m7 | FileCheck %s --check-prefix=CORTEX-M7-DOUBLE
@@ -157,8 +157,8 @@
 ; RUN: llc < %s -mtriple=armv8.1a-linux-gnueabi -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 | FileCheck %s  --check-prefix=CORTEX-A7-CHECK
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s  --check-prefix=CORTEX-A7-CHECK-FAST
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2d16sp,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
-; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2d16sp,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2sp,-vfp3,-vfp4,-neon,-fp16 | FileCheck %s --check-prefix=CORTEX-A7-NOFPU
+; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=-vfp2sp,-vfp3,-vfp4,-neon,-fp16  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-NOFPU-FAST
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon | FileCheck %s --check-prefix=CORTEX-A7-FPUV4
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -enable-sign-dependent-rounding-fp-math | FileCheck %s --check-prefix=DYN-ROUNDING
 ; RUN: llc < %s -mtriple=armv7-none-linux-gnueabi -mcpu=cortex-a7 -mattr=+vfp4,-neon  -enable-unsafe-fp-math -frame-pointer=all -enable-no-infs-fp-math -enable-no-nans-fp-math -fp-contract=fast | FileCheck %s --check-prefix=CORTEX-A7-FPUV4-FAST
@@ -230,7 +230,7 @@
 ; RUN: llc < %s -mtriple=armv5-none-linux-gnueabi -mcpu=arm1022e -mattr=+strict-align | FileCheck %s --check-prefix=STRICT-ALIGN
 
 ; ARMv8-R
-; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-vfp2d16sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU
+; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-vfp2sp,-fp16 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NOFPU
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 -mattr=-neon,-fp64,-d32 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-SP
 ; RUN: llc < %s -mtriple=arm-none-none-eabi -mcpu=cortex-r52 | FileCheck %s --check-prefix=ARMv8R --check-prefix=ARMv8R-NEON
 
diff --git a/llvm/test/CodeGen/ARM/constant-islands-cfg.mir b/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
index add9e2003a507..32548dfe67eb1 100644
--- a/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
+++ b/llvm/test/CodeGen/ARM/constant-islands-cfg.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=thumbv6m-apple-ios -run-pass=arm-cp-islands %s -o - | FileCheck %s
+# RUN: llc -mtriple=thumbv6m-apple-ios -run-pass=arm-cp-islands --verify-machine-dom-info %s -o - | FileCheck %s
 --- |
   ; Function Attrs: minsize nounwind optsize uwtable
   define arm_aapcscc double @test_split_cfg(double %a, double %b) local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/ARM/debug-info-blocks.ll b/llvm/test/CodeGen/ARM/debug-info-blocks.ll
index cc1a45f23da07..8b31e7a51d514 100644
--- a/llvm/test/CodeGen/ARM/debug-info-blocks.ll
+++ b/llvm/test/CodeGen/ARM/debug-info-blocks.ll
@@ -155,7 +155,7 @@ define hidden void @foobar_func_block_invoke_0(i8* %.block_descriptor, %0* %load
 !47 = !DIDerivedType(tag: DW_TAG_member, name: "DestroyFuncPtr", line: 307, size: 32, align: 32, offset: 96, file: !153, scope: !40, baseType: !46)
 !48 = !DIDerivedType(tag: DW_TAG_member, name: "mydata", line: 609, size: 32, align: 32, offset: 160, file: !152, scope: !24, baseType: !49)
 !49 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 32, scope: !0, baseType: !50)
-!50 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !152, scope: !24, elements: !51)
+!50 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, file: !152, scope: !24, elements: !51)
 !51 = !{!52, !53, !54, !55, !56, !57, !58}
 !52 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 32, align: 32, file: !152, scope: !24, baseType: !32)
 !53 = !DIDerivedType(tag: DW_TAG_member, name: "__forwarding", size: 32, align: 32, offset: 32, file: !152, scope: !24, baseType: !32)
diff --git a/llvm/test/CodeGen/ARM/ifcvt_triangleSameCvtNext.mir b/llvm/test/CodeGen/ARM/ifcvt_triangleSameCvtNext.mir
new file mode 100644
index 0000000000000..bef131df27c91
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/ifcvt_triangleSameCvtNext.mir
@@ -0,0 +1,29 @@
+# RUN: llc -mtriple=arm-apple-ios -run-pass=if-converter -verify-machineinstrs %s -o - | FileCheck %s
+...
+---
+name:            foo
+body:             |
+  bb.0:
+    Bcc %bb.2, 1, $cpsr
+
+  bb.1:
+    $sp = tADDspi $sp, 2, 14, _
+    B %bb.1
+
+  bb.2:
+    Bcc %bb.3, 0, $cpsr
+    B %bb.2
+
+  bb.3:
+    Bcc %bb.1, 1, $cpsr
+    B %bb.1
+...
+
+# Both branches in bb.3 jump to bb.1. IfConversion shouldn't treat this as a
+# tringle and insert the tADDspi in bb3, but leave it as it is.
+
+# CHECK:     bb.3:
+# CHECK:       successors: %bb.1
+# CHECK-NOT:   tADDspi
+# CHECK:       Bcc %bb.1, 1, $cpsr
+# CHECK:       B %bb.1
diff --git a/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll b/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
index ff8dba6e38c01..2b08e6d46fd72 100644
--- a/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
+++ b/llvm/test/CodeGen/ARM/inlineasm-X-allocation.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=armv7-none-eabi -mattr=-neon,-vfp2d16sp %s -o - | FileCheck %s  -check-prefixes=COMMON,NOVFP
+; RUN: llc -mtriple=armv7-none-eabi -mattr=-neon,-vfp2sp %s -o - | FileCheck %s  -check-prefixes=COMMON,NOVFP
 ; RUN: llc -mtriple=armv7-none-eabi -mattr=+neon %s -float-abi=hard -o - | FileCheck %s -check-prefixes=COMMON,VFP
 
 ; The intent here is to test "X", which says that any operand whatsoever is allowed.
diff --git a/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll b/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
index a7bf6b7efebf0..f70e9f378f3b0 100644
--- a/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
+++ b/llvm/test/CodeGen/ARM/softfp-constant-comparison.ll
@@ -43,4 +43,4 @@ land.end:                                         ; preds = %land.rhs, %entry
   ret void
 }
 
-attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m4" "target-features"="+armv7e-m,+dsp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2d16sp,+vfp3d16sp,+vfp4d16sp,-aes,-crc,-crypto,-dotprod,-fp16fml,-fullfp16,-hwdiv-arm,-lob,-mve,-mve.fp,-ras,-sb,-sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #0 = { noinline nounwind optnone "correctly-rounded-divide-sqrt-fp-math"="false" "denormal-fp-math"="preserve-sign" "disable-tail-calls"="false" "frame-pointer"="none" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="true" "no-jump-tables"="false" "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="cortex-m4" "target-features"="+armv7e-m,+dsp,+fp16,+fpregs,+hwdiv,+thumb-mode,+vfp2sp,+vfp3d16sp,+vfp4d16sp,-aes,-crc,-crypto,-dotprod,-fp16fml,-fullfp16,-hwdiv-arm,-lob,-mve,-mve.fp,-ras,-sb,-sha2" "unsafe-fp-math"="false" "use-soft-float"="false" }
diff --git a/llvm/test/CodeGen/BPF/BTF/array-size-0.ll b/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
index d397c2c3a2396..65a0841873433 100644
--- a/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
+++ b/llvm/test/CodeGen/BPF/BTF/array-size-0.ll
@@ -32,7 +32,7 @@
 ; CHECK-NEXT:        .long   0
 ; CHECK-NEXT:        .long   1
 ; CHECK-NEXT:        .long   3
-; CHECK-NEXT:        .long   0
+; CHECK-NEXT:        .long   10
 ; CHECK-NEXT:        .long   3                       # BTF_KIND_INT(id = 3)
 ; CHECK-NEXT:        .long   16777216                # 0x1000000
 ; CHECK-NEXT:        .long   4
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll
new file mode 100644
index 0000000000000..35415f6d9d3d4
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-load.ll
@@ -0,0 +1,82 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s
+;
+; Source Code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   struct s {int a; int b;};
+;   int test(struct s *arg) { return *(const int *)_(&arg->b); }
+; Compiler flag to generate IR:
+;   clang -target bpf -S -O2 -g -emit-llvm test.c
+
+%struct.s = type { i32, i32 }
+
+; Function Attrs: nounwind readonly
+define dso_local i32 @test(%struct.s* readonly %arg) local_unnamed_addr #0 !dbg !11 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !20, metadata !DIExpression()), !dbg !21
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1), !dbg !22, !llvm.preserve.access.index !15
+  %1 = load i32, i32* %0, align 4, !dbg !23, !tbaa !24
+  ret i32 %1, !dbg !28
+}
+
+; CHECK-LABEL: test
+; CHECK:       r2 = 4
+; CHECK:       r1 += r2
+; CHECK:       r0 = *(u32 *)(r1 + 0)
+; CHECK:       exit
+;
+; CHECK:       .long   1                       # BTF_KIND_STRUCT(id = 2)
+;
+; CHECK:       .byte   115                     # string offset=1
+; CHECK:       .ascii  ".text"                 # string offset=20
+; CHECK:       .ascii  "0:1"                   # string offset=26
+;
+; CHECK:       .long   12                      # OffsetReloc
+; CHECK-NEXT:  .long   20                      # Offset reloc section string offset=20
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   26
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!7, !8, !9}
+!llvm.ident = !{!10}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, retainedTypes: !3, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{!4}
+!4 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !5, size: 64)
+!5 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !6)
+!6 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!7 = !{i32 2, !"Dwarf Version", i32 4}
+!8 = !{i32 2, !"Debug Info Version", i32 3}
+!9 = !{i32 1, !"wchar_size", i32 4}
+!10 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)"}
+!11 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !12, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !19)
+!12 = !DISubroutineType(types: !13)
+!13 = !{!6, !14}
+!14 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !15, size: 64)
+!15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 2, size: 64, elements: !16)
+!16 = !{!17, !18}
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !15, file: !1, line: 2, baseType: !6, size: 32)
+!18 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !15, file: !1, line: 2, baseType: !6, size: 32, offset: 32)
+!19 = !{!20}
+!20 = !DILocalVariable(name: "arg", arg: 1, scope: !11, file: !1, line: 3, type: !14)
+!21 = !DILocation(line: 0, scope: !11)
+!22 = !DILocation(line: 3, column: 48, scope: !11)
+!23 = !DILocation(line: 3, column: 34, scope: !11)
+!24 = !{!25, !25, i64 0}
+!25 = !{!"int", !26, i64 0}
+!26 = !{!"omnipotent char", !27, i64 0}
+!27 = !{!"Simple C/C++ TBAA"}
+!28 = !DILocation(line: 3, column: 27, scope: !11)
diff --git a/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll
new file mode 100644
index 0000000000000..52a630027e932
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/CORE/offset-reloc-end-ret.ll
@@ -0,0 +1,76 @@
+; RUN: llc -march=bpfel -filetype=asm -o - %s | FileCheck %s 
+;
+; Source Code:
+;   #define _(x) (__builtin_preserve_access_index(x))
+;   struct s {int a; int b;};
+;   const void *test(struct s *arg) { return _(&arg->b); }
+; Compiler flag to generate IR:
+;   clang -target bpf -S -O2 -g -emit-llvm test.c
+
+%struct.s = type { i32, i32 }
+
+; Function Attrs: nounwind readnone
+define dso_local i8* @test(%struct.s* readnone %arg) local_unnamed_addr #0 !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata %struct.s* %arg, metadata !19, metadata !DIExpression()), !dbg !20
+  %0 = tail call i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s* %arg, i32 1, i32 1), !dbg !21, !llvm.preserve.access.index !13
+  %1 = bitcast i32* %0 to i8*, !dbg !21
+  ret i8* %1, !dbg !22
+}
+
+; CHECK-LABEL: test
+; CHECK:       r0 = r1
+; CHECK:       r1 = 4
+; CHECK:       r0 += r1
+; CHECK:       exit
+;
+; CHECK:       .long   1                       # BTF_KIND_STRUCT(id = 2)
+;
+; CHECK:       .byte   115                     # string offset=1
+; CHECK:       .ascii  ".text"                 # string offset=20
+; CHECK:       .ascii  "0:1"                   # string offset=63
+;
+; CHECK:       .long   12                      # OffsetReloc
+; CHECK-NEXT:  .long   20                      # Offset reloc section string offset=20
+; CHECK-NEXT:  .long   1
+; CHECK-NEXT:  .long   .Ltmp{{[0-9]+}}
+; CHECK-NEXT:  .long   2
+; CHECK-NEXT:  .long   63
+
+; Function Attrs: nounwind readnone
+declare i32* @llvm.preserve.struct.access.index.p0i32.p0s_struct.ss(%struct.s*, i32, i32) #1
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata) #2
+
+attributes #0 = { nounwind readnone "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { nounwind readnone }
+attributes #2 = { nounwind readnone speculatable willreturn }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.c", directory: "/tmp/home/yhs/work/tests/core")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0 (https://github.com/llvm/llvm-project.git 6e353b4df3aa452ed4741a5e5caea02b1a876d8c)"}
+!7 = distinct !DISubprogram(name: "test", scope: !1, file: !1, line: 3, type: !8, scopeLine: 3, flags: DIFlagPrototyped, isDefinition: true, isOptimized: true, unit: !0, retainedNodes: !18)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !12}
+!10 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_const_type, baseType: null)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "s", file: !1, line: 2, size: 64, elements: !14)
+!14 = !{!15, !17}
+!15 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !13, file: !1, line: 2, baseType: !16, size: 32)
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !DIDerivedType(tag: DW_TAG_member, name: "b", scope: !13, file: !1, line: 2, baseType: !16, size: 32, offset: 32)
+!18 = !{!19}
+!19 = !DILocalVariable(name: "arg", arg: 1, scope: !7, file: !1, line: 3, type: !12)
+!20 = !DILocation(line: 0, scope: !7)
+!21 = !DILocation(line: 3, column: 42, scope: !7)
+!22 = !DILocation(line: 3, column: 35, scope: !7)
diff --git a/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll b/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll
new file mode 100644
index 0000000000000..b0fbb1ceaf0f0
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/autohvx/isel-setcc-v256i1.ll
@@ -0,0 +1,15 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Check that this doesn't crash. The select should be broken up into two
+; vmux instructions.
+
+; CHECK-LABEL: foo:
+; CHECK: vmux
+; CHECK: vmux
+define <256 x i8> @foo(<256 x i8> %a0, <256 x i8> %a1) #0 {
+  %v0 = icmp slt <256 x i8> %a0, zeroinitializer
+  %v1 = select <256 x i1> %v0, <256 x i8> %a1, <256 x i8> %a0
+  ret <256 x i8> %v1
+}
+
+attributes #0 = { "target-cpu"="hexagonv62" "target-features"="+hvx,+hvx-length128b" }
diff --git a/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll
new file mode 100644
index 0000000000000..f45b1a77c6d7a
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/isel-bitcast-v8i8-v4i16.ll
@@ -0,0 +1,13 @@
+; RUN: llc -march=hexagon < %s | FileCheck %s
+
+; Check that this doesn't fail to select instructions.
+; CHECK: vsplath
+
+define <8 x i8> @fred(i16 %a0) #0 {
+  %t0 = insertelement <4 x i16> undef, i16 %a0, i32 0
+  %t1 = shufflevector <4 x i16> %t0, <4 x i16> undef, <4 x i32> zeroinitializer
+  %t2 = bitcast <4 x i16> %t1 to <8 x i8>
+  ret <8 x i8> %t2
+}
+
+attributes #0 = { readnone nounwind "target-cpu"="hexagonv62" }
diff --git a/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir b/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
index 5b953f13b1b48..aa553dccc7de7 100644
--- a/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
+++ b/llvm/test/CodeGen/Hexagon/pipeliner/swp-phi-start.mir
@@ -1,4 +1,4 @@
-# RUN: llc < %s -x mir -march=hexagon -run-pass=modulo-schedule-test | FileCheck %s
+# RUN: llc < %s -x mir -march=hexagon -run-pass=modulo-schedule-test -pipeliner-experimental-cg=true | FileCheck %s
 
 # Simple check for this sanity test; ensure all instructions are in stage 0 in
 # the prolog and stage 3 in the epilog.
diff --git a/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll b/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
index 5272faf8f9b8a..f89df6e55734d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-art-deps-rec.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
 
 ; RUN: llc -march=hexagon -mcpu=hexagonv65 -O3 -debug-only=pipeliner \
-; RUN: < %s 2>&1 | FileCheck %s
+; RUN: < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the artificial dependences are ignored while computing the
 ; circuits.
diff --git a/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll b/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
index 74f0647a72a4e..ee93e8b3468e2 100644
--- a/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-bad-sched.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: asserts
-; RUN: llc -march=hexagon -enable-pipeliner -enable-aa-sched-mi < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -enable-aa-sched-mi < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; CHECK: loop0(
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-1.ll b/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
index b33cf522115b2..e5b5be4d43004 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 -lsr-setupcost-depth-limit=1 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 -lsr-setupcost-depth-limit=1 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct code when a loop carried value
 ; is scheduled one stage earlier than it's use. The code in
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
index 8271e8b1b54ee..d9db8ec6194c8 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep1.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 # REQUIRES: asserts
 
 # Test that the loop carried dependence check correctly identifies a recurrence.
diff --git a/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir b/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
index 126e6aa462b4e..5271a2db7758e 100644
--- a/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
+++ b/llvm/test/CodeGen/Hexagon/swp-carried-dep2.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 | FileCheck %s
+# RUN: llc -mtriple=hexagon -run-pass pipeliner -debug-only=pipeliner %s -o /dev/null 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 # REQUIRES: asserts
 
 # Test that the loop carried dependence check correctly identifies a recurrence
diff --git a/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll b/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
index d0e72be778a1c..5695f3d61b740 100644
--- a/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-chain-refs.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=hexagon -enable-pipeliner=true -stats -o /dev/null < %s \
-; RUN:      2>&1 | FileCheck %s --check-prefix=STATS
+; RUN:      2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Test that we do not schedule chained references too far apart,
diff --git a/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll b/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
index 855f43e50125d..157bdd069f921 100644
--- a/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-change-dep1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=1 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=1 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we update the offset correctly for loads that are
 ; moved past stores. In these cases, we change the dependences
diff --git a/llvm/test/CodeGen/Hexagon/swp-change-deps.ll b/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
index e2ca071f5f5fb..1b35c633c52de 100644
--- a/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-change-deps.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct offsets for loads in the prolog
 ; after removing dependences on a post-increment instructions of the
diff --git a/llvm/test/CodeGen/Hexagon/swp-check-offset.ll b/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
index 220ebde0f86d6..6a7211df12d8f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-check-offset.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv62 -enable-pipeliner < %s | FileCheck --check-prefix=CHECK-V62 %s
-; RUN: llc -march=hexagon -mcpu=hexagonv65 -enable-pipeliner < %s | FileCheck --check-prefix=CHECK-V65 %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv62 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck --check-prefix=CHECK-V62 %s
+; RUN: llc -march=hexagon -mcpu=hexagonv65 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck --check-prefix=CHECK-V65 %s
 
 ;
 ; Make sure we pipeline the loop and that we generate the correct
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
index 95dfc37e30620..c785ee74513a9 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc1.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -march=hexagon -enable-pipeliner -enable-pipeliner-opt-size \
 ; RUN:     -verify-machineinstrs -hexagon-initial-cfg-cleanup=0 \
 ; RUN:     -enable-aa-sched-mi=false -hexagon-expand-condsets=0 \
-; RUN:     < %s | FileCheck %s
+; RUN:     < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Disable expand-condsets because it will assert on undefined registers.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
index 8b9f87f428d65..29d12bd14390d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we fixup a pipelined loop correctly when the number of
 ; stages is greater than the compile-time loop trip count. In this
diff --git a/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll b/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
index a8caebd09eb11..48a61428538c2 100644
--- a/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-const-tc3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner correctly fixes up the pipelined CFG when the loop
 ; has a constant trip count, and the trip count is less than the number of
diff --git a/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll b/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
index 48f33bd6d22cc..d14177cc684f2 100644
--- a/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-conv3x3-nested.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 ; XFAIL: *
 ; LSR changes required.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll b/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
index 69743407c148c..f511241a7c73b 100644
--- a/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-copytophi-dag.ll
@@ -1,7 +1,7 @@
 ; REQUIRES: asserts
 ;
 ; RUN: llc -march=hexagon -enable-pipeliner=true -debug-only=pipeliner < %s \
-; RUN: 2>&1 | FileCheck %s
+; RUN: 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the artificial dependence is created as a result of
 ; CopyToPhi DAG mutation.
diff --git a/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll b/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
index 7ba4286bf4144..cc19ce1ae4417 100644
--- a/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-dep-neg-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the code that changes the dependences does not allow
 ; a load with a negative offset to be overlapped with the post
diff --git a/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll b/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
index cbdc3ba36f650..5698d37cb23ba 100644
--- a/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-disable-Os.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 ; CHECK: loop0(.LBB0_{{[0-9]+}},#347)
 
 target triple = "hexagon"
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
index a54ac5825601d..f57f94bf03cec 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-numphis.ll
@@ -1,6 +1,6 @@
 ; XFAIL: *
 ; Needs some fixed in the pipeliner.
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; CHECK: endloop0
 ; CHECK: vmem
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
index b2a7dada33f41..b32fed97f26f1 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi2.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 %s.0 = type { i16, i8, i8, i16, i8, i8, i16, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i8, i32, i16, i8, i8, %s.1, [2 x [16 x %s.2]], i32 (i8*, i8*, i8*, i8*, i8*)*, %s.3*, %s.3*, [120 x i8], i8, i8, %s.4*, [2 x [120 x [8 x i8]]], [56 x i8], [2 x [121 x %s.5]], [2 x %s.5], %s.5*, %s.5*, i32, i32, i16, i8, i8, %s.7, %s.9, %s.11, %s.8*, %s.8* }
 %s.1 = type { i8, i8, i8, i8, i8, i8, i8, i8, i32, i8, [16 x i8], i8, [4 x i8], [32 x i16], [32 x i16], [2 x i8], [4 x i8], [2 x [4 x i8]], [2 x [4 x i8]], i32, i32, i16, i8 }
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
index e85ea7654e055..8b611cfe0b4f2 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi4.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct value for a Phi in the epilog
 ; that is for a value defined two stages earlier. An extra copy in the
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
index a524dc0d5bedf..72c05284d6953 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi5.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we use the correct name in an epilog phi for a phi value
 ; that is defined for the last time in the kernel. Previously, we
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
index 8cad174937df8..89a04d9f76ff5 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi7.ll
@@ -7,8 +7,8 @@
 
 ; CHECK: if ({{.*}}) jump
 ; CHECK: [[VREG:v([0-9]+)]]{{.*}} = {{.*}}vmem(r{{[0-9]+}}++#1)
-; CHECK: if ({{.*}}) {{jump|jump:nt}} [[EPLOG1:(.*)]]
-; CHECK: if ({{.*}}) {{jump|jump:nt}} [[EPLOG:(.*)]]
+; CHECK: if ({{.*}}) {{jump|jump:nt|jump:t}} [[EPLOG1:(.*)]]
+; CHECK: if ({{.*}}) {{jump|jump:nt|jump:t}} [[EPLOG:(.*)]]
 ; CHECK: [[EPLOG]]:
 ; CHECK: [[VREG1:v([0-9]+)]] = [[VREG]]
 ; CHECK: [[VREG]] = v{{[0-9]+}}
diff --git a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
index 370d31d92c707..214307e25137d 100644
--- a/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-epilog-phi8.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mno-pairing -mno-compound -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mno-pairing -mno-compound -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 ; XFAIL: *
 
 ; Test that we generate the correct phi names in the epilog when the pipeliner
diff --git a/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll b/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
index 681e749233785..c87479f6e97db 100644
--- a/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-kernel-phi1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner-opt-size -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner-opt-size -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct names for the phis in the kernel for the
 ; incoming values. In this case, the loop contains a phi and has another phi
diff --git a/llvm/test/CodeGen/Hexagon/swp-large-rec.ll b/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
index ee88aaffd5c01..45d40df4ec0ae 100644
--- a/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-large-rec.ll
@@ -1,6 +1,6 @@
 ; RUN: llc -march=hexagon -enable-pipeliner -stats \
 ; RUN:     -pipeliner-prune-loop-carried=false -fp-contract=fast \
-; RUN:     -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN:     -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; That that we do not pipeline this loop. The recurrence is too large. If
diff --git a/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll b/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
index 9b68cf956591e..d8e4f003d5375 100644
--- a/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-listen-loop3.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -pipeliner-ignore-recmii -pipeliner-max-stages=2 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -pipeliner-ignore-recmii -pipeliner-max-stages=2 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; This is a loop we pipeline to three packets, though we could do bettter.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll b/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
index b95e62419567b..9f145189786ef 100644
--- a/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-loop-carried-unknown.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner schedules a store before the load in which there is a
 ; loop carried dependence. Previously, the loop carried dependence wasn't added
diff --git a/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll b/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
index a657f92c5d5a0..631d02649d11e 100644
--- a/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-lots-deps.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; STATS: 1 pipeliner        - Number of loops software pipelined
diff --git a/llvm/test/CodeGen/Hexagon/swp-max.ll b/llvm/test/CodeGen/Hexagon/swp-max.ll
index 26238ea6fb374..32282204ec52a 100644
--- a/llvm/test/CodeGen/Hexagon/swp-max.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-max.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner \
-; RUN:     -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN:     -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 @A = global [8 x i32] [i32 4, i32 -3, i32 5, i32 -2, i32 -1, i32 2, i32 6, i32 -2], align 8
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-maxstart.ll b/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
index 811c94062a0f0..8d65e76913f36 100644
--- a/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-maxstart.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the MinStart computation, which is based upon the length
 ; of the chain edges, is computed correctly. A bug in the code allowed
diff --git a/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll b/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
index 81f4d22cfd543..20e39dd08fd72 100644
--- a/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-memrefs-epilog.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O2 -fp-contract=fast < %s | FileCheck %s
+; RUN: llc -march=hexagon -O2 -fp-contract=fast < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the memoperands for instructions in the epilog are updated
 ; correctly. Previously, the pipeliner updated the offset for the memoperands
diff --git a/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll b/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
index fc2576af8ac2c..5a2e7d4e14d5b 100644
--- a/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-multi-loops.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Make sure we attempt to pipeline all inner most loops.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-new-phi.ll b/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
index 0ba3e30731abc..d3c1058fe36cc 100644
--- a/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-new-phi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the generatePhi code doesn't rename a a Phi instruction that's defined
 ; in the same block.  The bug causes a Phi to incorrectly depend on another Phi.
diff --git a/llvm/test/CodeGen/Hexagon/swp-order-copies.ll b/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
index 5de0717654ffa..0a017c4ab7f6f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order-copies.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the instruction ordering code in the pipeliner fixes up dependences
 ; between post-increment register definitions and uses so that the register
diff --git a/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll b/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
index d1d852bcae778..4cd29a4a0baf8 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order-deps7.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner cause an assert and correctly pipelines the
 ; loop.
diff --git a/llvm/test/CodeGen/Hexagon/swp-order.ll b/llvm/test/CodeGen/Hexagon/swp-order.ll
index bc16b8835b7d8..14cc682eb7e57 100644
--- a/llvm/test/CodeGen/Hexagon/swp-order.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-order.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that when we order instructions in a packet we check for
 ; order dependences so that the source of an order dependence
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll b/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
index 68cb69ba2ac4d..31b98328a2fac 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-ch-offset.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct offsets after we removed unneeded
 ; chain dependences between Phis and generated a better pipeline.
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll b/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
index e3a6c9db616ec..3037dcc2d5e17 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-chains.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -debug-only=pipeliner < %s -o - 2>&1 | FileCheck %s
+; RUN: llc -march=hexagon -debug-only=pipeliner < %s -o - 2>&1 -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that there is a chain edge between two dependent Phis.
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll b/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
index 38b56c1126a00..ec7af41a31fa7 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-dep.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 -enable-pipeliner -pipeliner-max-stages=2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -hexagon-initial-cfg-cleanup=0 -enable-pipeliner -pipeliner-max-stages=2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Check that the pipelined code uses the proper address in the
 ; prolog and the kernel. The bug occurs when the address computation
diff --git a/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll b/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
index d39252141d2eb..be838e767aa07 100644
--- a/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-phi-ref.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -enable-bsb-sched=0 -join-liveintervals=false < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -enable-bsb-sched=0 -join-liveintervals=false < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; XFAIL: *
 ; This test is failing after post-ra machine sinking.
diff --git a/llvm/test/CodeGen/Hexagon/swp-pragma-disable.ii b/llvm/test/CodeGen/Hexagon/swp-pragma-disable.ii
index 80494f5ac10de..b97065c129b0f 100644
--- a/llvm/test/CodeGen/Hexagon/swp-pragma-disable.ii
+++ b/llvm/test/CodeGen/Hexagon/swp-pragma-disable.ii
@@ -1,5 +1,5 @@
 ; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
-; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test that checks if pipeliner disabled by pragma 
diff --git a/llvm/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii b/llvm/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii
index 6a4ba7eaccd31..2c6b606a99f73 100644
--- a/llvm/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii
+++ b/llvm/test/CodeGen/Hexagon/swp-pragma-initiation-interval.ii
@@ -1,5 +1,5 @@
 ; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
-; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test that checks if the II set by pragma was taken by pipeliner.
diff --git a/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll b/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
index 4a6fa5a6cfe73..14b04a1cfe647 100644
--- a/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-prolog-phi.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -rdf-opt=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -rdf-opt=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we generate the correct name for a value in a prolog block. The
 ; pipeliner was using an incorrect value for an instruction in the 2nd prolog
diff --git a/llvm/test/CodeGen/Hexagon/swp-rename.ll b/llvm/test/CodeGen/Hexagon/swp-rename.ll
index ab0cc11ec7e9e..eb60a0e38d0e7 100644
--- a/llvm/test/CodeGen/Hexagon/swp-rename.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-rename.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; A test that the Phi rewrite logic is correct.
 
diff --git a/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll b/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
index 1a9734860d5cb..1e2eefae6ac84 100644
--- a/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-resmii-1.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -debug-only=pipeliner < %s -o - 2>&1 > /dev/null | FileCheck %s
+; RUN: llc -march=hexagon -enable-pipeliner -debug-only=pipeliner < %s -o - 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 
 ; Test that checks that we compute the correct ResMII for haar.
diff --git a/llvm/test/CodeGen/Hexagon/swp-resmii.ll b/llvm/test/CodeGen/Hexagon/swp-resmii.ll
index 851d82ea50f09..99812af3be5d0 100644
--- a/llvm/test/CodeGen/Hexagon/swp-resmii.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-resmii.ll
@@ -1,5 +1,5 @@
 ; RUN: llc -disable-lsr -march=hexagon -enable-pipeliner  \
-; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null | FileCheck %s
+; RUN:     -debug-only=pipeliner < %s 2>&1 > /dev/null -pipeliner-experimental-cg=true | FileCheck %s
 ; REQUIRES: asserts
 ;
 ; Test that checks if the ResMII is 1.
diff --git a/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll b/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
index 7371ed10a7117..6883e51503b14 100644
--- a/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-reuse-phi-6.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon < %s | FileCheck %s
+; RUN: llc -march=hexagon < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that the pipeliner generates correct code when attempting to reuse
 ; an existing phi. This test case contains a phi that references another
diff --git a/llvm/test/CodeGen/Hexagon/swp-sigma.ll b/llvm/test/CodeGen/Hexagon/swp-sigma.ll
index 56742ca83a71b..968fafc449d72 100644
--- a/llvm/test/CodeGen/Hexagon/swp-sigma.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-sigma.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -O2 < %s | FileCheck %s
+; RUN: llc -march=hexagon -O2 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; We do not pipeline sigma yet, but the non-pipelined version
 ; with good scheduling is pretty fast. The compiler generates
diff --git a/llvm/test/CodeGen/Hexagon/swp-stages4.ll b/llvm/test/CodeGen/Hexagon/swp-stages4.ll
index 2d88094cf740b..1b96aca2a48c5 100644
--- a/llvm/test/CodeGen/Hexagon/swp-stages4.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-stages4.ll
@@ -1,11 +1,11 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -disable-block-placement=0 -hexagon-bit=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -disable-block-placement=0 -hexagon-bit=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Test that we rename registers correctly for multiple stages when there is a
 ; Phi and depends upon another Phi.
 
 ; CHECK: = and
 ; CHECK: = and
-; CHECK: = and
+; CHECK: r[[REGA:[0-9]+]] = memub(r{{[0-9]+}}+#1)
 ; CHECK: r[[REG0:[0-9]+]] = and(r[[REG1:[0-9]+]],#255)
 ; CHECK-NOT: r[[REG0]] = and(r[[REG1]],#255)
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
diff --git a/llvm/test/CodeGen/Hexagon/swp-stages5.ll b/llvm/test/CodeGen/Hexagon/swp-stages5.ll
index fdfb2101cd36c..1f8463fbc30f9 100644
--- a/llvm/test/CodeGen/Hexagon/swp-stages5.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-stages5.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -hexagon-bit=0 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner -pipeliner-max-stages=2 -hexagon-bit=0 < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Very similar to swp-stages4.ll, but the pipelined schedule is a little
 ; different.
diff --git a/llvm/test/CodeGen/Hexagon/swp-subreg.ll b/llvm/test/CodeGen/Hexagon/swp-subreg.ll
index d75b3afc7b72c..b9754f4eb3628 100644
--- a/llvm/test/CodeGen/Hexagon/swp-subreg.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-subreg.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; We're unable to pipeline a loop with a subreg as an operand of a Phi.
diff --git a/llvm/test/CodeGen/Hexagon/swp-swap.ll b/llvm/test/CodeGen/Hexagon/swp-swap.ll
index a8432cb7d2187..4cd073cb16b83 100644
--- a/llvm/test/CodeGen/Hexagon/swp-swap.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-swap.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Test that we don't pipeline, incorrectly, the swap operation.
diff --git a/llvm/test/CodeGen/Hexagon/swp-tfri.ll b/llvm/test/CodeGen/Hexagon/swp-tfri.ll
index 66b999e5590bd..f0c26045430c2 100644
--- a/llvm/test/CodeGen/Hexagon/swp-tfri.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-tfri.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 -stats -o /dev/null < %s 2>&1 | FileCheck %s --check-prefix=STATS
+; RUN: llc -march=hexagon -enable-pipeliner -hexagon-initial-cfg-cleanup=0 -stats -o /dev/null < %s 2>&1 -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=STATS
 ; REQUIRES: asserts
 
 ; Check that we handle the case when a value is first defined in the loop.
diff --git a/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll b/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
index 3ff88452499e0..4bd1a513429f0 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vect-dotprod.ll
@@ -1,6 +1,6 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -O2 < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -O3 < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O2 < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -O3 < %s -pipeliner-experimental-cg=true | FileCheck %s
 ;
 ; Check that we pipeline a vectorized dot product in a single packet.
 ;
diff --git a/llvm/test/CodeGen/Hexagon/swp-vmult.ll b/llvm/test/CodeGen/Hexagon/swp-vmult.ll
index dfc7dd913242b..fd9cdf9b38c90 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vmult.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vmult.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: to-be-fixed
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
 
 ; Multiply and accumulate
 ; CHECK: mpyi([[REG0:r([0-9]+)]],[[REG1:r([0-9]+)]])
diff --git a/llvm/test/CodeGen/Hexagon/swp-vsum.ll b/llvm/test/CodeGen/Hexagon/swp-vsum.ll
index 1c4d1c2ef0175..5dcd2824550d6 100644
--- a/llvm/test/CodeGen/Hexagon/swp-vsum.ll
+++ b/llvm/test/CodeGen/Hexagon/swp-vsum.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s | FileCheck %s
-; RUN: llc -march=hexagon -mcpu=hexagonv60 -enable-pipeliner < %s | FileCheck %s --check-prefix=CHECKV60
+; RUN: llc -march=hexagon -mcpu=hexagonv5 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s
+; RUN: llc -march=hexagon -mcpu=hexagonv60 -enable-pipeliner < %s -pipeliner-experimental-cg=true | FileCheck %s --check-prefix=CHECKV60
 
 ; Simple vector total.
 ; CHECK: loop0(.LBB0_[[LOOP:.]],
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir b/llvm/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir
index a93af1d54d39d..c92d35f923915 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/expected-target-index-name.mir
@@ -44,6 +44,6 @@ body: |
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
     $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir b/llvm/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir
index 31efedd379609..5a0e0309e417f 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/invalid-target-index-operand.mir
@@ -44,6 +44,6 @@ body: |
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
     $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/load-store-opt-dlc.mir b/llvm/test/CodeGen/MIR/AMDGPU/load-store-opt-dlc.mir
index e3bcac22f1367..8ad50b72c284f 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/load-store-opt-dlc.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/load-store-opt-dlc.mir
@@ -32,7 +32,7 @@
   }
 ...
 
-# CHECK: BUFFER_STORE_DWORDX2_OFFSET killed %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 8 into %ir.out.gep.1, align 4, addrspace 1)
+# CHECK: BUFFER_STORE_DWORDX2_OFFSET killed %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 8 into %ir.out.gep.1, align 4, addrspace 1)
 ---
 name: test1
 liveins:
@@ -56,14 +56,14 @@ body: |
     %5:vgpr_32 = COPY $vgpr0
     %6:vgpr_32 = COPY $vgpr1
 
-    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 
     S_ENDPGM 0
 ...
 
-# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 8, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 8, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 ---
 name: test2
 liveins:
@@ -87,14 +87,14 @@ body: |
     %5:vgpr_32 = COPY $vgpr0
     %6:vgpr_32 = COPY $vgpr1
 
-    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 
     S_ENDPGM 0
 ...
 
-# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 8, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+# CHECK: BUFFER_STORE_DWORD_OFFSET %{{[0-9]+}}, %{{[0-9]+}}, 0, 8, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 ---
 name: test3
 liveins:
@@ -118,13 +118,13 @@ body: |
     %5:vgpr_32 = COPY $vgpr0
     %6:vgpr_32 = COPY $vgpr1
 
-    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 0, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 
     S_ENDPGM 0
 ...
 
-# CHECK: BUFFER_STORE_DWORDX2_OFFSET killed %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 1, implicit $exec :: (store 8 into %ir.out.gep.1, align 4, addrspace 1)
+# CHECK: BUFFER_STORE_DWORDX2_OFFSET killed %{{[0-9]+}}, %{{[0-9]+}}, 0, 4, 0, 0, 0, 1, 0, implicit $exec :: (store 8 into %ir.out.gep.1, align 4, addrspace 1)
 ---
 name: test4
 liveins:
@@ -148,8 +148,8 @@ body: |
     %5:vgpr_32 = COPY $vgpr0
     %6:vgpr_32 = COPY $vgpr1
 
-    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
-    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 1, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %5, %4, 0, 4, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
+    BUFFER_STORE_DWORD_OFFSET %6, %4, 0, 8, 0, 0, 0, 1, 0, implicit $exec :: (store 4 into %ir.out.gep.1, addrspace 1)
 
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/mir-canon-multi.mir b/llvm/test/CodeGen/MIR/AMDGPU/mir-canon-multi.mir
index 005014d5e83f4..67bf92b60814b 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/mir-canon-multi.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/mir-canon-multi.mir
@@ -7,7 +7,7 @@
 # CHECK-NEXT: %namedVReg1353:vreg_64 = REG_SEQUENCE %namedVReg4354, %subreg.sub0, %namedVReg1352, %subreg.sub1
 # CHECK-NEXT: %namedVReg1354:sgpr_128 = REG_SEQUENCE %namedVReg4354, %subreg.sub0, %namedVReg1352, %subreg.sub1, %namedVReg1358, %subreg.sub2, %namedVReg1359, %subreg.sub3
 # This tests for the itereator invalidation fix (reviews.llvm.org/D62713)
-# CHECK-NEXT: BUFFER_STORE_DWORD_ADDR64 %namedVReg1352, %namedVReg1353, %namedVReg1354, 0, 0, 0, 0, 0, 0, implicit $exec
+# CHECK-NEXT: BUFFER_STORE_DWORD_ADDR64 %namedVReg1352, %namedVReg1353, %namedVReg1354, 0, 0, 0, 0, 0, 0, 0, implicit $exec
 ...
 ---
 name: foo
@@ -27,7 +27,7 @@ body:             |
     %vreg123_3:vgpr_32 = COPY %5
     %16:sgpr_128 = REG_SEQUENCE killed %vreg123_0, %subreg.sub0, %vreg123_1, %subreg.sub1, %vreg123_2, %subreg.sub2, %vreg123_3, %subreg.sub3
 
-    BUFFER_STORE_DWORD_ADDR64 %vreg123_1, %27, killed %16, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_ADDR64 %vreg123_1, %27, killed %16, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 
 ...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/parse-order-reserved-regs.mir b/llvm/test/CodeGen/MIR/AMDGPU/parse-order-reserved-regs.mir
index 629f7aefd6aff..c48b13e46e207 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/parse-order-reserved-regs.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/parse-order-reserved-regs.mir
@@ -12,7 +12,7 @@
 # CHECK: scratchRSrcReg:  '$sgpr0_sgpr1_sgpr2_sgpr3'
 # CHECK: scratchWaveOffsetReg: '$sgpr50'
 # CHECK: frameOffsetReg:  '$sgpr50'
-# CHECK: renamable $vgpr0 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr50, 4, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+# CHECK: renamable $vgpr0 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr50, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
 name: reserve_correct_register
 tracksRegLiveness: true
 machineFunctionInfo:
@@ -25,6 +25,6 @@ stack:
 
 body:             |
   bb.0:
-    renamable $vgpr0 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr50, 4, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
+    renamable $vgpr0 = BUFFER_LOAD_DWORD_OFFEN %stack.0, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr50, 4, 0, 0, 0, 0, 0, implicit $exec :: (load 4, addrspace 5)
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/target-index-operands.mir b/llvm/test/CodeGen/MIR/AMDGPU/target-index-operands.mir
index b43705eaf8c0d..1864f1cbec2fa 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/target-index-operands.mir
+++ b/llvm/test/CodeGen/MIR/AMDGPU/target-index-operands.mir
@@ -52,7 +52,7 @@ body: |
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
     $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
 ---
@@ -82,6 +82,6 @@ body: |
     $sgpr7 = S_MOV_B32 61440
     $sgpr6 = S_MOV_B32 -1
     $vgpr0 = V_MOV_B32_e32 killed $sgpr2, implicit $exec
-    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, implicit $exec
+    BUFFER_STORE_DWORD_OFFSET killed $vgpr0, $sgpr4_sgpr5_sgpr6_sgpr7, 0, 0, 0, 0, 0, 0, 0, implicit $exec
     S_ENDPGM 0
 ...
diff --git a/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir b/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
index 1d65b3eb5dbf7..bd5b2451a8d76 100644
--- a/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
+++ b/llvm/test/CodeGen/MIR/X86/call-site-info-error2.mir
@@ -1,5 +1,5 @@
 # RUN: not llc -mtriple=x86_64-- -run-pass none -debug-entry-values %s -o - 2>&1 | FileCheck %s
-# CHECK: baa call instruction offset out of range.Unable to reference instruction at bb: 0 at offset:1
+# CHECK: baa call instruction offset out of range. Unable to reference instruction at bb: 0 at offset:1
 --- |
   define dso_local i32 @baa(i32 %a) local_unnamed_addr {
   entry:
diff --git a/llvm/test/CodeGen/MSP430/interrupt.ll b/llvm/test/CodeGen/MSP430/interrupt.ll
index 94fb3bc457a35..dac3e14321d70 100644
--- a/llvm/test/CodeGen/MSP430/interrupt.ll
+++ b/llvm/test/CodeGen/MSP430/interrupt.ll
@@ -50,4 +50,13 @@ entry:
   ret void
 }
 
+; Functions without 'interrupt' attribute don't get a vector section.
+; CHECK-NOT: __interrupt_vector
+; CHECK-LABEL: NMI:
+; CHECK: reti
+define msp430_intrcc void @NMI() #1 {
+  ret void
+}
+
 attributes #0 = { noinline nounwind optnone "interrupt"="2" }
+attributes #1 = { noinline nounwind optnone }
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/var_arg.mir b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/var_arg.mir
new file mode 100644
index 0000000000000..9f07f742e972a
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/instruction-select/var_arg.mir
@@ -0,0 +1,127 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  @.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+  declare void @llvm.va_start(i8*) #0
+  declare void @llvm.va_copy(i8*, i8*) #0
+  declare i32 @printf(i8*, ...)
+
+  define void @testVaCopyArg(i8* %fmt, ...) {
+  entry:
+    %fmt.addr = alloca i8*, align 4
+    %ap = alloca i8*, align 4
+    %aq = alloca i8*, align 4
+    %s = alloca i8*, align 4
+    store i8* %fmt, i8** %fmt.addr, align 4
+    %ap1 = bitcast i8** %ap to i8*
+    call void @llvm.va_start(i8* %ap1)
+    %0 = bitcast i8** %aq to i8*
+    %1 = bitcast i8** %ap to i8*
+    call void @llvm.va_copy(i8* %0, i8* %1)
+    %argp.cur = load i8*, i8** %aq, align 4
+    %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+    store i8* %argp.next, i8** %aq, align 4
+    %2 = bitcast i8* %argp.cur to i8**
+    %3 = load i8*, i8** %2, align 4
+    store i8* %3, i8** %s, align 4
+    %4 = load i8*, i8** %s, align 4
+    %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+    ret void
+  }
+
+...
+---
+name:            testVaCopyArg
+alignment:       4
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$a0' }
+fixedStack:
+  - { id: 0, offset: 12, size: 4, alignment: 4, isImmutable: true }
+  - { id: 1, offset: 8, size: 4, alignment: 8, isImmutable: true }
+  - { id: 2, offset: 4, size: 4, alignment: 4, isImmutable: true }
+  - { id: 3, offset: 4, size: 4, alignment: 4, isImmutable: true }
+stack:
+  - { id: 0, name: fmt.addr, size: 4, alignment: 4 }
+  - { id: 1, name: ap, size: 4, alignment: 4 }
+  - { id: 2, name: aq, size: 4, alignment: 4 }
+  - { id: 3, name: s, size: 4, alignment: 4 }
+machineFunctionInfo: {}
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: testVaCopyArg
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:gpr32 = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gpr32 = COPY $a1
+    ; MIPS32: [[ADDiu:%[0-9]+]]:gpr32 = ADDiu %fixed-stack.1, 0
+    ; MIPS32: SW [[COPY1]], [[ADDiu]], 0 :: (store 4 into %fixed-stack.1)
+    ; MIPS32: [[COPY2:%[0-9]+]]:gpr32 = COPY $a2
+    ; MIPS32: [[ADDiu1:%[0-9]+]]:gpr32 = ADDiu %fixed-stack.2, 0
+    ; MIPS32: SW [[COPY2]], [[ADDiu1]], 0 :: (store 4 into %fixed-stack.2)
+    ; MIPS32: [[COPY3:%[0-9]+]]:gpr32 = COPY $a3
+    ; MIPS32: [[ADDiu2:%[0-9]+]]:gpr32 = ADDiu %fixed-stack.3, 0
+    ; MIPS32: SW [[COPY3]], [[ADDiu2]], 0 :: (store 4 into %fixed-stack.3)
+    ; MIPS32: [[LUi:%[0-9]+]]:gpr32 = LUi target-flags(mips-abs-hi) @.str
+    ; MIPS32: [[ADDiu3:%[0-9]+]]:gpr32 = ADDiu [[LUi]], target-flags(mips-abs-lo) @.str
+    ; MIPS32: [[ADDiu4:%[0-9]+]]:gpr32 = ADDiu %stack.0.fmt.addr, 0
+    ; MIPS32: [[ADDiu5:%[0-9]+]]:gpr32 = ADDiu %stack.1.ap, 0
+    ; MIPS32: [[ADDiu6:%[0-9]+]]:gpr32 = ADDiu %stack.2.aq, 0
+    ; MIPS32: [[ADDiu7:%[0-9]+]]:gpr32 = ADDiu %stack.3.s, 0
+    ; MIPS32: SW [[COPY]], [[ADDiu4]], 0 :: (store 4 into %ir.fmt.addr)
+    ; MIPS32: [[LEA_ADDiu:%[0-9]+]]:gpr32 = LEA_ADDiu %stack.0.fmt.addr, 0
+    ; MIPS32: SW [[LEA_ADDiu]], [[ADDiu5]], 0
+    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[ADDiu5]], 0 :: (load 4)
+    ; MIPS32: SW [[LW]], [[ADDiu6]], 0 :: (store 4)
+    ; MIPS32: [[LW1:%[0-9]+]]:gpr32 = LW [[ADDiu6]], 0 :: (load 4 from %ir.aq)
+    ; MIPS32: [[ORi:%[0-9]+]]:gpr32 = ORi $zero, 4
+    ; MIPS32: [[ADDu:%[0-9]+]]:gpr32 = ADDu [[LW1]], [[ORi]]
+    ; MIPS32: SW [[ADDu]], [[ADDiu6]], 0 :: (store 4 into %ir.aq)
+    ; MIPS32: [[LW2:%[0-9]+]]:gpr32 = LW [[LW1]], 0 :: (load 4 from %ir.2)
+    ; MIPS32: SW [[LW2]], [[ADDiu7]], 0 :: (store 4 into %ir.s)
+    ; MIPS32: [[LW3:%[0-9]+]]:gpr32 = LW [[ADDiu7]], 0 :: (load 4 from %ir.s)
+    ; MIPS32: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: $a0 = COPY [[ADDiu3]]
+    ; MIPS32: $a1 = COPY [[LW3]]
+    ; MIPS32: JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ; MIPS32: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: RetRA
+    %0:gprb(p0) = COPY $a0
+    %1:gprb(s32) = COPY $a1
+    %2:gprb(p0) = G_FRAME_INDEX %fixed-stack.2
+    G_STORE %1(s32), %2(p0) :: (store 4 into %fixed-stack.2)
+    %3:gprb(s32) = COPY $a2
+    %4:gprb(p0) = G_FRAME_INDEX %fixed-stack.1
+    G_STORE %3(s32), %4(p0) :: (store 4 into %fixed-stack.1)
+    %5:gprb(s32) = COPY $a3
+    %6:gprb(p0) = G_FRAME_INDEX %fixed-stack.0
+    G_STORE %5(s32), %6(p0) :: (store 4 into %fixed-stack.0)
+    %18:gprb(p0) = G_GLOBAL_VALUE @.str
+    %17:gprb(p0) = COPY %18(p0)
+    %7:gprb(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+    %8:gpr32(p0) = G_FRAME_INDEX %stack.1.ap
+    %9:gpr32(p0) = G_FRAME_INDEX %stack.2.aq
+    %10:gprb(p0) = G_FRAME_INDEX %stack.3.s
+    G_STORE %0(p0), %7(p0) :: (store 4 into %ir.fmt.addr)
+    G_VASTART %8(p0) :: (store 4 into %ir.ap1, align 1)
+    %19:gpr32 = LW %8(p0), 0 :: (load 4)
+    SW %19, %9(p0), 0 :: (store 4)
+    %11:gprb(p0) = G_LOAD %9(p0) :: (load 4 from %ir.aq)
+    %12:gprb(s32) = G_CONSTANT i32 4
+    %13:gprb(p0) = G_GEP %11, %12(s32)
+    G_STORE %13(p0), %9(p0) :: (store 4 into %ir.aq)
+    %14:gprb(p0) = G_LOAD %11(p0) :: (load 4 from %ir.2)
+    G_STORE %14(p0), %10(p0) :: (store 4 into %ir.s)
+    %15:gprb(p0) = G_LOAD %10(p0) :: (load 4 from %ir.s)
+    ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    $a0 = COPY %17(p0)
+    $a1 = COPY %15(p0)
+    JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    RetRA
+
+...
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll
new file mode 100644
index 0000000000000..16c6b3089fa7a
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/aggregate_struct_return.ll
@@ -0,0 +1,132 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+
+define { float, float } @add_complex_float({ float, float }* %a, { float, float }* %b) {
+  ; MIPS32-LABEL: name: add_complex_float
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+  ; MIPS32:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[COPY2]](p0) :: (load 4 from %ir..realp)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP]](p0) :: (load 4 from %ir..imagp)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY1]](p0)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[COPY3]](p0) :: (load 4 from %ir..realp1)
+  ; MIPS32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C]](s32)
+  ; MIPS32:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir..imagp3)
+  ; MIPS32:   [[FADD:%[0-9]+]]:_(s32) = G_FADD [[LOAD]], [[LOAD2]]
+  ; MIPS32:   [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[LOAD1]], [[LOAD3]]
+  ; MIPS32:   $f0 = COPY [[FADD]](s32)
+  ; MIPS32:   $f2 = COPY [[FADD1]](s32)
+  ; MIPS32:   RetRA implicit $f0, implicit $f2
+entry:
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 0
+  %.real = load float, float* %.realp, align 4
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 1
+  %.imag = load float, float* %.imagp, align 4
+  %.realp1 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 0
+  %.real2 = load float, float* %.realp1, align 4
+  %.imagp3 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 1
+  %.imag4 = load float, float* %.imagp3, align 4
+  %add.r = fadd float %.real, %.real2
+  %add.i = fadd float %.imag, %.imag4
+  %.fca.0.insert = insertvalue { float, float } undef, float %add.r, 0
+  %.fca.1.insert = insertvalue { float, float } %.fca.0.insert, float %add.i, 1
+  ret { float, float } %.fca.1.insert
+}
+
+define { double, double } @add_complex_double({ double, double }* %a, { double, double }* %b) {
+  ; MIPS32-LABEL: name: add_complex_double
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
+  ; MIPS32:   [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[COPY2]](p0) :: (load 8 from %ir..realp)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir..imagp)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY1]](p0)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(s64) = G_LOAD [[COPY3]](p0) :: (load 8 from %ir..realp1)
+  ; MIPS32:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[COPY1]], [[C]](s32)
+  ; MIPS32:   [[LOAD3:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir..imagp3)
+  ; MIPS32:   [[FADD:%[0-9]+]]:_(s64) = G_FADD [[LOAD]], [[LOAD2]]
+  ; MIPS32:   [[FADD1:%[0-9]+]]:_(s64) = G_FADD [[LOAD1]], [[LOAD3]]
+  ; MIPS32:   $d0 = COPY [[FADD]](s64)
+  ; MIPS32:   $d1 = COPY [[FADD1]](s64)
+  ; MIPS32:   RetRA implicit $d0, implicit $d1
+entry:
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 0
+  %.real = load double, double* %.realp, align 8
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 1
+  %.imag = load double, double* %.imagp, align 8
+  %.realp1 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 0
+  %.real2 = load double, double* %.realp1, align 8
+  %.imagp3 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 1
+  %.imag4 = load double, double* %.imagp3, align 8
+  %add.r = fadd double %.real, %.real2
+  %add.i = fadd double %.imag, %.imag4
+  %.fca.0.insert = insertvalue { double, double } undef, double %add.r, 0
+  %.fca.1.insert = insertvalue { double, double } %.fca.0.insert, double %add.i, 1
+  ret { double, double } %.fca.1.insert
+}
+
+declare { float, float } @ret_complex_float()
+define void @call_ret_complex_float({ float, float }* %z) {
+  ; MIPS32-LABEL: name: call_ret_complex_float
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   JAL @ret_complex_float, csr_o32, implicit-def $ra, implicit-def $sp, implicit-def $f0, implicit-def $f2
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $f0
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $f2
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; MIPS32:   G_STORE [[COPY1]](s32), [[COPY3]](p0) :: (store 4 into %ir..realp)
+  ; MIPS32:   G_STORE [[COPY2]](s32), [[GEP]](p0) :: (store 4 into %ir..imagp)
+  ; MIPS32:   RetRA
+entry:
+  %call = call { float, float } @ret_complex_float()
+  %0 = extractvalue { float, float } %call, 0
+  %1 = extractvalue { float, float } %call, 1
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 1
+  store float %0, float* %.realp, align 4
+  store float %1, float* %.imagp, align 4
+  ret void
+}
+
+declare { double, double } @ret_complex_double()
+define void @call_ret_complex_double({ double, double }* %z) {
+  ; MIPS32-LABEL: name: call_ret_complex_double
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   JAL @ret_complex_double, csr_o32, implicit-def $ra, implicit-def $sp, implicit-def $d0, implicit-def $d1
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s64) = COPY $d0
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s64) = COPY $d1
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[COPY]], [[C]](s32)
+  ; MIPS32:   G_STORE [[COPY1]](s64), [[COPY3]](p0) :: (store 8 into %ir..realp)
+  ; MIPS32:   G_STORE [[COPY2]](s64), [[GEP]](p0) :: (store 8 into %ir..imagp)
+  ; MIPS32:   RetRA
+entry:
+  %call = call { double, double } @ret_complex_double()
+  %0 = extractvalue { double, double } %call, 0
+  %1 = extractvalue { double, double } %call, 1
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 1
+  store double %0, double* %.realp, align 8
+  store double %1, double* %.imagp, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
index 0e1117b8955ee..4cc956c2040c7 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll
@@ -153,7 +153,7 @@ define void @call_symbol(i8* nocapture readonly %src, i8* nocapture %dest, i32 s
   ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
-  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32) :: (store 1 into %ir.dest), (load 1 from %ir.src)
+  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), 0 :: (store 1 into %ir.dest), (load 1 from %ir.src)
   ; MIPS32:   RetRA
   ; MIPS32_PIC-LABEL: name: call_symbol
   ; MIPS32_PIC: bb.1.entry:
@@ -161,7 +161,7 @@ define void @call_symbol(i8* nocapture readonly %src, i8* nocapture %dest, i32 s
   ; MIPS32_PIC:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32_PIC:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32_PIC:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
-  ; MIPS32_PIC:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32) :: (store 1 into %ir.dest), (load 1 from %ir.src)
+  ; MIPS32_PIC:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), 0 :: (store 1 into %ir.dest), (load 1 from %ir.src)
   ; MIPS32_PIC:   RetRA
 entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 %length, i1 false)
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll
new file mode 100644
index 0000000000000..b3b68175ad074
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/var_arg.ll
@@ -0,0 +1,66 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -O0 -mtriple=mipsel-linux-gnu -global-isel -stop-after=irtranslator -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+
+@.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+declare i32 @printf(i8*, ...)
+
+define void @testVaCopyArg(i8* %fmt, ...) {
+  ; MIPS32-LABEL: name: testVaCopyArg
+  ; MIPS32: bb.1.entry:
+  ; MIPS32:   liveins: $a0, $a1, $a2, $a3
+  ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+  ; MIPS32:   [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+  ; MIPS32:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
+  ; MIPS32:   G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store 4 into %fixed-stack.2)
+  ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+  ; MIPS32:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+  ; MIPS32:   G_STORE [[COPY2]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %fixed-stack.1)
+  ; MIPS32:   [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+  ; MIPS32:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0
+  ; MIPS32:   G_STORE [[COPY3]](s32), [[FRAME_INDEX2]](p0) :: (store 4 into %fixed-stack.0)
+  ; MIPS32:   [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @.str
+  ; MIPS32:   [[COPY4:%[0-9]+]]:_(p0) = COPY [[GV]](p0)
+  ; MIPS32:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+  ; MIPS32:   [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.ap
+  ; MIPS32:   [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.2.aq
+  ; MIPS32:   [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.s
+  ; MIPS32:   G_STORE [[COPY]](p0), [[FRAME_INDEX3]](p0) :: (store 4 into %ir.fmt.addr)
+  ; MIPS32:   G_VASTART [[FRAME_INDEX4]](p0) :: (store 4 into %ir.ap1, align 1)
+  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX5]](p0), [[FRAME_INDEX4]](p0)
+  ; MIPS32:   [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX5]](p0) :: (load 4 from %ir.aq)
+  ; MIPS32:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; MIPS32:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[LOAD]], [[C]](s32)
+  ; MIPS32:   G_STORE [[GEP]](p0), [[FRAME_INDEX5]](p0) :: (store 4 into %ir.aq)
+  ; MIPS32:   [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.2)
+  ; MIPS32:   G_STORE [[LOAD1]](p0), [[FRAME_INDEX6]](p0) :: (store 4 into %ir.s)
+  ; MIPS32:   [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX6]](p0) :: (load 4 from %ir.s)
+  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   $a0 = COPY [[COPY4]](p0)
+  ; MIPS32:   $a1 = COPY [[LOAD2]](p0)
+  ; MIPS32:   JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+  ; MIPS32:   [[COPY5:%[0-9]+]]:_(s32) = COPY $v0
+  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   RetRA
+entry:
+  %fmt.addr = alloca i8*, align 4
+  %ap = alloca i8*, align 4
+  %aq = alloca i8*, align 4
+  %s = alloca i8*, align 4
+  store i8* %fmt, i8** %fmt.addr, align 4
+  %ap1 = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* %ap1)
+  %0 = bitcast i8** %aq to i8*
+  %1 = bitcast i8** %ap to i8*
+  call void @llvm.va_copy(i8* %0, i8* %1)
+  %argp.cur = load i8*, i8** %aq, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %aq, align 4
+  %2 = bitcast i8* %argp.cur to i8**
+  %3 = load i8*, i8** %2, align 4
+  store i8* %3, i8** %s, align 4
+  %4 = load i8*, i8** %s, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/dyn_stackalloc.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/dyn_stackalloc.mir
index 75d247439bd68..50776623965c2 100644
--- a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/dyn_stackalloc.mir
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/dyn_stackalloc.mir
@@ -73,7 +73,7 @@ body:             |
     %8:_(s32) = G_CONSTANT i32 -8
     %9:_(s32) = G_AND %7, %8
     %10:_(p0) = G_DYN_STACKALLOC %9(s32), 0
-    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %10(p0), %0(s8), %1(s32) :: (store 1 into %ir.vla)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %10(p0), %0(s8), %1(s32), 0 :: (store 1 into %ir.vla)
     %11:_(p0) = G_GEP %10, %1(s32)
     %12:_(p0) = COPY %11(p0)
     G_STORE %13(s8), %12(p0) :: (store 1 into %ir.arrayidx)
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/legalizer/var_arg.mir b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/var_arg.mir
new file mode 100644
index 0000000000000..98a92d773c154
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/legalizer/var_arg.mir
@@ -0,0 +1,123 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  @.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+  declare void @llvm.va_start(i8*) #0
+  declare void @llvm.va_copy(i8*, i8*) #0
+  declare i32 @printf(i8*, ...)
+
+  define void @testVaCopyArg(i8* %fmt, ...) {
+  entry:
+    %fmt.addr = alloca i8*, align 4
+    %ap = alloca i8*, align 4
+    %aq = alloca i8*, align 4
+    %s = alloca i8*, align 4
+    store i8* %fmt, i8** %fmt.addr, align 4
+    %ap1 = bitcast i8** %ap to i8*
+    call void @llvm.va_start(i8* %ap1)
+    %0 = bitcast i8** %aq to i8*
+    %1 = bitcast i8** %ap to i8*
+    call void @llvm.va_copy(i8* %0, i8* %1)
+    %argp.cur = load i8*, i8** %aq, align 4
+    %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+    store i8* %argp.next, i8** %aq, align 4
+    %2 = bitcast i8* %argp.cur to i8**
+    %3 = load i8*, i8** %2, align 4
+    store i8* %3, i8** %s, align 4
+    %4 = load i8*, i8** %s, align 4
+    %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+    ret void
+  }
+
+...
+---
+name:            testVaCopyArg
+alignment:       4
+tracksRegLiveness: true
+liveins:
+  - { reg: '$a0' }
+fixedStack:
+  - { id: 0, offset: 12, size: 4, alignment: 4, isImmutable: true }
+  - { id: 1, offset: 8, size: 4, alignment: 8, isImmutable: true }
+  - { id: 2, offset: 4, size: 4, alignment: 4, isImmutable: true }
+  - { id: 3, offset: 4, size: 4, alignment: 4, isImmutable: true }
+stack:
+  - { id: 0, name: fmt.addr, size: 4, alignment: 4 }
+  - { id: 1, name: ap, size: 4, alignment: 4 }
+  - { id: 2, name: aq, size: 4, alignment: 4 }
+  - { id: 3, name: s, size: 4, alignment: 4 }
+machineFunctionInfo: {}
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: testVaCopyArg
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:_(s32) = COPY $a1
+    ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1
+    ; MIPS32: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store 4 into %fixed-stack.1)
+    ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
+    ; MIPS32: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.2
+    ; MIPS32: G_STORE [[COPY2]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %fixed-stack.2)
+    ; MIPS32: [[COPY3:%[0-9]+]]:_(s32) = COPY $a3
+    ; MIPS32: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.3
+    ; MIPS32: G_STORE [[COPY3]](s32), [[FRAME_INDEX2]](p0) :: (store 4 into %fixed-stack.3)
+    ; MIPS32: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @.str
+    ; MIPS32: [[COPY4:%[0-9]+]]:_(p0) = COPY [[GV]](p0)
+    ; MIPS32: [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+    ; MIPS32: [[FRAME_INDEX4:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.ap
+    ; MIPS32: [[FRAME_INDEX5:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.2.aq
+    ; MIPS32: [[FRAME_INDEX6:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.s
+    ; MIPS32: G_STORE [[COPY]](p0), [[FRAME_INDEX3]](p0) :: (store 4 into %ir.fmt.addr)
+    ; MIPS32: G_VASTART [[FRAME_INDEX4]](p0) :: (store 4 into %ir.ap1, align 1)
+    ; MIPS32: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX4]](p0) :: (load 4)
+    ; MIPS32: G_STORE [[LOAD]](p0), [[FRAME_INDEX5]](p0) :: (store 4)
+    ; MIPS32: [[LOAD1:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX5]](p0) :: (load 4 from %ir.aq)
+    ; MIPS32: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+    ; MIPS32: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[LOAD1]], [[C]](s32)
+    ; MIPS32: G_STORE [[GEP]](p0), [[FRAME_INDEX5]](p0) :: (store 4 into %ir.aq)
+    ; MIPS32: [[LOAD2:%[0-9]+]]:_(p0) = G_LOAD [[LOAD1]](p0) :: (load 4 from %ir.2)
+    ; MIPS32: G_STORE [[LOAD2]](p0), [[FRAME_INDEX6]](p0) :: (store 4 into %ir.s)
+    ; MIPS32: [[LOAD3:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX6]](p0) :: (load 4 from %ir.s)
+    ; MIPS32: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: $a0 = COPY [[COPY4]](p0)
+    ; MIPS32: $a1 = COPY [[LOAD3]](p0)
+    ; MIPS32: JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ; MIPS32: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(p0) = G_FRAME_INDEX %fixed-stack.2
+    G_STORE %1(s32), %2(p0) :: (store 4 into %fixed-stack.2)
+    %3:_(s32) = COPY $a2
+    %4:_(p0) = G_FRAME_INDEX %fixed-stack.1
+    G_STORE %3(s32), %4(p0) :: (store 4 into %fixed-stack.1)
+    %5:_(s32) = COPY $a3
+    %6:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    G_STORE %5(s32), %6(p0) :: (store 4 into %fixed-stack.0)
+    %18:_(p0) = G_GLOBAL_VALUE @.str
+    %17:_(p0) = COPY %18(p0)
+    %7:_(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+    %8:_(p0) = G_FRAME_INDEX %stack.1.ap
+    %9:_(p0) = G_FRAME_INDEX %stack.2.aq
+    %10:_(p0) = G_FRAME_INDEX %stack.3.s
+    G_STORE %0(p0), %7(p0) :: (store 4 into %ir.fmt.addr)
+    G_VASTART %8(p0) :: (store 4 into %ir.ap1, align 1)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), %9(p0), %8(p0)
+    %11:_(p0) = G_LOAD %9(p0) :: (load 4 from %ir.aq)
+    %12:_(s32) = G_CONSTANT i32 4
+    %13:_(p0) = G_GEP %11, %12(s32)
+    G_STORE %13(p0), %9(p0) :: (store 4 into %ir.aq)
+    %14:_(p0) = G_LOAD %11(p0) :: (load 4 from %ir.2)
+    G_STORE %14(p0), %10(p0) :: (store 4 into %ir.s)
+    %15:_(p0) = G_LOAD %10(p0) :: (load 4 from %ir.s)
+    ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    $a0 = COPY %17(p0)
+    $a1 = COPY %15(p0)
+    JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    RetRA
+
+...
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll
new file mode 100644
index 0000000000000..a9f49c025b953
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/aggregate_struct_return.ll
@@ -0,0 +1,114 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+define { float, float } @add_complex_float({ float, float }* %a, { float, float }* %b) {
+; MIPS32-LABEL: add_complex_float:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    lwc1 $f0, 0($4)
+; MIPS32-NEXT:    lwc1 $f1, 4($4)
+; MIPS32-NEXT:    lwc1 $f2, 0($5)
+; MIPS32-NEXT:    lwc1 $f3, 4($5)
+; MIPS32-NEXT:    add.s $f0, $f0, $f2
+; MIPS32-NEXT:    add.s $f2, $f1, $f3
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 0
+  %.real = load float, float* %.realp, align 4
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %a, i32 0, i32 1
+  %.imag = load float, float* %.imagp, align 4
+  %.realp1 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 0
+  %.real2 = load float, float* %.realp1, align 4
+  %.imagp3 = getelementptr inbounds { float, float }, { float, float }* %b, i32 0, i32 1
+  %.imag4 = load float, float* %.imagp3, align 4
+  %add.r = fadd float %.real, %.real2
+  %add.i = fadd float %.imag, %.imag4
+  %.fca.0.insert = insertvalue { float, float } undef, float %add.r, 0
+  %.fca.1.insert = insertvalue { float, float } %.fca.0.insert, float %add.i, 1
+  ret { float, float } %.fca.1.insert
+}
+
+define { double, double } @add_complex_double({ double, double }* %a, { double, double }* %b) {
+; MIPS32-LABEL: add_complex_double:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    ldc1 $f0, 0($4)
+; MIPS32-NEXT:    ldc1 $f2, 8($4)
+; MIPS32-NEXT:    ldc1 $f4, 0($5)
+; MIPS32-NEXT:    ldc1 $f6, 8($5)
+; MIPS32-NEXT:    add.d $f0, $f0, $f4
+; MIPS32-NEXT:    add.d $f2, $f2, $f6
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 0
+  %.real = load double, double* %.realp, align 8
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %a, i32 0, i32 1
+  %.imag = load double, double* %.imagp, align 8
+  %.realp1 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 0
+  %.real2 = load double, double* %.realp1, align 8
+  %.imagp3 = getelementptr inbounds { double, double }, { double, double }* %b, i32 0, i32 1
+  %.imag4 = load double, double* %.imagp3, align 8
+  %add.r = fadd double %.real, %.real2
+  %add.i = fadd double %.imag, %.imag4
+  %.fca.0.insert = insertvalue { double, double } undef, double %add.r, 0
+  %.fca.1.insert = insertvalue { double, double } %.fca.0.insert, double %add.i, 1
+  ret { double, double } %.fca.1.insert
+}
+
+declare { float, float } @ret_complex_float()
+define void @call_ret_complex_float({ float, float }* %z) {
+; MIPS32-LABEL: call_ret_complex_float:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    sw $4, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jal ret_complex_float
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    swc1 $f0, 0($1)
+; MIPS32-NEXT:    swc1 $f2, 4($1)
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %call = call { float, float } @ret_complex_float()
+  %0 = extractvalue { float, float } %call, 0
+  %1 = extractvalue { float, float } %call, 1
+  %.realp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { float, float }, { float, float }* %z, i32 0, i32 1
+  store float %0, float* %.realp, align 4
+  store float %1, float* %.imagp, align 4
+  ret void
+}
+
+declare { double, double } @ret_complex_double()
+define void @call_ret_complex_double({ double, double }* %z) {
+; MIPS32-LABEL: call_ret_complex_double:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -24
+; MIPS32-NEXT:    .cfi_def_cfa_offset 24
+; MIPS32-NEXT:    sw $ra, 20($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    sw $4, 16($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    jal ret_complex_double
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $1, 16($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    sdc1 $f0, 0($1)
+; MIPS32-NEXT:    sdc1 $f2, 8($1)
+; MIPS32-NEXT:    lw $ra, 20($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 24
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %call = call { double, double } @ret_complex_double()
+  %0 = extractvalue { double, double } %call, 0
+  %1 = extractvalue { double, double } %call, 1
+  %.realp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 0
+  %.imagp = getelementptr inbounds { double, double }, { double, double }* %z, i32 0, i32 1
+  store double %0, double* %.realp, align 8
+  store double %1, double* %.imagp, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll
new file mode 100644
index 0000000000000..91a82f2a7fccc
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/llvm-ir/var_arg.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc  -O0 -mtriple=mipsel-linux-gnu -global-isel  -verify-machineinstrs %s -o -| FileCheck %s -check-prefixes=MIPS32
+
+@.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+declare void @llvm.va_start(i8*)
+declare void @llvm.va_copy(i8*, i8*)
+declare i32 @printf(i8*, ...)
+
+define void @testVaCopyArg(i8* %fmt, ...) {
+; MIPS32-LABEL: testVaCopyArg:
+; MIPS32:       # %bb.0: # %entry
+; MIPS32-NEXT:    addiu $sp, $sp, -40
+; MIPS32-NEXT:    .cfi_def_cfa_offset 40
+; MIPS32-NEXT:    sw $ra, 36($sp) # 4-byte Folded Spill
+; MIPS32-NEXT:    .cfi_offset 31, -4
+; MIPS32-NEXT:    addiu $1, $sp, 44
+; MIPS32-NEXT:    sw $5, 0($1)
+; MIPS32-NEXT:    addiu $1, $sp, 48
+; MIPS32-NEXT:    sw $6, 0($1)
+; MIPS32-NEXT:    addiu $1, $sp, 52
+; MIPS32-NEXT:    sw $7, 0($1)
+; MIPS32-NEXT:    lui $1, %hi($.str)
+; MIPS32-NEXT:    addiu $1, $1, %lo($.str)
+; MIPS32-NEXT:    addiu $2, $sp, 32
+; MIPS32-NEXT:    addiu $3, $sp, 28
+; MIPS32-NEXT:    addiu $5, $sp, 24
+; MIPS32-NEXT:    addiu $6, $sp, 20
+; MIPS32-NEXT:    sw $4, 0($2)
+; MIPS32-NEXT:    addiu $2, $sp, 44
+; MIPS32-NEXT:    sw $2, 0($3)
+; MIPS32-NEXT:    lw $2, 0($3)
+; MIPS32-NEXT:    sw $2, 0($5)
+; MIPS32-NEXT:    lw $2, 0($5)
+; MIPS32-NEXT:    ori $3, $zero, 4
+; MIPS32-NEXT:    addu $3, $2, $3
+; MIPS32-NEXT:    sw $3, 0($5)
+; MIPS32-NEXT:    lw $2, 0($2)
+; MIPS32-NEXT:    sw $2, 0($6)
+; MIPS32-NEXT:    lw $5, 0($6)
+; MIPS32-NEXT:    move $4, $1
+; MIPS32-NEXT:    jal printf
+; MIPS32-NEXT:    nop
+; MIPS32-NEXT:    lw $ra, 36($sp) # 4-byte Folded Reload
+; MIPS32-NEXT:    addiu $sp, $sp, 40
+; MIPS32-NEXT:    jr $ra
+; MIPS32-NEXT:    nop
+entry:
+  %fmt.addr = alloca i8*, align 4
+  %ap = alloca i8*, align 4
+  %aq = alloca i8*, align 4
+  %s = alloca i8*, align 4
+  store i8* %fmt, i8** %fmt.addr, align 4
+  %ap1 = bitcast i8** %ap to i8*
+  call void @llvm.va_start(i8* %ap1)
+  %0 = bitcast i8** %aq to i8*
+  %1 = bitcast i8** %ap to i8*
+  call void @llvm.va_copy(i8* %0, i8* %1)
+  %argp.cur = load i8*, i8** %aq, align 4
+  %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+  store i8* %argp.next, i8** %aq, align 4
+  %2 = bitcast i8* %argp.cur to i8**
+  %3 = load i8*, i8** %2, align 4
+  store i8* %3, i8** %s, align 4
+  %4 = load i8*, i8** %s, align 4
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+  ret void
+}
diff --git a/llvm/test/CodeGen/Mips/GlobalISel/regbankselect/var_arg.mir b/llvm/test/CodeGen/Mips/GlobalISel/regbankselect/var_arg.mir
new file mode 100644
index 0000000000000..d249396854de5
--- /dev/null
+++ b/llvm/test/CodeGen/Mips/GlobalISel/regbankselect/var_arg.mir
@@ -0,0 +1,125 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -O0 -mtriple=mipsel-linux-gnu -run-pass=regbankselect -verify-machineinstrs %s -o - | FileCheck %s -check-prefixes=MIPS32
+--- |
+
+  @.str = private unnamed_addr constant [11 x i8] c"string %s\0A\00", align 1
+  declare void @llvm.va_start(i8*) #0
+  declare void @llvm.va_copy(i8*, i8*) #0
+  declare i32 @printf(i8*, ...)
+
+  define void @testVaCopyArg(i8* %fmt, ...) {
+  entry:
+    %fmt.addr = alloca i8*, align 4
+    %ap = alloca i8*, align 4
+    %aq = alloca i8*, align 4
+    %s = alloca i8*, align 4
+    store i8* %fmt, i8** %fmt.addr, align 4
+    %ap1 = bitcast i8** %ap to i8*
+    call void @llvm.va_start(i8* %ap1)
+    %0 = bitcast i8** %aq to i8*
+    %1 = bitcast i8** %ap to i8*
+    call void @llvm.va_copy(i8* %0, i8* %1)
+    %argp.cur = load i8*, i8** %aq, align 4
+    %argp.next = getelementptr inbounds i8, i8* %argp.cur, i32 4
+    store i8* %argp.next, i8** %aq, align 4
+    %2 = bitcast i8* %argp.cur to i8**
+    %3 = load i8*, i8** %2, align 4
+    store i8* %3, i8** %s, align 4
+    %4 = load i8*, i8** %s, align 4
+    %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i8* %4)
+    ret void
+  }
+
+...
+---
+name:            testVaCopyArg
+alignment:       4
+legalized:       true
+tracksRegLiveness: true
+liveins:
+  - { reg: '$a0' }
+fixedStack:
+  - { id: 0, offset: 12, size: 4, alignment: 4, isImmutable: true }
+  - { id: 1, offset: 8, size: 4, alignment: 8, isImmutable: true }
+  - { id: 2, offset: 4, size: 4, alignment: 4, isImmutable: true }
+  - { id: 3, offset: 4, size: 4, alignment: 4, isImmutable: true }
+stack:
+  - { id: 0, name: fmt.addr, size: 4, alignment: 4 }
+  - { id: 1, name: ap, size: 4, alignment: 4 }
+  - { id: 2, name: aq, size: 4, alignment: 4 }
+  - { id: 3, name: s, size: 4, alignment: 4 }
+machineFunctionInfo: {}
+body:             |
+  bb.1.entry:
+    liveins: $a0, $a1, $a2, $a3
+
+    ; MIPS32-LABEL: name: testVaCopyArg
+    ; MIPS32: liveins: $a0, $a1, $a2, $a3
+    ; MIPS32: [[COPY:%[0-9]+]]:gprb(p0) = COPY $a0
+    ; MIPS32: [[COPY1:%[0-9]+]]:gprb(s32) = COPY $a1
+    ; MIPS32: [[FRAME_INDEX:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %fixed-stack.1
+    ; MIPS32: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store 4 into %fixed-stack.1)
+    ; MIPS32: [[COPY2:%[0-9]+]]:gprb(s32) = COPY $a2
+    ; MIPS32: [[FRAME_INDEX1:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %fixed-stack.2
+    ; MIPS32: G_STORE [[COPY2]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %fixed-stack.2)
+    ; MIPS32: [[COPY3:%[0-9]+]]:gprb(s32) = COPY $a3
+    ; MIPS32: [[FRAME_INDEX2:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %fixed-stack.3
+    ; MIPS32: G_STORE [[COPY3]](s32), [[FRAME_INDEX2]](p0) :: (store 4 into %fixed-stack.3)
+    ; MIPS32: [[GV:%[0-9]+]]:gprb(p0) = G_GLOBAL_VALUE @.str
+    ; MIPS32: [[COPY4:%[0-9]+]]:gprb(p0) = COPY [[GV]](p0)
+    ; MIPS32: [[FRAME_INDEX3:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+    ; MIPS32: [[FRAME_INDEX4:%[0-9]+]]:gpr32(p0) = G_FRAME_INDEX %stack.1.ap
+    ; MIPS32: [[FRAME_INDEX5:%[0-9]+]]:gpr32(p0) = G_FRAME_INDEX %stack.2.aq
+    ; MIPS32: [[FRAME_INDEX6:%[0-9]+]]:gprb(p0) = G_FRAME_INDEX %stack.3.s
+    ; MIPS32: G_STORE [[COPY]](p0), [[FRAME_INDEX3]](p0) :: (store 4 into %ir.fmt.addr)
+    ; MIPS32: G_VASTART [[FRAME_INDEX4]](p0) :: (store 4 into %ir.ap1, align 1)
+    ; MIPS32: [[LW:%[0-9]+]]:gpr32 = LW [[FRAME_INDEX4]](p0), 0 :: (load 4)
+    ; MIPS32: SW [[LW]], [[FRAME_INDEX5]](p0), 0 :: (store 4)
+    ; MIPS32: [[LOAD:%[0-9]+]]:gprb(p0) = G_LOAD [[FRAME_INDEX5]](p0) :: (load 4 from %ir.aq)
+    ; MIPS32: [[C:%[0-9]+]]:gprb(s32) = G_CONSTANT i32 4
+    ; MIPS32: [[GEP:%[0-9]+]]:gprb(p0) = G_GEP [[LOAD]], [[C]](s32)
+    ; MIPS32: G_STORE [[GEP]](p0), [[FRAME_INDEX5]](p0) :: (store 4 into %ir.aq)
+    ; MIPS32: [[LOAD1:%[0-9]+]]:gprb(p0) = G_LOAD [[LOAD]](p0) :: (load 4 from %ir.2)
+    ; MIPS32: G_STORE [[LOAD1]](p0), [[FRAME_INDEX6]](p0) :: (store 4 into %ir.s)
+    ; MIPS32: [[LOAD2:%[0-9]+]]:gprb(p0) = G_LOAD [[FRAME_INDEX6]](p0) :: (load 4 from %ir.s)
+    ; MIPS32: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: $a0 = COPY [[COPY4]](p0)
+    ; MIPS32: $a1 = COPY [[LOAD2]](p0)
+    ; MIPS32: JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ; MIPS32: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    ; MIPS32: RetRA
+    %0:_(p0) = COPY $a0
+    %1:_(s32) = COPY $a1
+    %2:_(p0) = G_FRAME_INDEX %fixed-stack.2
+    G_STORE %1(s32), %2(p0) :: (store 4 into %fixed-stack.2)
+    %3:_(s32) = COPY $a2
+    %4:_(p0) = G_FRAME_INDEX %fixed-stack.1
+    G_STORE %3(s32), %4(p0) :: (store 4 into %fixed-stack.1)
+    %5:_(s32) = COPY $a3
+    %6:_(p0) = G_FRAME_INDEX %fixed-stack.0
+    G_STORE %5(s32), %6(p0) :: (store 4 into %fixed-stack.0)
+    %18:_(p0) = G_GLOBAL_VALUE @.str
+    %17:_(p0) = COPY %18(p0)
+    %7:_(p0) = G_FRAME_INDEX %stack.0.fmt.addr
+    %8:gpr32(p0) = G_FRAME_INDEX %stack.1.ap
+    %9:gpr32(p0) = G_FRAME_INDEX %stack.2.aq
+    %10:_(p0) = G_FRAME_INDEX %stack.3.s
+    G_STORE %0(p0), %7(p0) :: (store 4 into %ir.fmt.addr)
+    G_VASTART %8(p0) :: (store 4 into %ir.ap1, align 1)
+    %19:gpr32 = LW %8(p0), 0 :: (load 4)
+    SW %19, %9(p0), 0 :: (store 4)
+    %11:_(p0) = G_LOAD %9(p0) :: (load 4 from %ir.aq)
+    %12:_(s32) = G_CONSTANT i32 4
+    %13:_(p0) = G_GEP %11, %12(s32)
+    G_STORE %13(p0), %9(p0) :: (store 4 into %ir.aq)
+    %14:_(p0) = G_LOAD %11(p0) :: (load 4 from %ir.2)
+    G_STORE %14(p0), %10(p0) :: (store 4 into %ir.s)
+    %15:_(p0) = G_LOAD %10(p0) :: (load 4 from %ir.s)
+    ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
+    $a0 = COPY %17(p0)
+    $a1 = COPY %15(p0)
+    JAL @printf, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit-def $v0
+    ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+    RetRA
+
+...
diff --git a/llvm/test/CodeGen/Mips/address-selection.ll b/llvm/test/CodeGen/Mips/address-selection.ll
index 36a1d5117be7b..1e0a9a42e127b 100644
--- a/llvm/test/CodeGen/Mips/address-selection.ll
+++ b/llvm/test/CodeGen/Mips/address-selection.ll
@@ -1,8 +1,10 @@
 ; RUN: llc -march=mips < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS
-; RUN: llc -march=mips -relocation-model=pic -mxgot < %s -debug 2>&1 | FileCheck %s --check-prefix=MIPS-XGOT
+; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot < %s \
+; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MIPS-XGOT
 
 ; RUN: llc -march=mips -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM
-; RUN: llc -march=mips -relocation-model=pic -mxgot -mattr=+micromips < %s -debug 2>&1 | FileCheck %s --check-prefix=MM-XGOT
+; RUN: llc -march=mips -relocation-model=pic -mattr=+xgot,+micromips < %s \
+; RUN:     -debug 2>&1 | FileCheck %s --check-prefix=MM-XGOT
 
 ; REQUIRES: asserts
 
diff --git a/llvm/test/CodeGen/Mips/biggot.ll b/llvm/test/CodeGen/Mips/biggot.ll
index 305dcf8557298..aed1c487e7dba 100644
--- a/llvm/test/CodeGen/Mips/biggot.ll
+++ b/llvm/test/CodeGen/Mips/biggot.ll
@@ -1,9 +1,11 @@
-; RUN: llc -march=mipsel -mxgot -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mxgot -relocation-model=pic < %s | \
-; RUN: FileCheck %s -check-prefix=N64
-; RUN: llc -march=mipsel -mxgot -relocation-model=pic -fast-isel < %s | FileCheck %s -check-prefix=O32
-; RUN: llc -march=mips64el -mcpu=mips64r2 -mxgot -relocation-model=pic -fast-isel < %s | \
-; RUN: FileCheck %s -check-prefix=N64
+; RUN: llc -march=mipsel -mattr=+xgot \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
+; RUN: llc -march=mipsel -mattr=+xgot -fast-isel \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=O32
+; RUN: llc -march=mips64el -mcpu=mips64r2 -mattr=+xgot -fast-isel \
+; RUN:     -relocation-model=pic < %s | FileCheck %s -check-prefix=N64
 
 @v0 = external global i32
 
diff --git a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
index 2956a175dd70e..5b72917a48179 100644
--- a/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
+++ b/llvm/test/CodeGen/Mips/mirparser/target-flags-pic-mxgot-tls.mir
@@ -1,5 +1,5 @@
 # RUN: llc -march=mips64 -target-abi n64 -start-before=finalize-isel \
-# RUN:     -stop-after=finalize-isel -relocation-model=pic -mxgot \
+# RUN:     -stop-after=finalize-isel -relocation-model=pic -mattr=+xgot \
 # RUN:     -o /dev/null %s
 
 # A simple test to show that we can parse the target specific flags: gpoff-hi,
diff --git a/llvm/test/CodeGen/Mips/msa/arithmetic.ll b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
index 413f045662dc2..0caafb6cc0982 100644
--- a/llvm/test/CodeGen/Mips/msa/arithmetic.ll
+++ b/llvm/test/CodeGen/Mips/msa/arithmetic.ll
@@ -190,6 +190,20 @@ define void @sub_v16i8_i(<16 x i8>* %c, <16 x i8>* %a) nounwind {
   ret void
 }
 
+define void @sub_v16i8_i_negated(<16 x i8>* %c, <16 x i8>* %a) nounwind {
+; ALL-LABEL: sub_v16i8_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.b $w0, 0($5)
+; ALL-NEXT:    subvi.b $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.b $w0, 0($4)
+  %1 = load <16 x i8>, <16 x i8>* %a
+  %2 = add <16 x i8> %1, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+              i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+  store <16 x i8> %2, <16 x i8>* %c
+  ret void
+}
+
 define void @sub_v8i16_i(<8 x i16>* %c, <8 x i16>* %a) nounwind {
 ; ALL-LABEL: sub_v8i16_i:
 ; ALL:       # %bb.0:
@@ -204,6 +218,20 @@ define void @sub_v8i16_i(<8 x i16>* %c, <8 x i16>* %a) nounwind {
   ret void
 }
 
+define void @sub_v8i16_i_negated(<8 x i16>* %c, <8 x i16>* %a) nounwind {
+; ALL-LABEL: sub_v8i16_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.h $w0, 0($5)
+; ALL-NEXT:    subvi.h $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.h $w0, 0($4)
+  %1 = load <8 x i16>, <8 x i16>* %a
+  %2 = add <8 x i16> %1, <i16 -1, i16 -1, i16 -1, i16 -1,
+              i16 -1, i16 -1, i16 -1, i16 -1>
+  store <8 x i16> %2, <8 x i16>* %c
+  ret void
+}
+
 define void @sub_v4i32_i(<4 x i32>* %c, <4 x i32>* %a) nounwind {
 ; ALL-LABEL: sub_v4i32_i:
 ; ALL:       # %bb.0:
@@ -217,6 +245,19 @@ define void @sub_v4i32_i(<4 x i32>* %c, <4 x i32>* %a) nounwind {
   ret void
 }
 
+define void @sub_v4i32_i_negated(<4 x i32>* %c, <4 x i32>* %a) nounwind {
+; ALL-LABEL: sub_v4i32_i_negated:
+; ALL:       # %bb.0:
+; ALL-NEXT:    ld.w $w0, 0($5)
+; ALL-NEXT:    subvi.w $w0, $w0, 1
+; ALL-NEXT:    jr $ra
+; ALL-NEXT:    st.w $w0, 0($4)
+  %1 = load <4 x i32>, <4 x i32>* %a
+  %2 = add <4 x i32> %1, <i32 -1, i32 -1, i32 -1, i32 -1>
+  store <4 x i32> %2, <4 x i32>* %c
+  ret void
+}
+
 define void @sub_v2i64_i(<2 x i64>* %c, <2 x i64>* %a) nounwind {
 ; ALL-LABEL: sub_v2i64_i:
 ; ALL:       # %bb.0:
@@ -230,6 +271,29 @@ define void @sub_v2i64_i(<2 x i64>* %c, <2 x i64>* %a) nounwind {
   ret void
 }
 
+define void @sub_v2i64_i_negated(<2 x i64>* %c, <2 x i64>* %a) nounwind {
+; MIPS-LABEL: sub_v2i64_i_negated:
+; MIPS:       # %bb.0:
+; MIPS-NEXT:    ldi.b $w0, -1
+; MIPS-NEXT:    shf.w $w0, $w0, 177
+; MIPS-NEXT:    ld.d $w1, 0($5)
+; MIPS-NEXT:    addv.d $w0, $w1, $w0
+; MIPS-NEXT:    jr $ra
+; MIPS-NEXT:    st.d $w0, 0($4)
+;
+; MIPSEL-LABEL: sub_v2i64_i_negated:
+; MIPSEL:       # %bb.0:
+; MIPSEL-NEXT:    ldi.b $w0, -1
+; MIPSEL-NEXT:    ld.d $w1, 0($5)
+; MIPSEL-NEXT:    addv.d $w0, $w1, $w0
+; MIPSEL-NEXT:    jr $ra
+; MIPSEL-NEXT:    st.d $w0, 0($4)
+  %1 = load <2 x i64>, <2 x i64>* %a
+  %2 = add <2 x i64> %1, <i64 -1, i64 -1>
+  store <2 x i64> %2, <2 x i64>* %c
+  ret void
+}
+
 define void @mul_v16i8(<16 x i8>* %c, <16 x i8>* %a, <16 x i8>* %b) nounwind {
 ; ALL-LABEL: mul_v16i8:
 ; ALL:       # %bb.0:
diff --git a/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll b/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
index 75cac9030ef89..10a5e0d4371c8 100644
--- a/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
+++ b/llvm/test/CodeGen/PowerPC/CompareEliminationSpillIssue.ll
@@ -9,7 +9,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 @.str = private unnamed_addr constant [12 x i8] c"Value = %d\0A\00", align 1
 
 ; Function Attrs: noinline nounwind
diff --git a/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll b/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
index 937264f989baf..06b93420a2195 100644
--- a/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
+++ b/llvm/test/CodeGen/PowerPC/VSX-XForm-Scalars.ll
@@ -27,18 +27,16 @@ define void @testExpandPostRAPseudo(i32* nocapture readonly %ptr) {
 ;
 ; CHECK-P9-LABEL: testExpandPostRAPseudo:
 ; CHECK-P9:  # %bb.0: # %entry
-; CHECK-P9:    lfiwzx f0, 0, r3
 ; CHECK-P9:    addis r4, r2, .LC0@toc@ha
+; CHECK-P9:    lxvwsx vs0, 0, r3
 ; CHECK-P9:    ld r4, .LC0@toc@l(r4)
-; CHECK-P9:    xxpermdi vs0, f0, f0, 2
-; CHECK-P9:    xxspltw vs0, vs0, 3
 ; CHECK-P9:    stxvx vs0, 0, r4
+; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    lfiwax f0, 0, r3
 ; CHECK-P9:    addis r3, r2, .LC1@toc@ha
 ; CHECK-P9:    ld r3, .LC1@toc@l(r3)
 ; CHECK-P9:    xscvsxdsp f0, f0
 ; CHECK-P9:    ld r3, 0(r3)
-; CHECK-P9:    lis r4, 1024
 ; CHECK-P9:    stfsx f0, r3, r4
 ; CHECK-P9:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/aix-xcoff-common.ll b/llvm/test/CodeGen/PowerPC/aix-xcoff-common.ll
index a3e135f15b411..ffae4a42d1579 100644
--- a/llvm/test/CodeGen/PowerPC/aix-xcoff-common.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-xcoff-common.ll
@@ -21,6 +21,8 @@
 
 @array = common local_unnamed_addr global [33 x i8] zeroinitializer, align 1
 
+; CHECK-NOT: .toc
+
 ; CHECK:      .csect .text[PR]
 ; CHECK-NEXT:  .file
 ; CHECK-NEXT: .comm   a,4,2
diff --git a/llvm/test/CodeGen/PowerPC/build-vector-tests.ll b/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
index 92644d59e1d0a..26adda0b00550 100644
--- a/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
+++ b/llvm/test/CodeGen/PowerPC/build-vector-tests.ll
@@ -1278,16 +1278,12 @@ entry:
 define <4 x i32> @spltMemVali(i32* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemVali:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    lfiwzx f0, 0, r3
-; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; P9BE-NEXT:    xxspltw v2, vs0, 0
+; P9BE-NEXT:    lxvwsx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltMemVali:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lfiwzx f0, 0, r3
-; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
-; P9LE-NEXT:    xxspltw v2, vs0, 3
+; P9LE-NEXT:    lxvwsx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltMemVali:
@@ -2833,16 +2829,12 @@ entry:
 define <4 x i32> @spltMemValui(i32* nocapture readonly %ptr) {
 ; P9BE-LABEL: spltMemValui:
 ; P9BE:       # %bb.0: # %entry
-; P9BE-NEXT:    lfiwzx f0, 0, r3
-; P9BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; P9BE-NEXT:    xxspltw v2, vs0, 0
+; P9BE-NEXT:    lxvwsx v2, 0, r3
 ; P9BE-NEXT:    blr
 ;
 ; P9LE-LABEL: spltMemValui:
 ; P9LE:       # %bb.0: # %entry
-; P9LE-NEXT:    lfiwzx f0, 0, r3
-; P9LE-NEXT:    xxpermdi vs0, f0, f0, 2
-; P9LE-NEXT:    xxspltw v2, vs0, 3
+; P9LE-NEXT:    lxvwsx v2, 0, r3
 ; P9LE-NEXT:    blr
 ;
 ; P8BE-LABEL: spltMemValui:
diff --git a/llvm/test/CodeGen/PowerPC/csr-split.ll b/llvm/test/CodeGen/PowerPC/csr-split.ll
index 4e7db535089ec..cd981a27d2edb 100644
--- a/llvm/test/CodeGen/PowerPC/csr-split.ll
+++ b/llvm/test/CodeGen/PowerPC/csr-split.ll
@@ -6,7 +6,7 @@
 
 ; Check CSR split can work properly for tests below.
 
-@a = common dso_local local_unnamed_addr global i32 0, align 4
+@a = dso_local local_unnamed_addr global i32 0, align 4
 
 define dso_local signext i32 @test1(i32* %b) local_unnamed_addr  {
 ; CHECK-PWR9-LABEL: test1:
diff --git a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
index 4f904d6fe09f6..44acfcdd6e66a 100644
--- a/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
+++ b/llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll
@@ -4,13 +4,13 @@
 ; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4
 ; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result.
 
-@a = common local_unnamed_addr global i32 0, align 4
-@b = common local_unnamed_addr global i32 0, align 4
-@c = common local_unnamed_addr global i32 0, align 4
-@d = common local_unnamed_addr global i32 0, align 4
-@e = common local_unnamed_addr global i32 0, align 4
-@f = common local_unnamed_addr global i32 0, align 4
-@arr = common local_unnamed_addr global [5 x i32] zeroinitializer, align 4
+@a = local_unnamed_addr global i32 0, align 4
+@b = local_unnamed_addr global i32 0, align 4
+@c = local_unnamed_addr global i32 0, align 4
+@d = local_unnamed_addr global i32 0, align 4
+@e = local_unnamed_addr global i32 0, align 4
+@f = local_unnamed_addr global i32 0, align 4
+@arr = local_unnamed_addr global [5 x i32] zeroinitializer, align 4
 
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @testTripCount2(i32 signext %a) {
diff --git a/llvm/test/CodeGen/PowerPC/elf-common.ll b/llvm/test/CodeGen/PowerPC/elf-common.ll
new file mode 100644
index 0000000000000..cc73d9b58b54a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/elf-common.ll
@@ -0,0 +1,45 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -O0 < %s \
+; RUN:    | FileCheck -check-prefix=NOOPT %s
+; RUN: llc -relocation-model=static -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 < %s | FileCheck -check-prefix=STATIC %s
+; RUN: llc -relocation-model=pic -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:     -mcpu=pwr8 < %s | FileCheck -check-prefix=PIC %s
+
+; Test correct code generation for static and pic for loading and storing a common symbol 
+
+@comm_glob = common global i32 0, align 4
+
+define signext i32 @test_comm() nounwind {
+; NOOPT-LABEL: test_comm:
+; NOOPT:       # %bb.0: # %entry
+; NOOPT-NEXT:    addis 3, 2, comm_glob@toc@ha
+; NOOPT-NEXT:    addi 3, 3, comm_glob@toc@l
+; NOOPT-NEXT:    lwz 4, 0(3)
+; NOOPT-NEXT:    addi 5, 4, 1
+; NOOPT-NEXT:    stw 5, 0(3)
+; NOOPT-NEXT:    extsw 3, 4
+; NOOPT-NEXT:    blr
+;
+; STATIC-LABEL: test_comm:
+; STATIC:       # %bb.0: # %entry
+; STATIC-NEXT:    addis 4, 2, comm_glob@toc@ha
+; STATIC-NEXT:    lwa 3, comm_glob@toc@l(4)
+; STATIC-NEXT:    addi 5, 3, 1
+; STATIC-NEXT:    stw 5, comm_glob@toc@l(4)
+; STATIC-NEXT:    blr
+;
+; PIC-LABEL: test_comm:
+; PIC:       # %bb.0: # %entry
+; PIC-NEXT:    addis 3, 2, .LC0@toc@ha
+; PIC-NEXT:    ld 4, .LC0@toc@l(3)
+; PIC-NEXT:    lwa 3, 0(4)
+; PIC-NEXT:    addi 5, 3, 1
+; PIC-NEXT:    stw 5, 0(4)
+; PIC-NEXT:    blr
+entry:
+  %0 = load i32, i32* @comm_glob, align 4
+  %inc = add nsw i32 %0, 1
+  store i32 %inc, i32* @comm_glob, align 4
+  ret i32 %0
+}
diff --git a/llvm/test/CodeGen/PowerPC/expand-isel.ll b/llvm/test/CodeGen/PowerPC/expand-isel.ll
index d9b571eda47aa..1d1c00e5f5a5a 100644
--- a/llvm/test/CodeGen/PowerPC/expand-isel.ll
+++ b/llvm/test/CodeGen/PowerPC/expand-isel.ll
@@ -169,8 +169,8 @@ entry:
 }
 
 
-@b = common local_unnamed_addr global i32 0, align 4
-@a = common local_unnamed_addr global i32 0, align 4
+@b = local_unnamed_addr global i32 0, align 4
+@a = local_unnamed_addr global i32 0, align 4
 ; Function Attrs: norecurse nounwind readonly
 define signext i32 @testComplexISEL() #0 {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
index 30476508dd14c..8c21b85d0ce0a 100644
--- a/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-aggregates.ll
@@ -11,7 +11,7 @@
 %struct.With9fp128params = type { fp128, fp128, fp128, fp128, fp128, fp128,
                                   fp128, fp128, fp128 }
 
-@a1 = common local_unnamed_addr global [3 x fp128] zeroinitializer, align 16
+@a1 = local_unnamed_addr global [3 x fp128] zeroinitializer, align 16
 
 ; Function Attrs: norecurse nounwind readonly
 define fp128 @testArray_01(fp128* nocapture readonly %sa) {
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-call.ll b/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
index 8823beb96844b..d3a1bea87d131 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-call.ll
@@ -1,8 +1,4 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
-; RUN: llc < %s -O0 -relocation-model=pic -verify-machineinstrs -mattr=-vsx -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -ppc-late-peephole=true | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -relocation-model=pic -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -ppc-late-peephole=true | FileCheck %s --check-prefix=ELF64
 
 define i32 @t1(i8 signext %a) nounwind {
   %1 = sext i8 %a to i32
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-const.ll b/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
index 3987e54a8d1b6..c0b9fe43926a7 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-const.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
 define zeroext i1 @testi1(i8 %in) nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
index 80a733c5da8d9..cb9e1c9256d54 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-load-store.ll
@@ -1,8 +1,5 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
 ; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=-vsx -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
+; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=+vsx -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=VSX
 ; RUN: llc -relocation-model=static < %s -O0 -verify-machineinstrs -fast-isel -fast-isel-abort=1 -mattr=spe  -mtriple=powerpc-unknown-linux-gnu -mcpu=e500 | FileCheck %s --check-prefix=SPE
 
 ; This test verifies that load/store instructions are properly generated,
@@ -78,9 +75,11 @@ define double @t6() nounwind {
 ; SPE: t6
   %1 = load double, double* @f, align 8
 ; ELF64: lfd
+; VSX: lxsdx
 ; SPE: evldd
   %2 = fadd double %1, 1.0
 ; ELF64: fadd
+; VSX: xsadddp
 ; SPE: efdadd
   ret double %2
 }
@@ -150,6 +149,8 @@ define void @t12(double %v) nounwind {
   store double %1, double* @f, align 8
 ; ELF64: fadd
 ; ELF64: stfd
+; VSX: xsadddp
+; VSX: stxsdx
 ; SPE: efdadd
 ; SPE: evstdd
   ret void
diff --git a/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll b/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
index 4c499e44e4f04..5fe786129b574 100644
--- a/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
+++ b/llvm/test/CodeGen/PowerPC/fast-isel-ret.ll
@@ -1,8 +1,4 @@
-; FIXME: FastISel currently returns false if it hits code that uses VSX
-; registers and with -fast-isel-abort=1 turned on the test case will then fail.
-; When fastisel better supports VSX fix up this test case.
-;
-; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 -mattr=-vsx | FileCheck %s --check-prefix=ELF64
+; RUN: llc < %s -O0 -verify-machineinstrs -fast-isel-abort=1 -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr7 | FileCheck %s --check-prefix=ELF64
 
 define zeroext i1 @rettrue() nounwind {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/fdiv.ll b/llvm/test/CodeGen/PowerPC/fdiv.ll
new file mode 100644
index 0000000000000..183e514d994ae
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/fdiv.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s
+
+define dso_local float @foo(float %0, float %1) local_unnamed_addr {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xsresp 3, 2
+; CHECK-NEXT:    xsmulsp 0, 1, 3
+; CHECK-NEXT:    xsnmsubasp 1, 2, 0
+; CHECK-NEXT:    xsmaddasp 0, 3, 1
+; CHECK-NEXT:    fmr 1, 0
+; CHECK-NEXT:    blr
+  %3 = fdiv fast float %0, %1
+  ret float %3
+}
diff --git a/llvm/test/CodeGen/PowerPC/load-and-splat.ll b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
new file mode 100644
index 0000000000000..800b79e611776
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/load-and-splat.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mcpu=pwr9 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P9
+; RUN: llc -mcpu=pwr8 -ppc-asm-full-reg-names -ppc-vsr-nums-as-vr \
+; RUN:   -mtriple=powerpc64le-unknown-unknown < %s | FileCheck %s \
+; RUN:   -check-prefix=P8
+define dso_local void @test(<2 x double>* nocapture %c, double* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 24
+; P9-NEXT:    lxvdsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 24
+; P8-NEXT:    lxvdsx vs0, 0, r4
+; P8-NEXT:    stxvd2x vs0, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds double, double* %a, i64 3
+  %0 = load double, double* %arrayidx, align 8
+  %splat.splatinsert.i = insertelement <2 x double> undef, double %0, i32 0
+  %splat.splat.i = shufflevector <2 x double> %splat.splatinsert.i, <2 x double> undef, <2 x i32> zeroinitializer
+  store <2 x double> %splat.splat.i, <2 x double>* %c, align 16
+  ret void
+}
+
+define dso_local void @test2(<4 x float>* nocapture %c, float* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test2:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 12
+; P9-NEXT:    lxvwsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test2:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 12
+; P8-NEXT:    lfiwzx f0, 0, r4
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    stvx v2, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds float, float* %a, i64 3
+  %0 = load float, float* %arrayidx, align 4
+  %splat.splatinsert.i = insertelement <4 x float> undef, float %0, i32 0
+  %splat.splat.i = shufflevector <4 x float> %splat.splatinsert.i, <4 x float> undef, <4 x i32> zeroinitializer
+  store <4 x float> %splat.splat.i, <4 x float>* %c, align 16
+  ret void
+}
+
+define dso_local void @test3(<4 x i32>* nocapture %c, i32* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test3:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 12
+; P9-NEXT:    lxvwsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test3:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 12
+; P8-NEXT:    lfiwzx f0, 0, r4
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    stvx v2, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds i32, i32* %a, i64 3
+  %0 = load i32, i32* %arrayidx, align 4
+  %splat.splatinsert.i = insertelement <4 x i32> undef, i32 %0, i32 0
+  %splat.splat.i = shufflevector <4 x i32> %splat.splatinsert.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  store <4 x i32> %splat.splat.i, <4 x i32>* %c, align 16
+  ret void
+}
+
+define dso_local void @test4(<2 x i64>* nocapture %c, i64* nocapture readonly %a) local_unnamed_addr {
+; P9-LABEL: test4:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r4, r4, 24
+; P9-NEXT:    lxvdsx vs0, 0, r4
+; P9-NEXT:    stxv vs0, 0(r3)
+; P9-NEXT:    blr
+;
+; P8-LABEL: test4:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r4, r4, 24
+; P8-NEXT:    lxvdsx vs0, 0, r4
+; P8-NEXT:    stxvd2x vs0, 0, r3
+; P8-NEXT:    blr
+entry:
+  %arrayidx = getelementptr inbounds i64, i64* %a, i64 3
+  %0 = load i64, i64* %arrayidx, align 8
+  %splat.splatinsert.i = insertelement <2 x i64> undef, i64 %0, i32 0
+  %splat.splat.i = shufflevector <2 x i64> %splat.splatinsert.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  store <2 x i64> %splat.splat.i, <2 x i64>* %c, align 16
+  ret void
+}
+
+define <16 x i8> @unadjusted_lxvwsx(i32* %s, i32* %t) {
+; P9-LABEL: unadjusted_lxvwsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvwsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lfiwzx f0, 0, r3
+; P8-NEXT:    xxpermdi vs0, f0, f0, 2
+; P8-NEXT:    xxspltw v2, vs0, 3
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i32* %s to <4 x i8>*
+    %1 = load <4 x i8>, <4 x i8>* %0, align 4
+    %2 = shufflevector <4 x i8> %1, <4 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @adjusted_lxvwsx(i64* %s, i64* %t) {
+; P9-LABEL: adjusted_lxvwsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 4
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    ld r3, 0(r3)
+; P8-NEXT:    mtvsrd f0, r3
+; P8-NEXT:    xxswapd v2, vs0
+; P8-NEXT:    xxspltw v2, v2, 2
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i64* %s to <8 x i8>*
+    %1 = load <8 x i8>, <8 x i8>* %0, align 8
+    %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @unadjusted_lxvwsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: unadjusted_lxvwsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvwsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3, i32 0, i32 1, i32 2, i32 3>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 4
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 2
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8_2(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8_2:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 8
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8_2:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 1
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11, i32 8, i32 9, i32 10, i32 11>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvwsx_v16i8_3(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvwsx_v16i8_3:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 12
+; P9-NEXT:    lxvwsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvwsx_v16i8_3:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lvx v2, 0, r3
+; P8-NEXT:    xxspltw v2, v2, 0
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15, i32 12, i32 13, i32 14, i32 15>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @unadjusted_lxvdsx(i64* %s, i64* %t) {
+; P9-LABEL: unadjusted_lxvdsx:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvdsx:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = bitcast i64* %s to <8 x i8>*
+    %1 = load <8 x i8>, <8 x i8>* %0, align 8
+    %2 = shufflevector <8 x i8> %1, <8 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %2
+}
+
+define <16 x i8> @unadjusted_lxvdsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: unadjusted_lxvdsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: unadjusted_lxvdsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+    ret <16 x i8> %1
+}
+
+define <16 x i8> @adjusted_lxvdsx_v16i8(<16 x i8> *%s, <16 x i8> %t) {
+; P9-LABEL: adjusted_lxvdsx_v16i8:
+; P9:       # %bb.0: # %entry
+; P9-NEXT:    addi r3, r3, 8
+; P9-NEXT:    lxvdsx v2, 0, r3
+; P9-NEXT:    blr
+;
+; P8-LABEL: adjusted_lxvdsx_v16i8:
+; P8:       # %bb.0: # %entry
+; P8-NEXT:    addi r3, r3, 8
+; P8-NEXT:    lxvdsx v2, 0, r3
+; P8-NEXT:    blr
+  entry:
+    %0 = load <16 x i8>, <16 x i8>* %s, align 16
+    %1 = shufflevector <16 x i8> %0, <16 x i8> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+    ret <16 x i8> %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll
new file mode 100644
index 0000000000000..b368cb43dbbd2
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/lower-globaladdr32-aix-asm.ll
@@ -0,0 +1,21 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff \
+; RUN: -code-model=small < %s | FileCheck %s
+
+@b = common global i32 0
+@a = common global i32 0
+
+define void @test() {
+  %1 = load i32, i32* @b
+  store i32 %1, i32* @a
+  ret void
+}
+
+; CHECK-LABEL:   test
+; CHECK-DAG:       lwz [[REG1:[0-9]+]], LC0(2)
+; CHECK-DAG:       lwz [[REG2:[0-9]+]], LC1(2)
+; CHECK-DAG:       lwz [[REG3:[0-9]+]], 0([[REG1]])
+; CHECK:           stw [[REG3]], 0([[REG2]])
+; CHECK:           blr
+
+; TODO Update test when TOC-entry emission lands.
+; CHECK-NOT: .tc
diff --git a/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll b/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
index cf970928fdbd5..0b2910d6b9746 100644
--- a/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
+++ b/llvm/test/CodeGen/PowerPC/power9-moves-and-splats.ll
@@ -61,16 +61,12 @@ entry:
 define <4 x i32> @test4(i32* nocapture readonly %in) {
 ; CHECK-LABEL: test4:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test4:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -83,16 +79,12 @@ entry:
 define <4 x float> @test5(float* nocapture readonly %in) {
 ; CHECK-LABEL: test5:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test5:
 ; CHECK-BE:       # %bb.0: # %entry
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -107,18 +99,14 @@ define <4 x i32> @test6() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r3, r2, .LC0@toc@ha
 ; CHECK-NEXT:    ld r3, .LC0@toc@l(r3)
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test6:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r3, r2, .LC0@toc@ha
 ; CHECK-BE-NEXT:    ld r3, .LC0@toc@l(r3)
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
@@ -133,18 +121,14 @@ define <4 x float> @test7() {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    addis r3, r2, .LC1@toc@ha
 ; CHECK-NEXT:    ld r3, .LC1@toc@l(r3)
-; CHECK-NEXT:    lfiwzx f0, 0, r3
-; CHECK-NEXT:    xxpermdi vs0, f0, f0, 2
-; CHECK-NEXT:    xxspltw v2, vs0, 3
+; CHECK-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-NEXT:    blr
 ;
 ; CHECK-BE-LABEL: test7:
 ; CHECK-BE:       # %bb.0: # %entry
 ; CHECK-BE-NEXT:    addis r3, r2, .LC1@toc@ha
 ; CHECK-BE-NEXT:    ld r3, .LC1@toc@l(r3)
-; CHECK-BE-NEXT:    lfiwzx f0, 0, r3
-; CHECK-BE-NEXT:    xxsldwi vs0, f0, f0, 1
-; CHECK-BE-NEXT:    xxspltw v2, vs0, 0
+; CHECK-BE-NEXT:    lxvwsx v2, 0, r3
 ; CHECK-BE-NEXT:    blr
 
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
index 46e347becbb67..547f9273f5a4f 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-P9-mod.ll
@@ -2,14 +2,14 @@
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr9 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 | FileCheck %s -check-prefix=CHECK-PWR8 -implicit-check-not mod[us][wd]
 
-@mod_resultsw = common local_unnamed_addr global i32 0, align 4
-@mod_resultud = common local_unnamed_addr global i64 0, align 8
-@div_resultsw = common local_unnamed_addr global i32 0, align 4
-@mod_resultuw = common local_unnamed_addr global i32 0, align 4
-@div_resultuw = common local_unnamed_addr global i32 0, align 4
-@div_resultsd = common local_unnamed_addr global i64 0, align 8
-@mod_resultsd = common local_unnamed_addr global i64 0, align 8
-@div_resultud = common local_unnamed_addr global i64 0, align 8
+@mod_resultsw = local_unnamed_addr global i32 0, align 4
+@mod_resultud = local_unnamed_addr global i64 0, align 8
+@div_resultsw = local_unnamed_addr global i32 0, align 4
+@mod_resultuw = local_unnamed_addr global i32 0, align 4
+@div_resultuw = local_unnamed_addr global i32 0, align 4
+@div_resultsd = local_unnamed_addr global i64 0, align 8
+@mod_resultsd = local_unnamed_addr global i64 0, align 8
+@div_resultud = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind
 define void @modulo_sw(i32 signext %a, i32 signext %b) local_unnamed_addr {
diff --git a/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll b/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
index 032bad6f63d67..ab84707ce46ef 100644
--- a/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
+++ b/llvm/test/CodeGen/PowerPC/ppc64-pre-inc-no-extra-phi.ll
@@ -3,7 +3,7 @@
 ; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
 ; RUN: llc < %s -mtriple=powerpc64-unknown-linux-gnu -mcpu=pwr8 -verify-machineinstrs | FileCheck %s
 
-@perm = common local_unnamed_addr global [100 x i64] zeroinitializer, align 8
+@perm = local_unnamed_addr global [100 x i64] zeroinitializer, align 8
 
 define void @sort_basket() local_unnamed_addr {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/pr32140.ll b/llvm/test/CodeGen/PowerPC/pr32140.ll
index ddcf27232398e..1a0054b0fd778 100644
--- a/llvm/test/CodeGen/PowerPC/pr32140.ll
+++ b/llvm/test/CodeGen/PowerPC/pr32140.ll
@@ -2,10 +2,10 @@
 ; RUN: llc -mtriple=powerpc64le-linux-gnu -mcpu=pwr8 < %s | FileCheck %s --check-prefix CHECK-LE
 ; RUN: llc -mtriple=powerpc64-linux-gnu -mcpu=pwr8 < %s | FileCheck %s --check-prefix CHECK-BE
 
-@as = common local_unnamed_addr global i16 0, align 2
-@bs = common local_unnamed_addr global i16 0, align 2
-@ai = common local_unnamed_addr global i32 0, align 4
-@bi = common local_unnamed_addr global i32 0, align 4
+@as = local_unnamed_addr global i16 0, align 2
+@bs = local_unnamed_addr global i16 0, align 2
+@ai = local_unnamed_addr global i32 0, align 4
+@bi = local_unnamed_addr global i32 0, align 4
 
 define void @bswapStorei64Toi32() {
 ; CHECK-LABEL: bswapStorei64Toi32:
diff --git a/llvm/test/CodeGen/PowerPC/pr36068.ll b/llvm/test/CodeGen/PowerPC/pr36068.ll
index aac659bfb7093..ee56d020e3cc4 100644
--- a/llvm/test/CodeGen/PowerPC/pr36068.ll
+++ b/llvm/test/CodeGen/PowerPC/pr36068.ll
@@ -1,7 +1,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown < %s  | \
 ; RUN:   FileCheck %s
 
-@glob = common local_unnamed_addr global <4 x float> zeroinitializer, align 4
+@glob = local_unnamed_addr global <4 x float> zeroinitializer, align 4
 
 ; Function Attrs: norecurse nounwind
 define void @test(float %a, <4 x float>* nocapture readonly %b) {
diff --git a/llvm/test/CodeGen/PowerPC/pr38087.ll b/llvm/test/CodeGen/PowerPC/pr38087.ll
index 2736ffa723c1a..2ef4c689beab7 100644
--- a/llvm/test/CodeGen/PowerPC/pr38087.ll
+++ b/llvm/test/CodeGen/PowerPC/pr38087.ll
@@ -8,16 +8,16 @@ declare <4 x float> @llvm.fmuladd.v4f32(<4 x float>, <4 x float>, <4 x float>) #
 ; Function Attrs: nounwind readnone speculatable
 declare { i32, i1 } @llvm.usub.with.overflow.i32(i32, i32) #0
 
-define void @draw_llvm_vs_variant0() {
+define void @draw_llvm_vs_variant0(<4 x float> %x) {
 ; CHECK-LABEL: draw_llvm_vs_variant0:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lfd f0, 0(r3)
-; CHECK-NEXT:    xxpermdi v2, f0, f0, 2
-; CHECK-NEXT:    vmrglh v2, v2, v2
-; CHECK-NEXT:    vextsh2w v2, v2
-; CHECK-NEXT:    xvcvsxwsp vs0, v2
+; CHECK-NEXT:    xxpermdi v3, f0, f0, 2
+; CHECK-NEXT:    vmrglh v3, v3, v3
+; CHECK-NEXT:    vextsh2w v3, v3
+; CHECK-NEXT:    xvcvsxwsp vs0, v3
 ; CHECK-NEXT:    xxspltw vs0, vs0, 2
-; CHECK-NEXT:    xvmaddasp vs0, vs0, vs0
+; CHECK-NEXT:    xvmaddasp vs0, v2, v2
 ; CHECK-NEXT:    stxvx vs0, 0, r3
 ; CHECK-NEXT:    blr
 entry:
@@ -49,7 +49,7 @@ entry:
   %24 = and <4 x i32> %23, %22
   %25 = bitcast <4 x i32> %24 to <4 x float>
   %26 = shufflevector <4 x float> %25, <4 x float> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
-  %27 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> undef, <4 x float> %26)
+  %27 = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %x, <4 x float> %x, <4 x float> %26)
   store <4 x float> %27, <4 x float>* undef
   ret void
 }
diff --git a/llvm/test/CodeGen/PowerPC/pr39815.ll b/llvm/test/CodeGen/PowerPC/pr39815.ll
index 062e055167c42..badba31319544 100644
--- a/llvm/test/CodeGen/PowerPC/pr39815.ll
+++ b/llvm/test/CodeGen/PowerPC/pr39815.ll
@@ -1,8 +1,8 @@
 ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s \
 ; RUN:   -verify-machineinstrs | FileCheck %s
 
-@b = common dso_local local_unnamed_addr global i64* null, align 8
-@a = common dso_local local_unnamed_addr global i8 0, align 1
+@b = dso_local local_unnamed_addr global i64* null, align 8
+@a = dso_local local_unnamed_addr global i8 0, align 1
 
 define void @testADDEPromoteResult() {
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
index 1afd27262ba84..818738fa9998e 100644
--- a/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
+++ b/llvm/test/CodeGen/PowerPC/qpx-load-splat.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=powerpc64le-unknown-linux-gnu -ppc-vsr-nums-as-vr \
 ; RUN:   -ppc-asm-full-reg-names -verify-machineinstrs < %s | FileCheck %s
 
@@ -34,9 +35,9 @@ define <4 x double> @fooxu(double* nocapture readonly %a, i64 %idx, double** %pp
 ; CHECK-LABEL: fooxu:
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    sldi r4, r4, 3
-; CHECK-NEXT:    lfdux f0, r3, r4
-; CHECK-NEXT:    xxspltd v2, vs0, 0
-; CHECK-NEXT:    std r3, 0(r5)
+; CHECK-NEXT:    add r6, r3, r4
+; CHECK-NEXT:    std r6, 0(r5)
+; CHECK-NEXT:    lxvdsx v2, r3, r4
 ; CHECK-NEXT:    vmr v3, v2
 ; CHECK-NEXT:    blr
 entry:
diff --git a/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll b/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll
new file mode 100644
index 0000000000000..f7727d6f4ea10
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/reduce_scalarization02.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-unknown \
+; RUN:     -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-unknown \
+; RUN:     -mcpu=pwr9 -ppc-asm-full-reg-names \
+; RUN:     -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
+
+; Test reduce scalarization in fpext v2f32 to v2f64 from the extract_subvector v4f32 node.
+
+define dso_local void @test(<4 x float>* nocapture readonly %a, <2 x double>* nocapture %b, <2 x double>* nocapture %c) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv vs0, 0(r3)
+; CHECK-NEXT:    xxmrglw vs1, vs0, vs0
+; CHECK-NEXT:    xxmrghw vs0, vs0, vs0
+; CHECK-NEXT:    xvcvspdp vs1, vs1
+; CHECK-NEXT:    xvcvspdp vs0, vs0
+; CHECK-NEXT:    stxv vs1, 0(r4)
+; CHECK-NEXT:    stxv vs0, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xxmrghw vs1, vs0, vs0
+; CHECK-BE-NEXT:    xxmrglw vs0, vs0, vs0
+; CHECK-BE-NEXT:    xvcvspdp vs1, vs1
+; CHECK-BE-NEXT:    xvcvspdp vs0, vs0
+; CHECK-BE-NEXT:    stxv vs1, 0(r4)
+; CHECK-BE-NEXT:    stxv vs0, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <4 x float>, <4 x float>* %a, align 16
+  %shuffle = shufflevector <4 x float> %0, <4 x float> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle1 = shufflevector <4 x float> %0, <4 x float> undef, <2 x i32> <i32 2, i32 3>
+  %vecinit4 = fpext <2 x float> %shuffle to <2 x double>
+  %vecinit11 = fpext <2 x float> %shuffle1 to <2 x double>
+  store <2 x double> %vecinit4, <2 x double>* %b, align 16
+  store <2 x double> %vecinit11, <2 x double>* %c, align 16
+  ret void
+}
+
+; Ensure we don't crash for wider types
+
+define dso_local void @test2(<16 x float>* nocapture readonly %a, <2 x double>* nocapture %b, <2 x double>* nocapture %c) {
+; CHECK-LABEL: test2:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lxv vs0, 0(r3)
+; CHECK-NEXT:    xxsldwi vs1, vs0, vs0, 1
+; CHECK-NEXT:    xscvspdpn f2, vs0
+; CHECK-NEXT:    xxsldwi vs3, vs0, vs0, 3
+; CHECK-NEXT:    xxswapd vs0, vs0
+; CHECK-NEXT:    xscvspdpn f1, vs1
+; CHECK-NEXT:    xscvspdpn f3, vs3
+; CHECK-NEXT:    xscvspdpn f0, vs0
+; CHECK-NEXT:    xxmrghd vs0, vs0, vs3
+; CHECK-NEXT:    xxmrghd vs1, vs2, vs1
+; CHECK-NEXT:    stxv vs0, 0(r4)
+; CHECK-NEXT:    stxv vs1, 0(r5)
+; CHECK-NEXT:    blr
+;
+; CHECK-BE-LABEL: test2:
+; CHECK-BE:       # %bb.0: # %entry
+; CHECK-BE-NEXT:    lxv vs0, 0(r3)
+; CHECK-BE-NEXT:    xxswapd vs1, vs0
+; CHECK-BE-NEXT:    xxsldwi vs2, vs0, vs0, 3
+; CHECK-BE-NEXT:    xscvspdpn f3, vs0
+; CHECK-BE-NEXT:    xxsldwi vs0, vs0, vs0, 1
+; CHECK-BE-NEXT:    xscvspdpn f1, vs1
+; CHECK-BE-NEXT:    xscvspdpn f2, vs2
+; CHECK-BE-NEXT:    xscvspdpn f0, vs0
+; CHECK-BE-NEXT:    xxmrghd vs0, vs3, vs0
+; CHECK-BE-NEXT:    xxmrghd vs1, vs1, vs2
+; CHECK-BE-NEXT:    stxv vs0, 0(r4)
+; CHECK-BE-NEXT:    stxv vs1, 0(r5)
+; CHECK-BE-NEXT:    blr
+entry:
+  %0 = load <16 x float>, <16 x float>* %a, align 16
+  %shuffle = shufflevector <16 x float> %0, <16 x float> undef, <2 x i32> <i32 0, i32 1>
+  %shuffle1 = shufflevector <16 x float> %0, <16 x float> undef, <2 x i32> <i32 2, i32 3>
+  %vecinit4 = fpext <2 x float> %shuffle to <2 x double>
+  %vecinit11 = fpext <2 x float> %shuffle1 to <2 x double>
+  store <2 x double> %vecinit4, <2 x double>* %b, align 16
+  store <2 x double> %vecinit11, <2 x double>* %c, align 16
+  ret void
+}
diff --git a/llvm/test/CodeGen/PowerPC/sh-overflow.mir b/llvm/test/CodeGen/PowerPC/sh-overflow.mir
new file mode 100644
index 0000000000000..31cd710c39ea0
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/sh-overflow.mir
@@ -0,0 +1,58 @@
+# RUN: llc -O3 -mtriple=powerpc64le-unknown-linux-gnu -start-after ppc-mi-peepholes -ppc-late-peephole -ppc-asm-full-reg-names -verify-machineinstrs %s -o - | FileCheck %s
+
+---
+name:            special_right_shift32_0
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gprc }
+  - { id: 1, class: gprc }
+  - { id: 2, class: gprc }
+liveins:
+  - { reg: '$r3', virtual-reg: '%0' }
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $r3
+
+    ; Ensure we do not attempt to transform this into srwi $r3, $r3, 0 in the
+    ; form specified by ISA 3.0b (rlwinm $r3, $r3, 32 - 0, 0, 31)
+
+    ; CHECK-LABEL: special_right_shift32_0:
+    ; CHECK:         slwi r[[#]], r[[#]], 0
+
+    %0:gprc = COPY killed $r3
+    %1:gprc = LI 0
+    %2:gprc = SRW killed %0, killed %1
+    $r3 = COPY killed %2
+    BLR implicit $lr, implicit $rm, implicit killed $r3
+
+...
+---
+name:            special_right_shift64_0
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: g8rc }
+  - { id: 1, class: gprc }
+  - { id: 2, class: g8rc }
+liveins:
+  - { reg: '$x3', virtual-reg: '%0' }
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $x3
+
+    ; Ensure we do not attempt to transform this into srdi $r3, $r3, 0 in the
+    ; form specified by ISA 3.0b (rldicl $r3, $r3, 64 - 0, 0)
+
+    ; CHECK-LABEL: special_right_shift64_0:
+    ; CHECK:         rotldi r[[#]], r[[#]], 0
+
+    %0:g8rc = COPY killed $x3
+    %1:gprc = LI 0
+    %2:g8rc = SRD killed %0, killed %1
+    $x3 = COPY killed %2
+    BLR8 implicit $lr8, implicit $rm, implicit killed $x3
+
+...
diff --git a/llvm/test/CodeGen/PowerPC/sms-simple.ll b/llvm/test/CodeGen/PowerPC/sms-simple.ll
index 6b1f0e453e8f2..c4d6270ba99a2 100644
--- a/llvm/test/CodeGen/PowerPC/sms-simple.ll
+++ b/llvm/test/CodeGen/PowerPC/sms-simple.ll
@@ -4,7 +4,7 @@
 ; RUN:       | FileCheck %s
 
 @x = dso_local local_unnamed_addr global <{ i32, i32, i32, i32, [1020 x i32] }> <{ i32 1, i32 2, i32 3, i32 4, [1020 x i32] zeroinitializer }>, align 4
-@y = common dso_local global [1024 x i32] zeroinitializer, align 4
+@y = dso_local global [1024 x i32] zeroinitializer, align 4
 
 ; Function Attrs: norecurse nounwind
 define dso_local i32* @foo() local_unnamed_addr #0 {
diff --git a/llvm/test/CodeGen/PowerPC/swaps-le-7.ll b/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
index 3c8813bc99187..6f5445197ac49 100644
--- a/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
+++ b/llvm/test/CodeGen/PowerPC/swaps-le-7.ll
@@ -9,8 +9,8 @@
 @G4 = global <2 x double> <double 7.0, double 8.0>
 
 ; CHECK-LABEL: @zg
-; CHECK: xxspltd
-; CHECK-NEXT: xxspltd
+; CHECK: lxvdsx
+; CHECK-NEXT: lxvdsx
 ; CHECK-NEXT: xvmuldp
 ; CHECK-DAG: xvmuldp
 ; CHECK-DAG: xvsubdp
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
index 893b374d8521e..8b5cade8f8ca0 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
index d75485fcb6ff9..d405741726b20 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsi.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsi.c'
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll b/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
index f5627ff766dcc..9d6e0b6784d05 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqsll.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqsll.c'
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesieqss.ll b/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
index 2a5ee0c0f9d28..9acfcc4405a70 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesieqss.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesieqss.c'
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ieqss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequc.ll b/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
index 18243d36619ef..7f938dff8612b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequi.ll b/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
index afa447525efb3..b40e3af36caf5 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequi.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequi.c'
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequi(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequll.ll b/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
index 16399715386c9..6ef9db9bdc886 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequll.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequll.c'
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiequs.ll b/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
index 99b388313e121..90bcee8d70c44 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiequs.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testComparesiequs.c'
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iequs(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesc.ll b/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
index eb457951fb784..d9a397cf023bb 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_igesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_igesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesi.ll b/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
index 93e6c1e1f9cf6..bff6cb2435115 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_igesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_igesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigesll.ll b/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
index 0f0d792edcf4a..541ee87da1ff7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_igesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_igesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigess.ll b/llvm/test/CodeGen/PowerPC/testComparesigess.ll
index ac04e9e86a2e4..c0e1ab8ec214a 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigess.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigess.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_igess(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_igess:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll b/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
index 5047f0fbee159..3b9314bf6fdf1 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeui.ll b/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
index 48a0677a44a8c..13d99c4355bbd 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeui.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeull.ll b/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
index 53f0e2ac47d1b..4d430db0b68b6 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeull.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeull(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigeus.ll b/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
index ede0e42bcbf38..3e5063dc3fc3b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigeus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igeus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
index 9ae7e5d0a9096..19a0a1d96d0b2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
index ecdc2f7b4746d..21a773f1fd804 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll b/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
index d1f89d9aef1aa..f96a285142a33 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtsll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtss.ll b/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
index f6fa43a530498..fe951271986b1 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtss.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll b/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
index 51a29746c0f5d..2aca2dd2b0dff 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtuc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtui.ll b/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
index db80bef1569ba..aa1c2edc5754b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtui.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesigtus.ll b/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
index ffcd468ab0d42..877753c1a477e 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesigtus.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_igtus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesc.ll b/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
index 94118d642eda7..bd5a16fb5a807 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_ilesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_ilesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesi.ll b/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
index a5f2f84a5a695..8717e237eb0a0 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_ilesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_ilesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesilesll.ll b/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
index 532df4c3cdbc3..b1827661a935f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesilesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_ilesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_ilesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiless.ll b/llvm/test/CodeGen/PowerPC/testComparesiless.ll
index 45c887d21adca..b9b029496f921 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiless.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiless.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_iless(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iless:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileuc.ll b/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
index 9976dada86b3d..4461fb66cef31 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileui.ll b/llvm/test/CodeGen/PowerPC/testComparesileui.ll
index dc69277111c9c..ad6418c103c8c 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileui.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileull.ll b/llvm/test/CodeGen/PowerPC/testComparesileull.ll
index 4b18754d79668..54ee2561acbd7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileull.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileull(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesileus.ll b/llvm/test/CodeGen/PowerPC/testComparesileus.ll
index 0cb21e0b28e10..e4b68bb0328f8 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesileus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesileus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_ileus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
index 7702598c5ffb1..b1a1b96e66df2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsc.ll
@@ -8,7 +8,7 @@
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
index a97b734a8e41f..252e9c3238a22 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsi.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll b/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
index 0341c158fe2b2..0a31d051e0e58 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltsll.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltss.ll b/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
index 195a5cf58a819..f6a91251363a1 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltss.ll
@@ -8,7 +8,7 @@
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll b/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
index 64b4fa948a17a..de2dc1cbe19fb 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltui.ll b/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
index 005f3b8352929..a0d40b62aa0f4 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltui.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiltus.ll b/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
index 20cb9e141a288..98b8f35535324 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiltus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define signext i32 @test_iltus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesc.ll b/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
index f5ad934409e5f..8c81eadc10a89 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_inesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_inesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesi.ll b/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
index 3b0fe78e99c84..ae8d981cde22f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_inesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_inesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesinesll.ll b/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
index bfc0b9d3fbb47..c360fb99106ae 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesinesll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_inesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_inesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesiness.ll b/llvm/test/CodeGen/PowerPC/testComparesiness.ll
index 9e570949e1273..cdfeaa9d77e05 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesiness.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesiness.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_iness(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_iness:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineuc.ll b/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
index 4e7f5c05cf6de..e63ea46b05551 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineuc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define signext i32 @test_ineuc(i8 zeroext %a, i8 zeroext %b) {
 ; CHECK-LABEL: test_ineuc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineui.ll b/llvm/test/CodeGen/PowerPC/testComparesineui.ll
index e8fb3dd85be77..86e6fb9c7d263 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineui.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define signext i32 @test_ineui(i32 zeroext %a, i32 zeroext %b) {
 ; CHECK-LABEL: test_ineui:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineull.ll b/llvm/test/CodeGen/PowerPC/testComparesineull.ll
index e288988bdcde6..c0b04c2737c43 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineull.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define signext i32 @test_ineull(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_ineull:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesineus.ll b/llvm/test/CodeGen/PowerPC/testComparesineus.ll
index 4a2f851fa647c..df208eebf44a7 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesineus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesineus.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define signext i32 @test_ineus(i16 zeroext %a, i16 zeroext %b) {
 ; CHECK-LABEL: test_ineus:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
index 3e519edcd0cd3..7759f0ad9f3c1 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsc.ll
@@ -7,7 +7,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 ; ModuleID = 'ComparisonTestCases/testCompareslleqsc.c'
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsc(i8 signext %a, i8 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
index 4de1d588238cf..63612aec1672f 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsi(i32 signext %a, i32 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
index ac92d171a68bb..18b8d0b24fde1 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqsll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll b/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
index d65990f9dc936..13e0759e83ea4 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslleqss.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lleqss(i16 signext %a, i16 signext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequc.ll b/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
index 1672d19a4a3da..4c3c1d6115c2c 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequi.ll b/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
index fd1d13bae7f6b..16741ea0251ae 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequi(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequll.ll b/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
index a42b5e4c7b7b1..16bd2d1a4c0c4 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllequs.ll b/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
index e3dde77aba7e6..4470b4f959a79 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllequs.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llequs(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
index b97da250323bc..f7c79859666e0 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesc.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define i64 @test_llgesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_llgesc:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
index cc23babc00055..046c037bb7940 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesi.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define i64 @test_llgesi(i32 signext %a, i32 signext %b) {
 ; CHECK-LABEL: test_llgesi:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll b/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
index 1c119b950a35a..2b1dc4a689c96 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llgesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llgesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgess.ll b/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
index 400c11bef4b61..50583cd7fa94b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgess.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define i64 @test_llgess(i16 signext %a, i16 signext %b) {
 ; CHECK-LABEL: test_llgess:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
index 4c01f87952bf8..85a9df5c21883 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
index 5fc0323d8041f..19da9da01fa40 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeui.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
index 32849572ebb10..68d908a4e805f 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeull.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeull(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll b/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
index 552bdd46a1614..56099c63d5373 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgeus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgeus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
index 8948503c3b633..6f4c9b00b83f2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtsll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
index b885082190dfd..e535d30d2ebaa 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtuc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
index 4a093db0b9cd7..3c6b2f1c27599 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtui.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll b/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
index b148c6861440e..1dbd9ca80bec2 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllgtus.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llgtus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
index 1a5ad44ee080b..1ec226a86420c 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesc.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 define i64 @test_lllesc(i8 signext %a, i8 signext %b) {
 ; CHECK-LABEL: test_lllesc:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
index 9b79c1739a5c4..3ff4907741078 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesi.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 define i64 @test_lllesi(i32 signext %a, i32 signext %b)  {
 ; CHECK-LABEL: test_lllesi:
diff --git a/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll b/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
index 83ce2f812ae14..ca7ce5e56e3b2 100644
--- a/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testCompareslllesll.ll
@@ -5,7 +5,7 @@
 ; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu -O2 \
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_lllesll(i64 %a, i64 %b)  {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllless.ll b/llvm/test/CodeGen/PowerPC/testComparesllless.ll
index 6f8657dac4b1d..c67ef49e86617 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllless.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllless.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:   --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 define i64 @test_llless(i16 signext %a, i16 signext %b)  {
 ; CHECK-LABEL: test_llless:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
index f54027714da3b..9f0bf574e55cd 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleui.ll b/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
index d6bf2e26e1e9f..b6c43e4a346e4 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleui.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleull.ll b/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
index 2eca70ed014c7..412d264974442 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleull.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleull(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllleus.ll b/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
index 1cdb66d6cff81..8c852f16da10d 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllleus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llleus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll b/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
index 5bf437c920642..aeb1891122e34 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltsll.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltsll(i64 %a, i64 %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll b/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
index 47d196720c62d..dafbbc1f4d051 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltuc.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i8 0, align 1
+@glob = local_unnamed_addr global i8 0, align 1
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltuc(i8 zeroext %a, i8 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltui.ll b/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
index 3f5dfba99e883..50a0ff11961da 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltui.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i32 0, align 4
+@glob = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltui(i32 zeroext %a, i32 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllltus.ll b/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
index 2cbc1b16c84bd..217821b661f9b 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllltus.ll
@@ -8,7 +8,7 @@
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl \
 ; RUN:  --check-prefixes=CHECK,LE
 
-@glob = common local_unnamed_addr global i16 0, align 2
+@glob = local_unnamed_addr global i16 0, align 2
 
 ; Function Attrs: norecurse nounwind readnone
 define i64 @test_llltus(i16 zeroext %a, i16 zeroext %b) {
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll b/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
index 10bff2f7d94e4..76ea6262ddadb 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllnesll.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llnesll(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llnesll:
diff --git a/llvm/test/CodeGen/PowerPC/testComparesllneull.ll b/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
index 9e6be97e8c176..e809a0b91a589 100644
--- a/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
+++ b/llvm/test/CodeGen/PowerPC/testComparesllneull.ll
@@ -6,7 +6,7 @@
 ; RUN:   -ppc-gpr-icmps=all -ppc-asm-full-reg-names -mcpu=pwr8 < %s | FileCheck %s --check-prefix=CHECK-LE \
 ; RUN:  --implicit-check-not cmpw --implicit-check-not cmpd --implicit-check-not cmpl
 
-@glob = common local_unnamed_addr global i64 0, align 8
+@glob = local_unnamed_addr global i64 0, align 8
 
 define i64 @test_llneull(i64 %a, i64 %b) {
 ; CHECK-LABEL: test_llneull:
diff --git a/llvm/test/CodeGen/PowerPC/test_func_desc.ll b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
new file mode 100644
index 0000000000000..3cd16b94fb238
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/test_func_desc.ll
@@ -0,0 +1,74 @@
+; RUN: llc -mtriple powerpc-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,32BIT %s
+
+; RUN: llc -mtriple powerpc64-ibm-aix-xcoff < %s | \
+; RUN: FileCheck --check-prefixes=CHECK,64BIT %s
+
+
+define i32 @foo() {
+entry:
+  ret i32 3
+}
+
+define i32 @main() {
+entry:
+  %0 = call i32 @foo()
+  %1 = call i32 bitcast (i32 (...)* @extern_foo to i32 ()*)()
+  %2 = call i32 @static_foo()
+  %3 = add nsw i32 %0, %1
+  %4 = add nsw i32 %3, %2
+  ret i32 %4
+}
+
+declare i32 @extern_foo(...)
+
+define internal i32 @static_foo() {
+entry:
+  ret i32 3
+}
+
+; CHECK: .globl foo
+; CHECK: .globl .foo
+; CHECK: .csect foo[DS]
+; CHECK-NEXT: foo:
+; 32BIT: .long .foo
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .foo
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .foo:
+
+; CHECK: .globl main
+; CHECK: .globl .main
+; CHECK: .csect main[DS]
+; CHECK-NEXT: main:
+; 32BIT: .long .main
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .main
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .main:
+; CHECK: bl .foo
+; CHECK: bl .extern_foo
+; CHECK: bl .static_foo
+
+; CHECK: .lglobl .static_foo
+; CHECK: .csect static_foo[DS]
+; CHECK-NEXT: static_foo:
+; 32BIT: .long .static_foo
+; 32BIT-NEXT: .long TOC[TC0]
+; 32BIT-NEXT: .long 0
+; 64BIT: .llong .static_foo
+; 64BIT-NEXT: .llong TOC[TC0]
+; 64BIT-NEXT: .llong 0
+; CHECK-NEXT: .csect .text[PR]
+; CHECK-LABEL: .static_foo:
+
+; CHECK-NOT: .csect extern_foo
+
+; CHECK: .toc
+; CHECK-NOT: .tc
diff --git a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
index ed6b49dd7f8b8..301446a9ee14c 100644
--- a/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
+++ b/llvm/test/CodeGen/PowerPC/vsx-fma-mutate-undef.ll
@@ -3,12 +3,12 @@ target datalayout = "e-m:e-i64:64-n32:64"
 target triple = "powerpc64le-unknown-linux-gnu"
 
 ; Function Attrs: nounwind
-define void @acosh_float8(<4 x i32> %v1, <4 x i32> %v2) #0 {
+define void @acosh_float8(<4 x i32> %v1, <4 x i32> %v2, <4 x float> %x) #0 {
 entry:
   br i1 undef, label %if.then, label %if.end
 
 if.then:                                          ; preds = %entry
-  %0 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> undef, <4 x float> <float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000>, <4 x float> undef) #0
+  %0 = tail call <4 x float> @llvm.fmuladd.v4f32(<4 x float> %x, <4 x float> <float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000, float 0x3FE62E4200000000>, <4 x float> %x) #0
   %astype.i.i.74.i = bitcast <4 x float> %0 to <4 x i32>
   %and.i.i.76.i = and <4 x i32> %astype.i.i.74.i, %v1
   %or.i.i.79.i = or <4 x i32> %and.i.i.76.i, %v2
diff --git a/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll b/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
index b06a384d73b30..69474bd0805ad 100644
--- a/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
+++ b/llvm/test/CodeGen/PowerPC/zext-and-cmp.ll
@@ -4,7 +4,7 @@
 ; to zero can be simplifed by using the record form when one of its operands
 ; is known to be zero extended.
 
-@k = common local_unnamed_addr global i32 0, align 4
+@k = local_unnamed_addr global i32 0, align 4
 
 ; Function Attrs: norecurse nounwind
 define signext i32 @cmplwi(i32* nocapture readonly %p, i32* nocapture readonly %q, i32 signext %j, i32 signext %r10) {
diff --git a/llvm/test/CodeGen/RISCV/add-before-shl.ll b/llvm/test/CodeGen/RISCV/add-before-shl.ll
index 3279de3ea99b7..51e55923cdcfa 100644
--- a/llvm/test/CodeGen/RISCV/add-before-shl.ll
+++ b/llvm/test/CodeGen/RISCV/add-before-shl.ll
@@ -96,36 +96,36 @@ define i128 @add_wide_operand(i128 %a) nounwind {
 ; RV32I-LABEL: add_wide_operand:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    lw a2, 0(a1)
-; RV32I-NEXT:    srli a3, a2, 29
-; RV32I-NEXT:    lw a4, 4(a1)
-; RV32I-NEXT:    slli a5, a4, 3
-; RV32I-NEXT:    or a6, a5, a3
-; RV32I-NEXT:    srli a4, a4, 29
-; RV32I-NEXT:    lw a5, 8(a1)
-; RV32I-NEXT:    slli a3, a5, 3
-; RV32I-NEXT:    or a3, a3, a4
+; RV32I-NEXT:    lw a3, 4(a1)
+; RV32I-NEXT:    lw a6, 12(a1)
+; RV32I-NEXT:    lw a1, 8(a1)
+; RV32I-NEXT:    srli a5, a2, 29
+; RV32I-NEXT:    slli a4, a3, 3
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    srli a3, a3, 29
+; RV32I-NEXT:    slli a5, a1, 3
+; RV32I-NEXT:    or a3, a5, a3
+; RV32I-NEXT:    srli a1, a1, 29
+; RV32I-NEXT:    slli a5, a6, 3
+; RV32I-NEXT:    or a1, a5, a1
 ; RV32I-NEXT:    slli a2, a2, 3
+; RV32I-NEXT:    lui a5, 128
+; RV32I-NEXT:    add a1, a1, a5
 ; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    sw a3, 8(a0)
-; RV32I-NEXT:    sw a6, 4(a0)
-; RV32I-NEXT:    srli a2, a5, 29
-; RV32I-NEXT:    lw a1, 12(a1)
-; RV32I-NEXT:    slli a1, a1, 3
-; RV32I-NEXT:    or a1, a1, a2
-; RV32I-NEXT:    lui a2, 128
-; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    sw a4, 4(a0)
 ; RV32I-NEXT:    sw a1, 12(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: add_wide_operand:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    srli a2, a0, 61
+; RV64I-NEXT:    slli a1, a1, 3
 ; RV64I-NEXT:    or a1, a1, a2
+; RV64I-NEXT:    slli a0, a0, 3
 ; RV64I-NEXT:    addi a2, zero, 1
 ; RV64I-NEXT:    slli a2, a2, 51
 ; RV64I-NEXT:    add a1, a1, a2
-; RV64I-NEXT:    slli a0, a0, 3
 ; RV64I-NEXT:    ret
   %1 = add i128 %a, 5192296858534827628530496329220096
   %2 = shl i128 %1, 3
diff --git a/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll b/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
index 068e52f6175eb..5fd8261e27cc3 100644
--- a/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
+++ b/llvm/test/CodeGen/RISCV/addc-adde-sube-subc.ll
@@ -20,9 +20,9 @@ define i64 @addc_adde(i64 %a, i64 %b) nounwind {
 define i64 @subc_sube(i64 %a, i64 %b) nounwind {
 ; RV32I-LABEL: subc_sube:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    sltu a4, a0, a2
 ; RV32I-NEXT:    sub a1, a1, a3
-; RV32I-NEXT:    sltu a3, a0, a2
-; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a1, a1, a4
 ; RV32I-NEXT:    sub a0, a0, a2
 ; RV32I-NEXT:    ret
   %1 = sub i64 %a, %b
diff --git a/llvm/test/CodeGen/RISCV/addcarry.ll b/llvm/test/CodeGen/RISCV/addcarry.ll
index 5a25fb9850962..71711f9eb5054 100644
--- a/llvm/test/CodeGen/RISCV/addcarry.ll
+++ b/llvm/test/CodeGen/RISCV/addcarry.ll
@@ -10,17 +10,17 @@ define i64 @addcarry(i64 %x, i64 %y) nounwind {
 ; RISCV32-LABEL: addcarry:
 ; RISCV32:       # %bb.0:
 ; RISCV32-NEXT:    mul a4, a0, a3
-; RISCV32-NEXT:    mulhu a5, a0, a2
-; RISCV32-NEXT:    add a4, a5, a4
-; RISCV32-NEXT:    sltu a6, a4, a5
-; RISCV32-NEXT:    mulhu a5, a0, a3
-; RISCV32-NEXT:    add a6, a5, a6
-; RISCV32-NEXT:    mulhu a5, a1, a2
-; RISCV32-NEXT:    add a7, a6, a5
+; RISCV32-NEXT:    mulhu a7, a0, a2
+; RISCV32-NEXT:    add a4, a7, a4
 ; RISCV32-NEXT:    mul a5, a1, a2
 ; RISCV32-NEXT:    add a6, a4, a5
-; RISCV32-NEXT:    sltu a4, a6, a4
-; RISCV32-NEXT:    add a4, a7, a4
+; RISCV32-NEXT:    sltu t0, a6, a4
+; RISCV32-NEXT:    sltu a4, a4, a7
+; RISCV32-NEXT:    mulhu a5, a0, a3
+; RISCV32-NEXT:    add a4, a5, a4
+; RISCV32-NEXT:    mulhu a5, a1, a2
+; RISCV32-NEXT:    add a4, a4, a5
+; RISCV32-NEXT:    add a4, a4, t0
 ; RISCV32-NEXT:    mul a5, a1, a3
 ; RISCV32-NEXT:    add a5, a4, a5
 ; RISCV32-NEXT:    bgez a1, .LBB0_2
@@ -31,13 +31,13 @@ define i64 @addcarry(i64 %x, i64 %y) nounwind {
 ; RISCV32-NEXT:  # %bb.3:
 ; RISCV32-NEXT:    sub a5, a5, a0
 ; RISCV32-NEXT:  .LBB0_4:
+; RISCV32-NEXT:    slli a1, a5, 30
+; RISCV32-NEXT:    srli a3, a6, 2
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    slli a3, a6, 30
 ; RISCV32-NEXT:    mul a0, a0, a2
 ; RISCV32-NEXT:    srli a0, a0, 2
-; RISCV32-NEXT:    slli a1, a6, 30
-; RISCV32-NEXT:    or a0, a1, a0
-; RISCV32-NEXT:    srli a1, a6, 2
-; RISCV32-NEXT:    slli a2, a5, 30
-; RISCV32-NEXT:    or a1, a2, a1
+; RISCV32-NEXT:    or a0, a3, a0
 ; RISCV32-NEXT:    ret
   %tmp = call i64 @llvm.smul.fix.i64(i64 %x, i64 %y, i32 2);
   ret i64 %tmp;
diff --git a/llvm/test/CodeGen/RISCV/alloca.ll b/llvm/test/CodeGen/RISCV/alloca.ll
index 22dd42ccb7e3b..6cd9bf46376d4 100644
--- a/llvm/test/CodeGen/RISCV/alloca.ll
+++ b/llvm/test/CodeGen/RISCV/alloca.ll
@@ -82,8 +82,7 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    addi a1, zero, 10
 ; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    addi a1, zero, 9
-; RV32I-NEXT:    sw a1, 0(sp)
+; RV32I-NEXT:    addi t0, zero, 9
 ; RV32I-NEXT:    addi a1, zero, 2
 ; RV32I-NEXT:    addi a2, zero, 3
 ; RV32I-NEXT:    addi a3, zero, 4
@@ -91,6 +90,7 @@ define void @alloca_callframe(i32 %n) nounwind {
 ; RV32I-NEXT:    addi a5, zero, 6
 ; RV32I-NEXT:    addi a6, zero, 7
 ; RV32I-NEXT:    addi a7, zero, 8
+; RV32I-NEXT:    sw t0, 0(sp)
 ; RV32I-NEXT:    call func
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    addi sp, s0, -16
diff --git a/llvm/test/CodeGen/RISCV/alu64.ll b/llvm/test/CodeGen/RISCV/alu64.ll
index 0a44a380df72e..dd336065ef6c0 100644
--- a/llvm/test/CodeGen/RISCV/alu64.ll
+++ b/llvm/test/CodeGen/RISCV/alu64.ll
@@ -123,8 +123,8 @@ define i64 @slli(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: slli:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slli a1, a1, 7
 ; RV32I-NEXT:    srli a2, a0, 25
+; RV32I-NEXT:    slli a1, a1, 7
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    slli a0, a0, 7
 ; RV32I-NEXT:    ret
@@ -140,8 +140,8 @@ define i64 @srli(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: srli:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a0, a0, 8
 ; RV32I-NEXT:    slli a2, a1, 24
+; RV32I-NEXT:    srli a0, a0, 8
 ; RV32I-NEXT:    or a0, a0, a2
 ; RV32I-NEXT:    srli a1, a1, 8
 ; RV32I-NEXT:    ret
@@ -157,8 +157,8 @@ define i64 @srai(i64 %a) nounwind {
 ;
 ; RV32I-LABEL: srai:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    srli a0, a0, 9
 ; RV32I-NEXT:    slli a2, a1, 23
+; RV32I-NEXT:    srli a0, a0, 9
 ; RV32I-NEXT:    or a0, a0, a2
 ; RV32I-NEXT:    srai a1, a1, 9
 ; RV32I-NEXT:    ret
@@ -194,9 +194,9 @@ define i64 @sub(i64 %a, i64 %b) nounwind {
 ;
 ; RV32I-LABEL: sub:
 ; RV32I:       # %bb.0:
+; RV32I-NEXT:    sltu a4, a0, a2
 ; RV32I-NEXT:    sub a1, a1, a3
-; RV32I-NEXT:    sltu a3, a0, a2
-; RV32I-NEXT:    sub a1, a1, a3
+; RV32I-NEXT:    sub a1, a1, a4
 ; RV32I-NEXT:    sub a0, a0, a2
 ; RV32I-NEXT:    ret
   %1 = sub i64 %a, %b
@@ -218,13 +218,14 @@ define i64 @sll(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB11_2:
+; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    srli a4, a0, 1
 ; RV32I-NEXT:    srl a3, a4, a3
-; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    or a1, a1, a3
-; RV32I-NEXT:    sll a0, a0, a2
+; RV32I-NEXT:    sll a2, a0, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:    ret
   %1 = shl i64 %a, %b
   ret i64 %1
@@ -304,13 +305,14 @@ define i64 @srl(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB15_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    mv a1, a2
 ; RV32I-NEXT:    ret
   %1 = lshr i64 %a, %b
   ret i64 %1
@@ -331,11 +333,11 @@ define i64 @sra(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB16_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    sra a1, a1, a2
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
index 3fb6342133e38..4358fb5a1b8ea 100644
--- a/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
+++ b/llvm/test/CodeGen/RISCV/arith-with-overflow.ll
@@ -11,16 +11,10 @@ define i1 @sadd(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: sadd:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    add a3, a0, a1
+; RV32I-NEXT:    slt a0, a3, a0
+; RV32I-NEXT:    slti a1, a1, 0
+; RV32I-NEXT:    xor a0, a1, a0
 ; RV32I-NEXT:    sw a3, 0(a2)
-; RV32I-NEXT:    addi a2, zero, -1
-; RV32I-NEXT:    slt a1, a2, a1
-; RV32I-NEXT:    slt a0, a2, a0
-; RV32I-NEXT:    slt a2, a2, a3
-; RV32I-NEXT:    xor a2, a0, a2
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a0, a1
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.sadd.with.overflow.i32(i32 %a, i32 %b)
@@ -33,17 +27,11 @@ entry:
 define i1 @ssub(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: ssub:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    sub a3, a0, a1
-; RV32I-NEXT:    sw a3, 0(a2)
-; RV32I-NEXT:    addi a2, zero, -1
-; RV32I-NEXT:    slt a1, a2, a1
-; RV32I-NEXT:    slt a0, a2, a0
-; RV32I-NEXT:    slt a2, a2, a3
-; RV32I-NEXT:    xor a2, a0, a2
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    sgtz a3, a1
+; RV32I-NEXT:    sub a1, a0, a1
+; RV32I-NEXT:    slt a0, a1, a0
+; RV32I-NEXT:    xor a0, a3, a0
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.ssub.with.overflow.i32(i32 %a, i32 %b)
@@ -57,8 +45,8 @@ define i1 @uadd(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: uadd:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    add a1, a0, a1
-; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    sltu a0, a1, a0
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
@@ -72,8 +60,8 @@ define i1 @usub(i32 %a, i32 %b, i32* %c) nounwind {
 ; RV32I-LABEL: usub:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    sub a1, a0, a1
-; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    sltu a0, a0, a1
+; RV32I-NEXT:    sw a1, 0(a2)
 ; RV32I-NEXT:    ret
 entry:
   %x = call {i32, i1} @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
index b331b4b9926b6..082b92bb66986 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg-flag.ll
@@ -7,7 +7,6 @@
 ; higher bits were masked to zero for the comparison.
 
 define i1 @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 signext %cmp,
-        i32 signext %val) nounwind {
 ; RV64IA-LABEL: cmpxchg_i32_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0: # %entry
 ; RV64IA-NEXT:  .LBB0_1: # %entry
@@ -22,6 +21,7 @@ define i1 @cmpxchg_i32_seq_cst_seq_cst(i32* %ptr, i32 signext %cmp,
 ; RV64IA-NEXT:    xor a0, a3, a1
 ; RV64IA-NEXT:    seqz a0, a0
 ; RV64IA-NEXT:    ret
+        i32 signext %val) nounwind {
 entry:
   %0 = cmpxchg i32* %ptr, i32 %cmp, i32 %val seq_cst seq_cst
   %1 = extractvalue { i32, i1 } %0, 1
diff --git a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
index a4526b7f46e76..43da05ebe7c71 100644
--- a/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-cmpxchg.ll
@@ -24,24 +24,24 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB0_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB0_1
 ; RV32IA-NEXT:  .LBB0_3:
 ; RV32IA-NEXT:    ret
@@ -61,24 +61,24 @@ define void @cmpxchg_i8_monotonic_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i8_monotonic_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB0_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB0_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB0_1
 ; RV64IA-NEXT:  .LBB0_3:
 ; RV64IA-NEXT:    ret
@@ -102,24 +102,24 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB1_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-NEXT:  .LBB1_3:
 ; RV32IA-NEXT:    ret
@@ -139,24 +139,24 @@ define void @cmpxchg_i8_acquire_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acquire_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB1_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB1_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-NEXT:  .LBB1_3:
 ; RV64IA-NEXT:    ret
@@ -180,24 +180,24 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB2_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-NEXT:  .LBB2_3:
 ; RV32IA-NEXT:    ret
@@ -217,24 +217,24 @@ define void @cmpxchg_i8_acquire_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acquire_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB2_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB2_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-NEXT:  .LBB2_3:
 ; RV64IA-NEXT:    ret
@@ -258,24 +258,24 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_release_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB3_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-NEXT:  .LBB3_3:
 ; RV32IA-NEXT:    ret
@@ -295,24 +295,24 @@ define void @cmpxchg_i8_release_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_release_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB3_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB3_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-NEXT:  .LBB3_3:
 ; RV64IA-NEXT:    ret
@@ -336,24 +336,24 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_release_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB4_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-NEXT:  .LBB4_3:
 ; RV32IA-NEXT:    ret
@@ -373,24 +373,24 @@ define void @cmpxchg_i8_release_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_release_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB4_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB4_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-NEXT:  .LBB4_3:
 ; RV64IA-NEXT:    ret
@@ -414,24 +414,24 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB5_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-NEXT:  .LBB5_3:
 ; RV32IA-NEXT:    ret
@@ -451,24 +451,24 @@ define void @cmpxchg_i8_acq_rel_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acq_rel_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB5_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB5_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-NEXT:  .LBB5_3:
 ; RV64IA-NEXT:    ret
@@ -492,24 +492,24 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB6_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-NEXT:  .LBB6_3:
 ; RV32IA-NEXT:    ret
@@ -529,24 +529,24 @@ define void @cmpxchg_i8_acq_rel_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_acq_rel_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB6_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB6_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-NEXT:  .LBB6_3:
 ; RV64IA-NEXT:    ret
@@ -570,24 +570,24 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB7_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB7_1
 ; RV32IA-NEXT:  .LBB7_3:
 ; RV32IA-NEXT:    ret
@@ -607,24 +607,24 @@ define void @cmpxchg_i8_seq_cst_monotonic(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB7_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB7_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB7_1
 ; RV64IA-NEXT:  .LBB7_3:
 ; RV64IA-NEXT:    ret
@@ -648,24 +648,24 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB8_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB8_1
 ; RV32IA-NEXT:  .LBB8_3:
 ; RV32IA-NEXT:    ret
@@ -685,24 +685,24 @@ define void @cmpxchg_i8_seq_cst_acquire(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB8_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB8_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB8_1
 ; RV64IA-NEXT:  .LBB8_3:
 ; RV64IA-NEXT:    ret
@@ -726,24 +726,24 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV32IA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a4, a4, a3
-; RV32IA-NEXT:    andi a2, a2, 255
-; RV32IA-NEXT:    sll a2, a2, a3
+; RV32IA-NEXT:    sll a4, a4, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    andi a2, a2, 255
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV32IA-NEXT:    and a5, a3, a4
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a5, a2, a4
 ; RV32IA-NEXT:    bne a5, a1, .LBB9_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
-; RV32IA-NEXT:    xor a5, a3, a2
+; RV32IA-NEXT:    xor a5, a2, a0
 ; RV32IA-NEXT:    and a5, a5, a4
-; RV32IA-NEXT:    xor a5, a3, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a2, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV32IA-NEXT:    bnez a5, .LBB9_1
 ; RV32IA-NEXT:  .LBB9_3:
 ; RV32IA-NEXT:    ret
@@ -763,24 +763,24 @@ define void @cmpxchg_i8_seq_cst_seq_cst(i8* %ptr, i8 %cmp, i8 %val) nounwind {
 ;
 ; RV64IA-LABEL: cmpxchg_i8_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a4, a4, a3
-; RV64IA-NEXT:    andi a2, a2, 255
-; RV64IA-NEXT:    sllw a2, a2, a3
+; RV64IA-NEXT:    sllw a4, a4, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    andi a2, a2, 255
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a3, (a0)
-; RV64IA-NEXT:    and a5, a3, a4
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a5, a2, a4
 ; RV64IA-NEXT:    bne a5, a1, .LBB9_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB9_1 Depth=1
-; RV64IA-NEXT:    xor a5, a3, a2
+; RV64IA-NEXT:    xor a5, a2, a0
 ; RV64IA-NEXT:    and a5, a5, a4
-; RV64IA-NEXT:    xor a5, a3, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a2, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a3)
 ; RV64IA-NEXT:    bnez a5, .LBB9_1
 ; RV64IA-NEXT:  .LBB9_3:
 ; RV64IA-NEXT:    ret
@@ -804,26 +804,26 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ;
 ; RV32IA-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB10_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB10_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB10_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB10_1
 ; RV32IA-NEXT:  .LBB10_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -842,26 +842,26 @@ define void @cmpxchg_i16_monotonic_monotonic(i16* %ptr, i16 %cmp, i16 %val) noun
 ;
 ; RV64IA-LABEL: cmpxchg_i16_monotonic_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB10_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB10_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB10_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB10_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB10_1
 ; RV64IA-NEXT:  .LBB10_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val monotonic monotonic
@@ -884,26 +884,26 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB11_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB11_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB11_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB11_1
 ; RV32IA-NEXT:  .LBB11_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -922,26 +922,26 @@ define void @cmpxchg_i16_acquire_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acquire_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB11_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB11_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB11_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB11_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB11_1
 ; RV64IA-NEXT:  .LBB11_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire monotonic
@@ -964,26 +964,26 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB12_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB12_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB12_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB12_1
 ; RV32IA-NEXT:  .LBB12_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1002,26 +1002,26 @@ define void @cmpxchg_i16_acquire_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acquire_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB12_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB12_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB12_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB12_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB12_1
 ; RV64IA-NEXT:  .LBB12_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acquire acquire
@@ -1044,26 +1044,26 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_release_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB13_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB13_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB13_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB13_1
 ; RV32IA-NEXT:  .LBB13_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1082,26 +1082,26 @@ define void @cmpxchg_i16_release_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_release_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB13_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB13_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB13_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB13_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB13_1
 ; RV64IA-NEXT:  .LBB13_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release monotonic
@@ -1124,26 +1124,26 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_release_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB14_3
+; RV32IA-NEXT:    lr.w a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB14_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB14_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB14_1
 ; RV32IA-NEXT:  .LBB14_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1162,26 +1162,26 @@ define void @cmpxchg_i16_release_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_release_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB14_3
+; RV64IA-NEXT:    lr.w a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB14_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB14_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB14_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB14_1
 ; RV64IA-NEXT:  .LBB14_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val release acquire
@@ -1204,26 +1204,26 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB15_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB15_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB15_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB15_1
 ; RV32IA-NEXT:  .LBB15_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1242,26 +1242,26 @@ define void @cmpxchg_i16_acq_rel_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acq_rel_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB15_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB15_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB15_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB15_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB15_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB15_1
 ; RV64IA-NEXT:  .LBB15_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel monotonic
@@ -1284,26 +1284,26 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB16_3
+; RV32IA-NEXT:    lr.w.aq a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB16_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB16_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB16_1
 ; RV32IA-NEXT:  .LBB16_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1322,26 +1322,26 @@ define void @cmpxchg_i16_acq_rel_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_acq_rel_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB16_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB16_3
+; RV64IA-NEXT:    lr.w.aq a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB16_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB16_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB16_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.rl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB16_1
 ; RV64IA-NEXT:  .LBB16_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val acq_rel acquire
@@ -1364,26 +1364,26 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB17_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB17_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB17_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB17_1
 ; RV32IA-NEXT:  .LBB17_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1402,26 +1402,26 @@ define void @cmpxchg_i16_seq_cst_monotonic(i16* %ptr, i16 %cmp, i16 %val) nounwi
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB17_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB17_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB17_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB17_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB17_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB17_1
 ; RV64IA-NEXT:  .LBB17_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst monotonic
@@ -1444,26 +1444,26 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB18_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB18_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB18_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB18_1
 ; RV32IA-NEXT:  .LBB18_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1482,26 +1482,26 @@ define void @cmpxchg_i16_seq_cst_acquire(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB18_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB18_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB18_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB18_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB18_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB18_1
 ; RV64IA-NEXT:  .LBB18_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst acquire
@@ -1524,26 +1524,26 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV32IA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a3, 16
-; RV32IA-NEXT:    addi a3, a3, -1
-; RV32IA-NEXT:    and a1, a1, a3
-; RV32IA-NEXT:    and a2, a2, a3
-; RV32IA-NEXT:    slli a4, a0, 3
-; RV32IA-NEXT:    andi a4, a4, 24
-; RV32IA-NEXT:    sll a3, a3, a4
-; RV32IA-NEXT:    sll a2, a2, a4
-; RV32IA-NEXT:    sll a1, a1, a4
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a3, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a4, 16
+; RV32IA-NEXT:    addi a4, a4, -1
+; RV32IA-NEXT:    sll a5, a4, a0
+; RV32IA-NEXT:    and a1, a1, a4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    and a2, a2, a4
+; RV32IA-NEXT:    sll a0, a2, a0
 ; RV32IA-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a3
-; RV32IA-NEXT:    bne a5, a1, .LBB19_3
+; RV32IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV32IA-NEXT:    and a4, a2, a5
+; RV32IA-NEXT:    bne a4, a1, .LBB19_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a2
-; RV32IA-NEXT:    and a5, a5, a3
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV32IA-NEXT:    bnez a5, .LBB19_1
+; RV32IA-NEXT:    xor a4, a2, a0
+; RV32IA-NEXT:    and a4, a4, a5
+; RV32IA-NEXT:    xor a4, a2, a4
+; RV32IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV32IA-NEXT:    bnez a4, .LBB19_1
 ; RV32IA-NEXT:  .LBB19_3:
 ; RV32IA-NEXT:    ret
 ;
@@ -1562,26 +1562,26 @@ define void @cmpxchg_i16_seq_cst_seq_cst(i16* %ptr, i16 %cmp, i16 %val) nounwind
 ;
 ; RV64IA-LABEL: cmpxchg_i16_seq_cst_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a3, 16
-; RV64IA-NEXT:    addiw a3, a3, -1
-; RV64IA-NEXT:    and a1, a1, a3
-; RV64IA-NEXT:    and a2, a2, a3
-; RV64IA-NEXT:    slli a4, a0, 3
-; RV64IA-NEXT:    andi a4, a4, 24
-; RV64IA-NEXT:    sllw a3, a3, a4
-; RV64IA-NEXT:    sllw a2, a2, a4
-; RV64IA-NEXT:    sllw a1, a1, a4
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a3, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a4, 16
+; RV64IA-NEXT:    addiw a4, a4, -1
+; RV64IA-NEXT:    sllw a5, a4, a0
+; RV64IA-NEXT:    and a1, a1, a4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    and a2, a2, a4
+; RV64IA-NEXT:    sllw a0, a2, a0
 ; RV64IA-NEXT:  .LBB19_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a3
-; RV64IA-NEXT:    bne a5, a1, .LBB19_3
+; RV64IA-NEXT:    lr.w.aqrl a2, (a3)
+; RV64IA-NEXT:    and a4, a2, a5
+; RV64IA-NEXT:    bne a4, a1, .LBB19_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB19_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a2
-; RV64IA-NEXT:    and a5, a5, a3
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
-; RV64IA-NEXT:    bnez a5, .LBB19_1
+; RV64IA-NEXT:    xor a4, a2, a0
+; RV64IA-NEXT:    and a4, a4, a5
+; RV64IA-NEXT:    xor a4, a2, a4
+; RV64IA-NEXT:    sc.w.aqrl a4, a4, (a3)
+; RV64IA-NEXT:    bnez a4, .LBB19_1
 ; RV64IA-NEXT:  .LBB19_3:
 ; RV64IA-NEXT:    ret
   %res = cmpxchg i16* %ptr, i16 %cmp, i16 %val seq_cst seq_cst
@@ -2173,12 +2173,13 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
+; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    addi  a4, zero, 2
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2189,12 +2190,13 @@ define void @cmpxchg_i64_acquire_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2233,13 +2235,14 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2249,13 +2252,14 @@ define void @cmpxchg_i64_acquire_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2293,12 +2297,13 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2309,12 +2314,13 @@ define void @cmpxchg_i64_release_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2353,13 +2359,14 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 3
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2369,13 +2376,14 @@ define void @cmpxchg_i64_release_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 3
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2413,12 +2421,13 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 4
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2429,12 +2438,13 @@ define void @cmpxchg_i64_acq_rel_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 4
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2473,13 +2483,14 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2489,13 +2500,14 @@ define void @cmpxchg_i64_acq_rel_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2533,12 +2545,13 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a5, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
@@ -2549,12 +2562,13 @@ define void @cmpxchg_i64_seq_cst_monotonic(i64* %ptr, i64 %cmp, i64 %val) nounwi
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a5, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
@@ -2593,13 +2607,14 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2609,13 +2624,14 @@ define void @cmpxchg_i64_seq_cst_acquire(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
@@ -2653,13 +2669,14 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
+; RV32I-NEXT:    mv a6, a4
 ; RV32I-NEXT:    sw a2, 4(sp)
 ; RV32I-NEXT:    sw a1, 0(sp)
 ; RV32I-NEXT:    mv a1, sp
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    mv a3, a6
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -2669,13 +2686,14 @@ define void @cmpxchg_i64_seq_cst_seq_cst(i64* %ptr, i64 %cmp, i64 %val) nounwind
 ; RV32IA:       # %bb.0:
 ; RV32IA-NEXT:    addi sp, sp, -16
 ; RV32IA-NEXT:    sw ra, 12(sp)
+; RV32IA-NEXT:    mv a6, a4
 ; RV32IA-NEXT:    sw a2, 4(sp)
 ; RV32IA-NEXT:    sw a1, 0(sp)
 ; RV32IA-NEXT:    mv a1, sp
-; RV32IA-NEXT:    mv a2, a3
-; RV32IA-NEXT:    mv a3, a4
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a2, a3
+; RV32IA-NEXT:    mv a3, a6
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
 ; RV32IA-NEXT:    lw ra, 12(sp)
 ; RV32IA-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/atomic-rmw.ll b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
index c12a298b15e38..dc396efd530d5 100644
--- a/llvm/test/CodeGen/RISCV/atomic-rmw.ll
+++ b/llvm/test/CodeGen/RISCV/atomic-rmw.ll
@@ -21,23 +21,23 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB0_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_monotonic:
@@ -52,23 +52,23 @@ define i8 @atomicrmw_xchg_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB0_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB0_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b monotonic
   ret i8 %1
@@ -87,23 +87,23 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB1_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_acquire:
@@ -118,23 +118,23 @@ define i8 @atomicrmw_xchg_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB1_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB1_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acquire
   ret i8 %1
@@ -153,23 +153,23 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB2_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_release:
@@ -184,23 +184,23 @@ define i8 @atomicrmw_xchg_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB2_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB2_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b release
   ret i8 %1
@@ -219,23 +219,23 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB3_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_acq_rel:
@@ -250,23 +250,23 @@ define i8 @atomicrmw_xchg_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB3_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB3_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -285,23 +285,23 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB4_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i8_seq_cst:
@@ -316,23 +316,23 @@ define i8 @atomicrmw_xchg_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB4_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB4_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -351,23 +351,23 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB5_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_monotonic:
@@ -382,23 +382,23 @@ define i8 @atomicrmw_add_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB5_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB5_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b monotonic
   ret i8 %1
@@ -417,23 +417,23 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB6_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_acquire:
@@ -448,23 +448,23 @@ define i8 @atomicrmw_add_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB6_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB6_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acquire
   ret i8 %1
@@ -483,23 +483,23 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB7_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_release:
@@ -514,23 +514,23 @@ define i8 @atomicrmw_add_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB7_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB7_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b release
   ret i8 %1
@@ -549,23 +549,23 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB8_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_acq_rel:
@@ -580,23 +580,23 @@ define i8 @atomicrmw_add_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB8_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB8_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -615,23 +615,23 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    add a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB9_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i8_seq_cst:
@@ -646,23 +646,23 @@ define i8 @atomicrmw_add_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB9_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    add a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB9_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -681,23 +681,23 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB10_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_monotonic:
@@ -712,23 +712,23 @@ define i8 @atomicrmw_sub_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB10_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB10_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b monotonic
   ret i8 %1
@@ -747,23 +747,23 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB11_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_acquire:
@@ -778,23 +778,23 @@ define i8 @atomicrmw_sub_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB11_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB11_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acquire
   ret i8 %1
@@ -813,23 +813,23 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB12_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_release:
@@ -844,23 +844,23 @@ define i8 @atomicrmw_sub_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB12_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB12_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b release
   ret i8 %1
@@ -879,23 +879,23 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB13_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_acq_rel:
@@ -910,23 +910,23 @@ define i8 @atomicrmw_sub_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB13_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB13_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -945,23 +945,23 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    sub a5, a4, a1
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB14_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i8_seq_cst:
@@ -976,23 +976,23 @@ define i8 @atomicrmw_sub_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB14_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    sub a5, a4, a1
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB14_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1011,17 +1011,17 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_monotonic:
@@ -1036,17 +1036,17 @@ define i8 @atomicrmw_and_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1065,17 +1065,17 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_acquire:
@@ -1090,17 +1090,17 @@ define i8 @atomicrmw_and_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acquire
   ret i8 %1
@@ -1119,17 +1119,17 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_release:
@@ -1144,17 +1144,17 @@ define i8 @atomicrmw_and_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b release
   ret i8 %1
@@ -1173,17 +1173,17 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_acq_rel:
@@ -1198,17 +1198,17 @@ define i8 @atomicrmw_and_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1227,17 +1227,17 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    not a3, a3
+; RV32IA-NEXT:    andi a1, a1, 255
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:    or a1, a3, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i8_seq_cst:
@@ -1252,17 +1252,17 @@ define i8 @atomicrmw_and_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    not a3, a3
+; RV64IA-NEXT:    andi a1, a1, 255
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:    or a1, a3, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1281,24 +1281,24 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB20_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_monotonic:
@@ -1313,24 +1313,24 @@ define i8 @atomicrmw_nand_i8_monotonic(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB20_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB20_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1349,24 +1349,24 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB21_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_acquire:
@@ -1381,24 +1381,24 @@ define i8 @atomicrmw_nand_i8_acquire(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB21_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB21_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acquire
   ret i8 %1
@@ -1417,24 +1417,24 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB22_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_release:
@@ -1449,24 +1449,24 @@ define i8 @atomicrmw_nand_i8_release(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB22_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB22_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b release
   ret i8 %1
@@ -1485,24 +1485,24 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB23_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_acq_rel:
@@ -1517,24 +1517,24 @@ define i8 @atomicrmw_nand_i8_acq_rel(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB23_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB23_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1553,24 +1553,24 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a3, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV32IA-NEXT:    and a5, a4, a1
 ; RV32IA-NEXT:    not a5, a5
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB24_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i8_seq_cst:
@@ -1585,24 +1585,24 @@ define i8 @atomicrmw_nand_i8_seq_cst(i8* %a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a3, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB24_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a4, (a2)
 ; RV64IA-NEXT:    and a5, a4, a1
 ; RV64IA-NEXT:    not a5, a5
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB24_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1621,13 +1621,13 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_monotonic:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_monotonic:
@@ -1642,13 +1642,13 @@ define i8 @atomicrmw_or_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1667,13 +1667,13 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_acquire:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_acquire:
@@ -1688,13 +1688,13 @@ define i8 @atomicrmw_or_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acquire
   ret i8 %1
@@ -1713,13 +1713,13 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_release:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_release:
@@ -1734,13 +1734,13 @@ define i8 @atomicrmw_or_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_release:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b release
   ret i8 %1
@@ -1759,13 +1759,13 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_acq_rel:
@@ -1780,13 +1780,13 @@ define i8 @atomicrmw_or_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_acq_rel:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -1805,13 +1805,13 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i8_seq_cst:
@@ -1826,13 +1826,13 @@ define i8 @atomicrmw_or_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i8_seq_cst:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -1851,13 +1851,13 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_monotonic:
@@ -1872,13 +1872,13 @@ define i8 @atomicrmw_xor_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_monotonic:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b monotonic
   ret i8 %1
@@ -1897,13 +1897,13 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_acquire:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_acquire:
@@ -1918,13 +1918,13 @@ define i8 @atomicrmw_xor_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_acquire:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acquire
   ret i8 %1
@@ -1943,13 +1943,13 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_release:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_release:
@@ -1964,13 +1964,13 @@ define i8 @atomicrmw_xor_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_release:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b release
   ret i8 %1
@@ -1989,13 +1989,13 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_acq_rel:
@@ -2010,13 +2010,13 @@ define i8 @atomicrmw_xor_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_acq_rel:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -2035,13 +2035,13 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV32IA:       # %bb.0:
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i8_seq_cst:
@@ -2056,13 +2056,13 @@ define i8 @atomicrmw_xor_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i8_seq_cst:
 ; RV64IA:       # %bb.0:
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -2079,33 +2079,33 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB35_2
 ; RV32I-NEXT:  .LBB35_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB35_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB35_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB35_4
 ; RV32I-NEXT:  .LBB35_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB35_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB35_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB35_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB35_1
 ; RV32I-NEXT:  .LBB35_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2116,32 +2116,32 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB35_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB35_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB35_3: # in Loop: Header=BB35_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB35_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB35_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_monotonic:
@@ -2154,33 +2154,33 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB35_2
 ; RV64I-NEXT:  .LBB35_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB35_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB35_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB35_4
 ; RV64I-NEXT:  .LBB35_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB35_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB35_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB35_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB35_1
 ; RV64I-NEXT:  .LBB35_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2191,32 +2191,32 @@ define i8 @atomicrmw_max_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB35_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB35_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB35_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB35_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB35_3: # in Loop: Header=BB35_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB35_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB35_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b monotonic
   ret i8 %1
@@ -2233,33 +2233,33 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB36_2
 ; RV32I-NEXT:  .LBB36_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB36_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB36_4
 ; RV32I-NEXT:  .LBB36_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB36_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB36_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB36_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB36_1
 ; RV32I-NEXT:  .LBB36_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2270,32 +2270,32 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB36_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB36_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB36_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB36_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_acquire:
@@ -2308,33 +2308,33 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB36_2
 ; RV64I-NEXT:  .LBB36_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB36_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB36_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB36_4
 ; RV64I-NEXT:  .LBB36_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB36_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB36_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB36_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB36_1
 ; RV64I-NEXT:  .LBB36_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2345,32 +2345,32 @@ define i8 @atomicrmw_max_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB36_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB36_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB36_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB36_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB36_3: # in Loop: Header=BB36_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB36_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB36_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acquire
   ret i8 %1
@@ -2387,33 +2387,33 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB37_2
 ; RV32I-NEXT:  .LBB37_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB37_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB37_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB37_4
 ; RV32I-NEXT:  .LBB37_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB37_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB37_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB37_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB37_1
 ; RV32I-NEXT:  .LBB37_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2424,32 +2424,32 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB37_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB37_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB37_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB37_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_release:
@@ -2462,33 +2462,33 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB37_2
 ; RV64I-NEXT:  .LBB37_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB37_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB37_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB37_4
 ; RV64I-NEXT:  .LBB37_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB37_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB37_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB37_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB37_1
 ; RV64I-NEXT:  .LBB37_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2499,32 +2499,32 @@ define i8 @atomicrmw_max_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB37_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB37_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB37_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB37_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB37_3: # in Loop: Header=BB37_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB37_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB37_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b release
   ret i8 %1
@@ -2541,33 +2541,33 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB38_2
 ; RV32I-NEXT:  .LBB38_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB38_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB38_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB38_4
 ; RV32I-NEXT:  .LBB38_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB38_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB38_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB38_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB38_1
 ; RV32I-NEXT:  .LBB38_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2578,32 +2578,32 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB38_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB38_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB38_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB38_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_acq_rel:
@@ -2616,33 +2616,33 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB38_2
 ; RV64I-NEXT:  .LBB38_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB38_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB38_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB38_4
 ; RV64I-NEXT:  .LBB38_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB38_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB38_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB38_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB38_1
 ; RV64I-NEXT:  .LBB38_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2653,32 +2653,32 @@ define i8 @atomicrmw_max_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB38_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB38_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB38_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB38_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB38_3: # in Loop: Header=BB38_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB38_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB38_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -2695,33 +2695,33 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB39_2
 ; RV32I-NEXT:  .LBB39_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB39_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB39_4
 ; RV32I-NEXT:  .LBB39_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB39_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB39_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB39_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB39_1
 ; RV32I-NEXT:  .LBB39_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2732,32 +2732,32 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB39_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB39_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB39_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB39_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i8_seq_cst:
@@ -2770,33 +2770,33 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB39_2
 ; RV64I-NEXT:  .LBB39_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB39_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB39_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB39_4
 ; RV64I-NEXT:  .LBB39_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB39_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB39_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB39_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB39_1
 ; RV64I-NEXT:  .LBB39_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2807,32 +2807,32 @@ define i8 @atomicrmw_max_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB39_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB39_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB39_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB39_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB39_3: # in Loop: Header=BB39_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB39_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB39_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -2849,33 +2849,33 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB40_2
 ; RV32I-NEXT:  .LBB40_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB40_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB40_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB40_4
 ; RV32I-NEXT:  .LBB40_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB40_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB40_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB40_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB40_1
 ; RV32I-NEXT:  .LBB40_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -2886,32 +2886,32 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB40_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB40_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB40_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB40_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_monotonic:
@@ -2924,33 +2924,33 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB40_2
 ; RV64I-NEXT:  .LBB40_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB40_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB40_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB40_4
 ; RV64I-NEXT:  .LBB40_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB40_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB40_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB40_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB40_1
 ; RV64I-NEXT:  .LBB40_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -2961,32 +2961,32 @@ define i8 @atomicrmw_min_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB40_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB40_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB40_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB40_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB40_3: # in Loop: Header=BB40_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB40_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB40_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b monotonic
   ret i8 %1
@@ -3003,33 +3003,33 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB41_2
 ; RV32I-NEXT:  .LBB41_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB41_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB41_4
 ; RV32I-NEXT:  .LBB41_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB41_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB41_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB41_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB41_1
 ; RV32I-NEXT:  .LBB41_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3040,32 +3040,32 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB41_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB41_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB41_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB41_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_acquire:
@@ -3078,33 +3078,33 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB41_2
 ; RV64I-NEXT:  .LBB41_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB41_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB41_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB41_4
 ; RV64I-NEXT:  .LBB41_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB41_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB41_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB41_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB41_1
 ; RV64I-NEXT:  .LBB41_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3115,32 +3115,32 @@ define i8 @atomicrmw_min_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB41_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB41_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB41_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB41_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB41_3: # in Loop: Header=BB41_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB41_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB41_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acquire
   ret i8 %1
@@ -3157,33 +3157,33 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB42_2
 ; RV32I-NEXT:  .LBB42_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB42_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
+; RV32I-NEXT:    sb a1, 11(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB42_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB42_4
 ; RV32I-NEXT:  .LBB42_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB42_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB42_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB42_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB42_1
 ; RV32I-NEXT:  .LBB42_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3194,32 +3194,32 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB42_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB42_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB42_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB42_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_release:
@@ -3232,33 +3232,33 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB42_2
 ; RV64I-NEXT:  .LBB42_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB42_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
+; RV64I-NEXT:    sb a1, 7(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB42_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB42_4
 ; RV64I-NEXT:  .LBB42_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB42_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB42_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB42_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB42_1
 ; RV64I-NEXT:  .LBB42_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3269,32 +3269,32 @@ define i8 @atomicrmw_min_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB42_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB42_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB42_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB42_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB42_3: # in Loop: Header=BB42_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB42_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB42_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b release
   ret i8 %1
@@ -3311,33 +3311,33 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB43_2
 ; RV32I-NEXT:  .LBB43_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB43_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB43_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB43_4
 ; RV32I-NEXT:  .LBB43_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB43_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB43_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB43_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB43_1
 ; RV32I-NEXT:  .LBB43_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3348,32 +3348,32 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB43_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB43_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB43_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB43_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_acq_rel:
@@ -3386,33 +3386,33 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB43_2
 ; RV64I-NEXT:  .LBB43_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB43_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB43_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB43_4
 ; RV64I-NEXT:  .LBB43_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB43_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB43_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB43_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB43_1
 ; RV64I-NEXT:  .LBB43_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3423,32 +3423,32 @@ define i8 @atomicrmw_min_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB43_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB43_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB43_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB43_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB43_3: # in Loop: Header=BB43_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB43_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB43_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -3465,33 +3465,33 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 24
-; RV32I-NEXT:    srai s0, a1, 24
+; RV32I-NEXT:    lbu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 24
+; RV32I-NEXT:    srai s0, a0, 24
 ; RV32I-NEXT:    addi s3, sp, 11
 ; RV32I-NEXT:    j .LBB44_2
 ; RV32I-NEXT:  .LBB44_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a1, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB44_4
+; RV32I-NEXT:    lb a1, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB44_4
 ; RV32I-NEXT:  .LBB44_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 24
-; RV32I-NEXT:    srai a1, a1, 24
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB44_1
+; RV32I-NEXT:    slli a0, a1, 24
+; RV32I-NEXT:    srai a0, a0, 24
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB44_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB44_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB44_1
 ; RV32I-NEXT:  .LBB44_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3502,32 +3502,32 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 24
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    addi a4, zero, 255
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    addi a3, zero, 255
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 24
 ; RV32IA-NEXT:    srai a1, a1, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 24
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB44_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB44_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB44_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB44_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i8_seq_cst:
@@ -3540,33 +3540,33 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 56
-; RV64I-NEXT:    srai s0, a1, 56
+; RV64I-NEXT:    lbu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 56
+; RV64I-NEXT:    srai s0, a0, 56
 ; RV64I-NEXT:    addi s3, sp, 7
 ; RV64I-NEXT:    j .LBB44_2
 ; RV64I-NEXT:  .LBB44_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB44_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a1, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB44_4
+; RV64I-NEXT:    lb a1, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB44_4
 ; RV64I-NEXT:  .LBB44_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 56
-; RV64I-NEXT:    srai a1, a1, 56
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB44_1
+; RV64I-NEXT:    slli a0, a1, 56
+; RV64I-NEXT:    srai a0, a0, 56
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB44_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB44_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB44_1
 ; RV64I-NEXT:  .LBB44_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3577,32 +3577,32 @@ define i8 @atomicrmw_min_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 56
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    addi a4, zero, 255
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    addi a3, zero, 255
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 56
 ; RV64IA-NEXT:    srai a1, a1, 56
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 56
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB44_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB44_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB44_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB44_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB44_3: # in Loop: Header=BB44_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB44_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB44_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -3617,33 +3617,33 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB45_2
 ; RV32I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB45_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB45_4
 ; RV32I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB45_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB45_1
 ; RV32I-NEXT:  .LBB45_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3654,27 +3654,27 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB45_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB45_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB45_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_monotonic:
@@ -3685,33 +3685,33 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB45_2
 ; RV64I-NEXT:  .LBB45_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB45_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB45_4
 ; RV64I-NEXT:  .LBB45_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB45_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB45_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB45_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB45_1
 ; RV64I-NEXT:  .LBB45_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3722,27 +3722,27 @@ define i8 @atomicrmw_umax_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB45_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB45_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB45_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB45_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB45_3: # in Loop: Header=BB45_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB45_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b monotonic
   ret i8 %1
@@ -3757,33 +3757,33 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB46_2
 ; RV32I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB46_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB46_4
 ; RV32I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB46_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB46_1
 ; RV32I-NEXT:  .LBB46_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3794,27 +3794,27 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB46_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB46_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB46_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_acquire:
@@ -3825,33 +3825,33 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB46_2
 ; RV64I-NEXT:  .LBB46_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB46_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB46_4
 ; RV64I-NEXT:  .LBB46_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB46_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB46_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB46_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB46_1
 ; RV64I-NEXT:  .LBB46_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -3862,27 +3862,27 @@ define i8 @atomicrmw_umax_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB46_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB46_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB46_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB46_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB46_3: # in Loop: Header=BB46_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB46_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acquire
   ret i8 %1
@@ -3897,33 +3897,33 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB47_2
 ; RV32I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB47_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB47_4
 ; RV32I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB47_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB47_1
 ; RV32I-NEXT:  .LBB47_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -3934,27 +3934,27 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB47_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB47_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB47_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB47_3: # in Loop: Header=BB47_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB47_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_release:
@@ -3965,33 +3965,33 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB47_2
 ; RV64I-NEXT:  .LBB47_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB47_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB47_4
 ; RV64I-NEXT:  .LBB47_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB47_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB47_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB47_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB47_1
 ; RV64I-NEXT:  .LBB47_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4002,27 +4002,27 @@ define i8 @atomicrmw_umax_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB47_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB47_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB47_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB47_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB47_3: # in Loop: Header=BB47_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB47_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b release
   ret i8 %1
@@ -4037,33 +4037,33 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB48_2
 ; RV32I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB48_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB48_4
 ; RV32I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB48_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB48_1
 ; RV32I-NEXT:  .LBB48_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4074,27 +4074,27 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB48_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB48_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB48_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_acq_rel:
@@ -4105,33 +4105,33 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB48_2
 ; RV64I-NEXT:  .LBB48_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB48_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB48_4
 ; RV64I-NEXT:  .LBB48_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB48_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB48_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB48_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB48_1
 ; RV64I-NEXT:  .LBB48_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4142,27 +4142,27 @@ define i8 @atomicrmw_umax_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB48_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB48_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB48_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB48_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB48_3: # in Loop: Header=BB48_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB48_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -4177,33 +4177,33 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB49_2
 ; RV32I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB49_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB49_4
 ; RV32I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s0, a1, .LBB49_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB49_1
 ; RV32I-NEXT:  .LBB49_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4214,27 +4214,27 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a3, a1, .LBB49_3
+; RV32IA-NEXT:    bgeu a2, a1, .LBB49_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB49_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i8_seq_cst:
@@ -4245,33 +4245,33 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB49_2
 ; RV64I-NEXT:  .LBB49_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB49_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB49_4
 ; RV64I-NEXT:  .LBB49_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB49_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB49_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB49_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB49_1
 ; RV64I-NEXT:  .LBB49_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4282,27 +4282,27 @@ define i8 @atomicrmw_umax_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB49_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a3, a1, .LBB49_3
+; RV64IA-NEXT:    bgeu a2, a1, .LBB49_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB49_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB49_3: # in Loop: Header=BB49_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB49_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -4317,33 +4317,33 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB50_2
 ; RV32I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB50_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB50_4
 ; RV32I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB50_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB50_1
 ; RV32I-NEXT:  .LBB50_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4354,27 +4354,27 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB50_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB50_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB50_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_monotonic:
@@ -4385,33 +4385,33 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB50_2
 ; RV64I-NEXT:  .LBB50_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB50_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB50_4
 ; RV64I-NEXT:  .LBB50_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB50_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB50_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB50_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB50_1
 ; RV64I-NEXT:  .LBB50_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4422,27 +4422,27 @@ define i8 @atomicrmw_umin_i8_monotonic(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB50_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB50_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB50_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB50_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB50_3: # in Loop: Header=BB50_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB50_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b monotonic
   ret i8 %1
@@ -4457,33 +4457,33 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB51_2
 ; RV32I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB51_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB51_4
 ; RV32I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB51_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB51_1
 ; RV32I-NEXT:  .LBB51_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4494,27 +4494,27 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB51_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB51_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB51_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB51_3: # in Loop: Header=BB51_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB51_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_acquire:
@@ -4525,33 +4525,33 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB51_2
 ; RV64I-NEXT:  .LBB51_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB51_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB51_4
 ; RV64I-NEXT:  .LBB51_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB51_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB51_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB51_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB51_1
 ; RV64I-NEXT:  .LBB51_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4562,27 +4562,27 @@ define i8 @atomicrmw_umin_i8_acquire(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB51_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB51_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB51_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB51_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB51_3: # in Loop: Header=BB51_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB51_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acquire
   ret i8 %1
@@ -4597,33 +4597,33 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB52_2
 ; RV32I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB52_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB52_4
 ; RV32I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB52_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB52_1
 ; RV32I-NEXT:  .LBB52_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4634,27 +4634,27 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB52_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB52_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB52_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_release:
@@ -4665,33 +4665,33 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB52_2
 ; RV64I-NEXT:  .LBB52_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB52_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB52_4
 ; RV64I-NEXT:  .LBB52_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB52_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB52_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB52_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB52_1
 ; RV64I-NEXT:  .LBB52_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4702,27 +4702,27 @@ define i8 @atomicrmw_umin_i8_release(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB52_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB52_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB52_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB52_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB52_3: # in Loop: Header=BB52_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB52_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b release
   ret i8 %1
@@ -4737,33 +4737,33 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB53_2
 ; RV32I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB53_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB53_4
 ; RV32I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB53_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB53_1
 ; RV32I-NEXT:  .LBB53_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4774,27 +4774,27 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aq a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB53_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB53_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB53_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_acq_rel:
@@ -4805,33 +4805,33 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB53_2
 ; RV64I-NEXT:  .LBB53_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB53_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB53_4
 ; RV64I-NEXT:  .LBB53_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB53_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB53_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB53_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB53_1
 ; RV64I-NEXT:  .LBB53_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4842,27 +4842,27 @@ define i8 @atomicrmw_umin_i8_acq_rel(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB53_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aq a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB53_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB53_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB53_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB53_3: # in Loop: Header=BB53_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB53_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b acq_rel
   ret i8 %1
@@ -4877,33 +4877,33 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lbu a0, 0(a0)
-; RV32I-NEXT:    andi s0, a1, 255
-; RV32I-NEXT:    addi s3, sp, 11
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lbu a3, 0(a0)
+; RV32I-NEXT:    mv s3, a1
+; RV32I-NEXT:    andi s1, a1, 255
+; RV32I-NEXT:    addi s2, sp, 11
 ; RV32I-NEXT:    j .LBB54_2
 ; RV32I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV32I-NEXT:    sb a0, 11(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sb a3, 11(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_1
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lb a0, 11(sp)
-; RV32I-NEXT:    bnez a1, .LBB54_4
+; RV32I-NEXT:    lb a3, 11(sp)
+; RV32I-NEXT:    bnez a0, .LBB54_4
 ; RV32I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    andi a1, a0, 255
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s0, a1, .LBB54_1
+; RV32I-NEXT:    andi a0, a3, 255
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV32I-NEXT:    mv a2, s2
+; RV32I-NEXT:    mv a2, s3
 ; RV32I-NEXT:    j .LBB54_1
 ; RV32I-NEXT:  .LBB54_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -4914,27 +4914,27 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
 ; RV32IA-NEXT:    addi a3, zero, 255
-; RV32IA-NEXT:    sll a6, a3, a2
+; RV32IA-NEXT:    sll a3, a3, a0
 ; RV32IA-NEXT:    andi a1, a1, 255
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a3, a4, a6
+; RV32IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV32IA-NEXT:    and a2, a4, a3
 ; RV32IA-NEXT:    mv a5, a4
-; RV32IA-NEXT:    bgeu a1, a3, .LBB54_3
+; RV32IA-NEXT:    bgeu a1, a2, .LBB54_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
 ; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
+; RV32IA-NEXT:    and a5, a5, a3
 ; RV32IA-NEXT:    xor a5, a4, a5
 ; RV32IA-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB54_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a2
+; RV32IA-NEXT:    srl a0, a4, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i8_seq_cst:
@@ -4945,33 +4945,33 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lbu a0, 0(a0)
-; RV64I-NEXT:    andi s0, a1, 255
-; RV64I-NEXT:    addi s3, sp, 7
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lbu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    andi s1, a1, 255
+; RV64I-NEXT:    addi s2, sp, 7
 ; RV64I-NEXT:    j .LBB54_2
 ; RV64I-NEXT:  .LBB54_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV64I-NEXT:    sb a0, 7(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sb a3, 7(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_1
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lb a0, 7(sp)
-; RV64I-NEXT:    bnez a1, .LBB54_4
+; RV64I-NEXT:    lb a3, 7(sp)
+; RV64I-NEXT:    bnez a0, .LBB54_4
 ; RV64I-NEXT:  .LBB54_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    andi a1, a0, 255
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB54_1
+; RV64I-NEXT:    andi a0, a3, 255
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB54_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB54_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB54_1
 ; RV64I-NEXT:  .LBB54_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -4982,27 +4982,27 @@ define i8 @atomicrmw_umin_i8_seq_cst(i8 *%a, i8 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i8_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
 ; RV64IA-NEXT:    addi a3, zero, 255
-; RV64IA-NEXT:    sllw a6, a3, a2
+; RV64IA-NEXT:    sllw a3, a3, a0
 ; RV64IA-NEXT:    andi a1, a1, 255
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB54_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a3, a4, a6
+; RV64IA-NEXT:    lr.w.aqrl a4, (a6)
+; RV64IA-NEXT:    and a2, a4, a3
 ; RV64IA-NEXT:    mv a5, a4
-; RV64IA-NEXT:    bgeu a1, a3, .LBB54_3
+; RV64IA-NEXT:    bgeu a1, a2, .LBB54_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB54_1 Depth=1
 ; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
+; RV64IA-NEXT:    and a5, a5, a3
 ; RV64IA-NEXT:    xor a5, a4, a5
 ; RV64IA-NEXT:  .LBB54_3: # in Loop: Header=BB54_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB54_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a2
+; RV64IA-NEXT:    srlw a0, a4, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i8* %a, i8 %b seq_cst
   ret i8 %1
@@ -5021,24 +5021,24 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB55_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB55_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_monotonic:
@@ -5053,24 +5053,24 @@ define i16 @atomicrmw_xchg_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB55_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB55_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5089,24 +5089,24 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB56_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_acquire:
@@ -5121,24 +5121,24 @@ define i16 @atomicrmw_xchg_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB56_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB56_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acquire
   ret i16 %1
@@ -5157,24 +5157,24 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB57_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_release:
@@ -5189,24 +5189,24 @@ define i16 @atomicrmw_xchg_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB57_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB57_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b release
   ret i16 %1
@@ -5225,24 +5225,24 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB58_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_acq_rel:
@@ -5257,24 +5257,24 @@ define i16 @atomicrmw_xchg_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB58_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB58_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5293,24 +5293,24 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
 ; RV32IA-NEXT:    add a5, zero, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB59_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xchg_i16_seq_cst:
@@ -5325,24 +5325,24 @@ define i16 @atomicrmw_xchg_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xchg_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB59_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
 ; RV64IA-NEXT:    add a5, zero, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB59_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xchg i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -5361,24 +5361,24 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB60_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_monotonic:
@@ -5393,24 +5393,24 @@ define i16 @atomicrmw_add_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB60_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB60_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5429,24 +5429,24 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB61_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_acquire:
@@ -5461,24 +5461,24 @@ define i16 @atomicrmw_add_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB61_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB61_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acquire
   ret i16 %1
@@ -5497,24 +5497,24 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB62_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_release:
@@ -5529,24 +5529,24 @@ define i16 @atomicrmw_add_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB62_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB62_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b release
   ret i16 %1
@@ -5565,24 +5565,24 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB63_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_acq_rel:
@@ -5597,24 +5597,24 @@ define i16 @atomicrmw_add_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB63_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB63_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5633,24 +5633,24 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    add a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    add a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB64_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_add_i16_seq_cst:
@@ -5665,24 +5665,24 @@ define i16 @atomicrmw_add_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_add_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB64_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    add a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    add a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB64_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw add i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -5701,24 +5701,24 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB65_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_monotonic:
@@ -5733,24 +5733,24 @@ define i16 @atomicrmw_sub_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB65_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB65_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b monotonic
   ret i16 %1
@@ -5769,24 +5769,24 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB66_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_acquire:
@@ -5801,24 +5801,24 @@ define i16 @atomicrmw_sub_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB66_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB66_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acquire
   ret i16 %1
@@ -5837,24 +5837,24 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB67_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_release:
@@ -5869,24 +5869,24 @@ define i16 @atomicrmw_sub_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB67_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB67_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b release
   ret i16 %1
@@ -5905,24 +5905,24 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB68_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_acq_rel:
@@ -5937,24 +5937,24 @@ define i16 @atomicrmw_sub_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB68_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB68_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -5973,24 +5973,24 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    sub a5, a4, a1
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    sub a5, a3, a1
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB69_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_sub_i16_seq_cst:
@@ -6005,24 +6005,24 @@ define i16 @atomicrmw_sub_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_sub_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB69_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    sub a5, a4, a1
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    sub a5, a3, a1
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB69_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw sub i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6041,18 +6041,18 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_monotonic:
@@ -6067,18 +6067,18 @@ define i16 @atomicrmw_and_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6097,18 +6097,18 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_acquire:
@@ -6123,18 +6123,18 @@ define i16 @atomicrmw_and_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acquire
   ret i16 %1
@@ -6153,18 +6153,18 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_release:
@@ -6179,18 +6179,18 @@ define i16 @atomicrmw_and_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b release
   ret i16 %1
@@ -6209,18 +6209,18 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_acq_rel:
@@ -6235,18 +6235,18 @@ define i16 @atomicrmw_and_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6265,18 +6265,18 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    not a2, a2
-; RV32IA-NEXT:    or a1, a2, a1
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a3
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    not a4, a4
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    or a1, a4, a1
+; RV32IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_and_i16_seq_cst:
@@ -6291,18 +6291,18 @@ define i16 @atomicrmw_and_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_and_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    not a2, a2
-; RV64IA-NEXT:    or a1, a2, a1
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoand.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a3
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    not a4, a4
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    or a1, a4, a1
+; RV64IA-NEXT:    amoand.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw and i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6321,25 +6321,25 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB75_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_monotonic:
@@ -6354,25 +6354,25 @@ define i16 @atomicrmw_nand_i16_monotonic(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB75_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB75_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6391,25 +6391,25 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB76_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_acquire:
@@ -6424,25 +6424,25 @@ define i16 @atomicrmw_nand_i16_acquire(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB76_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB76_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acquire
   ret i16 %1
@@ -6461,25 +6461,25 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB77_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_release:
@@ -6494,25 +6494,25 @@ define i16 @atomicrmw_nand_i16_release(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB77_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB77_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b release
   ret i16 %1
@@ -6531,25 +6531,25 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aq a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB78_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_acq_rel:
@@ -6564,25 +6564,25 @@ define i16 @atomicrmw_nand_i16_acq_rel(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB78_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aq a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB78_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6601,25 +6601,25 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a2, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB79_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a5, a4, a1
+; RV32IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV32IA-NEXT:    and a5, a3, a1
 ; RV32IA-NEXT:    not a5, a5
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    and a5, a5, a2
-; RV32IA-NEXT:    xor a5, a4, a5
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV32IA-NEXT:    bnez a5, .LBB79_1
 ; RV32IA-NEXT:  # %bb.2:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_nand_i16_seq_cst:
@@ -6634,25 +6634,25 @@ define i16 @atomicrmw_nand_i16_seq_cst(i16* %a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_nand_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a2, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB79_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a5, a4, a1
+; RV64IA-NEXT:    lr.w.aqrl a3, (a2)
+; RV64IA-NEXT:    and a5, a3, a1
 ; RV64IA-NEXT:    not a5, a5
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    and a5, a5, a2
-; RV64IA-NEXT:    xor a5, a4, a5
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a2)
 ; RV64IA-NEXT:    bnez a5, .LBB79_1
 ; RV64IA-NEXT:  # %bb.2:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw nand i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6671,15 +6671,15 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_monotonic:
@@ -6694,15 +6694,15 @@ define i16 @atomicrmw_or_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6721,15 +6721,15 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_acquire:
@@ -6744,15 +6744,15 @@ define i16 @atomicrmw_or_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acquire
   ret i16 %1
@@ -6771,15 +6771,15 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_release:
@@ -6794,15 +6794,15 @@ define i16 @atomicrmw_or_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b release
   ret i16 %1
@@ -6821,15 +6821,15 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_acq_rel:
@@ -6844,15 +6844,15 @@ define i16 @atomicrmw_or_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -6871,15 +6871,15 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_or_i16_seq_cst:
@@ -6894,15 +6894,15 @@ define i16 @atomicrmw_or_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_or_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw or i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -6921,15 +6921,15 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_monotonic:
@@ -6944,15 +6944,15 @@ define i16 @atomicrmw_xor_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b monotonic
   ret i16 %1
@@ -6971,15 +6971,15 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_acquire:
@@ -6994,15 +6994,15 @@ define i16 @atomicrmw_xor_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aq a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aq a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acquire
   ret i16 %1
@@ -7021,15 +7021,15 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_release:
@@ -7044,15 +7044,15 @@ define i16 @atomicrmw_xor_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.rl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.rl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b release
   ret i16 %1
@@ -7071,15 +7071,15 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_acq_rel:
@@ -7094,15 +7094,15 @@ define i16 @atomicrmw_xor_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -7121,15 +7121,15 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
-; RV32IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV32IA-NEXT:    srl a0, a0, a2
+; RV32IA-NEXT:    andi a2, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV32IA-NEXT:    srl a0, a1, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_xor_i16_seq_cst:
@@ -7144,15 +7144,15 @@ define i16 @atomicrmw_xor_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_xor_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
-; RV64IA-NEXT:    amoxor.w.aqrl a0, a1, (a0)
-; RV64IA-NEXT:    srlw a0, a0, a2
+; RV64IA-NEXT:    andi a2, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    amoxor.w.aqrl a1, a1, (a2)
+; RV64IA-NEXT:    srlw a0, a1, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw xor i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -7169,33 +7169,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB90_2
 ; RV32I-NEXT:  .LBB90_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB90_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB90_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB90_4
 ; RV32I-NEXT:  .LBB90_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB90_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB90_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB90_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB90_1
 ; RV32I-NEXT:  .LBB90_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7206,33 +7206,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB90_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB90_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB90_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB90_3: # in Loop: Header=BB90_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB90_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB90_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_monotonic:
@@ -7245,33 +7245,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB90_2
 ; RV64I-NEXT:  .LBB90_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB90_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB90_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB90_4
 ; RV64I-NEXT:  .LBB90_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB90_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB90_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB90_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB90_1
 ; RV64I-NEXT:  .LBB90_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7282,33 +7282,33 @@ define i16 @atomicrmw_max_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB90_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB90_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB90_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB90_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB90_3: # in Loop: Header=BB90_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB90_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB90_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b monotonic
   ret i16 %1
@@ -7325,33 +7325,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB91_2
 ; RV32I-NEXT:  .LBB91_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB91_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB91_4
 ; RV32I-NEXT:  .LBB91_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB91_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB91_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB91_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB91_1
 ; RV32I-NEXT:  .LBB91_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7362,33 +7362,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB91_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB91_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB91_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB91_3: # in Loop: Header=BB91_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB91_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB91_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_acquire:
@@ -7401,33 +7401,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB91_2
 ; RV64I-NEXT:  .LBB91_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB91_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB91_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB91_4
 ; RV64I-NEXT:  .LBB91_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB91_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB91_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB91_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB91_1
 ; RV64I-NEXT:  .LBB91_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7438,33 +7438,33 @@ define i16 @atomicrmw_max_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB91_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB91_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB91_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB91_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB91_3: # in Loop: Header=BB91_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB91_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB91_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acquire
   ret i16 %1
@@ -7481,33 +7481,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB92_2
 ; RV32I-NEXT:  .LBB92_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB92_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB92_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB92_4
 ; RV32I-NEXT:  .LBB92_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB92_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB92_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB92_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB92_1
 ; RV32I-NEXT:  .LBB92_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7518,33 +7518,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB92_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB92_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB92_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB92_3: # in Loop: Header=BB92_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB92_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB92_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_release:
@@ -7557,33 +7557,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB92_2
 ; RV64I-NEXT:  .LBB92_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB92_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB92_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB92_4
 ; RV64I-NEXT:  .LBB92_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB92_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB92_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB92_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB92_1
 ; RV64I-NEXT:  .LBB92_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7594,33 +7594,33 @@ define i16 @atomicrmw_max_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB92_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB92_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB92_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB92_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB92_3: # in Loop: Header=BB92_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB92_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB92_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b release
   ret i16 %1
@@ -7637,33 +7637,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB93_2
 ; RV32I-NEXT:  .LBB93_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB93_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB93_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB93_4
 ; RV32I-NEXT:  .LBB93_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB93_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB93_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB93_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB93_1
 ; RV32I-NEXT:  .LBB93_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7674,33 +7674,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB93_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB93_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB93_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB93_3: # in Loop: Header=BB93_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB93_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB93_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_acq_rel:
@@ -7713,33 +7713,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB93_2
 ; RV64I-NEXT:  .LBB93_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB93_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB93_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB93_4
 ; RV64I-NEXT:  .LBB93_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB93_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB93_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB93_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB93_1
 ; RV64I-NEXT:  .LBB93_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7750,33 +7750,33 @@ define i16 @atomicrmw_max_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB93_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB93_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB93_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB93_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB93_3: # in Loop: Header=BB93_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB93_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB93_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -7793,33 +7793,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB94_2
 ; RV32I-NEXT:  .LBB94_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB94_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB94_4
 ; RV32I-NEXT:  .LBB94_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    blt s0, a1, .LBB94_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    blt s0, a0, .LBB94_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB94_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB94_1
 ; RV32I-NEXT:  .LBB94_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7830,33 +7830,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a4, a1, .LBB94_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a3, a1, .LBB94_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB94_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB94_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_max_i16_seq_cst:
@@ -7869,33 +7869,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB94_2
 ; RV64I-NEXT:  .LBB94_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB94_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB94_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB94_4
 ; RV64I-NEXT:  .LBB94_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB94_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    blt s0, a0, .LBB94_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB94_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB94_1
 ; RV64I-NEXT:  .LBB94_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -7906,33 +7906,33 @@ define i16 @atomicrmw_max_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_max_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB94_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a4, a1, .LBB94_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a3, a1, .LBB94_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB94_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB94_3: # in Loop: Header=BB94_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB94_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB94_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw max i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -7949,33 +7949,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB95_2
 ; RV32I-NEXT:  .LBB95_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB95_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB95_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB95_4
 ; RV32I-NEXT:  .LBB95_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB95_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB95_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB95_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB95_1
 ; RV32I-NEXT:  .LBB95_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -7986,33 +7986,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB95_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB95_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB95_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB95_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_monotonic:
@@ -8025,33 +8025,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB95_2
 ; RV64I-NEXT:  .LBB95_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB95_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB95_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB95_4
 ; RV64I-NEXT:  .LBB95_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB95_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB95_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB95_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB95_1
 ; RV64I-NEXT:  .LBB95_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8062,33 +8062,33 @@ define i16 @atomicrmw_min_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB95_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB95_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB95_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB95_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB95_3: # in Loop: Header=BB95_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB95_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB95_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b monotonic
   ret i16 %1
@@ -8105,33 +8105,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB96_2
 ; RV32I-NEXT:  .LBB96_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB96_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB96_4
 ; RV32I-NEXT:  .LBB96_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB96_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB96_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB96_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB96_1
 ; RV32I-NEXT:  .LBB96_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8142,33 +8142,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB96_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB96_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
-; RV32IA-NEXT:    sc.w a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB96_1
+; RV32IA-NEXT:    sc.w a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB96_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_acquire:
@@ -8181,33 +8181,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB96_2
 ; RV64I-NEXT:  .LBB96_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB96_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB96_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB96_4
 ; RV64I-NEXT:  .LBB96_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB96_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB96_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB96_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB96_1
 ; RV64I-NEXT:  .LBB96_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8218,33 +8218,33 @@ define i16 @atomicrmw_min_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB96_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB96_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB96_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB96_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB96_3: # in Loop: Header=BB96_1 Depth=1
-; RV64IA-NEXT:    sc.w a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB96_1
+; RV64IA-NEXT:    sc.w a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB96_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acquire
   ret i16 %1
@@ -8261,33 +8261,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB97_2
 ; RV32I-NEXT:  .LBB97_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB97_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
+; RV32I-NEXT:    sh a1, 10(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s1
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB97_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB97_4
 ; RV32I-NEXT:  .LBB97_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB97_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB97_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB97_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB97_1
 ; RV32I-NEXT:  .LBB97_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8298,33 +8298,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB97_3
+; RV32IA-NEXT:    lr.w a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB97_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB97_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB97_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_release:
@@ -8337,33 +8337,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB97_2
 ; RV64I-NEXT:  .LBB97_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB97_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
+; RV64I-NEXT:    sh a1, 6(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s1
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB97_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB97_4
 ; RV64I-NEXT:  .LBB97_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB97_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB97_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB97_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB97_1
 ; RV64I-NEXT:  .LBB97_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8374,33 +8374,33 @@ define i16 @atomicrmw_min_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB97_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB97_3
+; RV64IA-NEXT:    lr.w a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB97_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB97_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB97_3: # in Loop: Header=BB97_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB97_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB97_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b release
   ret i16 %1
@@ -8417,33 +8417,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB98_2
 ; RV32I-NEXT:  .LBB98_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB98_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB98_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB98_4
 ; RV32I-NEXT:  .LBB98_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB98_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB98_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB98_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB98_1
 ; RV32I-NEXT:  .LBB98_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8454,33 +8454,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB98_3
+; RV32IA-NEXT:    lr.w.aq a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB98_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB98_1
+; RV32IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB98_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_acq_rel:
@@ -8493,33 +8493,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB98_2
 ; RV64I-NEXT:  .LBB98_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB98_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB98_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB98_4
 ; RV64I-NEXT:  .LBB98_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB98_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB98_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB98_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB98_1
 ; RV64I-NEXT:  .LBB98_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8530,33 +8530,33 @@ define i16 @atomicrmw_min_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB98_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB98_3
+; RV64IA-NEXT:    lr.w.aq a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB98_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB98_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB98_3: # in Loop: Header=BB98_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB98_1
+; RV64IA-NEXT:    sc.w.rl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB98_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -8573,33 +8573,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s3, 12(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    slli a1, a1, 16
-; RV32I-NEXT:    srai s0, a1, 16
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    slli a0, s2, 16
+; RV32I-NEXT:    srai s0, a0, 16
 ; RV32I-NEXT:    addi s3, sp, 10
 ; RV32I-NEXT:    j .LBB99_2
 ; RV32I-NEXT:  .LBB99_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_2 Depth=1
-; RV32I-NEXT:    sh a0, 10(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 10(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 10(sp)
-; RV32I-NEXT:    bnez a1, .LBB99_4
+; RV32I-NEXT:    lh a1, 10(sp)
+; RV32I-NEXT:    bnez a0, .LBB99_4
 ; RV32I-NEXT:  .LBB99_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    slli a1, a0, 16
-; RV32I-NEXT:    srai a1, a1, 16
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bge s0, a1, .LBB99_1
+; RV32I-NEXT:    slli a0, a1, 16
+; RV32I-NEXT:    srai a0, a0, 16
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bge s0, a0, .LBB99_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB99_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB99_1
 ; RV32I-NEXT:  .LBB99_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -8610,33 +8610,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    slli a2, a0, 3
-; RV32IA-NEXT:    andi a2, a2, 24
-; RV32IA-NEXT:    addi a3, zero, 16
-; RV32IA-NEXT:    sub a6, a3, a2
-; RV32IA-NEXT:    lui a4, 16
-; RV32IA-NEXT:    addi a4, a4, -1
-; RV32IA-NEXT:    sll a7, a4, a2
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a7, a3, a0
 ; RV32IA-NEXT:    slli a1, a1, 16
 ; RV32IA-NEXT:    srai a1, a1, 16
-; RV32IA-NEXT:    sll a1, a1, a2
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    sll a1, a1, a0
+; RV32IA-NEXT:    addi a4, zero, 16
+; RV32IA-NEXT:    sub a4, a4, a0
 ; RV32IA-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV32IA-NEXT:    and a4, a5, a7
-; RV32IA-NEXT:    mv a3, a5
-; RV32IA-NEXT:    sll a4, a4, a6
-; RV32IA-NEXT:    sra a4, a4, a6
-; RV32IA-NEXT:    bge a1, a4, .LBB99_3
+; RV32IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV32IA-NEXT:    and a3, a5, a7
+; RV32IA-NEXT:    mv a2, a5
+; RV32IA-NEXT:    sll a3, a3, a4
+; RV32IA-NEXT:    sra a3, a3, a4
+; RV32IA-NEXT:    bge a1, a3, .LBB99_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
-; RV32IA-NEXT:    xor a3, a5, a1
-; RV32IA-NEXT:    and a3, a3, a7
-; RV32IA-NEXT:    xor a3, a5, a3
+; RV32IA-NEXT:    xor a2, a5, a1
+; RV32IA-NEXT:    and a2, a2, a7
+; RV32IA-NEXT:    xor a2, a5, a2
 ; RV32IA-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV32IA-NEXT:    bnez a3, .LBB99_1
+; RV32IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV32IA-NEXT:    bnez a2, .LBB99_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a5, a2
+; RV32IA-NEXT:    srl a0, a5, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_min_i16_seq_cst:
@@ -8649,33 +8649,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s3, 8(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    slli a1, a1, 48
-; RV64I-NEXT:    srai s0, a1, 48
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    slli a0, s2, 48
+; RV64I-NEXT:    srai s0, a0, 48
 ; RV64I-NEXT:    addi s3, sp, 6
 ; RV64I-NEXT:    j .LBB99_2
 ; RV64I-NEXT:  .LBB99_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB99_2 Depth=1
-; RV64I-NEXT:    sh a0, 6(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 6(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 6(sp)
-; RV64I-NEXT:    bnez a1, .LBB99_4
+; RV64I-NEXT:    lh a1, 6(sp)
+; RV64I-NEXT:    bnez a0, .LBB99_4
 ; RV64I-NEXT:  .LBB99_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    slli a1, a0, 48
-; RV64I-NEXT:    srai a1, a1, 48
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB99_1
+; RV64I-NEXT:    slli a0, a1, 48
+; RV64I-NEXT:    srai a0, a0, 48
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bge s0, a0, .LBB99_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB99_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB99_1
 ; RV64I-NEXT:  .LBB99_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -8686,33 +8686,33 @@ define i16 @atomicrmw_min_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_min_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    slli a2, a0, 3
-; RV64IA-NEXT:    andi a2, a2, 24
-; RV64IA-NEXT:    addi a3, zero, 48
-; RV64IA-NEXT:    sub a6, a3, a2
-; RV64IA-NEXT:    lui a4, 16
-; RV64IA-NEXT:    addiw a4, a4, -1
-; RV64IA-NEXT:    sllw a7, a4, a2
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a7, a3, a0
 ; RV64IA-NEXT:    slli a1, a1, 48
 ; RV64IA-NEXT:    srai a1, a1, 48
-; RV64IA-NEXT:    sllw a1, a1, a2
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    sllw a1, a1, a0
+; RV64IA-NEXT:    addi a4, zero, 48
+; RV64IA-NEXT:    sub a4, a4, a0
 ; RV64IA-NEXT:  .LBB99_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a5, (a0)
-; RV64IA-NEXT:    and a4, a5, a7
-; RV64IA-NEXT:    mv a3, a5
-; RV64IA-NEXT:    sll a4, a4, a6
-; RV64IA-NEXT:    sra a4, a4, a6
-; RV64IA-NEXT:    bge a1, a4, .LBB99_3
+; RV64IA-NEXT:    lr.w.aqrl a5, (a6)
+; RV64IA-NEXT:    and a3, a5, a7
+; RV64IA-NEXT:    mv a2, a5
+; RV64IA-NEXT:    sll a3, a3, a4
+; RV64IA-NEXT:    sra a3, a3, a4
+; RV64IA-NEXT:    bge a1, a3, .LBB99_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB99_1 Depth=1
-; RV64IA-NEXT:    xor a3, a5, a1
-; RV64IA-NEXT:    and a3, a3, a7
-; RV64IA-NEXT:    xor a3, a5, a3
+; RV64IA-NEXT:    xor a2, a5, a1
+; RV64IA-NEXT:    and a2, a2, a7
+; RV64IA-NEXT:    xor a2, a5, a2
 ; RV64IA-NEXT:  .LBB99_3: # in Loop: Header=BB99_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a3, a3, (a0)
-; RV64IA-NEXT:    bnez a3, .LBB99_1
+; RV64IA-NEXT:    sc.w.aqrl a2, a2, (a6)
+; RV64IA-NEXT:    bnez a2, .LBB99_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a5, a2
+; RV64IA-NEXT:    srlw a0, a5, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw min i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -8730,33 +8730,33 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB100_2
 ; RV32I-NEXT:  .LBB100_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB100_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB100_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB100_4
 ; RV32I-NEXT:  .LBB100_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB100_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB100_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB100_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB100_1
 ; RV32I-NEXT:  .LBB100_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -8768,28 +8768,28 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB100_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB100_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_monotonic:
@@ -8803,33 +8803,33 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB100_2
 ; RV64I-NEXT:  .LBB100_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB100_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB100_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB100_4
 ; RV64I-NEXT:  .LBB100_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB100_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB100_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB100_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB100_1
 ; RV64I-NEXT:  .LBB100_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -8841,28 +8841,28 @@ define i16 @atomicrmw_umax_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB100_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB100_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB100_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB100_3: # in Loop: Header=BB100_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB100_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b monotonic
   ret i16 %1
@@ -8880,33 +8880,33 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB101_2
 ; RV32I-NEXT:  .LBB101_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB101_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB101_4
 ; RV32I-NEXT:  .LBB101_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB101_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB101_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB101_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB101_1
 ; RV32I-NEXT:  .LBB101_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -8918,28 +8918,28 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB101_3
-; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB101_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_acquire:
@@ -8953,33 +8953,33 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB101_2
 ; RV64I-NEXT:  .LBB101_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB101_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB101_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB101_4
 ; RV64I-NEXT:  .LBB101_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB101_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB101_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB101_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB101_1
 ; RV64I-NEXT:  .LBB101_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -8991,28 +8991,28 @@ define i16 @atomicrmw_umax_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB101_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB101_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB101_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB101_3: # in Loop: Header=BB101_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB101_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acquire
   ret i16 %1
@@ -9030,33 +9030,33 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB102_2
 ; RV32I-NEXT:  .LBB102_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB102_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB102_4
 ; RV32I-NEXT:  .LBB102_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB102_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB102_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB102_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB102_1
 ; RV32I-NEXT:  .LBB102_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9068,28 +9068,28 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB102_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB102_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB102_3: # in Loop: Header=BB102_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB102_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_release:
@@ -9103,33 +9103,33 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB102_2
 ; RV64I-NEXT:  .LBB102_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB102_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB102_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB102_4
 ; RV64I-NEXT:  .LBB102_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB102_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB102_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB102_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB102_1
 ; RV64I-NEXT:  .LBB102_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9141,28 +9141,28 @@ define i16 @atomicrmw_umax_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB102_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB102_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB102_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB102_3: # in Loop: Header=BB102_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB102_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b release
   ret i16 %1
@@ -9180,33 +9180,33 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB103_2
 ; RV32I-NEXT:  .LBB103_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB103_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB103_4
 ; RV32I-NEXT:  .LBB103_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB103_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB103_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB103_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB103_1
 ; RV32I-NEXT:  .LBB103_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9218,28 +9218,28 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB103_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB103_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB103_3: # in Loop: Header=BB103_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB103_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_acq_rel:
@@ -9253,33 +9253,33 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB103_2
 ; RV64I-NEXT:  .LBB103_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB103_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB103_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB103_4
 ; RV64I-NEXT:  .LBB103_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB103_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB103_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB103_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB103_1
 ; RV64I-NEXT:  .LBB103_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9291,28 +9291,28 @@ define i16 @atomicrmw_umax_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB103_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB103_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB103_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB103_3: # in Loop: Header=BB103_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB103_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -9330,33 +9330,33 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB104_2
 ; RV32I-NEXT:  .LBB104_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB104_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB104_4
 ; RV32I-NEXT:  .LBB104_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bltu s1, a1, .LBB104_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bltu s1, a0, .LBB104_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB104_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB104_1
 ; RV32I-NEXT:  .LBB104_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9368,28 +9368,28 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a2, a1, .LBB104_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB104_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB104_3: # in Loop: Header=BB104_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB104_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umax_i16_seq_cst:
@@ -9403,33 +9403,33 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB104_2
 ; RV64I-NEXT:  .LBB104_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB104_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB104_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB104_4
 ; RV64I-NEXT:  .LBB104_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s1, a1, .LBB104_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bltu s1, a0, .LBB104_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB104_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB104_1
 ; RV64I-NEXT:  .LBB104_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9441,28 +9441,28 @@ define i16 @atomicrmw_umax_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umax_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB104_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a2, a1, .LBB104_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB104_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB104_3: # in Loop: Header=BB104_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB104_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umax i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -9480,33 +9480,33 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB105_2
 ; RV32I-NEXT:  .LBB105_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB105_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB105_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB105_4
 ; RV32I-NEXT:  .LBB105_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB105_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB105_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB105_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB105_1
 ; RV32I-NEXT:  .LBB105_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9518,28 +9518,28 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB105_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB105_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB105_3: # in Loop: Header=BB105_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB105_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_monotonic:
@@ -9553,33 +9553,33 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB105_2
 ; RV64I-NEXT:  .LBB105_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB105_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB105_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB105_4
 ; RV64I-NEXT:  .LBB105_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB105_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB105_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB105_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB105_1
 ; RV64I-NEXT:  .LBB105_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9591,28 +9591,28 @@ define i16 @atomicrmw_umin_i16_monotonic(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_monotonic:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB105_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB105_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB105_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB105_3: # in Loop: Header=BB105_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB105_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b monotonic
   ret i16 %1
@@ -9630,33 +9630,33 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB106_2
 ; RV32I-NEXT:  .LBB106_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB106_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB106_4
 ; RV32I-NEXT:  .LBB106_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB106_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB106_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB106_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB106_1
 ; RV32I-NEXT:  .LBB106_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9668,28 +9668,28 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_acquire:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB106_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB106_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB106_3: # in Loop: Header=BB106_1 Depth=1
-; RV32IA-NEXT:    sc.w a5, a5, (a0)
+; RV32IA-NEXT:    sc.w a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB106_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_acquire:
@@ -9703,33 +9703,33 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB106_2
 ; RV64I-NEXT:  .LBB106_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB106_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB106_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB106_4
 ; RV64I-NEXT:  .LBB106_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB106_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB106_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB106_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB106_1
 ; RV64I-NEXT:  .LBB106_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9741,28 +9741,28 @@ define i16 @atomicrmw_umin_i16_acquire(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_acquire:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB106_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB106_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB106_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB106_3: # in Loop: Header=BB106_1 Depth=1
-; RV64IA-NEXT:    sc.w a5, a5, (a0)
+; RV64IA-NEXT:    sc.w a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB106_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acquire
   ret i16 %1
@@ -9780,33 +9780,33 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB107_2
 ; RV32I-NEXT:  .LBB107_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
+; RV32I-NEXT:    sh a1, 6(sp)
+; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a0, s4
 ; RV32I-NEXT:    mv a1, s3
-; RV32I-NEXT:    addi a3, zero, 3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB107_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB107_4
 ; RV32I-NEXT:  .LBB107_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB107_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB107_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB107_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB107_1
 ; RV32I-NEXT:  .LBB107_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9818,28 +9818,28 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_release:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB107_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB107_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB107_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB107_3: # in Loop: Header=BB107_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB107_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_release:
@@ -9853,33 +9853,33 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB107_2
 ; RV64I-NEXT:  .LBB107_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB107_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
+; RV64I-NEXT:    sh a1, 14(sp)
+; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a0, s4
 ; RV64I-NEXT:    mv a1, s3
-; RV64I-NEXT:    addi a3, zero, 3
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB107_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB107_4
 ; RV64I-NEXT:  .LBB107_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB107_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB107_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB107_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB107_1
 ; RV64I-NEXT:  .LBB107_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -9891,28 +9891,28 @@ define i16 @atomicrmw_umin_i16_release(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_release:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB107_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB107_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB107_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB107_3: # in Loop: Header=BB107_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB107_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b release
   ret i16 %1
@@ -9930,33 +9930,33 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB108_2
 ; RV32I-NEXT:  .LBB108_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB108_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB108_4
 ; RV32I-NEXT:  .LBB108_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB108_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB108_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB108_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB108_1
 ; RV32I-NEXT:  .LBB108_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -9968,28 +9968,28 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aq a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aq a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB108_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB108_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB108_3: # in Loop: Header=BB108_1 Depth=1
-; RV32IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB108_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_acq_rel:
@@ -10003,33 +10003,33 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB108_2
 ; RV64I-NEXT:  .LBB108_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB108_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB108_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB108_4
 ; RV64I-NEXT:  .LBB108_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB108_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB108_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB108_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB108_1
 ; RV64I-NEXT:  .LBB108_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -10041,28 +10041,28 @@ define i16 @atomicrmw_umin_i16_acq_rel(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_acq_rel:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB108_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aq a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aq a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB108_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB108_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB108_3: # in Loop: Header=BB108_1 Depth=1
-; RV64IA-NEXT:    sc.w.rl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.rl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB108_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b acq_rel
   ret i16 %1
@@ -10080,33 +10080,33 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV32I-NEXT:    sw s4, 8(sp)
 ; RV32I-NEXT:    mv s2, a1
 ; RV32I-NEXT:    mv s4, a0
-; RV32I-NEXT:    lhu a0, 0(a0)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi s0, a1, -1
+; RV32I-NEXT:    lhu a1, 0(a0)
+; RV32I-NEXT:    lui a0, 16
+; RV32I-NEXT:    addi s0, a0, -1
 ; RV32I-NEXT:    and s1, s2, s0
 ; RV32I-NEXT:    addi s3, sp, 6
 ; RV32I-NEXT:    j .LBB109_2
 ; RV32I-NEXT:  .LBB109_1: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_2 Depth=1
-; RV32I-NEXT:    sh a0, 6(sp)
-; RV32I-NEXT:    mv a0, s4
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sh a1, 6(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s4
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_2
-; RV32I-NEXT:    mv a1, a0
-; RV32I-NEXT:    lh a0, 6(sp)
-; RV32I-NEXT:    bnez a1, .LBB109_4
+; RV32I-NEXT:    lh a1, 6(sp)
+; RV32I-NEXT:    bnez a0, .LBB109_4
 ; RV32I-NEXT:  .LBB109_2: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    and a1, a0, s0
-; RV32I-NEXT:    mv a2, a0
-; RV32I-NEXT:    bgeu s1, a1, .LBB109_1
+; RV32I-NEXT:    and a0, a1, s0
+; RV32I-NEXT:    mv a2, a1
+; RV32I-NEXT:    bgeu s1, a0, .LBB109_1
 ; RV32I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB109_2 Depth=1
 ; RV32I-NEXT:    mv a2, s2
 ; RV32I-NEXT:    j .LBB109_1
 ; RV32I-NEXT:  .LBB109_4: # %atomicrmw.end
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    lw s4, 8(sp)
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
@@ -10118,28 +10118,28 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV32IA-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV32IA:       # %bb.0:
-; RV32IA-NEXT:    lui a2, 16
-; RV32IA-NEXT:    addi a2, a2, -1
-; RV32IA-NEXT:    and a1, a1, a2
-; RV32IA-NEXT:    slli a3, a0, 3
-; RV32IA-NEXT:    andi a3, a3, 24
-; RV32IA-NEXT:    sll a6, a2, a3
-; RV32IA-NEXT:    sll a1, a1, a3
-; RV32IA-NEXT:    andi a0, a0, -4
+; RV32IA-NEXT:    andi a6, a0, -4
+; RV32IA-NEXT:    slli a0, a0, 3
+; RV32IA-NEXT:    andi a0, a0, 24
+; RV32IA-NEXT:    lui a3, 16
+; RV32IA-NEXT:    addi a3, a3, -1
+; RV32IA-NEXT:    sll a4, a3, a0
+; RV32IA-NEXT:    and a1, a1, a3
+; RV32IA-NEXT:    sll a1, a1, a0
 ; RV32IA-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
-; RV32IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV32IA-NEXT:    and a2, a4, a6
-; RV32IA-NEXT:    mv a5, a4
+; RV32IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV32IA-NEXT:    and a2, a3, a4
+; RV32IA-NEXT:    mv a5, a3
 ; RV32IA-NEXT:    bgeu a1, a2, .LBB109_3
 ; RV32IA-NEXT:  # %bb.2: # in Loop: Header=BB109_1 Depth=1
-; RV32IA-NEXT:    xor a5, a4, a1
-; RV32IA-NEXT:    and a5, a5, a6
-; RV32IA-NEXT:    xor a5, a4, a5
+; RV32IA-NEXT:    xor a5, a3, a1
+; RV32IA-NEXT:    and a5, a5, a4
+; RV32IA-NEXT:    xor a5, a3, a5
 ; RV32IA-NEXT:  .LBB109_3: # in Loop: Header=BB109_1 Depth=1
-; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV32IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV32IA-NEXT:    bnez a5, .LBB109_1
 ; RV32IA-NEXT:  # %bb.4:
-; RV32IA-NEXT:    srl a0, a4, a3
+; RV32IA-NEXT:    srl a0, a3, a0
 ; RV32IA-NEXT:    ret
 ;
 ; RV64I-LABEL: atomicrmw_umin_i16_seq_cst:
@@ -10153,33 +10153,33 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ; RV64I-NEXT:    sd s4, 16(sp)
 ; RV64I-NEXT:    mv s2, a1
 ; RV64I-NEXT:    mv s4, a0
-; RV64I-NEXT:    lhu a0, 0(a0)
-; RV64I-NEXT:    lui a1, 16
-; RV64I-NEXT:    addiw s0, a1, -1
+; RV64I-NEXT:    lhu a1, 0(a0)
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw s0, a0, -1
 ; RV64I-NEXT:    and s1, s2, s0
 ; RV64I-NEXT:    addi s3, sp, 14
 ; RV64I-NEXT:    j .LBB109_2
 ; RV64I-NEXT:  .LBB109_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB109_2 Depth=1
-; RV64I-NEXT:    sh a0, 14(sp)
-; RV64I-NEXT:    mv a0, s4
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sh a1, 14(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s4
+; RV64I-NEXT:    mv a1, s3
 ; RV64I-NEXT:    call __atomic_compare_exchange_2
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lh a0, 14(sp)
-; RV64I-NEXT:    bnez a1, .LBB109_4
+; RV64I-NEXT:    lh a1, 14(sp)
+; RV64I-NEXT:    bnez a0, .LBB109_4
 ; RV64I-NEXT:  .LBB109_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    and a1, a0, s0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s1, a1, .LBB109_1
+; RV64I-NEXT:    and a0, a1, s0
+; RV64I-NEXT:    mv a2, a1
+; RV64I-NEXT:    bgeu s1, a0, .LBB109_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB109_2 Depth=1
 ; RV64I-NEXT:    mv a2, s2
 ; RV64I-NEXT:    j .LBB109_1
 ; RV64I-NEXT:  .LBB109_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ld s4, 16(sp)
 ; RV64I-NEXT:    ld s3, 24(sp)
 ; RV64I-NEXT:    ld s2, 32(sp)
@@ -10191,28 +10191,28 @@ define i16 @atomicrmw_umin_i16_seq_cst(i16 *%a, i16 %b) nounwind {
 ;
 ; RV64IA-LABEL: atomicrmw_umin_i16_seq_cst:
 ; RV64IA:       # %bb.0:
-; RV64IA-NEXT:    lui a2, 16
-; RV64IA-NEXT:    addiw a2, a2, -1
-; RV64IA-NEXT:    and a1, a1, a2
-; RV64IA-NEXT:    slli a3, a0, 3
-; RV64IA-NEXT:    andi a3, a3, 24
-; RV64IA-NEXT:    sllw a6, a2, a3
-; RV64IA-NEXT:    sllw a1, a1, a3
-; RV64IA-NEXT:    andi a0, a0, -4
+; RV64IA-NEXT:    andi a6, a0, -4
+; RV64IA-NEXT:    slli a0, a0, 3
+; RV64IA-NEXT:    andi a0, a0, 24
+; RV64IA-NEXT:    lui a3, 16
+; RV64IA-NEXT:    addiw a3, a3, -1
+; RV64IA-NEXT:    sllw a4, a3, a0
+; RV64IA-NEXT:    and a1, a1, a3
+; RV64IA-NEXT:    sllw a1, a1, a0
 ; RV64IA-NEXT:  .LBB109_1: # =>This Inner Loop Header: Depth=1
-; RV64IA-NEXT:    lr.w.aqrl a4, (a0)
-; RV64IA-NEXT:    and a2, a4, a6
-; RV64IA-NEXT:    mv a5, a4
+; RV64IA-NEXT:    lr.w.aqrl a3, (a6)
+; RV64IA-NEXT:    and a2, a3, a4
+; RV64IA-NEXT:    mv a5, a3
 ; RV64IA-NEXT:    bgeu a1, a2, .LBB109_3
 ; RV64IA-NEXT:  # %bb.2: # in Loop: Header=BB109_1 Depth=1
-; RV64IA-NEXT:    xor a5, a4, a1
-; RV64IA-NEXT:    and a5, a5, a6
-; RV64IA-NEXT:    xor a5, a4, a5
+; RV64IA-NEXT:    xor a5, a3, a1
+; RV64IA-NEXT:    and a5, a5, a4
+; RV64IA-NEXT:    xor a5, a3, a5
 ; RV64IA-NEXT:  .LBB109_3: # in Loop: Header=BB109_1 Depth=1
-; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a0)
+; RV64IA-NEXT:    sc.w.aqrl a5, a5, (a6)
 ; RV64IA-NEXT:    bnez a5, .LBB109_1
 ; RV64IA-NEXT:  # %bb.4:
-; RV64IA-NEXT:    srlw a0, a4, a3
+; RV64IA-NEXT:    srlw a0, a3, a0
 ; RV64IA-NEXT:    ret
   %1 = atomicrmw umin i16* %a, i16 %b seq_cst
   ret i16 %1
@@ -11496,30 +11496,31 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB145_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB145_3
 ; RV32I-NEXT:  .LBB145_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB145_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB145_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB145_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB145_1
 ; RV32I-NEXT:  .LBB145_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB145_1
 ; RV32I-NEXT:  .LBB145_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11540,33 +11541,33 @@ define i32 @atomicrmw_max_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB145_2
 ; RV64I-NEXT:  .LBB145_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB145_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB145_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB145_4
 ; RV64I-NEXT:  .LBB145_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB145_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB145_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB145_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB145_1
 ; RV64I-NEXT:  .LBB145_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11591,30 +11592,31 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB146_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB146_3
 ; RV32I-NEXT:  .LBB146_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB146_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB146_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB146_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB146_1
 ; RV32I-NEXT:  .LBB146_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB146_1
 ; RV32I-NEXT:  .LBB146_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11635,33 +11637,33 @@ define i32 @atomicrmw_max_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB146_2
 ; RV64I-NEXT:  .LBB146_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB146_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB146_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB146_4
 ; RV64I-NEXT:  .LBB146_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB146_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB146_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB146_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB146_1
 ; RV64I-NEXT:  .LBB146_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11686,30 +11688,31 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB147_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB147_3
 ; RV32I-NEXT:  .LBB147_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB147_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB147_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB147_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB147_1
 ; RV32I-NEXT:  .LBB147_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB147_1
 ; RV32I-NEXT:  .LBB147_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11730,33 +11733,33 @@ define i32 @atomicrmw_max_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB147_2
 ; RV64I-NEXT:  .LBB147_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB147_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB147_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB147_4
 ; RV64I-NEXT:  .LBB147_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB147_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB147_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB147_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB147_1
 ; RV64I-NEXT:  .LBB147_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11781,30 +11784,31 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB148_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB148_3
 ; RV32I-NEXT:  .LBB148_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB148_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB148_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB148_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB148_1
 ; RV32I-NEXT:  .LBB148_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB148_1
 ; RV32I-NEXT:  .LBB148_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11825,33 +11829,33 @@ define i32 @atomicrmw_max_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB148_2
 ; RV64I-NEXT:  .LBB148_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB148_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB148_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB148_4
 ; RV64I-NEXT:  .LBB148_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB148_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB148_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB148_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB148_1
 ; RV64I-NEXT:  .LBB148_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11876,30 +11880,31 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB149_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB149_3
 ; RV32I-NEXT:  .LBB149_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB149_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB149_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB149_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB149_1
 ; RV32I-NEXT:  .LBB149_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB149_1
 ; RV32I-NEXT:  .LBB149_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -11920,33 +11925,33 @@ define i32 @atomicrmw_max_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB149_2
 ; RV64I-NEXT:  .LBB149_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB149_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB149_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB149_4
 ; RV64I-NEXT:  .LBB149_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    blt s0, a1, .LBB149_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a0, .LBB149_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB149_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB149_1
 ; RV64I-NEXT:  .LBB149_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -11971,30 +11976,31 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB150_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB150_3
 ; RV32I-NEXT:  .LBB150_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB150_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB150_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB150_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB150_1
 ; RV32I-NEXT:  .LBB150_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB150_1
 ; RV32I-NEXT:  .LBB150_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12015,33 +12021,33 @@ define i32 @atomicrmw_min_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB150_2
 ; RV64I-NEXT:  .LBB150_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB150_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB150_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB150_4
 ; RV64I-NEXT:  .LBB150_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB150_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB150_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB150_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB150_1
 ; RV64I-NEXT:  .LBB150_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12066,30 +12072,31 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB151_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB151_3
 ; RV32I-NEXT:  .LBB151_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB151_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB151_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB151_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB151_1
 ; RV32I-NEXT:  .LBB151_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB151_1
 ; RV32I-NEXT:  .LBB151_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12110,33 +12117,33 @@ define i32 @atomicrmw_min_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB151_2
 ; RV64I-NEXT:  .LBB151_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB151_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB151_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB151_4
 ; RV64I-NEXT:  .LBB151_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB151_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB151_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB151_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB151_1
 ; RV64I-NEXT:  .LBB151_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12161,30 +12168,31 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB152_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB152_3
 ; RV32I-NEXT:  .LBB152_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB152_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB152_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB152_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB152_1
 ; RV32I-NEXT:  .LBB152_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB152_1
 ; RV32I-NEXT:  .LBB152_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12205,33 +12213,33 @@ define i32 @atomicrmw_min_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB152_2
 ; RV64I-NEXT:  .LBB152_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB152_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB152_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB152_4
 ; RV64I-NEXT:  .LBB152_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB152_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB152_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB152_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB152_1
 ; RV64I-NEXT:  .LBB152_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12256,30 +12264,31 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB153_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB153_3
 ; RV32I-NEXT:  .LBB153_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB153_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB153_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB153_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB153_1
 ; RV32I-NEXT:  .LBB153_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB153_1
 ; RV32I-NEXT:  .LBB153_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12300,33 +12309,33 @@ define i32 @atomicrmw_min_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB153_2
 ; RV64I-NEXT:  .LBB153_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB153_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB153_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB153_4
 ; RV64I-NEXT:  .LBB153_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB153_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB153_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB153_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB153_1
 ; RV64I-NEXT:  .LBB153_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12351,30 +12360,31 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    blt s0, a2, .LBB154_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    blt s1, a3, .LBB154_3
 ; RV32I-NEXT:  .LBB154_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB154_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB154_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bge s0, a2, .LBB154_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bge s1, a3, .LBB154_1
 ; RV32I-NEXT:  .LBB154_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB154_1
 ; RV32I-NEXT:  .LBB154_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12395,33 +12405,33 @@ define i32 @atomicrmw_min_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB154_2
 ; RV64I-NEXT:  .LBB154_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB154_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB154_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB154_4
 ; RV64I-NEXT:  .LBB154_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bge s0, a1, .LBB154_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a0, .LBB154_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB154_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB154_1
 ; RV64I-NEXT:  .LBB154_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12446,30 +12456,31 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB155_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB155_3
 ; RV32I-NEXT:  .LBB155_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB155_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB155_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB155_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB155_1
 ; RV32I-NEXT:  .LBB155_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB155_1
 ; RV32I-NEXT:  .LBB155_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12490,33 +12501,33 @@ define i32 @atomicrmw_umax_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB155_2
 ; RV64I-NEXT:  .LBB155_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB155_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB155_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB155_4
 ; RV64I-NEXT:  .LBB155_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB155_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB155_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB155_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB155_1
 ; RV64I-NEXT:  .LBB155_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12541,30 +12552,31 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB156_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB156_3
 ; RV32I-NEXT:  .LBB156_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB156_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB156_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB156_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB156_1
 ; RV32I-NEXT:  .LBB156_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB156_1
 ; RV32I-NEXT:  .LBB156_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12585,33 +12597,33 @@ define i32 @atomicrmw_umax_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB156_2
 ; RV64I-NEXT:  .LBB156_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB156_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB156_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB156_4
 ; RV64I-NEXT:  .LBB156_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB156_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB156_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB156_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB156_1
 ; RV64I-NEXT:  .LBB156_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12636,30 +12648,31 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB157_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB157_3
 ; RV32I-NEXT:  .LBB157_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB157_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB157_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB157_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB157_1
 ; RV32I-NEXT:  .LBB157_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB157_1
 ; RV32I-NEXT:  .LBB157_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12680,33 +12693,33 @@ define i32 @atomicrmw_umax_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB157_2
 ; RV64I-NEXT:  .LBB157_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB157_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB157_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB157_4
 ; RV64I-NEXT:  .LBB157_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB157_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB157_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB157_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB157_1
 ; RV64I-NEXT:  .LBB157_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12731,30 +12744,31 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB158_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB158_3
 ; RV32I-NEXT:  .LBB158_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB158_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB158_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB158_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB158_1
 ; RV32I-NEXT:  .LBB158_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB158_1
 ; RV32I-NEXT:  .LBB158_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12775,33 +12789,33 @@ define i32 @atomicrmw_umax_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB158_2
 ; RV64I-NEXT:  .LBB158_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB158_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB158_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB158_4
 ; RV64I-NEXT:  .LBB158_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB158_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB158_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB158_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB158_1
 ; RV64I-NEXT:  .LBB158_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12826,30 +12840,31 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB159_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB159_3
 ; RV32I-NEXT:  .LBB159_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB159_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB159_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB159_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB159_1
 ; RV32I-NEXT:  .LBB159_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB159_1
 ; RV32I-NEXT:  .LBB159_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12870,33 +12885,33 @@ define i32 @atomicrmw_umax_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB159_2
 ; RV64I-NEXT:  .LBB159_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB159_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB159_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB159_4
 ; RV64I-NEXT:  .LBB159_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bltu s0, a1, .LBB159_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a0, .LBB159_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB159_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB159_1
 ; RV64I-NEXT:  .LBB159_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -12921,30 +12936,31 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB160_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB160_3
 ; RV32I-NEXT:  .LBB160_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a3, 12(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a3, zero
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB160_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB160_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB160_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB160_1
 ; RV32I-NEXT:  .LBB160_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB160_1
 ; RV32I-NEXT:  .LBB160_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -12965,33 +12981,33 @@ define i32 @atomicrmw_umin_i32_monotonic(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB160_2
 ; RV64I-NEXT:  .LBB160_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB160_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB160_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB160_4
 ; RV64I-NEXT:  .LBB160_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB160_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB160_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB160_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB160_1
 ; RV64I-NEXT:  .LBB160_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13016,30 +13032,31 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB161_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB161_3
 ; RV32I-NEXT:  .LBB161_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 2
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB161_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB161_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB161_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB161_1
 ; RV32I-NEXT:  .LBB161_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB161_1
 ; RV32I-NEXT:  .LBB161_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13060,33 +13077,33 @@ define i32 @atomicrmw_umin_i32_acquire(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB161_2
 ; RV64I-NEXT:  .LBB161_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB161_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB161_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB161_4
 ; RV64I-NEXT:  .LBB161_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB161_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB161_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB161_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB161_1
 ; RV64I-NEXT:  .LBB161_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13111,30 +13128,31 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB162_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB162_3
 ; RV32I-NEXT:  .LBB162_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB162_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB162_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB162_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB162_1
 ; RV32I-NEXT:  .LBB162_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB162_1
 ; RV32I-NEXT:  .LBB162_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13155,33 +13173,33 @@ define i32 @atomicrmw_umin_i32_release(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB162_2
 ; RV64I-NEXT:  .LBB162_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB162_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB162_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB162_4
 ; RV64I-NEXT:  .LBB162_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB162_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB162_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB162_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB162_1
 ; RV64I-NEXT:  .LBB162_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13206,30 +13224,31 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB163_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB163_3
 ; RV32I-NEXT:  .LBB163_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 4
 ; RV32I-NEXT:    addi a4, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB163_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB163_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB163_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB163_1
 ; RV32I-NEXT:  .LBB163_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB163_1
 ; RV32I-NEXT:  .LBB163_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13250,33 +13269,33 @@ define i32 @atomicrmw_umin_i32_acq_rel(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB163_2
 ; RV64I-NEXT:  .LBB163_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB163_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB163_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB163_4
 ; RV64I-NEXT:  .LBB163_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB163_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB163_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB163_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB163_1
 ; RV64I-NEXT:  .LBB163_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -13301,30 +13320,31 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV32I-NEXT:    sw s0, 24(sp)
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
-; RV32I-NEXT:    mv s0, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a2, 0(a0)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a3, 0(a0)
+; RV32I-NEXT:    mv s1, a1
 ; RV32I-NEXT:    addi s2, sp, 12
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bltu s0, a2, .LBB164_3
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bltu s1, a3, .LBB164_3
 ; RV32I-NEXT:  .LBB164_1: # %atomicrmw.start
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s2
+; RV32I-NEXT:    sw a3, 12(sp)
 ; RV32I-NEXT:    addi a3, zero, 5
 ; RV32I-NEXT:    addi a4, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s2
 ; RV32I-NEXT:    call __atomic_compare_exchange_4
-; RV32I-NEXT:    lw a2, 12(sp)
+; RV32I-NEXT:    lw a3, 12(sp)
 ; RV32I-NEXT:    bnez a0, .LBB164_4
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV32I-NEXT:    # in Loop: Header=BB164_1 Depth=1
-; RV32I-NEXT:    sw a2, 12(sp)
-; RV32I-NEXT:    bgeu s0, a2, .LBB164_1
+; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bgeu s1, a3, .LBB164_1
 ; RV32I-NEXT:  .LBB164_3: # %atomicrmw.start
-; RV32I-NEXT:    mv a2, s0
+; RV32I-NEXT:    mv a2, s1
 ; RV32I-NEXT:    j .LBB164_1
 ; RV32I-NEXT:  .LBB164_4: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
 ; RV32I-NEXT:    lw s0, 24(sp)
@@ -13345,33 +13365,33 @@ define i32 @atomicrmw_umin_i32_seq_cst(i32 *%a, i32 %b) nounwind {
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
 ; RV64I-NEXT:    sd s3, 8(sp)
-; RV64I-NEXT:    mv s2, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    lwu a0, 0(a0)
-; RV64I-NEXT:    sext.w s0, a1
-; RV64I-NEXT:    addi s3, sp, 4
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    lwu a3, 0(a0)
+; RV64I-NEXT:    mv s3, a1
+; RV64I-NEXT:    sext.w s1, a1
+; RV64I-NEXT:    addi s2, sp, 4
 ; RV64I-NEXT:    j .LBB164_2
 ; RV64I-NEXT:  .LBB164_1: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB164_2 Depth=1
-; RV64I-NEXT:    sw a0, 4(sp)
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s3
+; RV64I-NEXT:    sw a3, 4(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_4
-; RV64I-NEXT:    mv a1, a0
-; RV64I-NEXT:    lw a0, 4(sp)
-; RV64I-NEXT:    bnez a1, .LBB164_4
+; RV64I-NEXT:    lw a3, 4(sp)
+; RV64I-NEXT:    bnez a0, .LBB164_4
 ; RV64I-NEXT:  .LBB164_2: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sext.w a1, a0
-; RV64I-NEXT:    mv a2, a0
-; RV64I-NEXT:    bgeu s0, a1, .LBB164_1
+; RV64I-NEXT:    sext.w a0, a3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a0, .LBB164_1
 ; RV64I-NEXT:  # %bb.3: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB164_2 Depth=1
-; RV64I-NEXT:    mv a2, s2
+; RV64I-NEXT:    mv a2, s3
 ; RV64I-NEXT:    j .LBB164_1
 ; RV64I-NEXT:  .LBB164_4: # %atomicrmw.end
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s3, 8(sp)
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
@@ -14802,41 +14822,43 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB200_3
+; RV32I-NEXT:    bne a5, s1, .LBB200_3
 ; RV32I-NEXT:    j .LBB200_4
 ; RV32I-NEXT:  .LBB200_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB200_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB200_4
+; RV32I-NEXT:    beq a5, s1, .LBB200_4
 ; RV32I-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB200_5
 ; RV32I-NEXT:  .LBB200_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB200_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB200_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB200_1
 ; RV32I-NEXT:  .LBB200_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -14853,41 +14875,43 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB200_3
+; RV32IA-NEXT:    bne a5, s1, .LBB200_3
 ; RV32IA-NEXT:    j .LBB200_4
 ; RV32IA-NEXT:  .LBB200_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB200_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB200_4
+; RV32IA-NEXT:    beq a5, s1, .LBB200_4
 ; RV32IA-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB200_5
 ; RV32IA-NEXT:  .LBB200_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB200_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB200_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB200_1
 ; RV32IA-NEXT:  .LBB200_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -14903,30 +14927,31 @@ define i64 @atomicrmw_max_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB200_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB200_3
 ; RV64I-NEXT:  .LBB200_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB200_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB200_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB200_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB200_1
 ; RV64I-NEXT:  .LBB200_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB200_1
 ; RV64I-NEXT:  .LBB200_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -14951,41 +14976,43 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB201_3
+; RV32I-NEXT:    bne a5, s1, .LBB201_3
 ; RV32I-NEXT:    j .LBB201_4
 ; RV32I-NEXT:  .LBB201_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB201_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB201_4
+; RV32I-NEXT:    beq a5, s1, .LBB201_4
 ; RV32I-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB201_5
 ; RV32I-NEXT:  .LBB201_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB201_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB201_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB201_1
 ; RV32I-NEXT:  .LBB201_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15002,41 +15029,43 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB201_3
+; RV32IA-NEXT:    bne a5, s1, .LBB201_3
 ; RV32IA-NEXT:    j .LBB201_4
 ; RV32IA-NEXT:  .LBB201_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB201_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB201_4
+; RV32IA-NEXT:    beq a5, s1, .LBB201_4
 ; RV32IA-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB201_5
 ; RV32IA-NEXT:  .LBB201_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB201_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB201_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB201_1
 ; RV32IA-NEXT:  .LBB201_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15052,30 +15081,31 @@ define i64 @atomicrmw_max_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB201_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB201_3
 ; RV64I-NEXT:  .LBB201_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB201_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB201_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB201_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB201_1
 ; RV64I-NEXT:  .LBB201_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB201_1
 ; RV64I-NEXT:  .LBB201_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15100,41 +15130,43 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB202_3
+; RV32I-NEXT:    bne a5, s1, .LBB202_3
 ; RV32I-NEXT:    j .LBB202_4
 ; RV32I-NEXT:  .LBB202_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB202_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB202_4
+; RV32I-NEXT:    beq a5, s1, .LBB202_4
 ; RV32I-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB202_5
 ; RV32I-NEXT:  .LBB202_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB202_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB202_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB202_1
 ; RV32I-NEXT:  .LBB202_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15151,41 +15183,43 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB202_3
+; RV32IA-NEXT:    bne a5, s1, .LBB202_3
 ; RV32IA-NEXT:    j .LBB202_4
 ; RV32IA-NEXT:  .LBB202_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB202_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB202_4
+; RV32IA-NEXT:    beq a5, s1, .LBB202_4
 ; RV32IA-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB202_5
 ; RV32IA-NEXT:  .LBB202_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB202_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB202_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB202_1
 ; RV32IA-NEXT:  .LBB202_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15201,30 +15235,31 @@ define i64 @atomicrmw_max_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB202_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB202_3
 ; RV64I-NEXT:  .LBB202_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB202_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB202_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB202_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB202_1
 ; RV64I-NEXT:  .LBB202_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB202_1
 ; RV64I-NEXT:  .LBB202_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15249,41 +15284,43 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB203_3
+; RV32I-NEXT:    bne a5, s1, .LBB203_3
 ; RV32I-NEXT:    j .LBB203_4
 ; RV32I-NEXT:  .LBB203_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB203_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB203_4
+; RV32I-NEXT:    beq a5, s1, .LBB203_4
 ; RV32I-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB203_5
 ; RV32I-NEXT:  .LBB203_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB203_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB203_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB203_1
 ; RV32I-NEXT:  .LBB203_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15300,41 +15337,43 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB203_3
+; RV32IA-NEXT:    bne a5, s1, .LBB203_3
 ; RV32IA-NEXT:    j .LBB203_4
 ; RV32IA-NEXT:  .LBB203_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB203_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB203_4
+; RV32IA-NEXT:    beq a5, s1, .LBB203_4
 ; RV32IA-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB203_5
 ; RV32IA-NEXT:  .LBB203_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB203_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB203_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB203_1
 ; RV32IA-NEXT:  .LBB203_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15350,30 +15389,31 @@ define i64 @atomicrmw_max_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB203_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB203_3
 ; RV64I-NEXT:  .LBB203_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB203_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB203_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB203_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB203_1
 ; RV64I-NEXT:  .LBB203_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB203_1
 ; RV64I-NEXT:  .LBB203_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15398,41 +15438,43 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB204_3
+; RV32I-NEXT:    bne a5, s1, .LBB204_3
 ; RV32I-NEXT:    j .LBB204_4
 ; RV32I-NEXT:  .LBB204_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB204_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB204_4
+; RV32I-NEXT:    beq a5, s1, .LBB204_4
 ; RV32I-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB204_5
 ; RV32I-NEXT:  .LBB204_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB204_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB204_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB204_1
 ; RV32I-NEXT:  .LBB204_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15449,41 +15491,43 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB204_3
+; RV32IA-NEXT:    bne a5, s1, .LBB204_3
 ; RV32IA-NEXT:    j .LBB204_4
 ; RV32IA-NEXT:  .LBB204_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB204_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB204_4
+; RV32IA-NEXT:    beq a5, s1, .LBB204_4
 ; RV32IA-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB204_5
 ; RV32IA-NEXT:  .LBB204_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB204_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB204_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB204_1
 ; RV32IA-NEXT:  .LBB204_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15499,30 +15543,31 @@ define i64 @atomicrmw_max_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB204_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB204_3
 ; RV64I-NEXT:  .LBB204_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB204_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB204_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB204_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB204_1
 ; RV64I-NEXT:  .LBB204_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB204_1
 ; RV64I-NEXT:  .LBB204_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15547,42 +15592,44 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB205_3
+; RV32I-NEXT:    bne a5, s1, .LBB205_3
 ; RV32I-NEXT:    j .LBB205_4
 ; RV32I-NEXT:  .LBB205_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB205_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB205_4
+; RV32I-NEXT:    beq a5, s1, .LBB205_4
 ; RV32I-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB205_5
 ; RV32I-NEXT:  .LBB205_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB205_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB205_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB205_1
 ; RV32I-NEXT:  .LBB205_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15599,42 +15646,44 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB205_3
+; RV32IA-NEXT:    bne a5, s1, .LBB205_3
 ; RV32IA-NEXT:    j .LBB205_4
 ; RV32IA-NEXT:  .LBB205_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB205_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB205_4
+; RV32IA-NEXT:    beq a5, s1, .LBB205_4
 ; RV32IA-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB205_5
 ; RV32IA-NEXT:  .LBB205_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB205_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB205_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB205_1
 ; RV32IA-NEXT:  .LBB205_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15650,30 +15699,31 @@ define i64 @atomicrmw_min_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB205_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB205_3
 ; RV64I-NEXT:  .LBB205_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB205_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB205_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB205_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB205_1
 ; RV64I-NEXT:  .LBB205_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB205_1
 ; RV64I-NEXT:  .LBB205_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15698,42 +15748,44 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB206_3
+; RV32I-NEXT:    bne a5, s1, .LBB206_3
 ; RV32I-NEXT:    j .LBB206_4
 ; RV32I-NEXT:  .LBB206_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB206_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB206_4
+; RV32I-NEXT:    beq a5, s1, .LBB206_4
 ; RV32I-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB206_5
 ; RV32I-NEXT:  .LBB206_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB206_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB206_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB206_1
 ; RV32I-NEXT:  .LBB206_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15750,42 +15802,44 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB206_3
+; RV32IA-NEXT:    bne a5, s1, .LBB206_3
 ; RV32IA-NEXT:    j .LBB206_4
 ; RV32IA-NEXT:  .LBB206_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB206_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB206_4
+; RV32IA-NEXT:    beq a5, s1, .LBB206_4
 ; RV32IA-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB206_5
 ; RV32IA-NEXT:  .LBB206_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB206_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB206_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB206_1
 ; RV32IA-NEXT:  .LBB206_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15801,30 +15855,31 @@ define i64 @atomicrmw_min_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB206_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB206_3
 ; RV64I-NEXT:  .LBB206_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB206_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB206_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB206_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB206_1
 ; RV64I-NEXT:  .LBB206_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB206_1
 ; RV64I-NEXT:  .LBB206_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -15849,42 +15904,44 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB207_3
+; RV32I-NEXT:    bne a5, s1, .LBB207_3
 ; RV32I-NEXT:    j .LBB207_4
 ; RV32I-NEXT:  .LBB207_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB207_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB207_4
+; RV32I-NEXT:    beq a5, s1, .LBB207_4
 ; RV32I-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB207_5
 ; RV32I-NEXT:  .LBB207_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB207_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB207_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB207_1
 ; RV32I-NEXT:  .LBB207_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -15901,42 +15958,44 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB207_3
+; RV32IA-NEXT:    bne a5, s1, .LBB207_3
 ; RV32IA-NEXT:    j .LBB207_4
 ; RV32IA-NEXT:  .LBB207_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB207_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB207_4
+; RV32IA-NEXT:    beq a5, s1, .LBB207_4
 ; RV32IA-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB207_5
 ; RV32IA-NEXT:  .LBB207_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB207_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB207_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB207_1
 ; RV32IA-NEXT:  .LBB207_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -15952,30 +16011,31 @@ define i64 @atomicrmw_min_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB207_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB207_3
 ; RV64I-NEXT:  .LBB207_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB207_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB207_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB207_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB207_1
 ; RV64I-NEXT:  .LBB207_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB207_1
 ; RV64I-NEXT:  .LBB207_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16000,42 +16060,44 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB208_3
+; RV32I-NEXT:    bne a5, s1, .LBB208_3
 ; RV32I-NEXT:    j .LBB208_4
 ; RV32I-NEXT:  .LBB208_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB208_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB208_4
+; RV32I-NEXT:    beq a5, s1, .LBB208_4
 ; RV32I-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB208_5
 ; RV32I-NEXT:  .LBB208_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB208_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB208_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB208_1
 ; RV32I-NEXT:  .LBB208_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16052,42 +16114,44 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB208_3
+; RV32IA-NEXT:    bne a5, s1, .LBB208_3
 ; RV32IA-NEXT:    j .LBB208_4
 ; RV32IA-NEXT:  .LBB208_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB208_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB208_4
+; RV32IA-NEXT:    beq a5, s1, .LBB208_4
 ; RV32IA-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB208_5
 ; RV32IA-NEXT:  .LBB208_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB208_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB208_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB208_1
 ; RV32IA-NEXT:  .LBB208_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16103,30 +16167,31 @@ define i64 @atomicrmw_min_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB208_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB208_3
 ; RV64I-NEXT:  .LBB208_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB208_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB208_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB208_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB208_1
 ; RV64I-NEXT:  .LBB208_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB208_1
 ; RV64I-NEXT:  .LBB208_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16151,42 +16216,44 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB209_3
+; RV32I-NEXT:    bne a5, s1, .LBB209_3
 ; RV32I-NEXT:    j .LBB209_4
 ; RV32I-NEXT:  .LBB209_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB209_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB209_4
+; RV32I-NEXT:    beq a5, s1, .LBB209_4
 ; RV32I-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV32I-NEXT:    slt a0, s0, a1
+; RV32I-NEXT:    slt a0, s1, a5
 ; RV32I-NEXT:    j .LBB209_5
 ; RV32I-NEXT:  .LBB209_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB209_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB209_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB209_1
 ; RV32I-NEXT:  .LBB209_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16203,42 +16270,44 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB209_3
+; RV32IA-NEXT:    bne a5, s1, .LBB209_3
 ; RV32IA-NEXT:    j .LBB209_4
 ; RV32IA-NEXT:  .LBB209_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB209_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB209_4
+; RV32IA-NEXT:    beq a5, s1, .LBB209_4
 ; RV32IA-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV32IA-NEXT:    slt a0, s0, a1
+; RV32IA-NEXT:    slt a0, s1, a5
 ; RV32IA-NEXT:    j .LBB209_5
 ; RV32IA-NEXT:  .LBB209_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB209_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB209_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB209_1
 ; RV32IA-NEXT:  .LBB209_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16254,30 +16323,31 @@ define i64 @atomicrmw_min_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    blt s0, a2, .LBB209_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    blt s1, a3, .LBB209_3
 ; RV64I-NEXT:  .LBB209_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB209_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB209_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bge s0, a2, .LBB209_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bge s1, a3, .LBB209_1
 ; RV64I-NEXT:  .LBB209_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB209_1
 ; RV64I-NEXT:  .LBB209_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16302,41 +16372,43 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB210_3
+; RV32I-NEXT:    bne a5, s1, .LBB210_3
 ; RV32I-NEXT:    j .LBB210_4
 ; RV32I-NEXT:  .LBB210_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB210_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB210_4
+; RV32I-NEXT:    beq a5, s1, .LBB210_4
 ; RV32I-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB210_5
 ; RV32I-NEXT:  .LBB210_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB210_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB210_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB210_1
 ; RV32I-NEXT:  .LBB210_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16353,41 +16425,43 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB210_3
+; RV32IA-NEXT:    bne a5, s1, .LBB210_3
 ; RV32IA-NEXT:    j .LBB210_4
 ; RV32IA-NEXT:  .LBB210_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB210_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB210_4
+; RV32IA-NEXT:    beq a5, s1, .LBB210_4
 ; RV32IA-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB210_5
 ; RV32IA-NEXT:  .LBB210_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB210_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB210_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB210_1
 ; RV32IA-NEXT:  .LBB210_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16403,30 +16477,31 @@ define i64 @atomicrmw_umax_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB210_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB210_3
 ; RV64I-NEXT:  .LBB210_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB210_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB210_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB210_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB210_1
 ; RV64I-NEXT:  .LBB210_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB210_1
 ; RV64I-NEXT:  .LBB210_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16451,41 +16526,43 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB211_3
+; RV32I-NEXT:    bne a5, s1, .LBB211_3
 ; RV32I-NEXT:    j .LBB211_4
 ; RV32I-NEXT:  .LBB211_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB211_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB211_4
+; RV32I-NEXT:    beq a5, s1, .LBB211_4
 ; RV32I-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB211_5
 ; RV32I-NEXT:  .LBB211_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB211_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB211_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB211_1
 ; RV32I-NEXT:  .LBB211_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16502,41 +16579,43 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB211_3
+; RV32IA-NEXT:    bne a5, s1, .LBB211_3
 ; RV32IA-NEXT:    j .LBB211_4
 ; RV32IA-NEXT:  .LBB211_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB211_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB211_4
+; RV32IA-NEXT:    beq a5, s1, .LBB211_4
 ; RV32IA-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB211_5
 ; RV32IA-NEXT:  .LBB211_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB211_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB211_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB211_1
 ; RV32IA-NEXT:  .LBB211_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16552,30 +16631,31 @@ define i64 @atomicrmw_umax_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB211_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB211_3
 ; RV64I-NEXT:  .LBB211_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB211_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB211_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB211_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB211_1
 ; RV64I-NEXT:  .LBB211_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB211_1
 ; RV64I-NEXT:  .LBB211_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16600,41 +16680,43 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB212_3
+; RV32I-NEXT:    bne a5, s1, .LBB212_3
 ; RV32I-NEXT:    j .LBB212_4
 ; RV32I-NEXT:  .LBB212_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB212_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB212_4
+; RV32I-NEXT:    beq a5, s1, .LBB212_4
 ; RV32I-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB212_5
 ; RV32I-NEXT:  .LBB212_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB212_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB212_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB212_1
 ; RV32I-NEXT:  .LBB212_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16651,41 +16733,43 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB212_3
+; RV32IA-NEXT:    bne a5, s1, .LBB212_3
 ; RV32IA-NEXT:    j .LBB212_4
 ; RV32IA-NEXT:  .LBB212_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB212_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB212_4
+; RV32IA-NEXT:    beq a5, s1, .LBB212_4
 ; RV32IA-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB212_5
 ; RV32IA-NEXT:  .LBB212_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB212_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB212_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB212_1
 ; RV32IA-NEXT:  .LBB212_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16701,30 +16785,31 @@ define i64 @atomicrmw_umax_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB212_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB212_3
 ; RV64I-NEXT:  .LBB212_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB212_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB212_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB212_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB212_1
 ; RV64I-NEXT:  .LBB212_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB212_1
 ; RV64I-NEXT:  .LBB212_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16749,41 +16834,43 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB213_3
+; RV32I-NEXT:    bne a5, s1, .LBB213_3
 ; RV32I-NEXT:    j .LBB213_4
 ; RV32I-NEXT:  .LBB213_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB213_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB213_4
+; RV32I-NEXT:    beq a5, s1, .LBB213_4
 ; RV32I-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB213_5
 ; RV32I-NEXT:  .LBB213_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB213_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB213_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB213_1
 ; RV32I-NEXT:  .LBB213_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16800,41 +16887,43 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB213_3
+; RV32IA-NEXT:    bne a5, s1, .LBB213_3
 ; RV32IA-NEXT:    j .LBB213_4
 ; RV32IA-NEXT:  .LBB213_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB213_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB213_4
+; RV32IA-NEXT:    beq a5, s1, .LBB213_4
 ; RV32IA-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB213_5
 ; RV32IA-NEXT:  .LBB213_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB213_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB213_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB213_1
 ; RV32IA-NEXT:  .LBB213_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16850,30 +16939,31 @@ define i64 @atomicrmw_umax_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB213_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB213_3
 ; RV64I-NEXT:  .LBB213_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB213_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB213_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB213_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB213_1
 ; RV64I-NEXT:  .LBB213_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB213_1
 ; RV64I-NEXT:  .LBB213_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -16898,41 +16988,43 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB214_3
+; RV32I-NEXT:    bne a5, s1, .LBB214_3
 ; RV32I-NEXT:    j .LBB214_4
 ; RV32I-NEXT:  .LBB214_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB214_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB214_4
+; RV32I-NEXT:    beq a5, s1, .LBB214_4
 ; RV32I-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB214_5
 ; RV32I-NEXT:  .LBB214_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB214_5: # %atomicrmw.start
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB214_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB214_1
 ; RV32I-NEXT:  .LBB214_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -16949,41 +17041,43 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB214_3
+; RV32IA-NEXT:    bne a5, s1, .LBB214_3
 ; RV32IA-NEXT:    j .LBB214_4
 ; RV32IA-NEXT:  .LBB214_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB214_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB214_4
+; RV32IA-NEXT:    beq a5, s1, .LBB214_4
 ; RV32IA-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB214_5
 ; RV32IA-NEXT:  .LBB214_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB214_5: # %atomicrmw.start
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB214_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB214_1
 ; RV32IA-NEXT:  .LBB214_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -16999,30 +17093,31 @@ define i64 @atomicrmw_umax_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB214_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB214_3
 ; RV64I-NEXT:  .LBB214_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB214_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB214_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB214_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB214_1
 ; RV64I-NEXT:  .LBB214_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB214_1
 ; RV64I-NEXT:  .LBB214_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17047,42 +17142,44 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB215_3
+; RV32I-NEXT:    bne a5, s1, .LBB215_3
 ; RV32I-NEXT:    j .LBB215_4
 ; RV32I-NEXT:  .LBB215_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
+; RV32I-NEXT:    mv a0, s0
 ; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a4, zero
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB215_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB215_4
+; RV32I-NEXT:    beq a5, s1, .LBB215_4
 ; RV32I-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB215_5
 ; RV32I-NEXT:  .LBB215_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB215_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB215_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB215_1
 ; RV32I-NEXT:  .LBB215_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17099,42 +17196,44 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB215_3
+; RV32IA-NEXT:    bne a5, s1, .LBB215_3
 ; RV32IA-NEXT:    j .LBB215_4
 ; RV32IA-NEXT:  .LBB215_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
+; RV32IA-NEXT:    mv a0, s0
 ; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a4, zero
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB215_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB215_4
+; RV32IA-NEXT:    beq a5, s1, .LBB215_4
 ; RV32IA-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB215_5
 ; RV32IA-NEXT:  .LBB215_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB215_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB215_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB215_1
 ; RV32IA-NEXT:  .LBB215_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17150,30 +17249,31 @@ define i64 @atomicrmw_umin_i64_monotonic(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB215_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB215_3
 ; RV64I-NEXT:  .LBB215_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
+; RV64I-NEXT:    sd a3, 8(sp)
+; RV64I-NEXT:    mv a0, s0
 ; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a3, zero
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB215_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB215_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB215_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB215_1
 ; RV64I-NEXT:  .LBB215_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB215_1
 ; RV64I-NEXT:  .LBB215_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17198,42 +17298,44 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB216_3
+; RV32I-NEXT:    bne a5, s1, .LBB216_3
 ; RV32I-NEXT:    j .LBB216_4
 ; RV32I-NEXT:  .LBB216_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 2
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB216_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB216_4
+; RV32I-NEXT:    beq a5, s1, .LBB216_4
 ; RV32I-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB216_5
 ; RV32I-NEXT:  .LBB216_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB216_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB216_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB216_1
 ; RV32I-NEXT:  .LBB216_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17250,42 +17352,44 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB216_3
+; RV32IA-NEXT:    bne a5, s1, .LBB216_3
 ; RV32IA-NEXT:    j .LBB216_4
 ; RV32IA-NEXT:  .LBB216_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 2
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB216_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB216_4
+; RV32IA-NEXT:    beq a5, s1, .LBB216_4
 ; RV32IA-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB216_5
 ; RV32IA-NEXT:  .LBB216_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB216_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB216_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB216_1
 ; RV32IA-NEXT:  .LBB216_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17301,30 +17405,31 @@ define i64 @atomicrmw_umin_i64_acquire(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB216_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB216_3
 ; RV64I-NEXT:  .LBB216_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 2
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB216_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB216_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB216_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB216_1
 ; RV64I-NEXT:  .LBB216_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB216_1
 ; RV64I-NEXT:  .LBB216_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17349,42 +17454,44 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB217_3
+; RV32I-NEXT:    bne a5, s1, .LBB217_3
 ; RV32I-NEXT:    j .LBB217_4
 ; RV32I-NEXT:  .LBB217_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 3
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    mv a5, zero
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB217_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB217_4
+; RV32I-NEXT:    beq a5, s1, .LBB217_4
 ; RV32I-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB217_5
 ; RV32I-NEXT:  .LBB217_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB217_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB217_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB217_1
 ; RV32I-NEXT:  .LBB217_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17401,42 +17508,44 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB217_3
+; RV32IA-NEXT:    bne a5, s1, .LBB217_3
 ; RV32IA-NEXT:    j .LBB217_4
 ; RV32IA-NEXT:  .LBB217_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 3
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    mv a5, zero
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB217_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB217_4
+; RV32IA-NEXT:    beq a5, s1, .LBB217_4
 ; RV32IA-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB217_5
 ; RV32IA-NEXT:  .LBB217_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB217_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB217_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB217_1
 ; RV32IA-NEXT:  .LBB217_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17452,30 +17561,31 @@ define i64 @atomicrmw_umin_i64_release(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB217_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB217_3
 ; RV64I-NEXT:  .LBB217_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 3
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB217_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB217_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB217_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB217_1
 ; RV64I-NEXT:  .LBB217_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB217_1
 ; RV64I-NEXT:  .LBB217_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17500,42 +17610,44 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB218_3
+; RV32I-NEXT:    bne a5, s1, .LBB218_3
 ; RV32I-NEXT:    j .LBB218_4
 ; RV32I-NEXT:  .LBB218_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 4
 ; RV32I-NEXT:    addi a5, zero, 2
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB218_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB218_4
+; RV32I-NEXT:    beq a5, s1, .LBB218_4
 ; RV32I-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB218_5
 ; RV32I-NEXT:  .LBB218_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB218_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB218_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB218_1
 ; RV32I-NEXT:  .LBB218_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17552,42 +17664,44 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB218_3
+; RV32IA-NEXT:    bne a5, s1, .LBB218_3
 ; RV32IA-NEXT:    j .LBB218_4
 ; RV32IA-NEXT:  .LBB218_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 4
 ; RV32IA-NEXT:    addi a5, zero, 2
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB218_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB218_4
+; RV32IA-NEXT:    beq a5, s1, .LBB218_4
 ; RV32IA-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB218_5
 ; RV32IA-NEXT:  .LBB218_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB218_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB218_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB218_1
 ; RV32IA-NEXT:  .LBB218_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17603,30 +17717,31 @@ define i64 @atomicrmw_umin_i64_acq_rel(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB218_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB218_3
 ; RV64I-NEXT:  .LBB218_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 2
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB218_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB218_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB218_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB218_1
 ; RV64I-NEXT:  .LBB218_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB218_1
 ; RV64I-NEXT:  .LBB218_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
@@ -17651,42 +17766,44 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32I-NEXT:    sw s1, 20(sp)
 ; RV32I-NEXT:    sw s2, 16(sp)
 ; RV32I-NEXT:    sw s3, 12(sp)
-; RV32I-NEXT:    mv s0, a2
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    lw a5, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    mv s1, a2
 ; RV32I-NEXT:    mv s2, a1
-; RV32I-NEXT:    mv s1, a0
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    lw a2, 0(a0)
 ; RV32I-NEXT:    mv s3, sp
-; RV32I-NEXT:    bne a1, s0, .LBB219_3
+; RV32I-NEXT:    bne a5, s1, .LBB219_3
 ; RV32I-NEXT:    j .LBB219_4
 ; RV32I-NEXT:  .LBB219_1: # %atomicrmw.start
-; RV32I-NEXT:    sw a1, 4(sp)
-; RV32I-NEXT:    mv a0, s1
-; RV32I-NEXT:    mv a1, s3
+; RV32I-NEXT:    sw a4, 0(sp)
+; RV32I-NEXT:    sw a5, 4(sp)
 ; RV32I-NEXT:    addi a4, zero, 5
 ; RV32I-NEXT:    addi a5, zero, 5
+; RV32I-NEXT:    mv a0, s0
+; RV32I-NEXT:    mv a1, s3
 ; RV32I-NEXT:    call __atomic_compare_exchange_8
-; RV32I-NEXT:    lw a1, 4(sp)
-; RV32I-NEXT:    lw a2, 0(sp)
+; RV32I-NEXT:    lw a5, 4(sp)
+; RV32I-NEXT:    lw a4, 0(sp)
 ; RV32I-NEXT:    bnez a0, .LBB219_7
 ; RV32I-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32I-NEXT:    beq a1, s0, .LBB219_4
+; RV32I-NEXT:    beq a5, s1, .LBB219_4
 ; RV32I-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV32I-NEXT:    sltu a0, s0, a1
+; RV32I-NEXT:    sltu a0, s1, a5
 ; RV32I-NEXT:    j .LBB219_5
 ; RV32I-NEXT:  .LBB219_4:
-; RV32I-NEXT:    sltu a0, s2, a2
+; RV32I-NEXT:    sltu a0, s2, a4
 ; RV32I-NEXT:  .LBB219_5: # %atomicrmw.start
 ; RV32I-NEXT:    xori a0, a0, 1
-; RV32I-NEXT:    sw a2, 0(sp)
-; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    mv a2, a4
+; RV32I-NEXT:    mv a3, a5
 ; RV32I-NEXT:    bnez a0, .LBB219_1
 ; RV32I-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32I-NEXT:    mv a2, s2
-; RV32I-NEXT:    mv a3, s0
+; RV32I-NEXT:    mv a3, s1
 ; RV32I-NEXT:    j .LBB219_1
 ; RV32I-NEXT:  .LBB219_7: # %atomicrmw.end
-; RV32I-NEXT:    mv a0, a2
+; RV32I-NEXT:    mv a0, a4
+; RV32I-NEXT:    mv a1, a5
 ; RV32I-NEXT:    lw s3, 12(sp)
 ; RV32I-NEXT:    lw s2, 16(sp)
 ; RV32I-NEXT:    lw s1, 20(sp)
@@ -17703,42 +17820,44 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV32IA-NEXT:    sw s1, 20(sp)
 ; RV32IA-NEXT:    sw s2, 16(sp)
 ; RV32IA-NEXT:    sw s3, 12(sp)
-; RV32IA-NEXT:    mv s0, a2
+; RV32IA-NEXT:    mv s0, a0
+; RV32IA-NEXT:    lw a5, 4(a0)
+; RV32IA-NEXT:    lw a4, 0(a0)
+; RV32IA-NEXT:    mv s1, a2
 ; RV32IA-NEXT:    mv s2, a1
-; RV32IA-NEXT:    mv s1, a0
-; RV32IA-NEXT:    lw a1, 4(a0)
-; RV32IA-NEXT:    lw a2, 0(a0)
 ; RV32IA-NEXT:    mv s3, sp
-; RV32IA-NEXT:    bne a1, s0, .LBB219_3
+; RV32IA-NEXT:    bne a5, s1, .LBB219_3
 ; RV32IA-NEXT:    j .LBB219_4
 ; RV32IA-NEXT:  .LBB219_1: # %atomicrmw.start
-; RV32IA-NEXT:    sw a1, 4(sp)
-; RV32IA-NEXT:    mv a0, s1
-; RV32IA-NEXT:    mv a1, s3
+; RV32IA-NEXT:    sw a4, 0(sp)
+; RV32IA-NEXT:    sw a5, 4(sp)
 ; RV32IA-NEXT:    addi a4, zero, 5
 ; RV32IA-NEXT:    addi a5, zero, 5
+; RV32IA-NEXT:    mv a0, s0
+; RV32IA-NEXT:    mv a1, s3
 ; RV32IA-NEXT:    call __atomic_compare_exchange_8
-; RV32IA-NEXT:    lw a1, 4(sp)
-; RV32IA-NEXT:    lw a2, 0(sp)
+; RV32IA-NEXT:    lw a5, 4(sp)
+; RV32IA-NEXT:    lw a4, 0(sp)
 ; RV32IA-NEXT:    bnez a0, .LBB219_7
 ; RV32IA-NEXT:  # %bb.2: # %atomicrmw.start
-; RV32IA-NEXT:    beq a1, s0, .LBB219_4
+; RV32IA-NEXT:    beq a5, s1, .LBB219_4
 ; RV32IA-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV32IA-NEXT:    sltu a0, s0, a1
+; RV32IA-NEXT:    sltu a0, s1, a5
 ; RV32IA-NEXT:    j .LBB219_5
 ; RV32IA-NEXT:  .LBB219_4:
-; RV32IA-NEXT:    sltu a0, s2, a2
+; RV32IA-NEXT:    sltu a0, s2, a4
 ; RV32IA-NEXT:  .LBB219_5: # %atomicrmw.start
 ; RV32IA-NEXT:    xori a0, a0, 1
-; RV32IA-NEXT:    sw a2, 0(sp)
-; RV32IA-NEXT:    mv a3, a1
+; RV32IA-NEXT:    mv a2, a4
+; RV32IA-NEXT:    mv a3, a5
 ; RV32IA-NEXT:    bnez a0, .LBB219_1
 ; RV32IA-NEXT:  # %bb.6: # %atomicrmw.start
 ; RV32IA-NEXT:    mv a2, s2
-; RV32IA-NEXT:    mv a3, s0
+; RV32IA-NEXT:    mv a3, s1
 ; RV32IA-NEXT:    j .LBB219_1
 ; RV32IA-NEXT:  .LBB219_7: # %atomicrmw.end
-; RV32IA-NEXT:    mv a0, a2
+; RV32IA-NEXT:    mv a0, a4
+; RV32IA-NEXT:    mv a1, a5
 ; RV32IA-NEXT:    lw s3, 12(sp)
 ; RV32IA-NEXT:    lw s2, 16(sp)
 ; RV32IA-NEXT:    lw s1, 20(sp)
@@ -17754,30 +17873,31 @@ define i64 @atomicrmw_umin_i64_seq_cst(i64 *%a, i64 %b) nounwind {
 ; RV64I-NEXT:    sd s0, 32(sp)
 ; RV64I-NEXT:    sd s1, 24(sp)
 ; RV64I-NEXT:    sd s2, 16(sp)
-; RV64I-NEXT:    mv s0, a1
-; RV64I-NEXT:    mv s1, a0
-; RV64I-NEXT:    ld a2, 0(a0)
+; RV64I-NEXT:    mv s0, a0
+; RV64I-NEXT:    ld a3, 0(a0)
+; RV64I-NEXT:    mv s1, a1
 ; RV64I-NEXT:    addi s2, sp, 8
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bltu s0, a2, .LBB219_3
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bltu s1, a3, .LBB219_3
 ; RV64I-NEXT:  .LBB219_1: # %atomicrmw.start
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    mv a0, s1
-; RV64I-NEXT:    mv a1, s2
+; RV64I-NEXT:    sd a3, 8(sp)
 ; RV64I-NEXT:    addi a3, zero, 5
 ; RV64I-NEXT:    addi a4, zero, 5
+; RV64I-NEXT:    mv a0, s0
+; RV64I-NEXT:    mv a1, s2
 ; RV64I-NEXT:    call __atomic_compare_exchange_8
-; RV64I-NEXT:    ld a2, 8(sp)
+; RV64I-NEXT:    ld a3, 8(sp)
 ; RV64I-NEXT:    bnez a0, .LBB219_4
 ; RV64I-NEXT:  # %bb.2: # %atomicrmw.start
 ; RV64I-NEXT:    # in Loop: Header=BB219_1 Depth=1
-; RV64I-NEXT:    sd a2, 8(sp)
-; RV64I-NEXT:    bgeu s0, a2, .LBB219_1
+; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bgeu s1, a3, .LBB219_1
 ; RV64I-NEXT:  .LBB219_3: # %atomicrmw.start
-; RV64I-NEXT:    mv a2, s0
+; RV64I-NEXT:    mv a2, s1
 ; RV64I-NEXT:    j .LBB219_1
 ; RV64I-NEXT:  .LBB219_4: # %atomicrmw.end
-; RV64I-NEXT:    mv a0, a2
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    ld s2, 16(sp)
 ; RV64I-NEXT:    ld s1, 24(sp)
 ; RV64I-NEXT:    ld s0, 32(sp)
diff --git a/llvm/test/CodeGen/RISCV/bare-select.ll b/llvm/test/CodeGen/RISCV/bare-select.ll
index 1b8f2f63e99d5..cf8fe96742bfb 100644
--- a/llvm/test/CodeGen/RISCV/bare-select.ll
+++ b/llvm/test/CodeGen/RISCV/bare-select.ll
@@ -5,12 +5,12 @@
 define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 ; RV32I-LABEL: bare_select:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB0_2
+; RV32I-NEXT:    andi a3, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a3, .LBB0_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB0_2:
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = select i1 %a, i32 %b, i32 %c
   ret i32 %1
@@ -19,12 +19,12 @@ define i32 @bare_select(i1 %a, i32 %b, i32 %c) nounwind {
 define float @bare_select_float(i1 %a, float %b, float %c) nounwind {
 ; RV32I-LABEL: bare_select_float:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB1_2
+; RV32I-NEXT:    andi a3, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a3, .LBB1_2
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:  .LBB1_2:
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = select i1 %a, float %b, float %c
   ret float %1
diff --git a/llvm/test/CodeGen/RISCV/blockaddress.ll b/llvm/test/CodeGen/RISCV/blockaddress.ll
index 5b7f818764282..d70f314804131 100644
--- a/llvm/test/CodeGen/RISCV/blockaddress.ll
+++ b/llvm/test/CodeGen/RISCV/blockaddress.ll
@@ -9,11 +9,11 @@ define void @test_blockaddress() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    lui a0, %hi(.Ltmp0)
-; RV32I-NEXT:    addi a0, a0, %lo(.Ltmp0)
-; RV32I-NEXT:    lui a1, %hi(addr)
-; RV32I-NEXT:    sw a0, %lo(addr)(a1)
-; RV32I-NEXT:    lw a0, %lo(addr)(a1)
+; RV32I-NEXT:    lui a0, %hi(addr)
+; RV32I-NEXT:    lui a1, %hi(.Ltmp0)
+; RV32I-NEXT:    addi a1, a1, %lo(.Ltmp0)
+; RV32I-NEXT:    sw a1, %lo(addr)(a0)
+; RV32I-NEXT:    lw a0, %lo(addr)(a0)
 ; RV32I-NEXT:    jr a0
 ; RV32I-NEXT:  .Ltmp0: # Block address taken
 ; RV32I-NEXT:  .LBB0_1: # %block
diff --git a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
index 0a29f24a27732..d896d9fe071c0 100644
--- a/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
+++ b/llvm/test/CodeGen/RISCV/bswap-ctlz-cttz-ctpop.ll
@@ -29,10 +29,10 @@ define i16 @test_bswap_i16(i16 %a) nounwind {
 define i32 @test_bswap_i32(i32 %a) nounwind {
 ; RV32I-LABEL: test_bswap_i32:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -256
-; RV32I-NEXT:    srli a2, a0, 8
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 8
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -256
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    srli a2, a0, 24
 ; RV32I-NEXT:    or a1, a1, a2
 ; RV32I-NEXT:    slli a2, a0, 8
@@ -49,9 +49,9 @@ define i32 @test_bswap_i32(i32 %a) nounwind {
 define i64 @test_bswap_i64(i64 %a) nounwind {
 ; RV32I-LABEL: test_bswap_i64:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a2, 16
-; RV32I-NEXT:    addi a3, a2, -256
 ; RV32I-NEXT:    srli a2, a1, 8
+; RV32I-NEXT:    lui a3, 16
+; RV32I-NEXT:    addi a3, a3, -256
 ; RV32I-NEXT:    and a2, a2, a3
 ; RV32I-NEXT:    srli a4, a1, 24
 ; RV32I-NEXT:    or a2, a2, a4
@@ -87,10 +87,10 @@ define i8 @test_cttz_i8(i8 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -131,10 +131,10 @@ define i16 @test_cttz_i16(i16 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -172,10 +172,10 @@ define i32 @test_cttz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -220,11 +220,11 @@ define i32 @test_ctlz_i32(i32 %a) nounwind {
 ; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    srli a1, a0, 16
 ; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
 ; RV32I-NEXT:    not a0, a0
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -269,9 +269,9 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    not a1, s4
 ; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi s5, a1, 1365
 ; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s5, a2, 1365
 ; RV32I-NEXT:    and a1, a1, s5
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
@@ -282,12 +282,12 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 4112
-; RV32I-NEXT:    addi s6, a1, 257
 ; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi s1, a1, -241
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    addi s6, a1, -241
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
@@ -302,8 +302,8 @@ define i64 @test_cttz_i64(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    bnez s4, .LBB7_2
 ; RV32I-NEXT:  # %bb.1:
@@ -336,10 +336,10 @@ define i8 @test_cttz_i8_zero_undef(i8 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -371,10 +371,10 @@ define i16 @test_cttz_i16_zero_undef(i16 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -406,10 +406,10 @@ define i32 @test_cttz_i32_zero_undef(i32 %a) nounwind {
 ; RV32I-NEXT:    addi a1, a0, -1
 ; RV32I-NEXT:    not a0, a0
 ; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
@@ -450,9 +450,9 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    addi a0, a0, -1
 ; RV32I-NEXT:    not a1, s4
 ; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi s5, a1, 1365
 ; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi s5, a2, 1365
 ; RV32I-NEXT:    and a1, a1, s5
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
@@ -463,12 +463,12 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    lui a1, 4112
-; RV32I-NEXT:    addi s6, a1, 257
 ; RV32I-NEXT:    lui a1, 61681
-; RV32I-NEXT:    addi s1, a1, -241
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    addi s6, a1, -241
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    lui a1, 4112
+; RV32I-NEXT:    addi s1, a1, 257
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    mv s2, a0
 ; RV32I-NEXT:    addi a0, s3, -1
@@ -483,8 +483,8 @@ define i64 @test_cttz_i64_zero_undef(i64 %a) nounwind {
 ; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    srli a1, a0, 4
 ; RV32I-NEXT:    add a0, a0, a1
-; RV32I-NEXT:    and a0, a0, s1
-; RV32I-NEXT:    mv a1, s6
+; RV32I-NEXT:    and a0, a0, s6
+; RV32I-NEXT:    mv a1, s1
 ; RV32I-NEXT:    call __mulsi3
 ; RV32I-NEXT:    bnez s4, .LBB11_2
 ; RV32I-NEXT:  # %bb.1:
@@ -514,10 +514,10 @@ define i32 @test_ctpop_i32(i32 %a) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
-; RV32I-NEXT:    lui a1, 349525
-; RV32I-NEXT:    addi a1, a1, 1365
-; RV32I-NEXT:    srli a2, a0, 1
-; RV32I-NEXT:    and a1, a2, a1
+; RV32I-NEXT:    srli a1, a0, 1
+; RV32I-NEXT:    lui a2, 349525
+; RV32I-NEXT:    addi a2, a2, 1365
+; RV32I-NEXT:    and a1, a1, a2
 ; RV32I-NEXT:    sub a0, a0, a1
 ; RV32I-NEXT:    lui a1, 209715
 ; RV32I-NEXT:    addi a1, a1, 819
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
index ac1f4dafeb403..56d3ff04d1635 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr32s.ll
@@ -23,8 +23,8 @@ define void @callee() nounwind {
 ; ILP32-LP64-LABEL: callee:
 ; ILP32-LP64:       # %bb.0:
 ; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    flw ft0, %lo(var)(a0)
+; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    flw ft1, 4(a1)
 ; ILP32-LP64-NEXT:    flw ft2, 8(a1)
 ; ILP32-LP64-NEXT:    flw ft3, 12(a1)
@@ -52,14 +52,14 @@ define void @callee() nounwind {
 ; ILP32-LP64-NEXT:    flw fs5, 100(a1)
 ; ILP32-LP64-NEXT:    flw fs6, 104(a1)
 ; ILP32-LP64-NEXT:    flw fs7, 108(a1)
-; ILP32-LP64-NEXT:    flw fs8, 112(a1)
-; ILP32-LP64-NEXT:    flw fs9, 116(a1)
-; ILP32-LP64-NEXT:    flw fs10, 120(a1)
-; ILP32-LP64-NEXT:    flw fs11, 124(a1)
-; ILP32-LP64-NEXT:    fsw fs11, 124(a1)
-; ILP32-LP64-NEXT:    fsw fs10, 120(a1)
-; ILP32-LP64-NEXT:    fsw fs9, 116(a1)
-; ILP32-LP64-NEXT:    fsw fs8, 112(a1)
+; ILP32-LP64-NEXT:    flw fs8, 124(a1)
+; ILP32-LP64-NEXT:    flw fs9, 120(a1)
+; ILP32-LP64-NEXT:    flw fs10, 116(a1)
+; ILP32-LP64-NEXT:    flw fs11, 112(a1)
+; ILP32-LP64-NEXT:    fsw fs8, 124(a1)
+; ILP32-LP64-NEXT:    fsw fs9, 120(a1)
+; ILP32-LP64-NEXT:    fsw fs10, 116(a1)
+; ILP32-LP64-NEXT:    fsw fs11, 112(a1)
 ; ILP32-LP64-NEXT:    fsw fs7, 108(a1)
 ; ILP32-LP64-NEXT:    fsw fs6, 104(a1)
 ; ILP32-LP64-NEXT:    fsw fs5, 100(a1)
@@ -106,6 +106,7 @@ define void @callee() nounwind {
 ; ILP32F-LP64F-NEXT:    fsw fs10, 4(sp)
 ; ILP32F-LP64F-NEXT:    fsw fs11, 0(sp)
 ; ILP32F-LP64F-NEXT:    lui a0, %hi(var)
+; ILP32F-LP64F-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32F-LP64F-NEXT:    addi a1, a0, %lo(var)
 ;
 ; ILP32D-LP64D-LABEL: callee:
@@ -124,6 +125,7 @@ define void @callee() nounwind {
 ; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
 ; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
 ; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
+; ILP32D-LP64D-NEXT:    flw ft0, %lo(var)(a0)
 ; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
   %val = load [32 x float], [32 x float]* @var
   store volatile [32 x float] %val, [32 x float]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
index cc51b10fee184..f95bc45736af0 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-fpr64s.ll
@@ -19,8 +19,8 @@ define void @callee() nounwind {
 ; ILP32-LP64-LABEL: callee:
 ; ILP32-LP64:       # %bb.0:
 ; ILP32-LP64-NEXT:    lui a0, %hi(var)
-; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    fld ft0, %lo(var)(a0)
+; ILP32-LP64-NEXT:    addi a1, a0, %lo(var)
 ; ILP32-LP64-NEXT:    fld ft1, 8(a1)
 ; ILP32-LP64-NEXT:    fld ft2, 16(a1)
 ; ILP32-LP64-NEXT:    fld ft3, 24(a1)
@@ -48,14 +48,14 @@ define void @callee() nounwind {
 ; ILP32-LP64-NEXT:    fld fs5, 200(a1)
 ; ILP32-LP64-NEXT:    fld fs6, 208(a1)
 ; ILP32-LP64-NEXT:    fld fs7, 216(a1)
-; ILP32-LP64-NEXT:    fld fs8, 224(a1)
-; ILP32-LP64-NEXT:    fld fs9, 232(a1)
-; ILP32-LP64-NEXT:    fld fs10, 240(a1)
-; ILP32-LP64-NEXT:    fld fs11, 248(a1)
-; ILP32-LP64-NEXT:    fsd fs11, 248(a1)
-; ILP32-LP64-NEXT:    fsd fs10, 240(a1)
-; ILP32-LP64-NEXT:    fsd fs9, 232(a1)
-; ILP32-LP64-NEXT:    fsd fs8, 224(a1)
+; ILP32-LP64-NEXT:    fld fs8, 248(a1)
+; ILP32-LP64-NEXT:    fld fs9, 240(a1)
+; ILP32-LP64-NEXT:    fld fs10, 232(a1)
+; ILP32-LP64-NEXT:    fld fs11, 224(a1)
+; ILP32-LP64-NEXT:    fsd fs8, 248(a1)
+; ILP32-LP64-NEXT:    fsd fs9, 240(a1)
+; ILP32-LP64-NEXT:    fsd fs10, 232(a1)
+; ILP32-LP64-NEXT:    fsd fs11, 224(a1)
 ; ILP32-LP64-NEXT:    fsd fs7, 216(a1)
 ; ILP32-LP64-NEXT:    fsd fs6, 208(a1)
 ; ILP32-LP64-NEXT:    fsd fs5, 200(a1)
@@ -102,6 +102,7 @@ define void @callee() nounwind {
 ; ILP32D-LP64D-NEXT:    fsd fs10, 8(sp)
 ; ILP32D-LP64D-NEXT:    fsd fs11, 0(sp)
 ; ILP32D-LP64D-NEXT:    lui a0, %hi(var)
+; ILP32D-LP64D-NEXT:    fld ft0, %lo(var)(a0)
 ; ILP32D-LP64D-NEXT:    addi a1, a0, %lo(var)
   %val = load [32 x double], [32 x double]* @var
   store volatile [32 x double] %val, [32 x double]* @var
diff --git a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
index ad8f5a361be79..3cfb58d1edcf2 100644
--- a/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
+++ b/llvm/test/CodeGen/RISCV/callee-saved-gprs.ll
@@ -41,7 +41,9 @@ define void @callee() nounwind {
 ; RV32I-NEXT:    sw s10, 36(sp)
 ; RV32I-NEXT:    sw s11, 32(sp)
 ; RV32I-NEXT:    lui a0, %hi(var)
-; RV32I-NEXT:    addi a1, a0, %lo(var)
+; RV32I-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV32I-WITH-FP-LABEL: callee:
 ; RV32I-WITH-FP:       # %bb.0:
@@ -61,7 +63,9 @@ define void @callee() nounwind {
 ; RV32I-WITH-FP-NEXT:    sw s11, 28(sp)
 ; RV32I-WITH-FP-NEXT:    addi s0, sp, 80
 ; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV32I-WITH-FP-NEXT:    addi a1, a0, %lo(var)
+; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
+; RV32I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV64I-LABEL: callee:
 ; RV64I:       # %bb.0:
@@ -79,7 +83,9 @@ define void @callee() nounwind {
 ; RV64I-NEXT:    sd s10, 56(sp)
 ; RV64I-NEXT:    sd s11, 48(sp)
 ; RV64I-NEXT:    lui a0, %hi(var)
-; RV64I-NEXT:    addi a1, a0, %lo(var)
+; RV64I-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-NEXT:    sd a1, 40(sp)
+; RV64I-NEXT:    addi a2, a0, %lo(var)
 ;
 ; RV64I-WITH-FP-LABEL: callee:
 ; RV64I-WITH-FP:       # %bb.0:
@@ -99,7 +105,9 @@ define void @callee() nounwind {
 ; RV64I-WITH-FP-NEXT:    sd s11, 56(sp)
 ; RV64I-WITH-FP-NEXT:    addi s0, sp, 160
 ; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
-; RV64I-WITH-FP-NEXT:    addi a1, a0, %lo(var)
+; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
+; RV64I-WITH-FP-NEXT:    addi a2, a0, %lo(var)
   %val = load [32 x i32], [32 x i32]* @var
   store volatile [32 x i32] %val, [32 x i32]* @var
   ret void
@@ -111,36 +119,41 @@ define void @callee() nounwind {
 define void @caller() nounwind {
 ; RV32I-LABEL: caller:
 ; RV32I:         lui a0, %hi(var)
-; RV32I-NEXT:    addi s1, a0, %lo(var)
+; RV32I-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-NEXT:    sw a1, 88(sp)
+; RV32I-NEXT:    addi s0, a0, %lo(var)
+
 ; RV32I:         sw a0, 8(sp)
-; RV32I-NEXT:    lw s2, 84(s1)
-; RV32I-NEXT:    lw s3, 88(s1)
-; RV32I-NEXT:    lw s4, 92(s1)
-; RV32I-NEXT:    lw s5, 96(s1)
-; RV32I-NEXT:    lw s6, 100(s1)
-; RV32I-NEXT:    lw s7, 104(s1)
-; RV32I-NEXT:    lw s8, 108(s1)
-; RV32I-NEXT:    lw s9, 112(s1)
-; RV32I-NEXT:    lw s10, 116(s1)
-; RV32I-NEXT:    lw s11, 120(s1)
-; RV32I-NEXT:    lw s0, 124(s1)
+; RV32I-NEXT:    lw s2, 84(s0)
+; RV32I-NEXT:    lw s3, 88(s0)
+; RV32I-NEXT:    lw s4, 92(s0)
+; RV32I-NEXT:    lw s5, 96(s0)
+; RV32I-NEXT:    lw s6, 100(s0)
+; RV32I-NEXT:    lw s7, 104(s0)
+; RV32I-NEXT:    lw s8, 108(s0)
+; RV32I-NEXT:    lw s9, 112(s0)
+; RV32I-NEXT:    lw s10, 116(s0)
+; RV32I-NEXT:    lw s11, 120(s0)
+; RV32I-NEXT:    lw s1, 124(s0)
 ; RV32I-NEXT:    call callee
-; RV32I-NEXT:    sw s0, 124(s1)
-; RV32I-NEXT:    sw s11, 120(s1)
-; RV32I-NEXT:    sw s10, 116(s1)
-; RV32I-NEXT:    sw s9, 112(s1)
-; RV32I-NEXT:    sw s8, 108(s1)
-; RV32I-NEXT:    sw s7, 104(s1)
-; RV32I-NEXT:    sw s6, 100(s1)
-; RV32I-NEXT:    sw s5, 96(s1)
-; RV32I-NEXT:    sw s4, 92(s1)
-; RV32I-NEXT:    sw s3, 88(s1)
-; RV32I-NEXT:    sw s2, 84(s1)
+; RV32I-NEXT:    sw s1, 124(s0)
+; RV32I-NEXT:    sw s11, 120(s0)
+; RV32I-NEXT:    sw s10, 116(s0)
+; RV32I-NEXT:    sw s9, 112(s0)
+; RV32I-NEXT:    sw s8, 108(s0)
+; RV32I-NEXT:    sw s7, 104(s0)
+; RV32I-NEXT:    sw s6, 100(s0)
+; RV32I-NEXT:    sw s5, 96(s0)
+; RV32I-NEXT:    sw s4, 92(s0)
+; RV32I-NEXT:    sw s3, 88(s0)
+; RV32I-NEXT:    sw s2, 84(s0)
 ; RV32I-NEXT:    lw a0, 8(sp)
 ;
 ; RV32I-WITH-FP-LABEL: caller:
 ; RV32I-WITH-FP:         addi s0, sp, 144
 ; RV32I-WITH-FP-NEXT:    lui a0, %hi(var)
+; RV32I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV32I-WITH-FP-NEXT:    sw a1, -56(s0)
 ; RV32I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
 ; RV32I-WITH-FP:         sw a0, -140(s0)
 ; RV32I-WITH-FP-NEXT:    lw s5, 88(s1)
@@ -168,36 +181,40 @@ define void @caller() nounwind {
 ;
 ; RV64I-LABEL: caller:
 ; RV64I:         lui a0, %hi(var)
-; RV64I-NEXT:    addi s1, a0, %lo(var)
+; RV64I-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-NEXT:    sd a1, 160(sp)
+; RV64I-NEXT:    addi s0, a0, %lo(var)
 ; RV64I:         sd a0, 0(sp)
-; RV64I-NEXT:    lw s2, 84(s1)
-; RV64I-NEXT:    lw s3, 88(s1)
-; RV64I-NEXT:    lw s4, 92(s1)
-; RV64I-NEXT:    lw s5, 96(s1)
-; RV64I-NEXT:    lw s6, 100(s1)
-; RV64I-NEXT:    lw s7, 104(s1)
-; RV64I-NEXT:    lw s8, 108(s1)
-; RV64I-NEXT:    lw s9, 112(s1)
-; RV64I-NEXT:    lw s10, 116(s1)
-; RV64I-NEXT:    lw s11, 120(s1)
-; RV64I-NEXT:    lw s0, 124(s1)
+; RV64I-NEXT:    lw s2, 84(s0)
+; RV64I-NEXT:    lw s3, 88(s0)
+; RV64I-NEXT:    lw s4, 92(s0)
+; RV64I-NEXT:    lw s5, 96(s0)
+; RV64I-NEXT:    lw s6, 100(s0)
+; RV64I-NEXT:    lw s7, 104(s0)
+; RV64I-NEXT:    lw s8, 108(s0)
+; RV64I-NEXT:    lw s9, 112(s0)
+; RV64I-NEXT:    lw s10, 116(s0)
+; RV64I-NEXT:    lw s11, 120(s0)
+; RV64I-NEXT:    lw s1, 124(s0)
 ; RV64I-NEXT:    call callee
-; RV64I-NEXT:    sw s0, 124(s1)
-; RV64I-NEXT:    sw s11, 120(s1)
-; RV64I-NEXT:    sw s10, 116(s1)
-; RV64I-NEXT:    sw s9, 112(s1)
-; RV64I-NEXT:    sw s8, 108(s1)
-; RV64I-NEXT:    sw s7, 104(s1)
-; RV64I-NEXT:    sw s6, 100(s1)
-; RV64I-NEXT:    sw s5, 96(s1)
-; RV64I-NEXT:    sw s4, 92(s1)
-; RV64I-NEXT:    sw s3, 88(s1)
-; RV64I-NEXT:    sw s2, 84(s1)
+; RV64I-NEXT:    sw s1, 124(s0)
+; RV64I-NEXT:    sw s11, 120(s0)
+; RV64I-NEXT:    sw s10, 116(s0)
+; RV64I-NEXT:    sw s9, 112(s0)
+; RV64I-NEXT:    sw s8, 108(s0)
+; RV64I-NEXT:    sw s7, 104(s0)
+; RV64I-NEXT:    sw s6, 100(s0)
+; RV64I-NEXT:    sw s5, 96(s0)
+; RV64I-NEXT:    sw s4, 92(s0)
+; RV64I-NEXT:    sw s3, 88(s0)
+; RV64I-NEXT:    sw s2, 84(s0)
 ; RV64I-NEXT:    ld a0, 0(sp)
 ;
 ; RV64I-WITH-FP-LABEL: caller:
 ; RV64I-WITH-FP:         addi s0, sp, 288
 ; RV64I-WITH-FP-NEXT:    lui a0, %hi(var)
+; RV64I-WITH-FP-NEXT:    lw a1, %lo(var)(a0)
+; RV64I-WITH-FP-NEXT:    sd a1, -112(s0)
 ; RV64I-WITH-FP-NEXT:    addi s1, a0, %lo(var)
 ; RV64I-WITH-FP:         sd a0, -280(s0)
 ; RV64I-WITH-FP-NEXT:    lw s5, 88(s1)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
index c93edcaec25a3..24788e110b461 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-common.ll
@@ -61,8 +61,8 @@ define i32 @caller_double_in_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    lui a2, 262144
+; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    call callee_double_in_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -75,8 +75,8 @@ define i32 @caller_double_in_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    lui a2, 262144
+; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    call callee_double_in_regs
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -94,14 +94,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-FPELIM-LABEL: callee_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 0(a2)
-; RV32I-FPELIM-NEXT:    add a0, a0, a7
-; RV32I-FPELIM-NEXT:    lw a1, 0(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 16(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    lw a2, 0(sp)
+; RV32I-FPELIM-NEXT:    lw a3, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a4, 16(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a7
+; RV32I-FPELIM-NEXT:    add a0, a0, a2
+; RV32I-FPELIM-NEXT:    add a0, a0, a3
+; RV32I-FPELIM-NEXT:    add a0, a0, a4
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    ret
 ;
@@ -112,14 +112,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
-; RV32I-WITHFP-NEXT:    add a0, a0, a7
-; RV32I-WITHFP-NEXT:    lw a1, 0(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 8(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 16(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
+; RV32I-WITHFP-NEXT:    lw a2, 0(s0)
+; RV32I-WITHFP-NEXT:    lw a3, 8(s0)
+; RV32I-WITHFP-NEXT:    lw a4, 16(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a7
+; RV32I-WITHFP-NEXT:    add a0, a0, a2
+; RV32I-WITHFP-NEXT:    add a0, a0, a3
+; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -169,8 +169,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    addi a2, sp, 32
@@ -180,6 +179,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
+; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -215,8 +215,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -328
 ; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1311
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    addi a2, s0, -32
@@ -226,6 +225,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
+; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -241,8 +241,8 @@ define void @caller_aligned_stack() nounwind {
 define double @callee_small_scalar_ret() nounwind {
 ; RV32I-FPELIM-LABEL: callee_small_scalar_ret:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    mv a0, zero
 ; RV32I-FPELIM-NEXT:    lui a1, 261888
+; RV32I-FPELIM-NEXT:    mv a0, zero
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_small_scalar_ret:
@@ -251,8 +251,8 @@ define double @callee_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    mv a0, zero
 ; RV32I-WITHFP-NEXT:    lui a1, 261888
+; RV32I-WITHFP-NEXT:    mv a0, zero
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
index 10b0d54627da8..e86a8c7406f7c 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32-ilp32f-ilp32d-common.ll
@@ -82,22 +82,22 @@ define i32 @caller_i64_in_regs() nounwind {
 define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i64 %g, i32 %h) nounwind {
 ; RV32I-FPELIM-LABEL: callee_many_scalars:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw t0, 0(sp)
-; RV32I-FPELIM-NEXT:    xor a4, a4, t0
-; RV32I-FPELIM-NEXT:    xor a3, a3, a7
-; RV32I-FPELIM-NEXT:    or a3, a3, a4
-; RV32I-FPELIM-NEXT:    lui a4, 16
-; RV32I-FPELIM-NEXT:    addi a4, a4, -1
-; RV32I-FPELIM-NEXT:    and a1, a1, a4
-; RV32I-FPELIM-NEXT:    andi a0, a0, 255
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    lw t0, 4(sp)
+; RV32I-FPELIM-NEXT:    lw t1, 0(sp)
+; RV32I-FPELIM-NEXT:    andi t2, a0, 255
+; RV32I-FPELIM-NEXT:    lui a0, 16
+; RV32I-FPELIM-NEXT:    addi a0, a0, -1
+; RV32I-FPELIM-NEXT:    and a0, a1, a0
+; RV32I-FPELIM-NEXT:    add a0, t2, a0
 ; RV32I-FPELIM-NEXT:    add a0, a0, a2
-; RV32I-FPELIM-NEXT:    seqz a1, a3
+; RV32I-FPELIM-NEXT:    xor a1, a4, t1
+; RV32I-FPELIM-NEXT:    xor a2, a3, a7
+; RV32I-FPELIM-NEXT:    or a1, a2, a1
+; RV32I-FPELIM-NEXT:    seqz a1, a1
 ; RV32I-FPELIM-NEXT:    add a0, a1, a0
 ; RV32I-FPELIM-NEXT:    add a0, a0, a5
 ; RV32I-FPELIM-NEXT:    add a0, a0, a6
-; RV32I-FPELIM-NEXT:    lw a1, 4(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    add a0, a0, t0
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_many_scalars:
@@ -106,22 +106,22 @@ define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i64 %d, i32 %e, i32 %f, i
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw t0, 0(s0)
-; RV32I-WITHFP-NEXT:    xor a4, a4, t0
-; RV32I-WITHFP-NEXT:    xor a3, a3, a7
-; RV32I-WITHFP-NEXT:    or a3, a3, a4
-; RV32I-WITHFP-NEXT:    lui a4, 16
-; RV32I-WITHFP-NEXT:    addi a4, a4, -1
-; RV32I-WITHFP-NEXT:    and a1, a1, a4
-; RV32I-WITHFP-NEXT:    andi a0, a0, 255
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    lw t0, 4(s0)
+; RV32I-WITHFP-NEXT:    lw t1, 0(s0)
+; RV32I-WITHFP-NEXT:    andi t2, a0, 255
+; RV32I-WITHFP-NEXT:    lui a0, 16
+; RV32I-WITHFP-NEXT:    addi a0, a0, -1
+; RV32I-WITHFP-NEXT:    and a0, a1, a0
+; RV32I-WITHFP-NEXT:    add a0, t2, a0
 ; RV32I-WITHFP-NEXT:    add a0, a0, a2
-; RV32I-WITHFP-NEXT:    seqz a1, a3
+; RV32I-WITHFP-NEXT:    xor a1, a4, t1
+; RV32I-WITHFP-NEXT:    xor a2, a3, a7
+; RV32I-WITHFP-NEXT:    or a1, a2, a1
+; RV32I-WITHFP-NEXT:    seqz a1, a1
 ; RV32I-WITHFP-NEXT:    add a0, a1, a0
 ; RV32I-WITHFP-NEXT:    add a0, a0, a5
 ; RV32I-WITHFP-NEXT:    add a0, a0, a6
-; RV32I-WITHFP-NEXT:    lw a1, 4(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    add a0, a0, t0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -146,15 +146,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 8
 ; RV32I-FPELIM-NEXT:    sw a0, 4(sp)
-; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 3
 ; RV32I-FPELIM-NEXT:    addi a3, zero, 4
-; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 7
+; RV32I-FPELIM-NEXT:    sw zero, 0(sp)
+; RV32I-FPELIM-NEXT:    mv a4, zero
 ; RV32I-FPELIM-NEXT:    call callee_many_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
@@ -168,15 +168,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 8
 ; RV32I-WITHFP-NEXT:    sw a0, 4(sp)
-; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 3
 ; RV32I-WITHFP-NEXT:    addi a3, zero, 4
-; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 7
+; RV32I-WITHFP-NEXT:    sw zero, 0(sp)
+; RV32I-WITHFP-NEXT:    mv a4, zero
 ; RV32I-WITHFP-NEXT:    call callee_many_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -192,20 +192,20 @@ define i32 @caller_many_scalars() nounwind {
 define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_scalars:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw a2, 12(a1)
-; RV32I-FPELIM-NEXT:    lw a3, 12(a0)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a3, 4(a1)
-; RV32I-FPELIM-NEXT:    lw a4, 4(a0)
-; RV32I-FPELIM-NEXT:    xor a3, a4, a3
+; RV32I-FPELIM-NEXT:    lw a6, 0(a1)
+; RV32I-FPELIM-NEXT:    lw a7, 0(a0)
+; RV32I-FPELIM-NEXT:    lw a4, 4(a1)
+; RV32I-FPELIM-NEXT:    lw a5, 12(a1)
+; RV32I-FPELIM-NEXT:    lw a2, 12(a0)
+; RV32I-FPELIM-NEXT:    lw a3, 4(a0)
+; RV32I-FPELIM-NEXT:    lw a1, 8(a1)
+; RV32I-FPELIM-NEXT:    lw a0, 8(a0)
+; RV32I-FPELIM-NEXT:    xor a2, a2, a5
+; RV32I-FPELIM-NEXT:    xor a3, a3, a4
 ; RV32I-FPELIM-NEXT:    or a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a3, 8(a1)
-; RV32I-FPELIM-NEXT:    lw a4, 8(a0)
-; RV32I-FPELIM-NEXT:    xor a3, a4, a3
-; RV32I-FPELIM-NEXT:    lw a1, 0(a1)
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32I-FPELIM-NEXT:    xor a0, a0, a1
-; RV32I-FPELIM-NEXT:    or a0, a0, a3
+; RV32I-FPELIM-NEXT:    xor a1, a7, a6
+; RV32I-FPELIM-NEXT:    or a0, a1, a0
 ; RV32I-FPELIM-NEXT:    or a0, a0, a2
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
 ; RV32I-FPELIM-NEXT:    ret
@@ -216,20 +216,20 @@ define i32 @callee_large_scalars(i128 %a, fp128 %b) nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw a2, 12(a1)
-; RV32I-WITHFP-NEXT:    lw a3, 12(a0)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a3, 4(a1)
-; RV32I-WITHFP-NEXT:    lw a4, 4(a0)
-; RV32I-WITHFP-NEXT:    xor a3, a4, a3
+; RV32I-WITHFP-NEXT:    lw a6, 0(a1)
+; RV32I-WITHFP-NEXT:    lw a7, 0(a0)
+; RV32I-WITHFP-NEXT:    lw a4, 4(a1)
+; RV32I-WITHFP-NEXT:    lw a5, 12(a1)
+; RV32I-WITHFP-NEXT:    lw a2, 12(a0)
+; RV32I-WITHFP-NEXT:    lw a3, 4(a0)
+; RV32I-WITHFP-NEXT:    lw a1, 8(a1)
+; RV32I-WITHFP-NEXT:    lw a0, 8(a0)
+; RV32I-WITHFP-NEXT:    xor a2, a2, a5
+; RV32I-WITHFP-NEXT:    xor a3, a3, a4
 ; RV32I-WITHFP-NEXT:    or a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a3, 8(a1)
-; RV32I-WITHFP-NEXT:    lw a4, 8(a0)
-; RV32I-WITHFP-NEXT:    xor a3, a4, a3
-; RV32I-WITHFP-NEXT:    lw a1, 0(a1)
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a1
-; RV32I-WITHFP-NEXT:    or a0, a0, a3
+; RV32I-WITHFP-NEXT:    xor a1, a7, a6
+; RV32I-WITHFP-NEXT:    or a0, a1, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a2
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
@@ -255,10 +255,10 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 36(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 32(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 28(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
+; RV32I-FPELIM-NEXT:    addi a2, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 24
 ; RV32I-FPELIM-NEXT:    mv a1, sp
+; RV32I-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars
 ; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 48
@@ -278,10 +278,10 @@ define i32 @caller_large_scalars() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi a2, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a1, s0, -48
+; RV32I-WITHFP-NEXT:    sw a2, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars
 ; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 44(sp)
@@ -299,20 +299,20 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-FPELIM-LABEL: callee_large_scalars_exhausted_regs:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 4(sp)
-; RV32I-FPELIM-NEXT:    lw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    lw a2, 12(a7)
-; RV32I-FPELIM-NEXT:    xor a1, a2, a1
-; RV32I-FPELIM-NEXT:    lw a2, 4(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 4(a7)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    or a1, a2, a1
-; RV32I-FPELIM-NEXT:    lw a2, 8(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 8(a7)
-; RV32I-FPELIM-NEXT:    xor a2, a3, a2
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32I-FPELIM-NEXT:    lw a3, 0(a7)
-; RV32I-FPELIM-NEXT:    xor a0, a3, a0
-; RV32I-FPELIM-NEXT:    or a0, a0, a2
+; RV32I-FPELIM-NEXT:    lw a6, 0(a0)
+; RV32I-FPELIM-NEXT:    lw t0, 0(a7)
+; RV32I-FPELIM-NEXT:    lw a3, 4(a0)
+; RV32I-FPELIM-NEXT:    lw a4, 12(a0)
+; RV32I-FPELIM-NEXT:    lw a5, 12(a7)
+; RV32I-FPELIM-NEXT:    lw a1, 4(a7)
+; RV32I-FPELIM-NEXT:    lw a0, 8(a0)
+; RV32I-FPELIM-NEXT:    lw a2, 8(a7)
+; RV32I-FPELIM-NEXT:    xor a4, a5, a4
+; RV32I-FPELIM-NEXT:    xor a1, a1, a3
+; RV32I-FPELIM-NEXT:    or a1, a1, a4
+; RV32I-FPELIM-NEXT:    xor a0, a2, a0
+; RV32I-FPELIM-NEXT:    xor a2, t0, a6
+; RV32I-FPELIM-NEXT:    or a0, a2, a0
 ; RV32I-FPELIM-NEXT:    or a0, a0, a1
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
 ; RV32I-FPELIM-NEXT:    ret
@@ -324,20 +324,20 @@ define i32 @callee_large_scalars_exhausted_regs(i32 %a, i32 %b, i32 %c, i32 %d,
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 4(s0)
-; RV32I-WITHFP-NEXT:    lw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    lw a2, 12(a7)
-; RV32I-WITHFP-NEXT:    xor a1, a2, a1
-; RV32I-WITHFP-NEXT:    lw a2, 4(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 4(a7)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    or a1, a2, a1
-; RV32I-WITHFP-NEXT:    lw a2, 8(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 8(a7)
-; RV32I-WITHFP-NEXT:    xor a2, a3, a2
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
-; RV32I-WITHFP-NEXT:    lw a3, 0(a7)
-; RV32I-WITHFP-NEXT:    xor a0, a3, a0
-; RV32I-WITHFP-NEXT:    or a0, a0, a2
+; RV32I-WITHFP-NEXT:    lw a6, 0(a0)
+; RV32I-WITHFP-NEXT:    lw t0, 0(a7)
+; RV32I-WITHFP-NEXT:    lw a3, 4(a0)
+; RV32I-WITHFP-NEXT:    lw a4, 12(a0)
+; RV32I-WITHFP-NEXT:    lw a5, 12(a7)
+; RV32I-WITHFP-NEXT:    lw a1, 4(a7)
+; RV32I-WITHFP-NEXT:    lw a0, 8(a0)
+; RV32I-WITHFP-NEXT:    lw a2, 8(a7)
+; RV32I-WITHFP-NEXT:    xor a4, a5, a4
+; RV32I-WITHFP-NEXT:    xor a1, a1, a3
+; RV32I-WITHFP-NEXT:    or a1, a1, a4
+; RV32I-WITHFP-NEXT:    xor a0, a2, a0
+; RV32I-WITHFP-NEXT:    xor a2, t0, a6
+; RV32I-WITHFP-NEXT:    or a0, a2, a0
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
@@ -367,8 +367,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    sw zero, 52(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 48(sp)
 ; RV32I-FPELIM-NEXT:    sw zero, 44(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 8
-; RV32I-FPELIM-NEXT:    sw a0, 40(sp)
+; RV32I-FPELIM-NEXT:    addi t0, zero, 8
 ; RV32I-FPELIM-NEXT:    addi a7, sp, 40
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 2
@@ -377,6 +376,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 5
 ; RV32I-FPELIM-NEXT:    addi a5, zero, 6
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 7
+; RV32I-FPELIM-NEXT:    sw t0, 40(sp)
 ; RV32I-FPELIM-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -400,8 +400,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    sw zero, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -16(s0)
 ; RV32I-WITHFP-NEXT:    sw zero, -20(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 8
-; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi t0, zero, 8
 ; RV32I-WITHFP-NEXT:    addi a7, s0, -24
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 2
@@ -410,6 +409,7 @@ define i32 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 5
 ; RV32I-WITHFP-NEXT:    addi a5, zero, 6
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 7
+; RV32I-WITHFP-NEXT:    sw t0, -24(s0)
 ; RV32I-WITHFP-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -524,9 +524,9 @@ define i32 @caller_small_coerced_struct() nounwind {
 define i32 @callee_large_struct(%struct.large* byval align 4 %a) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_struct:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    lw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
+; RV32I-FPELIM-NEXT:    lw a1, 0(a0)
+; RV32I-FPELIM-NEXT:    lw a0, 12(a0)
+; RV32I-FPELIM-NEXT:    add a0, a1, a0
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_struct:
@@ -535,9 +535,9 @@ define i32 @callee_large_struct(%struct.large* byval align 4 %a) nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    lw a0, 0(a0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
+; RV32I-WITHFP-NEXT:    lw a1, 0(a0)
+; RV32I-WITHFP-NEXT:    lw a0, 12(a0)
+; RV32I-WITHFP-NEXT:    add a0, a1, a0
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -557,16 +557,16 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 44(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
 ; RV32I-FPELIM-NEXT:    sw a0, 24(sp)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 2
+; RV32I-FPELIM-NEXT:    sw a1, 28(sp)
+; RV32I-FPELIM-NEXT:    addi a2, zero, 3
+; RV32I-FPELIM-NEXT:    sw a2, 32(sp)
+; RV32I-FPELIM-NEXT:    addi a3, zero, 4
+; RV32I-FPELIM-NEXT:    sw a3, 36(sp)
 ; RV32I-FPELIM-NEXT:    sw a0, 8(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 2
-; RV32I-FPELIM-NEXT:    sw a0, 28(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 12(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 3
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 16(sp)
-; RV32I-FPELIM-NEXT:    addi a0, zero, 4
-; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
-; RV32I-FPELIM-NEXT:    sw a0, 20(sp)
+; RV32I-FPELIM-NEXT:    sw a1, 12(sp)
+; RV32I-FPELIM-NEXT:    sw a2, 16(sp)
+; RV32I-FPELIM-NEXT:    sw a3, 20(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct
 ; RV32I-FPELIM-NEXT:    lw ra, 44(sp)
@@ -581,16 +581,16 @@ define i32 @caller_large_struct() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 48
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 2
+; RV32I-WITHFP-NEXT:    sw a1, -20(s0)
+; RV32I-WITHFP-NEXT:    addi a2, zero, 3
+; RV32I-WITHFP-NEXT:    sw a2, -16(s0)
+; RV32I-WITHFP-NEXT:    addi a3, zero, 4
+; RV32I-WITHFP-NEXT:    sw a3, -12(s0)
 ; RV32I-WITHFP-NEXT:    sw a0, -40(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 2
-; RV32I-WITHFP-NEXT:    sw a0, -20(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -36(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 3
-; RV32I-WITHFP-NEXT:    sw a0, -16(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
-; RV32I-WITHFP-NEXT:    addi a0, zero, 4
-; RV32I-WITHFP-NEXT:    sw a0, -12(s0)
-; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
+; RV32I-WITHFP-NEXT:    sw a1, -36(s0)
+; RV32I-WITHFP-NEXT:    sw a2, -32(s0)
+; RV32I-WITHFP-NEXT:    sw a3, -28(s0)
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -40
 ; RV32I-WITHFP-NEXT:    call callee_large_struct
 ; RV32I-WITHFP-NEXT:    lw s0, 40(sp)
@@ -619,14 +619,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-FPELIM-LABEL: callee_aligned_stack:
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    lw a0, 0(a2)
-; RV32I-FPELIM-NEXT:    add a0, a0, a7
-; RV32I-FPELIM-NEXT:    lw a1, 0(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
-; RV32I-FPELIM-NEXT:    lw a1, 16(sp)
-; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    lw a2, 0(sp)
+; RV32I-FPELIM-NEXT:    lw a3, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a4, 16(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a7
+; RV32I-FPELIM-NEXT:    add a0, a0, a2
+; RV32I-FPELIM-NEXT:    add a0, a0, a3
+; RV32I-FPELIM-NEXT:    add a0, a0, a4
 ; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    ret
 ;
@@ -637,14 +637,14 @@ define i32 @callee_aligned_stack(i32 %a, i32 %b, fp128 %c, i32 %d, i32 %e, i64 %
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
 ; RV32I-WITHFP-NEXT:    lw a0, 0(a2)
-; RV32I-WITHFP-NEXT:    add a0, a0, a7
-; RV32I-WITHFP-NEXT:    lw a1, 0(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 8(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
-; RV32I-WITHFP-NEXT:    lw a1, 16(s0)
-; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw a1, 20(s0)
+; RV32I-WITHFP-NEXT:    lw a2, 0(s0)
+; RV32I-WITHFP-NEXT:    lw a3, 8(s0)
+; RV32I-WITHFP-NEXT:    lw a4, 16(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a7
+; RV32I-WITHFP-NEXT:    add a0, a0, a2
+; RV32I-WITHFP-NEXT:    add a0, a0, a3
+; RV32I-WITHFP-NEXT:    add a0, a0, a4
 ; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
@@ -690,8 +690,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32I-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32I-FPELIM-NEXT:    lui a0, 335544
-; RV32I-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32I-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32I-FPELIM-NEXT:    addi t0, a0, 1311
 ; RV32I-FPELIM-NEXT:    lui a0, 688509
 ; RV32I-FPELIM-NEXT:    addi a5, a0, -2048
 ; RV32I-FPELIM-NEXT:    addi a2, sp, 32
@@ -701,6 +700,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
 ; RV32I-FPELIM-NEXT:    addi a7, zero, 14
+; RV32I-FPELIM-NEXT:    sw t0, 32(sp)
 ; RV32I-FPELIM-NEXT:    call callee_aligned_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 64
@@ -733,8 +733,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a0, a0, -328
 ; RV32I-WITHFP-NEXT:    sw a0, -28(s0)
 ; RV32I-WITHFP-NEXT:    lui a0, 335544
-; RV32I-WITHFP-NEXT:    addi a0, a0, 1311
-; RV32I-WITHFP-NEXT:    sw a0, -32(s0)
+; RV32I-WITHFP-NEXT:    addi t0, a0, 1311
 ; RV32I-WITHFP-NEXT:    lui a0, 688509
 ; RV32I-WITHFP-NEXT:    addi a5, a0, -2048
 ; RV32I-WITHFP-NEXT:    addi a2, s0, -32
@@ -744,6 +743,7 @@ define void @caller_aligned_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 13
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
 ; RV32I-WITHFP-NEXT:    addi a7, zero, 14
+; RV32I-WITHFP-NEXT:    sw t0, -32(s0)
 ; RV32I-WITHFP-NEXT:    call callee_aligned_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 56(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 60(sp)
@@ -787,17 +787,15 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    sw s0, 8(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 56
-; RV32I-FPELIM-NEXT:    addi s0, a0, 580
 ; RV32I-FPELIM-NEXT:    call callee_small_scalar_ret
-; RV32I-FPELIM-NEXT:    xor a1, a1, s0
+; RV32I-FPELIM-NEXT:    lui a2, 56
+; RV32I-FPELIM-NEXT:    addi a2, a2, 580
+; RV32I-FPELIM-NEXT:    xor a1, a1, a2
 ; RV32I-FPELIM-NEXT:    lui a2, 200614
 ; RV32I-FPELIM-NEXT:    addi a2, a2, 647
 ; RV32I-FPELIM-NEXT:    xor a0, a0, a2
 ; RV32I-FPELIM-NEXT:    or a0, a0, a1
 ; RV32I-FPELIM-NEXT:    seqz a0, a0
-; RV32I-FPELIM-NEXT:    lw s0, 8(sp)
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 16
 ; RV32I-FPELIM-NEXT:    ret
@@ -807,18 +805,16 @@ define i32 @caller_small_scalar_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    addi sp, sp, -16
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
-; RV32I-WITHFP-NEXT:    sw s1, 4(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lui a0, 56
-; RV32I-WITHFP-NEXT:    addi s1, a0, 580
 ; RV32I-WITHFP-NEXT:    call callee_small_scalar_ret
-; RV32I-WITHFP-NEXT:    xor a1, a1, s1
+; RV32I-WITHFP-NEXT:    lui a2, 56
+; RV32I-WITHFP-NEXT:    addi a2, a2, 580
+; RV32I-WITHFP-NEXT:    xor a1, a1, a2
 ; RV32I-WITHFP-NEXT:    lui a2, 200614
 ; RV32I-WITHFP-NEXT:    addi a2, a2, 647
 ; RV32I-WITHFP-NEXT:    xor a0, a0, a2
 ; RV32I-WITHFP-NEXT:    or a0, a0, a1
 ; RV32I-WITHFP-NEXT:    seqz a0, a0
-; RV32I-WITHFP-NEXT:    lw s1, 4(sp)
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -946,14 +942,14 @@ define void @caller_large_scalar_ret() nounwind {
 define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) nounwind {
 ; RV32I-FPELIM-LABEL: callee_large_struct_ret:
 ; RV32I-FPELIM:       # %bb.0:
-; RV32I-FPELIM-NEXT:    addi a1, zero, 4
-; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
-; RV32I-FPELIM-NEXT:    addi a1, zero, 3
-; RV32I-FPELIM-NEXT:    sw a1, 8(a0)
-; RV32I-FPELIM-NEXT:    addi a1, zero, 2
-; RV32I-FPELIM-NEXT:    sw a1, 4(a0)
 ; RV32I-FPELIM-NEXT:    addi a1, zero, 1
 ; RV32I-FPELIM-NEXT:    sw a1, 0(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 2
+; RV32I-FPELIM-NEXT:    sw a1, 4(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 3
+; RV32I-FPELIM-NEXT:    sw a1, 8(a0)
+; RV32I-FPELIM-NEXT:    addi a1, zero, 4
+; RV32I-FPELIM-NEXT:    sw a1, 12(a0)
 ; RV32I-FPELIM-NEXT:    ret
 ;
 ; RV32I-WITHFP-LABEL: callee_large_struct_ret:
@@ -962,14 +958,14 @@ define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) no
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    addi a1, zero, 4
-; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
-; RV32I-WITHFP-NEXT:    addi a1, zero, 3
-; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
-; RV32I-WITHFP-NEXT:    addi a1, zero, 2
-; RV32I-WITHFP-NEXT:    sw a1, 4(a0)
 ; RV32I-WITHFP-NEXT:    addi a1, zero, 1
 ; RV32I-WITHFP-NEXT:    sw a1, 0(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 2
+; RV32I-WITHFP-NEXT:    sw a1, 4(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 3
+; RV32I-WITHFP-NEXT:    sw a1, 8(a0)
+; RV32I-WITHFP-NEXT:    addi a1, zero, 4
+; RV32I-WITHFP-NEXT:    sw a1, 12(a0)
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 16
@@ -992,9 +988,9 @@ define i32 @caller_large_struct_ret() nounwind {
 ; RV32I-FPELIM-NEXT:    sw ra, 28(sp)
 ; RV32I-FPELIM-NEXT:    addi a0, sp, 8
 ; RV32I-FPELIM-NEXT:    call callee_large_struct_ret
-; RV32I-FPELIM-NEXT:    lw a0, 20(sp)
-; RV32I-FPELIM-NEXT:    lw a1, 8(sp)
-; RV32I-FPELIM-NEXT:    add a0, a1, a0
+; RV32I-FPELIM-NEXT:    lw a0, 8(sp)
+; RV32I-FPELIM-NEXT:    lw a1, 20(sp)
+; RV32I-FPELIM-NEXT:    add a0, a0, a1
 ; RV32I-FPELIM-NEXT:    lw ra, 28(sp)
 ; RV32I-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32I-FPELIM-NEXT:    ret
@@ -1007,9 +1003,9 @@ define i32 @caller_large_struct_ret() nounwind {
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 32
 ; RV32I-WITHFP-NEXT:    addi a0, s0, -24
 ; RV32I-WITHFP-NEXT:    call callee_large_struct_ret
-; RV32I-WITHFP-NEXT:    lw a0, -12(s0)
-; RV32I-WITHFP-NEXT:    lw a1, -24(s0)
-; RV32I-WITHFP-NEXT:    add a0, a1, a0
+; RV32I-WITHFP-NEXT:    lw a0, -24(s0)
+; RV32I-WITHFP-NEXT:    lw a1, -12(s0)
+; RV32I-WITHFP-NEXT:    add a0, a0, a1
 ; RV32I-WITHFP-NEXT:    lw s0, 24(sp)
 ; RV32I-WITHFP-NEXT:    lw ra, 28(sp)
 ; RV32I-WITHFP-NEXT:    addi sp, sp, 32
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
index ddedab591d004..8ff3de7990847 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32.ll
@@ -107,15 +107,15 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-FPELIM:       # %bb.0:
 ; RV32I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32I-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32I-FPELIM-NEXT:    lui a0, 264704
-; RV32I-FPELIM-NEXT:    sw a0, 0(sp)
+; RV32I-FPELIM-NEXT:    lui a1, 264704
 ; RV32I-FPELIM-NEXT:    addi a0, zero, 1
-; RV32I-FPELIM-NEXT:    mv a1, zero
 ; RV32I-FPELIM-NEXT:    addi a2, zero, 2
-; RV32I-FPELIM-NEXT:    mv a3, zero
 ; RV32I-FPELIM-NEXT:    addi a4, zero, 3
-; RV32I-FPELIM-NEXT:    mv a5, zero
 ; RV32I-FPELIM-NEXT:    addi a6, zero, 4
+; RV32I-FPELIM-NEXT:    sw a1, 0(sp)
+; RV32I-FPELIM-NEXT:    mv a1, zero
+; RV32I-FPELIM-NEXT:    mv a3, zero
+; RV32I-FPELIM-NEXT:    mv a5, zero
 ; RV32I-FPELIM-NEXT:    mv a7, zero
 ; RV32I-FPELIM-NEXT:    call callee_float_on_stack
 ; RV32I-FPELIM-NEXT:    lw ra, 12(sp)
@@ -128,15 +128,15 @@ define i32 @caller_float_on_stack() nounwind {
 ; RV32I-WITHFP-NEXT:    sw ra, 12(sp)
 ; RV32I-WITHFP-NEXT:    sw s0, 8(sp)
 ; RV32I-WITHFP-NEXT:    addi s0, sp, 16
-; RV32I-WITHFP-NEXT:    lui a0, 264704
-; RV32I-WITHFP-NEXT:    sw a0, 0(sp)
+; RV32I-WITHFP-NEXT:    lui a1, 264704
 ; RV32I-WITHFP-NEXT:    addi a0, zero, 1
-; RV32I-WITHFP-NEXT:    mv a1, zero
 ; RV32I-WITHFP-NEXT:    addi a2, zero, 2
-; RV32I-WITHFP-NEXT:    mv a3, zero
 ; RV32I-WITHFP-NEXT:    addi a4, zero, 3
-; RV32I-WITHFP-NEXT:    mv a5, zero
 ; RV32I-WITHFP-NEXT:    addi a6, zero, 4
+; RV32I-WITHFP-NEXT:    sw a1, 0(sp)
+; RV32I-WITHFP-NEXT:    mv a1, zero
+; RV32I-WITHFP-NEXT:    mv a3, zero
+; RV32I-WITHFP-NEXT:    mv a5, zero
 ; RV32I-WITHFP-NEXT:    mv a7, zero
 ; RV32I-WITHFP-NEXT:    call callee_float_on_stack
 ; RV32I-WITHFP-NEXT:    lw s0, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
index 37dfa7bf9c893..5ad002f5c423f 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32d.ll
@@ -38,9 +38,9 @@ define i32 @caller_double_in_fpr() nounwind {
 define i32 @callee_double_in_fpr_exhausted_gprs(i64 %a, i64 %b, i64 %c, i64 %d, i32 %e, double %f) nounwind {
 ; RV32-ILP32D-LABEL: callee_double_in_fpr_exhausted_gprs:
 ; RV32-ILP32D:       # %bb.0:
-; RV32-ILP32D-NEXT:    fcvt.w.d a0, fa0, rtz
-; RV32-ILP32D-NEXT:    lw a1, 0(sp)
-; RV32-ILP32D-NEXT:    add a0, a1, a0
+; RV32-ILP32D-NEXT:    lw a0, 0(sp)
+; RV32-ILP32D-NEXT:    fcvt.w.d a1, fa0, rtz
+; RV32-ILP32D-NEXT:    add a0, a0, a1
 ; RV32-ILP32D-NEXT:    ret
   %f_fptosi = fptosi double %f to i32
   %1 = add i32 %e, %f_fptosi
@@ -52,18 +52,18 @@ define i32 @caller_double_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
-; RV32-ILP32D-NEXT:    addi a0, zero, 5
-; RV32-ILP32D-NEXT:    sw a0, 0(sp)
+; RV32-ILP32D-NEXT:    addi a1, zero, 5
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 2
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 3
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 4
+; RV32-ILP32D-NEXT:    sw a1, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_fpr_exhausted_gprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
@@ -82,9 +82,9 @@ define i32 @callee_double_in_gpr_exhausted_fprs(double %a, double %b, double %c,
 ; RV32-ILP32D-NEXT:    sw a0, 8(sp)
 ; RV32-ILP32D-NEXT:    sw a1, 12(sp)
 ; RV32-ILP32D-NEXT:    fld ft0, 8(sp)
-; RV32-ILP32D-NEXT:    fcvt.w.d a0, ft0, rtz
-; RV32-ILP32D-NEXT:    fcvt.w.d a1, fa7, rtz
-; RV32-ILP32D-NEXT:    add a0, a1, a0
+; RV32-ILP32D-NEXT:    fcvt.w.d a0, fa7, rtz
+; RV32-ILP32D-NEXT:    fcvt.w.d a1, ft0, rtz
+; RV32-ILP32D-NEXT:    add a0, a0, a1
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
 ; RV32-ILP32D-NEXT:    ret
   %h_fptosi = fptosi double %h to i32
@@ -100,21 +100,21 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI5_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI5_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI5_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI5_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI5_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI5_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI5_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI5_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI5_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI5_5)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a1)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_5)
 ; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_6)
@@ -122,8 +122,8 @@ define i32 @caller_double_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI5_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI5_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
-; RV32-ILP32D-NEXT:    mv a0, zero
 ; RV32-ILP32D-NEXT:    lui a1, 262688
+; RV32-ILP32D-NEXT:    mv a0, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_exhausted_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    addi sp, sp, 16
@@ -157,39 +157,39 @@ define i32 @caller_double_in_gpr_and_stack_almost_exhausted_gprs_fprs() nounwind
 ; RV32-ILP32D:       # %bb.0:
 ; RV32-ILP32D-NEXT:    addi sp, sp, -16
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
-; RV32-ILP32D-NEXT:    lui a0, 262816
-; RV32-ILP32D-NEXT:    sw a0, 0(sp)
+; RV32-ILP32D-NEXT:    lui a1, 262816
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_0)
-; RV32-ILP32D-NEXT:    addi a6, a0, %lo(.LCPI7_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI7_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI7_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI7_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI7_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI7_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI7_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI7_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI7_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI7_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI7_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_5)
+; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_6)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa5, 0(a1)
-; RV32-ILP32D-NEXT:    fld fa6, 0(a6)
+; RV32-ILP32D-NEXT:    fld fa6, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI7_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI7_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 3
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 5
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 7
+; RV32-ILP32D-NEXT:    sw a1, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_in_gpr_and_stack_almost_exhausted_gprs_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
@@ -223,38 +223,38 @@ define i32 @caller_double_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32D-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32D-NEXT:    lui a0, 262816
 ; RV32-ILP32D-NEXT:    sw a0, 4(sp)
-; RV32-ILP32D-NEXT:    sw zero, 0(sp)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_0)
-; RV32-ILP32D-NEXT:    addi a6, a0, %lo(.LCPI9_0)
-; RV32-ILP32D-NEXT:    lui a1, %hi(.LCPI9_1)
-; RV32-ILP32D-NEXT:    addi a1, a1, %lo(.LCPI9_1)
-; RV32-ILP32D-NEXT:    lui a2, %hi(.LCPI9_2)
-; RV32-ILP32D-NEXT:    addi a2, a2, %lo(.LCPI9_2)
-; RV32-ILP32D-NEXT:    lui a3, %hi(.LCPI9_3)
-; RV32-ILP32D-NEXT:    addi a3, a3, %lo(.LCPI9_3)
-; RV32-ILP32D-NEXT:    lui a4, %hi(.LCPI9_4)
-; RV32-ILP32D-NEXT:    addi a4, a4, %lo(.LCPI9_4)
-; RV32-ILP32D-NEXT:    lui a5, %hi(.LCPI9_5)
-; RV32-ILP32D-NEXT:    addi a5, a5, %lo(.LCPI9_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_0)
+; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_1)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_1)
+; RV32-ILP32D-NEXT:    fld fa1, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_2)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_2)
+; RV32-ILP32D-NEXT:    fld fa2, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_3)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_3)
+; RV32-ILP32D-NEXT:    fld fa3, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_4)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_4)
+; RV32-ILP32D-NEXT:    fld fa4, 0(a0)
+; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_5)
+; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_5)
+; RV32-ILP32D-NEXT:    fld fa5, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_6)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_6)
-; RV32-ILP32D-NEXT:    fld fa0, 0(a0)
-; RV32-ILP32D-NEXT:    fld fa1, 0(a5)
-; RV32-ILP32D-NEXT:    fld fa2, 0(a4)
-; RV32-ILP32D-NEXT:    fld fa3, 0(a3)
-; RV32-ILP32D-NEXT:    fld fa4, 0(a2)
-; RV32-ILP32D-NEXT:    fld fa5, 0(a1)
-; RV32-ILP32D-NEXT:    fld fa6, 0(a6)
+; RV32-ILP32D-NEXT:    fld fa6, 0(a0)
 ; RV32-ILP32D-NEXT:    lui a0, %hi(.LCPI9_7)
 ; RV32-ILP32D-NEXT:    addi a0, a0, %lo(.LCPI9_7)
 ; RV32-ILP32D-NEXT:    fld fa7, 0(a0)
 ; RV32-ILP32D-NEXT:    addi a0, zero, 1
-; RV32-ILP32D-NEXT:    mv a1, zero
 ; RV32-ILP32D-NEXT:    addi a2, zero, 3
-; RV32-ILP32D-NEXT:    mv a3, zero
 ; RV32-ILP32D-NEXT:    addi a4, zero, 5
-; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    addi a6, zero, 7
+; RV32-ILP32D-NEXT:    sw zero, 0(sp)
+; RV32-ILP32D-NEXT:    mv a1, zero
+; RV32-ILP32D-NEXT:    mv a3, zero
+; RV32-ILP32D-NEXT:    mv a5, zero
 ; RV32-ILP32D-NEXT:    mv a7, zero
 ; RV32-ILP32D-NEXT:    call callee_double_on_stack_exhausted_gprs_fprs
 ; RV32-ILP32D-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
index 5f3042dfe6e9d..70b25059f3353 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-ilp32f-ilp32d-common.ll
@@ -41,9 +41,9 @@ define i32 @caller_float_in_fpr() nounwind {
 define i32 @callee_float_in_fpr_exhausted_gprs(i64 %a, i64 %b, i64 %c, i64 %d, i32 %e, float %f) nounwind {
 ; RV32-ILP32FD-LABEL: callee_float_in_fpr_exhausted_gprs:
 ; RV32-ILP32FD:       # %bb.0:
-; RV32-ILP32FD-NEXT:    fcvt.w.s a0, fa0, rtz
-; RV32-ILP32FD-NEXT:    lw a1, 0(sp)
-; RV32-ILP32FD-NEXT:    add a0, a1, a0
+; RV32-ILP32FD-NEXT:    lw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    fcvt.w.s a1, fa0, rtz
+; RV32-ILP32FD-NEXT:    add a0, a0, a1
 ; RV32-ILP32FD-NEXT:    ret
   %f_fptosi = fptosi float %f to i32
   %1 = add i32 %e, %f_fptosi
@@ -55,18 +55,18 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
-; RV32-ILP32FD-NEXT:    addi a0, zero, 5
-; RV32-ILP32FD-NEXT:    sw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    addi a1, zero, 5
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI3_0)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI3_0)
 ; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
 ; RV32-ILP32FD-NEXT:    addi a0, zero, 1
-; RV32-ILP32FD-NEXT:    mv a1, zero
 ; RV32-ILP32FD-NEXT:    addi a2, zero, 2
-; RV32-ILP32FD-NEXT:    mv a3, zero
 ; RV32-ILP32FD-NEXT:    addi a4, zero, 3
-; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    addi a6, zero, 4
+; RV32-ILP32FD-NEXT:    sw a1, 0(sp)
+; RV32-ILP32FD-NEXT:    mv a1, zero
+; RV32-ILP32FD-NEXT:    mv a3, zero
+; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_in_fpr_exhausted_gprs
 ; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
@@ -81,10 +81,10 @@ define i32 @caller_float_in_fpr_exhausted_gprs() nounwind {
 define i32 @callee_float_in_gpr_exhausted_fprs(float %a, float %b, float %c, float %d, float %e, float %f, float %g, float %h, float %i) nounwind {
 ; RV32-ILP32FD-LABEL: callee_float_in_gpr_exhausted_fprs:
 ; RV32-ILP32FD:       # %bb.0:
-; RV32-ILP32FD-NEXT:    fcvt.w.s a1, fa7, rtz
 ; RV32-ILP32FD-NEXT:    fmv.w.x ft0, a0
-; RV32-ILP32FD-NEXT:    fcvt.w.s a0, ft0, rtz
-; RV32-ILP32FD-NEXT:    add a0, a1, a0
+; RV32-ILP32FD-NEXT:    fcvt.w.s a0, fa7, rtz
+; RV32-ILP32FD-NEXT:    fcvt.w.s a1, ft0, rtz
+; RV32-ILP32FD-NEXT:    add a0, a0, a1
 ; RV32-ILP32FD-NEXT:    ret
   %h_fptosi = fptosi float %h to i32
   %i_fptosi = fptosi float %i to i32
@@ -99,21 +99,21 @@ define i32 @caller_float_in_gpr_exhausted_fprs() nounwind {
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_0)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_0)
-; RV32-ILP32FD-NEXT:    lui a1, %hi(.LCPI5_1)
-; RV32-ILP32FD-NEXT:    addi a1, a1, %lo(.LCPI5_1)
-; RV32-ILP32FD-NEXT:    lui a2, %hi(.LCPI5_2)
-; RV32-ILP32FD-NEXT:    addi a2, a2, %lo(.LCPI5_2)
-; RV32-ILP32FD-NEXT:    lui a3, %hi(.LCPI5_3)
-; RV32-ILP32FD-NEXT:    addi a3, a3, %lo(.LCPI5_3)
-; RV32-ILP32FD-NEXT:    lui a4, %hi(.LCPI5_4)
-; RV32-ILP32FD-NEXT:    addi a4, a4, %lo(.LCPI5_4)
-; RV32-ILP32FD-NEXT:    lui a5, %hi(.LCPI5_5)
-; RV32-ILP32FD-NEXT:    addi a5, a5, %lo(.LCPI5_5)
-; RV32-ILP32FD-NEXT:    flw fa0, 0(a5)
-; RV32-ILP32FD-NEXT:    flw fa1, 0(a4)
-; RV32-ILP32FD-NEXT:    flw fa2, 0(a3)
-; RV32-ILP32FD-NEXT:    flw fa3, 0(a2)
-; RV32-ILP32FD-NEXT:    flw fa4, 0(a1)
+; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_1)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_1)
+; RV32-ILP32FD-NEXT:    flw fa1, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_2)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_2)
+; RV32-ILP32FD-NEXT:    flw fa2, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_3)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_3)
+; RV32-ILP32FD-NEXT:    flw fa3, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_4)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_4)
+; RV32-ILP32FD-NEXT:    flw fa4, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_5)
 ; RV32-ILP32FD-NEXT:    flw fa5, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI5_6)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI5_6)
@@ -151,39 +151,39 @@ define i32 @caller_float_on_stack_exhausted_gprs_fprs() nounwind {
 ; RV32-ILP32FD:       # %bb.0:
 ; RV32-ILP32FD-NEXT:    addi sp, sp, -16
 ; RV32-ILP32FD-NEXT:    sw ra, 12(sp)
-; RV32-ILP32FD-NEXT:    lui a0, 267520
-; RV32-ILP32FD-NEXT:    sw a0, 0(sp)
+; RV32-ILP32FD-NEXT:    lui a1, 267520
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_0)
-; RV32-ILP32FD-NEXT:    addi a6, a0, %lo(.LCPI7_0)
-; RV32-ILP32FD-NEXT:    lui a1, %hi(.LCPI7_1)
-; RV32-ILP32FD-NEXT:    addi a1, a1, %lo(.LCPI7_1)
-; RV32-ILP32FD-NEXT:    lui a2, %hi(.LCPI7_2)
-; RV32-ILP32FD-NEXT:    addi a2, a2, %lo(.LCPI7_2)
-; RV32-ILP32FD-NEXT:    lui a3, %hi(.LCPI7_3)
-; RV32-ILP32FD-NEXT:    addi a3, a3, %lo(.LCPI7_3)
-; RV32-ILP32FD-NEXT:    lui a4, %hi(.LCPI7_4)
-; RV32-ILP32FD-NEXT:    addi a4, a4, %lo(.LCPI7_4)
-; RV32-ILP32FD-NEXT:    lui a5, %hi(.LCPI7_5)
-; RV32-ILP32FD-NEXT:    addi a5, a5, %lo(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_0)
+; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_1)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_1)
+; RV32-ILP32FD-NEXT:    flw fa1, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_2)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_2)
+; RV32-ILP32FD-NEXT:    flw fa2, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_3)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_3)
+; RV32-ILP32FD-NEXT:    flw fa3, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_4)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_4)
+; RV32-ILP32FD-NEXT:    flw fa4, 0(a0)
+; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_5)
+; RV32-ILP32FD-NEXT:    flw fa5, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_6)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_6)
-; RV32-ILP32FD-NEXT:    flw fa0, 0(a0)
-; RV32-ILP32FD-NEXT:    flw fa1, 0(a5)
-; RV32-ILP32FD-NEXT:    flw fa2, 0(a4)
-; RV32-ILP32FD-NEXT:    flw fa3, 0(a3)
-; RV32-ILP32FD-NEXT:    flw fa4, 0(a2)
-; RV32-ILP32FD-NEXT:    flw fa5, 0(a1)
-; RV32-ILP32FD-NEXT:    flw fa6, 0(a6)
+; RV32-ILP32FD-NEXT:    flw fa6, 0(a0)
 ; RV32-ILP32FD-NEXT:    lui a0, %hi(.LCPI7_7)
 ; RV32-ILP32FD-NEXT:    addi a0, a0, %lo(.LCPI7_7)
 ; RV32-ILP32FD-NEXT:    flw fa7, 0(a0)
 ; RV32-ILP32FD-NEXT:    addi a0, zero, 1
-; RV32-ILP32FD-NEXT:    mv a1, zero
 ; RV32-ILP32FD-NEXT:    addi a2, zero, 3
-; RV32-ILP32FD-NEXT:    mv a3, zero
 ; RV32-ILP32FD-NEXT:    addi a4, zero, 5
-; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    addi a6, zero, 7
+; RV32-ILP32FD-NEXT:    sw a1, 0(sp)
+; RV32-ILP32FD-NEXT:    mv a1, zero
+; RV32-ILP32FD-NEXT:    mv a3, zero
+; RV32-ILP32FD-NEXT:    mv a5, zero
 ; RV32-ILP32FD-NEXT:    mv a7, zero
 ; RV32-ILP32FD-NEXT:    call callee_float_on_stack_exhausted_gprs_fprs
 ; RV32-ILP32FD-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
index 23bb630c67eac..95cf39e79d3cc 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64-lp64f-lp64d-common.ll
@@ -48,22 +48,22 @@ define i64 @caller_i128_in_regs() nounwind {
 define i32 @callee_many_scalars(i8 %a, i16 %b, i32 %c, i128 %d, i32 %e, i32 %f, i128 %g, i32 %h) nounwind {
 ; RV64I-LABEL: callee_many_scalars:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld t0, 0(sp)
-; RV64I-NEXT:    xor a4, a4, t0
-; RV64I-NEXT:    xor a3, a3, a7
-; RV64I-NEXT:    or a3, a3, a4
-; RV64I-NEXT:    lui a4, 16
-; RV64I-NEXT:    addiw a4, a4, -1
-; RV64I-NEXT:    and a1, a1, a4
-; RV64I-NEXT:    andi a0, a0, 255
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lw t0, 8(sp)
+; RV64I-NEXT:    ld t1, 0(sp)
+; RV64I-NEXT:    andi t2, a0, 255
+; RV64I-NEXT:    lui a0, 16
+; RV64I-NEXT:    addiw a0, a0, -1
+; RV64I-NEXT:    and a0, a1, a0
+; RV64I-NEXT:    add a0, t2, a0
 ; RV64I-NEXT:    add a0, a0, a2
-; RV64I-NEXT:    seqz a1, a3
+; RV64I-NEXT:    xor a1, a4, t1
+; RV64I-NEXT:    xor a2, a3, a7
+; RV64I-NEXT:    or a1, a2, a1
+; RV64I-NEXT:    seqz a1, a1
 ; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    add a0, a0, a5
 ; RV64I-NEXT:    add a0, a0, a6
-; RV64I-NEXT:    lw a1, 8(sp)
-; RV64I-NEXT:    addw a0, a0, a1
+; RV64I-NEXT:    addw a0, a0, t0
 ; RV64I-NEXT:    ret
   %a_ext = zext i8 %a to i32
   %b_ext = zext i16 %b to i32
@@ -85,15 +85,15 @@ define i32 @caller_many_scalars() nounwind {
 ; RV64I-NEXT:    sd ra, 24(sp)
 ; RV64I-NEXT:    addi a0, zero, 8
 ; RV64I-NEXT:    sd a0, 8(sp)
-; RV64I-NEXT:    sd zero, 0(sp)
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    addi a2, zero, 3
 ; RV64I-NEXT:    addi a3, zero, 4
-; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    addi a5, zero, 5
 ; RV64I-NEXT:    addi a6, zero, 6
 ; RV64I-NEXT:    addi a7, zero, 7
+; RV64I-NEXT:    sd zero, 0(sp)
+; RV64I-NEXT:    mv a4, zero
 ; RV64I-NEXT:    call callee_many_scalars
 ; RV64I-NEXT:    ld ra, 24(sp)
 ; RV64I-NEXT:    addi sp, sp, 32
@@ -107,20 +107,20 @@ define i32 @caller_many_scalars() nounwind {
 define i64 @callee_large_scalars(i256 %a, i256 %b) nounwind {
 ; RV64I-LABEL: callee_large_scalars:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a2, 24(a1)
-; RV64I-NEXT:    ld a3, 24(a0)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    ld a3, 8(a1)
-; RV64I-NEXT:    ld a4, 8(a0)
-; RV64I-NEXT:    xor a3, a4, a3
+; RV64I-NEXT:    ld a6, 0(a1)
+; RV64I-NEXT:    ld a7, 0(a0)
+; RV64I-NEXT:    ld a4, 8(a1)
+; RV64I-NEXT:    ld a5, 24(a1)
+; RV64I-NEXT:    ld a2, 24(a0)
+; RV64I-NEXT:    ld a3, 8(a0)
+; RV64I-NEXT:    ld a1, 16(a1)
+; RV64I-NEXT:    ld a0, 16(a0)
+; RV64I-NEXT:    xor a2, a2, a5
+; RV64I-NEXT:    xor a3, a3, a4
 ; RV64I-NEXT:    or a2, a3, a2
-; RV64I-NEXT:    ld a3, 16(a1)
-; RV64I-NEXT:    ld a4, 16(a0)
-; RV64I-NEXT:    xor a3, a4, a3
-; RV64I-NEXT:    ld a1, 0(a1)
-; RV64I-NEXT:    ld a0, 0(a0)
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a3
+; RV64I-NEXT:    xor a1, a7, a6
+; RV64I-NEXT:    or a0, a1, a0
 ; RV64I-NEXT:    or a0, a0, a2
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -134,18 +134,18 @@ define i64 @caller_large_scalars() nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -80
 ; RV64I-NEXT:    sd ra, 72(sp)
-; RV64I-NEXT:    addi a0, zero, 2
-; RV64I-NEXT:    sd a0, 0(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    sd zero, 16(sp)
 ; RV64I-NEXT:    sd zero, 8(sp)
+; RV64I-NEXT:    addi a0, zero, 2
+; RV64I-NEXT:    sd a0, 0(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
 ; RV64I-NEXT:    sd zero, 48(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
-; RV64I-NEXT:    addi a0, zero, 1
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    addi a2, zero, 1
 ; RV64I-NEXT:    addi a0, sp, 32
 ; RV64I-NEXT:    mv a1, sp
+; RV64I-NEXT:    sd a2, 32(sp)
 ; RV64I-NEXT:    call callee_large_scalars
 ; RV64I-NEXT:    ld ra, 72(sp)
 ; RV64I-NEXT:    addi sp, sp, 80
@@ -162,20 +162,20 @@ define i64 @callee_large_scalars_exhausted_regs(i64 %a, i64 %b, i64 %c, i64 %d,
 ; RV64I-LABEL: callee_large_scalars_exhausted_regs:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    ld a0, 8(sp)
-; RV64I-NEXT:    ld a1, 24(a0)
-; RV64I-NEXT:    ld a2, 24(a7)
-; RV64I-NEXT:    xor a1, a2, a1
-; RV64I-NEXT:    ld a2, 8(a0)
-; RV64I-NEXT:    ld a3, 8(a7)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    or a1, a2, a1
-; RV64I-NEXT:    ld a2, 16(a0)
-; RV64I-NEXT:    ld a3, 16(a7)
-; RV64I-NEXT:    xor a2, a3, a2
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    ld a3, 0(a7)
-; RV64I-NEXT:    xor a0, a3, a0
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    ld a6, 0(a0)
+; RV64I-NEXT:    ld t0, 0(a7)
+; RV64I-NEXT:    ld a3, 8(a0)
+; RV64I-NEXT:    ld a4, 24(a0)
+; RV64I-NEXT:    ld a5, 24(a7)
+; RV64I-NEXT:    ld a1, 8(a7)
+; RV64I-NEXT:    ld a0, 16(a0)
+; RV64I-NEXT:    ld a2, 16(a7)
+; RV64I-NEXT:    xor a4, a5, a4
+; RV64I-NEXT:    xor a1, a1, a3
+; RV64I-NEXT:    or a1, a1, a4
+; RV64I-NEXT:    xor a0, a2, a0
+; RV64I-NEXT:    xor a2, t0, a6
+; RV64I-NEXT:    or a0, a2, a0
 ; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    seqz a0, a0
 ; RV64I-NEXT:    ret
@@ -193,16 +193,15 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    sd a0, 8(sp)
 ; RV64I-NEXT:    addi a0, zero, 9
 ; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    addi a0, zero, 10
-; RV64I-NEXT:    sd a0, 16(sp)
 ; RV64I-NEXT:    sd zero, 40(sp)
 ; RV64I-NEXT:    sd zero, 32(sp)
 ; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    addi a0, zero, 10
+; RV64I-NEXT:    sd a0, 16(sp)
 ; RV64I-NEXT:    sd zero, 72(sp)
 ; RV64I-NEXT:    sd zero, 64(sp)
 ; RV64I-NEXT:    sd zero, 56(sp)
-; RV64I-NEXT:    addi a0, zero, 8
-; RV64I-NEXT:    sd a0, 48(sp)
+; RV64I-NEXT:    addi t0, zero, 8
 ; RV64I-NEXT:    addi a7, sp, 48
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
@@ -211,6 +210,7 @@ define i64 @caller_large_scalars_exhausted_regs() nounwind {
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    addi a5, zero, 6
 ; RV64I-NEXT:    addi a6, zero, 7
+; RV64I-NEXT:    sd t0, 48(sp)
 ; RV64I-NEXT:    call callee_large_scalars_exhausted_regs
 ; RV64I-NEXT:    ld ra, 88(sp)
 ; RV64I-NEXT:    addi sp, sp, 96
@@ -277,9 +277,9 @@ define i64 @caller_small_coerced_struct() nounwind {
 define i64 @callee_large_struct(%struct.large* byval align 8 %a) nounwind {
 ; RV64I-LABEL: callee_large_struct:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a1, 24(a0)
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ld a1, 0(a0)
+; RV64I-NEXT:    ld a0, 24(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr inbounds %struct.large, %struct.large* %a, i64 0, i32 0
   %2 = getelementptr inbounds %struct.large, %struct.large* %a, i64 0, i32 3
@@ -296,16 +296,16 @@ define i64 @caller_large_struct() nounwind {
 ; RV64I-NEXT:    sd ra, 72(sp)
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    sd a0, 40(sp)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    sd a1, 48(sp)
+; RV64I-NEXT:    addi a2, zero, 3
+; RV64I-NEXT:    sd a2, 56(sp)
+; RV64I-NEXT:    addi a3, zero, 4
+; RV64I-NEXT:    sd a3, 64(sp)
 ; RV64I-NEXT:    sd a0, 8(sp)
-; RV64I-NEXT:    addi a0, zero, 2
-; RV64I-NEXT:    sd a0, 48(sp)
-; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    addi a0, zero, 3
-; RV64I-NEXT:    sd a0, 56(sp)
-; RV64I-NEXT:    sd a0, 24(sp)
-; RV64I-NEXT:    addi a0, zero, 4
-; RV64I-NEXT:    sd a0, 64(sp)
-; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd a1, 16(sp)
+; RV64I-NEXT:    sd a2, 24(sp)
+; RV64I-NEXT:    sd a3, 32(sp)
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct
 ; RV64I-NEXT:    ld ra, 72(sp)
@@ -332,15 +332,15 @@ define i64 @callee_aligned_stack(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i128 %f
 ; should only be 8-byte aligned
 ; RV64I-LABEL: callee_aligned_stack:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    add a0, a5, a7
+; RV64I-NEXT:    ld a0, 40(sp)
 ; RV64I-NEXT:    ld a1, 0(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 16(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 32(sp)
-; RV64I-NEXT:    add a0, a0, a1
-; RV64I-NEXT:    ld a1, 40(sp)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    ld a2, 16(sp)
+; RV64I-NEXT:    ld a3, 32(sp)
+; RV64I-NEXT:    add a4, a5, a7
+; RV64I-NEXT:    add a1, a4, a1
+; RV64I-NEXT:    add a1, a1, a2
+; RV64I-NEXT:    add a1, a1, a3
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %f_trunc = trunc i128 %f to i64
   %1 = add i64 %f_trunc, %g
@@ -366,19 +366,19 @@ define void @caller_aligned_stack() nounwind {
 ; RV64I-NEXT:    sd a0, 40(sp)
 ; RV64I-NEXT:    addi a0, zero, 10
 ; RV64I-NEXT:    sd a0, 32(sp)
+; RV64I-NEXT:    sd zero, 24(sp)
 ; RV64I-NEXT:    addi a0, zero, 9
 ; RV64I-NEXT:    sd a0, 16(sp)
-; RV64I-NEXT:    addi a0, zero, 8
-; RV64I-NEXT:    sd a0, 0(sp)
-; RV64I-NEXT:    sd zero, 24(sp)
+; RV64I-NEXT:    addi a6, zero, 8
 ; RV64I-NEXT:    addi a0, zero, 1
 ; RV64I-NEXT:    addi a1, zero, 2
 ; RV64I-NEXT:    addi a2, zero, 3
 ; RV64I-NEXT:    addi a3, zero, 4
 ; RV64I-NEXT:    addi a4, zero, 5
 ; RV64I-NEXT:    addi a5, zero, 6
-; RV64I-NEXT:    mv a6, zero
 ; RV64I-NEXT:    addi a7, zero, 7
+; RV64I-NEXT:    sd a6, 0(sp)
+; RV64I-NEXT:    mv a6, zero
 ; RV64I-NEXT:    call callee_aligned_stack
 ; RV64I-NEXT:    ld ra, 56(sp)
 ; RV64I-NEXT:    addi sp, sp, 64
@@ -482,18 +482,18 @@ define void @caller_large_scalar_ret() nounwind {
 define void @callee_large_struct_ret(%struct.large* noalias sret %agg.result) nounwind {
 ; RV64I-LABEL: callee_large_struct_ret:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    addi a1, zero, 4
-; RV64I-NEXT:    sw a1, 24(a0)
-; RV64I-NEXT:    addi a1, zero, 3
-; RV64I-NEXT:    sw a1, 16(a0)
-; RV64I-NEXT:    addi a1, zero, 2
-; RV64I-NEXT:    sw a1, 8(a0)
-; RV64I-NEXT:    sw zero, 28(a0)
-; RV64I-NEXT:    sw zero, 20(a0)
-; RV64I-NEXT:    sw zero, 12(a0)
 ; RV64I-NEXT:    sw zero, 4(a0)
 ; RV64I-NEXT:    addi a1, zero, 1
 ; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    sw zero, 12(a0)
+; RV64I-NEXT:    addi a1, zero, 2
+; RV64I-NEXT:    sw a1, 8(a0)
+; RV64I-NEXT:    sw zero, 20(a0)
+; RV64I-NEXT:    addi a1, zero, 3
+; RV64I-NEXT:    sw a1, 16(a0)
+; RV64I-NEXT:    sw zero, 28(a0)
+; RV64I-NEXT:    addi a1, zero, 4
+; RV64I-NEXT:    sw a1, 24(a0)
 ; RV64I-NEXT:    ret
   %a = getelementptr inbounds %struct.large, %struct.large* %agg.result, i64 0, i32 0
   store i64 1, i64* %a, align 4
@@ -513,9 +513,9 @@ define i64 @caller_large_struct_ret() nounwind {
 ; RV64I-NEXT:    sd ra, 40(sp)
 ; RV64I-NEXT:    addi a0, sp, 8
 ; RV64I-NEXT:    call callee_large_struct_ret
-; RV64I-NEXT:    ld a0, 32(sp)
-; RV64I-NEXT:    ld a1, 8(sp)
-; RV64I-NEXT:    add a0, a1, a0
+; RV64I-NEXT:    ld a0, 8(sp)
+; RV64I-NEXT:    ld a1, 32(sp)
+; RV64I-NEXT:    add a0, a0, a1
 ; RV64I-NEXT:    ld ra, 40(sp)
 ; RV64I-NEXT:    addi sp, sp, 48
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
index 3e6941c6c2a8b..3135fcd162d14 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-lp64.ll
@@ -108,15 +108,15 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-FPELIM:       # %bb.0:
 ; RV64I-FPELIM-NEXT:    addi sp, sp, -16
 ; RV64I-FPELIM-NEXT:    sd ra, 8(sp)
-; RV64I-FPELIM-NEXT:    lui a0, 264704
-; RV64I-FPELIM-NEXT:    sd a0, 0(sp)
+; RV64I-FPELIM-NEXT:    lui a1, 264704
 ; RV64I-FPELIM-NEXT:    addi a0, zero, 1
-; RV64I-FPELIM-NEXT:    mv a1, zero
 ; RV64I-FPELIM-NEXT:    addi a2, zero, 2
-; RV64I-FPELIM-NEXT:    mv a3, zero
 ; RV64I-FPELIM-NEXT:    addi a4, zero, 3
-; RV64I-FPELIM-NEXT:    mv a5, zero
 ; RV64I-FPELIM-NEXT:    addi a6, zero, 4
+; RV64I-FPELIM-NEXT:    sd a1, 0(sp)
+; RV64I-FPELIM-NEXT:    mv a1, zero
+; RV64I-FPELIM-NEXT:    mv a3, zero
+; RV64I-FPELIM-NEXT:    mv a5, zero
 ; RV64I-FPELIM-NEXT:    mv a7, zero
 ; RV64I-FPELIM-NEXT:    call callee_float_on_stack
 ; RV64I-FPELIM-NEXT:    ld ra, 8(sp)
@@ -129,15 +129,15 @@ define i64 @caller_float_on_stack() nounwind {
 ; RV64I-WITHFP-NEXT:    sd ra, 24(sp)
 ; RV64I-WITHFP-NEXT:    sd s0, 16(sp)
 ; RV64I-WITHFP-NEXT:    addi s0, sp, 32
-; RV64I-WITHFP-NEXT:    lui a0, 264704
-; RV64I-WITHFP-NEXT:    sd a0, 0(sp)
+; RV64I-WITHFP-NEXT:    lui a1, 264704
 ; RV64I-WITHFP-NEXT:    addi a0, zero, 1
-; RV64I-WITHFP-NEXT:    mv a1, zero
 ; RV64I-WITHFP-NEXT:    addi a2, zero, 2
-; RV64I-WITHFP-NEXT:    mv a3, zero
 ; RV64I-WITHFP-NEXT:    addi a4, zero, 3
-; RV64I-WITHFP-NEXT:    mv a5, zero
 ; RV64I-WITHFP-NEXT:    addi a6, zero, 4
+; RV64I-WITHFP-NEXT:    sd a1, 0(sp)
+; RV64I-WITHFP-NEXT:    mv a1, zero
+; RV64I-WITHFP-NEXT:    mv a3, zero
+; RV64I-WITHFP-NEXT:    mv a5, zero
 ; RV64I-WITHFP-NEXT:    mv a7, zero
 ; RV64I-WITHFP-NEXT:    call callee_float_on_stack
 ; RV64I-WITHFP-NEXT:    ld s0, 16(sp)
diff --git a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
index 9a68f0006668d..bc86428d0836f 100644
--- a/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
+++ b/llvm/test/CodeGen/RISCV/calling-conv-rv32f-ilp32.ll
@@ -33,15 +33,15 @@ define float @caller_onstack_f32_noop(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
 ; RV32IF-NEXT:    sw a0, 4(sp)
-; RV32IF-NEXT:    lui a0, 264704
-; RV32IF-NEXT:    sw a0, 0(sp)
+; RV32IF-NEXT:    lui a1, 264704
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    mv a1, zero
 ; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    mv a3, zero
 ; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    addi a6, zero, 4
+; RV32IF-NEXT:    sw a1, 0(sp)
+; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
@@ -56,19 +56,19 @@ define float @caller_onstack_f32_fadd(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fmv.w.x ft1, a1
-; RV32IF-NEXT:    fsub.s ft2, ft1, ft0
-; RV32IF-NEXT:    fsw ft2, 4(sp)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fsw ft0, 0(sp)
+; RV32IF-NEXT:    fmv.w.x ft0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft2, ft1, ft0
+; RV32IF-NEXT:    fsub.s ft0, ft0, ft1
+; RV32IF-NEXT:    fsw ft0, 4(sp)
 ; RV32IF-NEXT:    addi a0, zero, 1
-; RV32IF-NEXT:    mv a1, zero
 ; RV32IF-NEXT:    addi a2, zero, 2
-; RV32IF-NEXT:    mv a3, zero
 ; RV32IF-NEXT:    addi a4, zero, 3
-; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    addi a6, zero, 4
+; RV32IF-NEXT:    fsw ft2, 0(sp)
+; RV32IF-NEXT:    mv a1, zero
+; RV32IF-NEXT:    mv a3, zero
+; RV32IF-NEXT:    mv a5, zero
 ; RV32IF-NEXT:    mv a7, zero
 ; RV32IF-NEXT:    call onstack_f32_noop
 ; RV32IF-NEXT:    lw ra, 12(sp)
diff --git a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
index 0706a61298732..6430a221e6872 100644
--- a/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/codemodel-lowering.ll
@@ -61,9 +61,9 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
 ; RV32I-SMALL-NEXT:    sw ra, 12(sp)
 ; RV32I-SMALL-NEXT:    lui a1, %hi(.Ltmp0)
 ; RV32I-SMALL-NEXT:    addi a1, a1, %lo(.Ltmp0)
+; RV32I-SMALL-NEXT:    addi a2, zero, 101
 ; RV32I-SMALL-NEXT:    sw a1, 8(sp)
-; RV32I-SMALL-NEXT:    addi a1, zero, 101
-; RV32I-SMALL-NEXT:    blt a0, a1, .LBB2_3
+; RV32I-SMALL-NEXT:    blt a0, a2, .LBB2_3
 ; RV32I-SMALL-NEXT:  # %bb.1: # %if.then
 ; RV32I-SMALL-NEXT:    lw a0, 8(sp)
 ; RV32I-SMALL-NEXT:    jr a0
@@ -86,9 +86,9 @@ define signext i32 @lower_blockaddress_displ(i32 signext %w) nounwind {
 ; RV32I-MEDIUM-NEXT:    # Label of block must be emitted
 ; RV32I-MEDIUM-NEXT:    auipc a1, %pcrel_hi(.Ltmp0)
 ; RV32I-MEDIUM-NEXT:    addi a1, a1, %pcrel_lo(.LBB2_5)
+; RV32I-MEDIUM-NEXT:    addi a2, zero, 101
 ; RV32I-MEDIUM-NEXT:    sw a1, 8(sp)
-; RV32I-MEDIUM-NEXT:    addi a1, zero, 101
-; RV32I-MEDIUM-NEXT:    blt a0, a1, .LBB2_3
+; RV32I-MEDIUM-NEXT:    blt a0, a2, .LBB2_3
 ; RV32I-MEDIUM-NEXT:  # %bb.1: # %if.then
 ; RV32I-MEDIUM-NEXT:    lw a0, 8(sp)
 ; RV32I-MEDIUM-NEXT:    jr a0
@@ -131,11 +131,11 @@ indirectgoto:
 define float @lower_constantpool(float %a) nounwind {
 ; RV32I-SMALL-LABEL: lower_constantpool:
 ; RV32I-SMALL:       # %bb.0:
-; RV32I-SMALL-NEXT:    fmv.w.x ft0, a0
-; RV32I-SMALL-NEXT:    lui a0, %hi(.LCPI3_0)
-; RV32I-SMALL-NEXT:    addi a0, a0, %lo(.LCPI3_0)
-; RV32I-SMALL-NEXT:    flw ft1, 0(a0)
-; RV32I-SMALL-NEXT:    fadd.s ft0, ft0, ft1
+; RV32I-SMALL-NEXT:    lui a1, %hi(.LCPI3_0)
+; RV32I-SMALL-NEXT:    addi a1, a1, %lo(.LCPI3_0)
+; RV32I-SMALL-NEXT:    flw ft0, 0(a1)
+; RV32I-SMALL-NEXT:    fmv.w.x ft1, a0
+; RV32I-SMALL-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32I-SMALL-NEXT:    fmv.x.w a0, ft0
 ; RV32I-SMALL-NEXT:    ret
 ;
diff --git a/llvm/test/CodeGen/RISCV/compress.ll b/llvm/test/CodeGen/RISCV/compress.ll
index 6ee215dad42f1..b58adb848fcc5 100644
--- a/llvm/test/CodeGen/RISCV/compress.ll
+++ b/llvm/test/CodeGen/RISCV/compress.ll
@@ -19,10 +19,10 @@
 
 define i32 @simple_arith(i32 %a, i32 %b) nounwind {
 ; RV32IC-LABEL: simple_arith:
-; RV32IC:         c.srai a1, 9
-; RV32IC-NEXT:    addi a2, a0, 1
+; RV32IC:         addi a2, a0, 1
 ; RV32IC-NEXT:    c.andi a2, 11
 ; RV32IC-NEXT:    c.slli a2, 7
+; RV32IC-NEXT:    c.srai a1, 9
 ; RV32IC-NEXT:    c.add a1, a2
 ; RV32IC-NEXT:    sub a0, a1, a0
 ; RV32IC-NEXT:    c.jr ra
diff --git a/llvm/test/CodeGen/RISCV/div.ll b/llvm/test/CodeGen/RISCV/div.ll
index 1fd0084f3ac3a..e3504b9278645 100644
--- a/llvm/test/CodeGen/RISCV/div.ll
+++ b/llvm/test/CodeGen/RISCV/div.ll
@@ -457,8 +457,8 @@ define i64 @sdiv64_sext_operands(i32 %a, i32 %b) nounwind {
 ;
 ; RV64IM-LABEL: sdiv64_sext_operands:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    div a0, a0, a1
 ; RV64IM-NEXT:    ret
   %1 = sext i32 %a to i64
diff --git a/llvm/test/CodeGen/RISCV/double-arith.ll b/llvm/test/CodeGen/RISCV/double-arith.ll
index 2b696210ae1df..ad68dca154dc2 100644
--- a/llvm/test/CodeGen/RISCV/double-arith.ll
+++ b/llvm/test/CodeGen/RISCV/double-arith.ll
@@ -473,13 +473,13 @@ define double @fmsub_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fmsub_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a2
-; RV64IFD-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV64IFD-NEXT:    fld ft1, 0(a2)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmv.d.x ft3, a2
+; RV64IFD-NEXT:    fadd.d ft0, ft3, ft0
 ; RV64IFD-NEXT:    fmsub.d ft0, ft2, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
@@ -496,18 +496,18 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
 ; RV32IFD-NEXT:    fld ft0, 8(sp)
-; RV32IFD-NEXT:    sw a0, 8(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    sw a4, 8(sp)
 ; RV32IFD-NEXT:    sw a5, 12(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft2, 8(sp)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI16_0)
 ; RV32IFD-NEXT:    addi a0, a0, %lo(.LCPI16_0)
 ; RV32IFD-NEXT:    fld ft3, 0(a0)
 ; RV32IFD-NEXT:    fadd.d ft2, ft2, ft3
 ; RV32IFD-NEXT:    fadd.d ft1, ft1, ft3
-; RV32IFD-NEXT:    fnmadd.d ft0, ft1, ft0, ft2
+; RV32IFD-NEXT:    fnmadd.d ft0, ft2, ft0, ft1
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -516,15 +516,15 @@ define double @fnmadd_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fnmadd_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a2
-; RV64IFD-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV64IFD-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV64IFD-NEXT:    fld ft1, 0(a2)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
-; RV64IFD-NEXT:    fmv.d.x ft2, a0
-; RV64IFD-NEXT:    fadd.d ft1, ft2, ft1
-; RV64IFD-NEXT:    fmv.d.x ft2, a1
-; RV64IFD-NEXT:    fnmadd.d ft0, ft1, ft2, ft0
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a2
+; RV64IFD-NEXT:    fmv.d.x ft3, a0
+; RV64IFD-NEXT:    fadd.d ft3, ft3, ft0
+; RV64IFD-NEXT:    fadd.d ft0, ft2, ft0
+; RV64IFD-NEXT:    fnmadd.d ft0, ft3, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %a_ = fadd double 0.0, %a
@@ -561,13 +561,13 @@ define double @fnmsub_d(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fnmsub_d:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV64IFD-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV64IFD-NEXT:    fld ft0, 0(a3)
 ; RV64IFD-NEXT:    fmv.d.x ft1, a2
 ; RV64IFD-NEXT:    fmv.d.x ft2, a1
+; RV64IFD-NEXT:    fmv.d.x ft3, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft3, ft0
 ; RV64IFD-NEXT:    fnmsub.d ft0, ft0, ft2, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
index 626a239e298a0..f80e140caece5 100644
--- a/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
+++ b/llvm/test/CodeGen/RISCV/double-bitmanip-dagcombines.ll
@@ -118,9 +118,9 @@ define double @fcopysign_fneg(double %a, double %b) nounwind {
 ;
 ; RV64I-LABEL: fcopysign_fneg:
 ; RV64I:       # %bb.0:
+; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    addi a2, zero, -1
 ; RV64I-NEXT:    slli a2, a2, 63
-; RV64I-NEXT:    not a1, a1
 ; RV64I-NEXT:    and a1, a1, a2
 ; RV64I-NEXT:    addi a2, a2, -1
 ; RV64I-NEXT:    and a0, a0, a2
diff --git a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
index 1c8b867bd41c9..356f632a3f341 100644
--- a/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-br-fcmp.ll
@@ -385,11 +385,11 @@ define void @br_fcmp_ord(double %a, double %b) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
 ; RV64IFD-NEXT:    ld ra, 8(sp)
@@ -712,11 +712,11 @@ define void @br_fcmp_uno(double %a, double %b) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    seqz a0, a0
 ; RV64IFD-NEXT:    bnez a0, .LBB15_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
diff --git a/llvm/test/CodeGen/RISCV/double-calling-conv.ll b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
index f3cb0feb5f449..837cb819ec9b8 100644
--- a/llvm/test/CodeGen/RISCV/double-calling-conv.ll
+++ b/llvm/test/CodeGen/RISCV/double-calling-conv.ll
@@ -76,16 +76,16 @@ define double @caller_double_split_reg_stack() nounwind {
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp)
 ; RV32IFD-NEXT:    lui a0, 262510
-; RV32IFD-NEXT:    addi a0, a0, 327
-; RV32IFD-NEXT:    sw a0, 0(sp)
+; RV32IFD-NEXT:    addi a2, a0, 327
 ; RV32IFD-NEXT:    lui a0, 262446
 ; RV32IFD-NEXT:    addi a6, a0, 327
 ; RV32IFD-NEXT:    lui a0, 713032
 ; RV32IFD-NEXT:    addi a5, a0, -1311
 ; RV32IFD-NEXT:    addi a0, zero, 1
 ; RV32IFD-NEXT:    addi a1, zero, 2
-; RV32IFD-NEXT:    mv a2, zero
 ; RV32IFD-NEXT:    addi a3, zero, 3
+; RV32IFD-NEXT:    sw a2, 0(sp)
+; RV32IFD-NEXT:    mv a2, zero
 ; RV32IFD-NEXT:    mv a4, zero
 ; RV32IFD-NEXT:    mv a7, a5
 ; RV32IFD-NEXT:    call callee_double_split_reg_stack
@@ -120,20 +120,20 @@ define double @caller_double_stack() nounwind {
 ; RV32IFD-NEXT:    lui a0, 262510
 ; RV32IFD-NEXT:    addi a0, a0, 327
 ; RV32IFD-NEXT:    sw a0, 4(sp)
+; RV32IFD-NEXT:    lui a0, 713032
+; RV32IFD-NEXT:    addi a1, a0, -1311
+; RV32IFD-NEXT:    sw a1, 0(sp)
 ; RV32IFD-NEXT:    lui a0, 262574
 ; RV32IFD-NEXT:    addi a0, a0, 327
 ; RV32IFD-NEXT:    sw a0, 12(sp)
-; RV32IFD-NEXT:    lui a0, 713032
-; RV32IFD-NEXT:    addi a0, a0, -1311
-; RV32IFD-NEXT:    sw a0, 0(sp)
-; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    addi a0, zero, 1
-; RV32IFD-NEXT:    mv a1, zero
 ; RV32IFD-NEXT:    addi a2, zero, 2
-; RV32IFD-NEXT:    mv a3, zero
 ; RV32IFD-NEXT:    addi a4, zero, 3
-; RV32IFD-NEXT:    mv a5, zero
 ; RV32IFD-NEXT:    addi a6, zero, 4
+; RV32IFD-NEXT:    sw a1, 8(sp)
+; RV32IFD-NEXT:    mv a1, zero
+; RV32IFD-NEXT:    mv a3, zero
+; RV32IFD-NEXT:    mv a5, zero
 ; RV32IFD-NEXT:    mv a7, zero
 ; RV32IFD-NEXT:    call callee_double_stack
 ; RV32IFD-NEXT:    lw ra, 28(sp)
diff --git a/llvm/test/CodeGen/RISCV/double-convert.ll b/llvm/test/CodeGen/RISCV/double-convert.ll
index 1d30fdadf4830..f7abdea68ca29 100644
--- a/llvm/test/CodeGen/RISCV/double-convert.ll
+++ b/llvm/test/CodeGen/RISCV/double-convert.ll
@@ -257,9 +257,9 @@ define double @fmv_d_x(i64 %a, i64 %b) nounwind {
 ;
 ; RV64IFD-LABEL: fmv_d_x:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = bitcast i64 %a to double
diff --git a/llvm/test/CodeGen/RISCV/double-fcmp.ll b/llvm/test/CodeGen/RISCV/double-fcmp.ll
index 0046b59d49635..28875226abb4b 100644
--- a/llvm/test/CodeGen/RISCV/double-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-fcmp.ll
@@ -197,11 +197,11 @@ define i32 @fcmp_ord(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: fcmp_ord:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    ret
   %1 = fcmp ord double %a, %b
   %2 = zext i1 %1 to i32
@@ -397,11 +397,11 @@ define i32 @fcmp_uno(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: fcmp_uno:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    feq.d a0, ft0, ft0
-; RV64IFD-NEXT:    and a0, a0, a1
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    feq.d a0, ft1, ft1
+; RV64IFD-NEXT:    feq.d a1, ft0, ft0
+; RV64IFD-NEXT:    and a0, a1, a0
 ; RV64IFD-NEXT:    seqz a0, a0
 ; RV64IFD-NEXT:    ret
   %1 = fcmp uno double %a, %b
diff --git a/llvm/test/CodeGen/RISCV/double-imm.ll b/llvm/test/CodeGen/RISCV/double-imm.ll
index 7f835644066ba..3cd397969a53f 100644
--- a/llvm/test/CodeGen/RISCV/double-imm.ll
+++ b/llvm/test/CodeGen/RISCV/double-imm.ll
@@ -49,11 +49,11 @@ define double @double_imm_op(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: double_imm_op:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IFD-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV64IFD-NEXT:    fld ft0, 0(a1)
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = fadd double %a, 1.0
diff --git a/llvm/test/CodeGen/RISCV/double-intrinsics.ll b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
index 29327452ff125..b8873883cd586 100644
--- a/llvm/test/CodeGen/RISCV/double-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/double-intrinsics.ll
@@ -111,28 +111,24 @@ define double @sincos_f64(double %a) nounwind {
 ; RV32IFD-NEXT:    sw ra, 28(sp)
 ; RV32IFD-NEXT:    sw s0, 24(sp)
 ; RV32IFD-NEXT:    sw s1, 20(sp)
-; RV32IFD-NEXT:    sw s2, 16(sp)
-; RV32IFD-NEXT:    sw s3, 12(sp)
 ; RV32IFD-NEXT:    mv s0, a1
 ; RV32IFD-NEXT:    mv s1, a0
 ; RV32IFD-NEXT:    call sin
-; RV32IFD-NEXT:    mv s2, a0
-; RV32IFD-NEXT:    mv s3, a1
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    fsd ft0, 0(sp)
 ; RV32IFD-NEXT:    mv a0, s1
 ; RV32IFD-NEXT:    mv a1, s0
 ; RV32IFD-NEXT:    call cos
-; RV32IFD-NEXT:    sw a0, 0(sp)
-; RV32IFD-NEXT:    sw a1, 4(sp)
-; RV32IFD-NEXT:    fld ft0, 0(sp)
-; RV32IFD-NEXT:    sw s2, 0(sp)
-; RV32IFD-NEXT:    sw s3, 4(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    fld ft1, 0(sp)
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV32IFD-NEXT:    fsd ft0, 0(sp)
-; RV32IFD-NEXT:    lw a0, 0(sp)
-; RV32IFD-NEXT:    lw a1, 4(sp)
-; RV32IFD-NEXT:    lw s3, 12(sp)
-; RV32IFD-NEXT:    lw s2, 16(sp)
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    lw s1, 20(sp)
 ; RV32IFD-NEXT:    lw s0, 24(sp)
 ; RV32IFD-NEXT:    lw ra, 28(sp)
@@ -144,17 +140,16 @@ define double @sincos_f64(double %a) nounwind {
 ; RV64IFD-NEXT:    addi sp, sp, -32
 ; RV64IFD-NEXT:    sd ra, 24(sp)
 ; RV64IFD-NEXT:    sd s0, 16(sp)
-; RV64IFD-NEXT:    sd s1, 8(sp)
 ; RV64IFD-NEXT:    mv s0, a0
 ; RV64IFD-NEXT:    call sin
-; RV64IFD-NEXT:    mv s1, a0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsd ft0, 8(sp)
 ; RV64IFD-NEXT:    mv a0, s0
 ; RV64IFD-NEXT:    call cos
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    fmv.d.x ft1, s1
+; RV64IFD-NEXT:    fld ft1, 8(sp)
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld s1, 8(sp)
 ; RV64IFD-NEXT:    ld s0, 16(sp)
 ; RV64IFD-NEXT:    ld ra, 24(sp)
 ; RV64IFD-NEXT:    addi sp, sp, 32
@@ -350,17 +345,17 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ; RV32IFD-LABEL: fmuladd_f64:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
+; RV32IFD-NEXT:    sw a4, 8(sp)
+; RV32IFD-NEXT:    sw a5, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fmul.d ft0, ft1, ft0
-; RV32IFD-NEXT:    sw a4, 8(sp)
-; RV32IFD-NEXT:    sw a5, 12(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV32IFD-NEXT:    fld ft2, 8(sp)
+; RV32IFD-NEXT:    fmul.d ft1, ft2, ft1
+; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -369,11 +364,11 @@ define double @fmuladd_f64(double %a, double %b, double %c) nounwind {
 ;
 ; RV64IFD-LABEL: fmuladd_f64:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a1
-; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    fmul.d ft0, ft1, ft0
-; RV64IFD-NEXT:    fmv.d.x ft1, a2
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV64IFD-NEXT:    fmv.d.x ft0, a2
+; RV64IFD-NEXT:    fmv.d.x ft1, a1
+; RV64IFD-NEXT:    fmv.d.x ft2, a0
+; RV64IFD-NEXT:    fmul.d ft1, ft2, ft1
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = call double @llvm.fmuladd.f64(double %a, double %b, double %c)
diff --git a/llvm/test/CodeGen/RISCV/double-mem.ll b/llvm/test/CodeGen/RISCV/double-mem.ll
index 76be198ff4d08..ad6a3afc1e626 100644
--- a/llvm/test/CodeGen/RISCV/double-mem.ll
+++ b/llvm/test/CodeGen/RISCV/double-mem.ll
@@ -8,9 +8,9 @@ define double @fld(double *%a) nounwind {
 ; RV32IFD-LABEL: fld:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    fld ft0, 24(a0)
-; RV32IFD-NEXT:    fld ft1, 0(a0)
-; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    fld ft0, 0(a0)
+; RV32IFD-NEXT:    fld ft1, 24(a0)
+; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
@@ -19,9 +19,9 @@ define double @fld(double *%a) nounwind {
 ;
 ; RV64IFD-LABEL: fld:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fld ft0, 24(a0)
-; RV64IFD-NEXT:    fld ft1, 0(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fld ft0, 0(a0)
+; RV64IFD-NEXT:    fld ft1, 24(a0)
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
 ; RV64IFD-NEXT:    ret
   %1 = load double, double* %a
@@ -44,8 +44,8 @@ define void @fsd(double *%a, double %b, double %c) nounwind {
 ; RV32IFD-NEXT:    sw a2, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
 ; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV32IFD-NEXT:    fsd ft0, 64(a0)
 ; RV32IFD-NEXT:    fsd ft0, 0(a0)
+; RV32IFD-NEXT:    fsd ft0, 64(a0)
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -54,8 +54,8 @@ define void @fsd(double *%a, double %b, double %c) nounwind {
 ; RV64IFD-NEXT:    fmv.d.x ft0, a2
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
 ; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
-; RV64IFD-NEXT:    fsd ft0, 64(a0)
 ; RV64IFD-NEXT:    fsd ft0, 0(a0)
+; RV64IFD-NEXT:    fsd ft0, 64(a0)
 ; RV64IFD-NEXT:    ret
 ; Use %b and %c in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
@@ -100,10 +100,10 @@ define double @fld_fsd_global(double %a, double %b) nounwind {
 ; RV64IFD-NEXT:    lui a0, %hi(G)
 ; RV64IFD-NEXT:    fld ft1, %lo(G)(a0)
 ; RV64IFD-NEXT:    fsd ft0, %lo(G)(a0)
-; RV64IFD-NEXT:    addi a0, a0, %lo(G)
-; RV64IFD-NEXT:    fld ft1, 72(a0)
-; RV64IFD-NEXT:    fsd ft0, 72(a0)
+; RV64IFD-NEXT:    addi a1, a0, %lo(G)
+; RV64IFD-NEXT:    fld ft1, 72(a1)
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    fsd ft0, 72(a1)
 ; RV64IFD-NEXT:    ret
 ; Use %a and %b in an FP op to ensure floating point registers are used, even
 ; for the soft float ABI
@@ -136,14 +136,14 @@ define double @fld_fsd_constant(double %a) nounwind {
 ;
 ; RV64IFD-LABEL: fld_fsd_constant:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    lui a0, 56
-; RV64IFD-NEXT:    addiw a0, a0, -1353
-; RV64IFD-NEXT:    slli a0, a0, 14
-; RV64IFD-NEXT:    fld ft1, -273(a0)
-; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
-; RV64IFD-NEXT:    fsd ft0, -273(a0)
+; RV64IFD-NEXT:    lui a1, 56
+; RV64IFD-NEXT:    addiw a1, a1, -1353
+; RV64IFD-NEXT:    slli a1, a1, 14
+; RV64IFD-NEXT:    fld ft0, -273(a1)
+; RV64IFD-NEXT:    fmv.d.x ft1, a0
+; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
+; RV64IFD-NEXT:    fsd ft0, -273(a1)
 ; RV64IFD-NEXT:    ret
   %1 = inttoptr i32 3735928559 to double*
   %2 = load volatile double, double* %1
@@ -159,22 +159,18 @@ define double @fld_stack(double %a) nounwind {
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -32
 ; RV32IFD-NEXT:    sw ra, 28(sp)
-; RV32IFD-NEXT:    sw s0, 24(sp)
-; RV32IFD-NEXT:    sw s1, 20(sp)
-; RV32IFD-NEXT:    mv s0, a1
-; RV32IFD-NEXT:    mv s1, a0
-; RV32IFD-NEXT:    addi a0, sp, 8
-; RV32IFD-NEXT:    call notdead
-; RV32IFD-NEXT:    sw s1, 0(sp)
-; RV32IFD-NEXT:    sw s0, 4(sp)
-; RV32IFD-NEXT:    fld ft0, 0(sp)
-; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    fsd ft0, 0(sp)
-; RV32IFD-NEXT:    lw a0, 0(sp)
-; RV32IFD-NEXT:    lw a1, 4(sp)
-; RV32IFD-NEXT:    lw s1, 20(sp)
-; RV32IFD-NEXT:    lw s0, 24(sp)
+; RV32IFD-NEXT:    addi a0, sp, 16
+; RV32IFD-NEXT:    call notdead
+; RV32IFD-NEXT:    fld ft0, 16(sp)
+; RV32IFD-NEXT:    fld ft1, 0(sp)
+; RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
+; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    lw a0, 8(sp)
+; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    lw ra, 28(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 32
 ; RV32IFD-NEXT:    ret
@@ -183,15 +179,14 @@ define double @fld_stack(double %a) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    addi sp, sp, -32
 ; RV64IFD-NEXT:    sd ra, 24(sp)
-; RV64IFD-NEXT:    sd s0, 16(sp)
-; RV64IFD-NEXT:    mv s0, a0
-; RV64IFD-NEXT:    addi a0, sp, 8
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    fsd ft0, 8(sp)
+; RV64IFD-NEXT:    addi a0, sp, 16
 ; RV64IFD-NEXT:    call notdead
-; RV64IFD-NEXT:    fmv.d.x ft0, s0
+; RV64IFD-NEXT:    fld ft0, 16(sp)
 ; RV64IFD-NEXT:    fld ft1, 8(sp)
-; RV64IFD-NEXT:    fadd.d ft0, ft1, ft0
+; RV64IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
-; RV64IFD-NEXT:    ld s0, 16(sp)
 ; RV64IFD-NEXT:    ld ra, 24(sp)
 ; RV64IFD-NEXT:    addi sp, sp, 32
 ; RV64IFD-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
index 168c6c2e67d10..eb58ba4c69a7a 100644
--- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll
+++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
@@ -17,8 +17,8 @@ define i32 @main() nounwind {
 ; RV32IFD:       # %bb.0: # %entry
 ; RV32IFD-NEXT:    addi sp, sp, -16
 ; RV32IFD-NEXT:    sw ra, 12(sp)
-; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    lui a1, 262144
+; RV32IFD-NEXT:    mv a0, zero
 ; RV32IFD-NEXT:    call test
 ; RV32IFD-NEXT:    sw a0, 0(sp)
 ; RV32IFD-NEXT:    sw a1, 4(sp)
diff --git a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
index 321adce3d7fb9..12789c05335ad 100644
--- a/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/double-select-fcmp.ll
@@ -298,23 +298,23 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ; RV32IFD-LABEL: select_fcmp_ueq:
 ; RV32IFD:       # %bb.0:
 ; RV32IFD-NEXT:    addi sp, sp, -16
-; RV32IFD-NEXT:    sw a0, 8(sp)
-; RV32IFD-NEXT:    sw a1, 12(sp)
-; RV32IFD-NEXT:    fld ft0, 8(sp)
 ; RV32IFD-NEXT:    sw a2, 8(sp)
 ; RV32IFD-NEXT:    sw a3, 12(sp)
+; RV32IFD-NEXT:    fld ft0, 8(sp)
+; RV32IFD-NEXT:    sw a0, 8(sp)
+; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    feq.d a0, ft1, ft1
+; RV32IFD-NEXT:    feq.d a0, ft1, ft0
 ; RV32IFD-NEXT:    feq.d a1, ft0, ft0
-; RV32IFD-NEXT:    and a0, a1, a0
-; RV32IFD-NEXT:    seqz a0, a0
-; RV32IFD-NEXT:    feq.d a1, ft0, ft1
-; RV32IFD-NEXT:    or a0, a1, a0
+; RV32IFD-NEXT:    feq.d a2, ft1, ft1
+; RV32IFD-NEXT:    and a1, a2, a1
+; RV32IFD-NEXT:    seqz a1, a1
+; RV32IFD-NEXT:    or a0, a0, a1
 ; RV32IFD-NEXT:    bnez a0, .LBB8_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    fmv.d ft0, ft1
+; RV32IFD-NEXT:    fmv.d ft1, ft0
 ; RV32IFD-NEXT:  .LBB8_2:
-; RV32IFD-NEXT:    fsd ft0, 8(sp)
+; RV32IFD-NEXT:    fsd ft1, 8(sp)
 ; RV32IFD-NEXT:    lw a0, 8(sp)
 ; RV32IFD-NEXT:    lw a1, 12(sp)
 ; RV32IFD-NEXT:    addi sp, sp, 16
@@ -322,14 +322,14 @@ define double @select_fcmp_ueq(double %a, double %b) nounwind {
 ;
 ; RV64IFD-LABEL: select_fcmp_ueq:
 ; RV64IFD:       # %bb.0:
-; RV64IFD-NEXT:    fmv.d.x ft0, a0
 ; RV64IFD-NEXT:    fmv.d.x ft1, a1
-; RV64IFD-NEXT:    feq.d a0, ft1, ft1
-; RV64IFD-NEXT:    feq.d a1, ft0, ft0
-; RV64IFD-NEXT:    and a0, a1, a0
-; RV64IFD-NEXT:    seqz a0, a0
-; RV64IFD-NEXT:    feq.d a1, ft0, ft1
-; RV64IFD-NEXT:    or a0, a1, a0
+; RV64IFD-NEXT:    fmv.d.x ft0, a0
+; RV64IFD-NEXT:    feq.d a0, ft0, ft1
+; RV64IFD-NEXT:    feq.d a1, ft1, ft1
+; RV64IFD-NEXT:    feq.d a2, ft0, ft0
+; RV64IFD-NEXT:    and a1, a2, a1
+; RV64IFD-NEXT:    seqz a1, a1
+; RV64IFD-NEXT:    or a0, a0, a1
 ; RV64IFD-NEXT:    bnez a0, .LBB8_2
 ; RV64IFD-NEXT:  # %bb.1:
 ; RV64IFD-NEXT:    fmv.d ft0, ft1
@@ -604,12 +604,12 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; RV32IFD-NEXT:    sw a0, 8(sp)
 ; RV32IFD-NEXT:    sw a1, 12(sp)
 ; RV32IFD-NEXT:    fld ft1, 8(sp)
-; RV32IFD-NEXT:    feq.d a0, ft1, ft0
-; RV32IFD-NEXT:    bnez a0, .LBB16_2
+; RV32IFD-NEXT:    feq.d a1, ft1, ft0
+; RV32IFD-NEXT:    mv a0, a4
+; RV32IFD-NEXT:    bnez a1, .LBB16_2
 ; RV32IFD-NEXT:  # %bb.1:
-; RV32IFD-NEXT:    mv a4, a5
+; RV32IFD-NEXT:    mv a0, a5
 ; RV32IFD-NEXT:  .LBB16_2:
-; RV32IFD-NEXT:    mv a0, a4
 ; RV32IFD-NEXT:    addi sp, sp, 16
 ; RV32IFD-NEXT:    ret
 ;
@@ -617,12 +617,12 @@ define i32 @i32_select_fcmp_oeq(double %a, double %b, i32 %c, i32 %d) nounwind {
 ; RV64IFD:       # %bb.0:
 ; RV64IFD-NEXT:    fmv.d.x ft0, a1
 ; RV64IFD-NEXT:    fmv.d.x ft1, a0
-; RV64IFD-NEXT:    feq.d a0, ft1, ft0
-; RV64IFD-NEXT:    bnez a0, .LBB16_2
+; RV64IFD-NEXT:    feq.d a1, ft1, ft0
+; RV64IFD-NEXT:    mv a0, a2
+; RV64IFD-NEXT:    bnez a1, .LBB16_2
 ; RV64IFD-NEXT:  # %bb.1:
-; RV64IFD-NEXT:    mv a2, a3
+; RV64IFD-NEXT:    mv a0, a3
 ; RV64IFD-NEXT:  .LBB16_2:
-; RV64IFD-NEXT:    mv a0, a2
 ; RV64IFD-NEXT:    ret
   %1 = fcmp oeq double %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
diff --git a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
index 28a9e12dadd0e..a595cd8d9e9ea 100644
--- a/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
+++ b/llvm/test/CodeGen/RISCV/double-stack-spill-restore.ll
@@ -37,10 +37,10 @@ define double @func(double %d, i32 %n) nounwind {
 ; RV64IFD:       # %bb.0: # %entry
 ; RV64IFD-NEXT:    addi sp, sp, -16
 ; RV64IFD-NEXT:    sd ra, 8(sp)
+; RV64IFD-NEXT:    slli a2, a1, 32
+; RV64IFD-NEXT:    srli a2, a2, 32
 ; RV64IFD-NEXT:    fmv.d.x ft0, a0
-; RV64IFD-NEXT:    slli a0, a1, 32
-; RV64IFD-NEXT:    srli a0, a0, 32
-; RV64IFD-NEXT:    beqz a0, .LBB0_2
+; RV64IFD-NEXT:    beqz a2, .LBB0_2
 ; RV64IFD-NEXT:  # %bb.1: # %if.else
 ; RV64IFD-NEXT:    addi a1, a1, -1
 ; RV64IFD-NEXT:    fmv.x.d a0, ft0
diff --git a/llvm/test/CodeGen/RISCV/float-arith.ll b/llvm/test/CodeGen/RISCV/float-arith.ll
index a668b7e494274..5244a69a6fad0 100644
--- a/llvm/test/CodeGen/RISCV/float-arith.ll
+++ b/llvm/test/CodeGen/RISCV/float-arith.ll
@@ -339,26 +339,26 @@ define float @fmadd_s(float %a, float %b, float %c) nounwind {
 define float @fmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fmsub_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a2
-; RV32IF-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV32IF-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV32IF-NEXT:    flw ft1, 0(a2)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fmv.w.x ft2, a0
+; RV32IF-NEXT:    fmv.w.x ft3, a2
+; RV32IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV32IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmsub_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a2
-; RV64IF-NEXT:    lui a2, %hi(.LCPI15_0)
-; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI15_0)
-; RV64IF-NEXT:    flw ft1, 0(a2)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a3, %hi(.LCPI15_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI15_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmv.w.x ft3, a2
+; RV64IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV64IF-NEXT:    fmsub.s ft0, ft2, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
@@ -371,29 +371,29 @@ define float @fmsub_s(float %a, float %b, float %c) nounwind {
 define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fnmadd_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a2
-; RV32IF-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV32IF-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV32IF-NEXT:    flw ft1, 0(a2)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fmv.w.x ft2, a0
-; RV32IF-NEXT:    fadd.s ft1, ft2, ft1
-; RV32IF-NEXT:    fmv.w.x ft2, a1
-; RV32IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
+; RV32IF-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fmv.w.x ft2, a2
+; RV32IF-NEXT:    fmv.w.x ft3, a0
+; RV32IF-NEXT:    fadd.s ft3, ft3, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft2, ft0
+; RV32IF-NEXT:    fnmadd.s ft0, ft3, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fnmadd_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a2
-; RV64IF-NEXT:    lui a2, %hi(.LCPI16_0)
-; RV64IF-NEXT:    addi a2, a2, %lo(.LCPI16_0)
-; RV64IF-NEXT:    flw ft1, 0(a2)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV64IF-NEXT:    fmv.w.x ft2, a0
-; RV64IF-NEXT:    fadd.s ft1, ft2, ft1
-; RV64IF-NEXT:    fmv.w.x ft2, a1
-; RV64IF-NEXT:    fnmadd.s ft0, ft1, ft2, ft0
+; RV64IF-NEXT:    lui a3, %hi(.LCPI16_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI16_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a2
+; RV64IF-NEXT:    fmv.w.x ft3, a0
+; RV64IF-NEXT:    fadd.s ft3, ft3, ft0
+; RV64IF-NEXT:    fadd.s ft0, ft2, ft0
+; RV64IF-NEXT:    fnmadd.s ft0, ft3, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %a_ = fadd float 0.0, %a
@@ -407,26 +407,26 @@ define float @fnmadd_s(float %a, float %b, float %c) nounwind {
 define float @fnmsub_s(float %a, float %b, float %c) nounwind {
 ; RV32IF-LABEL: fnmsub_s:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV32IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV32IF-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV32IF-NEXT:    flw ft0, 0(a3)
 ; RV32IF-NEXT:    fmv.w.x ft1, a2
 ; RV32IF-NEXT:    fmv.w.x ft2, a1
+; RV32IF-NEXT:    fmv.w.x ft3, a0
+; RV32IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV32IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fnmsub_s:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, %hi(.LCPI17_0)
-; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI17_0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a3, %hi(.LCPI17_0)
+; RV64IF-NEXT:    addi a3, a3, %lo(.LCPI17_0)
+; RV64IF-NEXT:    flw ft0, 0(a3)
 ; RV64IF-NEXT:    fmv.w.x ft1, a2
 ; RV64IF-NEXT:    fmv.w.x ft2, a1
+; RV64IF-NEXT:    fmv.w.x ft3, a0
+; RV64IF-NEXT:    fadd.s ft0, ft3, ft0
 ; RV64IF-NEXT:    fnmsub.s ft0, ft0, ft2, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
index 425da7383162f..a1e5b32eeaffc 100644
--- a/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-br-fcmp.ll
@@ -337,11 +337,11 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
 ; RV32IF-NEXT:    lw ra, 12(sp)
@@ -354,11 +354,11 @@ define void @br_fcmp_ord(float %a, float %b) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
 ; RV64IF-NEXT:    ld ra, 8(sp)
@@ -635,11 +635,11 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    bnez a0, .LBB15_2
 ; RV32IF-NEXT:  # %bb.1: # %if.else
@@ -653,11 +653,11 @@ define void @br_fcmp_uno(float %a, float %b) nounwind {
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    addi sp, sp, -16
 ; RV64IF-NEXT:    sd ra, 8(sp)
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    seqz a0, a0
 ; RV64IF-NEXT:    bnez a0, .LBB15_2
 ; RV64IF-NEXT:  # %bb.1: # %if.else
diff --git a/llvm/test/CodeGen/RISCV/float-convert.ll b/llvm/test/CodeGen/RISCV/float-convert.ll
index 296e9d468fe7e..35a71c4ee591a 100644
--- a/llvm/test/CodeGen/RISCV/float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/float-convert.ll
@@ -97,17 +97,17 @@ define float @fcvt_s_wu(i32 %a) nounwind {
 define float @fmv_w_x(i32 %a, i32 %b) nounwind {
 ; RV32IF-LABEL: fmv_w_x:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmv_w_x:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
 ; Ensure fmv.w.x is generated even for a soft float calling convention
diff --git a/llvm/test/CodeGen/RISCV/float-fcmp.ll b/llvm/test/CodeGen/RISCV/float-fcmp.ll
index 43a849bee3623..3a44020a2fd32 100644
--- a/llvm/test/CodeGen/RISCV/float-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-fcmp.ll
@@ -146,20 +146,20 @@ define i32 @fcmp_one(float %a, float %b) nounwind {
 define i32 @fcmp_ord(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fcmp_ord:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcmp_ord:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    ret
   %1 = fcmp ord float %a, %b
   %2 = zext i1 %1 to i32
@@ -303,21 +303,21 @@ define i32 @fcmp_une(float %a, float %b) nounwind {
 define i32 @fcmp_uno(float %a, float %b) nounwind {
 ; RV32IF-LABEL: fcmp_uno:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    feq.s a0, ft0, ft0
-; RV32IF-NEXT:    and a0, a0, a1
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    feq.s a0, ft1, ft1
+; RV32IF-NEXT:    feq.s a1, ft0, ft0
+; RV32IF-NEXT:    and a0, a1, a0
 ; RV32IF-NEXT:    seqz a0, a0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fcmp_uno:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    feq.s a0, ft0, ft0
-; RV64IF-NEXT:    and a0, a0, a1
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    feq.s a0, ft1, ft1
+; RV64IF-NEXT:    feq.s a1, ft0, ft0
+; RV64IF-NEXT:    and a0, a1, a0
 ; RV64IF-NEXT:    seqz a0, a0
 ; RV64IF-NEXT:    ret
   %1 = fcmp uno float %a, %b
diff --git a/llvm/test/CodeGen/RISCV/float-imm.ll b/llvm/test/CodeGen/RISCV/float-imm.ll
index 09d9c3c933a12..e0ec464ff03e7 100644
--- a/llvm/test/CodeGen/RISCV/float-imm.ll
+++ b/llvm/test/CodeGen/RISCV/float-imm.ll
@@ -26,21 +26,21 @@ define float @float_imm_op(float %a) nounwind {
 ; TODO: addi should be folded in to the flw
 ; RV32IF-LABEL: float_imm_op:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV32IF-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV32IF-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV32IF-NEXT:    flw ft0, 0(a1)
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: float_imm_op:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, %hi(.LCPI1_0)
-; RV64IF-NEXT:    addi a0, a0, %lo(.LCPI1_0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    lui a1, %hi(.LCPI1_0)
+; RV64IF-NEXT:    addi a1, a1, %lo(.LCPI1_0)
+; RV64IF-NEXT:    flw ft0, 0(a1)
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, 1.0
diff --git a/llvm/test/CodeGen/RISCV/float-intrinsics.ll b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
index 7ac2df6e89496..68bb95b634877 100644
--- a/llvm/test/CodeGen/RISCV/float-intrinsics.ll
+++ b/llvm/test/CodeGen/RISCV/float-intrinsics.ll
@@ -108,17 +108,16 @@ define float @sincos_f32(float %a) nounwind {
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
 ; RV32IF-NEXT:    sw s0, 8(sp)
-; RV32IF-NEXT:    sw s1, 4(sp)
 ; RV32IF-NEXT:    mv s0, a0
 ; RV32IF-NEXT:    call sinf
-; RV32IF-NEXT:    mv s1, a0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fsw ft0, 4(sp)
 ; RV32IF-NEXT:    mv a0, s0
 ; RV32IF-NEXT:    call cosf
 ; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    fmv.w.x ft1, s1
+; RV32IF-NEXT:    flw ft1, 4(sp)
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw s1, 4(sp)
 ; RV32IF-NEXT:    lw s0, 8(sp)
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
@@ -129,17 +128,16 @@ define float @sincos_f32(float %a) nounwind {
 ; RV64IF-NEXT:    addi sp, sp, -32
 ; RV64IF-NEXT:    sd ra, 24(sp)
 ; RV64IF-NEXT:    sd s0, 16(sp)
-; RV64IF-NEXT:    sd s1, 8(sp)
 ; RV64IF-NEXT:    mv s0, a0
 ; RV64IF-NEXT:    call sinf
-; RV64IF-NEXT:    mv s1, a0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsw ft0, 12(sp)
 ; RV64IF-NEXT:    mv a0, s0
 ; RV64IF-NEXT:    call cosf
 ; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    fmv.w.x ft1, s1
+; RV64IF-NEXT:    flw ft1, 12(sp)
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld s1, 8(sp)
 ; RV64IF-NEXT:    ld s0, 16(sp)
 ; RV64IF-NEXT:    ld ra, 24(sp)
 ; RV64IF-NEXT:    addi sp, sp, 32
@@ -324,21 +322,21 @@ define float @fmuladd_f32(float %a, float %b, float %c) nounwind {
 ; Use of fmadd depends on TargetLowering::isFMAFasterthanFMulAndFAdd
 ; RV32IF-LABEL: fmuladd_f32:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a1
-; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    fmul.s ft0, ft1, ft0
-; RV32IF-NEXT:    fmv.w.x ft1, a2
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV32IF-NEXT:    fmv.w.x ft0, a2
+; RV32IF-NEXT:    fmv.w.x ft1, a1
+; RV32IF-NEXT:    fmv.w.x ft2, a0
+; RV32IF-NEXT:    fmul.s ft1, ft2, ft1
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fmuladd_f32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fmul.s ft0, ft1, ft0
-; RV64IF-NEXT:    fmv.w.x ft1, a2
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
+; RV64IF-NEXT:    fmv.w.x ft0, a2
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fmv.w.x ft2, a0
+; RV64IF-NEXT:    fmul.s ft1, ft2, ft1
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
diff --git a/llvm/test/CodeGen/RISCV/float-mem.ll b/llvm/test/CodeGen/RISCV/float-mem.ll
index 2d1a7a7f96594..c989235e276fa 100644
--- a/llvm/test/CodeGen/RISCV/float-mem.ll
+++ b/llvm/test/CodeGen/RISCV/float-mem.ll
@@ -7,17 +7,17 @@
 define float @flw(float *%a) nounwind {
 ; RV32IF-LABEL: flw:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    flw ft0, 12(a0)
-; RV32IF-NEXT:    flw ft1, 0(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    flw ft0, 0(a0)
+; RV32IF-NEXT:    flw ft1, 12(a0)
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    flw ft0, 12(a0)
-; RV64IF-NEXT:    flw ft1, 0(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    flw ft0, 0(a0)
+; RV64IF-NEXT:    flw ft1, 12(a0)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = load float, float* %a
@@ -37,8 +37,8 @@ define void @fsw(float *%a, float %b, float %c) nounwind {
 ; RV32IF-NEXT:    fmv.w.x ft0, a2
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
 ; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
-; RV32IF-NEXT:    fsw ft0, 32(a0)
 ; RV32IF-NEXT:    fsw ft0, 0(a0)
+; RV32IF-NEXT:    fsw ft0, 32(a0)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: fsw:
@@ -46,8 +46,8 @@ define void @fsw(float *%a, float %b, float %c) nounwind {
 ; RV64IF-NEXT:    fmv.w.x ft0, a2
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
 ; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
-; RV64IF-NEXT:    fsw ft0, 32(a0)
 ; RV64IF-NEXT:    fsw ft0, 0(a0)
+; RV64IF-NEXT:    fsw ft0, 32(a0)
 ; RV64IF-NEXT:    ret
   %1 = fadd float %b, %c
   store float %1, float* %a
@@ -70,10 +70,10 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 ; RV32IF-NEXT:    lui a0, %hi(G)
 ; RV32IF-NEXT:    flw ft1, %lo(G)(a0)
 ; RV32IF-NEXT:    fsw ft0, %lo(G)(a0)
-; RV32IF-NEXT:    addi a0, a0, %lo(G)
-; RV32IF-NEXT:    flw ft1, 36(a0)
-; RV32IF-NEXT:    fsw ft0, 36(a0)
+; RV32IF-NEXT:    addi a1, a0, %lo(G)
+; RV32IF-NEXT:    flw ft1, 36(a1)
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
+; RV32IF-NEXT:    fsw ft0, 36(a1)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_fsw_global:
@@ -84,10 +84,10 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 ; RV64IF-NEXT:    lui a0, %hi(G)
 ; RV64IF-NEXT:    flw ft1, %lo(G)(a0)
 ; RV64IF-NEXT:    fsw ft0, %lo(G)(a0)
-; RV64IF-NEXT:    addi a0, a0, %lo(G)
-; RV64IF-NEXT:    flw ft1, 36(a0)
-; RV64IF-NEXT:    fsw ft0, 36(a0)
+; RV64IF-NEXT:    addi a1, a0, %lo(G)
+; RV64IF-NEXT:    flw ft1, 36(a1)
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    fsw ft0, 36(a1)
 ; RV64IF-NEXT:    ret
   %1 = fadd float %a, %b
   %2 = load volatile float, float* @G
@@ -102,24 +102,24 @@ define float @flw_fsw_global(float %a, float %b) nounwind {
 define float @flw_fsw_constant(float %a) nounwind {
 ; RV32IF-LABEL: flw_fsw_constant:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
-; RV32IF-NEXT:    lui a0, 912092
-; RV32IF-NEXT:    flw ft1, -273(a0)
-; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV32IF-NEXT:    fsw ft0, -273(a0)
+; RV32IF-NEXT:    lui a1, 912092
+; RV32IF-NEXT:    flw ft0, -273(a1)
+; RV32IF-NEXT:    fmv.w.x ft1, a0
+; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
+; RV32IF-NEXT:    fsw ft0, -273(a1)
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_fsw_constant:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
-; RV64IF-NEXT:    lui a0, 56
-; RV64IF-NEXT:    addiw a0, a0, -1353
-; RV64IF-NEXT:    slli a0, a0, 14
-; RV64IF-NEXT:    flw ft1, -273(a0)
-; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
-; RV64IF-NEXT:    fsw ft0, -273(a0)
+; RV64IF-NEXT:    lui a1, 56
+; RV64IF-NEXT:    addiw a1, a1, -1353
+; RV64IF-NEXT:    slli a1, a1, 14
+; RV64IF-NEXT:    flw ft0, -273(a1)
+; RV64IF-NEXT:    fmv.w.x ft1, a0
+; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
+; RV64IF-NEXT:    fsw ft0, -273(a1)
 ; RV64IF-NEXT:    ret
   %1 = inttoptr i32 3735928559 to float*
   %2 = load volatile float, float* %1
@@ -135,34 +135,32 @@ define float @flw_stack(float %a) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    addi sp, sp, -16
 ; RV32IF-NEXT:    sw ra, 12(sp)
-; RV32IF-NEXT:    sw s0, 8(sp)
-; RV32IF-NEXT:    mv s0, a0
-; RV32IF-NEXT:    addi a0, sp, 4
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    fsw ft0, 4(sp)
+; RV32IF-NEXT:    addi a0, sp, 8
 ; RV32IF-NEXT:    call notdead
-; RV32IF-NEXT:    fmv.w.x ft0, s0
+; RV32IF-NEXT:    flw ft0, 8(sp)
 ; RV32IF-NEXT:    flw ft1, 4(sp)
-; RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV32IF-NEXT:    fmv.x.w a0, ft0
-; RV32IF-NEXT:    lw s0, 8(sp)
 ; RV32IF-NEXT:    lw ra, 12(sp)
 ; RV32IF-NEXT:    addi sp, sp, 16
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: flw_stack:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    addi sp, sp, -32
-; RV64IF-NEXT:    sd ra, 24(sp)
-; RV64IF-NEXT:    sd s0, 16(sp)
-; RV64IF-NEXT:    mv s0, a0
-; RV64IF-NEXT:    addi a0, sp, 12
+; RV64IF-NEXT:    addi sp, sp, -16
+; RV64IF-NEXT:    sd ra, 8(sp)
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fsw ft0, 0(sp)
+; RV64IF-NEXT:    addi a0, sp, 4
 ; RV64IF-NEXT:    call notdead
-; RV64IF-NEXT:    fmv.w.x ft0, s0
-; RV64IF-NEXT:    flw ft1, 12(sp)
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    flw ft0, 4(sp)
+; RV64IF-NEXT:    flw ft1, 0(sp)
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
-; RV64IF-NEXT:    ld s0, 16(sp)
-; RV64IF-NEXT:    ld ra, 24(sp)
-; RV64IF-NEXT:    addi sp, sp, 32
+; RV64IF-NEXT:    ld ra, 8(sp)
+; RV64IF-NEXT:    addi sp, sp, 16
 ; RV64IF-NEXT:    ret
   %1 = alloca float, align 4
   %2 = bitcast float* %1 to i8*
diff --git a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
index ec81c53e1ff51..142e4f8e80b51 100644
--- a/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
+++ b/llvm/test/CodeGen/RISCV/float-select-fcmp.ll
@@ -240,14 +240,14 @@ define float @select_fcmp_ord(float %a, float %b) nounwind {
 define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ; RV32IF-LABEL: select_fcmp_ueq:
 ; RV32IF:       # %bb.0:
-; RV32IF-NEXT:    fmv.w.x ft0, a0
 ; RV32IF-NEXT:    fmv.w.x ft1, a1
-; RV32IF-NEXT:    feq.s a0, ft1, ft1
-; RV32IF-NEXT:    feq.s a1, ft0, ft0
-; RV32IF-NEXT:    and a0, a1, a0
-; RV32IF-NEXT:    seqz a0, a0
-; RV32IF-NEXT:    feq.s a1, ft0, ft1
-; RV32IF-NEXT:    or a0, a1, a0
+; RV32IF-NEXT:    fmv.w.x ft0, a0
+; RV32IF-NEXT:    feq.s a0, ft0, ft1
+; RV32IF-NEXT:    feq.s a1, ft1, ft1
+; RV32IF-NEXT:    feq.s a2, ft0, ft0
+; RV32IF-NEXT:    and a1, a2, a1
+; RV32IF-NEXT:    seqz a1, a1
+; RV32IF-NEXT:    or a0, a0, a1
 ; RV32IF-NEXT:    bnez a0, .LBB8_2
 ; RV32IF-NEXT:  # %bb.1:
 ; RV32IF-NEXT:    fmv.s ft0, ft1
@@ -257,14 +257,14 @@ define float @select_fcmp_ueq(float %a, float %b) nounwind {
 ;
 ; RV64IF-LABEL: select_fcmp_ueq:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a0
 ; RV64IF-NEXT:    fmv.w.x ft1, a1
-; RV64IF-NEXT:    feq.s a0, ft1, ft1
-; RV64IF-NEXT:    feq.s a1, ft0, ft0
-; RV64IF-NEXT:    and a0, a1, a0
-; RV64IF-NEXT:    seqz a0, a0
-; RV64IF-NEXT:    feq.s a1, ft0, ft1
-; RV64IF-NEXT:    or a0, a1, a0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    feq.s a0, ft0, ft1
+; RV64IF-NEXT:    feq.s a1, ft1, ft1
+; RV64IF-NEXT:    feq.s a2, ft0, ft0
+; RV64IF-NEXT:    and a1, a2, a1
+; RV64IF-NEXT:    seqz a1, a1
+; RV64IF-NEXT:    or a0, a0, a1
 ; RV64IF-NEXT:    bnez a0, .LBB8_2
 ; RV64IF-NEXT:  # %bb.1:
 ; RV64IF-NEXT:    fmv.s ft0, ft1
@@ -486,24 +486,24 @@ define i32 @i32_select_fcmp_oeq(float %a, float %b, i32 %c, i32 %d) nounwind {
 ; RV32IF:       # %bb.0:
 ; RV32IF-NEXT:    fmv.w.x ft0, a1
 ; RV32IF-NEXT:    fmv.w.x ft1, a0
-; RV32IF-NEXT:    feq.s a0, ft1, ft0
-; RV32IF-NEXT:    bnez a0, .LBB16_2
+; RV32IF-NEXT:    feq.s a1, ft1, ft0
+; RV32IF-NEXT:    mv a0, a2
+; RV32IF-NEXT:    bnez a1, .LBB16_2
 ; RV32IF-NEXT:  # %bb.1:
-; RV32IF-NEXT:    mv a2, a3
+; RV32IF-NEXT:    mv a0, a3
 ; RV32IF-NEXT:  .LBB16_2:
-; RV32IF-NEXT:    mv a0, a2
 ; RV32IF-NEXT:    ret
 ;
 ; RV64IF-LABEL: i32_select_fcmp_oeq:
 ; RV64IF:       # %bb.0:
 ; RV64IF-NEXT:    fmv.w.x ft0, a1
 ; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    feq.s a0, ft1, ft0
-; RV64IF-NEXT:    bnez a0, .LBB16_2
+; RV64IF-NEXT:    feq.s a1, ft1, ft0
+; RV64IF-NEXT:    mv a0, a2
+; RV64IF-NEXT:    bnez a1, .LBB16_2
 ; RV64IF-NEXT:  # %bb.1:
-; RV64IF-NEXT:    mv a2, a3
+; RV64IF-NEXT:    mv a0, a3
 ; RV64IF-NEXT:  .LBB16_2:
-; RV64IF-NEXT:    mv a0, a2
 ; RV64IF-NEXT:    ret
   %1 = fcmp oeq float %a, %b
   %2 = select i1 %1, i32 %c, i32 %d
diff --git a/llvm/test/CodeGen/RISCV/fp128.ll b/llvm/test/CodeGen/RISCV/fp128.ll
index a928d69fe9ed8..91b1702911af7 100644
--- a/llvm/test/CodeGen/RISCV/fp128.ll
+++ b/llvm/test/CodeGen/RISCV/fp128.ll
@@ -13,28 +13,28 @@ define i32 @test_load_and_cmp() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
-; RV32I-NEXT:    lui a0, %hi(y)
-; RV32I-NEXT:    lw a1, %lo(y)(a0)
-; RV32I-NEXT:    sw a1, 8(sp)
-; RV32I-NEXT:    lui a1, %hi(x)
-; RV32I-NEXT:    lw a2, %lo(x)(a1)
-; RV32I-NEXT:    sw a2, 24(sp)
-; RV32I-NEXT:    addi a0, a0, %lo(y)
-; RV32I-NEXT:    lw a2, 12(a0)
-; RV32I-NEXT:    sw a2, 20(sp)
-; RV32I-NEXT:    lw a2, 8(a0)
-; RV32I-NEXT:    sw a2, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    addi a0, a1, %lo(x)
-; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lui a0, %hi(x)
+; RV32I-NEXT:    addi a1, a0, %lo(x)
+; RV32I-NEXT:    lw a6, 4(a1)
+; RV32I-NEXT:    lw a7, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lui a4, %hi(y)
+; RV32I-NEXT:    addi a5, a4, %lo(y)
+; RV32I-NEXT:    lw a2, 4(a5)
+; RV32I-NEXT:    lw a3, 8(a5)
+; RV32I-NEXT:    lw a5, 12(a5)
+; RV32I-NEXT:    lw a4, %lo(y)(a4)
+; RV32I-NEXT:    sw a4, 8(sp)
+; RV32I-NEXT:    sw a0, 24(sp)
+; RV32I-NEXT:    sw a5, 20(sp)
+; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a2, 12(sp)
 ; RV32I-NEXT:    sw a1, 36(sp)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    sw a1, 32(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 28(sp)
+; RV32I-NEXT:    sw a7, 32(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a6, 28(sp)
 ; RV32I-NEXT:    call __netf2
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    lw ra, 44(sp)
@@ -52,39 +52,39 @@ define i32 @test_add_and_fptosi() nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -80
 ; RV32I-NEXT:    sw ra, 76(sp)
-; RV32I-NEXT:    lui a0, %hi(y)
-; RV32I-NEXT:    lw a1, %lo(y)(a0)
-; RV32I-NEXT:    sw a1, 24(sp)
-; RV32I-NEXT:    lui a1, %hi(x)
-; RV32I-NEXT:    lw a2, %lo(x)(a1)
-; RV32I-NEXT:    sw a2, 40(sp)
-; RV32I-NEXT:    addi a0, a0, %lo(y)
-; RV32I-NEXT:    lw a2, 12(a0)
-; RV32I-NEXT:    sw a2, 36(sp)
-; RV32I-NEXT:    lw a2, 8(a0)
+; RV32I-NEXT:    lui a0, %hi(x)
+; RV32I-NEXT:    addi a1, a0, %lo(x)
+; RV32I-NEXT:    lw a6, 4(a1)
+; RV32I-NEXT:    lw a7, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    lw a0, %lo(x)(a0)
+; RV32I-NEXT:    lui a4, %hi(y)
+; RV32I-NEXT:    addi a5, a4, %lo(y)
+; RV32I-NEXT:    lw a3, 4(a5)
+; RV32I-NEXT:    lw a2, 8(a5)
+; RV32I-NEXT:    lw a5, 12(a5)
+; RV32I-NEXT:    lw a4, %lo(y)(a4)
+; RV32I-NEXT:    sw a4, 24(sp)
+; RV32I-NEXT:    sw a0, 40(sp)
+; RV32I-NEXT:    sw a5, 36(sp)
 ; RV32I-NEXT:    sw a2, 32(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 28(sp)
-; RV32I-NEXT:    addi a0, a1, %lo(x)
-; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    sw a3, 28(sp)
 ; RV32I-NEXT:    sw a1, 52(sp)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    sw a1, 48(sp)
-; RV32I-NEXT:    lw a0, 4(a0)
-; RV32I-NEXT:    sw a0, 44(sp)
+; RV32I-NEXT:    sw a7, 48(sp)
 ; RV32I-NEXT:    addi a0, sp, 56
 ; RV32I-NEXT:    addi a1, sp, 40
 ; RV32I-NEXT:    addi a2, sp, 24
+; RV32I-NEXT:    sw a6, 44(sp)
 ; RV32I-NEXT:    call __addtf3
-; RV32I-NEXT:    lw a0, 68(sp)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 64(sp)
-; RV32I-NEXT:    sw a0, 16(sp)
+; RV32I-NEXT:    lw a1, 56(sp)
 ; RV32I-NEXT:    lw a0, 60(sp)
+; RV32I-NEXT:    lw a2, 64(sp)
+; RV32I-NEXT:    lw a3, 68(sp)
+; RV32I-NEXT:    sw a3, 20(sp)
+; RV32I-NEXT:    sw a2, 16(sp)
 ; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 56(sp)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    addi a0, sp, 8
+; RV32I-NEXT:    sw a1, 8(sp)
 ; RV32I-NEXT:    call __fixtfsi
 ; RV32I-NEXT:    lw ra, 76(sp)
 ; RV32I-NEXT:    addi sp, sp, 80
diff --git a/llvm/test/CodeGen/RISCV/frame-info.ll b/llvm/test/CodeGen/RISCV/frame-info.ll
index 9022fc395628e..754e8f55d53b1 100644
--- a/llvm/test/CodeGen/RISCV/frame-info.ll
+++ b/llvm/test/CodeGen/RISCV/frame-info.ll
@@ -38,12 +38,12 @@ define void @foo(i32 signext %size) {
 ; RV64-NEXT:    .cfi_offset s0, -16
 ; RV64-NEXT:    addi s0, sp, 16
 ; RV64-NEXT:    .cfi_def_cfa s0, 0
-; RV64-NEXT:    addi a1, zero, 1
-; RV64-NEXT:    slli a1, a1, 33
-; RV64-NEXT:    addi a1, a1, -16
 ; RV64-NEXT:    slli a0, a0, 32
 ; RV64-NEXT:    srli a0, a0, 32
 ; RV64-NEXT:    addi a0, a0, 15
+; RV64-NEXT:    addi a1, zero, 1
+; RV64-NEXT:    slli a1, a1, 33
+; RV64-NEXT:    addi a1, a1, -16
 ; RV64-NEXT:    and a0, a0, a1
 ; RV64-NEXT:    sub a0, sp, a0
 ; RV64-NEXT:    mv sp, a0
diff --git a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
index fe326b383ec0a..21923f0769080 100644
--- a/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
+++ b/llvm/test/CodeGen/RISCV/get-setcc-result-type.ll
@@ -6,21 +6,21 @@ define void @getSetCCResultType(<4 x i32>* %p, <4 x i32>* %q) nounwind {
 ; RV32I-LABEL: getSetCCResultType:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lw a2, 8(a0)
+; RV32I-NEXT:    lw a3, 4(a0)
+; RV32I-NEXT:    lw a4, 0(a0)
 ; RV32I-NEXT:    seqz a1, a1
+; RV32I-NEXT:    seqz a2, a2
+; RV32I-NEXT:    seqz a3, a3
+; RV32I-NEXT:    seqz a4, a4
+; RV32I-NEXT:    neg a4, a4
+; RV32I-NEXT:    neg a3, a3
+; RV32I-NEXT:    neg a2, a2
 ; RV32I-NEXT:    neg a1, a1
 ; RV32I-NEXT:    sw a1, 12(a0)
-; RV32I-NEXT:    lw a1, 8(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 8(a0)
-; RV32I-NEXT:    lw a1, 4(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a0)
-; RV32I-NEXT:    seqz a1, a1
-; RV32I-NEXT:    neg a1, a1
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
+; RV32I-NEXT:    sw a4, 0(a0)
 ; RV32I-NEXT:    ret
 entry:
   %0 = load <4 x i32>, <4 x i32>* %p, align 16
diff --git a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
index b00873ce486d6..df8b31170695d 100644
--- a/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
+++ b/llvm/test/CodeGen/RISCV/hoist-global-addr-base.ll
@@ -12,10 +12,10 @@ define dso_local void @multiple_stores() local_unnamed_addr nounwind {
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    lui a0, %hi(s)
 ; CHECK-NEXT:    addi a0, a0, %lo(s)
-; CHECK-NEXT:    addi a1, zero, 20
-; CHECK-NEXT:    sw a1, 164(a0)
 ; CHECK-NEXT:    addi a1, zero, 10
 ; CHECK-NEXT:    sw a1, 160(a0)
+; CHECK-NEXT:    addi a1, zero, 20
+; CHECK-NEXT:    sw a1, 164(a0)
 ; CHECK-NEXT:    ret
 entry:
   store i32 10, i32* getelementptr inbounds (%struct.S, %struct.S* @s, i32 0, i32 1), align 4
diff --git a/llvm/test/CodeGen/RISCV/imm-cse.ll b/llvm/test/CodeGen/RISCV/imm-cse.ll
index 1af27f0879891..a1ddcd6445629 100644
--- a/llvm/test/CodeGen/RISCV/imm-cse.ll
+++ b/llvm/test/CodeGen/RISCV/imm-cse.ll
@@ -10,19 +10,19 @@
 define void @imm32_cse() nounwind {
 ; RV32I-LABEL: imm32_cse:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lui a0, 1
-; RV32I-NEXT:    addi a0, a0, 1
-; RV32I-NEXT:    lui a1, %hi(src)
-; RV32I-NEXT:    lw a2, %lo(src)(a1)
-; RV32I-NEXT:    add a2, a2, a0
-; RV32I-NEXT:    lui a3, %hi(dst)
-; RV32I-NEXT:    sw a2, %lo(dst)(a3)
-; RV32I-NEXT:    lw a2, %lo(src)(a1)
-; RV32I-NEXT:    add a2, a2, a0
+; RV32I-NEXT:    lui a0, %hi(src)
+; RV32I-NEXT:    lw a1, %lo(src)(a0)
+; RV32I-NEXT:    lui a2, 1
 ; RV32I-NEXT:    addi a2, a2, 1
-; RV32I-NEXT:    sw a2, %lo(dst)(a3)
-; RV32I-NEXT:    lw a1, %lo(src)(a1)
-; RV32I-NEXT:    add a0, a1, a0
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    lui a3, %hi(dst)
+; RV32I-NEXT:    sw a1, %lo(dst)(a3)
+; RV32I-NEXT:    lw a1, %lo(src)(a0)
+; RV32I-NEXT:    add a1, a1, a2
+; RV32I-NEXT:    addi a1, a1, 1
+; RV32I-NEXT:    sw a1, %lo(dst)(a3)
+; RV32I-NEXT:    lw a0, %lo(src)(a0)
+; RV32I-NEXT:    add a0, a0, a2
 ; RV32I-NEXT:    addi a0, a0, 2
 ; RV32I-NEXT:    sw a0, %lo(dst)(a3)
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/imm.ll b/llvm/test/CodeGen/RISCV/imm.ll
index b733b5921658d..ce254f2b1265d 100644
--- a/llvm/test/CodeGen/RISCV/imm.ll
+++ b/llvm/test/CodeGen/RISCV/imm.ll
@@ -142,8 +142,8 @@ define i64 @imm64_2() nounwind {
 define i64 @imm64_3() nounwind {
 ; RV32I-LABEL: imm64_3:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    addi a1, zero, 1
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_3:
@@ -157,8 +157,8 @@ define i64 @imm64_3() nounwind {
 define i64 @imm64_4() nounwind {
 ; RV32I-LABEL: imm64_4:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_4:
@@ -172,8 +172,8 @@ define i64 @imm64_4() nounwind {
 define i64 @imm64_5() nounwind {
 ; RV32I-LABEL: imm64_5:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lui a1, 524288
+; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_5:
@@ -249,7 +249,7 @@ define i64 @imm64_9() nounwind {
 ; RV32I-LABEL: imm64_9:
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi a0, zero, -1
-; RV32I-NEXT:    addi  a1, zero, -1
+; RV32I-NEXT:    addi a1, zero, -1
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: imm64_9:
diff --git a/llvm/test/CodeGen/RISCV/indirectbr.ll b/llvm/test/CodeGen/RISCV/indirectbr.ll
index e734de3c8e496..01201e818fa90 100644
--- a/llvm/test/CodeGen/RISCV/indirectbr.ll
+++ b/llvm/test/CodeGen/RISCV/indirectbr.ll
@@ -8,7 +8,7 @@ define i32 @indirectbr(i8* %target) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
 ; RV32I-NEXT:    jr a0
-; RV32I-NEXT:  .LBB0_1:
+; RV32I-NEXT:  .LBB0_1: # %test_label
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
@@ -26,7 +26,7 @@ define i32 @indirectbr_with_offset(i8* %a) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp)
 ; RV32I-NEXT:    jr 1380(a0)
-; RV32I-NEXT:  .LBB1_1:
+; RV32I-NEXT:  .LBB1_1: # %test_label
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    lw ra, 12(sp)
 ; RV32I-NEXT:    addi sp, sp, 16
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
index 6d4f8da2c2544..37ce0f89e7ec5 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-d-constraint-f.ll
@@ -26,11 +26,11 @@ define double @constraint_f_double(double %a) nounwind {
 ;
 ; RV64F-LABEL: constraint_f_double:
 ; RV64F:       # %bb.0:
-; RV64F-NEXT:    fmv.d.x ft0, a0
-; RV64F-NEXT:    lui a0, %hi(gd)
-; RV64F-NEXT:    fld ft1, %lo(gd)(a0)
+; RV64F-NEXT:    lui a1, %hi(gd)
+; RV64F-NEXT:    fld ft0, %lo(gd)(a1)
+; RV64F-NEXT:    fmv.d.x ft1, a0
 ; RV64F-NEXT:    #APP
-; RV64F-NEXT:    fadd.d ft0, ft0, ft1
+; RV64F-NEXT:    fadd.d ft0, ft1, ft0
 ; RV64F-NEXT:    #NO_APP
 ; RV64F-NEXT:    fmv.x.d a0, ft0
 ; RV64F-NEXT:    ret
@@ -59,9 +59,9 @@ define double @constraint_f_double_abi_name(double %a) nounwind {
 ;
 ; RV64F-LABEL: constraint_f_double_abi_name:
 ; RV64F:       # %bb.0:
+; RV64F-NEXT:    lui a1, %hi(gd)
+; RV64F-NEXT:    fld fs0, %lo(gd)(a1)
 ; RV64F-NEXT:    fmv.d.x fa1, a0
-; RV64F-NEXT:    lui a0, %hi(gd)
-; RV64F-NEXT:    fld fs0, %lo(gd)(a0)
 ; RV64F-NEXT:    #APP
 ; RV64F-NEXT:    fadd.d ft0, fa1, fs0
 ; RV64F-NEXT:    #NO_APP
diff --git a/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll b/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
index c91b1ae672efa..25bce62952aed 100644
--- a/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
+++ b/llvm/test/CodeGen/RISCV/inline-asm-f-constraint-f.ll
@@ -9,22 +9,22 @@
 define float @constraint_f_float(float %a) nounwind {
 ; RV32F-LABEL: constraint_f_float:
 ; RV32F:       # %bb.0:
-; RV32F-NEXT:    fmv.w.x ft0, a0
-; RV32F-NEXT:    lui a0, %hi(gf)
-; RV32F-NEXT:    flw ft1, %lo(gf)(a0)
+; RV32F-NEXT:    lui a1, %hi(gf)
+; RV32F-NEXT:    flw ft0, %lo(gf)(a1)
+; RV32F-NEXT:    fmv.w.x ft1, a0
 ; RV32F-NEXT:    #APP
-; RV32F-NEXT:    fadd.s ft0, ft0, ft1
+; RV32F-NEXT:    fadd.s ft0, ft1, ft0
 ; RV32F-NEXT:    #NO_APP
 ; RV32F-NEXT:    fmv.x.w a0, ft0
 ; RV32F-NEXT:    ret
 ;
 ; RV64F-LABEL: constraint_f_float:
 ; RV64F:       # %bb.0:
-; RV64F-NEXT:    fmv.w.x ft0, a0
-; RV64F-NEXT:    lui a0, %hi(gf)
-; RV64F-NEXT:    flw ft1, %lo(gf)(a0)
+; RV64F-NEXT:    lui a1, %hi(gf)
+; RV64F-NEXT:    flw ft0, %lo(gf)(a1)
+; RV64F-NEXT:    fmv.w.x ft1, a0
 ; RV64F-NEXT:    #APP
-; RV64F-NEXT:    fadd.s ft0, ft0, ft1
+; RV64F-NEXT:    fadd.s ft0, ft1, ft0
 ; RV64F-NEXT:    #NO_APP
 ; RV64F-NEXT:    fmv.x.w a0, ft0
 ; RV64F-NEXT:    ret
@@ -36,9 +36,9 @@ define float @constraint_f_float(float %a) nounwind {
 define float @constraint_f_float_abi_name(float %a) nounwind {
 ; RV32F-LABEL: constraint_f_float_abi_name:
 ; RV32F:       # %bb.0:
+; RV32F-NEXT:    lui a1, %hi(gf)
+; RV32F-NEXT:    flw fs0, %lo(gf)(a1)
 ; RV32F-NEXT:    fmv.w.x fa0, a0
-; RV32F-NEXT:    lui a0, %hi(gf)
-; RV32F-NEXT:    flw fs0, %lo(gf)(a0)
 ; RV32F-NEXT:    #APP
 ; RV32F-NEXT:    fadd.s ft0, fa0, fs0
 ; RV32F-NEXT:    #NO_APP
@@ -47,9 +47,9 @@ define float @constraint_f_float_abi_name(float %a) nounwind {
 ;
 ; RV64F-LABEL: constraint_f_float_abi_name:
 ; RV64F:       # %bb.0:
+; RV64F-NEXT:    lui a1, %hi(gf)
+; RV64F-NEXT:    flw fs0, %lo(gf)(a1)
 ; RV64F-NEXT:    fmv.w.x fa0, a0
-; RV64F-NEXT:    lui a0, %hi(gf)
-; RV64F-NEXT:    flw fs0, %lo(gf)(a0)
 ; RV64F-NEXT:    #APP
 ; RV64F-NEXT:    fadd.s ft0, fa0, fs0
 ; RV64F-NEXT:    #NO_APP
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
index 91b6249d06de5..3b3ee617b1351 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr-nocall.ll
@@ -213,11 +213,11 @@ define void @foo_float() nounwind #0 {
 ; CHECK-RV32IF-NEXT:    sw a0, 12(sp)
 ; CHECK-RV32IF-NEXT:    fsw ft0, 8(sp)
 ; CHECK-RV32IF-NEXT:    fsw ft1, 4(sp)
-; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
-; CHECK-RV32IF-NEXT:    flw ft0, %lo(f)(a0)
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
-; CHECK-RV32IF-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IF-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
 ; CHECK-RV32IF-NEXT:    flw ft1, 4(sp)
@@ -232,11 +232,11 @@ define void @foo_float() nounwind #0 {
 ; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
-; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
-; CHECK-RV32IFD-NEXT:    flw ft0, %lo(f)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
-; CHECK-RV32IFD-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft1, ft0
+; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IFD-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
 ; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
@@ -312,11 +312,11 @@ define void @foo_fp_float() nounwind #1 {
 ; CHECK-RV32IF-NEXT:    fsw ft0, 16(sp)
 ; CHECK-RV32IF-NEXT:    fsw ft1, 12(sp)
 ; CHECK-RV32IF-NEXT:    addi s0, sp, 32
-; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
-; CHECK-RV32IF-NEXT:    flw ft0, %lo(f)(a0)
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(e)
-; CHECK-RV32IF-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32IF-NEXT:    fadd.s ft0, ft1, ft0
+; CHECK-RV32IF-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IF-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IF-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IF-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IF-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IF-NEXT:    fsw ft0, %lo(d)(a0)
 ; CHECK-RV32IF-NEXT:    flw ft1, 12(sp)
@@ -336,11 +336,11 @@ define void @foo_fp_float() nounwind #1 {
 ; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
 ; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
-; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
-; CHECK-RV32IFD-NEXT:    flw ft0, %lo(f)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(e)
-; CHECK-RV32IFD-NEXT:    flw ft1, %lo(e)(a0)
-; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft1, ft0
+; CHECK-RV32IFD-NEXT:    flw ft0, %lo(e)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(f)
+; CHECK-RV32IFD-NEXT:    flw ft1, %lo(f)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.s ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(d)
 ; CHECK-RV32IFD-NEXT:    fsw ft0, %lo(d)(a0)
 ; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
@@ -534,11 +534,11 @@ define void @foo_double() nounwind #0 {
 ; CHECK-RV32IFD-NEXT:    sw a0, 28(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, 16(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft1, 8(sp)
-; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
-; CHECK-RV32IFD-NEXT:    fld ft0, %lo(i)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
-; CHECK-RV32IFD-NEXT:    fld ft1, %lo(h)(a0)
-; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
+; CHECK-RV32IFD-NEXT:    fld ft1, %lo(i)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
 ; CHECK-RV32IFD-NEXT:    fld ft1, 8(sp)
@@ -738,11 +738,11 @@ define void @foo_fp_double() nounwind #1 {
 ; CHECK-RV32IFD-NEXT:    fsd ft0, 8(sp)
 ; CHECK-RV32IFD-NEXT:    fsd ft1, 0(sp)
 ; CHECK-RV32IFD-NEXT:    addi s0, sp, 32
-; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
-; CHECK-RV32IFD-NEXT:    fld ft0, %lo(i)(a0)
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(h)
-; CHECK-RV32IFD-NEXT:    fld ft1, %lo(h)(a0)
-; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft1, ft0
+; CHECK-RV32IFD-NEXT:    fld ft0, %lo(h)(a0)
+; CHECK-RV32IFD-NEXT:    lui a0, %hi(i)
+; CHECK-RV32IFD-NEXT:    fld ft1, %lo(i)(a0)
+; CHECK-RV32IFD-NEXT:    fadd.d ft0, ft0, ft1
 ; CHECK-RV32IFD-NEXT:    lui a0, %hi(g)
 ; CHECK-RV32IFD-NEXT:    fsd ft0, %lo(g)(a0)
 ; CHECK-RV32IFD-NEXT:    fld ft1, 0(sp)
diff --git a/llvm/test/CodeGen/RISCV/legalize-fneg.ll b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
index d2d0b9ea6c2c6..80f0a988a6a6c 100644
--- a/llvm/test/CodeGen/RISCV/legalize-fneg.ll
+++ b/llvm/test/CodeGen/RISCV/legalize-fneg.ll
@@ -15,9 +15,9 @@ define void @test1(float* %a, float* %b) nounwind {
 ;
 ; RV64-LABEL: test1:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    lw a1, 0(a1)
 ; RV64-NEXT:    addi a2, zero, 1
 ; RV64-NEXT:    slli a2, a2, 31
-; RV64-NEXT:    lw a1, 0(a1)
 ; RV64-NEXT:    xor a1, a1, a2
 ; RV64-NEXT:    sw a1, 0(a0)
 ; RV64-NEXT:    ret
@@ -33,17 +33,17 @@ define void @test2(double* %a, double* %b) nounwind {
 ; RV32:       # %bb.0: # %entry
 ; RV32-NEXT:    lw a2, 4(a1)
 ; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lui a3, 524288
+; RV32-NEXT:    xor a2, a2, a3
 ; RV32-NEXT:    sw a1, 0(a0)
-; RV32-NEXT:    lui a1, 524288
-; RV32-NEXT:    xor a1, a2, a1
-; RV32-NEXT:    sw a1, 4(a0)
+; RV32-NEXT:    sw a2, 4(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: test2:
 ; RV64:       # %bb.0: # %entry
+; RV64-NEXT:    ld a1, 0(a1)
 ; RV64-NEXT:    addi a2, zero, -1
 ; RV64-NEXT:    slli a2, a2, 63
-; RV64-NEXT:    ld a1, 0(a1)
 ; RV64-NEXT:    xor a1, a1, a2
 ; RV64-NEXT:    sd a1, 0(a0)
 ; RV64-NEXT:    ret
@@ -57,27 +57,27 @@ entry:
 define void @test3(fp128* %a, fp128* %b) nounwind {
 ; RV32-LABEL: test3:
 ; RV32:       # %bb.0: # %entry
-; RV32-NEXT:    lw a2, 12(a1)
-; RV32-NEXT:    lw a3, 4(a1)
-; RV32-NEXT:    lw a4, 0(a1)
-; RV32-NEXT:    lw a1, 8(a1)
-; RV32-NEXT:    sw a1, 8(a0)
-; RV32-NEXT:    sw a4, 0(a0)
-; RV32-NEXT:    sw a3, 4(a0)
-; RV32-NEXT:    lui a1, 524288
-; RV32-NEXT:    xor a1, a2, a1
-; RV32-NEXT:    sw a1, 12(a0)
+; RV32-NEXT:    lw a2, 4(a1)
+; RV32-NEXT:    lw a3, 12(a1)
+; RV32-NEXT:    lw a4, 8(a1)
+; RV32-NEXT:    lw a1, 0(a1)
+; RV32-NEXT:    lui a5, 524288
+; RV32-NEXT:    xor a3, a3, a5
+; RV32-NEXT:    sw a4, 8(a0)
+; RV32-NEXT:    sw a1, 0(a0)
+; RV32-NEXT:    sw a2, 4(a0)
+; RV32-NEXT:    sw a3, 12(a0)
 ; RV32-NEXT:    ret
 ;
 ; RV64-LABEL: test3:
 ; RV64:       # %bb.0: # %entry
 ; RV64-NEXT:    ld a2, 8(a1)
 ; RV64-NEXT:    ld a1, 0(a1)
+; RV64-NEXT:    addi a3, zero, -1
+; RV64-NEXT:    slli a3, a3, 63
+; RV64-NEXT:    xor a2, a2, a3
 ; RV64-NEXT:    sd a1, 0(a0)
-; RV64-NEXT:    addi a1, zero, -1
-; RV64-NEXT:    slli a1, a1, 63
-; RV64-NEXT:    xor a1, a2, a1
-; RV64-NEXT:    sd a1, 8(a0)
+; RV64-NEXT:    sd a2, 8(a0)
 ; RV64-NEXT:    ret
 entry:
   %0 = load fp128, fp128* %b
diff --git a/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll b/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
index 2f4c821d883d8..dbe9eb9f050af 100644
--- a/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
+++ b/llvm/test/CodeGen/RISCV/lsr-legaladdimm.ll
@@ -11,21 +11,21 @@
 define i32 @main() nounwind {
 ; RV32I-LABEL: main:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    lui a0, %hi(b)
-; RV32I-NEXT:    addi a0, a0, %lo(b)
-; RV32I-NEXT:    lui a1, %hi(a)
-; RV32I-NEXT:    addi a1, a1, %lo(a)
-; RV32I-NEXT:    lui a2, 1
-; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    mv a0, zero
+; RV32I-NEXT:    lui a1, %hi(b)
+; RV32I-NEXT:    addi a1, a1, %lo(b)
+; RV32I-NEXT:    lui a2, %hi(a)
+; RV32I-NEXT:    addi a2, a2, %lo(a)
+; RV32I-NEXT:    lui a3, 1
 ; RV32I-NEXT:  .LBB0_1: # %for.body
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    addi a4, a3, -2048
-; RV32I-NEXT:    sw a4, 0(a1)
+; RV32I-NEXT:    addi a4, a0, -2048
+; RV32I-NEXT:    sw a4, 0(a2)
+; RV32I-NEXT:    sw a0, 0(a1)
+; RV32I-NEXT:    addi a0, a0, 1
 ; RV32I-NEXT:    addi a1, a1, 4
-; RV32I-NEXT:    sw a3, 0(a0)
-; RV32I-NEXT:    addi a0, a0, 4
-; RV32I-NEXT:    addi a3, a3, 1
-; RV32I-NEXT:    bne a3, a2, .LBB0_1
+; RV32I-NEXT:    addi a2, a2, 4
+; RV32I-NEXT:    bne a0, a3, .LBB0_1
 ; RV32I-NEXT:  # %bb.2: # %for.end
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/mem.ll b/llvm/test/CodeGen/RISCV/mem.ll
index f873e842829b1..5a2eeee41d0b8 100644
--- a/llvm/test/CodeGen/RISCV/mem.ll
+++ b/llvm/test/CodeGen/RISCV/mem.ll
@@ -7,8 +7,9 @@
 define i32 @lb(i8 *%a) nounwind {
 ; RV32I-LABEL: lb:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
-; RV32I-NEXT:    lb a0, 1(a0)
+; RV32I-NEXT:    lb a1, 1(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 1
   %2 = load i8, i8* %1
@@ -21,8 +22,9 @@ define i32 @lb(i8 *%a) nounwind {
 define i32 @lh(i16 *%a) nounwind {
 ; RV32I-LABEL: lh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lh a1, 0(a0)
-; RV32I-NEXT:    lh a0, 4(a0)
+; RV32I-NEXT:    lh a1, 4(a0)
+; RV32I-NEXT:    lh a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 2
   %2 = load i16, i16* %1
@@ -35,8 +37,9 @@ define i32 @lh(i16 *%a) nounwind {
 define i32 @lw(i32 *%a) nounwind {
 ; RV32I-LABEL: lw:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lw a1, 0(a0)
-; RV32I-NEXT:    lw a0, 12(a0)
+; RV32I-NEXT:    lw a1, 12(a0)
+; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 3
   %2 = load i32, i32* %1
@@ -47,9 +50,9 @@ define i32 @lw(i32 *%a) nounwind {
 define i32 @lbu(i8 *%a) nounwind {
 ; RV32I-LABEL: lbu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lbu a1, 0(a0)
-; RV32I-NEXT:    lbu a0, 4(a0)
-; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lbu a1, 4(a0)
+; RV32I-NEXT:    lbu a0, 0(a0)
+; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 4
   %2 = load i8, i8* %1
@@ -63,9 +66,9 @@ define i32 @lbu(i8 *%a) nounwind {
 define i32 @lhu(i16 *%a) nounwind {
 ; RV32I-LABEL: lhu:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lhu a1, 0(a0)
-; RV32I-NEXT:    lhu a0, 10(a0)
-; RV32I-NEXT:    add a0, a0, a1
+; RV32I-NEXT:    lhu a1, 10(a0)
+; RV32I-NEXT:    lhu a0, 0(a0)
+; RV32I-NEXT:    add a0, a1, a0
 ; RV32I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 5
   %2 = load i16, i16* %1
@@ -81,8 +84,8 @@ define i32 @lhu(i16 *%a) nounwind {
 define void @sb(i8 *%a, i8 %b) nounwind {
 ; RV32I-LABEL: sb:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sb a1, 6(a0)
 ; RV32I-NEXT:    sb a1, 0(a0)
+; RV32I-NEXT:    sb a1, 6(a0)
 ; RV32I-NEXT:    ret
   store i8 %b, i8* %a
   %1 = getelementptr i8, i8* %a, i32 6
@@ -93,8 +96,8 @@ define void @sb(i8 *%a, i8 %b) nounwind {
 define void @sh(i16 *%a, i16 %b) nounwind {
 ; RV32I-LABEL: sh:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sh a1, 14(a0)
 ; RV32I-NEXT:    sh a1, 0(a0)
+; RV32I-NEXT:    sh a1, 14(a0)
 ; RV32I-NEXT:    ret
   store i16 %b, i16* %a
   %1 = getelementptr i16, i16* %a, i32 7
@@ -105,8 +108,8 @@ define void @sh(i16 *%a, i16 %b) nounwind {
 define void @sw(i32 *%a, i32 %b) nounwind {
 ; RV32I-LABEL: sw:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sw a1, 32(a0)
 ; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a1, 32(a0)
 ; RV32I-NEXT:    ret
   store i32 %b, i32* %a
   %1 = getelementptr i32, i32* %a, i32 8
@@ -118,10 +121,10 @@ define void @sw(i32 *%a, i32 %b) nounwind {
 define i32 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 ; RV32I-LABEL: load_sext_zext_anyext_i1:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
 ; RV32I-NEXT:    lbu a1, 1(a0)
-; RV32I-NEXT:    lbu a0, 2(a0)
-; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lbu a2, 2(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    sub a0, a2, a1
 ; RV32I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
@@ -140,10 +143,10 @@ define i32 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 define i16 @load_sext_zext_anyext_i1_i16(i1 *%a) nounwind {
 ; RV32I-LABEL: load_sext_zext_anyext_i1_i16:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    lb a1, 0(a0)
 ; RV32I-NEXT:    lbu a1, 1(a0)
-; RV32I-NEXT:    lbu a0, 2(a0)
-; RV32I-NEXT:    sub a0, a0, a1
+; RV32I-NEXT:    lbu a2, 2(a0)
+; RV32I-NEXT:    lb a0, 0(a0)
+; RV32I-NEXT:    sub a0, a2, a1
 ; RV32I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
diff --git a/llvm/test/CodeGen/RISCV/mem64.ll b/llvm/test/CodeGen/RISCV/mem64.ll
index 32669e8aef36b..522786b867f66 100644
--- a/llvm/test/CodeGen/RISCV/mem64.ll
+++ b/llvm/test/CodeGen/RISCV/mem64.ll
@@ -7,8 +7,9 @@
 define i64 @lb(i8 *%a) nounwind {
 ; RV64I-LABEL: lb:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
-; RV64I-NEXT:    lb a0, 1(a0)
+; RV64I-NEXT:    lb a1, 1(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 1
   %2 = load i8, i8* %1
@@ -21,8 +22,9 @@ define i64 @lb(i8 *%a) nounwind {
 define i64 @lh(i16 *%a) nounwind {
 ; RV64I-LABEL: lh:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lh a1, 0(a0)
-; RV64I-NEXT:    lh a0, 4(a0)
+; RV64I-NEXT:    lh a1, 4(a0)
+; RV64I-NEXT:    lh a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 2
   %2 = load i16, i16* %1
@@ -35,8 +37,9 @@ define i64 @lh(i16 *%a) nounwind {
 define i64 @lw(i32 *%a) nounwind {
 ; RV64I-LABEL: lw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lw a1, 0(a0)
-; RV64I-NEXT:    lw a0, 12(a0)
+; RV64I-NEXT:    lw a1, 12(a0)
+; RV64I-NEXT:    lw a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 3
   %2 = load i32, i32* %1
@@ -49,9 +52,9 @@ define i64 @lw(i32 *%a) nounwind {
 define i64 @lbu(i8 *%a) nounwind {
 ; RV64I-LABEL: lbu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lbu a1, 0(a0)
-; RV64I-NEXT:    lbu a0, 4(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lbu a1, 4(a0)
+; RV64I-NEXT:    lbu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i8, i8* %a, i32 4
   %2 = load i8, i8* %1
@@ -65,9 +68,9 @@ define i64 @lbu(i8 *%a) nounwind {
 define i64 @lhu(i16 *%a) nounwind {
 ; RV64I-LABEL: lhu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lhu a1, 0(a0)
-; RV64I-NEXT:    lhu a0, 10(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lhu a1, 10(a0)
+; RV64I-NEXT:    lhu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i16, i16* %a, i32 5
   %2 = load i16, i16* %1
@@ -81,9 +84,9 @@ define i64 @lhu(i16 *%a) nounwind {
 define i64 @lwu(i32 *%a) nounwind {
 ; RV64I-LABEL: lwu:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lwu a1, 0(a0)
-; RV64I-NEXT:    lwu a0, 24(a0)
-; RV64I-NEXT:    add a0, a0, a1
+; RV64I-NEXT:    lwu a1, 24(a0)
+; RV64I-NEXT:    lwu a0, 0(a0)
+; RV64I-NEXT:    add a0, a1, a0
 ; RV64I-NEXT:    ret
   %1 = getelementptr i32, i32* %a, i32 6
   %2 = load i32, i32* %1
@@ -99,8 +102,8 @@ define i64 @lwu(i32 *%a) nounwind {
 define void @sb(i8 *%a, i8 %b) nounwind {
 ; RV64I-LABEL: sb:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sb a1, 7(a0)
 ; RV64I-NEXT:    sb a1, 0(a0)
+; RV64I-NEXT:    sb a1, 7(a0)
 ; RV64I-NEXT:    ret
   store i8 %b, i8* %a
   %1 = getelementptr i8, i8* %a, i32 7
@@ -111,8 +114,8 @@ define void @sb(i8 *%a, i8 %b) nounwind {
 define void @sh(i16 *%a, i16 %b) nounwind {
 ; RV64I-LABEL: sh:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sh a1, 16(a0)
 ; RV64I-NEXT:    sh a1, 0(a0)
+; RV64I-NEXT:    sh a1, 16(a0)
 ; RV64I-NEXT:    ret
   store i16 %b, i16* %a
   %1 = getelementptr i16, i16* %a, i32 8
@@ -123,8 +126,8 @@ define void @sh(i16 *%a, i16 %b) nounwind {
 define void @sw(i32 *%a, i32 %b) nounwind {
 ; RV64I-LABEL: sw:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sw a1, 36(a0)
 ; RV64I-NEXT:    sw a1, 0(a0)
+; RV64I-NEXT:    sw a1, 36(a0)
 ; RV64I-NEXT:    ret
   store i32 %b, i32* %a
   %1 = getelementptr i32, i32* %a, i32 9
@@ -137,8 +140,9 @@ define void @sw(i32 *%a, i32 %b) nounwind {
 define i64 @ld(i64 *%a) nounwind {
 ; RV64I-LABEL: ld:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    ld a1, 0(a0)
-; RV64I-NEXT:    ld a0, 80(a0)
+; RV64I-NEXT:    ld a1, 80(a0)
+; RV64I-NEXT:    ld a0, 0(a0)
+; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
   %1 = getelementptr i64, i64* %a, i32 10
   %2 = load i64, i64* %1
@@ -149,8 +153,8 @@ define i64 @ld(i64 *%a) nounwind {
 define void @sd(i64 *%a, i64 %b) nounwind {
 ; RV64I-LABEL: sd:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sd a1, 88(a0)
 ; RV64I-NEXT:    sd a1, 0(a0)
+; RV64I-NEXT:    sd a1, 88(a0)
 ; RV64I-NEXT:    ret
   store i64 %b, i64* %a
   %1 = getelementptr i64, i64* %a, i32 11
@@ -162,10 +166,10 @@ define void @sd(i64 *%a, i64 %b) nounwind {
 define i64 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 ; RV64I-LABEL: load_sext_zext_anyext_i1:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
 ; RV64I-NEXT:    lbu a1, 1(a0)
-; RV64I-NEXT:    lbu a0, 2(a0)
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lbu a2, 2(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    sub a0, a2, a1
 ; RV64I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
@@ -184,10 +188,10 @@ define i64 @load_sext_zext_anyext_i1(i1 *%a) nounwind {
 define i16 @load_sext_zext_anyext_i1_i16(i1 *%a) nounwind {
 ; RV64I-LABEL: load_sext_zext_anyext_i1_i16:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    lb a1, 0(a0)
 ; RV64I-NEXT:    lbu a1, 1(a0)
-; RV64I-NEXT:    lbu a0, 2(a0)
-; RV64I-NEXT:    sub a0, a0, a1
+; RV64I-NEXT:    lbu a2, 2(a0)
+; RV64I-NEXT:    lb a0, 0(a0)
+; RV64I-NEXT:    sub a0, a2, a1
 ; RV64I-NEXT:    ret
   ; sextload i1
   %1 = getelementptr i1, i1* %a, i32 1
diff --git a/llvm/test/CodeGen/RISCV/mul.ll b/llvm/test/CodeGen/RISCV/mul.ll
index 9bf95bece61f1..2448580d39853 100644
--- a/llvm/test/CodeGen/RISCV/mul.ll
+++ b/llvm/test/CodeGen/RISCV/mul.ll
@@ -247,8 +247,8 @@ define i32 @mulhs(i32 %a, i32 %b) nounwind {
 ;
 ; RV64IM-LABEL: mulhs:
 ; RV64IM:       # %bb.0:
-; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    sext.w a0, a0
+; RV64IM-NEXT:    sext.w a1, a1
 ; RV64IM-NEXT:    mul a0, a0, a1
 ; RV64IM-NEXT:    srli a0, a0, 32
 ; RV64IM-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/remat.ll b/llvm/test/CodeGen/RISCV/remat.ll
index 41cb90ba3e88e..d8267e7a7ee41 100644
--- a/llvm/test/CodeGen/RISCV/remat.ll
+++ b/llvm/test/CodeGen/RISCV/remat.ll
@@ -37,19 +37,19 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    sw s9, 20(sp)
 ; RV32I-NEXT:    sw s10, 16(sp)
 ; RV32I-NEXT:    sw s11, 12(sp)
-; RV32I-NEXT:    lui s9, %hi(a)
-; RV32I-NEXT:    lw a0, %lo(a)(s9)
+; RV32I-NEXT:    lui s6, %hi(a)
+; RV32I-NEXT:    lw a0, %lo(a)(s6)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
 ; RV32I-NEXT:  # %bb.1: # %for.body.preheader
 ; RV32I-NEXT:    lui s2, %hi(l)
 ; RV32I-NEXT:    lui s3, %hi(k)
 ; RV32I-NEXT:    lui s4, %hi(j)
-; RV32I-NEXT:    lui s6, %hi(i)
-; RV32I-NEXT:    lui s5, %hi(h)
-; RV32I-NEXT:    lui s7, %hi(g)
-; RV32I-NEXT:    lui s8, %hi(f)
-; RV32I-NEXT:    lui s1, %hi(e)
-; RV32I-NEXT:    lui s0, %hi(d)
+; RV32I-NEXT:    lui s5, %hi(i)
+; RV32I-NEXT:    lui s1, %hi(d)
+; RV32I-NEXT:    lui s0, %hi(e)
+; RV32I-NEXT:    lui s7, %hi(f)
+; RV32I-NEXT:    lui s8, %hi(g)
+; RV32I-NEXT:    lui s9, %hi(h)
 ; RV32I-NEXT:    lui s10, %hi(c)
 ; RV32I-NEXT:    lui s11, %hi(b)
 ; RV32I-NEXT:    lw a1, %lo(l)(s2)
@@ -57,19 +57,19 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    j .LBB0_5
 ; RV32I-NEXT:  .LBB0_2: # %for.inc
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a0, %lo(a)(s9)
+; RV32I-NEXT:    lw a0, %lo(a)(s6)
 ; RV32I-NEXT:    addi a0, a0, -1
-; RV32I-NEXT:    sw a0, %lo(a)(s9)
+; RV32I-NEXT:    sw a0, %lo(a)(s6)
 ; RV32I-NEXT:    beqz a0, .LBB0_11
 ; RV32I-NEXT:  # %bb.3: # %for.body
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
 ; RV32I-NEXT:    lw a1, %lo(l)(s2)
 ; RV32I-NEXT:    beqz a1, .LBB0_5
 ; RV32I-NEXT:  .LBB0_4: # %if.then
-; RV32I-NEXT:    lw a4, %lo(e)(s1)
-; RV32I-NEXT:    lw a3, %lo(d)(s0)
-; RV32I-NEXT:    lw a2, %lo(c)(s10)
 ; RV32I-NEXT:    lw a1, %lo(b)(s11)
+; RV32I-NEXT:    lw a2, %lo(c)(s10)
+; RV32I-NEXT:    lw a3, %lo(d)(s1)
+; RV32I-NEXT:    lw a4, %lo(e)(s0)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_5: # %if.end
@@ -78,11 +78,11 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB0_7
 ; RV32I-NEXT:  # %bb.6: # %if.then3
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(f)(s8)
-; RV32I-NEXT:    lw a3, %lo(e)(s1)
-; RV32I-NEXT:    lw a2, %lo(d)(s0)
-; RV32I-NEXT:    lw a1, %lo(c)(s10)
 ; RV32I-NEXT:    lw a0, %lo(b)(s11)
+; RV32I-NEXT:    lw a1, %lo(c)(s10)
+; RV32I-NEXT:    lw a2, %lo(d)(s1)
+; RV32I-NEXT:    lw a3, %lo(e)(s0)
+; RV32I-NEXT:    lw a4, %lo(f)(s7)
 ; RV32I-NEXT:    addi a5, zero, 64
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_7: # %if.end5
@@ -91,24 +91,24 @@ define i32 @test() nounwind {
 ; RV32I-NEXT:    beqz a0, .LBB0_9
 ; RV32I-NEXT:  # %bb.8: # %if.then7
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(g)(s7)
-; RV32I-NEXT:    lw a3, %lo(f)(s8)
-; RV32I-NEXT:    lw a2, %lo(e)(s1)
-; RV32I-NEXT:    lw a1, %lo(d)(s0)
 ; RV32I-NEXT:    lw a0, %lo(c)(s10)
+; RV32I-NEXT:    lw a1, %lo(d)(s1)
+; RV32I-NEXT:    lw a2, %lo(e)(s0)
+; RV32I-NEXT:    lw a3, %lo(f)(s7)
+; RV32I-NEXT:    lw a4, %lo(g)(s8)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:  .LBB0_9: # %if.end9
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a0, %lo(i)(s6)
+; RV32I-NEXT:    lw a0, %lo(i)(s5)
 ; RV32I-NEXT:    beqz a0, .LBB0_2
 ; RV32I-NEXT:  # %bb.10: # %if.then11
 ; RV32I-NEXT:    # in Loop: Header=BB0_5 Depth=1
-; RV32I-NEXT:    lw a4, %lo(h)(s5)
-; RV32I-NEXT:    lw a3, %lo(g)(s7)
-; RV32I-NEXT:    lw a2, %lo(f)(s8)
-; RV32I-NEXT:    lw a1, %lo(e)(s1)
-; RV32I-NEXT:    lw a0, %lo(d)(s0)
+; RV32I-NEXT:    lw a0, %lo(d)(s1)
+; RV32I-NEXT:    lw a1, %lo(e)(s0)
+; RV32I-NEXT:    lw a2, %lo(f)(s7)
+; RV32I-NEXT:    lw a3, %lo(g)(s8)
+; RV32I-NEXT:    lw a4, %lo(h)(s9)
 ; RV32I-NEXT:    addi a5, zero, 32
 ; RV32I-NEXT:    call foo
 ; RV32I-NEXT:    j .LBB0_2
diff --git a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
index 5f8952085ae89..13e47242d2a53 100644
--- a/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
+++ b/llvm/test/CodeGen/RISCV/rv64f-float-convert.ll
@@ -113,9 +113,9 @@ define zeroext i32 @bcvt_f32_to_zext_i32(float %a, float %b) nounwind {
 define float @bcvt_i64_to_f32_via_i32(i64 %a, i64 %b) nounwind {
 ; RV64IF-LABEL: bcvt_i64_to_f32_via_i32:
 ; RV64IF:       # %bb.0:
-; RV64IF-NEXT:    fmv.w.x ft0, a1
-; RV64IF-NEXT:    fmv.w.x ft1, a0
-; RV64IF-NEXT:    fadd.s ft0, ft1, ft0
+; RV64IF-NEXT:    fmv.w.x ft0, a0
+; RV64IF-NEXT:    fmv.w.x ft1, a1
+; RV64IF-NEXT:    fadd.s ft0, ft0, ft1
 ; RV64IF-NEXT:    fmv.x.w a0, ft0
 ; RV64IF-NEXT:    ret
   %1 = trunc i64 %a to i32
diff --git a/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
index 89e3f414e5830..a545a13b9c395 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-complex-float.ll
@@ -13,16 +13,16 @@ define i64 @complex_float_add(i64 %a.coerce, i64 %b.coerce) nounwind {
 ; CHECK-NEXT:    sd s0, 16(sp)
 ; CHECK-NEXT:    sd s1, 8(sp)
 ; CHECK-NEXT:    sd s2, 0(sp)
-; CHECK-NEXT:    mv s0, a1
-; CHECK-NEXT:    mv s1, a0
+; CHECK-NEXT:    srli s2, a0, 32
+; CHECK-NEXT:    srli s1, a1, 32
 ; CHECK-NEXT:    call __addsf3
-; CHECK-NEXT:    mv s2, a0
-; CHECK-NEXT:    srli a0, s1, 32
-; CHECK-NEXT:    srli a1, s0, 32
+; CHECK-NEXT:    mv s0, a0
+; CHECK-NEXT:    mv a0, s2
+; CHECK-NEXT:    mv a1, s1
 ; CHECK-NEXT:    call __addsf3
-; CHECK-NEXT:    slli a1, s2, 32
-; CHECK-NEXT:    srli a1, a1, 32
 ; CHECK-NEXT:    slli a0, a0, 32
+; CHECK-NEXT:    slli a1, s0, 32
+; CHECK-NEXT:    srli a1, a1, 32
 ; CHECK-NEXT:    or a0, a0, a1
 ; CHECK-NEXT:    ld s2, 0(sp)
 ; CHECK-NEXT:    ld s1, 8(sp)
diff --git a/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
index 8b177469fe553..c4a4de7681b08 100644
--- a/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
+++ b/llvm/test/CodeGen/RISCV/rv64i-w-insts-legalization.ll
@@ -8,17 +8,17 @@ define signext i32 @addw(i32 signext %s, i32 signext %n, i32 signext %k) nounwin
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
 ; CHECK-NEXT:    not a2, a0
 ; CHECK-NEXT:    add a2, a2, a1
+; CHECK-NEXT:    addi a3, a0, 1
+; CHECK-NEXT:    mul a3, a2, a3
+; CHECK-NEXT:    slli a2, a2, 32
+; CHECK-NEXT:    srli a2, a2, 32
 ; CHECK-NEXT:    sub a1, a1, a0
 ; CHECK-NEXT:    addi a1, a1, -2
 ; CHECK-NEXT:    slli a1, a1, 32
 ; CHECK-NEXT:    srli a1, a1, 32
-; CHECK-NEXT:    slli a3, a2, 32
-; CHECK-NEXT:    srli a3, a3, 32
-; CHECK-NEXT:    mul a1, a3, a1
-; CHECK-NEXT:    addi a3, a0, 1
-; CHECK-NEXT:    mul a2, a2, a3
-; CHECK-NEXT:    add a0, a2, a0
+; CHECK-NEXT:    mul a1, a2, a1
 ; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    add a0, a3, a0
 ; CHECK-NEXT:    addw a0, a0, a1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB0_2:
@@ -54,18 +54,18 @@ define signext i32 @subw(i32 signext %s, i32 signext %n, i32 signext %k) nounwin
 ; CHECK:       # %bb.0: # %entry
 ; CHECK-NEXT:    bge a0, a1, .LBB1_2
 ; CHECK-NEXT:  # %bb.1: # %for.body.preheader
-; CHECK-NEXT:    sub a2, a1, a0
-; CHECK-NEXT:    addi a2, a2, -2
-; CHECK-NEXT:    slli a2, a2, 32
-; CHECK-NEXT:    srli a2, a2, 32
-; CHECK-NEXT:    not a3, a0
-; CHECK-NEXT:    add a1, a3, a1
-; CHECK-NEXT:    slli a4, a1, 32
-; CHECK-NEXT:    srli a4, a4, 32
-; CHECK-NEXT:    mul a2, a4, a2
-; CHECK-NEXT:    mul a1, a1, a3
-; CHECK-NEXT:    sub a0, a1, a0
-; CHECK-NEXT:    srli a1, a2, 1
+; CHECK-NEXT:    not a2, a0
+; CHECK-NEXT:    add a3, a2, a1
+; CHECK-NEXT:    mul a2, a3, a2
+; CHECK-NEXT:    slli a3, a3, 32
+; CHECK-NEXT:    srli a3, a3, 32
+; CHECK-NEXT:    sub a1, a1, a0
+; CHECK-NEXT:    addi a1, a1, -2
+; CHECK-NEXT:    slli a1, a1, 32
+; CHECK-NEXT:    srli a1, a1, 32
+; CHECK-NEXT:    mul a1, a3, a1
+; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    sub a0, a2, a0
 ; CHECK-NEXT:    subw a0, a0, a1
 ; CHECK-NEXT:    ret
 ; CHECK-NEXT:  .LBB1_2:
diff --git a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
index 3a18b4a54ea53..d38f3d5771472 100644
--- a/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
+++ b/llvm/test/CodeGen/RISCV/select-optimize-multiple.ll
@@ -40,27 +40,41 @@ define i128 @cmovcc128(i64 signext %a, i128 %b, i128 %c) nounwind {
 ; RV32I-LABEL: cmovcc128:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    xori a1, a1, 123
-; RV32I-NEXT:    or a1, a1, a2
-; RV32I-NEXT:    beqz a1, .LBB1_2
+; RV32I-NEXT:    or a2, a1, a2
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    beqz a2, .LBB1_2
 ; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a1, a4
+; RV32I-NEXT:  .LBB1_2: # %entry
+; RV32I-NEXT:    lw a6, 0(a1)
+; RV32I-NEXT:    beqz a2, .LBB1_6
+; RV32I-NEXT:  # %bb.3: # %entry
 ; RV32I-NEXT:    addi a1, a4, 4
-; RV32I-NEXT:    addi a2, a4, 8
-; RV32I-NEXT:    addi a5, a4, 12
-; RV32I-NEXT:    mv a3, a4
-; RV32I-NEXT:    j .LBB1_3
-; RV32I-NEXT:  .LBB1_2:
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    bnez a2, .LBB1_7
+; RV32I-NEXT:  .LBB1_4:
+; RV32I-NEXT:    addi a1, a3, 8
+; RV32I-NEXT:    lw a1, 0(a1)
+; RV32I-NEXT:    bnez a2, .LBB1_8
+; RV32I-NEXT:  .LBB1_5:
+; RV32I-NEXT:    addi a2, a3, 12
+; RV32I-NEXT:    j .LBB1_9
+; RV32I-NEXT:  .LBB1_6:
 ; RV32I-NEXT:    addi a1, a3, 4
-; RV32I-NEXT:    addi a2, a3, 8
-; RV32I-NEXT:    addi a5, a3, 12
-; RV32I-NEXT:  .LBB1_3: # %entry
-; RV32I-NEXT:    lw a4, 0(a5)
-; RV32I-NEXT:    sw a4, 12(a0)
-; RV32I-NEXT:    lw a2, 0(a2)
-; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    beqz a2, .LBB1_4
+; RV32I-NEXT:  .LBB1_7: # %entry
+; RV32I-NEXT:    addi a1, a4, 8
 ; RV32I-NEXT:    lw a1, 0(a1)
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a3)
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    beqz a2, .LBB1_5
+; RV32I-NEXT:  .LBB1_8: # %entry
+; RV32I-NEXT:    addi a2, a4, 12
+; RV32I-NEXT:  .LBB1_9: # %entry
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 0(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmovcc128:
@@ -83,24 +97,24 @@ entry:
 define i64 @cmov64(i1 %a, i64 %b, i64 %c) nounwind {
 ; RV32I-LABEL: cmov64:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB2_2
+; RV32I-NEXT:    andi a5, a0, 1
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    bnez a5, .LBB2_2
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    mv a0, a3
 ; RV32I-NEXT:    mv a2, a4
 ; RV32I-NEXT:  .LBB2_2: # %entry
-; RV32I-NEXT:    mv a0, a1
 ; RV32I-NEXT:    mv a1, a2
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmov64:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB2_2
+; RV64I-NEXT:    andi a3, a0, 1
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    bnez a3, .LBB2_2
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    mv a1, a2
+; RV64I-NEXT:    mv a0, a2
 ; RV64I-NEXT:  .LBB2_2: # %entry
-; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    ret
 entry:
   %cond = select i1 %a, i64 %b, i64 %c
@@ -110,38 +124,52 @@ entry:
 define i128 @cmov128(i1 %a, i128 %b, i128 %c) nounwind {
 ; RV32I-LABEL: cmov128:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    andi a1, a1, 1
-; RV32I-NEXT:    bnez a1, .LBB3_2
+; RV32I-NEXT:    andi a4, a1, 1
+; RV32I-NEXT:    mv a1, a2
+; RV32I-NEXT:    bnez a4, .LBB3_2
 ; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:  .LBB3_2: # %entry
+; RV32I-NEXT:    lw a6, 0(a1)
+; RV32I-NEXT:    bnez a4, .LBB3_6
+; RV32I-NEXT:  # %bb.3: # %entry
 ; RV32I-NEXT:    addi a1, a3, 4
-; RV32I-NEXT:    addi a4, a3, 8
-; RV32I-NEXT:    addi a5, a3, 12
-; RV32I-NEXT:    mv a2, a3
-; RV32I-NEXT:    j .LBB3_3
-; RV32I-NEXT:  .LBB3_2:
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    beqz a4, .LBB3_7
+; RV32I-NEXT:  .LBB3_4:
+; RV32I-NEXT:    addi a1, a2, 8
+; RV32I-NEXT:    lw a1, 0(a1)
+; RV32I-NEXT:    beqz a4, .LBB3_8
+; RV32I-NEXT:  .LBB3_5:
+; RV32I-NEXT:    addi a2, a2, 12
+; RV32I-NEXT:    j .LBB3_9
+; RV32I-NEXT:  .LBB3_6:
 ; RV32I-NEXT:    addi a1, a2, 4
-; RV32I-NEXT:    addi a4, a2, 8
-; RV32I-NEXT:    addi a5, a2, 12
-; RV32I-NEXT:  .LBB3_3: # %entry
-; RV32I-NEXT:    lw a3, 0(a5)
-; RV32I-NEXT:    sw a3, 12(a0)
-; RV32I-NEXT:    lw a3, 0(a4)
-; RV32I-NEXT:    sw a3, 8(a0)
+; RV32I-NEXT:    lw a5, 0(a1)
+; RV32I-NEXT:    bnez a4, .LBB3_4
+; RV32I-NEXT:  .LBB3_7: # %entry
+; RV32I-NEXT:    addi a1, a3, 8
 ; RV32I-NEXT:    lw a1, 0(a1)
-; RV32I-NEXT:    sw a1, 4(a0)
-; RV32I-NEXT:    lw a1, 0(a2)
-; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    bnez a4, .LBB3_5
+; RV32I-NEXT:  .LBB3_8: # %entry
+; RV32I-NEXT:    addi a2, a3, 12
+; RV32I-NEXT:  .LBB3_9: # %entry
+; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    sw a1, 8(a0)
+; RV32I-NEXT:    sw a5, 4(a0)
+; RV32I-NEXT:    sw a6, 0(a0)
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmov128:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB3_2
+; RV64I-NEXT:    andi a5, a0, 1
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    bnez a5, .LBB3_2
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    mv a1, a3
+; RV64I-NEXT:    mv a0, a3
 ; RV64I-NEXT:    mv a2, a4
 ; RV64I-NEXT:  .LBB3_2: # %entry
-; RV64I-NEXT:    mv a0, a1
 ; RV64I-NEXT:    mv a1, a2
 ; RV64I-NEXT:    ret
 entry:
@@ -280,39 +308,37 @@ entry:
 define i32 @cmovdiffcc(i1 %a, i1 %b, i32 %c, i32 %d, i32 %e, i32 %f) nounwind {
 ; RV32I-LABEL: cmovdiffcc:
 ; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    andi a0, a0, 1
 ; RV32I-NEXT:    andi a1, a1, 1
-; RV32I-NEXT:    beqz a1, .LBB7_3
+; RV32I-NEXT:    beqz a0, .LBB7_3
 ; RV32I-NEXT:  # %bb.1: # %entry
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    beqz a0, .LBB7_4
+; RV32I-NEXT:    beqz a1, .LBB7_4
 ; RV32I-NEXT:  .LBB7_2: # %entry
 ; RV32I-NEXT:    add a0, a2, a4
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB7_3: # %entry
-; RV32I-NEXT:    mv a4, a5
-; RV32I-NEXT:    andi a0, a0, 1
-; RV32I-NEXT:    bnez a0, .LBB7_2
-; RV32I-NEXT:  .LBB7_4: # %entry
 ; RV32I-NEXT:    mv a2, a3
+; RV32I-NEXT:    bnez a1, .LBB7_2
+; RV32I-NEXT:  .LBB7_4: # %entry
+; RV32I-NEXT:    mv a4, a5
 ; RV32I-NEXT:    add a0, a2, a4
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: cmovdiffcc:
 ; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    andi a0, a0, 1
 ; RV64I-NEXT:    andi a1, a1, 1
-; RV64I-NEXT:    beqz a1, .LBB7_3
+; RV64I-NEXT:    beqz a0, .LBB7_3
 ; RV64I-NEXT:  # %bb.1: # %entry
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    beqz a0, .LBB7_4
+; RV64I-NEXT:    beqz a1, .LBB7_4
 ; RV64I-NEXT:  .LBB7_2: # %entry
 ; RV64I-NEXT:    addw a0, a2, a4
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB7_3: # %entry
-; RV64I-NEXT:    mv a4, a5
-; RV64I-NEXT:    andi a0, a0, 1
-; RV64I-NEXT:    bnez a0, .LBB7_2
-; RV64I-NEXT:  .LBB7_4: # %entry
 ; RV64I-NEXT:    mv a2, a3
+; RV64I-NEXT:    bnez a1, .LBB7_2
+; RV64I-NEXT:  .LBB7_4: # %entry
+; RV64I-NEXT:    mv a4, a5
 ; RV64I-NEXT:    addw a0, a2, a4
 ; RV64I-NEXT:    ret
 entry:
diff --git a/llvm/test/CodeGen/RISCV/setcc-logic.ll b/llvm/test/CodeGen/RISCV/setcc-logic.ll
index 72f797db15a64..0c29bf505234e 100644
--- a/llvm/test/CodeGen/RISCV/setcc-logic.ll
+++ b/llvm/test/CodeGen/RISCV/setcc-logic.ll
@@ -7,17 +7,17 @@
 define i1 @and_icmp_eq(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV32I-LABEL: and_icmp_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a2, a2, a3
 ; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    xor a1, a2, a3
+; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    seqz a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: and_icmp_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a2, a2, a3
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    xor a1, a2, a3
+; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    seqz a0, a0
@@ -31,17 +31,17 @@ define i1 @and_icmp_eq(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 define i1 @or_icmp_ne(i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
 ; RV32I-LABEL: or_icmp_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a2, a2, a3
 ; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    xor a1, a2, a3
+; RV32I-NEXT:    or a0, a0, a1
 ; RV32I-NEXT:    snez a0, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_icmp_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a2, a2, a3
 ; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    xor a1, a2, a3
+; RV64I-NEXT:    or a0, a0, a1
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
 ; RV64I-NEXT:    snez a0, a0
@@ -102,22 +102,22 @@ define i1 @and_icmps_const_1bit_diff(i32 %x) nounwind {
 define i1 @and_icmps_const_not1bit_diff(i32 %x) nounwind {
 ; RV32I-LABEL: and_icmps_const_not1bit_diff:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xori a1, a0, 92
+; RV32I-NEXT:    xori a1, a0, 44
 ; RV32I-NEXT:    snez a1, a1
-; RV32I-NEXT:    xori a0, a0, 44
+; RV32I-NEXT:    xori a0, a0, 92
 ; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    and a0, a0, a1
+; RV32I-NEXT:    and a0, a1, a0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: and_icmps_const_not1bit_diff:
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    slli a0, a0, 32
 ; RV64I-NEXT:    srli a0, a0, 32
-; RV64I-NEXT:    xori a1, a0, 92
+; RV64I-NEXT:    xori a1, a0, 44
 ; RV64I-NEXT:    snez a1, a1
-; RV64I-NEXT:    xori a0, a0, 44
+; RV64I-NEXT:    xori a0, a0, 92
 ; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    and a0, a0, a1
+; RV64I-NEXT:    and a0, a1, a0
 ; RV64I-NEXT:    ret
   %a = icmp ne i32 %x, 44
   %b = icmp ne i32 %x, 92
diff --git a/llvm/test/CodeGen/RISCV/shifts.ll b/llvm/test/CodeGen/RISCV/shifts.ll
index 80cf136144d7f..54c49f3f3ef66 100644
--- a/llvm/test/CodeGen/RISCV/shifts.ll
+++ b/llvm/test/CodeGen/RISCV/shifts.ll
@@ -17,13 +17,14 @@ define i64 @lshr64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a1, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB0_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
-; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    srl a2, a1, a2
+; RV32I-NEXT:    mv a1, a2
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: lshr64:
@@ -62,11 +63,11 @@ define i64 @ashr64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    srai a1, a1, 31
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB2_2:
+; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    slli a4, a1, 1
 ; RV32I-NEXT:    sll a3, a4, a3
-; RV32I-NEXT:    srl a0, a0, a2
 ; RV32I-NEXT:    or a0, a0, a3
 ; RV32I-NEXT:    sra a1, a1, a2
 ; RV32I-NEXT:    ret
@@ -107,13 +108,14 @@ define i64 @shl64(i64 %a, i64 %b) nounwind {
 ; RV32I-NEXT:    mv a0, zero
 ; RV32I-NEXT:    ret
 ; RV32I-NEXT:  .LBB4_2:
+; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    addi a3, zero, 31
 ; RV32I-NEXT:    sub a3, a3, a2
 ; RV32I-NEXT:    srli a4, a0, 1
 ; RV32I-NEXT:    srl a3, a4, a3
-; RV32I-NEXT:    sll a1, a1, a2
 ; RV32I-NEXT:    or a1, a1, a3
-; RV32I-NEXT:    sll a0, a0, a2
+; RV32I-NEXT:    sll a2, a0, a2
+; RV32I-NEXT:    mv a0, a2
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: shl64:
@@ -148,27 +150,27 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __lshrti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -183,13 +185,14 @@ define i128 @lshr128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    mv a1, zero
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB6_2:
+; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    slli a4, a1, 1
 ; RV64I-NEXT:    sll a3, a4, a3
-; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    or a0, a0, a3
-; RV64I-NEXT:    srl a1, a1, a2
+; RV64I-NEXT:    srl a2, a1, a2
+; RV64I-NEXT:    mv a1, a2
 ; RV64I-NEXT:    ret
   %1 = lshr i128 %a, %b
   ret i128 %1
@@ -201,27 +204,27 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __ashrti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -236,11 +239,11 @@ define i128 @ashr128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    srai a1, a1, 63
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB7_2:
+; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    slli a4, a1, 1
 ; RV64I-NEXT:    sll a3, a4, a3
-; RV64I-NEXT:    srl a0, a0, a2
 ; RV64I-NEXT:    or a0, a0, a3
 ; RV64I-NEXT:    sra a1, a1, a2
 ; RV64I-NEXT:    ret
@@ -254,27 +257,27 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV32I-NEXT:    addi sp, sp, -48
 ; RV32I-NEXT:    sw ra, 44(sp)
 ; RV32I-NEXT:    sw s0, 40(sp)
-; RV32I-NEXT:    mv s0, a0
-; RV32I-NEXT:    lw a0, 12(a1)
-; RV32I-NEXT:    sw a0, 20(sp)
-; RV32I-NEXT:    lw a0, 8(a1)
-; RV32I-NEXT:    sw a0, 16(sp)
-; RV32I-NEXT:    lw a0, 4(a1)
-; RV32I-NEXT:    sw a0, 12(sp)
-; RV32I-NEXT:    lw a0, 0(a1)
-; RV32I-NEXT:    sw a0, 8(sp)
 ; RV32I-NEXT:    lw a2, 0(a2)
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    mv s0, a0
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a5, 16(sp)
+; RV32I-NEXT:    sw a4, 12(sp)
 ; RV32I-NEXT:    addi a0, sp, 24
 ; RV32I-NEXT:    addi a1, sp, 8
+; RV32I-NEXT:    sw a3, 8(sp)
 ; RV32I-NEXT:    call __ashlti3
 ; RV32I-NEXT:    lw a0, 36(sp)
+; RV32I-NEXT:    lw a1, 32(sp)
+; RV32I-NEXT:    lw a2, 28(sp)
+; RV32I-NEXT:    lw a3, 24(sp)
 ; RV32I-NEXT:    sw a0, 12(s0)
-; RV32I-NEXT:    lw a0, 32(sp)
-; RV32I-NEXT:    sw a0, 8(s0)
-; RV32I-NEXT:    lw a0, 28(sp)
-; RV32I-NEXT:    sw a0, 4(s0)
-; RV32I-NEXT:    lw a0, 24(sp)
-; RV32I-NEXT:    sw a0, 0(s0)
+; RV32I-NEXT:    sw a1, 8(s0)
+; RV32I-NEXT:    sw a2, 4(s0)
+; RV32I-NEXT:    sw a3, 0(s0)
 ; RV32I-NEXT:    lw s0, 40(sp)
 ; RV32I-NEXT:    lw ra, 44(sp)
 ; RV32I-NEXT:    addi sp, sp, 48
@@ -289,13 +292,14 @@ define i128 @shl128(i128 %a, i128 %b) nounwind {
 ; RV64I-NEXT:    mv a0, zero
 ; RV64I-NEXT:    ret
 ; RV64I-NEXT:  .LBB8_2:
+; RV64I-NEXT:    sll a1, a1, a2
 ; RV64I-NEXT:    addi a3, zero, 63
 ; RV64I-NEXT:    sub a3, a3, a2
 ; RV64I-NEXT:    srli a4, a0, 1
 ; RV64I-NEXT:    srl a3, a4, a3
-; RV64I-NEXT:    sll a1, a1, a2
 ; RV64I-NEXT:    or a1, a1, a3
-; RV64I-NEXT:    sll a0, a0, a2
+; RV64I-NEXT:    sll a2, a0, a2
+; RV64I-NEXT:    mv a0, a2
 ; RV64I-NEXT:    ret
   %1 = shl i128 %a, %b
   ret i128 %1
diff --git a/llvm/test/CodeGen/RISCV/split-offsets.ll b/llvm/test/CodeGen/RISCV/split-offsets.ll
index 731f6cacd6a58..8642425344f69 100644
--- a/llvm/test/CodeGen/RISCV/split-offsets.ll
+++ b/llvm/test/CodeGen/RISCV/split-offsets.ll
@@ -11,33 +11,33 @@
 define void @test1([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
 ; RV32I-LABEL: test1:
 ; RV32I:       # %bb.0: # %entry
+; RV32I-NEXT:    lw a0, 0(a0)
 ; RV32I-NEXT:    lui a2, 20
 ; RV32I-NEXT:    addi a2, a2, -1920
-; RV32I-NEXT:    lw a0, 0(a0)
+; RV32I-NEXT:    add a1, a1, a2
 ; RV32I-NEXT:    add a0, a0, a2
+; RV32I-NEXT:    addi a2, zero, 2
+; RV32I-NEXT:    sw a2, 0(a0)
 ; RV32I-NEXT:    addi a3, zero, 1
 ; RV32I-NEXT:    sw a3, 4(a0)
-; RV32I-NEXT:    addi a4, zero, 2
-; RV32I-NEXT:    sw a4, 0(a0)
-; RV32I-NEXT:    add a0, a1, a2
-; RV32I-NEXT:    sw a4, 4(a0)
-; RV32I-NEXT:    sw a3, 0(a0)
+; RV32I-NEXT:    sw a3, 0(a1)
+; RV32I-NEXT:    sw a2, 4(a1)
 ; RV32I-NEXT:    .cfi_def_cfa_offset 0
 ; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: test1:
 ; RV64I:       # %bb.0: # %entry
+; RV64I-NEXT:    ld a0, 0(a0)
 ; RV64I-NEXT:    lui a2, 20
 ; RV64I-NEXT:    addiw a2, a2, -1920
-; RV64I-NEXT:    ld a0, 0(a0)
+; RV64I-NEXT:    add a1, a1, a2
 ; RV64I-NEXT:    add a0, a0, a2
+; RV64I-NEXT:    addi a2, zero, 2
+; RV64I-NEXT:    sw a2, 0(a0)
 ; RV64I-NEXT:    addi a3, zero, 1
 ; RV64I-NEXT:    sw a3, 4(a0)
-; RV64I-NEXT:    addi a4, zero, 2
-; RV64I-NEXT:    sw a4, 0(a0)
-; RV64I-NEXT:    add a0, a1, a2
-; RV64I-NEXT:    sw a4, 4(a0)
-; RV64I-NEXT:    sw a3, 0(a0)
+; RV64I-NEXT:    sw a3, 0(a1)
+; RV64I-NEXT:    sw a2, 4(a1)
 ; RV64I-NEXT:    .cfi_def_cfa_offset 0
 ; RV64I-NEXT:    ret
 entry:
@@ -57,20 +57,20 @@ entry:
 define void @test2([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
 ; RV32I-LABEL: test2:
 ; RV32I:       # %bb.0: # %entry
-; RV32I-NEXT:    lui a3, 20
-; RV32I-NEXT:    addi a3, a3, -1920
-; RV32I-NEXT:    lw a0, 0(a0)
-; RV32I-NEXT:    add a0, a0, a3
-; RV32I-NEXT:    add a1, a1, a3
 ; RV32I-NEXT:    mv a3, zero
+; RV32I-NEXT:    lw a4, 0(a0)
+; RV32I-NEXT:    lui a0, 20
+; RV32I-NEXT:    addi a5, a0, -1920
+; RV32I-NEXT:    add a0, a1, a5
+; RV32I-NEXT:    add a1, a4, a5
 ; RV32I-NEXT:    bge a3, a2, .LBB1_2
 ; RV32I-NEXT:  .LBB1_1: # %while_body
 ; RV32I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    addi a4, a3, 1
-; RV32I-NEXT:    sw a4, 0(a0)
-; RV32I-NEXT:    sw a3, 4(a1)
 ; RV32I-NEXT:    sw a4, 0(a1)
+; RV32I-NEXT:    sw a3, 4(a1)
+; RV32I-NEXT:    sw a4, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
 ; RV32I-NEXT:    mv a3, a4
 ; RV32I-NEXT:    blt a3, a2, .LBB1_1
 ; RV32I-NEXT:  .LBB1_2: # %while_end
@@ -79,22 +79,22 @@ define void @test2([65536 x i32]** %sp, [65536 x i32]* %t, i32 %n) {
 ;
 ; RV64I-LABEL: test2:
 ; RV64I:       # %bb.0: # %entry
-; RV64I-NEXT:    lui a3, 20
-; RV64I-NEXT:    addiw a3, a3, -1920
-; RV64I-NEXT:    ld a0, 0(a0)
-; RV64I-NEXT:    add a0, a0, a3
-; RV64I-NEXT:    add a1, a1, a3
-; RV64I-NEXT:    sext.w a2, a2
 ; RV64I-NEXT:    mv a3, zero
+; RV64I-NEXT:    ld a4, 0(a0)
+; RV64I-NEXT:    lui a0, 20
+; RV64I-NEXT:    addiw a5, a0, -1920
+; RV64I-NEXT:    add a0, a1, a5
+; RV64I-NEXT:    add a1, a4, a5
+; RV64I-NEXT:    sext.w a2, a2
 ; RV64I-NEXT:    sext.w a4, a3
 ; RV64I-NEXT:    bge a4, a2, .LBB1_2
 ; RV64I-NEXT:  .LBB1_1: # %while_body
 ; RV64I-NEXT:    # =>This Inner Loop Header: Depth=1
-; RV64I-NEXT:    sw a3, 4(a0)
 ; RV64I-NEXT:    addi a4, a3, 1
-; RV64I-NEXT:    sw a4, 0(a0)
-; RV64I-NEXT:    sw a3, 4(a1)
 ; RV64I-NEXT:    sw a4, 0(a1)
+; RV64I-NEXT:    sw a3, 4(a1)
+; RV64I-NEXT:    sw a4, 0(a0)
+; RV64I-NEXT:    sw a3, 4(a0)
 ; RV64I-NEXT:    mv a3, a4
 ; RV64I-NEXT:    sext.w a4, a3
 ; RV64I-NEXT:    blt a4, a2, .LBB1_1
diff --git a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
index e0ac6483a5d65..30346a134a785 100644
--- a/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/RISCV/umulo-128-legalisation-lowering.ll
@@ -4,111 +4,113 @@
 define { i128, i8 } @muloti_test(i128 %l, i128 %r) #0 {
 ; RISCV32-LABEL: muloti_test:
 ; RISCV32:       # %bb.0: # %start
-; RISCV32-NEXT:    addi sp, sp, -80
-; RISCV32-NEXT:    sw ra, 76(sp)
-; RISCV32-NEXT:    sw s0, 72(sp)
-; RISCV32-NEXT:    sw s1, 68(sp)
-; RISCV32-NEXT:    sw s2, 64(sp)
-; RISCV32-NEXT:    sw s3, 60(sp)
-; RISCV32-NEXT:    sw s4, 56(sp)
-; RISCV32-NEXT:    sw s5, 52(sp)
-; RISCV32-NEXT:    sw s6, 48(sp)
-; RISCV32-NEXT:    mv s1, a2
-; RISCV32-NEXT:    mv s0, a1
-; RISCV32-NEXT:    mv s2, a0
-; RISCV32-NEXT:    sw zero, 12(sp)
-; RISCV32-NEXT:    sw zero, 8(sp)
-; RISCV32-NEXT:    sw zero, 28(sp)
-; RISCV32-NEXT:    sw zero, 24(sp)
-; RISCV32-NEXT:    lw s3, 4(a2)
-; RISCV32-NEXT:    sw s3, 4(sp)
-; RISCV32-NEXT:    lw s5, 0(a2)
-; RISCV32-NEXT:    sw s5, 0(sp)
-; RISCV32-NEXT:    lw s4, 4(a1)
-; RISCV32-NEXT:    sw s4, 20(sp)
-; RISCV32-NEXT:    lw s6, 0(a1)
-; RISCV32-NEXT:    sw s6, 16(sp)
-; RISCV32-NEXT:    addi a0, sp, 32
-; RISCV32-NEXT:    addi a1, sp, 16
-; RISCV32-NEXT:    mv a2, sp
+; RISCV32-NEXT:    addi sp, sp, -96
+; RISCV32-NEXT:    sw ra, 92(sp)
+; RISCV32-NEXT:    sw s0, 88(sp)
+; RISCV32-NEXT:    sw s1, 84(sp)
+; RISCV32-NEXT:    sw s2, 80(sp)
+; RISCV32-NEXT:    sw s3, 76(sp)
+; RISCV32-NEXT:    sw s4, 72(sp)
+; RISCV32-NEXT:    sw s5, 68(sp)
+; RISCV32-NEXT:    sw s6, 64(sp)
+; RISCV32-NEXT:    sw s7, 60(sp)
+; RISCV32-NEXT:    sw s8, 56(sp)
+; RISCV32-NEXT:    lw s2, 12(a1)
+; RISCV32-NEXT:    lw s6, 8(a1)
+; RISCV32-NEXT:    lw s3, 12(a2)
+; RISCV32-NEXT:    lw s7, 8(a2)
+; RISCV32-NEXT:    lw s0, 0(a1)
+; RISCV32-NEXT:    lw s8, 4(a1)
+; RISCV32-NEXT:    lw s1, 0(a2)
+; RISCV32-NEXT:    lw s5, 4(a2)
+; RISCV32-NEXT:    mv s4, a0
+; RISCV32-NEXT:    sw zero, 20(sp)
+; RISCV32-NEXT:    sw zero, 16(sp)
+; RISCV32-NEXT:    sw zero, 36(sp)
+; RISCV32-NEXT:    sw zero, 32(sp)
+; RISCV32-NEXT:    sw s5, 12(sp)
+; RISCV32-NEXT:    sw s1, 8(sp)
+; RISCV32-NEXT:    sw s8, 28(sp)
+; RISCV32-NEXT:    addi a0, sp, 40
+; RISCV32-NEXT:    addi a1, sp, 24
+; RISCV32-NEXT:    addi a2, sp, 8
+; RISCV32-NEXT:    sw s0, 24(sp)
 ; RISCV32-NEXT:    call __multi3
-; RISCV32-NEXT:    lw a0, 12(s0)
-; RISCV32-NEXT:    lw a1, 8(s0)
-; RISCV32-NEXT:    mul a2, s3, a1
-; RISCV32-NEXT:    mul a3, a0, s5
-; RISCV32-NEXT:    add a4, a3, a2
-; RISCV32-NEXT:    lw a2, 12(s1)
-; RISCV32-NEXT:    lw a3, 8(s1)
-; RISCV32-NEXT:    mul a5, s4, a3
-; RISCV32-NEXT:    mul s1, a2, s6
-; RISCV32-NEXT:    add a5, s1, a5
-; RISCV32-NEXT:    mul s1, a3, s6
-; RISCV32-NEXT:    mul s0, a1, s5
-; RISCV32-NEXT:    add s1, s0, s1
-; RISCV32-NEXT:    sltu s0, s1, s0
-; RISCV32-NEXT:    mulhu a6, a3, s6
-; RISCV32-NEXT:    add t1, a6, a5
-; RISCV32-NEXT:    mulhu t2, a1, s5
-; RISCV32-NEXT:    add t3, t2, a4
-; RISCV32-NEXT:    add a5, t3, t1
-; RISCV32-NEXT:    add a5, a5, s0
-; RISCV32-NEXT:    lw s0, 44(sp)
-; RISCV32-NEXT:    add a5, s0, a5
-; RISCV32-NEXT:    lw a4, 40(sp)
-; RISCV32-NEXT:    add a7, a4, s1
-; RISCV32-NEXT:    sltu t0, a7, a4
-; RISCV32-NEXT:    add a5, a5, t0
-; RISCV32-NEXT:    beq a5, s0, .LBB0_2
+; RISCV32-NEXT:    mul a0, s8, s7
+; RISCV32-NEXT:    mul a1, s3, s0
+; RISCV32-NEXT:    add a0, a1, a0
+; RISCV32-NEXT:    mulhu a5, s7, s0
+; RISCV32-NEXT:    add a0, a5, a0
+; RISCV32-NEXT:    mul a1, s5, s6
+; RISCV32-NEXT:    mul a2, s2, s1
+; RISCV32-NEXT:    add a1, a2, a1
+; RISCV32-NEXT:    mulhu t0, s6, s1
+; RISCV32-NEXT:    add t1, t0, a1
+; RISCV32-NEXT:    add a6, t1, a0
+; RISCV32-NEXT:    mul a1, s7, s0
+; RISCV32-NEXT:    mul a3, s6, s1
+; RISCV32-NEXT:    add a4, a3, a1
+; RISCV32-NEXT:    lw a1, 52(sp)
+; RISCV32-NEXT:    lw a2, 48(sp)
+; RISCV32-NEXT:    sltu a3, a4, a3
+; RISCV32-NEXT:    add a3, a6, a3
+; RISCV32-NEXT:    add a3, a1, a3
+; RISCV32-NEXT:    add a6, a2, a4
+; RISCV32-NEXT:    sltu a2, a6, a2
+; RISCV32-NEXT:    add a7, a3, a2
+; RISCV32-NEXT:    beq a7, a1, .LBB0_2
 ; RISCV32-NEXT:  # %bb.1: # %start
-; RISCV32-NEXT:    sltu t0, a5, s0
+; RISCV32-NEXT:    sltu a2, a7, a1
 ; RISCV32-NEXT:  .LBB0_2: # %start
-; RISCV32-NEXT:    snez a4, s3
-; RISCV32-NEXT:    snez s1, a0
-; RISCV32-NEXT:    and a4, s1, a4
-; RISCV32-NEXT:    snez s1, s4
-; RISCV32-NEXT:    snez s0, a2
-; RISCV32-NEXT:    and s1, s0, s1
-; RISCV32-NEXT:    mulhu s0, a2, s6
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or s1, s1, s0
-; RISCV32-NEXT:    mulhu s0, a0, s5
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or a4, a4, s0
-; RISCV32-NEXT:    sltu t2, t3, t2
-; RISCV32-NEXT:    mulhu s0, s3, a1
-; RISCV32-NEXT:    snez s0, s0
-; RISCV32-NEXT:    or t3, a4, s0
-; RISCV32-NEXT:    sltu s0, t1, a6
-; RISCV32-NEXT:    mulhu a4, s4, a3
+; RISCV32-NEXT:    sltu a0, a0, a5
+; RISCV32-NEXT:    snez a1, s8
+; RISCV32-NEXT:    snez a3, s3
+; RISCV32-NEXT:    and a1, a3, a1
+; RISCV32-NEXT:    mulhu a3, s3, s0
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    mulhu a3, s8, s7
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a1, a1, a3
+; RISCV32-NEXT:    or a0, a1, a0
+; RISCV32-NEXT:    sltu a1, t1, t0
+; RISCV32-NEXT:    snez a3, s5
+; RISCV32-NEXT:    snez a4, s2
+; RISCV32-NEXT:    and a3, a4, a3
+; RISCV32-NEXT:    mulhu a4, s2, s1
+; RISCV32-NEXT:    snez a4, a4
+; RISCV32-NEXT:    or a3, a3, a4
+; RISCV32-NEXT:    mulhu a4, s5, s6
+; RISCV32-NEXT:    snez a4, a4
+; RISCV32-NEXT:    or a3, a3, a4
+; RISCV32-NEXT:    or a1, a3, a1
+; RISCV32-NEXT:    or a3, s7, s3
+; RISCV32-NEXT:    snez a3, a3
+; RISCV32-NEXT:    or a4, s6, s2
 ; RISCV32-NEXT:    snez a4, a4
-; RISCV32-NEXT:    or a4, s1, a4
-; RISCV32-NEXT:    lw s1, 36(sp)
-; RISCV32-NEXT:    sw s1, 4(s2)
-; RISCV32-NEXT:    lw s1, 32(sp)
-; RISCV32-NEXT:    sw s1, 0(s2)
-; RISCV32-NEXT:    sw a7, 8(s2)
-; RISCV32-NEXT:    sw a5, 12(s2)
-; RISCV32-NEXT:    or a4, a4, s0
-; RISCV32-NEXT:    or a5, t3, t2
+; RISCV32-NEXT:    and a3, a4, a3
+; RISCV32-NEXT:    or a1, a3, a1
 ; RISCV32-NEXT:    or a0, a1, a0
-; RISCV32-NEXT:    or a1, a3, a2
-; RISCV32-NEXT:    snez a1, a1
-; RISCV32-NEXT:    snez a0, a0
-; RISCV32-NEXT:    and a0, a0, a1
-; RISCV32-NEXT:    or a0, a0, a5
-; RISCV32-NEXT:    or a0, a0, a4
-; RISCV32-NEXT:    or a0, a0, t0
+; RISCV32-NEXT:    lw a1, 44(sp)
+; RISCV32-NEXT:    lw a3, 40(sp)
+; RISCV32-NEXT:    or a0, a0, a2
 ; RISCV32-NEXT:    andi a0, a0, 1
-; RISCV32-NEXT:    sb a0, 16(s2)
-; RISCV32-NEXT:    lw s6, 48(sp)
-; RISCV32-NEXT:    lw s5, 52(sp)
-; RISCV32-NEXT:    lw s4, 56(sp)
-; RISCV32-NEXT:    lw s3, 60(sp)
-; RISCV32-NEXT:    lw s2, 64(sp)
-; RISCV32-NEXT:    lw s1, 68(sp)
-; RISCV32-NEXT:    lw s0, 72(sp)
-; RISCV32-NEXT:    lw ra, 76(sp)
-; RISCV32-NEXT:    addi sp, sp, 80
+; RISCV32-NEXT:    sw a1, 4(s4)
+; RISCV32-NEXT:    sw a3, 0(s4)
+; RISCV32-NEXT:    sw a6, 8(s4)
+; RISCV32-NEXT:    sw a7, 12(s4)
+; RISCV32-NEXT:    sb a0, 16(s4)
+; RISCV32-NEXT:    lw s8, 56(sp)
+; RISCV32-NEXT:    lw s7, 60(sp)
+; RISCV32-NEXT:    lw s6, 64(sp)
+; RISCV32-NEXT:    lw s5, 68(sp)
+; RISCV32-NEXT:    lw s4, 72(sp)
+; RISCV32-NEXT:    lw s3, 76(sp)
+; RISCV32-NEXT:    lw s2, 80(sp)
+; RISCV32-NEXT:    lw s1, 84(sp)
+; RISCV32-NEXT:    lw s0, 88(sp)
+; RISCV32-NEXT:    lw ra, 92(sp)
+; RISCV32-NEXT:    addi sp, sp, 96
 ; RISCV32-NEXT:    ret
 start:
   %0 = tail call { i128, i1 } @llvm.umul.with.overflow.i128(i128 %l, i128 %r) #2
diff --git a/llvm/test/CodeGen/RISCV/vararg.ll b/llvm/test/CodeGen/RISCV/vararg.ll
index b630a1b436f58..9b906cb2c12e6 100644
--- a/llvm/test/CodeGen/RISCV/vararg.ll
+++ b/llvm/test/CodeGen/RISCV/vararg.ll
@@ -46,9 +46,9 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
@@ -65,9 +65,9 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -83,9 +83,9 @@ define i32 @va1(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
@@ -150,9 +150,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 24
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
@@ -169,9 +169,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -187,9 +187,9 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 24
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
@@ -203,10 +203,10 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -223,10 +223,10 @@ define i32 @va1_va_arg(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -256,9 +256,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, -16(s0)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a1, 15
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sub a0, sp, a0
@@ -286,9 +286,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 16(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 8
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -16(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a1, 15
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -16
 ; ILP32-ILP32F-WITHFP-NEXT:    sub a0, sp, a0
@@ -316,9 +316,9 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 8(s0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 4(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, s0, 8
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, -16(s0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 4(s0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a1, 15
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sub a0, sp, a0
@@ -346,17 +346,17 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, s0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, -32(s0)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 33
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 32
-; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a1, a1, 32
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    and a0, a1, a0
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd s1, 8(s0)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a1, 32
+; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a0, a0, 32
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 15
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, zero, 1
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 33
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, -16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    mv sp, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call notdead
@@ -382,17 +382,17 @@ define i32 @va1_va_arg_alloca(i8* %fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -32(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 33
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -16
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a1, a1, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a1, a0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a1, 32
+; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a0, a0, 32
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 15
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, zero, 1
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 33
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -16
+; LP64-LP64F-LP64D-WITHFP-NEXT:    and a0, a0, a1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sub a0, sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv sp, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call notdead
@@ -419,9 +419,9 @@ define void @va1_caller() nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 2
+; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va1
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -433,9 +433,9 @@ define void @va1_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 2
+; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -446,9 +446,9 @@ define void @va1_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 2
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -498,10 +498,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 35
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a1, a0, -8
+; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 35
+; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a1)
 ; ILP32-ILP32F-FPELIM-NEXT:    ori a1, a1, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a1, 0(a1)
@@ -521,10 +521,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a1, a0, -8
+; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
+; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a1)
 ; ILP32-ILP32F-WITHFP-NEXT:    ori a1, a1, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a1, 0(a1)
@@ -543,10 +543,10 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 35
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a1, a0, -8
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 35
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a1)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a1, a1, 4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a1, 0(a1)
@@ -556,8 +556,6 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va2:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -80
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 24
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 72(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 64(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 56(sp)
@@ -565,6 +563,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 24
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a0, 32
@@ -585,8 +585,6 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 56(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 48(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 40(s0)
@@ -594,6 +592,8 @@ define i64 @va2(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, s0, 8
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a0, 32
@@ -705,10 +705,10 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 24
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 24(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -725,10 +725,10 @@ define i64 @va2_va_arg(i8 *%fmt, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, s0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 8(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
@@ -747,8 +747,8 @@ define void @va2_caller() nounwind {
 ; ILP32-ILP32F-FPELIM:       # %bb.0:
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, -16
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a3, 261888
+; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    call va2
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -760,8 +760,8 @@ define void @va2_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
-; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a3, 261888
+; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    call va2
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -772,8 +772,8 @@ define void @va2_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM:       # %bb.0:
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, -16
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a3, 261888
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -819,16 +819,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 20(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 27
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, sp, 19
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -8
-; ILP32-ILP32F-FPELIM-NEXT:    ori a3, a0, 4
-; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
-; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
-; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
+; ILP32-ILP32F-FPELIM-NEXT:    addi a3, sp, 27
+; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 4(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a0)
+; ILP32-ILP32F-FPELIM-NEXT:    ori a0, a0, 4
+; ILP32-ILP32F-FPELIM-NEXT:    lw a4, 0(a0)
+; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a4
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 32
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
@@ -844,16 +844,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a5, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 4(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 19
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, s0, 11
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -8
-; ILP32-ILP32F-WITHFP-NEXT:    ori a3, a0, 4
-; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
-; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
-; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
+; ILP32-ILP32F-WITHFP-NEXT:    addi a3, s0, 19
+; ILP32-ILP32F-WITHFP-NEXT:    sw a3, -12(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a0)
+; ILP32-ILP32F-WITHFP-NEXT:    ori a0, a0, 4
+; ILP32-ILP32F-WITHFP-NEXT:    lw a4, 0(a0)
+; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a4
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
@@ -868,16 +868,16 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 20(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a4, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 27
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 4(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, sp, 19
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -8
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a3, a0, 4
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 0(a3)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a3
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, sp, 27
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 4(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 0(a0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ori a0, a0, 4
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a4, 0(a0)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sltu a1, a0, a1
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a2, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 32
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
@@ -885,15 +885,15 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-LABEL: va3:
 ; LP64-LP64F-LP64D-FPELIM:       # %bb.0:
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, -64
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a7, 56(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a6, 48(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lw a0, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a2, a0, 32
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    srli a2, a2, 32
@@ -914,15 +914,15 @@ define i64 @va3(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi s0, sp, 32
-; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a7, 40(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a6, 32(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lw a0, -24(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 7
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a2, a0, 32
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    srli a2, a2, 32
@@ -973,9 +973,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, a3, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a4, 4(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-FPELIM-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-FPELIM-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 32
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
@@ -999,9 +999,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, a3, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a4, -12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw a3, 0(a3)
-; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    sltu a1, a0, a1
+; ILP32-ILP32F-WITHFP-NEXT:    add a2, a2, a3
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a2, a1
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 16(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 20(sp)
@@ -1023,9 +1023,9 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    fld ft0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    fsd ft0, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 12(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a3, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a2, a2, a0
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 8(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sltu a1, a0, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a2, a1
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
@@ -1039,11 +1039,11 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a5, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a4, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 24(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a0, a0, 8
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 16(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, sp, 16
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a3, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a0, a1, a2
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 64
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -1058,11 +1058,11 @@ define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a5, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a4, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 8(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a0, a0, 8
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 0(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a0, s0
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a3, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a0, a1, a2
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 80
@@ -1084,9 +1084,9 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, zero, 1111
+; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a2, zero
 ; ILP32-ILP32F-FPELIM-NEXT:    mv a4, zero
-; ILP32-ILP32F-FPELIM-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-FPELIM-NEXT:    call va3
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 16
@@ -1100,9 +1100,9 @@ define void @va3_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi s0, sp, 16
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, zero, 2
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, zero, 1111
+; ILP32-ILP32F-WITHFP-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a2, zero
 ; ILP32-ILP32F-WITHFP-NEXT:    mv a4, zero
-; ILP32-ILP32F-WITHFP-NEXT:    lui a5, 262144
 ; ILP32-ILP32F-WITHFP-NEXT:    call va3
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
@@ -1115,9 +1115,9 @@ define void @va3_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, zero, 2
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, zero, 1111
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 262144
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a2, zero
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    mv a4, zero
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a5, 262144
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va3
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 16
@@ -1190,9 +1190,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    andi a0, a0, -4
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a3, a0, 4
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 4(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, s0
 ; ILP32-ILP32F-FPELIM-NEXT:    add a1, a1, a2
-; ILP32-ILP32F-FPELIM-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-FPELIM-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-FPELIM-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 12(sp)
@@ -1233,9 +1233,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    andi a0, a0, -4
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a3, a0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, -16(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, s1
 ; ILP32-ILP32F-WITHFP-NEXT:    add a1, a1, a2
-; ILP32-ILP32F-WITHFP-NEXT:    lw a0, 0(a0)
 ; ILP32-ILP32F-WITHFP-NEXT:    add a0, a1, a0
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s1, 20(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 24(sp)
@@ -1275,9 +1275,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    andi a0, a0, -4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, a0, 4
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 4(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, s0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a1, a1, a2
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw a0, 0(a0)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    add a0, a1, a0
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw s0, 8(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 12(sp)
@@ -1317,9 +1317,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    andi a0, a0, -4
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a0, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 8(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, s0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    add a1, a1, a2
-; LP64-LP64F-LP64D-FPELIM-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addw a0, a1, a0
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 24(sp)
@@ -1361,9 +1361,9 @@ define i32 @va4_va_copy(i32 %argno, ...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    andi a0, a0, -4
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a0, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, -32(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, s1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    add a1, a1, a2
-; LP64-LP64F-LP64D-WITHFP-NEXT:    ld a0, 0(a0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addw a0, a1, a0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s1, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
@@ -1425,8 +1425,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, -328
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 36(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 335544
-; ILP32-ILP32F-FPELIM-NEXT:    addi a0, a0, 1311
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 32(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    addi a5, a0, 1311
 ; ILP32-ILP32F-FPELIM-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a2, sp, 32
@@ -1435,6 +1434,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a3, zero, 12
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a7, zero, 4
+; ILP32-ILP32F-FPELIM-NEXT:    sw a5, 32(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-FPELIM-NEXT:    lw ra, 60(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 64
@@ -1470,8 +1470,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, -328
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -28(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 335544
-; ILP32-ILP32F-WITHFP-NEXT:    addi a0, a0, 1311
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, -32(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    addi a5, a0, 1311
 ; ILP32-ILP32F-WITHFP-NEXT:    lui a0, 688509
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a6, a0, -2048
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a2, s0, -32
@@ -1480,6 +1479,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a3, zero, 12
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a4, zero, 13
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a7, zero, 4
+; ILP32-ILP32F-WITHFP-NEXT:    sw a5, -32(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 56(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 60(sp)
@@ -1514,8 +1514,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, -328
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 36(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 335544
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a0, a0, 1311
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 32(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a5, a0, 1311
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lui a0, 688509
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a6, a0, -2048
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a2, sp, 32
@@ -1524,6 +1523,7 @@ define void @va5_aligned_stack_caller() nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a3, zero, 12
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a4, zero, 13
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a7, zero, 4
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a5, 32(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    lw ra, 60(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 64
@@ -1546,33 +1546,33 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 14
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 655
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 1475
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 0(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi t0, a0, 1475
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 1192
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 381
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a6, a0, -2048
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 1048248
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 1311
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -1147
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 13
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a1, 512
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a1, a1, 73
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, -1311
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, a1, 1147
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a1, a1, 14
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 983
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 15
-; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a2, 1192
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a2, a2, 381
-; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a2, a2, 12
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a6, a2, -2048
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a2, a0, 1311
-; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a1, -1967
+; LP64-LP64F-LP64D-FPELIM-NEXT:    lui a0, 512
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addiw a0, a0, 73
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 15
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, -1311
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, a0, 1147
+; LP64-LP64F-LP64D-FPELIM-NEXT:    slli a0, a0, 14
+; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a3, a0, -1967
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, zero, 11
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a4, zero, 12
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a7, zero, 14
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd t0, 0(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ld ra, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 48
@@ -1597,33 +1597,33 @@ define void @va5_aligned_stack_caller() nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 14
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 655
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 1475
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(sp)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi t0, a0, 1475
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 1192
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 381
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a6, a0, -2048
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 1048248
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 1311
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -1147
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 13
-; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a1, 512
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a1, a1, 73
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, -1311
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, a1, 1147
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a1, a1, 14
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 983
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 15
-; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a2, 1192
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a2, a2, 381
-; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a2, a2, 12
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a6, a2, -2048
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a2, a0, 1311
-; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a1, -1967
+; LP64-LP64F-LP64D-WITHFP-NEXT:    lui a0, 512
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addiw a0, a0, 73
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 15
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, -1311
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 12
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, a0, 1147
+; LP64-LP64F-LP64D-WITHFP-NEXT:    slli a0, a0, 14
+; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a3, a0, -1967
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a0, zero, 1
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a1, zero, 11
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a4, zero, 12
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a5, zero, 13
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi a7, zero, 14
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd t0, 0(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    call va5_aligned_stack_callee
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 32(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 40(sp)
@@ -1650,9 +1650,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a3, 28(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a2, 24(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 20(sp)
+; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi a1, sp, 20
 ; ILP32-ILP32F-FPELIM-NEXT:    sw a1, 12(sp)
-; ILP32-ILP32F-FPELIM-NEXT:    sw a0, 16(sp)
 ; ILP32-ILP32F-FPELIM-NEXT:    addi sp, sp, 48
 ; ILP32-ILP32F-FPELIM-NEXT:    ret
 ;
@@ -1669,9 +1669,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a3, 12(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a2, 8(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, 4(s0)
+; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi a1, s0, 4
 ; ILP32-ILP32F-WITHFP-NEXT:    sw a1, -12(s0)
-; ILP32-ILP32F-WITHFP-NEXT:    sw a0, 0(s0)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw s0, 8(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    lw ra, 12(sp)
 ; ILP32-ILP32F-WITHFP-NEXT:    addi sp, sp, 48
@@ -1687,9 +1687,9 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a3, 28(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a2, 24(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 20(sp)
+; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi a1, sp, 20
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a1, 12(sp)
-; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    sw a0, 16(sp)
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    addi sp, sp, 48
 ; RV32D-ILP32-ILP32F-ILP32D-FPELIM-NEXT:    ret
 ;
@@ -1703,10 +1703,10 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a3, 40(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a2, 32(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 24(sp)
+; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi a1, sp, 16
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ori a1, a1, 8
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a1, 8(sp)
-; LP64-LP64F-LP64D-FPELIM-NEXT:    sd a0, 16(sp)
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    addi sp, sp, 80
 ; LP64-LP64F-LP64D-FPELIM-NEXT:    ret
 ;
@@ -1723,10 +1723,10 @@ define i32 @va6_no_fixed_args(...) nounwind {
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a3, 24(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a2, 16(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, 8(s0)
+; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    mv a1, s0
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ori a1, a1, 8
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a1, -24(s0)
-; LP64-LP64F-LP64D-WITHFP-NEXT:    sd a0, 0(s0)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld s0, 16(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    ld ra, 24(sp)
 ; LP64-LP64F-LP64D-WITHFP-NEXT:    addi sp, sp, 96
diff --git a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
index 12c7796995a81..7c4bbcdaa1e8f 100644
--- a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
@@ -45,10 +45,10 @@ define i32 @test_zext_i16() nounwind {
 ; RV32I-LABEL: test_zext_i16:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(shorts)
-; RV32I-NEXT:    lui a1, 16
-; RV32I-NEXT:    addi a1, a1, -120
-; RV32I-NEXT:    lhu a2, %lo(shorts)(a0)
-; RV32I-NEXT:    bne a2, a1, .LBB1_3
+; RV32I-NEXT:    lhu a1, %lo(shorts)(a0)
+; RV32I-NEXT:    lui a2, 16
+; RV32I-NEXT:    addi a2, a2, -120
+; RV32I-NEXT:    bne a1, a2, .LBB1_3
 ; RV32I-NEXT:  # %bb.1: # %entry
 ; RV32I-NEXT:    addi a0, a0, %lo(shorts)
 ; RV32I-NEXT:    lhu a0, 2(a0)
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-01.ll b/llvm/test/CodeGen/SystemZ/cond-move-01.ll
index fad5012be10e9..c6b0cd5393afc 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-01.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-01.ll
@@ -7,7 +7,7 @@
 ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 -verify-machineinstrs | FileCheck %s
 ;
 ; And again in the presence of the select instructions.
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 ; Test LOCR.
 define i32 @f1(i32 %a, i32 %b, i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-02.ll b/llvm/test/CodeGen/SystemZ/cond-move-02.ll
index c8ab5331216b0..362d7cc3865b6 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-02.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-02.ll
@@ -4,7 +4,7 @@
 ;
 ; Run the test again to make sure it still works the same even
 ; in the presence of the select instructions.
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 
 define i32 @f1(i32 %x) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-03.ll b/llvm/test/CodeGen/SystemZ/cond-move-03.ll
index 0f4d080d6bcb2..f668f1ac40718 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-03.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-03.ll
@@ -6,7 +6,7 @@
 ;
 ; Run the test again to make sure it still works the same even
 ; in the presence of the select instructions.
-; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=arch13 \
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z15 \
 ; RUN:   -no-integrated-as | FileCheck %s
 
 define void @f1(i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-06.ll b/llvm/test/CodeGen/SystemZ/cond-move-06.ll
index 3c133d9167185..bcedb3ae8e8d8 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-06.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-06.ll
@@ -1,6 +1,6 @@
 ; Test SELR and SELGR.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 -verify-machineinstrs | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 -verify-machineinstrs | FileCheck %s
 
 ; Test SELR.
 define i32 @f1(i32 %limit, i32 %a, i32 %b) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-07.ll b/llvm/test/CodeGen/SystemZ/cond-move-07.ll
index 87123b5337928..2bee8919b5198 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-07.ll
+++ b/llvm/test/CodeGen/SystemZ/cond-move-07.ll
@@ -1,7 +1,7 @@
 ; Test SELFHR.
 ; See comments in asm-18.ll about testing high-word operations.
 ;
-; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=arch13 \
+; RUN: llc < %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z15 \
 ; RUN:   -no-integrated-as | FileCheck %s
 
 define void @f1(i32 %limit) {
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-08.mir b/llvm/test/CodeGen/SystemZ/cond-move-08.mir
index 5808fc643bcae..aed6110360152 100644
--- a/llvm/test/CodeGen/SystemZ/cond-move-08.mir
+++ b/llvm/test/CodeGen/SystemZ/cond-move-08.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=s390x-linux-gnu -mcpu=arch13 -start-before=greedy %s -o - \
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z15 -start-before=greedy %s -o - \
 # RUN:   | FileCheck %s
 #
 # Test that regalloc manages (via regalloc hints) to avoid a LOCRMux jump
@@ -73,7 +73,7 @@
   ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
 
-  attributes #0 = { "target-cpu"="arch13" }
+  attributes #0 = { "target-cpu"="z15" }
   attributes #1 = { nounwind }
 
 ...
diff --git a/llvm/test/CodeGen/SystemZ/cond-move-09.mir b/llvm/test/CodeGen/SystemZ/cond-move-09.mir
new file mode 100644
index 0000000000000..9eb4885c2b4ca
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/cond-move-09.mir
@@ -0,0 +1,29 @@
+# RUN: llc -mtriple=s390x-linux-gnu -mcpu=z14 -O0 -start-after=regallocfast %s \
+# RUN: -o - | FileCheck %s
+#
+# Test that LOCRMux expansion works at -O0.
+
+--- |
+  define internal i32 @fun(i16 %arg) { bb: ret i32 0 }
+...
+
+# CHECK-LABEL: fun
+# CHECK: risbhg  %r13, %r13, 0, 159, 32
+
+---
+name:            'fun'
+alignment:       16
+tracksRegLiveness: true
+frameInfo:
+  maxAlignment:    8
+machineFunctionInfo: {}
+body:             |
+  bb.0.bb:
+    renamable $r0d = IMPLICIT_DEF
+    CGHI killed renamable $r0d, 8, implicit-def $cc
+    renamable $r13l = LHIMux 0
+    renamable $r13h = LHIMux -1
+    renamable $r13h = LOCRMux renamable $r13h, killed renamable $r13l, 14, 10, implicit $cc
+    renamable $r2l = COPY killed renamable $r13h
+    Return implicit killed $r2l
+...
diff --git a/llvm/test/CodeGen/SystemZ/ctpop-02.ll b/llvm/test/CodeGen/SystemZ/ctpop-02.ll
index 5b9d41f9af2e9..a2bddac8c5a42 100644
--- a/llvm/test/CodeGen/SystemZ/ctpop-02.ll
+++ b/llvm/test/CodeGen/SystemZ/ctpop-02.ll
@@ -1,6 +1,6 @@
-; Test population-count instruction on arch13
+; Test population-count instruction on z15
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i32 @llvm.ctpop.i32(i32 %a)
 declare i64 @llvm.ctpop.i64(i64 %a)
diff --git a/llvm/test/CodeGen/SystemZ/debuginstr-02.mir b/llvm/test/CodeGen/SystemZ/debuginstr-02.mir
index 7c4f6735cf536..2548482f5991a 100644
--- a/llvm/test/CodeGen/SystemZ/debuginstr-02.mir
+++ b/llvm/test/CodeGen/SystemZ/debuginstr-02.mir
@@ -1,13 +1,16 @@
 # Check that the backend can handle consecutive select instructions also in
-# the presence of DEBUG_VALUE machine instructions.
+# the presence of DEBUG_VALUE machine instructions, which should be moved.
 #
-# RUN: llc %s -verify-machineinstrs -mtriple=s390x-linux-gnu -mcpu=z13 \
-# RUN:   -start-before=finalize-isel -o - 2>&1 | FileCheck %s
+# RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z13 -run-pass=finalize-isel \
+# RUN:   -o - 2>&1 | FileCheck %s
 #
-# CHECK-LABEL: %bb.1:
-# CHECK:       ldr
-# CHECK-NEXT:  ldr
-# CHECK-NEXT:  ldr
+# CHECK-LABEL: bb.1 (%ir-block.0):
+# CHECK-NEXT: %5:fp32bit = PHI %1, %bb.0, %2, %bb.2
+# CHECK-NEXT: %6:fp32bit = PHI %3, %bb.0, %4, %bb.2
+# CHECK-NEXT: %7:fp32bit = PHI %1, %bb.0, %4, %bb.2
+# CHECK-NEXT: DBG_VALUE %5, $noreg, !5, !DIExpression(), debug-location !9
+# CHECK-NEXT: DBG_VALUE %6, $noreg, !5, !DIExpression(), debug-location !9
+# CHECK-NEXT: %8:fp32bit = AEBR %5, killed %6, implicit-def dead $cc, implicit $fpc
 
 --- |
   ; ModuleID = 'tc.ll'
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
index d69e64d7c96f1..cc6450a6c04b8 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-10.ll
@@ -18,19 +18,14 @@ define i32 @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    larl %r1, .LCPI0_0
-; CHECK-NEXT:    le %f2, 0(%r1)
-; CHECK-NEXT:    ler %f1, %f0
-; CHECK-NEXT:    sebr %f1, %f2
-; CHECK-NEXT:    cebr %f0, %f2
+; CHECK-NEXT:    le %f1, 0(%r1)
+; CHECK-NEXT:    cebr %f0, %f1
+; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    jl .LBB0_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    ler %f0, %f1
-; CHECK-NEXT:  .LBB0_2:
-; CHECK-NEXT:    lhi %r0, 0
-; CHECK-NEXT:    jl .LBB0_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    cfebr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
 ; CHECK-NEXT:    br %r14
@@ -44,19 +39,14 @@ define i32 @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    larl %r1, .LCPI1_0
-; CHECK-NEXT:    ldeb %f2, 0(%r1)
-; CHECK-NEXT:    ldr %f1, %f0
-; CHECK-NEXT:    sdbr %f1, %f2
-; CHECK-NEXT:    cdbr %f0, %f2
+; CHECK-NEXT:    ldeb %f1, 0(%r1)
+; CHECK-NEXT:    cdbr %f0, %f1
+; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    jl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    ldr %f0, %f1
-; CHECK-NEXT:  .LBB1_2:
-; CHECK-NEXT:    lhi %r0, 0
-; CHECK-NEXT:    jl .LBB1_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    cfdbr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
 ; CHECK-NEXT:    br %r14
@@ -72,19 +62,14 @@ define i32 @f3(fp128 *%src) {
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
 ; CHECK-NEXT:    larl %r1, .LCPI2_0
-; CHECK-NEXT:    lxeb %f4, 0(%r1)
-; CHECK-NEXT:    lxr %f1, %f0
-; CHECK-NEXT:    sxbr %f1, %f4
-; CHECK-NEXT:    cxbr %f0, %f4
+; CHECK-NEXT:    lxeb %f1, 0(%r1)
+; CHECK-NEXT:    cxbr %f0, %f1
+; CHECK-NEXT:    lhi %r0, 0
 ; CHECK-NEXT:    jl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lxr %f0, %f1
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    lhi %r0, 0
-; CHECK-NEXT:    jl .LBB2_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    llilh %r0, 32768
-; CHECK-NEXT:  .LBB2_4:
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    cfxbr %r2, 5, %f0
 ; CHECK-NEXT:    xr %r2, %r0
 ; CHECK-NEXT:    br %r14
diff --git a/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
index 2319c629daf9e..e9811cf2fd9c5 100644
--- a/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
+++ b/llvm/test/CodeGen/SystemZ/fp-strict-conv-12.ll
@@ -17,19 +17,14 @@ define i64 @f1(float %f) {
 ; CHECK-LABEL: f1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    larl %r1, .LCPI0_0
-; CHECK-NEXT:    le %f2, 0(%r1)
-; CHECK-NEXT:    ler %f1, %f0
-; CHECK-NEXT:    sebr %f1, %f2
-; CHECK-NEXT:    cebr %f0, %f2
+; CHECK-NEXT:    le %f1, 0(%r1)
+; CHECK-NEXT:    cebr %f0, %f1
+; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    jl .LBB0_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    ler %f0, %f1
-; CHECK-NEXT:  .LBB0_2:
-; CHECK-NEXT:    lghi %r0, 0
-; CHECK-NEXT:    jl .LBB0_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sebr %f0, %f1
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB0_4:
+; CHECK-NEXT:  .LBB0_2:
 ; CHECK-NEXT:    cgebr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
 ; CHECK-NEXT:    br %r14
@@ -43,19 +38,14 @@ define i64 @f2(double %f) {
 ; CHECK-LABEL: f2:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    larl %r1, .LCPI1_0
-; CHECK-NEXT:    ldeb %f2, 0(%r1)
-; CHECK-NEXT:    ldr %f1, %f0
-; CHECK-NEXT:    sdbr %f1, %f2
-; CHECK-NEXT:    cdbr %f0, %f2
+; CHECK-NEXT:    ldeb %f1, 0(%r1)
+; CHECK-NEXT:    cdbr %f0, %f1
+; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    jl .LBB1_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    ldr %f0, %f1
-; CHECK-NEXT:  .LBB1_2:
-; CHECK-NEXT:    lghi %r0, 0
-; CHECK-NEXT:    jl .LBB1_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sdbr %f0, %f1
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:  .LBB1_2:
 ; CHECK-NEXT:    cgdbr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
 ; CHECK-NEXT:    br %r14
@@ -71,19 +61,14 @@ define i64 @f3(fp128 *%src) {
 ; CHECK-NEXT:    ld %f0, 0(%r2)
 ; CHECK-NEXT:    ld %f2, 8(%r2)
 ; CHECK-NEXT:    larl %r1, .LCPI2_0
-; CHECK-NEXT:    lxeb %f4, 0(%r1)
-; CHECK-NEXT:    lxr %f1, %f0
-; CHECK-NEXT:    sxbr %f1, %f4
-; CHECK-NEXT:    cxbr %f0, %f4
+; CHECK-NEXT:    lxeb %f1, 0(%r1)
+; CHECK-NEXT:    cxbr %f0, %f1
+; CHECK-NEXT:    lghi %r0, 0
 ; CHECK-NEXT:    jl .LBB2_2
 ; CHECK-NEXT:  # %bb.1:
-; CHECK-NEXT:    lxr %f0, %f1
-; CHECK-NEXT:  .LBB2_2:
-; CHECK-NEXT:    lghi %r0, 0
-; CHECK-NEXT:    jl .LBB2_4
-; CHECK-NEXT:  # %bb.3:
+; CHECK-NEXT:    sxbr %f0, %f1
 ; CHECK-NEXT:    llihh %r0, 32768
-; CHECK-NEXT:  .LBB2_4:
+; CHECK-NEXT:  .LBB2_2:
 ; CHECK-NEXT:    cgxbr %r2, 5, %f0
 ; CHECK-NEXT:    xgr %r2, %r0
 ; CHECK-NEXT:    br %r14
diff --git a/llvm/test/CodeGen/SystemZ/mature-mc-support.ll b/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
index a01716c276701..bf82c363f43a7 100644
--- a/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
+++ b/llvm/test/CodeGen/SystemZ/mature-mc-support.ll
@@ -1,12 +1,10 @@
 ; Test that inline assembly is parsed by the MC layer when MC support is mature
 ; (even when the output is assembly).
-; FIXME: SystemZ doesn't use the integrated assembler by default so we only test
-; that -filetype=obj tries to parse the assembly.
 
-; SKIP: not llc -march=systemz < %s > /dev/null 2> %t1
-; SKIP: FileCheck %s < %t1
+; RUN: not llc -mtriple=s390x-linux-gnu < %s > /dev/null 2> %t1
+; RUN: FileCheck %s < %t1
 
-; RUN: not llc -march=systemz -filetype=obj < %s > /dev/null 2> %t2
+; RUN: not llc -mtriple=s390x-linux-gnu -filetype=obj < %s > /dev/null 2> %t2
 ; RUN: FileCheck %s < %t2
 
 
diff --git a/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll b/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll
new file mode 100644
index 0000000000000..38f1db537b52f
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mnop-mcount-01.ll
@@ -0,0 +1,26 @@
+; RUN: llc %s -mtriple=s390x-linux-gnu -mcpu=z10 -o - -verify-machineinstrs \
+; RUN:   | FileCheck %s
+
+define void @test1() #0 {
+entry:
+  ret void
+
+; CHECK-LABEL: @test1
+; CHECK: brasl %r0, __fentry__
+; CHECK-NOT: brcl 0, .Ltmp0
+; CHECK: br %r14
+}
+
+define void @test2() #1 {
+entry:
+  ret void
+
+; CHECK-LABEL: @test2
+; CHECK-NOT: brasl %r0, __fentry__
+; CHECK: brcl 0, .Ltmp0
+; CHECK: br %r14
+}
+
+attributes #0 = { "fentry-call"="true" }
+attributes #1 = { "fentry-call"="true" "mnop-mcount"="true" }
+
diff --git a/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll b/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll
new file mode 100644
index 0000000000000..19b1724d16adf
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/mnop-mcount-02.ll
@@ -0,0 +1,11 @@
+; RUN: not llc %s -mtriple=s390x-linux-gnu -o - 2>&1 | FileCheck %s
+;
+; CHECK: LLVM ERROR: mnop-mcount only supported with fentry-call
+
+define void @test1() #0 {
+entry:
+  ret void
+}
+
+attributes #0 = { "instrument-function-entry-inlined"="mcount" "mnop-mcount"="true" }
+
diff --git a/llvm/test/CodeGen/SystemZ/multiselect.ll b/llvm/test/CodeGen/SystemZ/multiselect.ll
index d2078ff291ba4..672a302a540e4 100644
--- a/llvm/test/CodeGen/SystemZ/multiselect.ll
+++ b/llvm/test/CodeGen/SystemZ/multiselect.ll
@@ -1,10 +1,11 @@
 ; Test that multiple select statements using the same condition are expanded
-; into a single conditional branch.
+; into a single conditional branch when possible.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -disable-block-placement | FileCheck %s
 
-define void @test(i32 signext %positive, double %base, double %offset, double* %rmin, double* %rmax) {
+define void @test0(i32 signext %positive, double %base, double %offset, double* %rmin, double* %rmax) {
 entry:
+; CHECK-LABEL: test0
 ; CHECK: cijlh %r2, 0,
 ; CHECK-NOT: cij
 ; CHECK-NOT: je
@@ -19,3 +20,51 @@ entry:
   ret void
 }
 
+; Two selects with an intervening instruction that doesn't clobber CC can
+; still be merged.
+define double @test1(i32 signext %positive, double %A, double %B, double %C) {
+entry:
+; CHECK-LABEL: test1
+; CHECK: cijhe {{.*}}LBB1_2
+; CHECK-NOT: cij
+; CHECK: br %r14
+
+  %tobool = icmp slt i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %mul = fmul double %A, %B
+  %s2  = select i1 %tobool, double %B, double %C
+  %add = fadd double %s1, %s2
+  %add2 = fadd double %add, %mul
+  ret double %add2
+}
+
+; Two selects with an intervening user of the first select can't be merged.
+define double @test2(i32 signext %positive, double %A, double %B) {
+entry:
+; CHECK-LABEL: test2
+; CHECK: cije {{.*}}LBB2_2
+; CHECK: cibe {{.*}}%r14
+; CHECK: br %r14
+
+  %tobool = icmp eq i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %add = fadd double %A, %s1
+  %s2  = select i1 %tobool, double %A, double %add
+  ret double %s2
+}
+
+; Two selects with different conditions can't be merged
+define double @test3(i32 signext %positive, double %A, double %B, double %C) {
+entry:
+; CHECK-LABEL: test3
+; CHECK: cijl {{.*}}LBB3_2
+; CHECK: cijl {{.*}}LBB3_4
+; CHECK: br %r14
+
+  %tobool = icmp slt i32 %positive, 0
+  %s1  = select i1 %tobool, double %A, double %B
+  %tobool2 = icmp slt i32 %positive, 2
+  %s2  = select i1 %tobool2, double %B, double %C
+  %add = fadd double %s1, %s2
+  ret double %add
+}
diff --git a/llvm/test/CodeGen/SystemZ/not-01.ll b/llvm/test/CodeGen/SystemZ/not-01.ll
index 3b9dbd1311f64..be2a5a88d73f0 100644
--- a/llvm/test/CodeGen/SystemZ/not-01.ll
+++ b/llvm/test/CodeGen/SystemZ/not-01.ll
@@ -1,6 +1,6 @@
-; Combined logical operations involving complement on arch13
+; Combined logical operations involving complement on z15
 ;
-; RUN: llc -mcpu=arch13 < %s -mtriple=s390x-linux-gnu | FileCheck %s
+; RUN: llc -mcpu=z15 < %s -mtriple=s390x-linux-gnu | FileCheck %s
 
 ; And-with-complement 32-bit.
 define i32 @f1(i32 %dummy, i32 %a, i32 %b) {
diff --git a/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll b/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
index 0290d425e5706..4633175830a40 100644
--- a/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
+++ b/llvm/test/CodeGen/SystemZ/tail-call-mem-intrinsics.ll
@@ -1,4 +1,4 @@
-; RUN: llc -march=systemz < %s | FileCheck %s
+; RUN: llc -mtriple=s390x-linux-gnu < %s | FileCheck %s
 
 ; CHECK-LABEL: tail_memcpy:
 ; CHECK: jg memcpy
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
index 8132108953a90..4d5e574033f34 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-01.ll
@@ -1,6 +1,6 @@
 ; Test loads of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
index ff7facc0910a7..b7e8a1c1711df 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-02.ll
@@ -1,6 +1,6 @@
 ; Test stores of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
 declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
index 9102c739b1248..b812308fc6d16 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-03.ll
@@ -1,6 +1,6 @@
 ; Test vector insertion of byte-swapped memory values.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
index 43787a48b2efd..0ab521dbb9cf5 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-04.ll
@@ -1,6 +1,6 @@
 ; Test vector extraction of byte-swapped value to memory.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
index 038a3f4f4f57e..4718aadbdd5dc 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-05.ll
@@ -1,6 +1,6 @@
 ; Test vector insertions of byte-swapped memory values into 0.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
index 0c78633ee7d11..a67b5756c4d52 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-06.ll
@@ -1,6 +1,6 @@
 ; Test insertions of byte-swapped memory values into a nonzero index of an undef.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll b/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
index 0190184c1e2f3..d3f38a0a6851a 100644
--- a/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-bswap-07.ll
@@ -1,6 +1,6 @@
 ; Test replications of a byte-swapped scalar memory value.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare i16 @llvm.bswap.i16(i16)
 declare i32 @llvm.bswap.i32(i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-conv-03.ll b/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
index 8398876821c19..df1e3a4e0dc33 100644
--- a/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-conv-03.ll
@@ -1,6 +1,6 @@
-; Test conversions between integer and float elements on arch13.
+; Test conversions between integer and float elements on z15.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test conversion of f32s to signed i32s.
 define <4 x i32> @f1(<4 x float> %floats) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll b/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
index 89b65392440a5..0e93f609eac1c 100644
--- a/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-eswap-01.ll
@@ -1,6 +1,6 @@
 ; Test loads of byte-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test v16i8 loads.
 define <16 x i8> @f1(<16 x i8> *%ptr) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll b/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
index 1eab5a06ff213..07e07258204e7 100644
--- a/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-eswap-02.ll
@@ -1,6 +1,6 @@
 ; Test stores of element-swapped vector elements.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; Test v16i8 stores.
 define void @f1(<16 x i8> %val, <16 x i8> *%ptr) {
diff --git a/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll b/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
index d192ad8948805..fa144cc93d350 100644
--- a/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-intrinsics-03.ll
@@ -1,6 +1,6 @@
-; Test vector intrinsics added with arch13.
+; Test vector intrinsics added with z15.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 declare <16 x i8> @llvm.s390.vsld(<16 x i8>, <16 x i8>, i32)
 declare <16 x i8> @llvm.s390.vsrd(<16 x i8>, <16 x i8>, i32)
diff --git a/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll b/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
index f42a2b4120279..6b9bc80cbdeb5 100644
--- a/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
+++ b/llvm/test/CodeGen/SystemZ/vec-strict-conv-03.ll
@@ -1,6 +1,6 @@
-; Test strict conversions between integer and float elements on arch13.
+; Test strict conversions between integer and float elements on z15.
 ;
-; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=arch13 | FileCheck %s
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z15 | FileCheck %s
 
 ; FIXME: llvm.experimental.constrained.[su]itofp does not yet exist
 
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
index f01e6ab19184b..79c81ca7a449c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
@@ -4,8 +4,8 @@
 ; CHECK: vector.body:
 ; CHECK: %index = phi i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.vctp8(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 16
-; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.vctp8(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]], <16 x i8> undef)
 ; CHECK: tail call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> {{.*}}, <16 x i8>* {{.*}}, i32 4, <16 x i1> [[VCTP]])
@@ -57,8 +57,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; CHECK: vector.body:
 ; CHECK: %index = phi i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 8
-; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 ; CHECK: tail call void @llvm.masked.store.v8i16.p0v8i16(<8 x i16> {{.*}}, <8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]])
@@ -109,8 +109,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; CHECK-LABEL: mul_v4i32
 ; CHECK: vector.body:
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
-; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
@@ -162,8 +162,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; CHECK: vector.body:
 ; CHECK: %index = phi i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <2 x i1> @llvm.arm.vctp64(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 2
-; CHECK: [[VCTP:%[^ ]+]] = call <2 x i1> @llvm.arm.vctp64(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <2 x i64> @llvm.masked.load.v2i64.p0v2i64(<2 x i64>* {{.*}}, i32 4, <2 x i1> [[VCTP]], <2 x i64> undef)
 ; CHECK: tail call void @llvm.masked.store.v2i64.p0v2i64(<2 x i64> [[LD0]], <2 x i64>* {{.*}}, i32 4, <2 x i1> [[VCTP]])
 define void @copy_v2i64(i64* %a, i64* %b, i32 %N) {
@@ -210,8 +210,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; CHECK: vector.body:
 ; CHECK: %index = phi i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
-; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
@@ -268,8 +268,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; One of the loads now uses ult predicate.
 ; CHECK-LABEL: mismatch_load_pred
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
-; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong, <4 x i32> undef)
 ; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]])
@@ -322,8 +322,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 ; CHECK-LABEL: mismatch_store_pred
 ; CHECK: %index = phi i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELEMS]])
 ; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 4
-; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REMAINING]])
 ; CHECK: [[LD0:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: [[LD1:%[^ ]+]] = tail call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %wrong)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
index 17b7b63c0c43f..a0f13f3af65e2 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/branch-targets.ll
@@ -14,9 +14,8 @@
 ; CHECK-END: .LBB0_1:
 ; CHECK-END:   b .LBB0_3
 ; CHECK-END: .LBB0_2:
-; CHECK-END:   sub.w lr, lr, #1
-; CHECK-END:   cmp.w lr, #0
-; CHECK-END:   bne.w .LBB0_3
+; CHECK-END:   subs.w lr, lr, #1
+; CHECK-END:   bne .LBB0_3
 ; CHECK-END:   b .LBB0_4
 ; CHECK-END: .LBB0_3:
 ; CHECK-END:   b .LBB0_2
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
index 1efe18e129c57..b17965cd3355c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-mov.mir
@@ -4,6 +4,9 @@
 # CHECK: $lr = t2LEUpdate renamable $lr, %bb.1
 
 --- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define i32 @do_copy(i32 %n, i32* nocapture %p, i32* nocapture readonly %q) {
   entry:
     %scevgep = getelementptr i32, i32* %q, i32 -1
@@ -15,10 +18,10 @@
     %lsr.iv4 = phi i32* [ %scevgep5, %while.body ], [ %scevgep3, %entry ]
     %lsr.iv = phi i32* [ %scevgep1, %while.body ], [ %scevgep, %entry ]
     %0 = phi i32 [ %n, %entry ], [ %2, %while.body ]
-    %scevgep7 = getelementptr i32, i32* %lsr.iv, i32 1
-    %scevgep4 = getelementptr i32, i32* %lsr.iv4, i32 1
-    %1 = load i32, i32* %scevgep7, align 4
-    store i32 %1, i32* %scevgep4, align 4
+    %scevgep6 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep2 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %1 = load i32, i32* %scevgep6, align 4
+    store i32 %1, i32* %scevgep2, align 4
     %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
     %scevgep5 = getelementptr i32, i32* %lsr.iv4, i32 1
     %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
@@ -44,7 +47,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -84,6 +87,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -96,9 +100,10 @@ body:             |
   
   bb.1.while.body:
     successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+    liveins: $lr, $r0, $r1
   
-    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep7)
-    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep4)
+    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep6)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep2)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.1, implicit-def dead $cpsr
     tB %bb.2, 14, $noreg
@@ -108,4 +113,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
new file mode 100644
index 0000000000000..5900dd9ac66a9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
@@ -0,0 +1,296 @@
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -enable-arm-maskedldst=true -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
+
+; CHECK-LABEL: vpsel_mul_reduce_add
+; CHECK:      dls lr, lr
+; CHECK:  [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      vstr p0, [sp
+; CHECK:      vpstt	
+; CHECK-NEXT: vldrwt.u32
+; CHECK-NEXT: vldrwt.u32
+; CHECK:      vcmp.i32
+; CHECK:      vpsel
+; CHECK:      vldr p0, [sp
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r0]
+; CHECK:      sub{{.*}} [[ELEMS]], [[ELEMS_OUT]], #4
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32	[[ELEMS_OUT]]
+; CHECK-NEXT: vpsel
+; CHECK-NEXT: vaddv.u32
+define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %rem = urem i32 %index, 16
+  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
+  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
+  %wide.masked.load = select <4 x i1> %cmp, <4 x i32> %wide.masked.load.b, <4 x i32> %wide.masked.load.c
+  %mul = mul nsw <4 x i32> %wide.masked.load, %wide.masked.load.a
+  %add = add nsw <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %tmp7 = icmp eq i32 %index.next, %n.vec
+  br i1 %tmp7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %tmp8 = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %tmp9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %tmp8)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %tmp9, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; CHECK-LABEL: vpsel_mul_reduce_add_2
+; CHECK:      dls lr, lr
+; CHECK:  [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      vstr p0, [sp
+; CHECK:      vpstt
+; CHECK-NEXT: vldrwt.u32
+; CHECK-NEXT: vldrwt.u32
+; CHECK;      vsub
+; CHECK:      vpst
+; CHECK-NEXT: vldrwt.u32
+; CHECK:      vcmp.i32
+; CHECK:      vpsel
+; CHECK:      vldr p0, [sp
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r0]
+; CHECK:      sub{{.*}} [[ELEMS]], [[ELEMS_OUT]], #4
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32	[[ELEMS_OUT]]
+; CHECK-NEXT: vpsel
+; CHECK-NEXT: vaddv.u32
+define dso_local i32 @vpsel_mul_reduce_add_2(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
+                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %rem = urem i32 %index, 16
+  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
+  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
+  %sel = select <4 x i1> %cmp, <4 x i32> %sub, <4 x i32> %wide.masked.load.b
+  %mul = mul  <4 x i32> %sel, %wide.masked.load.a
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; CHECK-LABEL: and_mul_reduce_add
+; CHECK:      dls lr, lr
+; CHECK:  [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      vpstt	
+; CHECK-NEXT: vldrwt.u32
+; CHECK-NEXT: vldrwt.u32
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      vpsttt
+; CHECK-NEXT: vcmpt.i32	eq, {{.*}}, zr
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r3]
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r2]
+; CHECK:      sub{{.*}} [[ELEMS]],{{.*}}#4
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32 [[ELEMS_OUT]]
+; CHECK:      vpsel
+define dso_local i32 @and_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
+                                         i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
+  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
+  %mask = and <4 x i1> %cmp, %tmp1
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; TODO: Why does p0 get reloaded from the stack into p0, just to be vmrs'd?
+; CHECK-LABEL: or_mul_reduce_add
+; CHECK:      dls lr, lr
+; CHECK:  [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      vstr p0, [sp
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      vpstt	
+; CHECK-NEXT: vldrwt.u32
+; CHECK-NEXT: vldrwt.u32
+; CHECK:      vcmp.i32	eq, {{.*}}, zr
+; CHECK:      vmrs [[VCMP:r[0-9]+]], p0
+; CHECK:      vldr p0, [sp
+; CHECK:      vmrs [[VCTP:r[0-9]+]], p0
+; CHECK:      orr{{.*}} [[VCMP]], [[VCTP]]
+; CHECK:      sub{{.*}} [[ELEMS:r[0-9]+]], [[ELEMS_OUT]], #4
+; CHECK-NEXT: vmsr p0
+; CHECK-NEXT: vpstt
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r3]
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r2]
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32 [[ELEMS_OUT]]
+; CHECK:      vpsel
+define dso_local i32 @or_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b,
+                                        i32* noalias nocapture readonly %c, i32* noalias nocapture readonly %d, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %tmp = getelementptr inbounds i32, i32* %a, i32 %index
+  %tmp1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %tmp2 = bitcast i32* %tmp to <4 x i32>*
+  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp2, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %tmp3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %tmp4 = bitcast i32* %tmp3 to <4 x i32>*
+  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp4, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
+  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
+  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
+  %mask = or <4 x i1> %cmp, %tmp1
+  %tmp5 = getelementptr inbounds i32, i32* %c, i32 %index
+  %tmp6 = bitcast i32* %tmp5 to <4 x i32>*
+  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp6, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %tmp7 = getelementptr inbounds i32, i32* %d, i32 %index
+  %tmp8 = bitcast i32* %tmp7 to <4 x i32>*
+  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %tmp8, i32 4, <4 x i1> %mask, <4 x i32> undef)
+  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
+  %add = add  <4 x i32> %mul, %vec.phi
+  %index.next = add i32 %index, 4
+  %cmp.exit = icmp eq i32 %index.next, %n.vec
+  br i1 %cmp.exit, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
+  %reduce = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %acc)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+
+; Function Attrs: nounwind readnone willreturn
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
index 2b12ec1490198..8fa138a6d987e 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/end-positive-offset.mir
@@ -3,13 +3,17 @@
 # CHECK-NOT: DoLoopStart
 # CHECK-NOT: DLS
 # CHECK: bb.1.for.body:
-# CHECK:   t2CMPri $lr, 0, 14, $noreg, implicit-def $cpsr
-# CHECK:   t2Bcc %bb.3, 1, $cpsr
-# CHECK:   tB %bb.2, 14, $noreg
+# CHECK:      $lr = t2SUBri killed renamable $lr, 1, 14, $noreg, def $cpsr
+# CHECK-NOT:  t2CMPri $lr
+# CHECK:      tBcc %bb.3, 1, $cpsr
+# CHECK:      tB %bb.2, 14, $noreg
 # CHECK: bb.2.for.cond.cleanup:
 # CHECK: bb.3.for.header:
 
---- |  
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define void @size_limit(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
   entry:
     call void @llvm.set.loop.iterations.i32(i32 %N)
@@ -45,9 +49,11 @@
   }
   
   ; Function Attrs: nounwind
-  declare i32 @llvm.arm.space(i32 immarg, i32) #0 
+  declare i32 @llvm.arm.space(i32 immarg, i32) #0
+  
   ; Function Attrs: noduplicate nounwind
-  declare void @llvm.set.loop.iterations.i32(i32) #1 
+  declare void @llvm.set.loop.iterations.i32(i32) #1
+  
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
@@ -63,7 +69,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -128,6 +134,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.3(0x80000000)
+    liveins: $r0, $r1, $r2, $r3, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -184,5 +191,3 @@ body:             |
     tB %bb.1, 14, $noreg
 
 ...
-
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
new file mode 100644
index 0000000000000..140656d728606
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
@@ -0,0 +1,602 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fpregs,+fullfp16 -enable-arm-maskedldst=true -disable-mve-tail-predication=false %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB0_11
+; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    mov.w r12, #1
+; CHECK-NEXT:    cset lr, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r5, r1, r3, lsl #2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq.w r6, r6, lr
+; CHECK-NEXT:    lslseq.w r6, r6, #31
+; CHECK-NEXT:    beq .LBB0_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    subs r6, r3, #1
+; CHECK-NEXT:    and r7, r3, #3
+; CHECK-NEXT:    cmp r6, #3
+; CHECK-NEXT:    bhs .LBB0_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB0_8
+; CHECK-NEXT:  .LBB0_4: @ %vector.ph
+; CHECK-NEXT:    adds r6, r3, #3
+; CHECK-NEXT:    bic r6, r6, #3
+; CHECK-NEXT:    subs r6, #4
+; CHECK-NEXT:    add.w lr, r12, r6, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q0, [r1]
+; CHECK-NEXT:    vldrwt.u32 q1, [r2]
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    le lr, .LBB0_5
+; CHECK-NEXT:    b .LBB0_11
+; CHECK-NEXT:  .LBB0_6: @ %for.body.preheader.new
+; CHECK-NEXT:    subs r3, r3, r7
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    subs r3, #4
+; CHECK-NEXT:    add.w lr, r12, r3, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r4, r1, r3
+; CHECK-NEXT:    adds r5, r2, r3
+; CHECK-NEXT:    adds r6, r0, r3
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    vldr s0, [r4]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vldr s2, [r5]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6]
+; CHECK-NEXT:    vldr s0, [r4, #4]
+; CHECK-NEXT:    vldr s2, [r5, #4]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #4]
+; CHECK-NEXT:    vldr s0, [r4, #8]
+; CHECK-NEXT:    vldr s2, [r5, #8]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #8]
+; CHECK-NEXT:    vldr s0, [r4, #12]
+; CHECK-NEXT:    vldr s2, [r5, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #12]
+; CHECK-NEXT:    le lr, .LBB0_7
+; CHECK-NEXT:  .LBB0_8: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r8, .LBB0_11
+; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    mov lr, r8
+; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    add r1, r3
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:  .LBB0_10: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r1, #4]
+; CHECK-NEXT:    adds r1, #4
+; CHECK-NEXT:    vldr s2, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r0, #4]
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    le lr, .LBB0_10
+; CHECK-NEXT:  .LBB0_11: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
+
+vector.memcheck:                                  ; preds = %entry
+  %scevgep = getelementptr float, float* %a, i32 %N
+  %scevgep13 = getelementptr float, float* %b, i32 %N
+  %scevgep16 = getelementptr float, float* %c, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %a
+  %bound1 = icmp ugt float* %scevgep, %b
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %a
+  %bound119 = icmp ugt float* %scevgep, %c
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %vector.memcheck
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert21 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat22 = shufflevector <4 x i32> %broadcast.splatinsert21, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds float, float* %b, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat22
+  %4 = bitcast float* %2 to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %3, <4 x float> undef)
+  %5 = getelementptr inbounds float, float* %c, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.masked.load23 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %6, i32 4, <4 x i1> %3, <4 x float> undef)
+  %7 = fmul fast <4 x float> %wide.masked.load23, %wide.masked.load
+  %8 = getelementptr inbounds float, float* %a, i32 %index
+  %9 = bitcast float* %8 to <4 x float>*
+  call void @llvm.masked.store.v4f32.p0v4f32(<4 x float> %7, <4 x float>* %9, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %10 = icmp eq i32 %index.next, %n.vec
+  br i1 %10, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds float, float* %b, i32 %i.09.epil
+  %11 = load float, float* %arrayidx.epil, align 4
+  %arrayidx1.epil = getelementptr inbounds float, float* %c, i32 %i.09.epil
+  %12 = load float, float* %arrayidx1.epil, align 4
+  %mul.epil = fmul fast float %12, %11
+  %arrayidx2.epil = getelementptr inbounds float, float* %a, i32 %i.09.epil
+  store float %mul.epil, float* %arrayidx2.epil, align 4
+  %inc.epil = add nuw i32 %i.09.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %b, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %c, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %mul = fmul fast float %14, %13
+  %arrayidx2 = getelementptr inbounds float, float* %a, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = or i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %c, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %mul.1 = fmul fast float %16, %15
+  %arrayidx2.1 = getelementptr inbounds float, float* %a, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = or i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %mul.2 = fmul fast float %18, %17
+  %arrayidx2.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = or i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %mul.3 = fmul fast float %20, %19
+  %arrayidx2.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @fast_float_mac(float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    cbz r2, .LBB1_4
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q1, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r2
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-NEXT:    vldrwt.u32 q2, [r1]
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    vmul.f32 q1, q2, q1
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vadd.f32 q1, q1, q0
+; CHECK-NEXT:    le lr, .LBB1_2
+; CHECK-NEXT:  @ %bb.3: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q1, q0
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:  .LBB1_4:
+; CHECK-NEXT:    vldr s0, .LCPI1_0
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.5:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds float, float* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast float* %0 to <4 x float>*
+  %wide.masked.load = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %2, i32 4, <4 x i1> %1, <4 x float> undef)
+  %3 = getelementptr inbounds float, float* %c, i32 %index
+  %4 = bitcast float* %3 to <4 x float>*
+  %wide.masked.load13 = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %4, i32 4, <4 x i1> %1, <4 x float> undef)
+  %5 = fmul fast <4 x float> %wide.masked.load13, %wide.masked.load
+  %6 = fadd fast <4 x float> %5, %vec.phi
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %8 = select <4 x i1> %1, <4 x float> %6, <4 x float> %vec.phi
+  %rdx.shuf = shufflevector <4 x float> %8, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bin.rdx = fadd fast <4 x float> %8, %rdx.shuf
+  %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
+  %9 = extractelement <4 x float> %bin.rdx15, i32 0
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %9, %middle.block ]
+  ret float %a.0.lcssa
+}
+
+define arm_aapcs_vfpcc float @fast_float_half_mac(half* nocapture readonly %b, half* nocapture readonly %c, i32 %N) {
+; CHECK-LABEL: fast_float_half_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    beq.w .LBB2_22
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    adr r3, .LCPI2_1
+; CHECK-NEXT:    vldrw.u32 q1, [r3]
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB2_4
+; CHECK-NEXT:  .LBB2_2: @ %cond.load24
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s16, s20
+; CHECK-NEXT:    vmov r3, s20
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q4[0], r3
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vldr.16 s20, [r1, #6]
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:  .LBB2_3: @ %else25
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmul.f16 q3, q4, q3
+; CHECK-NEXT:    sub.w lr, lr, #1
+; CHECK-NEXT:    vmovx.f16 s18, s13
+; CHECK-NEXT:    vmovx.f16 s16, s12
+; CHECK-NEXT:    vcvtb.f32.f16 s23, s18
+; CHECK-NEXT:    adds r0, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s22, s13
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vcvtb.f32.f16 s21, s16
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vcvtb.f32.f16 s20, s12
+; CHECK-NEXT:    vadd.f32 q3, q2, q5
+; CHECK-NEXT:    cmp.w lr, #0
+; CHECK-NEXT:    bne .LBB2_4
+; CHECK-NEXT:    b .LBB2_21
+; CHECK-NEXT:  .LBB2_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q2, q3
+; CHECK-NEXT:    vadd.i32 q3, q1, r12
+; CHECK-NEXT:    vcmp.u32 cs, q0, q3
+; CHECK-NEXT:    @ implicit-def: $q4
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r4, r3, #0
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    bfi r3, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r3, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r3, r4, #2, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    bfi r3, r2, #3, #1
+; CHECK-NEXT:    lsls r2, r3, #31
+; CHECK-NEXT:    bne .LBB2_9
+; CHECK-NEXT:  @ %bb.5: @ %else
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    lsls r2, r3, #30
+; CHECK-NEXT:    bpl .LBB2_10
+; CHECK-NEXT:  .LBB2_6: @ %cond.load5
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s12, [r0, #2]
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    vmovx.f16 s16, s17
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q3[0], r4
+; CHECK-NEXT:    vmov.16 q3[1], r2
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vmov.16 q3[2], r2
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:    lsls r2, r3, #29
+; CHECK-NEXT:    bmi .LBB2_11
+; CHECK-NEXT:  .LBB2_7: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q4, q3
+; CHECK-NEXT:    lsls r2, r3, #28
+; CHECK-NEXT:    bmi .LBB2_12
+; CHECK-NEXT:  .LBB2_8: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q3, q4
+; CHECK-NEXT:    b .LBB2_13
+; CHECK-NEXT:  .LBB2_9: @ %cond.load
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s16, [r0]
+; CHECK-NEXT:    lsls r2, r3, #30
+; CHECK-NEXT:    bmi .LBB2_6
+; CHECK-NEXT:  .LBB2_10: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q3, q4
+; CHECK-NEXT:    lsls r2, r3, #29
+; CHECK-NEXT:    bpl .LBB2_7
+; CHECK-NEXT:  .LBB2_11: @ %cond.load8
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s16, s12
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vldr.16 s20, [r0, #4]
+; CHECK-NEXT:    vmov r4, s16
+; CHECK-NEXT:    vmov.16 q4[0], r2
+; CHECK-NEXT:    vmovx.f16 s12, s13
+; CHECK-NEXT:    vmov.16 q4[1], r4
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    lsls r2, r3, #28
+; CHECK-NEXT:    bpl .LBB2_8
+; CHECK-NEXT:  .LBB2_12: @ %cond.load11
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s12, s16
+; CHECK-NEXT:    vmov r3, s16
+; CHECK-NEXT:    vmov r2, s12
+; CHECK-NEXT:    vmov.16 q3[0], r3
+; CHECK-NEXT:    vmov.16 q3[1], r2
+; CHECK-NEXT:    vmov r2, s17
+; CHECK-NEXT:    vldr.16 s16, [r0, #6]
+; CHECK-NEXT:    vmov.16 q3[2], r2
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q3[3], r2
+; CHECK-NEXT:  .LBB2_13: @ %else12
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    @ implicit-def: $q5
+; CHECK-NEXT:    and r3, r2, #1
+; CHECK-NEXT:    rsbs r4, r3, #0
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    bfi r3, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r2, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r3, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r2, #8, #1
+; CHECK-NEXT:    ubfx r2, r2, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r3, r4, #2, #1
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    bfi r3, r2, #3, #1
+; CHECK-NEXT:    lsls r2, r3, #31
+; CHECK-NEXT:    bne .LBB2_17
+; CHECK-NEXT:  @ %bb.14: @ %else16
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    lsls r2, r3, #30
+; CHECK-NEXT:    bpl .LBB2_18
+; CHECK-NEXT:  .LBB2_15: @ %cond.load18
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s16, [r1, #2]
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmovx.f16 s20, s21
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q4[0], r4
+; CHECK-NEXT:    vmov.16 q4[1], r2
+; CHECK-NEXT:    vmov r2, s21
+; CHECK-NEXT:    vmov.16 q4[2], r2
+; CHECK-NEXT:    vmov r2, s20
+; CHECK-NEXT:    vmov.16 q4[3], r2
+; CHECK-NEXT:    lsls r2, r3, #29
+; CHECK-NEXT:    bmi .LBB2_19
+; CHECK-NEXT:  .LBB2_16: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q5, q4
+; CHECK-NEXT:    lsls r2, r3, #28
+; CHECK-NEXT:    bmi.w .LBB2_2
+; CHECK-NEXT:    b .LBB2_20
+; CHECK-NEXT:  .LBB2_17: @ %cond.load15
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vldr.16 s20, [r1]
+; CHECK-NEXT:    lsls r2, r3, #30
+; CHECK-NEXT:    bmi .LBB2_15
+; CHECK-NEXT:  .LBB2_18: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q4, q5
+; CHECK-NEXT:    lsls r2, r3, #29
+; CHECK-NEXT:    bpl .LBB2_16
+; CHECK-NEXT:  .LBB2_19: @ %cond.load21
+; CHECK-NEXT:    @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmovx.f16 s20, s16
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vldr.16 s24, [r1, #4]
+; CHECK-NEXT:    vmov r4, s20
+; CHECK-NEXT:    vmov.16 q5[0], r2
+; CHECK-NEXT:    vmovx.f16 s16, s17
+; CHECK-NEXT:    vmov.16 q5[1], r4
+; CHECK-NEXT:    vmov r2, s24
+; CHECK-NEXT:    vmov.16 q5[2], r2
+; CHECK-NEXT:    vmov r2, s16
+; CHECK-NEXT:    vmov.16 q5[3], r2
+; CHECK-NEXT:    lsls r2, r3, #28
+; CHECK-NEXT:    bmi.w .LBB2_2
+; CHECK-NEXT:  .LBB2_20: @ in Loop: Header=BB2_4 Depth=1
+; CHECK-NEXT:    vmov q4, q5
+; CHECK-NEXT:    b .LBB2_3
+; CHECK-NEXT:  .LBB2_21: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vmov.f32 s4, s2
+; CHECK-NEXT:    vmov.f32 s5, s3
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    vmov.32 r0, q0[1]
+; CHECK-NEXT:    vdup.32 q1, r0
+; CHECK-NEXT:    vadd.f32 q0, q0, q1
+; CHECK-NEXT:    b .LBB2_23
+; CHECK-NEXT:  .LBB2_22:
+; CHECK-NEXT:    vldr s0, .LCPI2_0
+; CHECK-NEXT:  .LBB2_23: @ %for.cond.cleanup
+; CHECK-NEXT:    @ kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12}
+; CHECK-NEXT:    pop {r4, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.24:
+; CHECK-NEXT:  .LCPI2_1:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x float> [ zeroinitializer, %vector.ph ], [ %7, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds half, half* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast half* %0 to <4 x half>*
+  %wide.masked.load = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %2, i32 2, <4 x i1> %1, <4 x half> undef)
+  %3 = getelementptr inbounds half, half* %c, i32 %index
+  %4 = bitcast half* %3 to <4 x half>*
+  %wide.masked.load13 = call <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>* %4, i32 2, <4 x i1> %1, <4 x half> undef)
+  %5 = fmul fast <4 x half> %wide.masked.load13, %wide.masked.load
+  %6 = fpext <4 x half> %5 to <4 x float>
+  %7 = fadd fast <4 x float> %vec.phi, %6
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %9 = select <4 x i1> %1, <4 x float> %7, <4 x float> %vec.phi
+  %rdx.shuf = shufflevector <4 x float> %9, <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+  %bin.rdx = fadd fast <4 x float> %9, %rdx.shuf
+  %rdx.shuf14 = shufflevector <4 x float> %bin.rdx, <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+  %bin.rdx15 = fadd fast <4 x float> %bin.rdx, %rdx.shuf14
+  %10 = extractelement <4 x float> %bin.rdx15, i32 0
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %a.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %10, %middle.block ]
+  ret float %a.0.lcssa
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>*, i32 immarg, <4 x i1>, <4 x float>)
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.masked.store.v4f32.p0v4f32(<4 x float>, <4 x float>*, i32 immarg, <4 x i1>)
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x half> @llvm.masked.load.v4f16.p0v4f16(<4 x half>*, i32 immarg, <4 x i1>, <4 x half>)
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
index 92bd7344dfa2b..74fb334d34705 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/massive.mir
@@ -1,10 +1,15 @@
-# RUN: llc -mtriple=armv8.1m.main -mattr=+lob -run-pass=arm-low-overhead-loops %s -o - | FileCheck %s
+# RUN: llc -mtriple=armv8.1m.main -mattr=+lob -run-pass=arm-low-overhead-loops --verify-machineinstrs %s -o - | FileCheck %s
 # CHECK: for.body:
 # CHECK-NOT: t2DLS
 # CHECK-NOT: t2LEUpdate
 
---- |  
-  define dso_local arm_aapcscc void @massive(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) local_unnamed_addr {
+--- |
+  ; ModuleID = 'massive.ll'
+  source_filename = "massive.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define dso_local arm_aapcscc void @massive(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
   entry:
     %cmp8 = icmp eq i32 %N, 0
     br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
@@ -25,13 +30,13 @@
     %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
     %0 = phi i32 [ %N, %for.body.preheader ], [ %3, %for.body ]
     %size = call i32 @llvm.arm.space(i32 4096, i32 undef)
-    %scevgep1 = getelementptr i32, i32* %lsr.iv9, i32 1
-    %1 = load i32, i32* %scevgep1, align 4, !tbaa !3
-    %scevgep5 = getelementptr i32, i32* %lsr.iv5, i32 1
-    %2 = load i32, i32* %scevgep5, align 4, !tbaa !3
+    %scevgep3 = getelementptr i32, i32* %lsr.iv9, i32 1
+    %1 = load i32, i32* %scevgep3, align 4
+    %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+    %2 = load i32, i32* %scevgep7, align 4
     %mul = mul nsw i32 %2, %1
-    %scevgep9 = getelementptr i32, i32* %lsr.iv1, i32 1
-    store i32 %mul, i32* %scevgep9, align 4, !tbaa !3
+    %scevgep11 = getelementptr i32, i32* %lsr.iv1, i32 1
+    store i32 %mul, i32* %scevgep11, align 4
     %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
     %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
     %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
@@ -42,26 +47,18 @@
   
   ; Function Attrs: nounwind
   declare i32 @llvm.arm.space(i32 immarg, i32) #0
-
+  
   ; Function Attrs: noduplicate nounwind
   declare void @llvm.set.loop.iterations.i32(i32) #1
   
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #0
+  
   attributes #0 = { nounwind }
   attributes #1 = { noduplicate nounwind }
-  
-  !llvm.module.flags = !{!0, !1}
-  !llvm.ident = !{!2}
-  
-  !0 = !{i32 1, !"wchar_size", i32 4}
-  !1 = !{i32 1, !"min_enum_size", i32 4}
-  !2 = !{!"clang version 9.0.0 (http://llvm.org/git/clang.git a9c7c0fc5d468f3d18a5c6beb697ab0d5be2ff4c) (http://llvm.org/git/llvm.git f34bff0c141a04a5182d57e2cfb1e4bc582c81b0)"}
-  !3 = !{!4, !4, i64 0}
-  !4 = !{!"int", !5, i64 0}
-  !5 = !{!"omnipotent char", !6, i64 0}
-  !6 = !{!"Simple C/C++ TBAA"}
 
 ...
 ---
@@ -72,7 +69,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -113,6 +110,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r3, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -129,12 +127,13 @@ body:             |
   
   bb.1.for.body:
     successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+    liveins: $lr, $r0, $r1, $r2
   
     dead renamable $r3 = SPACE 4096, undef renamable $r0
-    renamable $r12, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep1, !tbaa !3)
-    renamable $r3, renamable $r2 = t2LDR_PRE killed renamable $r2, 4, 14, $noreg :: (load 4 from %ir.scevgep5, !tbaa !3)
+    renamable $r12, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep3)
+    renamable $r3, renamable $r2 = t2LDR_PRE killed renamable $r2, 4, 14, $noreg :: (load 4 from %ir.scevgep7)
     renamable $r3 = nsw t2MUL killed renamable $r3, killed renamable $r12, 14, $noreg
-    early-clobber renamable $r0 = t2STR_PRE killed renamable $r3, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep9, !tbaa !3)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r3, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep11)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.1, implicit-def dead $cpsr
     tB %bb.2, 14, $noreg
@@ -143,4 +142,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r7, def $pc
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir
new file mode 100644
index 0000000000000..47f933fdebd63
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mov-after-dls.mir
@@ -0,0 +1,115 @@
+# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops %s -verify-machineinstrs -o - | FileCheck %s
+# CHECK: $lr = t2DLS $r0
+# CHECK-NOT: $lr = tMOVr $r0
+# CHECK: $lr = t2LEUpdate renamable $lr, %bb.1
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define i32 @do_copy(i32 %n, i32* nocapture %p, i32* nocapture readonly %q) {
+  entry:
+    %scevgep = getelementptr i32, i32* %q, i32 -1
+    %scevgep3 = getelementptr i32, i32* %p, i32 -1
+    call void @llvm.set.loop.iterations.i32(i32 %n)
+    br label %while.body
+  
+  while.body:                                       ; preds = %while.body, %entry
+    %lsr.iv4 = phi i32* [ %scevgep5, %while.body ], [ %scevgep3, %entry ]
+    %lsr.iv = phi i32* [ %scevgep1, %while.body ], [ %scevgep, %entry ]
+    %0 = phi i32 [ %n, %entry ], [ %2, %while.body ]
+    %scevgep6 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep2 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %1 = load i32, i32* %scevgep6, align 4
+    store i32 %1, i32* %scevgep2, align 4
+    %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep5 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+    %3 = icmp ne i32 %2, 0
+    br i1 %3, label %while.body, label %while.end
+  
+  while.end:                                        ; preds = %while.body
+    ret i32 0
+  }
+  
+  declare void @llvm.set.loop.iterations.i32(i32) #0
+  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
+  
+  attributes #0 = { noduplicate nounwind }
+  attributes #1 = { nounwind }
+
+...
+---
+name:            do_copy
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+  
+    frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    t2DoLoopStart $r0
+    $lr = tMOVr killed $r0, 14, $noreg
+    renamable $r0, dead $cpsr = tSUBi3 killed renamable $r1, 4, 14, $noreg
+    renamable $r1, dead $cpsr = tSUBi3 killed renamable $r2, 4, 14, $noreg
+  
+  bb.1.while.body:
+    successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+    liveins: $lr, $r0, $r1
+  
+    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep6)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep2)
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd renamable $lr, %bb.1, implicit-def dead $cpsr
+    tB %bb.2, 14, $noreg
+  
+  bb.2.while.end:
+    $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
index bc5f88acfc6c8..2e3aa0d97baf4 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/multiblock-massive.mir
@@ -1,10 +1,15 @@
-# RUN: llc -mtriple=armv8.1m.main -mattr=+lob -run-pass=arm-low-overhead-loops %s -o - | FileCheck %s
+# RUN: llc -mtriple=armv8.1m.main -mattr=+lob -run-pass=arm-low-overhead-loops --verify-machineinstrs %s -o - | FileCheck %s
 # CHECK: for.body:
 # CHECK-NOT: t2DLS
 # CHECK-NOT: t2LEUpdate
 
 --- |
-  define dso_local arm_aapcscc void @size_limit(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) local_unnamed_addr {
+  ; ModuleID = 'multiblock-massive.ll'
+  source_filename = "multiblock-massive.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define void @size_limit(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
   entry:
     %cmp8 = icmp eq i32 %N, 0
     br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
@@ -53,6 +58,9 @@
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #0
+  
   attributes #0 = { nounwind }
   attributes #1 = { noduplicate nounwind }
 
@@ -65,7 +73,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -106,6 +114,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2, $r3, $r4, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -120,6 +129,7 @@ body:             |
   
   bb.1.for.end:
     successors: %bb.4(0x04000000), %bb.2(0x7c000000)
+    liveins: $lr, $r0, $r1, $r2
   
     renamable $r1, dead $cpsr = tADDi8 killed renamable $r1, 4, 14, $noreg
     renamable $r2, dead $cpsr = tADDi8 killed renamable $r2, 4, 14, $noreg
@@ -130,6 +140,7 @@ body:             |
   
   bb.2.for.body:
     successors: %bb.3(0x50000000), %bb.1(0x30000000)
+    liveins: $lr, $r0, $r1, $r2
   
     dead renamable $r3 = SPACE 3072, undef renamable $r0
     renamable $r3 = tLDRi renamable $r1, 0, 14, $noreg :: (load 4 from %ir.lsr.iv4)
@@ -141,6 +152,7 @@ body:             |
   
   bb.3.middle.block:
     successors: %bb.1(0x80000000)
+    liveins: $lr, $r0, $r1, $r2, $r3, $r12
   
     renamable $r3 = t2UDIV killed renamable $r12, killed renamable $r3, 14, $noreg
     tSTRi killed renamable $r3, renamable $r0, 0, 14, $noreg :: (store 4 into %ir.lsr.iv1)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
new file mode 100644
index 0000000000000..841b6631eba16
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-float-loops.ll
@@ -0,0 +1,2014 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fpregs,+fullfp16 %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc void @float_float_mul(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB0_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB0_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB0_4
+; CHECK-NEXT:  .LBB0_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB0_11
+; CHECK-NEXT:  .LBB0_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB0_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB0_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB0_6
+; CHECK-NEXT:  .LBB0_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB0_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB0_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB0_9
+; CHECK-NEXT:  .LBB0_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB0_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB0_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB0_4
+; CHECK-NEXT:    b .LBB0_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %mul.prol = fmul float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %mul.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fmul <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %mul = fmul float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %mul.1 = fmul float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %mul.2 = fmul float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %mul.3 = fmul float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_float_add(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB1_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB1_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB1_4
+; CHECK-NEXT:  .LBB1_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB1_11
+; CHECK-NEXT:  .LBB1_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB1_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB1_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB1_6
+; CHECK-NEXT:  .LBB1_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB1_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB1_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vadd.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB1_9
+; CHECK-NEXT:  .LBB1_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB1_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vadd.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB1_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB1_4
+; CHECK-NEXT:    b .LBB1_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %add.prol = fadd float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %add.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fadd <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %add = fadd float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %add.1 = fadd float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %add.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %add.2 = fadd float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %add.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %add.3 = fadd float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %add.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_float_sub(float* nocapture readonly %a, float* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_float_sub:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB2_10
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB2_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB2_4
+; CHECK-NEXT:  .LBB2_3: @ %vector.memcheck
+; CHECK-NEXT:    add.w r5, r2, r3, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r3, lsl #2
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r3, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r2
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r2
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB2_11
+; CHECK-NEXT:  .LBB2_4: @ %for.body.preheader22
+; CHECK-NEXT:    mvn.w r7, r12
+; CHECK-NEXT:    adds r5, r7, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB2_7
+; CHECK-NEXT:  @ %bb.5: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r7, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r7
+; CHECK-NEXT:    adds r6, r1, r7
+; CHECK-NEXT:    add r7, r2
+; CHECK-NEXT:  .LBB2_6: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r7, #4]
+; CHECK-NEXT:    adds r7, #4
+; CHECK-NEXT:    le lr, .LBB2_6
+; CHECK-NEXT:  .LBB2_7: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB2_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r8, r0, #8
+; CHECK-NEXT:    sub.w r10, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB2_9: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r7, r8, r12
+; CHECK-NEXT:    vldr s0, [r6, #4]
+; CHECK-NEXT:    add.w r3, r10, r12
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    add.w r11, r5, r12
+; CHECK-NEXT:    add.w r8, r8, #16
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r3, #12]
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #12]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vldr s2, [r7, #16]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r11, #16]
+; CHECK-NEXT:    vldr s0, [r6, #16]
+; CHECK-NEXT:    vldr s2, [r4, #16]
+; CHECK-NEXT:    vsub.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB2_9
+; CHECK-NEXT:  .LBB2_10: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+; CHECK-NEXT:  .LBB2_11: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_12: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vsub.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB2_12
+; CHECK-NEXT:  @ %bb.13: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne.w .LBB2_4
+; CHECK-NEXT:    b .LBB2_10
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader22, label %vector.memcheck
+
+for.body.preheader22:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader22, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader22 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader22 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds float, float* %b, i32 %i.09.prol
+  %3 = load float, float* %arrayidx1.prol, align 4
+  %sub.prol = fsub float %2, %3
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %sub.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader22
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader22 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %scevgep16 = getelementptr float, float* %b, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt float* %scevgep16, %c
+  %bound119 = icmp ugt float* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader22, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds float, float* %b, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  %wide.load21 = load <4 x float>, <4 x float>* %8, align 4
+  %9 = fsub <4 x float> %wide.load, %wide.load21
+  %10 = getelementptr inbounds float, float* %c, i32 %index
+  %11 = bitcast float* %10 to <4 x float>*
+  store <4 x float> %9, <4 x float>* %11, align 4
+  %index.next = add i32 %index, 4
+  %12 = icmp eq i32 %index.next, %n.vec
+  br i1 %12, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader22
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %13 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %b, i32 %i.09
+  %14 = load float, float* %arrayidx1, align 4
+  %sub = fsub float %13, %14
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %sub, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %15 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds float, float* %b, i32 %inc
+  %16 = load float, float* %arrayidx1.1, align 4
+  %sub.1 = fsub float %15, %16
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %sub.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %17 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds float, float* %b, i32 %inc.1
+  %18 = load float, float* %arrayidx1.2, align 4
+  %sub.2 = fsub float %17, %18
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %sub.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %19 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds float, float* %b, i32 %inc.2
+  %20 = load float, float* %arrayidx1.3, align 4
+  %sub.3 = fsub float %19, %20
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %sub.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_int_mul(float* nocapture readonly %a, i32* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_int_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq.w .LBB3_13
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bls .LBB3_6
+; CHECK-NEXT:  @ %bb.2: @ %vector.memcheck
+; CHECK-NEXT:    add.w r7, r0, r3, lsl #2
+; CHECK-NEXT:    cmp r7, r2
+; CHECK-NEXT:    itt hi
+; CHECK-NEXT:    addhi.w r7, r2, r3, lsl #2
+; CHECK-NEXT:    cmphi r7, r0
+; CHECK-NEXT:    bhi .LBB3_6
+; CHECK-NEXT:  @ %bb.3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    sub.w r7, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r5, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r4, #16]!
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    vmul.f32 q0, q1, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB3_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    bne .LBB3_7
+; CHECK-NEXT:    b .LBB3_13
+; CHECK-NEXT:  .LBB3_6:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:  .LBB3_7: @ %for.body.preheader16
+; CHECK-NEXT:    mvn.w r6, r12
+; CHECK-NEXT:    adds r5, r6, r3
+; CHECK-NEXT:    and lr, r3, #3
+; CHECK-NEXT:    wls lr, lr, .LBB3_10
+; CHECK-NEXT:  @ %bb.8: @ %for.body.prol.preheader
+; CHECK-NEXT:    mvn r4, #3
+; CHECK-NEXT:    add.w r6, r4, r12, lsl #2
+; CHECK-NEXT:    adds r4, r0, r6
+; CHECK-NEXT:    add r6, r2
+; CHECK-NEXT:  .LBB3_9: @ %for.body.prol
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r7, r1, r12, lsl #2
+; CHECK-NEXT:    add.w r12, r12, #1
+; CHECK-NEXT:    vldr s0, [r7]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r4, #4]
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r6, #4]
+; CHECK-NEXT:    adds r6, #4
+; CHECK-NEXT:    le lr, .LBB3_9
+; CHECK-NEXT:  .LBB3_10: @ %for.body.prol.loopexit
+; CHECK-NEXT:    cmp r5, #3
+; CHECK-NEXT:    blo .LBB3_13
+; CHECK-NEXT:  @ %bb.11: @ %for.body.preheader1
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r10, r0, #8
+; CHECK-NEXT:    sub.w r4, r1, #8
+; CHECK-NEXT:    sub.w r5, r2, #8
+; CHECK-NEXT:    subs r0, #4
+; CHECK-NEXT:    subs r1, #4
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    lsl.w r12, r12, #2
+; CHECK-NEXT:  .LBB3_12: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    add.w r3, r1, r12
+; CHECK-NEXT:    add.w r11, r0, r12
+; CHECK-NEXT:    add.w r9, r2, r12
+; CHECK-NEXT:    add.w r6, r4, r12
+; CHECK-NEXT:    vldr s0, [r3, #4]
+; CHECK-NEXT:    add.w r7, r10, r12
+; CHECK-NEXT:    add.w r8, r5, r12
+; CHECK-NEXT:    add.w r10, r10, #16
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r11, #4]
+; CHECK-NEXT:    adds r4, #16
+; CHECK-NEXT:    adds r5, #16
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    adds r2, #16
+; CHECK-NEXT:    subs.w lr, lr, #4
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #4]
+; CHECK-NEXT:    vldr s0, [r6, #12]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r7, #12]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r8, #12]
+; CHECK-NEXT:    vldr s2, [r6, #16]
+; CHECK-NEXT:    vldr s0, [r7, #16]
+; CHECK-NEXT:    vcvt.f32.s32 s2, s2
+; CHECK-NEXT:    vmul.f32 s0, s0, s2
+; CHECK-NEXT:    vstr s0, [r8, #16]
+; CHECK-NEXT:    vldr s0, [r3, #16]
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vldr s2, [r11, #16]
+; CHECK-NEXT:    vmul.f32 s0, s2, s0
+; CHECK-NEXT:    vstr s0, [r9, #16]
+; CHECK-NEXT:    bne .LBB3_12
+; CHECK-NEXT:  .LBB3_13: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader16, label %vector.memcheck
+
+for.body.preheader16:                             ; preds = %middle.block, %vector.memcheck, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %vector.memcheck ], [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  %0 = xor i32 %i.09.ph, -1
+  %1 = add i32 %0, %N
+  %xtraiter = and i32 %N, 3
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol:                                    ; preds = %for.body.preheader16, %for.body.prol
+  %i.09.prol = phi i32 [ %inc.prol, %for.body.prol ], [ %i.09.ph, %for.body.preheader16 ]
+  %prol.iter = phi i32 [ %prol.iter.sub, %for.body.prol ], [ %xtraiter, %for.body.preheader16 ]
+  %arrayidx.prol = getelementptr inbounds float, float* %a, i32 %i.09.prol
+  %2 = load float, float* %arrayidx.prol, align 4
+  %arrayidx1.prol = getelementptr inbounds i32, i32* %b, i32 %i.09.prol
+  %3 = load i32, i32* %arrayidx1.prol, align 4
+  %conv.prol = sitofp i32 %3 to float
+  %mul.prol = fmul float %2, %conv.prol
+  %arrayidx2.prol = getelementptr inbounds float, float* %c, i32 %i.09.prol
+  store float %mul.prol, float* %arrayidx2.prol, align 4
+  %inc.prol = add nuw i32 %i.09.prol, 1
+  %prol.iter.sub = add i32 %prol.iter, -1
+  %prol.iter.cmp = icmp eq i32 %prol.iter.sub, 0
+  br i1 %prol.iter.cmp, label %for.body.prol.loopexit, label %for.body.prol
+
+for.body.prol.loopexit:                           ; preds = %for.body.prol, %for.body.preheader16
+  %i.09.unr = phi i32 [ %i.09.ph, %for.body.preheader16 ], [ %inc.prol, %for.body.prol ]
+  %4 = icmp ult i32 %1, 3
+  br i1 %4, label %for.cond.cleanup, label %for.body
+
+vector.memcheck:                                  ; preds = %for.body.preheader
+  %scevgep = getelementptr float, float* %c, i32 %N
+  %scevgep13 = getelementptr float, float* %a, i32 %N
+  %bound0 = icmp ugt float* %scevgep13, %c
+  %bound1 = icmp ugt float* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  br i1 %found.conflict, label %for.body.preheader16, label %vector.ph
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %5 = getelementptr inbounds float, float* %a, i32 %index
+  %6 = bitcast float* %5 to <4 x float>*
+  %wide.load = load <4 x float>, <4 x float>* %6, align 4
+  %7 = getelementptr inbounds i32, i32* %b, i32 %index
+  %8 = bitcast i32* %7 to <4 x i32>*
+  %wide.load15 = load <4 x i32>, <4 x i32>* %8, align 4
+  %9 = sitofp <4 x i32> %wide.load15 to <4 x float>
+  %10 = fmul <4 x float> %wide.load, %9
+  %11 = getelementptr inbounds float, float* %c, i32 %index
+  %12 = bitcast float* %11 to <4 x float>*
+  store <4 x float> %10, <4 x float>* %12, align 4
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader16
+
+for.cond.cleanup:                                 ; preds = %for.body.prol.loopexit, %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.prol.loopexit, %for.body
+  %i.09 = phi i32 [ %inc.3, %for.body ], [ %i.09.unr, %for.body.prol.loopexit ]
+  %arrayidx = getelementptr inbounds float, float* %a, i32 %i.09
+  %14 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %15 = load i32, i32* %arrayidx1, align 4
+  %conv = sitofp i32 %15 to float
+  %mul = fmul float %14, %conv
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %mul, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds float, float* %a, i32 %inc
+  %16 = load float, float* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %b, i32 %inc
+  %17 = load i32, i32* %arrayidx1.1, align 4
+  %conv.1 = sitofp i32 %17 to float
+  %mul.1 = fmul float %16, %conv.1
+  %arrayidx2.1 = getelementptr inbounds float, float* %c, i32 %inc
+  store float %mul.1, float* %arrayidx2.1, align 4
+  %inc.1 = add nuw i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds float, float* %a, i32 %inc.1
+  %18 = load float, float* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %b, i32 %inc.1
+  %19 = load i32, i32* %arrayidx1.2, align 4
+  %conv.2 = sitofp i32 %19 to float
+  %mul.2 = fmul float %18, %conv.2
+  %arrayidx2.2 = getelementptr inbounds float, float* %c, i32 %inc.1
+  store float %mul.2, float* %arrayidx2.2, align 4
+  %inc.2 = add nuw i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds float, float* %a, i32 %inc.2
+  %20 = load float, float* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %b, i32 %inc.2
+  %21 = load i32, i32* %arrayidx1.3, align 4
+  %conv.3 = sitofp i32 %21 to float
+  %mul.3 = fmul float %20, %conv.3
+  %arrayidx2.3 = getelementptr inbounds float, float* %c, i32 %inc.2
+  store float %mul.3, float* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %exitcond.3 = icmp eq i32 %inc.3, %N
+  br i1 %exitcond.3, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @float_int_int_mul(i32* nocapture readonly %a, i32* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: float_int_int_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB4_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB4_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB4_6
+; CHECK-NEXT:  .LBB4_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #16
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #16
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB4_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldrw.u32 q0, [r4, #16]!
+; CHECK-NEXT:    vldrw.u32 q1, [r5, #16]!
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    vcvt.f32.s32 q0, q0
+; CHECK-NEXT:    vstrb.8 q0, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB4_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    popeq {r4, r5, r6, pc}
+; CHECK-NEXT:  .LBB4_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    mvn r3, #3
+; CHECK-NEXT:    add.w r3, r3, r12, lsl #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add r0, r3
+; CHECK-NEXT:    add r1, r3
+; CHECK-NEXT:    add r2, r3
+; CHECK-NEXT:  .LBB4_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r0, #4]!
+; CHECK-NEXT:    ldr r6, [r1, #4]!
+; CHECK-NEXT:    muls r3, r6, r3
+; CHECK-NEXT:    vmov s0, r3
+; CHECK-NEXT:    vcvt.f32.s32 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB4_7
+; CHECK-NEXT:  .LBB4_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r6, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = bitcast i32* %0 to <4 x i32>*
+  %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
+  %2 = getelementptr inbounds i32, i32* %b, i32 %index
+  %3 = bitcast i32* %2 to <4 x i32>*
+  %wide.load10 = load <4 x i32>, <4 x i32>* %3, align 4
+  %4 = mul nsw <4 x i32> %wide.load10, %wide.load
+  %5 = sitofp <4 x i32> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.09
+  %9 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %10 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %10, %9
+  %conv = sitofp i32 %mul to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_mul(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB5_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB5_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB5_6
+; CHECK-NEXT:  .LBB5_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vmul.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB5_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB5_8
+; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB5_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vmul.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB5_7
+; CHECK-NEXT:  .LBB5_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fmul <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %mul = fmul half %9, %10
+  %conv = fpext half %mul to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_add(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_add:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB6_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB6_6
+; CHECK-NEXT:  .LBB6_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vadd.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB6_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB6_8
+; CHECK-NEXT:  .LBB6_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB6_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vadd.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB6_7
+; CHECK-NEXT:  .LBB6_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fadd <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %add = fadd half %9, %10
+  %conv = fpext half %add to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_half_sub(half* nocapture readonly %a, half* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_half_sub:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB7_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB7_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB7_6
+; CHECK-NEXT:  .LBB7_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r5, #8]!
+; CHECK-NEXT:    ldr r7, [r4, #8]!
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q0[0], r7
+; CHECK-NEXT:    ldr r7, [r5, #4]
+; CHECK-NEXT:    ldr.w r8, [r4, #4]
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmov.32 q0[1], r8
+; CHECK-NEXT:    vsub.f16 q0, q0, q1
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB7_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB7_8
+; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader11
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB7_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s0, [r1, #2]
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vsub.f16 s0, s2, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB7_7
+; CHECK-NEXT:  .LBB7_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader11, label %vector.ph
+
+for.body.preheader11:                             ; preds = %middle.block, %for.body.preheader
+  %i.09.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds half, half* %b, i32 %index
+  %3 = bitcast half* %2 to <4 x half>*
+  %wide.load10 = load <4 x half>, <4 x half>* %3, align 2
+  %4 = fsub <4 x half> %wide.load, %wide.load10
+  %5 = fpext <4 x half> %4 to <4 x float>
+  %6 = getelementptr inbounds float, float* %c, i32 %index
+  %7 = bitcast float* %6 to <4 x float>*
+  store <4 x float> %5, <4 x float>* %7, align 4
+  %index.next = add i32 %index, 4
+  %8 = icmp eq i32 %index.next, %n.vec
+  br i1 %8, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader11
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader11, %for.body
+  %i.09 = phi i32 [ %inc, %for.body ], [ %i.09.ph, %for.body.preheader11 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.09
+  %9 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.09
+  %10 = load half, half* %arrayidx1, align 2
+  %sub = fsub half %9, %10
+  %conv = fpext half %sub to float
+  %arrayidx2 = getelementptr inbounds float, float* %c, i32 %i.09
+  store float %conv, float* %arrayidx2, align 4
+  %inc = add nuw i32 %i.09, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc void @half_short_mul(half* nocapture readonly %a, i16* nocapture readonly %b, float* nocapture %c, i32 %N) {
+; CHECK-LABEL: half_short_mul:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB8_8
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhi .LBB8_3
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB8_6
+; CHECK-NEXT:  .LBB8_3: @ %vector.ph
+; CHECK-NEXT:    bic r12, r3, #3
+; CHECK-NEXT:    movs r5, #1
+; CHECK-NEXT:    sub.w r6, r12, #4
+; CHECK-NEXT:    sub.w r4, r0, #8
+; CHECK-NEXT:    add.w lr, r5, r6, lsr #2
+; CHECK-NEXT:    sub.w r5, r1, #8
+; CHECK-NEXT:    sub.w r6, r2, #16
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB8_4: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r8, [r4, #8]!
+; CHECK-NEXT:    vldrh.u32 q0, [r5, #8]!
+; CHECK-NEXT:    vmov r7, s0
+; CHECK-NEXT:    vmov.16 q1[0], r7
+; CHECK-NEXT:    vmov r7, s1
+; CHECK-NEXT:    vmov.16 q1[1], r7
+; CHECK-NEXT:    vmov r7, s2
+; CHECK-NEXT:    vmov.16 q1[2], r7
+; CHECK-NEXT:    vmov r7, s3
+; CHECK-NEXT:    vmov.16 q1[3], r7
+; CHECK-NEXT:    ldr r7, [r4, #4]
+; CHECK-NEXT:    vcvt.f16.s16 q0, q1
+; CHECK-NEXT:    vmov.32 q1[0], r8
+; CHECK-NEXT:    vmov.32 q1[1], r7
+; CHECK-NEXT:    vmul.f16 q0, q1, q0
+; CHECK-NEXT:    vmovx.f16 s6, s1
+; CHECK-NEXT:    vmovx.f16 s4, s0
+; CHECK-NEXT:    vcvtb.f32.f16 s11, s6
+; CHECK-NEXT:    vcvtb.f32.f16 s10, s1
+; CHECK-NEXT:    vcvtb.f32.f16 s9, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s0
+; CHECK-NEXT:    vstrb.8 q2, [r6, #16]!
+; CHECK-NEXT:    le lr, .LBB8_4
+; CHECK-NEXT:  @ %bb.5: @ %middle.block
+; CHECK-NEXT:    cmp r12, r3
+; CHECK-NEXT:    beq .LBB8_8
+; CHECK-NEXT:  .LBB8_6: @ %for.body.preheader13
+; CHECK-NEXT:    sub.w lr, r3, r12
+; CHECK-NEXT:    sub.w r3, r12, #1
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:    add.w r0, r0, r3, lsl #1
+; CHECK-NEXT:    add.w r1, r1, r3, lsl #1
+; CHECK-NEXT:    add.w r2, r2, r3, lsl #2
+; CHECK-NEXT:  .LBB8_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrsh r3, [r1, #2]!
+; CHECK-NEXT:    vldr.16 s0, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vmov s2, r3
+; CHECK-NEXT:    vcvt.f16.s32 s2, s2
+; CHECK-NEXT:    vmul.f16 s0, s0, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s0, s0
+; CHECK-NEXT:    vstr s0, [r2, #4]
+; CHECK-NEXT:    adds r2, #4
+; CHECK-NEXT:    le lr, .LBB8_7
+; CHECK-NEXT:  .LBB8_8: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %min.iters.check = icmp ult i32 %N, 4
+  br i1 %min.iters.check, label %for.body.preheader13, label %vector.ph
+
+for.body.preheader13:                             ; preds = %middle.block, %for.body.preheader
+  %i.011.ph = phi i32 [ 0, %for.body.preheader ], [ %n.vec, %middle.block ]
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.preheader
+  %n.vec = and i32 %N, -4
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %0 = getelementptr inbounds half, half* %a, i32 %index
+  %1 = bitcast half* %0 to <4 x half>*
+  %wide.load = load <4 x half>, <4 x half>* %1, align 2
+  %2 = getelementptr inbounds i16, i16* %b, i32 %index
+  %3 = bitcast i16* %2 to <4 x i16>*
+  %wide.load12 = load <4 x i16>, <4 x i16>* %3, align 2
+  %4 = sitofp <4 x i16> %wide.load12 to <4 x half>
+  %5 = fmul <4 x half> %wide.load, %4
+  %6 = fpext <4 x half> %5 to <4 x float>
+  %7 = getelementptr inbounds float, float* %c, i32 %index
+  %8 = bitcast float* %7 to <4 x float>*
+  store <4 x float> %6, <4 x float>* %8, align 4
+  %index.next = add i32 %index, 4
+  %9 = icmp eq i32 %index.next, %n.vec
+  br i1 %9, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %cmp.n = icmp eq i32 %n.vec, %N
+  br i1 %cmp.n, label %for.cond.cleanup, label %for.body.preheader13
+
+for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body.preheader13, %for.body
+  %i.011 = phi i32 [ %inc, %for.body ], [ %i.011.ph, %for.body.preheader13 ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.011
+  %10 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %b, i32 %i.011
+  %11 = load i16, i16* %arrayidx1, align 2
+  %conv2 = sitofp i16 %11 to half
+  %mul = fmul half %10, %conv2
+  %conv3 = fpext half %mul to float
+  %arrayidx4 = getelementptr inbounds float, float* %c, i32 %i.011
+  store float %conv3, float* %arrayidx4, align 4
+  %inc = add nuw i32 %i.011, 1
+  %exitcond = icmp eq i32 %inc, %N
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_half_mac(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_half_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    cbz r2, .LBB9_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r4, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB9_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    mov r5, r4
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB9_6
+; CHECK-NEXT:  .LBB9_3:
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB9_4: @ %for.body.preheader.new
+; CHECK-NEXT:    subs r2, r2, r4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI9_0
+; CHECK-NEXT:    mov r5, r4
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    sub.w r3, r0, #8
+; CHECK-NEXT:    sub.w r2, r1, #8
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB9_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r2, #14]
+; CHECK-NEXT:    vldr.16 s4, [r3, #14]
+; CHECK-NEXT:    vldr.16 s6, [r3, #12]
+; CHECK-NEXT:    vldr.16 s8, [r3, #10]
+; CHECK-NEXT:    vmul.f16 s2, s4, s2
+; CHECK-NEXT:    vldr.16 s4, [r2, #12]
+; CHECK-NEXT:    vldr.16 s10, [r3, #8]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vmul.f16 s4, s6, s4
+; CHECK-NEXT:    vldr.16 s6, [r2, #10]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vmul.f16 s6, s8, s6
+; CHECK-NEXT:    vldr.16 s8, [r2, #8]
+; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vmul.f16 s8, s10, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB9_5
+; CHECK-NEXT:  .LBB9_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r5, .LBB9_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r2, #1
+; CHECK-NEXT:    mov lr, r5
+; CHECK-NEXT:    add.w r2, r2, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:  .LBB9_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r1, #2]
+; CHECK-NEXT:    vldr.16 s4, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vmul.f16 s2, s4, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB9_8
+; CHECK-NEXT:  .LBB9_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI9_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.010.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.09.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.010.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.09.epil = phi float [ %add.epil, %for.body.epil ], [ %res.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.010.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds half, half* %b, i32 %i.010.epil
+  %3 = load half, half* %arrayidx1.epil, align 2
+  %mul.epil = fmul half %2, %3
+  %conv.epil = fpext half %mul.epil to float
+  %add.epil = fadd float %res.09.epil, %conv.epil
+  %inc.epil = add nuw i32 %i.010.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.010 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.09 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.010
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.010
+  %5 = load half, half* %arrayidx1, align 2
+  %mul = fmul half %4, %5
+  %conv = fpext half %mul to float
+  %add = fadd float %res.09, %conv
+  %inc = or i32 %i.010, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds half, half* %b, i32 %inc
+  %7 = load half, half* %arrayidx1.1, align 2
+  %mul.1 = fmul half %6, %7
+  %conv.1 = fpext half %mul.1 to float
+  %add.1 = fadd float %add, %conv.1
+  %inc.1 = or i32 %i.010, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds half, half* %b, i32 %inc.1
+  %9 = load half, half* %arrayidx1.2, align 2
+  %mul.2 = fmul half %8, %9
+  %conv.2 = fpext half %mul.2 to float
+  %add.2 = fadd float %add.1, %conv.2
+  %inc.2 = or i32 %i.010, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds half, half* %b, i32 %inc.2
+  %11 = load half, half* %arrayidx1.3, align 2
+  %mul.3 = fmul half %10, %11
+  %conv.3 = fpext half %mul.3 to float
+  %add.3 = fadd float %add.2, %conv.3
+  %inc.3 = add nuw i32 %i.010, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_half_acc(half* nocapture readonly %a, half* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_half_acc:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r7, lr}
+; CHECK-NEXT:    cbz r2, .LBB10_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r4, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB10_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    mov r5, r4
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB10_6
+; CHECK-NEXT:  .LBB10_3:
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:  .LBB10_4: @ %for.body.preheader.new
+; CHECK-NEXT:    subs r2, r2, r4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI10_0
+; CHECK-NEXT:    mov r5, r4
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    sub.w r3, r0, #8
+; CHECK-NEXT:    sub.w r2, r1, #8
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB10_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r2, #14]
+; CHECK-NEXT:    vldr.16 s4, [r3, #14]
+; CHECK-NEXT:    vldr.16 s6, [r3, #12]
+; CHECK-NEXT:    vldr.16 s8, [r3, #10]
+; CHECK-NEXT:    vadd.f16 s2, s4, s2
+; CHECK-NEXT:    vldr.16 s4, [r2, #12]
+; CHECK-NEXT:    vldr.16 s10, [r3, #8]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vadd.f16 s4, s6, s4
+; CHECK-NEXT:    vldr.16 s6, [r2, #10]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    vadd.f16 s6, s8, s6
+; CHECK-NEXT:    vldr.16 s8, [r2, #8]
+; CHECK-NEXT:    adds r2, #8
+; CHECK-NEXT:    vadd.f16 s8, s10, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB10_5
+; CHECK-NEXT:  .LBB10_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r5, .LBB10_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r2, #1
+; CHECK-NEXT:    mov lr, r5
+; CHECK-NEXT:    add.w r2, r2, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:  .LBB10_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vldr.16 s2, [r1, #2]
+; CHECK-NEXT:    vldr.16 s4, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    adds r1, #2
+; CHECK-NEXT:    vadd.f16 s2, s4, s2
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB10_8
+; CHECK-NEXT:  .LBB10_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop {r4, r5, r7, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI10_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp9 = icmp eq i32 %N, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add2.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add2.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.010.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add2.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.010.epil = phi float [ %add2.epil, %for.body.epil ], [ %res.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.011.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds half, half* %b, i32 %i.011.epil
+  %3 = load half, half* %arrayidx1.epil, align 2
+  %add.epil = fadd half %2, %3
+  %conv.epil = fpext half %add.epil to float
+  %add2.epil = fadd float %res.010.epil, %conv.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add2.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add2.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.010 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add2.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.011
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %b, i32 %i.011
+  %5 = load half, half* %arrayidx1, align 2
+  %add = fadd half %4, %5
+  %conv = fpext half %add to float
+  %add2 = fadd float %res.010, %conv
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds half, half* %b, i32 %inc
+  %7 = load half, half* %arrayidx1.1, align 2
+  %add.1 = fadd half %6, %7
+  %conv.1 = fpext half %add.1 to float
+  %add2.1 = fadd float %add2, %conv.1
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds half, half* %b, i32 %inc.1
+  %9 = load half, half* %arrayidx1.2, align 2
+  %add.2 = fadd half %8, %9
+  %conv.2 = fpext half %add.2 to float
+  %add2.2 = fadd float %add2.1, %conv.2
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds half, half* %b, i32 %inc.2
+  %11 = load half, half* %arrayidx1.3, align 2
+  %add.3 = fadd half %10, %11
+  %conv.3 = fpext half %add.3 to float
+  %add2.3 = fadd float %add2.2, %conv.3
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc float @half_short_mac(half* nocapture readonly %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: half_short_mac:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, lr}
+; CHECK-NEXT:    cbz r2, .LBB11_3
+; CHECK-NEXT:  @ %bb.1: @ %for.body.preheader
+; CHECK-NEXT:    subs r3, r2, #1
+; CHECK-NEXT:    and r7, r2, #3
+; CHECK-NEXT:    cmp r3, #3
+; CHECK-NEXT:    bhs .LBB11_4
+; CHECK-NEXT:  @ %bb.2:
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB11_6
+; CHECK-NEXT:  .LBB11_3:
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:  .LBB11_4: @ %for.body.preheader.new
+; CHECK-NEXT:    subs r2, r2, r7
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vldr s0, .LCPI11_0
+; CHECK-NEXT:    mov r8, r7
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, r3, r2, lsr #2
+; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB11_5: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    adds r2, r1, r3
+; CHECK-NEXT:    adds r6, r0, r3
+; CHECK-NEXT:    vldr.16 s2, [r6, #6]
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    ldrsh.w r4, [r2, #2]
+; CHECK-NEXT:    ldrsh.w r5, [r2, #4]
+; CHECK-NEXT:    ldrsh.w r2, [r2, #6]
+; CHECK-NEXT:    vmov s8, r4
+; CHECK-NEXT:    vmov s6, r5
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vcvt.f16.s32 s4, s4
+; CHECK-NEXT:    vmul.f16 s2, s2, s4
+; CHECK-NEXT:    vldr.16 s4, [r6, #4]
+; CHECK-NEXT:    vcvt.f16.s32 s6, s6
+; CHECK-NEXT:    vmul.f16 s4, s4, s6
+; CHECK-NEXT:    vldr.16 s6, [r6, #2]
+; CHECK-NEXT:    vcvt.f16.s32 s8, s8
+; CHECK-NEXT:    ldrsh r2, [r1, r3]
+; CHECK-NEXT:    vmul.f16 s6, s6, s8
+; CHECK-NEXT:    vldr.16 s8, [r6]
+; CHECK-NEXT:    adds r3, #8
+; CHECK-NEXT:    vmov s10, r2
+; CHECK-NEXT:    vcvt.f16.s32 s10, s10
+; CHECK-NEXT:    vmul.f16 s8, s8, s10
+; CHECK-NEXT:    vcvtb.f32.f16 s8, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s6, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s8
+; CHECK-NEXT:    vcvtb.f32.f16 s4, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s6
+; CHECK-NEXT:    vadd.f32 s0, s0, s4
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB11_5
+; CHECK-NEXT:  .LBB11_6: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r8, .LBB11_9
+; CHECK-NEXT:  @ %bb.7: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r3, #1
+; CHECK-NEXT:    add.w r2, r3, r12, lsl #1
+; CHECK-NEXT:    add r0, r2
+; CHECK-NEXT:    add r1, r2
+; CHECK-NEXT:    mov lr, r8
+; CHECK-NEXT:  .LBB11_8: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrsh r2, [r1, #2]!
+; CHECK-NEXT:    vldr.16 s2, [r0, #2]
+; CHECK-NEXT:    adds r0, #2
+; CHECK-NEXT:    vmov s4, r2
+; CHECK-NEXT:    vcvt.f16.s32 s4, s4
+; CHECK-NEXT:    vmul.f16 s2, s2, s4
+; CHECK-NEXT:    vcvtb.f32.f16 s2, s2
+; CHECK-NEXT:    vadd.f32 s0, s0, s2
+; CHECK-NEXT:    le lr, .LBB11_8
+; CHECK-NEXT:  .LBB11_9: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.10:
+; CHECK-NEXT:  .LCPI11_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:                               ; preds = %entry
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %add.lcssa.ph = phi float [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.012.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %res.011.unr = phi float [ 0.000000e+00, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.012.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.012.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %res.011.epil = phi float [ %add.epil, %for.body.epil ], [ %res.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds half, half* %a, i32 %i.012.epil
+  %2 = load half, half* %arrayidx.epil, align 2
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %b, i32 %i.012.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sitofp i16 %3 to half
+  %mul.epil = fmul half %2, %conv2.epil
+  %conv3.epil = fpext half %mul.epil to float
+  %add.epil = fadd float %res.011.epil, %conv3.epil
+  %inc.epil = add nuw i32 %i.012.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  %res.0.lcssa = phi float [ 0.000000e+00, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret float %res.0.lcssa
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.012 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %res.011 = phi float [ 0.000000e+00, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %a, i32 %i.012
+  %4 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %b, i32 %i.012
+  %5 = load i16, i16* %arrayidx1, align 2
+  %conv2 = sitofp i16 %5 to half
+  %mul = fmul half %4, %conv2
+  %conv3 = fpext half %mul to float
+  %add = fadd float %res.011, %conv3
+  %inc = or i32 %i.012, 1
+  %arrayidx.1 = getelementptr inbounds half, half* %a, i32 %inc
+  %6 = load half, half* %arrayidx.1, align 2
+  %arrayidx1.1 = getelementptr inbounds i16, i16* %b, i32 %inc
+  %7 = load i16, i16* %arrayidx1.1, align 2
+  %conv2.1 = sitofp i16 %7 to half
+  %mul.1 = fmul half %6, %conv2.1
+  %conv3.1 = fpext half %mul.1 to float
+  %add.1 = fadd float %add, %conv3.1
+  %inc.1 = or i32 %i.012, 2
+  %arrayidx.2 = getelementptr inbounds half, half* %a, i32 %inc.1
+  %8 = load half, half* %arrayidx.2, align 2
+  %arrayidx1.2 = getelementptr inbounds i16, i16* %b, i32 %inc.1
+  %9 = load i16, i16* %arrayidx1.2, align 2
+  %conv2.2 = sitofp i16 %9 to half
+  %mul.2 = fmul half %8, %conv2.2
+  %conv3.2 = fpext half %mul.2 to float
+  %add.2 = fadd float %add.1, %conv3.2
+  %inc.2 = or i32 %i.012, 3
+  %arrayidx.3 = getelementptr inbounds half, half* %a, i32 %inc.2
+  %10 = load half, half* %arrayidx.3, align 2
+  %arrayidx1.3 = getelementptr inbounds i16, i16* %b, i32 %inc.2
+  %11 = load i16, i16* %arrayidx1.3, align 2
+  %conv2.3 = sitofp i16 %11 to half
+  %mul.3 = fmul half %10, %conv2.3
+  %conv3.3 = fpext half %mul.3 to float
+  %add.3 = fadd float %add.2, %conv3.3
+  %inc.3 = add nuw i32 %i.012, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
new file mode 100644
index 0000000000000..17f9d26ec0335
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
@@ -0,0 +1,1758 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false -enable-arm-maskedldst=true %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_char(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_char:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    sub.w lr, r3, #4
+; CHECK-NEXT:    adr r2, .LCPI0_0
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    add.w lr, r3, lr, lsr #2
+; CHECK-NEXT:    vmov.i32 q4, #0x0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB0_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q3, q4
+; CHECK-NEXT:    vadd.i32 q4, q1, r0
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    @ implicit-def: $q4
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r2, r3, #1
+; CHECK-NEXT:    rsbs r4, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r3, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r3, #8, #1
+; CHECK-NEXT:    ubfx r3, r3, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #3, #1
+; CHECK-NEXT:    lsls r3, r2, #31
+; CHECK-NEXT:    add.w r3, r1, r0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r4, [r3]
+; CHECK-NEXT:    vmovne.32 q4[0], r4
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r4, [r3, #1]
+; CHECK-NEXT:    vmovmi.32 q4[1], r4
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r4, [r3, #2]
+; CHECK-NEXT:    vmovmi.32 q4[2], r4
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r2, [r3, #3]
+; CHECK-NEXT:    vmovmi.32 q4[3], r2
+; CHECK-NEXT:    vand q5, q4, q2
+; CHECK-NEXT:    vmov q4, q3
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vmla.u32 q4, q5, r12
+; CHECK-NEXT:    le lr, .LBB0_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q4, q3
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop.w {r4, lr}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = zext i8 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i8* %0 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %2, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %4 = mul nuw nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nuw nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_short(i16 signext %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_short:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    sub.w lr, r3, #4
+; CHECK-NEXT:    adr r2, .LCPI1_0
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    add.w lr, r3, lr, lsr #2
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB1_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q2, q3
+; CHECK-NEXT:    vadd.i32 q3, q1, r0
+; CHECK-NEXT:    vcmp.u32 cs, q0, q3
+; CHECK-NEXT:    @ implicit-def: $q3
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r2, r3, #1
+; CHECK-NEXT:    rsbs r4, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r3, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r3, #8, #1
+; CHECK-NEXT:    ubfx r3, r3, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #3, #1
+; CHECK-NEXT:    lsls r3, r2, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r3, [r1]
+; CHECK-NEXT:    vmovne.32 q3[0], r3
+; CHECK-NEXT:    lsls r3, r2, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r3, [r1, #2]
+; CHECK-NEXT:    vmovmi.32 q3[1], r3
+; CHECK-NEXT:    lsls r3, r2, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r3, [r1, #4]
+; CHECK-NEXT:    vmovmi.32 q3[2], r3
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r2, [r1, #6]
+; CHECK-NEXT:    vmovmi.32 q3[3], r2
+; CHECK-NEXT:    vmovlb.s16 q4, q3
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vmla.u32 q3, q4, r12
+; CHECK-NEXT:    le lr, .LBB1_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop.w {r4, lr}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI1_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = sext i16 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = mul nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_uchar(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_uchar:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    sub.w lr, r3, #4
+; CHECK-NEXT:    adr r2, .LCPI2_0
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    add.w lr, r3, lr, lsr #2
+; CHECK-NEXT:    vmov.i32 q4, #0x0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB2_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q3, q4
+; CHECK-NEXT:    vadd.i32 q4, q1, r0
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    @ implicit-def: $q4
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r2, r3, #1
+; CHECK-NEXT:    rsbs r4, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r3, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r3, #8, #1
+; CHECK-NEXT:    ubfx r3, r3, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #3, #1
+; CHECK-NEXT:    lsls r3, r2, #31
+; CHECK-NEXT:    add.w r3, r1, r0
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r4, [r3]
+; CHECK-NEXT:    vmovne.32 q4[0], r4
+; CHECK-NEXT:    lsls r4, r2, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r4, [r3, #1]
+; CHECK-NEXT:    vmovmi.32 q4[1], r4
+; CHECK-NEXT:    lsls r4, r2, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r4, [r3, #2]
+; CHECK-NEXT:    vmovmi.32 q4[2], r4
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r2, [r3, #3]
+; CHECK-NEXT:    vmovmi.32 q4[3], r2
+; CHECK-NEXT:    vand q5, q4, q2
+; CHECK-NEXT:    vmov q4, q3
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vmla.u32 q4, q5, r12
+; CHECK-NEXT:    le lr, .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q4, q3
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    pop.w {r4, lr}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI2_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = zext i8 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i8, i8* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i8* %0 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %2, i32 1, <4 x i1> %1, <4 x i8> undef)
+  %3 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %4 = mul nuw nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nuw nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_ushort(i16 signext %a, i16* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_ushort:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    mov r12, r0
+; CHECK-NEXT:    movs r0, #0
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    it eq
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r4, lr}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    subs r2, #1
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    vdup.32 q0, r2
+; CHECK-NEXT:    sub.w lr, r3, #4
+; CHECK-NEXT:    adr r2, .LCPI3_0
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r2]
+; CHECK-NEXT:    add.w lr, r3, lr, lsr #2
+; CHECK-NEXT:    vmov.i32 q3, #0x0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB3_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmov q2, q3
+; CHECK-NEXT:    vadd.i32 q3, q1, r0
+; CHECK-NEXT:    vcmp.u32 cs, q0, q3
+; CHECK-NEXT:    @ implicit-def: $q3
+; CHECK-NEXT:    adds r0, #4
+; CHECK-NEXT:    vmrs r3, p0
+; CHECK-NEXT:    and r2, r3, #1
+; CHECK-NEXT:    rsbs r4, r2, #0
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    bfi r2, r4, #0, #1
+; CHECK-NEXT:    ubfx r4, r3, #4, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #1, #1
+; CHECK-NEXT:    ubfx r4, r3, #8, #1
+; CHECK-NEXT:    ubfx r3, r3, #12, #1
+; CHECK-NEXT:    rsbs r4, r4, #0
+; CHECK-NEXT:    bfi r2, r4, #2, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r2, r3, #3, #1
+; CHECK-NEXT:    lsls r3, r2, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r3, [r1]
+; CHECK-NEXT:    vmovne.32 q3[0], r3
+; CHECK-NEXT:    lsls r3, r2, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r3, [r1, #2]
+; CHECK-NEXT:    vmovmi.32 q3[1], r3
+; CHECK-NEXT:    lsls r3, r2, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r3, [r1, #4]
+; CHECK-NEXT:    vmovmi.32 q3[2], r3
+; CHECK-NEXT:    lsls r2, r2, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r2, [r1, #6]
+; CHECK-NEXT:    vmovmi.32 q3[3], r2
+; CHECK-NEXT:    vmovlb.u16 q4, q3
+; CHECK-NEXT:    vmov q3, q2
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vmla.u32 q3, q4, r12
+; CHECK-NEXT:    le lr, .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vpsel q0, q3, q2
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    pop.w {r4, lr}
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:  .LCPI3_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp7 = icmp eq i32 %N, 0
+  br i1 %cmp7, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv = sext i16 %a to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %conv, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %5, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat11
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = zext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = mul nsw <4 x i32> %broadcast.splat13, %3
+  %5 = add nsw <4 x i32> %4, %vec.phi
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %7 = select <4 x i1> %1, <4 x i32> %5, <4 x i32> %vec.phi
+  %8 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %7)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %8, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc i32 @test_acc_scalar_int(i32 %a, i32* nocapture readonly %b, i32 %N) {
+; CHECK-LABEL: test_acc_scalar_int:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    cmp r2, #0
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    moveq r0, #0
+; CHECK-NEXT:    bxeq lr
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    adds r3, r2, #3
+; CHECK-NEXT:    vmov.i32 q0, #0x0
+; CHECK-NEXT:    bic r3, r3, #3
+; CHECK-NEXT:    sub.w r12, r3, #4
+; CHECK-NEXT:    movs r3, #1
+; CHECK-NEXT:    add.w lr, r3, r12, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB4_1: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r2
+; CHECK-NEXT:    mov r3, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vldrwt.u32 q2, [r1]
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    subs r2, #4
+; CHECK-NEXT:    vmov q1, q0
+; CHECK-NEXT:    vmla.u32 q0, q2, r0
+; CHECK-NEXT:    le lr, .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %middle.block
+; CHECK-NEXT:    vctp.32 r3
+; CHECK-NEXT:    vpsel q0, q0, q1
+; CHECK-NEXT:    vaddv.u32 r0, q0
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %a, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %4, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat12
+  %4 = add nsw <4 x i32> %3, %vec.phi
+  %index.next = add i32 %index, 4
+  %5 = icmp eq i32 %index.next, %n.vec
+  br i1 %5, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %6 = select <4 x i1> %1, <4 x i32> %4, <4 x i32> %vec.phi
+  %7 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %6)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %7, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_char(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_char:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r12, [sp, #72]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB5_12
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r3
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB5_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and lr, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs.w .LBB5_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    b .LBB5_9
+; CHECK-NEXT:  .LBB5_4: @ %vector.ph
+; CHECK-NEXT:    add.w r7, r12, #3
+; CHECK-NEXT:    adr r5, .LCPI5_0
+; CHECK-NEXT:    bic r7, r7, #3
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r5]
+; CHECK-NEXT:    vdup.32 q0, r4
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB5_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q4, q1, r4
+; CHECK-NEXT:    @ implicit-def: $q5
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    @ implicit-def: $q4
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    add.w r6, r0, r4
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r7, [r6]
+; CHECK-NEXT:    vmovne.32 q4[0], r7
+; CHECK-NEXT:    lsls r7, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #1]
+; CHECK-NEXT:    vmovmi.32 q4[1], r7
+; CHECK-NEXT:    lsls r7, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #2]
+; CHECK-NEXT:    vmovmi.32 q4[2], r7
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r5, [r6, #3]
+; CHECK-NEXT:    vmovmi.32 q4[3], r5
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    vand q4, q4, q2
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    add.w r6, r1, r4
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r7, [r6]
+; CHECK-NEXT:    vmovne.32 q5[0], r7
+; CHECK-NEXT:    lsls r7, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #1]
+; CHECK-NEXT:    vmovmi.32 q5[1], r7
+; CHECK-NEXT:    lsls r7, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #2]
+; CHECK-NEXT:    vmovmi.32 q5[2], r7
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r5, [r6, #3]
+; CHECK-NEXT:    vmovmi.32 q5[3], r5
+; CHECK-NEXT:    vand q5, q5, q3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vmul.i32 q4, q5, q4
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vadd.i32 q4, q4, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q4, [r3]
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    le lr, .LBB5_5
+; CHECK-NEXT:    b .LBB5_12
+; CHECK-NEXT:  .LBB5_6: @ %for.body.preheader.new
+; CHECK-NEXT:    sub.w r12, lr, r12
+; CHECK-NEXT:    subs r4, r1, #3
+; CHECK-NEXT:    subs r5, r0, #3
+; CHECK-NEXT:    sub.w r7, r3, #16
+; CHECK-NEXT:    mov.w r9, #0
+; CHECK-NEXT:  .LBB5_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb.w r8, [r5, #3]
+; CHECK-NEXT:    sub.w r9, r9, #4
+; CHECK-NEXT:    ldrb r6, [r4, #3]
+; CHECK-NEXT:    cmp r12, r9
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #16]!
+; CHECK-NEXT:    ldrb r8, [r5, #4]!
+; CHECK-NEXT:    ldrb r6, [r4, #4]!
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #4]
+; CHECK-NEXT:    ldrb.w r8, [r5, #1]
+; CHECK-NEXT:    ldrb r6, [r4, #1]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #8]
+; CHECK-NEXT:    ldrb.w r8, [r5, #2]
+; CHECK-NEXT:    ldrb r6, [r4, #2]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #12]
+; CHECK-NEXT:    bne .LBB5_7
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup.loopexit.unr-lcssa.loopexit
+; CHECK-NEXT:    rsb.w r7, r9, #0
+; CHECK-NEXT:  .LBB5_9: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, lr, .LBB5_12
+; CHECK-NEXT:  @ %bb.10: @ %for.body.epil.preheader
+; CHECK-NEXT:    subs r7, #1
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add.w r3, r3, r7, lsl #2
+; CHECK-NEXT:  .LBB5_11: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb r7, [r0, #1]!
+; CHECK-NEXT:    ldrb r6, [r1, #1]!
+; CHECK-NEXT:    smlabb r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB5_11
+; CHECK-NEXT:  .LBB5_12: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.13:
+; CHECK-NEXT:  .LCPI5_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %res12 = bitcast i32* %res to i8*
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv3 = zext i8 %c to i32
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = bitcast i32* %scevgep to i8*
+  %scevgep14 = getelementptr i8, i8* %a, i32 %N
+  %scevgep15 = getelementptr i8, i8* %b, i32 %N
+  %bound0 = icmp ugt i8* %scevgep14, %res12
+  %bound1 = icmp ugt i8* %scevgep13, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound016 = icmp ugt i8* %scevgep15, %res12
+  %bound117 = icmp ugt i8* %scevgep13, %b
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx = or i1 %found.conflict, %found.conflict18
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %for.body.lr.ph
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert22 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat23 = shufflevector <4 x i32> %broadcast.splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i8, i8* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat20
+  %4 = bitcast i8* %2 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %4, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %5 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %6 = getelementptr inbounds i8, i8* %b, i32 %index
+  %7 = bitcast i8* %6 to <4 x i8>*
+  %wide.masked.load21 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %7, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %8 = zext <4 x i8> %wide.masked.load21 to <4 x i32>
+  %9 = mul nuw nsw <4 x i32> %8, %5
+  %10 = add nuw nsw <4 x i32> %9, %broadcast.splat23
+  %11 = getelementptr inbounds i32, i32* %res, i32 %index
+  %12 = bitcast i32* %11 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %10, <4 x i32>* %12, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i8, i8* %a, i32 %i.011.epil
+  %14 = load i8, i8* %arrayidx.epil, align 1
+  %conv.epil = zext i8 %14 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %b, i32 %i.011.epil
+  %15 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %15 to i32
+  %mul.epil = mul nuw nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nuw nsw i32 %mul.epil, %conv3
+  %arrayidx4.epil = getelementptr inbounds i32, i32* %res, i32 %i.011.epil
+  store i32 %add.epil, i32* %arrayidx4.epil, align 4
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.011
+  %16 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %16 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.011
+  %17 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %17 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add nuw nsw i32 %mul, %conv3
+  %arrayidx4 = getelementptr inbounds i32, i32* %res, i32 %i.011
+  store i32 %add, i32* %arrayidx4, align 4
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc
+  %18 = load i8, i8* %arrayidx.1, align 1
+  %conv.1 = zext i8 %18 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc
+  %19 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %19 to i32
+  %mul.1 = mul nuw nsw i32 %conv2.1, %conv.1
+  %add.1 = add nuw nsw i32 %mul.1, %conv3
+  %arrayidx4.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx4.1, align 4
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.1
+  %20 = load i8, i8* %arrayidx.2, align 1
+  %conv.2 = zext i8 %20 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.1
+  %21 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %21 to i32
+  %mul.2 = mul nuw nsw i32 %conv2.2, %conv.2
+  %add.2 = add nuw nsw i32 %mul.2, %conv3
+  %arrayidx4.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx4.2, align 4
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.2
+  %22 = load i8, i8* %arrayidx.3, align 1
+  %conv.3 = zext i8 %22 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.2
+  %23 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %23 to i32
+  %mul.3 = mul nuw nsw i32 %conv2.3, %conv.3
+  %add.3 = add nuw nsw i32 %mul.3, %conv3
+  %arrayidx4.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx4.3, align 4
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_short(i16* nocapture readonly %a, i16* nocapture readonly %b, i16 signext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_short:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r12, [sp, #28]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB6_3
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    add.w r5, r12, #3
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    bic r5, r5, #3
+; CHECK-NEXT:    subs r5, #4
+; CHECK-NEXT:    add.w lr, r4, r5, lsr #2
+; CHECK-NEXT:    adr r5, .LCPI6_0
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r5]
+; CHECK-NEXT:    vdup.32 q0, r4
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB6_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q2, q1, r4
+; CHECK-NEXT:    @ implicit-def: $q3
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vcmp.u32 cs, q0, q2
+; CHECK-NEXT:    @ implicit-def: $q2
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r6, [r0]
+; CHECK-NEXT:    vmovne.32 q2[0], r6
+; CHECK-NEXT:    lsls r6, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r0, #2]
+; CHECK-NEXT:    vmovmi.32 q2[1], r6
+; CHECK-NEXT:    lsls r6, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r0, #4]
+; CHECK-NEXT:    vmovmi.32 q2[2], r6
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r5, [r0, #6]
+; CHECK-NEXT:    vmovmi.32 q2[3], r5
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    vmovlb.s16 q2, q2
+; CHECK-NEXT:    adds r0, #8
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r6, [r1]
+; CHECK-NEXT:    vmovne.32 q3[0], r6
+; CHECK-NEXT:    lsls r6, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r1, #2]
+; CHECK-NEXT:    vmovmi.32 q3[1], r6
+; CHECK-NEXT:    lsls r6, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r1, #4]
+; CHECK-NEXT:    vmovmi.32 q3[2], r6
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r5, [r1, #6]
+; CHECK-NEXT:    vmovmi.32 q3[3], r5
+; CHECK-NEXT:    vmovlb.s16 q3, q3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vmul.i32 q2, q3, q2
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vadd.i32 q2, q2, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q2, [r3]
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    le lr, .LBB6_2
+; CHECK-NEXT:  .LBB6_3: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.4:
+; CHECK-NEXT:  .LCPI6_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv3 = sext i16 %c to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = sext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = getelementptr inbounds i16, i16* %b, i32 %index
+  %5 = bitcast i16* %4 to <4 x i16>*
+  %wide.masked.load14 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %5, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %6 = sext <4 x i16> %wide.masked.load14 to <4 x i32>
+  %7 = mul nsw <4 x i32> %6, %3
+  %8 = add nsw <4 x i32> %7, %broadcast.splat16
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_uchar(i8* nocapture readonly %a, i8* nocapture readonly %b, i8 zeroext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_uchar:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, lr}
+; CHECK-NEXT:    sub sp, #4
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r12, [sp, #72]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB7_12
+; CHECK-NEXT:  @ %bb.1: @ %for.body.lr.ph
+; CHECK-NEXT:    add.w r5, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r6, r1, r12
+; CHECK-NEXT:    cmp r5, r1
+; CHECK-NEXT:    add.w r4, r0, r12
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    cmp r5, r0
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r3
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    ands r5, r4
+; CHECK-NEXT:    lsls r5, r5, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq r7, r6
+; CHECK-NEXT:    lslseq.w r7, r7, #31
+; CHECK-NEXT:    beq .LBB7_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and lr, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs.w .LBB7_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    movs r7, #0
+; CHECK-NEXT:    b .LBB7_9
+; CHECK-NEXT:  .LBB7_4: @ %vector.ph
+; CHECK-NEXT:    add.w r7, r12, #3
+; CHECK-NEXT:    adr r5, .LCPI7_0
+; CHECK-NEXT:    bic r7, r7, #3
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    movs r6, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r5]
+; CHECK-NEXT:    vdup.32 q0, r4
+; CHECK-NEXT:    add.w lr, r6, r7, lsr #2
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    vmov.i32 q2, #0xff
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB7_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q4, q1, r4
+; CHECK-NEXT:    @ implicit-def: $q5
+; CHECK-NEXT:    vcmp.u32 cs, q0, q4
+; CHECK-NEXT:    @ implicit-def: $q4
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    add.w r6, r0, r4
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r7, [r6]
+; CHECK-NEXT:    vmovne.32 q4[0], r7
+; CHECK-NEXT:    lsls r7, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #1]
+; CHECK-NEXT:    vmovmi.32 q4[1], r7
+; CHECK-NEXT:    lsls r7, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #2]
+; CHECK-NEXT:    vmovmi.32 q4[2], r7
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r5, [r6, #3]
+; CHECK-NEXT:    vmovmi.32 q4[3], r5
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    vand q4, q4, q2
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    add.w r6, r1, r4
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r7, [r6]
+; CHECK-NEXT:    vmovne.32 q5[0], r7
+; CHECK-NEXT:    lsls r7, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #1]
+; CHECK-NEXT:    vmovmi.32 q5[1], r7
+; CHECK-NEXT:    lsls r7, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r7, [r6, #2]
+; CHECK-NEXT:    vmovmi.32 q5[2], r7
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrbmi r5, [r6, #3]
+; CHECK-NEXT:    vmovmi.32 q5[3], r5
+; CHECK-NEXT:    vand q5, q5, q3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vmul.i32 q4, q5, q4
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vadd.i32 q4, q4, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q4, [r3]
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    le lr, .LBB7_5
+; CHECK-NEXT:    b .LBB7_12
+; CHECK-NEXT:  .LBB7_6: @ %for.body.preheader.new
+; CHECK-NEXT:    sub.w r12, lr, r12
+; CHECK-NEXT:    subs r4, r1, #3
+; CHECK-NEXT:    subs r5, r0, #3
+; CHECK-NEXT:    sub.w r7, r3, #16
+; CHECK-NEXT:    mov.w r9, #0
+; CHECK-NEXT:  .LBB7_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb.w r8, [r5, #3]
+; CHECK-NEXT:    sub.w r9, r9, #4
+; CHECK-NEXT:    ldrb r6, [r4, #3]
+; CHECK-NEXT:    cmp r12, r9
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #16]!
+; CHECK-NEXT:    ldrb r8, [r5, #4]!
+; CHECK-NEXT:    ldrb r6, [r4, #4]!
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #4]
+; CHECK-NEXT:    ldrb.w r8, [r5, #1]
+; CHECK-NEXT:    ldrb r6, [r4, #1]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #8]
+; CHECK-NEXT:    ldrb.w r8, [r5, #2]
+; CHECK-NEXT:    ldrb r6, [r4, #2]
+; CHECK-NEXT:    smlabb r6, r6, r8, r2
+; CHECK-NEXT:    str r6, [r7, #12]
+; CHECK-NEXT:    bne .LBB7_7
+; CHECK-NEXT:  @ %bb.8: @ %for.cond.cleanup.loopexit.unr-lcssa.loopexit
+; CHECK-NEXT:    rsb.w r7, r9, #0
+; CHECK-NEXT:  .LBB7_9: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, lr, .LBB7_12
+; CHECK-NEXT:  @ %bb.10: @ %for.body.epil.preheader
+; CHECK-NEXT:    subs r7, #1
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add.w r3, r3, r7, lsl #2
+; CHECK-NEXT:  .LBB7_11: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldrb r7, [r0, #1]!
+; CHECK-NEXT:    ldrb r6, [r1, #1]!
+; CHECK-NEXT:    smlabb r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB7_11
+; CHECK-NEXT:  .LBB7_12: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.13:
+; CHECK-NEXT:  .LCPI7_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %res12 = bitcast i32* %res to i8*
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %for.body.lr.ph
+
+for.body.lr.ph:                                   ; preds = %entry
+  %conv3 = zext i8 %c to i32
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = bitcast i32* %scevgep to i8*
+  %scevgep14 = getelementptr i8, i8* %a, i32 %N
+  %scevgep15 = getelementptr i8, i8* %b, i32 %N
+  %bound0 = icmp ugt i8* %scevgep14, %res12
+  %bound1 = icmp ugt i8* %scevgep13, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound016 = icmp ugt i8* %scevgep15, %res12
+  %bound117 = icmp ugt i8* %scevgep13, %b
+  %found.conflict18 = and i1 %bound016, %bound117
+  %conflict.rdx = or i1 %found.conflict, %found.conflict18
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %for.body.lr.ph
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %for.body.lr.ph
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert19 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat20 = shufflevector <4 x i32> %broadcast.splatinsert19, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert22 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat23 = shufflevector <4 x i32> %broadcast.splatinsert22, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i8, i8* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat20
+  %4 = bitcast i8* %2 to <4 x i8>*
+  %wide.masked.load = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %4, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %5 = zext <4 x i8> %wide.masked.load to <4 x i32>
+  %6 = getelementptr inbounds i8, i8* %b, i32 %index
+  %7 = bitcast i8* %6 to <4 x i8>*
+  %wide.masked.load21 = call <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>* %7, i32 1, <4 x i1> %3, <4 x i8> undef)
+  %8 = zext <4 x i8> %wide.masked.load21 to <4 x i32>
+  %9 = mul nuw nsw <4 x i32> %8, %5
+  %10 = add nuw nsw <4 x i32> %9, %broadcast.splat23
+  %11 = getelementptr inbounds i32, i32* %res, i32 %index
+  %12 = bitcast i32* %11 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %10, <4 x i32>* %12, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %13 = icmp eq i32 %index.next, %n.vec
+  br i1 %13, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i8, i8* %a, i32 %i.011.epil
+  %14 = load i8, i8* %arrayidx.epil, align 1
+  %conv.epil = zext i8 %14 to i32
+  %arrayidx1.epil = getelementptr inbounds i8, i8* %b, i32 %i.011.epil
+  %15 = load i8, i8* %arrayidx1.epil, align 1
+  %conv2.epil = zext i8 %15 to i32
+  %mul.epil = mul nuw nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nuw nsw i32 %mul.epil, %conv3
+  %arrayidx4.epil = getelementptr inbounds i32, i32* %res, i32 %i.011.epil
+  store i32 %add.epil, i32* %arrayidx4.epil, align 4
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.011
+  %16 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %16 to i32
+  %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.011
+  %17 = load i8, i8* %arrayidx1, align 1
+  %conv2 = zext i8 %17 to i32
+  %mul = mul nuw nsw i32 %conv2, %conv
+  %add = add nuw nsw i32 %mul, %conv3
+  %arrayidx4 = getelementptr inbounds i32, i32* %res, i32 %i.011
+  store i32 %add, i32* %arrayidx4, align 4
+  %inc = or i32 %i.011, 1
+  %arrayidx.1 = getelementptr inbounds i8, i8* %a, i32 %inc
+  %18 = load i8, i8* %arrayidx.1, align 1
+  %conv.1 = zext i8 %18 to i32
+  %arrayidx1.1 = getelementptr inbounds i8, i8* %b, i32 %inc
+  %19 = load i8, i8* %arrayidx1.1, align 1
+  %conv2.1 = zext i8 %19 to i32
+  %mul.1 = mul nuw nsw i32 %conv2.1, %conv.1
+  %add.1 = add nuw nsw i32 %mul.1, %conv3
+  %arrayidx4.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx4.1, align 4
+  %inc.1 = or i32 %i.011, 2
+  %arrayidx.2 = getelementptr inbounds i8, i8* %a, i32 %inc.1
+  %20 = load i8, i8* %arrayidx.2, align 1
+  %conv.2 = zext i8 %20 to i32
+  %arrayidx1.2 = getelementptr inbounds i8, i8* %b, i32 %inc.1
+  %21 = load i8, i8* %arrayidx1.2, align 1
+  %conv2.2 = zext i8 %21 to i32
+  %mul.2 = mul nuw nsw i32 %conv2.2, %conv.2
+  %add.2 = add nuw nsw i32 %mul.2, %conv3
+  %arrayidx4.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx4.2, align 4
+  %inc.2 = or i32 %i.011, 3
+  %arrayidx.3 = getelementptr inbounds i8, i8* %a, i32 %inc.2
+  %22 = load i8, i8* %arrayidx.3, align 1
+  %conv.3 = zext i8 %22 to i32
+  %arrayidx1.3 = getelementptr inbounds i8, i8* %b, i32 %inc.2
+  %23 = load i8, i8* %arrayidx1.3, align 1
+  %conv2.3 = zext i8 %23 to i32
+  %mul.3 = mul nuw nsw i32 %conv2.3, %conv.3
+  %add.3 = add nuw nsw i32 %mul.3, %conv3
+  %arrayidx4.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx4.3, align 4
+  %inc.3 = add nuw i32 %i.011, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_ushort(i16* nocapture readonly %a, i16* nocapture readonly %b, i16 signext %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_ushort:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    ldr.w r12, [sp, #28]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB8_3
+; CHECK-NEXT:  @ %bb.1: @ %vector.ph
+; CHECK-NEXT:    add.w r5, r12, #3
+; CHECK-NEXT:    movs r4, #1
+; CHECK-NEXT:    bic r5, r5, #3
+; CHECK-NEXT:    subs r5, #4
+; CHECK-NEXT:    add.w lr, r4, r5, lsr #2
+; CHECK-NEXT:    adr r5, .LCPI8_0
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    vldrw.u32 q1, [r5]
+; CHECK-NEXT:    vdup.32 q0, r4
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB8_2: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vadd.i32 q2, q1, r4
+; CHECK-NEXT:    @ implicit-def: $q3
+; CHECK-NEXT:    adds r4, #4
+; CHECK-NEXT:    vcmp.u32 cs, q0, q2
+; CHECK-NEXT:    @ implicit-def: $q2
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r6, [r0]
+; CHECK-NEXT:    vmovne.32 q2[0], r6
+; CHECK-NEXT:    lsls r6, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r0, #2]
+; CHECK-NEXT:    vmovmi.32 q2[1], r6
+; CHECK-NEXT:    lsls r6, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r0, #4]
+; CHECK-NEXT:    vmovmi.32 q2[2], r6
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r5, [r0, #6]
+; CHECK-NEXT:    vmovmi.32 q2[3], r5
+; CHECK-NEXT:    vmrs r6, p0
+; CHECK-NEXT:    vmovlb.u16 q2, q2
+; CHECK-NEXT:    adds r0, #8
+; CHECK-NEXT:    and r5, r6, #1
+; CHECK-NEXT:    rsbs r7, r5, #0
+; CHECK-NEXT:    movs r5, #0
+; CHECK-NEXT:    bfi r5, r7, #0, #1
+; CHECK-NEXT:    ubfx r7, r6, #4, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #1, #1
+; CHECK-NEXT:    ubfx r7, r6, #8, #1
+; CHECK-NEXT:    ubfx r6, r6, #12, #1
+; CHECK-NEXT:    rsbs r7, r7, #0
+; CHECK-NEXT:    bfi r5, r7, #2, #1
+; CHECK-NEXT:    rsbs r6, r6, #0
+; CHECK-NEXT:    bfi r5, r6, #3, #1
+; CHECK-NEXT:    lsls r6, r5, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrhne r6, [r1]
+; CHECK-NEXT:    vmovne.32 q3[0], r6
+; CHECK-NEXT:    lsls r6, r5, #30
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r1, #2]
+; CHECK-NEXT:    vmovmi.32 q3[1], r6
+; CHECK-NEXT:    lsls r6, r5, #29
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r6, [r1, #4]
+; CHECK-NEXT:    vmovmi.32 q3[2], r6
+; CHECK-NEXT:    lsls r5, r5, #28
+; CHECK-NEXT:    itt mi
+; CHECK-NEXT:    ldrhmi r5, [r1, #6]
+; CHECK-NEXT:    vmovmi.32 q3[3], r5
+; CHECK-NEXT:    vmovlb.u16 q3, q3
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    vmul.i32 q2, q3, q2
+; CHECK-NEXT:    adds r1, #8
+; CHECK-NEXT:    vadd.i32 q2, q2, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q2, [r3]
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    le lr, .LBB8_2
+; CHECK-NEXT:  .LBB8_3: @ %for.cond.cleanup
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r4, r5, r6, r7, pc}
+; CHECK-NEXT:    .p2align 4
+; CHECK-NEXT:  @ %bb.4:
+; CHECK-NEXT:  .LCPI8_0:
+; CHECK-NEXT:    .long 0 @ 0x0
+; CHECK-NEXT:    .long 1 @ 0x1
+; CHECK-NEXT:    .long 2 @ 0x2
+; CHECK-NEXT:    .long 3 @ 0x3
+entry:
+  %cmp10 = icmp eq i32 %N, 0
+  br i1 %cmp10, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %conv3 = sext i16 %c to i32
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert12 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat13 = shufflevector <4 x i32> %broadcast.splatinsert12, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert15 = insertelement <4 x i32> undef, i32 %conv3, i32 0
+  %broadcast.splat16 = shufflevector <4 x i32> %broadcast.splatinsert15, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i16, i16* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat13
+  %2 = bitcast i16* %0 to <4 x i16>*
+  %wide.masked.load = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %2, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %3 = zext <4 x i16> %wide.masked.load to <4 x i32>
+  %4 = getelementptr inbounds i16, i16* %b, i32 %index
+  %5 = bitcast i16* %4 to <4 x i16>*
+  %wide.masked.load14 = call <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>* %5, i32 2, <4 x i1> %1, <4 x i16> undef)
+  %6 = zext <4 x i16> %wide.masked.load14 to <4 x i32>
+  %7 = mul nuw nsw <4 x i32> %6, %3
+  %8 = add nsw <4 x i32> %7, %broadcast.splat16
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+define arm_aapcs_vfpcc void @test_vec_mul_scalar_add_int(i32* nocapture readonly %a, i32* nocapture readonly %b, i32 %c, i32* nocapture %res, i32 %N) {
+; CHECK-LABEL: test_vec_mul_scalar_add_int:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, lr}
+; CHECK-NEXT:    ldr.w r12, [sp, #32]
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB9_11
+; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
+; CHECK-NEXT:    add.w r4, r3, r12, lsl #2
+; CHECK-NEXT:    add.w r5, r1, r12, lsl #2
+; CHECK-NEXT:    cmp r4, r1
+; CHECK-NEXT:    add.w r6, r0, r12, lsl #2
+; CHECK-NEXT:    cset r7, hi
+; CHECK-NEXT:    cmp r5, r3
+; CHECK-NEXT:    cset r5, hi
+; CHECK-NEXT:    cmp r4, r0
+; CHECK-NEXT:    cset r4, hi
+; CHECK-NEXT:    cmp r6, r3
+; CHECK-NEXT:    cset r6, hi
+; CHECK-NEXT:    mov.w lr, #1
+; CHECK-NEXT:    ands r6, r4
+; CHECK-NEXT:    lsls r6, r6, #31
+; CHECK-NEXT:    itt eq
+; CHECK-NEXT:    andeq.w r4, r5, r7
+; CHECK-NEXT:    lslseq.w r4, r4, #31
+; CHECK-NEXT:    beq .LBB9_4
+; CHECK-NEXT:  @ %bb.2: @ %for.body.preheader
+; CHECK-NEXT:    sub.w r4, r12, #1
+; CHECK-NEXT:    and r5, r12, #3
+; CHECK-NEXT:    cmp r4, #3
+; CHECK-NEXT:    bhs .LBB9_6
+; CHECK-NEXT:  @ %bb.3:
+; CHECK-NEXT:    mov r10, r5
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    b .LBB9_8
+; CHECK-NEXT:  .LBB9_4: @ %vector.ph
+; CHECK-NEXT:    add.w r4, r12, #3
+; CHECK-NEXT:    bic r4, r4, #3
+; CHECK-NEXT:    subs r4, #4
+; CHECK-NEXT:    add.w lr, lr, r4, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB9_5: @ %vector.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vctp.32 r12
+; CHECK-NEXT:    sub.w r12, r12, #4
+; CHECK-NEXT:    vpstt
+; CHECK-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-NEXT:    vldrwt.u32 q1, [r1]
+; CHECK-NEXT:    adds r0, #16
+; CHECK-NEXT:    vmul.i32 q0, q1, q0
+; CHECK-NEXT:    adds r1, #16
+; CHECK-NEXT:    vadd.i32 q0, q0, r2
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r3]
+; CHECK-NEXT:    adds r3, #16
+; CHECK-NEXT:    le lr, .LBB9_5
+; CHECK-NEXT:    b .LBB9_11
+; CHECK-NEXT:  .LBB9_6: @ %for.body.preheader.new
+; CHECK-NEXT:    sub.w r7, r12, r5
+; CHECK-NEXT:    mov r10, r5
+; CHECK-NEXT:    subs r7, #4
+; CHECK-NEXT:    movs r4, #0
+; CHECK-NEXT:    mov.w r12, #0
+; CHECK-NEXT:    add.w lr, lr, r7, lsr #2
+; CHECK-NEXT:    dls lr, lr
+; CHECK-NEXT:  .LBB9_7: @ %for.body
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r5, [r0, r4]
+; CHECK-NEXT:    add.w r9, r0, r4
+; CHECK-NEXT:    ldr r6, [r1, r4]
+; CHECK-NEXT:    adds r7, r1, r4
+; CHECK-NEXT:    add.w r12, r12, #4
+; CHECK-NEXT:    mla r5, r6, r5, r2
+; CHECK-NEXT:    str r5, [r3, r4]
+; CHECK-NEXT:    ldr.w r8, [r9, #4]
+; CHECK-NEXT:    ldr r6, [r7, #4]
+; CHECK-NEXT:    mla r8, r6, r8, r2
+; CHECK-NEXT:    adds r6, r3, r4
+; CHECK-NEXT:    adds r4, #16
+; CHECK-NEXT:    str.w r8, [r6, #4]
+; CHECK-NEXT:    ldr.w r8, [r9, #8]
+; CHECK-NEXT:    ldr r5, [r7, #8]
+; CHECK-NEXT:    mla r5, r5, r8, r2
+; CHECK-NEXT:    str r5, [r6, #8]
+; CHECK-NEXT:    ldr.w r5, [r9, #12]
+; CHECK-NEXT:    ldr r7, [r7, #12]
+; CHECK-NEXT:    mla r5, r7, r5, r2
+; CHECK-NEXT:    str r5, [r6, #12]
+; CHECK-NEXT:    le lr, .LBB9_7
+; CHECK-NEXT:  .LBB9_8: @ %for.cond.cleanup.loopexit.unr-lcssa
+; CHECK-NEXT:    wls lr, r10, .LBB9_11
+; CHECK-NEXT:  @ %bb.9: @ %for.body.epil.preheader
+; CHECK-NEXT:    mvn r7, #3
+; CHECK-NEXT:    mov lr, r10
+; CHECK-NEXT:    add.w r7, r7, r12, lsl #2
+; CHECK-NEXT:    add r0, r7
+; CHECK-NEXT:    add r1, r7
+; CHECK-NEXT:    add r3, r7
+; CHECK-NEXT:  .LBB9_10: @ %for.body.epil
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r7, [r0, #4]!
+; CHECK-NEXT:    ldr r6, [r1, #4]!
+; CHECK-NEXT:    mla r7, r6, r7, r2
+; CHECK-NEXT:    str r7, [r3, #4]!
+; CHECK-NEXT:    le lr, .LBB9_10
+; CHECK-NEXT:  .LBB9_11: @ %for.cond.cleanup
+; CHECK-NEXT:    pop.w {r4, r5, r6, r7, r8, r9, r10, pc}
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.memcheck
+
+vector.memcheck:                                  ; preds = %entry
+  %scevgep = getelementptr i32, i32* %res, i32 %N
+  %scevgep13 = getelementptr i32, i32* %a, i32 %N
+  %scevgep16 = getelementptr i32, i32* %b, i32 %N
+  %bound0 = icmp ugt i32* %scevgep13, %res
+  %bound1 = icmp ugt i32* %scevgep, %a
+  %found.conflict = and i1 %bound0, %bound1
+  %bound018 = icmp ugt i32* %scevgep16, %res
+  %bound119 = icmp ugt i32* %scevgep, %b
+  %found.conflict20 = and i1 %bound018, %bound119
+  %conflict.rdx = or i1 %found.conflict, %found.conflict20
+  br i1 %conflict.rdx, label %for.body.preheader, label %vector.ph
+
+for.body.preheader:                               ; preds = %vector.memcheck
+  %0 = add i32 %N, -1
+  %xtraiter = and i32 %N, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:                           ; preds = %for.body.preheader
+  %unroll_iter = sub i32 %N, %xtraiter
+  br label %for.body
+
+vector.ph:                                        ; preds = %vector.memcheck
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert21 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat22 = shufflevector <4 x i32> %broadcast.splatinsert21, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert24 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat25 = shufflevector <4 x i32> %broadcast.splatinsert24, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %2 = getelementptr inbounds i32, i32* %a, i32 %index
+  %3 = icmp ule <4 x i32> %induction, %broadcast.splat22
+  %4 = bitcast i32* %2 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %3, <4 x i32> undef)
+  %5 = getelementptr inbounds i32, i32* %b, i32 %index
+  %6 = bitcast i32* %5 to <4 x i32>*
+  %wide.masked.load23 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %6, i32 4, <4 x i1> %3, <4 x i32> undef)
+  %7 = mul nsw <4 x i32> %wide.masked.load23, %wide.masked.load
+  %8 = add nsw <4 x i32> %7, %broadcast.splat25
+  %9 = getelementptr inbounds i32, i32* %res, i32 %index
+  %10 = bitcast i32* %9 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %8, <4 x i32>* %10, i32 4, <4 x i1> %3)
+  %index.next = add i32 %index, 4
+  %11 = icmp eq i32 %index.next, %n.vec
+  br i1 %11, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup.loopexit.unr-lcssa:              ; preds = %for.body, %for.body.preheader
+  %i.09.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:                                    ; preds = %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil
+  %i.09.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.09.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %arrayidx.epil = getelementptr inbounds i32, i32* %a, i32 %i.09.epil
+  %12 = load i32, i32* %arrayidx.epil, align 4
+  %arrayidx1.epil = getelementptr inbounds i32, i32* %b, i32 %i.09.epil
+  %13 = load i32, i32* %arrayidx1.epil, align 4
+  %mul.epil = mul nsw i32 %13, %12
+  %add.epil = add nsw i32 %mul.epil, %c
+  %arrayidx2.epil = getelementptr inbounds i32, i32* %res, i32 %i.09.epil
+  store i32 %add.epil, i32* %arrayidx2.epil, align 4
+  %inc.epil = add nuw i32 %i.09.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:                                 ; preds = %vector.body, %for.cond.cleanup.loopexit.unr-lcssa, %for.body.epil, %entry
+  ret void
+
+for.body:                                         ; preds = %for.body, %for.body.preheader.new
+  %i.09 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %a, i32 %i.09
+  %14 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %b, i32 %i.09
+  %15 = load i32, i32* %arrayidx1, align 4
+  %mul = mul nsw i32 %15, %14
+  %add = add nsw i32 %mul, %c
+  %arrayidx2 = getelementptr inbounds i32, i32* %res, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %inc = or i32 %i.09, 1
+  %arrayidx.1 = getelementptr inbounds i32, i32* %a, i32 %inc
+  %16 = load i32, i32* %arrayidx.1, align 4
+  %arrayidx1.1 = getelementptr inbounds i32, i32* %b, i32 %inc
+  %17 = load i32, i32* %arrayidx1.1, align 4
+  %mul.1 = mul nsw i32 %17, %16
+  %add.1 = add nsw i32 %mul.1, %c
+  %arrayidx2.1 = getelementptr inbounds i32, i32* %res, i32 %inc
+  store i32 %add.1, i32* %arrayidx2.1, align 4
+  %inc.1 = or i32 %i.09, 2
+  %arrayidx.2 = getelementptr inbounds i32, i32* %a, i32 %inc.1
+  %18 = load i32, i32* %arrayidx.2, align 4
+  %arrayidx1.2 = getelementptr inbounds i32, i32* %b, i32 %inc.1
+  %19 = load i32, i32* %arrayidx1.2, align 4
+  %mul.2 = mul nsw i32 %19, %18
+  %add.2 = add nsw i32 %mul.2, %c
+  %arrayidx2.2 = getelementptr inbounds i32, i32* %res, i32 %inc.1
+  store i32 %add.2, i32* %arrayidx2.2, align 4
+  %inc.2 = or i32 %i.09, 3
+  %arrayidx.3 = getelementptr inbounds i32, i32* %a, i32 %inc.2
+  %20 = load i32, i32* %arrayidx.3, align 4
+  %arrayidx1.3 = getelementptr inbounds i32, i32* %b, i32 %inc.2
+  %21 = load i32, i32* %arrayidx1.3, align 4
+  %mul.3 = mul nsw i32 %21, %20
+  %add.3 = add nsw i32 %mul.3, %c
+  %arrayidx2.3 = getelementptr inbounds i32, i32* %res, i32 %inc.2
+  store i32 %add.3, i32* %arrayidx2.3, align 4
+  %inc.3 = add nuw i32 %i.09, 4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i8> @llvm.masked.load.v4i8.p0v4i8(<4 x i8>*, i32 immarg, <4 x i1>, <4 x i8>) #2
+
+; Function Attrs: nounwind readnone willreturn
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>) #3
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i16> @llvm.masked.load.v4i16.p0v4i16(<4 x i16>*, i32 immarg, <4 x i1>, <4 x i16>) #2
+
+; Function Attrs: argmemonly nounwind readonly willreturn
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>) #2
+
+; Function Attrs: argmemonly nounwind willreturn
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) #4
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
index a5e922858f2d0..6d5516249e239 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
@@ -69,8 +69,8 @@ for.cond.cleanup:                                 ; preds = %middle.block, %entr
 ; CHECK: phi
 ; CHECK: phi
 ; CHECK: [[IV:%[^ ]+]] = phi i32 [ %N, %for.cond1.preheader.us ], [ [[REM:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[IV]])
 ; CHECK: [[REM]] = sub i32 [[IV]], 4
-; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[REM]])
 ; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 ; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]], <4 x i32> undef)
 define void @mat_vec_i32(i32** nocapture readonly %A, i32* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir
new file mode 100644
index 0000000000000..5e327b0e08cc5
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-cbnz.mir
@@ -0,0 +1,201 @@
+# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-cp-islands --verify-machineinstrs -o - | FileCheck %s --check-prefix=CHECK-LOB
+# RUN: llc -mtriple=thumbv8.1m.main -mattr=-lob %s -run-pass=arm-cp-islands --verify-machineinstrs -o - | FileCheck %s --check-prefix=CHECK-NOLOB
+
+# CHECK-NOLOB-NOT: t2LE
+
+# CHECK-LOB: bb.3.land.rhs:
+# CHECK-LOB:   renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg
+# CHECK-LOB:   tCBNZ $r0, %bb.8
+# CHECK-LOB:   t2LE %bb.3
+# CHECK-LOB: bb.7.while.body19:
+# CHECK-LOB:   tCBZ $r0, %bb.8
+# CHECK-LOB:   t2LE %bb.6
+# CHECK-LOB: bb.8:
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  %struct.head_s = type { %struct.head_s*, %struct.data_s* }
+  %struct.data_s = type { i16, i16 }
+  
+  ; Function Attrs: norecurse nounwind readonly
+  define dso_local arm_aapcscc %struct.head_s* @search(%struct.head_s* readonly %list, %struct.data_s* nocapture readonly %info) local_unnamed_addr #0 {
+  entry:
+    %idx = getelementptr inbounds %struct.data_s, %struct.data_s* %info, i32 0, i32 1
+    %0 = load i16, i16* %idx, align 2
+    %cmp = icmp sgt i16 %0, -1
+    br i1 %cmp, label %while.cond.preheader, label %while.cond9.preheader
+  
+  while.cond9.preheader:                            ; preds = %entry
+    %1 = icmp eq %struct.head_s* %list, null
+    br i1 %1, label %return, label %land.rhs11.lr.ph
+  
+  land.rhs11.lr.ph:                                 ; preds = %while.cond9.preheader
+    %data16143 = bitcast %struct.data_s* %info to i16*
+    %2 = load i16, i16* %data16143, align 2
+    %conv15 = sext i16 %2 to i32
+    br label %land.rhs11
+  
+  while.cond.preheader:                             ; preds = %entry
+    %3 = icmp eq %struct.head_s* %list, null
+    br i1 %3, label %return, label %land.rhs.preheader
+  
+  land.rhs.preheader:                               ; preds = %while.cond.preheader
+    br label %land.rhs
+  
+  land.rhs:                                         ; preds = %land.rhs.preheader, %while.body
+    %list.addr.033 = phi %struct.head_s* [ %6, %while.body ], [ %list, %land.rhs.preheader ]
+    %info2 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.033, i32 0, i32 1
+    %4 = load %struct.data_s*, %struct.data_s** %info2, align 4
+    %idx3 = getelementptr inbounds %struct.data_s, %struct.data_s* %4, i32 0, i32 1
+    %5 = load i16, i16* %idx3, align 2
+    %cmp7 = icmp eq i16 %5, %0
+    br i1 %cmp7, label %return, label %while.body
+  
+  while.body:                                       ; preds = %land.rhs
+    %next4 = bitcast %struct.head_s* %list.addr.033 to %struct.head_s**
+    %6 = load %struct.head_s*, %struct.head_s** %next4, align 4
+    %tobool = icmp ne %struct.head_s* %6, null
+    br i1 %tobool, label %return, label %land.rhs
+  
+  land.rhs11:                                       ; preds = %while.body19, %land.rhs11.lr.ph
+    %list.addr.136 = phi %struct.head_s* [ %list, %land.rhs11.lr.ph ], [ %10, %while.body19 ]
+    %info12 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.136, i32 0, i32 1
+    %7 = load %struct.data_s*, %struct.data_s** %info12, align 4
+    %data165 = bitcast %struct.data_s* %7 to i16*
+    %8 = load i16, i16* %data165, align 2
+    %9 = and i16 %8, 255
+    %and = zext i16 %9 to i32
+    %cmp16 = icmp eq i32 %and, %conv15
+    br i1 %cmp16, label %return, label %while.body19
+  
+  while.body19:                                     ; preds = %land.rhs11
+    %next206 = bitcast %struct.head_s* %list.addr.136 to %struct.head_s**
+    %10 = load %struct.head_s*, %struct.head_s** %next206, align 4
+    %tobool10 = icmp eq %struct.head_s* %10, null
+    br i1 %tobool10, label %return, label %land.rhs11
+  
+  return:                                           ; preds = %while.body19, %land.rhs11, %while.body, %land.rhs, %while.cond.preheader, %while.cond9.preheader
+    %retval.0 = phi %struct.head_s* [ null, %while.cond.preheader ], [ null, %while.cond9.preheader ], [ %list.addr.033, %land.rhs ], [ null, %while.body ], [ %list.addr.136, %land.rhs11 ], [ null, %while.body19 ]
+    ret %struct.head_s* %retval.0
+  }
+  
+  attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv8.1-m.main,+hwdiv,+lob,+ras,+soft-float,+strict-align,+thumb-mode,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8d16,-fp-armv8d16sp,-fp-armv8sp,-fp16,-fp16fml,-fp64,-fpregs,-fullfp16,-neon,-vfp2,-vfp2d16,-vfp2d16sp,-vfp2sp,-vfp3,-vfp3d16,-vfp3d16sp,-vfp3sp,-vfp4,-vfp4d16,-vfp4d16sp,-vfp4sp" "unsafe-fp-math"="false" "use-soft-float"="true" }
+  
+...
+---
+name:            search
+alignment:       1
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.5(0x50000000), %bb.1(0x30000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = t2LDRSHi12 renamable $r1, 2, 14, $noreg :: (load 2 from %ir.idx)
+    t2CMPri renamable $r2, -1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.1, 13, killed $cpsr
+  
+  bb.5.while.cond.preheader:
+    successors: %bb.8(0x30000000), %bb.6(0x50000000)
+    liveins: $r0, $r2
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.8, 0, killed $cpsr
+  
+  bb.6.land.rhs.preheader:
+    successors: %bb.7(0x80000000)
+    liveins: $r0, $r2
+  
+    renamable $r1 = tUXTH killed renamable $r2, 14, $noreg
+  
+  bb.7.land.rhs:
+    successors: %bb.8(0x04000000), %bb.7(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info2)
+    renamable $r2 = tLDRHi killed renamable $r2, 1, 14, $noreg :: (load 2 from %ir.idx3)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 8, implicit-def $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next4)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.7, 0, killed $cpsr
+    t2B %bb.8, 14, $noreg
+  
+  bb.1.while.cond9.preheader:
+    successors: %bb.8(0x30000000), %bb.2(0x50000000)
+    liveins: $r0, $r1
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.8, 0, killed $cpsr
+  
+  bb.2.land.rhs11.lr.ph:
+    successors: %bb.3(0x80000000)
+    liveins: $r0, $r1
+  
+    renamable $r1 = t2LDRSHi12 killed renamable $r1, 0, 14, $noreg :: (load 2 from %ir.data16143)
+  
+  bb.3.land.rhs11:
+    successors: %bb.9(0x04000000), %bb.4(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info12)
+    renamable $r2 = tLDRBi killed renamable $r2, 0, 14, $noreg :: (load 1 from %ir.data165, align 2)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.9, 0, killed $cpsr
+  
+  bb.4.while.body19:
+    successors: %bb.8(0x04000000), %bb.3(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next206)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.3, 1, killed $cpsr
+  
+  bb.8:
+    successors: %bb.9(0x80000000)
+  
+    renamable $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+  
+  bb.9.return:
+    liveins: $r0
+  
+    tBX_RET 14, $noreg, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll
new file mode 100644
index 0000000000000..78ddb35e756e3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-le-simple.ll
@@ -0,0 +1,158 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main %s -o - | FileCheck %s
+
+define void @cbz_exit(i32* %in, i32* %res) {
+; CHECK-LABEL: cbz_exit:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB0_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r2, #4]!
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cbz r3, .LBB0_2
+; CHECK-NEXT:    le .LBB0_1
+; CHECK-NEXT:  .LBB0_2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit(i32* %in, i32* %res) {
+; CHECK-LABEL: cbnz_exit:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB1_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r3, [r2, #4]!
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cbnz r3, .LBB1_2
+; CHECK-NEXT:    le .LBB1_1
+; CHECK-NEXT:  .LBB1_2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit_too_large(i32* %in, i32* %res) {
+; CHECK-LABEL: cbnz_exit_too_large:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r2, r0, #4
+; CHECK-NEXT:    mov.w r0, #-1
+; CHECK-NEXT:  .LBB2_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr r12, [r2, #4]!
+; CHECK-NEXT:    .zero 4090
+; CHECK-NEXT:    adds r0, #1
+; CHECK-NEXT:    cmp.w r12, #0
+; CHECK-NEXT:    beq.w .LBB2_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  %size = call i32 @llvm.arm.space(i32 4090, i32 undef)
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbz_exit_minsize(i32* %in, i32* %res) #0 {
+; CHECK-LABEL: cbz_exit_minsize:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:  .LBB3_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr.w r3, [r0, r2, lsl #2]
+; CHECK-NEXT:    adds r2, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    bne .LBB3_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    subs r0, r2, #1
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp eq i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+define void @cbnz_exit_minsize(i32* %in, i32* %res) #0 {
+; CHECK-LABEL: cbnz_exit_minsize:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:  .LBB4_1: @ %loop
+; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    ldr.w r3, [r0, r2, lsl #2]
+; CHECK-NEXT:    adds r2, #1
+; CHECK-NEXT:    cmp r3, #0
+; CHECK-NEXT:    beq .LBB4_1
+; CHECK-NEXT:  @ %bb.2: @ %exit
+; CHECK-NEXT:    subs r0, r2, #1
+; CHECK-NEXT:    str r0, [r1]
+; CHECK-NEXT:    bx lr
+entry:
+  br label %loop
+
+loop:
+  %offset = phi i32 [ 0, %entry ], [ %next, %loop ]
+  %ptr = getelementptr i32, i32* %in, i32 %offset
+  %val = load i32, i32* %ptr
+  %next = add i32 %offset, 1
+  %cmp = icmp ne i32 %val, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store i32 %offset, i32* %res
+  ret void
+}
+
+attributes #0 = { minsize optsize }
+
+declare i32 @llvm.arm.space(i32 immarg, i32);
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir
new file mode 100644
index 0000000000000..ffabd94c93447
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec-reorder.mir
@@ -0,0 +1,184 @@
+# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-cp-islands --verify-machineinstrs -o - | FileCheck %s
+# CHECK-NOT: t2LE
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-unknown-unknown"
+  
+  %struct.head_s = type { %struct.head_s*, %struct.data_s* }
+  %struct.data_s = type { i16, i16 }
+  
+  ; Function Attrs: norecurse nounwind readonly
+  define dso_local arm_aapcscc %struct.head_s* @search(%struct.head_s* readonly %list, %struct.data_s* nocapture readonly %info) local_unnamed_addr #0 {
+  entry:
+    %idx = getelementptr inbounds %struct.data_s, %struct.data_s* %info, i32 0, i32 1
+    %tmp = load i16, i16* %idx, align 2
+    %cmp = icmp sgt i16 %tmp, -1
+    br i1 %cmp, label %while.cond.preheader, label %while.cond9.preheader
+  
+  while.cond9.preheader:                            ; preds = %entry
+    %0 = icmp eq %struct.head_s* %list, null
+    br i1 %0, label %return, label %land.rhs11.lr.ph
+  
+  land.rhs11.lr.ph:                                 ; preds = %while.cond9.preheader
+    %data16143 = bitcast %struct.data_s* %info to i16*
+    %tmp1 = load i16, i16* %data16143, align 2
+    %conv15 = sext i16 %tmp1 to i32
+    br label %land.rhs11
+  
+  while.cond.preheader:                             ; preds = %entry
+    %1 = icmp eq %struct.head_s* %list, null
+    br i1 %1, label %return, label %land.rhs.preheader
+  
+  land.rhs.preheader:                               ; preds = %while.cond.preheader
+    br label %land.rhs
+  
+  while.body:                                       ; preds = %land.rhs
+    %next4 = bitcast %struct.head_s* %list.addr.033 to %struct.head_s**
+    %tmp4 = load %struct.head_s*, %struct.head_s** %next4, align 4
+    %tobool = icmp eq %struct.head_s* %tmp4, null
+    br i1 %tobool, label %return, label %land.rhs
+  
+  land.rhs:                                         ; preds = %land.rhs.preheader, %while.body
+    %list.addr.033 = phi %struct.head_s* [ %tmp4, %while.body ], [ %list, %land.rhs.preheader ]
+    %info2 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.033, i32 0, i32 1
+    %tmp2 = load %struct.data_s*, %struct.data_s** %info2, align 4
+    %idx3 = getelementptr inbounds %struct.data_s, %struct.data_s* %tmp2, i32 0, i32 1
+    %tmp3 = load i16, i16* %idx3, align 2
+    %cmp7 = icmp eq i16 %tmp3, %tmp
+    br i1 %cmp7, label %return, label %while.body
+  
+  while.body19:                                     ; preds = %land.rhs11
+    %next205 = bitcast %struct.head_s* %list.addr.136 to %struct.head_s**
+    %tmp8 = load %struct.head_s*, %struct.head_s** %next205, align 4
+    %tobool10 = icmp eq %struct.head_s* %tmp8, null
+    br i1 %tobool10, label %return, label %land.rhs11
+  
+  land.rhs11:                                       ; preds = %while.body19, %land.rhs11.lr.ph
+    %list.addr.136 = phi %struct.head_s* [ %list, %land.rhs11.lr.ph ], [ %tmp8, %while.body19 ]
+    %info12 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.136, i32 0, i32 1
+    %tmp5 = load %struct.data_s*, %struct.data_s** %info12, align 4
+    %data166 = bitcast %struct.data_s* %tmp5 to i16*
+    %tmp6 = load i16, i16* %data166, align 2
+    %2 = and i16 %tmp6, 255
+    %and = zext i16 %2 to i32
+    %cmp16 = icmp eq i32 %and, %conv15
+    br i1 %cmp16, label %return, label %while.body19
+  
+  return:                                           ; preds = %land.rhs11, %while.body19, %land.rhs, %while.body, %while.cond.preheader, %while.cond9.preheader
+    %retval.0 = phi %struct.head_s* [ null, %while.cond.preheader ], [ null, %while.cond9.preheader ], [ %list.addr.033, %land.rhs ], [ null, %while.body ], [ %list.addr.136, %land.rhs11 ], [ null, %while.body19 ]
+    ret %struct.head_s* %retval.0
+  }
+  
+  attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv8.1-m.main,+hwdiv,+lob,+ras,+soft-float,+strict-align,+thumb-mode,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8d16,-fp-armv8d16sp,-fp-armv8sp,-fp16,-fp16fml,-fp64,-fpregs,-fullfp16,-neon,-vfp2,-vfp2d16,-vfp2d16sp,-vfp2sp,-vfp3,-vfp3d16,-vfp3d16sp,-vfp3sp,-vfp4,-vfp4d16,-vfp4d16sp,-vfp4sp" "unsafe-fp-math"="false" "use-soft-float"="true" }
+  
+...
+---
+name:            search
+alignment:       1
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.3(0x50000000), %bb.1(0x30000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = t2LDRSHi12 renamable $r1, 2, 14, $noreg :: (load 2 from %ir.idx)
+    t2CMPri renamable $r2, -1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.1, 13, killed $cpsr
+  
+  bb.3.while.cond.preheader:
+    successors: %bb.4(0x80000000)
+    liveins: $r0, $r2
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 4, implicit-def $itstate
+    renamable $r0 = tMOVi8 $noreg, 0, 0, $cpsr, implicit killed $r0, implicit $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    renamable $r1 = tUXTH killed renamable $r2, 14, $noreg
+  
+  bb.4.land.rhs:
+    successors: %bb.6(0x04000000), %bb.5(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info2)
+    renamable $r2 = tLDRHi killed renamable $r2, 1, 14, $noreg :: (load 2 from %ir.idx3)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.6, 0, killed $cpsr
+  
+  bb.5.while.body:
+    successors: %bb.4(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next4)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 4, implicit-def $itstate
+    renamable $r0 = tMOVi8 $noreg, 0, 0, $cpsr, implicit killed $r0, implicit $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    t2B %bb.4, 14, $noreg
+  
+  bb.6.return:
+    liveins: $r0
+  
+    tBX_RET 14, $noreg, implicit $r0
+  
+  bb.1.while.cond9.preheader:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $r1
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 4, implicit-def $itstate
+    renamable $r0 = tMOVi8 $noreg, 0, 0, $cpsr, implicit killed $r0, implicit $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    renamable $r1 = t2LDRSHi12 killed renamable $r1, 0, 14, $noreg :: (load 2 from %ir.data16143)
+  
+  bb.2.land.rhs11:
+    successors: %bb.2(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info12)
+    renamable $r2 = tLDRBi killed renamable $r2, 0, 14, $noreg :: (load 1 from %ir.data166, align 2)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 8, implicit-def $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next205)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 4, implicit-def $itstate
+    renamable $r0 = tMOVi8 $noreg, 0, 0, $cpsr, implicit killed $r0, implicit $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    t2B %bb.2, 14, $noreg
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir
new file mode 100644
index 0000000000000..6d27494341368
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/no-dec.mir
@@ -0,0 +1,201 @@
+# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-cp-islands --verify-machineinstrs -o - | FileCheck %s --check-prefix=CHECK-LOB
+# RUN: llc -mtriple=thumbv8.1m.main -mattr=-lob %s -run-pass=arm-cp-islands --verify-machineinstrs -o - | FileCheck %s --check-prefix=CHECK-NOLOB
+
+# CHECK-NOLOB-NOT: t2LE
+
+# CHECK-LOB: bb.3.land.rhs:
+# CHECK-LOB:   tCBZ $r0, %bb.8
+# CHECK-LOB:   t2LE %bb.3
+# CHECK-LOB: bb.6.land.rhs11:
+# CHECK-LOB: bb.7.while.body19:
+# CHECK-LOB:   tCBZ $r0, %bb.8
+# CHECK-LOB:   t2LE %bb.6
+# CHECK-LOB: bb.8:
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-unknown-unknown"
+  
+  %struct.head_s = type { %struct.head_s*, %struct.data_s* }
+  %struct.data_s = type { i16, i16 }
+  
+  ; Function Attrs: norecurse nounwind readonly
+  define dso_local arm_aapcscc %struct.head_s* @search(%struct.head_s* readonly %list, %struct.data_s* nocapture readonly %info) local_unnamed_addr #0 {
+  entry:
+    %idx = getelementptr inbounds %struct.data_s, %struct.data_s* %info, i32 0, i32 1
+    %0 = load i16, i16* %idx, align 2
+    %cmp = icmp sgt i16 %0, -1
+    br i1 %cmp, label %while.cond.preheader, label %while.cond9.preheader
+  
+  while.cond9.preheader:                            ; preds = %entry
+    %1 = icmp eq %struct.head_s* %list, null
+    br i1 %1, label %return, label %land.rhs11.lr.ph
+  
+  land.rhs11.lr.ph:                                 ; preds = %while.cond9.preheader
+    %data16143 = bitcast %struct.data_s* %info to i16*
+    %2 = load i16, i16* %data16143, align 2
+    %conv15 = sext i16 %2 to i32
+    br label %land.rhs11
+  
+  while.cond.preheader:                             ; preds = %entry
+    %3 = icmp eq %struct.head_s* %list, null
+    br i1 %3, label %return, label %land.rhs.preheader
+  
+  land.rhs.preheader:                               ; preds = %while.cond.preheader
+    br label %land.rhs
+  
+  land.rhs:                                         ; preds = %land.rhs.preheader, %while.body
+    %list.addr.033 = phi %struct.head_s* [ %6, %while.body ], [ %list, %land.rhs.preheader ]
+    %info2 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.033, i32 0, i32 1
+    %4 = load %struct.data_s*, %struct.data_s** %info2, align 4
+    %idx3 = getelementptr inbounds %struct.data_s, %struct.data_s* %4, i32 0, i32 1
+    %5 = load i16, i16* %idx3, align 2
+    %cmp7 = icmp eq i16 %5, %0
+    br i1 %cmp7, label %return, label %while.body
+  
+  while.body:                                       ; preds = %land.rhs
+    %next4 = bitcast %struct.head_s* %list.addr.033 to %struct.head_s**
+    %6 = load %struct.head_s*, %struct.head_s** %next4, align 4
+    %tobool = icmp eq %struct.head_s* %6, null
+    br i1 %tobool, label %return, label %land.rhs
+  
+  land.rhs11:                                       ; preds = %while.body19, %land.rhs11.lr.ph
+    %list.addr.136 = phi %struct.head_s* [ %list, %land.rhs11.lr.ph ], [ %10, %while.body19 ]
+    %info12 = getelementptr inbounds %struct.head_s, %struct.head_s* %list.addr.136, i32 0, i32 1
+    %7 = load %struct.data_s*, %struct.data_s** %info12, align 4
+    %data165 = bitcast %struct.data_s* %7 to i16*
+    %8 = load i16, i16* %data165, align 2
+    %9 = and i16 %8, 255
+    %and = zext i16 %9 to i32
+    %cmp16 = icmp eq i32 %and, %conv15
+    br i1 %cmp16, label %return, label %while.body19
+  
+  while.body19:                                     ; preds = %land.rhs11
+    %next206 = bitcast %struct.head_s* %list.addr.136 to %struct.head_s**
+    %10 = load %struct.head_s*, %struct.head_s** %next206, align 4
+    %tobool10 = icmp eq %struct.head_s* %10, null
+    br i1 %tobool10, label %return, label %land.rhs11
+  
+  return:                                           ; preds = %while.body19, %land.rhs11, %while.body, %land.rhs, %while.cond.preheader, %while.cond9.preheader
+    %retval.0 = phi %struct.head_s* [ null, %while.cond.preheader ], [ null, %while.cond9.preheader ], [ %list.addr.033, %land.rhs ], [ null, %while.body ], [ %list.addr.136, %land.rhs11 ], [ null, %while.body19 ]
+    ret %struct.head_s* %retval.0
+  }
+  
+  attributes #0 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+armv8.1-m.main,+hwdiv,+lob,+ras,+soft-float,+strict-align,+thumb-mode,-crypto,-d32,-dotprod,-fp-armv8,-fp-armv8d16,-fp-armv8d16sp,-fp-armv8sp,-fp16,-fp16fml,-fp64,-fpregs,-fullfp16,-neon,-vfp2,-vfp2d16,-vfp2d16sp,-vfp2sp,-vfp3,-vfp3d16,-vfp3d16sp,-vfp3sp,-vfp4,-vfp4d16,-vfp4d16sp,-vfp4sp" "unsafe-fp-math"="false" "use-soft-float"="true" }
+  
+...
+---
+name:            search
+alignment:       1
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:           []
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.5(0x50000000), %bb.1(0x30000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = t2LDRSHi12 renamable $r1, 2, 14, $noreg :: (load 2 from %ir.idx)
+    t2CMPri renamable $r2, -1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.1, 13, killed $cpsr
+  
+  bb.5.while.cond.preheader:
+    successors: %bb.8(0x30000000), %bb.6(0x50000000)
+    liveins: $r0, $r2
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.8, 0, killed $cpsr
+  
+  bb.6.land.rhs.preheader:
+    successors: %bb.7(0x80000000)
+    liveins: $r0, $r2
+  
+    renamable $r1 = tUXTH killed renamable $r2, 14, $noreg
+  
+  bb.7.land.rhs:
+    successors: %bb.8(0x04000000), %bb.7(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info2)
+    renamable $r2 = tLDRHi killed renamable $r2, 1, 14, $noreg :: (load 2 from %ir.idx3)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2IT 0, 8, implicit-def $itstate
+    tBX_RET 0, killed $cpsr, implicit $r0, implicit killed $itstate
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next4)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.7, 1, killed $cpsr
+    t2B %bb.8, 14, $noreg
+  
+  bb.1.while.cond9.preheader:
+    successors: %bb.8(0x30000000), %bb.2(0x50000000)
+    liveins: $r0, $r1
+  
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.8, 0, killed $cpsr
+  
+  bb.2.land.rhs11.lr.ph:
+    successors: %bb.3(0x80000000)
+    liveins: $r0, $r1
+  
+    renamable $r1 = t2LDRSHi12 killed renamable $r1, 0, 14, $noreg :: (load 2 from %ir.data16143)
+  
+  bb.3.land.rhs11:
+    successors: %bb.9(0x04000000), %bb.4(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r2 = tLDRi renamable $r0, 1, 14, $noreg :: (load 4 from %ir.info12)
+    renamable $r2 = tLDRBi killed renamable $r2, 0, 14, $noreg :: (load 1 from %ir.data165, align 2)
+    tCMPr killed renamable $r2, renamable $r1, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.9, 0, killed $cpsr
+  
+  bb.4.while.body19:
+    successors: %bb.8(0x04000000), %bb.3(0x7c000000)
+    liveins: $r0, $r1
+  
+    renamable $r0 = tLDRi killed renamable $r0, 0, 14, $noreg :: (load 4 from %ir.next206)
+    tCMPi8 renamable $r0, 0, 14, $noreg, implicit-def $cpsr
+    t2Bcc %bb.3, 1, killed $cpsr
+  
+  bb.8:
+    successors: %bb.9(0x80000000)
+  
+    renamable $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+  
+  bb.9.return:
+    liveins: $r0
+  
+    tBX_RET 14, $noreg, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
index 3ea75a295f63f..e0434c4e05039 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.mir
@@ -5,8 +5,6 @@
 # CHECK-NOT: t2LEUpdate
 
 --- |
-  ; ModuleID = '/home/sampar01/src/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.ll'
-  source_filename = "/home/sampar01/src/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-call.ll"
   target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
   target triple = "thumbv8.1m.main"
   
@@ -35,15 +33,9 @@
   
   declare i32 @bar(...) local_unnamed_addr #0
   
-  ; Function Attrs: noduplicate nounwind
   declare void @llvm.set.loop.iterations.i32(i32) #1
-  
-  ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
-  ; Function Attrs: nounwind
-  declare void @llvm.stackprotector(i8*, i8**) #2
-  
   attributes #0 = { "target-features"="+mve.fp" }
   attributes #1 = { noduplicate nounwind }
   attributes #2 = { nounwind }
@@ -57,7 +49,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -101,6 +93,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.4(0x30000000), %bb.1(0x50000000)
+    liveins: $r0, $r4, $r5, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r4, killed $r5, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 16
@@ -112,6 +105,7 @@ body:             |
   
   bb.1.while.body.preheader:
     successors: %bb.2(0x80000000)
+    liveins: $r0
   
     $lr = tMOVr $r0, 14, $noreg
     renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
@@ -119,6 +113,7 @@ body:             |
   
   bb.2.while.body:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r4
   
     $r5 = tMOVr killed $lr, 14, $noreg
     tBL 14, $noreg, @bar, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $r0
@@ -129,6 +124,8 @@ body:             |
     tB %bb.3, 14, $noreg
   
   bb.3.while.end:
+    liveins: $r4
+  
     $r0 = tMOVr killed $r4, 14, $noreg
     tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
   
@@ -138,4 +135,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
index b69d35859a7ab..fc2705797404e 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-read.mir
@@ -4,7 +4,10 @@
 # CHECK-NOT: t2DLS
 # CHECK-NOT: t2LEUpdate
 
---- |  
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define i32 @mov_between_dec_end(i32 %n) #0 {
   entry:
     %cmp6 = icmp eq i32 %n, 0
@@ -15,7 +18,6 @@
     br label %while.body
   
   while.body:                                       ; preds = %while.body, %while.body.preheader
-    %res.07 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
     %0 = phi i32 [ %n, %while.body.preheader ], [ %1, %while.body ]
     %1 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
     %add = add i32 %1, 0
@@ -27,10 +29,7 @@
     ret i32 %res.0.lcssa
   }
   
-  ; Function Attrs: noduplicate nounwind
   declare void @llvm.set.loop.iterations.i32(i32) #1
-  
-  ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
   attributes #0 = { "target-features"="+mve.fp" }
@@ -46,7 +45,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -56,11 +55,11 @@ frameInfo:
   isReturnAddressTaken: false
   hasStackMap:     false
   hasPatchPoint:   false
-  stackSize:       16
+  stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
-  adjustsStack:    true
-  hasCalls:        true
+  adjustsStack:    false
+  hasCalls:        false
   stackProtector:  ''
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
@@ -78,51 +77,46 @@ stack:
   - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
       stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 2, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r5', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 3, name: '', type: spill-slot, offset: -16, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r4', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 callSites:       []
 constants:       []
 machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.4(0x30000000), %bb.1(0x50000000)
+    liveins: $r0, $r7, $lr
   
-    frame-setup tPUSH 14, $noreg, killed $r4, killed $r5, killed $r7, killed $lr, implicit-def $sp, implicit $sp
-    frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
     frame-setup CFI_INSTRUCTION offset $r7, -8
-    frame-setup CFI_INSTRUCTION offset $r5, -12
-    frame-setup CFI_INSTRUCTION offset $r4, -16
     tCBZ $r0, %bb.4
   
   bb.1.while.body.preheader:
     successors: %bb.2(0x80000000)
+    liveins: $r0
   
     $lr = tMOVr $r0, 14, $noreg
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
     t2DoLoopStart killed $r0
   
   bb.2.while.body:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r4
   
-    renamable $r4 = nsw tADDhirr killed renamable $r4, killed renamable $r0, 14, $noreg
+    renamable $r4 = nsw tADDhirr killed renamable $r4, killed renamable $r4, 14, $noreg
     renamable $lr = t2LoopDec killed renamable $lr, 1
     renamable $r4 = tMOVr $lr, 14, $noreg
     t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
     tB %bb.3, 14, $noreg
   
   bb.3.while.end:
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
+    liveins: $lr
+  
+    $r0 = tMOVr killed $lr, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
   
   bb.4:
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
+    renamable $lr = t2MOVi 0, 14, $noreg, $noreg
+    $r0 = tMOVr killed $lr, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-spill.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-spill.mir
deleted file mode 100644
index 97ac75de49721..0000000000000
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-spill.mir
+++ /dev/null
@@ -1,136 +0,0 @@
-# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-low-overhead-loops --verify-machineinstrs -o - | FileCheck %s
-
-# CHECK: while.body:
-# CHECK-NOT: t2DLS
-# CHECK-NOT: t2LEUpdate
-
---- |  
-  define i32 @skip_spill(i32 %n) #0 {
-  entry:
-    %cmp6 = icmp eq i32 %n, 0
-    br i1 %cmp6, label %while.end, label %while.body.preheader
-  
-  while.body.preheader:                             ; preds = %entry
-    call void @llvm.set.loop.iterations.i32(i32 %n)
-    br label %while.body
-  
-  while.body:                                       ; preds = %while.body, %while.body.preheader
-    %res.07 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
-    %0 = phi i32 [ %n, %while.body.preheader ], [ %1, %while.body ]
-    %call = tail call i32 bitcast (i32 (...)* @bar to i32 ()*)()
-    %add = add nsw i32 %call, %res.07
-    %1 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
-    %2 = icmp ne i32 %1, 0
-    br i1 %2, label %while.body, label %while.end
-  
-  while.end:                                        ; preds = %while.body, %entry
-    %res.0.lcssa = phi i32 [ 0, %entry ], [ %add, %while.body ]
-    ret i32 %res.0.lcssa
-  }
-  
-  declare i32 @bar(...) local_unnamed_addr #0
-  
-  ; Function Attrs: noduplicate nounwind
-  declare void @llvm.set.loop.iterations.i32(i32) #1
-  
-  ; Function Attrs: noduplicate nounwind
-  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
-  
-  ; Function Attrs: nounwind
-  declare void @llvm.stackprotector(i8*, i8**) #2
-  
-  attributes #0 = { "target-features"="+mve.fp" }
-  attributes #1 = { noduplicate nounwind }
-  attributes #2 = { nounwind }
-
-...
----
-name:            skip_spill
-alignment:       2
-exposesReturnsTwice: false
-legalized:       false
-regBankSelected: false
-selected:        false
-failedISel:      false
-tracksRegLiveness: false
-hasWinCFI:       false
-registers:       []
-liveins:
-  - { reg: '$r0', virtual-reg: '' }
-frameInfo:
-  isFrameAddressTaken: false
-  isReturnAddressTaken: false
-  hasStackMap:     false
-  hasPatchPoint:   false
-  stackSize:       16
-  offsetAdjustment: 0
-  maxAlignment:    4
-  adjustsStack:    true
-  hasCalls:        true
-  stackProtector:  ''
-  maxCallFrameSize: 0
-  cvBytesOfCalleeSavedRegisters: 0
-  hasOpaqueSPAdjustment: false
-  hasVAStart:      false
-  hasMustTailInVarArgFunc: false
-  localFrameSize:  0
-  savePoint:       ''
-  restorePoint:    ''
-fixedStack:      []
-stack:
-  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 2, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r5', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 3, name: '', type: spill-slot, offset: -16, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r4', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-callSites:       []
-constants:       []
-machineFunctionInfo: {}
-body:             |
-  bb.0.entry:
-    successors: %bb.4(0x30000000), %bb.1(0x50000000)
-  
-    frame-setup tPUSH 14, $noreg, killed $r4, killed $r5, killed $r7, killed $lr, implicit-def $sp, implicit $sp
-    frame-setup CFI_INSTRUCTION def_cfa_offset 16
-    frame-setup CFI_INSTRUCTION offset $lr, -4
-    frame-setup CFI_INSTRUCTION offset $r7, -8
-    frame-setup CFI_INSTRUCTION offset $r5, -12
-    frame-setup CFI_INSTRUCTION offset $r4, -16
-    tCBZ $r0, %bb.4
-  
-  bb.1.while.body.preheader:
-    successors: %bb.2(0x80000000)
-  
-    $lr = tMOVr $r0, 14, $noreg
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
-    t2DoLoopStart killed $r0
-  
-  bb.2.while.body:
-    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
-  
-    $r5 = tMOVr killed $lr, 14, $noreg
-    tBL 14, $noreg, @bar, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $r0
-    $lr = tMOVr killed $r5, 14, $noreg
-    renamable $r4 = nsw tADDhirr killed renamable $r4, killed renamable $r0, 14, $noreg
-    renamable $lr = t2LoopDec killed renamable $lr, 1
-    t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
-    tB %bb.3, 14, $noreg
-  
-  bb.3.while.end:
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
-  
-  bb.4:
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
-
-...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
index 2d31c8ad63bb1..ca3a1122d982c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-after-write.mir
@@ -4,7 +4,10 @@
 # CHECK-NOT: t2DLS
 # CHECK-NOT: t2LEUpdate
 
---- |  
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define i32 @mov_between_dec_end(i32 %n) #0 {
   entry:
     %cmp6 = icmp eq i32 %n, 0
@@ -15,7 +18,6 @@
     br label %while.body
   
   while.body:                                       ; preds = %while.body, %while.body.preheader
-    %res.07 = phi i32 [ %add, %while.body ], [ 0, %while.body.preheader ]
     %0 = phi i32 [ %n, %while.body.preheader ], [ %1, %while.body ]
     %1 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
     %add = add i32 %1, 2
@@ -33,6 +35,9 @@
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #2
+  
   attributes #0 = { "target-features"="+mve.fp" }
   attributes #1 = { noduplicate nounwind }
   attributes #2 = { nounwind }
@@ -46,7 +51,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -56,11 +61,11 @@ frameInfo:
   isReturnAddressTaken: false
   hasStackMap:     false
   hasPatchPoint:   false
-  stackSize:       16
+  stackSize:       8
   offsetAdjustment: 0
   maxAlignment:    4
-  adjustsStack:    true
-  hasCalls:        true
+  adjustsStack:    false
+  hasCalls:        false
   stackProtector:  ''
   maxCallFrameSize: 0
   cvBytesOfCalleeSavedRegisters: 0
@@ -78,51 +83,45 @@ stack:
   - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
       stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 2, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r5', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 3, name: '', type: spill-slot, offset: -16, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r4', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 callSites:       []
 constants:       []
 machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.4(0x30000000), %bb.1(0x50000000)
+    liveins: $r0, $r7, $lr
   
-    frame-setup tPUSH 14, $noreg, killed $r4, killed $r5, killed $r7, killed $lr, implicit-def $sp, implicit $sp
-    frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
     frame-setup CFI_INSTRUCTION offset $r7, -8
-    frame-setup CFI_INSTRUCTION offset $r5, -12
-    frame-setup CFI_INSTRUCTION offset $r4, -16
     tCBZ $r0, %bb.4
   
   bb.1.while.body.preheader:
     successors: %bb.2(0x80000000)
+    liveins: $r0
   
     $lr = tMOVr $r0, 14, $noreg
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
     t2DoLoopStart killed $r0
   
   bb.2.while.body:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr
   
     $r4 = tMOVr $lr, 14, $noreg
     renamable $lr = t2LoopDec killed renamable $lr, 1
+    renamable $r0 = t2ADDri renamable $lr, 2, 14, $noreg, $noreg
     $lr = tMOVr $r4, 14, $noreg
     t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
     tB %bb.3, 14, $noreg
   
   bb.3.while.end:
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
+    liveins: $r0
+  
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
   
   bb.4:
-    renamable $r4, dead $cpsr = tMOVi8 0, 14, $noreg
-    $r0 = tMOVr killed $r4, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r5, def $r7, def $pc, implicit killed $r0
+    renamable $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
index 54973cc683b68..3837542c2b140 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-header.mir
@@ -1,13 +1,18 @@
-# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops %s -o - | FileCheck %s
+# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops --verify-machineinstrs %s -o - | FileCheck %s
 # CHECK-NOT: t2DLS
 # CHECK: bb.5.for.inc16:
-# CHECK:   $lr = t2SUBri killed renamable $lr, 1, 14
-# CHECK:   t2CMPri $lr, 0, 14
-# CHECK:   t2Bcc %bb.6, 1
-# CHECK:   tB %bb.2
+# CHECK:      $lr = t2SUBri killed renamable $lr, 1, 14, $noreg, def $cpsr
+# CHECK-NOT:  t2CMPri $lr
+# CHECK:      tBcc %bb.6, 1
+# CHECK:      tB %bb.2
 # CHECK: bb.6.for.cond4.preheader:
 
 --- |
+  ; ModuleID = 'revert-non-header.ll'
+  source_filename = "revert-non-header.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define void @header_not_target_unrolled_loop(i32* nocapture %v, i32 %n) {
   entry:
     %cmp56 = icmp sgt i32 %n, 1
@@ -48,16 +53,16 @@
     br label %land.rhs
   
   land.rhs:                                         ; preds = %land.rhs.preheader, %for.body8
-    %lsr.iv4 = phi i32 [ 0, %land.rhs.preheader ], [ %lsr.iv.next, %for.body8 ]
+    %lsr.iv4 = phi i32 [ %lsr.iv.next, %for.body8 ], [ 0, %land.rhs.preheader ]
     %j.051 = phi i32 [ %j.0, %for.body8 ], [ %j.048, %land.rhs.preheader ]
     %1 = bitcast i32* %lsr.iv2 to i8*
     %2 = bitcast i32* %lsr.iv to i8*
-    %uglygep10 = getelementptr i8, i8* %2, i32 %lsr.iv4
-    %uglygep1011 = bitcast i8* %uglygep10 to i32*
-    %tmp9 = load i32, i32* %uglygep1011, align 4
-    %uglygep7 = getelementptr i8, i8* %1, i32 %lsr.iv4
-    %uglygep78 = bitcast i8* %uglygep7 to i32*
-    %tmp12 = load i32, i32* %uglygep78, align 4
+    %uglygep3 = getelementptr i8, i8* %2, i32 %lsr.iv4
+    %uglygep34 = bitcast i8* %uglygep3 to i32*
+    %tmp9 = load i32, i32* %uglygep34, align 4
+    %uglygep1 = getelementptr i8, i8* %1, i32 %lsr.iv4
+    %uglygep12 = bitcast i8* %uglygep1 to i32*
+    %tmp12 = load i32, i32* %uglygep12, align 4
     %cmp7 = icmp sgt i32 %tmp9, %tmp12
     br i1 %cmp7, label %for.body8, label %for.inc16
   
@@ -108,7 +113,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -165,6 +170,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x40000000), %bb.9(0x40000000)
+    liveins: $r0, $r1, $r4, $r5, $r6, $r7, $r8, $r9, $r10, $lr
   
     $sp = frame-setup t2STMDB_UPD $sp, 14, $noreg, killed $r4, killed $r5, killed $r6, killed $r7, killed $r8, killed $r9, killed $r10, killed $lr
     frame-setup CFI_INSTRUCTION def_cfa_offset 32
@@ -181,18 +187,21 @@ body:             |
   
   bb.1:
     successors: %bb.3(0x80000000)
+    liveins: $r0, $r1
   
     $r9 = tMOVr $r1, 14, $noreg
     tB %bb.3, 14, $noreg
   
   bb.2.for.cond.loopexit:
     successors: %bb.3(0x7c000000), %bb.9(0x04000000)
+    liveins: $r0, $r1, $r9, $r12
   
     t2CMPri killed renamable $r12, 4, 14, $noreg, implicit-def $cpsr
     tBcc %bb.9, 11, killed $cpsr
   
   bb.3.for.cond1.preheader:
     successors: %bb.4(0x40000000), %bb.2(0x40000000)
+    liveins: $r0, $r1, $r9
   
     renamable $r2 = t2ADDrs $r9, $r9, 251, 14, $noreg, $noreg
     $r12 = tMOVr killed $r9, 14, $noreg
@@ -204,6 +213,7 @@ body:             |
   
   bb.4.for.cond4.preheader.preheader:
     successors: %bb.7(0x50000000), %bb.5(0x30000000)
+    liveins: $lr, $r0, $r1, $r9, $r12
   
     renamable $r3 = t2ADDrs renamable $r0, renamable $r9, 18, 14, $noreg, $noreg
     renamable $r10 = t2LSLri renamable $r9, 2, 14, $noreg, $noreg
@@ -214,6 +224,7 @@ body:             |
   
   bb.5.for.inc16:
     successors: %bb.6(0x7c000000), %bb.2(0x04000000)
+    liveins: $lr, $r0, $r1, $r3, $r5, $r8, $r9, $r10, $r12
   
     renamable $r3, dead $cpsr = tADDi8 killed renamable $r3, 4, 14, $noreg
     renamable $r5, dead $cpsr = tADDi8 killed renamable $r5, 4, 14, $noreg
@@ -224,20 +235,23 @@ body:             |
   
   bb.6.for.cond4.preheader:
     successors: %bb.7(0x50000000), %bb.5(0x30000000)
+    liveins: $lr, $r0, $r1, $r3, $r5, $r8, $r9, $r10, $r12
   
     renamable $r7 = nsw t2SUBrr renamable $r8, renamable $r9, 14, $noreg, def $cpsr
     tBcc %bb.5, 4, killed $cpsr
   
   bb.7.land.rhs.preheader:
     successors: %bb.8(0x80000000)
+    liveins: $lr, $r0, $r1, $r3, $r5, $r7, $r8, $r9, $r10, $r12
   
     renamable $r6, dead $cpsr = tMOVi8 0, 14, $noreg
   
   bb.8.land.rhs:
     successors: %bb.5(0x07e00000), %bb.8(0x78200000)
+    liveins: $lr, $r0, $r1, $r3, $r5, $r6, $r7, $r8, $r9, $r10, $r12
   
-    renamable $r4 = tLDRr renamable $r3, $r6, 14, $noreg :: (load 4 from %ir.uglygep78)
-    renamable $r2 = tLDRr renamable $r5, $r6, 14, $noreg :: (load 4 from %ir.uglygep1011)
+    renamable $r4 = tLDRr renamable $r3, $r6, 14, $noreg :: (load 4 from %ir.uglygep12)
+    renamable $r2 = tLDRr renamable $r5, $r6, 14, $noreg :: (load 4 from %ir.uglygep34)
     tCMPr renamable $r2, renamable $r4, 14, $noreg, implicit-def $cpsr
     t2IT 12, 1, implicit-def $itstate
     tSTRr killed renamable $r4, renamable $r5, $r6, 12, $cpsr, implicit $itstate :: (store 4 into %ir.5)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
index bcd80c70bb4c1..a7c1876d9c5d5 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-non-loop.mir
@@ -5,15 +5,18 @@
 # CHECK:   tBcc %bb.2, 3
 # CHECK: bb.1.not.preheader:
 # CHECK:   t2CMPri renamable $lr, 0, 14
-# CHECK:   t2Bcc %bb.4, 0
+# CHECK:   tBcc %bb.4, 0
 # CHECK:   tB %bb.2
 # CHECK: bb.3.while.body:
 # CHECK:   t2CMPri $lr, 0, 14
-# CHECK:   t2Bcc %bb.3, 1
+# CHECK:   tBcc %bb.3, 1
 # CHECK:   tB %bb.4
 # CHECK: bb.4.while.end:
 
 --- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define void @non_loop(i16* nocapture %a, i16* nocapture readonly %b, i32 %N) {
   entry:
     %cmp = icmp ugt i32 %N, 2
@@ -23,19 +26,19 @@
     %test = call i1 @llvm.test.set.loop.iterations.i32(i32 %N)
     br i1 %test, label %while.body.preheader, label %while.end
   
-  while.body.preheader:                             ; preds = %entry, %not.preheader
+  while.body.preheader:                             ; preds = %not.preheader, %entry
     %scevgep = getelementptr i16, i16* %a, i32 -1
     %scevgep3 = getelementptr i16, i16* %b, i32 -1
     br label %while.body
   
-  while.body:                                       ; preds = %while.body.preheader, %while.body
+  while.body:                                       ; preds = %while.body, %while.body.preheader
     %lsr.iv4 = phi i16* [ %scevgep3, %while.body.preheader ], [ %scevgep5, %while.body ]
     %lsr.iv = phi i16* [ %scevgep, %while.body.preheader ], [ %scevgep1, %while.body ]
     %count = phi i32 [ %count.next, %while.body ], [ %N, %while.body.preheader ]
-    %scevgep2 = getelementptr i16, i16* %lsr.iv, i32 1
-    %scevgep6 = getelementptr i16, i16* %lsr.iv4, i32 1
-    %load = load i16, i16* %scevgep6, align 2
-    store i16 %load, i16* %scevgep2, align 2
+    %scevgep7 = getelementptr i16, i16* %lsr.iv, i32 1
+    %scevgep4 = getelementptr i16, i16* %lsr.iv4, i32 1
+    %load = load i16, i16* %scevgep4, align 2
+    store i16 %load, i16* %scevgep7, align 2
     %count.next = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %count, i32 1)
     %cmp1 = icmp ne i32 %count.next, 0
     %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
@@ -46,13 +49,8 @@
     ret void
   }
   
-  ; Function Attrs: noduplicate nounwind
   declare i1 @llvm.test.set.loop.iterations.i32(i32) #0
-  
-  ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
-  
-  ; Function Attrs: nounwind
   declare void @llvm.stackprotector(i8*, i8**) #1
   
   attributes #0 = { noduplicate nounwind }
@@ -67,7 +65,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -107,6 +105,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -118,21 +117,24 @@ body:             |
   
   bb.1.not.preheader:
     successors: %bb.2(0x40000000), %bb.4(0x40000000)
+    liveins: $lr, $r0, $r1
   
     t2WhileLoopStart renamable $lr, %bb.4, implicit-def dead $cpsr
     tB %bb.2, 14, $noreg
   
   bb.2.while.body.preheader:
     successors: %bb.3(0x80000000)
+    liveins: $lr, $r0, $r1
   
     renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 2, 14, $noreg
     renamable $r0, dead $cpsr = tSUBi8 killed renamable $r0, 2, 14, $noreg
   
   bb.3.while.body:
     successors: %bb.3(0x7c000000), %bb.4(0x04000000)
+    liveins: $lr, $r0, $r1
   
-    renamable $r2, renamable $r1 = t2LDRH_PRE killed renamable $r1, 2, 14, $noreg :: (load 2 from %ir.scevgep6)
-    early-clobber renamable $r0 = t2STRH_PRE killed renamable $r2, killed renamable $r0, 2, 14, $noreg :: (store 2 into %ir.scevgep2)
+    renamable $r2, renamable $r1 = t2LDRH_PRE killed renamable $r1, 2, 14, $noreg :: (load 2 from %ir.scevgep4)
+    early-clobber renamable $r0 = t2STRH_PRE killed renamable $r2, killed renamable $r0, 2, 14, $noreg :: (store 2 into %ir.scevgep7)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.3, implicit-def dead $cpsr
     tB %bb.4, 14, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
index 754d35ea89bff..6e9dcaf3827cc 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/revert-while.mir
@@ -7,12 +7,14 @@
 # CHECK:      bb.1.do.body.preheader:
 # CHECK:        $lr = tMOVr killed $r3
 # CHECK:      bb.2.do.body:
-# CHECK:        $lr = t2SUBri killed renamable $lr, 1, 14
-# CHECK-NEXT:   t2CMPri $lr, 0, 14, $noreg, implicit-def $cpsr
+# CHECK:        $lr = t2SUBri killed renamable $lr, 1, 14, $noreg, def $cpsr
 # CHECK-NEXT:   t2Bcc %bb.2, 1, $cpsr
 # CHECK-NEXT:   tB %bb.3, 14
 
 --- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define void @ne_trip_count(i1 zeroext %t1, i32* nocapture %a, i32* nocapture readonly %b, i32 %N) #0 {
   entry:
     %0 = call i1 @llvm.test.set.loop.iterations.i32(i32 %N)
@@ -23,15 +25,15 @@
     %scevgep5 = getelementptr i32, i32* %b, i32 -1
     br label %do.body
   
-  do.body:                                          ; preds = %do.body.preheader, %do.body
+  do.body:                                          ; preds = %do.body, %do.body.preheader
     %lsr.iv6 = phi i32* [ %scevgep5, %do.body.preheader ], [ %scevgep7, %do.body ]
     %lsr.iv = phi i32* [ %scevgep2, %do.body.preheader ], [ %scevgep3, %do.body ]
     %1 = phi i32 [ %2, %do.body ], [ %N, %do.body.preheader ]
-    %scevgep8 = getelementptr i32, i32* %lsr.iv6, i32 1
-    %scevgep4 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep = getelementptr i32, i32* %lsr.iv6, i32 1
+    %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
     %size = call i32 @llvm.arm.space(i32 4096, i32 undef)
-    %tmp = load i32, i32* %scevgep8, align 4
-    store i32 %tmp, i32* %scevgep4, align 4
+    %tmp = load i32, i32* %scevgep, align 4
+    store i32 %tmp, i32* %scevgep1, align 4
     %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %1, i32 1)
     %3 = icmp ne i32 %2, 0
     %scevgep3 = getelementptr i32, i32* %lsr.iv, i32 1
@@ -51,9 +53,6 @@
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #2
   
-  ; Function Attrs: nounwind
-  declare void @llvm.stackprotector(i8*, i8**) #1
-  
   attributes #0 = { "target-features"="+lob" }
   attributes #1 = { nounwind }
   attributes #2 = { noduplicate nounwind }
@@ -67,7 +66,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -107,6 +106,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x40000000), %bb.3(0x40000000)
+    liveins: $r1, $r2, $r3, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -117,6 +117,7 @@ body:             |
   
   bb.1.do.body.preheader:
     successors: %bb.2(0x80000000)
+    liveins: $r1, $r2, $r3
   
     renamable $r0, dead $cpsr = tSUBi3 killed renamable $r2, 4, 14, $noreg
     renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 4, 14, $noreg
@@ -124,10 +125,11 @@ body:             |
   
   bb.2.do.body:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r0, $r1
   
     dead renamable $r2 = SPACE 4096, undef renamable $r0
-    renamable $r2, renamable $r0 = t2LDR_PRE killed renamable $r0, 4, 14, $noreg :: (load 4 from %ir.scevgep8)
-    early-clobber renamable $r1 = t2STR_PRE killed renamable $r2, killed renamable $r1, 4, 14, $noreg :: (store 4 into %ir.scevgep4)
+    renamable $r2, renamable $r0 = t2LDR_PRE killed renamable $r0, 4, 14, $noreg :: (load 4 from %ir.scevgep)
+    early-clobber renamable $r1 = t2STR_PRE killed renamable $r2, killed renamable $r1, 4, 14, $noreg :: (store 4 into %ir.scevgep1)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
     tB %bb.3, 14, $noreg
@@ -136,4 +138,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r7, def $pc
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir
new file mode 100644
index 0000000000000..efbee0e609cb8
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/safe-def-no-mov.mir
@@ -0,0 +1,124 @@
+# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops %s -verify-machineinstrs -o - | FileCheck %s
+# CHECK: $lr = t2DLS $r0
+# CHECK: $lr = tMOVr $r0, 14
+# CHECK: $lr = t2LEUpdate renamable $lr, %bb.2
+
+# TODO: Explore the preheader to remove the redundant tMOVr
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define i32 @do_copy(i32 %n, i32* nocapture %p, i32* nocapture readonly %q) {
+  entry:
+    %scevgep = getelementptr i32, i32* %q, i32 -1
+    %scevgep3 = getelementptr i32, i32* %p, i32 -1
+    call void @llvm.set.loop.iterations.i32(i32 %n)
+    br label %preheader
+
+  preheader:
+    br label %while.body
+  
+  while.body:                                       ; preds = %while.body, %entry
+    %lsr.iv4 = phi i32* [ %scevgep5, %while.body ], [ %scevgep3, %preheader ]
+    %lsr.iv = phi i32* [ %scevgep1, %while.body ], [ %scevgep, %preheader ]
+    %0 = phi i32 [ %n, %preheader ], [ %2, %while.body ]
+    %scevgep6 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep2 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %1 = load i32, i32* %scevgep6, align 4
+    store i32 %1, i32* %scevgep2, align 4
+    %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep5 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+    %3 = icmp ne i32 %2, 0
+    br i1 %3, label %while.body, label %while.end
+  
+  while.end:                                        ; preds = %while.body
+    ret i32 0
+  }
+  
+  declare void @llvm.set.loop.iterations.i32(i32) #0
+  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
+  
+  attributes #0 = { noduplicate nounwind }
+  attributes #1 = { nounwind }
+
+...
+---
+name:            do_copy
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+  
+    frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    t2DoLoopStart $r0
+    renamable $r0, dead $cpsr = tSUBi3 killed renamable $r1, 4, 14, $noreg
+    renamable $r1, dead $cpsr = tSUBi3 killed renamable $r2, 4, 14, $noreg
+
+  bb.1.preheader:
+    successors: %bb.2(0x80000000)
+    liveins: $r0
+    $lr = tMOVr $r0, 14, $noreg
+  
+  bb.2.while.body:
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r0, $r1
+  
+    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep6)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep2)
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14, $noreg
+  
+  bb.3.while.end:
+    $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
index 69a5bd33a96cf..acb7ebd35f3fd 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/size-limit.mir
@@ -1,10 +1,15 @@
-# RUN: llc -mtriple=armv8.1m.main -run-pass=arm-low-overhead-loops %s -o - | FileCheck %s
+# RUN: llc -mtriple=armv8.1m.main -run-pass=arm-low-overhead-loops --verify-machineinstrs %s -o - | FileCheck %s
 # CHECK: entry:
 # CHECK: $lr = t2DLS
 # CHECK: for.body:
 # CHECK: $lr = t2LEUpdate renamable $lr
 
---- |  
+--- |
+  ; ModuleID = 'size-limit.ll'
+  source_filename = "size-limit.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define dso_local arm_aapcscc void @size_limit(i32* nocapture %a, i32* nocapture readonly %b, i32* nocapture readonly %c, i32 %N) {
   entry:
     %cmp8 = icmp eq i32 %N, 0
@@ -26,13 +31,13 @@
     %lsr.iv1 = phi i32* [ %scevgep, %for.body.preheader ], [ %scevgep2, %for.body ]
     %0 = phi i32 [ %N, %for.body.preheader ], [ %3, %for.body ]
     %size = call i32 @llvm.arm.space(i32 4070, i32 undef)
-    %scevgep1 = getelementptr i32, i32* %lsr.iv9, i32 1
-    %1 = load i32, i32* %scevgep1, align 4
-    %scevgep5 = getelementptr i32, i32* %lsr.iv5, i32 1
-    %2 = load i32, i32* %scevgep5, align 4
+    %scevgep3 = getelementptr i32, i32* %lsr.iv9, i32 1
+    %1 = load i32, i32* %scevgep3, align 4
+    %scevgep7 = getelementptr i32, i32* %lsr.iv5, i32 1
+    %2 = load i32, i32* %scevgep7, align 4
     %mul = mul nsw i32 %2, %1
-    %scevgep9 = getelementptr i32, i32* %lsr.iv1, i32 1
-    store i32 %mul, i32* %scevgep9, align 4
+    %scevgep11 = getelementptr i32, i32* %lsr.iv1, i32 1
+    store i32 %mul, i32* %scevgep11, align 4
     %scevgep2 = getelementptr i32, i32* %lsr.iv1, i32 1
     %scevgep6 = getelementptr i32, i32* %lsr.iv5, i32 1
     %scevgep10 = getelementptr i32, i32* %lsr.iv9, i32 1
@@ -43,11 +48,16 @@
   
   ; Function Attrs: nounwind
   declare i32 @llvm.arm.space(i32 immarg, i32) #0
+  
   ; Function Attrs: noduplicate nounwind
   declare void @llvm.set.loop.iterations.i32(i32) #1
+  
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
   
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #0
+  
   attributes #0 = { nounwind }
   attributes #1 = { noduplicate nounwind }
 
@@ -60,7 +70,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -101,6 +111,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r3, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -117,12 +128,13 @@ body:             |
   
   bb.1.for.body:
     successors: %bb.1(0x7c000000), %bb.2(0x04000000)
+    liveins: $lr, $r0, $r1, $r2
   
     dead renamable $r3 = SPACE 4070, undef renamable $r0
-    renamable $r12, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep1)
-    renamable $r3, renamable $r2 = t2LDR_PRE killed renamable $r2, 4, 14, $noreg :: (load 4 from %ir.scevgep5)
+    renamable $r12, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep3)
+    renamable $r3, renamable $r2 = t2LDR_PRE killed renamable $r2, 4, 14, $noreg :: (load 4 from %ir.scevgep7)
     renamable $r3 = nsw t2MUL killed renamable $r3, killed renamable $r12, 14, $noreg
-    early-clobber renamable $r0 = t2STR_PRE killed renamable $r3, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep9)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r3, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep11)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.1, implicit-def dead $cpsr
     tB %bb.2, 14, $noreg
@@ -131,5 +143,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r7, def $pc
 
 ...
-
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
index 9d86f9291aa1b..a97da0c0f9e5e 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/switch.mir
@@ -1,4 +1,4 @@
-# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-low-overhead-loops -o -
+# RUN: llc -mtriple=thumbv8.1m.main %s -run-pass=arm-low-overhead-loops --verify-machineinstrs -o -
 # CHECK:      bb.1.for.body.preheader:
 # CHECK:        $lr = t2DLS
 # CHECK-NOT:    t2LoopDec
@@ -6,6 +6,11 @@
 # CHECK:        $lr = t2LEUpdate renamable $lr, %bb.2
 
 --- |
+  ; ModuleID = 'switch.ll'
+  source_filename = "switch.ll"
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
   define dso_local arm_aapcscc i32 @search(i8* nocapture readonly %c, i32 %N) {
   entry:
     %cmp11 = icmp eq i32 %N, 0
@@ -52,10 +57,17 @@
     br i1 %4, label %for.body, label %for.cond.cleanup
   }
   
-  declare void @llvm.set.loop.iterations.i32(i32) #1
-  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #1
-  attributes #1 = { noduplicate nounwind }
-  attributes #2 = { nounwind }
+  ; Function Attrs: noduplicate nounwind
+  declare void @llvm.set.loop.iterations.i32(i32) #0
+  
+  ; Function Attrs: noduplicate nounwind
+  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
+  
+  ; Function Attrs: nounwind
+  declare void @llvm.stackprotector(i8*, i8**) #1
+  
+  attributes #0 = { noduplicate nounwind }
+  attributes #1 = { nounwind }
 
 ...
 ---
@@ -66,7 +78,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -77,8 +89,8 @@ frameInfo:
   isReturnAddressTaken: false
   hasStackMap:     false
   hasPatchPoint:   false
-  stackSize:       16
-  offsetAdjustment: -8
+  stackSize:       8
+  offsetAdjustment: 0
   maxAlignment:    4
   adjustsStack:    false
   hasCalls:        false
@@ -97,12 +109,6 @@ stack:
       stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
   - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 2, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4, 
-      stack-id: default, callee-saved-register: '$r6', callee-saved-restored: true, 
-      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
-  - { id: 3, name: '', type: spill-slot, offset: -16, size: 4, alignment: 4, 
       stack-id: default, callee-saved-register: '$r4', callee-saved-restored: true, 
       debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
 callSites:       []
@@ -111,19 +117,17 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.7(0x30000000), %bb.1(0x50000000)
+    liveins: $r0, $r1, $r4, $lr
   
-    frame-setup tPUSH 14, $noreg, killed $r4, killed $r6, $r7, killed $lr, implicit-def $sp, implicit $sp
-    frame-setup CFI_INSTRUCTION def_cfa_offset 16
+    frame-setup tPUSH 14, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
     frame-setup CFI_INSTRUCTION offset $lr, -4
-    frame-setup CFI_INSTRUCTION offset $r7, -8
-    frame-setup CFI_INSTRUCTION offset $r6, -12
-    frame-setup CFI_INSTRUCTION offset $r4, -16
-    $r7 = frame-setup tADDrSPi $sp, 2, 14, $noreg
-    frame-setup CFI_INSTRUCTION def_cfa $r7, 8
+    frame-setup CFI_INSTRUCTION offset $r4, -8
     tCBZ $r1, %bb.7
   
   bb.1.for.body.preheader:
     successors: %bb.4(0x80000000)
+    liveins: $r0, $r1
   
     $lr = tMOVr $r1, 14, $noreg
     t2DoLoopStart killed $r1
@@ -134,6 +138,7 @@ body:             |
   
   bb.2.for.body:
     successors: %bb.3(0x80000000)
+    liveins: $lr, $r0, $r1, $r2, $r3, $r12
   
     tCMPi8 killed renamable $r3, 32, 14, $noreg, implicit-def $cpsr
     t2IT 0, 8, implicit-def $itstate
@@ -141,6 +146,7 @@ body:             |
   
   bb.3.for.inc:
     successors: %bb.4(0x7c000000), %bb.8(0x04000000)
+    liveins: $lr, $r0, $r1, $r2, $r12
   
     renamable $r0, dead $cpsr = tADDi8 killed renamable $r0, 1, 14, $noreg
     renamable $lr = t2LoopDec killed renamable $lr, 1
@@ -149,6 +155,7 @@ body:             |
   
   bb.4.for.body:
     successors: %bb.2(0x26666665), %bb.5(0x5999999b)
+    liveins: $lr, $r0, $r1, $r2, $r12
   
     renamable $r3 = tLDRBi renamable $r0, 0, 14, $noreg :: (load 1 from %ir.lsr.iv1)
     renamable $r4 = t2SUBri renamable $r3, 108, 14, $noreg, $noreg
@@ -157,6 +164,7 @@ body:             |
   
   bb.5.for.body:
     successors: %bb.6(0x6db6db6e), %bb.2(0x12492492)
+    liveins: $lr, $r0, $r1, $r2, $r3, $r4, $r12
   
     renamable $r4 = t2LSLrr renamable $r12, killed renamable $r4, 14, $noreg, $noreg
     t2TSTri killed renamable $r4, 25, 14, $noreg, implicit-def $cpsr
@@ -164,6 +172,7 @@ body:             |
   
   bb.6.sw.bb:
     successors: %bb.3(0x80000000)
+    liveins: $lr, $r0, $r1, $r2, $r12
   
     renamable $r2, dead $cpsr = nsw tADDi8 killed renamable $r2, 1, 14, $noreg
     tB %bb.3, 14, $noreg
@@ -175,8 +184,9 @@ body:             |
     renamable $r1, dead $cpsr = tMOVi8 0, 14, $noreg
   
   bb.8.for.cond.cleanup:
+    liveins: $r1, $r2
+  
     renamable $r0, dead $cpsr = nsw tSUBrr killed renamable $r2, killed renamable $r1, 14, $noreg
-    tPOP_RET 14, $noreg, def $r4, def $r6, def $r7, def $pc, implicit killed $r0
+    tPOP_RET 14, $noreg, def $r4, def $pc, implicit killed $r0
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
index c1b957b9657b2..70e272ffc0dce 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
@@ -50,8 +50,8 @@ for.cond.cleanup:                                 ; preds = %vector.body, %entry
 
 ; CHECK-LABEL: expand_v8i16_v4i32
 ; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS_REM:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]])
 ; CHECK: [[ELEMS_REM]] = sub i32 [[ELEMS]], 8
-; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS_REM]])
 ; CHECK: tail call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* {{.*}}, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 ; CHECK: %store.pred = icmp ule <4 x i32> %induction.store
 ; CHECK: tail call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> {{.*}}, <4 x i32>* {{.*}}, i32 4, <4 x i1> %store.pred)
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
index b30806001c1aa..7cdd28fd0f3cf 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
@@ -5,8 +5,8 @@
 ; CHECK: phi <8 x i16> [ zeroinitializer, %entry ]
 ; CHECK: phi i32
 ; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[PHI]])
 ; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8
-; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]])
 ; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 ; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp6, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 define i16 @reduction_i32(i16* nocapture readonly %A, i16* nocapture readonly %B, i32 %N) {
@@ -63,8 +63,8 @@ middle.block:                                     ; preds = %vector.body
 ; CHECK: phi <8 x i16> [ zeroinitializer, %entry ]
 ; CHECK: phi i32
 ; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %entry ], [ [[ELEMS:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[PHI]])
 ; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8
-; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.vctp16(i32 [[ELEMS]])
 ; CHECK: call <8 x i16> @llvm.masked.load.v8i16.p0v8i16(<8 x i16>* %tmp4, i32 4, <8 x i1> [[VCTP]], <8 x i16> undef)
 define i16 @reduction_i32_with_scalar(i16* nocapture readonly %A, i16 %B, i32 %N) local_unnamed_addr {
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-liveout.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-liveout.mir
new file mode 100644
index 0000000000000..2c96665e6388f
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-liveout.mir
@@ -0,0 +1,122 @@
+# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops %s -verify-machineinstrs -o - | FileCheck %s
+# CHECK-NOT: $lr = t2DLS
+# CHECK: $lr = tMOVr $r0, 14
+# CHECK-NOT: $lr = t2LEUpdate
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define i32 @do_copy(i32 %n, i32* nocapture %p, i32* nocapture readonly %q) {
+  entry:
+    %scevgep = getelementptr i32, i32* %q, i32 -1
+    %scevgep3 = getelementptr i32, i32* %p, i32 -1
+    call void @llvm.set.loop.iterations.i32(i32 %n)
+    br label %preheader
+
+  preheader:
+    br label %while.body
+  
+  while.body:                                       ; preds = %while.body, %entry
+    %lsr.iv4 = phi i32* [ %scevgep5, %while.body ], [ %scevgep3, %preheader ]
+    %lsr.iv = phi i32* [ %scevgep1, %while.body ], [ %scevgep, %preheader ]
+    %0 = phi i32 [ %n, %preheader ], [ %2, %while.body ]
+    %scevgep6 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep2 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %1 = load i32, i32* %scevgep6, align 4
+    store i32 %1, i32* %scevgep2, align 4
+    %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep5 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+    %3 = icmp ne i32 %2, 0
+    br i1 %3, label %while.body, label %while.end
+  
+  while.end:                                        ; preds = %while.body
+    ret i32 0
+  }
+  
+  declare void @llvm.set.loop.iterations.i32(i32) #0
+  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
+  
+  attributes #0 = { noduplicate nounwind }
+  attributes #1 = { nounwind }
+
+...
+---
+name:            do_copy
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+  
+    frame-setup tPUSH 14, $noreg, killed $r7, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    t2DoLoopStart $r0
+    renamable $r0, dead $cpsr = tSUBi3 killed renamable $r1, 4, 14, $noreg
+    renamable $r1, dead $cpsr = tSUBi3 killed renamable $r2, 4, 14, $noreg
+
+  bb.1.preheader:
+    successors: %bb.2(0x80000000)
+    liveins: $r0, $lr
+    $lr = tMOVr $r0, 14, $noreg
+  
+  bb.2.while.body:
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r0, $r1
+  
+    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep6)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep2)
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14, $noreg
+  
+  bb.3.while.end:
+    $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir
new file mode 100644
index 0000000000000..15a76e6270a81
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/unsafe-use-after.mir
@@ -0,0 +1,122 @@
+# RUN: llc -mtriple=thumbv8.1m.main -run-pass=arm-low-overhead-loops %s -verify-machineinstrs -o - | FileCheck %s
+# CHECK-NOT: $lr = t2DLS
+# CHECK: $lr = tMOVr $r0, 14
+# CHECK-NOT: $lr = t2LEUpdate
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main"
+  
+  define i32 @do_copy(i32 %n, i32* nocapture %p, i32* nocapture readonly %q) {
+  entry:
+    %scevgep = getelementptr i32, i32* %q, i32 -1
+    %scevgep3 = getelementptr i32, i32* %p, i32 -1
+    call void @llvm.set.loop.iterations.i32(i32 %n)
+    br label %preheader
+
+  preheader:
+    br label %while.body
+  
+  while.body:                                       ; preds = %while.body, %entry
+    %lsr.iv4 = phi i32* [ %scevgep5, %while.body ], [ %scevgep3, %preheader ]
+    %lsr.iv = phi i32* [ %scevgep1, %while.body ], [ %scevgep, %preheader ]
+    %0 = phi i32 [ %n, %preheader ], [ %2, %while.body ]
+    %scevgep6 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep2 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %1 = load i32, i32* %scevgep6, align 4
+    store i32 %1, i32* %scevgep2, align 4
+    %scevgep1 = getelementptr i32, i32* %lsr.iv, i32 1
+    %scevgep5 = getelementptr i32, i32* %lsr.iv4, i32 1
+    %2 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %0, i32 1)
+    %3 = icmp ne i32 %2, 0
+    br i1 %3, label %while.body, label %while.end
+  
+  while.end:                                        ; preds = %while.body
+    ret i32 0
+  }
+  
+  declare void @llvm.set.loop.iterations.i32(i32) #0
+  declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #0
+  
+  attributes #0 = { noduplicate nounwind }
+  attributes #1 = { nounwind }
+
+...
+---
+name:            do_copy
+alignment:       2
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+registers:       []
+liveins:
+  - { reg: '$r0', virtual-reg: '' }
+  - { reg: '$r1', virtual-reg: '' }
+  - { reg: '$r2', virtual-reg: '' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       8
+  offsetAdjustment: 0
+  maxAlignment:    4
+  adjustsStack:    false
+  hasCalls:        false
+  stackProtector:  ''
+  maxCallFrameSize: 0
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  localFrameSize:  0
+  savePoint:       ''
+  restorePoint:    ''
+fixedStack:      []
+stack:
+  - { id: 0, name: '', type: spill-slot, offset: -4, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: false, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+  - { id: 1, name: '', type: spill-slot, offset: -8, size: 4, alignment: 4, 
+      stack-id: default, callee-saved-register: '$r7', callee-saved-restored: true, 
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+callSites:       []
+constants:       []
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
+  
+    frame-setup tPUSH 14, $noreg, killed $r7, implicit-def $sp, implicit $sp
+    frame-setup CFI_INSTRUCTION def_cfa_offset 8
+    frame-setup CFI_INSTRUCTION offset $lr, -4
+    frame-setup CFI_INSTRUCTION offset $r7, -8
+    t2DoLoopStart $r0
+    renamable $r0 = t2SUBri killed renamable $lr, 4, 14, $noreg, def $cpsr
+    renamable $r1, dead $cpsr = tSUBi3 killed renamable $r2, 4, 14, $noreg
+
+  bb.1.preheader:
+    successors: %bb.2(0x80000000)
+    liveins: $r0
+    $lr = tMOVr $r0, 14, $noreg
+  
+  bb.2.while.body:
+    successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r0, $r1
+  
+    renamable $r2, renamable $r1 = t2LDR_PRE killed renamable $r1, 4, 14, $noreg :: (load 4 from %ir.scevgep6)
+    early-clobber renamable $r0 = t2STR_PRE killed renamable $r2, killed renamable $r0, 4, 14, $noreg :: (store 4 into %ir.scevgep2)
+    renamable $lr = t2LoopDec killed renamable $lr, 1
+    t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
+    tB %bb.3, 14, $noreg
+  
+  bb.3.while.end:
+    $r0, dead $cpsr = tMOVi8 0, 14, $noreg
+    tPOP_RET 14, $noreg, def $r7, def $pc, implicit killed $r0
+
+...
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
new file mode 100644
index 0000000000000..bd691963ad32c
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
@@ -0,0 +1,254 @@
+; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -enable-arm-maskedldst=true -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
+
+; CHECK-LABEL: mul_reduce_add
+; CHECK:      dls lr,
+; CHECK: [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32	[[ELEMS:r[0-9]+]]
+; CHECK:      vpstt	
+; CHECK-NEXT: vldrwt.u32
+; CHECK-NEXT: vldrwt.u32
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      sub{{.*}} [[ELEMS]], #4
+; CHECK:      le	lr, [[LOOP]]
+; CHECK:      vctp.32	[[ELEMS_OUT]]
+; CHECK:      vpsel
+; CHECK:      vaddv.u32	r0
+define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %6, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = getelementptr inbounds i32, i32* %b, i32 %index
+  %4 = bitcast i32* %3 to <4 x i32>*
+  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %4, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %5 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
+  %6 = add nsw <4 x i32> %5, %vec.phi
+  %index.next = add i32 %index, 4
+  %7 = icmp eq i32 %index.next, %n.vec
+  br i1 %7, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %8 = select <4 x i1> %1, <4 x i32> %6, <4 x i32> %vec.phi
+  %9 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %8)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %9, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; CHECK-LABEL: mul_reduce_add_const
+; CHECK:    dls lr
+; CHECK: [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r0]
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      sub{{.*}} [[ELEMS]], #4
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32 [[ELEMS_OUT]]
+; CHECK:      vpsel
+define dso_local i32 @mul_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
+  %index.next = add i32 %index, 4
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
+  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; CHECK-LABEL: add_reduce_add_const
+; CHECK:      dls lr, lr
+; CHECK: [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r0]
+; CHECK:      mov [[ELEMS_OUT:r[0-9]+]], [[ELEMS]]
+; CHECK:      sub{{.*}} [[ELEMS]], #4
+; CHECK:      vadd.i32
+; CHECK:      le lr, [[LOOP]]
+; CHECK:      vctp.32 [[ELEMS_OUT]]
+; CHECK:      vpsel
+define dso_local i32 @add_reduce_add_const(i32* noalias nocapture readonly %a, i32 %b, i32 %N) {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert9 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat10 = shufflevector <4 x i32> %broadcast.splatinsert9, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %3, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %a, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat10
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %vec.phi
+  %index.next = add i32 %index, 4
+  %4 = icmp eq i32 %index.next, %n.vec
+  br i1 %4, label %middle.block, label %vector.body
+
+middle.block:                                     ; preds = %vector.body
+  %5 = select <4 x i1> %1, <4 x i32> %3, <4 x i32> %vec.phi
+  %6 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %5)
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %6, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+
+; CHECK-LABEL: vector_mul_const
+; CHECK:      dls lr, lr
+; CHECK: [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      sub{{.*}} [[ELEMS]], #4
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r1]
+; CHECK:      vmul.i32
+; CHECK:      vpst	
+; CHECK-NEXT: vstrwt.32 q{{.*}}, [r0]
+; CHECK:      le lr, [[LOOP]]
+define dso_local void @vector_mul_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = mul nsw <4 x i32> %wide.masked.load, %broadcast.splat11
+  %4 = getelementptr inbounds i32, i32* %a, i32 %index
+  %5 = bitcast i32* %4 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+; CHECK-LABEL: vector_add_const
+; CHECK:      dls lr, lr
+; CHECK: [[LOOP:.LBB[0-9_]+]]:
+; CHECK:      vctp.32 [[ELEMS:r[0-9]+]]
+; CHECK:      sub{{.*}} [[ELEMS]], #4
+; CHECK:      vpst	
+; CHECK-NEXT: vldrwt.u32 q{{.*}}, [r1]
+; CHECK:      vadd.i32
+; CHECK:      vpst	
+; CHECK-NEXT: vstrwt.32 q{{.*}}, [r0]
+; CHECK:      le lr, [[LOOP]]
+define dso_local void @vector_add_const(i32* noalias nocapture %a, i32* noalias nocapture readonly %b, i32 %c, i32 %N) {
+entry:
+  %cmp6 = icmp eq i32 %N, 0
+  br i1 %cmp6, label %for.cond.cleanup, label %vector.ph
+
+vector.ph:                                        ; preds = %entry
+  %n.rnd.up = add i32 %N, 3
+  %n.vec = and i32 %n.rnd.up, -4
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
+  %broadcast.splatinsert10 = insertelement <4 x i32> undef, i32 %c, i32 0
+  %broadcast.splat11 = shufflevector <4 x i32> %broadcast.splatinsert10, <4 x i32> undef, <4 x i32> zeroinitializer
+  br label %vector.body
+
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %0 = getelementptr inbounds i32, i32* %b, i32 %index
+  %1 = icmp ule <4 x i32> %induction, %broadcast.splat9
+  %2 = bitcast i32* %0 to <4 x i32>*
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %2, i32 4, <4 x i1> %1, <4 x i32> undef)
+  %3 = add nsw <4 x i32> %wide.masked.load, %broadcast.splat11
+  %4 = getelementptr inbounds i32, i32* %a, i32 %index
+  %5 = bitcast i32* %4 to <4 x i32>*
+  call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %3, <4 x i32>* %5, i32 4, <4 x i1> %1)
+  %index.next = add i32 %index, 4
+  %6 = icmp eq i32 %index.next, %n.vec
+  br i1 %6, label %for.cond.cleanup, label %vector.body
+
+for.cond.cleanup:                                 ; preds = %vector.body, %entry
+  ret void
+}
+
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare void @llvm.masked.store.v4i32.p0v4i32(<4 x i32>, <4 x i32>*, i32 immarg, <4 x i1>) #4
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
new file mode 100644
index 0000000000000..dbf40f60cbd9a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll
@@ -0,0 +1,75 @@
+
+; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: vec_mul_reduce_add
+
+; CHECK: vector.body:
+; CHECK-NOT: phi i32 [ 0, %vector.ph ]
+; CHECK: [[ELTS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[SUB:%[^ ]+]], %vector.body ]
+; CHECK: [[VCTP:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELTS]])
+; CHECK: [[SUB]] = sub i32 [[ELTS]], 4
+; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]]
+; CHECK: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* {{.*}}, i32 4, <4 x i1> [[VCTP]],
+
+; CHECK: middle.block:
+; CHECK: [[VCTP_CLONE:%[^ ]+]] = call <4 x i1> @llvm.arm.vctp32(i32 [[ELTS]])
+; CHECK: [[VPSEL:%[^ ]+]] = select <4 x i1> [[VCTP_CLONE]],
+; CHECK: call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[VPSEL]])
+
+define i32 @vec_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
+entry:
+  %cmp8 = icmp eq i32 %N, 0
+  %0 = add i32 %N, 3
+  %1 = lshr i32 %0, 2
+  %2 = shl nuw i32 %1, 2
+  %3 = add i32 %2, -4
+  %4 = lshr i32 %3, 2
+  %5 = add nuw nsw i32 %4, 1
+  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
+  
+vector.ph:                                        ; preds = %entry
+  %trip.count.minus.1 = add i32 %N, -1
+  %broadcast.splatinsert11 = insertelement <4 x i32> undef, i32 %trip.count.minus.1, i32 0
+  %broadcast.splat12 = shufflevector <4 x i32> %broadcast.splatinsert11, <4 x i32> undef, <4 x i32> zeroinitializer
+  call void @llvm.set.loop.iterations.i32(i32 %5)
+  br label %vector.body
+  
+vector.body:                                      ; preds = %vector.body, %vector.ph
+  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+  %lsr.iv2 = phi i32* [ %scevgep3, %vector.body ], [ %a, %vector.ph ]
+  %lsr.iv = phi i32* [ %scevgep, %vector.body ], [ %b, %vector.ph ]
+  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %9, %vector.body ]
+  %6 = phi i32 [ %5, %vector.ph ], [ %10, %vector.body ]
+  %lsr.iv24 = bitcast i32* %lsr.iv2 to <4 x i32>*
+  %lsr.iv1 = bitcast i32* %lsr.iv to <4 x i32>*
+  %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %index, i32 0
+  %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
+  %induction = add <4 x i32> %broadcast.splat, <i32 0, i32 1, i32 2, i32 3>
+  %7 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv24, i32 4, <4 x i1> %7, <4 x i32> undef)
+  %wide.masked.load13 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>* %lsr.iv1, i32 4, <4 x i1> %7, <4 x i32> undef)
+  %8 = mul nsw <4 x i32> %wide.masked.load13, %wide.masked.load
+  %9 = add nsw <4 x i32> %8, %vec.phi
+  %index.next = add i32 %index, 4
+  %scevgep = getelementptr i32, i32* %lsr.iv, i32 4
+  %scevgep3 = getelementptr i32, i32* %lsr.iv2, i32 4
+  %10 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %6, i32 1)
+  %11 = icmp ne i32 %10, 0
+  br i1 %11, label %vector.body, label %middle.block
+  
+middle.block:                                     ; preds = %vector.body
+  %12 = icmp ule <4 x i32> %induction, %broadcast.splat12
+  %13 = select <4 x i1> %12, <4 x i32> %9, <4 x i32> %vec.phi
+  %14 = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> %13)
+  br label %for.cond.cleanup
+  
+for.cond.cleanup:                                 ; preds = %middle.block, %entry
+  %res.0.lcssa = phi i32 [ 0, %entry ], [ %14, %middle.block ]
+  ret i32 %res.0.lcssa
+}
+  
+declare <4 x i32> @llvm.masked.load.v4i32.p0v4i32(<4 x i32>*, i32 immarg, <4 x i1>, <4 x i32>)
+declare i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32>)
+declare void @llvm.set.loop.iterations.i32(i32)
+declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32)
+  
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
index 9195b9772241e..cfafa4277311c 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.mir
@@ -3,8 +3,6 @@
 # CHECK-NOT: WhileLoopStart
 
 --- |
-  ; ModuleID = '/home/sampar01/src/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while-negative-offset.ll'
-  source_filename = "while-size-limit.ll"
   target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
   target triple = "thumbv8.1m.main"
   
@@ -47,8 +45,10 @@
   
   ; Function Attrs: nounwind
   declare i32 @llvm.arm.space(i32 immarg, i32) #1
+  
   ; Function Attrs: noduplicate nounwind
   declare i1 @llvm.test.set.loop.iterations.i32(i32) #2
+  
   ; Function Attrs: noduplicate nounwind
   declare i32 @llvm.loop.decrement.reg.i32.i32.i32(i32, i32) #2
   
@@ -65,7 +65,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -130,6 +130,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.4(0x80000000)
+    liveins: $r0, $r1, $r2, $r3, $r4, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r4, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -192,5 +193,3 @@ body:             |
     tB %bb.2, 14, $noreg
 
 ...
-
-
diff --git a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
index e05adc6b4744a..b1559bd752b41 100644
--- a/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
+++ b/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.mir
@@ -10,8 +10,6 @@
 # CHECK:   $lr = t2LEUpdate renamable $lr
 
 --- |
-  ; ModuleID = '/home/sampar01/src/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.ll'
-  source_filename = "/home/sampar01/src/llvm/test/CodeGen/Thumb2/LowOverheadLoops/while.ll"
   target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
   target triple = "thumbv8.1m.main"
   
@@ -25,14 +23,14 @@
     %scevgep3 = getelementptr i16, i16* %b, i32 -1
     br label %while.body
   
-  while.body:                                       ; preds = %while.body.preheader, %while.body
+  while.body:                                       ; preds = %while.body, %while.body.preheader
     %lsr.iv4 = phi i16* [ %scevgep3, %while.body.preheader ], [ %scevgep5, %while.body ]
     %lsr.iv = phi i16* [ %scevgep, %while.body.preheader ], [ %scevgep1, %while.body ]
     %1 = phi i32 [ %3, %while.body ], [ %N, %while.body.preheader ]
-    %scevgep2 = getelementptr i16, i16* %lsr.iv, i32 1
-    %scevgep6 = getelementptr i16, i16* %lsr.iv4, i32 1
-    %2 = load i16, i16* %scevgep6, align 2, !tbaa !2
-    store i16 %2, i16* %scevgep2, align 2, !tbaa !2
+    %scevgep7 = getelementptr i16, i16* %lsr.iv, i32 1
+    %scevgep4 = getelementptr i16, i16* %lsr.iv4, i32 1
+    %2 = load i16, i16* %scevgep4, align 2
+    store i16 %2, i16* %scevgep7, align 2
     %3 = call i32 @llvm.loop.decrement.reg.i32.i32.i32(i32 %1, i32 1)
     %4 = icmp ne i32 %3, 0
     %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
@@ -48,15 +46,6 @@
   
   attributes #0 = { noduplicate nounwind }
   attributes #1 = { nounwind }
-  
-  !llvm.module.flags = !{!0, !1}
-  
-  !0 = !{i32 1, !"wchar_size", i32 4}
-  !1 = !{i32 1, !"min_enum_size", i32 4}
-  !2 = !{!3, !3, i64 0}
-  !3 = !{!"short", !4, i64 0}
-  !4 = !{!"omnipotent char", !5, i64 0}
-  !5 = !{!"Simple C/C++ TBAA"}
 
 ...
 ---
@@ -67,7 +56,7 @@ legalized:       false
 regBankSelected: false
 selected:        false
 failedISel:      false
-tracksRegLiveness: false
+tracksRegLiveness: true
 hasWinCFI:       false
 registers:       []
 liveins:
@@ -107,6 +96,7 @@ machineFunctionInfo: {}
 body:             |
   bb.0.entry:
     successors: %bb.1(0x40000000), %bb.3(0x40000000)
+    liveins: $r0, $r1, $r2, $r7, $lr
   
     frame-setup tPUSH 14, $noreg, killed $r7, killed $lr, implicit-def $sp, implicit $sp
     frame-setup CFI_INSTRUCTION def_cfa_offset 8
@@ -117,6 +107,7 @@ body:             |
   
   bb.1.while.body.preheader:
     successors: %bb.2(0x80000000)
+    liveins: $r0, $r1, $r2
   
     renamable $r1, dead $cpsr = tSUBi8 killed renamable $r1, 2, 14, $noreg
     renamable $r0, dead $cpsr = tSUBi8 killed renamable $r0, 2, 14, $noreg
@@ -124,9 +115,10 @@ body:             |
   
   bb.2.while.body:
     successors: %bb.2(0x7c000000), %bb.3(0x04000000)
+    liveins: $lr, $r0, $r1
   
-    renamable $r2, renamable $r1 = t2LDRH_PRE killed renamable $r1, 2, 14, $noreg :: (load 2 from %ir.scevgep6, !tbaa !2)
-    early-clobber renamable $r0 = t2STRH_PRE killed renamable $r2, killed renamable $r0, 2, 14, $noreg :: (store 2 into %ir.scevgep2, !tbaa !2)
+    renamable $r2, renamable $r1 = t2LDRH_PRE killed renamable $r1, 2, 14, $noreg :: (load 2 from %ir.scevgep4)
+    early-clobber renamable $r0 = t2STRH_PRE killed renamable $r2, killed renamable $r0, 2, 14, $noreg :: (store 2 into %ir.scevgep7)
     renamable $lr = t2LoopDec killed renamable $lr, 1
     t2LoopEnd renamable $lr, %bb.2, implicit-def dead $cpsr
     tB %bb.3, 14, $noreg
@@ -135,4 +127,3 @@ body:             |
     tPOP_RET 14, $noreg, def $r7, def $pc
 
 ...
-
diff --git a/llvm/test/CodeGen/Thumb2/constant-islands.ll b/llvm/test/CodeGen/Thumb2/constant-islands.ll
index 81a1f357e67df..ac323adfd3f1d 100644
--- a/llvm/test/CodeGen/Thumb2/constant-islands.ll
+++ b/llvm/test/CodeGen/Thumb2/constant-islands.ll
@@ -1,4 +1,4 @@
-; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O0 -filetype=obj -o %t.o
+; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O0 -filetype=obj -verify-machine-dom-info -o %t.o
 ; RUN: llc < %s -mtriple=thumb-apple-ios -mcpu=cortex-a8 -O0 -filetype=obj -o %t.o
 ; RUN: llc < %s -mtriple=arm-apple-ios   -mcpu=cortex-a8 -O2 -filetype=obj -verify-machineinstrs -o %t.o
 ; RUN: llc < %s -mtriple=thumb-apple-ios -mcpu=cortex-a8 -O2 -filetype=obj -verify-machineinstrs -o %t.o
diff --git a/llvm/test/CodeGen/Thumb2/float-ops.ll b/llvm/test/CodeGen/Thumb2/float-ops.ll
index cbd8ca557a062..cc3684c2ec0e1 100644
--- a/llvm/test/CodeGen/Thumb2/float-ops.ll
+++ b/llvm/test/CodeGen/Thumb2/float-ops.ll
@@ -130,7 +130,7 @@ define void @store_d(double* %a, double %b) {
 entry:
 ; CHECK-LABEL: store_d:
 ; NOREGS: strd r2, r3, [r0]
-; ONLYREGS: vstr d0, [r0]
+; ONLYREGS: strd r2, r3, [r0]
 ; HARD: vstr d0, [r0]
   store double %b, double* %a, align 8
   ret void
diff --git a/llvm/test/CodeGen/Thumb2/fp16-stacksplot.mir b/llvm/test/CodeGen/Thumb2/fp16-stacksplot.mir
new file mode 100644
index 0000000000000..856f307488cf3
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/fp16-stacksplot.mir
@@ -0,0 +1,95 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+fullfp16 -run-pass=stack-protector -run-pass=prologepilog | FileCheck %s
+---
+name: func0
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 2, alignment: 2,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -1200, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 1200, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -2, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: func0
+    ; CHECK: liveins: $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11, $lr
+    ; CHECK: $sp = frame-setup t2STMDB_UPD $sp, 14, $noreg, killed $r4, killed $r5, killed $r6, killed $r7, killed $r8, killed $r9, killed $r10, killed $r11, killed $lr
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 36
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r11, -8
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r10, -12
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r9, -16
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r8, -20
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r7, -24
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r6, -28
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r5, -32
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -36
+    ; CHECK: $sp = frame-setup t2SUBri killed $sp, 1208, 14, $noreg, $noreg
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 1244
+    ; CHECK: $r0 = IMPLICIT_DEF
+    ; CHECK: $r1 = IMPLICIT_DEF
+    ; CHECK: $r2 = IMPLICIT_DEF
+    ; CHECK: $r3 = IMPLICIT_DEF
+    ; CHECK: $r4 = IMPLICIT_DEF
+    ; CHECK: $r5 = IMPLICIT_DEF
+    ; CHECK: $r6 = IMPLICIT_DEF
+    ; CHECK: $r7 = IMPLICIT_DEF
+    ; CHECK: $r8 = IMPLICIT_DEF
+    ; CHECK: $r9 = IMPLICIT_DEF
+    ; CHECK: $r10 = IMPLICIT_DEF
+    ; CHECK: $r11 = IMPLICIT_DEF
+    ; CHECK: $r12 = IMPLICIT_DEF
+    ; CHECK: $lr = IMPLICIT_DEF
+    ; CHECK: t2STRi12 killed $r0, $sp, 0, 14, $noreg :: (store 4 into %stack.2)
+    ; CHECK: $r0 = t2ADDri killed $sp, 1024, 14, $noreg, $noreg
+    ; CHECK: renamable $s4 = VLDRH killed $r0, 91, 14, $noreg :: (dereferenceable load 2 from %stack.0)
+    ; CHECK: $r0 = t2LDRi12 $sp, 0, 14, $noreg :: (load 4 from %stack.2)
+    ; CHECK: KILL $r0
+    ; CHECK: KILL $r1
+    ; CHECK: KILL $r2
+    ; CHECK: KILL $r3
+    ; CHECK: KILL $r4
+    ; CHECK: KILL $r5
+    ; CHECK: KILL $r6
+    ; CHECK: KILL $r7
+    ; CHECK: KILL $r8
+    ; CHECK: KILL $r9
+    ; CHECK: KILL $r10
+    ; CHECK: KILL $r11
+    ; CHECK: KILL $r12
+    ; CHECK: KILL $lr
+    $r0 = IMPLICIT_DEF
+    $r1 = IMPLICIT_DEF
+    $r2 = IMPLICIT_DEF
+    $r3 = IMPLICIT_DEF
+    $r4 = IMPLICIT_DEF
+    $r5 = IMPLICIT_DEF
+    $r6 = IMPLICIT_DEF
+    $r7 = IMPLICIT_DEF
+    $r8 = IMPLICIT_DEF
+    $r9 = IMPLICIT_DEF
+    $r10 = IMPLICIT_DEF
+    $r11 = IMPLICIT_DEF
+    $r12 = IMPLICIT_DEF
+    $lr = IMPLICIT_DEF
+
+    renamable $s4 = VLDRH %stack.0, 0, 14, $noreg :: (dereferenceable load 2 from %stack.0)
+
+    KILL $r0
+    KILL $r1
+    KILL $r2
+    KILL $r3
+    KILL $r4
+    KILL $r5
+    KILL $r6
+    KILL $r7
+    KILL $r8
+    KILL $r9
+    KILL $r10
+    KILL $r11
+    KILL $r12
+    KILL $lr
+...
diff --git a/llvm/test/CodeGen/Thumb2/lsll0.ll b/llvm/test/CodeGen/Thumb2/lsll0.ll
new file mode 100644
index 0000000000000..f11e6696143be
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/lsll0.ll
@@ -0,0 +1,48 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s
+
+define void @_Z4loopPxS_iS_i(i64* %d) {
+; CHECK-LABEL: _Z4loopPxS_iS_i:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r0]
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    vmov r2, s0
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    rsbs r2, r2, #0
+; CHECK-NEXT:    sxth r1, r1
+; CHECK-NEXT:    sxth r2, r2
+; CHECK-NEXT:    asr.w r12, r1, #31
+; CHECK-NEXT:    asrs r3, r2, #31
+; CHECK-NEXT:    strd r2, r3, [r0]
+; CHECK-NEXT:    strd r1, r12, [r0, #8]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <2 x i64>, <2 x i64>* undef, align 8
+  %0 = trunc <2 x i64> %wide.load to <2 x i32>
+  %1 = shl <2 x i32> %0, <i32 16, i32 16>
+  %2 = ashr exact <2 x i32> %1, <i32 16, i32 16>
+  %3 = sub <2 x i32> %2, %0
+  %4 = and <2 x i32> %3, <i32 7, i32 7>
+  %5 = shl <2 x i32> %2, %4
+  %6 = extractelement <2 x i32> %5, i32 0
+  %7 = zext i32 %6 to i64
+  %8 = select i1 false, i64 %7, i64 undef
+  %9 = trunc i64 %8 to i16
+  %10 = sub i16 0, %9
+  %11 = sext i16 %10 to i64
+  %12 = getelementptr inbounds i64, i64* %d, i64 undef
+  store i64 %11, i64* %12, align 8
+  %13 = extractelement <2 x i32> %5, i32 1
+  %14 = zext i32 %13 to i64
+  %15 = select i1 false, i64 %14, i64 undef
+  %16 = trunc i64 %15 to i16
+  %17 = sub i16 0, %16
+  %18 = sext i16 %17 to i64
+  %19 = or i32 0, 1
+  %20 = sext i32 %19 to i64
+  %21 = getelementptr inbounds i64, i64* %d, i64 %20
+  store i64 %18, i64* %21, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll b/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll
new file mode 100644
index 0000000000000..99f9506c3a9d2
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-bitreverse.ll
@@ -0,0 +1,52 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @brv_2i64_t(<2 x i64> %src){
+; CHECK-LABEL: brv_2i64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev64.8 q1, q0
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q1, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <2 x i64> @llvm.bitreverse.v2i64(<2 x i64> %src)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @brv_4i32_t(<4 x i32> %src){
+; CHECK-LABEL: brv_4i32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.bitreverse.v4i32(<4 x i32> %src)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @brv_8i16_t(<8 x i16> %src){
+; CHECK-LABEL: brv_8i16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.bitreverse.v8i16(<8 x i16> %src)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @brv_16i8_t(<16 x i8> %src){
+; CHECK-LABEL: brv_16i8_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.bitreverse.v16i8(<16 x i8> %src)
+  ret <16 x i8> %0
+}
+
+declare <2 x i64> @llvm.bitreverse.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.bitreverse.v8i16(<8 x i16>)
+declare <16 x i8> @llvm.bitreverse.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-bswap.ll b/llvm/test/CodeGen/Thumb2/mve-bswap.ll
new file mode 100644
index 0000000000000..c7d3f844fd7e9
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-bswap.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <4 x i32> @bswap_4i32_t(<4 x i32> %src){
+; CHECK-LABEL: bswap_4i32_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <4 x i32> @llvm.bswap.v4i32(<4 x i32> %src)
+    ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @bswap_8i16_t(<8 x i16> %src){
+; CHECK-LABEL: bswap_8i16_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <8 x i16> @llvm.bswap.v8i16(<8 x i16> %src)
+    ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @bswap_2i64_t(<2 x i64> %src){
+; CHECK-LABEL: bswap_2i64_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev64.8 q1, q0
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+entry:
+    %0 = call <2 x i64> @llvm.bswap.v2i64(<2 x i64> %src)
+    ret <2 x i64> %0
+}
+
+declare <2 x i64> @llvm.bswap.v2i64(<2 x i64>)
+declare <4 x i32> @llvm.bswap.v4i32(<4 x i32>)
+declare <8 x i16> @llvm.bswap.v8i16(<8 x i16>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-ctlz.ll b/llvm/test/CodeGen/Thumb2/mve-ctlz.ll
new file mode 100644
index 0000000000000..0645fadd2c6e4
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-ctlz.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @ctlz_2i64_0_t(<2 x i64> %src){
+; CHECK-LABEL: ctlz_2i64_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI0_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %src, i1 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ctlz_4i32_0_t(<4 x i32> %src){
+; CHECK-LABEL: ctlz_4i32_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %src, i1 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ctlz_8i16_0_t(<8 x i16> %src){
+; CHECK-LABEL: ctlz_8i16_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %src, i1 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ctlz_16i8_0_t(<16 x i8> %src){
+; CHECK-LABEL: ctlz_16i8_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %src, i1 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @ctlz_2i64_1_t(<2 x i64> %src){
+; CHECK-LABEL: ctlz_2i64_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s3
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s2
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s1
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s0
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI4_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %src, i1 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @ctlz_4i32_1_t(<4 x i32> %src){
+; CHECK-LABEL: ctlz_4i32_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %src, i1 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @ctlz_8i16_1_t(<8 x i16> %src){
+; CHECK-LABEL: ctlz_8i16_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %src, i1 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @ctlz_16i8_1_t(<16 x i8> %src){
+; CHECK-LABEL: ctlz_16i8_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %src, i1 1)
+  ret <16 x i8> %0
+}
+
+
+declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1)
diff --git a/llvm/test/CodeGen/Thumb2/mve-cttz.ll b/llvm/test/CodeGen/Thumb2/mve-cttz.ll
new file mode 100644
index 0000000000000..d1034fb2aac9a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-cttz.ll
@@ -0,0 +1,160 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -verify-machineinstrs -mattr=+mve %s -o - | FileCheck %s
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_0_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI0_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI0_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 0)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_0_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 0)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_0_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 0)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_0_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_0_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 0)
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <2 x i64> @cttz_2i64_1_t(<2 x i64> %src){
+; CHECK-LABEL: cttz_2i64_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov r0, s2
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s3
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov r0, s0
+; CHECK-NEXT:    vmov s6, r1
+; CHECK-NEXT:    cmp r0, #0
+; CHECK-NEXT:    rbit r0, r0
+; CHECK-NEXT:    cset r1, ne
+; CHECK-NEXT:    lsls r1, r1, #31
+; CHECK-NEXT:    vmov r1, s1
+; CHECK-NEXT:    rbit r1, r1
+; CHECK-NEXT:    clz r1, r1
+; CHECK-NEXT:    add.w r1, r1, #32
+; CHECK-NEXT:    it ne
+; CHECK-NEXT:    clzne r1, r0
+; CHECK-NEXT:    vmov s4, r1
+; CHECK-NEXT:    vldr s5, .LCPI4_0
+; CHECK-NEXT:    vmov.f32 s7, s5
+; CHECK-NEXT:    vmov q0, q1
+; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    .p2align 2
+; CHECK-NEXT:  @ %bb.1:
+; CHECK-NEXT:  .LCPI4_0:
+; CHECK-NEXT:    .long 0 @ float 0
+entry:
+  %0 = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %src, i1 1)
+  ret <2 x i64> %0
+}
+
+define arm_aapcs_vfpcc <4 x i32> @cttz_4i32_1_t(<4 x i32> %src){
+; CHECK-LABEL: cttz_4i32_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #32
+; CHECK-NEXT:    vbrsr.32 q0, q0, r0
+; CHECK-NEXT:    vclz.i32 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %src, i1 1)
+  ret <4 x i32> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @cttz_8i16_1_t(<8 x i16> %src){
+; CHECK-LABEL: cttz_8i16_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #16
+; CHECK-NEXT:    vbrsr.16 q0, q0, r0
+; CHECK-NEXT:    vclz.i16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %src, i1 1)
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @cttz_16i8_1_t(<16 x i8> %src) {
+; CHECK-LABEL: cttz_16i8_1_t:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    movs r0, #8
+; CHECK-NEXT:    vbrsr.8 q0, q0, r0
+; CHECK-NEXT:    vclz.i8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %src, i1 1)
+  ret <16 x i8> %0
+}
+
+
+declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
+declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
+declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
+declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
index f7d6a3f3799ab..4302b9a8e2a64 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-ldst.ll
@@ -1,78 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
-; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 define void @foo_v4i32_v4i32(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
 ; CHECK-LABEL: foo_v4i32_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
-; CHECK-NEXT:    lsls r3, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrne r3, [r2]
-; CHECK-NEXT:    vmovne.32 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.32 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #8]
-; CHECK-NEXT:    vmovmi.32 q0[2], r3
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r1, [r2, #12]
-; CHECK-NEXT:    vmovmi.32 q0[3], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
@@ -85,26 +21,27 @@ entry:
 define void @foo_sext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
 ; CHECK-LABEL: foo_sext_v4i32_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
@@ -121,41 +58,12 @@ define void @foo_sext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #3]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    vmovlb.s8 q0, q0
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
@@ -168,26 +76,27 @@ entry:
 define void @foo_sext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
 ; CHECK-LABEL: foo_sext_v4i32_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrhne r3, [r2]
@@ -204,40 +113,11 @@ define void @foo_sext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16>
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r1, [r2, #6]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    vmovlb.s16 q0, q0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
@@ -250,27 +130,28 @@ entry:
 define void @foo_zext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%src) {
 ; CHECK-LABEL: foo_zext_v4i32_v4i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vmov.i32 q1, #0xff
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrbne r3, [r2]
@@ -287,40 +168,11 @@ define void @foo_zext_v4i32_v4i8(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i8> *%
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrbmi r1, [r2, #3]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    vand q0, q0, q1
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
@@ -333,26 +185,27 @@ entry:
 define void @foo_zext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16> *%src) {
 ; CHECK-LABEL: foo_zext_v4i32_v4i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s32 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r1, lr, #1
+; CHECK-NEXT:    ubfx r3, lr, #4, #1
+; CHECK-NEXT:    rsb.w r12, r1, #0
+; CHECK-NEXT:    movs r1, #0
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r12, #0, #1
+; CHECK-NEXT:    bfi r1, r3, #1, #1
+; CHECK-NEXT:    ubfx r3, lr, #8, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #2, #1
+; CHECK-NEXT:    ubfx r3, lr, #12, #1
+; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    bfi r1, r3, #3, #1
 ; CHECK-NEXT:    lsls r3, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    ldrhne r3, [r2]
@@ -369,40 +222,11 @@ define void @foo_zext_v4i32_v4i16(<4 x i32> *%dest, <4 x i32> *%mask, <4 x i16>
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    ldrhmi r1, [r2, #6]
 ; CHECK-NEXT:    vmovmi.32 q0[3], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
 ; CHECK-NEXT:    vmovlb.u16 q0, q0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne r2, s0
-; CHECK-NEXT:    strne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s1
-; CHECK-NEXT:    strmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r2, s2
-; CHECK-NEXT:    strmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi r1, s3
-; CHECK-NEXT:    strmi r1, [r0, #12]
-; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    bx lr
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
+; CHECK-NEXT:    add sp, #4
+; CHECK-NEXT:    pop {r7, pc}
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
   %1 = icmp sgt <4 x i32> %0, zeroinitializer
@@ -415,644 +239,213 @@ entry:
 define void @foo_v8i16_v8i16(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
 ; CHECK-LABEL: foo_v8i16_v8i16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %src, i32 2, <8 x i1> %1, <8 x i16> undef)
+  call void @llvm.masked.store.v8i16(<8 x i16> %2, <8 x i16>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_sext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
+; CHECK-LABEL: foo_sext_v8i16_v8i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
 ; CHECK-NEXT:    vcmp.s16 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    ubfx r1, lr, #2, #1
+; CHECK-NEXT:    rsb.w r12, r3, #0
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi r3, r12, #0, #1
 ; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
+; CHECK-NEXT:    ubfx r1, lr, #4, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
+; CHECK-NEXT:    ubfx r1, lr, #6, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
+; CHECK-NEXT:    ubfx r1, lr, #8, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
+; CHECK-NEXT:    ubfx r1, lr, #10, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
+; CHECK-NEXT:    ubfx r1, lr, #12, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r3
-; CHECK-NEXT:    lsls r3, r3, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrhne r3, [r2]
-; CHECK-NEXT:    vmovne.16 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #2]
-; CHECK-NEXT:    vmovmi.16 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.16 q0[2], r3
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #6]
-; CHECK-NEXT:    vmovmi.16 q0[3], r3
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #8]
-; CHECK-NEXT:    vmovmi.16 q0[4], r3
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #10]
-; CHECK-NEXT:    vmovmi.16 q0[5], r3
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #12]
-; CHECK-NEXT:    vmovmi.16 q0[6], r3
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r1, [r2, #14]
-; CHECK-NEXT:    vmovmi.16 q0[7], r1
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    and r3, r1, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #2, #1
-; CHECK-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #3, #1
-; CHECK-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #4, #1
-; CHECK-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #5, #1
-; CHECK-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r2, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r2
-; CHECK-NEXT:    lsls r2, r2, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-NEXT:    strhne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-NEXT:    strhmi r2, [r0, #2]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-NEXT:    strhmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-NEXT:    strhmi r2, [r0, #6]
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-NEXT:    strhmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-NEXT:    strhmi r2, [r0, #10]
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-NEXT:    strhmi r2, [r0, #12]
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-NEXT:    strhmi r1, [r0, #14]
-; CHECK-NEXT:    add sp, #16
-; CHECK-NEXT:    bx lr
-entry:
-  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
-  %1 = icmp sgt <8 x i16> %0, zeroinitializer
-  %2 = call <8 x i16> @llvm.masked.load.v8i16(<8 x i16>* %src, i32 2, <8 x i1> %1, <8 x i16> undef)
-  call void @llvm.masked.store.v8i16(<8 x i16> %2, <8 x i16>* %dest, i32 2, <8 x i1> %1)
-  ret void
-}
-
-define void @foo_sext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
-; CHECK-LABEL: foo_sext_v8i16_v8i8:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r3
-; CHECK-NEXT:    lsls r3, r3, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrbne r3, [r2]
-; CHECK-NEXT:    vmovne.16 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #1]
-; CHECK-NEXT:    vmovmi.16 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #2]
-; CHECK-NEXT:    vmovmi.16 q0[2], r3
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #3]
-; CHECK-NEXT:    vmovmi.16 q0[3], r3
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.16 q0[4], r3
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #5]
-; CHECK-NEXT:    vmovmi.16 q0[5], r3
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #6]
-; CHECK-NEXT:    vmovmi.16 q0[6], r3
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r1, [r2, #7]
-; CHECK-NEXT:    vmovmi.16 q0[7], r1
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    vmovlb.s8 q0, q0
-; CHECK-NEXT:    and r3, r1, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #2, #1
-; CHECK-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #3, #1
-; CHECK-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #4, #1
-; CHECK-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #5, #1
-; CHECK-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r2, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r2
-; CHECK-NEXT:    lsls r2, r2, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-NEXT:    strhne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-NEXT:    strhmi r2, [r0, #2]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-NEXT:    strhmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-NEXT:    strhmi r2, [r0, #6]
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-NEXT:    strhmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-NEXT:    strhmi r2, [r0, #10]
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-NEXT:    strhmi r2, [r0, #12]
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-NEXT:    strhmi r1, [r0, #14]
-; CHECK-NEXT:    add sp, #16
-; CHECK-NEXT:    bx lr
-entry:
-  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
-  %1 = icmp sgt <8 x i16> %0, zeroinitializer
-  %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
-  %3 = sext <8 x i8> %2 to <8 x i16>
-  call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
-  ret void
-}
-
-define void @foo_zext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
-; CHECK-LABEL: foo_zext_v8i16_v8i8:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
-; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r3
-; CHECK-NEXT:    lsls r3, r3, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrbne r3, [r2]
-; CHECK-NEXT:    vmovne.16 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #1]
-; CHECK-NEXT:    vmovmi.16 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #2]
-; CHECK-NEXT:    vmovmi.16 q0[2], r3
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #3]
-; CHECK-NEXT:    vmovmi.16 q0[3], r3
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.16 q0[4], r3
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #5]
-; CHECK-NEXT:    vmovmi.16 q0[5], r3
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #6]
-; CHECK-NEXT:    vmovmi.16 q0[6], r3
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r1, [r2, #7]
-; CHECK-NEXT:    vmovmi.16 q0[7], r1
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    vmovlb.u8 q0, q0
-; CHECK-NEXT:    and r3, r1, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #2, #1
-; CHECK-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #3, #1
-; CHECK-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #4, #1
-; CHECK-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #5, #1
-; CHECK-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r2, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r2
-; CHECK-NEXT:    lsls r2, r2, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-NEXT:    strhne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-NEXT:    strhmi r2, [r0, #2]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-NEXT:    strhmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-NEXT:    strhmi r2, [r0, #6]
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-NEXT:    strhmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-NEXT:    strhmi r2, [r0, #10]
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-NEXT:    strhmi r2, [r0, #12]
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-NEXT:    strhmi r1, [r0, #14]
-; CHECK-NEXT:    add sp, #16
-; CHECK-NEXT:    bx lr
-entry:
-  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
-  %1 = icmp sgt <8 x i16> %0, zeroinitializer
-  %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
-  %3 = zext <8 x i8> %2 to <8 x i16>
-  call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
-  ret void
-}
-
-define void @foo_v16i8_v16i8(<16 x i8> *%dest, <16 x i8> *%mask, <16 x i8> *%src) {
-; CHECK-LABEL: foo_v16i8_v16i8:
-; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-NEXT:    push {r4, r6, r7, lr}
-; CHECK-NEXT:    .setfp r7, sp, #8
-; CHECK-NEXT:    add r7, sp, #8
-; CHECK-NEXT:    .pad #32
-; CHECK-NEXT:    sub sp, #32
-; CHECK-NEXT:    mov r4, sp
-; CHECK-NEXT:    bfc r4, #0, #4
-; CHECK-NEXT:    mov sp, r4
-; CHECK-NEXT:    vldrb.u8 q0, [r1]
-; CHECK-NEXT:    sub.w r4, r7, #8
-; CHECK-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r3, p0
-; CHECK-NEXT:    uxth r1, r3
-; CHECK-NEXT:    lsls r3, r3, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrbne r3, [r2]
-; CHECK-NEXT:    vmovne.8 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #1]
-; CHECK-NEXT:    vmovmi.8 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #2]
-; CHECK-NEXT:    vmovmi.8 q0[2], r3
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #3]
-; CHECK-NEXT:    vmovmi.8 q0[3], r3
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.8 q0[4], r3
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #5]
-; CHECK-NEXT:    vmovmi.8 q0[5], r3
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #6]
-; CHECK-NEXT:    vmovmi.8 q0[6], r3
-; CHECK-NEXT:    lsls r3, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #7]
-; CHECK-NEXT:    vmovmi.8 q0[7], r3
-; CHECK-NEXT:    lsls r3, r1, #23
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #8]
-; CHECK-NEXT:    vmovmi.8 q0[8], r3
-; CHECK-NEXT:    lsls r3, r1, #22
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #9]
-; CHECK-NEXT:    vmovmi.8 q0[9], r3
-; CHECK-NEXT:    lsls r3, r1, #21
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #10]
-; CHECK-NEXT:    vmovmi.8 q0[10], r3
-; CHECK-NEXT:    lsls r3, r1, #20
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #11]
-; CHECK-NEXT:    vmovmi.8 q0[11], r3
-; CHECK-NEXT:    lsls r3, r1, #19
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #12]
-; CHECK-NEXT:    vmovmi.8 q0[12], r3
-; CHECK-NEXT:    lsls r3, r1, #18
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #13]
-; CHECK-NEXT:    vmovmi.8 q0[13], r3
-; CHECK-NEXT:    lsls r3, r1, #17
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r3, [r2, #14]
-; CHECK-NEXT:    vmovmi.8 q0[14], r3
-; CHECK-NEXT:    lsls r1, r1, #16
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrbmi r1, [r2, #15]
-; CHECK-NEXT:    vmovmi.8 q0[15], r1
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    uxth r1, r2
-; CHECK-NEXT:    lsls r2, r2, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    vmovne.u8 r2, q0[0]
-; CHECK-NEXT:    strbne r2, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[1]
-; CHECK-NEXT:    strbmi r2, [r0, #1]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[2]
-; CHECK-NEXT:    strbmi r2, [r0, #2]
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[3]
-; CHECK-NEXT:    strbmi r2, [r0, #3]
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[4]
-; CHECK-NEXT:    strbmi r2, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[5]
-; CHECK-NEXT:    strbmi r2, [r0, #5]
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[6]
-; CHECK-NEXT:    strbmi r2, [r0, #6]
-; CHECK-NEXT:    lsls r2, r1, #24
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[7]
-; CHECK-NEXT:    strbmi r2, [r0, #7]
-; CHECK-NEXT:    lsls r2, r1, #23
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[8]
-; CHECK-NEXT:    strbmi r2, [r0, #8]
-; CHECK-NEXT:    lsls r2, r1, #22
+; CHECK-NEXT:    bfi r3, r1, #6, #1
+; CHECK-NEXT:    ubfx r1, lr, #14, #1
+; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi r3, r1, #7, #1
+; CHECK-NEXT:    uxtb r1, r3
+; CHECK-NEXT:    lsls r3, r3, #31
+; CHECK-NEXT:    itt ne
+; CHECK-NEXT:    ldrbne r3, [r2]
+; CHECK-NEXT:    vmovne.16 q0[0], r3
+; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[9]
-; CHECK-NEXT:    strbmi r2, [r0, #9]
-; CHECK-NEXT:    lsls r2, r1, #21
+; CHECK-NEXT:    ldrbmi r3, [r2, #1]
+; CHECK-NEXT:    vmovmi.16 q0[1], r3
+; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[10]
-; CHECK-NEXT:    strbmi r2, [r0, #10]
-; CHECK-NEXT:    lsls r2, r1, #20
+; CHECK-NEXT:    ldrbmi r3, [r2, #2]
+; CHECK-NEXT:    vmovmi.16 q0[2], r3
+; CHECK-NEXT:    lsls r3, r1, #28
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[11]
-; CHECK-NEXT:    strbmi r2, [r0, #11]
-; CHECK-NEXT:    lsls r2, r1, #19
+; CHECK-NEXT:    ldrbmi r3, [r2, #3]
+; CHECK-NEXT:    vmovmi.16 q0[3], r3
+; CHECK-NEXT:    lsls r3, r1, #27
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[12]
-; CHECK-NEXT:    strbmi r2, [r0, #12]
-; CHECK-NEXT:    lsls r2, r1, #18
+; CHECK-NEXT:    ldrbmi r3, [r2, #4]
+; CHECK-NEXT:    vmovmi.16 q0[4], r3
+; CHECK-NEXT:    lsls r3, r1, #26
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[13]
-; CHECK-NEXT:    strbmi r2, [r0, #13]
-; CHECK-NEXT:    lsls r2, r1, #17
+; CHECK-NEXT:    ldrbmi r3, [r2, #5]
+; CHECK-NEXT:    vmovmi.16 q0[5], r3
+; CHECK-NEXT:    lsls r3, r1, #25
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r2, q0[14]
-; CHECK-NEXT:    strbmi r2, [r0, #14]
-; CHECK-NEXT:    lsls r1, r1, #16
+; CHECK-NEXT:    ldrbmi r3, [r2, #6]
+; CHECK-NEXT:    vmovmi.16 q0[6], r3
+; CHECK-NEXT:    lsls r1, r1, #24
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    vmovmi.u8 r1, q0[15]
-; CHECK-NEXT:    strbmi r1, [r0, #15]
-; CHECK-NEXT:    mov sp, r4
-; CHECK-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-NEXT:    ldrbmi r1, [r2, #7]
+; CHECK-NEXT:    vmovmi.16 q0[7], r1
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
 entry:
-  %0 = load <16 x i8>, <16 x i8>* %mask, align 1
-  %1 = icmp sgt <16 x i8> %0, zeroinitializer
-  %2 = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %src, i32 1, <16 x i1> %1, <16 x i8> undef)
-  call void @llvm.masked.store.v16i8(<16 x i8> %2, <16 x i8>* %dest, i32 1, <16 x i1> %1)
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %3 = sext <8 x i8> %2 to <8 x i16>
+  call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
   ret void
 }
 
-define void @foo_trunc_v8i8_v8i16(<8 x i8> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
-; CHECK-LABEL: foo_trunc_v8i8_v8i16:
+define void @foo_zext_v8i16_v8i8(<8 x i16> *%dest, <8 x i16> *%mask, <8 x i8> *%src) {
+; CHECK-LABEL: foo_zext_v8i16_v8i8:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
+; CHECK-NEXT:    .save {r7, lr}
+; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    vcmp.s16 gt, q0, zr
 ; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
+; CHECK-NEXT:    vmrs lr, p0
+; CHECK-NEXT:    and r3, lr, #1
+; CHECK-NEXT:    ubfx r1, lr, #2, #1
+; CHECK-NEXT:    rsb.w r12, r3, #0
+; CHECK-NEXT:    movs r3, #0
 ; CHECK-NEXT:    rsbs r1, r1, #0
+; CHECK-NEXT:    bfi r3, r12, #0, #1
 ; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
+; CHECK-NEXT:    ubfx r1, lr, #4, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
+; CHECK-NEXT:    ubfx r1, lr, #6, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
+; CHECK-NEXT:    ubfx r1, lr, #8, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
+; CHECK-NEXT:    ubfx r1, lr, #10, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
+; CHECK-NEXT:    ubfx r1, lr, #12, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
+; CHECK-NEXT:    ubfx r1, lr, #14, #1
 ; CHECK-NEXT:    rsbs r1, r1, #0
 ; CHECK-NEXT:    bfi r3, r1, #7, #1
 ; CHECK-NEXT:    uxtb r1, r3
 ; CHECK-NEXT:    lsls r3, r3, #31
 ; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrhne r3, [r2]
+; CHECK-NEXT:    ldrbne r3, [r2]
 ; CHECK-NEXT:    vmovne.16 q0[0], r3
 ; CHECK-NEXT:    lsls r3, r1, #30
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #2]
+; CHECK-NEXT:    ldrbmi r3, [r2, #1]
 ; CHECK-NEXT:    vmovmi.16 q0[1], r3
 ; CHECK-NEXT:    lsls r3, r1, #29
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #4]
+; CHECK-NEXT:    ldrbmi r3, [r2, #2]
 ; CHECK-NEXT:    vmovmi.16 q0[2], r3
 ; CHECK-NEXT:    lsls r3, r1, #28
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #6]
+; CHECK-NEXT:    ldrbmi r3, [r2, #3]
 ; CHECK-NEXT:    vmovmi.16 q0[3], r3
 ; CHECK-NEXT:    lsls r3, r1, #27
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #8]
+; CHECK-NEXT:    ldrbmi r3, [r2, #4]
 ; CHECK-NEXT:    vmovmi.16 q0[4], r3
 ; CHECK-NEXT:    lsls r3, r1, #26
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #10]
+; CHECK-NEXT:    ldrbmi r3, [r2, #5]
 ; CHECK-NEXT:    vmovmi.16 q0[5], r3
 ; CHECK-NEXT:    lsls r3, r1, #25
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r3, [r2, #12]
+; CHECK-NEXT:    ldrbmi r3, [r2, #6]
 ; CHECK-NEXT:    vmovmi.16 q0[6], r3
 ; CHECK-NEXT:    lsls r1, r1, #24
 ; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrhmi r1, [r2, #14]
+; CHECK-NEXT:    ldrbmi r1, [r2, #7]
 ; CHECK-NEXT:    vmovmi.16 q0[7], r1
-; CHECK-NEXT:    movs r2, #0
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vstrht.16 q0, [r0]
+; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    pop {r7, pc}
+entry:
+  %0 = load <8 x i16>, <8 x i16>* %mask, align 2
+  %1 = icmp sgt <8 x i16> %0, zeroinitializer
+  %2 = call <8 x i8> @llvm.masked.load.v8i8(<8 x i8>* %src, i32 1, <8 x i1> %1, <8 x i8> undef)
+  %3 = zext <8 x i8> %2 to <8 x i16>
+  call void @llvm.masked.store.v8i16(<8 x i16> %3, <8 x i16>* %dest, i32 2, <8 x i1> %1)
+  ret void
+}
+
+define void @foo_v16i8_v16i8(<16 x i8> *%dest, <16 x i8> *%mask, <16 x i8> *%src) {
+; CHECK-LABEL: foo_v16i8_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u8 q0, [r1]
+; CHECK-NEXT:    vptt.s8 gt, q0, zr
+; CHECK-NEXT:    vldrbt.u8 q0, [r2]
+; CHECK-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = load <16 x i8>, <16 x i8>* %mask, align 1
+  %1 = icmp sgt <16 x i8> %0, zeroinitializer
+  %2 = call <16 x i8> @llvm.masked.load.v16i8(<16 x i8>* %src, i32 1, <16 x i1> %1, <16 x i8> undef)
+  call void @llvm.masked.store.v16i8(<16 x i8> %2, <16 x i8>* %dest, i32 1, <16 x i1> %1)
+  ret void
+}
+
+define void @foo_trunc_v8i8_v8i16(<8 x i8> *%dest, <8 x i16> *%mask, <8 x i16> *%src) {
+; CHECK-LABEL: foo_trunc_v8i8_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .pad #8
+; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
 ; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    and r3, r1, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
+; CHECK-NEXT:    and r2, r1, #1
+; CHECK-NEXT:    rsbs r3, r2, #0
+; CHECK-NEXT:    movs r2, #0
 ; CHECK-NEXT:    bfi r2, r3, #0, #1
 ; CHECK-NEXT:    ubfx r3, r1, #2, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
@@ -1108,7 +501,7 @@ define void @foo_trunc_v8i8_v8i16(<8 x i8> *%dest, <8 x i16> *%mask, <8 x i16> *
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi.u16 r1, q0[7]
 ; CHECK-NEXT:    strbmi r1, [r0, #7]
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <8 x i16>, <8 x i16>* %mask, align 2
@@ -1122,46 +515,15 @@ entry:
 define void @foo_trunc_v4i8_v4i32(<4 x i8> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
 ; CHECK-LABEL: foo_trunc_v4i8_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
-; CHECK-NEXT:    lsls r3, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrne r3, [r2]
-; CHECK-NEXT:    vmovne.32 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.32 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #8]
-; CHECK-NEXT:    vmovmi.32 q0[2], r3
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r1, [r2, #12]
-; CHECK-NEXT:    vmovmi.32 q0[3], r1
+; CHECK-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
 ; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r1, r2, #1
+; CHECK-NEXT:    rsbs r3, r1, #0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-NEXT:    ubfx r3, r2, #4, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
@@ -1172,7 +534,6 @@ define void @foo_trunc_v4i8_v4i32(<4 x i8> *%dest, <4 x i32> *%mask, <4 x i32> *
 ; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
 ; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
 ; CHECK-NEXT:    lsls r2, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    vmovne r2, s0
@@ -1189,7 +550,7 @@ define void @foo_trunc_v4i8_v4i32(<4 x i8> *%dest, <4 x i32> *%mask, <4 x i32> *
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi r1, s3
 ; CHECK-NEXT:    strbmi r1, [r0, #3]
-; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
@@ -1203,46 +564,15 @@ entry:
 define void @foo_trunc_v4i16_v4i32(<4 x i16> *%dest, <4 x i32> *%mask, <4 x i32> *%src) {
 ; CHECK-LABEL: foo_trunc_v4i16_v4i32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
+; CHECK-NEXT:    .pad #4
+; CHECK-NEXT:    sub sp, #4
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
-; CHECK-NEXT:    lsls r3, r1, #31
-; CHECK-NEXT:    itt ne
-; CHECK-NEXT:    ldrne r3, [r2]
-; CHECK-NEXT:    vmovne.32 q0[0], r3
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #4]
-; CHECK-NEXT:    vmovmi.32 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r3, [r2, #8]
-; CHECK-NEXT:    vmovmi.32 q0[2], r3
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    itt mi
-; CHECK-NEXT:    ldrmi r1, [r2, #12]
-; CHECK-NEXT:    vmovmi.32 q0[3], r1
+; CHECK-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
 ; CHECK-NEXT:    vmrs r2, p0
+; CHECK-NEXT:    and r1, r2, #1
+; CHECK-NEXT:    rsbs r3, r1, #0
 ; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
 ; CHECK-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-NEXT:    ubfx r3, r2, #4, #1
 ; CHECK-NEXT:    rsbs r3, r3, #0
@@ -1253,7 +583,6 @@ define void @foo_trunc_v4i16_v4i32(<4 x i16> *%dest, <4 x i32> *%mask, <4 x i32>
 ; CHECK-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-NEXT:    rsbs r2, r2, #0
 ; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
 ; CHECK-NEXT:    lsls r2, r1, #31
 ; CHECK-NEXT:    itt ne
 ; CHECK-NEXT:    vmovne r2, s0
@@ -1270,7 +599,7 @@ define void @foo_trunc_v4i16_v4i32(<4 x i16> *%dest, <4 x i32> *%mask, <4 x i32>
 ; CHECK-NEXT:    itt mi
 ; CHECK-NEXT:    vmovmi r1, s3
 ; CHECK-NEXT:    strhmi r1, [r0, #6]
-; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    add sp, #4
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
@@ -1284,66 +613,10 @@ entry:
 define void @foo_v4f32_v4f32(<4 x float> *%dest, <4 x i32> *%mask, <4 x float> *%src) {
 ; CHECK-LABEL: foo_v4f32_v4f32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #8
-; CHECK-NEXT:    sub sp, #8
 ; CHECK-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    and r1, r3, #15
-; CHECK-NEXT:    lsls r3, r1, #31
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    vldrne s0, [r2]
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vldrmi s1, [r2, #4]
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vldrmi s2, [r2, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vldrmi s3, [r2, #12]
-; CHECK-NEXT:    vmrs r2, p0
-; CHECK-NEXT:    movs r1, #0
-; CHECK-NEXT:    and r3, r2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r1, r3, #2, #1
-; CHECK-NEXT:    rsbs r2, r2, #0
-; CHECK-NEXT:    bfi r1, r2, #3, #1
-; CHECK-NEXT:    and r1, r1, #15
-; CHECK-NEXT:    lsls r2, r1, #31
-; CHECK-NEXT:    it ne
-; CHECK-NEXT:    vstrne s0, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-NEXT:    lsls r1, r1, #28
-; CHECK-NEXT:    it mi
-; CHECK-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-NEXT:    add sp, #8
+; CHECK-NEXT:    vptt.s32 gt, q0, zr
+; CHECK-NEXT:    vldrwt.u32 q0, [r2]
+; CHECK-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <4 x i32>, <4 x i32>* %mask, align 4
@@ -1356,195 +629,10 @@ entry:
 define void @foo_v8f16_v8f16(<8 x half> *%dest, <8 x i16> *%mask, <8 x half> *%src) {
 ; CHECK-LABEL: foo_v8f16_v8f16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .pad #16
-; CHECK-NEXT:    sub sp, #16
 ; CHECK-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-NEXT:    movs r3, #0
-; CHECK-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-NEXT:    @ implicit-def: $q0
-; CHECK-NEXT:    vmrs r12, p0
-; CHECK-NEXT:    and r1, r12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #0, #1
-; CHECK-NEXT:    ubfx r1, r12, #2, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #1, #1
-; CHECK-NEXT:    ubfx r1, r12, #4, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #2, #1
-; CHECK-NEXT:    ubfx r1, r12, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #3, #1
-; CHECK-NEXT:    ubfx r1, r12, #8, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #4, #1
-; CHECK-NEXT:    ubfx r1, r12, #10, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #5, #1
-; CHECK-NEXT:    ubfx r1, r12, #12, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #6, #1
-; CHECK-NEXT:    ubfx r1, r12, #14, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r3, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r3
-; CHECK-NEXT:    lsls r3, r3, #31
-; CHECK-NEXT:    bne .LBB13_18
-; CHECK-NEXT:  @ %bb.1: @ %else
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    bmi .LBB13_19
-; CHECK-NEXT:  .LBB13_2: @ %else2
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    bmi .LBB13_20
-; CHECK-NEXT:  .LBB13_3: @ %else5
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    bmi .LBB13_21
-; CHECK-NEXT:  .LBB13_4: @ %else8
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    bmi .LBB13_22
-; CHECK-NEXT:  .LBB13_5: @ %else11
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    bmi .LBB13_23
-; CHECK-NEXT:  .LBB13_6: @ %else14
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    bmi .LBB13_24
-; CHECK-NEXT:  .LBB13_7: @ %else17
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    bpl .LBB13_9
-; CHECK-NEXT:  .LBB13_8: @ %cond.load19
-; CHECK-NEXT:    vldr.16 s4, [r2, #14]
-; CHECK-NEXT:    vmov r1, s4
-; CHECK-NEXT:    vmov.16 q0[7], r1
-; CHECK-NEXT:  .LBB13_9: @ %else20
-; CHECK-NEXT:    vmrs r1, p0
-; CHECK-NEXT:    movs r2, #0
-; CHECK-NEXT:    and r3, r1, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #0, #1
-; CHECK-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #1, #1
-; CHECK-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #2, #1
-; CHECK-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #3, #1
-; CHECK-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #4, #1
-; CHECK-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #5, #1
-; CHECK-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-NEXT:    rsbs r3, r3, #0
-; CHECK-NEXT:    bfi r2, r3, #6, #1
-; CHECK-NEXT:    rsbs r1, r1, #0
-; CHECK-NEXT:    bfi r2, r1, #7, #1
-; CHECK-NEXT:    uxtb r1, r2
-; CHECK-NEXT:    lsls r2, r2, #31
-; CHECK-NEXT:    bne .LBB13_25
-; CHECK-NEXT:  @ %bb.10: @ %else23
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    bmi .LBB13_26
-; CHECK-NEXT:  .LBB13_11: @ %else25
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    bmi .LBB13_27
-; CHECK-NEXT:  .LBB13_12: @ %else27
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    bmi .LBB13_28
-; CHECK-NEXT:  .LBB13_13: @ %else29
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    bmi .LBB13_29
-; CHECK-NEXT:  .LBB13_14: @ %else31
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    bmi .LBB13_30
-; CHECK-NEXT:  .LBB13_15: @ %else33
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    bmi .LBB13_31
-; CHECK-NEXT:  .LBB13_16: @ %else35
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    bmi .LBB13_32
-; CHECK-NEXT:  .LBB13_17: @ %else37
-; CHECK-NEXT:    add sp, #16
-; CHECK-NEXT:    bx lr
-; CHECK-NEXT:  .LBB13_18: @ %cond.load
-; CHECK-NEXT:    vldr.16 s0, [r2]
-; CHECK-NEXT:    lsls r3, r1, #30
-; CHECK-NEXT:    bpl .LBB13_2
-; CHECK-NEXT:  .LBB13_19: @ %cond.load1
-; CHECK-NEXT:    vldr.16 s4, [r2, #2]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[1], r3
-; CHECK-NEXT:    lsls r3, r1, #29
-; CHECK-NEXT:    bpl .LBB13_3
-; CHECK-NEXT:  .LBB13_20: @ %cond.load4
-; CHECK-NEXT:    vldr.16 s4, [r2, #4]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[2], r3
-; CHECK-NEXT:    lsls r3, r1, #28
-; CHECK-NEXT:    bpl .LBB13_4
-; CHECK-NEXT:  .LBB13_21: @ %cond.load7
-; CHECK-NEXT:    vldr.16 s4, [r2, #6]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[3], r3
-; CHECK-NEXT:    lsls r3, r1, #27
-; CHECK-NEXT:    bpl .LBB13_5
-; CHECK-NEXT:  .LBB13_22: @ %cond.load10
-; CHECK-NEXT:    vldr.16 s4, [r2, #8]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[4], r3
-; CHECK-NEXT:    lsls r3, r1, #26
-; CHECK-NEXT:    bpl .LBB13_6
-; CHECK-NEXT:  .LBB13_23: @ %cond.load13
-; CHECK-NEXT:    vldr.16 s4, [r2, #10]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[5], r3
-; CHECK-NEXT:    lsls r3, r1, #25
-; CHECK-NEXT:    bpl.w .LBB13_7
-; CHECK-NEXT:  .LBB13_24: @ %cond.load16
-; CHECK-NEXT:    vldr.16 s4, [r2, #12]
-; CHECK-NEXT:    vmov r3, s4
-; CHECK-NEXT:    vmov.16 q0[6], r3
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    bmi.w .LBB13_8
-; CHECK-NEXT:    b .LBB13_9
-; CHECK-NEXT:  .LBB13_25: @ %cond.store
-; CHECK-NEXT:    vstr.16 s0, [r0]
-; CHECK-NEXT:    lsls r2, r1, #30
-; CHECK-NEXT:    bpl .LBB13_11
-; CHECK-NEXT:  .LBB13_26: @ %cond.store24
-; CHECK-NEXT:    vmovx.f16 s4, s0
-; CHECK-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-NEXT:    lsls r2, r1, #29
-; CHECK-NEXT:    bpl .LBB13_12
-; CHECK-NEXT:  .LBB13_27: @ %cond.store26
-; CHECK-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-NEXT:    lsls r2, r1, #28
-; CHECK-NEXT:    bpl .LBB13_13
-; CHECK-NEXT:  .LBB13_28: @ %cond.store28
-; CHECK-NEXT:    vmovx.f16 s4, s1
-; CHECK-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-NEXT:    lsls r2, r1, #27
-; CHECK-NEXT:    bpl .LBB13_14
-; CHECK-NEXT:  .LBB13_29: @ %cond.store30
-; CHECK-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-NEXT:    lsls r2, r1, #26
-; CHECK-NEXT:    bpl .LBB13_15
-; CHECK-NEXT:  .LBB13_30: @ %cond.store32
-; CHECK-NEXT:    vmovx.f16 s4, s2
-; CHECK-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-NEXT:    lsls r2, r1, #25
-; CHECK-NEXT:    bpl .LBB13_16
-; CHECK-NEXT:  .LBB13_31: @ %cond.store34
-; CHECK-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-NEXT:    lsls r1, r1, #24
-; CHECK-NEXT:    bpl .LBB13_17
-; CHECK-NEXT:  .LBB13_32: @ %cond.store36
-; CHECK-NEXT:    vmovx.f16 s0, s3
-; CHECK-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-NEXT:    add sp, #16
+; CHECK-NEXT:    vptt.s16 gt, q0, zr
+; CHECK-NEXT:    vldrht.u16 q0, [r2]
+; CHECK-NEXT:    vstrht.16 q0, [r0]
 ; CHECK-NEXT:    bx lr
 entry:
   %0 = load <8 x i16>, <8 x i16>* %mask, align 2
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
index b1d048ecdbde5..534530b6fb086 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-load.ll
@@ -1,100 +1,20 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
-; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_zero(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_align4_zero:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    mov.w r12, #0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #3, #1
-; CHECK-LE-NEXT:    and r1, r2, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    beq .LBB0_2
-; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-LE-NEXT:    ldr r2, [r0]
-; CHECK-LE-NEXT:    vdup.32 q0, r12
-; CHECK-LE-NEXT:    vmov.32 q0[0], r2
-; CHECK-LE-NEXT:    b .LBB0_3
-; CHECK-LE-NEXT:  .LBB0_2:
-; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-LE-NEXT:  .LBB0_3: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_align4_zero:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    mov.w r12, #0
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #3, #1
-; CHECK-BE-NEXT:    and r1, r2, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    beq .LBB0_2
-; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-BE-NEXT:    ldr r2, [r0]
-; CHECK-BE-NEXT:    vdup.32 q1, r12
-; CHECK-BE-NEXT:    vmov.32 q1[0], r2
-; CHECK-BE-NEXT:    b .LBB0_3
-; CHECK-BE-NEXT:  .LBB0_2:
-; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-BE-NEXT:  .LBB0_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -105,84 +25,16 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_undef(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_align4_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_align4_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -193,84 +45,17 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align1_undef(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_align1_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_align1_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev32.8 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -281,82 +66,18 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @masked_v4i32_align4_other(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_align4_other:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_align4_other:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q1[2], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q1[3], r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -367,87 +88,19 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v4i32_preinc(i8* %x, i8* %y, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_preinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0, #4]
 ; CHECK-LE-NEXT:    adds r0, #4
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r2, r2, #15
-; CHECK-LE-NEXT:    lsls r3, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r3, [r0]
-; CHECK-LE-NEXT:    vmovne.32 q0[0], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r3, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r3, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r3
-; CHECK-LE-NEXT:    lsls r2, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r2
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_preinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0, #4]
 ; CHECK-BE-NEXT:    adds r0, #4
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r2, r2, #15
-; CHECK-BE-NEXT:    lsls r3, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r3, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r3, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q0[1], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r3, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q0[2], r3
-; CHECK-BE-NEXT:    lsls r2, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r2
 ; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -462,89 +115,19 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v4i32_postinc(i8* %x, i8* %y, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_postinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    add.w r12, r0, #4
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r3, r2, #15
-; CHECK-LE-NEXT:    lsls r2, r3, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov r0, r12
-; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_postinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    add.w r12, r0, #4
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r3, r2, #15
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.32 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.32 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.32 q0[2], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.32 q0[3], r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-BE-NEXT:    mov r0, r12
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -561,153 +144,21 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align4_zero(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_align4_zero:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    mov.w r12, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r2, r3, #0
-; CHECK-LE-NEXT:    movs r3, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-LE-NEXT:    ubfx r2, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r3, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r3
-; CHECK-LE-NEXT:    lsls r2, r3, #31
-; CHECK-LE-NEXT:    beq .LBB6_2
-; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-LE-NEXT:    ldrh r2, [r0]
-; CHECK-LE-NEXT:    vdup.16 q0, r12
-; CHECK-LE-NEXT:    vmov.16 q0[0], r2
-; CHECK-LE-NEXT:    b .LBB6_3
-; CHECK-LE-NEXT:  .LBB6_2:
-; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-LE-NEXT:  .LBB6_3: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_align4_zero:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    mov.w r12, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r2, r3, #0
-; CHECK-BE-NEXT:    movs r3, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-BE-NEXT:    ubfx r2, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r3, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r3
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    beq .LBB6_2
-; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-BE-NEXT:    ldrh r2, [r0]
-; CHECK-BE-NEXT:    vdup.16 q1, r12
-; CHECK-BE-NEXT:    vmov.16 q1[0], r2
-; CHECK-BE-NEXT:    b .LBB6_3
-; CHECK-BE-NEXT:  .LBB6_2:
-; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vrev32.16 q1, q0
-; CHECK-BE-NEXT:  .LBB6_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vrev32.16 q1, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -718,140 +169,16 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align4_undef(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_align4_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrhne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.16 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_align4_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -862,140 +189,17 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align1_undef(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_align1_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrhne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.16 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_align1_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev16.8 q1, q0
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -1006,138 +210,18 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @masked_v8i16_align4_other(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_align4_other:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrhne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.16 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_align4_other:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q1[7], r0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -1148,147 +232,23 @@ entry:
 define i8* @masked_v8i16_preinc(i8* %x, i8* %y, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_preinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r3, #0
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r2, r12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-LE-NEXT:    uxtb r2, r3
-; CHECK-LE-NEXT:    lsls r3, r3, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrhne r3, [r0]
-; CHECK-LE-NEXT:    vmovne.16 q0[0], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r3, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r3
-; CHECK-LE-NEXT:    lsls r2, r2, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r2
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    add sp, #8
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_preinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r3, #0
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r2, r12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-BE-NEXT:    uxtb r2, r3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r3, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q0[0], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q0[1], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q0[2], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q0[3], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q0[4], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q0[5], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r3, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q0[6], r3
-; CHECK-BE-NEXT:    lsls r2, r2, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q0[7], r2
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -1303,145 +263,19 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v8i16_postinc(i8* %x, i8* %y, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_postinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    add.w r12, r0, #4
-; CHECK-LE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-LE-NEXT:    uxtb r3, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrhne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.16 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.16 q0[7], r0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov r0, r12
-; CHECK-LE-NEXT:    add sp, #8
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_postinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    add.w r12, r0, #4
-; CHECK-BE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-BE-NEXT:    uxtb r3, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrhne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.16 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.16 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.16 q0[2], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.16 q0[3], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.16 q0[4], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.16 q0[5], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.16 q0[6], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrhmi r0, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.16 q0[7], r0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
-; CHECK-BE-NEXT:    mov r0, r12
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -1457,184 +291,22 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_zero(<16 x i8> *%dest, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_align4_zero:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    uxth r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    beq .LBB12_2
-; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    ldrb r3, [r0]
-; CHECK-LE-NEXT:    vdup.8 q0, r2
-; CHECK-LE-NEXT:    vmov.8 q0[0], r3
-; CHECK-LE-NEXT:    b .LBB12_3
-; CHECK-LE-NEXT:  .LBB12_2:
-; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-LE-NEXT:  .LBB12_3: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    vmovmi.8 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.8 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    vmovmi.8 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.8 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    vmovmi.8 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.8 q0[6], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    vmovmi.8 q0[7], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.8 q0[8], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    vmovmi.8 q0[9], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.8 q0[10], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    vmovmi.8 q0[11], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.8 q0[12], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    vmovmi.8 q0[13], r2
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    lsls r2, r1, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.8 q0[14], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-LE-NEXT:    vmovmi.8 q0[15], r0
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_align4_zero:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB12_2
-; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    ldrb r3, [r0]
-; CHECK-BE-NEXT:    vdup.8 q1, r2
-; CHECK-BE-NEXT:    vmov.8 q1[0], r3
-; CHECK-BE-NEXT:    b .LBB12_3
-; CHECK-BE-NEXT:  .LBB12_2:
-; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vrev32.8 q1, q0
-; CHECK-BE-NEXT:  .LBB12_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    vmovmi.8 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.8 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    vmovmi.8 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.8 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    vmovmi.8 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.8 q1[6], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    vmovmi.8 q1[7], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.8 q1[8], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    vmovmi.8 q1[9], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.8 q1[10], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    vmovmi.8 q1[11], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.8 q1[12], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    vmovmi.8 q1[13], r2
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    lsls r2, r1, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.8 q1[14], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-BE-NEXT:    vmovmi.8 q1[15], r0
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vrev32.8 q1, q1
+; CHECK-BE-NEXT:    vpt.s8 gt, q2, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.8 q0, q1
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <16 x i8> %a, zeroinitializer
   %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> zeroinitializer)
@@ -1644,171 +316,17 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_undef(<16 x i8> *%dest, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_align4_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    uxth r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrbne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.8 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    vmovmi.8 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.8 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    vmovmi.8 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.8 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    vmovmi.8 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.8 q0[6], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    vmovmi.8 q0[7], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.8 q0[8], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    vmovmi.8 q0[9], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.8 q0[10], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    vmovmi.8 q0[11], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.8 q0[12], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    vmovmi.8 q0[13], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.8 q0[14], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-LE-NEXT:    vmovmi.8 q0[15], r0
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_align4_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrbne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.8 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    vmovmi.8 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.8 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    vmovmi.8 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.8 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    vmovmi.8 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.8 q1[6], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    vmovmi.8 q1[7], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.8 q1[8], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    vmovmi.8 q1[9], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.8 q1[10], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    vmovmi.8 q1[11], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.8 q1[12], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    vmovmi.8 q1[13], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.8 q1[14], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-BE-NEXT:    vmovmi.8 q1[15], r0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.8 q0, q1
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <16 x i8> %a, zeroinitializer
   %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> undef)
@@ -1818,169 +336,19 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @masked_v16i8_align4_other(<16 x i8> *%dest, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_align4_other:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    uxth r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrbne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.8 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    vmovmi.8 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.8 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    vmovmi.8 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.8 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    vmovmi.8 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.8 q0[6], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    vmovmi.8 q0[7], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.8 q0[8], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    vmovmi.8 q0[9], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.8 q0[10], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    vmovmi.8 q0[11], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.8 q0[12], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    vmovmi.8 q0[13], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.8 q0[14], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-LE-NEXT:    vmovmi.8 q0[15], r0
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q1, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q1, q0
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_align4_other:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrbne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.8 q1[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    vmovmi.8 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.8 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    vmovmi.8 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.8 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    vmovmi.8 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.8 q1[6], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    vmovmi.8 q1[7], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.8 q1[8], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    vmovmi.8 q1[9], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.8 q1[10], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    vmovmi.8 q1[11], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.8 q1[12], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    vmovmi.8 q1[13], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.8 q1[14], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-BE-NEXT:    vmovmi.8 q1[15], r0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.8 q0, q1
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <16 x i8> %a, zeroinitializer
   %l = call <16 x i8> @llvm.masked.load.v16i8.p0v16i8(<16 x i8>* %dest, i32 1, <16 x i1> %c, <16 x i8> %a)
@@ -1990,174 +358,20 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v16i8_preinc(i8* %x, i8* %y, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_preinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0, #4]
 ; CHECK-LE-NEXT:    adds r0, #4
-; CHECK-LE-NEXT:    vmrs r3, p0
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    uxth r2, r3
-; CHECK-LE-NEXT:    lsls r3, r3, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrbne r3, [r0]
-; CHECK-LE-NEXT:    vmovne.8 q0[0], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #1]
-; CHECK-LE-NEXT:    vmovmi.8 q0[1], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.8 q0[2], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #3]
-; CHECK-LE-NEXT:    vmovmi.8 q0[3], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.8 q0[4], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #5]
-; CHECK-LE-NEXT:    vmovmi.8 q0[5], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.8 q0[6], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #7]
-; CHECK-LE-NEXT:    vmovmi.8 q0[7], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.8 q0[8], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #9]
-; CHECK-LE-NEXT:    vmovmi.8 q0[9], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.8 q0[10], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #11]
-; CHECK-LE-NEXT:    vmovmi.8 q0[11], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.8 q0[12], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #13]
-; CHECK-LE-NEXT:    vmovmi.8 q0[13], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r3, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.8 q0[14], r3
-; CHECK-LE-NEXT:    lsls r2, r2, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #15]
-; CHECK-LE-NEXT:    vmovmi.8 q0[15], r2
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_preinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    @ implicit-def: $q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0, #4]
 ; CHECK-BE-NEXT:    adds r0, #4
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vmrs r3, p0
-; CHECK-BE-NEXT:    uxth r2, r3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrbne r3, [r0]
-; CHECK-BE-NEXT:    vmovne.8 q0[0], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #1]
-; CHECK-BE-NEXT:    vmovmi.8 q0[1], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.8 q0[2], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #3]
-; CHECK-BE-NEXT:    vmovmi.8 q0[3], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.8 q0[4], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #5]
-; CHECK-BE-NEXT:    vmovmi.8 q0[5], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.8 q0[6], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #7]
-; CHECK-BE-NEXT:    vmovmi.8 q0[7], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.8 q0[8], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #9]
-; CHECK-BE-NEXT:    vmovmi.8 q0[9], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.8 q0[10], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #11]
-; CHECK-BE-NEXT:    vmovmi.8 q0[11], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.8 q0[12], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #13]
-; CHECK-BE-NEXT:    vmovmi.8 q0[13], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r3, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.8 q0[14], r3
-; CHECK-BE-NEXT:    lsls r2, r2, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #15]
-; CHECK-BE-NEXT:    vmovmi.8 q0[15], r2
 ; CHECK-BE-NEXT:    vstrb.8 q0, [r1]
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <16 x i8>*
@@ -2171,176 +385,20 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v16i8_postinc(i8* %x, i8* %y, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_postinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    add.w r12, r0, #4
-; CHECK-LE-NEXT:    uxth r3, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrbne r2, [r0]
-; CHECK-LE-NEXT:    vmovne.8 q0[0], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    vmovmi.8 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    vmovmi.8 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    vmovmi.8 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi.8 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    vmovmi.8 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    vmovmi.8 q0[6], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    vmovmi.8 q0[7], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi.8 q0[8], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    vmovmi.8 q0[9], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    vmovmi.8 q0[10], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    vmovmi.8 q0[11], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi.8 q0[12], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    vmovmi.8 q0[13], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    vmovmi.8 q0[14], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-LE-NEXT:    vmovmi.8 q0[15], r0
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov r0, r12
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_postinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    add.w r12, r0, #4
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r3, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrbne r2, [r0]
-; CHECK-BE-NEXT:    vmovne.8 q0[0], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    vmovmi.8 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    vmovmi.8 q0[2], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    vmovmi.8 q0[3], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi.8 q0[4], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    vmovmi.8 q0[5], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    vmovmi.8 q0[6], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    vmovmi.8 q0[7], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi.8 q0[8], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    vmovmi.8 q0[9], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    vmovmi.8 q0[10], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    vmovmi.8 q0[11], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi.8 q0[12], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    vmovmi.8 q0[13], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    vmovmi.8 q0[14], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrbmi r0, [r0, #15]
-; CHECK-BE-NEXT:    vmovmi.8 q0[15], r0
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrb.8 q0, [r1]
-; CHECK-BE-NEXT:    mov r0, r12
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
@@ -2355,101 +413,21 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_zero(<4 x float> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_align4_zero:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    beq .LBB17_2
-; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-LE-NEXT:    vldr s0, .LCPI17_0
-; CHECK-LE-NEXT:    vldr s4, [r0]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vdup.32 q0, r2
-; CHECK-LE-NEXT:    vmov.f32 s0, s4
-; CHECK-LE-NEXT:    b .LBB17_3
-; CHECK-LE-NEXT:  .LBB17_2:
-; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-LE-NEXT:  .LBB17_3: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s3, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:    .p2align 2
-; CHECK-LE-NEXT:  @ %bb.4:
-; CHECK-LE-NEXT:  .LCPI17_0:
-; CHECK-LE-NEXT:    .long 0 @ float 0
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align4_zero:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    beq .LBB17_2
-; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-BE-NEXT:    vldr s0, .LCPI17_0
-; CHECK-BE-NEXT:    vldr s2, [r0]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vdup.32 q1, r2
-; CHECK-BE-NEXT:    vmov.f32 s4, s2
-; CHECK-BE-NEXT:    b .LBB17_3
-; CHECK-BE-NEXT:  .LBB17_2:
 ; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
-; CHECK-BE-NEXT:  .LBB17_3: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s5, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s6, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s7, [r0, #12]
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:    .p2align 2
-; CHECK-BE-NEXT:  @ %bb.4:
-; CHECK-BE-NEXT:  .LCPI17_0:
-; CHECK-BE-NEXT:    .long 0 @ float 0
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
   %l = call <4 x float> @llvm.masked.load.v4f32.p0v4f32(<4 x float>* %dest, i32 4, <4 x i1> %c, <4 x float> zeroinitializer)
@@ -2459,76 +437,16 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_undef(<4 x float> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_align4_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vldrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s3, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align4_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vldrne s4, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s5, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s6, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s7, [r0, #12]
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -2539,84 +457,17 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align1_undef(<4 x float> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_align1_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    ldrne r2, [r0]
-; CHECK-LE-NEXT:    vmovne s0, r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-LE-NEXT:    vmovmi s1, r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-LE-NEXT:    vmovmi s2, r2
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-LE-NEXT:    vmovmi s3, r0
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align1_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    ldrne r2, [r0]
-; CHECK-BE-NEXT:    vmovne s4, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #4]
-; CHECK-BE-NEXT:    vmovmi s5, r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r2, [r0, #8]
-; CHECK-BE-NEXT:    vmovmi s6, r2
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    ldrmi r0, [r0, #12]
-; CHECK-BE-NEXT:    vmovmi s7, r0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev32.8 q1, q0
 ; CHECK-BE-NEXT:    vrev64.32 q0, q1
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -2627,76 +478,19 @@ entry:
 define arm_aapcs_vfpcc <4 x float> @masked_v4f32_align4_other(<4 x float> *%dest, <4 x i32> %a, <4 x float> %b) {
 ; CHECK-LE-LABEL: masked_v4f32_align4_other:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vldrne s4, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s5, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s6, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s7, [r0, #12]
-; CHECK-LE-NEXT:    vmov q0, q1
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align4_other:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
-; CHECK-BE-NEXT:    vrev64.32 q2, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q2, zr
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vldrne s8, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s9, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s10, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s11, [r0, #12]
-; CHECK-BE-NEXT:    vrev64.32 q0, q2
-; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q2
+; CHECK-BE-NEXT:    vrev64.32 q0, q1
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -2707,79 +501,19 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v4f32_preinc(i8* %x, i8* %y, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_preinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0, #4]
 ; CHECK-LE-NEXT:    adds r0, #4
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r2, r2, #15
-; CHECK-LE-NEXT:    lsls r3, r2, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vldrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r2, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s3, [r0, #12]
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_preinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0, #4]
 ; CHECK-BE-NEXT:    adds r0, #4
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r2, r2, #15
-; CHECK-BE-NEXT:    lsls r3, r2, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vldrne s0, [r0]
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r2, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s3, [r0, #12]
 ; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -2794,81 +528,19 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v4f32_postinc(i8* %x, i8* %y, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_postinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    add.w r12, r0, #4
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r3, r2, #15
-; CHECK-LE-NEXT:    lsls r2, r3, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vldrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vldrmi s3, [r0, #12]
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov r0, r12
-; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_postinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    add.w r12, r0, #4
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r3, r2, #15
-; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vldrne s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vldrmi s3, [r0, #12]
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vldrwt.u32 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-BE-NEXT:    mov r0, r12
-; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
@@ -2884,233 +556,22 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_zero(<8 x half> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_align4_zero:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    beq .LBB23_2
-; CHECK-LE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-LE-NEXT:    vldr.16 s0, .LCPI23_0
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vldr.16 s0, [r0]
-; CHECK-LE-NEXT:    vmov r3, s0
-; CHECK-LE-NEXT:    vdup.16 q0, r2
-; CHECK-LE-NEXT:    vmov.16 q0[0], r3
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB23_3
-; CHECK-LE-NEXT:    b .LBB23_4
-; CHECK-LE-NEXT:  .LBB23_2:
-; CHECK-LE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB23_4
-; CHECK-LE-NEXT:  .LBB23_3: @ %cond.load1
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r2
-; CHECK-LE-NEXT:  .LBB23_4: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB23_11
-; CHECK-LE-NEXT:  @ %bb.5: @ %else5
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB23_12
-; CHECK-LE-NEXT:  .LBB23_6: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB23_13
-; CHECK-LE-NEXT:  .LBB23_7: @ %else11
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB23_14
-; CHECK-LE-NEXT:  .LBB23_8: @ %else14
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB23_15
-; CHECK-LE-NEXT:  .LBB23_9: @ %else17
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB23_16
-; CHECK-LE-NEXT:  .LBB23_10: @ %else20
-; CHECK-LE-NEXT:    add sp, #8
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB23_11: @ %cond.load4
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB23_6
-; CHECK-LE-NEXT:  .LBB23_12: @ %cond.load7
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB23_7
-; CHECK-LE-NEXT:  .LBB23_13: @ %cond.load10
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB23_8
-; CHECK-LE-NEXT:  .LBB23_14: @ %cond.load13
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB23_9
-; CHECK-LE-NEXT:  .LBB23_15: @ %cond.load16
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB23_10
-; CHECK-LE-NEXT:  .LBB23_16: @ %cond.load19
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-LE-NEXT:    vmov r0, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:    .p2align 1
-; CHECK-LE-NEXT:  @ %bb.17:
-; CHECK-LE-NEXT:  .LCPI23_0:
-; CHECK-LE-NEXT:    .short 0 @ half 0
 ;
 ; CHECK-BE-LABEL: masked_v8f16_align4_zero:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    beq .LBB23_2
-; CHECK-BE-NEXT:  @ %bb.1: @ %cond.load
-; CHECK-BE-NEXT:    vldr.16 s0, .LCPI23_0
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vldr.16 s0, [r0]
-; CHECK-BE-NEXT:    vdup.16 q1, r2
-; CHECK-BE-NEXT:    vmov r3, s0
-; CHECK-BE-NEXT:    vmov.16 q1[0], r3
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB23_3
-; CHECK-BE-NEXT:    b .LBB23_4
-; CHECK-BE-NEXT:  .LBB23_2:
-; CHECK-BE-NEXT:    vmov.i32 q0, #0x0
-; CHECK-BE-NEXT:    vrev32.16 q1, q0
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB23_4
-; CHECK-BE-NEXT:  .LBB23_3: @ %cond.load1
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #2]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[1], r2
-; CHECK-BE-NEXT:  .LBB23_4: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB23_12
-; CHECK-BE-NEXT:  @ %bb.5: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB23_13
-; CHECK-BE-NEXT:  .LBB23_6: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB23_14
-; CHECK-BE-NEXT:  .LBB23_7: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB23_15
-; CHECK-BE-NEXT:  .LBB23_8: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB23_16
-; CHECK-BE-NEXT:  .LBB23_9: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB23_11
-; CHECK-BE-NEXT:  .LBB23_10: @ %cond.load19
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:    vmov r0, s0
-; CHECK-BE-NEXT:    vmov.16 q1[7], r0
-; CHECK-BE-NEXT:  .LBB23_11: @ %else20
+; CHECK-BE-NEXT:    vmov.i32 q1, #0x0
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vrev32.16 q1, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q1
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB23_12: @ %cond.load4
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #4]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB23_6
-; CHECK-BE-NEXT:  .LBB23_13: @ %cond.load7
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #6]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB23_7
-; CHECK-BE-NEXT:  .LBB23_14: @ %cond.load10
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #8]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB23_8
-; CHECK-BE-NEXT:  .LBB23_15: @ %cond.load13
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #10]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB23_9
-; CHECK-BE-NEXT:  .LBB23_16: @ %cond.load16
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #12]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB23_10
-; CHECK-BE-NEXT:    b .LBB23_11
-; CHECK-BE-NEXT:    .p2align 1
-; CHECK-BE-NEXT:  @ %bb.17:
-; CHECK-BE-NEXT:  .LCPI23_0:
-; CHECK-BE-NEXT:    .short 0 @ half 0
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> zeroinitializer)
@@ -3120,216 +581,17 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_undef(<8 x half> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_align4_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB24_9
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB24_10
-; CHECK-LE-NEXT:  .LBB24_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB24_11
-; CHECK-LE-NEXT:  .LBB24_3: @ %else5
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB24_12
-; CHECK-LE-NEXT:  .LBB24_4: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB24_13
-; CHECK-LE-NEXT:  .LBB24_5: @ %else11
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB24_14
-; CHECK-LE-NEXT:  .LBB24_6: @ %else14
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB24_15
-; CHECK-LE-NEXT:  .LBB24_7: @ %else17
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB24_16
-; CHECK-LE-NEXT:  .LBB24_8: @ %else20
-; CHECK-LE-NEXT:    add sp, #8
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB24_9: @ %cond.load
-; CHECK-LE-NEXT:    vldr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB24_2
-; CHECK-LE-NEXT:  .LBB24_10: @ %cond.load1
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB24_3
-; CHECK-LE-NEXT:  .LBB24_11: @ %cond.load4
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB24_4
-; CHECK-LE-NEXT:  .LBB24_12: @ %cond.load7
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB24_5
-; CHECK-LE-NEXT:  .LBB24_13: @ %cond.load10
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB24_6
-; CHECK-LE-NEXT:  .LBB24_14: @ %cond.load13
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB24_7
-; CHECK-LE-NEXT:  .LBB24_15: @ %cond.load16
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB24_8
-; CHECK-LE-NEXT:  .LBB24_16: @ %cond.load19
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-LE-NEXT:    vmov r0, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8f16_align4_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB24_10
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB24_11
-; CHECK-BE-NEXT:  .LBB24_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB24_12
-; CHECK-BE-NEXT:  .LBB24_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB24_13
-; CHECK-BE-NEXT:  .LBB24_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB24_14
-; CHECK-BE-NEXT:  .LBB24_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB24_15
-; CHECK-BE-NEXT:  .LBB24_6: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB24_16
-; CHECK-BE-NEXT:  .LBB24_7: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB24_9
-; CHECK-BE-NEXT:  .LBB24_8: @ %cond.load19
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:    vmov r0, s0
-; CHECK-BE-NEXT:    vmov.16 q1[7], r0
-; CHECK-BE-NEXT:  .LBB24_9: @ %else20
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q1, [r0]
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB24_10: @ %cond.load
-; CHECK-BE-NEXT:    vldr.16 s4, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB24_2
-; CHECK-BE-NEXT:  .LBB24_11: @ %cond.load1
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #2]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB24_3
-; CHECK-BE-NEXT:  .LBB24_12: @ %cond.load4
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #4]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB24_4
-; CHECK-BE-NEXT:  .LBB24_13: @ %cond.load7
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #6]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB24_5
-; CHECK-BE-NEXT:  .LBB24_14: @ %cond.load10
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #8]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB24_6
-; CHECK-BE-NEXT:  .LBB24_15: @ %cond.load13
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #10]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB24_7
-; CHECK-BE-NEXT:  .LBB24_16: @ %cond.load16
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #12]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB24_8
-; CHECK-BE-NEXT:    b .LBB24_9
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> undef)
@@ -3339,248 +601,18 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align1_undef(<8 x half> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_align1_undef:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #40
-; CHECK-LE-NEXT:    sub sp, #40
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB25_9
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB25_10
-; CHECK-LE-NEXT:  .LBB25_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB25_11
-; CHECK-LE-NEXT:  .LBB25_3: @ %else5
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB25_12
-; CHECK-LE-NEXT:  .LBB25_4: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB25_13
-; CHECK-LE-NEXT:  .LBB25_5: @ %else11
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB25_14
-; CHECK-LE-NEXT:  .LBB25_6: @ %else14
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB25_15
-; CHECK-LE-NEXT:  .LBB25_7: @ %else17
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB25_16
-; CHECK-LE-NEXT:  .LBB25_8: @ %else20
-; CHECK-LE-NEXT:    add sp, #40
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB25_9: @ %cond.load
-; CHECK-LE-NEXT:    ldrh r2, [r0]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #28]
-; CHECK-LE-NEXT:    vldr.16 s0, [sp, #28]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB25_2
-; CHECK-LE-NEXT:  .LBB25_10: @ %cond.load1
-; CHECK-LE-NEXT:    ldrh r2, [r0, #2]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #24]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #24]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB25_3
-; CHECK-LE-NEXT:  .LBB25_11: @ %cond.load4
-; CHECK-LE-NEXT:    ldrh r2, [r0, #4]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #20]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #20]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB25_4
-; CHECK-LE-NEXT:  .LBB25_12: @ %cond.load7
-; CHECK-LE-NEXT:    ldrh r2, [r0, #6]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #16]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #16]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB25_5
-; CHECK-LE-NEXT:  .LBB25_13: @ %cond.load10
-; CHECK-LE-NEXT:    ldrh r2, [r0, #8]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #12]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #12]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB25_6
-; CHECK-LE-NEXT:  .LBB25_14: @ %cond.load13
-; CHECK-LE-NEXT:    ldrh r2, [r0, #10]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #8]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #8]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB25_7
-; CHECK-LE-NEXT:  .LBB25_15: @ %cond.load16
-; CHECK-LE-NEXT:    ldrh r2, [r0, #12]
-; CHECK-LE-NEXT:    strh.w r2, [sp, #4]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp, #4]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB25_8
-; CHECK-LE-NEXT:  .LBB25_16: @ %cond.load19
-; CHECK-LE-NEXT:    ldrh r0, [r0, #14]
-; CHECK-LE-NEXT:    strh.w r0, [sp]
-; CHECK-LE-NEXT:    vldr.16 s4, [sp]
-; CHECK-LE-NEXT:    vmov r0, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r0
-; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrbt.u8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8f16_align1_undef:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #40
-; CHECK-BE-NEXT:    sub sp, #40
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q1
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB25_10
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB25_11
-; CHECK-BE-NEXT:  .LBB25_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB25_12
-; CHECK-BE-NEXT:  .LBB25_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB25_13
-; CHECK-BE-NEXT:  .LBB25_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB25_14
-; CHECK-BE-NEXT:  .LBB25_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB25_15
-; CHECK-BE-NEXT:  .LBB25_6: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB25_16
-; CHECK-BE-NEXT:  .LBB25_7: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB25_9
-; CHECK-BE-NEXT:  .LBB25_8: @ %cond.load19
-; CHECK-BE-NEXT:    ldrh r0, [r0, #14]
-; CHECK-BE-NEXT:    strh.w r0, [sp]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp]
-; CHECK-BE-NEXT:    vmov r0, s0
-; CHECK-BE-NEXT:    vmov.16 q1[7], r0
-; CHECK-BE-NEXT:  .LBB25_9: @ %else20
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrbt.u8 q0, [r0]
+; CHECK-BE-NEXT:    vrev16.8 q1, q0
 ; CHECK-BE-NEXT:    vrev64.16 q0, q1
-; CHECK-BE-NEXT:    add sp, #40
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB25_10: @ %cond.load
-; CHECK-BE-NEXT:    ldrh r2, [r0]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #28]
-; CHECK-BE-NEXT:    vldr.16 s4, [sp, #28]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB25_2
-; CHECK-BE-NEXT:  .LBB25_11: @ %cond.load1
-; CHECK-BE-NEXT:    ldrh r2, [r0, #2]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #24]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #24]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB25_3
-; CHECK-BE-NEXT:  .LBB25_12: @ %cond.load4
-; CHECK-BE-NEXT:    ldrh r2, [r0, #4]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #20]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #20]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB25_4
-; CHECK-BE-NEXT:  .LBB25_13: @ %cond.load7
-; CHECK-BE-NEXT:    ldrh r2, [r0, #6]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #16]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #16]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB25_5
-; CHECK-BE-NEXT:  .LBB25_14: @ %cond.load10
-; CHECK-BE-NEXT:    ldrh r2, [r0, #8]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #12]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #12]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB25_6
-; CHECK-BE-NEXT:  .LBB25_15: @ %cond.load13
-; CHECK-BE-NEXT:    ldrh r2, [r0, #10]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #8]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #8]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB25_7
-; CHECK-BE-NEXT:  .LBB25_16: @ %cond.load16
-; CHECK-BE-NEXT:    ldrh r2, [r0, #12]
-; CHECK-BE-NEXT:    strh.w r2, [sp, #4]
-; CHECK-BE-NEXT:    vldr.16 s0, [sp, #4]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q1[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB25_8
-; CHECK-BE-NEXT:    b .LBB25_9
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 1, <8 x i1> %c, <8 x half> undef)
@@ -3590,219 +622,20 @@ entry:
 define arm_aapcs_vfpcc <8 x half> @masked_v8f16_align4_other(<8 x half> *%dest, <8 x i16> %a, <8 x half> %b) {
 ; CHECK-LE-LABEL: masked_v8f16_align4_other:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB26_10
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB26_11
-; CHECK-LE-NEXT:  .LBB26_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB26_12
-; CHECK-LE-NEXT:  .LBB26_3: @ %else5
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB26_13
-; CHECK-LE-NEXT:  .LBB26_4: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB26_14
-; CHECK-LE-NEXT:  .LBB26_5: @ %else11
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB26_15
-; CHECK-LE-NEXT:  .LBB26_6: @ %else14
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB26_16
-; CHECK-LE-NEXT:  .LBB26_7: @ %else17
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB26_9
-; CHECK-LE-NEXT:  .LBB26_8: @ %cond.load19
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #14]
-; CHECK-LE-NEXT:    vmov r0, s0
-; CHECK-LE-NEXT:    vmov.16 q1[7], r0
-; CHECK-LE-NEXT:  .LBB26_9: @ %else20
-; CHECK-LE-NEXT:    vmov q0, q1
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    vpsel q0, q0, q1
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB26_10: @ %cond.load
-; CHECK-LE-NEXT:    vldr.16 s0, [r0]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[0], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB26_2
-; CHECK-LE-NEXT:  .LBB26_11: @ %cond.load1
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #2]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[1], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB26_3
-; CHECK-LE-NEXT:  .LBB26_12: @ %cond.load4
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #4]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[2], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB26_4
-; CHECK-LE-NEXT:  .LBB26_13: @ %cond.load7
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #6]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[3], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB26_5
-; CHECK-LE-NEXT:  .LBB26_14: @ %cond.load10
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #8]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[4], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB26_6
-; CHECK-LE-NEXT:  .LBB26_15: @ %cond.load13
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #10]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[5], r2
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB26_7
-; CHECK-LE-NEXT:  .LBB26_16: @ %cond.load16
-; CHECK-LE-NEXT:    vldr.16 s0, [r0, #12]
-; CHECK-LE-NEXT:    vmov r2, s0
-; CHECK-LE-NEXT:    vmov.16 q1[6], r2
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB26_8
-; CHECK-LE-NEXT:    b .LBB26_9
 ;
 ; CHECK-BE-LABEL: masked_v8f16_align4_other:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vrev64.16 q2, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q2, zr
 ; CHECK-BE-NEXT:    vrev64.16 q2, q1
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB26_10
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB26_11
-; CHECK-BE-NEXT:  .LBB26_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB26_12
-; CHECK-BE-NEXT:  .LBB26_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB26_13
-; CHECK-BE-NEXT:  .LBB26_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB26_14
-; CHECK-BE-NEXT:  .LBB26_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB26_15
-; CHECK-BE-NEXT:  .LBB26_6: @ %else14
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB26_16
-; CHECK-BE-NEXT:  .LBB26_7: @ %else17
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB26_9
-; CHECK-BE-NEXT:  .LBB26_8: @ %cond.load19
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:    vmov r0, s0
-; CHECK-BE-NEXT:    vmov.16 q2[7], r0
-; CHECK-BE-NEXT:  .LBB26_9: @ %else20
-; CHECK-BE-NEXT:    vrev64.16 q0, q2
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q1, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    vpsel q1, q0, q2
+; CHECK-BE-NEXT:    vrev64.16 q0, q1
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB26_10: @ %cond.load
-; CHECK-BE-NEXT:    vldr.16 s0, [r0]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[0], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB26_2
-; CHECK-BE-NEXT:  .LBB26_11: @ %cond.load1
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #2]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[1], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB26_3
-; CHECK-BE-NEXT:  .LBB26_12: @ %cond.load4
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #4]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[2], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB26_4
-; CHECK-BE-NEXT:  .LBB26_13: @ %cond.load7
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #6]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[3], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB26_5
-; CHECK-BE-NEXT:  .LBB26_14: @ %cond.load10
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #8]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[4], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB26_6
-; CHECK-BE-NEXT:  .LBB26_15: @ %cond.load13
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #10]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[5], r2
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB26_7
-; CHECK-BE-NEXT:  .LBB26_16: @ %cond.load16
-; CHECK-BE-NEXT:    vldr.16 s0, [r0, #12]
-; CHECK-BE-NEXT:    vmov r2, s0
-; CHECK-BE-NEXT:    vmov.16 q2[6], r2
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB26_8
-; CHECK-BE-NEXT:    b .LBB26_9
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
   %l = call <8 x half> @llvm.masked.load.v8f16.p0v8f16(<8 x half>* %dest, i32 2, <8 x i1> %c, <8 x half> %b)
@@ -3812,218 +645,20 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v8f16_preinc(i8* %x, i8* %y, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_preinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r3, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0, #4]
 ; CHECK-LE-NEXT:    adds r0, #4
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r2, r12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-LE-NEXT:    uxtb r2, r3
-; CHECK-LE-NEXT:    lsls r3, r3, #31
-; CHECK-LE-NEXT:    bne .LBB27_10
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    bmi .LBB27_11
-; CHECK-LE-NEXT:  .LBB27_2: @ %else2
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    bmi .LBB27_12
-; CHECK-LE-NEXT:  .LBB27_3: @ %else5
-; CHECK-LE-NEXT:    lsls r3, r2, #28
-; CHECK-LE-NEXT:    bmi .LBB27_13
-; CHECK-LE-NEXT:  .LBB27_4: @ %else8
-; CHECK-LE-NEXT:    lsls r3, r2, #27
-; CHECK-LE-NEXT:    bmi .LBB27_14
-; CHECK-LE-NEXT:  .LBB27_5: @ %else11
-; CHECK-LE-NEXT:    lsls r3, r2, #26
-; CHECK-LE-NEXT:    bmi .LBB27_15
-; CHECK-LE-NEXT:  .LBB27_6: @ %else14
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    bmi .LBB27_16
-; CHECK-LE-NEXT:  .LBB27_7: @ %else17
-; CHECK-LE-NEXT:    lsls r2, r2, #24
-; CHECK-LE-NEXT:    bpl .LBB27_9
-; CHECK-LE-NEXT:  .LBB27_8: @ %cond.load19
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r2
-; CHECK-LE-NEXT:  .LBB27_9: @ %else20
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    add sp, #8
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB27_10: @ %cond.load
-; CHECK-LE-NEXT:    vldr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r3, r2, #30
-; CHECK-LE-NEXT:    bpl .LBB27_2
-; CHECK-LE-NEXT:  .LBB27_11: @ %cond.load1
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    bpl .LBB27_3
-; CHECK-LE-NEXT:  .LBB27_12: @ %cond.load4
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[2], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #28
-; CHECK-LE-NEXT:    bpl .LBB27_4
-; CHECK-LE-NEXT:  .LBB27_13: @ %cond.load7
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #27
-; CHECK-LE-NEXT:    bpl .LBB27_5
-; CHECK-LE-NEXT:  .LBB27_14: @ %cond.load10
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[4], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #26
-; CHECK-LE-NEXT:    bpl .LBB27_6
-; CHECK-LE-NEXT:  .LBB27_15: @ %cond.load13
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r3
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    bpl .LBB27_7
-; CHECK-LE-NEXT:  .LBB27_16: @ %cond.load16
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-LE-NEXT:    vmov r3, s4
-; CHECK-LE-NEXT:    vmov.16 q0[6], r3
-; CHECK-LE-NEXT:    lsls r2, r2, #24
-; CHECK-LE-NEXT:    bmi .LBB27_8
-; CHECK-LE-NEXT:    b .LBB27_9
 ;
 ; CHECK-BE-LABEL: masked_v8f16_preinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r3, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0, #4]
 ; CHECK-BE-NEXT:    adds r0, #4
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    and r2, r12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-BE-NEXT:    uxtb r2, r3
-; CHECK-BE-NEXT:    lsls r3, r3, #31
-; CHECK-BE-NEXT:    bne .LBB27_10
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    bmi .LBB27_11
-; CHECK-BE-NEXT:  .LBB27_2: @ %else2
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    bmi .LBB27_12
-; CHECK-BE-NEXT:  .LBB27_3: @ %else5
-; CHECK-BE-NEXT:    lsls r3, r2, #28
-; CHECK-BE-NEXT:    bmi .LBB27_13
-; CHECK-BE-NEXT:  .LBB27_4: @ %else8
-; CHECK-BE-NEXT:    lsls r3, r2, #27
-; CHECK-BE-NEXT:    bmi .LBB27_14
-; CHECK-BE-NEXT:  .LBB27_5: @ %else11
-; CHECK-BE-NEXT:    lsls r3, r2, #26
-; CHECK-BE-NEXT:    bmi .LBB27_15
-; CHECK-BE-NEXT:  .LBB27_6: @ %else14
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    bmi .LBB27_16
-; CHECK-BE-NEXT:  .LBB27_7: @ %else17
-; CHECK-BE-NEXT:    lsls r2, r2, #24
-; CHECK-BE-NEXT:    bpl .LBB27_9
-; CHECK-BE-NEXT:  .LBB27_8: @ %cond.load19
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[7], r2
-; CHECK-BE-NEXT:  .LBB27_9: @ %else20
 ; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB27_10: @ %cond.load
-; CHECK-BE-NEXT:    vldr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r3, r2, #30
-; CHECK-BE-NEXT:    bpl .LBB27_2
-; CHECK-BE-NEXT:  .LBB27_11: @ %cond.load1
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[1], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    bpl .LBB27_3
-; CHECK-BE-NEXT:  .LBB27_12: @ %cond.load4
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[2], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #28
-; CHECK-BE-NEXT:    bpl .LBB27_4
-; CHECK-BE-NEXT:  .LBB27_13: @ %cond.load7
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[3], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #27
-; CHECK-BE-NEXT:    bpl .LBB27_5
-; CHECK-BE-NEXT:  .LBB27_14: @ %cond.load10
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[4], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #26
-; CHECK-BE-NEXT:    bpl .LBB27_6
-; CHECK-BE-NEXT:  .LBB27_15: @ %cond.load13
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[5], r3
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    bpl .LBB27_7
-; CHECK-BE-NEXT:  .LBB27_16: @ %cond.load16
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    vmov.16 q0[6], r3
-; CHECK-BE-NEXT:    lsls r2, r2, #24
-; CHECK-BE-NEXT:    bmi .LBB27_8
-; CHECK-BE-NEXT:    b .LBB27_9
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %z to <8 x half>*
@@ -4037,212 +672,20 @@ entry:
 define arm_aapcs_vfpcc i8* @masked_v8f16_postinc(i8* %x, i8* %y, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_postinc:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    @ implicit-def: $q0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-LE-NEXT:    uxtb r3, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB28_12
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    bmi .LBB28_13
-; CHECK-LE-NEXT:  .LBB28_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    bmi .LBB28_14
-; CHECK-LE-NEXT:  .LBB28_3: @ %else5
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    bmi .LBB28_15
-; CHECK-LE-NEXT:  .LBB28_4: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r3, #27
-; CHECK-LE-NEXT:    bmi .LBB28_16
-; CHECK-LE-NEXT:  .LBB28_5: @ %else11
-; CHECK-LE-NEXT:    lsls r2, r3, #26
-; CHECK-LE-NEXT:    bpl .LBB28_7
-; CHECK-LE-NEXT:  .LBB28_6: @ %cond.load13
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[5], r2
-; CHECK-LE-NEXT:  .LBB28_7: @ %else14
-; CHECK-LE-NEXT:    add.w r12, r0, #4
-; CHECK-LE-NEXT:    lsls r2, r3, #25
-; CHECK-LE-NEXT:    bpl .LBB28_9
-; CHECK-LE-NEXT:  @ %bb.8: @ %cond.load16
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[6], r2
-; CHECK-LE-NEXT:  .LBB28_9: @ %else17
-; CHECK-LE-NEXT:    lsls r2, r3, #24
-; CHECK-LE-NEXT:    bpl .LBB28_11
-; CHECK-LE-NEXT:  @ %bb.10: @ %cond.load19
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-LE-NEXT:    vmov r0, s4
-; CHECK-LE-NEXT:    vmov.16 q0[7], r0
-; CHECK-LE-NEXT:  .LBB28_11: @ %else20
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LE-NEXT:    mov r0, r12
-; CHECK-LE-NEXT:    add sp, #8
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB28_12: @ %cond.load
-; CHECK-LE-NEXT:    vldr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r3, #30
-; CHECK-LE-NEXT:    bpl .LBB28_2
-; CHECK-LE-NEXT:  .LBB28_13: @ %cond.load1
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[1], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #29
-; CHECK-LE-NEXT:    bpl .LBB28_3
-; CHECK-LE-NEXT:  .LBB28_14: @ %cond.load4
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[2], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #28
-; CHECK-LE-NEXT:    bpl .LBB28_4
-; CHECK-LE-NEXT:  .LBB28_15: @ %cond.load7
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[3], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #27
-; CHECK-LE-NEXT:    bpl .LBB28_5
-; CHECK-LE-NEXT:  .LBB28_16: @ %cond.load10
-; CHECK-LE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-LE-NEXT:    vmov r2, s4
-; CHECK-LE-NEXT:    vmov.16 q0[4], r2
-; CHECK-LE-NEXT:    lsls r2, r3, #26
-; CHECK-LE-NEXT:    bmi .LBB28_6
-; CHECK-LE-NEXT:    b .LBB28_7
 ;
 ; CHECK-BE-LABEL: masked_v8f16_postinc:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    @ implicit-def: $q0
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-BE-NEXT:    uxtb r3, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB28_12
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    bmi .LBB28_13
-; CHECK-BE-NEXT:  .LBB28_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    bmi .LBB28_14
-; CHECK-BE-NEXT:  .LBB28_3: @ %else5
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    bmi .LBB28_15
-; CHECK-BE-NEXT:  .LBB28_4: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r3, #27
-; CHECK-BE-NEXT:    bmi .LBB28_16
-; CHECK-BE-NEXT:  .LBB28_5: @ %else11
-; CHECK-BE-NEXT:    lsls r2, r3, #26
-; CHECK-BE-NEXT:    bpl .LBB28_7
-; CHECK-BE-NEXT:  .LBB28_6: @ %cond.load13
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #10]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[5], r2
-; CHECK-BE-NEXT:  .LBB28_7: @ %else14
-; CHECK-BE-NEXT:    add.w r12, r0, #4
-; CHECK-BE-NEXT:    lsls r2, r3, #25
-; CHECK-BE-NEXT:    bpl .LBB28_9
-; CHECK-BE-NEXT:  @ %bb.8: @ %cond.load16
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #12]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[6], r2
-; CHECK-BE-NEXT:  .LBB28_9: @ %else17
-; CHECK-BE-NEXT:    lsls r2, r3, #24
-; CHECK-BE-NEXT:    bpl .LBB28_11
-; CHECK-BE-NEXT:  @ %bb.10: @ %cond.load19
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #14]
-; CHECK-BE-NEXT:    vmov r0, s4
-; CHECK-BE-NEXT:    vmov.16 q0[7], r0
-; CHECK-BE-NEXT:  .LBB28_11: @ %else20
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vldrht.u16 q0, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    vstrh.16 q0, [r1]
-; CHECK-BE-NEXT:    mov r0, r12
-; CHECK-BE-NEXT:    add sp, #8
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB28_12: @ %cond.load
-; CHECK-BE-NEXT:    vldr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r3, #30
-; CHECK-BE-NEXT:    bpl .LBB28_2
-; CHECK-BE-NEXT:  .LBB28_13: @ %cond.load1
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[1], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #29
-; CHECK-BE-NEXT:    bpl .LBB28_3
-; CHECK-BE-NEXT:  .LBB28_14: @ %cond.load4
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #4]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[2], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #28
-; CHECK-BE-NEXT:    bpl .LBB28_4
-; CHECK-BE-NEXT:  .LBB28_15: @ %cond.load7
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #6]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[3], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #27
-; CHECK-BE-NEXT:    bpl .LBB28_5
-; CHECK-BE-NEXT:  .LBB28_16: @ %cond.load10
-; CHECK-BE-NEXT:    vldr.16 s4, [r0, #8]
-; CHECK-BE-NEXT:    vmov r2, s4
-; CHECK-BE-NEXT:    vmov.16 q0[4], r2
-; CHECK-BE-NEXT:    lsls r2, r3, #26
-; CHECK-BE-NEXT:    bmi .LBB28_6
-; CHECK-BE-NEXT:    b .LBB28_7
 entry:
   %z = getelementptr inbounds i8, i8* %x, i32 4
   %0 = bitcast i8* %x to <8 x half>*
diff --git a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
index 9d777dbedfda1..772231a458c1f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-masked-store.ll
@@ -1,85 +1,19 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
-; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-LE
+; RUN: llc -mtriple=thumbebv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -enable-arm-maskedldst -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-BE
 
 define arm_aapcs_vfpcc void @masked_v4i32(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne r2, s0
-; CHECK-LE-NEXT:    strne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s1
-; CHECK-LE-NEXT:    strmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s2
-; CHECK-LE-NEXT:    strmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
-; CHECK-LE-NEXT:    strmi r1, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s5
-; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s6
-; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -90,81 +24,16 @@ entry:
 define arm_aapcs_vfpcc void @masked_v4i32_align1(<4 x i32> *%dest, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_align1:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne r2, s0
-; CHECK-LE-NEXT:    strne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s1
-; CHECK-LE-NEXT:    strmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s2
-; CHECK-LE-NEXT:    strmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
-; CHECK-LE-NEXT:    strmi r1, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_align1:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s4
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s5
-; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s6
-; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s7
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    vrev32.8 q0, q1
+; CHECK-BE-NEXT:    vpt.s32 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <4 x i32> %a, zeroinitializer
@@ -175,89 +44,23 @@ entry:
 define i8* @masked_v4i32_pre(i8* %y, i8* %x, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_pre:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r1, r2, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne r2, s0
-; CHECK-LE-NEXT:    strne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s1
-; CHECK-LE-NEXT:    strmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s2
-; CHECK-LE-NEXT:    strmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
-; CHECK-LE-NEXT:    strmi r1, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_pre:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r1, r2, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r2, s0
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s1
-; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s2
-; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s3
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -272,91 +75,23 @@ entry:
 define i8* @masked_v4i32_post(i8* %y, i8* %x, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4i32_post:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r2, r2, #15
-; CHECK-LE-NEXT:    lsls r1, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne r1, s0
-; CHECK-LE-NEXT:    strne r1, [r0]
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r1, s1
-; CHECK-LE-NEXT:    strmi r1, [r0, #4]
-; CHECK-LE-NEXT:    adds r1, r0, #4
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r3, s2
-; CHECK-LE-NEXT:    strmi r3, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi r2, s3
-; CHECK-LE-NEXT:    strmi r2, [r0, #12]
-; CHECK-LE-NEXT:    mov r0, r1
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4i32_post:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r2, r2, #15
-; CHECK-BE-NEXT:    lsls r1, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne r1, s0
-; CHECK-BE-NEXT:    strne r1, [r0]
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r1, s1
-; CHECK-BE-NEXT:    strmi r1, [r0, #4]
-; CHECK-BE-NEXT:    adds r1, r0, #4
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r3, s2
-; CHECK-BE-NEXT:    strmi r3, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi r2, s3
-; CHECK-BE-NEXT:    strmi r2, [r0, #12]
-; CHECK-BE-NEXT:    mov r0, r1
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -372,137 +107,15 @@ entry:
 define arm_aapcs_vfpcc void @masked_v8i16(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-LE-NEXT:    strhne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r2, q1[0]
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[1]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[2]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[3]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[4]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[5]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[6]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q1[7]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -513,137 +126,16 @@ entry:
 define arm_aapcs_vfpcc void @masked_v8i16_align1(<8 x i16> *%dest, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_align1:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-LE-NEXT:    strhne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_align1:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r2, q1[0]
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[1]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[2]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[3]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[4]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[5]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q1[6]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q1[7]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev16.8 q0, q1
+; CHECK-BE-NEXT:    vpt.s16 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <8 x i16> %a, zeroinitializer
@@ -654,145 +146,23 @@ entry:
 define i8* @masked_v8i16_pre(i8* %y, i8* %x, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_pre:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-LE-NEXT:    strhne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_pre:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r2, q0[0]
-; CHECK-BE-NEXT:    strhne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[1]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[2]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[3]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[4]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[5]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[6]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[7]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -807,147 +177,23 @@ entry:
 define i8* @masked_v8i16_post(i8* %y, i8* %x, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8i16_post:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r3, #0
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r2, r12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-LE-NEXT:    lsls r1, r3, #31
-; CHECK-LE-NEXT:    uxtb r2, r3
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u16 r1, q0[0]
-; CHECK-LE-NEXT:    strhne r1, [r0]
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[1]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #2]
-; CHECK-LE-NEXT:    lsls r1, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[2]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r1, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[3]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #6]
-; CHECK-LE-NEXT:    lsls r1, r2, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[4]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r2, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r1, q0[5]
-; CHECK-LE-NEXT:    strhmi r1, [r0, #10]
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    add.w r1, r0, #4
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r3, q0[6]
-; CHECK-LE-NEXT:    strhmi r3, [r0, #12]
-; CHECK-LE-NEXT:    lsls r2, r2, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u16 r2, q0[7]
-; CHECK-LE-NEXT:    strhmi r2, [r0, #14]
-; CHECK-LE-NEXT:    mov r0, r1
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8i16_post:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r3, #0
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r2, r12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-BE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-BE-NEXT:    lsls r1, r3, #31
-; CHECK-BE-NEXT:    uxtb r2, r3
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u16 r1, q0[0]
-; CHECK-BE-NEXT:    strhne r1, [r0]
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[1]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #2]
-; CHECK-BE-NEXT:    lsls r1, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[2]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[3]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #6]
-; CHECK-BE-NEXT:    lsls r1, r2, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[4]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r2, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r1, q0[5]
-; CHECK-BE-NEXT:    strhmi r1, [r0, #10]
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    add.w r1, r0, #4
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r3, q0[6]
-; CHECK-BE-NEXT:    strhmi r3, [r0, #12]
-; CHECK-BE-NEXT:    lsls r2, r2, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u16 r2, q0[7]
-; CHECK-BE-NEXT:    strhmi r2, [r0, #14]
-; CHECK-BE-NEXT:    mov r0, r1
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -963,168 +209,16 @@ entry:
 define arm_aapcs_vfpcc void @masked_v16i8(<16 x i8> *%dest, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    uxth r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u8 r2, q0[0]
-; CHECK-LE-NEXT:    strbne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[1]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[2]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[3]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[4]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[5]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[6]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[7]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    lsls r2, r1, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[8]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[9]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    lsls r2, r1, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[10]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[11]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    lsls r2, r1, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[12]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r2, r1, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[13]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    lsls r2, r1, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[14]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    lsls r1, r1, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[15]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #15]
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
 ; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u8 r2, q1[0]
-; CHECK-BE-NEXT:    strbne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[1]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[2]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[3]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[4]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[5]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[6]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[7]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    lsls r2, r1, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[8]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[9]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    lsls r2, r1, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[10]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[11]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    lsls r2, r1, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[12]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r2, r1, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[13]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    lsls r2, r1, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q1[14]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    lsls r1, r1, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q1[15]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #15]
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    vpt.s8 gt, q1, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0]
+; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp sgt <16 x i8> %a, zeroinitializer
   call void @llvm.masked.store.v16i8.p0v16i8(<16 x i8> %a, <16 x i8>* %dest, i32 1, <16 x i1> %c)
@@ -1134,176 +228,24 @@ entry:
 define i8* @masked_v16i8_pre(i8* %y, i8* %x, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_pre:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vldr d1, [r7, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    uxth r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u8 r2, q0[0]
-; CHECK-LE-NEXT:    strbne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[1]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #1]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[2]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[3]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #3]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[4]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[5]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #5]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[6]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[7]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #7]
-; CHECK-LE-NEXT:    lsls r2, r1, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[8]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[9]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #9]
-; CHECK-LE-NEXT:    lsls r2, r1, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[10]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[11]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #11]
-; CHECK-LE-NEXT:    lsls r2, r1, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[12]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r2, r1, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[13]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #13]
-; CHECK-LE-NEXT:    lsls r2, r1, #17
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[14]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #14]
-; CHECK-LE-NEXT:    lsls r1, r1, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[15]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #15]
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q1, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_pre:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    vldr d1, [r7, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
-; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    vldrb.u8 q0, [r1]
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    uxth r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u8 r2, q0[0]
-; CHECK-BE-NEXT:    strbne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[1]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #1]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[2]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[3]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #3]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[4]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[5]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #5]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[6]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[7]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #7]
-; CHECK-BE-NEXT:    lsls r2, r1, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[8]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[9]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #9]
-; CHECK-BE-NEXT:    lsls r2, r1, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[10]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[11]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #11]
-; CHECK-BE-NEXT:    lsls r2, r1, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[12]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r2, r1, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[13]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #13]
-; CHECK-BE-NEXT:    lsls r2, r1, #17
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[14]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #14]
-; CHECK-BE-NEXT:    lsls r1, r1, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[15]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #15]
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
@@ -1317,178 +259,24 @@ entry:
 define i8* @masked_v16i8_post(i8* %y, i8* %x, <16 x i8> %a) {
 ; CHECK-LE-LABEL: masked_v16i8_post:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-LE-NEXT:    .setfp r7, sp, #8
-; CHECK-LE-NEXT:    add r7, sp, #8
-; CHECK-LE-NEXT:    .pad #16
-; CHECK-LE-NEXT:    sub sp, #16
-; CHECK-LE-NEXT:    mov r4, sp
-; CHECK-LE-NEXT:    bfc r4, #0, #4
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    vldr d1, [r7, #8]
-; CHECK-LE-NEXT:    sub.w r4, r7, #8
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    vcmp.s8 gt, q0, zr
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    uxth r2, r1
-; CHECK-LE-NEXT:    lsls r1, r1, #31
-; CHECK-LE-NEXT:    itt ne
-; CHECK-LE-NEXT:    vmovne.u8 r1, q0[0]
-; CHECK-LE-NEXT:    strbne r1, [r0]
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[1]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #1]
-; CHECK-LE-NEXT:    lsls r1, r2, #29
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[2]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #2]
-; CHECK-LE-NEXT:    lsls r1, r2, #28
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[3]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #3]
-; CHECK-LE-NEXT:    lsls r1, r2, #27
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[4]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r1, r2, #26
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[5]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #5]
-; CHECK-LE-NEXT:    lsls r1, r2, #25
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[6]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #6]
-; CHECK-LE-NEXT:    lsls r1, r2, #24
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[7]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #7]
-; CHECK-LE-NEXT:    lsls r1, r2, #23
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[8]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r2, #22
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[9]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #9]
-; CHECK-LE-NEXT:    lsls r1, r2, #21
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[10]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #10]
-; CHECK-LE-NEXT:    lsls r1, r2, #20
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[11]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #11]
-; CHECK-LE-NEXT:    lsls r1, r2, #19
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[12]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r2, #18
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r1, q0[13]
-; CHECK-LE-NEXT:    strbmi r1, [r0, #13]
-; CHECK-LE-NEXT:    lsls r3, r2, #17
-; CHECK-LE-NEXT:    add.w r1, r0, #4
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r3, q0[14]
-; CHECK-LE-NEXT:    strbmi r3, [r0, #14]
-; CHECK-LE-NEXT:    lsls r2, r2, #16
-; CHECK-LE-NEXT:    itt mi
-; CHECK-LE-NEXT:    vmovmi.u8 r2, q0[15]
-; CHECK-LE-NEXT:    strbmi r2, [r0, #15]
-; CHECK-LE-NEXT:    mov r0, r1
-; CHECK-LE-NEXT:    mov sp, r4
-; CHECK-LE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-LE-NEXT:    vpt.s8 gt, q0, zr
+; CHECK-LE-NEXT:    vstrbt.8 q1, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v16i8_post:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .save {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    push {r4, r6, r7, lr}
-; CHECK-BE-NEXT:    .setfp r7, sp, #8
-; CHECK-BE-NEXT:    add r7, sp, #8
-; CHECK-BE-NEXT:    .pad #16
-; CHECK-BE-NEXT:    sub sp, #16
-; CHECK-BE-NEXT:    mov r4, sp
-; CHECK-BE-NEXT:    bfc r4, #0, #4
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    vldr d1, [r7, #8]
-; CHECK-BE-NEXT:    sub.w r4, r7, #8
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrb.u8 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    vrev64.8 q1, q0
-; CHECK-BE-NEXT:    vldrb.u8 q0, [r1]
-; CHECK-BE-NEXT:    vcmp.s8 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    uxth r2, r1
-; CHECK-BE-NEXT:    lsls r1, r1, #31
-; CHECK-BE-NEXT:    itt ne
-; CHECK-BE-NEXT:    vmovne.u8 r1, q0[0]
-; CHECK-BE-NEXT:    strbne r1, [r0]
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[1]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #1]
-; CHECK-BE-NEXT:    lsls r1, r2, #29
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[2]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #2]
-; CHECK-BE-NEXT:    lsls r1, r2, #28
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[3]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #3]
-; CHECK-BE-NEXT:    lsls r1, r2, #27
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[4]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r2, #26
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[5]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #5]
-; CHECK-BE-NEXT:    lsls r1, r2, #25
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[6]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #6]
-; CHECK-BE-NEXT:    lsls r1, r2, #24
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[7]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #7]
-; CHECK-BE-NEXT:    lsls r1, r2, #23
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[8]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r2, #22
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[9]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #9]
-; CHECK-BE-NEXT:    lsls r1, r2, #21
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[10]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #10]
-; CHECK-BE-NEXT:    lsls r1, r2, #20
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[11]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #11]
-; CHECK-BE-NEXT:    lsls r1, r2, #19
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[12]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r2, #18
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r1, q0[13]
-; CHECK-BE-NEXT:    strbmi r1, [r0, #13]
-; CHECK-BE-NEXT:    lsls r3, r2, #17
-; CHECK-BE-NEXT:    add.w r1, r0, #4
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r3, q0[14]
-; CHECK-BE-NEXT:    strbmi r3, [r0, #14]
-; CHECK-BE-NEXT:    lsls r2, r2, #16
-; CHECK-BE-NEXT:    itt mi
-; CHECK-BE-NEXT:    vmovmi.u8 r2, q0[15]
-; CHECK-BE-NEXT:    strbmi r2, [r0, #15]
-; CHECK-BE-NEXT:    mov r0, r1
-; CHECK-BE-NEXT:    mov sp, r4
-; CHECK-BE-NEXT:    pop {r4, r6, r7, pc}
+; CHECK-BE-NEXT:    vrev64.8 q2, q0
+; CHECK-BE-NEXT:    vpt.s8 gt, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q1, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <16 x i8>*
@@ -1503,74 +291,16 @@ entry:
 define arm_aapcs_vfpcc void @masked_v4f32(<4 x float> *%dest, <4 x float> %a, <4 x i32> %b) {
 ; CHECK-LE-LABEL: masked_v4f32:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #4
-; CHECK-LE-NEXT:    sub sp, #4
-; CHECK-LE-NEXT:    vcmp.i32 ne, q1, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vstrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #4
+; CHECK-LE-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-LE-NEXT:    vstrwt.32 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #4
-; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne s4, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s5, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s6, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s7, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #4
+; CHECK-BE-NEXT:    vpt.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <4 x i32> %b, zeroinitializer
@@ -1581,90 +311,17 @@ entry:
 define arm_aapcs_vfpcc void @masked_v4f32_align1(<4 x float> *%dest, <4 x float> %a, <4 x i32> %b) {
 ; CHECK-LE-LABEL: masked_v4f32_align1:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #20
-; CHECK-LE-NEXT:    sub sp, #20
-; CHECK-LE-NEXT:    vcmp.i32 ne, q1, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    ittt ne
-; CHECK-LE-NEXT:    vstrne s0, [sp, #12]
-; CHECK-LE-NEXT:    ldrne r2, [sp, #12]
-; CHECK-LE-NEXT:    strne r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s1, [sp, #8]
-; CHECK-LE-NEXT:    ldrmi r2, [sp, #8]
-; CHECK-LE-NEXT:    strmi r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s2, [sp, #4]
-; CHECK-LE-NEXT:    ldrmi r2, [sp, #4]
-; CHECK-LE-NEXT:    strmi r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vstrmi s3, [sp]
-; CHECK-LE-NEXT:    ldrmi r1, [sp]
-; CHECK-LE-NEXT:    strmi r1, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #20
+; CHECK-LE-NEXT:    vpt.i32 ne, q1, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_align1:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #20
-; CHECK-BE-NEXT:    sub sp, #20
 ; CHECK-BE-NEXT:    vrev64.32 q2, q1
-; CHECK-BE-NEXT:    movs r1, #0
-; CHECK-BE-NEXT:    vcmp.i32 ne, q2, zr
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    and r3, r2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-BE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-BE-NEXT:    and r1, r1, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    ittt ne
-; CHECK-BE-NEXT:    vstrne s4, [sp, #12]
-; CHECK-BE-NEXT:    ldrne r2, [sp, #12]
-; CHECK-BE-NEXT:    strne r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s5, [sp, #8]
-; CHECK-BE-NEXT:    ldrmi r2, [sp, #8]
-; CHECK-BE-NEXT:    strmi r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s6, [sp, #4]
-; CHECK-BE-NEXT:    ldrmi r2, [sp, #4]
-; CHECK-BE-NEXT:    strmi r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    ittt mi
-; CHECK-BE-NEXT:    vstrmi s7, [sp]
-; CHECK-BE-NEXT:    ldrmi r1, [sp]
-; CHECK-BE-NEXT:    strmi r1, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #20
+; CHECK-BE-NEXT:    vrev32.8 q0, q1
+; CHECK-BE-NEXT:    vpt.i32 ne, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <4 x i32> %b, zeroinitializer
@@ -1675,81 +332,23 @@ entry:
 define i8* @masked_v4f32_pre(i8* %y, i8* %x, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_pre:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r1, r2, #15
-; CHECK-LE-NEXT:    lsls r2, r1, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vstrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r1, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_pre:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r1, r2, #15
-; CHECK-BE-NEXT:    lsls r2, r1, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r1, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -1764,83 +363,23 @@ entry:
 define i8* @masked_v4f32_post(i8* %y, i8* %x, <4 x i32> %a) {
 ; CHECK-LE-LABEL: masked_v4f32_post:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s32 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    and r2, r2, #15
-; CHECK-LE-NEXT:    lsls r1, r2, #31
-; CHECK-LE-NEXT:    it ne
-; CHECK-LE-NEXT:    vstrne s0, [r0]
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-LE-NEXT:    adds r1, r0, #4
-; CHECK-LE-NEXT:    lsls r3, r2, #29
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r2, #28
-; CHECK-LE-NEXT:    it mi
-; CHECK-LE-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-LE-NEXT:    mov r0, r1
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s32 gt, q0, zr
+; CHECK-LE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v4f32_post:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vcmp.s32 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    and r2, r2, #15
-; CHECK-BE-NEXT:    lsls r1, r2, #31
-; CHECK-BE-NEXT:    it ne
-; CHECK-BE-NEXT:    vstrne s0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s1, [r0, #4]
-; CHECK-BE-NEXT:    adds r1, r0, #4
-; CHECK-BE-NEXT:    lsls r3, r2, #29
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r2, #28
-; CHECK-BE-NEXT:    it mi
-; CHECK-BE-NEXT:    vstrmi s3, [r0, #12]
-; CHECK-BE-NEXT:    mov r0, r1
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.32 q2, q0
+; CHECK-BE-NEXT:    vpt.s32 gt, q2, zr
+; CHECK-BE-NEXT:    vstrwt.32 q1, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -1856,194 +395,16 @@ entry:
 define arm_aapcs_vfpcc void @masked_v8f16(<8 x half> *%dest, <8 x half> %a, <8 x i16> %b) {
 ; CHECK-LE-LABEL: masked_v8f16:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vcmp.i16 ne, q1, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB15_9
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB15_10
-; CHECK-LE-NEXT:  .LBB15_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB15_11
-; CHECK-LE-NEXT:  .LBB15_3: @ %else4
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB15_12
-; CHECK-LE-NEXT:  .LBB15_4: @ %else6
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB15_13
-; CHECK-LE-NEXT:  .LBB15_5: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB15_14
-; CHECK-LE-NEXT:  .LBB15_6: @ %else10
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB15_15
-; CHECK-LE-NEXT:  .LBB15_7: @ %else12
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB15_16
-; CHECK-LE-NEXT:  .LBB15_8: @ %else14
-; CHECK-LE-NEXT:    add sp, #8
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB15_9: @ %cond.store
-; CHECK-LE-NEXT:    vstr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB15_2
-; CHECK-LE-NEXT:  .LBB15_10: @ %cond.store1
-; CHECK-LE-NEXT:    vmovx.f16 s4, s0
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB15_3
-; CHECK-LE-NEXT:  .LBB15_11: @ %cond.store3
-; CHECK-LE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB15_4
-; CHECK-LE-NEXT:  .LBB15_12: @ %cond.store5
-; CHECK-LE-NEXT:    vmovx.f16 s4, s1
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB15_5
-; CHECK-LE-NEXT:  .LBB15_13: @ %cond.store7
-; CHECK-LE-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB15_6
-; CHECK-LE-NEXT:  .LBB15_14: @ %cond.store9
-; CHECK-LE-NEXT:    vmovx.f16 s4, s2
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB15_7
-; CHECK-LE-NEXT:  .LBB15_15: @ %cond.store11
-; CHECK-LE-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB15_8
-; CHECK-LE-NEXT:  .LBB15_16: @ %cond.store13
-; CHECK-LE-NEXT:    vmovx.f16 s0, s3
-; CHECK-LE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-LE-NEXT:    vstrht.16 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8f16:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
 ; CHECK-BE-NEXT:    vrev64.16 q2, q1
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.i16 ne, q2, zr
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB15_9
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB15_10
-; CHECK-BE-NEXT:  .LBB15_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB15_11
-; CHECK-BE-NEXT:  .LBB15_3: @ %else4
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB15_12
-; CHECK-BE-NEXT:  .LBB15_4: @ %else6
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB15_13
-; CHECK-BE-NEXT:  .LBB15_5: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB15_14
-; CHECK-BE-NEXT:  .LBB15_6: @ %else10
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB15_15
-; CHECK-BE-NEXT:  .LBB15_7: @ %else12
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB15_16
-; CHECK-BE-NEXT:  .LBB15_8: @ %else14
-; CHECK-BE-NEXT:    add sp, #8
-; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB15_9: @ %cond.store
-; CHECK-BE-NEXT:    vstr.16 s4, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB15_2
-; CHECK-BE-NEXT:  .LBB15_10: @ %cond.store1
-; CHECK-BE-NEXT:    vmovx.f16 s0, s4
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB15_3
-; CHECK-BE-NEXT:  .LBB15_11: @ %cond.store3
-; CHECK-BE-NEXT:    vstr.16 s5, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB15_4
-; CHECK-BE-NEXT:  .LBB15_12: @ %cond.store5
-; CHECK-BE-NEXT:    vmovx.f16 s0, s5
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB15_5
-; CHECK-BE-NEXT:  .LBB15_13: @ %cond.store7
-; CHECK-BE-NEXT:    vstr.16 s6, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB15_6
-; CHECK-BE-NEXT:  .LBB15_14: @ %cond.store9
-; CHECK-BE-NEXT:    vmovx.f16 s0, s6
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB15_7
-; CHECK-BE-NEXT:  .LBB15_15: @ %cond.store11
-; CHECK-BE-NEXT:    vstr.16 s7, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB15_8
-; CHECK-BE-NEXT:  .LBB15_16: @ %cond.store13
-; CHECK-BE-NEXT:    vmovx.f16 s0, s7
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vpt.i16 ne, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <8 x i16> %b, zeroinitializer
@@ -2054,226 +415,17 @@ entry:
 define arm_aapcs_vfpcc void @masked_v8f16_align1(<8 x half> *%dest, <8 x half> %a, <8 x i16> %b) {
 ; CHECK-LE-LABEL: masked_v8f16_align1:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #40
-; CHECK-LE-NEXT:    sub sp, #40
-; CHECK-LE-NEXT:    vcmp.i16 ne, q1, zr
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r3, r1, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-LE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    rsbs r1, r1, #0
-; CHECK-LE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB16_9
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB16_10
-; CHECK-LE-NEXT:  .LBB16_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB16_11
-; CHECK-LE-NEXT:  .LBB16_3: @ %else4
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB16_12
-; CHECK-LE-NEXT:  .LBB16_4: @ %else6
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB16_13
-; CHECK-LE-NEXT:  .LBB16_5: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB16_14
-; CHECK-LE-NEXT:  .LBB16_6: @ %else10
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB16_15
-; CHECK-LE-NEXT:  .LBB16_7: @ %else12
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB16_16
-; CHECK-LE-NEXT:  .LBB16_8: @ %else14
-; CHECK-LE-NEXT:    add sp, #40
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB16_9: @ %cond.store
-; CHECK-LE-NEXT:    vstr.16 s0, [sp, #28]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #28]
-; CHECK-LE-NEXT:    strh r2, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB16_2
-; CHECK-LE-NEXT:  .LBB16_10: @ %cond.store1
-; CHECK-LE-NEXT:    vmovx.f16 s4, s0
-; CHECK-LE-NEXT:    vstr.16 s4, [sp, #24]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #24]
-; CHECK-LE-NEXT:    strh r2, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB16_3
-; CHECK-LE-NEXT:  .LBB16_11: @ %cond.store3
-; CHECK-LE-NEXT:    vstr.16 s1, [sp, #20]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #20]
-; CHECK-LE-NEXT:    strh r2, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB16_4
-; CHECK-LE-NEXT:  .LBB16_12: @ %cond.store5
-; CHECK-LE-NEXT:    vmovx.f16 s4, s1
-; CHECK-LE-NEXT:    vstr.16 s4, [sp, #16]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #16]
-; CHECK-LE-NEXT:    strh r2, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB16_5
-; CHECK-LE-NEXT:  .LBB16_13: @ %cond.store7
-; CHECK-LE-NEXT:    vstr.16 s2, [sp, #12]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #12]
-; CHECK-LE-NEXT:    strh r2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB16_6
-; CHECK-LE-NEXT:  .LBB16_14: @ %cond.store9
-; CHECK-LE-NEXT:    vmovx.f16 s4, s2
-; CHECK-LE-NEXT:    vstr.16 s4, [sp, #8]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #8]
-; CHECK-LE-NEXT:    strh r2, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB16_7
-; CHECK-LE-NEXT:  .LBB16_15: @ %cond.store11
-; CHECK-LE-NEXT:    vstr.16 s3, [sp, #4]
-; CHECK-LE-NEXT:    ldrh.w r2, [sp, #4]
-; CHECK-LE-NEXT:    strh r2, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB16_8
-; CHECK-LE-NEXT:  .LBB16_16: @ %cond.store13
-; CHECK-LE-NEXT:    vmovx.f16 s0, s3
-; CHECK-LE-NEXT:    vstr.16 s0, [sp]
-; CHECK-LE-NEXT:    ldrh.w r1, [sp]
-; CHECK-LE-NEXT:    strh r1, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #40
+; CHECK-LE-NEXT:    vpt.i16 ne, q1, zr
+; CHECK-LE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8f16_align1:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #40
-; CHECK-BE-NEXT:    sub sp, #40
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vrev16.8 q0, q2
 ; CHECK-BE-NEXT:    vrev64.16 q2, q1
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vcmp.i16 ne, q2, zr
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #12, #1
-; CHECK-BE-NEXT:    ubfx r1, r1, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r1, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB16_9
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB16_10
-; CHECK-BE-NEXT:  .LBB16_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB16_11
-; CHECK-BE-NEXT:  .LBB16_3: @ %else4
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB16_12
-; CHECK-BE-NEXT:  .LBB16_4: @ %else6
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB16_13
-; CHECK-BE-NEXT:  .LBB16_5: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB16_14
-; CHECK-BE-NEXT:  .LBB16_6: @ %else10
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB16_15
-; CHECK-BE-NEXT:  .LBB16_7: @ %else12
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB16_16
-; CHECK-BE-NEXT:  .LBB16_8: @ %else14
-; CHECK-BE-NEXT:    add sp, #40
-; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB16_9: @ %cond.store
-; CHECK-BE-NEXT:    vstr.16 s4, [sp, #28]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #28]
-; CHECK-BE-NEXT:    strh r2, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB16_2
-; CHECK-BE-NEXT:  .LBB16_10: @ %cond.store1
-; CHECK-BE-NEXT:    vmovx.f16 s0, s4
-; CHECK-BE-NEXT:    vstr.16 s0, [sp, #24]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #24]
-; CHECK-BE-NEXT:    strh r2, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB16_3
-; CHECK-BE-NEXT:  .LBB16_11: @ %cond.store3
-; CHECK-BE-NEXT:    vstr.16 s5, [sp, #20]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #20]
-; CHECK-BE-NEXT:    strh r2, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB16_4
-; CHECK-BE-NEXT:  .LBB16_12: @ %cond.store5
-; CHECK-BE-NEXT:    vmovx.f16 s0, s5
-; CHECK-BE-NEXT:    vstr.16 s0, [sp, #16]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #16]
-; CHECK-BE-NEXT:    strh r2, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB16_5
-; CHECK-BE-NEXT:  .LBB16_13: @ %cond.store7
-; CHECK-BE-NEXT:    vstr.16 s6, [sp, #12]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #12]
-; CHECK-BE-NEXT:    strh r2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB16_6
-; CHECK-BE-NEXT:  .LBB16_14: @ %cond.store9
-; CHECK-BE-NEXT:    vmovx.f16 s0, s6
-; CHECK-BE-NEXT:    vstr.16 s0, [sp, #8]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #8]
-; CHECK-BE-NEXT:    strh r2, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB16_7
-; CHECK-BE-NEXT:  .LBB16_15: @ %cond.store11
-; CHECK-BE-NEXT:    vstr.16 s7, [sp, #4]
-; CHECK-BE-NEXT:    ldrh.w r2, [sp, #4]
-; CHECK-BE-NEXT:    strh r2, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB16_8
-; CHECK-BE-NEXT:  .LBB16_16: @ %cond.store13
-; CHECK-BE-NEXT:    vmovx.f16 s0, s7
-; CHECK-BE-NEXT:    vstr.16 s0, [sp]
-; CHECK-BE-NEXT:    ldrh.w r1, [sp]
-; CHECK-BE-NEXT:    strh r1, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #40
+; CHECK-BE-NEXT:    vpt.i16 ne, q2, zr
+; CHECK-BE-NEXT:    vstrbt.8 q0, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp ugt <8 x i16> %b, zeroinitializer
@@ -2284,201 +436,23 @@ entry:
 define i8* @masked_v8f16_pre(i8* %y, i8* %x, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_pre:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
-; CHECK-LE-NEXT:    adds r0, #4
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r2, #0
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r3, r12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-LE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-LE-NEXT:    uxtb r1, r2
-; CHECK-LE-NEXT:    lsls r2, r2, #31
-; CHECK-LE-NEXT:    bne .LBB17_9
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bmi .LBB17_10
-; CHECK-LE-NEXT:  .LBB17_2: @ %else2
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bmi .LBB17_11
-; CHECK-LE-NEXT:  .LBB17_3: @ %else4
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bmi .LBB17_12
-; CHECK-LE-NEXT:  .LBB17_4: @ %else6
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bmi .LBB17_13
-; CHECK-LE-NEXT:  .LBB17_5: @ %else8
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bmi .LBB17_14
-; CHECK-LE-NEXT:  .LBB17_6: @ %else10
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bmi .LBB17_15
-; CHECK-LE-NEXT:  .LBB17_7: @ %else12
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bmi .LBB17_16
-; CHECK-LE-NEXT:  .LBB17_8: @ %else14
-; CHECK-LE-NEXT:    add sp, #8
-; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB17_9: @ %cond.store
-; CHECK-LE-NEXT:    vstr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r2, r1, #30
-; CHECK-LE-NEXT:    bpl .LBB17_2
-; CHECK-LE-NEXT:  .LBB17_10: @ %cond.store1
-; CHECK-LE-NEXT:    vmovx.f16 s4, s0
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    lsls r2, r1, #29
-; CHECK-LE-NEXT:    bpl .LBB17_3
-; CHECK-LE-NEXT:  .LBB17_11: @ %cond.store3
-; CHECK-LE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r2, r1, #28
-; CHECK-LE-NEXT:    bpl .LBB17_4
-; CHECK-LE-NEXT:  .LBB17_12: @ %cond.store5
-; CHECK-LE-NEXT:    vmovx.f16 s4, s1
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    lsls r2, r1, #27
-; CHECK-LE-NEXT:    bpl .LBB17_5
-; CHECK-LE-NEXT:  .LBB17_13: @ %cond.store7
-; CHECK-LE-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r2, r1, #26
-; CHECK-LE-NEXT:    bpl .LBB17_6
-; CHECK-LE-NEXT:  .LBB17_14: @ %cond.store9
-; CHECK-LE-NEXT:    vmovx.f16 s4, s2
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:    lsls r2, r1, #25
-; CHECK-LE-NEXT:    bpl .LBB17_7
-; CHECK-LE-NEXT:  .LBB17_15: @ %cond.store11
-; CHECK-LE-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-LE-NEXT:    lsls r1, r1, #24
-; CHECK-LE-NEXT:    bpl .LBB17_8
-; CHECK-LE-NEXT:  .LBB17_16: @ %cond.store13
-; CHECK-LE-NEXT:    vmovx.f16 s0, s3
-; CHECK-LE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0, #4]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: masked_v8f16_pre:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
-; CHECK-BE-NEXT:    adds r0, #4
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r3, r12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #1, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #3, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #4, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #5, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #6, #1
-; CHECK-BE-NEXT:    ubfx r3, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-BE-NEXT:    bfi r2, r3, #7, #1
-; CHECK-BE-NEXT:    uxtb r1, r2
-; CHECK-BE-NEXT:    lsls r2, r2, #31
-; CHECK-BE-NEXT:    bne .LBB17_9
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB17_10
-; CHECK-BE-NEXT:  .LBB17_2: @ %else2
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bmi .LBB17_11
-; CHECK-BE-NEXT:  .LBB17_3: @ %else4
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bmi .LBB17_12
-; CHECK-BE-NEXT:  .LBB17_4: @ %else6
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bmi .LBB17_13
-; CHECK-BE-NEXT:  .LBB17_5: @ %else8
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bmi .LBB17_14
-; CHECK-BE-NEXT:  .LBB17_6: @ %else10
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bmi .LBB17_15
-; CHECK-BE-NEXT:  .LBB17_7: @ %else12
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bmi .LBB17_16
-; CHECK-BE-NEXT:  .LBB17_8: @ %else14
-; CHECK-BE-NEXT:    add sp, #8
-; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB17_9: @ %cond.store
-; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r2, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB17_2
-; CHECK-BE-NEXT:  .LBB17_10: @ %cond.store1
-; CHECK-BE-NEXT:    vmovx.f16 s4, s0
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r2, r1, #29
-; CHECK-BE-NEXT:    bpl .LBB17_3
-; CHECK-BE-NEXT:  .LBB17_11: @ %cond.store3
-; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r2, r1, #28
-; CHECK-BE-NEXT:    bpl .LBB17_4
-; CHECK-BE-NEXT:  .LBB17_12: @ %cond.store5
-; CHECK-BE-NEXT:    vmovx.f16 s4, s1
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-BE-NEXT:    lsls r2, r1, #27
-; CHECK-BE-NEXT:    bpl .LBB17_5
-; CHECK-BE-NEXT:  .LBB17_13: @ %cond.store7
-; CHECK-BE-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r2, r1, #26
-; CHECK-BE-NEXT:    bpl .LBB17_6
-; CHECK-BE-NEXT:  .LBB17_14: @ %cond.store9
-; CHECK-BE-NEXT:    vmovx.f16 s4, s2
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-BE-NEXT:    lsls r2, r1, #25
-; CHECK-BE-NEXT:    bpl .LBB17_7
-; CHECK-BE-NEXT:  .LBB17_15: @ %cond.store11
-; CHECK-BE-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-BE-NEXT:    lsls r1, r1, #24
-; CHECK-BE-NEXT:    bpl .LBB17_8
-; CHECK-BE-NEXT:  .LBB17_16: @ %cond.store13
-; CHECK-BE-NEXT:    vmovx.f16 s0, s3
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0, #4]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
@@ -2493,194 +467,24 @@ entry:
 define i8* @masked_v8f16_post(i8* %y, i8* %x, <8 x i16> %a) {
 ; CHECK-LE-LABEL: masked_v8f16_post:
 ; CHECK-LE:       @ %bb.0: @ %entry
-; CHECK-LE-NEXT:    .pad #8
-; CHECK-LE-NEXT:    sub sp, #8
-; CHECK-LE-NEXT:    vldr d1, [sp, #8]
+; CHECK-LE-NEXT:    vldr d1, [sp]
+; CHECK-LE-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-LE-NEXT:    vmov d0, r2, r3
-; CHECK-LE-NEXT:    movs r3, #0
-; CHECK-LE-NEXT:    vcmp.s16 gt, q0, zr
-; CHECK-LE-NEXT:    vmrs r12, p0
-; CHECK-LE-NEXT:    and r2, r12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-LE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    vldrw.u32 q0, [r1]
-; CHECK-LE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-LE-NEXT:    uxtb r2, r3
-; CHECK-LE-NEXT:    lsls r1, r3, #31
-; CHECK-LE-NEXT:    bne .LBB18_12
-; CHECK-LE-NEXT:  @ %bb.1: @ %else
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    bmi .LBB18_13
-; CHECK-LE-NEXT:  .LBB18_2: @ %else2
-; CHECK-LE-NEXT:    lsls r1, r2, #29
-; CHECK-LE-NEXT:    bmi .LBB18_14
-; CHECK-LE-NEXT:  .LBB18_3: @ %else4
-; CHECK-LE-NEXT:    lsls r1, r2, #28
-; CHECK-LE-NEXT:    bmi .LBB18_15
-; CHECK-LE-NEXT:  .LBB18_4: @ %else6
-; CHECK-LE-NEXT:    lsls r1, r2, #27
-; CHECK-LE-NEXT:    bmi .LBB18_16
-; CHECK-LE-NEXT:  .LBB18_5: @ %else8
-; CHECK-LE-NEXT:    lsls r1, r2, #26
-; CHECK-LE-NEXT:    bpl .LBB18_7
-; CHECK-LE-NEXT:  .LBB18_6: @ %cond.store9
-; CHECK-LE-NEXT:    vmovx.f16 s4, s2
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-LE-NEXT:  .LBB18_7: @ %else10
-; CHECK-LE-NEXT:    adds r1, r0, #4
-; CHECK-LE-NEXT:    lsls r3, r2, #25
-; CHECK-LE-NEXT:    bpl .LBB18_9
-; CHECK-LE-NEXT:  @ %bb.8: @ %cond.store11
-; CHECK-LE-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-LE-NEXT:  .LBB18_9: @ %else12
-; CHECK-LE-NEXT:    lsls r2, r2, #24
-; CHECK-LE-NEXT:    bpl .LBB18_11
-; CHECK-LE-NEXT:  @ %bb.10: @ %cond.store13
-; CHECK-LE-NEXT:    vmovx.f16 s0, s3
-; CHECK-LE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-LE-NEXT:  .LBB18_11: @ %else14
-; CHECK-LE-NEXT:    mov r0, r1
-; CHECK-LE-NEXT:    add sp, #8
+; CHECK-LE-NEXT:    vpt.s16 gt, q0, zr
+; CHECK-LE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-LE-NEXT:    adds r0, #4
 ; CHECK-LE-NEXT:    bx lr
-; CHECK-LE-NEXT:  .LBB18_12: @ %cond.store
-; CHECK-LE-NEXT:    vstr.16 s0, [r0]
-; CHECK-LE-NEXT:    lsls r1, r2, #30
-; CHECK-LE-NEXT:    bpl .LBB18_2
-; CHECK-LE-NEXT:  .LBB18_13: @ %cond.store1
-; CHECK-LE-NEXT:    vmovx.f16 s4, s0
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-LE-NEXT:    lsls r1, r2, #29
-; CHECK-LE-NEXT:    bpl .LBB18_3
-; CHECK-LE-NEXT:  .LBB18_14: @ %cond.store3
-; CHECK-LE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-LE-NEXT:    lsls r1, r2, #28
-; CHECK-LE-NEXT:    bpl .LBB18_4
-; CHECK-LE-NEXT:  .LBB18_15: @ %cond.store5
-; CHECK-LE-NEXT:    vmovx.f16 s4, s1
-; CHECK-LE-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-LE-NEXT:    lsls r1, r2, #27
-; CHECK-LE-NEXT:    bpl .LBB18_5
-; CHECK-LE-NEXT:  .LBB18_16: @ %cond.store7
-; CHECK-LE-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-LE-NEXT:    lsls r1, r2, #26
-; CHECK-LE-NEXT:    bmi .LBB18_6
-; CHECK-LE-NEXT:    b .LBB18_7
 ;
 ; CHECK-BE-LABEL: masked_v8f16_post:
 ; CHECK-BE:       @ %bb.0: @ %entry
-; CHECK-BE-NEXT:    .pad #8
-; CHECK-BE-NEXT:    sub sp, #8
-; CHECK-BE-NEXT:    vldr d1, [sp, #8]
+; CHECK-BE-NEXT:    vldr d1, [sp]
+; CHECK-BE-NEXT:    vldrh.u16 q1, [r1]
 ; CHECK-BE-NEXT:    vmov d0, r3, r2
-; CHECK-BE-NEXT:    movs r3, #0
-; CHECK-BE-NEXT:    vrev64.16 q1, q0
-; CHECK-BE-NEXT:    vcmp.s16 gt, q1, zr
-; CHECK-BE-NEXT:    vmrs r12, p0
-; CHECK-BE-NEXT:    and r2, r12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #4, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #6, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #3, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #8, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #4, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #10, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #5, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #12, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #6, #1
-; CHECK-BE-NEXT:    ubfx r2, r12, #14, #1
-; CHECK-BE-NEXT:    rsbs r2, r2, #0
-; CHECK-BE-NEXT:    vldrh.u16 q0, [r1]
-; CHECK-BE-NEXT:    bfi r3, r2, #7, #1
-; CHECK-BE-NEXT:    uxtb r2, r3
-; CHECK-BE-NEXT:    lsls r1, r3, #31
-; CHECK-BE-NEXT:    bne .LBB18_12
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    bmi .LBB18_13
-; CHECK-BE-NEXT:  .LBB18_2: @ %else2
-; CHECK-BE-NEXT:    lsls r1, r2, #29
-; CHECK-BE-NEXT:    bmi .LBB18_14
-; CHECK-BE-NEXT:  .LBB18_3: @ %else4
-; CHECK-BE-NEXT:    lsls r1, r2, #28
-; CHECK-BE-NEXT:    bmi .LBB18_15
-; CHECK-BE-NEXT:  .LBB18_4: @ %else6
-; CHECK-BE-NEXT:    lsls r1, r2, #27
-; CHECK-BE-NEXT:    bmi .LBB18_16
-; CHECK-BE-NEXT:  .LBB18_5: @ %else8
-; CHECK-BE-NEXT:    lsls r1, r2, #26
-; CHECK-BE-NEXT:    bpl .LBB18_7
-; CHECK-BE-NEXT:  .LBB18_6: @ %cond.store9
-; CHECK-BE-NEXT:    vmovx.f16 s4, s2
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #10]
-; CHECK-BE-NEXT:  .LBB18_7: @ %else10
-; CHECK-BE-NEXT:    adds r1, r0, #4
-; CHECK-BE-NEXT:    lsls r3, r2, #25
-; CHECK-BE-NEXT:    bpl .LBB18_9
-; CHECK-BE-NEXT:  @ %bb.8: @ %cond.store11
-; CHECK-BE-NEXT:    vstr.16 s3, [r0, #12]
-; CHECK-BE-NEXT:  .LBB18_9: @ %else12
-; CHECK-BE-NEXT:    lsls r2, r2, #24
-; CHECK-BE-NEXT:    bpl .LBB18_11
-; CHECK-BE-NEXT:  @ %bb.10: @ %cond.store13
-; CHECK-BE-NEXT:    vmovx.f16 s0, s3
-; CHECK-BE-NEXT:    vstr.16 s0, [r0, #14]
-; CHECK-BE-NEXT:  .LBB18_11: @ %else14
-; CHECK-BE-NEXT:    mov r0, r1
-; CHECK-BE-NEXT:    add sp, #8
+; CHECK-BE-NEXT:    vrev64.16 q2, q0
+; CHECK-BE-NEXT:    vpt.s16 gt, q2, zr
+; CHECK-BE-NEXT:    vstrht.16 q1, [r0]
+; CHECK-BE-NEXT:    adds r0, #4
 ; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB18_12: @ %cond.store
-; CHECK-BE-NEXT:    vstr.16 s0, [r0]
-; CHECK-BE-NEXT:    lsls r1, r2, #30
-; CHECK-BE-NEXT:    bpl .LBB18_2
-; CHECK-BE-NEXT:  .LBB18_13: @ %cond.store1
-; CHECK-BE-NEXT:    vmovx.f16 s4, s0
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #2]
-; CHECK-BE-NEXT:    lsls r1, r2, #29
-; CHECK-BE-NEXT:    bpl .LBB18_3
-; CHECK-BE-NEXT:  .LBB18_14: @ %cond.store3
-; CHECK-BE-NEXT:    vstr.16 s1, [r0, #4]
-; CHECK-BE-NEXT:    lsls r1, r2, #28
-; CHECK-BE-NEXT:    bpl .LBB18_4
-; CHECK-BE-NEXT:  .LBB18_15: @ %cond.store5
-; CHECK-BE-NEXT:    vmovx.f16 s4, s1
-; CHECK-BE-NEXT:    vstr.16 s4, [r0, #6]
-; CHECK-BE-NEXT:    lsls r1, r2, #27
-; CHECK-BE-NEXT:    bpl .LBB18_5
-; CHECK-BE-NEXT:  .LBB18_16: @ %cond.store7
-; CHECK-BE-NEXT:    vstr.16 s2, [r0, #8]
-; CHECK-BE-NEXT:    lsls r1, r2, #26
-; CHECK-BE-NEXT:    bmi .LBB18_6
-; CHECK-BE-NEXT:    b .LBB18_7
 entry:
   %z = getelementptr inbounds i8, i8* %y, i32 4
   %0 = bitcast i8* %x to <8 x half>*
@@ -2717,15 +521,11 @@ define arm_aapcs_vfpcc void @masked_v2i64(<2 x i64> *%dest, <2 x i64> %a) {
 ; CHECK-LE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-LE-NEXT:    and r1, r3, #3
 ; CHECK-LE-NEXT:    lsls r2, r3, #31
-; CHECK-LE-NEXT:    ittt ne
-; CHECK-LE-NEXT:    vmovne r2, s1
-; CHECK-LE-NEXT:    vmovne r3, s0
-; CHECK-LE-NEXT:    strdne r3, r2, [r0]
+; CHECK-LE-NEXT:    it ne
+; CHECK-LE-NEXT:    vstrne d0, [r0]
 ; CHECK-LE-NEXT:    lsls r1, r1, #30
-; CHECK-LE-NEXT:    ittt mi
-; CHECK-LE-NEXT:    vmovmi r1, s3
-; CHECK-LE-NEXT:    vmovmi r2, s2
-; CHECK-LE-NEXT:    strdmi r2, r1, [r0, #8]
+; CHECK-LE-NEXT:    it mi
+; CHECK-LE-NEXT:    vstrmi d1, [r0, #8]
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
@@ -2754,25 +554,11 @@ define arm_aapcs_vfpcc void @masked_v2i64(<2 x i64> *%dest, <2 x i64> %a) {
 ; CHECK-BE-NEXT:    bfi r3, r1, #0, #1
 ; CHECK-BE-NEXT:    and r1, r3, #3
 ; CHECK-BE-NEXT:    lsls r2, r3, #31
-; CHECK-BE-NEXT:    bne .LBB19_3
-; CHECK-BE-NEXT:  @ %bb.1: @ %else
-; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bmi .LBB19_4
-; CHECK-BE-NEXT:  .LBB19_2: @ %else2
-; CHECK-BE-NEXT:    add sp, #4
-; CHECK-BE-NEXT:    bx lr
-; CHECK-BE-NEXT:  .LBB19_3: @ %cond.store
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vmov r2, s5
-; CHECK-BE-NEXT:    vmov r3, s4
-; CHECK-BE-NEXT:    strd r3, r2, [r0]
+; CHECK-BE-NEXT:    it ne
+; CHECK-BE-NEXT:    vstrne d0, [r0]
 ; CHECK-BE-NEXT:    lsls r1, r1, #30
-; CHECK-BE-NEXT:    bpl .LBB19_2
-; CHECK-BE-NEXT:  .LBB19_4: @ %cond.store1
-; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    vmov r1, s7
-; CHECK-BE-NEXT:    vmov r2, s6
-; CHECK-BE-NEXT:    strd r2, r1, [r0, #8]
+; CHECK-BE-NEXT:    it mi
+; CHECK-BE-NEXT:    vstrmi d1, [r0, #8]
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
index 7578a842ec991..d227ebd35e57d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-and.ll
@@ -19,8 +19,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpnez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -35,8 +34,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsltz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -51,8 +49,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -67,8 +64,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpslez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -83,8 +79,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -112,8 +107,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpugtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -128,8 +122,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpulez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpulez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 cs, q1, zr
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -160,8 +153,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpeq_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -176,8 +168,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpne_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -192,8 +183,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpslt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -208,8 +198,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsgt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -224,8 +213,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsle_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -240,8 +228,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -256,8 +243,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpult_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpult_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 hi, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -272,8 +258,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpugt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpugt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 hi, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -288,8 +273,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpule_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpule_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -304,8 +288,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpuge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpuge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 cs, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -321,8 +304,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpeqr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpeqr_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -339,8 +321,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpner_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpner_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -357,9 +338,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsltr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpsltr_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
 ; CHECK-NEXT:    vdup.32 q2, r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -376,8 +356,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpsgtr_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -394,9 +373,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsler_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpsler_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
 ; CHECK-NEXT:    vdup.32 q2, r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -413,8 +391,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsger_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpsger_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -431,9 +408,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpultr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpultr_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
 ; CHECK-NEXT:    vdup.32 q2, r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 hi, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -450,8 +426,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpugtr_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpugtr_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 hi, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -468,9 +443,8 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpuler_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpuler_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
 ; CHECK-NEXT:    vdup.32 q2, r0
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 cs, q2, q1
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -487,8 +461,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpuger_v4i1(<4 x i32> %a, <4 x i32> %b, i32 %c) {
 ; CHECK-LABEL: cmpuger_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.u32 cs, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -522,8 +495,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
 ; CHECK-LABEL: cmpeq_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -538,8 +510,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @cmpeqr_v8i1(<8 x i16> %a, <8 x i16> %b, i16 %c) {
 ; CHECK-LABEL: cmpeqr_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i16 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 eq, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -572,8 +543,7 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
 ; CHECK-LABEL: cmpeq_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 eq, q1, q2
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
@@ -588,8 +558,7 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @cmpeqr_v16i1(<16 x i8> %a, <16 x i8> %b, i8 %c) {
 ; CHECK-LABEL: cmpeqr_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 eq, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i8 eq, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 eq, q1, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
index f8f2e0b56134a..61a78eb722e0c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-bitcast.ll
@@ -218,10 +218,10 @@ define arm_aapcs_vfpcc i4 @bitcast_from_v4i1(<4 x i32> %a) {
 ; CHECK-LE-NEXT:    .pad #4
 ; CHECK-LE-NEXT:    sub sp, #4
 ; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r2, r1, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
@@ -232,7 +232,6 @@ define arm_aapcs_vfpcc i4 @bitcast_from_v4i1(<4 x i32> %a) {
 ; CHECK-LE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-LE-NEXT:    rsbs r1, r1, #0
 ; CHECK-LE-NEXT:    bfi r0, r1, #3, #1
-; CHECK-LE-NEXT:    and r0, r0, #15
 ; CHECK-LE-NEXT:    add sp, #4
 ; CHECK-LE-NEXT:    bx lr
 ;
@@ -241,22 +240,21 @@ define arm_aapcs_vfpcc i4 @bitcast_from_v4i1(<4 x i32> %a) {
 ; CHECK-BE-NEXT:    .pad #4
 ; CHECK-BE-NEXT:    sub sp, #4
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
-; CHECK-BE-NEXT:    movs r3, #0
 ; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
-; CHECK-BE-NEXT:    vmrs r0, p0
-; CHECK-BE-NEXT:    and r2, r0, #1
-; CHECK-BE-NEXT:    ubfx r1, r0, #4, #1
+; CHECK-BE-NEXT:    vmrs r1, p0
+; CHECK-BE-NEXT:    and r0, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
+; CHECK-BE-NEXT:    movs r0, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r0, r2, #2, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
-; CHECK-BE-NEXT:    bfi r3, r1, #1, #1
-; CHECK-BE-NEXT:    ubfx r1, r0, #8, #1
-; CHECK-BE-NEXT:    ubfx r0, r0, #12, #1
-; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r3, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r0, r0, #0
-; CHECK-BE-NEXT:    bfi r3, r0, #3, #1
-; CHECK-BE-NEXT:    and r0, r3, #15
+; CHECK-BE-NEXT:    bfi r0, r1, #3, #1
 ; CHECK-BE-NEXT:    add sp, #4
 ; CHECK-BE-NEXT:    bx lr
 entry:
@@ -271,10 +269,10 @@ define arm_aapcs_vfpcc i8 @bitcast_from_v8i1(<8 x i16> %a) {
 ; CHECK-LE-NEXT:    .pad #8
 ; CHECK-LE-NEXT:    sub sp, #8
 ; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    vmrs r1, p0
-; CHECK-LE-NEXT:    and r2, r1, #1
-; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    and r0, r1, #1
+; CHECK-LE-NEXT:    rsbs r2, r0, #0
+; CHECK-LE-NEXT:    movs r0, #0
 ; CHECK-LE-NEXT:    bfi r0, r2, #0, #1
 ; CHECK-LE-NEXT:    ubfx r2, r1, #2, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
@@ -308,12 +306,12 @@ define arm_aapcs_vfpcc i8 @bitcast_from_v8i1(<8 x i16> %a) {
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
 ; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    ubfx r0, r1, #2, #1
-; CHECK-BE-NEXT:    rsbs r2, r0, #0
 ; CHECK-BE-NEXT:    and r0, r1, #1
-; CHECK-BE-NEXT:    rsbs r3, r0, #0
+; CHECK-BE-NEXT:    rsbs r2, r0, #0
 ; CHECK-BE-NEXT:    movs r0, #0
-; CHECK-BE-NEXT:    bfi r0, r3, #0, #1
+; CHECK-BE-NEXT:    bfi r0, r2, #0, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #2, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
 ; CHECK-BE-NEXT:    bfi r0, r2, #1, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
 ; CHECK-BE-NEXT:    rsbs r2, r2, #0
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
index 49d3e74e5a4c5..46b418daf3b3c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-build-var.ll
@@ -44,15 +44,10 @@ define arm_aapcs_vfpcc <4 x i32> @build_varN_v4i1(i32 %s, i32 %t, <4 x i32> %a,
 ; CHECK-LABEL: build_varN_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r1, #0
 ; CHECK-NEXT:    cset r0, lo
 ; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r1, r0, #0, #4
-; CHECK-NEXT:    bfi r1, r0, #4, #4
-; CHECK-NEXT:    bfi r1, r0, #8, #4
-; CHECK-NEXT:    bfi r1, r0, #12, #4
-; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -106,19 +101,10 @@ define arm_aapcs_vfpcc <8 x i16> @build_varN_v8i1(i32 %s, i32 %t, <8 x i16> %a,
 ; CHECK-LABEL: build_varN_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r1, #0
 ; CHECK-NEXT:    cset r0, lo
 ; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r1, r0, #0, #2
-; CHECK-NEXT:    bfi r1, r0, #2, #2
-; CHECK-NEXT:    bfi r1, r0, #4, #2
-; CHECK-NEXT:    bfi r1, r0, #6, #2
-; CHECK-NEXT:    bfi r1, r0, #8, #2
-; CHECK-NEXT:    bfi r1, r0, #10, #2
-; CHECK-NEXT:    bfi r1, r0, #12, #2
-; CHECK-NEXT:    bfi r1, r0, #14, #2
-; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
@@ -172,27 +158,10 @@ define arm_aapcs_vfpcc <16 x i8> @build_varN_v16i1(i32 %s, i32 %t, <16 x i8> %a,
 ; CHECK-LABEL: build_varN_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
 ; CHECK-NEXT:    cmp r0, r1
-; CHECK-NEXT:    mov.w r1, #0
 ; CHECK-NEXT:    cset r0, lo
 ; CHECK-NEXT:    and r0, r0, #1
 ; CHECK-NEXT:    rsbs r0, r0, #0
-; CHECK-NEXT:    bfi r1, r0, #0, #1
-; CHECK-NEXT:    bfi r1, r0, #1, #1
-; CHECK-NEXT:    bfi r1, r0, #2, #1
-; CHECK-NEXT:    bfi r1, r0, #3, #1
-; CHECK-NEXT:    bfi r1, r0, #4, #1
-; CHECK-NEXT:    bfi r1, r0, #5, #1
-; CHECK-NEXT:    bfi r1, r0, #6, #1
-; CHECK-NEXT:    bfi r1, r0, #7, #1
-; CHECK-NEXT:    bfi r1, r0, #8, #1
-; CHECK-NEXT:    bfi r1, r0, #9, #1
-; CHECK-NEXT:    bfi r1, r0, #10, #1
-; CHECK-NEXT:    bfi r1, r0, #11, #1
-; CHECK-NEXT:    bfi r1, r0, #12, #1
-; CHECK-NEXT:    bfi r1, r0, #13, #1
-; CHECK-NEXT:    bfi r1, r0, #14, #1
-; CHECK-NEXT:    bfi r1, r0, #15, #1
-; CHECK-NEXT:    vmsr p0, r1
+; CHECK-NEXT:    vmsr p0, r0
 ; CHECK-NEXT:    vpsel q0, q0, q1
 ; CHECK-NEXT:    bx lr
 entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
index d314d0525c44c..21af261170e6d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-loadstore.ll
@@ -176,44 +176,42 @@ define arm_aapcs_vfpcc void @store_v4i1(<4 x i1> *%dst, <4 x i32> %a) {
 ; CHECK-LE-LABEL: store_v4i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    vcmp.i32 eq, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
-; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #1, #1
-; CHECK-LE-NEXT:    ubfx r3, r2, #8, #1
-; CHECK-LE-NEXT:    ubfx r2, r2, #12, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
-; CHECK-LE-NEXT:    bfi r1, r3, #2, #1
+; CHECK-LE-NEXT:    movs r3, #0
+; CHECK-LE-NEXT:    vmrs r1, p0
+; CHECK-LE-NEXT:    and r2, r1, #1
 ; CHECK-LE-NEXT:    rsbs r2, r2, #0
-; CHECK-LE-NEXT:    bfi r1, r2, #3, #1
-; CHECK-LE-NEXT:    and r1, r1, #15
-; CHECK-LE-NEXT:    strb r1, [r0]
+; CHECK-LE-NEXT:    bfi r3, r2, #0, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #4, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-LE-NEXT:    ubfx r2, r1, #8, #1
+; CHECK-LE-NEXT:    ubfx r1, r1, #12, #1
+; CHECK-LE-NEXT:    rsbs r2, r2, #0
+; CHECK-LE-NEXT:    bfi r3, r2, #2, #1
+; CHECK-LE-NEXT:    rsbs r1, r1, #0
+; CHECK-LE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-LE-NEXT:    strb r3, [r0]
 ; CHECK-LE-NEXT:    bx lr
 ;
 ; CHECK-BE-LABEL: store_v4i1:
 ; CHECK-BE:       @ %bb.0: @ %entry
 ; CHECK-BE-NEXT:    vrev64.32 q1, q0
+; CHECK-BE-NEXT:    movs r3, #0
 ; CHECK-BE-NEXT:    vcmp.i32 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r1, p0
-; CHECK-BE-NEXT:    and r3, r1, #1
+; CHECK-BE-NEXT:    and r2, r1, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #0, #1
 ; CHECK-BE-NEXT:    ubfx r2, r1, #4, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
-; CHECK-BE-NEXT:    rsb.w r12, r2, #0
-; CHECK-BE-NEXT:    movs r2, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #0, #1
-; CHECK-BE-NEXT:    ubfx r3, r1, #8, #1
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #1, #1
+; CHECK-BE-NEXT:    ubfx r2, r1, #8, #1
 ; CHECK-BE-NEXT:    ubfx r1, r1, #12, #1
-; CHECK-BE-NEXT:    bfi r2, r12, #1, #1
-; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    rsbs r2, r2, #0
+; CHECK-BE-NEXT:    bfi r3, r2, #2, #1
 ; CHECK-BE-NEXT:    rsbs r1, r1, #0
-; CHECK-BE-NEXT:    bfi r2, r3, #2, #1
-; CHECK-BE-NEXT:    bfi r2, r1, #3, #1
-; CHECK-BE-NEXT:    and r1, r2, #15
-; CHECK-BE-NEXT:    strb r1, [r0]
+; CHECK-BE-NEXT:    bfi r3, r1, #3, #1
+; CHECK-BE-NEXT:    strb r3, [r0]
 ; CHECK-BE-NEXT:    bx lr
 entry:
   %c = icmp eq <4 x i32> %a, zeroinitializer
@@ -225,10 +223,10 @@ define arm_aapcs_vfpcc void @store_v8i1(<8 x i1> *%dst, <8 x i16> %a) {
 ; CHECK-LE-LABEL: store_v8i1:
 ; CHECK-LE:       @ %bb.0: @ %entry
 ; CHECK-LE-NEXT:    vcmp.i16 eq, q0, zr
-; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    vmrs r2, p0
-; CHECK-LE-NEXT:    and r3, r2, #1
-; CHECK-LE-NEXT:    rsbs r3, r3, #0
+; CHECK-LE-NEXT:    and r1, r2, #1
+; CHECK-LE-NEXT:    rsbs r3, r1, #0
+; CHECK-LE-NEXT:    movs r1, #0
 ; CHECK-LE-NEXT:    bfi r1, r3, #0, #1
 ; CHECK-LE-NEXT:    ubfx r3, r2, #2, #1
 ; CHECK-LE-NEXT:    rsbs r3, r3, #0
@@ -259,14 +257,14 @@ define arm_aapcs_vfpcc void @store_v8i1(<8 x i1> *%dst, <8 x i16> %a) {
 ; CHECK-BE-NEXT:    vrev64.16 q1, q0
 ; CHECK-BE-NEXT:    vcmp.i16 eq, q1, zr
 ; CHECK-BE-NEXT:    vmrs r2, p0
-; CHECK-BE-NEXT:    ubfx r1, r2, #2, #1
-; CHECK-BE-NEXT:    rsb.w r12, r1, #0
 ; CHECK-BE-NEXT:    and r1, r2, #1
 ; CHECK-BE-NEXT:    rsbs r3, r1, #0
 ; CHECK-BE-NEXT:    movs r1, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #0, #1
+; CHECK-BE-NEXT:    ubfx r3, r2, #2, #1
+; CHECK-BE-NEXT:    rsbs r3, r3, #0
+; CHECK-BE-NEXT:    bfi r1, r3, #1, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #4, #1
-; CHECK-BE-NEXT:    bfi r1, r12, #1, #1
 ; CHECK-BE-NEXT:    rsbs r3, r3, #0
 ; CHECK-BE-NEXT:    bfi r1, r3, #2, #1
 ; CHECK-BE-NEXT:    ubfx r3, r2, #6, #1
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
index 80b145ac04c72..bc94f8ba91876 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-not.ll
@@ -383,8 +383,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @vpnot_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: vpnot_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.s32 lt, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.s32 lt, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpst
diff --git a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
index 5dce995639992..26ff81c6f3941 100644
--- a/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-pred-or.ll
@@ -4,8 +4,7 @@
 define arm_aapcs_vfpcc <4 x i32> @cmpeqz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpeqz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -21,8 +20,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpnez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpnez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -38,8 +36,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsltz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsltz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 ge, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -55,8 +52,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -72,8 +68,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpslez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpslez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 gt, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -89,8 +84,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgez_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpsgez_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -120,8 +114,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpugtz_v4i1(<4 x i32> %a, <4 x i32> %b) {
 ; CHECK-LABEL: cmpugtz_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -170,8 +163,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpeq_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpeq_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 ne, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -187,8 +179,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpne_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpne_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i32 eq, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -204,8 +195,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpslt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpslt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q2, q1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -221,8 +211,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsgt_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsgt_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 le, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -238,8 +227,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsle_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsle_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q2, q1
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -255,8 +243,7 @@ entry:
 define arm_aapcs_vfpcc <4 x i32> @cmpsge_v4i1(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) {
 ; CHECK-LABEL: cmpsge_v4i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i32 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i32 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.s32 lt, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -351,8 +338,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @cmpeqz_v8i1(<8 x i16> %a, <8 x i16> %b) {
 ; CHECK-LABEL: cmpeqz_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 ne, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -368,8 +354,7 @@ entry:
 define arm_aapcs_vfpcc <8 x i16> @cmpeq_v8i1(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) {
 ; CHECK-LABEL: cmpeq_v8i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i16 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i16 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i16 ne, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -386,8 +371,7 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @cmpeqz_v16i1(<16 x i8> %a, <16 x i8> %b) {
 ; CHECK-LABEL: cmpeqz_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 ne, q1, zr
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
@@ -403,8 +387,7 @@ entry:
 define arm_aapcs_vfpcc <16 x i8> @cmpeq_v16i1(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) {
 ; CHECK-LABEL: cmpeq_v16i1:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    vcmp.i8 ne, q0, zr
-; CHECK-NEXT:    vpst
+; CHECK-NEXT:    vpt.i8 ne, q0, zr
 ; CHECK-NEXT:    vcmpt.i8 ne, q1, q2
 ; CHECK-NEXT:    vpnot
 ; CHECK-NEXT:    vpsel q0, q0, q1
diff --git a/llvm/test/CodeGen/Thumb2/mve-sext.ll b/llvm/test/CodeGen/Thumb2/mve-sext.ll
index 452e68405feb3..be4148821815c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-sext.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-sext.ll
@@ -32,6 +32,131 @@ entry:
   ret <4 x i32> %0
 }
 
+define arm_aapcs_vfpcc <16 x i16> @sext_v16i8_v16i16(<16 x i8> %src) {
+; CHECK-LABEL: sext_v16i8_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmovlb.s8 q2, q1
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <16 x i8> %src to <16 x i16>
+  ret <16 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i32> @sext_v8i16_v8i32(<8 x i16> %src) {
+; CHECK-LABEL: sext_v8i16_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmovlb.s16 q2, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <8 x i16> %src to <8 x i32>
+  ret <8 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i32> @sext_v16i8_v16i32(<16 x i8> %src) {
+; CHECK-LABEL: sext_v16i8_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q4, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.32 q3[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.32 q3[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.32 q3[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.32 q3[3], r0
+; CHECK-NEXT:    vmovlb.s8 q1, q1
+; CHECK-NEXT:    vmovlb.s8 q2, q2
+; CHECK-NEXT:    vmovlb.s8 q0, q3
+; CHECK-NEXT:    vmovlb.s16 q3, q0
+; CHECK-NEXT:    vmovlb.s16 q1, q1
+; CHECK-NEXT:    vmovlb.s16 q2, q2
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = sext <16 x i8> %src to <16 x i32>
+  ret <16 x i32> %0
+}
+
 define arm_aapcs_vfpcc <2 x i64> @sext_v2i32_v2i64(<2 x i32> %src) {
 ; CHECK-LABEL: sext_v2i32_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
@@ -82,16 +207,138 @@ entry:
   ret <4 x i32> %0
 }
 
+define arm_aapcs_vfpcc <16 x i16> @zext_v16i8_v16i16(<16 x i8> %src) {
+; CHECK-LABEL: zext_v16i8_v16i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmovlb.u8 q2, q1
+; CHECK-NEXT:    vmov.16 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.16 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.16 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.16 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.16 q1[4], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.16 q1[5], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.16 q1[6], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.16 q1[7], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.u8 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <16 x i8> %src to <16 x i16>
+  ret <16 x i16> %0
+}
+
+define arm_aapcs_vfpcc <8 x i32> @zext_v8i16_v8i32(<8 x i16> %src) {
+; CHECK-LABEL: zext_v8i16_v8i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[4]
+; CHECK-NEXT:    vmovlb.u16 q2, q1
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov q0, q2
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <8 x i16> %src to <8 x i32>
+  ret <8 x i32> %0
+}
+
+define arm_aapcs_vfpcc <16 x i32> @zext_v16i8_v16i32(<16 x i8> %src) {
+; CHECK-LABEL: zext_v16i8_v16i32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vmov.u8 r0, q0[0]
+; CHECK-NEXT:    vmov.i32 q3, #0xff
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[4]
+; CHECK-NEXT:    vand q4, q1, q3
+; CHECK-NEXT:    vmov.32 q1[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[8]
+; CHECK-NEXT:    vmov.32 q2[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[9]
+; CHECK-NEXT:    vmov.32 q2[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[10]
+; CHECK-NEXT:    vmov.32 q2[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[11]
+; CHECK-NEXT:    vmov.32 q2[3], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[12]
+; CHECK-NEXT:    vmov.32 q5[0], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[13]
+; CHECK-NEXT:    vmov.32 q5[1], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[14]
+; CHECK-NEXT:    vmov.32 q5[2], r0
+; CHECK-NEXT:    vmov.u8 r0, q0[15]
+; CHECK-NEXT:    vmov.32 q5[3], r0
+; CHECK-NEXT:    vand q1, q1, q3
+; CHECK-NEXT:    vand q2, q2, q3
+; CHECK-NEXT:    vand q3, q5, q3
+; CHECK-NEXT:    vmov q0, q4
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = zext <16 x i8> %src to <16 x i32>
+  ret <16 x i32> %0
+}
+
 define arm_aapcs_vfpcc <2 x i64> @zext_v2i32_v2i64(<2 x i32> %src) {
 ; CHECK-LABEL: zext_v2i32_v2i64:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    adr r0, .LCPI7_0
+; CHECK-NEXT:    adr r0, .LCPI13_0
 ; CHECK-NEXT:    vldrw.u32 q1, [r0]
 ; CHECK-NEXT:    vand q0, q0, q1
 ; CHECK-NEXT:    bx lr
 ; CHECK-NEXT:    .p2align 4
 ; CHECK-NEXT:  @ %bb.1:
-; CHECK-NEXT:  .LCPI7_0:
+; CHECK-NEXT:  .LCPI13_0:
 ; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
 ; CHECK-NEXT:    .long 0 @ 0x0
 ; CHECK-NEXT:    .long 4294967295 @ 0xffffffff
@@ -129,6 +376,119 @@ entry:
   ret <4 x i8> %0
 }
 
+define arm_aapcs_vfpcc <16 x i8> @trunc_v16i16_v16i8(<16 x i16> %src) {
+; CHECK-LABEL: trunc_v16i16_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov.u16 r0, q0[0]
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[1]
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[2]
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[3]
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[4]
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[5]
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[6]
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov.u16 r0, q2[7]
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[0]
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[1]
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[2]
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[3]
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[4]
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[5]
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[6]
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov.u16 r0, q1[7]
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <16 x i16> %src to <16 x i8>
+  ret <16 x i8> %0
+}
+
+define arm_aapcs_vfpcc <8 x i16> @trunc_v8i32_v8i16(<8 x i32> %src) {
+; CHECK-LABEL: trunc_v8i32_v8i16:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmov q2, q0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.16 q0[0], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.16 q0[1], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.16 q0[2], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.16 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.16 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.16 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.16 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.16 q0[7], r0
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <8 x i32> %src to <8 x i16>
+  ret <8 x i16> %0
+}
+
+define arm_aapcs_vfpcc <16 x i8> @trunc_v16i32_v16i8(<16 x i32> %src) {
+; CHECK-LABEL: trunc_v16i32_v16i8:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vmov q4, q0
+; CHECK-NEXT:    vmov r0, s16
+; CHECK-NEXT:    vmov.8 q0[0], r0
+; CHECK-NEXT:    vmov r0, s17
+; CHECK-NEXT:    vmov.8 q0[1], r0
+; CHECK-NEXT:    vmov r0, s18
+; CHECK-NEXT:    vmov.8 q0[2], r0
+; CHECK-NEXT:    vmov r0, s19
+; CHECK-NEXT:    vmov.8 q0[3], r0
+; CHECK-NEXT:    vmov r0, s4
+; CHECK-NEXT:    vmov.8 q0[4], r0
+; CHECK-NEXT:    vmov r0, s5
+; CHECK-NEXT:    vmov.8 q0[5], r0
+; CHECK-NEXT:    vmov r0, s6
+; CHECK-NEXT:    vmov.8 q0[6], r0
+; CHECK-NEXT:    vmov r0, s7
+; CHECK-NEXT:    vmov.8 q0[7], r0
+; CHECK-NEXT:    vmov r0, s8
+; CHECK-NEXT:    vmov.8 q0[8], r0
+; CHECK-NEXT:    vmov r0, s9
+; CHECK-NEXT:    vmov.8 q0[9], r0
+; CHECK-NEXT:    vmov r0, s10
+; CHECK-NEXT:    vmov.8 q0[10], r0
+; CHECK-NEXT:    vmov r0, s11
+; CHECK-NEXT:    vmov.8 q0[11], r0
+; CHECK-NEXT:    vmov r0, s12
+; CHECK-NEXT:    vmov.8 q0[12], r0
+; CHECK-NEXT:    vmov r0, s13
+; CHECK-NEXT:    vmov.8 q0[13], r0
+; CHECK-NEXT:    vmov r0, s14
+; CHECK-NEXT:    vmov.8 q0[14], r0
+; CHECK-NEXT:    vmov r0, s15
+; CHECK-NEXT:    vmov.8 q0[15], r0
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
+entry:
+  %0 = trunc <16 x i32> %src to <16 x i8>
+  ret <16 x i8> %0
+}
+
 define arm_aapcs_vfpcc <2 x i32> @trunc_v2i64_v2i32(<2 x i64> %src) {
 ; CHECK-LABEL: trunc_v2i64_v2i32:
 ; CHECK:       @ %bb.0: @ %entry
diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
new file mode 100644
index 0000000000000..74596ff399b1a
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffleext.ll
@@ -0,0 +1,94 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK
+
+define arm_aapcs_vfpcc <4 x i32> @sext_0246(<8 x i16> %src) {
+; CHECK-LABEL: sext_0246:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %out = sext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @sext_1357(<8 x i16> %src) {
+; CHECK-LABEL: sext_1357:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vmovlb.s16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %out = sext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_0246(<8 x i16> %src) {
+; CHECK-LABEL: zext_0246:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+  %out = zext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <4 x i32> @zext_1357(<8 x i16> %src) {
+; CHECK-LABEL: zext_1357:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev32.16 q0, q0
+; CHECK-NEXT:    vmovlb.u16 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <8 x i16> %src, <8 x i16> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+  %out = zext <4 x i16> %strided.vec to <4 x i32>
+  ret <4 x i32> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_02468101214(<16 x i8> %src) {
+; CHECK-LABEL: sext_02468101214:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %out = sext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @sext_13579111315(<16 x i8> %src) {
+; CHECK-LABEL: sext_13579111315:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    vmovlb.s8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %out = sext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_02468101214(<16 x i8> %src) {
+; CHECK-LABEL: zext_02468101214:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+  %out = zext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
+
+define arm_aapcs_vfpcc <8 x i16> @zext_13579111315(<16 x i8> %src) {
+; CHECK-LABEL: zext_13579111315:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vrev16.8 q0, q0
+; CHECK-NEXT:    vmovlb.u8 q0, q0
+; CHECK-NEXT:    bx lr
+entry:
+  %strided.vec = shufflevector <16 x i8> %src, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+  %out = zext <8 x i8> %strided.vec to <8 x i16>
+  ret <8 x i16> %out
+}
diff --git a/llvm/test/CodeGen/Thumb2/mve-stack.ll b/llvm/test/CodeGen/Thumb2/mve-stack.ll
index 0199c6da7832f..41cc1162ee92f 100644
--- a/llvm/test/CodeGen/Thumb2/mve-stack.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-stack.ll
@@ -77,10 +77,9 @@ define arm_aapcs_vfpcc void @vstrh32() {
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    vmov.i32 q0, #0x6
 ; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vmov.i32 q0, #0x6
 ; CHECK-NEXT:    vstrh.32 q0, [r0, #4]
-; CHECK-NEXT:    mov r0, sp
 ; CHECK-NEXT:    bl func
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r7, pc}
@@ -101,10 +100,9 @@ define arm_aapcs_vfpcc void @vstrb32() {
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    vmov.i32 q0, #0x6
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vstrb.32 q0, [r0, #6]
 ; CHECK-NEXT:    add r0, sp, #4
+; CHECK-NEXT:    vmov.i32 q0, #0x6
+; CHECK-NEXT:    vstrb.32 q0, [r0, #2]
 ; CHECK-NEXT:    bl func
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r7, pc}
@@ -125,10 +123,9 @@ define arm_aapcs_vfpcc void @vstrb16() {
 ; CHECK-NEXT:    push {r7, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    vstrb.16 q0, [r0, #2]
-; CHECK-NEXT:    mov r0, sp
 ; CHECK-NEXT:    bl func
 ; CHECK-NEXT:    add sp, #8
 ; CHECK-NEXT:    pop {r7, pc}
@@ -212,16 +209,16 @@ entry:
 define arm_aapcs_vfpcc <4 x i16> @vldrh32() {
 ; CHECK-LABEL: vldrh32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    bl func
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vldrh.u32 q0, [r0, #4]
+; CHECK-NEXT:    vldrh.u32 q0, [r4, #4]
 ; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %d = alloca [4 x i16], align 2
   %arraydecay = getelementptr inbounds [4 x i16], [4 x i16]* %d, i32 0, i32 0
@@ -235,16 +232,16 @@ entry:
 define arm_aapcs_vfpcc <4 x i8> @vldrb32() {
 ; CHECK-LABEL: vldrb32:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    add r0, sp, #4
+; CHECK-NEXT:    add r4, sp, #4
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    bl func
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vldrb.u32 q0, [r0, #6]
+; CHECK-NEXT:    vldrb.u32 q0, [r4, #2]
 ; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %d = alloca [4 x i8], align 2
   %arraydecay = getelementptr inbounds [4 x i8], [4 x i8]* %d, i32 0, i32 0
@@ -258,16 +255,16 @@ entry:
 define arm_aapcs_vfpcc <8 x i8> @vldrb16() {
 ; CHECK-LABEL: vldrb16:
 ; CHECK:       @ %bb.0: @ %entry
-; CHECK-NEXT:    .save {r7, lr}
-; CHECK-NEXT:    push {r7, lr}
+; CHECK-NEXT:    .save {r4, lr}
+; CHECK-NEXT:    push {r4, lr}
 ; CHECK-NEXT:    .pad #8
 ; CHECK-NEXT:    sub sp, #8
-; CHECK-NEXT:    mov r0, sp
+; CHECK-NEXT:    mov r4, sp
+; CHECK-NEXT:    mov r0, r4
 ; CHECK-NEXT:    bl func
-; CHECK-NEXT:    mov r0, sp
-; CHECK-NEXT:    vldrb.u16 q0, [r0, #2]
+; CHECK-NEXT:    vldrb.u16 q0, [r4, #2]
 ; CHECK-NEXT:    add sp, #8
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    pop {r4, pc}
 entry:
   %d = alloca [8 x i8], align 2
   %arraydecay = getelementptr inbounds [8 x i8], [8 x i8]* %d, i32 0, i32 0
diff --git a/llvm/test/CodeGen/Thumb2/mve-stacksplot.mir b/llvm/test/CodeGen/Thumb2/mve-stacksplot.mir
new file mode 100644
index 0000000000000..4add5c4f9a547
--- /dev/null
+++ b/llvm/test/CodeGen/Thumb2/mve-stacksplot.mir
@@ -0,0 +1,185 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -o - %s -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp -run-pass=stack-protector -run-pass=prologepilog | FileCheck %s
+---
+name: func0
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 16, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -16, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: func0
+    ; CHECK: liveins: $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11, $lr
+    ; CHECK: $sp = frame-setup t2STMDB_UPD $sp, 14, $noreg, killed $r4, killed $r5, killed $r6, killed $r7, killed $r8, killed $r9, killed $r10, killed $r11, killed $lr
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 36
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r11, -8
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r10, -12
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r9, -16
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r8, -20
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r7, -24
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r6, -28
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r5, -32
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -36
+    ; CHECK: $sp = frame-setup tSUBspi $sp, 5, 14, $noreg
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 56
+    ; CHECK: $r0 = IMPLICIT_DEF
+    ; CHECK: $r1 = IMPLICIT_DEF
+    ; CHECK: $r2 = IMPLICIT_DEF
+    ; CHECK: $r3 = IMPLICIT_DEF
+    ; CHECK: $r4 = IMPLICIT_DEF
+    ; CHECK: $r5 = IMPLICIT_DEF
+    ; CHECK: $r6 = IMPLICIT_DEF
+    ; CHECK: $r7 = IMPLICIT_DEF
+    ; CHECK: $r8 = IMPLICIT_DEF
+    ; CHECK: $r9 = IMPLICIT_DEF
+    ; CHECK: $r10 = IMPLICIT_DEF
+    ; CHECK: $r11 = IMPLICIT_DEF
+    ; CHECK: $r12 = IMPLICIT_DEF
+    ; CHECK: $lr = IMPLICIT_DEF
+    ; CHECK: t2STRi12 killed $r0, $sp, 0, 14, $noreg :: (store 4 into %stack.1)
+    ; CHECK: $r0 = tMOVr killed $sp, 14, $noreg
+    ; CHECK: renamable $q2 = MVE_VLDRBU32 killed $r0, 16, 0, $noreg :: (load 4 from %stack.0 + 12)
+    ; CHECK: $r0 = t2LDRi12 $sp, 0, 14, $noreg :: (load 4 from %stack.1)
+    ; CHECK: KILL $r0
+    ; CHECK: KILL $r1
+    ; CHECK: KILL $r2
+    ; CHECK: KILL $r3
+    ; CHECK: KILL $r4
+    ; CHECK: KILL $r5
+    ; CHECK: KILL $r6
+    ; CHECK: KILL $r7
+    ; CHECK: KILL $r8
+    ; CHECK: KILL $r9
+    ; CHECK: KILL $r10
+    ; CHECK: KILL $r11
+    ; CHECK: KILL $r12
+    ; CHECK: KILL $lr
+    $r0 = IMPLICIT_DEF
+    $r1 = IMPLICIT_DEF
+    $r2 = IMPLICIT_DEF
+    $r3 = IMPLICIT_DEF
+    $r4 = IMPLICIT_DEF
+    $r5 = IMPLICIT_DEF
+    $r6 = IMPLICIT_DEF
+    $r7 = IMPLICIT_DEF
+    $r8 = IMPLICIT_DEF
+    $r9 = IMPLICIT_DEF
+    $r10 = IMPLICIT_DEF
+    $r11 = IMPLICIT_DEF
+    $r12 = IMPLICIT_DEF
+    $lr = IMPLICIT_DEF
+
+    renamable $q2 = MVE_VLDRBU32 %stack.0, 12, 0, $noreg :: (load 4 from %stack.0 + 12)
+
+    KILL $r0
+    KILL $r1
+    KILL $r2
+    KILL $r3
+    KILL $r4
+    KILL $r5
+    KILL $r6
+    KILL $r7
+    KILL $r8
+    KILL $r9
+    KILL $r10
+    KILL $r11
+    KILL $r12
+    KILL $lr
+...
+---
+name: func1
+tracksRegLiveness: true
+stack:
+  - { id: 0, name: '', type: default, offset: 0, size: 16, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -1200, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+  - { id: 1, name: '', type: default, offset: 0, size: 1200, alignment: 4,
+      stack-id: default, callee-saved-register: '', callee-saved-restored: true,
+      local-offset: -1200, debug-info-variable: '', debug-info-expression: '',
+      debug-info-location: '' }
+body: |
+  bb.0:
+    ; CHECK-LABEL: name: func1
+    ; CHECK: liveins: $r4, $r5, $r6, $r7, $r8, $r9, $r10, $r11, $lr
+    ; CHECK: $sp = frame-setup t2STMDB_UPD $sp, 14, $noreg, killed $r4, killed $r5, killed $r6, killed $r7, killed $r8, killed $r9, killed $r10, killed $r11, killed $lr
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 36
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $lr, -4
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r11, -8
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r10, -12
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r9, -16
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r8, -20
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r7, -24
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r6, -28
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r5, -32
+    ; CHECK: frame-setup CFI_INSTRUCTION offset $r4, -36
+    ; CHECK: $sp = frame-setup t2SUBri killed $sp, 1216, 14, $noreg, $noreg
+    ; CHECK: $sp = frame-setup tSUBspi $sp, 1, 14, $noreg
+    ; CHECK: frame-setup CFI_INSTRUCTION def_cfa_offset 1256
+    ; CHECK: $r0 = IMPLICIT_DEF
+    ; CHECK: $r1 = IMPLICIT_DEF
+    ; CHECK: $r2 = IMPLICIT_DEF
+    ; CHECK: $r3 = IMPLICIT_DEF
+    ; CHECK: $r4 = IMPLICIT_DEF
+    ; CHECK: $r5 = IMPLICIT_DEF
+    ; CHECK: $r6 = IMPLICIT_DEF
+    ; CHECK: $r7 = IMPLICIT_DEF
+    ; CHECK: $r8 = IMPLICIT_DEF
+    ; CHECK: $r9 = IMPLICIT_DEF
+    ; CHECK: $r10 = IMPLICIT_DEF
+    ; CHECK: $r11 = IMPLICIT_DEF
+    ; CHECK: $r12 = IMPLICIT_DEF
+    ; CHECK: $lr = IMPLICIT_DEF
+    ; CHECK: t2STRi12 killed $r0, $sp, 0, 14, $noreg :: (store 4 into %stack.2)
+    ; CHECK: $r0 = t2ADDri killed $sp, 1152, 14, $noreg, $noreg
+    ; CHECK: renamable $q2 = MVE_VLDRBU8 killed $r0, 52, 0, $noreg :: (load 4 from %stack.0)
+    ; CHECK: $r0 = t2LDRi12 $sp, 0, 14, $noreg :: (load 4 from %stack.2)
+    ; CHECK: KILL $r0
+    ; CHECK: KILL $r1
+    ; CHECK: KILL $r2
+    ; CHECK: KILL $r3
+    ; CHECK: KILL $r4
+    ; CHECK: KILL $r5
+    ; CHECK: KILL $r6
+    ; CHECK: KILL $r7
+    ; CHECK: KILL $r8
+    ; CHECK: KILL $r9
+    ; CHECK: KILL $r10
+    ; CHECK: KILL $r11
+    ; CHECK: KILL $r12
+    ; CHECK: KILL $lr
+    $r0 = IMPLICIT_DEF
+    $r1 = IMPLICIT_DEF
+    $r2 = IMPLICIT_DEF
+    $r3 = IMPLICIT_DEF
+    $r4 = IMPLICIT_DEF
+    $r5 = IMPLICIT_DEF
+    $r6 = IMPLICIT_DEF
+    $r7 = IMPLICIT_DEF
+    $r8 = IMPLICIT_DEF
+    $r9 = IMPLICIT_DEF
+    $r10 = IMPLICIT_DEF
+    $r11 = IMPLICIT_DEF
+    $r12 = IMPLICIT_DEF
+    $lr = IMPLICIT_DEF
+
+    renamable $q2 = MVE_VLDRBU8 %stack.0, 0, 0, $noreg :: (load 4 from %stack.0)
+
+    KILL $r0
+    KILL $r1
+    KILL $r2
+    KILL $r3
+    KILL $r4
+    KILL $r5
+    KILL $r6
+    KILL $r7
+    KILL $r8
+    KILL $r9
+    KILL $r10
+    KILL $r11
+    KILL $r12
+    KILL $lr
+...
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
index a9176ece83684..0786849dae20e 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpf.ll
@@ -107,8 +107,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -380,8 +379,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -698,8 +696,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float>
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -753,8 +750,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float>
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -1021,8 +1017,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %s
 ;
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -1660,8 +1655,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %s
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -2410,8 +2404,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %s
 ;
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -2537,8 +2530,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %s
 ;
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q1
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
index 0a1bb3759cc56..608689dc46579 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfr.ll
@@ -112,8 +112,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vdup.32 q1, r0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -403,8 +402,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vdup.32 q1, r0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -742,8 +740,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vdup.32 q1, r0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, r0
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
@@ -801,8 +798,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, float %src2
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov r0, s4
 ; CHECK-MVEFP-NEXT:    vdup.32 q1, r0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q1, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q1, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q2, q3
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -1073,8 +1069,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
 ; CHECK-MVEFP-NEXT:    vmov r0, s12
 ; CHECK-MVEFP-NEXT:    vdup.16 q3, r0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -1725,8 +1720,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
 ; CHECK-MVEFP-NEXT:    vmov r0, s12
 ; CHECK-MVEFP-NEXT:    vdup.16 q3, r0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -2490,8 +2484,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
 ; CHECK-MVEFP-NEXT:    vmov r0, s12
 ; CHECK-MVEFP-NEXT:    vdup.16 q3, r0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, r0
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -2620,8 +2613,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, half* %src2p,
 ; CHECK-MVEFP-NEXT:    vldr.16 s12, [r0]
 ; CHECK-MVEFP-NEXT:    vmov r0, s12
 ; CHECK-MVEFP-NEXT:    vdup.16 q3, r0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, r0
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
index 4a03304f2cce0..126e00a31a8d8 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vcmpfz.ll
@@ -108,8 +108,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_one_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVEFP-LABEL: vcmp_one_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -382,8 +381,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ueq_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -701,8 +699,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_ord_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVEFP-LABEL: vcmp_ord_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -757,8 +754,7 @@ define arm_aapcs_vfpcc <4 x float> @vcmp_uno_v4f32(<4 x float> %src, <4 x float>
 ; CHECK-MVEFP-LABEL: vcmp_uno_v4f32:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f32 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f32 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f32 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -1018,8 +1014,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_one_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_one_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -1638,8 +1633,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ueq_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_ueq_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 le, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
@@ -2365,8 +2359,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_ord_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_ord_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpnot
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
@@ -2489,8 +2482,7 @@ define arm_aapcs_vfpcc <8 x half> @vcmp_uno_v8f16(<8 x half> %src, <8 x half> %a
 ; CHECK-MVEFP-LABEL: vcmp_uno_v8f16:
 ; CHECK-MVEFP:       @ %bb.0: @ %entry
 ; CHECK-MVEFP-NEXT:    vmov.i32 q3, #0x0
-; CHECK-MVEFP-NEXT:    vcmp.f16 le, q3, q0
-; CHECK-MVEFP-NEXT:    vpst
+; CHECK-MVEFP-NEXT:    vpt.f16 le, q3, q0
 ; CHECK-MVEFP-NEXT:    vcmpt.f16 lt, q0, q3
 ; CHECK-MVEFP-NEXT:    vpsel q0, q1, q2
 ; CHECK-MVEFP-NEXT:    bx lr
diff --git a/llvm/test/CodeGen/Thumb2/mve-vctp.ll b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
index d8b2a86305da9..8f7e1696e6790 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vctp.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vctp.ll
@@ -4,8 +4,8 @@
 define void @vctp8(i32 %arg, <16 x i8> *%in, <16 x i8>* %out) {
 ; CHECK-LABEL: vctp8:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vctp.8 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
@@ -20,8 +20,8 @@ define void @vctp8(i32 %arg, <16 x i8> *%in, <16 x i8>* %out) {
 define void @vctp16(i32 %arg, <8 x i16> *%in, <8 x i16>* %out) {
 ; CHECK-LABEL: vctp16:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vctp.16 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
@@ -36,8 +36,8 @@ define void @vctp16(i32 %arg, <8 x i16> *%in, <8 x i16>* %out) {
 define void @vctp32(i32 %arg, <4 x i32> *%in, <4 x i32>* %out) {
 ; CHECK-LABEL: vctp32:
 ; CHECK:       @ %bb.0:
-; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vctp.32 r0
+; CHECK-NEXT:    vldrw.u32 q1, [r1]
 ; CHECK-NEXT:    vmov.i32 q0, #0x0
 ; CHECK-NEXT:    vpsel q0, q1, q0
 ; CHECK-NEXT:    vstrw.32 q0, [r2]
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll b/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
index 8a7ae49f49ad6..4c705b75fd788 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmaxv.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
 
 declare i8 @llvm.experimental.vector.reduce.smax.v16i8(<16 x i8>)
diff --git a/llvm/test/CodeGen/Thumb2/mve-vmla.ll b/llvm/test/CodeGen/Thumb2/mve-vmla.ll
index b45efdf2edd84..b75fc964d015c 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vmla.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vmla.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve.fp %s -o - | FileCheck %s
 
 define arm_aapcs_vfpcc <4 x i32> @vmlau32(<4 x i32> %A, <4 x i32> %B, i32 %X) nounwind {
@@ -80,27 +81,26 @@ entry:
 
 define void @vmla32_in_loop(i32* %s1, i32 %x, i32* %d, i32 %n) {
 ; CHECK-LABEL: vmla32_in_loop:
-; CHECK:      .LBB6_1: @ %vector.body
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB6_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]!
 ; CHECK-NEXT:    vldrw.u32 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #4
 ; CHECK-NEXT:    vmla.u32 q1, q0, r1
 ; CHECK-NEXT:    vstrw.32 q1, [r2]
-; CHECK-NEXT:    le lr, .LBB6_1
+; CHECK-NEXT:    bne .LBB6_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    bx lr
 entry:
-  %cmp6 = icmp sgt i32 %n, 0
-  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
-
-vector.ph:                                        ; preds = %for.body.preheader
-  %n.vec = and i32 %n, -4
   %broadcast.splatinsert8 = insertelement <4 x i32> undef, i32 %x, i32 0
   %broadcast.splat9 = shufflevector <4 x i32> %broadcast.splatinsert8, <4 x i32> undef, <4 x i32> zeroinitializer
   br label %vector.body
 
-vector.body:                                      ; preds = %vector.body, %vector.ph
-  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i32, i32* %s1, i32 %index
   %1 = bitcast i32* %0 to <4 x i32>*
   %wide.load = load <4 x i32>, <4 x i32>* %1, align 4
@@ -112,36 +112,35 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %6 = bitcast i32* %3 to <4 x i32>*
   store <4 x i32> %5, <4 x i32>* %6, align 4
   %index.next = add i32 %index, 4
-  %7 = icmp eq i32 %index.next, %n.vec
+  %7 = icmp eq i32 %index.next, %n
   br i1 %7, label %for.cond.cleanup, label %vector.body
 
-for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+for.cond.cleanup:
   ret void
 }
 
 define void @vmla16_in_loop(i16* %s1, i16 %x, i16* %d, i32 %n) {
 ; CHECK-LABEL: vmla16_in_loop:
-; CHECK:  .LBB7_1: @ %vector.body
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB7_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
 ; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #8
 ; CHECK-NEXT:    vmla.u16 q1, q0, r1
 ; CHECK-NEXT:    vstrh.16 q1, [r2]
-; CHECK-NEXT:    le lr, .LBB7_1
+; CHECK-NEXT:    bne .LBB7_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    bx lr
 entry:
-  %cmp6 = icmp sgt i32 %n, 0
-  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
-
-vector.ph:                                        ; preds = %for.body.preheader
-  %n.vec = and i32 %n, -8
   %broadcast.splatinsert11 = insertelement <8 x i16> undef, i16 %x, i32 0
   %broadcast.splat12 = shufflevector <8 x i16> %broadcast.splatinsert11, <8 x i16> undef, <8 x i32> zeroinitializer
   br label %vector.body
 
-vector.body:                                      ; preds = %vector.body, %vector.ph
-  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i16, i16* %s1, i32 %index
   %1 = bitcast i16* %0 to <8 x i16>*
   %wide.load = load <8 x i16>, <8 x i16>* %1, align 2
@@ -153,36 +152,35 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %6 = bitcast i16* %3 to <8 x i16>*
   store <8 x i16> %5, <8 x i16>* %6, align 2
   %index.next = add i32 %index, 8
-  %7 = icmp eq i32 %index.next, %n.vec
+  %7 = icmp eq i32 %index.next, %n
   br i1 %7, label %for.cond.cleanup, label %vector.body
 
-for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+for.cond.cleanup:
   ret void
 }
 
 define void @vmla8_in_loop(i8* %s1, i8 %x, i8* %d, i32 %n) {
 ; CHECK-LABEL: vmla8_in_loop:
-; CHECK:  .LBB8_1: @ %vector.body
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    subs r0, #16
+; CHECK-NEXT:    subs r2, #16
+; CHECK-NEXT:  .LBB8_1: @ %vector.body
 ; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    vldrh.u16 q0, [r0, #8]!
-; CHECK-NEXT:    vldrh.u16 q1, [r2, #8]!
+; CHECK-NEXT:    vldrh.u16 q0, [r0, #16]!
+; CHECK-NEXT:    vldrh.u16 q1, [r2, #16]!
+; CHECK-NEXT:    subs r3, #16
 ; CHECK-NEXT:    vmla.u8 q1, q0, r1
 ; CHECK-NEXT:    vstrh.16 q1, [r2]
-; CHECK-NEXT:    le lr, .LBB8_1
+; CHECK-NEXT:    bne .LBB8_1
 ; CHECK-NEXT:  @ %bb.2: @ %for.cond.cleanup
-; CHECK-NEXT:    pop {r7, pc}
+; CHECK-NEXT:    bx lr
 entry:
-  %cmp6 = icmp sgt i32 %n, 0
-  br i1 %cmp6, label %vector.ph, label %for.cond.cleanup
-
-vector.ph:                                        ; preds = %for.body.preheader
-  %n.vec = and i32 %n, -8
   %broadcast.splatinsert11 = insertelement <16 x i8> undef, i8 %x, i32 0
   %broadcast.splat12 = shufflevector <16 x i8> %broadcast.splatinsert11, <16 x i8> undef, <16 x i32> zeroinitializer
   br label %vector.body
 
-vector.body:                                      ; preds = %vector.body, %vector.ph
-  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+vector.body:
+  %index = phi i32 [ 0, %entry ], [ %index.next, %vector.body ]
   %0 = getelementptr inbounds i8, i8* %s1, i32 %index
   %1 = bitcast i8* %0 to <16 x i8>*
   %wide.load = load <16 x i8>, <16 x i8>* %1, align 2
@@ -193,10 +191,10 @@ vector.body:                                      ; preds = %vector.body, %vecto
   %5 = add <16 x i8> %2, %wide.load13
   %6 = bitcast i8* %3 to <16 x i8>*
   store <16 x i8> %5, <16 x i8>* %6, align 2
-  %index.next = add i32 %index, 8
-  %7 = icmp eq i32 %index.next, %n.vec
+  %index.next = add i32 %index, 16
+  %7 = icmp eq i32 %index.next, %n
   br i1 %7, label %for.cond.cleanup, label %vector.body
 
-for.cond.cleanup:                                 ; preds = %for.body, %middle.block, %entry
+for.cond.cleanup:
   ret void
 }
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block.mir
index 6c9da8f854a6c..1880e5ece2294 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block.mir
@@ -65,9 +65,9 @@ body:             |
     ; CHECK-LABEL: name: test_vminnmq_m_f32_v2
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $vpr, implicit killed $q0 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, killed renamable $q2, 1, killed renamable $vpr, killed renamable $q0
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $q0 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, killed renamable $q2, 1, internal killed renamable $vpr, killed renamable $q0
     ; CHECK: }
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
     $vpr = VMSR_P0 killed $r0, 14, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block2.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block2.mir
index ae70ed8336656..59499908bf545 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block2.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block2.mir
@@ -68,10 +68,10 @@ body:             |
     ; CHECK-LABEL: name: test_vminnmq_m_f32_v2
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q2, implicit killed $q3, implicit killed $vpr, implicit killed $q0, implicit killed $q1 {
-    ; CHECK:   MVE_VPST 4, implicit-def $p0
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, killed renamable $q0
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, killed renamable $q3, 1, killed renamable $vpr, killed renamable $q1
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
+    ; CHECK:   MVE_VPST 4, implicit-def $vpr
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, internal renamable $vpr, killed renamable $q0
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, killed renamable $q3, 1, internal killed renamable $vpr, killed renamable $q1
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q1, killed $q1, 0, $noreg, undef $q0
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block3.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block3.mir
index c23f6abe67c69..d04a0db1d3c1a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block3.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block3.mir
@@ -69,12 +69,12 @@ body:             |
     ; CHECK-LABEL: name: test_vminnmq_m_f32_v2
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q2, implicit killed $q3, implicit killed $vpr, implicit killed $q0, implicit killed $q1 {
-    ; CHECK:   MVE_VPST 1, implicit-def $p0
-    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, undef renamable $q2
-    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, renamable $vpr, internal undef renamable $q2
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, renamable $q3, 1, renamable $vpr, killed renamable $q0
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, killed renamable $q3, 1, killed renamable $vpr, killed renamable $q1
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def dead $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q2, implicit killed $q3, implicit killed $q0, implicit killed $q1 {
+    ; CHECK:   MVE_VPST 1, implicit-def $vpr
+    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, internal renamable $vpr, undef renamable $q2
+    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, internal renamable $vpr, internal undef renamable $q2
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, renamable $q3, 1, internal renamable $vpr, killed renamable $q0
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, killed renamable $q3, 1, internal killed renamable $vpr, killed renamable $q1
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q1, killed $q1, 0, $noreg, undef $q0
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block4.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block4.mir
index 6b8d471d9bab6..1dca80f91060d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block4.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block4.mir
@@ -70,16 +70,16 @@ body:             |
     ; CHECK-LABEL: name: test_vminnmq_m_f32_v2
     ; CHECK: liveins: $q0, $q1, $q2, $q3, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q2, implicit $q3, implicit $vpr, implicit killed $q0 {
-    ; CHECK:   MVE_VPST 1, implicit-def $p0
-    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, renamable $vpr, undef renamable $q2
-    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, renamable $vpr, internal undef renamable $q2
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, renamable $q3, 1, renamable $vpr, killed renamable $q0
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, renamable $q3, 1, renamable $vpr, internal undef renamable $q0
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def dead $q2, implicit-def $d4, implicit-def $s8, implicit-def $s9, implicit-def $d5, implicit-def $s10, implicit-def $s11, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q2, implicit $q3, implicit killed $q0 {
+    ; CHECK:   MVE_VPST 1, implicit-def $vpr
+    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q2, renamable $q3, 1, internal renamable $vpr, undef renamable $q2
+    ; CHECK:   renamable $q2 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, internal renamable $q2, 1, internal renamable $vpr, internal undef renamable $q2
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q2, renamable $q3, 1, internal renamable $vpr, killed renamable $q0
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q0, renamable $q3, 1, internal renamable $vpr, internal undef renamable $q0
     ; CHECK: }
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q0, implicit killed $q3, implicit killed $vpr, implicit killed $q1 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q0, killed renamable $q3, 1, killed renamable $vpr, killed renamable $q1
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q0, implicit killed $q3, implicit killed $q1 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q0, killed renamable $q3, 1, internal killed renamable $vpr, killed renamable $q1
     ; CHECK: }
     ; CHECK: $q0 = MVE_VORR killed $q1, killed $q1, 0, $noreg, undef $q0
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block5.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block5.mir
index 245e604ef15e8..753e5ce9c38b9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block5.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block5.mir
@@ -68,16 +68,16 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q1, implicit $q2, implicit $vpr, implicit killed $q3 {
-    ; CHECK:   MVE_VPST 4, implicit-def $p0
-    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 1, renamable $vpr, undef renamable $q1
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK:   MVE_VPST 4, implicit-def $vpr
+    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 1, internal renamable $vpr, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $vpr, implicit killed $q3, implicit killed $q0 {
-    ; CHECK:   MVE_VPST 4, implicit-def $p0
-    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, killed renamable $q0
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+    ; CHECK:   MVE_VPST 4, implicit-def $vpr
+    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, internal killed renamable $vpr, killed renamable $q0
     ; CHECK: }
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
     $vpr = VMSR_P0 killed $r0, 14, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block6.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block6.mir
index 7ff04eb700fb4..d68c93ceb3cc5 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block6.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block6.mir
@@ -68,14 +68,14 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0, $r1
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q1, implicit $q2, implicit killed $vpr, implicit killed $q3 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, killed renamable $vpr, killed renamable $q3
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal killed renamable $vpr, killed renamable $q3
     ; CHECK: }
     ; CHECK: $vpr = VMSR_P0 killed $r1, 14, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q3, implicit killed $q2, implicit killed $vpr, implicit killed $q0 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, killed renamable $q0
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q3, implicit killed $q2, implicit killed $q0 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, killed renamable $q2, 1, internal killed renamable $vpr, killed renamable $q0
     ; CHECK: }
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
     $vpr = VMSR_P0 killed $r0, 14, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block7.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block7.mir
index 765e054118c04..ac5dec7970263 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block7.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block7.mir
@@ -69,17 +69,17 @@ body:             |
   ; CHECK:   liveins: $q0, $q1, $q2, $r0
   ; CHECK:   $vpr = VMSR_P0 killed $r0, 14, $noreg
   ; CHECK:   $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-  ; CHECK:   BUNDLE implicit-def $p0, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q1, implicit $q2, implicit $vpr, implicit killed $q3 {
-  ; CHECK:     MVE_VPST 4, implicit-def $p0
-  ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-  ; CHECK:     renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 1, renamable $vpr, undef renamable $q1
+  ; CHECK:   BUNDLE implicit-def $vpr, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q1, implicit $q2, implicit killed $q3 {
+  ; CHECK:     MVE_VPST 4, implicit-def $vpr
+  ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+  ; CHECK:     renamable $q1 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, internal renamable $q3, 1, internal renamable $vpr, undef renamable $q1
   ; CHECK:   }
   ; CHECK: bb.1.bb2:
   ; CHECK:   liveins: $q0, $q1, $q2, $q3, $vpr
-  ; CHECK:   BUNDLE implicit-def $p0, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $vpr, implicit killed $q3, implicit killed $q0 {
-  ; CHECK:     MVE_VPST 4, implicit-def $p0
-  ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-  ; CHECK:     renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, killed renamable $q0
+  ; CHECK:   BUNDLE implicit-def dead $vpr, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+  ; CHECK:     MVE_VPST 4, implicit-def $vpr
+  ; CHECK:     renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+  ; CHECK:     renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, internal killed renamable $vpr, killed renamable $q0
   ; CHECK:   }
   ; CHECK:   tBX_RET 14, $noreg, implicit $q0
   bb.0.entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-block8.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-block8.mir
index 13878923ab5f1..db76d8b25cf1b 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-block8.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-block8.mir
@@ -68,20 +68,20 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2, $r0
     ; CHECK: $vpr = VMSR_P0 killed $r0, 14, $noreg
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q1, implicit $q2, implicit $vpr, implicit killed $q3 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit killed $q1, implicit $q2, implicit killed $q3 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q3, implicit $vpr, implicit undef $q1 {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, renamable $q3, 1, renamable $vpr, undef renamable $q1
+    ; CHECK: BUNDLE implicit-def $vpr, implicit-def $q1, implicit-def $d2, implicit-def $s4, implicit-def $s5, implicit-def $d3, implicit-def $s6, implicit-def $s7, implicit killed $q3, implicit undef $q1 {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $q1 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q3, renamable $q3, 1, internal renamable $vpr, undef renamable $q1
     ; CHECK: }
     ; CHECK: $q3 = MVE_VORR $q0, $q0, 0, $noreg, undef $q3
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $vpr, implicit killed $q3, implicit killed $q0 {
-    ; CHECK:   MVE_VPST 4, implicit-def $p0
-    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, renamable $vpr, killed renamable $q3
-    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, killed renamable $vpr, killed renamable $q0
+    ; CHECK: BUNDLE implicit-def dead $vpr, implicit-def dead $q3, implicit-def $d6, implicit-def $s12, implicit-def $s13, implicit-def $d7, implicit-def $s14, implicit-def $s15, implicit-def $q0, implicit-def $d0, implicit-def $s0, implicit-def $s1, implicit-def $d1, implicit-def $s2, implicit-def $s3, implicit killed $q1, implicit killed $q2, implicit killed $q3, implicit killed $q0 {
+    ; CHECK:   MVE_VPST 4, implicit-def $vpr
+    ; CHECK:   renamable $q3 = nnan ninf nsz MVE_VMINNMf32 killed renamable $q1, renamable $q2, 1, internal renamable $vpr, killed renamable $q3
+    ; CHECK:   renamable $q0 = nnan ninf nsz MVE_VMINNMf32 internal killed renamable $q3, killed renamable $q2, 1, internal killed renamable $vpr, killed renamable $q0
     ; CHECK: }
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
     $vpr = VMSR_P0 killed $r0, 14, $noreg
diff --git a/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir b/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
index 037e86571268a..08273302a0276 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
+++ b/llvm/test/CodeGen/Thumb2/mve-vpt-nots.mir
@@ -61,15 +61,14 @@ body:             |
 
     ; CHECK-LABEL: name: vpnot
     ; CHECK: liveins: $q0, $q1, $q2
-    ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1 {
+    ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, $zr, 11, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit killed $q2, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $q2, implicit $zr {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
@@ -94,17 +93,16 @@ body:             |
   ; CHECK: bb.0.entry:
   ; CHECK:   successors: %bb.1(0x80000000)
   ; CHECK:   liveins: $q0, $q1, $q2
-  ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
-  ; CHECK:   BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr {
-  ; CHECK:     MVE_VPST 8, implicit-def $p0
-  ; CHECK:     renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+  ; CHECK:   BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1 {
+  ; CHECK:     MVE_VPTv4s32r 8, renamable $q0, $zr, 11, implicit-def $vpr
+  ; CHECK:     renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
   ; CHECK:   }
   ; CHECK:   renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
   ; CHECK: bb.1.bb2:
   ; CHECK:   liveins: $q0, $q1, $q2, $vpr
-  ; CHECK:   BUNDLE implicit-def $p0, implicit-def $vpr, implicit killed $q2, implicit $zr, implicit killed $vpr {
-  ; CHECK:     MVE_VPST 8, implicit-def $p0
-  ; CHECK:     renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+  ; CHECK:   BUNDLE implicit-def $vpr, implicit killed $q2, implicit $zr {
+  ; CHECK:     MVE_VPST 8, implicit-def $vpr
+  ; CHECK:     renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
   ; CHECK:   }
   ; CHECK:   renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
   ; CHECK:   tBX_RET 14, $noreg, implicit $q0
@@ -137,16 +135,15 @@ body:             |
 
     ; CHECK-LABEL: name: vpnot_two
     ; CHECK: liveins: $q0, $q1, $q2
-    ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1 {
+    ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, $zr, 11, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit killed $q2, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $q2, implicit $zr {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
@@ -173,19 +170,18 @@ body:             |
 
     ; CHECK-LABEL: name: vpnot_lots
     ; CHECK: liveins: $q0, $q1, $q2
-    ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q0, implicit $zr, implicit $q1 {
+    ; CHECK:   MVE_VPTv4s32r 8, renamable $q0, $zr, 11, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit killed $q2, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $q2, implicit $zr {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
     ; CHECK: tBX_RET 14, $noreg, implicit $q0
@@ -217,9 +213,9 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr, implicit killed $q2 {
-    ; CHECK:   MVE_VPST 4, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $q2 {
+    ; CHECK:   MVE_VPST 4, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
     ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
@@ -248,14 +244,14 @@ body:             |
     ; CHECK: liveins: $q0, $q1, $q2
     ; CHECK: renamable $vpr = MVE_VCMPs32r renamable $q0, $zr, 11, 0, $noreg
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit $q1, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit $q1, implicit $zr {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPs32r renamable $q1, $zr, 12, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
-    ; CHECK: BUNDLE implicit-def $p0, implicit-def $vpr, implicit killed $q2, implicit $zr, implicit killed $vpr {
-    ; CHECK:   MVE_VPST 8, implicit-def $p0
-    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, killed renamable $vpr
+    ; CHECK: BUNDLE implicit-def $vpr, implicit killed $q2, implicit $zr {
+    ; CHECK:   MVE_VPST 8, implicit-def $vpr
+    ; CHECK:   renamable $vpr = MVE_VCMPi32r killed renamable $q2, $zr, 0, 1, internal killed renamable $vpr
     ; CHECK: }
     ; CHECK: renamable $vpr = MVE_VPNOT killed renamable $vpr, 0, $noreg
     ; CHECK: renamable $q0 = MVE_VPSEL killed renamable $q0, killed renamable $q1, 0, killed renamable $vpr
diff --git a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
index 7b0a64c7ba6f1..4c3b4d3e1b374 100644
--- a/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-widen-narrow.ll
@@ -41,6 +41,57 @@ entry:
   ret void
 }
 
+
+define void @foo_int8_int32_double(<16 x i8>* %dest, <16 x i32>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int8_int32_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vldrw.u32 q2, [r1, #32]
+; CHECK-NEXT:    vldrw.u32 q3, [r1, #48]
+; CHECK-NEXT:    vstrb.32 q1, [r0, #4]
+; CHECK-NEXT:    vstrb.32 q0, [r0]
+; CHECK-NEXT:    vstrb.32 q3, [r0, #12]
+; CHECK-NEXT:    vstrb.32 q2, [r0, #8]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i32>, <16 x i32>* %src, align 4
+  %0 = trunc <16 x i32> %wide.load to <16 x i8>
+  store <16 x i8> %0, <16 x i8>* %dest, align 1
+  ret void
+}
+
+define void @foo_int16_int32_double(<8 x i16>* %dest, <8 x i32>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int16_int32_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrw.u32 q0, [r1]
+; CHECK-NEXT:    vldrw.u32 q1, [r1, #16]
+; CHECK-NEXT:    vstrh.32 q1, [r0, #8]
+; CHECK-NEXT:    vstrh.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i32>, <8 x i32>* %src, align 4
+  %0 = trunc <8 x i32> %wide.load to <8 x i16>
+  store <8 x i16> %0, <8 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_int8_int16_double(<16 x i8>* %dest, <16 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int8_int16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u16 q0, [r1]
+; CHECK-NEXT:    vldrh.u16 q1, [r1, #16]
+; CHECK-NEXT:    vstrb.16 q1, [r0, #8]
+; CHECK-NEXT:    vstrb.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i16>, <16 x i16>* %src, align 2
+  %0 = trunc <16 x i16> %wide.load to <16 x i8>
+  store <16 x i8> %0, <16 x i8>* %dest, align 1
+  ret void
+}
+
+
 define void @foo_int32_int8(<4 x i32>* %dest, <4 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_int32_int8:
 ; CHECK:       @ %bb.0: @ %entry
@@ -80,6 +131,56 @@ entry:
   ret void
 }
 
+define void @foo_int32_int8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s32 q0, [r1, #4]
+; CHECK-NEXT:    vldrb.s32 q1, [r1]
+; CHECK-NEXT:    vldrb.s32 q2, [r1, #12]
+; CHECK-NEXT:    vldrb.s32 q3, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define void @foo_int16_int8_double(<16 x i16>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int16_int8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
+; CHECK-NEXT:    vstrh.16 q1, [r0, #16]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i16>
+  store <16 x i16> %0, <16 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_int32_int16_double(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1]
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i16>, <8 x i16>* %src, align 2
+  %0 = sext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret void
+}
+
+
 define void @foo_uint32_uint8(<4 x i32>* %dest, <4 x i8>* readonly %src, i32 %n) {
 ; CHECK-LABEL: foo_uint32_uint8:
 ; CHECK:       @ %bb.0: @ %entry
@@ -120,6 +221,147 @@ entry:
 }
 
 
+define void @foo_uint32_uint8_double(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u32 q0, [r1, #4]
+; CHECK-NEXT:    vldrb.u32 q1, [r1]
+; CHECK-NEXT:    vldrb.u32 q2, [r1, #12]
+; CHECK-NEXT:    vldrb.u32 q3, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0]
+; CHECK-NEXT:    vstrw.32 q0, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = zext <16 x i8> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define void @foo_uint16_uint8_double(<16 x i16>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint16_uint8_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.u16 q0, [r1]
+; CHECK-NEXT:    vldrb.u16 q1, [r1, #8]
+; CHECK-NEXT:    vstrh.16 q1, [r0, #16]
+; CHECK-NEXT:    vstrh.16 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = zext <16 x i8> %wide.load to <16 x i16>
+  store <16 x i16> %0, <16 x i16>* %dest, align 2
+  ret void
+}
+
+define void @foo_uint32_uint16_double(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_double:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.u32 q0, [r1]
+; CHECK-NEXT:    vldrh.u32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <8 x i16>, <8 x i16>* %src, align 2
+  %0 = zext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret void
+}
+
+
+define void @foo_int32_int8_both(<16 x i32>* %dest, <16 x i8>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_int32_int8_both:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrb.s16 q1, [r1, #8]
+; CHECK-NEXT:    vmov.u16 r2, q1[4]
+; CHECK-NEXT:    vmov.32 q0[0], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[5]
+; CHECK-NEXT:    vmov.32 q0[1], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[6]
+; CHECK-NEXT:    vmov.32 q0[2], r2
+; CHECK-NEXT:    vmov.u16 r2, q1[7]
+; CHECK-NEXT:    vmov.32 q0[3], r2
+; CHECK-NEXT:    vmovlb.u16 q2, q0
+; CHECK-NEXT:    vldrb.s16 q0, [r1]
+; CHECK-NEXT:    vmov.u16 r1, q1[0]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vmov.32 q2[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[1]
+; CHECK-NEXT:    vmov.32 q2[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[2]
+; CHECK-NEXT:    vmov.32 q2[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q1[3]
+; CHECK-NEXT:    vmov.32 q2[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[4]
+; CHECK-NEXT:    vmovlb.u16 q1, q2
+; CHECK-NEXT:    vstrw.32 q1, [r0, #32]
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[5]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[6]
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[7]
+; CHECK-NEXT:    vmov.32 q1[3], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[0]
+; CHECK-NEXT:    vmovlb.u16 q1, q1
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.32 q1[0], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[1]
+; CHECK-NEXT:    vmov.32 q1[1], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[2]
+; CHECK-NEXT:    vmov.32 q1[2], r1
+; CHECK-NEXT:    vmov.u16 r1, q0[3]
+; CHECK-NEXT:    vmov.32 q1[3], r1
+; CHECK-NEXT:    vmovlb.u16 q0, q1
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    bx lr
+entry:
+  %wide.load = load <16 x i8>, <16 x i8>* %src, align 1
+  %0 = sext <16 x i8> %wide.load to <16 x i16>
+  %1 = zext <16 x i16> %0 to <16 x i32>
+  store <16 x i32> %1, <16 x i32>* %dest, align 4
+  ret void
+}
+
+define <8 x i16>* @foo_uint32_uint16_double_offset(<8 x i32>* %dest, <8 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_double_offset:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #16]!
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds <8 x i16>, <8 x i16>* %src, i32 1
+  %wide.load = load <8 x i16>, <8 x i16>* %z, align 2
+  %0 = sext <8 x i16> %wide.load to <8 x i32>
+  store <8 x i32> %0, <8 x i32>* %dest, align 4
+  ret <8 x i16>* %z
+}
+
+define <16 x i16>* @foo_uint32_uint16_quad_offset(<16 x i32>* %dest, <16 x i16>* readonly %src, i32 %n) {
+; CHECK-LABEL: foo_uint32_uint16_quad_offset:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    vldrh.s32 q0, [r1, #32]!
+; CHECK-NEXT:    vldrh.s32 q1, [r1, #8]
+; CHECK-NEXT:    vldrh.s32 q2, [r1, #24]
+; CHECK-NEXT:    vldrh.s32 q3, [r1, #16]
+; CHECK-NEXT:    vstrw.32 q0, [r0]
+; CHECK-NEXT:    vstrw.32 q2, [r0, #48]
+; CHECK-NEXT:    vstrw.32 q1, [r0, #16]
+; CHECK-NEXT:    vstrw.32 q3, [r0, #32]
+; CHECK-NEXT:    mov r0, r1
+; CHECK-NEXT:    bx lr
+entry:
+  %z = getelementptr inbounds <16 x i16>, <16 x i16>* %src, i32 1
+  %wide.load = load <16 x i16>, <16 x i16>* %z, align 2
+  %0 = sext <16 x i16> %wide.load to <16 x i32>
+  store <16 x i32> %0, <16 x i32>* %dest, align 4
+  ret <16 x i16>* %z
+}
 
 
 define void @foo_int16_int32_align1(<4 x i16>* %dest, <4 x i32>* readonly %src, i32 %n) {
diff --git a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
index 6574511089b74..89ea7e8f826e2 100644
--- a/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
+++ b/llvm/test/CodeGen/WebAssembly/bulk-memory.ll
@@ -142,8 +142,8 @@ define void @memset_1024(i8* %dest, i8 %val) {
 }
 
 ; The following tests check that frame index elimination works for
-; bulk memory instructions. The stack pointer is bumped by 16 instead
-; of 10 because the stack pointer in WebAssembly is currently always
+; bulk memory instructions. The stack pointer is bumped by 112 instead
+; of 100 because the stack pointer in WebAssembly is currently always
 ; 16-byte aligned, even in leaf functions, although it is not written
 ; back to the global in this case.
 
@@ -156,17 +156,17 @@ define void @memset_1024(i8* %dest, i8 %val) {
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_src (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.copy 0, 0, $0, $pop[[L4]], $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_src(i8* %dst) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %p, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dst, i8* %p, i32 100, i1 false)
   ret void
 }
 
@@ -174,17 +174,17 @@ define void @memcpy_alloca_src(i8* %dst) {
 ; NO-BULK-MEM-NOT: memory.copy
 ; BULK-MEM-NEXT: .functype memcpy_alloca_dst (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.copy 0, 0, $pop[[L4]], $0, $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memcpy_alloca_dst(i8* %src) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %src, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memcpy.p0i8.p0i8.i32(i8* %p, i8* %src, i32 100, i1 false)
   ret void
 }
 
@@ -192,16 +192,16 @@ define void @memcpy_alloca_dst(i8* %src) {
 ; NO-BULK-MEM-NOT: memory.fill
 ; BULK-MEM-NEXT: .functype memset_alloca (i32) -> ()
 ; BULK-MEM-NEXT: global.get $push[[L0:[0-9]+]]=, __stack_pointer
-; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 16
+; BULK-MEM-NEXT: i32.const $push[[L1:[0-9]+]]=, 112
 ; BULK-MEM-NEXT: i32.sub $push[[L2:[0-9]+]]=, $pop[[L0]], $pop[[L1]]
-; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 6
+; BULK-MEM-NEXT: i32.const $push[[L3:[0-9]+]]=, 12
 ; BULK-MEM-NEXT: i32.add $push[[L4:[0-9]+]]=, $pop[[L2]], $pop[[L3]]
-; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 10
+; BULK-MEM-NEXT: i32.const $push[[L5:[0-9]+]]=, 100
 ; BULK-MEM-NEXT: memory.fill 0, $pop[[L4]], $0, $pop[[L5]]
 ; BULK-MEM-NEXT: return
 define void @memset_alloca(i8 %val) {
-  %a = alloca [10 x i8]
-  %p = bitcast [10 x i8]* %a to i8*
-  call void @llvm.memset.p0i8.i32(i8* %p, i8 %val, i32 10, i1 false)
+  %a = alloca [100 x i8]
+  %p = bitcast [100 x i8]* %a to i8*
+  call void @llvm.memset.p0i8.i32(i8* %p, i8 %val, i32 100, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
index 9cb5a057ef14f..191c3d785521f 100644
--- a/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
+++ b/llvm/test/CodeGen/WebAssembly/cfg-stackify-eh.ll
@@ -436,13 +436,17 @@ try.cont:                                         ; preds = %catch.start1, %catc
 ; the right destination (label4), from which we rethrow the exception to the
 ; caller.
 
+; And the return value of 'baz' should NOT be stackified because the BB is split
+; during fixing unwind mismatches.
+
 ; NOSORT-LABEL: test6
 ; NOSORT:   try
 ; NOSORT:     call      foo
 ; --- Nested try/catch/end_try starts
 ; NOSORT:     try
 ; NOSORT:       call      bar
-; NOSORT:       call      bar
+; NOSORT:       i32.call  ${{[0-9]+}}=, baz
+; NOSORT-NOT:   i32.call  $push{{.*}}=, baz
 ; NOSORT:     catch     $[[REG:[0-9]+]]=
 ; NOSORT:       br        1                            # 1: down to label35
 ; NOSORT:     end_try
@@ -460,7 +464,8 @@ bb0:
 
 bb1:                                              ; preds = %bb0
   call void @bar()
-  call void @bar()
+  %call = call i32 @baz()
+  call void @nothrow(i32 %call) #0
   ret void
 
 catch.dispatch0:                                  ; preds = %bb0
@@ -616,8 +621,54 @@ try.cont:                                         ; preds = %catch.start1, %catc
   ret void
 }
 
+; In CFGSort, EH pads should be sorted as soon as it is available and
+; 'Preferred' queue and should NOT be entered into 'Ready' queue unless we are
+; in the middle of sorting another region that does not contain the EH pad. In
+; this example, 'catch.start' should be sorted right after 'if.then' is sorted
+; (before 'cont' is sorted) and there should not be any unwind destination
+; mismatches in CFGStackify.
+
+; NOOPT: block
+; NOOPT:   try
+; NOOPT:     call      foo
+; NOOPT:   catch
+; NOOPT:   end_try
+; NOOPT:   call      foo
+; NOOPT: end_block
+; NOOPT: return
+define void @test9(i32 %arg) personality i8* bitcast (i32 (...)* @__gxx_wasm_personality_v0 to i8*) {
+entry:
+  %tobool = icmp ne i32 %arg, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+catch.dispatch:                                   ; preds = %if.then
+  %0 = catchswitch within none [label %catch.start] unwind to caller
+
+catch.start:                                      ; preds = %catch.dispatch
+  %1 = catchpad within %0 [i8* null]
+  %2 = call i8* @llvm.wasm.get.exception(token %1)
+  %3 = call i32 @llvm.wasm.get.ehselector(token %1)
+  %4 = call i8* @__cxa_begin_catch(i8* %2) [ "funclet"(token %1) ]
+  call void @__cxa_end_catch() [ "funclet"(token %1) ]
+  catchret from %1 to label %if.end
+
+if.then:                                          ; preds = %entry
+  invoke void @foo()
+          to label %cont unwind label %catch.dispatch
+
+cont:                                             ; preds = %if.then
+  call void @foo()
+  br label %if.end
+
+if.end:                                           ; preds = %cont, %catch.start, %entry
+  ret void
+}
+
 declare void @foo()
 declare void @bar()
+declare i32 @baz()
+; Function Attrs: nounwind
+declare void @nothrow(i32) #0
 declare i32 @__gxx_wasm_personality_v0(...)
 declare i8* @llvm.wasm.get.exception(token)
 declare i32 @llvm.wasm.get.ehselector(token)
@@ -627,3 +678,5 @@ declare i8* @__cxa_begin_catch(i8*)
 declare void @__cxa_end_catch()
 declare void @__clang_call_terminate(i8*)
 declare void @_ZSt9terminatev()
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/WebAssembly/cpus.ll b/llvm/test/CodeGen/WebAssembly/cpus.ll
index 8ede6cbb5a716..01964e9c85abc 100644
--- a/llvm/test/CodeGen/WebAssembly/cpus.ll
+++ b/llvm/test/CodeGen/WebAssembly/cpus.ll
@@ -1,16 +1,17 @@
 ; This tests that llc accepts all valid WebAssembly CPUs.
 
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=mvp 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=mvp 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=mvp 2>&1 | FileCheck %s --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=generic 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=generic 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=generic 2>&1 | FileCheck %s  --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=bleeding-edge 2>&1 | FileCheck %s
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=bleeding-edge 2>&1 | FileCheck %s
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=bleeding-edge 2>&1 | FileCheck %s  --check-prefix=WASM64
 ; RUN: llc < %s -asm-verbose=false -mtriple=wasm32-unknown-unknown -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
-; RUN: llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=INVALID
+; RUN: not llc < %s -asm-verbose=false -mtriple=wasm64-unknown-unknown-wasm -mcpu=invalidcpu 2>&1 | FileCheck %s --check-prefix=WASM64
 
 ; CHECK-NOT: is not a recognized processor for this target
 ; INVALID: {{.+}} is not a recognized processor for this target
+; WASM64: 64-bit WebAssembly (wasm64) is not currently supported
 
 define i32 @f(i32 %i_like_the_web) {
   ret i32 %i_like_the_web
diff --git a/llvm/test/CodeGen/WebAssembly/simd-arith.ll b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
index 410b2e6f5ab9d..acbb1f9d5c483 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-arith.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-arith.ll
@@ -223,6 +223,23 @@ define <16 x i8> @not_v16i8(<16 x i8> %x) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: andnot_v16i8:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128.andnot
+; SIMD128-NEXT: .functype andnot_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <16 x i8> @andnot_v16i8(<16 x i8> %x, <16 x i8> %y) {
+ %inv_y = xor <16 x i8> %y,
+   <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1,
+    i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
+ %a = and <16 x i8> %x, %inv_y
+ ret <16 x i8> %a
+}
+
 ; CHECK-LABEL: bitselect_v16i8:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v16i8 (v128, v128, v128) -> (v128){{$}}
@@ -445,6 +462,22 @@ define <8 x i16> @not_v8i16(<8 x i16> %x) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: andnot_v8i16:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <8 x i16> @andnot_v8i16(<8 x i16> %x, <8 x i16> %y) {
+ %inv_y = xor <8 x i16> %y,
+   <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
+ %a = and <8 x i16> %x, %inv_y
+ ret <8 x i16> %a
+}
+
 ; CHECK-LABEL: bitselect_v8i16:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v8i16 (v128, v128, v128) -> (v128){{$}}
@@ -652,6 +685,21 @@ define <4 x i32> @not_v4i32(<4 x i32> %x) {
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: andnot_v4i32:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v4i32 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <4 x i32> @andnot_v4i32(<4 x i32> %x, <4 x i32> %y) {
+ %inv_y = xor <4 x i32> %y, <i32 -1, i32 -1, i32 -1, i32 -1>
+ %a = and <4 x i32> %x, %inv_y
+ ret <4 x i32> %a
+}
+
 ; CHECK-LABEL: bitselect_v4i32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype bitselect_v4i32 (v128, v128, v128) -> (v128){{$}}
@@ -962,6 +1010,21 @@ define <2 x i64> @not_v2i64(<2 x i64> %x) {
   ret <2 x i64> %a
 }
 
+; CHECK-LABEL: andnot_v2i64:
+; SIMD128-VM-NOT: v128.andnot
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype andnot_v2i64 (v128, v128) -> (v128){{$}}
+; SIMD128-SLOW-NEXT: v128.andnot $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-SLOW-NEXT: return $pop[[R]]{{$}}
+; SIMD128-FAST-NEXT: v128.not
+; SIMD128-FAST-NEXT: v128.and
+; SIMD128-FAST-NEXT: return
+define <2 x i64> @andnot_v2i64(<2 x i64> %x, <2 x i64> %y) {
+ %inv_y = xor <2 x i64> %y, <i64 -1, i64 -1>
+ %a = and <2 x i64> %x, %inv_y
+ ret <2 x i64> %a
+}
+
 ; CHECK-LABEL: bitselect_v2i64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
diff --git a/llvm/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll b/llvm/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll
deleted file mode 100644
index 39bcc1d0bde84..0000000000000
--- a/llvm/test/CodeGen/WebAssembly/simd-ext-load-trunc-store.ll
+++ /dev/null
@@ -1,57 +0,0 @@
-; RUN: llc < %s -asm-verbose=false -verify-machineinstrs -disable-wasm-fallthrough-return-opt -wasm-disable-explicit-locals -wasm-keep-registers -mattr=+unimplemented-simd128 | FileCheck %s
-
-; Check that store in memory with smaller lanes are loaded and stored
-; as expected. This is a regression test for part of bug 39275.
-
-target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
-target triple = "wasm32-unknown-unknown"
-
-; CHECK-LABEL: load_ext_2xi32:
-; CHECK-NEXT: .functype load_ext_2xi32 (i32) -> (v128){{$}}
-; CHECK-NEXT: i64.load32_u $push[[L0:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: i64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]{{$}}
-; CHECK-NEXT: i64.load32_u $push[[L2:[0-9]+]]=, 4($0){{$}}
-; CHECK-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $pop[[L1]], 1, $pop[[L2]]{{$}}
-; CHECK-NEXT: return $pop[[R]]{{$}}
-define <2 x i32> @load_ext_2xi32(<2 x i32>* %p) {
-  %1 = load <2 x i32>, <2 x i32>* %p, align 4
-  ret <2 x i32> %1
-}
-
-; CHECK-LABEL: load_zext_2xi32:
-; CHECK-NEXT: .functype load_zext_2xi32 (i32) -> (v128){{$}}
-; CHECK-NEXT: i64.load32_u $push[[L0:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: i64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]{{$}}
-; CHECK-NEXT: i64.load32_u $push[[L2:[0-9]+]]=, 4($0){{$}}
-; CHECK-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $pop[[L1]], 1, $pop[[L2]]{{$}}
-; CHECK-NEXT: return $pop[[R]]{{$}}
-define <2 x i64> @load_zext_2xi32(<2 x i32>* %p) {
-  %1 = load <2 x i32>, <2 x i32>* %p, align 4
-  %2 = zext <2 x i32> %1 to <2 x i64>
-  ret <2 x i64> %2
-}
-
-; CHECK-LABEL: load_sext_2xi32:
-; CHECK-NEXT: .functype load_sext_2xi32 (i32) -> (v128){{$}}
-; CHECK-NEXT: i64.load32_s $push[[L0:[0-9]+]]=, 0($0){{$}}
-; CHECK-NEXT: i64x2.splat $push[[L1:[0-9]+]]=, $pop[[L0]]{{$}}
-; CHECK-NEXT: i64.load32_s $push[[L2:[0-9]+]]=, 4($0){{$}}
-; CHECK-NEXT: i64x2.replace_lane $push[[R:[0-9]+]]=, $pop[[L1]], 1, $pop[[L2]]{{$}}
-; CHECK-NEXT: return $pop[[R]]{{$}}
-define <2 x i64> @load_sext_2xi32(<2 x i32>* %p) {
-  %1 = load <2 x i32>, <2 x i32>* %p, align 4
-  %2 = sext <2 x i32> %1 to <2 x i64>
-  ret <2 x i64> %2
-}
-
-; CHECK-LABEL: store_trunc_2xi32:
-; CHECK-NEXT: .functype store_trunc_2xi32 (i32, v128) -> (){{$}}
-; CHECK-NEXT: i64x2.extract_lane $push[[L0:[0-9]+]]=, $1, 1
-; CHECK-NEXT: i64.store32 4($0), $pop[[L0]]
-; CHECK-NEXT: i64x2.extract_lane $push[[L1:[0-9]+]]=, $1, 0
-; CHECK-NEXT: i64.store32 0($0), $pop[[L1]]
-; CHECK-NEXT: return
-define void @store_trunc_2xi32(<2 x i32>* %p, <2 x i32> %x) {
-  store <2 x i32> %x, <2 x i32>* %p, align 4
-  ret void
-}
diff --git a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
index 2077cf8c70e69..f88b8b454268c 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-intrinsics.ll
@@ -87,6 +87,30 @@ define <16 x i8> @bitselect_v16i8(<16 x i8> %v1, <16 x i8> %v2, <16 x i8> %c) {
   ret <16 x i8> %a
 }
 
+; CHECK-LABEL: narrow_signed_v16i8:
+; SIMD128-NEXT: .functype narrow_signed_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.narrow_i16x8_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(<8 x i16>, <8 x i16>)
+define <16 x i8> @narrow_signed_v16i8(<8 x i16> %low, <8 x i16> %high) {
+  %a = call <16 x i8> @llvm.wasm.narrow.signed.v16i8.v8i16(
+    <8 x i16> %low, <8 x i16> %high
+  )
+  ret <16 x i8> %a
+}
+
+; CHECK-LABEL: narrow_unsigned_v16i8:
+; SIMD128-NEXT: .functype narrow_unsigned_v16i8 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i8x16.narrow_i16x8_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(<8 x i16>, <8 x i16>)
+define <16 x i8> @narrow_unsigned_v16i8(<8 x i16> %low, <8 x i16> %high) {
+  %a = call <16 x i8> @llvm.wasm.narrow.unsigned.v16i8.v8i16(
+    <8 x i16> %low, <8 x i16> %high
+  )
+  ret <16 x i8> %a
+}
+
 ; ==============================================================================
 ; 8 x i16
 ; ==============================================================================
@@ -166,6 +190,70 @@ define <8 x i16> @bitselect_v8i16(<8 x i16> %v1, <8 x i16> %v2, <8 x i16> %c) {
   ret <8 x i16> %a
 }
 
+; CHECK-LABEL: narrow_signed_v8i16:
+; SIMD128-NEXT: .functype narrow_signed_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.narrow_i32x4_s $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(<4 x i32>, <4 x i32>)
+define <8 x i16> @narrow_signed_v8i16(<4 x i32> %low, <4 x i32> %high) {
+  %a = call <8 x i16> @llvm.wasm.narrow.signed.v8i16.v4i32(
+    <4 x i32> %low, <4 x i32> %high
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: narrow_unsigned_v8i16:
+; SIMD128-NEXT: .functype narrow_unsigned_v8i16 (v128, v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.narrow_i32x4_u $push[[R:[0-9]+]]=, $0, $1{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(<4 x i32>, <4 x i32>)
+define <8 x i16> @narrow_unsigned_v8i16(<4 x i32> %low, <4 x i32> %high) {
+  %a = call <8 x i16> @llvm.wasm.narrow.unsigned.v8i16.v4i32(
+    <4 x i32> %low, <4 x i32> %high
+  )
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_low_signed_v8i16:
+; SIMD128-NEXT: .functype widen_low_signed_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_low_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_low_signed_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.low.signed.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_high_signed_v8i16:
+; SIMD128-NEXT: .functype widen_high_signed_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_high_i8x16_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_high_signed_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.high.signed.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_low_unsigned_v8i16:
+; SIMD128-NEXT: .functype widen_low_unsigned_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_low_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_low_unsigned_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.low.unsigned.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
+; CHECK-LABEL: widen_high_unsigned_v8i16:
+; SIMD128-NEXT: .functype widen_high_unsigned_v8i16 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.widen_high_i8x16_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8>)
+define <8 x i16> @widen_high_unsigned_v8i16(<16 x i8> %v) {
+  %a = call <8 x i16> @llvm.wasm.widen.high.unsigned.v8i16.v16i8(<16 x i8> %v)
+  ret <8 x i16> %a
+}
+
 ; ==============================================================================
 ; 4 x i32
 ; ==============================================================================
@@ -223,6 +311,46 @@ define <4 x i32> @trunc_sat_u_v4i32(<4 x float> %x) {
   ret <4 x i32> %a
 }
 
+; CHECK-LABEL: widen_low_signed_v4i32:
+; SIMD128-NEXT: .functype widen_low_signed_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_low_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_low_signed_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.low.signed.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_high_signed_v4i32:
+; SIMD128-NEXT: .functype widen_high_signed_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_high_i16x8_s $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_high_signed_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.high.signed.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_low_unsigned_v4i32:
+; SIMD128-NEXT: .functype widen_low_unsigned_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_low_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_low_unsigned_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.low.unsigned.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
+; CHECK-LABEL: widen_high_unsigned_v4i32:
+; SIMD128-NEXT: .functype widen_high_unsigned_v4i32 (v128) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.widen_high_i16x8_u $push[[R:[0-9]+]]=, $0{{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+declare <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16>)
+define <4 x i32> @widen_high_unsigned_v4i32(<8 x i16> %v) {
+  %a = call <4 x i32> @llvm.wasm.widen.high.unsigned.v4i32.v8i16(<8 x i16> %v)
+  ret <4 x i32> %a
+}
+
 ; ==============================================================================
 ; 2 x i64
 ; ==============================================================================
diff --git a/llvm/test/CodeGen/WebAssembly/simd-offset.ll b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
index 61177ae924c91..03b6ca7c225b9 100644
--- a/llvm/test/CodeGen/WebAssembly/simd-offset.ll
+++ b/llvm/test/CodeGen/WebAssembly/simd-offset.ll
@@ -20,6 +20,19 @@ define <16 x i8> @load_v16i8(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8:
+; SIMD128-VM-NOT: v8x16.load_splat
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8(i8* %p) {
+  %e = load i8, i8* %p
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_folded_offset (i32) -> (v128){{$}}
@@ -33,6 +46,21 @@ define <16 x i8> @load_v16i8_with_folded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_folded_offset(i8* %p) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -44,6 +72,19 @@ define <16 x i8> @load_v16i8_with_folded_gep_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 1($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_folded_gep_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -57,6 +98,21 @@ define <16 x i8> @load_v16i8_with_unfolded_gep_negative_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -1{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_gep_negative_offset(i8* %p) {
+  %s = getelementptr inbounds i8, i8* %p, i32 -1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
@@ -72,6 +128,23 @@ define <16 x i8> @load_v16i8_with_unfolded_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_offset(i8* %p) {
+  %q = ptrtoint i8* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -85,6 +158,21 @@ define <16 x i8> @load_v16i8_with_unfolded_gep_offset(<16 x i8>* %p) {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 1{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_with_unfolded_gep_offset(i8* %p) {
+  %s = getelementptr i8, i8* %p, i32 1
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_from_numeric_address () -> (v128){{$}}
@@ -97,6 +185,20 @@ define <16 x i8> @load_v16i8_from_numeric_address() {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <16 x i8> @load_splat_v16i8_from_numeric_address() {
+  %s = inttoptr i32 32 to i8*
+  %e = load i8, i8* %s
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: load_v16i8_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v16i8_from_global_address () -> (v128){{$}}
@@ -109,6 +211,20 @@ define <16 x i8> @load_v16i8_from_global_address() {
   ret <16 x i8> %v
 }
 
+; CHECK-LABEL: load_splat_v16i8_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v16i8_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v8x16.load_splat $push[[R:[0-9]+]]=, gv_i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i8 = global i8 42
+define <16 x i8> @load_splat_v16i8_from_global_address() {
+  %e = load i8, i8* @gv_i8
+  %v1 = insertelement <16 x i8> undef, i8 %e, i32 0
+  %v2 = shufflevector <16 x i8> %v1, <16 x i8> undef, <16 x i32> zeroinitializer
+  ret <16 x i8> %v2
+}
+
 ; CHECK-LABEL: store_v16i8:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v16i8 (v128, i32) -> (){{$}}
@@ -210,6 +326,50 @@ define <8 x i16> @load_v8i16(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16(i16* %p) {
+  %e = load i16, i16* %p
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16(<8 x i8>* %p) {
+  %v = load <8 x i8>, <8 x i8>* %p
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_folded_offset (i32) -> (v128){{$}}
@@ -223,6 +383,62 @@ define <8 x i16> @load_v8i16_with_folded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_folded_offset(i16* %p) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_folded_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_folded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -234,6 +450,54 @@ define <8 x i16> @load_v8i16_with_folded_gep_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 2($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_folded_gep_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_folded_gep_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_folded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -247,6 +511,62 @@ define <8 x i16> @load_v8i16_with_unfolded_gep_negative_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -2{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_gep_negative_offset(i16* %p) {
+  %s = getelementptr inbounds i16, i16* %p, i32 -1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_gep_negative_offset(<8 x i8>* %p) {
+  %s = getelementptr inbounds <8 x i8>, <8 x i8>* %p, i32 -1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
@@ -262,6 +582,70 @@ define <8 x i16> @load_v8i16_with_unfolded_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_offset(i16* %p) {
+  %q = ptrtoint i16* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[L0:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[L0]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_offset(<8 x i8>* %p) {
+  %q = ptrtoint <8 x i8>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -275,6 +659,62 @@ define <8 x i16> @load_v8i16_with_unfolded_gep_offset(<8 x i16>* %p) {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 2{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_with_unfolded_gep_offset(i16* %p) {
+  %s = getelementptr i16, i16* %p, i32 1
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_with_unfolded_gep_offset(<8 x i8>* %p) {
+  %s = getelementptr <8 x i8>, <8 x i8>* %p, i32 1
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_from_numeric_address () -> (v128){{$}}
@@ -287,6 +727,58 @@ define <8 x i16> @load_v8i16_from_numeric_address() {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_splat_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to i16*
+  %e = load i16, i16* %s
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_sext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_from_numeric_address:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_from_numeric_address() {
+  %s = inttoptr i32 32 to <8 x i8>*
+  %v = load <8 x i8>, <8 x i8>* %s
+  ret <8 x i8> %v
+}
+
 ; CHECK-LABEL: load_v8i16_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v8i16_from_global_address () -> (v128){{$}}
@@ -299,6 +791,57 @@ define <8 x i16> @load_v8i16_from_global_address() {
   ret <8 x i16> %v
 }
 
+; CHECK-LABEL: load_splat_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v16x8.load_splat $push[[R:[0-9]+]]=, gv_i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i16 = global i16 42
+define <8 x i16> @load_splat_v8i16_from_global_address() {
+  %e = load i16, i16* @gv_i16
+  %v1 = insertelement <8 x i16> undef, i16 %e, i32 0
+  %v2 = shufflevector <8 x i16> %v1, <8 x i16> undef, <8 x i32> zeroinitializer
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_sext_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_s $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v8i8 = global <8 x i8> <i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42, i8 42>
+define <8 x i16> @load_sext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  %v2 = sext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_zext_v8i16_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i16> @load_zext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  %v2 = zext <8 x i8> %v to <8 x i16>
+  ret <8 x i16> %v2
+}
+
+; CHECK-LABEL: load_ext_v8i16_from_global_address:
+; NO-SIMD128-NOT: load8x8
+; SIMD128-NEXT: .functype load_ext_v8i16_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i16x8.load8x8_u $push[[R:[0-9]+]]=, gv_v8i8($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <8 x i8> @load_ext_v8i16_from_global_address() {
+  %v = load <8 x i8>, <8 x i8>* @gv_v8i8
+  ret <8 x i8> %v
+}
+
+
 ; CHECK-LABEL: store_v8i16:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v8i16 (v128, i32) -> (){{$}}
@@ -400,6 +943,49 @@ define <4 x i32> @load_v4i32(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat
+define <4 x i32> @load_splat_v4i32(i32* %addr) {
+  %e = load i32, i32* %addr, align 4
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32(<4 x i16>* %p) {
+  %v = load <4 x i16>, <4 x i16>* %p
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_folded_offset (i32) -> (v128){{$}}
@@ -413,6 +999,62 @@ define <4 x i32> @load_v4i32_with_folded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_folded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_folded_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_folded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -424,6 +1066,54 @@ define <4 x i32> @load_v4i32_with_folded_gep_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_folded_gep_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_folded_gep_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_folded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -437,6 +1127,62 @@ define <4 x i32> @load_v4i32_with_unfolded_gep_negative_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_gep_negative_offset(i32* %p) {
+  %s = getelementptr inbounds i32, i32* %p, i32 -1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_gep_negative_offset(<4 x i16>* %p) {
+  %s = getelementptr inbounds <4 x i16>, <4 x i16>* %p, i32 -1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
@@ -452,6 +1198,70 @@ define <4 x i32> @load_v4i32_with_unfolded_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_offset(i32* %p) {
+  %q = ptrtoint i32* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_offset(<4 x i16>* %p) {
+  %q = ptrtoint <4 x i16>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -465,6 +1275,62 @@ define <4 x i32> @load_v4i32_with_unfolded_gep_offset(<4 x i32>* %p) {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_with_unfolded_gep_offset(i32* %p) {
+  %s = getelementptr i32, i32* %p, i32 1
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_with_unfolded_gep_offset(<4 x i16>* %p) {
+  %s = getelementptr <4 x i16>, <4 x i16>* %p, i32 1
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_from_numeric_address () -> (v128){{$}}
@@ -477,6 +1343,58 @@ define <4 x i32> @load_v4i32_from_numeric_address() {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_splat_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to i32*
+  %e = load i32, i32* %s
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_sext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_from_numeric_address:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_from_numeric_address() {
+  %s = inttoptr i32 32 to <4 x i16>*
+  %v = load <4 x i16>, <4 x i16>* %s
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: load_v4i32_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4i32_from_global_address () -> (v128){{$}}
@@ -489,6 +1407,56 @@ define <4 x i32> @load_v4i32_from_global_address() {
   ret <4 x i32> %v
 }
 
+; CHECK-LABEL: load_splat_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i32 = global i32 42
+define <4 x i32> @load_splat_v4i32_from_global_address() {
+  %e = load i32, i32* @gv_i32
+  %v1 = insertelement <4 x i32> undef, i32 %e, i32 0
+  %v2 = shufflevector <4 x i32> %v1, <4 x i32> undef, <4 x i32> zeroinitializer
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_sext_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_s $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v4i16 = global <4 x i16> <i16 42, i16 42, i16 42, i16 42>
+define <4 x i32> @load_sext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  %v2 = sext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_zext_v4i32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i32> @load_zext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  %v2 = zext <4 x i16> %v to <4 x i32>
+  ret <4 x i32> %v2
+}
+
+; CHECK-LABEL: load_ext_v4i32_from_global_address:
+; NO-SIMD128-NOT: load16x4
+; SIMD128-NEXT: .functype load_ext_v4i32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i32x4.load16x4_u $push[[R:[0-9]+]]=, gv_v4i16($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x i16> @load_ext_v4i32_from_global_address() {
+  %v = load <4 x i16>, <4 x i16>* @gv_v4i16
+  ret <4 x i16> %v
+}
+
 ; CHECK-LABEL: store_v4i32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v4i32 (v128, i32) -> (){{$}}
@@ -586,9 +1554,57 @@ define void @store_v4i32_to_global_address(<4 x i32> %v) {
 ; SIMD128-NEXT: .functype load_v2i64 (i32) -> (v128){{$}}
 ; SIMD128-NEXT: v128.load $push[[R:[0-9]+]]=, 0($0){{$}}
 ; SIMD128-NEXT: return $pop[[R]]{{$}}
-define <2 x i64> @load_v2i64(<2 x i64>* %p) {
-  %v = load <2 x i64>, <2 x i64>* %p
-  ret <2 x i64> %v
+define <2 x i64> @load_v2i64(<2 x i64>* %p) {
+  %v = load <2 x i64>, <2 x i64>* %p
+  ret <2 x i64> %v
+}
+
+; CHECK-LABEL: load_splat_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64(i64* %p) {
+  %e = load i64, i64* %p
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64(<2 x i32>* %p) {
+  %v = load <2 x i32>, <2 x i32>* %p
+  ret <2 x i32> %v
 }
 
 ; CHECK-LABEL: load_v2i64_with_folded_offset:
@@ -605,6 +1621,66 @@ define <2 x i64> @load_v2i64_with_folded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_folded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_folded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -617,6 +1693,58 @@ define <2 x i64> @load_v2i64_with_folded_gep_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_folded_gep_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_folded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -631,6 +1759,66 @@ define <2 x i64> @load_v2i64_with_unfolded_gep_negative_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_gep_negative_offset(i64* %p) {
+  %s = getelementptr inbounds i64, i64* %p, i32 -1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_gep_negative_offset(<2 x i32>* %p) {
+  %s = getelementptr inbounds <2 x i32>, <2 x i32>* %p, i32 -1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -647,6 +1835,74 @@ define <2 x i64> @load_v2i64_with_unfolded_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_offset(i64* %p) {
+  %q = ptrtoint i64* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_offset(<2 x i32>* %p) {
+  %q = ptrtoint <2 x i32>* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -661,6 +1917,66 @@ define <2 x i64> @load_v2i64_with_unfolded_gep_offset(<2 x i64>* %p) {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_with_unfolded_gep_offset(i64* %p) {
+  %s = getelementptr i64, i64* %p, i32 1
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_with_unfolded_gep_offset(<2 x i32>* %p) {
+  %s = getelementptr <2 x i32>, <2 x i32>* %p, i32 1
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -674,6 +1990,62 @@ define <2 x i64> @load_v2i64_from_numeric_address() {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_splat_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to i64*
+  %e = load i64, i64* %s
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_sext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_from_numeric_address() {
+  %s = inttoptr i32 32 to <2 x i32>*
+  %v = load <2 x i32>, <2 x i32>* %s
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: load_v2i64_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -687,6 +2059,60 @@ define <2 x i64> @load_v2i64_from_global_address() {
   ret <2 x i64> %v
 }
 
+; CHECK-LABEL: load_splat_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_i64($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_i64 = global i64 42
+define <2 x i64> @load_splat_v2i64_from_global_address() {
+  %e = load i64, i64* @gv_i64
+  %v1 = insertelement <2 x i64> undef, i64 %e, i32 0
+  %v2 = shufflevector <2 x i64> %v1, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_sext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_sext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_s $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_v2i32 = global <2 x i32> <i32 42, i32 42>
+define <2 x i64> @load_sext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  %v2 = sext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_zext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_zext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i64> @load_zext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  %v2 = zext <2 x i32> %v to <2 x i64>
+  ret <2 x i64> %v2
+}
+
+; CHECK-LABEL: load_ext_v2i64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: load32x2
+; SIMD128-NEXT: .functype load_ext_v2i64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: i64x2.load32x2_u $push[[R:[0-9]+]]=, gv_v2i32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x i32> @load_ext_v2i64_from_global_address() {
+  %v = load <2 x i32>, <2 x i32>* @gv_v2i32
+  ret <2 x i32> %v
+}
+
 ; CHECK-LABEL: store_v2i64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -796,6 +2222,18 @@ define <4 x float> @load_v4f32(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32(float* %p) {
+  %e = load float, float* %p
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_folded_offset (i32) -> (v128){{$}}
@@ -809,6 +2247,21 @@ define <4 x float> @load_v4f32_with_folded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_folded_offset(float* %p) {
+  %q = ptrtoint float* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
@@ -820,6 +2273,19 @@ define <4 x float> @load_v4f32_with_folded_gep_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 4($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_folded_gep_offset(float* %p) {
+  %s = getelementptr inbounds float, float* %p, i32 1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
@@ -833,6 +2299,21 @@ define <4 x float> @load_v4f32_with_unfolded_gep_negative_offset(<4 x float>* %p
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_gep_negative_offset(float* %p) {
+  %s = getelementptr inbounds float, float* %p, i32 -1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
@@ -848,6 +2329,23 @@ define <4 x float> @load_v4f32_with_unfolded_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_offset(float* %p) {
+  %q = ptrtoint float* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
@@ -861,6 +2359,21 @@ define <4 x float> @load_v4f32_with_unfolded_gep_offset(<4 x float>* %p) {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 4{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_with_unfolded_gep_offset(float* %p) {
+  %s = getelementptr float, float* %p, i32 1
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_from_numeric_address () -> (v128){{$}}
@@ -873,6 +2386,20 @@ define <4 x float> @load_v4f32_from_numeric_address() {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <4 x float> @load_splat_v4f32_from_numeric_address() {
+  %s = inttoptr i32 32 to float*
+  %e = load float, float* %s
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: load_v4f32_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype load_v4f32_from_global_address () -> (v128){{$}}
@@ -885,6 +2412,20 @@ define <4 x float> @load_v4f32_from_global_address() {
   ret <4 x float> %v
 }
 
+; CHECK-LABEL: load_splat_v4f32_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v4f32_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v32x4.load_splat $push[[R:[0-9]+]]=, gv_f32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_f32 = global float 42.
+define <4 x float> @load_splat_v4f32_from_global_address() {
+  %e = load float, float* @gv_f32
+  %v1 = insertelement <4 x float> undef, float %e, i32 0
+  %v2 = shufflevector <4 x float> %v1, <4 x float> undef, <4 x i32> zeroinitializer
+  ret <4 x float> %v2
+}
+
 ; CHECK-LABEL: store_v4f32:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-NEXT: .functype store_v4f32 (v128, i32) -> (){{$}}
@@ -987,6 +2528,19 @@ define <2 x double> @load_v2f64(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64 (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64(double* %p) {
+  %e = load double, double* %p
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_folded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1001,6 +2555,22 @@ define <2 x double> @load_v2f64_with_folded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_folded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 16($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_folded_offset(double* %p) {
+  %q = ptrtoint double* %p to i32
+  %r = add nuw i32 %q, 16
+  %s = inttoptr i32 %r to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_folded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1013,6 +2583,20 @@ define <2 x double> @load_v2f64_with_folded_gep_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_folded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_folded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 8($0){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_folded_gep_offset(double* %p) {
+  %s = getelementptr inbounds double, double* %p, i32 1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_gep_negative_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1027,6 +2611,22 @@ define <2 x double> @load_v2f64_with_unfolded_gep_negative_offset(<2 x double>*
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_negative_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_negative_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, -8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_gep_negative_offset(double* %p) {
+  %s = getelementptr inbounds double, double* %p, i32 -1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1043,6 +2643,24 @@ define <2 x double> @load_v2f64_with_unfolded_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 16{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_offset(double* %p) {
+  %q = ptrtoint double* %p to i32
+  %r = add nsw i32 %q, 16
+  %s = inttoptr i32 %r to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_with_unfolded_gep_offset:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1057,6 +2675,22 @@ define <2 x double> @load_v2f64_with_unfolded_gep_offset(<2 x double>* %p) {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_with_unfolded_gep_offset:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_with_unfolded_gep_offset (i32) -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 8{{$}}
+; SIMD128-NEXT: i32.add $push[[L1:[0-9]+]]=, $0, $pop[[L0]]{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 0($pop[[L1]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_with_unfolded_gep_offset(double* %p) {
+  %s = getelementptr double, double* %p, i32 1
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_from_numeric_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1070,6 +2704,21 @@ define <2 x double> @load_v2f64_from_numeric_address() {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_from_numeric_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_from_numeric_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, 32($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+define <2 x double> @load_splat_v2f64_from_numeric_address() {
+  %s = inttoptr i32 32 to double*
+  %e = load double, double* %s
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: load_v2f64_from_global_address:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
@@ -1083,6 +2732,21 @@ define <2 x double> @load_v2f64_from_global_address() {
   ret <2 x double> %v
 }
 
+; CHECK-LABEL: load_splat_v2f64_from_global_address:
+; NO-SIMD128-NOT: v128
+; SIMD128-VM-NOT: v128
+; SIMD128-NEXT: .functype load_splat_v2f64_from_global_address () -> (v128){{$}}
+; SIMD128-NEXT: i32.const $push[[L0:[0-9]+]]=, 0{{$}}
+; SIMD128-NEXT: v64x2.load_splat $push[[R:[0-9]+]]=, gv_f64($pop[[L0]]){{$}}
+; SIMD128-NEXT: return $pop[[R]]{{$}}
+@gv_f64 = global double 42.
+define <2 x double> @load_splat_v2f64_from_global_address() {
+  %e = load double, double* @gv_f64
+  %v1 = insertelement <2 x double> undef, double %e, i32 0
+  %v2 = shufflevector <2 x double> %v1, <2 x double> undef, <2 x i32> zeroinitializer
+  ret <2 x double> %v2
+}
+
 ; CHECK-LABEL: store_v2f64:
 ; NO-SIMD128-NOT: v128
 ; SIMD128-VM-NOT: v128
diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
index 4d70bbec05ce2..2dd84b6639b19 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll
@@ -19,7 +19,7 @@ define float @test_return_f1(float %f.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.f
   ; ALL:   G_STORE [[TRUNC]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
@@ -49,7 +49,7 @@ define double @test_return_d1(double %d.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d
   ; ALL:   G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.coerce.dive2)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 8), (load 1 from %ir.1, align 8)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 8), (load 1 from %ir.1, align 8)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
@@ -82,7 +82,7 @@ define { double, double } @test_return_d2(double %d.coerce0, double %d.coerce1)
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
   ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
   ; ALL:   G_STORE [[TRUNC1]](s64), [[GEP]](p0) :: (store 8 into %ir.2)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.3, align 8), (load 1 from %ir.4, align 8)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.3, align 8), (load 1 from %ir.4, align 8)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5)
   ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8)
@@ -116,7 +116,7 @@ define i32 @test_return_i1(i32 %i.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.0, align 4), (load 1 from %ir.1, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   $eax = COPY [[LOAD]](s32)
   ; ALL:   RET 0, implicit $eax
@@ -142,7 +142,7 @@ define i64 @test_return_i2(i64 %i.coerce) {
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.0, align 4)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.1, align 4), (load 1 from %ir.2, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.1, align 4), (load 1 from %ir.2, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.3, align 4)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   RET 0, implicit $rax
@@ -174,9 +174,9 @@ define { i64, i32 } @test_return_i3(i64 %i.coerce0, i32 %i.coerce1) {
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
   ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C1]](s64)
   ; ALL:   G_STORE [[COPY1]](s32), [[GEP]](p0) :: (store 4 into %ir.1)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX1]](p0), [[FRAME_INDEX2]](p0), [[C]](s64) :: (store 1 into %ir.2, align 4), (load 1 from %ir.3, align 4)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.4, align 4), (load 1 from %ir.5, align 4)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX3]](p0), [[FRAME_INDEX]](p0), [[C]](s64) :: (store 1 into %ir.6, align 8), (load 1 from %ir.7, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX1]](p0), [[FRAME_INDEX2]](p0), [[C]](s64), 0 :: (store 1 into %ir.2, align 4), (load 1 from %ir.3, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.4, align 4), (load 1 from %ir.5, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX3]](p0), [[FRAME_INDEX]](p0), [[C]](s64), 0 :: (store 1 into %ir.6, align 8), (load 1 from %ir.7, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 8 from %ir.tmp)
   ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.tmp + 8, align 8)
@@ -218,7 +218,7 @@ define { i64, i64 } @test_return_i4(i64 %i.coerce0, i64 %i.coerce1) {
   ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
   ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
   ; ALL:   G_STORE [[COPY1]](s64), [[GEP]](p0) :: (store 8 into %ir.2, align 4)
-  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64) :: (store 1 into %ir.3, align 4), (load 1 from %ir.4, align 4)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), 0 :: (store 1 into %ir.3, align 4), (load 1 from %ir.4, align 4)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5, align 4)
   ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8, align 4)
diff --git a/llvm/test/CodeGen/X86/O3-pipeline.ll b/llvm/test/CodeGen/X86/O3-pipeline.ll
index 48bc2992a7edd..d7569ccb2da24 100644
--- a/llvm/test/CodeGen/X86/O3-pipeline.ll
+++ b/llvm/test/CodeGen/X86/O3-pipeline.ll
@@ -84,6 +84,7 @@
 ; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Natural Loop Construction
 ; CHECK-NEXT:       Early Machine Loop Invariant Code Motion
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Common Subexpression Elimination
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
@@ -130,6 +131,7 @@
 ; CHECK-NEXT:       X86 FP Stackifier
 ; CHECK-NEXT:       PostRA Machine Sink
 ; CHECK-NEXT:       Machine Block Frequency Analysis
+; CHECK-NEXT:       MachineDominator Tree Construction
 ; CHECK-NEXT:       MachinePostDominator Tree Construction
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
index aa68cfd70a88e..5721c69985248 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-x86.ll
@@ -2192,3 +2192,35 @@ define <8 x float>  @test_gather_mask(<8 x float> %a0, float* %a, <8 x i32> %idx
 
   ret <8 x float> %res
 }
+
+define <2 x i64> @test_mask_demanded_bits(<2 x i64> %a0, i8* %a1, <2 x i64> %idx, <2 x i1> %mask) {
+; X86-AVX-LABEL: test_mask_demanded_bits:
+; X86-AVX:       # %bb.0:
+; X86-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX-NEXT:    vpsllq $63, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X86-AVX-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
+; X86-AVX-NEXT:    retl # encoding: [0xc3]
+;
+; X86-AVX512VL-LABEL: test_mask_demanded_bits:
+; X86-AVX512VL:       # %bb.0:
+; X86-AVX512VL-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
+; X86-AVX512VL-NEXT:    vpsllq $63, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X86-AVX512VL-NEXT:    vpgatherqq %xmm2, (%eax,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x48]
+; X86-AVX512VL-NEXT:    retl # encoding: [0xc3]
+;
+; X64-AVX-LABEL: test_mask_demanded_bits:
+; X64-AVX:       # %bb.0:
+; X64-AVX-NEXT:    vpsllq $63, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X64-AVX-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
+; X64-AVX-NEXT:    retq # encoding: [0xc3]
+;
+; X64-AVX512VL-LABEL: test_mask_demanded_bits:
+; X64-AVX512VL:       # %bb.0:
+; X64-AVX512VL-NEXT:    vpsllq $63, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0x73,0xf2,0x3f]
+; X64-AVX512VL-NEXT:    vpgatherqq %xmm2, (%rdi,%xmm1,2), %xmm0 # encoding: [0xc4,0xe2,0xe9,0x91,0x04,0x4f]
+; X64-AVX512VL-NEXT:    retq # encoding: [0xc3]
+  %mask1 = sext <2 x i1> %mask to <2 x i64>
+  %res = call <2 x i64> @llvm.x86.avx2.gather.q.q(<2 x i64> %a0,
+                            i8* %a1, <2 x i64> %idx, <2 x i64> %mask1, i8 2) ;
+  ret <2 x i64> %res
+}
diff --git a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
index fb097dda114c6..aa07a4db8d597 100644
--- a/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
+++ b/llvm/test/CodeGen/X86/avx512-broadcast-unfold.ll
@@ -4448,3 +4448,188 @@ bb1:                                              ; preds = %bb1, %bb
 bb10:                                             ; preds = %bb1
   ret void
 }
+
+define void @bcast_unfold_cmp_v8f32_refold(float* nocapture %0) {
+; CHECK-LABEL: bcast_unfold_cmp_v8f32_refold:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm0 = [2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0,2.0E+0]
+; CHECK-NEXT:    vbroadcastss {{.*#+}} ymm1 = [3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0,3.0E+0]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB126_1: # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vcmpgtps 4096(%rdi,%rax), %ymm0, %k1
+; CHECK-NEXT:    vblendmps {{.*}}(%rip){1to8}, %ymm1, %ymm2 {%k1}
+; CHECK-NEXT:    vmovups %ymm2, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB126_1
+; CHECK-NEXT:  # %bb.2:
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  br label %2
+
+2:                                                ; preds = %2, %1
+  %3 = phi i64 [ 0, %1 ], [ %10, %2 ]
+  %4 = getelementptr inbounds float, float* %0, i64 %3
+  %5 = bitcast float* %4 to <8 x float>*
+  %6 = load <8 x float>, <8 x float>* %5, align 4
+  %7 = fcmp olt <8 x float> %6, <float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00, float 2.000000e+00>
+  %8 = select <8 x i1> %7, <8 x float> <float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00, float 4.000000e+00>, <8 x float> <float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00, float 3.000000e+00>
+  %9 = bitcast float* %4 to <8 x float>*
+  store <8 x float> %8, <8 x float>* %9, align 4
+  %10 = add i64 %3, 8
+  %11 = icmp eq i64 %10, 1024
+  br i1 %11, label %12, label %2
+
+12:                                               ; preds = %2
+  ret void
+}
+
+define void @bcast_unfold_ptestm_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestm_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB127_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vptestmd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB127_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp5 = icmp ne <4 x i32> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestnm_v4i32(i32* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestnm_v4i32:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-4096, %rax # imm = 0xF000
+; CHECK-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB128_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 4096(%rdi,%rax), %xmm1
+; CHECK-NEXT:    vptestnmd %xmm0, %xmm1, %k1
+; CHECK-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k1}
+; CHECK-NEXT:    vmovdqu %xmm1, 4096(%rdi,%rax)
+; CHECK-NEXT:    addq $16, %rax
+; CHECK-NEXT:    jne .LBB128_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i32, i32* %arg, i64 %tmp
+  %tmp3 = bitcast i32* %tmp2 to <4 x i32>*
+  %tmp4 = load <4 x i32>, <4 x i32>* %tmp3, align 4
+  %tmp4b = and <4 x i32> %tmp4, <i32 2, i32 2, i32 2, i32 2>
+  %tmp5 = icmp eq <4 x i32> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i32> <i32 3, i32 3, i32 3, i32 3>, <4 x i32> %tmp4
+  %tmp7 = bitcast i32* %tmp2 to <4 x i32>*
+  store <4 x i32> %tmp6, <4 x i32>* %tmp7, align 4
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestm_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestm_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB129_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vptestmq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB129_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp5 = icmp ne <4 x i64> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
+
+define void @bcast_unfold_ptestnm_v4i64(i64* %arg) {
+; CHECK-LABEL: bcast_unfold_ptestnm_v4i64:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movq $-8192, %rax # imm = 0xE000
+; CHECK-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [2,2,2,2]
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB130_1: # %bb1
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    vmovdqu 8192(%rdi,%rax), %ymm1
+; CHECK-NEXT:    vptestnmq %ymm0, %ymm1, %k1
+; CHECK-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k1}
+; CHECK-NEXT:    vmovdqu %ymm1, 8192(%rdi,%rax)
+; CHECK-NEXT:    addq $32, %rax
+; CHECK-NEXT:    jne .LBB130_1
+; CHECK-NEXT:  # %bb.2: # %bb10
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  br label %bb1
+
+bb1:                                              ; preds = %bb1, %bb
+  %tmp = phi i64 [ 0, %bb ], [ %tmp8, %bb1 ]
+  %tmp2 = getelementptr inbounds i64, i64* %arg, i64 %tmp
+  %tmp3 = bitcast i64* %tmp2 to <4 x i64>*
+  %tmp4 = load <4 x i64>, <4 x i64>* %tmp3, align 8
+  %tmp4b = and <4 x i64> %tmp4, <i64 2, i64 2, i64 2, i64 2>
+  %tmp5 = icmp eq <4 x i64> %tmp4b, zeroinitializer
+  %tmp6 = select <4 x i1> %tmp5, <4 x i64> <i64 3, i64 3, i64 3, i64 3>, <4 x i64> %tmp4
+  %tmp7 = bitcast i64* %tmp2 to <4 x i64>*
+  store <4 x i64> %tmp6, <4 x i64>* %tmp7, align 8
+  %tmp8 = add i64 %tmp, 4
+  %tmp9 = icmp eq i64 %tmp8, 1024
+  br i1 %tmp9, label %bb10, label %bb1
+
+bb10:                                             ; preds = %bb1
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/avx512-calling-conv.ll b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
index 5bbe76c609a6e..b13c27e0d470b 100644
--- a/llvm/test/CodeGen/X86/avx512-calling-conv.ll
+++ b/llvm/test/CodeGen/X86/avx512-calling-conv.ll
@@ -520,3 +520,2804 @@ define void @test15(<64 x i8>* %x) {
   ret void
 }
 declare <64 x i8> @test15_callee(<64 x i8>)
+
+define <17 x i1> @test16(<17 x i1> %a, <17 x i1> %b) nounwind {
+; KNL-LABEL: test16:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    pushq %rbp
+; KNL-NEXT:    pushq %r15
+; KNL-NEXT:    pushq %r14
+; KNL-NEXT:    pushq %r13
+; KNL-NEXT:    pushq %r12
+; KNL-NEXT:    pushq %rbx
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $7, %k1, %k1
+; KNL-NEXT:    kshiftlw $8, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $8, %k1, %k1
+; KNL-NEXT:    kshiftlw $9, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kshiftlw $10, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kshiftlw $11, %k0, %k6
+; KNL-NEXT:    korw %k1, %k6, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kshiftlw $12, %k0, %k5
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kshiftlw $13, %k0, %k4
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k1
+; KNL-NEXT:    kshiftlw $13, %k1, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k1
+; KNL-NEXT:    korw %k0, %k1, %k0
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %edx, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kmovw %esi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $15, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $2, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %r8d, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %r9d, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $8, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $9, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $10, %k2, %k2
+; KNL-NEXT:    korw %k2, %k6, %k2
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $11, %k2, %k2
+; KNL-NEXT:    korw %k2, %k5, %k2
+; KNL-NEXT:    xorl %ecx, %ecx
+; KNL-NEXT:    testb $1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    movl $65535, %edx ## imm = 0xFFFF
+; KNL-NEXT:    movl $0, %esi
+; KNL-NEXT:    cmovnel %edx, %esi
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $12, %k2, %k2
+; KNL-NEXT:    korw %k2, %k4, %k2
+; KNL-NEXT:    testb $1, {{[0-9]+}}(%rsp)
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $13, %k2, %k2
+; KNL-NEXT:    korw %k2, %k3, %k2
+; KNL-NEXT:    cmovnel %edx, %ecx
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; KNL-NEXT:    kmovw %edx, %k2
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dl
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL-NEXT:    kandw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kandw %k1, %k2, %k1
+; KNL-NEXT:    kmovw %k1, %r8d
+; KNL-NEXT:    kshiftrw $1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r10d
+; KNL-NEXT:    kshiftrw $3, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r11d
+; KNL-NEXT:    kshiftrw $4, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r12d
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r15d
+; KNL-NEXT:    kshiftrw $6, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r14d
+; KNL-NEXT:    kshiftrw $7, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r13d
+; KNL-NEXT:    kshiftrw $8, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ebx
+; KNL-NEXT:    kshiftrw $9, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %esi
+; KNL-NEXT:    kshiftrw $10, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ebp
+; KNL-NEXT:    kshiftrw $11, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ecx
+; KNL-NEXT:    kshiftrw $12, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edx
+; KNL-NEXT:    kshiftrw $13, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edi
+; KNL-NEXT:    kshiftrw $14, %k0, %k1
+; KNL-NEXT:    andl $1, %r8d
+; KNL-NEXT:    movb %r8b, 2(%rax)
+; KNL-NEXT:    kmovw %k0, %r8d
+; KNL-NEXT:    andl $1, %r8d
+; KNL-NEXT:    andl $1, %r9d
+; KNL-NEXT:    leal (%r8,%r9,2), %r8d
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    andl $1, %r10d
+; KNL-NEXT:    leal (%r8,%r10,4), %r8d
+; KNL-NEXT:    kmovw %k0, %r10d
+; KNL-NEXT:    andl $1, %r11d
+; KNL-NEXT:    leal (%r8,%r11,8), %r8d
+; KNL-NEXT:    andl $1, %r12d
+; KNL-NEXT:    shll $4, %r12d
+; KNL-NEXT:    orl %r8d, %r12d
+; KNL-NEXT:    andl $1, %r15d
+; KNL-NEXT:    shll $5, %r15d
+; KNL-NEXT:    orl %r12d, %r15d
+; KNL-NEXT:    andl $1, %r14d
+; KNL-NEXT:    shll $6, %r14d
+; KNL-NEXT:    andl $1, %r13d
+; KNL-NEXT:    shll $7, %r13d
+; KNL-NEXT:    orl %r14d, %r13d
+; KNL-NEXT:    andl $1, %ebx
+; KNL-NEXT:    shll $8, %ebx
+; KNL-NEXT:    orl %r13d, %ebx
+; KNL-NEXT:    andl $1, %esi
+; KNL-NEXT:    shll $9, %esi
+; KNL-NEXT:    orl %ebx, %esi
+; KNL-NEXT:    andl $1, %ebp
+; KNL-NEXT:    shll $10, %ebp
+; KNL-NEXT:    orl %esi, %ebp
+; KNL-NEXT:    orl %r15d, %ebp
+; KNL-NEXT:    andl $1, %ecx
+; KNL-NEXT:    shll $11, %ecx
+; KNL-NEXT:    andl $1, %edx
+; KNL-NEXT:    shll $12, %edx
+; KNL-NEXT:    orl %ecx, %edx
+; KNL-NEXT:    andl $1, %edi
+; KNL-NEXT:    shll $13, %edi
+; KNL-NEXT:    orl %edx, %edi
+; KNL-NEXT:    andl $1, %r9d
+; KNL-NEXT:    shll $14, %r9d
+; KNL-NEXT:    orl %edi, %r9d
+; KNL-NEXT:    andl $1, %r10d
+; KNL-NEXT:    shll $15, %r10d
+; KNL-NEXT:    orl %r9d, %r10d
+; KNL-NEXT:    orl %ebp, %r10d
+; KNL-NEXT:    movw %r10w, (%rax)
+; KNL-NEXT:    popq %rbx
+; KNL-NEXT:    popq %r12
+; KNL-NEXT:    popq %r13
+; KNL-NEXT:    popq %r14
+; KNL-NEXT:    popq %r15
+; KNL-NEXT:    popq %rbp
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: test16:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    pushq %rbp
+; SKX-NEXT:    pushq %r15
+; SKX-NEXT:    pushq %r14
+; SKX-NEXT:    pushq %r13
+; SKX-NEXT:    pushq %r12
+; SKX-NEXT:    pushq %rbx
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    movq %rdi, %rax
+; SKX-NEXT:    kshiftld $31, %k0, %k0
+; SKX-NEXT:    kshiftrd $31, %k0, %k1
+; SKX-NEXT:    kshiftld $2, %k0, %k0
+; SKX-NEXT:    kord %k0, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $30, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $3, %k1, %k2
+; SKX-NEXT:    kshiftld $3, %k2, %k2
+; SKX-NEXT:    kshiftld $30, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrd $30, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $29, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $4, %k1, %k2
+; SKX-NEXT:    kshiftld $4, %k2, %k2
+; SKX-NEXT:    kshiftld $29, %k1, %k1
+; SKX-NEXT:    kshiftrd $29, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $28, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $5, %k1, %k2
+; SKX-NEXT:    kshiftld $5, %k2, %k2
+; SKX-NEXT:    kshiftld $28, %k1, %k1
+; SKX-NEXT:    kshiftrd $28, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $27, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $6, %k1, %k2
+; SKX-NEXT:    kshiftld $6, %k2, %k2
+; SKX-NEXT:    kshiftld $27, %k1, %k1
+; SKX-NEXT:    kshiftrd $27, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $26, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $7, %k1, %k2
+; SKX-NEXT:    kshiftld $7, %k2, %k2
+; SKX-NEXT:    kshiftld $26, %k1, %k1
+; SKX-NEXT:    kshiftrd $26, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $25, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $8, %k1, %k2
+; SKX-NEXT:    kshiftld $8, %k2, %k2
+; SKX-NEXT:    kshiftld $25, %k1, %k1
+; SKX-NEXT:    kshiftrd $25, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $24, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $9, %k1, %k2
+; SKX-NEXT:    kshiftld $9, %k2, %k2
+; SKX-NEXT:    kshiftld $24, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrd $24, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $23, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $10, %k1, %k2
+; SKX-NEXT:    kshiftld $10, %k2, %k2
+; SKX-NEXT:    kshiftld $23, %k1, %k1
+; SKX-NEXT:    kshiftrd $23, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $22, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $11, %k1, %k2
+; SKX-NEXT:    kshiftld $11, %k2, %k2
+; SKX-NEXT:    kshiftld $22, %k1, %k1
+; SKX-NEXT:    kshiftrd $22, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $21, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $12, %k1, %k2
+; SKX-NEXT:    kshiftld $12, %k2, %k2
+; SKX-NEXT:    kshiftld $21, %k1, %k1
+; SKX-NEXT:    kshiftrd $21, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $20, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $13, %k1, %k2
+; SKX-NEXT:    kshiftld $13, %k2, %k2
+; SKX-NEXT:    kshiftld $20, %k1, %k1
+; SKX-NEXT:    kshiftrd $20, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $19, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $14, %k1, %k2
+; SKX-NEXT:    kshiftld $14, %k2, %k2
+; SKX-NEXT:    kshiftld $19, %k1, %k1
+; SKX-NEXT:    kshiftrd $19, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $18, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $15, %k1, %k2
+; SKX-NEXT:    kshiftld $15, %k2, %k2
+; SKX-NEXT:    kshiftld $18, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrd $18, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $17, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $16, %k1, %k2
+; SKX-NEXT:    kshiftld $16, %k2, %k2
+; SKX-NEXT:    kshiftld $17, %k1, %k1
+; SKX-NEXT:    kshiftrd $17, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $16, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kshiftrd $17, %k1, %k2
+; SKX-NEXT:    kshiftld $17, %k2, %k2
+; SKX-NEXT:    kshiftld $16, %k1, %k1
+; SKX-NEXT:    kshiftrd $16, %k1, %k1
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $15, %k2, %k2
+; SKX-NEXT:    kord %k1, %k2, %k1
+; SKX-NEXT:    kmovd %esi, %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $31, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kmovd %edx, %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $30, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $3, %k0, %k2
+; SKX-NEXT:    kshiftld $3, %k2, %k2
+; SKX-NEXT:    kshiftld $30, %k0, %k0
+; SKX-NEXT:    kshiftrd $30, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovd %ecx, %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $29, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $4, %k0, %k2
+; SKX-NEXT:    kshiftld $4, %k2, %k2
+; SKX-NEXT:    kshiftld $29, %k0, %k0
+; SKX-NEXT:    kshiftrd $29, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovd %r8d, %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $28, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $5, %k0, %k2
+; SKX-NEXT:    kshiftld $5, %k2, %k2
+; SKX-NEXT:    kshiftld $28, %k0, %k0
+; SKX-NEXT:    kshiftrd $28, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovd %r9d, %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $27, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $6, %k0, %k2
+; SKX-NEXT:    kshiftld $6, %k2, %k2
+; SKX-NEXT:    kshiftld $27, %k0, %k0
+; SKX-NEXT:    kshiftrd $27, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $26, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $7, %k0, %k2
+; SKX-NEXT:    kshiftld $7, %k2, %k2
+; SKX-NEXT:    kshiftld $26, %k0, %k0
+; SKX-NEXT:    kshiftrd $26, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $25, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $8, %k0, %k2
+; SKX-NEXT:    kshiftld $8, %k2, %k2
+; SKX-NEXT:    kshiftld $25, %k0, %k0
+; SKX-NEXT:    kshiftrd $25, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $24, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $9, %k0, %k2
+; SKX-NEXT:    kshiftld $9, %k2, %k2
+; SKX-NEXT:    kshiftld $24, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrd $24, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $23, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $10, %k0, %k2
+; SKX-NEXT:    kshiftld $10, %k2, %k2
+; SKX-NEXT:    kshiftld $23, %k0, %k0
+; SKX-NEXT:    kshiftrd $23, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $22, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $11, %k0, %k2
+; SKX-NEXT:    kshiftld $11, %k2, %k2
+; SKX-NEXT:    kshiftld $22, %k0, %k0
+; SKX-NEXT:    kshiftrd $22, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $21, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $12, %k0, %k2
+; SKX-NEXT:    kshiftld $12, %k2, %k2
+; SKX-NEXT:    kshiftld $21, %k0, %k0
+; SKX-NEXT:    kshiftrd $21, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $20, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $13, %k0, %k2
+; SKX-NEXT:    kshiftld $13, %k2, %k2
+; SKX-NEXT:    kshiftld $20, %k0, %k0
+; SKX-NEXT:    kshiftrd $20, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $19, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $14, %k0, %k2
+; SKX-NEXT:    kshiftld $14, %k2, %k2
+; SKX-NEXT:    kshiftld $19, %k0, %k0
+; SKX-NEXT:    kshiftrd $19, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $18, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $15, %k0, %k2
+; SKX-NEXT:    kshiftld $15, %k2, %k2
+; SKX-NEXT:    kshiftld $18, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrd $18, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $17, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $16, %k0, %k2
+; SKX-NEXT:    kshiftld $16, %k2, %k2
+; SKX-NEXT:    kshiftld $17, %k0, %k0
+; SKX-NEXT:    kshiftrd $17, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k3, %k2
+; SKX-NEXT:    kshiftrd $16, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kshiftrd $17, %k0, %k2
+; SKX-NEXT:    kshiftld $17, %k2, %k2
+; SKX-NEXT:    kshiftld $16, %k0, %k0
+; SKX-NEXT:    kshiftrd $16, %k0, %k0
+; SKX-NEXT:    kord %k2, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $15, %k2, %k2
+; SKX-NEXT:    kord %k0, %k2, %k0
+; SKX-NEXT:    kandd %k1, %k0, %k0
+; SKX-NEXT:    kshiftrd $16, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r8d
+; SKX-NEXT:    kshiftrd $1, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrd $2, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r10d
+; SKX-NEXT:    kshiftrd $3, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r11d
+; SKX-NEXT:    kshiftrd $4, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r12d
+; SKX-NEXT:    kshiftrd $5, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r15d
+; SKX-NEXT:    kshiftrd $6, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r14d
+; SKX-NEXT:    kshiftrd $7, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r13d
+; SKX-NEXT:    kshiftrd $8, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ebx
+; SKX-NEXT:    kshiftrd $9, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %esi
+; SKX-NEXT:    kshiftrd $10, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ebp
+; SKX-NEXT:    kshiftrd $11, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ecx
+; SKX-NEXT:    kshiftrd $12, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edx
+; SKX-NEXT:    kshiftrd $13, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edi
+; SKX-NEXT:    kshiftrd $14, %k0, %k1
+; SKX-NEXT:    andl $1, %r8d
+; SKX-NEXT:    movb %r8b, 2(%rax)
+; SKX-NEXT:    kmovd %k0, %r8d
+; SKX-NEXT:    andl $1, %r8d
+; SKX-NEXT:    andl $1, %r9d
+; SKX-NEXT:    leal (%r8,%r9,2), %r8d
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrd $15, %k0, %k0
+; SKX-NEXT:    andl $1, %r10d
+; SKX-NEXT:    leal (%r8,%r10,4), %r8d
+; SKX-NEXT:    kmovd %k0, %r10d
+; SKX-NEXT:    andl $1, %r11d
+; SKX-NEXT:    leal (%r8,%r11,8), %r8d
+; SKX-NEXT:    andl $1, %r12d
+; SKX-NEXT:    shll $4, %r12d
+; SKX-NEXT:    orl %r8d, %r12d
+; SKX-NEXT:    andl $1, %r15d
+; SKX-NEXT:    shll $5, %r15d
+; SKX-NEXT:    orl %r12d, %r15d
+; SKX-NEXT:    andl $1, %r14d
+; SKX-NEXT:    shll $6, %r14d
+; SKX-NEXT:    andl $1, %r13d
+; SKX-NEXT:    shll $7, %r13d
+; SKX-NEXT:    orl %r14d, %r13d
+; SKX-NEXT:    andl $1, %ebx
+; SKX-NEXT:    shll $8, %ebx
+; SKX-NEXT:    orl %r13d, %ebx
+; SKX-NEXT:    andl $1, %esi
+; SKX-NEXT:    shll $9, %esi
+; SKX-NEXT:    orl %ebx, %esi
+; SKX-NEXT:    andl $1, %ebp
+; SKX-NEXT:    shll $10, %ebp
+; SKX-NEXT:    orl %esi, %ebp
+; SKX-NEXT:    orl %r15d, %ebp
+; SKX-NEXT:    andl $1, %ecx
+; SKX-NEXT:    shll $11, %ecx
+; SKX-NEXT:    andl $1, %edx
+; SKX-NEXT:    shll $12, %edx
+; SKX-NEXT:    orl %ecx, %edx
+; SKX-NEXT:    andl $1, %edi
+; SKX-NEXT:    shll $13, %edi
+; SKX-NEXT:    orl %edx, %edi
+; SKX-NEXT:    andl $1, %r9d
+; SKX-NEXT:    shll $14, %r9d
+; SKX-NEXT:    orl %edi, %r9d
+; SKX-NEXT:    andl $1, %r10d
+; SKX-NEXT:    shll $15, %r10d
+; SKX-NEXT:    orl %r9d, %r10d
+; SKX-NEXT:    orl %ebp, %r10d
+; SKX-NEXT:    movw %r10w, (%rax)
+; SKX-NEXT:    popq %rbx
+; SKX-NEXT:    popq %r12
+; SKX-NEXT:    popq %r13
+; SKX-NEXT:    popq %r14
+; SKX-NEXT:    popq %r15
+; SKX-NEXT:    popq %rbp
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test16:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %ebp
+; KNL_X32-NEXT:    pushl %ebx
+; KNL_X32-NEXT:    pushl %edi
+; KNL_X32-NEXT:    pushl %esi
+; KNL_X32-NEXT:    subl $20, %esp
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $2, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $1, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $3, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $2, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $4, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $3, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $5, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $4, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $6, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $5, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $7, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $6, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $9, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $9, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $7, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $8, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $8, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $8, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $8, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $9, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $7, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $9, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k2
+; KNL_X32-NEXT:    kmovw %k2, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $6, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $10, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k6
+; KNL_X32-NEXT:    korw %k1, %k6, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $5, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $11, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k5
+; KNL_X32-NEXT:    korw %k1, %k5, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $4, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $12, %k1, %k1
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k4
+; KNL_X32-NEXT:    korw %k1, %k4, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $3, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k2
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k1
+; KNL_X32-NEXT:    kshiftlw $13, %k1, %k0
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftlw $2, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k2
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k1
+; KNL_X32-NEXT:    korw %k0, %k1, %k0
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftlw $1, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    kmovw %k0, (%esp) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k2 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $15, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $2, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $9, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $9, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $8, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $8, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $8, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $9, %k2, %k2
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL_X32-NEXT:    korw %k2, %k7, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k6, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k5, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k4, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k3, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k2, %k2
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    xorl %eax, %eax
+; KNL_X32-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; KNL_X32-NEXT:    movl $65535, %ecx ## imm = 0xFFFF
+; KNL_X32-NEXT:    movl $0, %edx
+; KNL_X32-NEXT:    cmovnel %ecx, %edx
+; KNL_X32-NEXT:    kshiftlw $2, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $1, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %bl
+; KNL_X32-NEXT:    kmovw %ebx, %k1
+; KNL_X32-NEXT:    kshiftlw $15, %k1, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw %edx, %k1
+; KNL_X32-NEXT:    testb $1, {{[0-9]+}}(%esp)
+; KNL_X32-NEXT:    cmovnel %ecx, %eax
+; KNL_X32-NEXT:    kmovw (%esp), %k2 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k2, %k0, %k0
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kandw %k1, %k2, %k1
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ecx
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    movb %bl, 2(%eax)
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    leal (%ebx,%esi,2), %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    leal (%esi,%edi,4), %esi
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $8, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    leal (%esi,%ebp,8), %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $9, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edx
+; KNL_X32-NEXT:    shll $4, %edx
+; KNL_X32-NEXT:    orl %esi, %edx
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ecx
+; KNL_X32-NEXT:    shll $5, %ecx
+; KNL_X32-NEXT:    orl %edx, %ecx
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    shll $6, %ebx
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    shll $7, %edi
+; KNL_X32-NEXT:    orl %ebx, %edi
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    shll $8, %ebp
+; KNL_X32-NEXT:    orl %edi, %ebp
+; KNL_X32-NEXT:    kmovw %k1, %edi
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    shll $9, %esi
+; KNL_X32-NEXT:    orl %ebp, %esi
+; KNL_X32-NEXT:    kmovw %k1, %ebp
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_X32-NEXT:    andl $1, %edx
+; KNL_X32-NEXT:    shll $10, %edx
+; KNL_X32-NEXT:    orl %esi, %edx
+; KNL_X32-NEXT:    kmovw %k1, %esi
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    orl %ecx, %edx
+; KNL_X32-NEXT:    kmovw %k0, %ecx
+; KNL_X32-NEXT:    andl $1, %ebx
+; KNL_X32-NEXT:    shll $11, %ebx
+; KNL_X32-NEXT:    andl $1, %edi
+; KNL_X32-NEXT:    shll $12, %edi
+; KNL_X32-NEXT:    orl %ebx, %edi
+; KNL_X32-NEXT:    andl $1, %ebp
+; KNL_X32-NEXT:    shll $13, %ebp
+; KNL_X32-NEXT:    orl %edi, %ebp
+; KNL_X32-NEXT:    andl $1, %esi
+; KNL_X32-NEXT:    shll $14, %esi
+; KNL_X32-NEXT:    orl %ebp, %esi
+; KNL_X32-NEXT:    andl $1, %ecx
+; KNL_X32-NEXT:    shll $15, %ecx
+; KNL_X32-NEXT:    orl %esi, %ecx
+; KNL_X32-NEXT:    orl %edx, %ecx
+; KNL_X32-NEXT:    movw %cx, (%eax)
+; KNL_X32-NEXT:    addl $20, %esp
+; KNL_X32-NEXT:    popl %esi
+; KNL_X32-NEXT:    popl %edi
+; KNL_X32-NEXT:    popl %ebx
+; KNL_X32-NEXT:    popl %ebp
+; KNL_X32-NEXT:    retl $4
+  %c = and <17 x i1> %a, %b
+  ret <17 x i1> %c
+}
+
+define <7 x i1> @test17(<7 x i1> %a, <7 x i1> %b, <7 x i1> %c, <7 x i1> %d, <7 x i1>%e, <7 x i1>%f, <7 x i1> %g, <7 x i1> %h, <7 x i1> %i) nounwind {
+; KNL-LABEL: test17:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $14, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $14, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $14, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k3
+; KNL-NEXT:    kshiftlw $3, %k3, %k3
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $13, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k3
+; KNL-NEXT:    kshiftlw $4, %k3, %k3
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $12, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k3
+; KNL-NEXT:    kshiftlw $5, %k3, %k3
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $11, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k3
+; KNL-NEXT:    kshiftlw $6, %k3, %k3
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $10, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k3
+; KNL-NEXT:    kshiftlw $7, %k3, %k3
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k3
+; KNL-NEXT:    kshiftlw $15, %k3, %k3
+; KNL-NEXT:    kshiftrw $9, %k3, %k3
+; KNL-NEXT:    korw %k0, %k3, %k3
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $14, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k4
+; KNL-NEXT:    kshiftlw $3, %k4, %k4
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $13, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k4
+; KNL-NEXT:    kshiftlw $4, %k4, %k4
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $12, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k4
+; KNL-NEXT:    kshiftlw $5, %k4, %k4
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $11, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k4
+; KNL-NEXT:    kshiftlw $6, %k4, %k4
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $10, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k4
+; KNL-NEXT:    kshiftlw $7, %k4, %k4
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k4
+; KNL-NEXT:    kshiftlw $15, %k4, %k4
+; KNL-NEXT:    kshiftrw $9, %k4, %k4
+; KNL-NEXT:    korw %k0, %k4, %k4
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $14, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k5
+; KNL-NEXT:    kshiftlw $3, %k5, %k5
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $13, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k5
+; KNL-NEXT:    kshiftlw $4, %k5, %k5
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $12, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k5
+; KNL-NEXT:    kshiftlw $5, %k5, %k5
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $11, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k5
+; KNL-NEXT:    kshiftlw $6, %k5, %k5
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $10, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k5
+; KNL-NEXT:    kshiftlw $7, %k5, %k5
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $9, %k5, %k5
+; KNL-NEXT:    korw %k0, %k5, %k5
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $14, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k6
+; KNL-NEXT:    kshiftlw $3, %k6, %k6
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $13, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k6
+; KNL-NEXT:    kshiftlw $4, %k6, %k6
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $12, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k6
+; KNL-NEXT:    kshiftlw $5, %k6, %k6
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $11, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k6
+; KNL-NEXT:    kshiftlw $6, %k6, %k6
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $10, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k6
+; KNL-NEXT:    kshiftlw $7, %k6, %k6
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k6
+; KNL-NEXT:    kshiftlw $15, %k6, %k6
+; KNL-NEXT:    kshiftrw $9, %k6, %k6
+; KNL-NEXT:    korw %k0, %k6, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $14, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k7
+; KNL-NEXT:    kshiftlw $3, %k7, %k7
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $13, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k7
+; KNL-NEXT:    kshiftlw $4, %k7, %k7
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $12, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k7
+; KNL-NEXT:    kshiftlw $5, %k7, %k7
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $11, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k7
+; KNL-NEXT:    kshiftlw $6, %k7, %k7
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $10, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k7
+; KNL-NEXT:    kshiftlw $7, %k7, %k7
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %dil
+; KNL-NEXT:    kmovw %edi, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $9, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k7
+; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %edx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $14, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %r8d, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %r9d, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $15, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $14, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftrw $3, %k1, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kshiftlw $14, %k1, %k1
+; KNL-NEXT:    kshiftrw $14, %k1, %k1
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftrw $4, %k1, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kshiftlw $13, %k1, %k1
+; KNL-NEXT:    kshiftrw $13, %k1, %k1
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftrw $5, %k1, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kshiftrw $12, %k1, %k1
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftrw $6, %k1, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kshiftrw $11, %k1, %k1
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftrw $7, %k1, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kshiftrw $10, %k1, %k1
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %cl
+; KNL-NEXT:    kmovw %ecx, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kandw %k7, %k0, %k0
+; KNL-NEXT:    kandw %k6, %k0, %k0
+; KNL-NEXT:    kandw %k5, %k0, %k0
+; KNL-NEXT:    kandw %k4, %k0, %k0
+; KNL-NEXT:    kandw %k3, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    kandw %k1, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r8d
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r9d
+; KNL-NEXT:    kshiftrw $4, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %r10d
+; KNL-NEXT:    kshiftrw $3, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edi
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %ecx
+; KNL-NEXT:    kshiftrw $1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, %edx
+; KNL-NEXT:    kmovw %k0, %esi
+; KNL-NEXT:    andb $1, %sil
+; KNL-NEXT:    andb $1, %dl
+; KNL-NEXT:    addb %dl, %dl
+; KNL-NEXT:    orb %sil, %dl
+; KNL-NEXT:    andb $1, %cl
+; KNL-NEXT:    shlb $2, %cl
+; KNL-NEXT:    orb %dl, %cl
+; KNL-NEXT:    andb $1, %dil
+; KNL-NEXT:    shlb $3, %dil
+; KNL-NEXT:    orb %cl, %dil
+; KNL-NEXT:    andb $1, %r10b
+; KNL-NEXT:    shlb $4, %r10b
+; KNL-NEXT:    orb %dil, %r10b
+; KNL-NEXT:    andb $1, %r9b
+; KNL-NEXT:    shlb $5, %r9b
+; KNL-NEXT:    orb %r10b, %r9b
+; KNL-NEXT:    shlb $6, %r8b
+; KNL-NEXT:    orb %r9b, %r8b
+; KNL-NEXT:    andb $127, %r8b
+; KNL-NEXT:    movb %r8b, (%rax)
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: test17:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    movq %rdi, %rax
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k1
+; SKX-NEXT:    kshiftlb $2, %k0, %k0
+; SKX-NEXT:    korb %k0, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k2
+; SKX-NEXT:    kshiftlb $3, %k2, %k2
+; SKX-NEXT:    kshiftlb $6, %k1, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k1
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kshiftrb $4, %k1, %k2
+; SKX-NEXT:    kshiftlb $4, %k2, %k2
+; SKX-NEXT:    kshiftlb $5, %k1, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k1
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kshiftrb $5, %k1, %k2
+; SKX-NEXT:    kshiftlb $5, %k2, %k2
+; SKX-NEXT:    kshiftlb $4, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrb $4, %k1, %k1
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k3, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kshiftrb $6, %k1, %k2
+; SKX-NEXT:    kshiftlb $6, %k2, %k2
+; SKX-NEXT:    kshiftlb $3, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kshiftlb $7, %k3, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kshiftrb $7, %k1, %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftlb $2, %k1, %k1
+; SKX-NEXT:    kshiftrb $2, %k1, %k1
+; SKX-NEXT:    korb %k2, %k1, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $1, %k2, %k2
+; SKX-NEXT:    korb %k1, %k2, %k1
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k2
+; SKX-NEXT:    kshiftlb $7, %k2, %k2
+; SKX-NEXT:    kshiftrb $7, %k2, %k2
+; SKX-NEXT:    korb %k0, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $6, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k3
+; SKX-NEXT:    kshiftlb $3, %k3, %k3
+; SKX-NEXT:    kshiftlb $6, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k3
+; SKX-NEXT:    kshiftlb $4, %k3, %k3
+; SKX-NEXT:    kshiftlb $5, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k3
+; SKX-NEXT:    kshiftlb $5, %k3, %k3
+; SKX-NEXT:    kshiftlb $4, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k3
+; SKX-NEXT:    kshiftlb $6, %k3, %k3
+; SKX-NEXT:    kshiftlb $3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kshiftlb $7, %k4, %k3
+; SKX-NEXT:    kshiftrb $2, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $7, %k2, %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftlb $2, %k2, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kshiftlb $7, %k4, %k3
+; SKX-NEXT:    kshiftrb $1, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kandb %k1, %k2, %k1
+; SKX-NEXT:    kshiftlb $7, %k4, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftrb $7, %k2, %k2
+; SKX-NEXT:    korb %k0, %k2, %k2
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $6, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k3
+; SKX-NEXT:    kshiftlb $3, %k3, %k3
+; SKX-NEXT:    kshiftlb $6, %k2, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kshiftlb $7, %k4, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k3
+; SKX-NEXT:    kshiftlb $4, %k3, %k3
+; SKX-NEXT:    kshiftlb $5, %k2, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $5, %k2, %k3
+; SKX-NEXT:    kshiftlb $5, %k3, %k3
+; SKX-NEXT:    kshiftlb $4, %k2, %k2
+; SKX-NEXT:    kshiftrb $4, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $6, %k2, %k3
+; SKX-NEXT:    kshiftlb $6, %k3, %k3
+; SKX-NEXT:    kshiftlb $3, %k2, %k2
+; SKX-NEXT:    kshiftrb $3, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $2, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kshiftrb $7, %k2, %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftlb $2, %k2, %k2
+; SKX-NEXT:    kshiftrb $2, %k2, %k2
+; SKX-NEXT:    korb %k3, %k2, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $1, %k3, %k3
+; SKX-NEXT:    korb %k2, %k3, %k2
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k3
+; SKX-NEXT:    kshiftlb $7, %k3, %k3
+; SKX-NEXT:    kshiftrb $7, %k3, %k3
+; SKX-NEXT:    korb %k0, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k4
+; SKX-NEXT:    kshiftlb $3, %k4, %k4
+; SKX-NEXT:    kshiftlb $6, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftrb $6, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kshiftlb $7, %k5, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k4
+; SKX-NEXT:    kshiftlb $4, %k4, %k4
+; SKX-NEXT:    kshiftlb $5, %k3, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kshiftlb $7, %k5, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k4
+; SKX-NEXT:    kshiftlb $5, %k4, %k4
+; SKX-NEXT:    kshiftlb $4, %k3, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $6, %k3, %k4
+; SKX-NEXT:    kshiftlb $6, %k4, %k4
+; SKX-NEXT:    kshiftlb $3, %k3, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $2, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $7, %k3, %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftlb $2, %k3, %k3
+; SKX-NEXT:    kshiftrb $2, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $1, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $7, %k4, %k4
+; SKX-NEXT:    korb %k0, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $6, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k5
+; SKX-NEXT:    kshiftlb $3, %k5, %k5
+; SKX-NEXT:    kshiftlb $6, %k4, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $5, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k5
+; SKX-NEXT:    kshiftlb $4, %k5, %k5
+; SKX-NEXT:    kshiftlb $5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    kshiftrb $5, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k6, %k5
+; SKX-NEXT:    kshiftrb $4, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k5
+; SKX-NEXT:    kshiftlb $5, %k5, %k5
+; SKX-NEXT:    kshiftlb $4, %k4, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k6, %k5
+; SKX-NEXT:    kshiftrb $3, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k5
+; SKX-NEXT:    kshiftlb $6, %k5, %k5
+; SKX-NEXT:    kshiftlb $3, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $2, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $7, %k4, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftlb $2, %k4, %k4
+; SKX-NEXT:    kshiftrb $2, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $1, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kandb %k3, %k4, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kandb %k2, %k3, %k2
+; SKX-NEXT:    kshiftlb $7, %k4, %k3
+; SKX-NEXT:    kshiftrb $7, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    korb %k0, %k3, %k3
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k4
+; SKX-NEXT:    kshiftlb $3, %k4, %k4
+; SKX-NEXT:    kshiftlb $6, %k3, %k3
+; SKX-NEXT:    kshiftrb $6, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k4
+; SKX-NEXT:    kshiftlb $4, %k4, %k4
+; SKX-NEXT:    kshiftlb $5, %k3, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $5, %k3, %k4
+; SKX-NEXT:    kshiftlb $5, %k4, %k4
+; SKX-NEXT:    kshiftlb $4, %k3, %k3
+; SKX-NEXT:    kshiftrb $4, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $6, %k3, %k4
+; SKX-NEXT:    kshiftlb $6, %k4, %k4
+; SKX-NEXT:    kshiftlb $3, %k3, %k3
+; SKX-NEXT:    kshiftrb $3, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $2, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftrb $7, %k3, %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftlb $2, %k3, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftrb $2, %k3, %k3
+; SKX-NEXT:    korb %k4, %k3, %k3
+; SKX-NEXT:    kshiftlb $7, %k5, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftrb $1, %k4, %k4
+; SKX-NEXT:    korb %k3, %k4, %k3
+; SKX-NEXT:    kshiftlb $7, %k5, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftrb $7, %k4, %k4
+; SKX-NEXT:    korb %k0, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $6, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k5
+; SKX-NEXT:    kshiftlb $3, %k5, %k5
+; SKX-NEXT:    kshiftlb $6, %k4, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k6, %k5
+; SKX-NEXT:    kshiftrb $5, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k5
+; SKX-NEXT:    kshiftlb $4, %k5, %k5
+; SKX-NEXT:    kshiftlb $5, %k4, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $4, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k5
+; SKX-NEXT:    kshiftlb $5, %k5, %k5
+; SKX-NEXT:    kshiftlb $4, %k4, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $3, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k5
+; SKX-NEXT:    kshiftlb $6, %k5, %k5
+; SKX-NEXT:    kshiftlb $3, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $2, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $7, %k4, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftlb $2, %k4, %k4
+; SKX-NEXT:    kshiftrb $2, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $1, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kandb %k3, %k4, %k3
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k4
+; SKX-NEXT:    kshiftlb $7, %k4, %k4
+; SKX-NEXT:    kshiftrb $7, %k4, %k4
+; SKX-NEXT:    korb %k0, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $6, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k5
+; SKX-NEXT:    kshiftlb $3, %k5, %k5
+; SKX-NEXT:    kshiftlb $6, %k4, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $5, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k5
+; SKX-NEXT:    kshiftlb $4, %k5, %k5
+; SKX-NEXT:    kshiftlb $5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    kshiftrb $5, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k6, %k5
+; SKX-NEXT:    kshiftrb $4, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $5, %k4, %k5
+; SKX-NEXT:    kshiftlb $5, %k5, %k5
+; SKX-NEXT:    kshiftlb $4, %k4, %k4
+; SKX-NEXT:    kshiftrb $4, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k6
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kshiftlb $7, %k6, %k5
+; SKX-NEXT:    kshiftrb $3, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $6, %k4, %k5
+; SKX-NEXT:    kshiftlb $6, %k5, %k5
+; SKX-NEXT:    kshiftlb $3, %k4, %k4
+; SKX-NEXT:    kshiftrb $3, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $2, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kshiftrb $7, %k4, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftlb $2, %k4, %k4
+; SKX-NEXT:    kshiftrb $2, %k4, %k4
+; SKX-NEXT:    korb %k5, %k4, %k4
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $1, %k5, %k5
+; SKX-NEXT:    korb %k4, %k5, %k4
+; SKX-NEXT:    kmovd %esi, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $7, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kmovd %edx, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $6, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kshiftrb $3, %k0, %k5
+; SKX-NEXT:    kshiftlb $3, %k5, %k5
+; SKX-NEXT:    kshiftlb $6, %k0, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k0
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovd %ecx, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $5, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kshiftrb $4, %k0, %k5
+; SKX-NEXT:    kshiftlb $4, %k5, %k5
+; SKX-NEXT:    kshiftlb $5, %k0, %k0
+; SKX-NEXT:    kshiftrb $5, %k0, %k0
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovd %r8d, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $4, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kshiftrb $5, %k0, %k5
+; SKX-NEXT:    kshiftlb $5, %k5, %k5
+; SKX-NEXT:    kshiftlb $4, %k0, %k0
+; SKX-NEXT:    kshiftrb $4, %k0, %k0
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovd %r9d, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $3, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k5
+; SKX-NEXT:    kshiftlb $6, %k5, %k5
+; SKX-NEXT:    kshiftlb $3, %k0, %k0
+; SKX-NEXT:    kshiftrb $3, %k0, %k0
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $2, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kshiftrb $7, %k0, %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftlb $2, %k0, %k0
+; SKX-NEXT:    kshiftrb $2, %k0, %k0
+; SKX-NEXT:    korb %k5, %k0, %k0
+; SKX-NEXT:    kmovb {{[0-9]+}}(%rsp), %k5
+; SKX-NEXT:    kshiftlb $7, %k5, %k5
+; SKX-NEXT:    kshiftrb $1, %k5, %k5
+; SKX-NEXT:    korb %k0, %k5, %k0
+; SKX-NEXT:    kandb %k4, %k0, %k0
+; SKX-NEXT:    kandb %k3, %k0, %k0
+; SKX-NEXT:    kandb %k2, %k0, %k0
+; SKX-NEXT:    kandb %k1, %k0, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r8d
+; SKX-NEXT:    kshiftrb $5, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r9d
+; SKX-NEXT:    kshiftrb $4, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %r10d
+; SKX-NEXT:    kshiftrb $3, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edi
+; SKX-NEXT:    kshiftrb $2, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %ecx
+; SKX-NEXT:    kshiftrb $1, %k0, %k1
+; SKX-NEXT:    kmovd %k1, %edx
+; SKX-NEXT:    kmovd %k0, %esi
+; SKX-NEXT:    andb $1, %sil
+; SKX-NEXT:    andb $1, %dl
+; SKX-NEXT:    addb %dl, %dl
+; SKX-NEXT:    orb %sil, %dl
+; SKX-NEXT:    andb $1, %cl
+; SKX-NEXT:    shlb $2, %cl
+; SKX-NEXT:    orb %dl, %cl
+; SKX-NEXT:    andb $1, %dil
+; SKX-NEXT:    shlb $3, %dil
+; SKX-NEXT:    orb %cl, %dil
+; SKX-NEXT:    andb $1, %r10b
+; SKX-NEXT:    shlb $4, %r10b
+; SKX-NEXT:    orb %dil, %r10b
+; SKX-NEXT:    andb $1, %r9b
+; SKX-NEXT:    shlb $5, %r9b
+; SKX-NEXT:    orb %r10b, %r9b
+; SKX-NEXT:    shlb $6, %r8b
+; SKX-NEXT:    orb %r9b, %r8b
+; SKX-NEXT:    andb $127, %r8b
+; SKX-NEXT:    movb %r8b, (%rax)
+; SKX-NEXT:    retq
+;
+; KNL_X32-LABEL: test17:
+; KNL_X32:       ## %bb.0:
+; KNL_X32-NEXT:    pushl %ebx
+; KNL_X32-NEXT:    pushl %eax
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftlw $2, %k0, %k1
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $14, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $9, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kmovw %k0, {{[-0-9]+}}(%e{{[sb]}}p) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $14, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $9, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kmovw %k0, (%esp) ## 2-byte Spill
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $14, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k3
+; KNL_X32-NEXT:    kshiftlw $3, %k3, %k3
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $13, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k3
+; KNL_X32-NEXT:    kshiftlw $4, %k3, %k3
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $12, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k3
+; KNL_X32-NEXT:    kshiftlw $5, %k3, %k3
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $11, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k3
+; KNL_X32-NEXT:    kshiftlw $6, %k3, %k3
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $10, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k3
+; KNL_X32-NEXT:    kshiftlw $7, %k3, %k3
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k3, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k3
+; KNL_X32-NEXT:    kshiftlw $15, %k3, %k3
+; KNL_X32-NEXT:    kshiftrw $9, %k3, %k3
+; KNL_X32-NEXT:    korw %k0, %k3, %k3
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $14, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k4
+; KNL_X32-NEXT:    kshiftlw $3, %k4, %k4
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $13, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k4
+; KNL_X32-NEXT:    kshiftlw $4, %k4, %k4
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $12, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k4
+; KNL_X32-NEXT:    kshiftlw $5, %k4, %k4
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $11, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k4
+; KNL_X32-NEXT:    kshiftlw $6, %k4, %k4
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $10, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k4
+; KNL_X32-NEXT:    kshiftlw $7, %k4, %k4
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k4, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k4
+; KNL_X32-NEXT:    kshiftlw $15, %k4, %k4
+; KNL_X32-NEXT:    kshiftrw $9, %k4, %k4
+; KNL_X32-NEXT:    korw %k0, %k4, %k4
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $14, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k5
+; KNL_X32-NEXT:    kshiftlw $3, %k5, %k5
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $13, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k5
+; KNL_X32-NEXT:    kshiftlw $4, %k5, %k5
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $12, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k5
+; KNL_X32-NEXT:    kshiftlw $5, %k5, %k5
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $11, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k5
+; KNL_X32-NEXT:    kshiftlw $6, %k5, %k5
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $10, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k5
+; KNL_X32-NEXT:    kshiftlw $7, %k5, %k5
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k5, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k5
+; KNL_X32-NEXT:    kshiftlw $15, %k5, %k5
+; KNL_X32-NEXT:    kshiftrw $9, %k5, %k5
+; KNL_X32-NEXT:    korw %k0, %k5, %k5
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $14, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k6
+; KNL_X32-NEXT:    kshiftlw $3, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $13, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k6
+; KNL_X32-NEXT:    kshiftlw $4, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $12, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k6
+; KNL_X32-NEXT:    kshiftlw $5, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $11, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k6
+; KNL_X32-NEXT:    kshiftlw $6, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $10, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k6
+; KNL_X32-NEXT:    kshiftlw $7, %k6, %k6
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k6, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k6
+; KNL_X32-NEXT:    kshiftlw $15, %k6, %k6
+; KNL_X32-NEXT:    kshiftrw $9, %k6, %k6
+; KNL_X32-NEXT:    korw %k0, %k6, %k6
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $14, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k7
+; KNL_X32-NEXT:    kshiftlw $3, %k7, %k7
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $13, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k7
+; KNL_X32-NEXT:    kshiftlw $4, %k7, %k7
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $12, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k7
+; KNL_X32-NEXT:    kshiftlw $5, %k7, %k7
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $11, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k7
+; KNL_X32-NEXT:    kshiftlw $6, %k7, %k7
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $10, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k7
+; KNL_X32-NEXT:    kshiftlw $7, %k7, %k7
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k7, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k7
+; KNL_X32-NEXT:    kshiftlw $15, %k7, %k7
+; KNL_X32-NEXT:    kshiftrw $9, %k7, %k7
+; KNL_X32-NEXT:    korw %k0, %k7, %k7
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k0
+; KNL_X32-NEXT:    kshiftlw $15, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $15, %k0, %k0
+; KNL_X32-NEXT:    korw %k1, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $14, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $14, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $13, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $12, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $11, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    kshiftrw $7, %k0, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $10, %k0, %k0
+; KNL_X32-NEXT:    korw %k2, %k0, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $9, %k2, %k2
+; KNL_X32-NEXT:    korw %k0, %k2, %k0
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $15, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $14, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kshiftrw $3, %k1, %k2
+; KNL_X32-NEXT:    kshiftlw $3, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $14, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $14, %k1, %k1
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $13, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kshiftrw $4, %k1, %k2
+; KNL_X32-NEXT:    kshiftlw $4, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $13, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $13, %k1, %k1
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $12, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kshiftrw $5, %k1, %k2
+; KNL_X32-NEXT:    kshiftlw $5, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $12, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $12, %k1, %k1
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $11, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kshiftrw $6, %k1, %k2
+; KNL_X32-NEXT:    kshiftlw $6, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $11, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $11, %k1, %k1
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $10, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kshiftrw $7, %k1, %k2
+; KNL_X32-NEXT:    kshiftlw $7, %k2, %k2
+; KNL_X32-NEXT:    kshiftlw $10, %k1, %k1
+; KNL_X32-NEXT:    kshiftrw $10, %k1, %k1
+; KNL_X32-NEXT:    korw %k2, %k1, %k1
+; KNL_X32-NEXT:    movb {{[0-9]+}}(%esp), %al
+; KNL_X32-NEXT:    kmovw %eax, %k2
+; KNL_X32-NEXT:    kshiftlw $15, %k2, %k2
+; KNL_X32-NEXT:    kshiftrw $9, %k2, %k2
+; KNL_X32-NEXT:    korw %k1, %k2, %k1
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kandw %k7, %k0, %k0
+; KNL_X32-NEXT:    kandw %k6, %k0, %k0
+; KNL_X32-NEXT:    kandw %k5, %k0, %k0
+; KNL_X32-NEXT:    kandw %k4, %k0, %k0
+; KNL_X32-NEXT:    kandw %k3, %k0, %k0
+; KNL_X32-NEXT:    kmovw (%esp), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kmovw {{[-0-9]+}}(%e{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL_X32-NEXT:    kandw %k1, %k0, %k0
+; KNL_X32-NEXT:    kshiftrw $6, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %ecx
+; KNL_X32-NEXT:    kshiftrw $5, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %eax
+; KNL_X32-NEXT:    kshiftrw $1, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $2, %k0, %k1
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    andb $1, %dl
+; KNL_X32-NEXT:    addb %dl, %dl
+; KNL_X32-NEXT:    orb %bl, %dl
+; KNL_X32-NEXT:    kmovw %k1, %ebx
+; KNL_X32-NEXT:    kshiftrw $3, %k0, %k1
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    shlb $2, %bl
+; KNL_X32-NEXT:    orb %dl, %bl
+; KNL_X32-NEXT:    kmovw %k1, %edx
+; KNL_X32-NEXT:    kshiftrw $4, %k0, %k0
+; KNL_X32-NEXT:    andb $1, %dl
+; KNL_X32-NEXT:    shlb $3, %dl
+; KNL_X32-NEXT:    orb %bl, %dl
+; KNL_X32-NEXT:    kmovw %k0, %ebx
+; KNL_X32-NEXT:    andb $1, %bl
+; KNL_X32-NEXT:    shlb $4, %bl
+; KNL_X32-NEXT:    orb %dl, %bl
+; KNL_X32-NEXT:    andb $1, %al
+; KNL_X32-NEXT:    shlb $5, %al
+; KNL_X32-NEXT:    orb %bl, %al
+; KNL_X32-NEXT:    shlb $6, %cl
+; KNL_X32-NEXT:    orb %al, %cl
+; KNL_X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_X32-NEXT:    andb $127, %cl
+; KNL_X32-NEXT:    movb %cl, (%eax)
+; KNL_X32-NEXT:    addl $4, %esp
+; KNL_X32-NEXT:    popl %ebx
+; KNL_X32-NEXT:    retl $4
+  %j = and <7 x i1> %a, %b
+  %k = and <7 x i1> %j, %c
+  %l = and <7 x i1> %k, %d
+  %m = and <7 x i1> %l, %e
+  %n = and <7 x i1> %m, %f
+  %o = and <7 x i1> %n, %g
+  %p = and <7 x i1> %o, %h
+  %q = and <7 x i1> %p, %i
+  ret <7 x i1> %q
+}
diff --git a/llvm/test/CodeGen/X86/avx512-cmp.ll b/llvm/test/CodeGen/X86/avx512-cmp.ll
index ecdca99bea312..3f3141e8876c7 100644
--- a/llvm/test/CodeGen/X86/avx512-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-cmp.ll
@@ -70,9 +70,8 @@ define float @test5(float %p) #0 {
 ; ALL-NEXT:    retq
 ; ALL-NEXT:  LBB3_1: ## %if.end
 ; ALL-NEXT:    vcmpltss %xmm0, %xmm1, %k1
-; ALL-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; ALL-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; ALL-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; ALL-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1}
 ; ALL-NEXT:    retq
 entry:
   %cmp = fcmp oeq float %p, 0.000000e+00
diff --git a/llvm/test/CodeGen/X86/avx512-ext.ll b/llvm/test/CodeGen/X86/avx512-ext.ll
index 1e484da2fffc9..fcb07a504067b 100644
--- a/llvm/test/CodeGen/X86/avx512-ext.ll
+++ b/llvm/test/CodeGen/X86/avx512-ext.ll
@@ -1886,22 +1886,496 @@ define void @extload_v8i64(<8 x i8>* %a, <8 x i64>* %res) {
 define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ; KNL-LABEL: test21:
 ; KNL:       # %bb.0:
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; KNL-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; KNL-NEXT:    vpsllw $15, %ymm4, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
-; KNL-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; KNL-NEXT:    vpsllw $15, %ymm5, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    kmovw %edi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k3
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k4
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k5
+; KNL-NEXT:    kmovw %r8d, %k1
+; KNL-NEXT:    kshiftlw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k6
+; KNL-NEXT:    kmovw %r9d, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k6, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k7
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $7, %k1, %k1
+; KNL-NEXT:    kshiftlw $8, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $8, %k1, %k1
+; KNL-NEXT:    kshiftlw $9, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kshiftlw $10, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kshiftlw $11, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kshiftlw $12, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kshiftlw $13, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $13, %k1, %k1
+; KNL-NEXT:    kshiftlw $14, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    korw %k0, %k1, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $2, %k1, %k1
+; KNL-NEXT:    kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $4, %k1, %k1
+; KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k6, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $8, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $13, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $14, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k6, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $7, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $8, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $13, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $14, %k1, %k1
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $15, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    korw %k0, %k1, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $15, %k7, %k7
+; KNL-NEXT:    korw %k0, %k7, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $2, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    korw %k7, %k1, %k7
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $3, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    korw %k7, %k1, %k7
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $4, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    korw %k7, %k1, %k7
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k7
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $6, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    korw %k7, %k1, %k7
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k2, %k7
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k3, %k7
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k4, %k7
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    korw %k7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k5, %k6
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    korw %k6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $11, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k6, %k2, %k5
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    korw %k5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kshiftlw $12, %k5, %k5
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k5, %k2, %k4
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    korw %k4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k4
+; KNL-NEXT:    kshiftlw $13, %k4, %k4
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k4, %k2, %k3
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    korw %k3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k3
+; KNL-NEXT:    kshiftlw $14, %k3, %k3
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k2 # 2-byte Reload
+; KNL-NEXT:    korw %k3, %k2, %k2
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k2
+; KNL-NEXT:    vpternlogd $255, %zmm4, %zmm4, %zmm4 {%k2} {z}
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    vpternlogd $255, %zmm5, %zmm5, %zmm5 {%k1} {z}
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    vpternlogd $255, %zmm6, %zmm6, %zmm6 {%k1} {z}
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; KNL-NEXT:    vpternlogd $255, %zmm7, %zmm7, %zmm7 {%k1} {z}
+; KNL-NEXT:    vpmovdw %zmm4, %ymm4
 ; KNL-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; KNL-NEXT:    vpsllw $15, %ymm6, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    vpmovdw %zmm5, %ymm4
 ; KNL-NEXT:    vpand %ymm2, %ymm4, %ymm2
-; KNL-NEXT:    vpsllw $15, %ymm7, %ymm4
-; KNL-NEXT:    vpsraw $15, %ymm4, %ymm4
+; KNL-NEXT:    vpmovdw %zmm6, %ymm4
 ; KNL-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; KNL-NEXT:    vpmovdw %zmm7, %ymm4
+; KNL-NEXT:    vpand %ymm0, %ymm4, %ymm0
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: test21:
@@ -1915,22 +2389,496 @@ define <64 x i16> @test21(<64 x i16> %x , <64 x i1> %mask) nounwind readnone {
 ;
 ; AVX512DQNOBW-LABEL: test21:
 ; AVX512DQNOBW:       # %bb.0:
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm7 = xmm7[0],zero,xmm7[1],zero,xmm7[2],zero,xmm7[3],zero,xmm7[4],zero,xmm7[5],zero,xmm7[6],zero,xmm7[7],zero,xmm7[8],zero,xmm7[9],zero,xmm7[10],zero,xmm7[11],zero,xmm7[12],zero,xmm7[13],zero,xmm7[14],zero,xmm7[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; AVX512DQNOBW-NEXT:    vpmovzxbw {{.*#+}} ymm4 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm4, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
-; AVX512DQNOBW-NEXT:    vpand %ymm0, %ymm4, %ymm0
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm5, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    kmovw %edi, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    kmovw %esi, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k0, %k3
+; AVX512DQNOBW-NEXT:    kmovw %edx, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k3, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k0, %k4
+; AVX512DQNOBW-NEXT:    kmovw %ecx, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k4, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k0, %k5
+; AVX512DQNOBW-NEXT:    kmovw %r8d, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k5, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k0, %k6
+; AVX512DQNOBW-NEXT:    kmovw %r9d, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k6, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k0, %k7
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k1, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k7, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k0, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    korw %k1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k0, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k1, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k1
+; AVX512DQNOBW-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k0, %k1, %k0
+; AVX512DQNOBW-NEXT:    korw %k0, %k2, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k2, %k3, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k2, %k4, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k2, %k5, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k6, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k3, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k4, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k5, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k6, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k3, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k4, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k5, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k0
+; AVX512DQNOBW-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k2, %k0, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    kshiftrw $15, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k2, %k7, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k7, %k7
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k7, %k7
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $13, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k7, %k7
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $12, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k6, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $11, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k7, %k7
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k7, %k0, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $10, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k1, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $9, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k3, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $8, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $9, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k4, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $7, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k7, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k7
+; AVX512DQNOBW-NEXT:    kshiftlw $10, %k7, %k7
+; AVX512DQNOBW-NEXT:    korw %k7, %k5, %k6
+; AVX512DQNOBW-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $6, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k6, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k6
+; AVX512DQNOBW-NEXT:    kshiftlw $11, %k6, %k6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k6, %k1, %k5
+; AVX512DQNOBW-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $5, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k5, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k5
+; AVX512DQNOBW-NEXT:    kshiftlw $12, %k5, %k5
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k5, %k1, %k4
+; AVX512DQNOBW-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $4, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k4, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k4
+; AVX512DQNOBW-NEXT:    kshiftlw $13, %k4, %k4
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k4, %k1, %k3
+; AVX512DQNOBW-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $3, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k3, %k2, %k2
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k3
+; AVX512DQNOBW-NEXT:    kshiftlw $14, %k3, %k3
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    korw %k3, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQNOBW-NEXT:    kshiftrw $2, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k1, %k2, %k1
+; AVX512DQNOBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512DQNOBW-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQNOBW-NEXT:    kmovw %eax, %k2
+; AVX512DQNOBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQNOBW-NEXT:    korw %k2, %k1, %k1
+; AVX512DQNOBW-NEXT:    vpmovm2d %k1, %zmm4
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm5
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm6
+; AVX512DQNOBW-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 # 2-byte Reload
+; AVX512DQNOBW-NEXT:    vpmovm2d %k0, %zmm7
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm4, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm1, %ymm4, %ymm1
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm6, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm5, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm2, %ymm4, %ymm2
-; AVX512DQNOBW-NEXT:    vpsllw $15, %ymm7, %ymm4
-; AVX512DQNOBW-NEXT:    vpsraw $15, %ymm4, %ymm4
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm6, %ymm4
 ; AVX512DQNOBW-NEXT:    vpand %ymm3, %ymm4, %ymm3
+; AVX512DQNOBW-NEXT:    vpmovdw %zmm7, %ymm4
+; AVX512DQNOBW-NEXT:    vpand %ymm0, %ymm4, %ymm0
 ; AVX512DQNOBW-NEXT:    retq
   %ret = select <64 x i1> %mask, <64 x i16> %x, <64 x i16> zeroinitializer
   ret <64 x i16> %ret
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index 37f16489df4a1..6e36bd1bb0eb1 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -302,12 +302,15 @@ define i16 @test16(i1 *%addr, i16 %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    movb (%rdi), %al
 ; KNL-NEXT:    kmovw %esi, %k0
-; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kshiftrw $10, %k0, %k2
-; KNL-NEXT:    kxorw %k1, %k2, %k1
-; KNL-NEXT:    kshiftlw $15, %k1, %k1
-; KNL-NEXT:    kshiftrw $5, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $5, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; KNL-NEXT:    retq
@@ -316,11 +319,14 @@ define i16 @test16(i1 *%addr, i16 %a) {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
 ; SKX-NEXT:    kmovd %esi, %k1
-; SKX-NEXT:    kshiftrw $10, %k1, %k2
-; SKX-NEXT:    kxorw %k0, %k2, %k0
+; SKX-NEXT:    kshiftrw $11, %k1, %k2
+; SKX-NEXT:    kshiftlw $11, %k2, %k2
+; SKX-NEXT:    kshiftlw $6, %k1, %k1
+; SKX-NEXT:    kshiftrw $6, %k1, %k1
 ; SKX-NEXT:    kshiftlw $15, %k0, %k0
 ; SKX-NEXT:    kshiftrw $5, %k0, %k0
-; SKX-NEXT:    kxorw %k0, %k1, %k0
+; SKX-NEXT:    korw %k0, %k2, %k0
+; SKX-NEXT:    korw %k0, %k1, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $ax killed $ax killed $eax
 ; SKX-NEXT:    retq
@@ -336,12 +342,15 @@ define i8 @test17(i1 *%addr, i8 %a) {
 ; KNL:       ## %bb.0:
 ; KNL-NEXT:    movb (%rdi), %al
 ; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %eax, %k1
-; KNL-NEXT:    kshiftrw $4, %k0, %k2
-; KNL-NEXT:    kxorw %k1, %k2, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $11, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k0, %k1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $al killed $al killed $eax
 ; KNL-NEXT:    retq
@@ -350,11 +359,14 @@ define i8 @test17(i1 *%addr, i8 %a) {
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovb (%rdi), %k0
 ; SKX-NEXT:    kmovd %esi, %k1
-; SKX-NEXT:    kshiftrb $4, %k1, %k2
-; SKX-NEXT:    kxorb %k0, %k2, %k0
+; SKX-NEXT:    kshiftrb $5, %k1, %k2
+; SKX-NEXT:    kshiftlb $5, %k2, %k2
+; SKX-NEXT:    kshiftlb $4, %k1, %k1
+; SKX-NEXT:    kshiftrb $4, %k1, %k1
 ; SKX-NEXT:    kshiftlb $7, %k0, %k0
 ; SKX-NEXT:    kshiftrb $3, %k0, %k0
-; SKX-NEXT:    kxorb %k0, %k1, %k0
+; SKX-NEXT:    korb %k0, %k2, %k0
+; SKX-NEXT:    korb %k0, %k1, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $al killed $al killed $eax
 ; SKX-NEXT:    retq
@@ -790,12 +802,15 @@ define i32 @test_insertelement_v32i1(i32 %a, i32 %b, <32 x i32> %x , <32 x i32>
 ; KNL-NEXT:    kmovw %k0, %ecx
 ; KNL-NEXT:    shll $16, %ecx
 ; KNL-NEXT:    vpcmpltud %zmm2, %zmm0, %k0
-; KNL-NEXT:    kshiftrw $4, %k0, %k1
+; KNL-NEXT:    kshiftrw $5, %k0, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
 ; KNL-NEXT:    kmovw %eax, %k2
-; KNL-NEXT:    kxorw %k2, %k1, %k1
-; KNL-NEXT:    kshiftlw $15, %k1, %k1
-; KNL-NEXT:    kshiftrw $11, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    orl %ecx, %eax
 ; KNL-NEXT:    vzeroupper
@@ -808,12 +823,15 @@ define i32 @test_insertelement_v32i1(i32 %a, i32 %b, <32 x i32> %x , <32 x i32>
 ; SKX-NEXT:    vpcmpltud %zmm2, %zmm0, %k0
 ; SKX-NEXT:    vpcmpltud %zmm3, %zmm1, %k1
 ; SKX-NEXT:    kunpckwd %k0, %k1, %k0
-; SKX-NEXT:    kshiftrd $4, %k0, %k1
+; SKX-NEXT:    kshiftrd $5, %k0, %k1
+; SKX-NEXT:    kshiftld $5, %k1, %k1
+; SKX-NEXT:    kshiftld $28, %k0, %k0
+; SKX-NEXT:    kshiftrd $28, %k0, %k0
 ; SKX-NEXT:    kmovd %eax, %k2
-; SKX-NEXT:    kxord %k2, %k1, %k1
-; SKX-NEXT:    kshiftld $31, %k1, %k1
-; SKX-NEXT:    kshiftrd $27, %k1, %k1
-; SKX-NEXT:    kxord %k1, %k0, %k0
+; SKX-NEXT:    kshiftld $31, %k2, %k2
+; SKX-NEXT:    kshiftrd $27, %k2, %k2
+; SKX-NEXT:    kord %k2, %k1, %k1
+; SKX-NEXT:    kord %k1, %k0, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
@@ -832,12 +850,15 @@ define i8 @test_iinsertelement_v4i1(i32 %a, i32 %b, <4 x i32> %x , <4 x i32> %y)
 ; KNL-NEXT:    cmpl %esi, %edi
 ; KNL-NEXT:    setb %al
 ; KNL-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
-; KNL-NEXT:    kshiftrw $2, %k0, %k1
-; KNL-NEXT:    kmovw %eax, %k2
-; KNL-NEXT:    kxorw %k2, %k1, %k1
+; KNL-NEXT:    kshiftrw $3, %k0, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %eax, %k1
 ; KNL-NEXT:    kshiftlw $15, %k1, %k1
 ; KNL-NEXT:    kshiftrw $13, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    korw %k0, %k1, %k0
 ; KNL-NEXT:    kmovw %k0, %eax
 ; KNL-NEXT:    ## kill: def $al killed $al killed $eax
 ; KNL-NEXT:    vzeroupper
@@ -848,12 +869,15 @@ define i8 @test_iinsertelement_v4i1(i32 %a, i32 %b, <4 x i32> %x , <4 x i32> %y)
 ; SKX-NEXT:    cmpl %esi, %edi
 ; SKX-NEXT:    setb %al
 ; SKX-NEXT:    vpcmpltud %xmm1, %xmm0, %k0
-; SKX-NEXT:    kshiftrb $2, %k0, %k1
-; SKX-NEXT:    kmovd %eax, %k2
-; SKX-NEXT:    kxorb %k2, %k1, %k1
+; SKX-NEXT:    kshiftrb $3, %k0, %k1
+; SKX-NEXT:    kshiftlb $3, %k1, %k1
+; SKX-NEXT:    kshiftlb $6, %k0, %k0
+; SKX-NEXT:    kshiftrb $6, %k0, %k0
+; SKX-NEXT:    korw %k1, %k0, %k0
+; SKX-NEXT:    kmovd %eax, %k1
 ; SKX-NEXT:    kshiftlb $7, %k1, %k1
 ; SKX-NEXT:    kshiftrb $5, %k1, %k1
-; SKX-NEXT:    kxorw %k1, %k0, %k0
+; SKX-NEXT:    korw %k0, %k1, %k0
 ; SKX-NEXT:    kmovd %k0, %eax
 ; SKX-NEXT:    ## kill: def $al killed $al killed $eax
 ; SKX-NEXT:    retq
@@ -2295,7 +2319,7 @@ define void @test_concat_v2i1(<2 x half>* %arg, <2 x half>* %arg1, <2 x half>* %
 ; KNL-NEXT:    movswl 2(%rsi), %eax
 ; KNL-NEXT:    vmovd %eax, %xmm1
 ; KNL-NEXT:    vcvtph2ps %xmm1, %xmm1
-; KNL-NEXT:    vmovss %xmm1, %xmm0, %xmm1 {%k2} {z}
+; KNL-NEXT:    vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
 ; KNL-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
 ; KNL-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; KNL-NEXT:    vmovd %xmm0, %eax
@@ -2343,7 +2367,7 @@ define void @test_concat_v2i1(<2 x half>* %arg, <2 x half>* %arg1, <2 x half>* %
 ; SKX-NEXT:    movswl 2(%rsi), %eax
 ; SKX-NEXT:    vmovd %eax, %xmm1
 ; SKX-NEXT:    vcvtph2ps %xmm1, %xmm1
-; SKX-NEXT:    vmovss %xmm1, %xmm0, %xmm1 {%k2} {z}
+; SKX-NEXT:    vmovss %xmm1, %xmm1, %xmm1 {%k2} {z}
 ; SKX-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
 ; SKX-NEXT:    vcvtps2ph $4, %xmm0, %xmm0
 ; SKX-NEXT:    vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 20327658a7b83..a526518c3fe69 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -29,11 +29,10 @@ define i16 @unpckbw_test(i16 %a0, i16 %a1) {
 define <16 x i32>@test_int_x86_avx512_mask_pbroadcastd_gpr_512(i32 %x0, <16 x i32> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcastd_gpr_512:
 ; X86:       ## %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x7c,0xc8]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x4c,0x24,0x01]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %zmm2 {%k1} {z} ## encoding: [0x62,0xf2,0x7d,0xc9,0x7c,0xd0]
+; X86-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf1,0x7d,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %zmm1, %zmm2 {%k1} {z} ## encoding: [0x62,0xf1,0x7d,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %zmm2, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0x7d,0x48,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %zmm0, %zmm1, %zmm0 ## encoding: [0x62,0xf1,0x75,0x48,0xfe,0xc0]
 ; X86-NEXT:    retl ## encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics.ll b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
index b2d6ce4dfc9de..e94f16a562994 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics.ll
@@ -4798,6 +4798,22 @@ define <4 x float>@test_int_x86_avx512_mask_getmant_ss(<4 x float> %x0, <4 x flo
   ret <4 x float> %res13
 }
 
+define <4 x float> @test_int_x86_avx512_mask_getmant_ss_load(<4 x float> %x0, <4 x float>* %x1p) {
+; X64-LABEL: test_int_x86_avx512_mask_getmant_ss_load:
+; X64:       # %bb.0:
+; X64-NEXT:    vgetmantss $11, (%rdi), %xmm0, %xmm0
+; X64-NEXT:    retq
+;
+; X86-LABEL: test_int_x86_avx512_mask_getmant_ss_load:
+; X86:       # %bb.0:
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    vgetmantss $11, (%eax), %xmm0, %xmm0
+; X86-NEXT:    retl
+  %x1 = load <4 x float>, <4 x float>* %x1p
+  %res = call <4 x float> @llvm.x86.avx512.mask.getmant.ss(<4 x float> %x0, <4 x float> %x1, i32 11, <4 x float> undef, i8 -1, i32 4)
+  ret <4 x float> %res
+}
+
 declare <8 x double> @llvm.x86.avx512.vpermilvar.pd.512(<8 x double>, <8 x i64>)
 
 define <8 x double>@test_int_x86_avx512_vpermilvar_pd_512(<8 x double> %x0, <8 x i64> %x1) {
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index f9d754aa8c62c..19f9a2a2bd544 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -1069,12 +1069,16 @@ define <64 x i8> @test16(i64 %x) {
 ; KNL-NEXT:    kmovw %ecx, %k1
 ; KNL-NEXT:    kmovw %eax, %k2
 ; KNL-NEXT:    kmovw %edi, %k3
-; KNL-NEXT:    kshiftrw $5, %k0, %k4
-; KNL-NEXT:    kxnorw %k0, %k0, %k5
-; KNL-NEXT:    kxorw %k5, %k4, %k4
-; KNL-NEXT:    kshiftlw $15, %k4, %k4
-; KNL-NEXT:    kshiftrw $10, %k4, %k4
-; KNL-NEXT:    kxorw %k4, %k0, %k4
+; KNL-NEXT:    kshiftrw $6, %k0, %k4
+; KNL-NEXT:    kshiftlw $6, %k4, %k4
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    movb $1, %al
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $10, %k5, %k5
+; KNL-NEXT:    korw %k5, %k4, %k4
+; KNL-NEXT:    korw %k4, %k0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
@@ -1091,24 +1095,32 @@ define <64 x i8> @test16(i64 %x) {
 ; SKX-LABEL: test16:
 ; SKX:       ## %bb.0:
 ; SKX-NEXT:    kmovq %rdi, %k0
-; SKX-NEXT:    kxnorw %k0, %k0, %k1
-; SKX-NEXT:    kshiftrq $5, %k0, %k2
-; SKX-NEXT:    kxorq %k1, %k2, %k1
-; SKX-NEXT:    kshiftlq $63, %k1, %k1
-; SKX-NEXT:    kshiftrq $58, %k1, %k1
-; SKX-NEXT:    kxorq %k1, %k0, %k0
+; SKX-NEXT:    kshiftrq $6, %k0, %k1
+; SKX-NEXT:    kshiftlq $6, %k1, %k1
+; SKX-NEXT:    kshiftlq $59, %k0, %k0
+; SKX-NEXT:    kshiftrq $59, %k0, %k0
+; SKX-NEXT:    movb $1, %al
+; SKX-NEXT:    kmovd %eax, %k2
+; SKX-NEXT:    kshiftlq $63, %k2, %k2
+; SKX-NEXT:    kshiftrq $58, %k2, %k2
+; SKX-NEXT:    korq %k2, %k1, %k1
+; SKX-NEXT:    korq %k1, %k0, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %zmm0
 ; SKX-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test16:
 ; AVX512BW:       ## %bb.0:
 ; AVX512BW-NEXT:    kmovq %rdi, %k0
-; AVX512BW-NEXT:    kxnorw %k0, %k0, %k1
-; AVX512BW-NEXT:    kshiftrq $5, %k0, %k2
-; AVX512BW-NEXT:    kxorq %k1, %k2, %k1
-; AVX512BW-NEXT:    kshiftlq $63, %k1, %k1
-; AVX512BW-NEXT:    kshiftrq $58, %k1, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $6, %k0, %k1
+; AVX512BW-NEXT:    kshiftlq $6, %k1, %k1
+; AVX512BW-NEXT:    kshiftlq $59, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $59, %k0, %k0
+; AVX512BW-NEXT:    movb $1, %al
+; AVX512BW-NEXT:    kmovd %eax, %k2
+; AVX512BW-NEXT:    kshiftlq $63, %k2, %k2
+; AVX512BW-NEXT:    kshiftrq $58, %k2, %k2
+; AVX512BW-NEXT:    korq %k2, %k1, %k1
+; AVX512BW-NEXT:    korq %k1, %k0, %k0
 ; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1116,27 +1128,31 @@ define <64 x i8> @test16(i64 %x) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    movl %edi, %ecx
-; AVX512DQ-NEXT:    kmovw %edi, %k0
+; AVX512DQ-NEXT:    kmovw %edi, %k1
 ; AVX512DQ-NEXT:    shrq $32, %rdi
 ; AVX512DQ-NEXT:    shrq $48, %rax
 ; AVX512DQ-NEXT:    shrl $16, %ecx
-; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
 ; AVX512DQ-NEXT:    kmovw %eax, %k2
 ; AVX512DQ-NEXT:    kmovw %edi, %k3
-; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k4
-; AVX512DQ-NEXT:    kxnorw %k0, %k0, %k5
-; AVX512DQ-NEXT:    kxorw %k5, %k4, %k4
-; AVX512DQ-NEXT:    kshiftlw $15, %k4, %k4
-; AVX512DQ-NEXT:    kshiftrw $10, %k4, %k4
-; AVX512DQ-NEXT:    kxorw %k4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $6, %k1, %k4
+; AVX512DQ-NEXT:    kshiftlw $6, %k4, %k4
+; AVX512DQ-NEXT:    kshiftlw $11, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $11, %k1, %k1
+; AVX512DQ-NEXT:    movb $1, %al
+; AVX512DQ-NEXT:    kmovw %eax, %k5
+; AVX512DQ-NEXT:    kshiftlw $15, %k5, %k5
+; AVX512DQ-NEXT:    kshiftrw $10, %k5, %k5
+; AVX512DQ-NEXT:    korw %k5, %k4, %k4
+; AVX512DQ-NEXT:    korw %k4, %k1, %k1
 ; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm1
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm2
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
@@ -1145,12 +1161,16 @@ define <64 x i8> @test16(i64 %x) {
 ; X86-LABEL: test16:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k0
-; X86-NEXT:    kshiftrq $5, %k0, %k1
-; X86-NEXT:    kxnorw %k0, %k0, %k2
-; X86-NEXT:    kxorq %k2, %k1, %k1
-; X86-NEXT:    kshiftlq $63, %k1, %k1
-; X86-NEXT:    kshiftrq $58, %k1, %k1
-; X86-NEXT:    kxorq %k1, %k0, %k0
+; X86-NEXT:    kshiftrq $6, %k0, %k1
+; X86-NEXT:    kshiftlq $6, %k1, %k1
+; X86-NEXT:    kshiftlq $59, %k0, %k0
+; X86-NEXT:    kshiftrq $59, %k0, %k0
+; X86-NEXT:    movb $1, %al
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kshiftlq $63, %k2, %k2
+; X86-NEXT:    kshiftrq $58, %k2, %k2
+; X86-NEXT:    korq %k2, %k1, %k1
+; X86-NEXT:    korq %k1, %k0, %k0
 ; X86-NEXT:    vpmovm2b %k0, %zmm0
 ; X86-NEXT:    retl
   %a = bitcast i64 %x to <64 x i1>
@@ -1174,12 +1194,15 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; KNL-NEXT:    kmovw %edi, %k3
 ; KNL-NEXT:    cmpl %edx, %esi
 ; KNL-NEXT:    setg %al
-; KNL-NEXT:    kshiftrw $5, %k0, %k4
+; KNL-NEXT:    kshiftrw $6, %k0, %k4
+; KNL-NEXT:    kshiftlw $6, %k4, %k4
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
 ; KNL-NEXT:    kmovw %eax, %k5
-; KNL-NEXT:    kxorw %k5, %k4, %k4
-; KNL-NEXT:    kshiftlw $15, %k4, %k4
-; KNL-NEXT:    kshiftrw $10, %k4, %k4
-; KNL-NEXT:    kxorw %k4, %k0, %k4
+; KNL-NEXT:    kshiftlw $15, %k5, %k5
+; KNL-NEXT:    kshiftrw $10, %k5, %k5
+; KNL-NEXT:    korw %k5, %k4, %k4
+; KNL-NEXT:    korw %k4, %k0, %k4
 ; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k3} {z}
 ; KNL-NEXT:    vpmovdb %zmm0, %xmm0
 ; KNL-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k2} {z}
@@ -1198,12 +1221,15 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; SKX-NEXT:    kmovq %rdi, %k0
 ; SKX-NEXT:    cmpl %edx, %esi
 ; SKX-NEXT:    setg %al
-; SKX-NEXT:    kmovd %eax, %k1
-; SKX-NEXT:    kshiftrq $5, %k0, %k2
-; SKX-NEXT:    kxorq %k1, %k2, %k1
-; SKX-NEXT:    kshiftlq $63, %k1, %k1
-; SKX-NEXT:    kshiftrq $58, %k1, %k1
-; SKX-NEXT:    kxorq %k1, %k0, %k0
+; SKX-NEXT:    kshiftrq $6, %k0, %k1
+; SKX-NEXT:    kshiftlq $6, %k1, %k1
+; SKX-NEXT:    kshiftlq $59, %k0, %k0
+; SKX-NEXT:    kshiftrq $59, %k0, %k0
+; SKX-NEXT:    kmovd %eax, %k2
+; SKX-NEXT:    kshiftlq $63, %k2, %k2
+; SKX-NEXT:    kshiftrq $58, %k2, %k2
+; SKX-NEXT:    korq %k2, %k1, %k1
+; SKX-NEXT:    korq %k1, %k0, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %zmm0
 ; SKX-NEXT:    retq
 ;
@@ -1212,12 +1238,15 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; AVX512BW-NEXT:    kmovq %rdi, %k0
 ; AVX512BW-NEXT:    cmpl %edx, %esi
 ; AVX512BW-NEXT:    setg %al
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    kshiftrq $5, %k0, %k2
-; AVX512BW-NEXT:    kxorq %k1, %k2, %k1
-; AVX512BW-NEXT:    kshiftlq $63, %k1, %k1
-; AVX512BW-NEXT:    kshiftrq $58, %k1, %k1
-; AVX512BW-NEXT:    kxorq %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $6, %k0, %k1
+; AVX512BW-NEXT:    kshiftlq $6, %k1, %k1
+; AVX512BW-NEXT:    kshiftlq $59, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $59, %k0, %k0
+; AVX512BW-NEXT:    kmovd %eax, %k2
+; AVX512BW-NEXT:    kshiftlq $63, %k2, %k2
+; AVX512BW-NEXT:    kshiftrq $58, %k2, %k2
+; AVX512BW-NEXT:    korq %k2, %k1, %k1
+; AVX512BW-NEXT:    korq %k1, %k0, %k0
 ; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
@@ -1225,29 +1254,32 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; AVX512DQ:       ## %bb.0:
 ; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    movl %edi, %ecx
-; AVX512DQ-NEXT:    kmovw %edi, %k0
+; AVX512DQ-NEXT:    kmovw %edi, %k1
 ; AVX512DQ-NEXT:    shrq $32, %rdi
 ; AVX512DQ-NEXT:    shrq $48, %rax
 ; AVX512DQ-NEXT:    shrl $16, %ecx
-; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
 ; AVX512DQ-NEXT:    kmovw %eax, %k2
 ; AVX512DQ-NEXT:    kmovw %edi, %k3
 ; AVX512DQ-NEXT:    cmpl %edx, %esi
 ; AVX512DQ-NEXT:    setg %al
-; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k4
+; AVX512DQ-NEXT:    kshiftrw $6, %k1, %k4
+; AVX512DQ-NEXT:    kshiftlw $6, %k4, %k4
+; AVX512DQ-NEXT:    kshiftlw $11, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $11, %k1, %k1
 ; AVX512DQ-NEXT:    kmovw %eax, %k5
-; AVX512DQ-NEXT:    kxorw %k5, %k4, %k4
-; AVX512DQ-NEXT:    kshiftlw $15, %k4, %k4
-; AVX512DQ-NEXT:    kshiftrw $10, %k4, %k4
-; AVX512DQ-NEXT:    kxorw %k4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $15, %k5, %k5
+; AVX512DQ-NEXT:    kshiftrw $10, %k5, %k5
+; AVX512DQ-NEXT:    korw %k5, %k4, %k4
+; AVX512DQ-NEXT:    korw %k4, %k1, %k1
 ; AVX512DQ-NEXT:    vpmovm2d %k3, %zmm0
 ; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
 ; AVX512DQ-NEXT:    vpmovm2d %k2, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm1
+; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm1
 ; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovm2d %k1, %zmm2
+; AVX512DQ-NEXT:    vpmovm2d %k0, %zmm2
 ; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
 ; AVX512DQ-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
 ; AVX512DQ-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
@@ -1259,12 +1291,15 @@ define <64 x i8> @test17(i64 %x, i32 %y, i32 %z) {
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k0
 ; X86-NEXT:    cmpl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    setg %al
-; X86-NEXT:    kmovd %eax, %k1
-; X86-NEXT:    kshiftrq $5, %k0, %k2
-; X86-NEXT:    kxorq %k1, %k2, %k1
-; X86-NEXT:    kshiftlq $63, %k1, %k1
-; X86-NEXT:    kshiftrq $58, %k1, %k1
-; X86-NEXT:    kxorq %k1, %k0, %k0
+; X86-NEXT:    kshiftrq $6, %k0, %k1
+; X86-NEXT:    kshiftlq $6, %k1, %k1
+; X86-NEXT:    kshiftlq $59, %k0, %k0
+; X86-NEXT:    kshiftrq $59, %k0, %k0
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kshiftlq $63, %k2, %k2
+; X86-NEXT:    kshiftrq $58, %k2, %k2
+; X86-NEXT:    korq %k2, %k1, %k1
+; X86-NEXT:    korq %k1, %k0, %k0
 ; X86-NEXT:    vpmovm2b %k0, %zmm0
 ; X86-NEXT:    retl
   %a = bitcast i64 %x to <64 x i1>
@@ -1281,11 +1316,12 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; KNL-NEXT:    kmovw %esi, %k1
 ; KNL-NEXT:    kshiftrw $8, %k1, %k2
 ; KNL-NEXT:    kshiftrw $9, %k1, %k1
-; KNL-NEXT:    kshiftrw $6, %k0, %k3
-; KNL-NEXT:    kxorw %k1, %k3, %k1
-; KNL-NEXT:    kshiftlw $15, %k1, %k1
-; KNL-NEXT:    kshiftrw $9, %k1, %k1
-; KNL-NEXT:    kxorw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k3
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
 ; KNL-NEXT:    kshiftlw $9, %k0, %k0
 ; KNL-NEXT:    kshiftrw $9, %k0, %k0
 ; KNL-NEXT:    kshiftlw $7, %k2, %k1
@@ -1302,11 +1338,12 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; SKX-NEXT:    kmovd %esi, %k1
 ; SKX-NEXT:    kshiftrw $8, %k1, %k2
 ; SKX-NEXT:    kshiftrw $9, %k1, %k1
-; SKX-NEXT:    kshiftrb $6, %k0, %k3
-; SKX-NEXT:    kxorb %k1, %k3, %k1
-; SKX-NEXT:    kshiftlb $7, %k1, %k1
-; SKX-NEXT:    kshiftrb $1, %k1, %k1
-; SKX-NEXT:    kxorb %k1, %k0, %k0
+; SKX-NEXT:    kshiftlb $2, %k0, %k0
+; SKX-NEXT:    kshiftrb $2, %k0, %k0
+; SKX-NEXT:    kshiftlb $7, %k0, %k3
+; SKX-NEXT:    kshiftlb $6, %k1, %k1
+; SKX-NEXT:    korb %k1, %k3, %k1
+; SKX-NEXT:    korb %k1, %k0, %k0
 ; SKX-NEXT:    kshiftlb $1, %k0, %k0
 ; SKX-NEXT:    kshiftrb $1, %k0, %k0
 ; SKX-NEXT:    kshiftlb $7, %k2, %k1
@@ -1320,11 +1357,12 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; AVX512BW-NEXT:    kmovd %esi, %k1
 ; AVX512BW-NEXT:    kshiftrw $8, %k1, %k2
 ; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512BW-NEXT:    kshiftrw $6, %k0, %k3
-; AVX512BW-NEXT:    kxorw %k1, %k3, %k1
-; AVX512BW-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512BW-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512BW-NEXT:    kxorw %k1, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512BW-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512BW-NEXT:    kshiftlw $7, %k0, %k3
+; AVX512BW-NEXT:    kshiftlw $6, %k1, %k1
+; AVX512BW-NEXT:    korw %k1, %k3, %k1
+; AVX512BW-NEXT:    korw %k1, %k0, %k0
 ; AVX512BW-NEXT:    kshiftlw $9, %k0, %k0
 ; AVX512BW-NEXT:    kshiftrw $9, %k0, %k0
 ; AVX512BW-NEXT:    kshiftlw $7, %k2, %k1
@@ -1340,11 +1378,12 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; AVX512DQ-NEXT:    kmovw %esi, %k1
 ; AVX512DQ-NEXT:    kshiftrw $8, %k1, %k2
 ; AVX512DQ-NEXT:    kshiftrw $9, %k1, %k1
-; AVX512DQ-NEXT:    kshiftrb $6, %k0, %k3
-; AVX512DQ-NEXT:    kxorb %k1, %k3, %k1
-; AVX512DQ-NEXT:    kshiftlb $7, %k1, %k1
-; AVX512DQ-NEXT:    kshiftrb $1, %k1, %k1
-; AVX512DQ-NEXT:    kxorb %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrb $2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlb $7, %k0, %k3
+; AVX512DQ-NEXT:    kshiftlb $6, %k1, %k1
+; AVX512DQ-NEXT:    korb %k1, %k3, %k1
+; AVX512DQ-NEXT:    korb %k1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftlb $1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftrb $1, %k0, %k0
 ; AVX512DQ-NEXT:    kshiftlb $7, %k2, %k1
@@ -1361,11 +1400,12 @@ define <8 x i1> @test18(i8 %a, i16 %y) {
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1
 ; X86-NEXT:    kshiftrw $8, %k1, %k2
 ; X86-NEXT:    kshiftrw $9, %k1, %k1
-; X86-NEXT:    kshiftrb $6, %k0, %k3
-; X86-NEXT:    kxorb %k1, %k3, %k1
-; X86-NEXT:    kshiftlb $7, %k1, %k1
-; X86-NEXT:    kshiftrb $1, %k1, %k1
-; X86-NEXT:    kxorb %k1, %k0, %k0
+; X86-NEXT:    kshiftlb $7, %k0, %k3
+; X86-NEXT:    kshiftlb $2, %k0, %k0
+; X86-NEXT:    kshiftrb $2, %k0, %k0
+; X86-NEXT:    kshiftlb $6, %k1, %k1
+; X86-NEXT:    korb %k1, %k3, %k1
+; X86-NEXT:    korb %k1, %k0, %k0
 ; X86-NEXT:    kshiftlb $1, %k0, %k0
 ; X86-NEXT:    kshiftrb $1, %k0, %k0
 ; X86-NEXT:    kshiftlb $7, %k2, %k1
@@ -1968,8 +2008,7 @@ define <64 x i8> @test_build_vec_v64i1(<64 x i8> %x) {
 define void @ktest_1(<8 x double> %in, double * %base) {
 ; KNL-LABEL: ktest_1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovupd (%rdi), %zmm1
-; KNL-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; KNL-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; KNL-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; KNL-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; KNL-NEXT:    kmovw %k0, %eax
@@ -1986,8 +2025,7 @@ define void @ktest_1(<8 x double> %in, double * %base) {
 ;
 ; SKX-LABEL: ktest_1:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovupd (%rdi), %zmm1
-; SKX-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; SKX-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; SKX-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; SKX-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; SKX-NEXT:    kortestb %k0, %k0
@@ -2003,8 +2041,7 @@ define void @ktest_1(<8 x double> %in, double * %base) {
 ;
 ; AVX512BW-LABEL: ktest_1:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    vmovupd (%rdi), %zmm1
-; AVX512BW-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; AVX512BW-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; AVX512BW-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; AVX512BW-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; AVX512BW-NEXT:    kmovd %k0, %eax
@@ -2021,8 +2058,7 @@ define void @ktest_1(<8 x double> %in, double * %base) {
 ;
 ; AVX512DQ-LABEL: ktest_1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vmovupd (%rdi), %zmm1
-; AVX512DQ-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; AVX512DQ-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1
 ; AVX512DQ-NEXT:    vmovupd 8(%rdi), %zmm1 {%k1} {z}
 ; AVX512DQ-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; AVX512DQ-NEXT:    kortestb %k0, %k0
@@ -2039,8 +2075,7 @@ define void @ktest_1(<8 x double> %in, double * %base) {
 ; X86-LABEL: ktest_1:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovupd (%eax), %zmm1
-; X86-NEXT:    vcmpltpd %zmm0, %zmm1, %k1
+; X86-NEXT:    vcmpgtpd (%eax), %zmm0, %k1
 ; X86-NEXT:    vmovupd 8(%eax), %zmm1 {%k1} {z}
 ; X86-NEXT:    vcmpltpd %zmm1, %zmm0, %k0 {%k1}
 ; X86-NEXT:    kortestb %k0, %k0
@@ -2084,10 +2119,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; KNL-LABEL: ktest_2:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vmovups (%rdi), %zmm2
-; KNL-NEXT:    vmovups 64(%rdi), %zmm3
-; KNL-NEXT:    vcmpltps %zmm1, %zmm3, %k1
-; KNL-NEXT:    vcmpltps %zmm0, %zmm2, %k2
+; KNL-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k1
+; KNL-NEXT:    vcmpgtps (%rdi), %zmm0, %k2
 ; KNL-NEXT:    vmovups 4(%rdi), %zmm2 {%k2} {z}
 ; KNL-NEXT:    vmovups 68(%rdi), %zmm3 {%k1} {z}
 ; KNL-NEXT:    vcmpltps %zmm3, %zmm1, %k0
@@ -2112,10 +2145,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; SKX-LABEL: ktest_2:
 ; SKX:       ## %bb.0:
-; SKX-NEXT:    vmovups (%rdi), %zmm2
-; SKX-NEXT:    vmovups 64(%rdi), %zmm3
-; SKX-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; SKX-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; SKX-NEXT:    vcmpgtps (%rdi), %zmm0, %k1
+; SKX-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k2
 ; SKX-NEXT:    kunpckwd %k1, %k2, %k0
 ; SKX-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z}
 ; SKX-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z}
@@ -2137,10 +2168,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; AVX512BW-LABEL: ktest_2:
 ; AVX512BW:       ## %bb.0:
-; AVX512BW-NEXT:    vmovups (%rdi), %zmm2
-; AVX512BW-NEXT:    vmovups 64(%rdi), %zmm3
-; AVX512BW-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; AVX512BW-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; AVX512BW-NEXT:    vcmpgtps (%rdi), %zmm0, %k1
+; AVX512BW-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k2
 ; AVX512BW-NEXT:    kunpckwd %k1, %k2, %k0
 ; AVX512BW-NEXT:    vmovups 68(%rdi), %zmm2 {%k2} {z}
 ; AVX512BW-NEXT:    vmovups 4(%rdi), %zmm3 {%k1} {z}
@@ -2162,10 +2191,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ;
 ; AVX512DQ-LABEL: ktest_2:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vmovups (%rdi), %zmm2
-; AVX512DQ-NEXT:    vmovups 64(%rdi), %zmm3
-; AVX512DQ-NEXT:    vcmpltps %zmm1, %zmm3, %k1
-; AVX512DQ-NEXT:    vcmpltps %zmm0, %zmm2, %k2
+; AVX512DQ-NEXT:    vcmpgtps 64(%rdi), %zmm1, %k1
+; AVX512DQ-NEXT:    vcmpgtps (%rdi), %zmm0, %k2
 ; AVX512DQ-NEXT:    vmovups 4(%rdi), %zmm2 {%k2} {z}
 ; AVX512DQ-NEXT:    vmovups 68(%rdi), %zmm3 {%k1} {z}
 ; AVX512DQ-NEXT:    vcmpltps %zmm3, %zmm1, %k0
@@ -2191,10 +2218,8 @@ define void @ktest_2(<32 x float> %in, float * %base) {
 ; X86-LABEL: ktest_2:
 ; X86:       ## %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    vmovups (%eax), %zmm2
-; X86-NEXT:    vmovups 64(%eax), %zmm3
-; X86-NEXT:    vcmpltps %zmm0, %zmm2, %k1
-; X86-NEXT:    vcmpltps %zmm1, %zmm3, %k2
+; X86-NEXT:    vcmpgtps (%eax), %zmm0, %k1
+; X86-NEXT:    vcmpgtps 64(%eax), %zmm1, %k2
 ; X86-NEXT:    kunpckwd %k1, %k2, %k0
 ; X86-NEXT:    vmovups 68(%eax), %zmm2 {%k2} {z}
 ; X86-NEXT:    vmovups 4(%eax), %zmm3 {%k1} {z}
@@ -2768,23 +2793,489 @@ define void @store_64i1(<64 x i1>* %a, <64 x i1> %v) {
 ;
 ; KNL-LABEL: store_64i1:
 ; KNL:       ## %bb.0:
-; KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL-NEXT:    vpmovsxbd %xmm1, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpmovsxbd %xmm2, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
-; KNL-NEXT:    vpmovsxbd %xmm3, %zmm0
-; KNL-NEXT:    vpslld $31, %zmm0, %zmm0
-; KNL-NEXT:    vptestmd %zmm0, %zmm0, %k3
-; KNL-NEXT:    kmovw %k3, 6(%rdi)
-; KNL-NEXT:    kmovw %k2, 4(%rdi)
-; KNL-NEXT:    kmovw %k1, 2(%rdi)
+; KNL-NEXT:    kmovw %esi, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    kmovw %edx, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k3
+; KNL-NEXT:    kmovw %ecx, %k1
+; KNL-NEXT:    kshiftlw $2, %k1, %k1
+; KNL-NEXT:    korw %k1, %k3, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k4
+; KNL-NEXT:    kmovw %r8d, %k1
+; KNL-NEXT:    kshiftlw $3, %k1, %k1
+; KNL-NEXT:    korw %k1, %k4, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k5
+; KNL-NEXT:    kmovw %r9d, %k1
+; KNL-NEXT:    kshiftlw $4, %k1, %k1
+; KNL-NEXT:    korw %k1, %k5, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k6
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $5, %k1, %k1
+; KNL-NEXT:    korw %k1, %k6, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k7
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $6, %k1, %k1
+; KNL-NEXT:    korw %k1, %k7, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $7, %k1, %k1
+; KNL-NEXT:    kshiftlw $8, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $8, %k1, %k1
+; KNL-NEXT:    kshiftlw $9, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $9, %k1, %k1
+; KNL-NEXT:    kshiftlw $10, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $10, %k1, %k1
+; KNL-NEXT:    kshiftlw $11, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $11, %k1, %k1
+; KNL-NEXT:    kshiftlw $12, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $12, %k1, %k1
+; KNL-NEXT:    kshiftlw $13, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $13, %k1, %k1
+; KNL-NEXT:    kshiftlw $14, %k0, %k2
+; KNL-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k1
+; KNL-NEXT:    kshiftlw $14, %k1, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k1
+; KNL-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k0, %k1, %k0
+; KNL-NEXT:    korw %k0, %k2, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $1, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $2, %k2, %k2
+; KNL-NEXT:    kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k2, %k3, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k2, %k4, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k2, %k5, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    korw %k2, %k6, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $8, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $9, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $10, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $11, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $12, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $13, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k0
+; KNL-NEXT:    kshiftlw $15, %k0, %k0
+; KNL-NEXT:    kshiftrw $15, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $1, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $14, %k0, %k0
+; KNL-NEXT:    kshiftrw $14, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $2, %k2, %k2
+; KNL-NEXT:    korw %k2, %k3, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $13, %k0, %k0
+; KNL-NEXT:    kshiftrw $13, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    korw %k2, %k4, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $12, %k0, %k0
+; KNL-NEXT:    kshiftrw $12, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    korw %k2, %k5, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $11, %k0, %k0
+; KNL-NEXT:    kshiftrw $11, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    korw %k2, %k6, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $10, %k0, %k0
+; KNL-NEXT:    kshiftrw $10, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $9, %k0, %k0
+; KNL-NEXT:    kshiftrw $9, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k1, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $8, %k0, %k0
+; KNL-NEXT:    kshiftrw $8, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $8, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k3, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $7, %k0, %k0
+; KNL-NEXT:    kshiftrw $7, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $9, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k4, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $6, %k0, %k0
+; KNL-NEXT:    kshiftrw $6, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $10, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k5, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $5, %k0, %k0
+; KNL-NEXT:    kshiftrw $5, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $11, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $4, %k0, %k0
+; KNL-NEXT:    kshiftrw $4, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $12, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $3, %k0, %k0
+; KNL-NEXT:    kshiftrw $3, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $13, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $2, %k0, %k0
+; KNL-NEXT:    kshiftrw $2, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k0, %k0
+; KNL-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $1, %k2, %k2
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    korw %k2, %k0, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $15, %k7, %k7
+; KNL-NEXT:    kshiftrw $15, %k7, %k7
+; KNL-NEXT:    korw %k2, %k7, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $2, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    korw %k7, %k0, %k7
+; KNL-NEXT:    kshiftlw $14, %k2, %k2
+; KNL-NEXT:    kshiftrw $14, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $3, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    korw %k7, %k0, %k7
+; KNL-NEXT:    kshiftlw $13, %k2, %k2
+; KNL-NEXT:    kshiftrw $13, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $4, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    korw %k7, %k0, %k7
+; KNL-NEXT:    kshiftlw $12, %k2, %k2
+; KNL-NEXT:    kshiftrw $12, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $5, %k7, %k7
+; KNL-NEXT:    korw %k7, %k6, %k7
+; KNL-NEXT:    kshiftlw $11, %k2, %k2
+; KNL-NEXT:    kshiftrw $11, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $6, %k7, %k7
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    korw %k7, %k0, %k7
+; KNL-NEXT:    kshiftlw $10, %k2, %k2
+; KNL-NEXT:    kshiftrw $10, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $7, %k7, %k7
+; KNL-NEXT:    korw %k7, %k1, %k7
+; KNL-NEXT:    kshiftlw $9, %k2, %k2
+; KNL-NEXT:    kshiftrw $9, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $8, %k7, %k7
+; KNL-NEXT:    korw %k7, %k3, %k7
+; KNL-NEXT:    kshiftlw $8, %k2, %k2
+; KNL-NEXT:    kshiftrw $8, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $9, %k7, %k7
+; KNL-NEXT:    korw %k7, %k4, %k7
+; KNL-NEXT:    kshiftlw $7, %k2, %k2
+; KNL-NEXT:    kshiftrw $7, %k2, %k2
+; KNL-NEXT:    korw %k7, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k7
+; KNL-NEXT:    kshiftlw $10, %k7, %k7
+; KNL-NEXT:    korw %k7, %k5, %k6
+; KNL-NEXT:    kshiftlw $6, %k2, %k2
+; KNL-NEXT:    kshiftrw $6, %k2, %k2
+; KNL-NEXT:    korw %k6, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k6
+; KNL-NEXT:    kshiftlw $11, %k6, %k6
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k6, %k1, %k5
+; KNL-NEXT:    kshiftlw $5, %k2, %k2
+; KNL-NEXT:    kshiftrw $5, %k2, %k2
+; KNL-NEXT:    korw %k5, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k5
+; KNL-NEXT:    kshiftlw $12, %k5, %k5
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k5, %k1, %k4
+; KNL-NEXT:    kshiftlw $4, %k2, %k2
+; KNL-NEXT:    kshiftrw $4, %k2, %k2
+; KNL-NEXT:    korw %k4, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k4
+; KNL-NEXT:    kshiftlw $13, %k4, %k4
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k4, %k1, %k3
+; KNL-NEXT:    kshiftlw $3, %k2, %k2
+; KNL-NEXT:    kshiftrw $3, %k2, %k2
+; KNL-NEXT:    korw %k3, %k2, %k2
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k3
+; KNL-NEXT:    kshiftlw $14, %k3, %k3
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; KNL-NEXT:    korw %k3, %k1, %k1
+; KNL-NEXT:    kshiftlw $2, %k2, %k2
+; KNL-NEXT:    kshiftrw $2, %k2, %k2
+; KNL-NEXT:    korw %k1, %k2, %k1
+; KNL-NEXT:    kshiftlw $1, %k1, %k1
+; KNL-NEXT:    kshiftrw $1, %k1, %k1
+; KNL-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; KNL-NEXT:    kmovw %eax, %k2
+; KNL-NEXT:    kshiftlw $15, %k2, %k2
+; KNL-NEXT:    korw %k2, %k1, %k1
+; KNL-NEXT:    kmovw %k1, 6(%rdi)
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    kmovw %k0, 4(%rdi)
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; KNL-NEXT:    kmovw %k0, 2(%rdi)
+; KNL-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; KNL-NEXT:    kmovw %k0, (%rdi)
-; KNL-NEXT:    vzeroupper
 ; KNL-NEXT:    retq
 ;
 ; SKX-LABEL: store_64i1:
@@ -2805,23 +3296,489 @@ define void @store_64i1(<64 x i1>* %a, <64 x i1> %v) {
 ;
 ; AVX512DQ-LABEL: store_64i1:
 ; AVX512DQ:       ## %bb.0:
-; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
-; AVX512DQ-NEXT:    vpmovsxbd %xmm1, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
-; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
-; AVX512DQ-NEXT:    vpmovsxbd %xmm3, %zmm0
-; AVX512DQ-NEXT:    vpslld $31, %zmm0, %zmm0
-; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
-; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
-; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
-; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw %esi, %k0
+; AVX512DQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $2, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    kmovw %edx, %k1
+; AVX512DQ-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $3, %k0, %k3
+; AVX512DQ-NEXT:    kmovw %ecx, %k1
+; AVX512DQ-NEXT:    kshiftlw $2, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k3, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $4, %k0, %k4
+; AVX512DQ-NEXT:    kmovw %r8d, %k1
+; AVX512DQ-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k4, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $5, %k0, %k5
+; AVX512DQ-NEXT:    kmovw %r9d, %k1
+; AVX512DQ-NEXT:    kshiftlw $4, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k5, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $6, %k0, %k6
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $5, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k6, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $7, %k0, %k7
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $6, %k1, %k1
+; AVX512DQ-NEXT:    korw %k1, %k7, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $7, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $8, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $8, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $9, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $9, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $10, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $10, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $11, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $11, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $12, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $12, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $13, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $13, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $14, %k0, %k2
+; AVX512DQ-NEXT:    kmovw %k2, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $2, %k0, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k1
+; AVX512DQ-NEXT:    kshiftlw $14, %k1, %k0
+; AVX512DQ-NEXT:    kshiftlw $15, %k0, %k1
+; AVX512DQ-NEXT:    kmovw %k1, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k0, %k1, %k0
+; AVX512DQ-NEXT:    korw %k0, %k2, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQ-NEXT:    kmovw %k3, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k2, %k3, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQ-NEXT:    kmovw %k4, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k2, %k4, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQ-NEXT:    kmovw %k5, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k2, %k5, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k6, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQ-NEXT:    kmovw %k7, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k0
+; AVX512DQ-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k3, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k4, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $12, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $12, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k5, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $11, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $11, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k6, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $10, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $10, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $9, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $9, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k1, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $8, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $8, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k3 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k3, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $7, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $7, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k4 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k4, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $6, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $6, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k5 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k5, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $5, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $5, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $4, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $4, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $3, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $3, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $2, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k7 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, {{[-0-9]+}}(%r{{[sb]}}p) ## 2-byte Spill
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k2, %k0, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $15, %k7, %k7
+; AVX512DQ-NEXT:    kshiftrw $15, %k7, %k7
+; AVX512DQ-NEXT:    korw %k2, %k7, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $2, %k7, %k7
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k7, %k0, %k7
+; AVX512DQ-NEXT:    kshiftlw $14, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $3, %k7, %k7
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k7, %k0, %k7
+; AVX512DQ-NEXT:    kshiftlw $13, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $13, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $4, %k7, %k7
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k7, %k0, %k7
+; AVX512DQ-NEXT:    kshiftlw $12, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $12, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $5, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k6, %k7
+; AVX512DQ-NEXT:    kshiftlw $11, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $11, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $6, %k7, %k7
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k7, %k0, %k7
+; AVX512DQ-NEXT:    kshiftlw $10, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $10, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $7, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k1, %k7
+; AVX512DQ-NEXT:    kshiftlw $9, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $9, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $8, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k3, %k7
+; AVX512DQ-NEXT:    kshiftlw $8, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $8, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $9, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k4, %k7
+; AVX512DQ-NEXT:    kshiftlw $7, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $7, %k2, %k2
+; AVX512DQ-NEXT:    korw %k7, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k7
+; AVX512DQ-NEXT:    kshiftlw $10, %k7, %k7
+; AVX512DQ-NEXT:    korw %k7, %k5, %k6
+; AVX512DQ-NEXT:    kshiftlw $6, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $6, %k2, %k2
+; AVX512DQ-NEXT:    korw %k6, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k6
+; AVX512DQ-NEXT:    kshiftlw $11, %k6, %k6
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k6, %k1, %k5
+; AVX512DQ-NEXT:    kshiftlw $5, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $5, %k2, %k2
+; AVX512DQ-NEXT:    korw %k5, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k5
+; AVX512DQ-NEXT:    kshiftlw $12, %k5, %k5
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k5, %k1, %k4
+; AVX512DQ-NEXT:    kshiftlw $4, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $4, %k2, %k2
+; AVX512DQ-NEXT:    korw %k4, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k4
+; AVX512DQ-NEXT:    kshiftlw $13, %k4, %k4
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k4, %k1, %k3
+; AVX512DQ-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $3, %k2, %k2
+; AVX512DQ-NEXT:    korw %k3, %k2, %k2
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k3
+; AVX512DQ-NEXT:    kshiftlw $14, %k3, %k3
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 ## 2-byte Reload
+; AVX512DQ-NEXT:    korw %k3, %k1, %k1
+; AVX512DQ-NEXT:    kshiftlw $2, %k2, %k2
+; AVX512DQ-NEXT:    kshiftrw $2, %k2, %k2
+; AVX512DQ-NEXT:    korw %k1, %k2, %k1
+; AVX512DQ-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512DQ-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512DQ-NEXT:    movb {{[0-9]+}}(%rsp), %al
+; AVX512DQ-NEXT:    kmovw %eax, %k2
+; AVX512DQ-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512DQ-NEXT:    korw %k2, %k1, %k1
+; AVX512DQ-NEXT:    kmovw %k1, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    kmovw %k0, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
+; AVX512DQ-NEXT:    kmovw %k0, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k0 ## 2-byte Reload
 ; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
-; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
 ; X86-LABEL: store_64i1:
@@ -4326,3 +5283,80 @@ bar:
 exit:
   ret void
 }
+
+define <64 x i1> @mask64_insert(i32 %a) {
+; KNL-LABEL: mask64_insert:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    movq %rdi, %rax
+; KNL-NEXT:    movw $-4, %cx
+; KNL-NEXT:    kmovw %ecx, %k0
+; KNL-NEXT:    kshiftrw $1, %k0, %k0
+; KNL-NEXT:    kshiftlw $1, %k0, %k0
+; KNL-NEXT:    andl $1, %esi
+; KNL-NEXT:    kmovw %esi, %k1
+; KNL-NEXT:    korw %k1, %k0, %k0
+; KNL-NEXT:    kmovw %k0, (%rdi)
+; KNL-NEXT:    movw $-3, 6(%rdi)
+; KNL-NEXT:    movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
+; KNL-NEXT:    retq
+;
+; SKX-LABEL: mask64_insert:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    kmovd %edi, %k0
+; SKX-NEXT:    kshiftlq $63, %k0, %k0
+; SKX-NEXT:    kshiftrq $63, %k0, %k0
+; SKX-NEXT:    movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; SKX-NEXT:    kmovq %rax, %k1
+; SKX-NEXT:    kshiftrq $1, %k1, %k1
+; SKX-NEXT:    kshiftlq $1, %k1, %k1
+; SKX-NEXT:    korq %k0, %k1, %k0
+; SKX-NEXT:    vpmovm2b %k0, %zmm0
+; SKX-NEXT:    retq
+;
+; AVX512BW-LABEL: mask64_insert:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    kmovd %edi, %k0
+; AVX512BW-NEXT:    kshiftlq $63, %k0, %k0
+; AVX512BW-NEXT:    kshiftrq $63, %k0, %k0
+; AVX512BW-NEXT:    movabsq $-562958543486980, %rax ## imm = 0xFFFDFFFDFFFDFFFC
+; AVX512BW-NEXT:    kmovq %rax, %k1
+; AVX512BW-NEXT:    kshiftrq $1, %k1, %k1
+; AVX512BW-NEXT:    kshiftlq $1, %k1, %k1
+; AVX512BW-NEXT:    korq %k0, %k1, %k0
+; AVX512BW-NEXT:    vpmovm2b %k0, %zmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512DQ-LABEL: mask64_insert:
+; AVX512DQ:       ## %bb.0:
+; AVX512DQ-NEXT:    movq %rdi, %rax
+; AVX512DQ-NEXT:    movw $-4, %cx
+; AVX512DQ-NEXT:    kmovw %ecx, %k0
+; AVX512DQ-NEXT:    kshiftrw $1, %k0, %k0
+; AVX512DQ-NEXT:    kshiftlw $1, %k0, %k0
+; AVX512DQ-NEXT:    andl $1, %esi
+; AVX512DQ-NEXT:    kmovw %esi, %k1
+; AVX512DQ-NEXT:    korw %k1, %k0, %k0
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
+; AVX512DQ-NEXT:    movw $-3, 6(%rdi)
+; AVX512DQ-NEXT:    movl $-131075, 2(%rdi) ## imm = 0xFFFDFFFD
+; AVX512DQ-NEXT:    retq
+;
+; X86-LABEL: mask64_insert:
+; X86:       ## %bb.0:
+; X86-NEXT:    kmovb {{[0-9]+}}(%esp), %k0
+; X86-NEXT:    movl $-131076, %eax ## imm = 0xFFFDFFFC
+; X86-NEXT:    kmovd %eax, %k1
+; X86-NEXT:    movl $-131075, %eax ## imm = 0xFFFDFFFD
+; X86-NEXT:    kmovd %eax, %k2
+; X86-NEXT:    kunpckdq %k1, %k2, %k1
+; X86-NEXT:    kshiftrq $1, %k1, %k1
+; X86-NEXT:    kshiftlq $1, %k1, %k1
+; X86-NEXT:    kshiftlq $63, %k0, %k0
+; X86-NEXT:    kshiftrq $63, %k0, %k0
+; X86-NEXT:    korq %k0, %k1, %k0
+; X86-NEXT:    vpmovm2b %k0, %zmm0
+; X86-NEXT:    retl
+  %a_i = trunc i32 %a to i1
+  %maskv = insertelement <64 x i1> <i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>, i1 %a_i, i32 0
+  ret <64 x i1> %maskv
+}
diff --git a/llvm/test/CodeGen/X86/avx512-select.ll b/llvm/test/CodeGen/X86/avx512-select.ll
index 31484c16dd5a4..cd4a70c4f73c4 100644
--- a/llvm/test/CodeGen/X86/avx512-select.ll
+++ b/llvm/test/CodeGen/X86/avx512-select.ll
@@ -74,7 +74,7 @@ define float @select02(float %a, float %b, float %c, float %eps) {
 ; X64-LABEL: select02:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vcmpless %xmm0, %xmm3, %k1
-; X64-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; X64-NEXT:    vmovaps %xmm1, %xmm0
 ; X64-NEXT:    retq
   %cmp = fcmp oge float %a, %eps
@@ -96,7 +96,7 @@ define double @select03(double %a, double %b, double %c, double %eps) {
 ; X64-LABEL: select03:
 ; X64:       # %bb.0:
 ; X64-NEXT:    vcmplesd %xmm0, %xmm3, %k1
-; X64-NEXT:    vmovsd %xmm2, %xmm0, %xmm1 {%k1}
+; X64-NEXT:    vmovsd %xmm2, %xmm1, %xmm1 {%k1}
 ; X64-NEXT:    vmovapd %xmm1, %xmm0
 ; X64-NEXT:    retq
   %cmp = fcmp oge double %a, %eps
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 023aef9df3060..0b518ea6a5bc2 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=KNL
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=KNL
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512 --check-prefix=AVX512BW
+; RUN: llc < %s -disable-peephole -mtriple=x86_64-apple-darwin -show-mc-encoding -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=SKX
 
 define <16 x float> @test1(<16 x float> %x, <16 x float> %y) nounwind {
 ; CHECK-LABEL: test1:
@@ -614,6 +614,30 @@ define <2 x double> @test31(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp
   ret <2 x double> %max
 }
 
+define <2 x double> @test31_commute(<2 x double> %x, <2 x double> %x1, <2 x double>* %yp) nounwind {
+; AVX512-LABEL: test31_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vmovupd (%rdi), %xmm2 ## encoding: [0xc5,0xf9,0x10,0x17]
+; AVX512-NEXT:    vcmpltpd %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0xed,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test31_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi), %xmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x08,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <2 x double>, <2 x double>* %yp, align 4
+  %mask = fcmp olt <2 x double> %y, %x
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
 define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
 ; AVX512-LABEL: test32:
 ; AVX512:       ## %bb.0:
@@ -637,6 +661,29 @@ define <4 x double> @test32(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp
   ret <4 x double> %max
 }
 
+define <4 x double> @test32_commute(<4 x double> %x, <4 x double> %x1, <4 x double>* %yp) nounwind {
+; AVX512-LABEL: test32_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vmovupd (%rdi), %ymm2 ## encoding: [0xc5,0xfd,0x10,0x17]
+; AVX512-NEXT:    vcmpltpd %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0xed,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test32_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi), %ymm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x28,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <4 x double>, <4 x double>* %yp, align 4
+  %mask = fcmp ogt <4 x double> %x, %y
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
 define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
 ; CHECK-LABEL: test33:
 ; CHECK:       ## %bb.0:
@@ -649,6 +696,18 @@ define <8 x double> @test33(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp
   ret <8 x double> %max
 }
 
+define <8 x double> @test33_commute(<8 x double> %x, <8 x double> %x1, <8 x double>* %yp) nounwind {
+; CHECK-LABEL: test33_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtpd (%rdi), %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %y = load <8 x double>, <8 x double>* %yp, align 4
+  %mask = fcmp olt <8 x double> %y, %x
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
 ; AVX512-LABEL: test34:
 ; AVX512:       ## %bb.0:
@@ -672,6 +731,29 @@ define <4 x float> @test34(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) no
   ret <4 x float> %max
 }
 
+define <4 x float> @test34_commute(<4 x float> %x, <4 x float> %x1, <4 x float>* %yp) nounwind {
+; AVX512-LABEL: test34_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vmovups (%rdi), %xmm2 ## encoding: [0xc5,0xf8,0x10,0x17]
+; AVX512-NEXT:    vcmpltps %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0x6c,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test34_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi), %xmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x08,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+  %y = load <4 x float>, <4 x float>* %yp, align 4
+  %mask = fcmp olt <4 x float> %y, %x
+  %max = select <4 x i1> %mask, <4 x float> %x, <4 x float> %x1
+  ret <4 x float> %max
+}
+
 define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
 ; AVX512-LABEL: test35:
 ; AVX512:       ## %bb.0:
@@ -695,6 +777,29 @@ define <8 x float> @test35(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) no
   ret <8 x float> %max
 }
 
+define <8 x float> @test35_commute(<8 x float> %x, <8 x float> %x1, <8 x float>* %yp) nounwind {
+; AVX512-LABEL: test35_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vmovups (%rdi), %ymm2 ## encoding: [0xc5,0xfc,0x10,0x17]
+; AVX512-NEXT:    vcmpltps %zmm0, %zmm2, %k1 ## encoding: [0x62,0xf1,0x6c,0x48,0xc2,0xc8,0x01]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test35_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi), %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %y = load <8 x float>, <8 x float>* %yp, align 4
+  %mask = fcmp ogt <8 x float> %x, %y
+  %max = select <8 x i1> %mask, <8 x float> %x, <8 x float> %x1
+  ret <8 x float> %max
+}
+
 define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
 ; CHECK-LABEL: test36:
 ; CHECK:       ## %bb.0:
@@ -707,6 +812,18 @@ define <16 x float> @test36(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp
   ret <16 x float> %max
 }
 
+define <16 x float> @test36_commute(<16 x float> %x, <16 x float> %x1, <16 x float>* %yp) nounwind {
+; CHECK-LABEL: test36_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtps (%rdi), %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+  %y = load <16 x float>, <16 x float>* %yp, align 4
+  %mask = fcmp olt <16 x float> %y, %x
+  %max = select <16 x i1> %mask, <16 x float> %x, <16 x float> %x1
+  ret <16 x float> %max
+}
+
 define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
 ; CHECK-LABEL: test37:
 ; CHECK:       ## %bb.0:
@@ -723,13 +840,28 @@ define <8 x double> @test37(<8 x double> %x, <8 x double> %x1, double* %ptr) nou
   ret <8 x double> %max
 }
 
+define <8 x double> @test37_commute(<8 x double> %x, <8 x double> %x1, double* %ptr) nounwind {
+; CHECK-LABEL: test37_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+
+  %mask = fcmp ogt <8 x double> %x, %shuffle
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
 ; AVX512-LABEL: test38:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT:    vbroadcastsd (%rdi), %ymm2 ## encoding: [0xc4,0xe2,0x7d,0x19,0x17]
-; AVX512-NEXT:    vcmpltpd %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -749,14 +881,37 @@ define <4 x double> @test38(<4 x double> %x, <4 x double> %x1, double* %ptr) nou
   ret <4 x double> %max
 }
 
+define <4 x double> @test38_commute(<4 x double> %x, <4 x double> %x1, double* %ptr) nounwind {
+; AVX512-LABEL: test38_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test38_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi){1to4}, %ymm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x38,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <4 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <4 x double> %v, <4 x double> undef, <4 x i32> zeroinitializer
+
+  %mask = fcmp ogt <4 x double> %x, %shuffle
+  %max = select <4 x i1> %mask, <4 x double> %x, <4 x double> %x1
+  ret <4 x double> %max
+}
+
 define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
 ; AVX512-LABEL: test39:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT:    vmovddup (%rdi), %xmm2 ## encoding: [0xc5,0xfb,0x12,0x17]
-; AVX512-NEXT:    ## xmm2 = mem[0,0]
-; AVX512-NEXT:    vcmpltpd %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
@@ -777,6 +932,32 @@ define <2 x double> @test39(<2 x double> %x, <2 x double> %x1, double* %ptr) nou
   ret <2 x double> %max
 }
 
+define <2 x double> @test39_commute(<2 x double> %x, <2 x double> %x1, double* %ptr) nounwind {
+; AVX512-LABEL: test39_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test39_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtpd (%rdi){1to2}, %xmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x18,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <2 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <2 x double> %v, <2 x double> undef, <2 x i32> <i32 0, i32 0>
+
+  %mask = fcmp ogt <2 x double> %x, %shuffle
+  %max = select <2 x i1> %mask, <2 x double> %x, <2 x double> %x1
+  ret <2 x double> %max
+}
+
 
 define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
 ; CHECK-LABEL: test40:
@@ -794,13 +975,28 @@ define <16  x float> @test40(<16  x float> %x, <16  x float> %x1, float* %ptr) n
   ret <16  x float> %max
 }
 
+define <16  x float> @test40_commute(<16  x float> %x, <16  x float> %x1, float* %ptr) nounwind {
+; CHECK-LABEL: test40_commute:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; CHECK-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; CHECK-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <16  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <16  x float> %v, <16  x float> undef, <16 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <16  x float> %x, %shuffle
+  %max = select <16 x i1> %mask, <16  x float> %x, <16  x float> %x1
+  ret <16  x float> %max
+}
+
 define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
 ; AVX512-LABEL: test41:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512-NEXT:    vbroadcastss (%rdi), %ymm2 ## encoding: [0xc4,0xe2,0x7d,0x18,0x17]
-; AVX512-NEXT:    vcmpltps %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
@@ -820,13 +1016,37 @@ define <8  x float> @test41(<8  x float> %x, <8  x float> %x1, float* %ptr) noun
   ret <8  x float> %max
 }
 
+define <8  x float> @test41_commute(<8  x float> %x, <8  x float> %x1, float* %ptr) nounwind {
+; AVX512-LABEL: test41_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 killed $zmm0
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test41_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi){1to8}, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x38,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %ymm0, %ymm1, %ymm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x29,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <8  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <8  x float> %v, <8  x float> undef, <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <8  x float> %x, %shuffle
+  %max = select <8 x i1> %mask, <8  x float> %x, <8  x float> %x1
+  ret <8  x float> %max
+}
+
 define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
 ; AVX512-LABEL: test42:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512-NEXT:    vbroadcastss (%rdi), %xmm2 ## encoding: [0xc4,0xe2,0x79,0x18,0x17]
-; AVX512-NEXT:    vcmpltps %zmm2, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x48,0xc2,0xca,0x01]
+; AVX512-NEXT:    vcmpltps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x01]
 ; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
 ; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
 ; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
@@ -847,6 +1067,32 @@ define <4  x float> @test42(<4  x float> %x, <4  x float> %x1, float* %ptr) noun
   ret <4  x float> %max
 }
 
+define <4  x float> @test42_commute(<4  x float> %x, <4  x float> %x1, float* %ptr) nounwind {
+; AVX512-LABEL: test42_commute:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $xmm1 killed $xmm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
+; AVX512-NEXT:    vcmpgtps (%rdi){1to16}, %zmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x58,0xc2,0x0f,0x0e]
+; AVX512-NEXT:    vblendmps %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x49,0x65,0xc0]
+; AVX512-NEXT:    ## kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test42_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vcmpgtps (%rdi){1to4}, %xmm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x18,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmps %xmm0, %xmm1, %xmm0 {%k1} ## encoding: [0x62,0xf2,0x75,0x09,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load float, float* %ptr
+  %v = insertelement <4  x float> undef, float %a, i32 0
+  %shuffle = shufflevector <4  x float> %v, <4  x float> undef, <4 x i32> <i32 0, i32 0, i32 0, i32 0>
+
+  %mask = fcmp ogt <4  x float> %x, %shuffle
+  %max = select <4 x i1> %mask, <4  x float> %x, <4  x float> %x1
+  ret <4  x float> %max
+}
+
 define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
 ; KNL-LABEL: test43:
 ; KNL:       ## %bb.0:
@@ -883,6 +1129,42 @@ define <8 x double> @test43(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x
   ret <8 x double> %max
 }
 
+define <8 x double> @test43_commute(<8 x double> %x, <8 x double> %x1, double* %ptr,<8 x i1> %mask_in) nounwind {
+; KNL-LABEL: test43_commute:
+; KNL:       ## %bb.0:
+; KNL-NEXT:    vpmovsxwq %xmm2, %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x24,0xd2]
+; KNL-NEXT:    vpsllq $63, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0x73,0xf2,0x3f]
+; KNL-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 ## encoding: [0x62,0xf1,0xfd,0x58,0xc2,0x0f,0x0e]
+; KNL-NEXT:    vptestmq %zmm2, %zmm2, %k1 {%k1} ## encoding: [0x62,0xf2,0xed,0x49,0x27,0xca]
+; KNL-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; KNL-NEXT:    retq ## encoding: [0xc3]
+;
+; AVX512BW-LABEL: test43_commute:
+; AVX512BW:       ## %bb.0:
+; AVX512BW-NEXT:    vpsllw $15, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0x71,0xf2,0x0f]
+; AVX512BW-NEXT:    vpmovw2m %zmm2, %k1 ## encoding: [0x62,0xf2,0xfe,0x48,0x29,0xca]
+; AVX512BW-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xc2,0x0f,0x0e]
+; AVX512BW-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; AVX512BW-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: test43_commute:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vpsllw $15, %xmm2, %xmm2 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0x71,0xf2,0x0f]
+; SKX-NEXT:    vpmovw2m %xmm2, %k1 ## encoding: [0x62,0xf2,0xfe,0x08,0x29,0xca]
+; SKX-NEXT:    vcmpgtpd (%rdi){1to8}, %zmm0, %k1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xc2,0x0f,0x0e]
+; SKX-NEXT:    vblendmpd %zmm0, %zmm1, %zmm0 {%k1} ## encoding: [0x62,0xf2,0xf5,0x49,0x65,0xc0]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+
+  %a = load double, double* %ptr
+  %v = insertelement <8 x double> undef, double %a, i32 0
+  %shuffle = shufflevector <8 x double> %v, <8 x double> undef, <8 x i32> zeroinitializer
+
+  %mask_cmp = fcmp ogt <8 x double> %x, %shuffle
+  %mask = and <8 x i1> %mask_cmp, %mask_in
+  %max = select <8 x i1> %mask, <8 x double> %x, <8 x double> %x1
+  ret <8 x double> %max
+}
+
 define <4 x i32> @test44(<4 x i16> %x, <4 x i16> %y) #0 {
 ; AVX512-LABEL: test44:
 ; AVX512:       ## %bb.0:
@@ -907,7 +1189,7 @@ define <2 x i64> @test45(<2 x i16> %x, <2 x i16> %y) #0 {
 ; AVX512-NEXT:    vpmovzxwq %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x34,0xc0]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
 ; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xdb,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI46_0-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI59_0-4, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test45:
@@ -928,7 +1210,7 @@ define <2 x i64> @test46(<2 x float> %x, <2 x float> %y) #0 {
 ; AVX512-NEXT:    vpermilps $212, %xmm0, %xmm0 ## encoding: [0xc4,0xe3,0x79,0x04,0xc0,0xd4]
 ; AVX512-NEXT:    ## xmm0 = xmm0[0,1,1,3]
 ; AVX512-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0 ## encoding: [0xc5,0xf8,0x54,0x05,A,A,A,A]
-; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI47_0-4, kind: reloc_riprel_4byte
+; AVX512-NEXT:    ## fixup A - offset: 4, value: LCPI60_0-4, kind: reloc_riprel_4byte
 ; AVX512-NEXT:    retq ## encoding: [0xc3]
 ;
 ; SKX-LABEL: test46:
@@ -1112,3 +1394,33 @@ define <2 x i64> @PR41066(<2 x i64> %t0, <2 x double> %x, <2 x double> %y) {
   %t2 = select <2 x i1> %t1, <2 x i64> <i64 undef, i64 0>, <2 x i64> zeroinitializer
   ret <2 x i64> %t2
 }
+
+define <4 x i32> @zext_bool_logic(<4 x i64> %cond1, <4 x i64> %cond2, <4 x i32> %x) {
+; AVX512-LABEL: zext_bool_logic:
+; AVX512:       ## %bb.0:
+; AVX512-NEXT:    ## kill: def $ymm1 killed $ymm1 def $zmm1
+; AVX512-NEXT:    ## kill: def $ymm0 killed $ymm0 def $zmm0
+; AVX512-NEXT:    vptestnmq %zmm0, %zmm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x48,0x27,0xc0]
+; AVX512-NEXT:    vptestnmq %zmm1, %zmm1, %k1 ## encoding: [0x62,0xf2,0xf6,0x48,0x27,0xc9]
+; AVX512-NEXT:    korw %k1, %k0, %k1 ## encoding: [0xc5,0xfc,0x45,0xc9]
+; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf3,0x7d,0xc9,0x25,0xc0,0xff]
+; AVX512-NEXT:    vpsubd %xmm0, %xmm2, %xmm0 ## encoding: [0xc5,0xe9,0xfa,0xc0]
+; AVX512-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; AVX512-NEXT:    retq ## encoding: [0xc3]
+;
+; SKX-LABEL: zext_bool_logic:
+; SKX:       ## %bb.0:
+; SKX-NEXT:    vptestnmq %ymm0, %ymm0, %k0 ## encoding: [0x62,0xf2,0xfe,0x28,0x27,0xc0]
+; SKX-NEXT:    vptestnmq %ymm1, %ymm1, %k1 ## encoding: [0x62,0xf2,0xf6,0x28,0x27,0xc9]
+; SKX-NEXT:    korw %k1, %k0, %k0 ## encoding: [0xc5,0xfc,0x45,0xc1]
+; SKX-NEXT:    vpmovm2d %k0, %xmm0 ## encoding: [0x62,0xf2,0x7e,0x08,0x38,0xc0]
+; SKX-NEXT:    vpsubd %xmm0, %xmm2, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfa,0xc0]
+; SKX-NEXT:    vzeroupper ## encoding: [0xc5,0xf8,0x77]
+; SKX-NEXT:    retq ## encoding: [0xc3]
+  %a = icmp eq <4 x i64> %cond1, zeroinitializer
+  %b = icmp eq <4 x i64> %cond2, zeroinitializer
+  %c = or <4 x i1> %a, %b
+  %d = zext <4 x i1> %c to <4 x i32>
+  %e = add <4 x i32> %d, %x
+  ret <4 x i32> %e
+}
diff --git a/llvm/test/CodeGen/X86/avx512-vselect.ll b/llvm/test/CodeGen/X86/avx512-vselect.ll
index d61e4e13df9ca..7ee4e6674e0e7 100644
--- a/llvm/test/CodeGen/X86/avx512-vselect.ll
+++ b/llvm/test/CodeGen/X86/avx512-vselect.ll
@@ -47,3 +47,174 @@ entry:
   %ret = select <16 x i1> %m.or, <16 x double> %a, <16 x double> %b
   ret <16 x double> %ret
 }
+
+define <16 x i64> @test3(<16 x i8> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-SKX-LABEL: test3:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vpshufd {{.*#+}} xmm5 = xmm0[2,3,0,1]
+; CHECK-SKX-NEXT:    vptestnmb %xmm5, %xmm5, %k1
+; CHECK-SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k2
+; CHECK-SKX-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test3:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqb %xmm5, %xmm0, %xmm0
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-KNL-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <16 x i8> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <16 x i64> @test4(<16 x i16> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-SKX-LABEL: test4:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; CHECK-SKX-NEXT:    vptestnmw %xmm5, %xmm5, %k1
+; CHECK-SKX-NEXT:    vptestnmw %xmm0, %xmm0, %k2
+; CHECK-SKX-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test4:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm5, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k1}
+; CHECK-KNL-NEXT:    kshiftrw $8, %k1, %k1
+; CHECK-KNL-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <16 x i16> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <16 x i64> @test5(<16 x i32> %x, <16 x i64> %a, <16 x i64> %b) {
+; CHECK-SKX-LABEL: test5:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-SKX-NEXT:    vptestnmd %ymm5, %ymm5, %k1
+; CHECK-SKX-NEXT:    vptestnmd %ymm0, %ymm0, %k2
+; CHECK-SKX-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test5:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-KNL-NEXT:    vptestnmd %zmm5, %zmm5, %k1
+; CHECK-KNL-NEXT:    vptestnmd %zmm0, %zmm0, %k2
+; CHECK-KNL-NEXT:    vpblendmq %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-KNL-NEXT:    vpblendmq %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <16 x i32> %x, zeroinitializer
+  %ret = select <16 x i1> %c, <16 x i64> %a, <16 x i64> %b
+  ret <16 x i64> %ret
+}
+
+define <32 x i32> @test6(<32 x i8> %x, <32 x i32> %a, <32 x i32> %b) {
+; CHECK-SKX-LABEL: test6:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; CHECK-SKX-NEXT:    vptestnmb %xmm5, %xmm5, %k1
+; CHECK-SKX-NEXT:    vptestnmb %xmm0, %xmm0, %k2
+; CHECK-SKX-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test6:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm5, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm5, %zmm5
+; CHECK-KNL-NEXT:    vptestmd %zmm5, %zmm5, %k1
+; CHECK-KNL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; CHECK-KNL-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-KNL-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <32 x i8> %x, zeroinitializer
+  %ret = select <32 x i1> %c, <32 x i32> %a, <32 x i32> %b
+  ret <32 x i32> %ret
+}
+
+define <32 x i32> @test7(<32 x i16> %x, <32 x i32> %a, <32 x i32> %b) {
+; CHECK-SKX-LABEL: test7:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-SKX-NEXT:    vptestnmw %ymm5, %ymm5, %k1
+; CHECK-SKX-NEXT:    vptestnmw %ymm0, %ymm0, %k2
+; CHECK-SKX-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test7:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-KNL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm6, %ymm5, %ymm5
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm5, %zmm5
+; CHECK-KNL-NEXT:    vptestmd %zmm5, %zmm5, %k1
+; CHECK-KNL-NEXT:    vpcmpeqw %ymm6, %ymm0, %ymm0
+; CHECK-KNL-NEXT:    vpmovsxwd %ymm0, %zmm0
+; CHECK-KNL-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; CHECK-KNL-NEXT:    vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-KNL-NEXT:    vpblendmd %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <32 x i16> %x, zeroinitializer
+  %ret = select <32 x i1> %c, <32 x i32> %a, <32 x i32> %b
+  ret <32 x i32> %ret
+}
+
+define <64 x i16> @test8(<64 x i8> %x, <64 x i16> %a, <64 x i16> %b) {
+; CHECK-SKX-LABEL: test8:
+; CHECK-SKX:       # %bb.0:
+; CHECK-SKX-NEXT:    vextracti64x4 $1, %zmm0, %ymm5
+; CHECK-SKX-NEXT:    vptestnmb %ymm5, %ymm5, %k1
+; CHECK-SKX-NEXT:    vptestnmb %ymm0, %ymm0, %k2
+; CHECK-SKX-NEXT:    vpblendmw %zmm1, %zmm3, %zmm0 {%k2}
+; CHECK-SKX-NEXT:    vpblendmw %zmm2, %zmm4, %zmm1 {%k1}
+; CHECK-SKX-NEXT:    retq
+;
+; CHECK-KNL-LABEL: test8:
+; CHECK-KNL:       # %bb.0:
+; CHECK-KNL-NEXT:    pushq %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-KNL-NEXT:    .cfi_offset %rbp, -16
+; CHECK-KNL-NEXT:    movq %rsp, %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa_register %rbp
+; CHECK-KNL-NEXT:    andq $-32, %rsp
+; CHECK-KNL-NEXT:    subq $32, %rsp
+; CHECK-KNL-NEXT:    vextracti64x4 $1, %zmm0, %ymm8
+; CHECK-KNL-NEXT:    vmovdqa 16(%rbp), %ymm9
+; CHECK-KNL-NEXT:    vpxor %xmm10, %xmm10, %xmm10
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm10, %ymm0, %ymm11
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm11, %ymm0
+; CHECK-KNL-NEXT:    vpblendvb %ymm0, %ymm1, %ymm5, %ymm0
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm11, %xmm1
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm1, %ymm1
+; CHECK-KNL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm6, %ymm1
+; CHECK-KNL-NEXT:    vpcmpeqb %ymm10, %ymm8, %ymm5
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm5, %ymm2
+; CHECK-KNL-NEXT:    vpblendvb %ymm2, %ymm3, %ymm7, %ymm2
+; CHECK-KNL-NEXT:    vextracti128 $1, %ymm5, %xmm3
+; CHECK-KNL-NEXT:    vpmovsxbw %xmm3, %ymm3
+; CHECK-KNL-NEXT:    vpblendvb %ymm3, %ymm4, %ymm9, %ymm3
+; CHECK-KNL-NEXT:    movq %rbp, %rsp
+; CHECK-KNL-NEXT:    popq %rbp
+; CHECK-KNL-NEXT:    .cfi_def_cfa %rsp, 8
+; CHECK-KNL-NEXT:    retq
+  %c = icmp eq <64 x i8> %x, zeroinitializer
+  %ret = select <64 x i1> %c, <64 x i16> %a, <64 x i16> %b
+  ret <64 x i16> %ret
+}
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index beea392fc1730..0ca1f7be88529 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -49,11 +49,10 @@ declare <64 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.512(i8, <64 x i8>, i64)
   define <64 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_512(i8 %x0, <64 x i8> %x1, i64 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -79,11 +78,10 @@ declare <32 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.512(i16, <32 x i16>, i
   define <32 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_512(i16 %x0, <32 x i16> %x1, i32 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_512:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x7b,0xc8]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %zmm1 # encoding: [0x62,0xf2,0x7d,0x48,0x79,0x4c,0x24,0x02]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastw %eax, %zmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %zmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xc9,0x7b,0xd0]
+; X86-NEXT:    vmovdqu16 %zmm1, %zmm0 {%k1} # encoding: [0x62,0xf1,0xff,0x49,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %zmm1, %zmm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0xc9,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %zmm2, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %zmm0, %zmm1, %zmm0 # encoding: [0x62,0xf1,0x75,0x48,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
index bd7969e996996..8907578cd33b7 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
@@ -7,11 +7,10 @@ declare <16 x i8> @llvm.x86.avx512.mask.pbroadcast.b.gpr.128(i8, <16 x i8>, i16)
 define <16 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_128(i8 %x0, <16 x i8> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -39,12 +38,11 @@ declare <8 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.128(i16, <8 x i16>, i8)
 define <8 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_128(i16 %x0, <8 x i16> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7b,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovd %ecx, %k1 # encoding: [0xc5,0xfb,0x92,0xc9]
-; X86-NEXT:    vpbroadcastw %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7b,0xd0]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x79,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
+; X86-NEXT:    vmovdqu16 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0xff,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -72,11 +70,10 @@ define <8 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_128(i16 %x0, <8 x i16
   define <32 x i8>@test_int_x86_avx512_mask_pbroadcast_b_gpr_256(i8 %x0, <32 x i8> %x1, i32 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_b_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movb {{[0-9]+}}(%esp), %al # encoding: [0x8a,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastb %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7a,0xc8]
+; X86-NEXT:    vpbroadcastb {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x78,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastb %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7a,0xc0]
-; X86-NEXT:    vpbroadcastb %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7a,0xd0]
+; X86-NEXT:    vmovdqu8 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0x7f,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqu8 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x7f,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddb %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfc,0xc2]
 ; X86-NEXT:    vpaddb %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfc,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -105,11 +102,10 @@ declare <16 x i16> @llvm.x86.avx512.mask.pbroadcast.w.gpr.256(i16, <16 x i16>, i
   define <16 x i16>@test_int_x86_avx512_mask_pbroadcast_w_gpr_256(i16 %x0, <16 x i16> %x1, i16 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_w_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movzwl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb7,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastw %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7b,0xc8]
+; X86-NEXT:    vpbroadcastw {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x79,0x4c,0x24,0x04]
 ; X86-NEXT:    kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x08]
-; X86-NEXT:    vpbroadcastw %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7b,0xc0]
-; X86-NEXT:    vpbroadcastw %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7b,0xd0]
+; X86-NEXT:    vmovdqu16 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0xff,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqu16 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0xff,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddw %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfd,0xc2]
 ; X86-NEXT:    vpaddw %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfd,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll b/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
index 5b1c69b680d1e..b76f991662476 100644
--- a/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
+++ b/llvm/test/CodeGen/X86/avx512f-vec-test-testn.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=X64
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+avx512f | FileCheck %s --check-prefix=CHECK --check-prefix=X86
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_test_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
@@ -9,7 +10,7 @@ define zeroext i8 @TEST_mm512_test_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) lo
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> %and.i.i, zeroinitializer
@@ -25,7 +26,7 @@ define zeroext i8 @TEST_mm512_test_epi64_mask_2(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> zeroinitializer, %and.i.i
@@ -41,7 +42,7 @@ define zeroext i16 @TEST_mm512_test_epi32_mask(<8 x i64> %__A, <8 x i64> %__B) l
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -52,14 +53,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_mask_test_epi64_mask(i8 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_test_epi64_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestmq %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andb %dil, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_test_epi64_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestmq %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andb %dil, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_test_epi64_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestmq %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp ne <8 x i64> %and.i.i, zeroinitializer
@@ -71,14 +81,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i16 @TEST_mm512_mask_test_epi32_mask(i16 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_test_epi32_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestmd %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andl %edi, %eax
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_test_epi32_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestmd %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_test_epi32_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestmd %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -97,7 +116,7 @@ define zeroext i8 @TEST_mm512_testn_epi64_mask(<8 x i64> %__A, <8 x i64> %__B) l
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and.i.i, zeroinitializer
@@ -113,7 +132,7 @@ define zeroext i8 @TEST_mm512_testn_epi64_mask_2(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $al killed $al killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> zeroinitializer, %and.i.i
@@ -129,7 +148,7 @@ define zeroext i16 @TEST_mm512_testn_epi32_mask(<8 x i64> %__A, <8 x i64> %__B)
 ; CHECK-NEXT:    kmovw %k0, %eax
 ; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
 ; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; CHECK-NEXT:    ret{{[l|q]}}
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -140,14 +159,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i8 @TEST_mm512_mask_testn_epi64_mask(i8 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_testn_epi64_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestnmq %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andb %dil, %al
-; CHECK-NEXT:    # kill: def $al killed $al killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_testn_epi64_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andb %dil, %al
+; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_testn_epi64_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestnmq %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = icmp eq <8 x i64> %and.i.i, zeroinitializer
@@ -159,14 +187,23 @@ entry:
 
 ; Function Attrs: norecurse nounwind readnone
 define zeroext i16 @TEST_mm512_mask_testn_epi32_mask(i16 %__U, <8 x i64> %__A, <8 x i64> %__B) local_unnamed_addr #0 {
-; CHECK-LABEL: TEST_mm512_mask_testn_epi32_mask:
-; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    vptestnmd %zmm0, %zmm1, %k0
-; CHECK-NEXT:    kmovw %k0, %eax
-; CHECK-NEXT:    andl %edi, %eax
-; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
-; CHECK-NEXT:    vzeroupper
-; CHECK-NEXT:    retq
+; X64-LABEL: TEST_mm512_mask_testn_epi32_mask:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X64-NEXT:    kmovw %k0, %eax
+; X64-NEXT:    andl %edi, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    vzeroupper
+; X64-NEXT:    retq
+;
+; X86-LABEL: TEST_mm512_mask_testn_epi32_mask:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    vptestnmd %zmm0, %zmm1, %k0
+; X86-NEXT:    kmovw %k0, %eax
+; X86-NEXT:    andw {{[0-9]+}}(%esp), %ax
+; X86-NEXT:    # kill: def $ax killed $ax killed $eax
+; X86-NEXT:    vzeroupper
+; X86-NEXT:    retl
 entry:
   %and.i.i = and <8 x i64> %__B, %__A
   %0 = bitcast <8 x i64> %and.i.i to <16 x i32>
@@ -177,3 +214,19 @@ entry:
   ret i16 %4
 }
 
+define <2 x i64> @setcc_commute(<2 x i64> %a) {
+; CHECK-LABEL: setcc_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
+; CHECK-NEXT:    vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT:    vpsubq %xmm0, %xmm1, %xmm1
+; CHECK-NEXT:    vptestnmq %zmm0, %zmm0, %k1
+; CHECK-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; CHECK-NEXT:    vmovdqa %xmm1, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = sub <2 x i64> zeroinitializer, %a
+  %2 = icmp eq <2 x i64> %a, zeroinitializer
+  %3 = select <2 x i1> %2, <2 x i64> %a, <2 x i64> %1
+  ret <2 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index 022ec6c685584..8684d1f568fdb 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -7,12 +7,11 @@ declare <4 x i32> @llvm.x86.avx512.mask.pbroadcast.d.gpr.128(i32, <4 x i32>, i8)
 define <4 x i32>@test_int_x86_avx512_mask_pbroadcast_d_gpr_128(i32 %x0, <4 x i32> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_d_gpr_128:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %xmm1 # encoding: [0x62,0xf2,0x7d,0x08,0x7c,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vpbroadcastd %eax, %xmm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %xmm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0x89,0x7c,0xd0]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %xmm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x58,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %xmm1, %xmm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0x89,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %xmm2, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %xmm0, %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
@@ -72,12 +71,11 @@ define <2 x i64>@test_int_x86_avx512_mask_pbroadcast_q_gpr_128(i64 %x0, <2 x i64
   define <8 x i32>@test_int_x86_avx512_mask_pbroadcast_d_gpr_256(i32 %x0, <8 x i32> %x1, i8 %mask) {
 ; X86-LABEL: test_int_x86_avx512_mask_pbroadcast_d_gpr_256:
 ; X86:       # %bb.0:
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax # encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT:    vpbroadcastd %eax, %ymm1 # encoding: [0x62,0xf2,0x7d,0x28,0x7c,0xc8]
-; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %ecx # encoding: [0x0f,0xb6,0x4c,0x24,0x08]
-; X86-NEXT:    kmovw %ecx, %k1 # encoding: [0xc5,0xf8,0x92,0xc9]
-; X86-NEXT:    vpbroadcastd %eax, %ymm0 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x7c,0xc0]
-; X86-NEXT:    vpbroadcastd %eax, %ymm2 {%k1} {z} # encoding: [0x62,0xf2,0x7d,0xa9,0x7c,0xd0]
+; X86-NEXT:    vpbroadcastd {{[0-9]+}}(%esp), %ymm1 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x58,0x4c,0x24,0x04]
+; X86-NEXT:    movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x08]
+; X86-NEXT:    kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
+; X86-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0x6f,0xc1]
+; X86-NEXT:    vmovdqa32 %ymm1, %ymm2 {%k1} {z} # encoding: [0x62,0xf1,0x7d,0xa9,0x6f,0xd1]
 ; X86-NEXT:    vpaddd %ymm2, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xfe,0xc2]
 ; X86-NEXT:    vpaddd %ymm0, %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfe,0xc0]
 ; X86-NEXT:    retl # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
index 5024e67fdb791..55e127e4f4184 100644
--- a/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-vec-cmp.ll
@@ -378,8 +378,7 @@ define <4 x i64> @test256_13(<4 x i64> %x, <4 x i64> %x1, i64* %yb.ptr) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpeqq %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -402,8 +401,7 @@ define <8 x i32> @test256_14(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpled %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -429,8 +427,7 @@ define <8 x i32> @test256_15(<8 x i32> %x, i32* %yb.ptr, <8 x i32> %x1, <8 x i32
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltd %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpgtd %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -458,8 +455,7 @@ define <4 x i64> @test256_16(<4 x i64> %x, i64* %yb.ptr, <4 x i64> %x1, <4 x i64
 ; NoVLX-NEXT:    # kill: def $ymm1 killed $ymm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltq %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm2
-; NoVLX-NEXT:    vpcmpgtq %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -937,8 +933,7 @@ define <2 x i64> @test128_13(<2 x i64> %x, <2 x i64> %x1, i64* %yb.ptr) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpeqq %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -961,8 +956,7 @@ define <4 x i32> @test128_14(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1) nounwind
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpled %zmm2, %zmm0, %k1
+; NoVLX-NEXT:    vpcmpled (%rdi){1to16}, %zmm0, %k1
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -988,8 +982,7 @@ define <4 x i32> @test128_15(<4 x i32> %x, i32* %yb.ptr, <4 x i32> %x1, <4 x i32
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltd %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpgtd %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmd %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
@@ -1017,8 +1010,7 @@ define <2 x i64> @test128_16(<2 x i64> %x, i64* %yb.ptr, <2 x i64> %x1, <2 x i64
 ; NoVLX-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vpcmpnltq %zmm2, %zmm1, %k1
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm2
-; NoVLX-NEXT:    vpcmpgtq %zmm2, %zmm0, %k1 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k1 {%k1}
 ; NoVLX-NEXT:    vpblendmq %zmm0, %zmm1, %zmm0 {%k1}
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
 ; NoVLX-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
index 6e353d0b9040f..f3f65c5456400 100644
--- a/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-vec-masked-cmp.ll
@@ -1191,8 +1191,7 @@ define zeroext i8 @test_vpcmpeqd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b)
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1222,9 +1221,8 @@ define zeroext i8 @test_masked_vpcmpeqd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1382,8 +1380,7 @@ define zeroext i16 @test_vpcmpeqd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1413,9 +1410,8 @@ define zeroext i16 @test_masked_vpcmpeqd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1564,8 +1560,7 @@ define zeroext i32 @test_vpcmpeqd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1593,9 +1588,8 @@ define zeroext i32 @test_masked_vpcmpeqd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1743,8 +1737,7 @@ define zeroext i64 @test_vpcmpeqd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1772,9 +1765,8 @@ define zeroext i64 @test_masked_vpcmpeqd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1934,8 +1926,7 @@ define zeroext i16 @test_vpcmpeqd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -1966,9 +1957,8 @@ define zeroext i16 @test_masked_vpcmpeqd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2119,8 +2109,7 @@ define zeroext i32 @test_vpcmpeqd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2149,9 +2138,8 @@ define zeroext i32 @test_masked_vpcmpeqd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2301,8 +2289,7 @@ define zeroext i64 @test_vpcmpeqd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %__
 ; NoVLX-LABEL: test_vpcmpeqd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2331,9 +2318,8 @@ define zeroext i64 @test_masked_vpcmpeqd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2796,8 +2782,7 @@ define zeroext i4 @test_vpcmpeqq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2825,9 +2810,8 @@ define zeroext i4 @test_masked_vpcmpeqq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -2984,8 +2968,7 @@ define zeroext i8 @test_vpcmpeqq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3015,9 +2998,8 @@ define zeroext i8 @test_masked_vpcmpeqq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <2
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3175,8 +3157,7 @@ define zeroext i16 @test_vpcmpeqq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3206,9 +3187,8 @@ define zeroext i16 @test_masked_vpcmpeqq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3357,8 +3337,7 @@ define zeroext i32 @test_vpcmpeqq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3386,9 +3365,8 @@ define zeroext i32 @test_masked_vpcmpeqq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3536,8 +3514,7 @@ define zeroext i64 @test_vpcmpeqq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3565,9 +3542,8 @@ define zeroext i64 @test_masked_vpcmpeqq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3729,8 +3705,7 @@ define zeroext i8 @test_vpcmpeqq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b)
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3761,9 +3736,8 @@ define zeroext i8 @test_masked_vpcmpeqq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <4
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3926,8 +3900,7 @@ define zeroext i16 @test_vpcmpeqq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -3958,9 +3931,8 @@ define zeroext i16 @test_masked_vpcmpeqq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4114,8 +4086,7 @@ define zeroext i32 @test_vpcmpeqq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4144,9 +4115,8 @@ define zeroext i32 @test_masked_vpcmpeqq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4299,8 +4269,7 @@ define zeroext i64 @test_vpcmpeqq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %__
 ; NoVLX-LABEL: test_vpcmpeqq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpeqq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -4329,9 +4298,8 @@ define zeroext i64 @test_masked_vpcmpeqq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpeqq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpeqq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6027,8 +5995,7 @@ define zeroext i8 @test_vpcmpsgtd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6058,9 +6025,8 @@ define zeroext i8 @test_masked_vpcmpsgtd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6218,8 +6184,7 @@ define zeroext i16 @test_vpcmpsgtd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6249,9 +6214,8 @@ define zeroext i16 @test_masked_vpcmpsgtd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6400,8 +6364,7 @@ define zeroext i32 @test_vpcmpsgtd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6429,9 +6392,8 @@ define zeroext i32 @test_masked_vpcmpsgtd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6579,8 +6541,7 @@ define zeroext i64 @test_vpcmpsgtd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6608,9 +6569,8 @@ define zeroext i64 @test_masked_vpcmpsgtd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6770,8 +6730,7 @@ define zeroext i16 @test_vpcmpsgtd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6802,9 +6761,8 @@ define zeroext i16 @test_masked_vpcmpsgtd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6955,8 +6913,7 @@ define zeroext i32 @test_vpcmpsgtd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -6985,9 +6942,8 @@ define zeroext i32 @test_masked_vpcmpsgtd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7137,8 +7093,7 @@ define zeroext i64 @test_vpcmpsgtd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsgtd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7167,9 +7122,8 @@ define zeroext i64 @test_masked_vpcmpsgtd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7632,8 +7586,7 @@ define zeroext i4 @test_vpcmpsgtq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7661,9 +7614,8 @@ define zeroext i4 @test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7820,8 +7772,7 @@ define zeroext i8 @test_vpcmpsgtq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -7851,9 +7802,8 @@ define zeroext i8 @test_masked_vpcmpsgtq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8011,8 +7961,7 @@ define zeroext i16 @test_vpcmpsgtq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8042,9 +7991,8 @@ define zeroext i16 @test_masked_vpcmpsgtq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8193,8 +8141,7 @@ define zeroext i32 @test_vpcmpsgtq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8222,9 +8169,8 @@ define zeroext i32 @test_masked_vpcmpsgtq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8372,8 +8318,7 @@ define zeroext i64 @test_vpcmpsgtq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8401,9 +8346,8 @@ define zeroext i64 @test_masked_vpcmpsgtq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8565,8 +8509,7 @@ define zeroext i8 @test_vpcmpsgtq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8597,9 +8540,8 @@ define zeroext i8 @test_masked_vpcmpsgtq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8762,8 +8704,7 @@ define zeroext i16 @test_vpcmpsgtq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8794,9 +8735,8 @@ define zeroext i16 @test_masked_vpcmpsgtq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8950,8 +8890,7 @@ define zeroext i32 @test_vpcmpsgtq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -8980,9 +8919,8 @@ define zeroext i32 @test_masked_vpcmpsgtq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -9135,8 +9073,7 @@ define zeroext i64 @test_vpcmpsgtq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgtq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpgtq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -9165,9 +9102,8 @@ define zeroext i64 @test_masked_vpcmpsgtq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgtq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpgtq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpgtq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -10923,8 +10859,7 @@ define zeroext i8 @test_vpcmpsged_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -10954,9 +10889,8 @@ define zeroext i8 @test_masked_vpcmpsged_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11114,8 +11048,7 @@ define zeroext i16 @test_vpcmpsged_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11145,9 +11078,8 @@ define zeroext i16 @test_masked_vpcmpsged_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11296,8 +11228,7 @@ define zeroext i32 @test_vpcmpsged_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11325,9 +11256,8 @@ define zeroext i32 @test_masked_vpcmpsged_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11475,8 +11405,7 @@ define zeroext i64 @test_vpcmpsged_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11504,9 +11433,8 @@ define zeroext i64 @test_masked_vpcmpsged_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11666,8 +11594,7 @@ define zeroext i16 @test_vpcmpsged_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11698,9 +11625,8 @@ define zeroext i16 @test_masked_vpcmpsged_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11851,8 +11777,7 @@ define zeroext i32 @test_vpcmpsged_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -11881,9 +11806,8 @@ define zeroext i32 @test_masked_vpcmpsged_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12033,8 +11957,7 @@ define zeroext i64 @test_vpcmpsged_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpsged_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltd (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12063,9 +11986,8 @@ define zeroext i64 @test_masked_vpcmpsged_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsged_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltd (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12528,8 +12450,7 @@ define zeroext i4 @test_vpcmpsgeq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12557,9 +12478,8 @@ define zeroext i4 @test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12716,8 +12636,7 @@ define zeroext i8 @test_vpcmpsgeq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12747,9 +12666,8 @@ define zeroext i8 @test_masked_vpcmpsgeq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12907,8 +12825,7 @@ define zeroext i16 @test_vpcmpsgeq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -12938,9 +12855,8 @@ define zeroext i16 @test_masked_vpcmpsgeq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13089,8 +13005,7 @@ define zeroext i32 @test_vpcmpsgeq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13118,9 +13033,8 @@ define zeroext i32 @test_masked_vpcmpsgeq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13268,8 +13182,7 @@ define zeroext i64 @test_vpcmpsgeq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13297,9 +13210,8 @@ define zeroext i64 @test_masked_vpcmpsgeq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13461,8 +13373,7 @@ define zeroext i8 @test_vpcmpsgeq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13493,9 +13404,8 @@ define zeroext i8 @test_masked_vpcmpsgeq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13658,8 +13568,7 @@ define zeroext i16 @test_vpcmpsgeq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13690,9 +13599,8 @@ define zeroext i16 @test_masked_vpcmpsgeq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13846,8 +13754,7 @@ define zeroext i32 @test_vpcmpsgeq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -13876,9 +13783,8 @@ define zeroext i32 @test_masked_vpcmpsgeq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -14031,8 +13937,7 @@ define zeroext i64 @test_vpcmpsgeq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpsgeq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpnltq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -14061,9 +13966,8 @@ define zeroext i64 @test_masked_vpcmpsgeq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpsgeq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpnltq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpnltq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -15839,8 +15743,7 @@ define zeroext i8 @test_vpcmpultd_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, i32* %__b
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -15870,9 +15773,8 @@ define zeroext i8 @test_masked_vpcmpultd_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16030,8 +15932,7 @@ define zeroext i16 @test_vpcmpultd_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16061,9 +15962,8 @@ define zeroext i16 @test_masked_vpcmpultd_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16212,8 +16112,7 @@ define zeroext i32 @test_vpcmpultd_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16241,9 +16140,8 @@ define zeroext i32 @test_masked_vpcmpultd_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16391,8 +16289,7 @@ define zeroext i64 @test_vpcmpultd_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16420,9 +16317,8 @@ define zeroext i64 @test_masked_vpcmpultd_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16582,8 +16478,7 @@ define zeroext i16 @test_vpcmpultd_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16614,9 +16509,8 @@ define zeroext i16 @test_masked_vpcmpultd_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16767,8 +16661,7 @@ define zeroext i32 @test_vpcmpultd_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16797,9 +16690,8 @@ define zeroext i32 @test_masked_vpcmpultd_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16949,8 +16841,7 @@ define zeroext i64 @test_vpcmpultd_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, i32* %_
 ; NoVLX-LABEL: test_vpcmpultd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltud (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -16979,9 +16870,8 @@ define zeroext i64 @test_masked_vpcmpultd_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultd_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastd (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltud %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltud (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17444,8 +17334,7 @@ define zeroext i4 @test_vpcmpultq_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17473,9 +17362,8 @@ define zeroext i4 @test_masked_vpcmpultq_v2i1_v4i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17632,8 +17520,7 @@ define zeroext i8 @test_vpcmpultq_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17663,9 +17550,8 @@ define zeroext i8 @test_masked_vpcmpultq_v2i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17823,8 +17709,7 @@ define zeroext i16 @test_vpcmpultq_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -17854,9 +17739,8 @@ define zeroext i16 @test_masked_vpcmpultq_v2i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18005,8 +17889,7 @@ define zeroext i32 @test_vpcmpultq_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18034,9 +17917,8 @@ define zeroext i32 @test_masked_vpcmpultq_v2i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18184,8 +18066,7 @@ define zeroext i64 @test_vpcmpultq_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %xmm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18213,9 +18094,8 @@ define zeroext i64 @test_masked_vpcmpultq_v2i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %xmm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18377,8 +18257,7 @@ define zeroext i8 @test_vpcmpultq_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, i64* %__b
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18409,9 +18288,8 @@ define zeroext i8 @test_masked_vpcmpultq_v4i1_v8i1_mask_mem_b(i8 zeroext %__u, <
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18574,8 +18452,7 @@ define zeroext i16 @test_vpcmpultq_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18606,9 +18483,8 @@ define zeroext i16 @test_masked_vpcmpultq_v4i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18762,8 +18638,7 @@ define zeroext i32 @test_vpcmpultq_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18792,9 +18667,8 @@ define zeroext i32 @test_masked_vpcmpultq_v4i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18947,8 +18821,7 @@ define zeroext i64 @test_vpcmpultq_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, i64* %_
 ; NoVLX-LABEL: test_vpcmpultq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rdi), %ymm1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vpcmpltuq (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -18977,9 +18850,8 @@ define zeroext i64 @test_masked_vpcmpultq_v4i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vpcmpultq_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vpbroadcastq (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vpcmpltuq %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vpcmpltuq (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19555,8 +19427,7 @@ define zeroext i8 @test_vcmpoeqps_v4i1_v8i1_mask_mem_b(<2 x i64> %__a, float* %_
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19652,8 +19523,7 @@ define zeroext i8 @test_masked_vcmpoeqps_v4i1_v8i1_mask_mem_b(i4 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19743,8 +19613,7 @@ define zeroext i16 @test_vcmpoeqps_v4i1_v16i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19840,8 +19709,7 @@ define zeroext i16 @test_masked_vcmpoeqps_v4i1_v16i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -19926,8 +19794,7 @@ define zeroext i32 @test_vcmpoeqps_v4i1_v32i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20017,8 +19884,7 @@ define zeroext i32 @test_masked_vcmpoeqps_v4i1_v32i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20102,8 +19968,7 @@ define zeroext i64 @test_vcmpoeqps_v4i1_v64i1_mask_mem_b(<2 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20193,8 +20058,7 @@ define zeroext i64 @test_masked_vcmpoeqps_v4i1_v64i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastss (%rsi), %xmm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20286,8 +20150,7 @@ define zeroext i16 @test_vcmpoeqps_v8i1_v16i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20385,9 +20248,8 @@ define zeroext i16 @test_masked_vcmpoeqps_v8i1_v16i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20475,8 +20337,7 @@ define zeroext i32 @test_vcmpoeqps_v8i1_v32i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20568,9 +20429,8 @@ define zeroext i32 @test_masked_vcmpoeqps_v8i1_v32i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20657,8 +20517,7 @@ define zeroext i64 @test_vcmpoeqps_v8i1_v64i1_mask_mem_b(<4 x i64> %__a, float*
 ; NoVLX-LABEL: test_vcmpoeqps_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqps (%rdi){1to16}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -20750,9 +20609,8 @@ define zeroext i64 @test_masked_vcmpoeqps_v8i1_v64i1_mask_mem_b(i8 zeroext %__u,
 ; NoVLX-LABEL: test_masked_vcmpoeqps_v8i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastss (%rsi), %ymm1
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vcmpeqps %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqps (%rsi){1to16}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $8, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21243,8 +21101,7 @@ define zeroext i4 @test_vcmpoeqpd_v2i1_v4i1_mask_mem_b(<2 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v4i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21334,8 +21191,7 @@ define zeroext i4 @test_masked_vcmpoeqpd_v2i1_v4i1_mask_mem_b(i2 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21424,8 +21280,7 @@ define zeroext i8 @test_vcmpoeqpd_v2i1_v8i1_mask_mem_b(<2 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21521,8 +21376,7 @@ define zeroext i8 @test_masked_vcmpoeqpd_v2i1_v8i1_mask_mem_b(i2 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21612,8 +21466,7 @@ define zeroext i16 @test_vcmpoeqpd_v2i1_v16i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21709,8 +21562,7 @@ define zeroext i16 @test_masked_vcmpoeqpd_v2i1_v16i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21795,8 +21647,7 @@ define zeroext i32 @test_vcmpoeqpd_v2i1_v32i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21886,8 +21737,7 @@ define zeroext i32 @test_masked_vcmpoeqpd_v2i1_v32i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -21971,8 +21821,7 @@ define zeroext i64 @test_vcmpoeqpd_v2i1_v64i1_mask_mem_b(<2 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v2i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22062,8 +21911,7 @@ define zeroext i64 @test_masked_vcmpoeqpd_v2i1_v64i1_mask_mem_b(i2 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vmovddup {{.*#+}} xmm1 = mem[0,0]
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $14, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $14, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22155,8 +22003,7 @@ define zeroext i8 @test_vcmpoeqpd_v4i1_v8i1_mask_mem_b(<4 x i64> %__a, double* %
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v8i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22255,8 +22102,7 @@ define zeroext i8 @test_masked_vcmpoeqpd_v4i1_v8i1_mask_mem_b(i4 zeroext %__u, <
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22349,8 +22195,7 @@ define zeroext i16 @test_vcmpoeqpd_v4i1_v16i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v16i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22449,8 +22294,7 @@ define zeroext i16 @test_masked_vcmpoeqpd_v4i1_v16i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22538,8 +22382,7 @@ define zeroext i32 @test_vcmpoeqpd_v4i1_v32i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v32i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22632,8 +22475,7 @@ define zeroext i32 @test_masked_vcmpoeqpd_v4i1_v32i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22720,8 +22562,7 @@ define zeroext i64 @test_vcmpoeqpd_v4i1_v64i1_mask_mem_b(<4 x i64> %__a, double*
 ; NoVLX-LABEL: test_vcmpoeqpd_v4i1_v64i1_mask_mem_b:
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; NoVLX-NEXT:    vbroadcastsd (%rdi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0
+; NoVLX-NEXT:    vcmpeqpd (%rdi){1to8}, %zmm0, %k0
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -22814,8 +22655,7 @@ define zeroext i64 @test_masked_vcmpoeqpd_v4i1_v64i1_mask_mem_b(i4 zeroext %__u,
 ; NoVLX:       # %bb.0: # %entry
 ; NoVLX-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
 ; NoVLX-NEXT:    kmovw %edi, %k1
-; NoVLX-NEXT:    vbroadcastsd (%rsi), %ymm1
-; NoVLX-NEXT:    vcmpeqpd %zmm1, %zmm0, %k0 {%k1}
+; NoVLX-NEXT:    vcmpeqpd (%rsi){1to8}, %zmm0, %k0 {%k1}
 ; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
 ; NoVLX-NEXT:    kshiftrw $12, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
@@ -23526,8 +23366,7 @@ define i8 @mask_zero_lower(<4 x i32> %a) {
 ; NoVLX:       # %bb.0:
 ; NoVLX-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; NoVLX-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; NoVLX-NEXT:    kshiftlw $12, %k0, %k0
-; NoVLX-NEXT:    kshiftrw $8, %k0, %k0
+; NoVLX-NEXT:    kshiftlw $4, %k0, %k0
 ; NoVLX-NEXT:    kmovw %k0, %eax
 ; NoVLX-NEXT:    # kill: def $al killed $al killed $eax
 ; NoVLX-NEXT:    vzeroupper
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
index 07e5f7a029c98..af1abe71e2f2f 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-zext.ll
@@ -496,12 +496,12 @@ define <32 x i8> @ext_i32_32i8(i32 %a0) {
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    shrl $16, %edi
 ; AVX512F-NEXT:    kmovw %edi, %k2
-; AVX512F-NEXT:    movl {{.*}}(%rip), %eax
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0 {%k1} {z}
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k2} {z}
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k2} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm1, %ymm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i32_32i8:
@@ -898,18 +898,18 @@ define <64 x i8> @ext_i64_64i8(i64 %a0) {
 ; AVX512F-NEXT:    kmovw %ecx, %k2
 ; AVX512F-NEXT:    kmovw %eax, %k3
 ; AVX512F-NEXT:    kmovw %edi, %k4
-; AVX512F-NEXT:    movl {{.*}}(%rip), %eax
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0 {%k4} {z}
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k3} {z}
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm1 {%k1} {z}
+; AVX512F-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k4} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vpbroadcastd %eax, %zmm2 {%k2} {z}
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k3} {z}
 ; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
 ; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1} {z}
+; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT:    vmovdqa32 %zmm0, %zmm0 {%k2} {z}
+; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: ext_i64_64i8:
diff --git a/llvm/test/CodeGen/X86/bitcast-mmx.ll b/llvm/test/CodeGen/X86/bitcast-mmx.ll
index 343c230324c2e..566cb1b8503c5 100644
--- a/llvm/test/CodeGen/X86/bitcast-mmx.ll
+++ b/llvm/test/CodeGen/X86/bitcast-mmx.ll
@@ -34,10 +34,10 @@ define i64 @t1(i64 %x, i32 %n) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movd 16(%ebp), %mm0
-; X86-NEXT:    movq 8(%ebp), %mm1
-; X86-NEXT:    psllq %mm0, %mm1
-; X86-NEXT:    movq %mm1, (%esp)
+; X86-NEXT:    movq 8(%ebp), %mm0
+; X86-NEXT:    movd 16(%ebp), %mm1
+; X86-NEXT:    psllq %mm1, %mm0
+; X86-NEXT:    movq %mm0, (%esp)
 ; X86-NEXT:    movl (%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl %ebp, %esp
@@ -46,10 +46,10 @@ define i64 @t1(i64 %x, i32 %n) nounwind {
 ;
 ; X64-LABEL: t1:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movd %esi, %mm0
-; X64-NEXT:    movq %rdi, %mm1
-; X64-NEXT:    psllq %mm0, %mm1
-; X64-NEXT:    movq %mm1, %rax
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    movd %esi, %mm1
+; X64-NEXT:    psllq %mm1, %mm0
+; X64-NEXT:    movq %mm0, %rax
 ; X64-NEXT:    retq
 entry:
   %0 = bitcast i64 %x to x86_mmx
@@ -65,11 +65,11 @@ define i64 @t2(i64 %x, i32 %n, i32 %w) nounwind {
 ; X86-NEXT:    movl %esp, %ebp
 ; X86-NEXT:    andl $-8, %esp
 ; X86-NEXT:    subl $8, %esp
-; X86-NEXT:    movd 16(%ebp), %mm0
-; X86-NEXT:    movd 20(%ebp), %mm1
-; X86-NEXT:    psllq %mm0, %mm1
-; X86-NEXT:    por 8(%ebp), %mm1
-; X86-NEXT:    movq %mm1, (%esp)
+; X86-NEXT:    movd 20(%ebp), %mm0
+; X86-NEXT:    movd 16(%ebp), %mm1
+; X86-NEXT:    psllq %mm1, %mm0
+; X86-NEXT:    por 8(%ebp), %mm0
+; X86-NEXT:    movq %mm0, (%esp)
 ; X86-NEXT:    movl (%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
 ; X86-NEXT:    movl %ebp, %esp
@@ -78,12 +78,12 @@ define i64 @t2(i64 %x, i32 %n, i32 %w) nounwind {
 ;
 ; X64-LABEL: t2:
 ; X64:       # %bb.0: # %entry
-; X64-NEXT:    movd %esi, %mm0
-; X64-NEXT:    movd %edx, %mm1
-; X64-NEXT:    psllq %mm0, %mm1
-; X64-NEXT:    movq %rdi, %mm0
-; X64-NEXT:    por %mm1, %mm0
-; X64-NEXT:    movq %mm0, %rax
+; X64-NEXT:    movd %edx, %mm0
+; X64-NEXT:    movd %esi, %mm1
+; X64-NEXT:    psllq %mm1, %mm0
+; X64-NEXT:    movq %rdi, %mm1
+; X64-NEXT:    por %mm0, %mm1
+; X64-NEXT:    movq %mm1, %rax
 ; X64-NEXT:    retq
 entry:
   %0 = insertelement <2 x i32> undef, i32 %w, i32 0
diff --git a/llvm/test/CodeGen/X86/bmi-x86_64.ll b/llvm/test/CodeGen/X86/bmi-x86_64.ll
index 81ac313298733..7646fd6bbb068 100644
--- a/llvm/test/CodeGen/X86/bmi-x86_64.ll
+++ b/llvm/test/CodeGen/X86/bmi-x86_64.ll
@@ -86,9 +86,9 @@ define i64 @bextr64d(i64 %a) {
 ;
 ; BMI2-SLOW-LABEL: bextr64d:
 ; BMI2-SLOW:       # %bb.0: # %entry
-; BMI2-SLOW-NEXT:    shrq $2, %rdi
-; BMI2-SLOW-NEXT:    movb $33, %al
+; BMI2-SLOW-NEXT:    movl $35, %eax
 ; BMI2-SLOW-NEXT:    bzhiq %rax, %rdi, %rax
+; BMI2-SLOW-NEXT:    shrq $2, %rax
 ; BMI2-SLOW-NEXT:    retq
 ;
 ; BEXTR-FAST-LABEL: bextr64d:
@@ -102,6 +102,34 @@ entry:
   ret i64 %and
 }
 
+define i64 @bextr64d_load(i64* %aptr) {
+; BMI1-SLOW-LABEL: bextr64d_load:
+; BMI1-SLOW:       # %bb.0: # %entry
+; BMI1-SLOW-NEXT:    movq (%rdi), %rax
+; BMI1-SLOW-NEXT:    shrq $2, %rax
+; BMI1-SLOW-NEXT:    movl $8448, %ecx # imm = 0x2100
+; BMI1-SLOW-NEXT:    bextrq %rcx, %rax, %rax
+; BMI1-SLOW-NEXT:    retq
+;
+; BMI2-SLOW-LABEL: bextr64d_load:
+; BMI2-SLOW:       # %bb.0: # %entry
+; BMI2-SLOW-NEXT:    movl $35, %eax
+; BMI2-SLOW-NEXT:    bzhiq %rax, (%rdi), %rax
+; BMI2-SLOW-NEXT:    shrq $2, %rax
+; BMI2-SLOW-NEXT:    retq
+;
+; BEXTR-FAST-LABEL: bextr64d_load:
+; BEXTR-FAST:       # %bb.0: # %entry
+; BEXTR-FAST-NEXT:    movl $8450, %eax # imm = 0x2102
+; BEXTR-FAST-NEXT:    bextrq %rax, (%rdi), %rax
+; BEXTR-FAST-NEXT:    retq
+entry:
+  %a = load i64, i64* %aptr, align 8
+  %shr = lshr i64 %a, 2
+  %and = and i64 %shr, 8589934591
+  ret i64 %and
+}
+
 define i64 @non_bextr64(i64 %x) {
 ; CHECK-LABEL: non_bextr64:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/bmi.ll b/llvm/test/CodeGen/X86/bmi.ll
index d5603431e756c..9c3a74fccbe63 100644
--- a/llvm/test/CodeGen/X86/bmi.ll
+++ b/llvm/test/CodeGen/X86/bmi.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X86,X86-SLOW-BEXTR,X86-SLOW-BEXTR-BMI2
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2 | FileCheck %s --check-prefixes=CHECK,X64,X64-SLOW-BEXTR,X64-SLOW-BEXTR-BMI2
 ; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+cmov,+bmi,+fast-bextr | FileCheck %s --check-prefixes=CHECK,X86,X86-FAST-BEXTR
 ; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+fast-bextr | FileCheck %s --check-prefixes=CHECK,X64,X64-FAST-BEXTR
 
diff --git a/llvm/test/CodeGen/X86/bswap_tree.ll b/llvm/test/CodeGen/X86/bswap_tree.ll
index 537acdb245be9..79a45050b98f0 100644
--- a/llvm/test/CodeGen/X86/bswap_tree.ll
+++ b/llvm/test/CodeGen/X86/bswap_tree.ll
@@ -70,3 +70,47 @@ define i32 @test2(i32 %x) nounwind {
   %result = or i32 %or0, %or1
   ret i32 %result
 }
+
+declare i32 @llvm.bswap.i32(i32)
+
+; Match a 32-bit packed halfword bswap, with some subtree
+; already converted to a bswap.
+define i32 @test3(i32 %x) nounwind {
+; CHECK-LABEL: test3:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT:    movl %eax, %ecx
+; CHECK-NEXT:    andl $16711680, %ecx # imm = 0xFF0000
+; CHECK-NEXT:    movl %eax, %edx
+; CHECK-NEXT:    andl $-16777216, %edx # imm = 0xFF000000
+; CHECK-NEXT:    shll $8, %ecx
+; CHECK-NEXT:    shrl $8, %edx
+; CHECK-NEXT:    orl %ecx, %edx
+; CHECK-NEXT:    bswapl %eax
+; CHECK-NEXT:    shrl $16, %eax
+; CHECK-NEXT:    orl %edx, %eax
+; CHECK-NEXT:    retl
+;
+; CHECK64-LABEL: test3:
+; CHECK64:       # %bb.0:
+; CHECK64-NEXT:    movl %edi, %eax
+; CHECK64-NEXT:    andl $16711680, %eax # imm = 0xFF0000
+; CHECK64-NEXT:    movl %edi, %ecx
+; CHECK64-NEXT:    andl $-16777216, %ecx # imm = 0xFF000000
+; CHECK64-NEXT:    shll $8, %eax
+; CHECK64-NEXT:    shrl $8, %ecx
+; CHECK64-NEXT:    addl %ecx, %eax
+; CHECK64-NEXT:    bswapl %edi
+; CHECK64-NEXT:    shrl $16, %edi
+; CHECK64-NEXT:    orl %edi, %eax
+; CHECK64-NEXT:    retq
+  %byte2 = and i32 %x, 16711680   ; 0x00ff0000
+  %byte3 = and i32 %x, 4278190080 ; 0xff000000
+  %1 = shl  i32 %byte2, 8
+  %2 = lshr i32 %byte3, 8
+  %or = or i32 %1, %2
+  %bswap = call i32 @llvm.bswap.i32(i32 %x)
+  %3 = lshr i32 %bswap, 16
+  %result = or i32 %or, %3
+  ret i32 %result
+}
diff --git a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
index 11fc0df2443ce..14a71050e94aa 100644
--- a/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
+++ b/llvm/test/CodeGen/X86/bypass-slow-division-64.ll
@@ -75,3 +75,13 @@ define i64 @Test_get_quotient_and_remainder(i64 %a, i64 %b) nounwind {
   %result = add i64 %resultdiv, %resultrem
   ret i64 %result
 }
+
+define void @PR43514(i32 %x, i32 %y) {
+; CHECK-LABEL: PR43514:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    retq
+  %z1 = zext i32 %x to i64
+  %z2 = zext i32 %y to i64
+  %s = srem i64 %z1, %z2
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/codegen-prepare-replacephi2.mir b/llvm/test/CodeGen/X86/codegen-prepare-replacephi2.mir
new file mode 100644
index 0000000000000..42986adcdca8f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/codegen-prepare-replacephi2.mir
@@ -0,0 +1,68 @@
+# RUN: llc -run-pass=codegenprepare -o - %s | FileCheck %s
+
+# REQUIRES: default_triple
+
+# This testcase without the accompanying fix triggers the assert
+# "Replacement PHI node is already replaced."
+
+--- |
+
+  define void @f1() {
+  entry:
+    %arrayidx = getelementptr inbounds [2 x i16], [2 x i16]* undef, i16 0, i16 2
+    %0 = bitcast i16* %arrayidx to i32*
+    %1 = bitcast [2 x i16]* undef to i32*
+    br label %for.cond
+
+  for.cond.loopexit:
+    br label %for.cond
+
+  for.cond:
+    %2 = phi i32* [ %0, %entry ], [ %12, %for.cond.loopexit ]
+    %3 = phi i32* [ %0, %entry ], [ %14, %for.cond.loopexit ]
+    br label %for.body
+
+  for.body:
+    %4 = phi i32* [ %3, %for.cond ], [ %14, %cleanup ]
+    %5 = phi i32* [ %2, %for.cond ], [ %13, %cleanup ]
+    %6 = phi i32* [ %2, %for.cond ], [ %12, %cleanup ]
+    br i1 undef, label %for.cond2.preheader, label %if.then
+
+  for.cond2.preheader:
+    br label %for.cond2
+
+  if.then:
+    store i32 undef, i32* %4, align 1
+    br label %cleanup
+
+  for.cond2:
+    %7 = phi i32* [ %10, %for.inc ], [ %6, %for.cond2.preheader ]
+    %8 = phi i32* [ %11, %for.inc ], [ %5, %for.cond2.preheader ]
+    %9 = phi i32* [ %11, %for.inc ], [ %4, %for.cond2.preheader ]
+    br i1 undef, label %for.inc, label %if.then5
+
+  if.then5:
+    br i1 icmp ne (i16 2, i16 0), label %cleanup.loopexit, label %if.end
+
+  if.end:
+    br label %for.inc
+
+  for.inc:
+    %10 = phi i32* [ %7, %if.end ], [ %1, %for.cond2 ]
+    %11 = phi i32* [ %8, %if.end ], [ %1, %for.cond2 ]
+    br label %for.cond2
+
+  cleanup.loopexit:
+    br label %cleanup
+
+  cleanup:
+    %12 = phi i32* [ %6, %if.then ], [ %7, %cleanup.loopexit ]
+    %13 = phi i32* [ %5, %if.then ], [ %8, %cleanup.loopexit ]
+    %14 = phi i32* [ %4, %if.then ], [ %9, %cleanup.loopexit ]
+    br i1 true, label %for.cond.loopexit, label %for.body
+  }
+
+...
+
+# Sanity check to verify that something got through.
+# CHECK-LABEL: entry:
diff --git a/llvm/test/CodeGen/X86/combine-bitselect.ll b/llvm/test/CodeGen/X86/combine-bitselect.ll
index 8cb6a4dca0913..b65a7504b24aa 100644
--- a/llvm/test/CodeGen/X86/combine-bitselect.ll
+++ b/llvm/test/CodeGen/X86/combine-bitselect.ll
@@ -329,9 +329,7 @@ define <8 x i64> @bitselect_v8i64_rr(<8 x i64>, <8 x i64>) {
 ;
 ; AVX512F-LABEL: bitselect_v8i64_rr:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = and <8 x i64> %0, <i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890, i64 4294967296, i64 12884901890, i64 12884901890, i64 12884901890>
   %4 = and <8 x i64> %1, <i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891, i64 -4294967297, i64 -12884901891, i64 -12884901891, i64 -12884901891>
@@ -405,9 +403,7 @@ define <8 x i64> @bitselect_v8i64_rm(<8 x i64>, <8 x i64>* nocapture readonly) {
 ; AVX512F-LABEL: bitselect_v8i64_rm:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %1
   %4 = and <8 x i64> %0, <i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3, i64 8589934593, i64 3>
@@ -482,9 +478,7 @@ define <8 x i64> @bitselect_v8i64_mr(<8 x i64>* nocapture readonly, <8 x i64>) {
 ; AVX512F-LABEL: bitselect_v8i64_mr:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %0
   %4 = and <8 x i64> %3, <i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296, i64 12884901890, i64 4294967296>
@@ -554,11 +548,9 @@ define <8 x i64> @bitselect_v8i64_mm(<8 x i64>* nocapture readonly, <8 x i64>* n
 ;
 ; AVX512F-LABEL: bitselect_v8i64_mm:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512F-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022,18446744073709551612,18446744065119617022]
+; AVX512F-NEXT:    vpternlogq $202, (%rdi), %zmm1, %zmm0
 ; AVX512F-NEXT:    retq
   %3 = load <8 x i64>, <8 x i64>* %0
   %4 = load <8 x i64>, <8 x i64>* %1
@@ -616,10 +608,8 @@ define <4 x i1> @bitselect_v4i1_loop(<4 x i32> %a0, <4 x i32> %a1) {
 ; AVX512F:       # %bb.0: # %bb
 ; AVX512F-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [12,12,12,12]
-; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm1, %k1
-; AVX512F-NEXT:    vpbroadcastd {{.*#+}} xmm2 = [15,15,15,15]
-; AVX512F-NEXT:    vpcmpeqd %zmm2, %zmm1, %k2
+; AVX512F-NEXT:    vpcmpeqd {{.*}}(%rip){1to16}, %zmm1, %k1
+; AVX512F-NEXT:    vpcmpeqd {{.*}}(%rip){1to16}, %zmm1, %k2
 ; AVX512F-NEXT:    vptestnmd %zmm0, %zmm0, %k0 {%k2}
 ; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1 {%k1}
 ; AVX512F-NEXT:    korw %k0, %k1, %k1
diff --git a/llvm/test/CodeGen/X86/combine-mulo.ll b/llvm/test/CodeGen/X86/combine-mulo.ll
index 99f69a9a100d4..0eddf2f81c010 100644
--- a/llvm/test/CodeGen/X86/combine-mulo.ll
+++ b/llvm/test/CodeGen/X86/combine-mulo.ll
@@ -34,30 +34,21 @@ define <4 x i32> @combine_vec_smul_two(<4 x i32> %a0, <4 x i32> %a1) {
 ; SSE-LABEL: combine_vec_smul_two:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    movdqa %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm0, %xmm0
-; SSE-NEXT:    pxor %xmm3, %xmm3
+; SSE-NEXT:    paddd %xmm0, %xmm2
+; SSE-NEXT:    movdqa %xmm0, %xmm3
 ; SSE-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    paddd %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm0
-; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
-; SSE-NEXT:    blendvps %xmm0, %xmm2, %xmm1
-; SSE-NEXT:    movaps %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm0, %xmm3
+; SSE-NEXT:    movdqa %xmm3, %xmm0
+; SSE-NEXT:    blendvps %xmm0, %xmm1, %xmm2
+; SSE-NEXT:    movaps %xmm2, %xmm0
 ; SSE-NEXT:    retq
 ;
 ; AVX-LABEL: combine_vec_smul_two:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm3
+; AVX-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX-NEXT:    retq
   %1 = call {<4 x i32>, <4 x i1>} @llvm.smul.with.overflow.v4i32(<4 x i32> %a0, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
   %2 = extractvalue {<4 x i32>, <4 x i1>} %1, 0
diff --git a/llvm/test/CodeGen/X86/commute-fcmp.ll b/llvm/test/CodeGen/X86/commute-fcmp.ll
index 4da8a45567f3a..390cec3a5712f 100644
--- a/llvm/test/CodeGen/X86/commute-fcmp.ll
+++ b/llvm/test/CodeGen/X86/commute-fcmp.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+sse2 -disable-peephole | FileCheck %s --check-prefix=SSE
 ; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 -disable-peephole | FileCheck %s --check-prefix=AVX
-; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl -disable-peephole | FileCheck %s --check-prefix=AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512vl,+avx512dq -disable-peephole | FileCheck %s --check-prefix=AVX512
 
 ;
 ; Float Comparisons
@@ -788,3 +788,523 @@ define <4 x i64> @commute_cmppd_le_ymmm(<4 x double>* %a0, <4 x double> %a1) {
   %3 = sext <4 x i1> %2 to <4 x i64>
   ret <4 x i64> %3
 }
+
+define <16 x i32> @commute_cmpps_eq_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_eq_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpeqps (%rdi), %xmm0
+; SSE-NEXT:    cmpeqps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpeqps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpeqps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_eq_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_eq_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp oeq <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ne_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ne_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpneqps (%rdi), %xmm0
+; SSE-NEXT:    cmpneqps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpneqps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpneqps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ne_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ne_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp une <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ord_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ord_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpordps (%rdi), %xmm0
+; SSE-NEXT:    cmpordps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpordps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpordps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ord_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpordps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpordps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ord_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpordps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ord <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_uno_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_uno_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpunordps (%rdi), %xmm0
+; SSE-NEXT:    cmpunordps 16(%rdi), %xmm1
+; SSE-NEXT:    cmpunordps 32(%rdi), %xmm2
+; SSE-NEXT:    cmpunordps 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_uno_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpunordps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpunordps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_uno_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp uno <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_ueq_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_ueq_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm7
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    movaps %xmm7, %xmm4
+; SSE-NEXT:    cmpeqps %xmm0, %xmm4
+; SSE-NEXT:    cmpunordps %xmm7, %xmm0
+; SSE-NEXT:    orps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm4
+; SSE-NEXT:    cmpeqps %xmm1, %xmm4
+; SSE-NEXT:    cmpunordps %xmm5, %xmm1
+; SSE-NEXT:    orps %xmm4, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm4
+; SSE-NEXT:    cmpeqps %xmm2, %xmm4
+; SSE-NEXT:    cmpunordps %xmm6, %xmm2
+; SSE-NEXT:    orps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm4
+; SSE-NEXT:    cmpeqps %xmm3, %xmm4
+; SSE-NEXT:    cmpunordps %xmm8, %xmm3
+; SSE-NEXT:    orps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_ueq_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeq_uqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_ueq_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeq_uqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ueq <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_one_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_one_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm7
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm8
+; SSE-NEXT:    movaps %xmm7, %xmm4
+; SSE-NEXT:    cmpneqps %xmm0, %xmm4
+; SSE-NEXT:    cmpordps %xmm7, %xmm0
+; SSE-NEXT:    andps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm4
+; SSE-NEXT:    cmpneqps %xmm1, %xmm4
+; SSE-NEXT:    cmpordps %xmm5, %xmm1
+; SSE-NEXT:    andps %xmm4, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm4
+; SSE-NEXT:    cmpneqps %xmm2, %xmm4
+; SSE-NEXT:    cmpordps %xmm6, %xmm2
+; SSE-NEXT:    andps %xmm4, %xmm2
+; SSE-NEXT:    movaps %xmm8, %xmm4
+; SSE-NEXT:    cmpneqps %xmm3, %xmm4
+; SSE-NEXT:    cmpordps %xmm8, %xmm3
+; SSE-NEXT:    andps %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_one_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneq_oqps (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqps 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_one_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneq_oqps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp one <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_lt_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_lt_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm4
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm7
+; SSE-NEXT:    cmpltps %xmm0, %xmm4
+; SSE-NEXT:    cmpltps %xmm1, %xmm5
+; SSE-NEXT:    cmpltps %xmm2, %xmm6
+; SSE-NEXT:    cmpltps %xmm3, %xmm7
+; SSE-NEXT:    movaps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm2
+; SSE-NEXT:    movaps %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_lt_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm2
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpltps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpltps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_lt_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgtps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp olt <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <16 x i32> @commute_cmpps_le_zmm(<16 x float>* %a0, <16 x float> %a1) {
+; SSE-LABEL: commute_cmpps_le_zmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movaps (%rdi), %xmm4
+; SSE-NEXT:    movaps 16(%rdi), %xmm5
+; SSE-NEXT:    movaps 32(%rdi), %xmm6
+; SSE-NEXT:    movaps 48(%rdi), %xmm7
+; SSE-NEXT:    cmpleps %xmm0, %xmm4
+; SSE-NEXT:    cmpleps %xmm1, %xmm5
+; SSE-NEXT:    cmpleps %xmm2, %xmm6
+; SSE-NEXT:    cmpleps %xmm3, %xmm7
+; SSE-NEXT:    movaps %xmm4, %xmm0
+; SSE-NEXT:    movaps %xmm5, %xmm1
+; SSE-NEXT:    movaps %xmm6, %xmm2
+; SSE-NEXT:    movaps %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmpps_le_zmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovaps (%rdi), %ymm2
+; AVX-NEXT:    vmovaps 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpleps %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpleps %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmpps_le_zmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgeps (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2d %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <16 x float>, <16 x float>* %a0
+  %2 = fcmp ole <16 x float> %1, %a1
+  %3 = sext <16 x i1> %2 to <16 x i32>
+  ret <16 x i32> %3
+}
+
+define <8 x i64> @commute_cmppd_eq_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_eq_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpeqpd (%rdi), %xmm0
+; SSE-NEXT:    cmpeqpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpeqpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpeqpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_eq_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_eq_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp oeq <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ne_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ne_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpneqpd (%rdi), %xmm0
+; SSE-NEXT:    cmpneqpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpneqpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpneqpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ne_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ne_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp une <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ord_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ord_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpordpd (%rdi), %xmm0
+; SSE-NEXT:    cmpordpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpordpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpordpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ord_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpordpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpordpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ord_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpordpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ord <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_uno_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_uno_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    cmpunordpd (%rdi), %xmm0
+; SSE-NEXT:    cmpunordpd 16(%rdi), %xmm1
+; SSE-NEXT:    cmpunordpd 32(%rdi), %xmm2
+; SSE-NEXT:    cmpunordpd 48(%rdi), %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_uno_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpunordpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpunordpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_uno_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpunordpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp uno <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_ueq_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_ueq_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm7
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm8
+; SSE-NEXT:    movapd %xmm7, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm0, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm7, %xmm0
+; SSE-NEXT:    orpd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm1, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm5, %xmm1
+; SSE-NEXT:    orpd %xmm4, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm2, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm6, %xmm2
+; SSE-NEXT:    orpd %xmm4, %xmm2
+; SSE-NEXT:    movapd %xmm8, %xmm4
+; SSE-NEXT:    cmpeqpd %xmm3, %xmm4
+; SSE-NEXT:    cmpunordpd %xmm8, %xmm3
+; SSE-NEXT:    orpd %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_ueq_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpeq_uqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpeq_uqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_ueq_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpeq_uqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ueq <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_one_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_one_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm7
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm8
+; SSE-NEXT:    movapd %xmm7, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm0, %xmm4
+; SSE-NEXT:    cmpordpd %xmm7, %xmm0
+; SSE-NEXT:    andpd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm1, %xmm4
+; SSE-NEXT:    cmpordpd %xmm5, %xmm1
+; SSE-NEXT:    andpd %xmm4, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm2, %xmm4
+; SSE-NEXT:    cmpordpd %xmm6, %xmm2
+; SSE-NEXT:    andpd %xmm4, %xmm2
+; SSE-NEXT:    movapd %xmm8, %xmm4
+; SSE-NEXT:    cmpneqpd %xmm3, %xmm4
+; SSE-NEXT:    cmpordpd %xmm8, %xmm3
+; SSE-NEXT:    andpd %xmm4, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_one_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vcmpneq_oqpd (%rdi), %ymm0, %ymm0
+; AVX-NEXT:    vcmpneq_oqpd 32(%rdi), %ymm1, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_one_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpneq_oqpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp one <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_lt_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_lt_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm4
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm7
+; SSE-NEXT:    cmpltpd %xmm0, %xmm4
+; SSE-NEXT:    cmpltpd %xmm1, %xmm5
+; SSE-NEXT:    cmpltpd %xmm2, %xmm6
+; SSE-NEXT:    cmpltpd %xmm3, %xmm7
+; SSE-NEXT:    movapd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm2
+; SSE-NEXT:    movapd %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_lt_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %ymm2
+; AVX-NEXT:    vmovapd 32(%rdi), %ymm3
+; AVX-NEXT:    vcmpltpd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmpltpd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_lt_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgtpd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp olt <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
+
+define <8 x i64> @commute_cmppd_le_zmmm(<8 x double>* %a0, <8 x double> %a1) {
+; SSE-LABEL: commute_cmppd_le_zmmm:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movapd (%rdi), %xmm4
+; SSE-NEXT:    movapd 16(%rdi), %xmm5
+; SSE-NEXT:    movapd 32(%rdi), %xmm6
+; SSE-NEXT:    movapd 48(%rdi), %xmm7
+; SSE-NEXT:    cmplepd %xmm0, %xmm4
+; SSE-NEXT:    cmplepd %xmm1, %xmm5
+; SSE-NEXT:    cmplepd %xmm2, %xmm6
+; SSE-NEXT:    cmplepd %xmm3, %xmm7
+; SSE-NEXT:    movapd %xmm4, %xmm0
+; SSE-NEXT:    movapd %xmm5, %xmm1
+; SSE-NEXT:    movapd %xmm6, %xmm2
+; SSE-NEXT:    movapd %xmm7, %xmm3
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: commute_cmppd_le_zmmm:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovapd (%rdi), %ymm2
+; AVX-NEXT:    vmovapd 32(%rdi), %ymm3
+; AVX-NEXT:    vcmplepd %ymm0, %ymm2, %ymm0
+; AVX-NEXT:    vcmplepd %ymm1, %ymm3, %ymm1
+; AVX-NEXT:    retq
+;
+; AVX512-LABEL: commute_cmppd_le_zmmm:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vcmpgepd (%rdi), %zmm0, %k0
+; AVX512-NEXT:    vpmovm2q %k0, %zmm0
+; AVX512-NEXT:    retq
+  %1 = load <8 x double>, <8 x double>* %a0
+  %2 = fcmp ole <8 x double> %1, %a1
+  %3 = sext <8 x i1> %2 to <8 x i64>
+  ret <8 x i64> %3
+}
diff --git a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir
index 583ff9d4bb197..97c609163973d 100755
--- a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir
+++ b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir
@@ -355,13 +355,13 @@ body: |
   $ymm0 = VMAXCPSZ256rm                        $ymm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $ymm0 = VMAXCPSYrr                  $ymm0, $ymm1
   $ymm0 = VMAXCPSZ256rr                        $ymm0, $ymm1                                   
-  ; CHECK: $ymm0 = VMAXCPDYrm                  $ymm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $ymm0 = VMAXPDYrm                   $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VMAXPDZ256rm                         $ymm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $ymm0 = VMAXCPDYrr                  $ymm0, $ymm1
+  ; CHECK: $ymm0 = VMAXPDYrr                   $ymm0, $ymm1
   $ymm0 = VMAXPDZ256rr                         $ymm0, $ymm1                                   
-  ; CHECK: $ymm0 = VMAXCPSYrm                  $ymm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $ymm0 = VMAXPSYrm                   $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VMAXPSZ256rm                         $ymm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $ymm0 = VMAXCPSYrr                  $ymm0, $ymm1
+  ; CHECK: $ymm0 = VMAXPSYrr                   $ymm0, $ymm1
   $ymm0 = VMAXPSZ256rr                         $ymm0, $ymm1                                   
   ; CHECK: $ymm0 = VMINCPDYrm                  $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VMINCPDZ256rm                        $ymm0, $rip, 1, $rax, 0, $noreg
@@ -371,13 +371,13 @@ body: |
   $ymm0 = VMINCPSZ256rm                        $ymm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $ymm0 = VMINCPSYrr                  $ymm0, $ymm1
   $ymm0 = VMINCPSZ256rr                        $ymm0, $ymm1                                   
-  ; CHECK: $ymm0 = VMINCPDYrm                  $ymm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $ymm0 = VMINPDYrm                   $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VMINPDZ256rm                         $ymm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $ymm0 = VMINCPDYrr                  $ymm0, $ymm1
+  ; CHECK: $ymm0 = VMINPDYrr                   $ymm0, $ymm1
   $ymm0 = VMINPDZ256rr                         $ymm0, $ymm1                                   
-  ; CHECK: $ymm0 = VMINCPSYrm                  $ymm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $ymm0 = VMINPSYrm                   $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VMINPSZ256rm                         $ymm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $ymm0 = VMINCPSYrr                  $ymm0, $ymm1
+  ; CHECK: $ymm0 = VMINPSYrr                   $ymm0, $ymm1
   $ymm0 = VMINPSZ256rr                         $ymm0, $ymm1                                   
   ; CHECK: $ymm0 = VXORPDYrm                   $ymm0, $rip, 1, $rax, 0, $noreg
   $ymm0 = VXORPDZ256rm                         $ymm0, $rip, 1, $rax, 0, $noreg
@@ -1083,13 +1083,13 @@ body: |
   $xmm0 = VMAXCPSZ128rm                        $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMAXCPSrr                   $xmm0, $xmm1
   $xmm0 = VMAXCPSZ128rr                        $xmm0, $xmm1                                        
-  ; CHECK: $xmm0 = VMAXCPDrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMAXPDrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXPDZ128rm                         $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMAXCPDrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMAXPDrr                    $xmm0, $xmm1
   $xmm0 = VMAXPDZ128rr                         $xmm0, $xmm1                                        
-  ; CHECK: $xmm0 = VMAXCPSrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMAXPSrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXPSZ128rm                         $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMAXCPSrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMAXPSrr                    $xmm0, $xmm1
   $xmm0 = VMAXPSZ128rr                         $xmm0, $xmm1                                        
   ; CHECK: $xmm0 = VMINCPDrm                   $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINCPDZ128rm                        $xmm0, $rip, 1, $rax, 0, $noreg
@@ -1099,13 +1099,13 @@ body: |
   $xmm0 = VMINCPSZ128rm                        $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMINCPSrr                   $xmm0, $xmm1
   $xmm0 = VMINCPSZ128rr                        $xmm0, $xmm1                                        
-  ; CHECK: $xmm0 = VMINCPDrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMINPDrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINPDZ128rm                         $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMINCPDrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMINPDrr                    $xmm0, $xmm1
   $xmm0 = VMINPDZ128rr                         $xmm0, $xmm1                                        
-  ; CHECK: $xmm0 = VMINCPSrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMINPSrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINPSZ128rm                         $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMINCPSrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMINPSrr                    $xmm0, $xmm1
   $xmm0 = VMINPSZ128rr                         $xmm0, $xmm1                                        
   ; CHECK: $xmm0 = VMULPDrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMULPDZ128rm                         $xmm0, $rip, 1, $rax, 0, $noreg
@@ -1850,19 +1850,19 @@ body: |
   $xmm0 = VMAXCSSZrm                           $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMAXCSSrr                   $xmm0, $xmm1
   $xmm0 = VMAXCSSZrr                           $xmm0, $xmm1                                            
-  ; CHECK: $xmm0 = VMAXCSDrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMAXSDrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXSDZrm                            $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMAXSDrm_Int                $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXSDZrm_Int                        $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMAXCSDrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMAXSDrr                    $xmm0, $xmm1
   $xmm0 = VMAXSDZrr                            $xmm0, $xmm1                                            
   ; CHECK: $xmm0 = VMAXSDrr_Int                $xmm0, $xmm1
   $xmm0 = VMAXSDZrr_Int                        $xmm0, $xmm1                                            
-  ; CHECK: $xmm0 = VMAXCSSrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMAXSSrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXSSZrm                            $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMAXSSrm_Int                $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMAXSSZrm_Int                        $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMAXCSSrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMAXSSrr                    $xmm0, $xmm1
   $xmm0 = VMAXSSZrr                            $xmm0, $xmm1                                            
   ; CHECK: $xmm0 = VMAXSSrr_Int                $xmm0, $xmm1
   $xmm0 = VMAXSSZrr_Int                        $xmm0, $xmm1                                            
@@ -1874,19 +1874,19 @@ body: |
   $xmm0 = VMINCSSZrm                           $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMINCSSrr                   $xmm0, $xmm1
   $xmm0 = VMINCSSZrr                           $xmm0, $xmm1                                            
-  ; CHECK: $xmm0 = VMINCSDrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMINSDrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINSDZrm                            $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMINSDrm_Int                $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINSDZrm_Int                        $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMINCSDrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMINSDrr                    $xmm0, $xmm1
   $xmm0 = VMINSDZrr                            $xmm0, $xmm1                                            
   ; CHECK: $xmm0 = VMINSDrr_Int                $xmm0, $xmm1
   $xmm0 = VMINSDZrr_Int                        $xmm0, $xmm1                                            
-  ; CHECK: $xmm0 = VMINCSSrm                   $xmm0, $rip, 1, $rax, 0, $noreg
+  ; CHECK: $xmm0 = VMINSSrm                    $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINSSZrm                            $xmm0, $rip, 1, $rax, 0, $noreg
   ; CHECK: $xmm0 = VMINSSrm_Int                $xmm0, $rip, 1, $rax, 0, $noreg
   $xmm0 = VMINSSZrm_Int                        $xmm0, $rip, 1, $rax, 0, $noreg
-  ; CHECK: $xmm0 = VMINCSSrr                   $xmm0, $xmm1
+  ; CHECK: $xmm0 = VMINSSrr                    $xmm0, $xmm1
   $xmm0 = VMINSSZrr                            $xmm0, $xmm1                                            
   ; CHECK: $xmm0 = VMINSSrr_Int                $xmm0, $xmm1
   $xmm0 = VMINSSZrr_Int                        $xmm0, $xmm1                                            
diff --git a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
index aab74bbdece53..354c695656a65 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select-pseudo-cmov.ll
@@ -27,7 +27,7 @@ define float @select_fcmp_one_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_one_f32:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vcmpneq_oqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp one float %a, %b
@@ -55,7 +55,7 @@ define double @select_fcmp_one_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_one_f64:
 ; AVX512:       ## %bb.0:
 ; AVX512-NEXT:    vcmpneq_oqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp one double %a, %b
@@ -87,7 +87,7 @@ define float @select_icmp_eq_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    sete %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -128,7 +128,7 @@ define float @select_icmp_ne_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setne %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -169,7 +169,7 @@ define float @select_icmp_ugt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    seta %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -210,7 +210,7 @@ define float @select_icmp_uge_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setae %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -251,7 +251,7 @@ define float @select_icmp_ult_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setb %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -292,7 +292,7 @@ define float @select_icmp_ule_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setbe %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -333,7 +333,7 @@ define float @select_icmp_sgt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setg %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -374,7 +374,7 @@ define float @select_icmp_sge_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setge %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -415,7 +415,7 @@ define float @select_icmp_slt_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setl %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
@@ -456,7 +456,7 @@ define float @select_icmp_sle_f32(i64 %a, i64 %b, float %c, float %d) {
 ; AVX512-ISEL-NEXT:    cmpq %rsi, %rdi
 ; AVX512-ISEL-NEXT:    setle %al
 ; AVX512-ISEL-NEXT:    kmovd %eax, %k1
-; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-ISEL-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-ISEL-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-ISEL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/fast-isel-select-sse.ll b/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
index e91b925a38e62..54ef4fdc114e3 100644
--- a/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-select-sse.ll
@@ -26,7 +26,7 @@ define float @select_fcmp_oeq_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_oeq_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oeq float %a, %b
@@ -52,7 +52,7 @@ define double @select_fcmp_oeq_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_oeq_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpeqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oeq double %a, %b
@@ -79,7 +79,7 @@ define float @select_fcmp_ogt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ogt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltss %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ogt float %a, %b
@@ -106,7 +106,7 @@ define double @select_fcmp_ogt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ogt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltsd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ogt double %a, %b
@@ -133,7 +133,7 @@ define float @select_fcmp_oge_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_oge_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpless %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oge float %a, %b
@@ -160,7 +160,7 @@ define double @select_fcmp_oge_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_oge_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplesd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp oge double %a, %b
@@ -186,7 +186,7 @@ define float @select_fcmp_olt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_olt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp olt float %a, %b
@@ -212,7 +212,7 @@ define double @select_fcmp_olt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_olt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp olt double %a, %b
@@ -238,7 +238,7 @@ define float @select_fcmp_ole_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ole_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpless %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ole float %a, %b
@@ -264,7 +264,7 @@ define double @select_fcmp_ole_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ole_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmplesd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ole double %a, %b
@@ -290,7 +290,7 @@ define float @select_fcmp_ord_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ord_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpordss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ord float %a, %b
@@ -316,7 +316,7 @@ define double @select_fcmp_ord_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ord_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpordsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ord double %a, %b
@@ -342,7 +342,7 @@ define float @select_fcmp_uno_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_uno_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpunordss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uno float %a, %b
@@ -368,7 +368,7 @@ define double @select_fcmp_uno_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_uno_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpunordsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uno double %a, %b
@@ -394,7 +394,7 @@ define float @select_fcmp_ugt_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ugt_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnless %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ugt float %a, %b
@@ -420,7 +420,7 @@ define double @select_fcmp_ugt_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ugt_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnlesd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ugt double %a, %b
@@ -446,7 +446,7 @@ define float @select_fcmp_uge_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_uge_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uge float %a, %b
@@ -472,7 +472,7 @@ define double @select_fcmp_uge_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_uge_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp uge double %a, %b
@@ -499,7 +499,7 @@ define float @select_fcmp_ult_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ult_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnless %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ult float %a, %b
@@ -526,7 +526,7 @@ define double @select_fcmp_ult_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ult_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnlesd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ult double %a, %b
@@ -553,7 +553,7 @@ define float @select_fcmp_ule_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_ule_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltss %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ule float %a, %b
@@ -580,7 +580,7 @@ define double @select_fcmp_ule_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_ule_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpnltsd %xmm0, %xmm1, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp ule double %a, %b
@@ -606,7 +606,7 @@ define float @select_fcmp_une_f32(float %a, float %b, float %c, float %d) {
 ; AVX512-LABEL: select_fcmp_une_f32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpneqss %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp une float %a, %b
@@ -632,7 +632,7 @@ define double @select_fcmp_une_f64(double %a, double %b, double %c, double %d) {
 ; AVX512-LABEL: select_fcmp_une_f64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vcmpneqsd %xmm1, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm2, %xmm0, %xmm3 {%k1}
+; AVX512-NEXT:    vmovsd %xmm2, %xmm3, %xmm3 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm3, %xmm0
 ; AVX512-NEXT:    retq
   %1 = fcmp une double %a, %b
diff --git a/llvm/test/CodeGen/X86/fmaxnum.ll b/llvm/test/CodeGen/X86/fmaxnum.ll
index e308412f7cada..23de2667a3ece 100644
--- a/llvm/test/CodeGen/X86/fmaxnum.ll
+++ b/llvm/test/CodeGen/X86/fmaxnum.ll
@@ -44,7 +44,7 @@ define float @test_fmaxf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @fmaxf(float %x, float %y) readnone
@@ -85,7 +85,7 @@ define double @test_fmax(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @fmax(double %x, double %y) readnone
@@ -133,7 +133,7 @@ define float @test_intrinsic_fmaxf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @llvm.maxnum.f32(float %x, float %y) readnone
@@ -164,7 +164,7 @@ define double @test_intrinsic_fmax(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vmaxsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @llvm.maxnum.f64(double %x, double %y) readnone
diff --git a/llvm/test/CodeGen/X86/fminnum.ll b/llvm/test/CodeGen/X86/fminnum.ll
index 33accf2e49c1f..1667a5cd0a75f 100644
--- a/llvm/test/CodeGen/X86/fminnum.ll
+++ b/llvm/test/CodeGen/X86/fminnum.ll
@@ -44,7 +44,7 @@ define float @test_fminf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @fminf(float %x, float %y) readnone
@@ -85,7 +85,7 @@ define double @test_fmin(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @fmin(double %x, double %y) readnone
@@ -133,7 +133,7 @@ define float @test_intrinsic_fminf(float %x, float %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminss %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordss %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovss %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call float @llvm.minnum.f32(float %x, float %y) readnone
@@ -164,7 +164,7 @@ define double @test_intrinsic_fmin(double %x, double %y) {
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vminsd %xmm0, %xmm1, %xmm2
 ; AVX512-NEXT:    vcmpunordsd %xmm0, %xmm0, %k1
-; AVX512-NEXT:    vmovsd %xmm1, %xmm0, %xmm2 {%k1}
+; AVX512-NEXT:    vmovsd %xmm1, %xmm2, %xmm2 {%k1}
 ; AVX512-NEXT:    vmovapd %xmm2, %xmm0
 ; AVX512-NEXT:    retq
   %z = call double @llvm.minnum.f64(double %x, double %y) readnone
diff --git a/llvm/test/CodeGen/X86/gpr-to-mask.ll b/llvm/test/CodeGen/X86/gpr-to-mask.ll
index ff237799783f1..8b6a607402476 100644
--- a/llvm/test/CodeGen/X86/gpr-to-mask.ll
+++ b/llvm/test/CodeGen/X86/gpr-to-mask.ll
@@ -13,7 +13,7 @@ define void @test_fcmp_storefloat(i1 %cond, float* %fptr, float %f1, float %f2,
 ; X86-64-NEXT:  .LBB0_2: # %else
 ; X86-64-NEXT:    vcmpeqss %xmm5, %xmm4, %k1
 ; X86-64-NEXT:  .LBB0_3: # %exit
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
@@ -112,7 +112,7 @@ define void @test_load_add(i1 %cond, float* %fptr, i1* %iptr1, i1* %iptr2, float
 ; X86-64-NEXT:    movb (%rcx), %al
 ; X86-64-NEXT:  .LBB2_3: # %exit
 ; X86-64-NEXT:    kmovd %eax, %k1
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
@@ -167,7 +167,7 @@ define void @test_load_i1(i1 %cond, float* %fptr, i1* %iptr1, i1* %iptr2, float
 ; X86-64-NEXT:  .LBB3_2: # %else
 ; X86-64-NEXT:    kmovb (%rcx), %k1
 ; X86-64-NEXT:  .LBB3_3: # %exit
-; X86-64-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; X86-64-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; X86-64-NEXT:    vmovss %xmm1, (%rsi)
 ; X86-64-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/haddsub-broadcast.ll b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
new file mode 100644
index 0000000000000..a37cf80ff79e4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/haddsub-broadcast.ll
@@ -0,0 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-- -mattr=avx2  | FileCheck %s
+
+; The broadcast node takes a vector operand as input and changes its length.
+
+define <4 x double> @PR43402(i64 %x) {
+; CHECK-LABEL: PR43402:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovsd {{.*#+}} xmm0 = mem[0],zero
+; CHECK-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1]
+; CHECK-NEXT:    vsubpd {{\.LCPI.*}}, %xmm0, %xmm0
+; CHECK-NEXT:    vhaddpd %xmm0, %xmm0, %xmm0
+; CHECK-NEXT:    vbroadcastsd %xmm0, %ymm0
+; CHECK-NEXT:    retl
+  %conv = uitofp i64 %x to double
+  %t2 = insertelement <4 x double> undef, double %conv, i32 0
+  %t3 = shufflevector <4 x double> %t2, <4 x double> undef, <4 x i32> zeroinitializer
+  ret <4 x double> %t3
+}
+
diff --git a/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll b/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll
new file mode 100644
index 0000000000000..b43e2e5a5fbd1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/isel-blendi-gettargetconstant.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=sse4.2 | FileCheck %s
+
+define void @csrot_(float* %0) {
+; CHECK-LABEL: csrot_:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; CHECK-NEXT:    movss {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; CHECK-NEXT:    xorps %xmm0, %xmm1
+; CHECK-NEXT:    blendps {{.*#+}} xmm1 = xmm1[0],mem[1,2,3]
+; CHECK-NEXT:    movlps %xmm1, (%rax)
+; CHECK-NEXT:    retq
+1:
+  %2 = load float, float* %0, align 4
+  %3 = fsub float -0.000000e+00, %2
+  %4 = insertelement <2 x float> <float undef, float -0.000000e+00>, float %3, i32 0
+  store <2 x float> %4, <2 x float>* undef, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index a899d1e71e4a8..9bea63f3055ff 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -1626,75 +1626,62 @@ declare <3 x i32> @llvm.masked.gather.v3i32.v3p0i32(<3 x i32*>, i32, <3 x i1>, <
 define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x i32> %src0) {
 ; KNL_64-LABEL: test30:
 ; KNL_64:       # %bb.0:
-; KNL_64-NEXT:    vpslld $31, %xmm2, %xmm2
-; KNL_64-NEXT:    vptestmd %zmm2, %zmm2, %k0
-; KNL_64-NEXT:    kshiftrw $1, %k0, %k1
-; KNL_64-NEXT:    kshiftrw $2, %k0, %k2
-; KNL_64-NEXT:    kmovw %k0, %eax
-; KNL_64-NEXT:    andb $1, %al
-; KNL_64-NEXT:    kmovw %k1, %ecx
-; KNL_64-NEXT:    andb $1, %cl
-; KNL_64-NEXT:    addb %cl, %cl
-; KNL_64-NEXT:    orb %al, %cl
-; KNL_64-NEXT:    kmovw %k2, %eax
-; KNL_64-NEXT:    andb $1, %al
-; KNL_64-NEXT:    shlb $2, %al
-; KNL_64-NEXT:    orb %cl, %al
+; KNL_64-NEXT:    andb $1, %dil
+; KNL_64-NEXT:    andb $1, %sil
+; KNL_64-NEXT:    addb %sil, %sil
+; KNL_64-NEXT:    orb %dil, %sil
+; KNL_64-NEXT:    andb $1, %dl
+; KNL_64-NEXT:    shlb $2, %dl
+; KNL_64-NEXT:    orb %sil, %dl
 ; KNL_64-NEXT:    vpmovsxdq %xmm1, %ymm1
 ; KNL_64-NEXT:    vpsllq $2, %ymm1, %ymm1
 ; KNL_64-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; KNL_64-NEXT:    testb $1, %al
+; KNL_64-NEXT:    testb $1, %dl
 ; KNL_64-NEXT:    jne .LBB31_1
 ; KNL_64-NEXT:  # %bb.2: # %else
-; KNL_64-NEXT:    testb $2, %al
+; KNL_64-NEXT:    testb $2, %dl
 ; KNL_64-NEXT:    jne .LBB31_3
 ; KNL_64-NEXT:  .LBB31_4: # %else2
-; KNL_64-NEXT:    testb $4, %al
+; KNL_64-NEXT:    testb $4, %dl
 ; KNL_64-NEXT:    jne .LBB31_5
 ; KNL_64-NEXT:  .LBB31_6: # %else5
-; KNL_64-NEXT:    vmovdqa %xmm3, %xmm0
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ; KNL_64-NEXT:  .LBB31_1: # %cond.load
-; KNL_64-NEXT:    vmovq %xmm0, %rcx
-; KNL_64-NEXT:    vpinsrd $0, (%rcx), %xmm3, %xmm3
-; KNL_64-NEXT:    testb $2, %al
+; KNL_64-NEXT:    vmovq %xmm0, %rax
+; KNL_64-NEXT:    vpinsrd $0, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    testb $2, %dl
 ; KNL_64-NEXT:    je .LBB31_4
 ; KNL_64-NEXT:  .LBB31_3: # %cond.load1
-; KNL_64-NEXT:    vpextrq $1, %xmm0, %rcx
-; KNL_64-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
-; KNL_64-NEXT:    testb $4, %al
+; KNL_64-NEXT:    vpextrq $1, %xmm0, %rax
+; KNL_64-NEXT:    vpinsrd $1, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    testb $4, %dl
 ; KNL_64-NEXT:    je .LBB31_6
 ; KNL_64-NEXT:  .LBB31_5: # %cond.load4
 ; KNL_64-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; KNL_64-NEXT:    vmovq %xmm0, %rax
-; KNL_64-NEXT:    vpinsrd $2, (%rax), %xmm3, %xmm3
-; KNL_64-NEXT:    vmovdqa %xmm3, %xmm0
+; KNL_64-NEXT:    vpinsrd $2, (%rax), %xmm2, %xmm2
+; KNL_64-NEXT:    vmovdqa %xmm2, %xmm0
 ; KNL_64-NEXT:    vzeroupper
 ; KNL_64-NEXT:    retq
 ;
 ; KNL_32-LABEL: test30:
 ; KNL_32:       # %bb.0:
-; KNL_32-NEXT:    subl $12, %esp
-; KNL_32-NEXT:    .cfi_def_cfa_offset 16
-; KNL_32-NEXT:    vmovdqa %xmm0, %xmm3
-; KNL_32-NEXT:    vpslld $31, %xmm2, %xmm0
-; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k0
-; KNL_32-NEXT:    kshiftrw $1, %k0, %k1
-; KNL_32-NEXT:    kshiftrw $2, %k0, %k2
-; KNL_32-NEXT:    kmovw %k0, %eax
+; KNL_32-NEXT:    pushl %eax
+; KNL_32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; KNL_32-NEXT:    andb $1, %al
-; KNL_32-NEXT:    kmovw %k1, %ecx
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; KNL_32-NEXT:    andb $1, %cl
 ; KNL_32-NEXT:    addb %cl, %cl
 ; KNL_32-NEXT:    orb %al, %cl
-; KNL_32-NEXT:    kmovw %k2, %eax
+; KNL_32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; KNL_32-NEXT:    andb $1, %al
 ; KNL_32-NEXT:    shlb $2, %al
 ; KNL_32-NEXT:    orb %cl, %al
-; KNL_32-NEXT:    vmovdqa {{[0-9]+}}(%esp), %xmm0
 ; KNL_32-NEXT:    vpslld $2, %xmm1, %xmm1
-; KNL_32-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; KNL_32-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; KNL_32-NEXT:    testb $1, %al
 ; KNL_32-NEXT:    jne .LBB31_1
 ; KNL_32-NEXT:  # %bb.2: # %else
@@ -1702,102 +1689,86 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; KNL_32-NEXT:    jne .LBB31_3
 ; KNL_32-NEXT:  .LBB31_4: # %else2
 ; KNL_32-NEXT:    testb $4, %al
-; KNL_32-NEXT:    jne .LBB31_5
+; KNL_32-NEXT:    je .LBB31_6
+; KNL_32-NEXT:  .LBB31_5: # %cond.load4
+; KNL_32-NEXT:    vpextrd $2, %xmm0, %eax
+; KNL_32-NEXT:    vpinsrd $2, (%eax), %xmm2, %xmm2
 ; KNL_32-NEXT:  .LBB31_6: # %else5
-; KNL_32-NEXT:    addl $12, %esp
+; KNL_32-NEXT:    vmovdqa %xmm2, %xmm0
+; KNL_32-NEXT:    popl %eax
 ; KNL_32-NEXT:    .cfi_def_cfa_offset 4
-; KNL_32-NEXT:    vzeroupper
 ; KNL_32-NEXT:    retl
 ; KNL_32-NEXT:  .LBB31_1: # %cond.load
-; KNL_32-NEXT:    .cfi_def_cfa_offset 16
-; KNL_32-NEXT:    vmovd %xmm1, %ecx
-; KNL_32-NEXT:    vpinsrd $0, (%ecx), %xmm0, %xmm0
+; KNL_32-NEXT:    .cfi_def_cfa_offset 8
+; KNL_32-NEXT:    vmovd %xmm0, %ecx
+; KNL_32-NEXT:    vpinsrd $0, (%ecx), %xmm2, %xmm2
 ; KNL_32-NEXT:    testb $2, %al
 ; KNL_32-NEXT:    je .LBB31_4
 ; KNL_32-NEXT:  .LBB31_3: # %cond.load1
-; KNL_32-NEXT:    vpextrd $1, %xmm1, %ecx
-; KNL_32-NEXT:    vpinsrd $1, (%ecx), %xmm0, %xmm0
+; KNL_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; KNL_32-NEXT:    vpinsrd $1, (%ecx), %xmm2, %xmm2
 ; KNL_32-NEXT:    testb $4, %al
-; KNL_32-NEXT:    je .LBB31_6
-; KNL_32-NEXT:  .LBB31_5: # %cond.load4
-; KNL_32-NEXT:    vpextrd $2, %xmm1, %eax
-; KNL_32-NEXT:    vpinsrd $2, (%eax), %xmm0, %xmm0
-; KNL_32-NEXT:    addl $12, %esp
-; KNL_32-NEXT:    .cfi_def_cfa_offset 4
-; KNL_32-NEXT:    vzeroupper
-; KNL_32-NEXT:    retl
+; KNL_32-NEXT:    jne .LBB31_5
+; KNL_32-NEXT:    jmp .LBB31_6
 ;
 ; SKX-LABEL: test30:
 ; SKX:       # %bb.0:
-; SKX-NEXT:    vpslld $31, %xmm2, %xmm2
-; SKX-NEXT:    vpmovd2m %xmm2, %k0
-; SKX-NEXT:    kshiftrb $1, %k0, %k1
-; SKX-NEXT:    kshiftrb $2, %k0, %k2
-; SKX-NEXT:    kmovw %k0, %eax
-; SKX-NEXT:    andb $1, %al
-; SKX-NEXT:    kmovw %k1, %ecx
-; SKX-NEXT:    andb $1, %cl
-; SKX-NEXT:    addb %cl, %cl
-; SKX-NEXT:    orb %al, %cl
-; SKX-NEXT:    kmovw %k2, %eax
-; SKX-NEXT:    andb $1, %al
-; SKX-NEXT:    shlb $2, %al
-; SKX-NEXT:    orb %cl, %al
+; SKX-NEXT:    andb $1, %dil
+; SKX-NEXT:    andb $1, %sil
+; SKX-NEXT:    addb %sil, %sil
+; SKX-NEXT:    orb %dil, %sil
+; SKX-NEXT:    andb $1, %dl
+; SKX-NEXT:    shlb $2, %dl
+; SKX-NEXT:    orb %sil, %dl
 ; SKX-NEXT:    vpmovsxdq %xmm1, %ymm1
 ; SKX-NEXT:    vpsllq $2, %ymm1, %ymm1
 ; SKX-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; SKX-NEXT:    testb $1, %al
+; SKX-NEXT:    testb $1, %dl
 ; SKX-NEXT:    jne .LBB31_1
 ; SKX-NEXT:  # %bb.2: # %else
-; SKX-NEXT:    testb $2, %al
+; SKX-NEXT:    testb $2, %dl
 ; SKX-NEXT:    jne .LBB31_3
 ; SKX-NEXT:  .LBB31_4: # %else2
-; SKX-NEXT:    testb $4, %al
+; SKX-NEXT:    testb $4, %dl
 ; SKX-NEXT:    jne .LBB31_5
 ; SKX-NEXT:  .LBB31_6: # %else5
-; SKX-NEXT:    vmovdqa %xmm3, %xmm0
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ; SKX-NEXT:  .LBB31_1: # %cond.load
-; SKX-NEXT:    vmovq %xmm0, %rcx
-; SKX-NEXT:    vpinsrd $0, (%rcx), %xmm3, %xmm3
-; SKX-NEXT:    testb $2, %al
+; SKX-NEXT:    vmovq %xmm0, %rax
+; SKX-NEXT:    vpinsrd $0, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    testb $2, %dl
 ; SKX-NEXT:    je .LBB31_4
 ; SKX-NEXT:  .LBB31_3: # %cond.load1
-; SKX-NEXT:    vpextrq $1, %xmm0, %rcx
-; SKX-NEXT:    vpinsrd $1, (%rcx), %xmm3, %xmm3
-; SKX-NEXT:    testb $4, %al
+; SKX-NEXT:    vpextrq $1, %xmm0, %rax
+; SKX-NEXT:    vpinsrd $1, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    testb $4, %dl
 ; SKX-NEXT:    je .LBB31_6
 ; SKX-NEXT:  .LBB31_5: # %cond.load4
 ; SKX-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; SKX-NEXT:    vmovq %xmm0, %rax
-; SKX-NEXT:    vpinsrd $2, (%rax), %xmm3, %xmm3
-; SKX-NEXT:    vmovdqa %xmm3, %xmm0
+; SKX-NEXT:    vpinsrd $2, (%rax), %xmm2, %xmm2
+; SKX-NEXT:    vmovdqa %xmm2, %xmm0
 ; SKX-NEXT:    vzeroupper
 ; SKX-NEXT:    retq
 ;
 ; SKX_32-LABEL: test30:
 ; SKX_32:       # %bb.0:
-; SKX_32-NEXT:    subl $12, %esp
-; SKX_32-NEXT:    .cfi_def_cfa_offset 16
-; SKX_32-NEXT:    vmovdqa %xmm0, %xmm3
-; SKX_32-NEXT:    vpslld $31, %xmm2, %xmm0
-; SKX_32-NEXT:    vpmovd2m %xmm0, %k0
-; SKX_32-NEXT:    kshiftrb $1, %k0, %k1
-; SKX_32-NEXT:    kshiftrb $2, %k0, %k2
-; SKX_32-NEXT:    kmovw %k0, %eax
+; SKX_32-NEXT:    pushl %eax
+; SKX_32-NEXT:    .cfi_def_cfa_offset 8
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; SKX_32-NEXT:    andb $1, %al
-; SKX_32-NEXT:    kmovw %k1, %ecx
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %cl
 ; SKX_32-NEXT:    andb $1, %cl
 ; SKX_32-NEXT:    addb %cl, %cl
 ; SKX_32-NEXT:    orb %al, %cl
-; SKX_32-NEXT:    kmovw %k2, %eax
+; SKX_32-NEXT:    movb {{[0-9]+}}(%esp), %al
 ; SKX_32-NEXT:    andb $1, %al
 ; SKX_32-NEXT:    shlb $2, %al
 ; SKX_32-NEXT:    orb %cl, %al
-; SKX_32-NEXT:    vmovdqa {{[0-9]+}}(%esp), %xmm0
 ; SKX_32-NEXT:    vpslld $2, %xmm1, %xmm1
-; SKX_32-NEXT:    vpaddd %xmm1, %xmm3, %xmm1
+; SKX_32-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
 ; SKX_32-NEXT:    testb $1, %al
 ; SKX_32-NEXT:    jne .LBB31_1
 ; SKX_32-NEXT:  # %bb.2: # %else
@@ -1805,28 +1776,27 @@ define <3 x i32> @test30(<3 x i32*> %base, <3 x i32> %ind, <3 x i1> %mask, <3 x
 ; SKX_32-NEXT:    jne .LBB31_3
 ; SKX_32-NEXT:  .LBB31_4: # %else2
 ; SKX_32-NEXT:    testb $4, %al
-; SKX_32-NEXT:    jne .LBB31_5
+; SKX_32-NEXT:    je .LBB31_6
+; SKX_32-NEXT:  .LBB31_5: # %cond.load4
+; SKX_32-NEXT:    vpextrd $2, %xmm0, %eax
+; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm2, %xmm2
 ; SKX_32-NEXT:  .LBB31_6: # %else5
-; SKX_32-NEXT:    addl $12, %esp
+; SKX_32-NEXT:    vmovdqa %xmm2, %xmm0
+; SKX_32-NEXT:    popl %eax
 ; SKX_32-NEXT:    .cfi_def_cfa_offset 4
 ; SKX_32-NEXT:    retl
 ; SKX_32-NEXT:  .LBB31_1: # %cond.load
-; SKX_32-NEXT:    .cfi_def_cfa_offset 16
-; SKX_32-NEXT:    vmovd %xmm1, %ecx
-; SKX_32-NEXT:    vpinsrd $0, (%ecx), %xmm0, %xmm0
+; SKX_32-NEXT:    .cfi_def_cfa_offset 8
+; SKX_32-NEXT:    vmovd %xmm0, %ecx
+; SKX_32-NEXT:    vpinsrd $0, (%ecx), %xmm2, %xmm2
 ; SKX_32-NEXT:    testb $2, %al
 ; SKX_32-NEXT:    je .LBB31_4
 ; SKX_32-NEXT:  .LBB31_3: # %cond.load1
-; SKX_32-NEXT:    vpextrd $1, %xmm1, %ecx
-; SKX_32-NEXT:    vpinsrd $1, (%ecx), %xmm0, %xmm0
+; SKX_32-NEXT:    vpextrd $1, %xmm0, %ecx
+; SKX_32-NEXT:    vpinsrd $1, (%ecx), %xmm2, %xmm2
 ; SKX_32-NEXT:    testb $4, %al
-; SKX_32-NEXT:    je .LBB31_6
-; SKX_32-NEXT:  .LBB31_5: # %cond.load4
-; SKX_32-NEXT:    vpextrd $2, %xmm1, %eax
-; SKX_32-NEXT:    vpinsrd $2, (%eax), %xmm0, %xmm0
-; SKX_32-NEXT:    addl $12, %esp
-; SKX_32-NEXT:    .cfi_def_cfa_offset 4
-; SKX_32-NEXT:    retl
+; SKX_32-NEXT:    jne .LBB31_5
+; SKX_32-NEXT:    jmp .LBB31_6
 
   %sext_ind = sext <3 x i32> %ind to <3 x i64>
   %gep.random = getelementptr i32, <3 x i32*> %base, <3 x i64> %sext_ind
@@ -3026,3 +2996,236 @@ define void @zero_mask(<2 x double>%a1, <2 x double*> %ptr) {
   call void @llvm.masked.scatter.v2f64.v2p0f64(<2 x double> %a1, <2 x double*> %ptr, i32 4, <2 x i1> zeroinitializer)
   ret void
 }
+
+define <2 x i64> @gather_2i64_constant_indices(i64* %ptr, <2 x i1> %mask) {
+; KNL_64-LABEL: gather_2i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_64-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_64-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,4294967294,u,u,u,u,u,u>
+; KNL_64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_64-NEXT:    vpgatherdq (%rdi,%ymm1,8), %zmm0 {%k1}
+; KNL_64-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: gather_2i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_32-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,4294967294,u,u,u,u,u,u>
+; KNL_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_32-NEXT:    vpgatherdq (%eax,%ymm1,8), %zmm0 {%k1}
+; KNL_32-NEXT:    # kill: def $xmm0 killed $xmm0 killed $zmm0
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: gather_2i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4294967294,u,u>
+; SKX_SMALL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: gather_2i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm1
+; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpgatherdq (%rdi,%xmm1,8), %xmm0 {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: gather_2i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_32-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,4294967294,u,u>
+; SKX_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_32-NEXT:    vpgatherdq (%eax,%xmm1,8), %xmm0 {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i64, i64* %ptr, <2 x i64> <i64 0, i64 -2>
+  %res = tail call <2 x i64> @llvm.masked.gather.v2i64.v2p0i64(<2 x i64*> %gep, i32 8, <2 x i1> %mask, <2 x i64> zeroinitializer) #1
+  ret <2 x i64> %res
+}
+
+define <16 x i32> @gather_16i64_constant_indices(i32* %ptr, <16 x i1> %mask) {
+; KNL_64-LABEL: gather_16i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_64-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_64-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_64-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_64-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: gather_16i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; KNL_32-NEXT:    vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: gather_16i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_SMALL-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_SMALL-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_SMALL-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: gather_16i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm1
+; SKX_LARGE-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: gather_16i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_32-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_32-NEXT:    vpxor %xmm0, %xmm0, %xmm0
+; SKX_32-NEXT:    vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <16 x i64> <i64 0, i64 -2, i64 1, i64 -8, i64 10, i64 20, i64 50, i64 65536, i64 16777215, i64 2147483647, i64 100, i64 -2000, i64 -2147483648, i64 76897723, i64 7, i64 -67897687>
+  %res = tail call <16 x i32> @llvm.masked.gather.v16i32.v16p0i32(<16 x i32*> %gep, i32 4, <16 x i1> %mask, <16 x i32> zeroinitializer) #1
+  ret <16 x i32> %res
+}
+
+define void @scatter_2i64_constant_indices(i32* %ptr, <2 x i1> %mask, <2 x i32> %src0)  {
+; KNL_64-LABEL: scatter_2i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_64-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_64-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_64-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_64-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,4294967294,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; KNL_64-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: scatter_2i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
+; KNL_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; KNL_32-NEXT:    vptestmq %zmm0, %zmm0, %k0
+; KNL_32-NEXT:    kshiftlw $14, %k0, %k0
+; KNL_32-NEXT:    kshiftrw $14, %k0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,4294967294,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; KNL_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: scatter_2i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_SMALL-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4294967294,u,u>
+; SKX_SMALL-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: scatter_2i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_LARGE-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa (%rax), %xmm0
+; SKX_LARGE-NEXT:    vpscatterdd %xmm1, (%rdi,%xmm0,4) {%k1}
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: scatter_2i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpsllq $63, %xmm0, %xmm0
+; SKX_32-NEXT:    vpmovq2m %xmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,4294967294,u,u>
+; SKX_32-NEXT:    vpscatterdd %xmm1, (%eax,%xmm0,4) {%k1}
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <2 x i64> <i64 0, i64 -2>
+  call void @llvm.masked.scatter.v2i32.v2p0i32(<2 x i32> %src0, <2 x i32*> %gep, i32 4, <2 x i1> %mask)
+  ret void
+}
+
+define void @scatter_16i64_constant_indices(i32* %ptr, <16 x i1> %mask, <16 x i32> %src0)  {
+; KNL_64-LABEL: scatter_16i64_constant_indices:
+; KNL_64:       # %bb.0:
+; KNL_64-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_64-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_64-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_64-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_64-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; KNL_64-NEXT:    vzeroupper
+; KNL_64-NEXT:    retq
+;
+; KNL_32-LABEL: scatter_16i64_constant_indices:
+; KNL_32:       # %bb.0:
+; KNL_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; KNL_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; KNL_32-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; KNL_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; KNL_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; KNL_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; KNL_32-NEXT:    vzeroupper
+; KNL_32-NEXT:    retl
+;
+; SKX_SMALL-LABEL: scatter_16i64_constant_indices:
+; SKX_SMALL:       # %bb.0:
+; SKX_SMALL-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_SMALL-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_SMALL-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_SMALL-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_SMALL-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; SKX_SMALL-NEXT:    vzeroupper
+; SKX_SMALL-NEXT:    retq
+;
+; SKX_LARGE-LABEL: scatter_16i64_constant_indices:
+; SKX_LARGE:       # %bb.0:
+; SKX_LARGE-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_LARGE-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_LARGE-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_LARGE-NEXT:    movabsq ${{\.LCPI.*}}, %rax
+; SKX_LARGE-NEXT:    vmovdqa64 (%rax), %zmm0
+; SKX_LARGE-NEXT:    vpscatterdd %zmm1, (%rdi,%zmm0,4) {%k1}
+; SKX_LARGE-NEXT:    vzeroupper
+; SKX_LARGE-NEXT:    retq
+;
+; SKX_32-LABEL: scatter_16i64_constant_indices:
+; SKX_32:       # %bb.0:
+; SKX_32-NEXT:    vpmovsxbd %xmm0, %zmm0
+; SKX_32-NEXT:    vpslld $31, %zmm0, %zmm0
+; SKX_32-NEXT:    vpmovd2m %zmm0, %k1
+; SKX_32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; SKX_32-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
+; SKX_32-NEXT:    vpscatterdd %zmm1, (%eax,%zmm0,4) {%k1}
+; SKX_32-NEXT:    vzeroupper
+; SKX_32-NEXT:    retl
+  %gep = getelementptr i32, i32* %ptr, <16 x i64> <i64 0, i64 -2, i64 1, i64 -8, i64 10, i64 20, i64 50, i64 65536, i64 16777215, i64 2147483647, i64 100, i64 -2000, i64 -2147483648, i64 76897723, i64 7, i64 -67897687>
+  call void @llvm.masked.scatter.v16i32.v16p0i32(<16 x i32> %src0, <16 x i32*> %gep, i32 4, <16 x i1> %mask)
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/masked_store.ll b/llvm/test/CodeGen/X86/masked_store.ll
index 1a6be01d804b2..164826d871559 100644
--- a/llvm/test/CodeGen/X86/masked_store.ll
+++ b/llvm/test/CodeGen/X86/masked_store.ll
@@ -4913,12 +4913,30 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ; AVX512F-LABEL: widen_masked_store:
 ; AVX512F:       ## %bb.0:
 ; AVX512F-NEXT:    ## kill: def $xmm0 killed $xmm0 def $zmm0
-; AVX512F-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k1
-; AVX512F-NEXT:    vpternlogd $255, %zmm1, %zmm1, %zmm1 {%k1} {z}
-; AVX512F-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    andl $1, %esi
+; AVX512F-NEXT:    kmovw %esi, %k0
+; AVX512F-NEXT:    kxorw %k0, %k0, %k1
+; AVX512F-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512F-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512F-NEXT:    korw %k0, %k1, %k0
+; AVX512F-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512F-NEXT:    kshiftlw $2, %k1, %k1
+; AVX512F-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512F-NEXT:    kmovw %edx, %k2
+; AVX512F-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512F-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512F-NEXT:    korw %k2, %k1, %k1
+; AVX512F-NEXT:    korw %k1, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512F-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512F-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512F-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512F-NEXT:    korw %k1, %k0, %k0
+; AVX512F-NEXT:    kmovw %ecx, %k1
+; AVX512F-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512F-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512F-NEXT:    korw %k0, %k1, %k0
 ; AVX512F-NEXT:    kshiftlw $12, %k0, %k0
 ; AVX512F-NEXT:    kshiftrw $12, %k0, %k1
 ; AVX512F-NEXT:    vmovdqu32 %zmm0, (%rdi) {%k1}
@@ -4927,24 +4945,60 @@ define void @widen_masked_store(<3 x i32> %v, <3 x i32>* %p, <3 x i1> %mask) {
 ;
 ; AVX512VLDQ-LABEL: widen_masked_store:
 ; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vpmovd2m %xmm1, %k0
-; AVX512VLDQ-NEXT:    vpmovm2d %k0, %xmm1
-; AVX512VLDQ-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLDQ-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512VLDQ-NEXT:    vpmovd2m %xmm1, %k1
+; AVX512VLDQ-NEXT:    kmovw %esi, %k0
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512VLDQ-NEXT:    kxorw %k0, %k0, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $1, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $1, %k1, %k1
+; AVX512VLDQ-NEXT:    korb %k0, %k1, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $2, %k0, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $2, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $7, %k0, %k0
+; AVX512VLDQ-NEXT:    kmovw %edx, %k2
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k2, %k2
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k2, %k2
+; AVX512VLDQ-NEXT:    korb %k2, %k1, %k1
+; AVX512VLDQ-NEXT:    korb %k1, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $3, %k0, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $3, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    kshiftrb $6, %k0, %k0
+; AVX512VLDQ-NEXT:    korw %k1, %k0, %k0
+; AVX512VLDQ-NEXT:    kmovw %ecx, %k1
+; AVX512VLDQ-NEXT:    kshiftlb $7, %k1, %k1
+; AVX512VLDQ-NEXT:    kshiftrb $5, %k1, %k1
+; AVX512VLDQ-NEXT:    korw %k0, %k1, %k1
 ; AVX512VLDQ-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
 ; AVX512VLDQ-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: widen_masked_store:
 ; AVX512VLBW:       ## %bb.0:
-; AVX512VLBW-NEXT:    vpslld $31, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vptestmd %xmm1, %xmm1, %k1
-; AVX512VLBW-NEXT:    vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX512VLBW-NEXT:    vmovdqa32 %xmm1, %xmm1 {%k1} {z}
-; AVX512VLBW-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512VLBW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1,2],xmm2[3]
-; AVX512VLBW-NEXT:    vptestmd %xmm1, %xmm1, %k1
+; AVX512VLBW-NEXT:    andl $1, %esi
+; AVX512VLBW-NEXT:    kmovw %esi, %k0
+; AVX512VLBW-NEXT:    kxorw %k0, %k0, %k1
+; AVX512VLBW-NEXT:    kshiftrw $1, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftlw $1, %k1, %k1
+; AVX512VLBW-NEXT:    korw %k0, %k1, %k0
+; AVX512VLBW-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512VLBW-NEXT:    kshiftlw $2, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $15, %k0, %k0
+; AVX512VLBW-NEXT:    kmovd %edx, %k2
+; AVX512VLBW-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512VLBW-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512VLBW-NEXT:    korw %k2, %k1, %k1
+; AVX512VLBW-NEXT:    korw %k1, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $3, %k0, %k1
+; AVX512VLBW-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512VLBW-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512VLBW-NEXT:    korw %k1, %k0, %k0
+; AVX512VLBW-NEXT:    kmovd %ecx, %k1
+; AVX512VLBW-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512VLBW-NEXT:    kshiftrw $13, %k1, %k1
+; AVX512VLBW-NEXT:    korw %k0, %k1, %k1
 ; AVX512VLBW-NEXT:    vmovdqa32 %xmm0, (%rdi) {%k1}
 ; AVX512VLBW-NEXT:    retq
   call void @llvm.masked.store.v3i32.p0v3i32(<3 x i32> %v, <3 x i32>* %p, i32 16, <3 x i1> %mask)
diff --git a/llvm/test/CodeGen/X86/memcpy.ll b/llvm/test/CodeGen/X86/memcpy.ll
index 597b776e2291a..531117dbcda45 100644
--- a/llvm/test/CodeGen/X86/memcpy.ll
+++ b/llvm/test/CodeGen/X86/memcpy.ll
@@ -1,6 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2 | FileCheck %s -check-prefix=LINUX
-; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=core2 | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-apple-darwin      -mcpu=core2     | FileCheck %s -check-prefix=DARWIN
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=core2     | FileCheck %s -check-prefix=LINUX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skylake   | FileCheck %s -check-prefix=LINUX-SKL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=skx       | FileCheck %s -check-prefix=LINUX-SKX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=knl       | FileCheck %s -check-prefix=LINUX-KNL
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=avx512bw | FileCheck %s -check-prefix=LINUX-AVX512BW
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture, i8* nocapture, i64, i1) nounwind
 declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 addrspace(256)* nocapture, i64, i1) nounwind
@@ -8,13 +12,29 @@ declare void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* nocapture, i8 add
 
 ; Variable memcpy's should lower to calls.
 define i8* @test1(i8* %a, i8* %b, i64 %n) nounwind {
+; DARWIN-LABEL: test1:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test1:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test1:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test1:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-SKX-LABEL: test1:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-KNL-LABEL: test1:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-AVX512BW-LABEL: test1:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
 entry:
 	tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 %n, i1 0 )
 	ret i8* %a
@@ -22,13 +42,29 @@ entry:
 
 ; Variable memcpy's should lower to calls.
 define i8* @test2(i64* %a, i64* %b, i64 %n) nounwind {
+; DARWIN-LABEL: test2:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test2:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test2:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test2:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-SKX-LABEL: test2:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-KNL-LABEL: test2:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-AVX512BW-LABEL: test2:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    jmp memcpy # TAILCALL
 entry:
 	%tmp14 = bitcast i64* %a to i8*
 	%tmp25 = bitcast i64* %b to i8*
@@ -43,11 +79,6 @@ entry:
 ; hurting performance so it should just ignore optsize when expanding memcpy.
 ; rdar://8821501
 define void @test3(i8* nocapture %A, i8* nocapture %B) nounwind optsize noredzone {
-; LINUX-LABEL: test3:
-; LINUX:       # %bb.0: # %entry
-; LINUX-NEXT:    movl $64, %edx
-; LINUX-NEXT:    jmp memcpy # TAILCALL
-;
 ; DARWIN-LABEL: test3:
 ; DARWIN:       ## %bb.0: ## %entry
 ; DARWIN-NEXT:    movq 56(%rsi), %rax
@@ -67,45 +98,163 @@ define void @test3(i8* nocapture %A, i8* nocapture %B) nounwind optsize noredzon
 ; DARWIN-NEXT:    movq %rcx, 8(%rdi)
 ; DARWIN-NEXT:    movq %rax, (%rdi)
 ; DARWIN-NEXT:    retq
+;
+; LINUX-LABEL: test3:
+; LINUX:       # %bb.0: # %entry
+; LINUX-NEXT:    movl $64, %edx
+; LINUX-NEXT:    jmp memcpy # TAILCALL
+;
+; LINUX-SKL-LABEL: test3:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 define void @test3_minsize(i8* nocapture %A, i8* nocapture %B) nounwind minsize noredzone {
+; DARWIN-LABEL: test3_minsize:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    pushq $64
+; DARWIN-NEXT:    popq %rdx
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test3_minsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test3_minsize:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test3_minsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3_minsize:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 define void @test3_minsize_optsize(i8* nocapture %A, i8* nocapture %B) nounwind optsize minsize noredzone {
+; DARWIN-LABEL: test3_minsize_optsize:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    pushq $64
+; DARWIN-NEXT:    popq %rdx
+; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+;
 ; LINUX-LABEL: test3_minsize_optsize:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    pushq $64
 ; LINUX-NEXT:    popq %rdx
 ; LINUX-NEXT:    jmp memcpy # TAILCALL
 ;
-; DARWIN-LABEL: test3_minsize_optsize:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    pushq $64
-; DARWIN-NEXT:    popq %rdx
-; DARWIN-NEXT:    jmp _memcpy ## TAILCALL
+; LINUX-SKL-LABEL: test3_minsize_optsize:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test3_minsize_optsize:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test3_minsize_optsize:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test3_minsize_optsize:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
 }
 
 ; Large constant memcpy's should be inlined when not optimizing for size.
 define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone {
+; DARWIN-LABEL: test4:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movq 56(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 56(%rdi)
+; DARWIN-NEXT:    movq 48(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 48(%rdi)
+; DARWIN-NEXT:    movq 40(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 40(%rdi)
+; DARWIN-NEXT:    movq 32(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 32(%rdi)
+; DARWIN-NEXT:    movq 24(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 24(%rdi)
+; DARWIN-NEXT:    movq 16(%rsi), %rax
+; DARWIN-NEXT:    movq %rax, 16(%rdi)
+; DARWIN-NEXT:    movq (%rsi), %rax
+; DARWIN-NEXT:    movq 8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, 8(%rdi)
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test4:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movq 56(%rsi), %rax
@@ -126,25 +275,36 @@ define void @test4(i8* nocapture %A, i8* nocapture %B) nounwind noredzone {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test4:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movq 56(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 56(%rdi)
-; DARWIN-NEXT:    movq 48(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 48(%rdi)
-; DARWIN-NEXT:    movq 40(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 40(%rdi)
-; DARWIN-NEXT:    movq 32(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 32(%rdi)
-; DARWIN-NEXT:    movq 24(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 24(%rdi)
-; DARWIN-NEXT:    movq 16(%rsi), %rax
-; DARWIN-NEXT:    movq %rax, 16(%rdi)
-; DARWIN-NEXT:    movq (%rsi), %rax
-; DARWIN-NEXT:    movq 8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, 8(%rdi)
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test4:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKL-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKL-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKL-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKL-NEXT:    vzeroupper
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test4:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups (%rsi), %ymm0
+; LINUX-SKX-NEXT:    vmovups 32(%rsi), %ymm1
+; LINUX-SKX-NEXT:    vmovups %ymm1, 32(%rdi)
+; LINUX-SKX-NEXT:    vmovups %ymm0, (%rdi)
+; LINUX-SKX-NEXT:    vzeroupper
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test4:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-KNL-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test4:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %zmm0
+; LINUX-AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    vzeroupper
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %A, i8* %B, i64 64, i1 false)
   ret void
@@ -154,6 +314,14 @@ entry:
 @.str = private unnamed_addr constant [30 x i8] c"\00aaaaaaaaaaaaaaaaaaaaaaaaaaaa\00", align 1
 
 define void @test5(i8* nocapture %C) nounwind uwtable ssp {
+; DARWIN-LABEL: test5:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movabsq $7016996765293437281, %rax ## imm = 0x6161616161616161
+; DARWIN-NEXT:    movq %rax, 8(%rdi)
+; DARWIN-NEXT:    movabsq $7016996765293437184, %rax ## imm = 0x6161616161616100
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test5:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movabsq $7016996765293437281, %rax # imm = 0x6161616161616161
@@ -162,13 +330,29 @@ define void @test5(i8* nocapture %C) nounwind uwtable ssp {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test5:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movabsq $7016996765293437281, %rax ## imm = 0x6161616161616161
-; DARWIN-NEXT:    movq %rax, 8(%rdi)
-; DARWIN-NEXT:    movabsq $7016996765293437184, %rax ## imm = 0x6161616161616100
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test5:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test5:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test5:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test5:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    vmovups {{.*}}(%rip), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %C, i8* getelementptr inbounds ([30 x i8], [30 x i8]* @.str, i64 0, i64 0), i64 16, i1 false)
   ret void
@@ -179,17 +363,41 @@ entry:
 @.str2 = private unnamed_addr constant [2 x i8] c"x\00", align 1
 
 define void @test6() nounwind uwtable {
+; DARWIN-LABEL: test6:
+; DARWIN:       ## %bb.0: ## %entry
+; DARWIN-NEXT:    movw $0, 8
+; DARWIN-NEXT:    movq $120, 0
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: test6:
 ; LINUX:       # %bb.0: # %entry
 ; LINUX-NEXT:    movw $0, 8
 ; LINUX-NEXT:    movq $120, 0
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: test6:
-; DARWIN:       ## %bb.0: ## %entry
-; DARWIN-NEXT:    movw $0, 8
-; DARWIN-NEXT:    movq $120, 0
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: test6:
+; LINUX-SKL:       # %bb.0: # %entry
+; LINUX-SKL-NEXT:    movw $0, 8
+; LINUX-SKL-NEXT:    movq $120, 0
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: test6:
+; LINUX-SKX:       # %bb.0: # %entry
+; LINUX-SKX-NEXT:    movw $0, 8
+; LINUX-SKX-NEXT:    movq $120, 0
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: test6:
+; LINUX-KNL:       # %bb.0: # %entry
+; LINUX-KNL-NEXT:    movw $0, 8
+; LINUX-KNL-NEXT:    movq $120, 0
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: test6:
+; LINUX-AVX512BW:       # %bb.0: # %entry
+; LINUX-AVX512BW-NEXT:    movw $0, 8
+; LINUX-AVX512BW-NEXT:    movq $120, 0
+; LINUX-AVX512BW-NEXT:    retq
 entry:
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* null, i8* getelementptr inbounds ([2 x i8], [2 x i8]* @.str2, i64 0, i64 0), i64 10, i1 false)
   ret void
@@ -198,6 +406,16 @@ entry:
 define void @PR15348(i8* %a, i8* %b) {
 ; Ensure that alignment of '0' in an @llvm.memcpy intrinsic results in
 ; unaligned loads and stores.
+; DARWIN-LABEL: PR15348:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movb 16(%rsi), %al
+; DARWIN-NEXT:    movb %al, 16(%rdi)
+; DARWIN-NEXT:    movq (%rsi), %rax
+; DARWIN-NEXT:    movq 8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, 8(%rdi)
+; DARWIN-NEXT:    movq %rax, (%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: PR15348:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    movb 16(%rsi), %al
@@ -208,15 +426,37 @@ define void @PR15348(i8* %a, i8* %b) {
 ; LINUX-NEXT:    movq %rax, (%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: PR15348:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    movb 16(%rsi), %al
-; DARWIN-NEXT:    movb %al, 16(%rdi)
-; DARWIN-NEXT:    movq (%rsi), %rax
-; DARWIN-NEXT:    movq 8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, 8(%rdi)
-; DARWIN-NEXT:    movq %rax, (%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: PR15348:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    movb 16(%rsi), %al
+; LINUX-SKL-NEXT:    movb %al, 16(%rdi)
+; LINUX-SKL-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: PR15348:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    movb 16(%rsi), %al
+; LINUX-SKX-NEXT:    movb %al, 16(%rdi)
+; LINUX-SKX-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: PR15348:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    movb 16(%rsi), %al
+; LINUX-KNL-NEXT:    movb %al, 16(%rdi)
+; LINUX-KNL-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: PR15348:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    movb 16(%rsi), %al
+; LINUX-AVX512BW-NEXT:    movb %al, 16(%rdi)
+; LINUX-AVX512BW-NEXT:    vmovups (%rsi), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, (%rdi)
+; LINUX-AVX512BW-NEXT:    retq
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %a, i8* %b, i64 17, i1 false)
   ret void
 }
@@ -224,6 +464,14 @@ define void @PR15348(i8* %a, i8* %b) {
 ; Memcpys from / to address space 256 should be lowered to appropriate loads /
 ; stores if small enough.
 define void @addrspace256(i8 addrspace(256)* %a, i8 addrspace(256)* %b) nounwind {
+; DARWIN-LABEL: addrspace256:
+; DARWIN:       ## %bb.0:
+; DARWIN-NEXT:    movq %gs:(%rsi), %rax
+; DARWIN-NEXT:    movq %gs:8(%rsi), %rcx
+; DARWIN-NEXT:    movq %rcx, %gs:8(%rdi)
+; DARWIN-NEXT:    movq %rax, %gs:(%rdi)
+; DARWIN-NEXT:    retq
+;
 ; LINUX-LABEL: addrspace256:
 ; LINUX:       # %bb.0:
 ; LINUX-NEXT:    movq %gs:(%rsi), %rax
@@ -232,13 +480,29 @@ define void @addrspace256(i8 addrspace(256)* %a, i8 addrspace(256)* %b) nounwind
 ; LINUX-NEXT:    movq %rax, %gs:(%rdi)
 ; LINUX-NEXT:    retq
 ;
-; DARWIN-LABEL: addrspace256:
-; DARWIN:       ## %bb.0:
-; DARWIN-NEXT:    movq %gs:(%rsi), %rax
-; DARWIN-NEXT:    movq %gs:8(%rsi), %rcx
-; DARWIN-NEXT:    movq %rcx, %gs:8(%rdi)
-; DARWIN-NEXT:    movq %rax, %gs:(%rdi)
-; DARWIN-NEXT:    retq
+; LINUX-SKL-LABEL: addrspace256:
+; LINUX-SKL:       # %bb.0:
+; LINUX-SKL-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-SKL-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-SKL-NEXT:    retq
+;
+; LINUX-SKX-LABEL: addrspace256:
+; LINUX-SKX:       # %bb.0:
+; LINUX-SKX-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-SKX-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-SKX-NEXT:    retq
+;
+; LINUX-KNL-LABEL: addrspace256:
+; LINUX-KNL:       # %bb.0:
+; LINUX-KNL-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-KNL-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-KNL-NEXT:    retq
+;
+; LINUX-AVX512BW-LABEL: addrspace256:
+; LINUX-AVX512BW:       # %bb.0:
+; LINUX-AVX512BW-NEXT:    vmovups %gs:(%rsi), %xmm0
+; LINUX-AVX512BW-NEXT:    vmovups %xmm0, %gs:(%rdi)
+; LINUX-AVX512BW-NEXT:    retq
   tail call void @llvm.memcpy.p256i8.p256i8.i64(i8 addrspace(256)* align 8 %a, i8 addrspace(256)* align 8 %b, i64 16, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/memset-nonzero.ll b/llvm/test/CodeGen/X86/memset-nonzero.ll
index 8166d009d72ff..6e3dedbf83039 100644
--- a/llvm/test/CodeGen/X86/memset-nonzero.ll
+++ b/llvm/test/CodeGen/X86/memset-nonzero.ll
@@ -1,9 +1,14 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse | FileCheck %s --check-prefix=SSE
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse  | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2 | FileCheck %s --check-prefix=SSE
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=sse2,-slow-unaligned-mem-16 | FileCheck %s --check-prefix=SSE2FAST
-; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx  | FileCheck %s --check-prefix=AVX --check-prefix=AVX1
 ; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx2 | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512dq -mattr=+prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX2
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512f  -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512F
+; RUN: llc -mtriple=x86_64-unknown-unknown < %s -mattr=avx512bw -mattr=-prefer-256-bit | FileCheck %s --check-prefix=AVX --check-prefix=AVX512 --check-prefix=AVX512BW
 
 ; https://llvm.org/bugs/show_bug.cgi?id=27100
 
@@ -80,13 +85,36 @@ define void @memset_64_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_64_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_64_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_64_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
+; AVX512NW-NEXT: retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 64, i64 -1)
   ret void
 }
@@ -126,15 +154,41 @@ define void @memset_128_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_128_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_128_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_128_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 128, i64 -1)
   ret void
 }
@@ -172,19 +226,53 @@ define void @memset_256_nonzero_bytes(i8* %x) {
 ; SSE2FAST-NEXT:    movups %xmm0, (%rdi)
 ; SSE2FAST-NEXT:    retq
 ;
-; AVX-LABEL: memset_256_nonzero_bytes:
-; AVX:       # %bb.0:
-; AVX-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
-; AVX-NEXT:    vmovups %ymm0, 224(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 192(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 160(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 128(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 96(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 64(%rdi)
-; AVX-NEXT:    vmovups %ymm0, 32(%rdi)
-; AVX-NEXT:    vmovups %ymm0, (%rdi)
-; AVX-NEXT:    vzeroupper
-; AVX-NEXT:    retq
+; AVX1-LABEL: memset_256_nonzero_bytes:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX1-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX1-NEXT:    vmovups %ymm0, (%rdi)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: memset_256_nonzero_bytes:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovaps {{.*#+}} ymm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX2-NEXT:    vmovups %ymm0, 224(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 192(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 160(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 128(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 96(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 64(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, 32(%rdi)
+; AVX2-NEXT:    vmovups %ymm0, (%rdi)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonzero_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vbroadcastss {{.*#+}} zmm0 = [707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378,707406378]
+; AVX512F-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonzero_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovaps {{.*#+}} zmm0 = [42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42,42]
+; AVX512BW-NEXT:    vmovups %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovups %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   %call = tail call i8* @__memset_chk(i8* %x, i32 42, i64 256, i64 -1)
   ret void
 }
@@ -227,6 +315,13 @@ define void @memset_16_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vpbroadcastb %xmm0, %xmm0
 ; AVX2-NEXT:    vmovdqu %xmm0, (%rdi)
 ; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: memset_16_nonconst_bytes:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %esi, %xmm0
+; AVX512-NEXT:    vpbroadcastb %xmm0, %xmm0
+; AVX512-NEXT:    vmovdqu %xmm0, (%rdi)
+; AVX512-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 16, i1 false)
   ret void
 }
@@ -270,6 +365,14 @@ define void @memset_32_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512-LABEL: memset_32_nonconst_bytes:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    vmovd %esi, %xmm0
+; AVX512-NEXT:    vpbroadcastb %xmm0, %ymm0
+; AVX512-NEXT:    vmovdqu %ymm0, (%rdi)
+; AVX512-NEXT:    vzeroupper
+; AVX512-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 32, i1 false)
   ret void
 }
@@ -322,6 +425,22 @@ define void @memset_64_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_64_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_64_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 64, i1 false)
   ret void
 }
@@ -390,6 +509,24 @@ define void @memset_128_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_128_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_128_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 128, i1 false)
   ret void
 }
@@ -455,6 +592,28 @@ define void @memset_256_nonconst_bytes(i8* %x, i8 %c) {
 ; AVX2-NEXT:    vmovdqu %ymm0, (%rdi)
 ; AVX2-NEXT:    vzeroupper
 ; AVX2-NEXT:    retq
+;
+; AVX512F-LABEL: memset_256_nonconst_bytes:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movzbl %sil, %eax
+; AVX512F-NEXT:    imull $16843009, %eax, %eax # imm = 0x1010101
+; AVX512F-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512F-NEXT:    vzeroupper
+; AVX512F-NEXT:    retq
+;
+; AVX512BW-LABEL: memset_256_nonconst_bytes:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpbroadcastb %esi, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 192(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 128(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, 64(%rdi)
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
+; AVX512BW-NEXT:    retq
   tail call void @llvm.memset.p0i8.i64(i8* %x, i8 %c, i64 256, i1 false)
   ret void
 }
diff --git a/llvm/test/CodeGen/X86/memset-zero.ll b/llvm/test/CodeGen/X86/memset-zero.ll
index fb28742e688c9..c6465bd462a5b 100644
--- a/llvm/test/CodeGen/X86/memset-zero.ll
+++ b/llvm/test/CodeGen/X86/memset-zero.ll
@@ -4,6 +4,7 @@
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=nehalem | FileCheck %s --check-prefix=NEHALEM
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=sandybridge | FileCheck %s --check-prefix=SANDYBRIDGE
 ; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=skylake | FileCheck %s --check-prefix=SKYLAKE
+; RUN: llc < %s -mtriple=x86_64-unknown-linux -mcpu=knl | FileCheck %s --check-prefix=KNL
 
 
 declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1) nounwind
@@ -28,6 +29,10 @@ define void @memset_0(i8* %a) nounwind  {
 ; SKYLAKE-LABEL: memset_0:
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_0:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 0, i1 false)
 	ret void
@@ -59,6 +64,11 @@ define void @memset_4(i8* %a) nounwind  {
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_4:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 4, i1 false)
 	ret void
@@ -95,6 +105,12 @@ define void @memset_5(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movb $0, 4(%rdi)
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_5:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movb $0, 4(%rdi)
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 5, i1 false)
 	ret void
@@ -131,6 +147,12 @@ define void @memset_7(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 3(%rdi)
 ; SKYLAKE-NEXT:    movl $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_7:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, 3(%rdi)
+; KNL-NEXT:    movl $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 7, i1 false)
 	ret void
@@ -163,6 +185,11 @@ define void @memset_8(i8* %a) nounwind  {
 ; SKYLAKE:       # %bb.0: # %entry
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_8:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 8, i1 false)
 	ret void
@@ -200,6 +227,12 @@ define void @memset_11(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 7(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_11:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movl $0, 7(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 11, i1 false)
 	ret void
@@ -238,6 +271,12 @@ define void @memset_13(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movq $0, 5(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_13:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, 5(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 13, i1 false)
 	ret void
@@ -276,6 +315,12 @@ define void @memset_15(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movq $0, 7(%rdi)
 ; SKYLAKE-NEXT:    movq $0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_15:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    movq $0, 7(%rdi)
+; KNL-NEXT:    movq $0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 15, i1 false)
 	ret void
@@ -314,6 +359,12 @@ define void @memset_16(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_16:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 16, i1 false)
 	ret void
@@ -357,6 +408,13 @@ define void @memset_17(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    movb $0, 16(%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_17:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    movb $0, 16(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 17, i1 false)
 	ret void
@@ -400,6 +458,13 @@ define void @memset_19(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    movl $0, 15(%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_19:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    movl $0, 15(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 19, i1 false)
 	ret void
@@ -447,6 +512,13 @@ define void @memset_31(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %xmm0, 15(%rdi)
 ; SKYLAKE-NEXT:    vmovups %xmm0, (%rdi)
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_31:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %xmm0, 15(%rdi)
+; KNL-NEXT:    vmovups %xmm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 31, i1 false)
 	ret void
@@ -494,6 +566,12 @@ define void @memset_32(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovups %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_32:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %ymm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 32, i1 false)
 	ret void
@@ -540,6 +618,12 @@ define void @memset_32_align32(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    vmovaps %ymm0, (%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_32_align32:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovaps %ymm0, (%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* align 32 %a, i8 0, i64 32, i1 false)
 	ret void
@@ -593,7 +677,150 @@ define void @memset_35(i8* %a) nounwind  {
 ; SKYLAKE-NEXT:    movl $0, 31(%rdi)
 ; SKYLAKE-NEXT:    vzeroupper
 ; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_35:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %ymm0, (%rdi)
+; KNL-NEXT:    movl $0, 31(%rdi)
+; KNL-NEXT:    retq
 entry:
 	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 35, i1 false)
 	ret void
 }
+
+define void @memset_64(i8* %a) nounwind  {
+; X86-LABEL: memset_64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $0, 60(%eax)
+; X86-NEXT:    movl $0, 56(%eax)
+; X86-NEXT:    movl $0, 52(%eax)
+; X86-NEXT:    movl $0, 48(%eax)
+; X86-NEXT:    movl $0, 44(%eax)
+; X86-NEXT:    movl $0, 40(%eax)
+; X86-NEXT:    movl $0, 36(%eax)
+; X86-NEXT:    movl $0, 32(%eax)
+; X86-NEXT:    movl $0, 28(%eax)
+; X86-NEXT:    movl $0, 24(%eax)
+; X86-NEXT:    movl $0, 20(%eax)
+; X86-NEXT:    movl $0, 16(%eax)
+; X86-NEXT:    movl $0, 12(%eax)
+; X86-NEXT:    movl $0, 8(%eax)
+; X86-NEXT:    movl $0, 4(%eax)
+; X86-NEXT:    movl $0, (%eax)
+; X86-NEXT:    retl
+;
+; CORE2-LABEL: memset_64:
+; CORE2:       # %bb.0: # %entry
+; CORE2-NEXT:    movq $0, 56(%rdi)
+; CORE2-NEXT:    movq $0, 48(%rdi)
+; CORE2-NEXT:    movq $0, 40(%rdi)
+; CORE2-NEXT:    movq $0, 32(%rdi)
+; CORE2-NEXT:    movq $0, 24(%rdi)
+; CORE2-NEXT:    movq $0, 16(%rdi)
+; CORE2-NEXT:    movq $0, 8(%rdi)
+; CORE2-NEXT:    movq $0, (%rdi)
+; CORE2-NEXT:    retq
+;
+; NEHALEM-LABEL: memset_64:
+; NEHALEM:       # %bb.0: # %entry
+; NEHALEM-NEXT:    xorps %xmm0, %xmm0
+; NEHALEM-NEXT:    movups %xmm0, 48(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, 32(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, 16(%rdi)
+; NEHALEM-NEXT:    movups %xmm0, (%rdi)
+; NEHALEM-NEXT:    retq
+;
+; SANDYBRIDGE-LABEL: memset_64:
+; SANDYBRIDGE:       # %bb.0: # %entry
+; SANDYBRIDGE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 16(%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, (%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 48(%rdi)
+; SANDYBRIDGE-NEXT:    vmovups %xmm0, 32(%rdi)
+; SANDYBRIDGE-NEXT:    retq
+;
+; SKYLAKE-LABEL: memset_64:
+; SKYLAKE:       # %bb.0: # %entry
+; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SKYLAKE-NEXT:    vmovups %ymm0, 32(%rdi)
+; SKYLAKE-NEXT:    vmovups %ymm0, (%rdi)
+; SKYLAKE-NEXT:    vzeroupper
+; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_64:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovups %zmm0, (%rdi)
+; KNL-NEXT:    retq
+entry:
+	call void @llvm.memset.p0i8.i64(i8* %a, i8 0, i64 64, i1 false)
+	ret void
+}
+
+define void @memset_64_align64(i8* %a) nounwind  {
+; X86-LABEL: memset_64_align64:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl $0, 60(%eax)
+; X86-NEXT:    movl $0, 56(%eax)
+; X86-NEXT:    movl $0, 52(%eax)
+; X86-NEXT:    movl $0, 48(%eax)
+; X86-NEXT:    movl $0, 44(%eax)
+; X86-NEXT:    movl $0, 40(%eax)
+; X86-NEXT:    movl $0, 36(%eax)
+; X86-NEXT:    movl $0, 32(%eax)
+; X86-NEXT:    movl $0, 28(%eax)
+; X86-NEXT:    movl $0, 24(%eax)
+; X86-NEXT:    movl $0, 20(%eax)
+; X86-NEXT:    movl $0, 16(%eax)
+; X86-NEXT:    movl $0, 12(%eax)
+; X86-NEXT:    movl $0, 8(%eax)
+; X86-NEXT:    movl $0, 4(%eax)
+; X86-NEXT:    movl $0, (%eax)
+; X86-NEXT:    retl
+;
+; CORE2-LABEL: memset_64_align64:
+; CORE2:       # %bb.0: # %entry
+; CORE2-NEXT:    xorps %xmm0, %xmm0
+; CORE2-NEXT:    movaps %xmm0, 48(%rdi)
+; CORE2-NEXT:    movaps %xmm0, 32(%rdi)
+; CORE2-NEXT:    movaps %xmm0, 16(%rdi)
+; CORE2-NEXT:    movaps %xmm0, (%rdi)
+; CORE2-NEXT:    retq
+;
+; NEHALEM-LABEL: memset_64_align64:
+; NEHALEM:       # %bb.0: # %entry
+; NEHALEM-NEXT:    xorps %xmm0, %xmm0
+; NEHALEM-NEXT:    movaps %xmm0, 48(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, 32(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, 16(%rdi)
+; NEHALEM-NEXT:    movaps %xmm0, (%rdi)
+; NEHALEM-NEXT:    retq
+;
+; SANDYBRIDGE-LABEL: memset_64_align64:
+; SANDYBRIDGE:       # %bb.0: # %entry
+; SANDYBRIDGE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SANDYBRIDGE-NEXT:    vmovaps %ymm0, 32(%rdi)
+; SANDYBRIDGE-NEXT:    vmovaps %ymm0, (%rdi)
+; SANDYBRIDGE-NEXT:    vzeroupper
+; SANDYBRIDGE-NEXT:    retq
+;
+; SKYLAKE-LABEL: memset_64_align64:
+; SKYLAKE:       # %bb.0: # %entry
+; SKYLAKE-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; SKYLAKE-NEXT:    vmovaps %ymm0, 32(%rdi)
+; SKYLAKE-NEXT:    vmovaps %ymm0, (%rdi)
+; SKYLAKE-NEXT:    vzeroupper
+; SKYLAKE-NEXT:    retq
+;
+; KNL-LABEL: memset_64_align64:
+; KNL:       # %bb.0: # %entry
+; KNL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
+; KNL-NEXT:    vmovaps %zmm0, (%rdi)
+; KNL-NEXT:    retq
+entry:
+	call void @llvm.memset.p0i8.i64(i8* align 64 %a, i8 0, i64 64, i1 false)
+	ret void
+}
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index b69525deb41ef..deb261151402d 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -982,3 +982,95 @@ define void @sext_v16i8_v16i64(<16 x i8> %x, <16 x i64>* %y) nounwind "min-legal
   store <16 x i64> %a, <16 x i64>* %y
   ret void
 }
+
+define void @vselect_split_v8i16_setcc(<8 x i16> %s, <8 x i16> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v8i16_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k2
+; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %p
+  %y = load <8 x i64>, <8 x i64>* %q
+  %a = icmp eq <8 x i16> %s, %t
+  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
+  store <8 x i64> %b, <8 x i64>* %r
+  ret void
+}
+
+define void @vselect_split_v8i32_setcc(<8 x i32> %s, <8 x i32> %t, <8 x i64>* %p, <8 x i64>* %q, <8 x i64>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v8i32_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %k2
+; CHECK-NEXT:    vmovdqa64 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa64 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <8 x i64>, <8 x i64>* %p
+  %y = load <8 x i64>, <8 x i64>* %q
+  %a = icmp eq <8 x i32> %s, %t
+  %b = select <8 x i1> %a, <8 x i64> %x, <8 x i64> %y
+  store <8 x i64> %b, <8 x i64>* %r
+  ret void
+}
+
+define void @vselect_split_v16i8_setcc(<16 x i8> %s, <16 x i8> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v16i8_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; CHECK-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
+; CHECK-NEXT:    vpcmpeqb %xmm1, %xmm0, %k2
+; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <16 x i32>, <16 x i32>* %p
+  %y = load <16 x i32>, <16 x i32>* %q
+  %a = icmp eq <16 x i8> %s, %t
+  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
+  store <16 x i32> %b, <16 x i32>* %r
+  ret void
+}
+
+define void @vselect_split_v16i16_setcc(<16 x i16> %s, <16 x i16> %t, <16 x i32>* %p, <16 x i32>* %q, <16 x i32>* %r) "min-legal-vector-width"="256" {
+; CHECK-LABEL: vselect_split_v16i16_setcc:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovdqa (%rsi), %ymm2
+; CHECK-NEXT:    vmovdqa 32(%rsi), %ymm3
+; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k1
+; CHECK-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; CHECK-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; CHECK-NEXT:    vpcmpeqw %xmm1, %xmm0, %k2
+; CHECK-NEXT:    vmovdqa32 32(%rdi), %ymm3 {%k2}
+; CHECK-NEXT:    vmovdqa32 (%rdi), %ymm2 {%k1}
+; CHECK-NEXT:    vmovdqa %ymm2, (%rdx)
+; CHECK-NEXT:    vmovdqa %ymm3, 32(%rdx)
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+  %x = load <16 x i32>, <16 x i32>* %p
+  %y = load <16 x i32>, <16 x i32>* %q
+  %a = icmp eq <16 x i16> %s, %t
+  %b = select <16 x i1> %a, <16 x i32> %x, <16 x i32> %y
+  store <16 x i32> %b, <16 x i32>* %r
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/mmx-fold-load.ll b/llvm/test/CodeGen/X86/mmx-fold-load.ll
index 71b8b40fc5d7e..5ad2d50c1d133 100644
--- a/llvm/test/CodeGen/X86/mmx-fold-load.ll
+++ b/llvm/test/CodeGen/X86/mmx-fold-load.ll
@@ -585,22 +585,22 @@ define void @test_psrlq_by_volatile_shift_amount(x86_mmx* %t) nounwind {
 ; X86-NEXT:    pushl %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl $1, (%esp)
-; X86-NEXT:    movd (%esp), %mm0
 ; X86-NEXT:    movl $255, %ecx
-; X86-NEXT:    movd %ecx, %mm1
-; X86-NEXT:    psrlq %mm0, %mm1
-; X86-NEXT:    movq %mm1, (%eax)
+; X86-NEXT:    movd %ecx, %mm0
+; X86-NEXT:    movd (%esp), %mm1
+; X86-NEXT:    psrlq %mm1, %mm0
+; X86-NEXT:    movq %mm0, (%eax)
 ; X86-NEXT:    popl %eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: test_psrlq_by_volatile_shift_amount:
 ; X64:       # %bb.0: # %entry
 ; X64-NEXT:    movl $1, -{{[0-9]+}}(%rsp)
-; X64-NEXT:    movd -{{[0-9]+}}(%rsp), %mm0
 ; X64-NEXT:    movl $255, %eax
-; X64-NEXT:    movd %eax, %mm1
-; X64-NEXT:    psrlq %mm0, %mm1
-; X64-NEXT:    movq %mm1, (%rdi)
+; X64-NEXT:    movd %eax, %mm0
+; X64-NEXT:    movd -{{[0-9]+}}(%rsp), %mm1
+; X64-NEXT:    psrlq %mm1, %mm0
+; X64-NEXT:    movq %mm0, (%rdi)
 ; X64-NEXT:    retq
 entry:
   %0 = alloca i32, align 4
diff --git a/llvm/test/CodeGen/X86/mmx-intrinsics.ll b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
index b9655830619ed..48d4ad0490f76 100644
--- a/llvm/test/CodeGen/X86/mmx-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/mmx-intrinsics.ll
@@ -1453,3 +1453,68 @@ define void @test90() {
 }
 
 declare void @llvm.x86.mmx.emms()
+
+define <1 x i64> @test_mm_insert_pi16(<1 x i64> %a.coerce, i32 %d) nounwind {
+; X86-LABEL: test_mm_insert_pi16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $16, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT:    movq {{[0-9]+}}(%esp), %mm0
+; X86-NEXT:    pinsrw $2, 16(%ebp), %mm0
+; X86-NEXT:    movq %mm0, (%esp)
+; X86-NEXT:    movl (%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_insert_pi16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    pinsrw $2, %esi, %mm0
+; X64-NEXT:    movq %mm0, %rax
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <1 x i64> %a.coerce to x86_mmx
+  %1 = tail call x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx %0, i32 %d, i32 2)
+  %2 = bitcast x86_mmx %1 to <1 x i64>
+  ret <1 x i64> %2
+}
+
+declare x86_mmx @llvm.x86.mmx.pinsr.w(x86_mmx, i32, i32 immarg)
+
+define i32 @test_mm_extract_pi16(<1 x i64> %a.coerce) nounwind {
+; X86-LABEL: test_mm_extract_pi16:
+; X86:       # %bb.0: # %entry
+; X86-NEXT:    pushl %ebp
+; X86-NEXT:    movl %esp, %ebp
+; X86-NEXT:    andl $-8, %esp
+; X86-NEXT:    subl $8, %esp
+; X86-NEXT:    movl 8(%ebp), %eax
+; X86-NEXT:    movl 12(%ebp), %ecx
+; X86-NEXT:    movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT:    movl %eax, (%esp)
+; X86-NEXT:    movq (%esp), %mm0
+; X86-NEXT:    pextrw $2, %mm0, %eax
+; X86-NEXT:    movl %ebp, %esp
+; X86-NEXT:    popl %ebp
+; X86-NEXT:    retl
+;
+; X64-LABEL: test_mm_extract_pi16:
+; X64:       # %bb.0: # %entry
+; X64-NEXT:    movq %rdi, %mm0
+; X64-NEXT:    pextrw $2, %mm0, %eax
+; X64-NEXT:    retq
+entry:
+  %0 = bitcast <1 x i64> %a.coerce to x86_mmx
+  %1 = tail call i32 @llvm.x86.mmx.pextr.w(x86_mmx %0, i32 2)
+  ret i32 %1
+}
+
+declare i32 @llvm.x86.mmx.pextr.w(x86_mmx, i32 immarg)
diff --git a/llvm/test/CodeGen/X86/mulo-pow2.ll b/llvm/test/CodeGen/X86/mulo-pow2.ll
index f5be144ff8e1c..de7dc5673274f 100644
--- a/llvm/test/CodeGen/X86/mulo-pow2.ll
+++ b/llvm/test/CodeGen/X86/mulo-pow2.ll
@@ -98,15 +98,10 @@ define <4 x i32> @smul_v4i32_1(<4 x i32> %a, <4 x i32> %b) nounwind {
 define <4 x i32> @smul_v4i32_2(<4 x i32> %a, <4 x i32> %b) nounwind {
 ; AVX-LABEL: smul_v4i32_2:
 ; AVX:       # %bb.0:
-; AVX-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm3
-; AVX-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm2
-; AVX-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX-NEXT:    vpcmpeqd %xmm2, %xmm3, %xmm2
-; AVX-NEXT:    vblendvps %xmm2, %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    vpaddd %xmm0, %xmm0, %xmm2
+; AVX-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm3
+; AVX-NEXT:    vpxor %xmm3, %xmm0, %xmm0
+; AVX-NEXT:    vblendvps %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX-NEXT:    retq
     %x = call { <4 x i32>, <4 x i1> } @llvm.smul.with.overflow.v4i32(<4 x i32> %a, <4 x i32> <i32 2, i32 2, i32 2, i32 2>)
     %y = extractvalue { <4 x i32>, <4 x i1> } %x, 0
diff --git a/llvm/test/CodeGen/X86/no-plt-libcalls.ll b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
index 876c8f10e428b..615b7bc330903 100644
--- a/llvm/test/CodeGen/X86/no-plt-libcalls.ll
+++ b/llvm/test/CodeGen/X86/no-plt-libcalls.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Check if "RtLibUseGOT" works correctly when lib calls are simplified.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -5,10 +6,13 @@
 @hello_world = constant [13 x i8] c"hello world\0A\00"
 declare i32 @printf(i8*, ...)
 define void @printf_call() {
+; CHECK-LABEL: @printf_call(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i64 0, i64 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK:  call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i64 0, i64 0))
   ret void
 }
 
diff --git a/llvm/test/CodeGen/X86/pr38803.ll b/llvm/test/CodeGen/X86/pr38803.ll
index a1ca27d89b038..a2fc19e0cde90 100644
--- a/llvm/test/CodeGen/X86/pr38803.ll
+++ b/llvm/test/CodeGen/X86/pr38803.ll
@@ -13,15 +13,14 @@ define float @_Z3fn2v() {
 ; CHECK-NEXT:    callq _Z1av
 ; CHECK-NEXT:    # kill: def $al killed $al def $eax
 ; CHECK-NEXT:    kmovd %eax, %k1
-; CHECK-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; CHECK-NEXT:    vmovss %xmm0, %xmm0, %xmm0 {%k1} {z}
+; CHECK-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1} {z}
 ; CHECK-NEXT:    cmpl $0, {{.*}}(%rip)
 ; CHECK-NEXT:    je .LBB0_2
 ; CHECK-NEXT:  # %bb.1: # %if.then
 ; CHECK-NEXT:    vcvtsi2ssl {{.*}}(%rip), %xmm1, %xmm1
 ; CHECK-NEXT:    kmovd %eax, %k1
 ; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
-; CHECK-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; CHECK-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; CHECK-NEXT:    vmovss %xmm1, {{.*}}(%rip)
 ; CHECK-NEXT:  .LBB0_2: # %if.end
 ; CHECK-NEXT:    popq %rax
diff --git a/llvm/test/CodeGen/X86/pr43507.ll b/llvm/test/CodeGen/X86/pr43507.ll
new file mode 100644
index 0000000000000..ec18d3c13ba81
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43507.ll
@@ -0,0 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s
+
+define <8 x i1> @ham(i64 %arg) {
+; CHECK-LABEL: ham:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    testb $1, %dil
+; CHECK-NEXT:    movl $255, %ecx
+; CHECK-NEXT:    cmovel %eax, %ecx
+; CHECK-NEXT:    kmovd %ecx, %k0
+; CHECK-NEXT:    vpmovm2w %k0, %xmm0
+; CHECK-NEXT:    retq
+  %tmp = trunc i64 %arg to i1
+  %tmp1 = insertelement <8 x i1> undef, i1 %tmp, i32 0
+  %tmp2 = shufflevector <8 x i1> %tmp1, <8 x i1> undef, <8 x i32> zeroinitializer
+  ret <8 x i1> %tmp2
+}
diff --git a/llvm/test/CodeGen/X86/pr43509.ll b/llvm/test/CodeGen/X86/pr43509.ll
new file mode 100644
index 0000000000000..4243764d08cf7
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43509.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=skx | FileCheck %s
+
+define <8 x i8> @foo(<8 x float> %arg) {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    vcmpgtps {{.*}}(%rip){1to8}, %ymm0, %k0
+; CHECK-NEXT:    vpmovm2b %k0, %xmm1
+; CHECK-NEXT:    vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT:    vcmpltps %ymm2, %ymm0, %k1
+; CHECK-NEXT:    vmovdqu8 {{.*}}(%rip), %xmm0 {%k1} {z}
+; CHECK-NEXT:    vpand %xmm0, %xmm1, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+bb:
+  %tmp = xor <8 x i8> zeroinitializer, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
+  %tmp1 = fcmp reassoc nsz contract ogt <8 x float> %arg, <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
+  %tmp2 = zext <8 x i1> %tmp1 to <8 x i8>
+  %tmp3 = and <8 x i8> %tmp, %tmp2
+  %tmp4 = fcmp reassoc nsz contract ogt <8 x float> zeroinitializer, %arg
+  %tmp5 = or <8 x i1> zeroinitializer, %tmp4
+  %tmp6 = zext <8 x i1> %tmp5 to <8 x i8>
+  %tmp7 = and <8 x i8> %tmp3, %tmp6
+  ret <8 x i8> %tmp7
+}
diff --git a/llvm/test/CodeGen/X86/pr43529.ll b/llvm/test/CodeGen/X86/pr43529.ll
new file mode 100644
index 0000000000000..afccf5e46d7c6
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr43529.ll
@@ -0,0 +1,39 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu | FileCheck %s
+
+define i32 @a() nounwind {
+; CHECK-LABEL: a:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    pushl %esi
+; CHECK-NEXT:    subl $8, %esp
+; CHECK-NEXT:    leal {{[0-9]+}}(%esp), %esi
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    subl $a, %eax
+; CHECK-NEXT:    calll d
+; CHECK-NEXT:    cmpl $a, %esi
+; CHECK-NEXT:    jbe .LBB0_2
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_1: # %for.cond
+; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:  .LBB0_2: # %for.end.split
+; CHECK-NEXT:    addl $8, %esp
+; CHECK-NEXT:    popl %esi
+; CHECK-NEXT:    retl
+entry:
+  %b = alloca i32, align 4
+  %0 = bitcast i32* %b to i8*
+  %1 = ptrtoint i32* %b to i32
+  %sub = sub nsw i32 %1, ptrtoint (i32 ()* @a to i32)
+  %call = call i32 bitcast (i32 (...)* @d to i32 (i32)*)(i32 inreg %sub)
+  %cmp = icmp ugt i32* %b, bitcast (i32 ()* @a to i32*)
+  br i1 %cmp, label %for.cond, label %for.end.split
+
+for.cond:                                         ; preds = %entry, %for.cond
+  br label %for.cond
+
+for.end.split:                                    ; preds = %entry
+  ret i32 undef
+}
+
+declare i32 @d(...)
diff --git a/llvm/test/CodeGen/X86/rot16.ll b/llvm/test/CodeGen/X86/rot16.ll
index b0221c48532a5..81838f84e3af4 100644
--- a/llvm/test/CodeGen/X86/rot16.ll
+++ b/llvm/test/CodeGen/X86/rot16.ll
@@ -177,3 +177,57 @@ define i16 @xbu(i16 %x, i16 %y, i16 %z) nounwind {
 	%t2 = or i16 %t0, %t1
 	ret i16 %t2
 }
+
+define i32 @rot16_demandedbits(i32 %x, i32 %y) nounwind {
+; X32-LABEL: rot16_demandedbits:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $11, %ecx
+; X32-NEXT:    shll $5, %eax
+; X32-NEXT:    orl %ecx, %eax
+; X32-NEXT:    andl $65536, %eax # imm = 0x10000
+; X32-NEXT:    retl
+;
+; X64-LABEL: rot16_demandedbits:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    shrl $11, %ecx
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    andl $65536, %eax # imm = 0x10000
+; X64-NEXT:    retq
+	%t0 = lshr i32 %x, 11
+	%t1 = shl i32 %x, 5
+	%t2 = or i32 %t0, %t1
+	%t3 = and i32 %t2, 65536
+	ret i32 %t3
+}
+
+define i16 @rot16_trunc(i32 %x, i32 %y) nounwind {
+; X32-LABEL: rot16_trunc:
+; X32:       # %bb.0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl %eax, %ecx
+; X32-NEXT:    shrl $11, %ecx
+; X32-NEXT:    shll $5, %eax
+; X32-NEXT:    orl %ecx, %eax
+; X32-NEXT:    # kill: def $ax killed $ax killed $eax
+; X32-NEXT:    retl
+;
+; X64-LABEL: rot16_trunc:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    movl %edi, %ecx
+; X64-NEXT:    shrl $11, %ecx
+; X64-NEXT:    shll $5, %eax
+; X64-NEXT:    orl %ecx, %eax
+; X64-NEXT:    # kill: def $ax killed $ax killed $eax
+; X64-NEXT:    retq
+	%t0 = lshr i32 %x, 11
+	%t1 = shl i32 %x, 5
+	%t2 = or i32 %t0, %t1
+	%t3 = trunc i32 %t2 to i16
+	ret i16 %t3
+}
diff --git a/llvm/test/CodeGen/X86/sadd_sat.ll b/llvm/test/CodeGen/X86/sadd_sat.ll
index e395bb4e04bbc..697ff90c59375 100644
--- a/llvm/test/CodeGen/X86/sadd_sat.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat.ll
@@ -183,28 +183,20 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
+; X64-NEXT:    pxor %xmm2, %xmm2
 ; X64-NEXT:    pxor %xmm3, %xmm3
-; X64-NEXT:    pxor %xmm4, %xmm4
-; X64-NEXT:    pcmpgtd %xmm1, %xmm4
-; X64-NEXT:    pcmpeqd %xmm2, %xmm2
-; X64-NEXT:    pxor %xmm2, %xmm4
-; X64-NEXT:    pxor %xmm5, %xmm5
-; X64-NEXT:    pcmpgtd %xmm0, %xmm5
-; X64-NEXT:    pxor %xmm2, %xmm5
-; X64-NEXT:    pcmpeqd %xmm5, %xmm4
-; X64-NEXT:    paddd %xmm1, %xmm0
-; X64-NEXT:    pcmpgtd %xmm0, %xmm3
-; X64-NEXT:    pxor %xmm3, %xmm2
-; X64-NEXT:    pcmpeqd %xmm5, %xmm2
-; X64-NEXT:    pandn %xmm4, %xmm2
-; X64-NEXT:    movdqa %xmm3, %xmm1
-; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
-; X64-NEXT:    psrld $1, %xmm3
-; X64-NEXT:    por %xmm1, %xmm3
-; X64-NEXT:    pand %xmm2, %xmm3
-; X64-NEXT:    pandn %xmm0, %xmm2
+; X64-NEXT:    pcmpgtd %xmm1, %xmm3
+; X64-NEXT:    paddd %xmm0, %xmm1
+; X64-NEXT:    pcmpgtd %xmm1, %xmm0
+; X64-NEXT:    pxor %xmm3, %xmm0
+; X64-NEXT:    pcmpgtd %xmm1, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm3
+; X64-NEXT:    pandn {{.*}}(%rip), %xmm3
+; X64-NEXT:    psrld $1, %xmm2
 ; X64-NEXT:    por %xmm3, %xmm2
-; X64-NEXT:    movdqa %xmm2, %xmm0
+; X64-NEXT:    pand %xmm0, %xmm2
+; X64-NEXT:    pandn %xmm1, %xmm0
+; X64-NEXT:    por %xmm2, %xmm0
 ; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y);
   ret <4 x i32> %tmp;
diff --git a/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
index 4d18515381014..94c6e46ea9628 100644
--- a/llvm/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
@@ -569,133 +569,88 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    psrld $1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm3, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    psrld $1, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    paddd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    pandn %xmm4, %xmm3
-; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    paddd %xmm1, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.sadd.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -704,133 +659,88 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
+; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm3, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm2, %xmm3
-; SSE2-NEXT:    pandn %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm3
+; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    psrld $1, %xmm2
 ; SSE2-NEXT:    por %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm3, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm2, %xmm3
-; SSSE3-NEXT:    pandn %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm3
+; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    psrld $1, %xmm2
 ; SSSE3-NEXT:    por %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm3, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    paddd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    pandn %xmm4, %xmm3
-; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    paddd %xmm1, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvps %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvps %xmm2, %xmm3, %xmm4, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvps %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i32> @llvm.sadd.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -839,214 +749,135 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm8
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm7
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSE2-NEXT:    paddd %xmm2, %xmm8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    paddd %xmm0, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm7, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm2, %xmm7
-; SSE2-NEXT:    pandn %xmm4, %xmm7
-; SSE2-NEXT:    psrld $1, %xmm2
-; SSE2-NEXT:    por %xmm7, %xmm2
-; SSE2-NEXT:    pand %xmm0, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm0
-; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm5, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrld $1, %xmm5
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    pand %xmm0, %xmm5
+; SSE2-NEXT:    pandn %xmm2, %xmm0
+; SSE2-NEXT:    por %xmm5, %xmm0
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
 ; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm4, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm1, %xmm5
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    paddd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm3, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm8
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm7
-; SSSE3-NEXT:    paddd %xmm2, %xmm8
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    paddd %xmm0, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
 ; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm7, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm2, %xmm7
-; SSSE3-NEXT:    pandn %xmm4, %xmm7
-; SSSE3-NEXT:    psrld $1, %xmm2
-; SSSE3-NEXT:    por %xmm7, %xmm2
-; SSSE3-NEXT:    pand %xmm0, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm0
-; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm7
+; SSSE3-NEXT:    pandn %xmm6, %xmm7
+; SSSE3-NEXT:    psrld $1, %xmm5
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    pand %xmm0, %xmm5
+; SSSE3-NEXT:    pandn %xmm2, %xmm0
+; SSSE3-NEXT:    por %xmm5, %xmm0
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    paddd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm6, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm4, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm1, %xmm5
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    paddd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm3, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm7
-; SSE41-NEXT:    paddd %xmm2, %xmm6
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm7, %xmm2
-; SSE41-NEXT:    movaps {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm5 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm5, %xmm7
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm7
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm6
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    paddd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE41-NEXT:    pandn %xmm0, %xmm4
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    paddd %xmm2, %xmm5
+; SSE41-NEXT:    movaps {{.*#+}} xmm8 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm6, %xmm7
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm4
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    movaps %xmm6, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    paddd %xmm3, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm8, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnps %ymm8, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm5
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm5, {{.*}}(%rip), %ymm6, %ymm6
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vblendvps %ymm0, %ymm6, %ymm5, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
 ; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vblendvps %ymm0, %ymm3, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
+; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -1055,378 +886,230 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; SSE2-LABEL: v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm11
+; SSE2-NEXT:    pxor %xmm8, %xmm8
+; SSE2-NEXT:    pxor %xmm9, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE2-NEXT:    paddd %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm9, %xmm0
 ; SSE2-NEXT:    pxor %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm12, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm12
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm12
-; SSE2-NEXT:    paddd %xmm4, %xmm11
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm10
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm10, %xmm11
+; SSE2-NEXT:    pandn %xmm9, %xmm11
+; SSE2-NEXT:    psrld $1, %xmm10
+; SSE2-NEXT:    por %xmm11, %xmm10
+; SSE2-NEXT:    pand %xmm0, %xmm10
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm10, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm11, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm12, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm12, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    paddd %xmm1, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm4, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm10
+; SSE2-NEXT:    pandn %xmm9, %xmm10
 ; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm11, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm11
-; SSE2-NEXT:    pxor %xmm9, %xmm11
+; SSE2-NEXT:    por %xmm10, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm5, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSE2-NEXT:    paddd %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm11, %xmm1
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm12, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm5
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    pand %xmm1, %xmm5
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    paddd %xmm2, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    paddd %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm8
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2-NEXT:    pandn %xmm4, %xmm8
-; SSE2-NEXT:    movdqa %xmm6, %xmm4
-; SSE2-NEXT:    pandn %xmm12, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm5
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pandn %xmm6, %xmm2
+; SSE2-NEXT:    por %xmm4, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    paddd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm9
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE2-NEXT:    pandn %xmm2, %xmm9
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
-; SSE2-NEXT:    pandn %xmm12, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm10
-; SSE2-NEXT:    por %xmm2, %xmm10
-; SSE2-NEXT:    pand %xmm9, %xmm10
-; SSE2-NEXT:    pandn %xmm3, %xmm9
-; SSE2-NEXT:    por %xmm10, %xmm9
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSE2-NEXT:    paddd %xmm3, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT:    pxor %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT:    movdqa %xmm8, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm4
+; SSE2-NEXT:    psrld $1, %xmm8
+; SSE2-NEXT:    por %xmm4, %xmm8
+; SSE2-NEXT:    pand %xmm3, %xmm8
+; SSE2-NEXT:    pandn %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm8, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v16i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm11
+; SSSE3-NEXT:    pxor %xmm8, %xmm8
+; SSSE3-NEXT:    pxor %xmm9, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSSE3-NEXT:    paddd %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
 ; SSSE3-NEXT:    pxor %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm12, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm12
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm9, %xmm12
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm12
-; SSSE3-NEXT:    paddd %xmm4, %xmm11
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm10
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm10, %xmm11
+; SSSE3-NEXT:    pandn %xmm9, %xmm11
+; SSSE3-NEXT:    psrld $1, %xmm10
+; SSSE3-NEXT:    por %xmm11, %xmm10
+; SSSE3-NEXT:    pand %xmm0, %xmm10
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm10, %xmm0
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm12, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm12, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    paddd %xmm1, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm4, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm10
+; SSSE3-NEXT:    pandn %xmm9, %xmm10
 ; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pand %xmm0, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm0
-; SSSE3-NEXT:    por %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm11, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm11
-; SSSE3-NEXT:    pxor %xmm9, %xmm11
+; SSSE3-NEXT:    por %xmm10, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm5, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSSE3-NEXT:    paddd %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm11, %xmm1
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pandn %xmm12, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm5
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    pand %xmm1, %xmm5
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    paddd %xmm2, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm4, %xmm2
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    paddd %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    movdqa %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm9, %xmm8
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSSE3-NEXT:    pandn %xmm4, %xmm8
-; SSSE3-NEXT:    movdqa %xmm6, %xmm4
-; SSSE3-NEXT:    pandn %xmm12, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm5
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pandn %xmm6, %xmm2
+; SSSE3-NEXT:    por %xmm4, %xmm2
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSSE3-NEXT:    paddd %xmm7, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm9
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSSE3-NEXT:    pandn %xmm2, %xmm9
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
-; SSSE3-NEXT:    pandn %xmm12, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm10
-; SSSE3-NEXT:    por %xmm2, %xmm10
-; SSSE3-NEXT:    pand %xmm9, %xmm10
-; SSSE3-NEXT:    pandn %xmm3, %xmm9
-; SSSE3-NEXT:    por %xmm10, %xmm9
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
+; SSSE3-NEXT:    paddd %xmm3, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSSE3-NEXT:    pxor %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSSE3-NEXT:    movdqa %xmm8, %xmm4
+; SSSE3-NEXT:    pandn %xmm9, %xmm4
+; SSSE3-NEXT:    psrld $1, %xmm8
+; SSSE3-NEXT:    por %xmm4, %xmm8
+; SSSE3-NEXT:    pand %xmm3, %xmm8
+; SSSE3-NEXT:    pandn %xmm7, %xmm3
+; SSSE3-NEXT:    por %xmm8, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v16i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm2, %xmm12
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    pxor %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm11, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm11
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm11
-; SSE41-NEXT:    pxor %xmm12, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm11
 ; SSE41-NEXT:    paddd %xmm4, %xmm9
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
-; SSE41-NEXT:    pandn %xmm11, %xmm4
-; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm11, %xmm13
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm13
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm9
-; SSE41-NEXT:    xorps %xmm13, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm13
-; SSE41-NEXT:    pxor %xmm10, %xmm13
-; SSE41-NEXT:    pxor %xmm14, %xmm14
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm14
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    pcmpeqd %xmm14, %xmm13
-; SSE41-NEXT:    paddd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSE41-NEXT:    pandn %xmm13, %xmm4
-; SSE41-NEXT:    movaps %xmm11, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm13, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm13
-; SSE41-NEXT:    pxor %xmm10, %xmm13
-; SSE41-NEXT:    xorps %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSE41-NEXT:    paddd %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm13, %xmm4
-; SSE41-NEXT:    movaps %xmm11, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm10, %xmm2
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm9
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    paddd %xmm5, %xmm4
+; SSE41-NEXT:    movaps %xmm10, %xmm2
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT:    paddd %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm10
-; SSE41-NEXT:    pandn %xmm0, %xmm10
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm4
+; SSE41-NEXT:    movdqa %xmm12, %xmm3
+; SSE41-NEXT:    paddd %xmm6, %xmm3
+; SSE41-NEXT:    movaps %xmm10, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm12
+; SSE41-NEXT:    pxor %xmm6, %xmm12
+; SSE41-NEXT:    movdqa %xmm12, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm8, %xmm5
+; SSE41-NEXT:    paddd %xmm7, %xmm5
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm10
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE41-NEXT:    pxor %xmm7, %xmm8
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm11
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    blendvps %xmm0, %xmm10, %xmm5
 ; SSE41-NEXT:    movaps %xmm9, %xmm0
+; SSE41-NEXT:    movaps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm10
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm6, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm10, %xmm10
-; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
-; AVX1-NEXT:    vandnps %ymm8, %ymm4, %ymm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm7
 ; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm10 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm10, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm4, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm7, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
-; AVX1-NEXT:    vpaddd %xmm4, %xmm2, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm11, %xmm12, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm12, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vandnps %ymm9, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvps %ymm3, %ymm8, %ymm10, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm9 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm9, %ymm10
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vblendvps %ymm0, %ymm10, %ymm7, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm6
+; AVX1-NEXT:    vblendvps %ymm6, %ymm8, %ymm9, %ymm7
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vblendvps %ymm1, %ymm7, %ymm6, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v16i32:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqd %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm8 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvps %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vblendvps %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm4
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm4, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vblendvps %ymm0, %ymm7, %ymm4, %ymm0
+; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvps %ymm2, %ymm5, %ymm6, %ymm4
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vblendvps %ymm1, %ymm4, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k0
+; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -1436,152 +1119,120 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE2-LABEL: v2i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
 ; SSE2-NEXT:    paddq %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
 ; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm1
 ; SSE2-NEXT:    pxor %xmm5, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm2, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm3, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSE2-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSE2-NEXT:    por %xmm2, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm3
 ; SSE2-NEXT:    pandn %xmm0, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm3, %xmm1
 ; SSE2-NEXT:    movdqa %xmm1, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
 ; SSSE3-NEXT:    paddq %xmm1, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
 ; SSSE3-NEXT:    pxor %xmm2, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm1
 ; SSSE3-NEXT:    pxor %xmm5, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm2, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm3, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
-; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm3
+; SSSE3-NEXT:    por %xmm2, %xmm3
+; SSSE3-NEXT:    pand %xmm1, %xmm3
 ; SSSE3-NEXT:    pandn %xmm0, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm3, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    paddq %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm1
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
 ; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    pandn %xmm4, %xmm1
 ; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -1591,56 +1242,36 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ;
 ; AVX1-LABEL: v2i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm2
 ; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpandn %xmm3, %xmm2, %xmm2
+; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm2
 ; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm2, {{.*}}(%rip), %xmm3, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm1, %xmm0
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm3, %xmm2, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k0
+; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i64> @llvm.sadd.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -1649,369 +1280,279 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE2-LABEL: v4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm9
-; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE2-NEXT:    paddq %xmm2, %xmm9
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm0
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    paddq %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm5, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
 ; SSE2-NEXT:    por %xmm0, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pxor %xmm6, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm8, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
 ; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm8, %xmm5
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    pand %xmm0, %xmm5
-; SSE2-NEXT:    pandn %xmm9, %xmm0
-; SSE2-NEXT:    por %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
 ; SSE2-NEXT:    paddq %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm3
-; SSE2-NEXT:    movdqa %xmm10, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm6, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    movdqa %xmm10, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE2-NEXT:    pand %xmm6, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm8, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm11, %xmm3
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pand %xmm2, %xmm4
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm9
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSSE3-NEXT:    paddq %xmm2, %xmm9
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm0
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    paddq %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm0
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm5, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm0, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm7
-; SSSE3-NEXT:    pxor %xmm6, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm0
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm8, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
 ; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm8, %xmm5
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    pand %xmm0, %xmm5
-; SSSE3-NEXT:    pandn %xmm9, %xmm0
-; SSSE3-NEXT:    por %xmm5, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
 ; SSSE3-NEXT:    paddq %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm3
-; SSSE3-NEXT:    movdqa %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm2
+; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm6, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    movdqa %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSSE3-NEXT:    pand %xmm6, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm8, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm11, %xmm3
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pand %xmm2, %xmm4
+; SSSE3-NEXT:    pand %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pandn %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm4, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    paddq %xmm2, %xmm9
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm2
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    paddq %xmm2, %xmm8
+; SSE41-NEXT:    movdqa %xmm8, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm7, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE41-NEXT:    pand %xmm7, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm2, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm8 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm8, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm7, %xmm2
-; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm5
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm0
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm9 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm6, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm9
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
 ; SSE41-NEXT:    paddq %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm10, %xmm3
-; SSE41-NEXT:    movdqa %xmm10, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
-; SSE41-NEXT:    pandn %xmm2, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm11
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm7
+; SSE41-NEXT:    por %xmm0, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm3
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm10, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm6
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    movapd %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddq %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnpd %ymm8, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT:    vpaddq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm5
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm5, {{.*}}(%rip), %ymm6, %ymm6
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm1, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm6, %ymm5, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpandn %ymm3, %ymm2, %ymm2
+; AVX2-NEXT:    vpaddq %ymm1, %ymm0, %ymm2
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
 ; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm4, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm3, %ymm2, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k0
+; AVX512-NEXT:    vpaddq %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -2021,687 +1562,513 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE2-LABEL: v8i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
+; SSE2-NEXT:    movdqa %xmm0, %xmm12
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE2-NEXT:    paddq %xmm4, %xmm13
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm1
 ; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    paddq %xmm4, %xmm12
+; SSE2-NEXT:    movdqa %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm14
-; SSE2-NEXT:    pxor %xmm10, %xmm14
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm11
-; SSE2-NEXT:    movdqa %xmm13, %xmm0
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm10
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm9, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm0
-; SSE2-NEXT:    pandn %xmm11, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm8, %xmm1
-; SSE2-NEXT:    paddq %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm10, %xmm0
 ; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm15
-; SSE2-NEXT:    pxor %xmm10, %xmm15
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm13
+; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm10, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm11, %xmm4
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa %xmm8, %xmm1
 ; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
+; SSE2-NEXT:    paddq %xmm5, %xmm8
+; SSE2-NEXT:    movdqa %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm1
+; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm12
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
 ; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm10, %xmm4
+; SSE2-NEXT:    pand %xmm11, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
 ; SSE2-NEXT:    paddq %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm8
 ; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
 ; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm8, %xmm6
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm6, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm6
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm4
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
 ; SSE2-NEXT:    paddq %xmm7, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm9, %xmm2
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
 ; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm7, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm7, %xmm10
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm10
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
-; SSE2-NEXT:    pand %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm2, %xmm5
-; SSE2-NEXT:    movdqa %xmm7, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    pand %xmm12, %xmm7
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pand %xmm5, %xmm7
-; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm6
 ; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm10, %xmm2
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm13
+; SSSE3-NEXT:    movdqa %xmm0, %xmm12
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSSE3-NEXT:    paddq %xmm4, %xmm13
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
 ; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    paddq %xmm4, %xmm12
+; SSSE3-NEXT:    movdqa %xmm12, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm11, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm14
-; SSSE3-NEXT:    pxor %xmm10, %xmm14
-; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm1[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm11
-; SSSE3-NEXT:    movdqa %xmm13, %xmm0
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm10
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm9, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm14, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm0
-; SSSE3-NEXT:    pandn %xmm11, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    pand %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm10, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm11, %xmm4
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
 ; SSSE3-NEXT:    paddq %xmm5, %xmm8
+; SSSE3-NEXT:    movdqa %xmm8, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm12
 ; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm9, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm15
-; SSSE3-NEXT:    pxor %xmm10, %xmm15
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm13
-; SSSE3-NEXT:    movdqa %xmm8, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm14, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm12, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
 ; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm10, %xmm4
+; SSSE3-NEXT:    pand %xmm11, %xmm5
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm1, %xmm5
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
 ; SSSE3-NEXT:    paddq %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm8
 ; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm12, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm8
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm8, %xmm6
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm6, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm6
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
 ; SSSE3-NEXT:    paddq %xmm7, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm9, %xmm2
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
 ; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm7, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pand %xmm8, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    pxor %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm7, %xmm10
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm10
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm10[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm2, %xmm5
-; SSSE3-NEXT:    movdqa %xmm7, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    pand %xmm12, %xmm7
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pand %xmm5, %xmm7
-; SSSE3-NEXT:    pandn %xmm3, %xmm5
-; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm6
 ; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pandn %xmm10, %xmm2
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
-; SSE41-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    paddq %xmm4, %xmm8
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    movdqa %xmm8, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm12
+; SSE41-NEXT:    por %xmm0, %xmm12
+; SSE41-NEXT:    pxor %xmm9, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
 ; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm4
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE41-NEXT:    pxor %xmm9, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm11
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm11
 ; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm11
-; SSE41-NEXT:    pxor %xmm9, %xmm11
-; SSE41-NEXT:    pcmpeqq %xmm11, %xmm12
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm11, %xmm4
-; SSE41-NEXT:    pandn %xmm12, %xmm4
-; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm11, %xmm13
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm13
+; SSE41-NEXT:    movdqa %xmm9, %xmm12
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm12
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm12, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm10, %xmm12
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm12
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm13, %xmm8
-; SSE41-NEXT:    movdqa %xmm1, %xmm14
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm8
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    paddq %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm15
-; SSE41-NEXT:    pxor %xmm9, %xmm15
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm1, %xmm12
+; SSE41-NEXT:    pxor %xmm9, %xmm12
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm14[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm5
+; SSE41-NEXT:    pand %xmm4, %xmm13
+; SSE41-NEXT:    por %xmm0, %xmm13
 ; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm15
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm15, %xmm4
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pand %xmm14, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm13, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm13
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm13
+; SSE41-NEXT:    movdqa %xmm9, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm13, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm10, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    paddq %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm5
-; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm9, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm6, %xmm5
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
+; SSE41-NEXT:    movdqa %xmm2, %xmm12
+; SSE41-NEXT:    pxor %xmm9, %xmm12
 ; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm6, %xmm4
-; SSE41-NEXT:    pandn %xmm5, %xmm4
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm5
+; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm6
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm5
+; SSE41-NEXT:    movdqa %xmm9, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm10, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    paddq %xmm7, %xmm3
-; SSE41-NEXT:    pxor %xmm10, %xmm7
-; SSE41-NEXT:    movdqa %xmm10, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
 ; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm10, %xmm0
-; SSE41-NEXT:    movdqa %xmm10, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    pxor %xmm0, %xmm9
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm9
-; SSE41-NEXT:    pandn %xmm4, %xmm9
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm9, %xmm7
 ; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm7, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSE41-NEXT:    pand %xmm12, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm6
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm10
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
 ; SSE41-NEXT:    movapd %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm12, %xmm12, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm10
-; AVX1-NEXT:    vpcmpeqq %xmm8, %xmm10, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm6, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpaddq %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm10, %xmm10
-; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm12, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
-; AVX1-NEXT:    vandnpd %ymm8, %ymm4, %ymm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm5
+; AVX1-NEXT:    vpaddq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpaddq %xmm2, %xmm0, %xmm6
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm6, %ymm7
 ; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm10 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm10, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm4, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm7, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm11, %ymm9
-; AVX1-NEXT:    vpaddq %xmm4, %xmm2, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm11, %xmm12, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm12, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm5, %xmm7, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm2
-; AVX1-NEXT:    vandnpd %ymm9, %ymm2, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm3, %ymm8, %ymm10, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm9 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm9, %ymm10
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm2, %ymm0
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm10, %ymm7, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm4
+; AVX1-NEXT:    vpaddq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpaddq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm5, %ymm6
+; AVX1-NEXT:    vblendvpd %ymm6, %ymm8, %ymm9, %ymm7
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vxorpd %ymm1, %ymm3, %ymm1
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm7, %ymm6, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i64:
 ; AVX2:       # %bb.0:
-; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqq %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpandn %ymm2, %ymm4, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpaddq %ymm2, %ymm0, %ymm4
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm4, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm7, %ymm4, %ymm0
+; AVX2-NEXT:    vpaddq %ymm3, %ymm1, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm5, %ymm6, %ymm4
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm3, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm4, %ymm2, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k0
+; AVX512-NEXT:    vpaddq %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
index 7dc44f182cb1f..158bab5f2f1d5 100644
--- a/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
+++ b/llvm/test/CodeGen/X86/scalar-fp-to-i64.ll
@@ -90,7 +90,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; AVX512F_32_WIN-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX512F_32_WIN-NEXT:    vcmpltss %xmm1, %xmm0, %k1
 ; AVX512F_32_WIN-NEXT:    vsubss %xmm1, %xmm0, %xmm2
-; AVX512F_32_WIN-NEXT:    vmovss %xmm0, %xmm0, %xmm2 {%k1}
+; AVX512F_32_WIN-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
 ; AVX512F_32_WIN-NEXT:    vmovss %xmm2, (%esp)
 ; AVX512F_32_WIN-NEXT:    flds (%esp)
 ; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
@@ -111,7 +111,7 @@ define i64 @f_to_u64(float %a) nounwind {
 ; AVX512F_32_LIN-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; AVX512F_32_LIN-NEXT:    vcmpltss %xmm1, %xmm0, %k1
 ; AVX512F_32_LIN-NEXT:    vsubss %xmm1, %xmm0, %xmm2
-; AVX512F_32_LIN-NEXT:    vmovss %xmm0, %xmm0, %xmm2 {%k1}
+; AVX512F_32_LIN-NEXT:    vmovss %xmm0, %xmm2, %xmm2 {%k1}
 ; AVX512F_32_LIN-NEXT:    vmovss %xmm2, (%esp)
 ; AVX512F_32_LIN-NEXT:    flds (%esp)
 ; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
@@ -580,7 +580,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; AVX512F_32_WIN-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX512F_32_WIN-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
 ; AVX512F_32_WIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; AVX512F_32_WIN-NEXT:    vmovsd %xmm0, %xmm0, %xmm2 {%k1}
+; AVX512F_32_WIN-NEXT:    vmovsd %xmm0, %xmm2, %xmm2 {%k1}
 ; AVX512F_32_WIN-NEXT:    vmovsd %xmm2, (%esp)
 ; AVX512F_32_WIN-NEXT:    fldl (%esp)
 ; AVX512F_32_WIN-NEXT:    fisttpll (%esp)
@@ -601,7 +601,7 @@ define i64 @d_to_u64(double %a) nounwind {
 ; AVX512F_32_LIN-NEXT:    vmovsd {{.*#+}} xmm1 = mem[0],zero
 ; AVX512F_32_LIN-NEXT:    vcmpltsd %xmm1, %xmm0, %k1
 ; AVX512F_32_LIN-NEXT:    vsubsd %xmm1, %xmm0, %xmm2
-; AVX512F_32_LIN-NEXT:    vmovsd %xmm0, %xmm0, %xmm2 {%k1}
+; AVX512F_32_LIN-NEXT:    vmovsd %xmm0, %xmm2, %xmm2 {%k1}
 ; AVX512F_32_LIN-NEXT:    vmovsd %xmm2, (%esp)
 ; AVX512F_32_LIN-NEXT:    fldl (%esp)
 ; AVX512F_32_LIN-NEXT:    fisttpll (%esp)
diff --git a/llvm/test/CodeGen/X86/scalarize-fp.ll b/llvm/test/CodeGen/X86/scalarize-fp.ll
index 650b948d3abd0..5b98272fdce5e 100644
--- a/llvm/test/CodeGen/X86/scalarize-fp.ll
+++ b/llvm/test/CodeGen/X86/scalarize-fp.ll
@@ -773,3 +773,26 @@ define <8 x float> @splat0_fdiv_const_op0_v8f32(<8 x float> %vx) {
   %r = shufflevector <8 x float> %b, <8 x float> undef, <8 x i32> zeroinitializer
   ret <8 x float> %r
 }
+
+define <4 x float> @multi_use_binop(<4 x float> %x, <4 x float> %y) {
+; SSE-LABEL: multi_use_binop:
+; SSE:       # %bb.0:
+; SSE-NEXT:    mulps %xmm1, %xmm0
+; SSE-NEXT:    movlhps {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE-NEXT:    shufps {{.*#+}} xmm0 = xmm0[0,1,2,0]
+; SSE-NEXT:    addps %xmm1, %xmm0
+; SSE-NEXT:    retq
+;
+; AVX-LABEL: multi_use_binop:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmulps %xmm1, %xmm0, %xmm0
+; AVX-NEXT:    vpermilps {{.*#+}} xmm1 = xmm0[0,1,2,0]
+; AVX-NEXT:    vmovddup {{.*#+}} xmm0 = xmm0[0,0]
+; AVX-NEXT:    vaddps %xmm0, %xmm1, %xmm0
+; AVX-NEXT:    retq
+  %mul = fmul <4 x float> %x, %y
+  %mul0 = shufflevector <4 x float> %mul, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 0>
+  %mul1 = shufflevector <4 x float> %mul, <4 x float> undef, <4 x i32> <i32 undef, i32 undef, i32 undef, i32 1>
+  %r = fadd <4 x float> %mul0, %mul1
+  ret <4 x float> %r
+}
diff --git a/llvm/test/CodeGen/X86/select-of-fp-constants.ll b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
index cc9ac9e56cda7..f492b255d887d 100644
--- a/llvm/test/CodeGen/X86/select-of-fp-constants.ll
+++ b/llvm/test/CodeGen/X86/select-of-fp-constants.ll
@@ -61,7 +61,7 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ; X32_AVX512F-LABEL: fcmp_select_fp_constants:
 ; X32_AVX512F:       # %bb.0:
 ; X32_AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X32_AVX512F-NEXT:    vcmpneqss {{[0-9]+}}(%esp), %xmm0, %k0
+; X32_AVX512F-NEXT:    vcmpneqss {{\.LCPI.*}}, %xmm0, %k0
 ; X32_AVX512F-NEXT:    kmovw %k0, %eax
 ; X32_AVX512F-NEXT:    flds {{\.LCPI.*}}(,%eax,4)
 ; X32_AVX512F-NEXT:    retl
@@ -84,10 +84,9 @@ define float @fcmp_select_fp_constants(float %x) nounwind readnone {
 ;
 ; X64_AVX512F-LABEL: fcmp_select_fp_constants:
 ; X64_AVX512F:       # %bb.0:
-; X64_AVX512F-NEXT:    vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; X64_AVX512F-NEXT:    vcmpneqss {{.*}}(%rip), %xmm0, %k1
 ; X64_AVX512F-NEXT:    vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; X64_AVX512F-NEXT:    vmovss %xmm1, %xmm0, %xmm0 {%k1}
+; X64_AVX512F-NEXT:    vmovss {{.*}}(%rip), %xmm0 {%k1}
 ; X64_AVX512F-NEXT:    retq
  %c = fcmp une float %x, -4.0
  %r = select i1 %c, float 42.0, float 23.0
diff --git a/llvm/test/CodeGen/X86/shift-by-signext.ll b/llvm/test/CodeGen/X86/shift-by-signext.ll
new file mode 100644
index 0000000000000..85ca2bd6e1d39
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shift-by-signext.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+bmi,+bmi2,+sse,+sse2,+avx,+avx2 | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+bmi,+bmi2,+sse,+sse2,+avx,+avx2 | FileCheck %s --check-prefixes=CHECK,X64
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t0_shl:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t0_shl:
+; X64:       # %bb.0:
+; X64-NEXT:    shlxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t1_lshr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t1_lshr:
+; X64:       # %bb.0:
+; X64-NEXT:    shrxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) nounwind {
+; X86-LABEL: t2_ashr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %al
+; X86-NEXT:    sarxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: t2_ashr:
+; X64:       # %bb.0:
+; X64-NEXT:    sarxl %esi, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <4 x i32> @t3_vec_shl(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t3_vec_shl:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsllvd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = shl <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t4_vec_lshr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t4_vec_lshr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsrlvd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = lshr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+define <4 x i32> @t5_vec_ashr(<4 x i32> %x, <4 x i8> %shamt) nounwind {
+; CHECK-LABEL: t5_vec_ashr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpmovsxbd %xmm1, %xmm1
+; CHECK-NEXT:    vpsravd %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    ret{{[l|q]}}
+  %shamt_wide = sext <4 x i8> %shamt to <4 x i32>
+  %r = ashr <4 x i32> %x, %shamt_wide
+  ret <4 x i32> %r
+}
+
+; This is not valid for funnel shifts
+declare i32 @llvm.fshl.i32(i32 %a, i32 %b, i32 %c)
+declare i32 @llvm.fshr.i32(i32 %a, i32 %b, i32 %c)
+define i32 @n6_fshl(i32 %x, i32 %y, i8 %shamt) nounwind {
+; X86-LABEL: n6_fshl:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shldl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n6_fshl:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %edi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shldl %cl, %esi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+define i32 @n7_fshr(i32 %x, i32 %y, i8 %shamt) nounwind {
+; X86-LABEL: n7_fshr:
+; X86:       # %bb.0:
+; X86-NEXT:    movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    shrdl %cl, %edx, %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n7_fshr:
+; X64:       # %bb.0:
+; X64-NEXT:    movl %edx, %ecx
+; X64-NEXT:    movl %esi, %eax
+; X64-NEXT:    # kill: def $cl killed $cl killed $ecx
+; X64-NEXT:    shrdl %cl, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  %r = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %shamt_wide)
+  ret i32 %r
+}
+
+define i32 @n8_extrause(i32 %x, i8 %shamt, i32* %shamt_wide_store) nounwind {
+; X86-LABEL: n8_extrause:
+; X86:       # %bb.0:
+; X86-NEXT:    movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl %eax, (%ecx)
+; X86-NEXT:    shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    retl
+;
+; X64-LABEL: n8_extrause:
+; X64:       # %bb.0:
+; X64-NEXT:    movsbl %sil, %eax
+; X64-NEXT:    movl %eax, (%rdx)
+; X64-NEXT:    shlxl %eax, %edi, %eax
+; X64-NEXT:    retq
+  %shamt_wide = sext i8 %shamt to i32
+  store i32 %shamt_wide, i32* %shamt_wide_store, align 4
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
diff --git a/llvm/test/CodeGen/X86/sqrt-fastmath.ll b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
index 6e0273d513fc6..6aad4e8f69eb9 100644
--- a/llvm/test/CodeGen/X86/sqrt-fastmath.ll
+++ b/llvm/test/CodeGen/X86/sqrt-fastmath.ll
@@ -94,7 +94,7 @@ define float @finite_f32_estimate(float %f) #1 {
 ; AVX512-NEXT:    vmulss %xmm1, %xmm2, %xmm1
 ; AVX512-NEXT:    vxorps %xmm2, %xmm2, %xmm2
 ; AVX512-NEXT:    vcmpeqss %xmm2, %xmm0, %k1
-; AVX512-NEXT:    vmovss %xmm2, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm2, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %call = tail call float @__sqrtf_finite(float %f) #2
@@ -165,7 +165,7 @@ define float @sqrtf_check_denorms(float %x) #3 {
 ; AVX512-NEXT:    vandps %xmm2, %xmm0, %xmm0
 ; AVX512-NEXT:    vcmpltss {{.*}}(%rip), %xmm0, %k1
 ; AVX512-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; AVX512-NEXT:    vmovss %xmm0, %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovss %xmm0, %xmm1, %xmm1 {%k1}
 ; AVX512-NEXT:    vmovaps %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %call = tail call float @__sqrtf_finite(float %x) #2
diff --git a/llvm/test/CodeGen/X86/sqrt-partial.ll b/llvm/test/CodeGen/X86/sqrt-partial.ll
index d017b5c360085..7ed68c1084998 100644
--- a/llvm/test/CodeGen/X86/sqrt-partial.ll
+++ b/llvm/test/CodeGen/X86/sqrt-partial.ll
@@ -67,7 +67,6 @@ define double @minsize(double %x, double %y) minsize {
 ; SSE-NEXT:    mulsd %xmm0, %xmm0
 ; SSE-NEXT:    mulsd %xmm1, %xmm1
 ; SSE-NEXT:    addsd %xmm0, %xmm1
-; SSE-NEXT:    xorps %xmm0, %xmm0
 ; SSE-NEXT:    sqrtsd %xmm1, %xmm0
 ; SSE-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/ssub_sat.ll b/llvm/test/CodeGen/X86/ssub_sat.ll
index dafae884eb1ce..dcfa086603b2e 100644
--- a/llvm/test/CodeGen/X86/ssub_sat.ll
+++ b/llvm/test/CodeGen/X86/ssub_sat.ll
@@ -183,30 +183,20 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
 ;
 ; X64-LABEL: vec:
 ; X64:       # %bb.0:
-; X64-NEXT:    movdqa %xmm0, %xmm2
-; X64-NEXT:    pxor %xmm3, %xmm3
-; X64-NEXT:    pxor %xmm0, %xmm0
-; X64-NEXT:    pcmpgtd %xmm1, %xmm0
-; X64-NEXT:    pcmpeqd %xmm4, %xmm4
-; X64-NEXT:    pxor %xmm4, %xmm0
-; X64-NEXT:    pxor %xmm5, %xmm5
-; X64-NEXT:    pcmpgtd %xmm2, %xmm5
-; X64-NEXT:    pxor %xmm4, %xmm5
-; X64-NEXT:    pcmpeqd %xmm5, %xmm0
-; X64-NEXT:    psubd %xmm1, %xmm2
-; X64-NEXT:    pcmpgtd %xmm2, %xmm3
-; X64-NEXT:    movdqa %xmm3, %xmm1
-; X64-NEXT:    pxor %xmm4, %xmm1
-; X64-NEXT:    pcmpeqd %xmm5, %xmm1
-; X64-NEXT:    pxor %xmm4, %xmm1
-; X64-NEXT:    pandn %xmm1, %xmm0
-; X64-NEXT:    movdqa %xmm3, %xmm1
+; X64-NEXT:    pxor %xmm2, %xmm2
+; X64-NEXT:    movdqa %xmm0, %xmm3
+; X64-NEXT:    psubd %xmm1, %xmm3
+; X64-NEXT:    pcmpgtd %xmm2, %xmm1
+; X64-NEXT:    pcmpgtd %xmm3, %xmm0
+; X64-NEXT:    pxor %xmm1, %xmm0
+; X64-NEXT:    pcmpgtd %xmm3, %xmm2
+; X64-NEXT:    movdqa %xmm2, %xmm1
 ; X64-NEXT:    pandn {{.*}}(%rip), %xmm1
-; X64-NEXT:    psrld $1, %xmm3
-; X64-NEXT:    por %xmm1, %xmm3
-; X64-NEXT:    pand %xmm0, %xmm3
-; X64-NEXT:    pandn %xmm2, %xmm0
-; X64-NEXT:    por %xmm3, %xmm0
+; X64-NEXT:    psrld $1, %xmm2
+; X64-NEXT:    por %xmm1, %xmm2
+; X64-NEXT:    pand %xmm0, %xmm2
+; X64-NEXT:    pandn %xmm3, %xmm0
+; X64-NEXT:    por %xmm2, %xmm0
 ; X64-NEXT:    retq
   %tmp = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %tmp
diff --git a/llvm/test/CodeGen/X86/ssub_sat_vec.ll b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
index dd8d546506206..43e8fca83044d 100644
--- a/llvm/test/CodeGen/X86/ssub_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/ssub_sat_vec.ll
@@ -569,141 +569,94 @@ define <16 x i1> @v16i1(<16 x i1> %x, <16 x i1> %y) nounwind {
 define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 ; SSE2-LABEL: v2i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE2-NEXT:    psubd %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v2i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSSE3-NEXT:    psubd %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v2i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pandn %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
 ; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i32> @llvm.ssub.sat.v2i32(<2 x i32> %x, <2 x i32> %y)
   ret <2 x i32> %z
@@ -712,141 +665,94 @@ define <2 x i32> @v2i32(<2 x i32> %x, <2 x i32> %y) nounwind {
 define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 ; SSE2-LABEL: v4i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSE2-NEXT:    psubd %xmm1, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm4, %xmm1
-; SSE2-NEXT:    pandn %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm3, %xmm1
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSE2-NEXT:    psrld $1, %xmm3
-; SSE2-NEXT:    por %xmm1, %xmm3
-; SSE2-NEXT:    pand %xmm0, %xmm3
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    por %xmm3, %xmm0
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm1, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm3, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
-; SSSE3-NEXT:    psubd %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm4, %xmm1
-; SSSE3-NEXT:    pandn %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm3, %xmm1
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm1
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm1
-; SSSE3-NEXT:    psrld $1, %xmm3
-; SSSE3-NEXT:    por %xmm1, %xmm3
-; SSSE3-NEXT:    pand %xmm0, %xmm3
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    por %xmm3, %xmm0
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm1, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm3, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
 ; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    psubd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pandn %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm1, %xmm2
 ; SSE41-NEXT:    movaps {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm2
-; SSE41-NEXT:    movaps %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, {{.*}}(%rip), %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
+; SSE41-NEXT:    movaps %xmm3, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovaps {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovaps {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqd %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm4 = [2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %xmm1, %xmm3, %xmm4, %xmm1
-; AVX2-NEXT:    vblendvps %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %xmm1, %xmm2, %xmm3, %xmm2
+; AVX2-NEXT:    vblendvps %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm1 = [2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i32> @llvm.ssub.sat.v4i32(<4 x i32> %x, <4 x i32> %y)
   ret <4 x i32> %z
@@ -855,226 +761,144 @@ define <4 x i32> @v4i32(<4 x i32> %x, <4 x i32> %y) nounwind {
 define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 ; SSE2-LABEL: v8i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE2-NEXT:    pxor %xmm8, %xmm0
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm8, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSE2-NEXT:    psubd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm6, %xmm2
-; SSE2-NEXT:    pandn %xmm7, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm6
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pand %xmm0, %xmm6
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    por %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm8, %xmm2
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm8, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT:    psubd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pxor %xmm8, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm8, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm7, %xmm3
-; SSE2-NEXT:    psrld $1, %xmm5
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
-; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm5
+; SSE2-NEXT:    psubd %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm2, %xmm7
+; SSE2-NEXT:    pandn %xmm6, %xmm7
+; SSE2-NEXT:    psrld $1, %xmm2
+; SSE2-NEXT:    por %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm5, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    psubd %xmm3, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pxor %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm3
+; SSE2-NEXT:    pandn %xmm6, %xmm3
+; SSE2-NEXT:    psrld $1, %xmm4
+; SSE2-NEXT:    por %xmm3, %xmm4
+; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pandn %xmm2, %xmm1
+; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSSE3-NEXT:    pxor %xmm8, %xmm0
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm8, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm0
-; SSSE3-NEXT:    psubd %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm6, %xmm2
-; SSSE3-NEXT:    pandn %xmm7, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm6
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pand %xmm0, %xmm6
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    por %xmm6, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm8, %xmm2
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm8, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
-; SSSE3-NEXT:    psubd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pxor %xmm8, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm8, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm7, %xmm3
-; SSSE3-NEXT:    psrld $1, %xmm5
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm2, %xmm5
-; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm5
+; SSSE3-NEXT:    psubd %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm7
+; SSSE3-NEXT:    pandn %xmm6, %xmm7
+; SSSE3-NEXT:    psrld $1, %xmm2
+; SSSE3-NEXT:    por %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm5, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    psubd %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pxor %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm3
+; SSSE3-NEXT:    pandn %xmm6, %xmm3
+; SSSE3-NEXT:    psrld $1, %xmm4
+; SSSE3-NEXT:    por %xmm3, %xmm4
+; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pandn %xmm2, %xmm1
+; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v8i32:
 ; SSE41:       # %bb.0:
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm8, %xmm8
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm8, %xmm8
-; SSE41-NEXT:    pxor %xmm8, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pxor %xmm8, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
 ; SSE41-NEXT:    psubd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pandn %xmm2, %xmm6
-; SSE41-NEXT:    movaps {{.*#+}} xmm9 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm7 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm7, %xmm2
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm2
-; SSE41-NEXT:    movdqa %xmm6, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm5
-; SSE41-NEXT:    xorps %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm8, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm8, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT:    psubd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    movdqa %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm8, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm8, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
+; SSE41-NEXT:    pxor %xmm2, %xmm4
+; SSE41-NEXT:    movaps {{.*#+}} xmm7 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm6 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm2
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm9, %xmm7
+; SSE41-NEXT:    blendvps %xmm0, %xmm2, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm2
+; SSE41-NEXT:    psubd %xmm3, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm8, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pxor %xmm3, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm1
+; SSE41-NEXT:    blendvps %xmm0, %xmm7, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm6, %xmm2
 ; SSE41-NEXT:    movaps %xmm5, %xmm0
+; SSE41-NEXT:    movaps %xmm2, %xmm1
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnps %ymm2, %ymm8, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm2 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm1, {{.*}}(%rip), %ymm2, %ymm2
+; AVX1-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm4 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvps %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm2 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm3 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvps %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} ymm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i32> @llvm.ssub.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
   ret <8 x i32> %z
@@ -1083,399 +907,244 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
 define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
 ; SSE2-LABEL: v16i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm12
-; SSE2-NEXT:    pxor %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm0, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm0
-; SSE2-NEXT:    pxor %xmm11, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm11
-; SSE2-NEXT:    pxor %xmm10, %xmm11
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE2-NEXT:    psubd %xmm4, %xmm12
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm12, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    psrld $1, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm12, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    pxor %xmm12, %xmm12
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm12
-; SSE2-NEXT:    pxor %xmm10, %xmm12
-; SSE2-NEXT:    pcmpeqd %xmm12, %xmm1
-; SSE2-NEXT:    psubd %xmm5, %xmm8
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm12, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
 ; SSE2-NEXT:    pxor %xmm8, %xmm8
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSE2-NEXT:    pxor %xmm10, %xmm8
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSE2-NEXT:    psubd %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm9
+; SSE2-NEXT:    psubd %xmm4, %xmm9
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE2-NEXT:    pxor %xmm4, %xmm0
 ; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSE2-NEXT:    movdqa %xmm4, %xmm11
+; SSE2-NEXT:    pandn %xmm10, %xmm11
 ; SSE2-NEXT:    psrld $1, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm4, %xmm8
+; SSE2-NEXT:    por %xmm11, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    psubd %xmm5, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm1
 ; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSE2-NEXT:    psubd %xmm7, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm9
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    psrld $1, %xmm9
-; SSE2-NEXT:    por %xmm2, %xmm9
-; SSE2-NEXT:    pand %xmm5, %xmm9
-; SSE2-NEXT:    pandn %xmm3, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm9
+; SSE2-NEXT:    pandn %xmm10, %xmm9
+; SSE2-NEXT:    psrld $1, %xmm5
 ; SSE2-NEXT:    por %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
+; SSE2-NEXT:    pand %xmm1, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    psubd %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE2-NEXT:    pxor %xmm6, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pandn %xmm10, %xmm6
+; SSE2-NEXT:    psrld $1, %xmm5
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm4, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psubd %xmm7, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm7, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSE2-NEXT:    movdqa %xmm8, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    psrld $1, %xmm8
+; SSE2-NEXT:    por %xmm5, %xmm8
+; SSE2-NEXT:    pand %xmm3, %xmm8
+; SSE2-NEXT:    pandn %xmm4, %xmm3
+; SSE2-NEXT:    por %xmm8, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v16i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm12
-; SSSE3-NEXT:    pxor %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm0, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm0
-; SSSE3-NEXT:    pxor %xmm11, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm11
-; SSSE3-NEXT:    pxor %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSSE3-NEXT:    psubd %xmm4, %xmm12
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm12, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm11, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    psrld $1, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm12, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm1, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm1
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    pxor %xmm12, %xmm12
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm12
-; SSSE3-NEXT:    pxor %xmm10, %xmm12
-; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm1
-; SSSE3-NEXT:    psubd %xmm5, %xmm8
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm12, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
 ; SSSE3-NEXT:    pxor %xmm8, %xmm8
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm8
-; SSSE3-NEXT:    pxor %xmm10, %xmm8
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm8
-; SSSE3-NEXT:    psubd %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm9
+; SSSE3-NEXT:    psubd %xmm4, %xmm9
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSSE3-NEXT:    pxor %xmm4, %xmm0
 ; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm8
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [2147483648,2147483648,2147483648,2147483648]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm11
+; SSSE3-NEXT:    pandn %xmm10, %xmm11
 ; SSSE3-NEXT:    psrld $1, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm4, %xmm8
+; SSSE3-NEXT:    por %xmm11, %xmm4
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm9, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    psubd %xmm5, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm1
 ; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
-; SSSE3-NEXT:    psubd %xmm7, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm9
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    psrld $1, %xmm9
-; SSSE3-NEXT:    por %xmm2, %xmm9
-; SSSE3-NEXT:    pand %xmm5, %xmm9
-; SSSE3-NEXT:    pandn %xmm3, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm9
+; SSSE3-NEXT:    pandn %xmm10, %xmm9
+; SSSE3-NEXT:    psrld $1, %xmm5
 ; SSSE3-NEXT:    por %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
+; SSSE3-NEXT:    pand %xmm1, %xmm5
+; SSSE3-NEXT:    pandn %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    psubd %xmm6, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSSE3-NEXT:    pxor %xmm6, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pandn %xmm10, %xmm6
+; SSSE3-NEXT:    psrld $1, %xmm5
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm4, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psubd %xmm7, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm7, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm8
+; SSSE3-NEXT:    movdqa %xmm8, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
+; SSSE3-NEXT:    psrld $1, %xmm8
+; SSSE3-NEXT:    por %xmm5, %xmm8
+; SSSE3-NEXT:    pand %xmm3, %xmm8
+; SSSE3-NEXT:    pandn %xmm4, %xmm3
+; SSSE3-NEXT:    por %xmm8, %xmm3
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v16i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    pxor %xmm8, %xmm8
+; SSE41-NEXT:    movdqa %xmm3, %xmm8
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
 ; SSE41-NEXT:    pxor %xmm10, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE41-NEXT:    pxor %xmm11, %xmm10
-; SSE41-NEXT:    pxor %xmm12, %xmm12
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE41-NEXT:    pxor %xmm11, %xmm12
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm10
+; SSE41-NEXT:    movdqa %xmm0, %xmm9
 ; SSE41-NEXT:    psubd %xmm4, %xmm9
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm10
-; SSE41-NEXT:    movaps {{.*#+}} xmm13 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483648,2147483648,2147483648,2147483648]
-; SSE41-NEXT:    movaps %xmm12, %xmm4
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm4
-; SSE41-NEXT:    movdqa %xmm10, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm3
+; SSE41-NEXT:    pxor %xmm4, %xmm3
+; SSE41-NEXT:    movaps {{.*#+}} xmm12 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT:    movaps {{.*#+}} xmm11 = [2147483648,2147483648,2147483648,2147483648]
+; SSE41-NEXT:    movaps %xmm11, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm4
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
 ; SSE41-NEXT:    blendvps %xmm0, %xmm4, %xmm9
-; SSE41-NEXT:    xorps %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm10
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm10
-; SSE41-NEXT:    pxor %xmm11, %xmm10
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm4
-; SSE41-NEXT:    psubd %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
-; SSE41-NEXT:    movaps %xmm12, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psubd %xmm5, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm1
+; SSE41-NEXT:    movaps %xmm11, %xmm3
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    xorps %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE41-NEXT:    psubd %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm11, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm11, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
-; SSE41-NEXT:    movaps %xmm12, %xmm5
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm5
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pxor %xmm11, %xmm4
-; SSE41-NEXT:    pxor %xmm0, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT:    psubd %xmm7, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm8
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm3
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm3, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    psubd %xmm6, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
+; SSE41-NEXT:    pxor %xmm6, %xmm2
+; SSE41-NEXT:    movaps %xmm11, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm1
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm1, %xmm3
 ; SSE41-NEXT:    movdqa %xmm8, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm11, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
+; SSE41-NEXT:    psubd %xmm7, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE41-NEXT:    pxor %xmm7, %xmm8
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm11
 ; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm13, %xmm12
-; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvps %xmm0, %xmm12, %xmm3
+; SSE41-NEXT:    blendvps %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movaps %xmm9, %xmm0
+; SSE41-NEXT:    movaps %xmm4, %xmm1
+; SSE41-NEXT:    movaps %xmm3, %xmm2
+; SSE41-NEXT:    movaps %xmm5, %xmm3
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm12
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm12, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm11, %xmm5, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpsubd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
-; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm10, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm5, %ymm5
-; AVX1-NEXT:    vandnps %ymm5, %ymm8, %ymm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm8 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX1-NEXT:    vmovaps {{.*#+}} ymm11 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX1-NEXT:    vblendvps %ymm7, %ymm8, %ymm11, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvps %ymm5, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX1-NEXT:    vpsubd %xmm5, %xmm2, %xmm12
-; AVX1-NEXT:    vpcmpgtd %xmm12, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpeqd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm10, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
-; AVX1-NEXT:    vandnps %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvps %ymm3, %ymm8, %ymm11, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm12, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm4 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX1-NEXT:    vmovaps {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX1-NEXT:    vblendvps %ymm2, %ymm4, %ymm6, %ymm7
+; AVX1-NEXT:    vblendvps %ymm0, %ymm7, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpsubd %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
+; AVX1-NEXT:    vxorps %ymm1, %ymm5, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvps %ymm2, %ymm4, %ymm6, %ymm3
+; AVX1-NEXT:    vblendvps %ymm1, %ymm3, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v16i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqd %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm7, %ymm5, %ymm5
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm7 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
-; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm8 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX2-NEXT:    vblendvps %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvps %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vblendvps %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvps %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm5 = [2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647,2147483647]
+; AVX2-NEXT:    vbroadcastss {{.*#+}} ymm6 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX2-NEXT:    vblendvps %ymm2, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vblendvps %ymm0, %ymm7, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendvps %ymm3, %ymm5, %ymm6, %ymm2
+; AVX2-NEXT:    vblendvps %ymm1, %ymm2, %ymm3, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtd %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm1 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
-; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtd %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastd {{.*#+}} zmm0 = [2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648,2147483648]
+; AVX512-NEXT:    vpbroadcastd {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa32 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <16 x i32> @llvm.ssub.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
   ret <16 x i32> %z
@@ -1485,50 +1154,38 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE2-LABEL: v2i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    pxor %xmm2, %xmm4
 ; SSE2-NEXT:    psubq %xmm1, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    pxor %xmm2, %xmm3
+; SSE2-NEXT:    movdqa %xmm4, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
 ; SSE2-NEXT:    pxor %xmm2, %xmm1
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm1
+; SSE2-NEXT:    pxor %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    pand %xmm5, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    por %xmm2, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pandn {{.*}}(%rip), %xmm2
 ; SSE2-NEXT:    pand {{.*}}(%rip), %xmm3
 ; SSE2-NEXT:    por %xmm2, %xmm3
@@ -1541,50 +1198,38 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSSE3-LABEL: v2i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    pxor %xmm2, %xmm4
 ; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm0, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm3
+; SSSE3-NEXT:    movdqa %xmm4, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pxor %xmm2, %xmm1
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm1
+; SSSE3-NEXT:    pxor %xmm5, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
 ; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    pand %xmm5, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    por %xmm2, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
 ; SSSE3-NEXT:    pandn {{.*}}(%rip), %xmm2
 ; SSSE3-NEXT:    pand {{.*}}(%rip), %xmm3
 ; SSSE3-NEXT:    por %xmm2, %xmm3
@@ -1597,46 +1242,32 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; SSE41-LABEL: v2i64:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm2
-; SSE41-NEXT:    movdqa {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
+; SSE41-NEXT:    movdqa {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm3, %xmm0
 ; SSE41-NEXT:    psubq %xmm1, %xmm2
-; SSE41-NEXT:    pxor %xmm0, %xmm1
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm1
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm0, %xmm3
+; SSE41-NEXT:    movdqa %xmm2, %xmm4
+; SSE41-NEXT:    pxor %xmm3, %xmm4
 ; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pxor %xmm0, %xmm3
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
+; SSE41-NEXT:    pxor %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm3, %xmm1
+; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE41-NEXT:    pand %xmm5, %xmm1
+; SSE41-NEXT:    por %xmm0, %xmm1
+; SSE41-NEXT:    pxor %xmm6, %xmm1
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
 ; SSE41-NEXT:    por %xmm3, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm1
 ; SSE41-NEXT:    movapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
 ; SSE41-NEXT:    blendvpd %xmm0, {{.*}}(%rip), %xmm3
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
@@ -1647,57 +1278,39 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 ; AVX1-LABEL: v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX1-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX1-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX1-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX1-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX1-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
-; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm1
-; AVX2-NEXT:    vpxor %xmm4, %xmm1, %xmm2
-; AVX2-NEXT:    vpcmpeqq %xmm2, %xmm5, %xmm2
-; AVX2-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX2-NEXT:    vpandn %xmm2, %xmm3, %xmm2
-; AVX2-NEXT:    vmovapd {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm3, %xmm1
-; AVX2-NEXT:    vblendvpd %xmm2, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
+; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
+; AVX2-NEXT:    vmovapd {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %xmm1, {{.*}}(%rip), %xmm2, %xmm2
+; AVX2-NEXT:    vblendvpd %xmm0, %xmm2, %xmm1, %xmm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %xmm0, %xmm2, %k2
-; AVX512-NEXT:    vmovdqa {{.*#+}} xmm1 = [9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %xmm1, %xmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %xmm2, %xmm1, %k0
+; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k2
+; AVX512-NEXT:    vmovdqa {{.*#+}} xmm0 = [9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vmovdqa64 {{.*}}(%rip), %xmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %xmm0, %xmm1 {%k1}
+; AVX512-NEXT:    vmovdqa %xmm1, %xmm0
 ; AVX512-NEXT:    retq
   %z = call <2 x i64> @llvm.ssub.sat.v2i64(<2 x i64> %x, <2 x i64> %y)
   ret <2 x i64> %z
@@ -1706,381 +1319,285 @@ define <2 x i64> @v2i64(<2 x i64> %x, <2 x i64> %y) nounwind {
 define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
 ; SSE2-LABEL: v4i64:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa %xmm0, %xmm10
-; SSE2-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSE2-NEXT:    psubq %xmm2, %xmm10
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm0, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT:    pxor %xmm8, %xmm0
+; SSE2-NEXT:    psubq %xmm2, %xmm4
+; SSE2-NEXT:    movdqa %xmm4, %xmm6
+; SSE2-NEXT:    pxor %xmm8, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm7
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm5, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm0
-; SSE2-NEXT:    movdqa %xmm5, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm0
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSE2-NEXT:    pand %xmm7, %xmm0
-; SSE2-NEXT:    movdqa %xmm10, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    movdqa %xmm5, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE2-NEXT:    por %xmm2, %xmm0
+; SSE2-NEXT:    pxor %xmm5, %xmm0
+; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    pandn %xmm2, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pandn %xmm8, %xmm2
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm0, %xmm4
-; SSE2-NEXT:    pandn %xmm10, %xmm0
-; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pand %xmm5, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm2
+; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pandn %xmm9, %xmm5
+; SSE2-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807]
+; SSE2-NEXT:    pand %xmm7, %xmm2
+; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    pand %xmm0, %xmm2
+; SSE2-NEXT:    pandn %xmm4, %xmm0
+; SSE2-NEXT:    por %xmm2, %xmm0
 ; SSE2-NEXT:    movdqa %xmm1, %xmm2
+; SSE2-NEXT:    pxor %xmm8, %xmm2
 ; SSE2-NEXT:    psubq %xmm3, %xmm1
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm1, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm6, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm8, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm2
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm5, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pand %xmm6, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSE2-NEXT:    por %xmm3, %xmm2
+; SSE2-NEXT:    pxor %xmm5, %xmm2
+; SSE2-NEXT:    movdqa %xmm8, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movdqa %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm8, %xmm3
-; SSE2-NEXT:    pand %xmm11, %xmm5
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pand %xmm2, %xmm5
+; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    pandn %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm7, %xmm3
+; SSE2-NEXT:    por %xmm4, %xmm3
+; SSE2-NEXT:    pand %xmm2, %xmm3
 ; SSE2-NEXT:    pandn %xmm1, %xmm2
-; SSE2-NEXT:    por %xmm5, %xmm2
+; SSE2-NEXT:    por %xmm3, %xmm2
 ; SSE2-NEXT:    movdqa %xmm2, %xmm1
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v4i64:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa %xmm0, %xmm10
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
-; SSSE3-NEXT:    psubq %xmm2, %xmm10
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm0, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSSE3-NEXT:    pxor %xmm8, %xmm0
+; SSSE3-NEXT:    psubq %xmm2, %xmm4
+; SSSE3-NEXT:    movdqa %xmm4, %xmm6
+; SSSE3-NEXT:    pxor %xmm8, %xmm6
+; SSSE3-NEXT:    movdqa %xmm0, %xmm7
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm5, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm9
-; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm0
-; SSSE3-NEXT:    movdqa %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm0
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm7, %xmm0
-; SSSE3-NEXT:    movdqa %xmm10, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    movdqa %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSSE3-NEXT:    por %xmm2, %xmm0
+; SSSE3-NEXT:    pxor %xmm5, %xmm0
+; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    pandn %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm8 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pandn %xmm8, %xmm2
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm11, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm0, %xmm4
-; SSSE3-NEXT:    pandn %xmm10, %xmm0
-; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pand %xmm5, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pandn %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm7 = [9223372036854775807,9223372036854775807]
+; SSSE3-NEXT:    pand %xmm7, %xmm2
+; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    pand %xmm0, %xmm2
+; SSSE3-NEXT:    pandn %xmm4, %xmm0
+; SSSE3-NEXT:    por %xmm2, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm2
+; SSSE3-NEXT:    pxor %xmm8, %xmm2
 ; SSSE3-NEXT:    psubq %xmm3, %xmm1
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm1, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm6, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm8, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm8, %xmm2
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm3
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm2
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pand %xmm6, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
+; SSSE3-NEXT:    por %xmm3, %xmm2
+; SSSE3-NEXT:    pxor %xmm5, %xmm2
+; SSSE3-NEXT:    movdqa %xmm8, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm8, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm3, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movdqa %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm8, %xmm3
-; SSSE3-NEXT:    pand %xmm11, %xmm5
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pand %xmm2, %xmm5
+; SSSE3-NEXT:    pand %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    pandn %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm7, %xmm3
+; SSSE3-NEXT:    por %xmm4, %xmm3
+; SSSE3-NEXT:    pand %xmm2, %xmm3
 ; SSSE3-NEXT:    pandn %xmm1, %xmm2
-; SSSE3-NEXT:    por %xmm5, %xmm2
+; SSSE3-NEXT:    por %xmm3, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm1
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: v4i64:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa %xmm0, %xmm9
-; SSE41-NEXT:    movdqa {{.*#+}} xmm11 = [2147483648,2147483648]
-; SSE41-NEXT:    psubq %xmm2, %xmm9
-; SSE41-NEXT:    pxor %xmm11, %xmm2
-; SSE41-NEXT:    movdqa %xmm11, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; SSE41-NEXT:    movdqa %xmm0, %xmm8
+; SSE41-NEXT:    movdqa {{.*#+}} xmm5 = [2147483648,2147483648]
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    psubq %xmm2, %xmm8
+; SSE41-NEXT:    movdqa %xmm8, %xmm6
+; SSE41-NEXT:    pxor %xmm5, %xmm6
+; SSE41-NEXT:    movdqa %xmm0, %xmm7
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm7, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE41-NEXT:    por %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm4, %xmm2
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; SSE41-NEXT:    por %xmm5, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm4, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm2
-; SSE41-NEXT:    movapd {{.*#+}} xmm8 = [9223372036854775807,9223372036854775807]
-; SSE41-NEXT:    movapd {{.*#+}} xmm7 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm7, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm9
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    psubq %xmm3, %xmm1
-; SSE41-NEXT:    pxor %xmm11, %xmm3
-; SSE41-NEXT:    movdqa %xmm11, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm2[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm4, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
 ; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm2
-; SSE41-NEXT:    pxor %xmm10, %xmm2
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pand %xmm7, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm4, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm5, %xmm7
+; SSE41-NEXT:    pcmpgtd %xmm6, %xmm7
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm10, %xmm3
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm2
+; SSE41-NEXT:    por %xmm7, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm9 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm6 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm6, %xmm4
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm4
+; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm11, %xmm0
-; SSE41-NEXT:    movdqa %xmm11, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm3, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm2
-; SSE41-NEXT:    blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    psubq %xmm3, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
+; SSE41-NEXT:    movdqa %xmm0, %xmm2
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm2, %xmm7
+; SSE41-NEXT:    por %xmm0, %xmm7
+; SSE41-NEXT:    pxor %xmm5, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
+; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSE41-NEXT:    pand %xmm10, %xmm2
+; SSE41-NEXT:    por %xmm0, %xmm2
+; SSE41-NEXT:    pxor %xmm7, %xmm2
+; SSE41-NEXT:    movdqa %xmm5, %xmm3
+; SSE41-NEXT:    pcmpeqd %xmm4, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT:    pand %xmm3, %xmm0
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm9, %xmm6
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT:    movapd %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm6, %xmm1
+; SSE41-NEXT:    movapd %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v4i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtq %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqq %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubq %xmm2, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm7, %xmm2
-; AVX1-NEXT:    vpxor %xmm5, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm3, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm4, %xmm3
-; AVX1-NEXT:    vpxor %xmm5, %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-NEXT:    vandnpd %ymm2, %ymm8, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm1, %ymm1
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm3, %ymm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm4, %xmm4
+; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm3, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm1, {{.*}}(%rip), %ymm2, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v4i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm3, %ymm5, %ymm3
-; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm2, %ymm1
-; AVX2-NEXT:    vpxor %ymm4, %ymm1, %ymm2
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpxor %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vpandn %ymm2, %ymm3, %ymm2
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm1, %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm1, %ymm2
+; AVX2-NEXT:    vpsubq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm3, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm2, %ymm1, %ymm0
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v4i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    vpcmpnltq %ymm2, %ymm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %ymm0, %ymm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %ymm1, %ymm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %ymm2, %ymm1, %k0
+; AVX512-NEXT:    vpsubq %ymm1, %ymm0, %ymm1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %ymm1, %ymm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %ymm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %ymm0, %ymm1 {%k1}
+; AVX512-NEXT:    vmovdqa %ymm1, %ymm0
 ; AVX512-NEXT:    retq
   %z = call <4 x i64> @llvm.ssub.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
   ret <4 x i64> %z
@@ -2090,414 +1607,324 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE2-LABEL: v8i64:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movdqa %xmm1, %xmm8
-; SSE2-NEXT:    movdqa %xmm0, %xmm13
+; SSE2-NEXT:    movdqa %xmm0, %xmm12
 ; SSE2-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE2-NEXT:    psubq %xmm4, %xmm13
+; SSE2-NEXT:    pxor %xmm9, %xmm0
+; SSE2-NEXT:    psubq %xmm4, %xmm12
+; SSE2-NEXT:    movdqa %xmm12, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
+; SSE2-NEXT:    movdqa %xmm0, %xmm10
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm10
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT:    pand %xmm11, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSE2-NEXT:    por %xmm0, %xmm10
 ; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm4, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm10, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE2-NEXT:    pxor %xmm10, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm0
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSE2-NEXT:    pand %xmm11, %xmm4
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm11, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm15
-; SSE2-NEXT:    pxor %xmm10, %xmm15
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSE2-NEXT:    pand %xmm1, %xmm0
-; SSE2-NEXT:    movdqa %xmm13, %xmm1
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm11
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT:    por %xmm4, %xmm0
+; SSE2-NEXT:    pxor %xmm10, %xmm0
+; SSE2-NEXT:    movdqa %xmm9, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm12, %xmm14
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSE2-NEXT:    por %xmm14, %xmm1
-; SSE2-NEXT:    movdqa %xmm1, %xmm11
-; SSE2-NEXT:    pxor %xmm10, %xmm11
-; SSE2-NEXT:    pcmpeqd %xmm15, %xmm11
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm10, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSE2-NEXT:    movdqa %xmm4, %xmm1
+; SSE2-NEXT:    pandn %xmm10, %xmm1
+; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
 ; SSE2-NEXT:    pand %xmm11, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pandn %xmm4, %xmm0
-; SSE2-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE2-NEXT:    movdqa %xmm1, %xmm4
-; SSE2-NEXT:    pandn %xmm11, %xmm4
-; SSE2-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE2-NEXT:    pand %xmm12, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
-; SSE2-NEXT:    pand %xmm0, %xmm1
-; SSE2-NEXT:    pandn %xmm13, %xmm0
-; SSE2-NEXT:    por %xmm1, %xmm0
+; SSE2-NEXT:    por %xmm1, %xmm4
+; SSE2-NEXT:    pand %xmm0, %xmm4
+; SSE2-NEXT:    pandn %xmm12, %xmm0
+; SSE2-NEXT:    por %xmm4, %xmm0
 ; SSE2-NEXT:    movdqa %xmm8, %xmm1
+; SSE2-NEXT:    pxor %xmm9, %xmm1
 ; SSE2-NEXT:    psubq %xmm5, %xmm8
+; SSE2-NEXT:    movdqa %xmm8, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    movdqa %xmm1, %xmm12
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT:    pand %xmm13, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT:    por %xmm1, %xmm12
 ; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm1
+; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm1
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm1
-; SSE2-NEXT:    movdqa %xmm8, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm4
-; SSE2-NEXT:    movdqa %xmm9, %xmm13
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT:    por %xmm5, %xmm1
+; SSE2-NEXT:    pxor %xmm12, %xmm1
+; SSE2-NEXT:    movdqa %xmm9, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm14, %xmm15
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
-; SSE2-NEXT:    por %xmm15, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm13
-; SSE2-NEXT:    pxor %xmm10, %xmm13
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
-; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pandn %xmm5, %xmm1
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm1, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm5, %xmm4
+; SSE2-NEXT:    pandn %xmm10, %xmm4
+; SSE2-NEXT:    pand %xmm11, %xmm5
+; SSE2-NEXT:    por %xmm4, %xmm5
+; SSE2-NEXT:    pand %xmm1, %xmm5
 ; SSE2-NEXT:    pandn %xmm8, %xmm1
-; SSE2-NEXT:    por %xmm4, %xmm1
+; SSE2-NEXT:    por %xmm5, %xmm1
 ; SSE2-NEXT:    movdqa %xmm2, %xmm4
+; SSE2-NEXT:    pxor %xmm9, %xmm4
 ; SSE2-NEXT:    psubq %xmm6, %xmm2
+; SSE2-NEXT:    movdqa %xmm2, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm8
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm12, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSE2-NEXT:    por %xmm4, %xmm8
 ; SSE2-NEXT:    pxor %xmm9, %xmm6
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm6, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm6, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm4
+; SSE2-NEXT:    pand %xmm12, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    por %xmm6, %xmm4
+; SSE2-NEXT:    pxor %xmm8, %xmm4
 ; SSE2-NEXT:    movdqa %xmm9, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE2-NEXT:    por %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm8
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pxor %xmm9, %xmm5
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    pand %xmm13, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pxor %xmm10, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm8
-; SSE2-NEXT:    movdqa %xmm4, %xmm5
-; SSE2-NEXT:    pandn %xmm11, %xmm5
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm5, %xmm4
-; SSE2-NEXT:    pand %xmm8, %xmm4
-; SSE2-NEXT:    pandn %xmm2, %xmm8
-; SSE2-NEXT:    por %xmm4, %xmm8
-; SSE2-NEXT:    movdqa %xmm3, %xmm2
-; SSE2-NEXT:    psubq %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm9, %xmm7
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm6
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm5
+; SSE2-NEXT:    pandn %xmm10, %xmm5
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pand %xmm4, %xmm6
+; SSE2-NEXT:    pandn %xmm2, %xmm4
 ; SSE2-NEXT:    por %xmm6, %xmm4
-; SSE2-NEXT:    pxor %xmm10, %xmm4
-; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm5
+; SSE2-NEXT:    movdqa %xmm3, %xmm5
+; SSE2-NEXT:    pxor %xmm9, %xmm5
+; SSE2-NEXT:    psubq %xmm7, %xmm3
 ; SSE2-NEXT:    movdqa %xmm3, %xmm2
 ; SSE2-NEXT:    pxor %xmm9, %xmm2
-; SSE2-NEXT:    movdqa %xmm9, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSE2-NEXT:    movdqa %xmm5, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm5, %xmm6
+; SSE2-NEXT:    pxor %xmm9, %xmm7
+; SSE2-NEXT:    movdqa %xmm7, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE2-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSE2-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT:    pand %xmm8, %xmm7
+; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSE2-NEXT:    por %xmm7, %xmm5
+; SSE2-NEXT:    pxor %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm9, %xmm6
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSE2-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSE2-NEXT:    pand %xmm7, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pxor %xmm10, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm6
-; SSE2-NEXT:    pxor %xmm10, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm5
-; SSE2-NEXT:    movdqa %xmm4, %xmm2
-; SSE2-NEXT:    pandn %xmm11, %xmm2
-; SSE2-NEXT:    pand %xmm12, %xmm4
-; SSE2-NEXT:    por %xmm2, %xmm4
-; SSE2-NEXT:    pand %xmm5, %xmm4
+; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    movdqa %xmm6, %xmm2
+; SSE2-NEXT:    pandn %xmm10, %xmm2
+; SSE2-NEXT:    pand %xmm11, %xmm6
+; SSE2-NEXT:    por %xmm2, %xmm6
+; SSE2-NEXT:    pand %xmm5, %xmm6
 ; SSE2-NEXT:    pandn %xmm3, %xmm5
-; SSE2-NEXT:    por %xmm4, %xmm5
-; SSE2-NEXT:    movdqa %xmm8, %xmm2
+; SSE2-NEXT:    por %xmm6, %xmm5
+; SSE2-NEXT:    movdqa %xmm4, %xmm2
 ; SSE2-NEXT:    movdqa %xmm5, %xmm3
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: v8i64:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movdqa %xmm1, %xmm8
-; SSSE3-NEXT:    movdqa %xmm0, %xmm13
+; SSSE3-NEXT:    movdqa %xmm0, %xmm12
 ; SSSE3-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSSE3-NEXT:    psubq %xmm4, %xmm13
+; SSSE3-NEXT:    pxor %xmm9, %xmm0
+; SSSE3-NEXT:    psubq %xmm4, %xmm12
+; SSSE3-NEXT:    movdqa %xmm12, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
+; SSSE3-NEXT:    movdqa %xmm0, %xmm10
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm10
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm11, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
+; SSSE3-NEXT:    por %xmm0, %xmm10
 ; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm10, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSSE3-NEXT:    pxor %xmm10, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm0
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm11 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm0
+; SSSE3-NEXT:    pand %xmm11, %xmm4
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm11, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm15
-; SSSE3-NEXT:    pxor %xmm10, %xmm15
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm1, %xmm0
-; SSSE3-NEXT:    movdqa %xmm13, %xmm1
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm11
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSSE3-NEXT:    por %xmm4, %xmm0
+; SSSE3-NEXT:    pxor %xmm10, %xmm0
+; SSSE3-NEXT:    movdqa %xmm9, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm10 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm12, %xmm14
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm11[1,1,3,3]
-; SSSE3-NEXT:    por %xmm14, %xmm1
-; SSSE3-NEXT:    movdqa %xmm1, %xmm11
-; SSSE3-NEXT:    pxor %xmm10, %xmm11
-; SSSE3-NEXT:    pcmpeqd %xmm15, %xmm11
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm11[1,0,3,2]
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm10, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSSE3-NEXT:    movdqa %xmm4, %xmm1
+; SSSE3-NEXT:    pandn %xmm10, %xmm1
+; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
 ; SSSE3-NEXT:    pand %xmm11, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pandn %xmm4, %xmm0
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSSE3-NEXT:    movdqa %xmm1, %xmm4
-; SSSE3-NEXT:    pandn %xmm11, %xmm4
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSSE3-NEXT:    pand %xmm12, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
-; SSSE3-NEXT:    pand %xmm0, %xmm1
-; SSSE3-NEXT:    pandn %xmm13, %xmm0
-; SSSE3-NEXT:    por %xmm1, %xmm0
+; SSSE3-NEXT:    por %xmm1, %xmm4
+; SSSE3-NEXT:    pand %xmm0, %xmm4
+; SSSE3-NEXT:    pandn %xmm12, %xmm0
+; SSSE3-NEXT:    por %xmm4, %xmm0
 ; SSSE3-NEXT:    movdqa %xmm8, %xmm1
+; SSSE3-NEXT:    pxor %xmm9, %xmm1
 ; SSSE3-NEXT:    psubq %xmm5, %xmm8
+; SSSE3-NEXT:    movdqa %xmm8, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    movdqa %xmm1, %xmm12
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm12
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm13, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSSE3-NEXT:    por %xmm1, %xmm12
 ; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm1
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm1[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm1
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm1
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm1
-; SSSE3-NEXT:    movdqa %xmm8, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
-; SSSE3-NEXT:    movdqa %xmm9, %xmm13
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSSE3-NEXT:    por %xmm5, %xmm1
+; SSSE3-NEXT:    pxor %xmm12, %xmm1
+; SSSE3-NEXT:    movdqa %xmm9, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm5[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm15 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm14, %xmm15
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm13[1,1,3,3]
-; SSSE3-NEXT:    por %xmm15, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm13
-; SSSE3-NEXT:    pxor %xmm10, %xmm13
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm13
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pandn %xmm5, %xmm1
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm1, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm5, %xmm4
+; SSSE3-NEXT:    pandn %xmm10, %xmm4
+; SSSE3-NEXT:    pand %xmm11, %xmm5
+; SSSE3-NEXT:    por %xmm4, %xmm5
+; SSSE3-NEXT:    pand %xmm1, %xmm5
 ; SSSE3-NEXT:    pandn %xmm8, %xmm1
-; SSSE3-NEXT:    por %xmm4, %xmm1
+; SSSE3-NEXT:    por %xmm5, %xmm1
 ; SSSE3-NEXT:    movdqa %xmm2, %xmm4
+; SSSE3-NEXT:    pxor %xmm9, %xmm4
 ; SSSE3-NEXT:    psubq %xmm6, %xmm2
+; SSSE3-NEXT:    movdqa %xmm2, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm8
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm8
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm8[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm12, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSSE3-NEXT:    por %xmm4, %xmm8
 ; SSSE3-NEXT:    pxor %xmm9, %xmm6
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm6, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm6, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm4
+; SSSE3-NEXT:    pand %xmm12, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    por %xmm6, %xmm4
+; SSSE3-NEXT:    pxor %xmm8, %xmm4
 ; SSSE3-NEXT:    movdqa %xmm9, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm6
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSSE3-NEXT:    por %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm8
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pxor %xmm9, %xmm5
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm5
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm13, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pxor %xmm10, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm8
-; SSSE3-NEXT:    movdqa %xmm4, %xmm5
-; SSSE3-NEXT:    pandn %xmm11, %xmm5
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm5, %xmm4
-; SSSE3-NEXT:    pand %xmm8, %xmm4
-; SSSE3-NEXT:    pandn %xmm2, %xmm8
-; SSSE3-NEXT:    por %xmm4, %xmm8
-; SSSE3-NEXT:    movdqa %xmm3, %xmm2
-; SSSE3-NEXT:    psubq %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm9, %xmm7
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm6
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm5
+; SSSE3-NEXT:    pandn %xmm10, %xmm5
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pand %xmm4, %xmm6
+; SSSE3-NEXT:    pandn %xmm2, %xmm4
 ; SSSE3-NEXT:    por %xmm6, %xmm4
-; SSSE3-NEXT:    pxor %xmm10, %xmm4
-; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm5
+; SSSE3-NEXT:    movdqa %xmm3, %xmm5
+; SSSE3-NEXT:    pxor %xmm9, %xmm5
+; SSSE3-NEXT:    psubq %xmm7, %xmm3
 ; SSSE3-NEXT:    movdqa %xmm3, %xmm2
 ; SSSE3-NEXT:    pxor %xmm9, %xmm2
-; SSSE3-NEXT:    movdqa %xmm9, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm4[0,0,2,2]
+; SSSE3-NEXT:    movdqa %xmm5, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm5, %xmm6
+; SSSE3-NEXT:    pxor %xmm9, %xmm7
+; SSSE3-NEXT:    movdqa %xmm7, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm8 = xmm5[0,0,2,2]
+; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSSE3-NEXT:    pand %xmm8, %xmm7
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
+; SSSE3-NEXT:    por %xmm7, %xmm5
+; SSSE3-NEXT:    pxor %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm9, %xmm6
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
 ; SSSE3-NEXT:    pcmpeqd %xmm9, %xmm2
 ; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
 ; SSSE3-NEXT:    pand %xmm7, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pxor %xmm10, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm6, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm6
-; SSSE3-NEXT:    pxor %xmm10, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm5
-; SSSE3-NEXT:    movdqa %xmm4, %xmm2
-; SSSE3-NEXT:    pandn %xmm11, %xmm2
-; SSSE3-NEXT:    pand %xmm12, %xmm4
-; SSSE3-NEXT:    por %xmm2, %xmm4
-; SSSE3-NEXT:    pand %xmm5, %xmm4
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    movdqa %xmm6, %xmm2
+; SSSE3-NEXT:    pandn %xmm10, %xmm2
+; SSSE3-NEXT:    pand %xmm11, %xmm6
+; SSSE3-NEXT:    por %xmm2, %xmm6
+; SSSE3-NEXT:    pand %xmm5, %xmm6
 ; SSSE3-NEXT:    pandn %xmm3, %xmm5
-; SSSE3-NEXT:    por %xmm4, %xmm5
-; SSSE3-NEXT:    movdqa %xmm8, %xmm2
+; SSSE3-NEXT:    por %xmm6, %xmm5
+; SSSE3-NEXT:    movdqa %xmm4, %xmm2
 ; SSSE3-NEXT:    movdqa %xmm5, %xmm3
 ; SSSE3-NEXT:    retq
 ;
@@ -2505,293 +1932,198 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movdqa %xmm0, %xmm8
 ; SSE41-NEXT:    movdqa {{.*#+}} xmm9 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    psubq %xmm4, %xmm8
+; SSE41-NEXT:    movdqa %xmm8, %xmm10
+; SSE41-NEXT:    pxor %xmm9, %xmm10
+; SSE41-NEXT:    movdqa %xmm0, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm12
+; SSE41-NEXT:    por %xmm0, %xmm12
 ; SSE41-NEXT:    pxor %xmm9, %xmm4
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm4[1,1,3,3]
-; SSE41-NEXT:    pand %xmm10, %xmm12
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm12, %xmm15
-; SSE41-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE41-NEXT:    pxor %xmm10, %xmm15
-; SSE41-NEXT:    pxor %xmm9, %xmm11
-; SSE41-NEXT:    movdqa %xmm9, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm11, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm11 = xmm11[1,1,3,3]
-; SSE41-NEXT:    pand %xmm12, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[1,1,3,3]
-; SSE41-NEXT:    por %xmm11, %xmm12
-; SSE41-NEXT:    pxor %xmm10, %xmm12
-; SSE41-NEXT:    pcmpeqq %xmm12, %xmm15
-; SSE41-NEXT:    movdqa %xmm8, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE41-NEXT:    pand %xmm11, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm12, %xmm4
 ; SSE41-NEXT:    movdqa %xmm9, %xmm11
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm11
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pcmpeqq %xmm12, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm15
-; SSE41-NEXT:    movapd {{.*#+}} xmm12 = [9223372036854775807,9223372036854775807]
-; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775808,9223372036854775808]
-; SSE41-NEXT:    movapd %xmm11, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm4
-; SSE41-NEXT:    movdqa %xmm15, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm4, %xmm8
+; SSE41-NEXT:    pcmpeqd %xmm10, %xmm11
+; SSE41-NEXT:    movdqa %xmm9, %xmm12
+; SSE41-NEXT:    pcmpgtd %xmm10, %xmm12
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm12[0,0,2,2]
+; SSE41-NEXT:    pand %xmm11, %xmm0
+; SSE41-NEXT:    por %xmm12, %xmm0
+; SSE41-NEXT:    movapd {{.*#+}} xmm11 = [9223372036854775807,9223372036854775807]
+; SSE41-NEXT:    movapd {{.*#+}} xmm10 = [9223372036854775808,9223372036854775808]
+; SSE41-NEXT:    movapd %xmm10, %xmm12
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm12
+; SSE41-NEXT:    movdqa %xmm4, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm8
 ; SSE41-NEXT:    movdqa %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    psubq %xmm5, %xmm1
+; SSE41-NEXT:    movdqa %xmm1, %xmm12
+; SSE41-NEXT:    pxor %xmm9, %xmm12
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm13
+; SSE41-NEXT:    por %xmm0, %xmm13
 ; SSE41-NEXT:    pxor %xmm9, %xmm5
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm4[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm5, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm5, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE41-NEXT:    pand %xmm14, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm13, %xmm4
+; SSE41-NEXT:    movdqa %xmm9, %xmm13
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm13
 ; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm5
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
 ; SSE41-NEXT:    pand %xmm13, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm14
-; SSE41-NEXT:    pxor %xmm10, %xmm14
-; SSE41-NEXT:    pcmpeqq %xmm14, %xmm4
-; SSE41-NEXT:    movdqa %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm13
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm13
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm13[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm15 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm15
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
-; SSE41-NEXT:    por %xmm15, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm14, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pandn %xmm5, %xmm4
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    por %xmm5, %xmm0
+; SSE41-NEXT:    movapd %xmm10, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm1
 ; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    psubq %xmm6, %xmm2
+; SSE41-NEXT:    movdqa %xmm2, %xmm12
+; SSE41-NEXT:    pxor %xmm9, %xmm12
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm5
+; SSE41-NEXT:    por %xmm0, %xmm5
 ; SSE41-NEXT:    pxor %xmm9, %xmm6
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm6, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE41-NEXT:    pand %xmm13, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm5, %xmm4
 ; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pcmpeqd %xmm12, %xmm5
 ; SSE41-NEXT:    movdqa %xmm9, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm14 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm14
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm14, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
-; SSE41-NEXT:    movapd %xmm11, %xmm5
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm12, %xmm6
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT:    pand %xmm5, %xmm0
+; SSE41-NEXT:    por %xmm6, %xmm0
+; SSE41-NEXT:    movapd %xmm10, %xmm5
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm5
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
 ; SSE41-NEXT:    blendvpd %xmm0, %xmm5, %xmm2
 ; SSE41-NEXT:    movdqa %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm9, %xmm0
 ; SSE41-NEXT:    psubq %xmm7, %xmm3
+; SSE41-NEXT:    movdqa %xmm3, %xmm5
+; SSE41-NEXT:    pxor %xmm9, %xmm5
+; SSE41-NEXT:    movdqa %xmm0, %xmm4
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
+; SSE41-NEXT:    pand %xmm4, %xmm6
+; SSE41-NEXT:    por %xmm0, %xmm6
 ; SSE41-NEXT:    pxor %xmm9, %xmm7
-; SSE41-NEXT:    movdqa %xmm9, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE41-NEXT:    movdqa %xmm7, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
 ; SSE41-NEXT:    pcmpeqd %xmm9, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm6, %xmm4
-; SSE41-NEXT:    pxor %xmm10, %xmm4
-; SSE41-NEXT:    pxor %xmm9, %xmm0
-; SSE41-NEXT:    movdqa %xmm9, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm10, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm3, %xmm0
-; SSE41-NEXT:    pxor %xmm9, %xmm0
+; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSE41-NEXT:    pand %xmm12, %xmm4
+; SSE41-NEXT:    por %xmm0, %xmm4
+; SSE41-NEXT:    pxor %xmm6, %xmm4
 ; SSE41-NEXT:    movdqa %xmm9, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pshufd {{.*#+}} xmm13 = xmm6[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm9, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm7 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm13, %xmm7
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
-; SSE41-NEXT:    por %xmm7, %xmm0
-; SSE41-NEXT:    movdqa %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm10, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm4
-; SSE41-NEXT:    blendvpd %xmm0, %xmm12, %xmm11
+; SSE41-NEXT:    pcmpeqd %xmm5, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm9
+; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm9[0,0,2,2]
+; SSE41-NEXT:    pand %xmm6, %xmm0
+; SSE41-NEXT:    por %xmm9, %xmm0
+; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm10
 ; SSE41-NEXT:    movdqa %xmm4, %xmm0
-; SSE41-NEXT:    blendvpd %xmm0, %xmm11, %xmm3
+; SSE41-NEXT:    blendvpd %xmm0, %xmm10, %xmm3
 ; SSE41-NEXT:    movapd %xmm8, %xmm0
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: v8i64:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm9
-; AVX1-NEXT:    vpxor %xmm10, %xmm10, %xmm10
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtq %xmm7, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm12
-; AVX1-NEXT:    vpcmpeqq %xmm8, %xmm12, %xmm8
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm11
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm11, %xmm5, %xmm11
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm11, %ymm8
-; AVX1-NEXT:    vpsubq %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm9, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm12, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm11
-; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm10, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm11, %ymm5, %ymm5
-; AVX1-NEXT:    vandnpd %ymm5, %ymm8, %ymm5
-; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm2, %ymm7
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm8 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX1-NEXT:    vmovapd {{.*#+}} ymm11 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX1-NEXT:    vblendvpd %ymm7, %ymm8, %ymm11, %ymm7
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm0, %ymm0
-; AVX1-NEXT:    vblendvpd %ymm5, %ymm7, %ymm0, %ymm0
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm5
-; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm10, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm7, %xmm6, %xmm9
-; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm10, %xmm7
-; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqq %xmm12, %xmm7, %xmm12
-; AVX1-NEXT:    vinsertf128 $1, %xmm9, %ymm12, %ymm9
-; AVX1-NEXT:    vpsubq %xmm5, %xmm2, %xmm12
-; AVX1-NEXT:    vpcmpgtq %xmm12, %xmm10, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm2
-; AVX1-NEXT:    vpcmpeqq %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm10, %xmm3
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm6
-; AVX1-NEXT:    vpcmpeqq %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm4, %xmm6, %xmm4
-; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm4, %ymm2
-; AVX1-NEXT:    vandnpd %ymm2, %ymm9, %ymm2
-; AVX1-NEXT:    vinsertf128 $1, %xmm5, %ymm3, %ymm3
-; AVX1-NEXT:    vblendvpd %ymm3, %ymm8, %ymm11, %ymm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm12, %ymm1, %ymm1
-; AVX1-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX1-NEXT:    vpsubq %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtq %xmm4, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubq %xmm2, %xmm0, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm0
+; AVX1-NEXT:    vxorpd %ymm0, %ymm6, %ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm2, %ymm2
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm4 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX1-NEXT:    vmovapd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm4, %ymm6, %ymm7
+; AVX1-NEXT:    vblendvpd %ymm0, %ymm7, %ymm2, %ymm0
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm2, %xmm7
+; AVX1-NEXT:    vpcmpgtq %xmm5, %xmm3, %xmm5
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
+; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
+; AVX1-NEXT:    vpsubq %xmm2, %xmm7, %xmm2
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm7, %xmm7
+; AVX1-NEXT:    vpsubq %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm7, %ymm1, %ymm1
+; AVX1-NEXT:    vxorpd %ymm1, %ymm5, %ymm1
+; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
+; AVX1-NEXT:    vblendvpd %ymm2, %ymm4, %ymm6, %ymm3
+; AVX1-NEXT:    vblendvpd %ymm1, %ymm3, %ymm2, %ymm1
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: v8i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqq %ymm5, %ymm7, %ymm5
-; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm0, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm8
-; AVX2-NEXT:    vpcmpeqq %ymm8, %ymm7, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpandn %ymm7, %ymm5, %ymm5
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
-; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm7, %ymm8, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm5, %ymm2, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm4, %ymm2
-; AVX2-NEXT:    vpxor %ymm6, %ymm2, %ymm2
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqq %ymm2, %ymm5, %ymm2
-; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpgtq %ymm1, %ymm4, %ymm3
-; AVX2-NEXT:    vpxor %ymm6, %ymm3, %ymm4
-; AVX2-NEXT:    vpcmpeqq %ymm4, %ymm5, %ymm4
-; AVX2-NEXT:    vpxor %ymm6, %ymm4, %ymm4
-; AVX2-NEXT:    vpandn %ymm4, %ymm2, %ymm2
-; AVX2-NEXT:    vblendvpd %ymm3, %ymm7, %ymm8, %ymm3
-; AVX2-NEXT:    vblendvpd %ymm2, %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm2, %ymm5
+; AVX2-NEXT:    vpsubq %ymm2, %ymm0, %ymm2
+; AVX2-NEXT:    vpcmpgtq %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [9223372036854775807,9223372036854775807,9223372036854775807,9223372036854775807]
+; AVX2-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX2-NEXT:    vblendvpd %ymm2, %ymm5, %ymm6, %ymm7
+; AVX2-NEXT:    vblendvpd %ymm0, %ymm7, %ymm2, %ymm0
+; AVX2-NEXT:    vpcmpgtq %ymm4, %ymm3, %ymm2
+; AVX2-NEXT:    vpsubq %ymm3, %ymm1, %ymm3
+; AVX2-NEXT:    vpcmpgtq %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm2, %ymm1
+; AVX2-NEXT:    vblendvpd %ymm3, %ymm5, %ymm6, %ymm2
+; AVX2-NEXT:    vblendvpd %ymm1, %ymm2, %ymm3, %ymm1
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: v8i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
-; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT:    vpcmpnltq %zmm2, %zmm0, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
-; AVX512-NEXT:    vpcmpgtq %zmm0, %zmm2, %k2
-; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm1 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm1 {%k2}
-; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512-NEXT:    vpcmpgtq %zmm2, %zmm1, %k0
+; AVX512-NEXT:    vpsubq %zmm1, %zmm0, %zmm1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %zmm1, %zmm2, %k2
+; AVX512-NEXT:    vpbroadcastq {{.*#+}} zmm0 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
+; AVX512-NEXT:    vpbroadcastq {{.*}}(%rip), %zmm0 {%k2}
+; AVX512-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512-NEXT:    vmovdqa64 %zmm1, %zmm0
 ; AVX512-NEXT:    retq
   %z = call <8 x i64> @llvm.ssub.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
   ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll b/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
index 3b69f2d9c524f..6625cc4f07a27 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-avx1.ll
@@ -2876,7 +2876,6 @@ define double @stack_fold_roundsd_minsize(double %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    vxorps %xmm0, %xmm0, %xmm0
 ; CHECK-NEXT:    vroundsd $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
diff --git a/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll b/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
index 0c81f1ac73a45..306ee31098090 100644
--- a/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-fp-sse42.ll
@@ -583,7 +583,6 @@ define float @stack_fold_cvtsd2ss(double %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtsd2ss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 8-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
@@ -975,7 +974,6 @@ define double @stack_fold_cvtss2sd(float %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    cvtss2sd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
@@ -2012,7 +2010,6 @@ define float @stack_fold_roundss(float %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    roundss $9, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
@@ -2181,7 +2178,6 @@ define float @stack_fold_sqrtss(float %a0) minsize {
 ; CHECK-NEXT:    #APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    #NO_APP
-; CHECK-NEXT:    xorps %xmm0, %xmm0
 ; CHECK-NEXT:    sqrtss {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 4-byte Folded Reload
 ; CHECK-NEXT:    retq
   %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm1},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{flags}"()
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
index ce6b1b5fb9cbe..e176f3a32c970 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512.ll
@@ -6013,6 +6013,20 @@ define <8 x i64> @stack_fold_psadbw(<64 x i8> %a0, <64 x i8> %a1) {
 }
 declare <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8>, <64 x i8>) nounwind readnone
 
+define <8 x i64> @stack_fold_psadbw_commute(<64 x i8> %a0, <64 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> %a1, <64 x i8> %a0)
+  ret <8 x i64> %2
+}
+
 define <64 x i8> @stack_fold_pshufb_zmm(<64 x i8> %a0, <64 x i8> %a1) {
 ; CHECK-LABEL: stack_fold_pshufb_zmm:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll b/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
index f3873a9e491f9..e1eaa1a52db35 100644
--- a/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-int-avx512vl.ll
@@ -2896,6 +2896,20 @@ define <2 x i64> @stack_fold_psadbw(<16 x i8> %a0, <16 x i8> %a1) {
 }
 declare <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8>, <16 x i8>) nounwind readnone
 
+define <2 x i64> @stack_fold_psadbw_commute(<16 x i8> %a0, <16 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm0 # 16-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> %a1, <16 x i8> %a0)
+  ret <2 x i64> %2
+}
+
 define <4 x i64> @stack_fold_psadbw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 ; CHECK-LABEL: stack_fold_psadbw_ymm:
 ; CHECK:       # %bb.0:
@@ -2911,6 +2925,20 @@ define <4 x i64> @stack_fold_psadbw_ymm(<32 x i8> %a0, <32 x i8> %a1) {
 }
 declare <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8>, <32 x i8>) nounwind readnone
 
+define <4 x i64> @stack_fold_psadbw_ymm_commute(<32 x i8> %a0, <32 x i8> %a1) {
+; CHECK-LABEL: stack_fold_psadbw_ymm_commute:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    vpsadbw {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; CHECK-NEXT:    retq
+  %1 = tail call <2 x i64> asm sideeffect "nop", "=x,~{xmm2},~{xmm2},~{xmm3},~{xmm4},~{xmm5},~{xmm6},~{xmm7},~{xmm8},~{xmm9},~{xmm10},~{xmm11},~{xmm12},~{xmm13},~{xmm14},~{xmm15},~{xmm16},~{xmm17},~{xmm18},~{xmm19},~{xmm20},~{xmm21},~{xmm22},~{xmm23},~{xmm24},~{xmm25},~{xmm26},~{xmm27},~{xmm28},~{xmm29},~{xmm30},~{xmm31},~{flags}"()
+  %2 = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> %a1, <32 x i8> %a0)
+  ret <4 x i64> %2
+}
+
 define <16 x i8> @stack_fold_pshufb(<16 x i8> %a0, <16 x i8> %a1) {
 ; CHECK-LABEL: stack_fold_pshufb:
 ; CHECK:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/stack-folding-mmx.ll b/llvm/test/CodeGen/X86/stack-folding-mmx.ll
index 071e73c2aeb18..8b6c2e687941a 100644
--- a/llvm/test/CodeGen/X86/stack-folding-mmx.ll
+++ b/llvm/test/CodeGen/X86/stack-folding-mmx.ll
@@ -96,9 +96,67 @@ define x86_mmx @stack_fold_cvttps2pi(<4 x float> %a0) {
 declare x86_mmx @llvm.x86.sse.cvttps2pi(<4 x float>) nounwind readnone
 
 ; TODO stack_fold_movd_load
-; TODO stack_fold_movd_store
+
+; padd forces execution on mmx
+define i32 @stack_fold_movd_store(x86_mmx %a0) nounwind {
+; CHECK-LABEL: stack_fold_movd_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    paddb %mm0, %mm0
+; CHECK-NEXT:    movd %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %1 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a0, x86_mmx %a0)
+  %2 = bitcast x86_mmx %1 to <2 x i32>
+  %3 = extractelement <2 x i32> %2, i32 0
+  %4 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  ret i32 %3
+}
+
 ; TODO stack_fold_movq_load
-; TODO stack_fold_movq_store
+
+; padd forces execution on mmx
+define i64 @stack_fold_movq_store(x86_mmx %a0) nounwind {
+; CHECK-LABEL: stack_fold_movq_store:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rbp
+; CHECK-NEXT:    pushq %r15
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %r13
+; CHECK-NEXT:    pushq %r12
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    paddb %mm0, %mm0
+; CHECK-NEXT:    movq %mm0, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT:    #APP
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    #NO_APP
+; CHECK-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r12
+; CHECK-NEXT:    popq %r13
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    popq %r15
+; CHECK-NEXT:    popq %rbp
+; CHECK-NEXT:    retq
+  %1 = call x86_mmx @llvm.x86.mmx.padd.b(x86_mmx %a0, x86_mmx %a0)
+  %2 = bitcast x86_mmx %1 to i64
+  %3 = tail call i64 asm sideeffect "nop", "=x,~{rax},~{rbx},~{rcx},~{rdx},~{rsi},~{rdi},~{rbp},~{r8},~{r9},~{r10},~{r11},~{r12},~{r13},~{r14},~{r15}"()
+  ret i64 %2
+}
 
 define x86_mmx @stack_fold_pabsb(x86_mmx %a0) {
 ; CHECK-LABEL: stack_fold_pabsb:
diff --git a/llvm/test/Transforms/StackProtector/X86/captures.ll b/llvm/test/CodeGen/X86/stack-protector-2.ll
similarity index 70%
rename from llvm/test/Transforms/StackProtector/X86/captures.ll
rename to llvm/test/CodeGen/X86/stack-protector-2.ll
index 03920f5beb095..dc86b93cdcca8 100644
--- a/llvm/test/Transforms/StackProtector/X86/captures.ll
+++ b/llvm/test/CodeGen/X86/stack-protector-2.ll
@@ -1,6 +1,5 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -mtriple=x86_64-pc-linux-gnu -stack-protector < %s | FileCheck %s
-; Bug 42238: Test some situations missed by old, custom capture tracking.
+; RUN: llc -mtriple=x86_64-pc-linux-gnu -start-before=stack-protector -stop-after=stack-protector -o - < %s | FileCheck %s
+; Bugs 42238/43308: Test some additional situations not caught previously.
 
 define void @store_captures() #0 {
 ; CHECK-LABEL: @store_captures(
@@ -38,26 +37,19 @@ entry:
   ret void
 }
 
-define i32* @return_captures() #0 {
-; CHECK-LABEL: @return_captures(
+define i32* @non_captures() #0 {
+; load, atomicrmw, and ret do not trigger a stack protector.
+; CHECK-LABEL: @non_captures(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
-; CHECK-NEXT:    [[J:%.*]] = alloca i32*, align 8
-; CHECK-NEXT:    store i32 0, i32* [[RETVAL]]
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LOAD]], 1
-; CHECK-NEXT:    store i32 [[ADD]], i32* [[A]], align 4
+; CHECK-NEXT:    [[ATOM:%.*]] = atomicrmw add i32* [[A]], i32 1 seq_cst
 ; CHECK-NEXT:    ret i32* [[A]]
 ;
 entry:
-  %retval = alloca i32, align 4
   %a = alloca i32, align 4
-  %j = alloca i32*, align 8
-  store i32 0, i32* %retval
   %load = load i32, i32* %a, align 4
-  %add = add nsw i32 %load, 1
-  store i32 %add, i32* %a, align 4
+  %atom = atomicrmw add i32* %a, i32 1 seq_cst
   ret i32* %a
 }
 
@@ -112,7 +104,7 @@ define void @cmpxchg_captures() #0 {
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, i32* [[A]], align 4
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[LOAD]], 1
 ; CHECK-NEXT:    store i32 [[ADD]], i32* [[A]], align 4
-; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32** [[J]], i32* [[A]], i32* null seq_cst monotonic
+; CHECK-NEXT:    [[TMP0:%.*]] = cmpxchg i32** [[J]], i32* null, i32* [[A]] seq_cst monotonic
 ; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
 ; CHECK-NEXT:    [[TMP1:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP1]]
@@ -132,8 +124,42 @@ entry:
   %add = add nsw i32 %load, 1
   store i32 %add, i32* %a, align 4
 
-  cmpxchg i32** %j, i32* %a, i32* null seq_cst monotonic
+  cmpxchg i32** %j, i32* null, i32* %a seq_cst monotonic
   ret void
 }
 
+define void @memset_captures(i64 %c) #0 {
+; CHECK-LABEL: @memset_captures(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[STACKGUARDSLOT:%.*]] = alloca i8*
+; CHECK-NEXT:    [[STACKGUARD:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    call void @llvm.stackprotector(i8* [[STACKGUARD]], i8** [[STACKGUARDSLOT]])
+; CHECK-NEXT:    [[CADDR:%.*]] = alloca i64, align 8
+; CHECK-NEXT:    store i64 %c, i64* [[CADDR]], align 8
+; CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[IPTR:%.*]] = bitcast i32* [[I]] to i8*
+; CHECK-NEXT:    [[COUNT:%.*]] = load i64, i64* [[CADDR]], align 8
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 [[IPTR]], i8 0, i64 [[COUNT]], i1 false)
+; CHECK-NEXT:    [[STACKGUARD1:%.*]] = load volatile i8*, i8* addrspace(257)* inttoptr (i32 40 to i8* addrspace(257)*)
+; CHECK-NEXT:    [[TMP1:%.*]] = load volatile i8*, i8** [[STACKGUARDSLOT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i8* [[STACKGUARD1]], [[TMP1]]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[SP_RETURN:%.*]], label [[CALLSTACKCHECKFAILBLK:%.*]], !prof !0
+; CHECK:       SP_return:
+; CHECK-NEXT:    ret void
+; CHECK:       CallStackCheckFailBlk:
+; CHECK-NEXT:    call void @__stack_chk_fail()
+; CHECK-NEXT:    unreachable
+;
+entry:
+  %c.addr = alloca i64, align 8
+  store i64 %c, i64* %c.addr, align 8
+  %i = alloca i32, align 4
+  %i.ptr = bitcast i32* %i to i8*
+  %count = load i64, i64* %c.addr, align 8
+  call void @llvm.memset.p0i8.i64(i8* align 4 %i.ptr, i8 0, i64 %count, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p0i8.i64(i8* nocapture writeonly, i8, i64, i1 immarg)
+
 attributes #0 = { sspstrong }
diff --git a/llvm/test/CodeGen/X86/stack-protector.ll b/llvm/test/CodeGen/X86/stack-protector.ll
index 874666fa4ce58..1dacd9eb7c79a 100644
--- a/llvm/test/CodeGen/X86/stack-protector.ll
+++ b/llvm/test/CodeGen/X86/stack-protector.ll
@@ -4087,8 +4087,8 @@ define i32 @IgnoreIntrinsicTest() #1 {
   %1 = alloca i32, align 4
   %2 = bitcast i32* %1 to i8*
   call void @llvm.lifetime.start.p0i8(i64 4, i8* nonnull %2)
-  store i32 1, i32* %1, align 4
-  %3 = load i32, i32* %1, align 4
+  store volatile i32 1, i32* %1, align 4
+  %3 = load volatile i32, i32* %1, align 4
   %4 = mul nsw i32 %3, 42
   call void @llvm.lifetime.end.p0i8(i64 4, i8* nonnull %2)
   ret i32 %4
diff --git a/llvm/test/CodeGen/X86/subcarry.ll b/llvm/test/CodeGen/X86/subcarry.ll
index 449391616aa9d..75ee6ef5921b5 100644
--- a/llvm/test/CodeGen/X86/subcarry.ll
+++ b/llvm/test/CodeGen/X86/subcarry.ll
@@ -162,3 +162,24 @@ entry:
   %40 = insertvalue %S undef, [4 x i64] %39, 0
   ret %S %40
 }
+
+declare {i64, i1} @llvm.uadd.with.overflow(i64, i64)
+declare {i64, i1} @llvm.usub.with.overflow(i64, i64)
+
+define i64 @sub_from_carry(i64 %x, i64 %y, i64* %valout, i64 %z) {
+; CHECK-LABEL: sub_from_carry:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq %rcx, %rax
+; CHECK-NEXT:    negq %rax
+; CHECK-NEXT:    addq %rsi, %rdi
+; CHECK-NEXT:    movq %rdi, (%rdx)
+; CHECK-NEXT:    adcq $0, %rax
+; CHECK-NEXT:    retq
+  %agg = call {i64, i1} @llvm.uadd.with.overflow(i64 %x, i64 %y)
+  %val = extractvalue {i64, i1} %agg, 0
+  %ov = extractvalue {i64, i1} %agg, 1
+  store i64 %val, i64* %valout, align 4
+  %carry = zext i1 %ov to i64
+  %res = sub i64 %carry, %z
+  ret i64 %res
+}
diff --git a/llvm/test/CodeGen/X86/switch-bt.ll b/llvm/test/CodeGen/X86/switch-bt.ll
index 958d053c22f0f..965cdbf17f508 100644
--- a/llvm/test/CodeGen/X86/switch-bt.ll
+++ b/llvm/test/CodeGen/X86/switch-bt.ll
@@ -155,3 +155,42 @@ sw.epilog:
 ; CHECK: cmpl $60
 ; CHECK: jne
 }
+
+
+; Omit the range check when the default case is unreachable, see PR43129.
+declare void @g(i32)
+define void @test5(i32 %x) {
+
+; CHECK-LABEL: test5
+; CHECK-NOT: cmp
+
+; 73 = 2^0 + 2^3 + 2^6
+; CHECK:      movl $73
+; CHECK-NEXT: btl
+; CHECK-NEXT: jb
+
+; 146 = 2^1 + 2^4 + 2^7
+; CHECK-NEXT: movl $146
+; CHECK-NEXT: btl
+; CHECK-NEXT: jae
+
+
+entry:
+  switch i32 %x, label %return [
+    i32 0, label %bb0
+    i32 3, label %bb0
+    i32 6, label %bb0
+
+    i32 1, label %bb1
+    i32 4, label %bb1
+    i32 7, label %bb1
+
+    i32 2, label %bb2
+    i32 5, label %bb2
+    i32 8, label %bb2
+  ]
+bb0: tail call void @g(i32 0) br label %return
+bb1: tail call void @g(i32 1) br label %return
+bb2: tail call void @g(i32 2) br label %return
+return: unreachable
+}
diff --git a/llvm/test/CodeGen/X86/vec-copysign-avx512.ll b/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
index b08b15ce004eb..2443327dbfd61 100644
--- a/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
+++ b/llvm/test/CodeGen/X86/vec-copysign-avx512.ll
@@ -3,55 +3,28 @@
 ; RUN: llc < %s -mtriple=x86_64-apple-macosx10.10.0 -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefix=CHECK --check-prefix=AVX512VLDQ
 
 define <4 x float> @v4f32(<4 x float> %a, <4 x float> %b) nounwind {
-; AVX512VL-LABEL: v4f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm1, %xmm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to4}, %xmm0, %xmm0
-; AVX512VL-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v4f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm1, %xmm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to4}, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    vorps %xmm1, %xmm0, %xmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v4f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to4}, %xmm1, %xmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <4 x float> @llvm.copysign.v4f32( <4 x float> %a, <4 x float> %b )
   ret <4 x float> %tmp
 }
 
 define <8 x float> @v8f32(<8 x float> %a, <8 x float> %b) nounwind {
-; AVX512VL-LABEL: v8f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v8f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v8f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to8}, %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %tmp = tail call <8 x float> @llvm.copysign.v8f32( <8 x float> %a, <8 x float> %b )
   ret <8 x float> %tmp
 }
 
 define <16 x float> @v16f32(<16 x float> %a, <16 x float> %b) nounwind {
-; AVX512VL-LABEL: v16f32:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VL-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v16f32:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512VLDQ-NEXT:    vandps {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    vorps %zmm1, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v16f32:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogd $228, {{.*}}(%rip){1to16}, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <16 x float> @llvm.copysign.v16f32( <16 x float> %a, <16 x float> %b )
   ret <16 x float> %tmp
 }
@@ -59,46 +32,26 @@ define <16 x float> @v16f32(<16 x float> %a, <16 x float> %b) nounwind {
 define <2 x double> @v2f64(<2 x double> %a, <2 x double> %b) nounwind {
 ; CHECK-LABEL: v2f64:
 ; CHECK:       ## %bb.0:
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm1, %xmm1
-; CHECK-NEXT:    vandps {{.*}}(%rip), %xmm0, %xmm0
-; CHECK-NEXT:    vorps %xmm1, %xmm0, %xmm0
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip), %xmm1, %xmm0
 ; CHECK-NEXT:    retq
   %tmp = tail call <2 x double> @llvm.copysign.v2f64( <2 x double> %a, <2 x double> %b )
   ret <2 x double> %tmp
 }
 
 define <4 x double> @v4f64(<4 x double> %a, <4 x double> %b) nounwind {
-; AVX512VL-LABEL: v4f64:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to4}, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to4}, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v4f64:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to4}, %ymm1, %ymm1
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to4}, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    vorpd %ymm1, %ymm0, %ymm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v4f64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip){1to4}, %ymm1, %ymm0
+; CHECK-NEXT:    retq
   %tmp = tail call <4 x double> @llvm.copysign.v4f64( <4 x double> %a, <4 x double> %b )
   ret <4 x double> %tmp
 }
 
 define <8 x double> @v8f64(<8 x double> %a, <8 x double> %b) nounwind {
-; AVX512VL-LABEL: v8f64:
-; AVX512VL:       ## %bb.0:
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm1, %zmm1
-; AVX512VL-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512VL-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT:    retq
-;
-; AVX512VLDQ-LABEL: v8f64:
-; AVX512VLDQ:       ## %bb.0:
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm1, %zmm1
-; AVX512VLDQ-NEXT:    vandpd {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    vorpd %zmm1, %zmm0, %zmm0
-; AVX512VLDQ-NEXT:    retq
+; CHECK-LABEL: v8f64:
+; CHECK:       ## %bb.0:
+; CHECK-NEXT:    vpternlogq $228, {{.*}}(%rip){1to8}, %zmm1, %zmm0
+; CHECK-NEXT:    retq
   %tmp = tail call <8 x double> @llvm.copysign.v8f64( <8 x double> %a, <8 x double> %b )
   ret <8 x double> %tmp
 }
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index c8c2b19b41a2b..314a379e03e09 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -52,67 +52,40 @@ define <2 x i32> @saddo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) noun
 ; SSE-LABEL: saddo_v2i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movq %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    paddd %xmm0, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movq %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: saddo_v2i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovq %xmm1, (%rdi)
@@ -129,82 +102,45 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; SSE2-LABEL: saddo_v3i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE2-NEXT:    paddd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, 8(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE2-NEXT:    paddd %xmm0, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE2-NEXT:    pxor %xmm2, %xmm0
+; SSE2-NEXT:    movq %xmm1, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: saddo_v3i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    paddd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSSE3-NEXT:    paddd %xmm0, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSSE3-NEXT:    pxor %xmm2, %xmm0
+; SSSE3-NEXT:    movq %xmm1, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: saddo_v3i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    paddd %xmm1, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm3, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE41-NEXT:    paddd %xmm0, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm1, 8(%rdi)
+; SSE41-NEXT:    movq %xmm1, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v3i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -212,17 +148,10 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v3i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -230,13 +159,10 @@ define <3 x i32> @saddo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX512-LABEL: saddo_v3i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
@@ -254,67 +180,40 @@ define <4 x i32> @saddo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) noun
 ; SSE-LABEL: saddo_v4i32:
 ; SSE:       # %bb.0:
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    paddd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
+; SSE-NEXT:    paddd %xmm0, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: saddo_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -331,164 +230,118 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE2-LABEL: saddo_v6i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSE2-NEXT:    movd %r8d, %xmm0
 ; SSE2-NEXT:    movd %ecx, %xmm1
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSE2-NEXT:    movd %edx, %xmm0
-; SSE2-NEXT:    movd %esi, %xmm4
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
 ; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSE2-NEXT:    movd %r9d, %xmm3
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT:    movd %r9d, %xmm0
+; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    paddd %xmm4, %xmm0
-; SSE2-NEXT:    pxor %xmm4, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pxor %xmm5, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE2-NEXT:    pandn %xmm6, %xmm4
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    paddd %xmm2, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
 ; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    paddd %xmm3, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    pandn %xmm6, %xmm2
-; SSE2-NEXT:    movq %xmm1, 16(%rcx)
-; SSE2-NEXT:    movdqa %xmm0, (%rcx)
-; SSE2-NEXT:    movq %xmm2, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm4, (%rdi)
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSE2-NEXT:    pxor %xmm3, %xmm6
+; SSE2-NEXT:    movdqa %xmm0, %xmm2
+; SSE2-NEXT:    paddd %xmm1, %xmm2
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE2-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT:    pxor %xmm0, %xmm5
+; SSE2-NEXT:    movq %xmm2, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm4, (%rcx)
+; SSE2-NEXT:    movq %xmm5, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm6, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: saddo_v6i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
 ; SSSE3-NEXT:    movd %r8d, %xmm0
 ; SSSE3-NEXT:    movd %ecx, %xmm1
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; SSSE3-NEXT:    movd %edx, %xmm0
-; SSSE3-NEXT:    movd %esi, %xmm4
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm1[0]
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm1[0]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; SSSE3-NEXT:    movd %r9d, %xmm3
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSSE3-NEXT:    movd %r9d, %xmm0
+; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    paddd %xmm4, %xmm0
-; SSSE3-NEXT:    pxor %xmm4, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pxor %xmm5, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSSE3-NEXT:    pandn %xmm6, %xmm4
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    paddd %xmm2, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
 ; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    paddd %xmm3, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    pandn %xmm6, %xmm2
-; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
-; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
-; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm4, (%rdi)
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm6
+; SSSE3-NEXT:    pxor %xmm3, %xmm6
+; SSSE3-NEXT:    movdqa %xmm0, %xmm2
+; SSSE3-NEXT:    paddd %xmm1, %xmm2
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm5
+; SSSE3-NEXT:    pxor %xmm0, %xmm5
+; SSSE3-NEXT:    movq %xmm2, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
+; SSSE3-NEXT:    movq %xmm5, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm6, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: saddo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm4
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm4
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm4
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm4
-; SSE41-NEXT:    movd %r9d, %xmm2
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
-; SSE41-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm1
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm0
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
-; SSE41-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm1
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
+; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pxor %xmm3, %xmm3
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    paddd %xmm4, %xmm1
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE41-NEXT:    pandn %xmm6, %xmm4
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    paddd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
 ; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm2, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    paddd %xmm2, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm5, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE41-NEXT:    pandn %xmm6, %xmm3
-; SSE41-NEXT:    movq %xmm0, 16(%rcx)
-; SSE41-NEXT:    movdqa %xmm1, (%rcx)
-; SSE41-NEXT:    movq %xmm3, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm4, (%rdi)
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
+; SSE41-NEXT:    pxor %xmm1, %xmm6
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE41-NEXT:    paddd %xmm0, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE41-NEXT:    pxor %xmm5, %xmm0
+; SSE41-NEXT:    movq %xmm2, 16(%rcx)
+; SSE41-NEXT:    movdqa %xmm4, (%rcx)
+; SSE41-NEXT:    movq %xmm0, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm6, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v6i32:
@@ -496,28 +349,15 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm8, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -525,17 +365,10 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v6i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -544,13 +377,10 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX512-LABEL: saddo_v6i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
 ; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
@@ -568,37 +398,18 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
 ; SSE-LABEL: saddo_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm1, %xmm1
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE-NEXT:    pxor %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE-NEXT:    pxor %xmm5, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE-NEXT:    paddd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm5, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    pandn %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE-NEXT:    pxor %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm7
-; SSE-NEXT:    pxor %xmm5, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE-NEXT:    paddd %xmm3, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm1
-; SSE-NEXT:    pxor %xmm5, %xmm1
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm1
-; SSE-NEXT:    pandn %xmm6, %xmm1
-; SSE-NEXT:    movdqa %xmm4, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm4
+; SSE-NEXT:    pxor %xmm5, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
+; SSE-NEXT:    paddd %xmm0, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE-NEXT:    pxor %xmm5, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm4
+; SSE-NEXT:    paddd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE-NEXT:    pxor %xmm4, %xmm1
+; SSE-NEXT:    movdqa %xmm3, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm2, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v8i32:
@@ -606,28 +417,15 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
 ; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpaddd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpaddd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm8, %ymm0, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -635,30 +433,20 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX2-LABEL: saddo_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm2
 ; AVX2-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpandn %ymm3, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: saddo_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm2, %k0
 ; AVX512-NEXT:    vpaddd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
@@ -674,132 +462,70 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
 ; SSE-LABEL: saddo_v16i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    movdqa %xmm3, %xmm8
-; SSE-NEXT:    pxor %xmm3, %xmm3
-; SSE-NEXT:    pxor %xmm11, %xmm11
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm11
-; SSE-NEXT:    pcmpeqd %xmm10, %xmm10
-; SSE-NEXT:    pxor %xmm10, %xmm11
-; SSE-NEXT:    pxor %xmm12, %xmm12
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm12
-; SSE-NEXT:    pxor %xmm10, %xmm12
-; SSE-NEXT:    pcmpeqd %xmm12, %xmm11
-; SSE-NEXT:    paddd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm8, %xmm8
 ; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE-NEXT:    pxor %xmm10, %xmm9
-; SSE-NEXT:    pcmpeqd %xmm12, %xmm9
-; SSE-NEXT:    pandn %xmm11, %xmm9
-; SSE-NEXT:    pxor %xmm12, %xmm12
-; SSE-NEXT:    pcmpgtd %xmm5, %xmm12
-; SSE-NEXT:    pxor %xmm10, %xmm12
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm12
-; SSE-NEXT:    paddd %xmm5, %xmm1
-; SSE-NEXT:    pxor %xmm11, %xmm11
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm11
-; SSE-NEXT:    pxor %xmm10, %xmm11
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm11
-; SSE-NEXT:    pandn %xmm12, %xmm11
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE-NEXT:    pxor %xmm10, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    paddd %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pxor %xmm10, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE-NEXT:    pandn %xmm4, %xmm6
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm7, %xmm4
-; SSE-NEXT:    pxor %xmm10, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm8, %xmm5
-; SSE-NEXT:    pxor %xmm10, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    paddd %xmm7, %xmm8
-; SSE-NEXT:    pcmpgtd %xmm8, %xmm3
-; SSE-NEXT:    pxor %xmm10, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    pandn %xmm4, %xmm3
-; SSE-NEXT:    movdqa %xmm8, 48(%rdi)
-; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm9, %xmm0
-; SSE-NEXT:    movdqa %xmm11, %xmm1
-; SSE-NEXT:    movdqa %xmm6, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm9
+; SSE-NEXT:    paddd %xmm0, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm0
+; SSE-NEXT:    pxor %xmm9, %xmm0
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm9
+; SSE-NEXT:    paddd %xmm1, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm1
+; SSE-NEXT:    pxor %xmm9, %xmm1
+; SSE-NEXT:    pxor %xmm9, %xmm9
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm9
+; SSE-NEXT:    paddd %xmm2, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm2
+; SSE-NEXT:    pxor %xmm9, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm8
+; SSE-NEXT:    paddd %xmm3, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm7, %xmm3
+; SSE-NEXT:    pxor %xmm8, %xmm3
+; SSE-NEXT:    movdqa %xmm7, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm6, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm5, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm4, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v16i32:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm9
+; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
 ; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm6, %xmm6
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm4, %xmm8
-; AVX1-NEXT:    vpaddd %xmm9, %xmm7, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm9, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm4
-; AVX1-NEXT:    vpandn %xmm8, %xmm4, %xmm8
+; AVX1-NEXT:    vpaddd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm6, %xmm6
 ; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm4
-; AVX1-NEXT:    vpxor %xmm6, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
-; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm5, %xmm1
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm8, %xmm1, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm5, %xmm3
-; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm3, %xmm7
-; AVX1-NEXT:    vpaddd %xmm4, %xmm1, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm5, %xmm1
-; AVX1-NEXT:    vpxor %xmm6, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT:    vpandn %xmm7, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm3
-; AVX1-NEXT:    vpxor %xmm6, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm5, %xmm7
-; AVX1-NEXT:    vpxor %xmm6, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm7, %xmm3
+; AVX1-NEXT:    vpaddd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm5, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpaddd %xmm6, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm5
 ; AVX1-NEXT:    vpaddd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm6, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm7, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[1,1,2,3]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
-; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
-; AVX1-NEXT:    vpshufd {{.*#+}} xmm3 = xmm1[2,3,0,1]
-; AVX1-NEXT:    vpmovsxbd %xmm3, %xmm3
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[2,3,0,1]
+; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm3, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm9, 48(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm10, 32(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm4, 16(%rdi)
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm6, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
@@ -807,28 +533,15 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
 ; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
 ; AVX2-NEXT:    vpaddd %ymm3, %ymm1, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpandn %ymm5, %ymm1, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm5, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm4
 ; AVX2-NEXT:    vpaddd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
-; AVX2-NEXT:    vpandn %ymm5, %ymm0, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
@@ -842,13 +555,10 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX512-LABEL: saddo_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm2, %k0
 ; AVX512-NEXT:    vpaddd %zmm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
 ; AVX512-NEXT:    retq
@@ -1094,157 +804,42 @@ define <8 x i32> @saddo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 }
 
 define <2 x i32> @saddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
-; SSE2-LABEL: saddo_v2i64:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    paddq %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, (%rdi)
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pandn %xmm3, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: saddo_v2i64:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    paddq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
-; SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pandn %xmm3, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: saddo_v2i64:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    paddq %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm2
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
-; SSE41-NEXT:    pandn %xmm4, %xmm2
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[0,2,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: saddo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    paddq %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    pand %xmm5, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE-NEXT:    por %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    por %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: saddo_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm2
 ; AVX1-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -1252,17 +847,10 @@ define <2 x i32> @saddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX2-LABEL: saddo_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm2
 ; AVX2-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpandn %xmm3, %xmm0, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -1270,13 +858,10 @@ define <2 x i32> @saddo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-LABEL: saddo_v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm2, %k0
 ; AVX512-NEXT:    vpaddq %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandnw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vec_smulo.ll b/llvm/test/CodeGen/X86/vec_smulo.ll
index 025934d63cd98..436e48729f9f6 100644
--- a/llvm/test/CodeGen/X86/vec_smulo.ll
+++ b/llvm/test/CodeGen/X86/vec_smulo.ll
@@ -1742,12 +1742,14 @@ define <2 x i32> @smulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; AVX512-NEXT:    seto %al
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kmovd %edx, %k1
-; AVX512-NEXT:    kshiftrw $1, %k1, %k2
-; AVX512-NEXT:    kxorw %k0, %k2, %k0
 ; AVX512-NEXT:    kshiftlw $15, %k0, %k0
 ; AVX512-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512-NEXT:    kxorw %k0, %k1, %k1
+; AVX512-NEXT:    kmovd %edx, %k1
+; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftrw $15, %k1, %k1
+; AVX512-NEXT:    kshiftlw $2, %k0, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -2248,7 +2250,9 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512-NEXT:    kshiftrw $15, %k0, %k1
+; AVX512-NEXT:    kshiftlw $2, %k0, %k0
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    imulb %sil
 ; AVX512-NEXT:    movl %eax, %edx
@@ -2261,11 +2265,12 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $14, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512-NEXT:    kshiftlw $3, %k0, %k2
 ; AVX512-NEXT:    movl %r11d, %eax
 ; AVX512-NEXT:    imulb %bl
 ; AVX512-NEXT:    movl %eax, %esi
@@ -2277,13 +2282,12 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kmovd %eax, %k3
+; AVX512-NEXT:    kshiftlw $2, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $13, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftlw $13, %k0, %k0
-; AVX512-NEXT:    kshiftrw $13, %k0, %k0
 ; AVX512-NEXT:    movl %r10d, %eax
 ; AVX512-NEXT:    imulb %r9b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
@@ -2295,30 +2299,37 @@ define <4 x i32> @smulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %bl
 ; AVX512-NEXT:    orb %cl, %bl
 ; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    kmovd %ecx, %k1
-; AVX512-NEXT:    kshiftlw $3, %k1, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
+; AVX512-NEXT:    kmovd %ecx, %k2
+; AVX512-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    kmovd %r8d, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512-NEXT:    kmovd %edx, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512-NEXT:    kmovd %esi, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kmovd %r8d, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    kshiftrw $15, %k1, %k1
+; AVX512-NEXT:    kmovd %edx, %k2
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k0
+; AVX512-NEXT:    korw %k0, %k1, %k0
 ; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512-NEXT:    kmovd %esi, %k2
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $13, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512-NEXT:    kshiftlw $4, %k1, %k1
+; AVX512-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    korw %k0, %k1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/vec_ssubo.ll b/llvm/test/CodeGen/X86/vec_ssubo.ll
index f8dcb6e0fb518..48834a7de1666 100644
--- a/llvm/test/CodeGen/X86/vec_ssubo.ll
+++ b/llvm/test/CodeGen/X86/vec_ssubo.ll
@@ -51,71 +51,42 @@ define <1 x i32> @ssubo_v1i32(<1 x i32> %a0, <1 x i32> %a1, <1 x i32>* %p2) noun
 define <2 x i32> @ssubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v2i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm3, %xmm3
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    psubd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movq %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    psubd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    movq %xmm3, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v2i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v2i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v2i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovq %xmm1, (%rdi)
@@ -131,87 +102,49 @@ define <2 x i32> @ssubo_v2i32(<2 x i32> %a0, <2 x i32> %a1, <2 x i32>* %p2) noun
 define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) nounwind {
 ; SSE2-LABEL: ssubo_v3i32:
 ; SSE2:       # %bb.0:
-; SSE2-NEXT:    pxor %xmm3, %xmm3
 ; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE2-NEXT:    pxor %xmm4, %xmm2
-; SSE2-NEXT:    pxor %xmm5, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE2-NEXT:    pxor %xmm4, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    psubd %xmm1, %xmm0
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE2-NEXT:    pxor %xmm4, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm2
-; SSE2-NEXT:    movq %xmm0, (%rdi)
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSE2-NEXT:    movd %xmm0, 8(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, %xmm0
+; SSE2-NEXT:    movdqa %xmm0, %xmm3
+; SSE2-NEXT:    psubd %xmm1, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT:    pxor %xmm1, %xmm0
+; SSE2-NEXT:    movq %xmm3, (%rdi)
+; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSE2-NEXT:    movd %xmm1, 8(%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: ssubo_v3i32:
 ; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
 ; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSSE3-NEXT:    pxor %xmm4, %xmm2
-; SSSE3-NEXT:    pxor %xmm5, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSSE3-NEXT:    pxor %xmm4, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    psubd %xmm1, %xmm0
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSSE3-NEXT:    pxor %xmm4, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm2
-; SSSE3-NEXT:    movq %xmm0, (%rdi)
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[2,3,0,1]
-; SSSE3-NEXT:    movd %xmm0, 8(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, %xmm0
+; SSSE3-NEXT:    movdqa %xmm0, %xmm3
+; SSSE3-NEXT:    psubd %xmm1, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSSE3-NEXT:    pxor %xmm1, %xmm0
+; SSSE3-NEXT:    movq %xmm3, (%rdi)
+; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[2,3,0,1]
+; SSSE3-NEXT:    movd %xmm1, 8(%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ssubo_v3i32:
 ; SSE41:       # %bb.0:
-; SSE41-NEXT:    pxor %xmm3, %xmm3
 ; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm4, %xmm2
-; SSE41-NEXT:    pxor %xmm5, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT:    pxor %xmm4, %xmm5
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE41-NEXT:    psubd %xmm1, %xmm0
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE41-NEXT:    pxor %xmm4, %xmm3
-; SSE41-NEXT:    pandn %xmm3, %xmm2
-; SSE41-NEXT:    pextrd $2, %xmm0, 8(%rdi)
-; SSE41-NEXT:    movq %xmm0, (%rdi)
-; SSE41-NEXT:    movdqa %xmm2, %xmm0
+; SSE41-NEXT:    movdqa %xmm0, %xmm3
+; SSE41-NEXT:    psubd %xmm1, %xmm3
+; SSE41-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE41-NEXT:    pxor %xmm1, %xmm0
+; SSE41-NEXT:    pextrd $2, %xmm3, 8(%rdi)
+; SSE41-NEXT:    movq %xmm3, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v3i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX1-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -219,18 +152,10 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v3i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
 ; AVX2-NEXT:    vmovq %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -238,13 +163,10 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 ; AVX512-LABEL: ssubo_v3i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vpextrd $2, %xmm1, 8(%rdi)
@@ -261,71 +183,42 @@ define <3 x i32> @ssubo_v3i32(<3 x i32> %a0, <3 x i32> %a1, <3 x i32>* %p2) noun
 define <4 x i32> @ssubo_v4i32(<4 x i32> %a0, <4 x i32> %a1, <4 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v4i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm3, %xmm3
 ; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm4
-; SSE-NEXT:    pxor %xmm4, %xmm2
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm5
-; SSE-NEXT:    pxor %xmm4, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE-NEXT:    psubd %xmm1, %xmm0
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm3
-; SSE-NEXT:    pxor %xmm4, %xmm3
-; SSE-NEXT:    pandn %xmm3, %xmm2
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    psubd %xmm1, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pcmpgtd %xmm3, %xmm0
+; SSE-NEXT:    pxor %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm3, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v4i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v4i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtd %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqd %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtd %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqd %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v4i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltd %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -342,201 +235,132 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; SSE2-LABEL: ssubo_v6i32:
 ; SSE2:       # %bb.0:
 ; SSE2-NEXT:    movq %rdi, %rax
-; SSE2-NEXT:    movd %r8d, %xmm0
-; SSE2-NEXT:    movd %ecx, %xmm1
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE2-NEXT:    movd %edx, %xmm2
-; SSE2-NEXT:    movd %esi, %xmm0
+; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE2-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE2-NEXT:    movd %r8d, %xmm1
+; SSE2-NEXT:    movd %ecx, %xmm2
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSE2-NEXT:    movd %edx, %xmm1
+; SSE2-NEXT:    movd %esi, %xmm3
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
 ; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSE2-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
-; SSE2-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
 ; SSE2-NEXT:    movd %r9d, %xmm1
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSE2-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSE2-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; SSE2-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE2-NEXT:    pxor %xmm3, %xmm3
-; SSE2-NEXT:    pxor %xmm2, %xmm2
-; SSE2-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT:    pxor %xmm5, %xmm2
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE2-NEXT:    psubd %xmm6, %xmm0
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pandn %xmm6, %xmm2
-; SSE2-NEXT:    pxor %xmm6, %xmm6
-; SSE2-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSE2-NEXT:    pxor %xmm5, %xmm6
-; SSE2-NEXT:    pxor %xmm7, %xmm7
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE2-NEXT:    pxor %xmm5, %xmm7
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE2-NEXT:    psubd %xmm4, %xmm1
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSE2-NEXT:    pxor %xmm5, %xmm3
-; SSE2-NEXT:    pandn %xmm3, %xmm6
-; SSE2-NEXT:    movq %xmm1, 16(%rcx)
-; SSE2-NEXT:    movdqa %xmm0, (%rcx)
-; SSE2-NEXT:    movq %xmm6, 16(%rdi)
-; SSE2-NEXT:    movdqa %xmm2, (%rdi)
+; SSE2-NEXT:    movdqa %xmm3, %xmm4
+; SSE2-NEXT:    psubd %xmm0, %xmm4
+; SSE2-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT:    pxor %xmm5, %xmm5
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE2-NEXT:    pxor %xmm3, %xmm0
+; SSE2-NEXT:    movdqa %xmm1, %xmm3
+; SSE2-NEXT:    psubd %xmm2, %xmm3
+; SSE2-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSE2-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE2-NEXT:    pxor %xmm1, %xmm2
+; SSE2-NEXT:    movq %xmm3, 16(%rcx)
+; SSE2-NEXT:    movdqa %xmm4, (%rcx)
+; SSE2-NEXT:    movq %xmm2, 16(%rdi)
+; SSE2-NEXT:    movdqa %xmm0, (%rdi)
 ; SSE2-NEXT:    retq
 ;
 ; SSSE3-LABEL: ssubo_v6i32:
 ; SSSE3:       # %bb.0:
 ; SSSE3-NEXT:    movq %rdi, %rax
-; SSSE3-NEXT:    movd %r8d, %xmm0
-; SSSE3-NEXT:    movd %ecx, %xmm1
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSSE3-NEXT:    movd %edx, %xmm2
-; SSSE3-NEXT:    movd %esi, %xmm0
+; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSSE3-NEXT:    movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
 ; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-NEXT:    movd %r8d, %xmm1
+; SSSE3-NEXT:    movd %ecx, %xmm2
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; SSSE3-NEXT:    movd %edx, %xmm1
+; SSSE3-NEXT:    movd %esi, %xmm3
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
 ; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
-; SSSE3-NEXT:    punpcklqdq {{.*#+}} xmm6 = xmm6[0],xmm2[0]
 ; SSSE3-NEXT:    movd %r9d, %xmm1
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSSE3-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
 ; SSSE3-NEXT:    movd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; SSSE3-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSSE3-NEXT:    punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
 ; SSSE3-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSSE3-NEXT:    pxor %xmm3, %xmm3
-; SSSE3-NEXT:    pxor %xmm2, %xmm2
-; SSSE3-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSSE3-NEXT:    pxor %xmm5, %xmm2
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSSE3-NEXT:    psubd %xmm6, %xmm0
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pandn %xmm6, %xmm2
-; SSSE3-NEXT:    pxor %xmm6, %xmm6
-; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm6
-; SSSE3-NEXT:    pxor %xmm5, %xmm6
-; SSSE3-NEXT:    pxor %xmm7, %xmm7
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSSE3-NEXT:    pxor %xmm5, %xmm7
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSSE3-NEXT:    psubd %xmm4, %xmm1
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    pcmpeqd %xmm7, %xmm3
-; SSSE3-NEXT:    pxor %xmm5, %xmm3
-; SSSE3-NEXT:    pandn %xmm3, %xmm6
-; SSSE3-NEXT:    movq %xmm1, 16(%rcx)
-; SSSE3-NEXT:    movdqa %xmm0, (%rcx)
-; SSSE3-NEXT:    movq %xmm6, 16(%rdi)
-; SSSE3-NEXT:    movdqa %xmm2, (%rdi)
+; SSSE3-NEXT:    movdqa %xmm3, %xmm4
+; SSSE3-NEXT:    psubd %xmm0, %xmm4
+; SSSE3-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSSE3-NEXT:    pxor %xmm5, %xmm5
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSSE3-NEXT:    pxor %xmm3, %xmm0
+; SSSE3-NEXT:    movdqa %xmm1, %xmm3
+; SSSE3-NEXT:    psubd %xmm2, %xmm3
+; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm1
+; SSSE3-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSSE3-NEXT:    pxor %xmm1, %xmm2
+; SSSE3-NEXT:    movq %xmm3, 16(%rcx)
+; SSSE3-NEXT:    movdqa %xmm4, (%rcx)
+; SSSE3-NEXT:    movq %xmm2, 16(%rdi)
+; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
 ; SSSE3-NEXT:    retq
 ;
 ; SSE41-LABEL: ssubo_v6i32:
 ; SSE41:       # %bb.0:
 ; SSE41-NEXT:    movq %rdi, %rax
-; SSE41-NEXT:    movd %esi, %xmm0
-; SSE41-NEXT:    pinsrd $1, %edx, %xmm0
-; SSE41-NEXT:    pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT:    pinsrd $3, %r8d, %xmm0
-; SSE41-NEXT:    movd %r9d, %xmm1
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm1
+; SSE41-NEXT:    movd %esi, %xmm1
+; SSE41-NEXT:    pinsrd $1, %edx, %xmm1
+; SSE41-NEXT:    pinsrd $2, %ecx, %xmm1
+; SSE41-NEXT:    pinsrd $3, %r8d, %xmm1
+; SSE41-NEXT:    movd %r9d, %xmm0
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm0
+; SSE41-NEXT:    movd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm2
 ; SSE41-NEXT:    movd {{.*#+}} xmm3 = mem[0],zero,zero,zero
 ; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm3
-; SSE41-NEXT:    movd {{.*#+}} xmm6 = mem[0],zero,zero,zero
-; SSE41-NEXT:    pinsrd $1, {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm6
-; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm6
+; SSE41-NEXT:    pinsrd $2, {{[0-9]+}}(%rsp), %xmm3
+; SSE41-NEXT:    pinsrd $3, {{[0-9]+}}(%rsp), %xmm3
 ; SSE41-NEXT:    movq {{[0-9]+}}(%rsp), %rcx
-; SSE41-NEXT:    pxor %xmm4, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm2
-; SSE41-NEXT:    pcmpgtd %xmm6, %xmm2
-; SSE41-NEXT:    pcmpeqd %xmm5, %xmm5
-; SSE41-NEXT:    pxor %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE41-NEXT:    psubd %xmm6, %xmm0
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pandn %xmm6, %xmm2
-; SSE41-NEXT:    pxor %xmm6, %xmm6
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT:    pxor %xmm5, %xmm6
-; SSE41-NEXT:    pxor %xmm7, %xmm7
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE41-NEXT:    pxor %xmm5, %xmm7
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm6
-; SSE41-NEXT:    psubd %xmm3, %xmm1
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE41-NEXT:    pxor %xmm5, %xmm4
-; SSE41-NEXT:    pandn %xmm4, %xmm6
+; SSE41-NEXT:    movdqa %xmm1, %xmm4
+; SSE41-NEXT:    psubd %xmm3, %xmm4
+; SSE41-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE41-NEXT:    pxor %xmm5, %xmm5
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm3
+; SSE41-NEXT:    pxor %xmm1, %xmm3
+; SSE41-NEXT:    movdqa %xmm0, %xmm1
+; SSE41-NEXT:    psubd %xmm2, %xmm1
+; SSE41-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE41-NEXT:    pcmpgtd %xmm1, %xmm0
+; SSE41-NEXT:    pxor %xmm2, %xmm0
 ; SSE41-NEXT:    movq %xmm1, 16(%rcx)
-; SSE41-NEXT:    movdqa %xmm0, (%rcx)
-; SSE41-NEXT:    movq %xmm6, 16(%rdi)
-; SSE41-NEXT:    movdqa %xmm2, (%rdi)
+; SSE41-NEXT:    movdqa %xmm4, (%rcx)
+; SSE41-NEXT:    movq %xmm0, 16(%rdi)
+; SSE41-NEXT:    movdqa %xmm3, (%rdi)
 ; SSE41-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v6i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -544,18 +368,10 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v6i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
 ; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX2-NEXT:    vmovq %xmm2, 16(%rdi)
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -564,13 +380,10 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 ; AVX512-LABEL: ssubo_v6i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
 ; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vextracti128 $1, %ymm1, %xmm2
@@ -588,70 +401,35 @@ define <6 x i32> @ssubo_v6i32(<6 x i32> %a0, <6 x i32> %a1, <6 x i32>* %p2) noun
 define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v8i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm5, %xmm5
 ; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm6
-; SSE-NEXT:    pxor %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm7
-; SSE-NEXT:    pxor %xmm6, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm4
-; SSE-NEXT:    psubd %xmm2, %xmm0
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pandn %xmm2, %xmm4
-; SSE-NEXT:    pxor %xmm2, %xmm2
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm7, %xmm7
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm7
-; SSE-NEXT:    pxor %xmm6, %xmm7
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm2
-; SSE-NEXT:    psubd %xmm3, %xmm1
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm7, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm5
-; SSE-NEXT:    pandn %xmm5, %xmm2
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm4, %xmm0
-; SSE-NEXT:    movdqa %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm0, %xmm5
+; SSE-NEXT:    psubd %xmm2, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm0
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm2
+; SSE-NEXT:    psubd %xmm3, %xmm2
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm3
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm1
+; SSE-NEXT:    pxor %xmm3, %xmm1
+; SSE-NEXT:    movdqa %xmm2, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm5, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v8i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm2
 ; AVX1-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm3, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm7, %xmm8
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm9, %xmm4, %xmm9
-; AVX1-NEXT:    vinsertf128 $1, %xmm8, %ymm9, %ymm8
-; AVX1-NEXT:    vpsubd %xmm2, %xmm6, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm3, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm7, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm2, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm3
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm3
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm2, %xmm4, %xmm2
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm4, %xmm4
 ; AVX1-NEXT:    vpsubd %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm3, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vinsertf128 $1, %xmm6, %ymm0, %ymm0
-; AVX1-NEXT:    vandnps %ymm0, %ymm8, %ymm0
+; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %ymm0, %ymm3, %ymm0
 ; AVX1-NEXT:    vmovdqa %xmm2, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -659,31 +437,20 @@ define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 ; AVX2-LABEL: ssubo_v8i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm3
-; AVX2-NEXT:    vpcmpeqd %ymm4, %ymm4, %ymm4
-; AVX2-NEXT:    vpxor %ymm4, %ymm3, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm2, %ymm5
-; AVX2-NEXT:    vpxor %ymm4, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm1, %ymm2
 ; AVX2-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm2, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm5, %ymm0
-; AVX2-NEXT:    vpxor %ymm4, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm3, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm2, %ymm0
 ; AVX2-NEXT:    vmovdqa %ymm1, (%rdi)
 ; AVX2-NEXT:    retq
 ;
 ; AVX512-LABEL: ssubo_v8i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k0
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %ymm2, %ymm1, %k0
 ; AVX512-NEXT:    vpsubd %ymm1, %ymm0, %ymm1
-; AVX512-NEXT:    vpcmpnltd %ymm2, %ymm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %ymm1, %ymm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %ymm0, %ymm0, %ymm0
 ; AVX512-NEXT:    vmovdqa32 %ymm0, %ymm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %ymm1, (%rdi)
@@ -699,128 +466,59 @@ define <8 x i32> @ssubo_v8i32(<8 x i32> %a0, <8 x i32> %a1, <8 x i32>* %p2) noun
 define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2) nounwind {
 ; SSE-LABEL: ssubo_v16i32:
 ; SSE:       # %bb.0:
-; SSE-NEXT:    pxor %xmm10, %xmm10
-; SSE-NEXT:    pxor %xmm8, %xmm8
-; SSE-NEXT:    pcmpgtd %xmm4, %xmm8
-; SSE-NEXT:    pcmpeqd %xmm11, %xmm11
-; SSE-NEXT:    pxor %xmm11, %xmm8
-; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm9
-; SSE-NEXT:    pxor %xmm11, %xmm9
-; SSE-NEXT:    pcmpeqd %xmm9, %xmm8
-; SSE-NEXT:    psubd %xmm4, %xmm0
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm9, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pandn %xmm4, %xmm8
 ; SSE-NEXT:    pxor %xmm9, %xmm9
-; SSE-NEXT:    pcmpgtd %xmm5, %xmm9
-; SSE-NEXT:    pxor %xmm11, %xmm9
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm9
-; SSE-NEXT:    psubd %xmm5, %xmm1
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm1, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm4, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pandn %xmm5, %xmm9
-; SSE-NEXT:    pxor %xmm4, %xmm4
-; SSE-NEXT:    pcmpgtd %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm11, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE-NEXT:    psubd %xmm6, %xmm2
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm2, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm5, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pandn %xmm6, %xmm4
-; SSE-NEXT:    pxor %xmm5, %xmm5
-; SSE-NEXT:    pcmpgtd %xmm7, %xmm5
-; SSE-NEXT:    pxor %xmm11, %xmm5
-; SSE-NEXT:    pxor %xmm6, %xmm6
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm6
-; SSE-NEXT:    pxor %xmm11, %xmm6
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm5
-; SSE-NEXT:    psubd %xmm7, %xmm3
-; SSE-NEXT:    pcmpgtd %xmm3, %xmm10
-; SSE-NEXT:    pxor %xmm11, %xmm10
-; SSE-NEXT:    pcmpeqd %xmm6, %xmm10
-; SSE-NEXT:    pxor %xmm11, %xmm10
-; SSE-NEXT:    pandn %xmm10, %xmm5
-; SSE-NEXT:    movdqa %xmm3, 48(%rdi)
-; SSE-NEXT:    movdqa %xmm2, 32(%rdi)
-; SSE-NEXT:    movdqa %xmm1, 16(%rdi)
-; SSE-NEXT:    movdqa %xmm0, (%rdi)
-; SSE-NEXT:    movdqa %xmm8, %xmm0
-; SSE-NEXT:    movdqa %xmm9, %xmm1
-; SSE-NEXT:    movdqa %xmm4, %xmm2
-; SSE-NEXT:    movdqa %xmm5, %xmm3
+; SSE-NEXT:    movdqa %xmm0, %xmm8
+; SSE-NEXT:    psubd %xmm4, %xmm8
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm8, %xmm0
+; SSE-NEXT:    pxor %xmm4, %xmm0
+; SSE-NEXT:    movdqa %xmm1, %xmm4
+; SSE-NEXT:    psubd %xmm5, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm4, %xmm1
+; SSE-NEXT:    pxor %xmm5, %xmm1
+; SSE-NEXT:    movdqa %xmm2, %xmm5
+; SSE-NEXT:    psubd %xmm6, %xmm5
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm5, %xmm2
+; SSE-NEXT:    pxor %xmm6, %xmm2
+; SSE-NEXT:    movdqa %xmm3, %xmm6
+; SSE-NEXT:    psubd %xmm7, %xmm6
+; SSE-NEXT:    pcmpgtd %xmm9, %xmm7
+; SSE-NEXT:    pcmpgtd %xmm6, %xmm3
+; SSE-NEXT:    pxor %xmm7, %xmm3
+; SSE-NEXT:    movdqa %xmm6, 48(%rdi)
+; SSE-NEXT:    movdqa %xmm5, 32(%rdi)
+; SSE-NEXT:    movdqa %xmm4, 16(%rdi)
+; SSE-NEXT:    movdqa %xmm8, (%rdi)
 ; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v16i32:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vextractf128 $1, %ymm3, %xmm4
-; AVX1-NEXT:    vpxor %xmm9, %xmm9, %xmm9
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm8
+; AVX1-NEXT:    vpxor %xmm5, %xmm5, %xmm5
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm4, %xmm6
 ; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm8, %xmm6, %xmm8
-; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm10
-; AVX1-NEXT:    vpcmpgtd %xmm10, %xmm9, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm6, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpandn %xmm6, %xmm8, %xmm6
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm7
-; AVX1-NEXT:    vpxor %xmm5, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm1, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm7, %xmm4, %xmm7
+; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm8
+; AVX1-NEXT:    vpcmpgtd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT:    vpxor %xmm7, %xmm6, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm3, %xmm7
 ; AVX1-NEXT:    vpsubd %xmm3, %xmm1, %xmm3
-; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm9, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm1, %xmm4, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpandn %xmm1, %xmm7, %xmm1
-; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm8
-; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm4, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpcmpeqd %xmm6, %xmm1, %xmm6
-; AVX1-NEXT:    vpsubd %xmm4, %xmm7, %xmm7
-; AVX1-NEXT:    vpcmpgtd %xmm7, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm1, %xmm1
-; AVX1-NEXT:    vpxor %xmm5, %xmm1, %xmm1
-; AVX1-NEXT:    vpandn %xmm1, %xmm6, %xmm1
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm4
-; AVX1-NEXT:    vpxor %xmm5, %xmm4, %xmm4
-; AVX1-NEXT:    vpcmpgtd %xmm0, %xmm9, %xmm6
-; AVX1-NEXT:    vpxor %xmm5, %xmm6, %xmm6
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm6, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT:    vpxor %xmm1, %xmm7, %xmm1
+; AVX1-NEXT:    vpackssdw %xmm6, %xmm1, %xmm1
+; AVX1-NEXT:    vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm6, %xmm7
+; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT:    vpsubd %xmm6, %xmm4, %xmm6
+; AVX1-NEXT:    vpcmpgtd %xmm6, %xmm4, %xmm4
+; AVX1-NEXT:    vpxor %xmm4, %xmm7, %xmm4
+; AVX1-NEXT:    vpcmpgtd %xmm5, %xmm2, %xmm5
 ; AVX1-NEXT:    vpsubd %xmm2, %xmm0, %xmm2
-; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm9, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqd %xmm0, %xmm6, %xmm0
-; AVX1-NEXT:    vpxor %xmm5, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm4, %xmm0
-; AVX1-NEXT:    vpackssdw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT:    vpacksswb %xmm8, %xmm0, %xmm1
+; AVX1-NEXT:    vpcmpgtd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm5, %xmm0
+; AVX1-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
+; AVX1-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm4 = xmm1[1,1,2,3]
 ; AVX1-NEXT:    vpmovsxbd %xmm4, %xmm4
@@ -830,40 +528,25 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[3,3,0,1]
 ; AVX1-NEXT:    vpmovsxbd %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
-; AVX1-NEXT:    vmovdqa %xmm10, 48(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm8, 48(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm3, 32(%rdi)
-; AVX1-NEXT:    vmovdqa %xmm7, 16(%rdi)
+; AVX1-NEXT:    vmovdqa %xmm6, 16(%rdi)
 ; AVX1-NEXT:    vmovdqa %xmm2, (%rdi)
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: ssubo_v16i32:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm5
-; AVX2-NEXT:    vpcmpeqd %ymm6, %ymm6, %ymm6
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm1, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm3, %ymm5
 ; AVX2-NEXT:    vpsubd %ymm3, %ymm1, %ymm3
-; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm4, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpcmpeqd %ymm1, %ymm7, %ymm1
-; AVX2-NEXT:    vpxor %ymm6, %ymm1, %ymm1
-; AVX2-NEXT:    vpandn %ymm1, %ymm5, %ymm1
+; AVX2-NEXT:    vpcmpgtd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT:    vpxor %ymm1, %ymm5, %ymm1
 ; AVX2-NEXT:    vextracti128 $1, %ymm1, %xmm5
 ; AVX2-NEXT:    vpackssdw %xmm5, %xmm1, %xmm1
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm5
-; AVX2-NEXT:    vpxor %ymm6, %ymm5, %ymm5
-; AVX2-NEXT:    vpcmpgtd %ymm0, %ymm4, %ymm7
-; AVX2-NEXT:    vpxor %ymm6, %ymm7, %ymm7
-; AVX2-NEXT:    vpcmpeqd %ymm5, %ymm7, %ymm5
+; AVX2-NEXT:    vpcmpgtd %ymm4, %ymm2, %ymm4
 ; AVX2-NEXT:    vpsubd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm4, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpcmpeqd %ymm0, %ymm7, %ymm0
-; AVX2-NEXT:    vpxor %ymm6, %ymm0, %ymm0
-; AVX2-NEXT:    vpandn %ymm0, %ymm5, %ymm0
+; AVX2-NEXT:    vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT:    vpxor %ymm0, %ymm4, %ymm0
 ; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
 ; AVX2-NEXT:    vpackssdw %xmm4, %xmm0, %xmm0
 ; AVX2-NEXT:    vpacksswb %xmm1, %xmm0, %xmm1
@@ -877,13 +560,10 @@ define <16 x i32> @ssubo_v16i32(<16 x i32> %a0, <16 x i32> %a1, <16 x i32>* %p2)
 ; AVX512-LABEL: ssubo_v16i32:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k0
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtd %zmm2, %zmm1, %k0
 ; AVX512-NEXT:    vpsubd %zmm1, %zmm0, %zmm1
-; AVX512-NEXT:    vpcmpnltd %zmm2, %zmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtd %zmm1, %zmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa64 %zmm1, (%rdi)
 ; AVX512-NEXT:    retq
@@ -1129,161 +809,42 @@ define <8 x i32> @ssubo_v8i16(<8 x i16> %a0, <8 x i16> %a1, <8 x i16>* %p2) noun
 }
 
 define <2 x i32> @ssubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) nounwind {
-; SSE2-LABEL: ssubo_v2i64:
-; SSE2:       # %bb.0:
-; SSE2-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT:    movdqa %xmm0, %xmm3
-; SSE2-NEXT:    psubq %xmm1, %xmm0
-; SSE2-NEXT:    pxor %xmm2, %xmm1
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT:    pand %xmm5, %xmm1
-; SSE2-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm1, %xmm4
-; SSE2-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE2-NEXT:    pxor %xmm1, %xmm4
-; SSE2-NEXT:    pxor %xmm2, %xmm3
-; SSE2-NEXT:    movdqa %xmm2, %xmm5
-; SSE2-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE2-NEXT:    por %xmm3, %xmm5
-; SSE2-NEXT:    pxor %xmm1, %xmm5
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSE2-NEXT:    pand %xmm4, %xmm3
-; SSE2-NEXT:    movdqa %xmm0, (%rdi)
-; SSE2-NEXT:    pxor %xmm2, %xmm0
-; SSE2-NEXT:    movdqa %xmm2, %xmm4
-; SSE2-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSE2-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE2-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE2-NEXT:    pand %xmm6, %xmm0
-; SSE2-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT:    por %xmm0, %xmm2
-; SSE2-NEXT:    pxor %xmm1, %xmm2
-; SSE2-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSE2-NEXT:    pand %xmm2, %xmm0
-; SSE2-NEXT:    pxor %xmm1, %xmm0
-; SSE2-NEXT:    pandn %xmm0, %xmm3
-; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSE2-NEXT:    retq
-;
-; SSSE3-LABEL: ssubo_v2i64:
-; SSSE3:       # %bb.0:
-; SSSE3-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSSE3-NEXT:    movdqa %xmm0, %xmm3
-; SSSE3-NEXT:    psubq %xmm1, %xmm0
-; SSSE3-NEXT:    pxor %xmm2, %xmm1
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm5, %xmm1
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm1, %xmm4
-; SSSE3-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSSE3-NEXT:    pxor %xmm1, %xmm4
-; SSSE3-NEXT:    pxor %xmm2, %xmm3
-; SSSE3-NEXT:    movdqa %xmm2, %xmm5
-; SSSE3-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSSE3-NEXT:    por %xmm3, %xmm5
-; SSSE3-NEXT:    pxor %xmm1, %xmm5
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm4, %xmm3
-; SSSE3-NEXT:    movdqa %xmm0, (%rdi)
-; SSSE3-NEXT:    pxor %xmm2, %xmm0
-; SSSE3-NEXT:    movdqa %xmm2, %xmm4
-; SSSE3-NEXT:    pcmpgtd %xmm0, %xmm4
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSSE3-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSSE3-NEXT:    pand %xmm6, %xmm0
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSSE3-NEXT:    por %xmm0, %xmm2
-; SSSE3-NEXT:    pxor %xmm1, %xmm2
-; SSSE3-NEXT:    pcmpeqd %xmm5, %xmm2
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm2[1,0,3,2]
-; SSSE3-NEXT:    pand %xmm2, %xmm0
-; SSSE3-NEXT:    pxor %xmm1, %xmm0
-; SSSE3-NEXT:    pandn %xmm0, %xmm3
-; SSSE3-NEXT:    pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
-; SSSE3-NEXT:    retq
-;
-; SSE41-LABEL: ssubo_v2i64:
-; SSE41:       # %bb.0:
-; SSE41-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE41-NEXT:    movdqa %xmm0, %xmm3
-; SSE41-NEXT:    psubq %xmm1, %xmm0
-; SSE41-NEXT:    pxor %xmm2, %xmm1
-; SSE41-NEXT:    movdqa %xmm2, %xmm4
-; SSE41-NEXT:    pcmpgtd %xmm1, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT:    pand %xmm5, %xmm1
-; SSE41-NEXT:    pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT:    por %xmm1, %xmm4
-; SSE41-NEXT:    pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT:    pxor %xmm1, %xmm4
-; SSE41-NEXT:    pxor %xmm2, %xmm3
-; SSE41-NEXT:    movdqa %xmm2, %xmm5
-; SSE41-NEXT:    pcmpgtd %xmm3, %xmm5
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT:    por %xmm3, %xmm5
-; SSE41-NEXT:    pxor %xmm1, %xmm5
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm4
-; SSE41-NEXT:    movdqa %xmm0, (%rdi)
-; SSE41-NEXT:    pxor %xmm2, %xmm0
-; SSE41-NEXT:    movdqa %xmm2, %xmm3
-; SSE41-NEXT:    pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT:    pshufd {{.*#+}} xmm6 = xmm3[0,0,2,2]
-; SSE41-NEXT:    pcmpeqd %xmm2, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT:    pand %xmm6, %xmm0
-; SSE41-NEXT:    pshufd {{.*#+}} xmm2 = xmm3[1,1,3,3]
-; SSE41-NEXT:    por %xmm0, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm2
-; SSE41-NEXT:    pcmpeqq %xmm5, %xmm2
-; SSE41-NEXT:    pxor %xmm1, %xmm2
-; SSE41-NEXT:    pandn %xmm2, %xmm4
-; SSE41-NEXT:    pshufd {{.*#+}} xmm0 = xmm4[0,2,2,3]
-; SSE41-NEXT:    retq
+; SSE-LABEL: ssubo_v2i64:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
+; SSE-NEXT:    movdqa %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm3
+; SSE-NEXT:    psubq %xmm1, %xmm0
+; SSE-NEXT:    movdqa %xmm0, (%rdi)
+; SSE-NEXT:    pxor %xmm2, %xmm0
+; SSE-NEXT:    movdqa %xmm3, %xmm4
+; SSE-NEXT:    pcmpgtd %xmm0, %xmm4
+; SSE-NEXT:    pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    pand %xmm5, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
+; SSE-NEXT:    por %xmm0, %xmm3
+; SSE-NEXT:    pxor %xmm2, %xmm1
+; SSE-NEXT:    movdqa %xmm1, %xmm0
+; SSE-NEXT:    pcmpgtd %xmm2, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
+; SSE-NEXT:    pcmpeqd %xmm2, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE-NEXT:    pand %xmm4, %xmm1
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE-NEXT:    por %xmm1, %xmm0
+; SSE-NEXT:    pxor %xmm3, %xmm0
+; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE-NEXT:    retq
 ;
 ; AVX1-LABEL: ssubo_v2i64:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX1-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX1-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX1-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX1-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX1-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
 ; AVX1-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX1-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX1-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX1-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX1-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX1-NEXT:    retq
@@ -1291,18 +852,10 @@ define <2 x i32> @ssubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX2-LABEL: ssubo_v2i64:
 ; AVX2:       # %bb.0:
 ; AVX2-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm3
-; AVX2-NEXT:    vpcmpeqd %xmm4, %xmm4, %xmm4
-; AVX2-NEXT:    vpxor %xmm4, %xmm3, %xmm3
-; AVX2-NEXT:    vpcmpgtq %xmm0, %xmm2, %xmm5
-; AVX2-NEXT:    vpxor %xmm4, %xmm5, %xmm5
-; AVX2-NEXT:    vpcmpeqq %xmm3, %xmm5, %xmm3
+; AVX2-NEXT:    vpcmpgtq %xmm2, %xmm1, %xmm2
 ; AVX2-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm2, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpcmpeqq %xmm0, %xmm5, %xmm0
-; AVX2-NEXT:    vpxor %xmm4, %xmm0, %xmm0
-; AVX2-NEXT:    vpandn %xmm0, %xmm3, %xmm0
+; AVX2-NEXT:    vpcmpgtq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpxor %xmm0, %xmm2, %xmm0
 ; AVX2-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
 ; AVX2-NEXT:    vmovdqa %xmm1, (%rdi)
 ; AVX2-NEXT:    retq
@@ -1310,13 +863,10 @@ define <2 x i32> @ssubo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-LABEL: ssubo_v2i64:
 ; AVX512:       # %bb.0:
 ; AVX512-NEXT:    vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k0
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm0, %k1
-; AVX512-NEXT:    kxorw %k0, %k1, %k0
+; AVX512-NEXT:    vpcmpgtq %xmm2, %xmm1, %k0
 ; AVX512-NEXT:    vpsubq %xmm1, %xmm0, %xmm1
-; AVX512-NEXT:    vpcmpnltq %xmm2, %xmm1, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kandw %k1, %k0, %k1
+; AVX512-NEXT:    vpcmpgtq %xmm1, %xmm0, %k1
+; AVX512-NEXT:    kxorw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
diff --git a/llvm/test/CodeGen/X86/vec_umulo.ll b/llvm/test/CodeGen/X86/vec_umulo.ll
index 61c470a3ce413..c859ce7b74bb8 100644
--- a/llvm/test/CodeGen/X86/vec_umulo.ll
+++ b/llvm/test/CodeGen/X86/vec_umulo.ll
@@ -1546,12 +1546,14 @@ define <2 x i32> @umulo_v2i64(<2 x i64> %a0, <2 x i64> %a1, <2 x i64>* %p2) noun
 ; AVX512-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
 ; AVX512-NEXT:    seto %al
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kmovd %r9d, %k1
-; AVX512-NEXT:    kshiftrw $1, %k1, %k2
-; AVX512-NEXT:    kxorw %k0, %k2, %k0
 ; AVX512-NEXT:    kshiftlw $15, %k0, %k0
 ; AVX512-NEXT:    kshiftrw $14, %k0, %k0
-; AVX512-NEXT:    kxorw %k0, %k1, %k1
+; AVX512-NEXT:    kmovd %r9d, %k1
+; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftrw $15, %k1, %k1
+; AVX512-NEXT:    kshiftlw $2, %k0, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
 ; AVX512-NEXT:    vmovdqa %xmm1, (%rdi)
@@ -1986,7 +1988,9 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
+; AVX512-NEXT:    kshiftlw $15, %k0, %k0
+; AVX512-NEXT:    kshiftrw $15, %k0, %k1
+; AVX512-NEXT:    kshiftlw $2, %k0, %k0
 ; AVX512-NEXT:    movl %edx, %eax
 ; AVX512-NEXT:    mulb %sil
 ; AVX512-NEXT:    movl %eax, %edx
@@ -1996,11 +2000,12 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
 ; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kshiftlw $1, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $14, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
+; AVX512-NEXT:    kshiftlw $3, %k0, %k2
 ; AVX512-NEXT:    movl %r11d, %eax
 ; AVX512-NEXT:    mulb %bl
 ; AVX512-NEXT:    movl %eax, %esi
@@ -2009,13 +2014,12 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %cl
 ; AVX512-NEXT:    orb %al, %cl
 ; AVX512-NEXT:    setne %al
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
+; AVX512-NEXT:    kmovd %eax, %k3
+; AVX512-NEXT:    kshiftlw $2, %k3, %k3
+; AVX512-NEXT:    korw %k3, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $13, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftlw $13, %k0, %k0
-; AVX512-NEXT:    kshiftrw $13, %k0, %k0
 ; AVX512-NEXT:    movl %r9d, %eax
 ; AVX512-NEXT:    mulb %r10b
 ; AVX512-NEXT:    # kill: def $al killed $al def $eax
@@ -2024,30 +2028,37 @@ define <4 x i32> @umulo_v4i1(<4 x i1> %a0, <4 x i1> %a1, <4 x i1>* %p2) nounwind
 ; AVX512-NEXT:    setne %bl
 ; AVX512-NEXT:    orb %cl, %bl
 ; AVX512-NEXT:    setne %cl
-; AVX512-NEXT:    kmovd %ecx, %k1
-; AVX512-NEXT:    kshiftlw $3, %k1, %k1
-; AVX512-NEXT:    korw %k1, %k0, %k1
+; AVX512-NEXT:    kmovd %ecx, %k2
+; AVX512-NEXT:    kshiftlw $3, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
 ; AVX512-NEXT:    vpcmpeqd %xmm0, %xmm0, %xmm0
 ; AVX512-NEXT:    vmovdqa32 %xmm0, %xmm0 {%k1} {z}
-; AVX512-NEXT:    kmovd %r8d, %k0
-; AVX512-NEXT:    kshiftrw $1, %k0, %k1
-; AVX512-NEXT:    kmovd %edx, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
-; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $14, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
-; AVX512-NEXT:    kshiftrw $2, %k0, %k1
-; AVX512-NEXT:    kmovd %esi, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kmovd %r8d, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
-; AVX512-NEXT:    kshiftrw $13, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    kshiftrw $15, %k1, %k1
+; AVX512-NEXT:    kmovd %edx, %k2
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $14, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k0, %k0
+; AVX512-NEXT:    korw %k0, %k1, %k0
 ; AVX512-NEXT:    kshiftrw $3, %k0, %k1
-; AVX512-NEXT:    kmovd %eax, %k2
-; AVX512-NEXT:    kxorw %k2, %k1, %k1
+; AVX512-NEXT:    kshiftlw $3, %k1, %k1
+; AVX512-NEXT:    kshiftlw $14, %k0, %k0
+; AVX512-NEXT:    kshiftrw $14, %k0, %k0
+; AVX512-NEXT:    kmovd %esi, %k2
+; AVX512-NEXT:    kshiftlw $15, %k2, %k2
+; AVX512-NEXT:    kshiftrw $13, %k2, %k2
+; AVX512-NEXT:    korw %k2, %k1, %k1
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    kshiftrw $4, %k0, %k1
+; AVX512-NEXT:    kshiftlw $4, %k1, %k1
+; AVX512-NEXT:    kshiftlw $13, %k0, %k0
+; AVX512-NEXT:    kshiftrw $13, %k0, %k0
+; AVX512-NEXT:    korw %k1, %k0, %k0
+; AVX512-NEXT:    kmovd %eax, %k1
 ; AVX512-NEXT:    kshiftlw $15, %k1, %k1
 ; AVX512-NEXT:    kshiftrw $12, %k1, %k1
-; AVX512-NEXT:    kxorw %k1, %k0, %k0
+; AVX512-NEXT:    korw %k0, %k1, %k0
 ; AVX512-NEXT:    kmovd %k0, %eax
 ; AVX512-NEXT:    movb %al, (%rdi)
 ; AVX512-NEXT:    popq %rbx
diff --git a/llvm/test/CodeGen/X86/vector-bitreverse.ll b/llvm/test/CodeGen/X86/vector-bitreverse.ll
index d28af7ad751ad..954d7a2c52a28 100644
--- a/llvm/test/CodeGen/X86/vector-bitreverse.ll
+++ b/llvm/test/CodeGen/X86/vector-bitreverse.ll
@@ -1911,30 +1911,26 @@ define <16 x i32> @test_bitreverse_v16i32(<16 x i32> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v16i32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrld $24, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsrld $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm2, %zmm2
-; AVX512F-NEXT:    vpord %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpslld $24, %zmm0, %zmm2
-; AVX512F-NEXT:    vpslld $8, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm0, %zmm2, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $4, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $4, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $2, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpslld $1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandd {{.*}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrld $1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpord %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12,3,2,1,0,7,6,5,4,11,10,9,8,15,14,13,12]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
+; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v16i32:
@@ -2219,42 +2215,26 @@ define <8 x i64> @test_bitreverse_v8i64(<8 x i64> %a) nounwind {
 ;
 ; AVX512F-LABEL: test_bitreverse_v8i64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpsrlq $56, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsrlq $40, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsrlq $24, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsrlq $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vporq %zmm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vpsllq $8, %zmm0, %zmm2
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm2, %zmm2
-; AVX512F-NEXT:    vpsllq $24, %zmm0, %zmm3
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm3, %zmm3
-; AVX512F-NEXT:    vporq %zmm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vpsllq $56, %zmm0, %zmm3
-; AVX512F-NEXT:    vpsllq $40, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm0, %zmm3, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $4, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $4, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $2, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm1
-; AVX512F-NEXT:    vpsllq $1, %zmm1, %zmm1
-; AVX512F-NEXT:    vpandq {{.*}}(%rip){1to8}, %zmm0, %zmm0
-; AVX512F-NEXT:    vpsrlq $1, %zmm0, %zmm0
-; AVX512F-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240,0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
+; AVX512F-NEXT:    vpshufb %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15,0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm6, %ymm1
+; AVX512F-NEXT:    vpor %ymm1, %ymm4, %ymm1
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm2
+; AVX512F-NEXT:    vpshufb %ymm2, %ymm5, %ymm2
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand %ymm3, %ymm0, %ymm0
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm6, %ymm0
+; AVX512F-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: test_bitreverse_v8i64:
diff --git a/llvm/test/CodeGen/X86/vector-compare-results.ll b/llvm/test/CodeGen/X86/vector-compare-results.ll
index 2d9fb798267bc..47a6ffb0aed5f 100644
--- a/llvm/test/CodeGen/X86/vector-compare-results.ll
+++ b/llvm/test/CodeGen/X86/vector-compare-results.ll
@@ -827,27 +827,49 @@ define <64 x i1> @test_cmp_v64i8(<64 x i8> %a0, <64 x i8> %a1) nounwind {
 ;
 ; AVX512F-LABEL: test_cmp_v64i8:
 ; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    movq %rdi, %rax
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512F-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512F-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
 ; AVX512F-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512F-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512F-NEXT:    # kill: def $xmm2 killed $xmm2 killed $ymm2
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm1
+; AVX512F-NEXT:    vptestmd %zmm1, %zmm1, %k0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm0
+; AVX512F-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k3
+; AVX512F-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512F-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512F-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512F-NEXT:    kmovw %k0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_cmp_v64i8:
 ; AVX512DQ:       # %bb.0:
+; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512DQ-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512DQ-NEXT:    vpcmpgtb %ymm2, %ymm3, %ymm2
 ; AVX512DQ-NEXT:    vpcmpgtb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-NEXT:    # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512DQ-NEXT:    # kill: def $xmm2 killed $xmm2 killed $ymm2
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm1
+; AVX512DQ-NEXT:    vpmovd2m %zmm1, %k0
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpmovsxbd %xmm2, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
+; AVX512DQ-NEXT:    vextracti128 $1, %ymm2, %xmm0
+; AVX512DQ-NEXT:    vpmovsxbd %xmm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
+; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1453,35 +1475,45 @@ define <64 x i1> @test_cmp_v64i16(<64 x i16> %a0, <64 x i16> %a1) nounwind {
 ;
 ; AVX512F-LABEL: test_cmp_v64i16:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm1
+; AVX512F-NEXT:    movq %rdi, %rax
 ; AVX512F-NEXT:    vpcmpgtw %ymm4, %ymm0, %ymm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512F-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512F-NEXT:    vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k1
+; AVX512F-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k2
+; AVX512F-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm0
+; AVX512F-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512F-NEXT:    vptestmd %zmm0, %zmm0, %k3
+; AVX512F-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512F-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512F-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512F-NEXT:    kmovw %k0, (%rdi)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512DQ-LABEL: test_cmp_v64i16:
 ; AVX512DQ:       # %bb.0:
-; AVX512DQ-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm3
-; AVX512DQ-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm2
-; AVX512DQ-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm1
+; AVX512DQ-NEXT:    movq %rdi, %rax
 ; AVX512DQ-NEXT:    vpcmpgtw %ymm4, %ymm0, %ymm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm0, %xmm0
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm1, %xmm1
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero,ymm2[8],zero,ymm2[9],zero,ymm2[10],zero,ymm2[11],zero,ymm2[12],zero,ymm2[13],zero,ymm2[14],zero,ymm2[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm2, %xmm2
-; AVX512DQ-NEXT:    vpmovzxwd {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero,ymm3[8],zero,ymm3[9],zero,ymm3[10],zero,ymm3[11],zero,ymm3[12],zero,ymm3[13],zero,ymm3[14],zero,ymm3[15],zero
-; AVX512DQ-NEXT:    vpmovdb %zmm3, %xmm3
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k0
+; AVX512DQ-NEXT:    vpcmpgtw %ymm5, %ymm1, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k1
+; AVX512DQ-NEXT:    vpcmpgtw %ymm6, %ymm2, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k2
+; AVX512DQ-NEXT:    vpcmpgtw %ymm7, %ymm3, %ymm0
+; AVX512DQ-NEXT:    vpmovsxwd %ymm0, %zmm0
+; AVX512DQ-NEXT:    vpmovd2m %zmm0, %k3
+; AVX512DQ-NEXT:    kmovw %k3, 6(%rdi)
+; AVX512DQ-NEXT:    kmovw %k2, 4(%rdi)
+; AVX512DQ-NEXT:    kmovw %k1, 2(%rdi)
+; AVX512DQ-NEXT:    kmovw %k0, (%rdi)
 ; AVX512DQ-NEXT:    vzeroupper
 ; AVX512DQ-NEXT:    retq
 ;
@@ -1674,10 +1706,12 @@ define <128 x i1> @test_cmp_v128i8(<128 x i8> %a0, <128 x i8> %a1) nounwind {
 ;
 ; AVX512BW-LABEL: test_cmp_v128i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpcmpgtb %zmm3, %zmm1, %k0
-; AVX512BW-NEXT:    vpcmpgtb %zmm2, %zmm0, %k1
-; AVX512BW-NEXT:    vpmovm2b %k1, %zmm0
-; AVX512BW-NEXT:    vpmovm2b %k0, %zmm1
+; AVX512BW-NEXT:    movq %rdi, %rax
+; AVX512BW-NEXT:    vpcmpgtb %zmm2, %zmm0, %k0
+; AVX512BW-NEXT:    vpcmpgtb %zmm3, %zmm1, %k1
+; AVX512BW-NEXT:    kmovq %k1, 8(%rdi)
+; AVX512BW-NEXT:    kmovq %k0, (%rdi)
+; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = icmp sgt <128 x i8> %a0, %a1
   ret <128 x i1> %1
diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll
index b8bbc5b2bd567..12a5f2bc2cc6a 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll
@@ -2980,14 +2980,53 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index f969aef6c980d..cf8a80cf9db9b 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -2503,14 +2503,53 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 9f2cad9f954b8..b6c5d9f744ef4 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -1548,53 +1548,41 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandn %ymm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm1, %ymm4, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
index 706cdf016f0b0..ce521ad8896d9 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll
@@ -1835,14 +1835,37 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index eb4304e3845c4..ca624b0a82ead 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -436,20 +436,16 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1497,14 +1493,37 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index f6302313c0bde..8cb0f36a17626 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -172,47 +172,39 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
-; AVX512VL-NEXT:    vpand %ymm6, %ymm2, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
+; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm7, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm8
-; AVX512VL-NEXT:    vpand %ymm7, %ymm8, %ymm8
-; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm7
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm8, %ymm7
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm7, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm9
-; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpor %ymm5, %ymm9, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm5, %ymm3
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpand %ymm6, %ymm1, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpand %ymm4, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm7, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand %ymm8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -826,35 +818,27 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll
index d1ac70aa7ccf4..00f5d73a4def1 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll
@@ -3001,14 +3001,53 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm1, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VL-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %xmm0, %xmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %xmm1, %xmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm2, %xmm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 9f74d39077a9b..8898373bfe817 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -2504,14 +2504,53 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VBMI2:       # %bb.0:
+; AVX512VBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512VBMI2-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VBMI2-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLBW-NEXT:    retq
+;
+; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLVBMI2:       # %bb.0:
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %ymm1, %ymm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm0
+; AVX512VLVBMI2-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 893260e1dd6bd..ca559a6911a32 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -1532,53 +1532,41 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpsllw $4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpand %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpor %ymm3, %ymm2, %ymm2
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm4 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpandn %ymm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm1, %ymm4, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VBMI2:       # %bb.0:
-; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VBMI2-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLVBMI2:       # %bb.0:
-; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLVBMI2-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLVBMI2-NEXT:    vpsllw $4, %zmm0, %zmm2
+; AVX512VLVBMI2-NEXT:    vpsrlw $4, %zmm1, %zmm0
+; AVX512VLVBMI2-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm2, %zmm0
 ; AVX512VLVBMI2-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
index 7e3d7fc6cf86c..d88a2a214caec 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll
@@ -1917,14 +1917,37 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_funnnel_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index f0c9d1e37cfdd..bf7c057965b32 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -475,22 +475,18 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
 ;
 ; AVX512VL-LABEL: var_funnnel_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512VL-NEXT:    vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT:    vpsubb %ymm1, %ymm3, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
+; AVX512VL-NEXT:    vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm2, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1575,14 +1571,37 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind {
 ; AVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; AVX2-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_funnnel_v32i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512-NEXT:    vpor %ymm1, %ymm0, %ymm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_funnnel_v32i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512F-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsrlw $4, %ymm0, %ymm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpsllw $4, %ymm0, %ymm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_funnnel_v32i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8:
 ; XOPAVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 84d14b6e3cf80..3838dfd4dd149 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -181,50 +181,42 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vpxor %xmm6, %xmm6, %xmm6
-; AVX512VL-NEXT:    vpsubb %ymm2, %ymm6, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpxor %xmm4, %xmm4, %xmm4
+; AVX512VL-NEXT:    vpsubb %ymm2, %ymm4, %ymm2
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7]
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm8, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm9
-; AVX512VL-NEXT:    vpand %ymm8, %ymm9, %ymm9
-; AVX512VL-NEXT:    vpor %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm5
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm8
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm5, %ymm9, %ymm8
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
-; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm9 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm9, %ymm4, %ymm4
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm8, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm8 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm8, %ymm5, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm10
-; AVX512VL-NEXT:    vpor %ymm4, %ymm10, %ymm4
+; AVX512VL-NEXT:    vpor %ymm5, %ymm10, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm5, %ymm3
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
-; AVX512VL-NEXT:    vpsubb %ymm1, %ymm6, %ymm1
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm5
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm5
+; AVX512VL-NEXT:    vpsubb %ymm1, %ymm4, %ymm1
 ; AVX512VL-NEXT:    vpand %ymm7, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm5, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm8, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm8, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm9, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand %ymm9, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm8, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -846,35 +838,27 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8:
 ; AVX512VLBW:       # %bb.0:
-; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
-; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm1, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>)
   ret <64 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index d42a7f8401889..88b0f0b467948 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -1800,17 +1800,13 @@ define i8 @test_v16i8(<16 x i8> %a0) {
 ; AVX2-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
 ; AVX2-NEXT:    vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
 ; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vmovdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255]
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrlq $32, %xmm0, %xmm2
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT:    vpsrld $16, %xmm0, %xmm2
-; AVX2-NEXT:    vpmullw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT:    vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT:    vpshufd {{.*#+}} xmm1 = xmm0[1,1,2,3]
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[2],zero,xmm0[2],zero,xmm0[4],zero,xmm0[6],zero,xmm0[8],zero,xmm0[10],zero,xmm0[12],zero,xmm0[14],zero
+; AVX2-NEXT:    vpmullw %xmm1, %xmm0, %xmm0
 ; AVX2-NEXT:    vpextrb $0, %xmm0, %eax
 ; AVX2-NEXT:    # kill: def $al killed $al killed $eax
 ; AVX2-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll
index e9802e2586b13..3acdca7cda574 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-128.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll
@@ -1801,14 +1801,37 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind {
 ; AVX-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; AVX-NEXT:    retq
 ;
-; AVX512-LABEL: splatconstant_rotate_v16i8:
-; AVX512:       # %bb.0:
-; AVX512-NEXT:    vpsllw $4, %xmm0, %xmm1
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
-; AVX512-NEXT:    vpsrlw $4, %xmm0, %xmm0
-; AVX512-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512-NEXT:    retq
+; AVX512F-LABEL: splatconstant_rotate_v16i8:
+; AVX512F:       # %bb.0:
+; AVX512F-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512F-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512F-NEXT:    retq
+;
+; AVX512VL-LABEL: splatconstant_rotate_v16i8:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VL-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VL-NEXT:    retq
+;
+; AVX512BW-LABEL: splatconstant_rotate_v16i8:
+; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm1, %xmm1
+; AVX512BW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512BW-NEXT:    vpand {{.*}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX512BW-NEXT:    retq
+;
+; AVX512VLBW-LABEL: splatconstant_rotate_v16i8:
+; AVX512VLBW:       # %bb.0:
+; AVX512VLBW-NEXT:    vpsllw $4, %xmm0, %xmm1
+; AVX512VLBW-NEXT:    vpsrlw $4, %xmm0, %xmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %xmm1, %xmm0
+; AVX512VLBW-NEXT:    retq
 ;
 ; XOP-LABEL: splatconstant_rotate_v16i8:
 ; XOP:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 9478db6c4ae80..df76a7738f8ea 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -432,20 +432,16 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
 ;
 ; AVX512VL-LABEL: var_rotate_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm2, %ymm2
-; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
-; AVX512VL-NEXT:    vpor %ymm2, %ymm3, %ymm2
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm2
+; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm2, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm3, %ymm3
@@ -1509,11 +1505,9 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v32i8:
@@ -1528,10 +1522,8 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind {
 ; AVX512VLBW-LABEL: splatconstant_rotate_v32i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %ymm0, %ymm1
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VLBW-NEXT:    vpor %ymm0, %ymm1, %ymm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VLBW-NEXT:    retq
 ;
 ; XOPAVX1-LABEL: splatconstant_rotate_v32i8:
@@ -1795,11 +1787,9 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind {
 ;
 ; AVX512VL-LABEL: splatconstant_rotate_mask_v32i8:
 ; AVX512VL:       # %bb.0:
-; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm1
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm1, %ymm1
-; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm1
+; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $216, {{.*}}(%rip), %ymm1, %ymm0
 ; AVX512VL-NEXT:    vpand {{.*}}(%rip), %ymm0, %ymm0
 ; AVX512VL-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index 2c3dbd8bb9210..d92d73a220d8b 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -167,44 +167,36 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm1, %ymm2
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm5, %ymm4
-; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm6
-; AVX512VL-NEXT:    vpand %ymm5, %ymm6, %ymm6
-; AVX512VL-NEXT:    vpor %ymm4, %ymm6, %ymm4
+; AVX512VL-NEXT:    vpsllw $4, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm6, %ymm5
 ; AVX512VL-NEXT:    vpsllw $5, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm6 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm6, %ymm4
-; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm7
-; AVX512VL-NEXT:    vpand %ymm6, %ymm7, %ymm7
-; AVX512VL-NEXT:    vpor %ymm4, %ymm7, %ymm4
+; AVX512VL-NEXT:    vpsllw $2, %ymm3, %ymm5
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm7, %ymm5
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
-; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm3, %ymm4
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm7 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512VL-NEXT:    vpand %ymm7, %ymm4, %ymm4
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm3, %ymm3, %ymm8
 ; AVX512VL-NEXT:    vpor %ymm4, %ymm8, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm2, %ymm2, %ymm2
 ; AVX512VL-NEXT:    vpblendvb %ymm2, %ymm4, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm5, %ymm3
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm5, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm6, %ymm4
 ; AVX512VL-NEXT:    vpsllw $5, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $6, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpandn %ymm3, %ymm6, %ymm3
 ; AVX512VL-NEXT:    vpsllw $2, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpand %ymm6, %ymm4, %ymm4
-; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
+; AVX512VL-NEXT:    vpternlogq $226, %ymm3, %ymm7, %ymm4
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm3, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpblendvb %ymm1, %ymm4, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vpsrlw $7, %ymm0, %ymm3
-; AVX512VL-NEXT:    vpand %ymm7, %ymm3, %ymm3
+; AVX512VL-NEXT:    vpand %ymm5, %ymm3, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm0, %ymm0, %ymm4
 ; AVX512VL-NEXT:    vpor %ymm3, %ymm4, %ymm3
 ; AVX512VL-NEXT:    vpaddb %ymm1, %ymm1, %ymm1
@@ -815,35 +807,27 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm2
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
 ;
 ; AVX512BW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512BW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512BW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512VLBW-LABEL: splatconstant_rotate_v64i8:
 ; AVX512VLBW:       # %bb.0:
 ; AVX512VLBW-NEXT:    vpsllw $4, %zmm0, %zmm1
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm1, %zmm1
 ; AVX512VLBW-NEXT:    vpsrlw $4, %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vpandq {{.*}}(%rip), %zmm0, %zmm0
-; AVX512VLBW-NEXT:    vporq %zmm0, %zmm1, %zmm0
+; AVX512VLBW-NEXT:    vpternlogq $216, {{.*}}(%rip), %zmm1, %zmm0
 ; AVX512VLBW-NEXT:    retq
   %shl = shl <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
   %lshr = lshr <64 x i8> %a, <i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4, i8 4>
@@ -964,18 +948,14 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind {
 ; AVX512VL:       # %bb.0:
 ; AVX512VL-NEXT:    vextracti64x4 $1, %zmm0, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm1, %ymm2
-; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
-; AVX512VL-NEXT:    vpandn %ymm2, %ymm3, %ymm2
 ; AVX512VL-NEXT:    vpsllw $4, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpand %ymm3, %ymm1, %ymm1
-; AVX512VL-NEXT:    vpor %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm3 = [240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240,240]
+; AVX512VL-NEXT:    vpternlogq $226, %ymm2, %ymm3, %ymm1
 ; AVX512VL-NEXT:    vmovdqa {{.*#+}} ymm2 = [39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39]
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm1, %ymm1
 ; AVX512VL-NEXT:    vpsrlw $4, %ymm0, %ymm4
-; AVX512VL-NEXT:    vpandn %ymm4, %ymm3, %ymm4
 ; AVX512VL-NEXT:    vpsllw $4, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpand %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpternlogq $226, %ymm4, %ymm3, %ymm0
 ; AVX512VL-NEXT:    vpand %ymm2, %ymm0, %ymm0
 ; AVX512VL-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512VL-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
index 84d68e1c58dfb..7db60b9226445 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v4.ll
@@ -658,74 +658,21 @@ define <4 x double> @shuffle_v4f64_0456(<4 x double> %a, <4 x double> %b) {
 }
 
 define <4 x double> @shuffle_v4f64_0z3z(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: shuffle_v4f64_0z3z:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: shuffle_v4f64_0z3z:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX2-SLOW-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: shuffle_v4f64_0z3z:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: shuffle_v4f64_0z3z:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,3,2]
-; AVX512VL-SLOW-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512VL-SLOW-NEXT:    vblendpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v4f64_0z3z:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6,7],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[24,25,26,27,28,29,30,31],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-FAST-NEXT:    retq
+; ALL-LABEL: shuffle_v4f64_0z3z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[3],ymm1[2]
+; ALL-NEXT:    retq
   %shuffle = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 0, i32 4, i32 3, i32 4>
   ret <4 x double> %shuffle
 }
 
 define <4 x double> @shuffle_v4f64_1z2z(<4 x double> %a, <4 x double> %b) {
-; AVX1-LABEL: shuffle_v4f64_1z2z:
-; AVX1:       # %bb.0:
-; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    retq
-;
-; AVX2-SLOW-LABEL: shuffle_v4f64_1z2z:
-; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,2,0]
-; AVX2-SLOW-NEXT:    retq
-;
-; AVX2-FAST-LABEL: shuffle_v4f64_1z2z:
-; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX2-FAST-NEXT:    retq
-;
-; AVX512VL-SLOW-LABEL: shuffle_v4f64_1z2z:
-; AVX512VL-SLOW:       # %bb.0:
-; AVX512VL-SLOW-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX512VL-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[1,0,2,0]
-; AVX512VL-SLOW-NEXT:    retq
-;
-; AVX512VL-FAST-LABEL: shuffle_v4f64_1z2z:
-; AVX512VL-FAST:       # %bb.0:
-; AVX512VL-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[16,17,18,19,20,21,22,23],zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512VL-FAST-NEXT:    retq
+; ALL-LABEL: shuffle_v4f64_1z2z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[2]
+; ALL-NEXT:    retq
   %1 = shufflevector <4 x double> %a, <4 x double> <double 0.000000e+00, double undef, double undef, double undef>, <4 x i32> <i32 1, i32 4, i32 2, i32 4>
   ret <4 x double> %1
 }
@@ -1776,9 +1723,8 @@ define <4 x i64> @shuffle_v4i64_1230(<4 x i64> %a) {
 define <4 x i64> @shuffle_v4i64_z0z3(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_z0z3:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,3]
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[3]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i64_z0z3:
@@ -1812,11 +1758,7 @@ define <4 x i64> @shuffle_v4i64_1z2z(<4 x i64> %a, <4 x i64> %b) {
 ; AVX1-LABEL: shuffle_v4i64_1z2z:
 ; AVX1:       # %bb.0:
 ; AVX1-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vblendpd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3]
-; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT:    vunpcklpd {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; AVX1-NEXT:    vpermilpd {{.*#+}} xmm0 = xmm0[1,0]
-; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm0[1],ymm1[0],ymm0[2],ymm1[2]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-SLOW-LABEL: shuffle_v4i64_1z2z:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
index 52b04fdc8dd18..dfe2d4db18f16 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-512-v8.ll
@@ -841,19 +841,11 @@ define <8 x double> @shuffle_v8f64_f511235a(<8 x double> %a, <8 x double> %b) {
 }
 
 define <8 x double> @shuffle_v8f64_1z2z5z6z(<8 x double> %a, <8 x double> %b) {
-; AVX512F-LABEL: shuffle_v8f64_1z2z5z6z:
-; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovapd {{.*#+}} zmm2 = [1,8,2,8,5,8,6,8]
-; AVX512F-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    retq
-;
-; AVX512F-32-LABEL: shuffle_v8f64_1z2z5z6z:
-; AVX512F-32:       # %bb.0:
-; AVX512F-32-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
-; AVX512F-32-NEXT:    vmovapd {{.*#+}} zmm2 = [1,0,8,0,2,0,8,0,5,0,8,0,6,0,8,0]
-; AVX512F-32-NEXT:    vpermt2pd %zmm1, %zmm2, %zmm0
-; AVX512F-32-NEXT:    retl
+; ALL-LABEL: shuffle_v8f64_1z2z5z6z:
+; ALL:       # %bb.0:
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[1],zmm1[0],zmm0[2],zmm1[2],zmm0[5],zmm1[4],zmm0[6],zmm1[6]
+; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> %a, <8 x double> <double 0.000000e+00, double undef, double undef, double undef, double undef, double undef, double undef, double undef>, <8 x i32> <i32 1, i32 8, i32 2, i32 8, i32 5, i32 8, i32 6, i32 8>
   ret <8 x double> %shuffle
 }
@@ -1767,8 +1759,8 @@ define <8 x double> @shuffle_v8f64_0z2z4z6z(<8 x double> %a, <8 x double> %b) {
 ;
 ; ALL-LABEL: shuffle_v8f64_0z2z4z6z:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; ALL-NEXT:    vunpcklpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; ALL-NEXT:    vxorpd %xmm1, %xmm1, %xmm1
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
 ; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> %a, <8 x double> zeroinitializer, <8 x i32><i32 0, i32 8, i32 2, i32 8, i32 4, i32 8, i32 6, i32 8>
   ret <8 x double> %shuffle
@@ -1809,8 +1801,8 @@ define <8 x double> @shuffle_v8f64_z9zbzdzf(<8 x double> %a, <8 x double> %b) {
 ;
 ; ALL-LABEL: shuffle_v8f64_z9zbzdzf:
 ; ALL:       # %bb.0:
-; ALL-NEXT:    vxorps %xmm0, %xmm0, %xmm0
-; ALL-NEXT:    vunpckhpd {{.*#+}} zmm0 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
+; ALL-NEXT:    vxorpd %xmm0, %xmm0, %xmm0
+; ALL-NEXT:    vshufpd {{.*#+}} zmm0 = zmm0[0],zmm1[1],zmm0[2],zmm1[3],zmm0[4],zmm1[5],zmm0[6],zmm1[7]
 ; ALL-NEXT:    ret{{[l|q]}}
   %shuffle = shufflevector <8 x double> zeroinitializer, <8 x double> %b, <8 x i32><i32 0, i32 9, i32 0, i32 11, i32 0, i32 13, i32 0, i32 15>
   ret <8 x double> %shuffle
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
index df2c2f55af45e..871321e391eee 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll
@@ -2910,3 +2910,74 @@ define <8 x i16> @shuffle_extract_insert_double(<8 x i16> %a, <8 x i16> %b) {
   %7 = insertelement <8 x i16> %6, i16 %b15, i32 7
   ret <8 x i16> %7
 }
+
+define <8 x i16> @shuffle_extract_concat_insert(<4 x i16> %lhsa, <4 x i16> %rhsa, <8 x i16> %b) {
+; SSE2-LABEL: shuffle_extract_concat_insert:
+; SSE2:       # %bb.0:
+; SSE2-NEXT:    movd %xmm1, %eax
+; SSE2-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE2-NEXT:    pextrw $5, %xmm2, %edx
+; SSE2-NEXT:    pextrw $7, %xmm2, %esi
+; SSE2-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT:    pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
+; SSE2-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT:    pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7]
+; SSE2-NEXT:    pinsrw $4, %ecx, %xmm0
+; SSE2-NEXT:    pinsrw $5, %edx, %xmm0
+; SSE2-NEXT:    pinsrw $6, %eax, %xmm0
+; SSE2-NEXT:    pinsrw $7, %esi, %xmm0
+; SSE2-NEXT:    retq
+;
+; SSSE3-LABEL: shuffle_extract_concat_insert:
+; SSSE3:       # %bb.0:
+; SSSE3-NEXT:    pextrw $2, %xmm1, %eax
+; SSSE3-NEXT:    pextrw $5, %xmm2, %ecx
+; SSSE3-NEXT:    pextrw $7, %xmm2, %edx
+; SSSE3-NEXT:    movd %xmm1, %esi
+; SSSE3-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,0,1,14,15,8,9,14,15,12,13,14,15]
+; SSSE3-NEXT:    pinsrw $4, %eax, %xmm0
+; SSSE3-NEXT:    pinsrw $5, %ecx, %xmm0
+; SSSE3-NEXT:    pinsrw $6, %esi, %xmm0
+; SSSE3-NEXT:    pinsrw $7, %edx, %xmm0
+; SSSE3-NEXT:    retq
+;
+; SSE41-LABEL: shuffle_extract_concat_insert:
+; SSE41:       # %bb.0:
+; SSE41-NEXT:    movd %xmm1, %eax
+; SSE41-NEXT:    pextrw $2, %xmm1, %ecx
+; SSE41-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE41-NEXT:    pshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,0,1,14,15,8,9,14,15,12,13,14,15]
+; SSE41-NEXT:    pinsrw $4, %ecx, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5],xmm0[6,7]
+; SSE41-NEXT:    pinsrw $6, %eax, %xmm0
+; SSE41-NEXT:    pblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm2[7]
+; SSE41-NEXT:    retq
+;
+; AVX-LABEL: shuffle_extract_concat_insert:
+; AVX:       # %bb.0:
+; AVX-NEXT:    vmovd %xmm1, %eax
+; AVX-NEXT:    vpextrw $2, %xmm1, %ecx
+; AVX-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,2,3,0,1,14,15,8,9,14,15,12,13,14,15]
+; AVX-NEXT:    vpinsrw $4, %ecx, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm2[5],xmm0[6,7]
+; AVX-NEXT:    vpinsrw $6, %eax, %xmm0, %xmm0
+; AVX-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6],xmm2[7]
+; AVX-NEXT:    retq
+  %a = shufflevector <4 x i16> %lhsa, <4 x i16> %rhsa, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %a0 = extractelement <8 x i16> %a, i32 0
+  %a4 = extractelement <8 x i16> %a, i32 4
+  %a6 = extractelement <8 x i16> %a, i32 6
+  %b11 = extractelement <8 x i16> %b, i32 3
+  %b13 = extractelement <8 x i16> %b, i32 5
+  %b15 = extractelement <8 x i16> %b, i32 7
+  %1 = shufflevector <8 x i16> %a, <8 x i16> %b, <8 x i32> <i32 2, i32 8, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+  %2 = insertelement <8 x i16> %1, i16 %a0, i32 2
+  %3 = insertelement <8 x i16> %2, i16 %b11, i32 3
+  %4 = insertelement <8 x i16> %3, i16 %a6, i32 4
+  %5 = insertelement <8 x i16> %4, i16 %b13, i32 5
+  %6 = insertelement <8 x i16> %5, i16 %a4, i32 6
+  %7 = insertelement <8 x i16> %6, i16 %b15, i32 7
+  ret <8 x i16> %7
+}
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index fa03c8984ad70..f1c19d91cd323 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -156,8 +156,7 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512F-LABEL: trunc_usat_v4i64_v4i32:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512F-NEXT:    vpcmpltuq %zmm1, %zmm0, %k1
+; AVX512F-NEXT:    vpcmpltuq {{.*}}(%rip){1to8}, %zmm0, %k1
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,429496729]
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vpmovqd %zmm1, %ymm0
@@ -177,8 +176,7 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
 ; AVX512BW-LABEL: trunc_usat_v4i64_v4i32:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT:    vpbroadcastq {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,4294967295]
-; AVX512BW-NEXT:    vpcmpltuq %zmm1, %zmm0, %k1
+; AVX512BW-NEXT:    vpcmpltuq {{.*}}(%rip){1to8}, %zmm0, %k1
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [4294967295,4294967295,4294967295,429496729]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vpmovqd %zmm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
new file mode 100644
index 0000000000000..1750cbf2cf62a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/vshli-simplify-demanded-bits.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+; A combine forming X86ISD::VSHLI was missing a test and not using
+; TargetConstant for the RHS operand.
+; https://bugs.chromium.org/p/chromium/issues/detail?id=1005750
+
+define <8 x i8> @vshli_target_constant(<8 x i16> %arg, <8 x i32> %arg1) {
+; CHECK-LABEL: vshli_target_constant:
+; CHECK:       # %bb.0: # %bb
+; CHECK-NEXT:    movdqa {{.*#+}} xmm0 = [2863311531,2863311531,2863311531,2863311531]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm3
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm3[1,3,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; CHECK-NEXT:    pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm2
+; CHECK-NEXT:    pshufd {{.*#+}} xmm2 = xmm2[1,3,2,3]
+; CHECK-NEXT:    pmuludq %xmm0, %xmm1
+; CHECK-NEXT:    pshufd {{.*#+}} xmm0 = xmm1[1,3,2,3]
+; CHECK-NEXT:    punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; CHECK-NEXT:    movq {{.*#+}} xmm1 = mem[0],zero
+; CHECK-NEXT:    pslld $15, %xmm2
+; CHECK-NEXT:    psrad $16, %xmm2
+; CHECK-NEXT:    pslld $15, %xmm4
+; CHECK-NEXT:    psrad $16, %xmm4
+; CHECK-NEXT:    packssdw %xmm2, %xmm4
+; CHECK-NEXT:    pand {{.*}}(%rip), %xmm4
+; CHECK-NEXT:    pxor %xmm0, %xmm0
+; CHECK-NEXT:    punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-NEXT:    pmullw %xmm4, %xmm1
+; CHECK-NEXT:    movdqa %xmm1, %xmm0
+; CHECK-NEXT:    punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; CHECK-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [128,128,128,128]
+; CHECK-NEXT:    paddd %xmm2, %xmm1
+; CHECK-NEXT:    paddd %xmm2, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm0
+; CHECK-NEXT:    psrld $8, %xmm1
+; CHECK-NEXT:    movdqa {{.*#+}} xmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; CHECK-NEXT:    pand %xmm2, %xmm1
+; CHECK-NEXT:    pand %xmm2, %xmm0
+; CHECK-NEXT:    packuswb %xmm1, %xmm0
+; CHECK-NEXT:    packuswb %xmm0, %xmm0
+; CHECK-NEXT:    retq
+bb:
+  %tmp = udiv <8 x i32> %arg1, <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+  %tmp2 = and <8 x i32> %tmp, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+  %tmp3 = load <8 x i8>, <8 x i8>* undef, align 1
+  %tmp4 = zext <8 x i8> %tmp3 to <8 x i32>
+  %tmp5 = mul nuw nsw <8 x i32> %tmp2, %tmp4
+  %tmp6 = add nuw nsw <8 x i32> %tmp5, <i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128, i32 128>
+  %tmp7 = lshr <8 x i32> %tmp6, <i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
+  %tmp8 = trunc <8 x i32> %tmp7 to <8 x i8>
+  ret <8 x i8> %tmp8
+}
diff --git a/llvm/test/DebugInfo/Generic/block-asan.ll b/llvm/test/DebugInfo/Generic/block-asan.ll
index ee9a52522cfe6..1d706fc6b84e6 100644
--- a/llvm/test/DebugInfo/Generic/block-asan.ll
+++ b/llvm/test/DebugInfo/Generic/block-asan.ll
@@ -70,7 +70,7 @@ attributes #3 = { nounwind }
 !10 = !{i32 1, !"PIC Level", i32 2}
 !11 = !{!"clang version 3.6.0 (trunk 223120) (llvm/trunk 223119)"}
 !12 = !DILocalVariable(name: "x", line: 4, scope: !4, file: !5, type: !13)
-!13 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, flags: DIFlagBlockByrefStruct, file: !1, scope: !5, elements: !14)
+!13 = !DICompositeType(tag: DW_TAG_structure_type, size: 224, file: !1, scope: !5, elements: !14)
 !14 = !{!15, !17, !18, !20, !21}
 !15 = !DIDerivedType(tag: DW_TAG_member, name: "__isa", size: 64, align: 64, file: !1, scope: !5, baseType: !16)
 !16 = !DIDerivedType(tag: DW_TAG_pointer_type, size: 64, align: 64, baseType: null)
diff --git a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
index 70c5501ff421c..f4857ccd7e843 100644
--- a/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
+++ b/llvm/test/DebugInfo/MIR/X86/dbgcall-site-interpretation.mir
@@ -21,10 +21,8 @@
 # CHECK-NEXT:     DW_AT_low_pc
 # CHECK-EMPTY:
 # CHECK-NEXT:     DW_TAG_GNU_call_site_parameter
-# CHECK-NEXT:       DW_AT_location      (DW_OP_reg2 RCX)
-# CHECK-NEXT:       DW_AT_GNU_call_site_value   (DW_OP_fbreg +8, DW_OP_deref)
-# CHECK-EMPTY:
-# CHECK-NEXT:     DW_TAG_GNU_call_site_parameter
+# RCX loads memory location. We can't rely that memory location won't be changed.
+# CHECK-NOT:       DW_AT_location      (DW_OP_reg2 RCX)
 # CHECK-NEXT:       DW_AT_location      (DW_OP_reg4 RSI)
 # CHECK-NEXT:       DW_AT_GNU_call_site_value   (DW_OP_lit4)
 # CHECK-EMPTY:
diff --git a/llvm/test/DebugInfo/X86/dwarfdump-debug-loc-error-cases.s b/llvm/test/DebugInfo/X86/dwarfdump-debug-loc-error-cases.s
index 78196bfecf46b..1d43037fd6eea 100644
--- a/llvm/test/DebugInfo/X86/dwarfdump-debug-loc-error-cases.s
+++ b/llvm/test/DebugInfo/X86/dwarfdump-debug-loc-error-cases.s
@@ -16,8 +16,13 @@
 # RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux --defsym CASE6=0 -o %t6.o
 # RUN: llvm-dwarfdump -debug-loc %t6.o 2>&1 | FileCheck %s
 
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux --defsym CASE7=0 -o %t7.o
+# RUN: llvm-dwarfdump -debug-loc %t7.o 2>&1 | FileCheck %s --check-prefix=UNKNOWN-REG
+
 # CHECK: error: unexpected end of data
 
+# UNKNOWN-REG: [0x0000000000000000,  0x0000000000000001): DW_OP_regx 0xdeadbeef
+
 .section  .debug_loc,"",@progbits
 .ifdef CASE1
   .byte  1                       # bogus
@@ -45,6 +50,17 @@
   .quad  1                       # ending offset
   .word  0xffff                  # Loc expr size
 .endif
+.ifdef CASE7
+  .quad  0                       # starting offset
+  .quad  1                       # ending offset
+  .word  2f-1f                   # Loc expr size
+1:
+  .byte  0x90                    # DW_OP_regx
+  .uleb128 0xdeadbeef
+2:
+  .quad  0                       # starting offset
+  .quad  0                       # ending offset
+.endif
 
 # A minimal compile unit is needed to deduce the address size of the location
 # lists
diff --git a/llvm/test/DebugInfo/X86/gnu-public-names.ll b/llvm/test/DebugInfo/X86/gnu-public-names.ll
index 6ea2fd5135cbf..d7fe13d797048 100644
--- a/llvm/test/DebugInfo/X86/gnu-public-names.ll
+++ b/llvm/test/DebugInfo/X86/gnu-public-names.ll
@@ -354,7 +354,7 @@ attributes #1 = { nounwind readnone speculatable }
 
 !0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
 !1 = distinct !DIGlobalVariable(name: "static_member_variable", linkageName: "_ZN1C22static_member_variableE", scope: !2, file: !3, line: 7, type: !13, isLocal: false, isDefinition: true, declaration: !22)
-!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !3, producer: "clang version 9.0.0 (trunk 363288) (llvm/trunk 363294)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !16, globals: !17, imports: !54, nameTableKind: GNU)
+!2 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_03, file: !3, producer: "clang version 9.0.0 (trunk 363288) (llvm/trunk 363294)", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !4, retainedTypes: !16, globals: !17, imports: !54, nameTableKind: GNU)
 !3 = !DIFile(filename: "names.cpp", directory: "/usr/local/google/home/blaikie/dev/scratch")
 !4 = !{!5, !9, !12}
 !5 = !DICompositeType(tag: DW_TAG_enumeration_type, file: !3, line: 49, baseType: !6, size: 32, elements: !7)
diff --git a/llvm/test/Demangle/invalid-manglings.test b/llvm/test/Demangle/invalid-manglings.test
index 543c116916e6d..b77288488b2db 100644
--- a/llvm/test/Demangle/invalid-manglings.test
+++ b/llvm/test/Demangle/invalid-manglings.test
@@ -374,3 +374,8 @@
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ??_P
 ; CHECK-NEXT: error: Invalid mangled name
+
+.?AUBase@@@8
+; CHECK-EMPTY:
+; CHECK-NEXT: .?AUBase@@@8
+; CHECK-NEXT: error: Invalid mangled name
diff --git a/llvm/test/Demangle/ms-operators.test b/llvm/test/Demangle/ms-operators.test
index 71a3a1305b456..b940488786631 100644
--- a/llvm/test/Demangle/ms-operators.test
+++ b/llvm/test/Demangle/ms-operators.test
@@ -209,6 +209,9 @@
 ??_R0?AUBase@@@8
 ; CHECK: struct Base `RTTI Type Descriptor'
 
+.?AUBase@@
+; CHECK: struct Base `RTTI Type Descriptor Name'
+
 ??_R1A@?0A@EA@Base@@8
 ; CHECK: Base::`RTTI Base Class Descriptor at (0, -1, 0, 64)'
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
new file mode 100644
index 0000000000000..d2acbf6bdf415
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca-compat.ll
@@ -0,0 +1,17 @@
+; Test that alloca instrumentation with old API levels does not use short granules.
+;
+; RUN: opt < %s -hwasan -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+declare void @use32(i32*)
+
+define void @test_alloca() sanitize_hwaddress {
+; CHECK-LABEL: @test_alloca(
+; CHECK: %[[X_TAG:[^ ]*]] = trunc i64 {{.*}} to i8
+; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 {{.*}}, i8 %[[X_TAG]], i64 1, i1 false)
+  %x = alloca i32, align 4
+  call void @use32(i32* nonnull %x)
+  ret void
+}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
index 3525f4ddb9498..15a1c6828af90 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/alloca.ll
@@ -5,7 +5,7 @@
 ; RUN: opt < %s -hwasan -hwasan-with-ifunc=1 -hwasan-uar-retag-to-zero=0 -S | FileCheck %s --check-prefixes=CHECK,DYNAMIC-SHADOW,UAR-TAGS
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android"
+target triple = "aarch64--linux-android10000"
 
 declare void @use32(i32*)
 
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
new file mode 100644
index 0000000000000..22626ed50c27e
--- /dev/null
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic-compat.ll
@@ -0,0 +1,13 @@
+; Test that the old outlined check is used with old API levels.
+
+; RUN: opt < %s -hwasan -S | FileCheck %s
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64--linux-android"
+
+define i8 @test_load8(i8* %a) sanitize_hwaddress {
+; CHECK-LABEL: @test_load8(
+; CHECK: call void @llvm.hwasan.check.memaccess(i8* {{.*}}, i8* {{.*}}, i32 0)
+  %b = load i8, i8* %a, align 4
+  ret i8 %b
+}
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
index 79e414a5e4ad3..d5406117c8741 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/basic.ll
@@ -14,7 +14,7 @@
 ; CHECK: @llvm.global_ctors = appending global [1 x { i32, void ()*, i8* }] [{ i32, void ()*, i8* } { i32 0, void ()* @hwasan.module_ctor, i8* bitcast (void ()* @hwasan.module_ctor to i8*) }]
 
 target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-target triple = "aarch64--linux-android"
+target triple = "aarch64--linux-android10000"
 
 define i8 @test_load8(i8* %a) sanitize_hwaddress {
 ; CHECK-LABEL: @test_load8(
@@ -56,8 +56,8 @@ define i8 @test_load8(i8* %a) sanitize_hwaddress {
 
 ; RECOVER: [[CONT]]:
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 0)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 0)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 0)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 0)
 
 ; CHECK: %[[G:[^ ]*]] = load i8, i8* %a, align 4
 ; CHECK: ret i8 %[[G]]
@@ -108,8 +108,8 @@ define i16 @test_load16(i16* %a) sanitize_hwaddress {
 ; RECOVER: [[CONT]]:
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 1)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 1)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 1)
 
 ; CHECK: %[[G:[^ ]*]] = load i16, i16* %a, align 4
 ; CHECK: ret i16 %[[G]]
@@ -136,8 +136,8 @@ define i32 @test_load32(i32* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 2)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 2)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 2)
 
 ; CHECK: %[[G:[^ ]*]] = load i32, i32* %a, align 4
 ; CHECK: ret i32 %[[G]]
@@ -164,8 +164,8 @@ define i64 @test_load64(i64* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 3)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 3)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 3)
 
 ; CHECK: %[[G:[^ ]*]] = load i64, i64* %a, align 8
 ; CHECK: ret i64 %[[G]]
@@ -192,8 +192,8 @@ define i128 @test_load128(i128* %a) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 4)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 4)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 4)
 
 ; CHECK: %[[G:[^ ]*]] = load i128, i128* %a, align 16
 ; CHECK: ret i128 %[[G]]
@@ -232,8 +232,8 @@ define void @test_store8(i8* %a, i8 %b) sanitize_hwaddress {
 ; RECOVER: call void asm sideeffect "brk #2352", "{x0}"(i64 %[[A]])
 ; RECOVER: br label
 
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %a, i32 16)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %a, i32 16)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %a, i32 16)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %a, i32 16)
 
 ; CHECK: store i8 %b, i8* %a, align 4
 ; CHECK: ret void
@@ -260,8 +260,8 @@ define void @test_store16(i16* %a, i16 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i16* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 17)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 17)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 17)
 
 ; CHECK: store i16 %b, i16* %a, align 4
 ; CHECK: ret void
@@ -288,8 +288,8 @@ define void @test_store32(i32* %a, i32 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i32* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 18)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 18)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 18)
 
 ; CHECK: store i32 %b, i32* %a, align 4
 ; CHECK: ret void
@@ -316,8 +316,8 @@ define void @test_store64(i64* %a, i64 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i64* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 19)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 19)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 19)
 
 ; CHECK: store i64 %b, i64* %a, align 8
 ; CHECK: ret void
@@ -344,8 +344,8 @@ define void @test_store128(i128* %a, i128 %b) sanitize_hwaddress {
 ; RECOVER: br label
 
 ; ABORT: %[[A:[^ ]*]] = bitcast i128* %a to i8*
-; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
-; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess(i8* null, i8* %[[A]], i32 20)
+; ABORT-DYNAMIC-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* %.hwasan.shadow, i8* %[[A]], i32 20)
+; ABORT-ZERO-BASED-SHADOW: call void @llvm.hwasan.check.memaccess.shortgranules(i8* null, i8* %[[A]], i32 20)
 
 ; CHECK: store i128 %b, i128* %a, align 16
 ; CHECK: ret void
diff --git a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
index af95f9a455806..1a4c04947a2b1 100644
--- a/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/PR23499.ll
@@ -14,14 +14,14 @@ $_Z3barIvEvv = comdat any
 @__profn__Z3barIvEvv = linkonce_odr hidden constant [11 x i8] c"_Z3barIvEvv", align 1
 
 ; CHECK-NOT: __profn__Z3barIvEvv
-; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat($__profv__Z3barIvEvv), align 8
-; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat($__profv__Z3barIvEvv), align 8
+; CHECK: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}__llvm_prf_cnts", comdat, align 8
+; CHECK: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}__llvm_prf_data{{.*}}", comdat, align 8
 ; CHECK: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"
 
 
 ; COFF-NOT: __profn__Z3barIvEvv
-; COFF: @__profc__Z3barIvEvv = linkonce_odr dso_local global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
-; COFF: @__profd__Z3barIvEvv = internal global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat($__profc__Z3barIvEvv), align 8
+; COFF: @__profc__Z3barIvEvv = linkonce_odr hidden global [1 x i64] zeroinitializer, section "{{.*}}prfc$M", comdat, align 8
+; COFF: @__profd__Z3barIvEvv = linkonce_odr hidden global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 4947693190065689389, i64 0, i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__Z3barIvEvv, i32 0, i32 0), i8*{{.*}}, i8* null, i32 1, [2 x i16] zeroinitializer }, section "{{.*}}prfd{{.*}}", comdat, align 8
 
 
 declare void @llvm.instrprof.increment(i8*, i64, i32, i32) #1
diff --git a/llvm/test/Instrumentation/InstrProfiling/comdat.ll b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
index 03ee905fa65c1..3f169c4b73423 100644
--- a/llvm/test/Instrumentation/InstrProfiling/comdat.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/comdat.ll
@@ -15,10 +15,10 @@ $foo_inline = comdat any
 
 @__profn_foo_inline = linkonce_odr hidden constant [10 x i8] c"foo_inline"
 
-; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_inline), align 8
-; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_inline), align 8
-; COFF: @__profc_foo_inline = linkonce_odr dso_local global{{.*}}, section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_inline = internal global{{.*}}, section ".lprfd$M", comdat($__profc_foo_inline), align 8
+; ELF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
+; COFF: @__profc_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_inline = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define weak_odr void @foo_inline() comdat {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_inline, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
@@ -28,10 +28,10 @@ $foo_extern = comdat any
 
 @__profn_foo_extern = linkonce_odr hidden constant [10 x i8] c"foo_extern"
 
-; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat($__profv_foo_extern)
-; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat($__profv_foo_extern)
-; COFF: @__profc_foo_extern = linkonce_odr dso_local global{{.*}}, section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_extern = internal global{{.*}}, section ".lprfd$M", comdat($__profc_foo_extern), align 8
+; ELF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_cnts", comdat, align 8
+; ELF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section "__llvm_prf_data", comdat, align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = linkonce_odr hidden global{{.*}}, section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
diff --git a/llvm/test/Instrumentation/InstrProfiling/linkage.ll b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
index 29ae5d6e90cb5..8a4dee2f70c4f 100644
--- a/llvm/test/Instrumentation/InstrProfiling/linkage.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/linkage.ll
@@ -53,12 +53,12 @@ define linkonce_odr void @foo_inline() {
   ret void
 }
 
-; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat($__profv_foo_extern), align 8
-; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat($__profv_foo_extern), align 8
+; LINUX: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_cnts", comdat, align 8
+; LINUX: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section "__llvm_prf_data", comdat, align 8
 ; MACHO: @__profc_foo_extern = linkonce_odr hidden global
 ; MACHO: @__profd_foo_extern = linkonce_odr hidden global
-; COFF: @__profc_foo_extern = linkonce_odr dso_local global {{.*}}section ".lprfc$M", comdat, align 8
-; COFF: @__profd_foo_extern = internal global {{.*}}section ".lprfd$M", comdat($__profc_foo_extern), align 8
+; COFF: @__profc_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfc$M", comdat, align 8
+; COFF: @__profd_foo_extern = linkonce_odr hidden global {{.*}}section ".lprfd$M", comdat, align 8
 define available_externally void @foo_extern() {
   call void @llvm.instrprof.increment(i8* getelementptr inbounds ([10 x i8], [10 x i8]* @__profn_foo_extern, i32 0, i32 0), i64 0, i32 1, i32 0)
   ret void
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
new file mode 100644
index 0000000000000..63de8663e0775
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_launder_invariant.ll
@@ -0,0 +1,38 @@
+; Make sure MSan handles llvm.launder.invariant.group correctly.
+
+; RUN: opt < %s -msan -msan-kernel=1 -O1 -S | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -O1 -S | FileCheck -check-prefixes=CHECK %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%class.Foo = type { i32 (...)** }
+@flag = dso_local local_unnamed_addr global i8 0, align 1
+
+define dso_local %class.Foo* @_Z1fv() local_unnamed_addr #0 {
+entry:
+  %p = alloca i8*, align 8
+  %0 = bitcast i8** %p to i8*
+  call void @llvm.lifetime.start.p0i8(i64 8, i8* nonnull %0)
+  %1 = load i8, i8* @flag, align 1
+  %tobool = icmp ne i8 %1, 0
+  %call = call zeroext i1 @_Z2f1PPvb(i8** nonnull %p, i1 zeroext %tobool)
+  %2 = load i8*, i8** %p, align 8
+  %3 = call i8* @llvm.launder.invariant.group.p0i8(i8* %2)
+  %4 = bitcast i8* %3 to %class.Foo*
+  %retval.0 = select i1 %call, %class.Foo* %4, %class.Foo* null
+  call void @llvm.lifetime.end.p0i8(i64 8, i8* nonnull %0)
+  ret %class.Foo* %retval.0
+}
+
+; CHECK-NOT: call void @__msan_warning_noreturn
+
+declare dso_local zeroext i1 @_Z2f1PPvb(i8**, i1 zeroext) local_unnamed_addr
+
+declare i8* @llvm.launder.invariant.group.p0i8(i8*)
+
+declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture)
+
+declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture)
+
+attributes #0 = { sanitize_memory uwtable }
diff --git a/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
new file mode 100644
index 0000000000000..f3b5c0d722c8a
--- /dev/null
+++ b/llvm/test/Instrumentation/MemorySanitizer/msan_llvm_strip_invariant.ll
@@ -0,0 +1,21 @@
+; Make sure MSan handles llvm.launder.invariant.group correctly.
+
+; RUN: opt < %s -msan -msan-kernel=1 -O1 -S | FileCheck -check-prefixes=CHECK %s
+; RUN: opt < %s -msan -O1 -S | FileCheck -check-prefixes=CHECK %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@flag = dso_local local_unnamed_addr global i8 0, align 1
+
+define dso_local i8* @f(i8* %x) local_unnamed_addr #0 {
+entry:
+  %0 = call i8* @llvm.strip.invariant.group.p0i8(i8* %x)
+  ret i8* %0
+}
+
+; CHECK-NOT: call void @__msan_warning_noreturn
+
+declare i8* @llvm.strip.invariant.group.p0i8(i8*)
+
+attributes #0 = { sanitize_memory uwtable }
diff --git a/llvm/test/MC/AMDGPU/expressions-gfx10.s b/llvm/test/MC/AMDGPU/expressions-gfx10.s
index bca8444f80bff..b3f051b819b7f 100644
--- a/llvm/test/MC/AMDGPU/expressions-gfx10.s
+++ b/llvm/test/MC/AMDGPU/expressions-gfx10.s
@@ -1,4 +1,5 @@
-// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64 -show-encoding %s | FileCheck %s --check-prefix=GFX10
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64 -show-encoding %s | FileCheck %s --check-prefix=GFX10
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1010 -mattr=-WavefrontSize32,+WavefrontSize64 %s 2>&1 | FileCheck -check-prefix=NOGFX10 %s
 
 i1=1
 
@@ -47,3 +48,26 @@ v_cmp_f_i32 s[10:11], u+1, v2
 v_lshlrev_b64 v[5:6], u-1, v[2:3]
 // GFX10: v_lshlrev_b64 v[5:6], u-1, v[2:3] ; encoding: [0x05,0x00,0xff,0xd6,0xff,0x04,0x02,0x00,A,A,A,A]
 // GFX10-NEXT:                              ;   fixup A - offset: 8, value: u-1, kind: FK_Data_4
+
+//===----------------------------------------------------------------------===//
+// Instructions can use only one literal.
+// Relocatable expressions are counted as literals.
+//===----------------------------------------------------------------------===//
+
+s_sub_u32 s0, 123, u
+// NOGFX10: error: only one literal operand is allowed
+
+s_sub_u32 s0, u, u
+// NOGFX10: error: only one literal operand is allowed
+
+s_sub_u32 s0, u, u1
+// NOGFX10: error: only one literal operand is allowed
+
+v_bfe_u32 v0, v2, 123, u
+// NOGFX10: error: invalid literal operand
+
+v_bfe_u32 v0, v2, u, u
+// NOGFX10: error: invalid literal operand
+
+v_bfe_u32 v0, v2, u, u1
+// NOGFX10: error: invalid literal operand
diff --git a/llvm/test/MC/AMDGPU/expressions-gfx9.s b/llvm/test/MC/AMDGPU/expressions-gfx9.s
index b1be72b019afe..a52887596af60 100644
--- a/llvm/test/MC/AMDGPU/expressions-gfx9.s
+++ b/llvm/test/MC/AMDGPU/expressions-gfx9.s
@@ -8,6 +8,23 @@
 v_mov_b32_sdwa v1, sext(u)
 // NOGFX9: error: expected an absolute expression
 
+//===----------------------------------------------------------------------===//
+// Instructions can use only one literal.
+// Relocatable expressions are counted as literals.
+//===----------------------------------------------------------------------===//
+
+s_sub_u32 s0, 123, u
+// NOGFX9: error: only one literal operand is allowed
+
+s_sub_u32 s0, u, u
+// NOGFX9: error: only one literal operand is allowed
+
+s_sub_u32 s0, u, u1
+// NOGFX9: error: only one literal operand is allowed
+
+v_bfe_u32 v0, v2, v3, u
+// NOGFX9: error: invalid literal operand
+
 //===----------------------------------------------------------------------===//
 // Constant expressions may be used with 'sext' modifier
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/reg-syntax-err.s b/llvm/test/MC/AMDGPU/reg-syntax-err.s
index 39fc1ebcee972..4376f5b896ff4 100644
--- a/llvm/test/MC/AMDGPU/reg-syntax-err.s
+++ b/llvm/test/MC/AMDGPU/reg-syntax-err.s
@@ -62,3 +62,12 @@ s_mov_b32 s1, xnack_mask_lo s1
 
 exp mrt0 v1, v2, v3, v4000 off
 // NOVI: error: failed parsing operand
+
+v_add_f64 v[0:1], v[0:1], v[0xF00000001:0x2]
+// NOVI: error: failed parsing operand
+
+v_add_f64 v[0:1], v[0:1], v[0x1:0xF00000002]
+// NOVI: error: failed parsing operand
+
+s_mov_b32 s1, s[0:-1]
+// NOVI: error: failed parsing operand
diff --git a/llvm/test/MC/ARM/directive-arch-armv2.s b/llvm/test/MC/ARM/directive-arch-armv2.s
index ceec7cfeda9f7..8985a0e19ff63 100644
--- a/llvm/test/MC/ARM/directive-arch-armv2.s
+++ b/llvm/test/MC/ARM/directive-arch-armv2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv2
diff --git a/llvm/test/MC/ARM/directive-arch-armv2a.s b/llvm/test/MC/ARM/directive-arch-armv2a.s
index c07718e2edbec..7ee434dde61c3 100644
--- a/llvm/test/MC/ARM/directive-arch-armv2a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv2a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv2a
diff --git a/llvm/test/MC/ARM/directive-arch-armv3.s b/llvm/test/MC/ARM/directive-arch-armv3.s
index ad773c9cfaf7a..6e514b69c08e9 100644
--- a/llvm/test/MC/ARM/directive-arch-armv3.s
+++ b/llvm/test/MC/ARM/directive-arch-armv3.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv3
diff --git a/llvm/test/MC/ARM/directive-arch-armv3m.s b/llvm/test/MC/ARM/directive-arch-armv3m.s
index be58782774992..08843c834b77d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv3m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv3m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv3m
diff --git a/llvm/test/MC/ARM/directive-arch-armv4.s b/llvm/test/MC/ARM/directive-arch-armv4.s
index 78e97e7aee44a..d49d4fd6c751a 100644
--- a/llvm/test/MC/ARM/directive-arch-armv4.s
+++ b/llvm/test/MC/ARM/directive-arch-armv4.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv4
diff --git a/llvm/test/MC/ARM/directive-arch-armv4t.s b/llvm/test/MC/ARM/directive-arch-armv4t.s
index 11f06ef2025a6..a4eafb1e238aa 100644
--- a/llvm/test/MC/ARM/directive-arch-armv4t.s
+++ b/llvm/test/MC/ARM/directive-arch-armv4t.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv4t
diff --git a/llvm/test/MC/ARM/directive-arch-armv5.s b/llvm/test/MC/ARM/directive-arch-armv5.s
index a3cbaa93b95d0..8f94af66d6fb8 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5
diff --git a/llvm/test/MC/ARM/directive-arch-armv5t.s b/llvm/test/MC/ARM/directive-arch-armv5t.s
index fc1370f98cd0f..234180ecb581b 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5t.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5t.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5t
diff --git a/llvm/test/MC/ARM/directive-arch-armv5te.s b/llvm/test/MC/ARM/directive-arch-armv5te.s
index 42229206087a9..2c4c148a363b8 100644
--- a/llvm/test/MC/ARM/directive-arch-armv5te.s
+++ b/llvm/test/MC/ARM/directive-arch-armv5te.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv5te
diff --git a/llvm/test/MC/ARM/directive-arch-armv6-m.s b/llvm/test/MC/ARM/directive-arch-armv6-m.s
index bc33f330eae6f..c37b8b1680c58 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv6.s b/llvm/test/MC/ARM/directive-arch-armv6.s
index 04a963edaaa99..2c631500bb9a5 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6
diff --git a/llvm/test/MC/ARM/directive-arch-armv6k.s b/llvm/test/MC/ARM/directive-arch-armv6k.s
index 1af24942e13c3..ba6adea597e0e 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6k.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6k.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6k
diff --git a/llvm/test/MC/ARM/directive-arch-armv6t2.s b/llvm/test/MC/ARM/directive-arch-armv6t2.s
index dd9905cf1b81e..0be5169b37769 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6t2.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6t2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6t2
diff --git a/llvm/test/MC/ARM/directive-arch-armv6z.s b/llvm/test/MC/ARM/directive-arch-armv6z.s
index 509040f45789e..efebe951e269c 100644
--- a/llvm/test/MC/ARM/directive-arch-armv6z.s
+++ b/llvm/test/MC/ARM/directive-arch-armv6z.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv6z
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-a.s b/llvm/test/MC/ARM/directive-arch-armv7-a.s
index 6c1f8fcafd6f9..c3bd86de1349b 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-m.s b/llvm/test/MC/ARM/directive-arch-armv7-m.s
index ac41bab5ca0dd..96df23a96b2e0 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7-r.s b/llvm/test/MC/ARM/directive-arch-armv7-r.s
index a7b57542610c6..b67aee3f2ea22 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7-r.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7-r.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7-r
diff --git a/llvm/test/MC/ARM/directive-arch-armv7.s b/llvm/test/MC/ARM/directive-arch-armv7.s
index 104c7fec54800..1ea2bcf89fbe1 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7
diff --git a/llvm/test/MC/ARM/directive-arch-armv7a.s b/llvm/test/MC/ARM/directive-arch-armv7a.s
index 9189dedcbb5b3..e17e170fb057b 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7a
diff --git a/llvm/test/MC/ARM/directive-arch-armv7e-m.s b/llvm/test/MC/ARM/directive-arch-armv7e-m.s
index bebf2dcda65a1..39f9184fd4213 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7e-m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7e-m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
   .syntax  unified
   .arch  armv7e-m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7em.s b/llvm/test/MC/ARM/directive-arch-armv7em.s
index ac6d1cc59dfc0..10f9effebcc4d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7em.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7em.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
   .syntax  unified
   .arch  armv7em
diff --git a/llvm/test/MC/ARM/directive-arch-armv7m.s b/llvm/test/MC/ARM/directive-arch-armv7m.s
index aafb3ee31e843..5b623af34e6a2 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7m.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7m.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7m
diff --git a/llvm/test/MC/ARM/directive-arch-armv7r.s b/llvm/test/MC/ARM/directive-arch-armv7r.s
index d99b5da5074c8..f4461483b5353 100644
--- a/llvm/test/MC/ARM/directive-arch-armv7r.s
+++ b/llvm/test/MC/ARM/directive-arch-armv7r.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv7r
diff --git a/llvm/test/MC/ARM/directive-arch-armv8-a.s b/llvm/test/MC/ARM/directive-arch-armv8-a.s
index 1b5152a4fd465..fd567c8e4b3a6 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv8.2-a.s b/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
index 1118b1261713e..8cd7c2f740a5d 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8.2-a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8.2-a
diff --git a/llvm/test/MC/ARM/directive-arch-armv8a.s b/llvm/test/MC/ARM/directive-arch-armv8a.s
index 0226231afc962..e9a5bef89d546 100644
--- a/llvm/test/MC/ARM/directive-arch-armv8a.s
+++ b/llvm/test/MC/ARM/directive-arch-armv8a.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	armv8a
diff --git a/llvm/test/MC/ARM/directive-arch-iwmmxt.s b/llvm/test/MC/ARM/directive-arch-iwmmxt.s
index 2a15bbe537c18..2a9317ddf4623 100644
--- a/llvm/test/MC/ARM/directive-arch-iwmmxt.s
+++ b/llvm/test/MC/ARM/directive-arch-iwmmxt.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	iwmmxt
diff --git a/llvm/test/MC/ARM/directive-arch-iwmmxt2.s b/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
index dec5abad8b7ca..90b2181ee2922 100644
--- a/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
+++ b/llvm/test/MC/ARM/directive-arch-iwmmxt2.s
@@ -6,7 +6,7 @@
 @ RUN: llvm-mc -triple arm-eabi -filetype asm %s \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ASM
 @ RUN: llvm-mc -triple arm-eabi -filetype obj %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax	unified
 	.arch	iwmmxt2
diff --git a/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s b/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
index b659cd2c285cf..194c77e6a75cf 100644
--- a/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
+++ b/llvm/test/MC/ARM/directive-eabi_attribute-overwrite.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s -check-prefix CHECK-ATTR
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s -check-prefix CHECK-ATTR
 
 	.syntax unified
 	.thumb
diff --git a/llvm/test/MC/ARM/directive-eabi_attribute.s b/llvm/test/MC/ARM/directive-eabi_attribute.s
index c25b0ae152af2..445bc215cfb7e 100644
--- a/llvm/test/MC/ARM/directive-eabi_attribute.s
+++ b/llvm/test/MC/ARM/directive-eabi_attribute.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc < %s -triple armv7-unknown-linux-gnueabi -filetype=obj -o - \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s --check-prefix=CHECK-OBJ
 
         .syntax unified
         .thumb
diff --git a/llvm/test/MC/ARM/directive-fpu-multiple.s b/llvm/test/MC/ARM/directive-fpu-multiple.s
index eaa5af97f7a50..4f474f0a95543 100644
--- a/llvm/test/MC/ARM/directive-fpu-multiple.s
+++ b/llvm/test/MC/ARM/directive-fpu-multiple.s
@@ -3,7 +3,7 @@
 @ The later .fpu directive should overwrite the earlier one.
 @ We also check here that all the .fpu directives that we expect to work do work
 
-@ RUN: llvm-mc -triple arm-eabi -filetype obj %s | llvm-readobj --arm-attributes \
+@ RUN: llvm-mc -triple arm-eabi -filetype obj %s | llvm-readobj --arch-specific \
 @ RUN:   | FileCheck %s -check-prefix CHECK-ATTR
 
 	.fpu none
diff --git a/llvm/test/MC/ARM/directive-object_arch-2.s b/llvm/test/MC/ARM/directive-object_arch-2.s
index 9f661182ffc57..b536f53e7f37b 100644
--- a/llvm/test/MC/ARM/directive-object_arch-2.s
+++ b/llvm/test/MC/ARM/directive-object_arch-2.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s
 
 	.syntax unified
 
diff --git a/llvm/test/MC/ARM/directive-object_arch.s b/llvm/test/MC/ARM/directive-object_arch.s
index 3b09940855543..74de1734a40d9 100644
--- a/llvm/test/MC/ARM/directive-object_arch.s
+++ b/llvm/test/MC/ARM/directive-object_arch.s
@@ -1,5 +1,5 @@
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes | FileCheck %s
+@ RUN:   | llvm-readobj --arch-specific | FileCheck %s
 
 	.syntax unified
 
diff --git a/llvm/test/MC/ARM/vfp-aliases-diagnostics.s b/llvm/test/MC/ARM/vfp-aliases-diagnostics.s
index 30fc6bb2e3f49..e72a8d41519c4 100644
--- a/llvm/test/MC/ARM/vfp-aliases-diagnostics.s
+++ b/llvm/test/MC/ARM/vfp-aliases-diagnostics.s
@@ -13,17 +13,17 @@ aliases:
 	fldmeax sp!, {s0}
 
 @ CHECK-LABEL: aliases
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK:	fstmeax sp!, {s0}
 @ CHECK:                     ^
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK:	fldmfdx sp!, {s0}
 @ CHECK:                     ^
 
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK:	fstmfdx sp!, {s0}
 @ CHECK:                     ^
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK:	fldmeax sp!, {s0}
 @ CHECK:                     ^
 
@@ -31,16 +31,16 @@ aliases:
 	fstmiaxhs r0, {s0}
 	fstmiaxls r0, {s0}
 	fstmiaxvs r0, {s0}
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK: 	fstmiaxcs r0, {s0}
 @ CHECK:                      ^
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK: 	fstmiaxhs r0, {s0}
 @ CHECK:                      ^
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK: 	fstmiaxls r0, {s0}
 @ CHECK:                      ^
-@ CHECK: error: operand must be a list of registers in range [d0, d31]
+@ CHECK: error: operand must be a list of registers in range [d0, d15]
 @ CHECK: 	fstmiaxvs r0, {s0}
 @ CHECK:                      ^
 
diff --git a/llvm/test/MC/ARM/vscclrm-asm.s b/llvm/test/MC/ARM/vscclrm-asm.s
index d86de90f849f1..3f679c7fdd097 100644
--- a/llvm/test/MC/ARM/vscclrm-asm.s
+++ b/llvm/test/MC/ARM/vscclrm-asm.s
@@ -1,6 +1,9 @@
 // RUN: not llvm-mc -triple=thumbv8.1m.main-none-eabi -mattr=+8msecext -show-encoding < %s 2>%t \
 // RUN: | FileCheck --check-prefix=CHECK %s
 // RUN:   FileCheck --check-prefix=ERROR < %t %s
+// RUN: not llvm-mc -triple=thumbv8.1m.main-none-eabi -mattr=+mve.fp,+8msecext -show-encoding < %s 2>%t \
+// RUN: | FileCheck --check-prefix=CHECK %s
+// RUN:   FileCheck --check-prefix=ERROR < %t %s
 // RUN: not llvm-mc -triple=thumbv8.1m.main-arm-none-eabi -mattr=-8msecext < %s 2>%t
 // RUN: FileCheck --check-prefix=NOSEC < %t %s
 
@@ -21,6 +24,9 @@ vscclrm {s31, vpr}
 // CHECK: vscclrm            {d0, d1, vpr}  @ encoding: [0x9f,0xec,0x04,0x0b]
 vscclrm {d0-d1, vpr}
 
+// CHECK: vscclrm            {d0, d1, d2, d3, vpr}  @ encoding: [0x9f,0xec,0x08,0x0b]
+vscclrm {d0-d3, vpr}
+
 // CHECK: vscclrm            {d5, d6, d7, vpr}  @ encoding: [0x9f,0xec,0x06,0x5b]
 vscclrm {d5-d7, vpr}
 
diff --git a/llvm/test/MC/Disassembler/ARM/vscclrm.txt b/llvm/test/MC/Disassembler/ARM/vscclrm.txt
index 7c162d6551525..8a89cfb76e4a4 100644
--- a/llvm/test/MC/Disassembler/ARM/vscclrm.txt
+++ b/llvm/test/MC/Disassembler/ARM/vscclrm.txt
@@ -1,5 +1,5 @@
 # RUN: llvm-mc -disassemble -triple=thumbv8.1m.main-none-eabi -mattr=+8msecext -show-encoding %s 2>&1 | FileCheck %s
-
+# RUN: llvm-mc -disassemble -triple=thumbv8.1m.main-none-eabi -mattr=+mve.fp,+8msecext -show-encoding %s 2>&1 | FileCheck %s
 
 [0x9f 0xec 0x04 0x0a]
 # CHECK: vscclrm {s0, s1, s2, s3, vpr}
@@ -16,6 +16,9 @@
 [0x9f,0xec,0x04,0x0b]
 # CHECK: vscclrm            {d0, d1, vpr}  @ encoding: [0x9f,0xec,0x04,0x0b]
 
+[0x9f,0xec,0x08,0x0b]
+# CHECK: vscclrm            {d0, d1, d2, d3, vpr}  @ encoding: [0x9f,0xec,0x08,0x0b]
+
 [0x9f,0xec,0x06,0x5b]
 # CHECK: vscclrm            {d5, d6, d7, vpr}  @ encoding: [0x9f,0xec,0x06,0x5b]
 
diff --git a/llvm/test/MC/Disassembler/SystemZ/insns-arch13.txt b/llvm/test/MC/Disassembler/SystemZ/insns-z15.txt
similarity index 99%
rename from llvm/test/MC/Disassembler/SystemZ/insns-arch13.txt
rename to llvm/test/MC/Disassembler/SystemZ/insns-z15.txt
index 3ed0435510e0c..1742a9208b6ed 100644
--- a/llvm/test/MC/Disassembler/SystemZ/insns-arch13.txt
+++ b/llvm/test/MC/Disassembler/SystemZ/insns-z15.txt
@@ -1,5 +1,5 @@
-# Test arch13 instructions that don't have PC-relative operands.
-# RUN: llvm-mc --disassemble %s -triple=s390x-linux-gnu -mcpu=arch13 \
+# Test z15 instructions that don't have PC-relative operands.
+# RUN: llvm-mc --disassemble %s -triple=s390x-linux-gnu -mcpu=z15 \
 # RUN:   | FileCheck %s
 
 # CHECK: dfltcc %r2, %r2, %r2
diff --git a/llvm/test/MC/Mips/mips-expansions.s b/llvm/test/MC/Mips/mips-expansions.s
index f4f3b1f6cf6e3..09b45c34937fa 100644
--- a/llvm/test/MC/Mips/mips-expansions.s
+++ b/llvm/test/MC/Mips/mips-expansions.s
@@ -1,7 +1,9 @@
-# RUN: llvm-mc %s -triple=mipsel-unknown-linux -show-encoding -mcpu=mips32r2 | \
-# RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-LE
-# RUN: llvm-mc %s -triple=mips-unknown-linux -show-encoding -mcpu=mips32r2 | \
-# RUN:   FileCheck %s --check-prefixes=CHECK,CHECK-BE
+# RUN: llvm-mc %s -triple=mipsel-unknown-linux -mcpu=mips32r2 -mattr=-xgot \
+# RUN:            -show-encoding | FileCheck %s --check-prefixes=CHECK,CHECK-LE,GOT
+# RUN: llvm-mc %s -triple=mipsel-unknown-linux -mcpu=mips32r2 -mattr=+xgot \
+# RUN:            -show-encoding | FileCheck %s --check-prefixes=CHECK,CHECK-LE,XGOT
+# RUN: llvm-mc %s -triple=mips-unknown-linux -mcpu=mips32r2 \
+# RUN:            -show-encoding | FileCheck %s --check-prefixes=CHECK,CHECK-BE
 
 # Check that the IAS expands macro instructions in the same way as GAS.
 
@@ -110,56 +112,120 @@
   .option pic2
   .set noat
   lw $10, symbol($4)
-# CHECK-LE: lw      $10, %got(symbol)($gp)    # encoding: [A,A,0x8a,0x8f]
-# CHECK-LE:                                   #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE: addu    $10, $10, $4              # encoding: [0x21,0x50,0x44,0x01]
-# CHECK-LE: lw      $10, 0($10)               # encoding: [0x00,0x00,0x4a,0x8d]
+# GOT:      lw      $10, %got(symbol)($gp)  # encoding: [A,A,0x8a,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
+# GOT-NEXT: lw      $10, 0($10)             # encoding: [0x00,0x00,0x4a,0x8d]
+
+# XGOT:      lui    $10, %got_hi(symbol)     # encoding: [A,A,0x0a,0x3c]
+# XGOT-NEXT:                                 #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $10, $10, $gp            # encoding: [0x21,0x50,0x5c,0x01]
+# XGOT-NEXT: lw     $10, %got_lo(symbol)($10) # encoding: [A,A,0x4a,0x8d]
+# XGOT-NEXT:                                 #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: addu   $10, $10, $4             # encoding: [0x21,0x50,0x44,0x01]
+# XGOT-NEXT: lw     $10, 0($10)              # encoding: [0x00,0x00,0x4a,0x8d]
   .set at
   sw $10, symbol($9)
-# CHECK-LE: lw      $1, %got(symbol)($gp)     # encoding: [A,A,0x81,0x8f]
-# CHECK-LE:                                   #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE: addu    $1, $1, $9                # encoding: [0x21,0x08,0x29,0x00]
-# CHECK-LE: sw      $10, 0($1)                # encoding: [0x00,0x00,0x2a,0xac]
+# GOT:      lw      $1, %got(symbol)($gp)   # encoding: [A,A,0x81,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# GOT-NEXT: sw      $10, 0($1)              # encoding: [0x00,0x00,0x2a,0xac]
+
+# XGOT:      lui    $1, %got_hi(symbol)     # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $1, $1, $gp             # encoding: [0x21,0x08,0x3c,0x00]
+# XGOT-NEXT: lw     $1, %got_lo(symbol)($1) # encoding: [A,A,0x21,0x8c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: addu   $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# XGOT-NEXT: sw     $10, 0($1)              # encoding: [0x00,0x00,0x2a,0xac]
 
   lw $8, 1f+8
-# CHECK-LE:      lw    $8, %got($tmp0)($gp)   # encoding: [A,A,0x88,0x8f]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
-# CHECK-LE-NEXT: addiu $8, $8, %lo($tmp0)     # encoding: [A,A,0x08,0x25]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
-# CHECK-LE-NEXT: lw    $8, 8($8)              # encoding: [0x08,0x00,0x08,0x8d]
+# GOT:      lw      $8, %got($tmp0)($gp)    # encoding: [A,A,0x88,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
+# GOT-NEXT: addiu   $8, $8, %lo($tmp0)      # encoding: [A,A,0x08,0x25]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
+# GOT-NEXT: lw      $8, 8($8)               # encoding: [0x08,0x00,0x08,0x8d]
+
+# XGOT:      lw     $8, %got($tmp0)($gp)    # encoding: [A,A,0x88,0x8f]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
+# XGOT-NEXT: addiu  $8, $8, %lo($tmp0)      # encoding: [A,A,0x08,0x25]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
+# XGOT-NEXT: lw     $8, 8($8)               # encoding: [0x08,0x00,0x08,0x8d]
   sw $8, 1f+8
-# CHECK-LE:      lw    $1, %got($tmp0)($gp)   # encoding: [A,A,0x81,0x8f]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
-# CHECK-LE-NEXT: addiu $1, $1, %lo($tmp0)     # encoding: [A,A,0x21,0x24]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
-# CHECK-LE-NEXT: sw    $8, 8($1)              # encoding: [0x08,0x00,0x28,0xac]
+# GOT:      lw      $1, %got($tmp0)($gp)    # encoding: [A,A,0x81,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
+# GOT-NEXT: addiu   $1, $1, %lo($tmp0)      # encoding: [A,A,0x21,0x24]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
+# GOT-NEXT: sw      $8, 8($1)               # encoding: [0x08,0x00,0x28,0xac]
+
+# XGOT:      lw     $1, %got($tmp0)($gp)    # encoding: [A,A,0x81,0x8f]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got($tmp0), kind: fixup_Mips_GOT
+# XGOT-NEXT: addiu  $1, $1, %lo($tmp0)      # encoding: [A,A,0x21,0x24]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %lo($tmp0), kind: fixup_Mips_LO16
+# XGOT-NEXT: sw     $8, 8($1)               # encoding: [0x08,0x00,0x28,0xac]
 
   lw $10, 655483($4)
-# CHECK-LE: lui     $10, 10                   # encoding: [0x0a,0x00,0x0a,0x3c]
-# CHECK-LE: addu    $10, $10, $4              # encoding: [0x21,0x50,0x44,0x01]
-# CHECK-LE: lw      $10, 123($10)             # encoding: [0x7b,0x00,0x4a,0x8d]
+# GOT:      lui     $10, 10                 # encoding: [0x0a,0x00,0x0a,0x3c]
+# GOT-NEXT: addu    $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
+# GOT-NEXT: lw      $10, 123($10)           # encoding: [0x7b,0x00,0x4a,0x8d]
+
+# XGOT:      lui    $10, 10                 # encoding: [0x0a,0x00,0x0a,0x3c]
+# XGOT-NEXT: addu   $10, $10, $4            # encoding: [0x21,0x50,0x44,0x01]
+# XGOT-NEXT: lw     $10, 123($10)           # encoding: [0x7b,0x00,0x4a,0x8d]
   sw $10, 123456($9)
-# CHECK-LE: lui     $1, 2                     # encoding: [0x02,0x00,0x01,0x3c]
-# CHECK-LE: addu    $1, $1, $9                # encoding: [0x21,0x08,0x29,0x00]
-# CHECK-LE: sw      $10, -7616($1)            # encoding: [0x40,0xe2,0x2a,0xac]
+# GOT:      lui     $1, 2                   # encoding: [0x02,0x00,0x01,0x3c]
+# GOT-NEXT: addu    $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# GOT-NEXT: sw      $10, -7616($1)          # encoding: [0x40,0xe2,0x2a,0xac]
+
+# XGOT:      lui    $1, 2                   # encoding: [0x02,0x00,0x01,0x3c]
+# XGOT-NEXT: addu   $1, $1, $9              # encoding: [0x21,0x08,0x29,0x00]
+# XGOT-NEXT: sw     $10, -7616($1)          # encoding: [0x40,0xe2,0x2a,0xac]
 
   lw $8, symbol+8
-# CHECK-LE:      lw $8, %got(symbol)($gp)     # encoding: [A,A,0x88,0x8f]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE-NEXT: lw $8, 8($8)                 # encoding: [0x08,0x00,0x08,0x8d]
+# GOT:      lw      $8, %got(symbol)($gp)   # encoding: [A,A,0x88,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: lw      $8, 8($8)               # encoding: [0x08,0x00,0x08,0x8d]
+
+# XGOT:      lui    $8, %got_hi(symbol)     # encoding: [A,A,0x08,0x3c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $8, $8, $gp             # encoding: [0x21,0x40,0x1c,0x01]
+# XGOT-NEXT: lw     $8, %got_lo(symbol)($8) # encoding: [A,A,0x08,0x8d]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: lw     $8, 8($8)               # encoding: [0x08,0x00,0x08,0x8d]
   sw $8, symbol+8
-# CHECK-LE:      lw $1, %got(symbol)($gp)     # encoding: [A,A,0x81,0x8f]
-# CHECK-LE-NEXT:                              #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE-NEXT: sw $8, 8($1)                 # encoding: [0x08,0x00,0x28,0xac]
+# GOT:      lw      $1, %got(symbol)($gp)   # encoding: [A,A,0x81,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: sw $8, 8($1)                    # encoding: [0x08,0x00,0x28,0xac]
+
+# XGOT:      lui    $1, %got_hi(symbol)     # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $1, $1, $gp             # encoding: [0x21,0x08,0x3c,0x00]
+# XGOT-NEXT: lw     $1, %got_lo(symbol)($1) # encoding: [A,A,0x21,0x8c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: sw     $8, 8($1)               # encoding: [0x08,0x00,0x28,0xac]
 
   ldc1 $f0, symbol
-# CHECK-LE: lw      $1, %got(symbol)($gp)     # encoding: [A,A,0x81,0x8f]
-# CHECK-LE:                                   #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE: ldc1    $f0, 0($1)                # encoding: [0x00,0x00,0x20,0xd4]
+# GOT:      lw      $1, %got(symbol)($gp)   # encoding: [A,A,0x81,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: ldc1    $f0, 0($1)              # encoding: [0x00,0x00,0x20,0xd4]
+
+# XGOT:      lui    $1, %got_hi(symbol)     # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $1, $1, $gp             # encoding: [0x21,0x08,0x3c,0x00]
+# XGOT-NEXT: lw     $1, %got_lo(symbol)($1) # encoding: [A,A,0x21,0x8c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: ldc1   $f0, 0($1)              # encoding: [0x00,0x00,0x20,0xd4]
   sdc1 $f0, symbol
-# CHECK-LE: lw      $1, %got(symbol)($gp)     # encoding: [A,A,0x81,0x8f]
-# CHECK-LE:                                   #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
-# CHECK-LE: sdc1    $f0, 0($1)                # encoding: [0x00,0x00,0x20,0xf4]
+# GOT:      lw      $1, %got(symbol)($gp)   # encoding: [A,A,0x81,0x8f]
+# GOT-NEXT:                                 #   fixup A - offset: 0, value: %got(symbol), kind: fixup_Mips_GOT
+# GOT-NEXT: sdc1    $f0, 0($1)              # encoding: [0x00,0x00,0x20,0xf4]
+
+# XGOT:      lui    $1, %got_hi(symbol)     # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: addu   $1, $1, $gp             # encoding: [0x21,0x08,0x3c,0x00]
+# XGOT-NEXT: lw     $1, %got_lo(symbol)($1) # encoding: [A,A,0x21,0x8c]
+# XGOT-NEXT:                                #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: sdc1   $f0, 0($1)              # encoding: [0x00,0x00,0x20,0xf4]
   .option pic0
 
 # Test BNE with an immediate as the 2nd operand.
diff --git a/llvm/test/MC/Mips/mips64-expansions.s b/llvm/test/MC/Mips/mips64-expansions.s
index 044a90e34df76..c586c81aa6e8e 100644
--- a/llvm/test/MC/Mips/mips64-expansions.s
+++ b/llvm/test/MC/Mips/mips64-expansions.s
@@ -1,4 +1,7 @@
-# RUN: llvm-mc %s -triple=mips64el-unknown-linux -show-encoding -mcpu=mips64r2 | FileCheck %s
+# RUN: llvm-mc %s -triple=mips64el-unknown-linux -mcpu=mips64r2 -mattr=-xgot \
+# RUN:            -show-encoding | FileCheck --check-prefixes=CHECK,GOT %s
+# RUN: llvm-mc %s -triple=mips64el-unknown-linux -mcpu=mips64r2 -mattr=+xgot \
+# RUN:            -show-encoding | FileCheck --check-prefixes=CHECK,XGOT %s
 
 # Check that signed negative 32-bit immediates are loaded correctly:
   li $10, ~(0x101010)
@@ -470,48 +473,112 @@ sym:
 # LW/SW and LDC1/SDC1 of symbol address, done by MipsAsmParser::expandMemInst():
   .option pic2
   lw $10, symbol($4)
-# CHECK: ld      $10, %got_disp(symbol)($gp) # encoding: [A,A,0x8a,0xdf]
-# CHECK:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK: daddu   $10, $10, $4               # encoding: [0x2d,0x50,0x44,0x01]
-# CHECK: lw      $10, 0($10)                # encoding: [0x00,0x00,0x4a,0x8d]
+# GOT:      ld      $10, %got_disp(symbol)($gp) # encoding: [A,A,0x8a,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: daddu   $10, $10, $4                # encoding: [0x2d,0x50,0x44,0x01]
+# GOT-NEXT: lw      $10, 0($10)                 # encoding: [0x00,0x00,0x4a,0x8d]
+
+# XGOT:      lui    $10, %got_hi(symbol)        # encoding: [A,A,0x0a,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $10, $10, $gp               # encoding: [0x2d,0x50,0x5c,0x01]
+# XGOT-NEXT: ld     $10, %got_lo(symbol)($10)   # encoding: [A,A,0x4a,0xdd]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: daddu  $10, $10, $4                # encoding: [0x2d,0x50,0x44,0x01]
+# XGOT-NEXT: lw     $10, 0($10)                 # encoding: [0x00,0x00,0x4a,0x8d]
+
   sw $10, symbol($9)
-# CHECK: ld      $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
-# CHECK:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK: daddu   $1, $1, $9                 # encoding: [0x2d,0x08,0x29,0x00]
-# CHECK: sw      $10, 0($1)                 # encoding: [0x00,0x00,0x2a,0xac]
+# GOT:      ld      $1, %got_disp(symbol)($gp)  # encoding: [A,A,0x81,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: daddu   $1, $1, $9                  # encoding: [0x2d,0x08,0x29,0x00]
+# GOT-NEXT: sw      $10, 0($1)                  # encoding: [0x00,0x00,0x2a,0xac]
+
+# XGOT:      lui    $1, %got_hi(symbol)         # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $1, $1, $gp                 # encoding: [0x2d,0x08,0x3c,0x00]
+# XGOT-NEXT: ld     $1, %got_lo(symbol)($1)     # encoding: [A,A,0x21,0xdc]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: daddu  $1, $1, $9                  # encoding: [0x2d,0x08,0x29,0x00]
+# XGOT-NEXT: sw     $10, 0($1)                  # encoding: [0x00,0x00,0x2a,0xac]
 
   lw $8, sym+8
-# CHECK:      ld $8, %got_disp(sym)($gp)    # encoding: [A,A,0x88,0xdf]
-# CHECK-NEXT:                               #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
-# CHECK-NEXT: lw $8, 8($8)                  # encoding: [0x08,0x00,0x08,0x8d]
+# GOT:      ld      $8, %got_disp(sym)($gp)     # encoding: [A,A,0x88,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: lw      $8, 8($8)                   # encoding: [0x08,0x00,0x08,0x8d]
+
+# XGOT:      ld     $8, %got_disp(sym)($gp)     # encoding: [A,A,0x88,0xdf]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
+# XGOT-NEXT: lw     $8, 8($8)                   # encoding: [0x08,0x00,0x08,0x8d]
+
   sw $8, sym+8
-# CHECK:      ld $1, %got_disp(sym)($gp)    # encoding: [A,A,0x81,0xdf]
-# CHECK-NEXT:                               #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
-# CHECK-NEXT: sw $8, 8($1)                  # encoding: [0x08,0x00,0x28,0xac]
+# GOT:      ld      $1, %got_disp(sym)($gp)     # encoding: [A,A,0x81,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: sw      $8, 8($1)                   # encoding: [0x08,0x00,0x28,0xac]
+
+# XGOT:      ld     $1, %got_disp(sym)($gp)     # encoding: [A,A,0x81,0xdf]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_disp(sym), kind: fixup_Mips_GOT_DISP
+# XGOT-NEXT: sw     $8, 8($1)                   # encoding: [0x08,0x00,0x28,0xac]
 
   lw $10, 655483($4)
-# CHECK: lui     $10, 10                    # encoding: [0x0a,0x00,0x0a,0x3c]
-# CHECK: daddu   $10, $10, $4               # encoding: [0x2d,0x50,0x44,0x01]
-# CHECK: lw      $10, 123($10)              # encoding: [0x7b,0x00,0x4a,0x8d]
+# GOT:      lui     $10, 10                     # encoding: [0x0a,0x00,0x0a,0x3c]
+# GOT-NEXT: daddu   $10, $10, $4                # encoding: [0x2d,0x50,0x44,0x01]
+# GOT-NEXT: lw      $10, 123($10)               # encoding: [0x7b,0x00,0x4a,0x8d]
+
+# XGOT:      lui    $10, 10                     # encoding: [0x0a,0x00,0x0a,0x3c]
+# XGOT-NEXT: daddu  $10, $10, $4                # encoding: [0x2d,0x50,0x44,0x01]
+# XGOT-NEXT: lw     $10, 123($10)               # encoding: [0x7b,0x00,0x4a,0x8d]
   sw $10, 123456($9)
-# CHECK: lui     $1, 2                      # encoding: [0x02,0x00,0x01,0x3c]
-# CHECK: daddu   $1, $1, $9                 # encoding: [0x2d,0x08,0x29,0x00]
-# CHECK: sw      $10, -7616($1)             # encoding: [0x40,0xe2,0x2a,0xac]
+# GOT:      lui     $1, 2                       # encoding: [0x02,0x00,0x01,0x3c]
+# GOT-NEXT: daddu   $1, $1, $9                  # encoding: [0x2d,0x08,0x29,0x00]
+# GOT-NEXT: sw      $10, -7616($1)              # encoding: [0x40,0xe2,0x2a,0xac]
+
+# XGOT:      lui    $1, 2                       # encoding: [0x02,0x00,0x01,0x3c]
+# XGOT-NEXT: daddu  $1, $1, $9                  # encoding: [0x2d,0x08,0x29,0x00]
+# XGOT-NEXT: sw     $10, -7616($1)              # encoding: [0x40,0xe2,0x2a,0xac]
 
   lw $8, symbol+8
-# CHECK:      ld $8, %got_disp(symbol)($gp) # encoding: [A,A,0x88,0xdf]
-# CHECK-NEXT:                               #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK-NEXT: lw $8, 8($8)                  # encoding: [0x08,0x00,0x08,0x8d]
+# GOT:      ld      $8, %got_disp(symbol)($gp)  # encoding: [A,A,0x88,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: lw      $8, 8($8)                   # encoding: [0x08,0x00,0x08,0x8d]
+
+# XGOT:      lui    $8, %got_hi(symbol)         # encoding: [A,A,0x08,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $8, $8, $gp                 # encoding: [0x2d,0x40,0x1c,0x01]
+# XGOT-NEXT: ld     $8, %got_lo(symbol)($8)     # encoding: [A,A,0x08,0xdd]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: lw     $8, 8($8)                   # encoding: [0x08,0x00,0x08,0x8d]
+
   sw $8, symbol+8
-# CHECK:      ld $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
-# CHECK-NEXT:                               #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK-NEXT: sw $8, 8($1)                  # encoding: [0x08,0x00,0x28,0xac]
+# GOT:      ld      $1, %got_disp(symbol)($gp)  # encoding: [A,A,0x81,0xdf]
+# GOT-NEXT:                                     #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: sw $8, 8($1)                        # encoding: [0x08,0x00,0x28,0xac]
+
+# XGOT:      lui    $1, %got_hi(symbol)         # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $1, $1, $gp                 # encoding: [0x2d,0x08,0x3c,0x00]
+# XGOT-NEXT: ld     $1, %got_lo(symbol)($1)     # encoding: [A,A,0x21,0xdc]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: sw     $8, 8($1)                   # encoding: [0x08,0x00,0x28,0xac]
 
   ldc1 $f0, symbol
-# CHECK: ld      $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
-# CHECK:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK: ldc1    $f0, 0($1)                 # encoding: [0x00,0x00,0x20,0xd4]
+# GOT:      ld      $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
+# GOT-NEXT:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: ldc1    $f0, 0($1)                 # encoding: [0x00,0x00,0x20,0xd4]
+
+# XGOT:      lui    $1, %got_hi(symbol)         # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $1, $1, $gp                 # encoding: [0x2d,0x08,0x3c,0x00]
+# XGOT-NEXT: ld     $1, %got_lo(symbol)($1)     # encoding: [A,A,0x21,0xdc]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: ldc1   $f0, 0($1)                  # encoding: [0x00,0x00,0x20,0xd4]
+
   sdc1 $f0, symbol
-# CHECK: ld      $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
-# CHECK:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
-# CHECK: sdc1    $f0, 0($1)                 # encoding: [0x00,0x00,0x20,0xf4]
+# GOT:      ld      $1, %got_disp(symbol)($gp) # encoding: [A,A,0x81,0xdf]
+# GOT-NEXT:                                    #   fixup A - offset: 0, value: %got_disp(symbol), kind: fixup_Mips_GOT_DISP
+# GOT-NEXT: sdc1    $f0, 0($1)                 # encoding: [0x00,0x00,0x20,0xf4]
+
+# XGOT:      lui    $1, %got_hi(symbol)         # encoding: [A,A,0x01,0x3c]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_hi(symbol), kind: fixup_Mips_GOT_HI16
+# XGOT-NEXT: daddu  $1, $1, $gp                 # encoding: [0x2d,0x08,0x3c,0x00]
+# XGOT-NEXT: ld     $1, %got_lo(symbol)($1)     # encoding: [A,A,0x21,0xdc]
+# XGOT-NEXT:                                    #   fixup A - offset: 0, value: %got_lo(symbol), kind: fixup_Mips_GOT_LO16
+# XGOT-NEXT: sdc1   $f0, 0($1)                  # encoding: [0x00,0x00,0x20,0xf4]
diff --git a/llvm/test/MC/SystemZ/insn-bad-arch13.s b/llvm/test/MC/SystemZ/insn-bad-z15.s
similarity index 99%
rename from llvm/test/MC/SystemZ/insn-bad-arch13.s
rename to llvm/test/MC/SystemZ/insn-bad-z15.s
index ad84e55b69f89..484dfd342dab0 100644
--- a/llvm/test/MC/SystemZ/insn-bad-arch13.s
+++ b/llvm/test/MC/SystemZ/insn-bad-z15.s
@@ -1,4 +1,6 @@
-# For arch13 only.
+# For z15 only.
+# RUN: not llvm-mc -triple s390x-linux-gnu -mcpu=z15 < %s 2> %t
+# RUN: FileCheck < %t %s
 # RUN: not llvm-mc -triple s390x-linux-gnu -mcpu=arch13 < %s 2> %t
 # RUN: FileCheck < %t %s
 
diff --git a/llvm/test/MC/SystemZ/insn-good-arch13.s b/llvm/test/MC/SystemZ/insn-good-z15.s
similarity index 99%
rename from llvm/test/MC/SystemZ/insn-good-arch13.s
rename to llvm/test/MC/SystemZ/insn-good-z15.s
index 0bbb8a546387e..1eb3b743cc61d 100644
--- a/llvm/test/MC/SystemZ/insn-good-arch13.s
+++ b/llvm/test/MC/SystemZ/insn-good-z15.s
@@ -1,4 +1,6 @@
-# For arch13 and above.
+# For z15 and above.
+# RUN: llvm-mc -triple s390x-linux-gnu -mcpu=z15 -show-encoding %s \
+# RUN:   | FileCheck %s
 # RUN: llvm-mc -triple s390x-linux-gnu -mcpu=arch13 -show-encoding %s \
 # RUN:   | FileCheck %s
 
diff --git a/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll b/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll
new file mode 100644
index 0000000000000..7e9b28ac243b0
--- /dev/null
+++ b/llvm/test/MC/WebAssembly/data-symbol-in-text-section.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -filetype=obj %s -o /dev/null 2>&1 | FileCheck %s
+; CHECK: data symbols must live in a data section: data_symbol
+
+target datalayout = "e-m:e-p:32:32-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+@data_symbol = constant [1024 x i32] zeroinitializer, section ".text", align 16
+
+define hidden i32 @main() local_unnamed_addr #0 {
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([1024 x i32], [1024 x i32]* @data_symbol, i32 0, i32 10)
+  ret i32 %0
+}
diff --git a/llvm/test/MC/WebAssembly/simd-encodings.s b/llvm/test/MC/WebAssembly/simd-encodings.s
index 491b4844d7fa5..2ee9f9ea8db00 100644
--- a/llvm/test/MC/WebAssembly/simd-encodings.s
+++ b/llvm/test/MC/WebAssembly/simd-encodings.s
@@ -463,4 +463,73 @@ main:
     # CHECK: f64x2.convert_i64x2_u # encoding: [0xfd,0xb2,0x01]
     f64x2.convert_i64x2_u
 
+    # CHECK: v8x16.load_splat 48 # encoding: [0xfd,0xc2,0x01,0x00,0x30]
+    v8x16.load_splat 48
+
+    # CHECK: v16x8.load_splat 48 # encoding: [0xfd,0xc3,0x01,0x01,0x30]
+    v16x8.load_splat 48
+
+    # CHECK: v32x4.load_splat 48 # encoding: [0xfd,0xc4,0x01,0x02,0x30]
+    v32x4.load_splat 48
+
+    # CHECK: v64x2.load_splat 48 # encoding: [0xfd,0xc5,0x01,0x03,0x30]
+    v64x2.load_splat 48
+
+    # CHECK: i8x16.narrow_i16x8_s # encoding: [0xfd,0xc6,0x01]
+    i8x16.narrow_i16x8_s
+
+    # CHECK: i8x16.narrow_i16x8_u # encoding: [0xfd,0xc7,0x01]
+    i8x16.narrow_i16x8_u
+
+    # CHECK: i16x8.narrow_i32x4_s # encoding: [0xfd,0xc8,0x01]
+    i16x8.narrow_i32x4_s
+
+    # CHECK: i16x8.narrow_i32x4_u # encoding: [0xfd,0xc9,0x01]
+    i16x8.narrow_i32x4_u
+
+    # CHECK: i16x8.widen_low_i8x16_s # encoding: [0xfd,0xca,0x01]
+    i16x8.widen_low_i8x16_s
+
+    # CHECK: i16x8.widen_high_i8x16_s # encoding: [0xfd,0xcb,0x01]
+    i16x8.widen_high_i8x16_s
+
+    # CHECK: i16x8.widen_low_i8x16_u # encoding: [0xfd,0xcc,0x01]
+    i16x8.widen_low_i8x16_u
+
+    # CHECK: i16x8.widen_high_i8x16_u # encoding: [0xfd,0xcd,0x01]
+    i16x8.widen_high_i8x16_u
+
+    # CHECK: i32x4.widen_low_i16x8_s # encoding: [0xfd,0xce,0x01]
+    i32x4.widen_low_i16x8_s
+
+    # CHECK: i32x4.widen_high_i16x8_s # encoding: [0xfd,0xcf,0x01]
+    i32x4.widen_high_i16x8_s
+
+    # CHECK: i32x4.widen_low_i16x8_u # encoding: [0xfd,0xd0,0x01]
+    i32x4.widen_low_i16x8_u
+
+    # CHECK: i32x4.widen_high_i16x8_u # encoding: [0xfd,0xd1,0x01]
+    i32x4.widen_high_i16x8_u
+
+    # CHECK: i16x8.load8x8_s 32 # encoding: [0xfd,0xd2,0x01,0x03,0x20]
+    i16x8.load8x8_s 32
+
+    # CHECK: i16x8.load8x8_u 32 # encoding: [0xfd,0xd3,0x01,0x03,0x20]
+    i16x8.load8x8_u 32
+
+    # CHECK: i32x4.load16x4_s 32 # encoding: [0xfd,0xd4,0x01,0x03,0x20]
+    i32x4.load16x4_s 32
+
+    # CHECK: i32x4.load16x4_u 32 # encoding: [0xfd,0xd5,0x01,0x03,0x20]
+    i32x4.load16x4_u 32
+
+    # CHECK: i64x2.load32x2_s 32 # encoding: [0xfd,0xd6,0x01,0x03,0x20]
+    i64x2.load32x2_s 32
+
+    # CHECK: i64x2.load32x2_u 32 # encoding: [0xfd,0xd7,0x01,0x03,0x20]
+    i64x2.load32x2_u 32
+
+    # CHECK: v128.andnot # encoding: [0xfd,0xd8,0x01]
+    v128.andnot
+
     end_function
diff --git a/llvm/test/MachineVerifier/test_memccpy_intrinsics.mir b/llvm/test/MachineVerifier/test_memccpy_intrinsics.mir
new file mode 100644
index 0000000000000..acdd9dd2b28ce
--- /dev/null
+++ b/llvm/test/MachineVerifier/test_memccpy_intrinsics.mir
@@ -0,0 +1,27 @@
+# RUN: not llc -o - -march=aarch64 -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s
+# REQUIRES: aarch64-registered-target
+
+---
+name:            test_memcpy_et_al
+legalized:       true
+regBankSelected: false
+selected:        false
+tracksRegLiveness: true
+liveins:
+body:             |
+  bb.0:
+
+    %0:_(p0) = G_IMPLICIT_DEF
+    %1:_(s64) = G_IMPLICIT_DEF
+    %2:_(s1) = G_IMPLICIT_DEF
+
+    ; CHECK: Bad machine code: Expected memcpy intrinsic to have 5 operands
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %0(p0), %1(s64)
+
+    ; CHECK: Bad machine code: Expected memmove intrinsic to have 5 operands
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %0(p0), %1(s64)
+
+    ; CHECK: Bad machine code: Expected memset intrinsic to have 5 operands
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %0(p0), %1(s64)
+
+...
diff --git a/llvm/test/Object/archive-extract.test b/llvm/test/Object/archive-extract.test
index 44d76b4dd1fba..11ca12e3353b1 100644
--- a/llvm/test/Object/archive-extract.test
+++ b/llvm/test/Object/archive-extract.test
@@ -58,4 +58,4 @@ RUN: llvm-ar p %p/Inputs/thin.a evenlen | FileCheck %s --check-prefix=EVENLEN
 EVENLEN: evenlen
 
 RUN: not llvm-ar p %p/Inputs/thin-path.a t/test2.o 2>&1 | FileCheck %s --check-prefix=MISSING
-MISSING: {{N|n}}o such file or directory.
+MISSING: error: {{N|n}}o such file or directory
diff --git a/llvm/test/Object/archive-format.test b/llvm/test/Object/archive-format.test
index b555bf88faca9..77e7a5de009f8 100644
--- a/llvm/test/Object/archive-format.test
+++ b/llvm/test/Object/archive-format.test
@@ -76,7 +76,7 @@ THIN-PATH-NEXT: /41             0           0     0     644     4         `
 THIN-PATH-NEXT: /65             0           0     0     644     4         `
 
 RUN: not llvm-ar --format=bsd rcT bad.a 0123456789abcde 0123456789abcdef 2>&1 | FileCheck --check-prefix=BSD-THIN %s
-BSD-THIN: Only the gnu format has a thin mode.
+BSD-THIN: error: only the gnu format has a thin mode
 
 If an archive has an object with no symbols, the linker and some other
 tools on some versions of Solaris will abort operations if there is no
diff --git a/llvm/test/Object/archive-update.test b/llvm/test/Object/archive-update.test
index eb68e8dca4816..e152da5705c16 100644
--- a/llvm/test/Object/archive-update.test
+++ b/llvm/test/Object/archive-update.test
@@ -48,4 +48,4 @@ BOTH-FILES: bar
 RUN: rm -f %t/tmp.a
 RUN: llvm-ar --format=gnu rc %t/tmp.a foo
 RUN: not llvm-ar --format=gnu rcT %t/tmp.a bar 2>&1 | FileCheck --check-prefix=ERROR %s
-ERROR: Cannot convert a regular archive to a thin one.
+ERROR: error: cannot convert a regular archive to a thin one
diff --git a/llvm/test/Object/mri-addlib.test b/llvm/test/Object/mri-addlib.test
deleted file mode 100644
index 745bcf65e7f49..0000000000000
--- a/llvm/test/Object/mri-addlib.test
+++ /dev/null
@@ -1,14 +0,0 @@
-; RUN: echo create %t.a > %t.mri
-; RUN: echo addlib %p/Inputs/GNU.a >> %t.mri
-; RUN: echo addlib %p/Inputs/archive-test.a-gnu-minimal >> %t.mri
-; RUN: echo save >> %t.mri
-; RUN: echo end >> %t.mri
-
-; RUN: llvm-ar -M  < %t.mri
-; RUN: llvm-ar t %t.a | FileCheck %s
-
-; CHECK: evenlen
-; CHECK-NEXT: oddlen
-; CHECK-NEXT: very_long_bytecode_file_name.bc
-; CHECK-NEXT: IsNAN.o
-; CHECK-NEXT: test
diff --git a/llvm/test/Object/mri-addmod.test b/llvm/test/Object/mri-addmod.test
deleted file mode 100644
index 4a0372ad42c00..0000000000000
--- a/llvm/test/Object/mri-addmod.test
+++ /dev/null
@@ -1,82 +0,0 @@
-# RUN: yaml2obj -docnum=1 %s > %t.elf-x86-64
-# RUN: echo create %t.a > %t.mri
-# RUN: echo 'addmod "%t.elf-x86-64"' >> %t.mri
-# RUN: echo save >> %t.mri
-# RUN: echo end >> %t.mri
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-Sections:
-  - Name:  .text
-    Type:  SHT_PROGBITS
-    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
-Symbols:
-  - Name:    main
-    Type:    STT_FUNC
-    Section: .text
-    Binding: STB_GLOBAL
-    Size:    0x0000000000000024
-  - Name:    SomeOtherFunction
-    Binding: STB_GLOBAL
-  - Name:    puts
-    Binding: STB_GLOBAL
-
-# RUN: llvm-ar -M  < %t.mri
-# RUN: llvm-nm -M %t.a | FileCheck %s
-
-# CHECK:      Archive map
-# CHECK-NEXT: main in {{.*}}elf-x86-64
-
-# CHECK:      {{.*}}elf-x86-64:
-# CHECK-NEXT:                  U SomeOtherFunction
-# CHECK-NEXT: 0000000000000000 T main
-# CHECK-NEXT:                  U puts
-
-## Now test that CREATE overwrites an existing file.
-# RUN: yaml2obj -docnum=2 %s > %t2.elf-x86-64
-
-# RUN: echo create %t.a > %t2.mri
-# RUN: echo addmod %t2.elf-x86-64 >> %t2.mri
-# RUN: echo save >> %t2.mri
-# RUN: echo end >> %t2.mri
-
-# RUN: llvm-ar -M  < %t2.mri
-# RUN: llvm-nm -M %t.a | FileCheck --check-prefix=NEW %s
-
-# NEW: Archive map
-# NEW-NEXT: foo in {{.*}}2.elf-x86-64
-# NEW-NEXT: main in {{.*}}2.elf-x86-64
-
-# NEW: {{.*}}2.elf-x86-64:
-# NEW-NEXT: 0000000000000000 t bar
-# NEW-NEXT: 0000000000000006 T foo
-# NEW-NEXT: 0000000000000016 T main
-
---- !ELF
-FileHeader:
-  Class:   ELFCLASS64
-  Data:    ELFDATA2LSB
-  Type:    ET_REL
-  Machine: EM_X86_64
-Sections:
-  - Name:  .text
-    Type:  SHT_PROGBITS
-    Flags: [ SHF_ALLOC, SHF_EXECINSTR ]
-Symbols:
-  - Name:    bar
-    Type:    STT_FUNC
-    Section: .text
-  - Name:    foo
-    Type:    STT_FUNC
-    Section: .text
-    Binding: STB_GLOBAL
-    Value:   0x0000000000000006
-  - Name:    main
-    Type:    STT_FUNC
-    Section: .text
-    Binding: STB_GLOBAL
-    Value:   0x0000000000000016
diff --git a/llvm/test/Other/cgscc-libcall-update.ll b/llvm/test/Other/cgscc-libcall-update.ll
index 5aa61df6a5728..43de193a640a8 100644
--- a/llvm/test/Other/cgscc-libcall-update.ll
+++ b/llvm/test/Other/cgscc-libcall-update.ll
@@ -14,11 +14,12 @@ bb:
   %tmp = alloca [1024 x i8], align 16
   %tmp2 = getelementptr inbounds [1024 x i8], [1024 x i8]* %tmp, i64 0, i64 0
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %tmp2, i8* %arg1, i64 1024, i1 false)
-; CHECK:         call void @llvm.memcpy
+; CHECK:         call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(1024) 
   %tmp3 = call i64 @llvm.objectsize.i64.p0i8(i8* %tmp2, i1 false, i1 true, i1 false)
   %tmp4 = call i8* @__strncpy_chk(i8* %arg2, i8* %tmp2, i64 1023, i64 %tmp3)
 ; CHECK-NOT:     call
-; CHECK:         call i8* @strncpy(i8* %arg2, i8* nonnull %tmp2, i64 1023)
+; CHECK:         call i8* @strncpy(i8* nonnull dereferenceable(1) %arg2, i8* nonnull dereferenceable(1) %tmp2, i64 1023)
+
 ; CHECK-NOT:     call
 
   ret i8* %tmp4
@@ -27,6 +28,7 @@ bb:
 
 define i8* @strncpy(i8* %arg1, i8* %arg2, i64 %size) noinline {
 bb:
+; CHECK:         call i8* @my_special_strncpy(i8* %arg1, i8* %arg2, i64 %size)
   %result = call i8* @my_special_strncpy(i8* %arg1, i8* %arg2, i64 %size)
   ret i8* %result
 }
diff --git a/llvm/test/Other/opt-O2-pipeline.ll b/llvm/test/Other/opt-O2-pipeline.ll
index b4acce3cb8b47..f8603ef12a7b8 100644
--- a/llvm/test/Other/opt-O2-pipeline.ll
+++ b/llvm/test/Other/opt-O2-pipeline.ll
@@ -185,8 +185,8 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Dominator Tree Construction
+; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Memory SSA
diff --git a/llvm/test/Other/opt-O3-pipeline.ll b/llvm/test/Other/opt-O3-pipeline.ll
index b1f68aed30297..5f02df4ea7076 100644
--- a/llvm/test/Other/opt-O3-pipeline.ll
+++ b/llvm/test/Other/opt-O3-pipeline.ll
@@ -190,8 +190,8 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Dominator Tree Construction
+; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Memory SSA
diff --git a/llvm/test/Other/opt-Os-pipeline.ll b/llvm/test/Other/opt-Os-pipeline.ll
index b9902685b88de..127ccede06093 100644
--- a/llvm/test/Other/opt-Os-pipeline.ll
+++ b/llvm/test/Other/opt-Os-pipeline.ll
@@ -172,8 +172,8 @@
 ; CHECK-NEXT:     CallGraph Construction
 ; CHECK-NEXT:     Globals Alias Analysis
 ; CHECK-NEXT:     FunctionPass Manager
-; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Dominator Tree Construction
+; CHECK-NEXT:       Float to int
 ; CHECK-NEXT:       Basic Alias Analysis (stateless AA impl)
 ; CHECK-NEXT:       Function Alias Analysis Results
 ; CHECK-NEXT:       Memory SSA
diff --git a/llvm/test/Reduce/Inputs/remove-bbs.py b/llvm/test/Reduce/Inputs/remove-bbs.py
new file mode 100755
index 0000000000000..71f099daaba06
--- /dev/null
+++ b/llvm/test/Reduce/Inputs/remove-bbs.py
@@ -0,0 +1,15 @@
+import sys
+
+InterestingBBs = 0
+input = open(sys.argv[1], "r")
+for line in input:
+  i = line.find(';')
+  if i >= 0:
+    line = line[:i]
+  if line.startswith("interesting") or "%interesting" in line:
+    InterestingBBs += 1
+
+if InterestingBBs == 6:
+  sys.exit(0) # interesting!
+
+sys.exit(1) # IR isn't interesting
diff --git a/llvm/test/Reduce/Inputs/remove-global-vars.py b/llvm/test/Reduce/Inputs/remove-global-vars.py
index 81fbea937ddf2..1ae8b0e6e76ca 100755
--- a/llvm/test/Reduce/Inputs/remove-global-vars.py
+++ b/llvm/test/Reduce/Inputs/remove-global-vars.py
@@ -2,9 +2,17 @@
 
 import sys
 
+InterestingVar = 0
+
 input = open(sys.argv[1], "r")
 for line in input:
-  if "@interesting = global" in line:
-    sys.exit(0)
+  i = line.find(';')
+  if i >= 0:
+    line = line[:i]
+  if line.startswith("@interesting = global") or "@interesting" in line:
+    InterestingVar += 1
+
+if InterestingVar == 4:
+  sys.exit(0) # interesting!
 
-sys.exit(1) # IR isn't interesting
+sys.exit(1)
diff --git a/llvm/test/Reduce/Inputs/remove-instructions.py b/llvm/test/Reduce/Inputs/remove-instructions.py
new file mode 100755
index 0000000000000..9717c73b01aa7
--- /dev/null
+++ b/llvm/test/Reduce/Inputs/remove-instructions.py
@@ -0,0 +1,17 @@
+import sys
+
+InterestingInstructions = 0
+
+input = open(sys.argv[1], "r")
+for line in input:
+  i = line.find(';')
+  if i >= 0:
+    line = line[:i]
+  if "%interesting" in line:
+    InterestingInstructions += 1
+  print(InterestingInstructions)
+
+if InterestingInstructions == 5:
+  sys.exit(0) # interesting!
+
+sys.exit(1)
diff --git a/llvm/test/Reduce/remove-args.ll b/llvm/test/Reduce/remove-args.ll
index 4823f751923ff..8d6130262bf50 100644
--- a/llvm/test/Reduce/remove-args.ll
+++ b/llvm/test/Reduce/remove-args.ll
@@ -1,16 +1,8 @@
 ; Test that llvm-reduce can remove uninteresting function arguments from function definitions as well as their calls.
 ;
 ; RUN: rm -rf %t
-; RUN: mkdir %t
-; get the python path from lit
-; RUN: echo "#!" %python > %t/test.py
-; then include the rest of the test script
-; RUN: cat %p/Inputs/remove-args.py >> %t/test.py
-; RUN: chmod +x %t/test.py
-
-; RUN: llvm-reduce --test %t/test.py %s -o %t/out.ll
-; RUN: cat %t/out.ll | FileCheck -implicit-check-not=uninteresting %s
-; REQUIRES: shell
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-args.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
 ; CHECK: @interesting(i32 %interesting)
 define void @interesting(i32 %uninteresting1, i32 %interesting, i32 %uninteresting2) {
diff --git a/llvm/test/Reduce/remove-bbs.ll b/llvm/test/Reduce/remove-bbs.ll
new file mode 100644
index 0000000000000..9e49bc531688e
--- /dev/null
+++ b/llvm/test/Reduce/remove-bbs.ll
@@ -0,0 +1,29 @@
+; Test that llvm-reduce can remove uninteresting Basic Blocks, and remove them from instructions (i.e. SwitchInst, BranchInst and IndirectBrInst)
+; Note: if an uninteresting BB is the default case for a switch, the instruction is removed altogether (since the default case cannot be replaced)
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-bbs.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+
+define void @main() {
+interesting:
+  ; CHECK-NOT: switch i32 0, label %uninteresting
+  switch i32 0, label %uninteresting [
+    i32 0, label %uninteresting
+  ]
+
+uninteresting:
+  ret void
+
+interesting2:
+  ; CHECK: switch i32 1, label %interesting3
+  switch i32 1, label %interesting3 [
+    ; CHECK-NOT: i32 0, label %uninteresting
+    i32 0, label %uninteresting
+    ; CHECK: i32 1, label %interesting3
+    i32 1, label %interesting3
+  ]
+
+interesting3:
+  ; CHECK: br label %interesting2
+  br i1 true, label %interesting2, label %uninteresting
+}
diff --git a/llvm/test/Reduce/remove-funcs.ll b/llvm/test/Reduce/remove-funcs.ll
index 6b3c86bea87bd..8e9b2579a974b 100644
--- a/llvm/test/Reduce/remove-funcs.ll
+++ b/llvm/test/Reduce/remove-funcs.ll
@@ -2,16 +2,8 @@
 ; their InstCalls.
 ;
 ; RUN: rm -rf %t
-; RUN: mkdir %t
-; get the python path from lit
-; RUN: echo "#!" %python > %t/test.py
-; then include the rest of the test script
-; RUN: cat %p/Inputs/remove-funcs.py >> %t/test.py
-; RUN: chmod +x %t/test.py
-
-; RUN: llvm-reduce --test %t/test.py %s -o %t/out.ll
-; RUN: cat %t/out.ll | FileCheck -implicit-check-not=uninteresting %s
-; REQUIRES: shell
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-funcs.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
 define i32 @uninteresting1() {
 entry:
diff --git a/llvm/test/Reduce/remove-global-vars.ll b/llvm/test/Reduce/remove-global-vars.ll
index 900774762c199..921083fc93b0f 100644
--- a/llvm/test/Reduce/remove-global-vars.ll
+++ b/llvm/test/Reduce/remove-global-vars.ll
@@ -2,16 +2,8 @@
 ; their direct uses (which in turn are replaced with 'undef').
 ;
 ; RUN: rm -rf %t
-; RUN: mkdir %t
-; get the python path from lit
-; RUN: echo "#!" %python > %t/test.py
-; then include the rest of the test script
-; RUN: cat %p/Inputs/remove-global-vars.py >> %t/test.py
-; RUN: chmod +x %t/test.py
-
-; RUN: llvm-reduce --test %t/test.py %s -o %t/out.ll
-; RUN: cat %t/out.ll | FileCheck -implicit-check-not=uninteresting %s
-; REQUIRES: shell
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-global-vars.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
 
 ; CHECK: @interesting = global
 @interesting = global i32 0, align 4
@@ -27,9 +19,7 @@ entry:
   %1 = load i32, i32* @interesting, align 4
   store i32 %1, i32* @uninteresting, align 4
 
-  ; CHECK: %inc = add nsw i32 undef, 1
-  %inc = add nsw i32 %0, 1
-  ; CHECK: store i32 %inc, i32* @interesting, align 4
-  store i32 %inc, i32* @interesting, align 4
+  ; CHECK: store i32 5, i32* @interesting, align 4
+  store i32 5, i32* @interesting, align 4
   ret i32 0
 }
diff --git a/llvm/test/Reduce/remove-instructions.ll b/llvm/test/Reduce/remove-instructions.ll
new file mode 100644
index 0000000000000..101351a89e1e0
--- /dev/null
+++ b/llvm/test/Reduce/remove-instructions.ll
@@ -0,0 +1,23 @@
+; Test that llvm-reduce can remove uninteresting instructions.
+;
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-instructions.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=uninteresting %s
+; REQUIRES: plugins
+
+; We're testing all direct uses of %interesting are conserved
+; CHECK-COUNT-5: %interesting
+define i32 @main() #0 {
+entry:
+  %uninteresting1 = alloca i32, align 4
+  %interesting = alloca i32, align 4
+  %uninteresting2 = alloca i32, align 4
+  store i32 0, i32* %uninteresting1, align 4
+  store i32 0, i32* %interesting, align 4
+  %0 = load i32, i32* %interesting, align 4
+  %uninteresting3 = add nsw i32 %0, 1
+  store i32 %uninteresting3, i32* %interesting, align 4
+  %1 = load i32, i32* %interesting, align 4
+  store i32 %1, i32* %uninteresting2, align 4
+  ; CHECK-NOT: ret
+  ret i32 0
+}
diff --git a/llvm/test/Reduce/remove-metadata.ll b/llvm/test/Reduce/remove-metadata.ll
index b887737f3b347..da7d5a2f16b48 100644
--- a/llvm/test/Reduce/remove-metadata.ll
+++ b/llvm/test/Reduce/remove-metadata.ll
@@ -2,16 +2,8 @@
 ; The Metadata pass erases named & unnamed metadata nodes.
 ;
 ; RUN: rm -rf %t
-; RUN: mkdir %t
-; get the python path from lit
-; RUN: echo "#!" %python > %t/test.py
-; then include the rest of the test script
-; RUN: cat %p/Inputs/remove-metadata.py >> %t/test.py
-; RUN: chmod +x %t/test.py
-
-; RUN: llvm-reduce --test %t/test.py %s -o %t/out.ll
-; RUN: cat %t/out.ll | FileCheck -implicit-check-not=! %s
-; REQUIRES: shell
+; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-metadata.py %s -o %t
+; RUN: cat %t | FileCheck -implicit-check-not=! %s
 
 @global = global i32 0, !dbg !0
 
diff --git a/llvm/test/TableGen/BigEncoder.td b/llvm/test/TableGen/BigEncoder.td
new file mode 100644
index 0000000000000..5c4bc016e269c
--- /dev/null
+++ b/llvm/test/TableGen/BigEncoder.td
@@ -0,0 +1,57 @@
+// RUN: llvm-tblgen -gen-emitter -I %p/../../include %s | FileCheck %s
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+    let InstructionSet = archInstrInfo;
+}
+
+def  Myi32  : Operand<i32> {
+  let DecoderMethod = "DecodeMyi32";
+}
+
+
+let OutOperandList = (outs), Size = 2 in {
+
+def foo : Instruction {
+    let InOperandList = (ins i32imm:$factor);
+    field bits<65> Inst;
+    bits<32> factor;
+    let Inst{7-0} = 0xAA;
+    let Inst{14-8} = factor{6-0}; // no offset
+    let AsmString = "foo  $factor";
+    field bits<16> SoftFail = 0;
+    }
+
+def bar : Instruction {
+    let InOperandList = (ins i32imm:$factor);
+    field bits<65> Inst;
+    bits<32> factor;
+    let Inst{7-0} = 0xBB;
+    let Inst{15-8} = factor{10-3}; // offset by 3
+    let AsmString = "bar  $factor";
+    field bits<16> SoftFail = 0;
+    }
+
+def biz : Instruction {
+    let InOperandList = (ins i32imm:$factor);
+    field bits<65> Inst;
+    bits<32> factor;
+    let Inst{7-0} = 0xCC;
+    let Inst{11-8,15-12} = factor{10-3}; // offset by 3, multipart
+    let AsmString = "biz  $factor";
+    field bits<16> SoftFail = 0;
+    }
+
+}
+// CHECK-LABEL: case ::biz: {
+// CHECK:      Value.insertBits(op.extractBitsAsZExtValue(4, 3), 12, 4);
+// CHECK-NEXT: Value.insertBits(op.extractBitsAsZExtValue(4, 7), 8, 4);
+
+// CHECK-LABEL: case ::foo: {
+// CHECK:      Value.insertBits(op.extractBitsAsZExtValue(7, 0), 8, 7);
+
+// CHECK-LABEL: case ::bar: {
+// CHECK:      Value.insertBits(op.extractBitsAsZExtValue(8, 3), 8, 8);
diff --git a/llvm/test/TableGen/HwModeEncodeDecode.td b/llvm/test/TableGen/HwModeEncodeDecode.td
new file mode 100644
index 0000000000000..b76d856465193
--- /dev/null
+++ b/llvm/test/TableGen/HwModeEncodeDecode.td
@@ -0,0 +1,81 @@
+// RUN: llvm-tblgen -gen-emitter -I %p/../../include %s | FileCheck %s --check-prefix=ENCODER
+// RUN: llvm-tblgen -gen-disassembler -I %p/../../include %s | FileCheck %s --check-prefix=DECODER
+
+include "llvm/Target/Target.td"
+
+def archInstrInfo : InstrInfo { }
+
+def arch : Target {
+    let InstructionSet = archInstrInfo;
+}
+
+def  Myi32  : Operand<i32> {
+  let DecoderMethod = "DecodeMyi32";
+}
+
+def ModeA : HwMode<"+a">;
+def ModeB : HwMode<"+b">;
+
+
+def fooTypeEncA : InstructionEncoding {
+  let Size = 4;
+  field bits<32> SoftFail = 0;
+  bits<32> Inst;
+  bits<8> factor;
+  let Inst{7-0} = factor;
+  let Inst{3-2} = 0b11;
+  let Inst{1-0} = 0b00;
+}
+
+def fooTypeEncB : InstructionEncoding {
+  let Size = 4;
+  field bits<32> SoftFail = 0;
+  bits<32> Inst;
+  bits<8> factor;
+  let Inst{15-8} = factor;
+  let Inst{1-0} = 0b11;
+}
+
+let OutOperandList = (outs) in {
+def foo : Instruction {
+  let InOperandList = (ins i32imm:$factor);
+  let EncodingInfos = EncodingByHwMode<
+    [ModeA, ModeB], [fooTypeEncA,
+                      fooTypeEncB]
+  >;
+  let AsmString = "foo  $factor";
+}
+
+def bar: Instruction {
+  let InOperandList = (ins i32imm:$factor);
+  let Size = 4;
+  bits<32> Inst;
+  bits<32> SoftFail;
+  bits<8> factor;
+  let Inst{31-24} = factor;
+  let Inst{1-0} = 0b10;
+  let AsmString = "bar  $factor";
+}
+}
+
+// DECODER-LABEL: DecoderTable_ModeA32[] =
+// DECODER-DAG: Opcode: fooTypeEncA:foo
+// DECODER-DAG: Opcode: bar
+// DECODER-LABEL: DecoderTable_ModeB32[] =
+// Note that the comment says fooTypeEncA but this is actually fooTypeEncB; plumbing
+// the correct comment text through the decoder is nontrivial.
+// DECODER-DAG: Opcode: fooTypeEncA:foo
+// DECODER-DAG: Opcode: bar
+
+// ENCODER-LABEL:   static const uint64_t InstBits_ModeA[] = {
+// ENCODER:         UINT64_C(2),        // bar
+// ENCODER:         UINT64_C(12),       // foo
+
+// ENCODER-LABEL:   static const uint64_t InstBits_ModeB[] = {
+// ENCODER:         UINT64_C(2),        // bar
+// ENCODER:         UINT64_C(3),        // foo
+
+// ENCODER:     case ::foo: {
+// ENCODER:      switch (HwMode) {
+// ENCODER:      default: llvm_unreachable("Unhandled HwMode");
+// ENCODER:      case 1: {
diff --git a/llvm/test/TableGen/RegisterEncoder.td b/llvm/test/TableGen/RegisterEncoder.td
index a0472c5feffac..b5fe2d206729c 100644
--- a/llvm/test/TableGen/RegisterEncoder.td
+++ b/llvm/test/TableGen/RegisterEncoder.td
@@ -30,6 +30,7 @@ def foo : Instruction {
 
 // CHECK: case ::foo: {
 // CHECK:   op = barEncoder
-// CHECK:   Value |= op & UINT64_C(255);
+// CHECK:   op &= UINT64_C(255);
+// CHECK:   Value |= op;
 // CHECK:   break;
-// CHECK: }
\ No newline at end of file
+// CHECK: }
diff --git a/llvm/test/TableGen/get-operand-type.td b/llvm/test/TableGen/get-operand-type.td
index 69bcde38c7ef2..27607f57a8d48 100644
--- a/llvm/test/TableGen/get-operand-type.td
+++ b/llvm/test/TableGen/get-operand-type.td
@@ -16,6 +16,8 @@ def RegClass : RegisterClass<"foo", [i32], 0, (add Reg)>;
 def OpA : Operand<i32>;
 def OpB : Operand<i32>;
 
+def RegOp : RegisterOperand<RegClass>;
+
 def InstA : Instruction {
   let Size = 1;
   let OutOperandList = (outs OpA:$a);
@@ -34,7 +36,17 @@ def InstB : Instruction {
   let Namespace = "MyNamespace";
 }
 
+def InstC : Instruction {
+  let Size = 1;
+  let OutOperandList = (outs RegClass:$d);
+  let InOperandList = (ins RegOp:$x);
+  field bits<8> Inst;
+  field bits<8> SoftFail = 0;
+  let Namespace = "MyNamespace";
+}
+
 // CHECK: #ifdef GET_INSTRINFO_OPERAND_TYPE
 // CHECK:        OpTypes::OpA, OpTypes::OpB, OpTypes::i32imm,
 // CHECK-NEXT:   OpTypes::i32imm, -1,
-// CHECK: #endif //GET_INSTRINFO_OPERAND_TYPE
+// CHECK-NEXT:   OpTypes::RegClass, OpTypes::RegOp,
+// CHECK: #endif // GET_INSTRINFO_OPERAND_TYPE
diff --git a/llvm/test/TableGen/immarg.td b/llvm/test/TableGen/immarg.td
new file mode 100644
index 0000000000000..407f06c3a40ec
--- /dev/null
+++ b/llvm/test/TableGen/immarg.td
@@ -0,0 +1,31 @@
+// RUN: llvm-tblgen -gen-global-isel -optimize-match-table=false -I %p/Common -I %p/../../include %s -o - < %s | FileCheck -check-prefix=GISEL %s
+
+include "llvm/Target/Target.td"
+include "GlobalISelEmitterCommon.td"
+
+let TargetPrefix = "mytarget" in {
+def int_mytarget_sleep0 : Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]>;
+def int_mytarget_sleep1 : Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]>;
+}
+
+// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
+// GISEL-NEXT: // MIs[0] Operand 0
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep0,
+// GISEL-NEXT: // MIs[0] src
+// GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
+// GISEL-NEXT: // (intrinsic_void {{[0-9]+}}:{ *:[iPTR] }, (timm:{ *:[i32] }):$src)  =>  (SLEEP0 (timm:{ *:[i32] }):$src)
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SLEEP0,
+// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
+def SLEEP0 : I<(outs), (ins i32imm:$src),
+  [(int_mytarget_sleep0 timm:$src)]
+>;
+
+// Test for situation which was crashing in ARM patterns.
+def p_imm : Operand<i32>;
+def SLEEP1 : I<(outs), (ins p_imm:$src), []>;
+
+// FIXME: This should not crash, but should it work or be an error?
+// def : Pat <
+//   (int_mytarget_sleep1 timm:$src),
+//   (SLEEP1 imm:$src)
+// >;
diff --git a/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll b/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll
new file mode 100644
index 0000000000000..bcd49aefe1585
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/Inputs/devirt_promote.ll
@@ -0,0 +1,39 @@
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+%struct.B = type { %struct.A }
+
+@_ZTV1B = constant { [4 x i8*] } { [4 x i8*] [i8* null, i8* undef, i8* bitcast (i32 (%struct.B*, i32)* @_ZN1B1fEi to i8*), i8* bitcast (i32 (%struct.A*, i32)* @_ZN1A1nEi to i8*)] }, !type !0, !type !1
+
+define i32 @_ZN1B1fEi(%struct.B* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define internal i32 @_ZN1A1nEi(%struct.A* %this, i32 %a) #0 {
+   ret i32 0;
+}
+
+define i32 @test2(%struct.B* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.B* %obj to i8***
+  %vtable2 = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable2 to i8*
+  %p2 = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1B")
+  call void @llvm.assume(i1 %p2)
+
+  %fptrptr = getelementptr i8*, i8** %vtable2, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.B*, i32)**
+  %fptr33 = load i32 (%struct.B*, i32)*, i32 (%struct.B*, i32)** %2, align 8
+
+  %call4 = tail call i32 %fptr33(%struct.B* nonnull %obj, i32 %a)
+  ret i32 %call4
+}
+
+attributes #0 = { noinline optnone }
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+!0 = !{i64 16, !"_ZTS1A"}
+!1 = !{i64 16, !"_ZTS1B"}
diff --git a/llvm/test/ThinLTO/X86/devirt_promote.ll b/llvm/test/ThinLTO/X86/devirt_promote.ll
new file mode 100644
index 0000000000000..563ed994157a2
--- /dev/null
+++ b/llvm/test/ThinLTO/X86/devirt_promote.ll
@@ -0,0 +1,72 @@
+; REQUIRES: x86-registered-target
+
+; Test devirtualization requiring promotion of local targets, where the
+; promotion is required by one devirtualization and needs to be updated
+; for a second devirtualization in the defining module as a post-pass
+; update.
+
+; Generate unsplit module with summary for ThinLTO index-based WPD.
+; RUN: opt -thinlto-bc -o %t3.o %s
+; RUN: opt -thinlto-bc -o %t4.o %p/Inputs/devirt_promote.ll
+
+; RUN: llvm-lto2 run %t3.o %t4.o -save-temps -use-new-pm -pass-remarks=. \
+; RUN:   -wholeprogramdevirt-print-index-based \
+; RUN:   -o %t5 \
+; RUN:   -r=%t3.o,test,px \
+; RUN:   -r=%t4.o,_ZN1B1fEi,p \
+; RUN:   -r=%t4.o,test2,px \
+; RUN:   -r=%t4.o,_ZTV1B,px \
+; RUN:   2>&1 | FileCheck %s --check-prefix=REMARK --check-prefix=PRINT
+; RUN: llvm-dis %t5.1.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR1
+; RUN: llvm-dis %t5.2.4.opt.bc -o - | FileCheck %s --check-prefix=CHECK-IR2
+; RUN: llvm-nm %t5.1 | FileCheck %s --check-prefix=NM-INDEX1
+; RUN: llvm-nm %t5.2 | FileCheck %s --check-prefix=NM-INDEX2
+
+; NM-INDEX1: U _ZN1A1nEi.llvm.
+
+; Make sure that not only did _ZN1A1nEi get promoted (due to the
+; devirtualization in the other module) but the reference due to the
+; devirtualization in its defining module should be to the promoted
+; symbol.
+; NM-INDEX2-NOT: U _ZN1A1nEi
+; NM-INDEX2: T _ZN1A1nEi.llvm.
+; NM-INDEX2-NOT: U _ZN1A1nEi
+
+; We should devirt call to _ZN1A1nEi once in importing module and once
+; in original (exporting) module.
+; REMARK-COUNT-2: single-impl: devirtualized a call to _ZN1A1nEi.llvm.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+%struct.A = type { i32 (...)** }
+
+; CHECK-IR1-LABEL: define i32 @test
+define i32 @test(%struct.A* %obj, i32 %a) {
+entry:
+  %0 = bitcast %struct.A* %obj to i8***
+  %vtable = load i8**, i8*** %0
+  %1 = bitcast i8** %vtable to i8*
+  %p = call i1 @llvm.type.test(i8* %1, metadata !"_ZTS1A")
+  call void @llvm.assume(i1 %p)
+  %fptrptr = getelementptr i8*, i8** %vtable, i32 1
+  %2 = bitcast i8** %fptrptr to i32 (%struct.A*, i32)**
+  %fptr1 = load i32 (%struct.A*, i32)*, i32 (%struct.A*, i32)** %2, align 8
+
+  ; Check that the call was devirtualized.
+  ; CHECK-IR1: %call = tail call i32 bitcast (void ()* @_ZN1A1nEi
+  %call = tail call i32 %fptr1(%struct.A* nonnull %obj, i32 %a)
+
+  ret i32 %call
+}
+; CHECK-IR1-LABEL: ret i32
+; CHECK-IR1-LABEL: }
+
+; CHECK-IR2: define i32 @test2
+; Check that the call was devirtualized.
+; CHECK-IR2:   %call4 = tail call i32 @_ZN1A1nEi
+
+declare i1 @llvm.type.test(i8*, metadata)
+declare void @llvm.assume(i1)
+
+attributes #0 = { noinline optnone }
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
index f4c1af5ed46cf..09545e268d77c 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/memset_chk-simplify-nobuiltin.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -disable-simplify-libcalls -codegenprepare < %s | FileCheck %s
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 
@@ -8,9 +9,11 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 ; - attribute nobuiltin
 ; - TLI::has (always returns false thanks to -disable-simplify-libcalls)
 
-; CHECK-NOT: _chk
-; CHECK: call void @llvm.memset.p0i8.i64(i8* align 1 %dst, i8 0, i64 %len, i1 false)
 define void @test_nobuiltin(i8* %dst, i64 %len) {
+; CHECK-LABEL: @test_nobuiltin(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* [[DST:%.*]], i8 0, i64 [[LEN:%.*]], i1 false) #1
+; CHECK-NEXT:    ret void
+;
   call i8* @__memset_chk(i8* %dst, i32 0, i64 %len, i64 -1) nobuiltin
   ret void
 }
diff --git a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
index 792ef9c0ece68..7c43d87ffa591 100644
--- a/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
+++ b/llvm/test/Transforms/DivRemPairs/X86/div-expanded-rem-pair.ll
@@ -168,3 +168,39 @@ end:
   %ret = phi i128 [ %rem, %if ], [ 3, %entry ]
   ret i128 %ret
 }
+
+; Even in expanded form, we can end up with div and rem in different basic
+; blocks neither of which dominates each another.
+define i32 @can_have_divrem_in_mutually_nondominating_bbs(i1 %cmp, i32 %a, i32 %b) {
+; CHECK-LABEL: @can_have_divrem_in_mutually_nondominating_bbs(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[CMP:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[T0:%.*]] = udiv i32 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = mul nuw i32 [[T0]], [[B]]
+; CHECK-NEXT:    [[T2_RECOMPOSED:%.*]] = urem i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[T3:%.*]] = udiv i32 [[A]], [[B]]
+; CHECK-NEXT:    br label [[END]]
+; CHECK:       end:
+; CHECK-NEXT:    [[RET:%.*]] = phi i32 [ [[T2_RECOMPOSED]], [[IF_THEN]] ], [ [[T3]], [[IF_ELSE]] ]
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+entry:
+  br i1 %cmp, label %if.then, label %if.else
+
+if.then:
+  %t0 = udiv i32 %a, %b
+  %t1 = mul nuw i32 %t0, %b
+  %t2 = sub i32 %a, %t1
+  br label %end
+
+if.else:
+  %t3 = udiv i32 %a, %b
+  br label %end
+
+end:
+  %ret = phi i32 [ %t2, %if.then ], [ %t3, %if.else ]
+  ret i32 %ret
+}
diff --git a/llvm/test/Transforms/Float2Int/basic.ll b/llvm/test/Transforms/Float2Int/basic.ll
index e1eddc668ea69..615cc601a92f9 100644
--- a/llvm/test/Transforms/Float2Int/basic.ll
+++ b/llvm/test/Transforms/Float2Int/basic.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -float2int -S | FileCheck %s
 ; RUN: opt < %s -passes='float2int' -S | FileCheck %s
 
@@ -5,108 +6,116 @@
 ; Positive tests
 ;
 
-; CHECK-LABEL: @simple1
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = add i32 %1, 1
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple1(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd float %1, 1.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd float %t1, 1.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @simple2
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 %1, 1
-; CHECK:  %3 = trunc i32 %2 to i8
-; CHECK:  ret i8 %3
 define i8 @simple2(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fsub float %1, 1.0
-  %3 = fptoui float %2 to i8
-  ret i8 %3
+; CHECK-LABEL: @simple2(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i8
+; CHECK-NEXT:    ret i8 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fsub float %t1, 1.0
+  %t3 = fptoui float %t2 to i8
+  ret i8 %t3
 }
 
-; CHECK-LABEL: @simple3
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 %1, 1
-; CHECK:  ret i32 %2
 define i32 @simple3(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fsub float %1, 1.0
-  %3 = fptoui float %2 to i32
-  ret i32 %3
+; CHECK-LABEL: @simple3(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 [[TMP1]], 1
+; CHECK-NEXT:    ret i32 [[T21]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fsub float %t1, 1.0
+  %t3 = fptoui float %t2 to i32
+  ret i32 %t3
 }
 
-; CHECK-LABEL: @cmp
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = icmp slt i32 %1, %2
-; CHECK:  ret i1 %3
 define i1 @cmp(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fcmp ult float %1, %2
-  ret i1 %3
+; CHECK-LABEL: @cmp(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = icmp slt i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i1 [[T31]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fcmp ult float %t1, %t2
+  ret i1 %t3
 }
 
-; CHECK-LABEL: @simple4
-; CHECK:  %1 = zext i32 %a to i64
-; CHECK:  %2 = add i64 %1, 1
-; CHECK:  %3 = trunc i64 %2 to i32
-; CHECK:  ret i32 %3
 define i32 @simple4(i32 %a) {
-  %1 = uitofp i32 %a to double
-  %2 = fadd double %1, 1.0
-  %3 = fptoui double %2 to i32
-  ret i32 %3
+; CHECK-LABEL: @simple4(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[A:%.*]] to i64
+; CHECK-NEXT:    [[T21:%.*]] = add i64 [[TMP1]], 1
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[T21]] to i32
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
+  %t1 = uitofp i32 %a to double
+  %t2 = fadd double %t1, 1.0
+  %t3 = fptoui double %t2 to i32
+  ret i32 %t3
 }
 
-; CHECK-LABEL: @simple5
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = add i32 %1, 1
-; CHECK:  %4 = mul i32 %3, %2
-; CHECK:  ret i32 %4
 define i32 @simple5(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fadd float %1, 1.0
-  %4 = fmul float %3, %2
-  %5 = fptoui float %4 to i32
-  ret i32 %5
+; CHECK-LABEL: @simple5(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = add i32 [[TMP1]], 1
+; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[T42]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fadd float %t1, 1.0
+  %t4 = fmul float %t3, %t2
+  %t5 = fptoui float %t4 to i32
+  ret i32 %t5
 }
 
-; CHECK-LABEL: @simple6
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %3 = sub i32 0, %1
-; CHECK:  %4 = mul i32 %3, %2
-; CHECK:  ret i32 %4
 define i32 @simple6(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fneg float %1
-  %4 = fmul float %3, %2
-  %5 = fptoui float %4 to i32
-  ret i32 %5
+; CHECK-LABEL: @simple6(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[T31:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[T42:%.*]] = mul i32 [[T31]], [[TMP2]]
+; CHECK-NEXT:    ret i32 [[T42]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fneg float %t1
+  %t4 = fmul float %t3, %t2
+  %t5 = fptoui float %t4 to i32
+  ret i32 %t5
 }
 
 ; The two chains don't interact - failure of one shouldn't
 ; cause failure of the other.
 
-; CHECK-LABEL: @multi1
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = zext i8 %b to i32
-; CHECK:  %fc = uitofp i8 %c to float
-; CHECK:  %x1 = add i32 %1, %2
-; CHECK:  %z = fadd float %fc, %d
-; CHECK:  %w = fptoui float %z to i32
-; CHECK:  %r = add i32 %x1, %w
-; CHECK:  ret i32 %r
 define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
+; CHECK-LABEL: @multi1(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i8 [[B:%.*]] to i32
+; CHECK-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; CHECK-NEXT:    [[X1:%.*]] = add i32 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; CHECK-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[X1]], [[W]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i8 %a to float
   %fb = uitofp i8 %b to float
   %fc = uitofp i8 %c to float
@@ -118,25 +127,27 @@ define i32 @multi1(i8 %a, i8 %b, i8 %c, float %d) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @simple_negzero
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = add i32 %1, 0
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple_negzero(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd fast float %1, -0.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple_negzero(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = add i32 [[TMP1]], 0
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd fast float %t1, -0.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @simple_negative
-; CHECK: %1 = sext i8 %call to i32
-; CHECK: %mul1 = mul i32 %1, -3
-; CHECK: %2 = trunc i32 %mul1 to i8
-; CHECK: %conv3 = sext i8 %2 to i32
-; CHECK: ret i32 %conv3
 define i32 @simple_negative(i8 %call) {
+; CHECK-LABEL: @simple_negative(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i8 [[CALL:%.*]] to i32
+; CHECK-NEXT:    [[MUL1:%.*]] = mul i32 [[TMP1]], -3
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[MUL1]] to i8
+; CHECK-NEXT:    [[CONV3:%.*]] = sext i8 [[TMP2]] to i32
+; CHECK-NEXT:    ret i32 [[CONV3]]
+;
   %conv1 = sitofp i8 %call to float
   %mul = fmul float %conv1, -3.000000e+00
   %conv2 = fptosi float %mul to i8
@@ -144,34 +155,37 @@ define i32 @simple_negative(i8 %call) {
   ret i32 %conv3
 }
 
-; CHECK-LABEL: @simple_fneg
-; CHECK:  %1 = zext i8 %a to i32
-; CHECK:  %2 = sub i32 0, %1
-; CHECK:  %3 = trunc i32 %2 to i16
-; CHECK:  ret i16 %3
 define i16 @simple_fneg(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fneg fast float %1
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @simple_fneg(
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[A:%.*]] to i32
+; CHECK-NEXT:    [[T21:%.*]] = sub i32 0, [[TMP1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc i32 [[T21]] to i16
+; CHECK-NEXT:    ret i16 [[TMP2]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fneg fast float %t1
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
 ;
 ; Negative tests
 ;
 
-; CHECK-LABEL: @neg_multi1
-; CHECK:  %fa = uitofp i8 %a to float
-; CHECK:  %fc = uitofp i8 %c to float
-; CHECK:  %x = fadd float %fa, %fc
-; CHECK:  %y = fptoui float %x to i32
-; CHECK:  %z = fadd float %fc, %d
-; CHECK:  %w = fptoui float %z to i32
-; CHECK:  %r = add i32 %y, %w
-; CHECK:  ret i32 %r
 ; The two chains intersect, which means because one fails, no
 ; transform can occur.
+
 define i32 @neg_multi1(i8 %a, i8 %b, i8 %c, float %d) {
+; CHECK-LABEL: @neg_multi1(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[FC:%.*]] = uitofp i8 [[C:%.*]] to float
+; CHECK-NEXT:    [[X:%.*]] = fadd float [[FA]], [[FC]]
+; CHECK-NEXT:    [[Y:%.*]] = fptoui float [[X]] to i32
+; CHECK-NEXT:    [[Z:%.*]] = fadd float [[FC]], [[D:%.*]]
+; CHECK-NEXT:    [[W:%.*]] = fptoui float [[Z]] to i32
+; CHECK-NEXT:    [[R:%.*]] = add i32 [[Y]], [[W]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i8 %a to float
   %fc = uitofp i8 %c to float
   %x = fadd float %fa, %fc
@@ -182,15 +196,17 @@ define i32 @neg_multi1(i8 %a, i8 %b, i8 %c, float %d) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @neg_muld
-; CHECK:  %fa = uitofp i32 %a to double
-; CHECK:  %fb = uitofp i32 %b to double
-; CHECK:  %mul = fmul double %fa, %fb
-; CHECK:  %r = fptoui double %mul to i64
-; CHECK:  ret i64 %r
 ; The i32 * i32 = i64, which has 64 bits, which is greater than the 52 bits
 ; that can be exactly represented in a double.
+
 define i64 @neg_muld(i32 %a, i32 %b) {
+; CHECK-LABEL: @neg_muld(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i32 [[A:%.*]] to double
+; CHECK-NEXT:    [[FB:%.*]] = uitofp i32 [[B:%.*]] to double
+; CHECK-NEXT:    [[MUL:%.*]] = fmul double [[FA]], [[FB]]
+; CHECK-NEXT:    [[R:%.*]] = fptoui double [[MUL]] to i64
+; CHECK-NEXT:    ret i64 [[R]]
+;
   %fa = uitofp i32 %a to double
   %fb = uitofp i32 %b to double
   %mul = fmul double %fa, %fb
@@ -198,16 +214,18 @@ define i64 @neg_muld(i32 %a, i32 %b) {
   ret i64 %r
 }
 
-; CHECK-LABEL: @neg_mulf
-; CHECK:  %fa = uitofp i16 %a to float
-; CHECK:  %fb = uitofp i16 %b to float
-; CHECK:  %mul = fmul float %fa, %fb
-; CHECK:  %r = fptoui float %mul to i32
-; CHECK:  ret i32 %r
 ; The i16 * i16 = i32, which can't be represented in a float, but can in a
 ; double. This should fail, as the written code uses floats, not doubles so
 ; the original result may be inaccurate.
+
 define i32 @neg_mulf(i16 %a, i16 %b) {
+; CHECK-LABEL: @neg_mulf(
+; CHECK-NEXT:    [[FA:%.*]] = uitofp i16 [[A:%.*]] to float
+; CHECK-NEXT:    [[FB:%.*]] = uitofp i16 [[B:%.*]] to float
+; CHECK-NEXT:    [[MUL:%.*]] = fmul float [[FA]], [[FB]]
+; CHECK-NEXT:    [[R:%.*]] = fptoui float [[MUL]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
   %fa = uitofp i16 %a to float
   %fb = uitofp i16 %b to float
   %mul = fmul float %fa, %fb
@@ -215,80 +233,120 @@ define i32 @neg_mulf(i16 %a, i16 %b) {
   ret i32 %r
 }
 
-; CHECK-LABEL: @neg_cmp
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = uitofp i8 %b to float
-; CHECK:  %3 = fcmp false float %1, %2
-; CHECK:  ret i1 %3
 ; "false" doesn't have an icmp equivalent.
+
 define i1 @neg_cmp(i8 %a, i8 %b) {
-  %1 = uitofp i8 %a to float
-  %2 = uitofp i8 %b to float
-  %3 = fcmp false float %1, %2
-  ret i1 %3
+; CHECK-LABEL: @neg_cmp(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = uitofp i8 [[B:%.*]] to float
+; CHECK-NEXT:    [[T3:%.*]] = fcmp false float [[T1]], [[T2]]
+; CHECK-NEXT:    ret i1 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = uitofp i8 %b to float
+  %t3 = fcmp false float %t1, %t2
+  ret i1 %t3
 }
 
-; CHECK-LABEL: @neg_div
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = fdiv float %1, 1.0
-; CHECK:  %3 = fptoui float %2 to i16
-; CHECK:  ret i16 %3
 ; Division isn't a supported operator.
+
 define i16 @neg_div(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fdiv float %1, 1.0
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @neg_div(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fdiv float [[T1]], 1.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fptoui float [[T2]] to i16
+; CHECK-NEXT:    ret i16 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fdiv float %t1, 1.0
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @neg_remainder
-; CHECK:  %1 = uitofp i8 %a to float
-; CHECK:  %2 = fadd float %1, 1.2
-; CHECK:  %3 = fptoui float %2 to i16
-; CHECK:  ret i16 %3
 ; 1.2 is not an integer.
+
 define i16 @neg_remainder(i8 %a) {
-  %1 = uitofp i8 %a to float
-  %2 = fadd float %1, 1.25
-  %3 = fptoui float %2 to i16
-  ret i16 %3
+; CHECK-LABEL: @neg_remainder(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i8 [[A:%.*]] to float
+; CHECK-NEXT:    [[T2:%.*]] = fadd float [[T1]], 1.250000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fptoui float [[T2]] to i16
+; CHECK-NEXT:    ret i16 [[T3]]
+;
+  %t1 = uitofp i8 %a to float
+  %t2 = fadd float %t1, 1.25
+  %t3 = fptoui float %t2 to i16
+  ret i16 %t3
 }
 
-; CHECK-LABEL: @neg_toolarge
-; CHECK:  %1 = uitofp i80 %a to fp128
-; CHECK:  %2 = fadd fp128 %1, %1
-; CHECK:  %3 = fptoui fp128 %2 to i80
-; CHECK:  ret i80 %3
 ; i80 > i64, which is the largest bitwidth handleable by default.
+
 define i80 @neg_toolarge(i80 %a) {
-  %1 = uitofp i80 %a to fp128
-  %2 = fadd fp128 %1, %1
-  %3 = fptoui fp128 %2 to i80
-  ret i80 %3
+; CHECK-LABEL: @neg_toolarge(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp i80 [[A:%.*]] to fp128
+; CHECK-NEXT:    [[T2:%.*]] = fadd fp128 [[T1]], [[T1]]
+; CHECK-NEXT:    [[T3:%.*]] = fptoui fp128 [[T2]] to i80
+; CHECK-NEXT:    ret i80 [[T3]]
+;
+  %t1 = uitofp i80 %a to fp128
+  %t2 = fadd fp128 %t1, %t1
+  %t3 = fptoui fp128 %t2 to i80
+  ret i80 %t3
 }
 
-; CHECK-LABEL: @neg_calluser
-; CHECK: sitofp
-; CHECK: fcmp
-; The sequence %1..%3 cannot be converted because %4 uses %2.
+; The sequence %t1..%t3 cannot be converted because %t4 uses %t2.
+
 define i32 @neg_calluser(i32 %value) {
-  %1 = sitofp i32 %value to double
-  %2 = fadd double %1, 1.0
-  %3 = fcmp olt double %2, 0.000000e+00
-  %4 = tail call double @g(double %2)
-  %5 = fptosi double %4 to i32
-  %6 = zext i1 %3 to i32
-  %7 = add i32 %6, %5
-  ret i32 %7
+; CHECK-LABEL: @neg_calluser(
+; CHECK-NEXT:    [[T1:%.*]] = sitofp i32 [[VALUE:%.*]] to double
+; CHECK-NEXT:    [[T2:%.*]] = fadd double [[T1]], 1.000000e+00
+; CHECK-NEXT:    [[T3:%.*]] = fcmp olt double [[T2]], 0.000000e+00
+; CHECK-NEXT:    [[T4:%.*]] = tail call double @g(double [[T2]])
+; CHECK-NEXT:    [[T5:%.*]] = fptosi double [[T4]] to i32
+; CHECK-NEXT:    [[T6:%.*]] = zext i1 [[T3]] to i32
+; CHECK-NEXT:    [[T7:%.*]] = add i32 [[T6]], [[T5]]
+; CHECK-NEXT:    ret i32 [[T7]]
+;
+  %t1 = sitofp i32 %value to double
+  %t2 = fadd double %t1, 1.0
+  %t3 = fcmp olt double %t2, 0.000000e+00
+  %t4 = tail call double @g(double %t2)
+  %t5 = fptosi double %t4 to i32
+  %t6 = zext i1 %t3 to i32
+  %t7 = add i32 %t6, %t5
+  ret i32 %t7
 }
+
 declare double @g(double)
 
-; CHECK-LABEL: @neg_vector
-; CHECK:  %1 = uitofp <4 x i8> %a to <4 x float>
-; CHECK:  %2 = fptoui <4 x float> %1 to <4 x i16>
-; CHECK:  ret <4 x i16> %2
 define <4 x i16> @neg_vector(<4 x i8> %a) {
-  %1 = uitofp <4 x i8> %a to <4 x float>
-  %2 = fptoui <4 x float> %1 to <4 x i16>
-  ret <4 x i16> %2
+; CHECK-LABEL: @neg_vector(
+; CHECK-NEXT:    [[T1:%.*]] = uitofp <4 x i8> [[A:%.*]] to <4 x float>
+; CHECK-NEXT:    [[T2:%.*]] = fptoui <4 x float> [[T1]] to <4 x i16>
+; CHECK-NEXT:    ret <4 x i16> [[T2]]
+;
+  %t1 = uitofp <4 x i8> %a to <4 x float>
+  %t2 = fptoui <4 x float> %t1 to <4 x i16>
+  ret <4 x i16> %t2
+}
+
+; Don't crash while processing unreachable (non-standard) IR.
+
+define void @PR38502() {
+; CHECK-LABEL: @PR38502(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret void
+; CHECK:       bogusBB:
+; CHECK-NEXT:    [[INC1:%.*]] = fadd double [[INC:%.*]], 1.000000e+00
+; CHECK-NEXT:    [[INC]] = fadd double [[INC1]], 1.000000e+00
+; CHECK-NEXT:    [[TOBOOL:%.*]] = fcmp une double [[INC]], 0.000000e+00
+; CHECK-NEXT:    br label [[BOGUSBB:%.*]]
+;
+entry:
+  ret void
+
+bogusBB:                                          ; preds = %bogusBB
+  %inc1 = fadd double %inc, 1.000000e+00
+  %inc = fadd double %inc1, 1.000000e+00
+  %tobool = fcmp une double %inc, 0.000000e+00
+  br label %bogusBB
 }
diff --git a/llvm/test/Transforms/FunctionAttrs/align.ll b/llvm/test/Transforms/FunctionAttrs/align.ll
index 5e8671b64a2e6..fc5ffc61e8ead 100644
--- a/llvm/test/Transforms/FunctionAttrs/align.ll
+++ b/llvm/test/Transforms/FunctionAttrs/align.ll
@@ -1,4 +1,4 @@
-; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
index 6e57475c57966..57d2713d278be 100644
--- a/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/arg_returned.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=9 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=6 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 ; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -functionattrs -S < %s | FileCheck %s --check-prefix=BOTH
 ;
 ; Test cases specifically designed for the "returned" argument attribute.
@@ -827,6 +827,17 @@ define i32 @exact(i32* %a) {
   ret i32 %add3
 }
 
+@G = external global i8
+define i32* @ret_const() #0 {
+  %bc = bitcast i8* @G to i32*
+  ret i32* %bc
+}
+define i32* @use_const() #0 {
+  %c = call i32* @ret_const()
+  ; CHECK: ret i32* bitcast (i8* @G to i32*)
+  ret i32* %c
+}
+
 attributes #0 = { noinline nounwind uwtable }
 
 ; BOTH-NOT: attributes #
diff --git a/llvm/test/Transforms/FunctionAttrs/dereferenceable.ll b/llvm/test/Transforms/FunctionAttrs/dereferenceable.ll
index 852b2a283517e..d7b576f506acb 100644
--- a/llvm/test/Transforms/FunctionAttrs/dereferenceable.ll
+++ b/llvm/test/Transforms/FunctionAttrs/dereferenceable.ll
@@ -1,4 +1,4 @@
-; RUN: opt -attributor -attributor-manifest-internal --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
+; RUN: opt -attributor -attributor-manifest-internal --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR
 
 
 declare void @deref_phi_user(i32* %a);
diff --git a/llvm/test/Transforms/FunctionAttrs/fn_noreturn.ll b/llvm/test/Transforms/FunctionAttrs/fn_noreturn.ll
index d8258be7d2cce..e3a0f0b2504cf 100644
--- a/llvm/test/Transforms/FunctionAttrs/fn_noreturn.ll
+++ b/llvm/test/Transforms/FunctionAttrs/fn_noreturn.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 -S < %s | FileCheck %s
+; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s
 ;
 ; Test cases specifically designed for the "no-return" function attribute.
 ; We use FIXME's to indicate problems and missing attributes.
diff --git a/llvm/test/Transforms/FunctionAttrs/heap_to_stack.ll b/llvm/test/Transforms/FunctionAttrs/heap_to_stack.ll
new file mode 100644
index 0000000000000..26a778be841ab
--- /dev/null
+++ b/llvm/test/Transforms/FunctionAttrs/heap_to_stack.ll
@@ -0,0 +1,318 @@
+; RUN: opt -passes=attributor --attributor-disable=false -S < %s | FileCheck %s
+
+declare noalias i8* @malloc(i64)
+
+declare void @nocapture_func_frees_pointer(i8* nocapture)
+
+declare void @func_throws(...)
+
+declare void @sync_func(i8* %p)
+
+declare void @sync_will_return(i8* %p) willreturn
+
+declare void @no_sync_func(i8* nocapture %p) nofree nosync willreturn
+
+declare void @nofree_func(i8* nocapture %p) nofree  nosync willreturn
+
+declare void @foo(i32* %p)
+
+declare void @foo_nounw(i32* %p) nounwind nofree
+
+declare i32 @no_return_call() noreturn
+
+declare void @free(i8* nocapture)
+
+declare void @llvm.lifetime.start.p0i8(i64, i8* nocapture) nounwind
+
+; TEST 1 - negative, pointer freed in another function.
+
+define void @test1() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: @malloc(i64 4)
+  ; CHECK-NEXT: @nocapture_func_frees_pointer(i8* noalias nocapture %1)
+  tail call void @nocapture_func_frees_pointer(i8* %1)
+  tail call void (...) @func_throws()
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 2 - negative, call to a sync function.
+
+define void @test2() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: @malloc(i64 4)
+  ; CHECK-NEXT: @sync_func(i8* %1)
+  tail call void @sync_func(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 3 - 1 malloc, 1 free
+
+define void @test3() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+declare noalias i8* @calloc(i64, i64)
+
+define void @test0() {
+  %1 = tail call noalias i8* @calloc(i64 2, i64 4)
+  ; CHECK: %1 = alloca i8, i64 8
+  ; CHECK-NEXT: %calloc_bc = bitcast i8* %1 to i8*
+  ; CHECK-NEXT: call void @llvm.memset.p0i8.i64(i8* %calloc_bc, i8 0, i64 8, i1 false)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 4 
+define void @test4() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @nofree_func(i8* noalias nocapture %1)
+  tail call void @nofree_func(i8* %1)
+  ret void
+}
+
+; TEST 5 - not all exit paths have a call to free, but all uses of malloc
+; are in nofree functions and are not captured
+
+define void @test5(i32) {
+  %2 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %2 = alloca i8, i64 4
+  ; CHECK-NEXT: icmp eq i32 %0, 0
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %5, label %4
+
+4:                                                ; preds = %1
+  tail call void @nofree_func(i8* %2)
+  br label %6
+
+5:                                                ; preds = %1
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+6:                                                ; preds = %5, %4
+  ret void
+}
+
+; TEST 6 - all exit paths have a call to free
+
+define void @test6(i32) {
+  %2 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %2 = alloca i8, i64 4
+  ; CHECK-NEXT: icmp eq i32 %0, 0
+  %3 = icmp eq i32 %0, 0
+  br i1 %3, label %5, label %4
+
+4:                                                ; preds = %1
+  tail call void @nofree_func(i8* %2)
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+5:                                                ; preds = %1
+  tail call void @free(i8* %2)
+  ; CHECK-NOT: @free(i8* %2)
+  br label %6
+
+6:                                                ; preds = %5, %4
+  ret void
+}
+
+; TEST 7 - free is dead.
+
+define void @test7() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: alloca i8, i64 4
+  ; CHECK-NEXT: tail call i32 @no_return_call()
+  tail call i32 @no_return_call()
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 8 - Negative: bitcast pointer used in capture function
+
+define void @test8() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @foo(i32* %2)
+  ; CHECK: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 9 - FIXME: malloc should be converted.
+define void @test9() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @foo_nounw(i32* %2)
+  ; CHECK: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 10 - 1 malloc, 1 free
+
+define i32 @test10() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret i32 %3
+}
+
+define i32 @test_lifetime() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: %1 = alloca i8, i64 4
+  ; CHECK-NEXT: @no_sync_func(i8* noalias nocapture %1)
+  tail call void @no_sync_func(i8* %1)
+  call void @llvm.lifetime.start.p0i8(i64 4, i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  ; CHECK-NOT: @free(i8* %1)
+  tail call void @free(i8* %1)
+  ret i32 %3
+}
+
+; TEST 11 
+; FIXME: should be ok
+
+define void @test11() {
+  %1 = tail call noalias i8* @malloc(i64 4)
+  ; CHECK: @malloc(i64 4)
+  ; CHECK-NEXT: @sync_will_return(i8* %1)
+  tail call void @sync_will_return(i8* %1)
+  tail call void @free(i8* %1)
+  ret void
+}
+
+; TEST 12
+define i32 @irreducible_cfg(i32 %0) {
+  %2 = alloca i32, align 4
+  %3 = alloca i32*, align 8
+  %4 = alloca i32, align 4
+  store i32 %0, i32* %2, align 4
+  %5 = call noalias i8* @malloc(i64 4) #2
+  ; CHECK: alloca i8, i64 4
+  ; CHECK-NEXT: %6 = bitcast
+  %6 = bitcast i8* %5 to i32*
+  store i32* %6, i32** %3, align 8
+  %7 = load i32*, i32** %3, align 8
+  store i32 10, i32* %7, align 4
+  %8 = load i32, i32* %2, align 4
+  %9 = icmp eq i32 %8, 1
+  br i1 %9, label %10, label %13
+
+10:                                               ; preds = %1
+  %11 = load i32, i32* %2, align 4
+  %12 = add nsw i32 %11, 5
+  store i32 %12, i32* %2, align 4
+  br label %20
+
+13:                                               ; preds = %1
+  store i32 1, i32* %2, align 4
+  br label %14
+
+14:                                               ; preds = %20, %13
+  %15 = load i32*, i32** %3, align 8
+  %16 = load i32, i32* %15, align 4
+  %17 = add nsw i32 %16, -1
+  store i32 %17, i32* %15, align 4
+  %18 = icmp ne i32 %16, 0
+  br i1 %18, label %19, label %23
+
+19:                                               ; preds = %14
+  br label %20
+
+20:                                               ; preds = %19, %10
+  %21 = load i32, i32* %2, align 4
+  %22 = add nsw i32 %21, 1
+  store i32 %22, i32* %2, align 4
+  br label %14
+
+23:                                               ; preds = %14
+  %24 = load i32*, i32** %3, align 8
+  %25 = load i32, i32* %24, align 4
+  store i32 %25, i32* %4, align 4
+  %26 = load i32*, i32** %3, align 8
+  %27 = bitcast i32* %26 to i8*
+  call void @free(i8* %27) #2
+  %28 = load i32*, i32** %3, align 8
+  %29 = load i32, i32* %28, align 4
+  ret i32 %29
+}
+
+define i32 @malloc_in_loop(i32 %0) {
+  %2 = alloca i32, align 4
+  %3 = alloca i32*, align 8
+  store i32 %0, i32* %2, align 4
+  br label %4
+
+4:                                                ; preds = %8, %1
+  %5 = load i32, i32* %2, align 4
+  %6 = add nsw i32 %5, -1
+  store i32 %6, i32* %2, align 4
+  %7 = icmp sgt i32 %6, 0
+  br i1 %7, label %8, label %11
+
+8:                                                ; preds = %4
+  %9 = call noalias i8* @malloc(i64 4)
+  ; CHECK: alloca i8, i64 4
+  %10 = bitcast i8* %9 to i32*
+  store i32* %10, i32** %3, align 8
+  br label %4
+
+11:                                               ; preds = %4
+  ret i32 5
+}
+
+; Malloc/Calloc too large
+define i32 @test13() {
+  %1 = tail call noalias i8* @malloc(i64 256)
+  ; CHECK: %1 = tail call noalias i8* @malloc(i64 256)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias %1)
+  tail call void @no_sync_func(i8* %1)
+  %2 = bitcast i8* %1 to i32*
+  store i32 10, i32* %2
+  %3 = load i32, i32* %2
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret i32 %3
+}
+
+define void @test14() {
+  %1 = tail call noalias i8* @calloc(i64 64, i64 4)
+  ; CHECK: %1 = tail call noalias i8* @calloc(i64 64, i64 4)
+  ; CHECK-NEXT: @no_sync_func(i8* noalias %1)
+  tail call void @no_sync_func(i8* %1)
+  tail call void @free(i8* %1)
+  ; CHECK: tail call void @free(i8* noalias %1)
+  ret void
+}
diff --git a/llvm/test/Transforms/FunctionAttrs/internal-noalias.ll b/llvm/test/Transforms/FunctionAttrs/internal-noalias.ll
index 7449c301783f6..1118fa194f3c5 100644
--- a/llvm/test/Transforms/FunctionAttrs/internal-noalias.ll
+++ b/llvm/test/Transforms/FunctionAttrs/internal-noalias.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=6 < %s | FileCheck %s
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=5 < %s | FileCheck %s
 
 define dso_local i32 @visible(i32* noalias %A, i32* noalias %B) #0 {
 entry:
@@ -10,7 +10,7 @@ entry:
 
 ; FIXME: Should be something like this.
 ; define internal i32 @noalias_args(i32* nocapture readonly %A, i32* noalias nocapture readonly %B)
-; CHECK: define internal i32 @noalias_args(i32* nocapture %A, i32* nocapture %B)
+; CHECK: define internal i32 @noalias_args(i32* nocapture %A, i32* noalias nocapture %B)
 
 define internal i32 @noalias_args(i32* %A, i32* %B) #0 {
 entry:
diff --git a/llvm/test/Transforms/FunctionAttrs/liveness.ll b/llvm/test/Transforms/FunctionAttrs/liveness.ll
index 93aac99995074..771c99309bb1a 100644
--- a/llvm/test/Transforms/FunctionAttrs/liveness.ll
+++ b/llvm/test/Transforms/FunctionAttrs/liveness.ll
@@ -1,4 +1,4 @@
-; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=11 -S < %s | FileCheck %s
+; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s
 
 declare void @no_return_call() nofree noreturn nounwind readnone
 
diff --git a/llvm/test/Transforms/FunctionAttrs/noalias_returned.ll b/llvm/test/Transforms/FunctionAttrs/noalias_returned.ll
index 2f71d1e57916e..7174c3dc97cb7 100644
--- a/llvm/test/Transforms/FunctionAttrs/noalias_returned.ll
+++ b/llvm/test/Transforms/FunctionAttrs/noalias_returned.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=5 < %s | FileCheck %s
+; RUN: opt -S -passes=attributor -aa-pipeline='basic-aa' -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=1 < %s | FileCheck %s
 
 ; TEST 1 - negative.
 
@@ -203,7 +203,7 @@ define void @test12_1() {
 ; CHECK-NEXT:    [[B:%.*]] = tail call noalias i8* @malloc(i64 4)
 ; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nonnull align 4 dereferenceable(1) [[A]])
 ; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nonnull align 4 dereferenceable(1) [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias [[B]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias nocapture [[B]])
 ; CHECK-NEXT:    tail call void @use_nocapture(i8* noalias [[B]])
 ; CHECK-NEXT:    ret void
 ;
@@ -220,7 +220,7 @@ define void @test12_2(){
 ; CHECK-LABEL: @test12_2(
 ; CHECK-NEXT:    [[A:%.*]] = tail call noalias i8* @malloc(i64 4)
 ; FIXME: This should be @use_nocapture(i8* noalias [[A]])
-; CHECK-NEXT:    tail call void @use_nocapture(i8* [[A]])
+; CHECK-NEXT:    tail call void @use_nocapture(i8* nocapture [[A]])
 ; FIXME: This should be @use_nocapture(i8* noalias [[A]])
 ; CHECK-NEXT:    tail call void @use_nocapture(i8* [[A]])
 ; CHECK-NEXT:    tail call void @use(i8* [[A]])
@@ -239,7 +239,7 @@ declare void @two_args(i8* nocapture , i8* nocapture)
 define void @test12_3(){
 ; CHECK-LABEL: @test12_3(
   %A = tail call noalias i8* @malloc(i64 4)
-; CHECK: tail call void @two_args(i8* %A, i8* %A)
+; CHECK: tail call void @two_args(i8* nocapture %A, i8* %A)
   tail call void @two_args(i8* %A, i8* %A)
   ret void
 }
@@ -252,18 +252,17 @@ define void @test12_4(){
   %A_1 = getelementptr i8, i8* %A, i64 1
   %B_0 = getelementptr i8, i8* %B, i64 0
 
-; FIXME: This should be @two_args(i8* noalias %A, i8* noalias %B)
-; CHECK: tail call void @two_args(i8* %A, i8* %B)
+; CHECK: tail call void @two_args(i8* noalias %A, i8* noalias %B)
   tail call void @two_args(i8* %A, i8* %B)
 
-; CHECK: tail call void @two_args(i8* %A, i8* %A_0)
+; CHECK: tail call void @two_args(i8* %A, i8* nocapture %A_0)
   tail call void @two_args(i8* %A, i8* %A_0)
 
 ; CHECK: tail call void @two_args(i8* %A, i8* %A_1)
   tail call void @two_args(i8* %A, i8* %A_1)
 
 ; FIXME: This should be @two_args(i8* noalias %A_0, i8* noalias %B_0)
-; CHECK: tail call void @two_args(i8* %A_0, i8* %B_0)
+; CHECK: tail call void @two_args(i8* %A_0, i8* nocapture %B_0)
   tail call void @two_args(i8* %A_0, i8* %B_0)
   ret void
 }
diff --git a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
index bfaa98eccf752..f76b429aa4b33 100644
--- a/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nofree-attributor.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -functionattrs --disable-nofree-inference=false -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=4 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
diff --git a/llvm/test/Transforms/FunctionAttrs/norecurse.ll b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
index 0293938e479f6..ed086341c4719 100644
--- a/llvm/test/Transforms/FunctionAttrs/norecurse.ll
+++ b/llvm/test/Transforms/FunctionAttrs/norecurse.ll
@@ -1,81 +1,88 @@
-; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s
-; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s
+; RUN: opt < %s -basicaa -functionattrs -rpo-functionattrs -S | FileCheck %s --check-prefixes=CHECK,BOTH
+; RUN: opt < %s -aa-pipeline=basic-aa -passes='cgscc(function-attrs),rpo-functionattrs' -S | FileCheck %s --check-prefixes=CHECK,BOTH
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefixes=ATTRIBUTOR,BOTH
 
 ; CHECK: Function Attrs
 ; CHECK-SAME: norecurse nounwind readnone
-; CHECK-NEXT: define i32 @leaf()
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind willreturn
+; BOTH-NEXT: define i32 @leaf()
 define i32 @leaf() {
   ret i32 1
 }
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
-; CHECK-NEXT: define i32 @self_rec()
+; BOTH-NOT: norecurse
+; BOTH-NEXT: define i32 @self_rec()
 define i32 @self_rec() {
   %a = call i32 @self_rec()
   ret i32 4
 }
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
-; CHECK-NEXT: define i32 @indirect_rec()
+; BOTH-NOT: norecurse
+; BOTH-NEXT: define i32 @indirect_rec()
 define i32 @indirect_rec() {
   %a = call i32 @indirect_rec2()
   ret i32 %a
 }
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
-; CHECK-NEXT: define i32 @indirect_rec2()
+; BOTH-NOT: norecurse
+; BOTH-NEXT: define i32 @indirect_rec2()
 define i32 @indirect_rec2() {
   %a = call i32 @indirect_rec()
   ret i32 %a
 }
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: readnone
-; CHECK-NOT: norecurse
-; CHECK-NEXT: define i32 @extern()
+; BOTH-NOT: norecurse
+; BOTH-NEXT: define i32 @extern()
 define i32 @extern() {
   %a = call i32 @k()
   ret i32 %a
 }
 
-; CHECK: Function Attrs
-; CHECK-NEXT: declare i32 @k()
+; BOTH: Function Attrs
+; BOTH-NEXT: declare i32 @k()
 declare i32 @k() readnone
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: nounwind
-; CHECK-NOT: norecurse
+; BOTH-NOT: norecurse
 ; CHECK-NEXT: define void @intrinsic(i8* nocapture %dest, i8* nocapture readonly %src, i32 %len)
+; ATTRIBUTOR-NEXT: define void @intrinsic(i8* nocapture %dest, i8* nocapture %src, i32 %len)
 define void @intrinsic(i8* %dest, i8* %src, i32 %len) {
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %dest, i8* %src, i32 %len, i1 false)
   ret void
 }
 
-; CHECK: Function Attrs
-; CHECK-NEXT: declare void @llvm.memcpy.p0i8.p0i8.i32
+; BOTH: Function Attrs
+; BOTH-NEXT: declare void @llvm.memcpy.p0i8.p0i8.i32
 declare void @llvm.memcpy.p0i8.p0i8.i32(i8*, i8*, i32, i1)
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: norecurse readnone
+; FIXME: missing "norecurse"
+; ATTRIBUTOR-SAME: nosync
 ; CHECK-NEXT: define internal i32 @called_by_norecurse()
 define internal i32 @called_by_norecurse() {
   %a = call i32 @k()
   ret i32 %a
 }
-; CHECK: Function Attrs
-; CHECK-NEXT: define void @m()
+; BOTH: Function Attrs
+; BOTH-NEXT: define void @m()
 define void @m() norecurse {
   %a = call i32 @called_by_norecurse()
   ret void
 }
 
-; CHECK: Function Attrs
+; BOTH: Function Attrs
 ; CHECK-SAME: norecurse readnone
+; FIXME: missing "norecurse"
+; ATTRIBUTOR-SAME: nosync
 ; CHECK-NEXT: define internal i32 @called_by_norecurse_indirectly()
 define internal i32 @called_by_norecurse_indirectly() {
   %a = call i32 @k()
@@ -89,3 +96,54 @@ define void @p() norecurse {
   call void @o()
   ret void
 }
+
+; ATTRIBUTOR: Function Attrs: nofree nosync nounwind
+; ATTRIBUTOR-NEXT: define void @f(i32 %x)
+define void @f(i32 %x)  {
+entry:
+  %x.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4
+  %tobool = icmp ne i32 %0, 0
+  br i1 %tobool, label %if.then, label %if.end
+
+if.then:
+  call void @g() norecurse
+  br label %if.end
+
+if.end:
+  ret void
+}
+
+; BOTH: define void @g()
+define void @g() norecurse {
+entry:
+  call void @f(i32 0)
+  ret void
+}
+
+; ATTRIBUTOR-NOT: Function Attrs
+; ATTRIBUTOR: define linkonce_odr i32 @leaf_redefinable()
+define linkonce_odr i32 @leaf_redefinable() {
+  ret i32 1
+}
+
+; Call through a function pointer
+; ATTRIBUTOR-NOT: Function Attrs
+; ATTRIBUTOR: define i32 @eval_func(i32 (i32)* nocapture %0, i32 %1)
+define i32 @eval_func(i32 (i32)* , i32) local_unnamed_addr {
+  %3 = tail call i32 %0(i32 %1) #2
+  ret i32 %3
+}
+
+declare void @unknown()
+; Call an unknown function in a dead block.
+; ATTRIBUTOR: Function Attrs: nofree norecurse nosync nounwind willreturn
+; ATTRIBUTOR: define i32 @call_unknown_in_dead_block()
+define i32 @call_unknown_in_dead_block() local_unnamed_addr {
+  ret i32 0
+Dead:
+  tail call void @unknown()
+  ret i32 1
+}
+
diff --git a/llvm/test/Transforms/FunctionAttrs/noreturn_async.ll b/llvm/test/Transforms/FunctionAttrs/noreturn_async.ll
index 1e34ca4f94861..e9e423688df90 100644
--- a/llvm/test/Transforms/FunctionAttrs/noreturn_async.ll
+++ b/llvm/test/Transforms/FunctionAttrs/noreturn_async.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 -S < %s | FileCheck %s
+; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s
 ;
 ; This file is the same as noreturn_sync.ll but with a personality which
 ; indicates that the exception handler *can* catch asynchronous exceptions. As
diff --git a/llvm/test/Transforms/FunctionAttrs/noreturn_sync.ll b/llvm/test/Transforms/FunctionAttrs/noreturn_sync.ll
index 4c62ea57f5099..5d644965fb273 100644
--- a/llvm/test/Transforms/FunctionAttrs/noreturn_sync.ll
+++ b/llvm/test/Transforms/FunctionAttrs/noreturn_sync.ll
@@ -1,4 +1,4 @@
-; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=5 -S < %s | FileCheck %s
+; RUN: opt -functionattrs -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s
 ;
 ; This file is the same as noreturn_async.ll but with a personality which
 ; indicates that the exception handler *cannot* catch asynchronous exceptions.
diff --git a/llvm/test/Transforms/FunctionAttrs/nosync.ll b/llvm/test/Transforms/FunctionAttrs/nosync.ll
index 2807e89e98612..d948048db129f 100644
--- a/llvm/test/Transforms/FunctionAttrs/nosync.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nosync.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=3 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -attributor -attributor-manifest-internal -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 
 ; Test cases designed for the nosync function attribute.
diff --git a/llvm/test/Transforms/FunctionAttrs/nounwind.ll b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
index 0e6b607d54612..8587805892283 100644
--- a/llvm/test/Transforms/FunctionAttrs/nounwind.ll
+++ b/llvm/test/Transforms/FunctionAttrs/nounwind.ll
@@ -1,5 +1,5 @@
 ; RUN: opt < %s -functionattrs -S | FileCheck %s
-; RUN: opt < %s -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=5 -S | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt < %s -attributor -attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S | FileCheck %s --check-prefix=ATTRIBUTOR
 
 ; TEST 1
 ; CHECK: Function Attrs: norecurse nounwind readnone
diff --git a/llvm/test/Transforms/FunctionAttrs/willreturn.ll b/llvm/test/Transforms/FunctionAttrs/willreturn.ll
index 1b9b3c91b31b4..a1f28e03f191d 100644
--- a/llvm/test/Transforms/FunctionAttrs/willreturn.ll
+++ b/llvm/test/Transforms/FunctionAttrs/willreturn.ll
@@ -1,5 +1,5 @@
 ; RUN: opt -functionattrs -S < %s | FileCheck %s --check-prefix=FNATTR
-; RUN: opt -attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=7 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
+; RUN: opt -passes=attributor --attributor-disable=false -attributor-max-iterations-verify -attributor-max-iterations=2 -S < %s | FileCheck %s --check-prefix=ATTRIBUTOR
 
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
@@ -11,7 +11,7 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 ; TEST 1 (positive case)
 ; FNATTR: Function Attrs: noinline norecurse nounwind readnone uwtable
 ; FNATTR-NEXT: define void @only_return()
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable willreturn
 ; ATTRIBUTOR-NEXT: define void @only_return()
 define void @only_return() #0 {
     ret void
@@ -59,7 +59,7 @@ define i32 @fib(i32 %0) local_unnamed_addr #0 {
 ; FNATTR: Function Attrs: noinline norecurse nounwind readnone uwtable
 ; FNATTR-NOT: willreturn
 ; FNATTR-NEXT: define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NOT: willreturn
 ; ATTRIBUTOR-NEXT: define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr
 define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
@@ -95,7 +95,7 @@ define i32 @fact_maybe_not_halt(i32 %0) local_unnamed_addr #0 {
 ; FIXME: missing willreturn
 ; FNATTR: Function Attrs: noinline norecurse nounwind readnone uwtable
 ; FNATTR-NEXT: define i32 @fact_loop(i32 %0)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NEXT: define i32 @fact_loop(i32 %0) local_unnamed_addr
 define i32 @fact_loop(i32 %0) local_unnamed_addr #0 {
   %2 = icmp slt i32 %0, 1
@@ -253,27 +253,24 @@ define void @call_maybe_noreturn() #0 {
 ; TEST 8 (positive case)
 ; Check propagation.
 
-; FNATTR: Function Attrs: willreturn
+; FNATTR: Function Attrs: norecurse willreturn
 ; FNATTR-NEXT: declare void @will_return()
-; ATTRIBUTOR: Function Attrs: willreturn
+; ATTRIBUTOR: Function Attrs: norecurse willreturn
 ; ATTRIBUTOR-NEXT: declare void @will_return()
-declare void @will_return() willreturn
+declare void @will_return() willreturn norecurse
 
-; FIXME: missing willreturn
-; FNATTR: Function Attrs: noinline nounwind uwtable
+; FNATTR: Function Attrs: noinline norecurse nounwind uwtable
 ; FNATTR-NEXT: define void @f1()
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
 ; ATTRIBUTOR-NEXT: define void @f1()
 define void @f1() #0 {
     tail call void @will_return()
     ret void
 }
 
-; FIXME: missing willreturn
-; FNATTR: Function Attrs: noinline nounwind uwtable
+; FNATTR: Function Attrs: noinline norecurse nounwind uwtable
 ; FNATTR-NEXT: define void @f2()
-; FIXME: missing willreturn
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
 ; ATTRIBUTOR-NEXT: define void @f2()
 define void @f2() #0 {
     tail call void @f1()
@@ -284,10 +281,10 @@ define void @f2() #0 {
 ; TEST 9 (negative case)
 ; call willreturn function in endless loop.
 
-; FNATTR: Function Attrs: noinline nounwind uwtable
+; FNATTR: Function Attrs: noinline norecurse nounwind uwtable
 ; FNATTR-NOT: willreturn
 ; FNATTR-NEXT: define void @call_will_return_but_has_loop()
-; ATTRIBUTOR: Function Attrs: noinline noreturn nounwind uwtable
+; ATTRIBUTOR: Function Attrs: noinline norecurse noreturn nounwind uwtable
 ; ATTRIBUTOR-NOT: willreturn
 ; ATTRIBUTOR-NEXT: define void @call_will_return_but_has_loop()
 define void @call_will_return_but_has_loop() #0 {
@@ -340,7 +337,7 @@ declare i32 @__gxx_personality_v0(...)
 ; FIXME: missing willreturn
 ; FNATTR: Function Attrs: noinline norecurse nounwind readonly uwtable
 ; FNATTR-NEXT: define i32 @loop_constant_trip_count(i32* nocapture readonly %0)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NEXT: define i32 @loop_constant_trip_count(i32* nocapture readonly %0)
 define i32 @loop_constant_trip_count(i32* nocapture readonly %0) #0 {
   br label %3
@@ -373,7 +370,7 @@ define i32 @loop_constant_trip_count(i32* nocapture readonly %0) #0 {
 ; FNATTR: Function Attrs: noinline norecurse nounwind readonly uwtable
 ; FNATTR-NOT: willreturn
 ; FNATTR-NEXT: define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NOT: willreturn
 ; ATTRIBUTOR-NEXT: define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr
 define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2, i32 %3) local_unnamed_addr #0 {
@@ -411,7 +408,7 @@ define i32 @loop_trip_count_unbound(i32 %0, i32 %1, i32* nocapture readonly %2,
 ; FIXME: missing willreturn
 ; FNATTR: Function Attrs: noinline norecurse nounwind readonly uwtable
 ; FNATTR-NEXT: define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NEXT: define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr
 
 define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr #0 {
@@ -442,7 +439,7 @@ define i32 @loop_trip_dec(i32 %0, i32* nocapture readonly %1) local_unnamed_addr
 
 ; FNATTR: Function Attrs: noinline norecurse nounwind readnone uwtable
 ; FNATTR-NEXT: define i32 @multiple_return(i32 %a)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable willreturn
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable willreturn
 ; ATTRIBUTOR-NEXT: define i32 @multiple_return(i32 %a)
 define i32 @multiple_return(i32 %a) #0 {
   %b =  icmp eq i32 %a, 0
@@ -460,7 +457,7 @@ f:
 ; 15.1 (positive case)
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-NEXT: define void @unreachable_exit_positive1()
-; ATTRIBUTOR: Function Attrs: noinline nounwind uwtable willreturn
+; ATTRIBUTOR: Function Attrs: noinline norecurse nounwind uwtable willreturn
 ; ATTRIBUTOR-NEXT: define void @unreachable_exit_positive1()
 define void @unreachable_exit_positive1() #0 {
   tail call void @will_return()
@@ -474,7 +471,7 @@ unreachable_label:
 ; FIXME: missing willreturn
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-NEXT: define i32 @unreachable_exit_positive2(i32 %0)
-; ATTRIBUTOR: Function Attrs: nofree noinline nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse nosync nounwind uwtable
 ; ATTRIBUTOR-NEXT: define i32 @unreachable_exit_positive2(i32 %0)
 define i32 @unreachable_exit_positive2(i32) local_unnamed_addr #0 {
   %2 = icmp slt i32 %0, 1
@@ -518,7 +515,7 @@ unreachable_label:
 ; FNATTR: Function Attrs: noinline nounwind uwtable
 ; FNATTR-NOT: willreturn
 ; FNATTR-NEXT: define void @unreachable_exit_negative2()
-; ATTRIBUTOR: Function Attrs: nofree noinline noreturn nosync nounwind uwtable
+; ATTRIBUTOR: Function Attrs: nofree noinline norecurse noreturn nosync nounwind uwtable
 ; ATTRIBUTOR-NOT: willreturn
 ; ATTRIBUTOR-NEXT: define void @unreachable_exit_negative2()
 define void @unreachable_exit_negative2() #0 {
diff --git a/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
new file mode 100644
index 0000000000000..811b50783a5c5
--- /dev/null
+++ b/llvm/test/Transforms/HotColdSplit/assumption-cache-invalidation.ll
@@ -0,0 +1,38 @@
+; RUN: opt -instcombine -hotcoldsplit -instsimplify %s -o /dev/null
+
+target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+target triple = "aarch64"
+
+%a = type { i64, i64 }
+%b = type { i64 }
+
+define void @f() {
+entry:
+  %0 = getelementptr inbounds %a, %a* null, i64 0, i32 1
+  br label %label
+
+label:
+  %1 = bitcast i64* %0 to %b**
+  %2 = load %b*, %b** %1, align 8
+  %3 = getelementptr inbounds %b, %b* %2, i64 undef, i32 0
+  %4 = load i64, i64* %3, align 8
+  %5 = icmp ugt i64 %4, 1
+  br i1 %5, label %if.then, label %if.else
+
+if.then:
+  unreachable
+
+if.else:
+  call void @g(i8* undef)
+  %6 = load i64, i64* undef, align 8
+  %7 = and i64 %6, -16
+  %8 = inttoptr i64 %7 to i8*
+  %9 = icmp eq i64 %4, 0
+  call void @llvm.assume(i1 %9)
+  unreachable
+}
+
+declare void @g(i8*)
+
+declare void @llvm.assume(i1)
+
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
new file mode 100644
index 0000000000000..06d5f719d1fbd
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value.ll
@@ -0,0 +1,92 @@
+; RUN: opt %s -indvars -verify -S -o - | FileCheck %s
+
+; Hand-reduced from this example:
+;
+; #include <stdio.h>
+
+; int main( int argc, char **argv )
+; {
+;    for( int ArgIndex = 1; ArgIndex < argc; ArgIndex += 1 )
+;    {
+;       printf("\n Argument %d:  %s\n", ArgIndex, argv[ArgIndex] );
+;    }
+; }
+
+; clang++ -g -O -mllvm -disable-llvm-optzns -gno-column-info
+; opt  -mem2reg -loop-rotate -scalar-evolution
+
+; CHECK: @main
+; CHECK: llvm.dbg.value(metadata i32 1, metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK: %[[VAR_NAME:.*]] = add nuw nsw i64
+; CHECK: llvm.dbg.value(metadata i64 %[[VAR_NAME]], metadata [[METADATA_IDX1]], metadata !DIExpression())
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "ArgIndex"
+
+source_filename = "test.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [20 x i8] c"\0A Argument %d:  %s\0A\00", align 1
+
+define dso_local i32 @main(i32 %argc, i8** %argv) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 1, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp1 = icmp slt i32 1, %argc, !dbg !19
+  br i1 %cmp1, label %for.body.lr.ph, label %for.cond.cleanup, !dbg !19
+
+for.body.lr.ph:                                   ; preds = %entry
+  br label %for.body, !dbg !19
+
+for.cond.for.cond.cleanup_crit_edge:              ; preds = %for.inc
+  br label %for.cond.cleanup, !dbg !19
+
+for.cond.cleanup:                                 ; preds = %for.cond.for.cond.cleanup_crit_edge, %entry
+  br label %for.end
+
+for.body:                                         ; preds = %for.body.lr.ph, %for.inc
+  %ArgIndex.02 = phi i32 [ 1, %for.body.lr.ph ], [ %add, %for.inc ]
+  call void @llvm.dbg.value(metadata i32 %ArgIndex.02, metadata !17, metadata !DIExpression()), !dbg !19
+  %idxprom = sext i32 %ArgIndex.02 to i64, !dbg !19
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 %idxprom, !dbg !19
+  %0 = load i8*, i8** %arrayidx, align 8, !dbg !19
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([20 x i8], [20 x i8]* @.str, i64 0, i64 0), i32 %ArgIndex.02, i8* %0), !dbg !19
+  br label %for.inc, !dbg !19
+
+for.inc:                                          ; preds = %for.body
+  %add = add nsw i32 %ArgIndex.02, 1, !dbg !19
+  call void @llvm.dbg.value(metadata i32 %add, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp = icmp slt i32 %add, %argc, !dbg !19
+  br i1 %cmp, label %for.body, label %for.cond.for.cond.cleanup_crit_edge, !dbg !19, !llvm.loop !19
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret i32 0, !dbg !19
+}
+
+declare dso_local i32 @printf(i8*, ...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!14 = !{!15, !16, !17}
+!15 = !DILocalVariable(name: "argc", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!16 = !DILocalVariable(name: "argv", arg: 2, scope: !7, file: !1, line: 4, type: !11)
+!17 = !DILocalVariable(name: "ArgIndex", scope: !18, file: !1, line: 6, type: !10)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 6)
+!19 = !DILocation(line: 0, scope: !7)
diff --git a/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
new file mode 100644
index 0000000000000..3859111eb2a9b
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/indvar-debug-value2.ll
@@ -0,0 +1,90 @@
+; RUN: opt %s -indvars -verify -S -o - | FileCheck %s
+
+; Hand-reduced from this example:
+;
+; #include <stdio.h>
+
+; int main( int argc, char **argv )
+; {
+;    for( int ArgIndex = 1; ArgIndex < argc; ArgIndex += 1 )
+;    {
+;       printf("\n Argument %d:  %s\n", ArgIndex, argv[ArgIndex] );
+;    }
+; }
+
+; clang++ -g -O -mllvm -disable-llvm-optzns -gno-column-info
+; opt  -mem2reg -scalar-evolution
+
+; CHECK: @main
+; CHECK: llvm.dbg.value(metadata i32 1, metadata [[METADATA_IDX1:![0-9]+]]
+; CHECK: %[[VAR_NAME:.*]] = add nuw nsw i64
+; CHECK: llvm.dbg.value(metadata i64 %[[VAR_NAME]], metadata [[METADATA_IDX1]], metadata !DIExpression())
+; CHECK: DICompileUnit
+; CHECK: [[METADATA_IDX1]] = !DILocalVariable(name: "ArgIndex"
+
+source_filename = "test.cpp"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@.str = private unnamed_addr constant [20 x i8] c"\0A Argument %d:  %s\0A\00", align 1
+
+define dso_local i32 @main(i32 %argc, i8** %argv) !dbg !7 {
+entry:
+  call void @llvm.dbg.value(metadata i32 %argc, metadata !15, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i8** %argv, metadata !16, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 1, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %for.cond, !dbg !19
+
+for.cond:                                         ; preds = %for.inc, %entry
+  %ArgIndex.0 = phi i32 [ 1, %entry ], [ %add, %for.inc ], !dbg !19
+  call void @llvm.dbg.value(metadata i32 %ArgIndex.0, metadata !17, metadata !DIExpression()), !dbg !19
+  %cmp = icmp slt i32 %ArgIndex.0, %argc, !dbg !19
+  br i1 %cmp, label %for.body, label %for.cond.cleanup, !dbg !19
+
+for.cond.cleanup:                                 ; preds = %for.cond
+  br label %for.end
+
+for.body:                                         ; preds = %for.cond
+  %idxprom = sext i32 %ArgIndex.0 to i64, !dbg !19
+  %arrayidx = getelementptr inbounds i8*, i8** %argv, i64 %idxprom, !dbg !19
+  %0 = load i8*, i8** %arrayidx, align 8, !dbg !19
+  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([20 x i8], [20 x i8]* @.str, i64 0, i64 0), i32 %ArgIndex.0, i8* %0), !dbg !19
+  br label %for.inc, !dbg !19
+
+for.inc:                                          ; preds = %for.body
+  %add = add nsw i32 %ArgIndex.0, 1, !dbg !19
+  call void @llvm.dbg.value(metadata i32 %add, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %for.cond, !dbg !19, !llvm.loop !19
+
+for.end:                                          ; preds = %for.cond.cleanup
+  ret i32 0, !dbg !19
+}
+
+declare dso_local i32 @printf(i8*, ...)
+
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, file: !1, producer: "clang version", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang"}
+!7 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 4, type: !8, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !10, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !13, size: 64)
+!13 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!14 = !{!15, !16, !17}
+!15 = !DILocalVariable(name: "argc", arg: 1, scope: !7, file: !1, line: 4, type: !10)
+!16 = !DILocalVariable(name: "argv", arg: 2, scope: !7, file: !1, line: 4, type: !11)
+!17 = !DILocalVariable(name: "ArgIndex", scope: !18, file: !1, line: 6, type: !10)
+!18 = distinct !DILexicalBlock(scope: !7, file: !1, line: 6)
+!19 = !DILocation(line: 0, scope: !7)
diff --git a/llvm/test/Transforms/IndVarSimplify/loop-predication.ll b/llvm/test/Transforms/IndVarSimplify/loop-predication.ll
new file mode 100644
index 0000000000000..c5fe11a34460c
--- /dev/null
+++ b/llvm/test/Transforms/IndVarSimplify/loop-predication.ll
@@ -0,0 +1,790 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -indvars -indvars-predicate-loops=1 -S | FileCheck %s
+
+declare void @prevent_merging()
+
+; Base case
+define i32 @test1(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Has side effect which must be reflected
+define i32 @neg_store(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @neg_store(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    store i32 0, i32* [[ARRAY_I_PTR]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  store i32 0, i32* %array.i.ptr
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+declare void @maythrow()
+
+; May exit through implicit exception edge
+define i32 @neg_implicit_exit(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @neg_implicit_exit(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    call void @maythrow()
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N:%.*]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  call void @maythrow()
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+
+
+; Base case, but in LFTR form (just for sanity checking)
+define i32 @test2(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[N:%.*]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP0]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP1]], i32 [[LENGTH]], i32 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP2]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ne i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ne i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ne i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; br (and rcheck1, rcheck2)
+define i32 @two_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32 %n) {
+; CHECK-LABEL: @two_range_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[LENGTH_2]], [[LENGTH_1]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP1]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP2]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i32 [[UMIN1]], [[TMP3]]
+; CHECK-NEXT:    [[UMIN2:%.*]] = select i1 [[TMP4]], i32 [[UMIN1]], i32 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[UMIN]], [[UMIN2]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds = and i1 %within.bounds.1, %within.bounds.2
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.2.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+define i32 @three_range_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @three_range_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_3:%.*]], [[LENGTH_2:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_3]], i32 [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[UMIN]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP1]], i32 [[UMIN]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH_3]], [[LENGTH_2]]
+; CHECK-NEXT:    [[UMIN2:%.*]] = select i1 [[TMP2]], i32 [[LENGTH_3]], i32 [[LENGTH_2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[UMIN2]], [[LENGTH_1]]
+; CHECK-NEXT:    [[UMIN3:%.*]] = select i1 [[TMP3]], i32 [[UMIN2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP4]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp ult i32 [[UMIN3]], [[TMP5]]
+; CHECK-NEXT:    [[UMIN4:%.*]] = select i1 [[TMP6]], i32 [[UMIN3]], i32 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp ne i32 [[UMIN1]], [[UMIN4]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP7]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    [[ARRAY_2_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_2:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_2_I:%.*]] = load i32, i32* [[ARRAY_2_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_2:%.*]] = add i32 [[LOOP_ACC_1]], [[ARRAY_2_I]]
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_2]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  %within.bounds.3 = icmp ult i32 %i, %length.3
+  %within.bounds.1.and.2 = and i1 %within.bounds.1, %within.bounds.2
+  %within.bounds = and i1 %within.bounds.1.and.2, %within.bounds.3
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %array.2.i.ptr = getelementptr inbounds i32, i32* %array.2, i64 %i.i64
+  %array.2.i = load i32, i32* %array.2.i.ptr, align 4
+  %loop.acc.2 = add i32 %loop.acc.1, %array.2.i
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.2, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded, %entry
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Analogous to the above, but with two distinct branches (on different conditions)
+define i32 @distinct_checks(i32* %array.1, i32 %length.1, i32* %array.2, i32 %length.2, i32* %array.3, i32 %length.3, i32 %n) {
+; CHECK-LABEL: @distinct_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[LENGTH_2:%.*]], [[LENGTH_1:%.*]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP0]], i32 [[LENGTH_2]], i32 [[LENGTH_1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP1]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[UMIN]], [[TMP2]]
+; CHECK-NEXT:    [[UMIN1:%.*]] = select i1 [[TMP3]], i32 [[UMIN]], i32 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[LENGTH_1]], [[UMIN1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[LENGTH_2]], [[UMIN1]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED1:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED1]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED1]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED1]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded4, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded1 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded1 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length.1
+  br i1 %within.bounds.1, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length.2
+  br i1 %within.bounds.2, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded1:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded1 ]
+  ret i32 %result
+}
+
+define i32 @duplicate_checks(i32* %array.1, i32* %array.2, i32* %array.3, i32 %length, i32 %n) {
+; CHECK-LABEL: @duplicate_checks(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i32 [[N:%.*]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i32 [[N]], i32 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[LENGTH:%.*]], [[TMP1]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP2]], i32 [[LENGTH]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne i32 [[LENGTH]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED1:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED1]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP3]], label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_1_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_1:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_1_I:%.*]] = load i32, i32* [[ARRAY_1_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_1:%.*]] = add i32 [[LOOP_ACC]], [[ARRAY_1_I]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[GUARDED1]], label [[DEOPT2:%.*]], !prof !0
+; CHECK:       deopt2:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded1:
+; CHECK-NEXT:    [[ARRAY_3_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY_3:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_3_I:%.*]] = load i32, i32* [[ARRAY_3_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC_1]], [[ARRAY_3_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw i32 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i32 [[I_NEXT]], [[N]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED1]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:                                   ; preds = %entry
+  br label %loop
+
+loop:                                             ; preds = %guarded4, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded1 ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded1 ], [ 0, %loop.preheader ]
+  %within.bounds.1 = icmp ult i32 %i, %length
+  br i1 %within.bounds.1, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.1.i.ptr = getelementptr inbounds i32, i32* %array.1, i64 %i.i64
+  %array.1.i = load i32, i32* %array.1.i.ptr, align 4
+  %loop.acc.1 = add i32 %loop.acc, %array.1.i
+  %within.bounds.2 = icmp ult i32 %i, %length
+  br i1 %within.bounds.2, label %guarded1, label %deopt2, !prof !0
+
+deopt2:                                           ; preds = %guarded
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded1:                                         ; preds = %guarded1
+  %array.3.i.ptr = getelementptr inbounds i32, i32* %array.3, i64 %i.i64
+  %array.3.i = load i32, i32* %array.3.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc.1, %array.3.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp ult i32 %i.next, %n
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded1 ]
+  ret i32 %result
+}
+
+
+define i32 @provably_taken(i32* %array, i32* %length.ptr) {
+; CHECK-LABEL: @provably_taken(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 false, label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[I_I64:%.*]] = zext i32 [[I]] to i64
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I_I64]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i32 [[I]], 1
+; CHECK-NEXT:    br i1 true, label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:
+  %length = load i32, i32* %length.ptr, !range !2
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i32 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.i64 = zext i32 %i to i64
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i.i64
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i32 %i, 1
+  %continue = icmp slt i32 %i.next, 200
+  br i1 %continue, label %loop, label %exit
+
+exit:                                             ; preds = %guarded
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; Non-latch exits can still be predicated
+define i32 @unconditional_latch(i32* %a, i32 %length) {
+; CHECK-LABEL: @unconditional_latch(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    br i1 false, label [[GUARDED:%.*]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    br label [[LOOP]]
+;
+loop.preheader:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %i = phi i32 [ %i.next, %guarded ], [ 400, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  %i.next = add i32 %i, 1
+  br label %loop
+}
+
+; Side effect in loop must run proper number of times
+define i32 @unconditional_latch_with_side_effect(i32* %a, i32 %length) {
+; CHECK-LABEL: @unconditional_latch_with_side_effect(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[I_NEXT:%.*]], [[GUARDED:%.*]] ], [ 400, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[I]], [[LENGTH:%.*]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    store volatile i32 0, i32* [[A:%.*]]
+; CHECK-NEXT:    [[I_NEXT]] = add i32 [[I]], 1
+; CHECK-NEXT:    br label [[LOOP]]
+;
+loop.preheader:
+  br label %loop
+
+loop:                                             ; preds = %guarded, %loop.preheader
+  %i = phi i32 [ %i.next, %guarded ], [ 400, %loop.preheader ]
+  %within.bounds = icmp ult i32 %i, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:                                            ; preds = %loop
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:                                          ; preds = %loop
+  store volatile i32 0, i32* %a
+  %i.next = add i32 %i, 1
+  br label %loop
+}
+
+; Demonstrate that this approach works with IVs of different steps, and types
+; This version uses a manually lftred exit condition to work around an issue described
+; in detail on next test.
+define i32 @different_ivs(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @different_ivs(
+; CHECK-NEXT:  loop.preheader:
+; CHECK-NEXT:    [[N64:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP0:%.*]] = icmp ugt i64 [[N64]], 1
+; CHECK-NEXT:    [[UMAX:%.*]] = select i1 [[TMP0]], i64 [[N64]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = add nsw i64 [[UMAX]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i32 [[LENGTH:%.*]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    [[UMIN:%.*]] = select i1 [[TMP3]], i64 [[TMP1]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = zext i32 [[LENGTH]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], [[UMIN]]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i64 [[I_NEXT]], [[N64]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT:%.*]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+loop.preheader:
+  %j.start = sub nuw nsw i32 %length, 1
+  %n64 = zext i32 %n to i64
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i64 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %j.start, %loop.preheader ]
+  %within.bounds = icmp ne i32 %j, -1
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i64 %i, 1
+  %j.next = sub nuw i32 %j, 1
+  %continue = icmp ult i64 %i.next, %n64
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ]
+  ret i32 %result
+}
+
+; TODO: We're failing to compute an exit count for the bounds check.
+; From some quick analysis, it looks like we don't handle -1 step
+; in howManyLessThans.  Should be a simple fix.
+define i32 @different_ivs2(i32* %array, i32 %length, i32 %n) {
+; CHECK-LABEL: @different_ivs2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[POS_LENGTH:%.*]] = icmp sgt i32 [[LENGTH:%.*]], 0
+; CHECK-NEXT:    br i1 [[POS_LENGTH]], label [[LOOP_PREHEADER:%.*]], label [[EXIT:%.*]]
+; CHECK:       loop.preheader:
+; CHECK-NEXT:    [[J_START:%.*]] = sub nuw nsw i32 [[LENGTH]], 1
+; CHECK-NEXT:    [[N64:%.*]] = zext i32 [[N:%.*]] to i64
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[LOOP_ACC:%.*]] = phi i32 [ [[LOOP_ACC_NEXT:%.*]], [[GUARDED:%.*]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[I:%.*]] = phi i64 [ [[I_NEXT:%.*]], [[GUARDED]] ], [ 0, [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[J:%.*]] = phi i32 [ [[J_NEXT:%.*]], [[GUARDED]] ], [ [[J_START]], [[LOOP_PREHEADER]] ]
+; CHECK-NEXT:    [[WITHIN_BOUNDS:%.*]] = icmp ult i32 [[J]], [[LENGTH]]
+; CHECK-NEXT:    br i1 [[WITHIN_BOUNDS]], label [[GUARDED]], label [[DEOPT:%.*]], !prof !0
+; CHECK:       deopt:
+; CHECK-NEXT:    call void @prevent_merging()
+; CHECK-NEXT:    ret i32 -1
+; CHECK:       guarded:
+; CHECK-NEXT:    [[ARRAY_I_PTR:%.*]] = getelementptr inbounds i32, i32* [[ARRAY:%.*]], i64 [[I]]
+; CHECK-NEXT:    [[ARRAY_I:%.*]] = load i32, i32* [[ARRAY_I_PTR]], align 4
+; CHECK-NEXT:    [[LOOP_ACC_NEXT]] = add i32 [[LOOP_ACC]], [[ARRAY_I]]
+; CHECK-NEXT:    [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT:    [[J_NEXT]] = sub nuw i32 [[J]], 1
+; CHECK-NEXT:    [[CONTINUE:%.*]] = icmp ult i64 [[I_NEXT]], [[N64]]
+; CHECK-NEXT:    br i1 [[CONTINUE]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    [[LOOP_ACC_NEXT_LCSSA:%.*]] = phi i32 [ [[LOOP_ACC_NEXT]], [[GUARDED]] ]
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[RESULT:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[LOOP_ACC_NEXT_LCSSA]], [[EXIT_LOOPEXIT]] ]
+; CHECK-NEXT:    ret i32 [[RESULT]]
+;
+entry:
+  %pos_length = icmp sgt i32 %length, 0
+  br i1 %pos_length, label %loop.preheader, label %exit
+
+loop.preheader:
+  %j.start = sub nuw nsw i32 %length, 1
+  %n64 = zext i32 %n to i64
+  br label %loop
+
+loop:
+  %loop.acc = phi i32 [ %loop.acc.next, %guarded ], [ 0, %loop.preheader ]
+  %i = phi i64 [ %i.next, %guarded ], [ 0, %loop.preheader ]
+  %j = phi i32 [ %j.next, %guarded ], [ %j.start, %loop.preheader ]
+  %within.bounds = icmp ult i32 %j, %length
+  br i1 %within.bounds, label %guarded, label %deopt, !prof !0
+
+deopt:
+  call void @prevent_merging()
+  ret i32 -1
+
+guarded:
+  %array.i.ptr = getelementptr inbounds i32, i32* %array, i64 %i
+  %array.i = load i32, i32* %array.i.ptr, align 4
+  %loop.acc.next = add i32 %loop.acc, %array.i
+  %i.next = add nuw i64 %i, 1
+  %j.next = sub nuw i32 %j, 1
+  %continue = icmp ult i64 %i.next, %n64
+  br i1 %continue, label %loop, label %exit
+
+exit:
+  %result = phi i32 [ %loop.acc.next, %guarded ], [0, %entry]
+  ret i32 %result
+}
+
+
+
+declare i32 @llvm.experimental.deoptimize.i32(...)
+
+!0 = !{!"branch_weights", i32 1048576, i32 1}
+!1 = !{i32 1, i32 -2147483648}
+!2 = !{i32 0, i32 50}
diff --git a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
index a1e1d0131248a..6edacc17518b9 100644
--- a/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
+++ b/llvm/test/Transforms/IndVarSimplify/loop_evaluate_1.ll
@@ -28,14 +28,10 @@ define i32 @test2(i32 %arg) {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:  bb:
 ; CHECK-NEXT:    [[TMP:%.*]] = icmp ugt i32 [[ARG:%.*]], 10
-; CHECK-NEXT:    br i1 [[TMP]], label [[BB1_PREHEADER:%.*]], label [[BB7:%.*]]
-; CHECK:       bb1.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i32 [[ARG]], -11
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw i32 [[TMP1]], 1
-; CHECK-NEXT:    br label [[BB7]]
-; CHECK:       bb7:
-; CHECK-NEXT:    [[TMP8:%.*]] = phi i32 [ 0, [[BB:%.*]] ], [ [[TMP2]], [[BB1_PREHEADER]] ]
+; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP]], i32 [[TMP2]], i32 0
 ; CHECK-NEXT:    ret i32 [[TMP8]]
 ;
 bb:
diff --git a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
index f3b0cd4364045..8054802ff0219 100644
--- a/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
+++ b/llvm/test/Transforms/InferFunctionAttrs/annotate.ll
@@ -619,7 +619,7 @@ declare i8* @malloc(i64)
 ; CHECK-LINUX: declare noalias i8* @memalign(i64, i64) [[G0]]
 declare i8* @memalign(i64, i64)
 
-; CHECK: declare i8* @memccpy(i8*, i8* nocapture readonly, i32, i64) [[G1]]
+; CHECK: declare i8* @memccpy(i8* noalias, i8* noalias nocapture readonly, i32, i64) [[G1]]
 declare i8* @memccpy(i8*, i8*, i32, i64)
 
 ; CHECK: declare i8* @memchr(i8*, i32, i64) [[G2]]
@@ -631,7 +631,7 @@ declare i32 @memcmp(i8*, i8*, i64)
 ; CHECK: declare i8* @memcpy(i8* noalias returned, i8* noalias nocapture readonly, i64) [[G1]]
 declare i8* @memcpy(i8*, i8*, i64)
 
-; CHECK: declare i8* @mempcpy(i8*, i8* nocapture readonly, i64) [[G1]]
+; CHECK: declare i8* @mempcpy(i8* noalias, i8* noalias nocapture readonly, i64) [[G1]]
 declare i8* @mempcpy(i8*, i8*, i64)
 
 ; CHECK: declare i8* @memmove(i8* returned, i8* nocapture readonly, i64) [[G1]]
diff --git a/llvm/test/Transforms/Inline/X86/switch.ll b/llvm/test/Transforms/Inline/X86/switch.ll
new file mode 100644
index 0000000000000..5565c7aba5036
--- /dev/null
+++ b/llvm/test/Transforms/Inline/X86/switch.ll
@@ -0,0 +1,160 @@
+; RUN: opt < %s -inline -inline-threshold=1 -S -mtriple=x86_64-unknown-linux-gnu  | FileCheck %s
+; RUN: opt < %s -passes='cgscc(inline)' -inline-threshold=1 -S -mtriple=x86_64-unknown-linux-gnu | FileCheck %s
+
+define i32 @callee_range(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1000, label %sw.bb1
+    i32 2000, label %sw.bb1
+    i32 3000, label %sw.bb1
+    i32 4000, label %sw.bb1
+    i32 5000, label %sw.bb1
+    i32 6000, label %sw.bb1
+    i32 7000, label %sw.bb1
+    i32 8000, label %sw.bb1
+    i32 9000, label %sw.bb1
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+return:
+  ret i32 42
+}
+
+define i32 @caller_range(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_range(
+; CHECK: call i32 @callee_range
+  %r = call i32 @callee_range(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_bittest(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 0, label %sw.bb0
+    i32 1, label %sw.bb1
+    i32 2, label %sw.bb2
+    i32 3, label %sw.bb0
+    i32 4, label %sw.bb1
+    i32 5, label %sw.bb2
+    i32 6, label %sw.bb0
+    i32 7, label %sw.bb1
+    i32 8, label %sw.bb2
+  ]
+
+sw.default:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb0:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb1:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb2:
+  br label %return
+
+return:
+  ret i32 42
+}
+
+
+define i32 @caller_bittest(i32 %a, i32* %P) {
+; CHECK-LABEL: @caller_bittest(
+; CHECK-NOT: call i32 @callee_bittest
+  %r= call i32 @callee_bittest(i32 %a, i32* %P)
+  ret i32 %r
+}
+
+define i32 @callee_jumptable(i32 %a, i32* %P) {
+  switch i32 %a, label %sw.default [
+    i32 1001, label %sw.bb101
+    i32 1002, label %sw.bb102
+    i32 1003, label %sw.bb103
+    i32 1004, label %sw.bb104
+    i32 1005, label %sw.bb101
+    i32 1006, label %sw.bb102
+    i32 1007, label %sw.bb103
+    i32 1008, label %sw.bb104
+    i32 1009, label %sw.bb101
+    i32 1010, label %sw.bb102
+    i32 1011, label %sw.bb103
+    i32 1012, label %sw.bb104
+ ]
+
+sw.default:
+  br label %return
+
+sw.bb101:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb102:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb103:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+sw.bb104:
+  store volatile i32 %a, i32* %P
+  br label %return
+
+return:
+  ret i32 42
+}
+
+define i32 @caller_jumptable(i32 %a, i32 %b, i32* %P) {
+; CHECK-LABEL: @caller_jumptable(
+; CHECK: call i32 @callee_jumptable
+  %r = call i32 @callee_jumptable(i32 %b, i32* %P)
+  ret i32 %r
+}
+
+
+define internal i32 @callee_negativeCost(i32 %t)  {
+entry:
+  switch i32 %t, label %sw.default [
+    i32 1, label %sw.bb
+    i32 0, label %sw.bb1
+    i32 42, label %sw.bb2
+    i32 43, label %sw.bb3
+  ]
+
+sw.bb:                                            ; preds = %entry
+  br label %cleanup
+
+sw.bb1:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb2:                                           ; preds = %entry
+  br label %cleanup
+
+sw.bb3:                                           ; preds = %entry
+  br label %cleanup
+
+sw.default:                                       ; preds = %entry
+  br label %cleanup
+
+cleanup:                                          ; preds = %sw.default, %sw.bb3, %sw.bb2, %sw.bb1, %sw.bb
+  %retval.0 = phi i32 [ 1, %sw.default ], [ 3, %sw.bb3 ], [ 2, %sw.bb2 ], [ 0, %sw.bb1 ], [ 0, %sw.bb ]
+  ret i32 %retval.0
+}
+
+define i32 @caller_negativeCost(i32 %t) {
+; CHECK-LABEL: @caller_negativeCost(
+; CHECK-NOT: call i32 @callee_negativeCost
+entry:
+  %call = call i32 @callee_negativeCost(i32 %t)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll b/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll
deleted file mode 100644
index 666b02e8ed09d..0000000000000
--- a/llvm/test/Transforms/InstCombine/2009-01-31-Pressure.ll
+++ /dev/null
@@ -1,22 +0,0 @@
-; RUN: opt < %s -instcombine -S | grep "%B = add i8 %b, %x"
-; PR2698
-
-declare void @use1(i1)
-declare void @use8(i8)
-
-define void @test1(i8 %a, i8 %b, i8 %x) {
-  %A = add i8 %a, %x
-  %B = add i8 %b, %x
-  %C = icmp eq i8 %A, %B
-  call void @use1(i1 %C)
-  ret void
-}
-
-define void @test2(i8 %a, i8 %b, i8 %x) {
-  %A = add i8 %a, %x
-  %B = add i8 %b, %x
-  %C = icmp eq i8 %A, %B
-  call void @use1(i1 %C)
-  call void @use8(i8 %A)
-  ret void
-}
diff --git a/llvm/test/Transforms/InstCombine/ARM/strcmp.ll b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
index 0a07d0a8ac8fe..a7f3d21a065d1 100644
--- a/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
+++ b/llvm/test/Transforms/InstCombine/ARM/strcmp.ll
@@ -67,7 +67,7 @@ define arm_aapcscc i32 @test4() {
 define arm_aapcscc i32 @test5(i1 %b) {
 ; CHECK-LABEL: @test5(
 ; CHECK-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; CHECK-NEXT:    ret i32 [[MEMCMP]]
 ;
 
@@ -145,7 +145,7 @@ define arm_aapcs_vfpcc i32 @test4_vfp() {
 define arm_aapcs_vfpcc i32 @test5_vfp(i1 %b) {
 ; CHECK-LABEL: @test5_vfp(
 ; CHECK-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; CHECK-NEXT:    ret i32 [[MEMCMP]]
 ;
 
diff --git a/llvm/test/Transforms/InstCombine/align-addr.ll b/llvm/test/Transforms/InstCombine/align-addr.ll
index 80449918b9549..d6dc5e91da45e 100644
--- a/llvm/test/Transforms/InstCombine/align-addr.ll
+++ b/llvm/test/Transforms/InstCombine/align-addr.ll
@@ -114,7 +114,7 @@ define void @test3(%struct.s* sret %a4) {
 ; Check that the alignment is bumped up the alignment of the sret type.
 ; CHECK-LABEL: @test3(
 ; CHECK-NEXT:    [[A4_CAST:%.*]] = bitcast %struct.s* [[A4:%.*]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 dereferenceable(16) [[A4_CAST]], i8 0, i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(16) [[A4_CAST]], i8 0, i64 16, i1 false)
 ; CHECK-NEXT:    call void @use(i8* [[A4_CAST]])
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/InstCombine/bcopy.ll b/llvm/test/Transforms/InstCombine/bcopy.ll
new file mode 100644
index 0000000000000..6a53bad7eeb09
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bcopy.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+declare void @bcopy(i8* nocapture readonly, i8* nocapture, i32)
+
+define void @bcopy_memmove(i8* nocapture readonly %a, i8* nocapture %b) {
+; CHECK-LABEL: @bcopy_memmove(
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[A:%.*]] to i64*
+; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8* [[B:%.*]] to i64*
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]], align 1
+; CHECK-NEXT:    store i64 [[TMP3]], i64* [[TMP2]], align 1
+; CHECK-NEXT:    ret void
+;
+  tail call void @bcopy(i8* %a, i8* %b, i32 8)
+  ret void
+}
+
+define void @bcopy_memmove2(i8* nocapture readonly %a, i8* nocapture %b, i32 %len) {
+; CHECK-LABEL: @bcopy_memmove2(
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* align 1 [[B:%.*]], i8* align 1 [[A:%.*]], i32 [[LEN:%.*]], i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @bcopy(i8* %a, i8* %b, i32 %len)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
index f188510dfdf02..efaf1cd1604d5 100644
--- a/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
+++ b/llvm/test/Transforms/InstCombine/deref-alloc-fns.ll
@@ -6,6 +6,9 @@ declare noalias i8* @calloc(i64, i64)
 declare noalias i8* @realloc(i8* nocapture, i64)
 declare noalias nonnull i8* @_Znam(i64) ; throwing version of 'new'
 declare noalias nonnull i8* @_Znwm(i64) ; throwing version of 'new'
+declare noalias i8* @strdup(i8*)
+
+@.str = private unnamed_addr constant [6 x i8] c"hello\00", align 1
 
 define noalias i8* @malloc_nonconstant_size(i64 %n) {
 ; CHECK-LABEL: @malloc_nonconstant_size(
@@ -206,3 +209,21 @@ define noalias i8* @op_new_constant_zero_size() {
   %call = tail call i8* @_Znam(i64 0)
   ret i8* %call
 }
+
+define noalias i8* @strdup_constant_str() {
+; CHECK-LABEL: @strdup_constant_str(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str, i64 0, i64 0))
+  ret i8* %call
+}
+
+define noalias i8* @strdup_notconstant_str(i8 * %str) {
+; CHECK-LABEL: @strdup_notconstant_str(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call noalias i8* @strdup(i8* [[STR:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = tail call noalias i8* @strdup(i8* %str)
+  ret i8* %call
+}
diff --git a/llvm/test/Transforms/InstCombine/fma.ll b/llvm/test/Transforms/InstCombine/fma.ll
index 89fdc6bc9a5a8..f123f5d56b807 100644
--- a/llvm/test/Transforms/InstCombine/fma.ll
+++ b/llvm/test/Transforms/InstCombine/fma.ll
@@ -182,7 +182,7 @@ define float @fmuladd_unary_fneg_x_unary_fneg_y(float %x, float %y, float %z) {
 
 define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_fneg_x_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -194,7 +194,7 @@ define float @fmuladd_fneg_x_fneg_y_fast(float %x, float %y, float %z) {
 
 define float @fmuladd_unary_fneg_x_unary_fneg_y_fast(float %x, float %y, float %z) {
 ; CHECK-LABEL: @fmuladd_unary_fneg_x_unary_fneg_y_fast(
-; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fmul fast float [[Y:%.*]], [[X:%.*]]
 ; CHECK-NEXT:    [[FMULADD:%.*]] = fadd fast float [[TMP1]], [[Z:%.*]]
 ; CHECK-NEXT:    ret float [[FMULADD]]
 ;
@@ -372,8 +372,7 @@ define float @fmuladd_x_1_z_fast(float %x, float %z) {
 define <2 x double> @fmuladd_a_0_b(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @fmuladd_a_0_b(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
 ;
 entry:
   %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %a, <2 x double> zeroinitializer, <2 x double> %b)
@@ -383,8 +382,7 @@ entry:
 define <2 x double> @fmuladd_0_a_b(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @fmuladd_0_a_b(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
 ;
 entry:
   %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> zeroinitializer, <2 x double> %a, <2 x double> %b)
@@ -407,8 +405,7 @@ declare <2 x double> @llvm.fmuladd.v2f64(<2 x double>, <2 x double>, <2 x double
 define <2 x double> @fma_a_0_b(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @fma_a_0_b(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
 ;
 entry:
   %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %a, <2 x double> zeroinitializer, <2 x double> %b)
@@ -418,8 +415,7 @@ entry:
 define <2 x double> @fma_0_a_b(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @fma_0_a_b(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[A:%.*]], <2 x double> zeroinitializer, <2 x double> [[B:%.*]])
-; CHECK-NEXT:    ret <2 x double> [[RES]]
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
 ;
 entry:
   %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> zeroinitializer, <2 x double> %a, <2 x double> %b)
@@ -440,8 +436,7 @@ entry:
 define <2 x double> @fma_sqrt(<2 x double> %a, <2 x double> %b) {
 ; CHECK-LABEL: @fma_sqrt(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[SQRT:%.*]] = call fast <2 x double> @llvm.sqrt.v2f64(<2 x double> [[A:%.*]])
-; CHECK-NEXT:    [[RES:%.*]] = call fast <2 x double> @llvm.fma.v2f64(<2 x double> [[SQRT]], <2 x double> [[SQRT]], <2 x double> [[B:%.*]])
+; CHECK-NEXT:    [[RES:%.*]] = fadd fast <2 x double> [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    ret <2 x double> [[RES]]
 ;
 entry:
@@ -450,7 +445,273 @@ entry:
   ret <2 x double> %res
 }
 
+; We do not fold constant multiplies in FMAs, as they could require rounding, unless either constant is 0.0 or 1.0.
+define <2 x double> @fma_const_fmul(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x4131233302898702, double 0x40C387800000D6C0>, <2 x double> <double 1.291820e-08, double 9.123000e-06>, <2 x double> [[B:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_zero(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_zero(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0, double 0.0>, <2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_zero2(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_zero2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> [[B:%.*]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0, double 0.0>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_one(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_one(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x4131233302898702, double 0x40C387800000D6C0>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1.0, double 1.0>, <2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_const_fmul_one2(<2 x double> %b) {
+; CHECK-LABEL: @fma_const_fmul_one2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x4131233302898702, double 0x40C387800000D6C0>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 1.0, double 1.0>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_and_const_0(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
 
+define <2 x double> @fma_nan_and_const_1(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_and_const_2(<2 x double> %b) {
+; CHECK-LABEL: @fma_nan_and_const_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double undef, double undef>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double undef, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double undef, double undef>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> <double undef, double 0x4068E00A137F38C5>, <2 x double> [[C:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double undef, double 199.00123>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> <double 0x4068E00A137F38C5, double undef>, <2 x double> [[C:%.*]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double 199.00123, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_partial_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_partial_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> [[B:%.*]], <2 x double> [[C:%.*]], <2 x double> <double 0x4068E00A137F38C5, double undef>)
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 199.00123, double undef>)
+  ret <2 x double> %res
+}
+
+
+define <2 x double> @fma_nan_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+define <2 x double> @fma_nan_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fma_nan_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fma_nan_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fma.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_const_fmul(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_const_fmul(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[RES:%.*]] = fadd nnan nsz <2 x double> [[B:%.*]], <double 0x3F8DB6C076AD949B, double 0x3FB75A405B6E6D69>
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 1123123.0099110012314, double 9999.0000001>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_0(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_1(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_and_const_2(<2 x double> %b) {
+; CHECK-LABEL: @fmuladd_nan_and_const_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0.0000000129182, double 0.000009123>, <2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_nan_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_nan_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_nan_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_0(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_0(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> <double undef, double undef>, <2 x double> %b, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_1(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> <double undef, double undef>, <2 x double> %c)
+  ret <2 x double> %res
+}
+
+define <2 x double> @fmuladd_undef_2(<2 x double> %b, <2 x double> %c) {
+; CHECK-LABEL: @fmuladd_undef_2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    ret <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>
+;
+entry:
+  %res = call nnan nsz <2 x double> @llvm.fmuladd.v2f64(<2 x double> %b, <2 x double> %c, <2 x double> <double 0x7FF8000000000000, double 0x7FF8000000000000>)
+  ret <2 x double> %res
+}
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>)
 declare <2 x double> @llvm.sqrt.v2f64(<2 x double>)
 
diff --git a/llvm/test/Transforms/InstCombine/fortify-folding.ll b/llvm/test/Transforms/InstCombine/fortify-folding.ll
index 68be219ed1ad9..ee81557615a54 100644
--- a/llvm/test/Transforms/InstCombine/fortify-folding.ll
+++ b/llvm/test/Transforms/InstCombine/fortify-folding.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s --dump-input-on-failure
 
 target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
@@ -9,9 +10,10 @@ target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
 %struct.__va_list_tag = type { i32, i32, i8*, i8* }
 
 define i8* @test_memccpy() {
-  ; CHECK-LABEL: define i8* @test_memccpy
-  ; CHECK-NEXT: call i8* @memccpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_memccpy(
+; CHECK-NEXT:    [[MEMCCPY:%.*]] = call i8* @memccpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60)
+; CHECK-NEXT:    ret i8* [[MEMCCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__memccpy_chk(i8* %dst, i8* %src, i32 0, i64 60, i64 -1)
@@ -19,9 +21,10 @@ define i8* @test_memccpy() {
 }
 
 define i8* @test_not_memccpy() {
-  ; CHECK-LABEL: define i8* @test_not_memccpy
-  ; CHECK-NEXT: call i8* @__memccpy_chk
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_memccpy(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__memccpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i32 0, i64 60, i64 59)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__memccpy_chk(i8* %dst, i8* %src, i32 0, i64 60, i64 59)
@@ -29,9 +32,10 @@ define i8* @test_not_memccpy() {
 }
 
 define i32 @test_snprintf() {
-  ; CHECK-LABEL: define i32 @test_snprintf
-  ; CHECK-NEXT: call i32 (i8*, i64, i8*, ...) @snprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_snprintf(
+; CHECK-NEXT:    [[SNPRINTF:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[SNPRINTF]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* %dst, i64 60, i32 0, i64 -1, i8* %fmt)
@@ -39,10 +43,11 @@ define i32 @test_snprintf() {
 }
 
 define i32 @test_not_snprintf() {
-  ; CHECK-LABEL: define i32 @test_not_snprintf
-  ; CHECK-NEXT: call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk
-  ; CHECK-NEXT: call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_not_snprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i32 0, i64 59, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 60, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i64, i32, i64, i8*, ...) @__snprintf_chk(i8* %dst, i64 60, i32 0, i64 59, i8* %fmt)
@@ -51,9 +56,10 @@ define i32 @test_not_snprintf() {
 }
 
 define i32 @test_sprintf() {
-  ; CHECK-LABEL: define i32 @test_sprintf
-  ; CHECK-NEXT: call i32 (i8*, i8*, ...) @sprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_sprintf(
+; CHECK-NEXT:    [[SPRINTF:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[SPRINTF]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* %dst, i32 0, i64 -1, i8* %fmt)
@@ -61,10 +67,11 @@ define i32 @test_sprintf() {
 }
 
 define i32 @test_not_sprintf() {
-  ; CHECK-LABEL: define i32 @test_not_sprintf
-  ; CHECK-NEXT: call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk
-  ; CHECK-NEXT: call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk
-  ; CHECK-NEXT: ret i32
+; CHECK-LABEL: @test_not_sprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 59, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    [[IGNORED:%.*]] = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %fmt = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i32 (i8*, i32, i64, i8*, ...) @__sprintf_chk(i8* %dst, i32 0, i64 59, i8* %fmt)
@@ -73,9 +80,10 @@ define i32 @test_not_sprintf() {
 }
 
 define i8* @test_strcat() {
-  ; CHECK-LABEL: define i8* @test_strcat
-  ; CHECK-NEXT: call i8* @strcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_strcat(
+; CHECK-NEXT:    [[STRCAT:%.*]] = call i8* @strcat(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strcat_chk(i8* %dst, i8* %src, i64 -1)
@@ -83,9 +91,10 @@ define i8* @test_strcat() {
 }
 
 define i8* @test_not_strcat() {
-  ; CHECK-LABEL: define i8* @test_not_strcat
-  ; CHECK-NEXT: call i8* @__strcat_chk
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_strcat(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strcat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strcat_chk(i8* %dst, i8* %src, i64 0)
@@ -93,9 +102,10 @@ define i8* @test_not_strcat() {
 }
 
 define i64 @test_strlcat() {
-  ; CHECK-LABEL: define i64 @test_strlcat
-  ; CHECK-NEXT: call i64 @strlcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_strlcat(
+; CHECK-NEXT:    [[STRLCAT:%.*]] = call i64 @strlcat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i64 [[STRLCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcat_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -103,9 +113,10 @@ define i64 @test_strlcat() {
 }
 
 define i64 @test_not_strlcat() {
-  ; CHECK-LABEL: define i64 @test_not_strlcat
-  ; CHECK-NEXT: call i64 @__strlcat_chk
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_not_strlcat(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @__strlcat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 0)
+; CHECK-NEXT:    ret i64 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcat_chk(i8* %dst, i8* %src, i64 22, i64 0)
@@ -113,9 +124,10 @@ define i64 @test_not_strlcat() {
 }
 
 define i8* @test_strncat() {
-  ; CHECK-LABEL: define i8* @test_strncat
-  ; CHECK-NEXT: call i8* @strncat(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_strncat(
+; CHECK-NEXT:    [[STRNCAT:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i8* [[STRNCAT]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strncat_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -123,9 +135,10 @@ define i8* @test_strncat() {
 }
 
 define i8* @test_not_strncat() {
-  ; CHECK-LABEL: define i8* @test_not_strncat
-  ; CHECK-NEXT: call i8* @__strncat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 3)
-  ; CHECK-NEXT: ret i8*
+; CHECK-LABEL: @test_not_strncat(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @__strncat_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 3)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i8* @__strncat_chk(i8* %dst, i8* %src, i64 22, i64 3)
@@ -133,9 +146,10 @@ define i8* @test_not_strncat() {
 }
 
 define i64 @test_strlcpy() {
-  ; CHECK-LABEL: define i64 @test_strlcpy
-  ; CHECK-NEXT: call i64 @strlcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_strlcpy(
+; CHECK-NEXT:    [[STRLCPY:%.*]] = call i64 @strlcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22)
+; CHECK-NEXT:    ret i64 [[STRLCPY]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcpy_chk(i8* %dst, i8* %src, i64 22, i64 -1)
@@ -143,9 +157,10 @@ define i64 @test_strlcpy() {
 }
 
 define i64 @test_not_strlcpy() {
-  ; CHECK-LABEL: define i64 @test_not_strlcpy
-  ; CHECK-NEXT: call i64 @__strlcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 2)
-  ; CHECK-NEXT: ret i64
+; CHECK-LABEL: @test_not_strlcpy(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @__strlcpy_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), i64 22, i64 2)
+; CHECK-NEXT:    ret i64 [[RET]]
+;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
   %ret = call i64 @__strlcpy_chk(i8* %dst, i8* %src, i64 22, i64 2)
@@ -153,8 +168,10 @@ define i64 @test_not_strlcpy() {
 }
 
 define i32 @test_vsnprintf() {
-  ; CHECK-LABEL: define i32 @test_vsnprintf
-  ; CHECK-NEXT: call i32 @vsnprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_vsnprintf(
+; CHECK-NEXT:    [[VSNPRINTF:%.*]] = call i32 @vsnprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[VSNPRINTF]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -163,9 +180,11 @@ define i32 @test_vsnprintf() {
 }
 
 define i32 @test_not_vsnprintf() {
-  ; CHECK-LABEL: define i32 @test_not_vsnprintf
-  ; CHECK-NEXT: call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
-  ; CHECK-NEXT: call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_not_vsnprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 @__vsnprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i64 4, i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -175,8 +194,10 @@ define i32 @test_not_vsnprintf() {
 }
 
 define i32 @test_vsprintf() {
-  ; CHECK-LABEL: define i32 @test_vsprintf
-  ; CHECK-NEXT: call i32 @vsprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_vsprintf(
+; CHECK-NEXT:    [[VSPRINTF:%.*]] = call i32 @vsprintf(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[VSPRINTF]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
@@ -185,9 +206,11 @@ define i32 @test_vsprintf() {
 }
 
 define i32 @test_not_vsprintf() {
-  ; CHECK-LABEL: define i32 @test_not_vsprintf
-  ; CHECK-NEXT: call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
-  ; CHECK-NEXT: call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-LABEL: @test_not_vsprintf(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 0, i64 3, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    [[IGN:%.*]] = call i32 @__vsprintf_chk(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i64 0, i64 0), i32 1, i64 -1, i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i64 0, i64 0), %struct.__va_list_tag* null)
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   ; ret i32
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
   %src = getelementptr inbounds [60 x i8], [60 x i8]* @b, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/gc.relocate.ll b/llvm/test/Transforms/InstCombine/gc.relocate.ll
index 78b3b5f42f91a..f8eade6f11f53 100644
--- a/llvm/test/Transforms/InstCombine/gc.relocate.ll
+++ b/llvm/test/Transforms/InstCombine/gc.relocate.ll
@@ -52,6 +52,17 @@ entry:
   ret <2 x i8 addrspace(1)*> %obj.relocated
 }
 
+define i32 addrspace(1)* @canonical_base(i32 addrspace(1)* %dparam) gc "statepoint-example" {
+; Checks that a nonnull pointer
+; CHECK-LABEL: @canonical_base
+; CHECK: (token %tok, i32 7, i32 7) ; (%dparam, %dparam)
+entry:
+  %tok = tail call token (i64, i32, i1 ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_i1f(i64 0, i32 0, i1 ()* @return_i1, i32 0, i32 0, i32 0, i32 0, i32 addrspace(1)* %dparam, i32 addrspace(1)* %dparam)
+  %relocate = call i32 addrspace(1)* @llvm.experimental.gc.relocate.p1i32(token %tok,  i32 7, i32 8)
+  ret i32 addrspace(1)* %relocate
+}
+
+
 declare void @do_safepoint()
 
 declare token @llvm.experimental.gc.statepoint.p0f_isVoidf(i64, i32, void ()*, i32, i32, ...)
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index c9a2bac8d781d..52116308c241e 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -491,7 +491,7 @@ bb10:
 	%tmp.0.reg2mem.0.rec = mul i32 %indvar, -1
 	%tmp12.rec = add i32 %tmp.0.reg2mem.0.rec, -1
 	%tmp12 = getelementptr inbounds %struct.x, %struct.x* %tmp45, i32 %tmp12.rec
-	%tmp16 = call i32 (i8*, ...) @printf( i8* getelementptr ([12 x i8], [12 x i8]* @.str1, i32 0, i32 0), %struct.x* %tmp12 ) nounwind
+	%tmp16 = call i32 (i8*, ...) @printf( i8* nonnull dereferenceable(1) getelementptr ([12 x i8], [12 x i8]* @.str1, i32 0, i32 0), %struct.x* %tmp12 ) nounwind
 	%tmp84 = icmp eq %struct.x* %tmp12, %orientations62
 	%indvar.next = add i32 %indvar, 1
 	br i1 %tmp84, label %bb17, label %bb10
@@ -652,7 +652,7 @@ define i32 @test35() nounwind {
              i8* getelementptr (%t1, %t1* bitcast (%t0* @s to %t1*), i32 0, i32 1, i32 0)) nounwind
   ret i32 0
 ; CHECK-LABEL: @test35(
-; CHECK: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([17 x i8], [17 x i8]* @"\01LC8", i64 0, i64 0), i8* getelementptr inbounds (%t0, %t0* @s, i64 0, i32 1, i64 0)) [[$NUW:#[0-9]+]]
+; CHECK: call i32 (i8*, ...) @printf(i8*  nonnull dereferenceable(1) getelementptr inbounds ([17 x i8], [17 x i8]* @"\01LC8", i64 0, i64 0), i8* getelementptr inbounds (%t0, %t0* @s, i64 0, i32 1, i64 0)) [[$NUW:#[0-9]+]]
 }
 
 ; Don't treat signed offsets as unsigned.
diff --git a/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll b/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll
new file mode 100644
index 0000000000000..034c285dd1ff1
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/high-bit-signmask-with-trunc.ll
@@ -0,0 +1,148 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i32 @t0(i64 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t1_exact(i64 %x) {
+; CHECK-LABEL: @t1_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr exact i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t2(i64 %x) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = ashr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+define i32 @t3_exact(i64 %x) {
+; CHECK-LABEL: @t3_exact(
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = ashr exact i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define <2 x i32> @t4(<2 x i64> %x) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[X:%.*]], <i64 63, i64 63>
+; CHECK-NEXT:    [[R:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 63>
+  %t1 = trunc <2 x i64> %t0 to <2 x i32>
+  %r = sub <2 x i32> zeroinitializer, %t1
+  ret <2 x i32> %r
+}
+
+define <2 x i32> @t5(<2 x i64> %x) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i64> [[X:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT:    [[T1:%.*]] = trunc <2 x i64> [[T0]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i32> <i32 0, i32 undef>, [[T1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 undef>
+  %t1 = trunc <2 x i64> %t0 to <2 x i32>
+  %r = sub <2 x i32> <i32 0, i32 undef>, %t1
+  ret <2 x i32> %r
+}
+
+declare void @use64(i64)
+declare void @use32(i32)
+
+define i32 @t6(i64 %x) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    [[R:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n7(i64 %x) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  call void @use32(i32 %t1)
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n8(i64 %x) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %t1 = trunc i64 %t0 to i32
+  call void @use32(i32 %t1)
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n9(i64 %x) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 62
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i32 0, [[T1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 62
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 0, %t1
+  ret i32 %r
+}
+
+define i32 @n10(i64 %x) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[T1:%.*]] = trunc i64 [[T0]] to i32
+; CHECK-NEXT:    [[R:%.*]] = xor i32 [[T1]], 1
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %t1 = trunc i64 %t0 to i32
+  %r = sub i32 1, %t1
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/high-bit-signmask.ll b/llvm/test/Transforms/InstCombine/high-bit-signmask.ll
new file mode 100644
index 0000000000000..4a1b395ca35f9
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/high-bit-signmask.ll
@@ -0,0 +1,126 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+define i64 @t0(i64 %x) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t0_exact(i64 %x) {
+; CHECK-LABEL: @t0_exact(
+; CHECK-NEXT:    [[R:%.*]] = ashr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr exact i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t2(i64 %x) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[R:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = ashr i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+define i64 @t3_exact(i64 %x) {
+; CHECK-LABEL: @t3_exact(
+; CHECK-NEXT:    [[R:%.*]] = lshr exact i64 [[X:%.*]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = ashr exact i64 %x, 63
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define <2 x i64> @t4(<2 x i64> %x) {
+; CHECK-LABEL: @t4(
+; CHECK-NEXT:    [[R:%.*]] = ashr <2 x i64> [[X:%.*]], <i64 63, i64 63>
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 63>
+  %r = sub <2 x i64> zeroinitializer, %t0
+  ret <2 x i64> %r
+}
+
+define <2 x i64> @t5(<2 x i64> %x) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <2 x i64> [[X:%.*]], <i64 63, i64 undef>
+; CHECK-NEXT:    [[R:%.*]] = sub <2 x i64> <i64 0, i64 undef>, [[T0]]
+; CHECK-NEXT:    ret <2 x i64> [[R]]
+;
+  %t0 = lshr <2 x i64> %x, <i64 63, i64 undef>
+  %r = sub <2 x i64> <i64 0, i64 undef>, %t0
+  ret <2 x i64> %r
+}
+
+declare void @use64(i64)
+declare void @use32(i64)
+
+define i64 @t6(i64 %x) {
+; CHECK-LABEL: @t6(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n7(i64 %x) {
+; CHECK-LABEL: @n7(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use32(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use32(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n8(i64 %x) {
+; CHECK-LABEL: @n8(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    call void @use64(i64 [[T0]])
+; CHECK-NEXT:    call void @use32(i64 [[T0]])
+; CHECK-NEXT:    [[R:%.*]] = ashr i64 [[X]], 63
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  call void @use64(i64 %t0)
+  call void @use32(i64 %t0)
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n9(i64 %x) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 62
+; CHECK-NEXT:    [[R:%.*]] = sub nsw i64 0, [[T0]]
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 62
+  %r = sub i64 0, %t0
+  ret i64 %r
+}
+
+define i64 @n10(i64 %x) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i64 [[X:%.*]], 63
+; CHECK-NEXT:    [[R:%.*]] = xor i64 [[T0]], 1
+; CHECK-NEXT:    ret i64 [[R]]
+;
+  %t0 = lshr i64 %x, 63
+  %r = sub i64 1, %t0
+  ret i64 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp-add.ll b/llvm/test/Transforms/InstCombine/icmp-add.ll
index 86d4d7c5735de..1e3875b112c3d 100644
--- a/llvm/test/Transforms/InstCombine/icmp-add.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-add.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
+declare void @use(i32)
+
 ; PR1949
 
 define i1 @test1(i32 %a) {
@@ -463,3 +465,159 @@ define i1 @sum_ult_op_uses(i8 %x, i8 %y, i8* %p) {
   ret i1 %c
 }
 
+; X + Z >s Y + Z -> X > Y if there is no overflow.
+define i1 @common_op_nsw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %z
+  %rhs = add nsw i32 %y, %z
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nsw_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = add nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = add nsw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @use(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %z
+  call void @use(i32 %lhs)
+  %rhs = add nsw i32 %y, %z
+  call void @use(i32 %rhs)
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Z >u Z + Y -> X > Y if there is no overflow.
+define i1 @common_op_nuw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw(
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %z
+  %rhs = add nuw i32 %z, %y
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nuw_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = add nuw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = add nuw i32 [[Z]], [[Y:%.*]]
+; CHECK-NEXT:    call void @use(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %z
+  call void @use(i32 %lhs)
+  %rhs = add nuw i32 %z, %y
+  call void @use(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nsw_commute(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nsw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %z, %x
+  %rhs = add nsw i32 %y, %z
+  %c = icmp slt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+define i1 @common_op_nuw_commute(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @common_op_nuw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp ult i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %z, %x
+  %rhs = add nuw i32 %z, %y
+  %c = icmp ult i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @common_op_test29(i32 %x, i32 %y) {
+; CHECK-LABEL: @common_op_test29(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X + Y > X -> Y > 0 if there is no overflow.
+define i1 @sum_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nuw(
+; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %lhs, %x
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @sum_nsw_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nsw_commute(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %rhs = add nsw i32 %x, %y
+  %c = icmp sgt i32 %x, %rhs
+  ret i1 %c
+}
+
+; X > X + Y -> 0 > Y if there is no overflow.
+define i1 @sum_nuw_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sum_nuw_commute(
+; CHECK-NEXT:    ret i1 false
+;
+  %rhs = add nuw i32 %x, %y
+  %c = icmp ugt i32 %x, %rhs
+  ret i1 %c
+}
+
+; PR2698 - https://bugs.llvm.org/show_bug.cgi?id=2698
+
+declare void @use1(i1)
+declare void @use8(i8)
+
+define void @bzip1(i8 %a, i8 %b, i8 %x) {
+; CHECK-LABEL: @bzip1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    ret void
+;
+  %add1 = add i8 %a, %x
+  %add2 = add i8 %b, %x
+  %cmp = icmp eq i8 %add1, %add2
+  call void @use1(i1 %cmp)
+  ret void
+}
+
+define void @bzip2(i8 %a, i8 %b, i8 %x) {
+; CHECK-LABEL: @bzip2(
+; CHECK-NEXT:    [[ADD1:%.*]] = add i8 [[A:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[A]], [[B:%.*]]
+; CHECK-NEXT:    call void @use1(i1 [[CMP]])
+; CHECK-NEXT:    call void @use8(i8 [[ADD1]])
+; CHECK-NEXT:    ret void
+;
+  %add1 = add i8 %a, %x
+  %add2 = add i8 %b, %x
+  %cmp = icmp eq i8 %add1, %add2
+  call void @use1(i1 %cmp)
+  call void @use8(i8 %add1)
+  ret void
+}
diff --git a/llvm/test/Transforms/InstCombine/icmp.ll b/llvm/test/Transforms/InstCombine/icmp.ll
index 3fecf97e89dbd..9008db9f897dc 100644
--- a/llvm/test/Transforms/InstCombine/icmp.ll
+++ b/llvm/test/Transforms/InstCombine/icmp.ll
@@ -521,37 +521,31 @@ define i1 @test24_as1(i64 %i) {
   ret i1 %cmp
 }
 
-define i1 @test25(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test25(
+; X - Z > Y - Z -> X > Y if there is no overflow.
+define i1 @test27(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test27(
 ; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
-  %lhs = add nsw i32 %x, %z
-  %rhs = add nsw i32 %y, %z
+  %lhs = sub nsw i32 %x, %z
+  %rhs = sub nsw i32 %y, %z
   %c = icmp sgt i32 %lhs, %rhs
   ret i1 %c
 }
 
-; X + Z > Y + Z -> X > Y if there is no overflow.
-define i1 @test26(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test26(
-; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[C]]
-;
-  %lhs = add nuw i32 %x, %z
-  %rhs = add nuw i32 %y, %z
-  %c = icmp ugt i32 %lhs, %rhs
-  ret i1 %c
-}
-
-; X - Z > Y - Z -> X > Y if there is no overflow.
-define i1 @test27(i32 %x, i32 %y, i32 %z) {
-; CHECK-LABEL: @test27(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+define i1 @test27_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test27_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
   %lhs = sub nsw i32 %x, %z
+  call void @foo(i32 %lhs)
   %rhs = sub nsw i32 %y, %z
+  call void @foo(i32 %rhs)
   %c = icmp sgt i32 %lhs, %rhs
   ret i1 %c
 }
@@ -568,47 +562,49 @@ define i1 @test28(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
-; X + Y > X -> Y > 0 if there is no overflow.
-define i1 @test29(i32 %x, i32 %y) {
-; CHECK-LABEL: @test29(
-; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Y:%.*]], 0
+define i1 @test28_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test28_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[Y:%.*]], [[Z]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[X]], [[Y]]
 ; CHECK-NEXT:    ret i1 [[C]]
 ;
-  %lhs = add nsw i32 %x, %y
-  %c = icmp sgt i32 %lhs, %x
+  %lhs = sub nuw i32 %x, %z
+  call void @foo(i32 %lhs)
+  %rhs = sub nuw i32 %y, %z
+  call void @foo(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
   ret i1 %c
 }
 
-; X + Y > X -> Y > 0 if there is no overflow.
-define i1 @test30(i32 %x, i32 %y) {
-; CHECK-LABEL: @test30(
-; CHECK-NEXT:    [[C:%.*]] = icmp ne i32 [[Y:%.*]], 0
-; CHECK-NEXT:    ret i1 [[C]]
-;
-  %lhs = add nuw i32 %x, %y
-  %c = icmp ugt i32 %lhs, %x
-  ret i1 %c
-}
+; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
 
-; X > X + Y -> 0 > Y if there is no overflow.
-define i1 @test31(i32 %x, i32 %y) {
-; CHECK-LABEL: @test31(
-; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], 0
-; CHECK-NEXT:    ret i1 [[C]]
+define i1 @ugt_sub(i32 %xsrc, i32 %y) {
+; CHECK-LABEL: @ugt_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
 ;
-  %rhs = add nsw i32 %x, %y
-  %c = icmp sgt i32 %x, %rhs
-  ret i1 %c
+  %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
+  %sub = sub i32 %x, %y
+  %cmp = icmp ugt i32 %sub, %x
+  ret i1 %cmp
 }
 
-; X > X + Y -> 0 > Y if there is no overflow.
-define i1 @test32(i32 %x, i32 %y) {
-; CHECK-LABEL: @test32(
-; CHECK-NEXT:    ret i1 false
+; Swap operands and predicate. Try a vector type to verify that works too.
+
+define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
+; CHECK-LABEL: @ult_sub(
+; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
 ;
-  %rhs = add nuw i32 %x, %y
-  %c = icmp ugt i32 %x, %rhs
-  ret i1 %c
+  %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
+  %sub = sub <2 x i8> %x, %y
+  %cmp = icmp ult <2 x i8> %x, %sub
+  ret <2 x i1> %cmp
 }
 
 ; X - Y > X -> 0 > Y if there is no overflow.
@@ -654,34 +650,6 @@ define i1 @test36(i32 %x, i32 %y) {
   ret i1 %c
 }
 
-; PR36969 - https://bugs.llvm.org/show_bug.cgi?id=36969
-
-define i1 @ugt_sub(i32 %xsrc, i32 %y) {
-; CHECK-LABEL: @ugt_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv i32 [[XSRC:%.*]], 42
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 [[X]], [[Y:%.*]]
-; CHECK-NEXT:    ret i1 [[CMP]]
-;
-  %x = udiv i32 %xsrc, 42 ; thwart complexity-based canonicalization
-  %sub = sub i32 %x, %y
-  %cmp = icmp ugt i32 %sub, %x
-  ret i1 %cmp
-}
-
-; Swap operands and predicate. Try a vector type to verify that works too.
-
-define <2 x i1> @ult_sub(<2 x i8> %xsrc, <2 x i8> %y) {
-; CHECK-LABEL: @ult_sub(
-; CHECK-NEXT:    [[X:%.*]] = udiv <2 x i8> [[XSRC:%.*]], <i8 42, i8 -42>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult <2 x i8> [[X]], [[Y:%.*]]
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
-;
-  %x = udiv <2 x i8> %xsrc, <i8 42, i8 -42> ; thwart complexity-based canonicalization
-  %sub = sub <2 x i8> %x, %y
-  %cmp = icmp ult <2 x i8> %x, %sub
-  ret <2 x i1> %cmp
-}
-
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test37(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test37(
@@ -694,6 +662,53 @@ define i1 @test37(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
+define i1 @test37_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test37_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt i32 [[Z]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nsw i32 %x, %y
+  call void @foo(i32 %lhs)
+  %rhs = sub nsw i32 %x, %z
+  call void @foo(i32 %rhs)
+  %c = icmp sgt i32 %lhs, %rhs
+  ret i1 %c
+}
+
+; TODO: Min/max pattern should not prevent the fold.
+
+define i32 @neg_max_s32(i32 %x, i32 %y) {
+; CHECK-LABEL: @neg_max_s32(
+; CHECK-NEXT:    [[C:%.*]] = icmp slt i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[S_V:%.*]] = select i1 [[C]], i32 [[Y]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[S_V]]
+;
+  %nx = sub nsw i32 0, %x
+  %ny = sub nsw i32 0, %y
+  %c = icmp slt i32 %nx, %ny
+  %s = select i1 %c, i32 %ny, i32 %nx
+  %r = sub nsw i32 0, %s
+  ret i32 %r
+}
+
+define <4 x i32> @neg_max_v4s32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @neg_max_v4s32(
+; CHECK-NEXT:    [[C:%.*]] = icmp sgt <4 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[S_V:%.*]] = select <4 x i1> [[C]], <4 x i32> [[X]], <4 x i32> [[Y]]
+; CHECK-NEXT:    ret <4 x i32> [[S_V]]
+;
+  %nx = sub nsw <4 x i32> zeroinitializer, %x
+  %ny = sub nsw <4 x i32> zeroinitializer, %y
+  %c = icmp sgt <4 x i32> %nx, %ny
+  %s = select <4 x i1> %c, <4 x i32> %nx, <4 x i32> %ny
+  %r = sub <4 x i32> zeroinitializer, %s
+  ret <4 x i32> %r
+}
+
 ; X - Y > X - Z -> Z > Y if there is no overflow.
 define i1 @test38(i32 %x, i32 %y, i32 %z) {
 ; CHECK-LABEL: @test38(
@@ -706,6 +721,23 @@ define i1 @test38(i32 %x, i32 %y, i32 %z) {
   ret i1 %c
 }
 
+define i1 @test38_extra_uses(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @test38_extra_uses(
+; CHECK-NEXT:    [[LHS:%.*]] = sub nuw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[LHS]])
+; CHECK-NEXT:    [[RHS:%.*]] = sub nuw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    call void @foo(i32 [[RHS]])
+; CHECK-NEXT:    [[C:%.*]] = icmp ugt i32 [[Z]], [[Y]]
+; CHECK-NEXT:    ret i1 [[C]]
+;
+  %lhs = sub nuw i32 %x, %y
+  call void @foo(i32 %lhs)
+  %rhs = sub nuw i32 %x, %z
+  call void @foo(i32 %rhs)
+  %c = icmp ugt i32 %lhs, %rhs
+  ret i1 %c
+}
+
 ; PR9343 #1
 define i1 @test39(i32 %X, i32 %Y) {
 ; CHECK-LABEL: @test39(
diff --git a/llvm/test/Transforms/InstCombine/load-cmp.ll b/llvm/test/Transforms/InstCombine/load-cmp.ll
index 5746b7aa28d54..7b24ce309a98c 100644
--- a/llvm/test/Transforms/InstCombine/load-cmp.ll
+++ b/llvm/test/Transforms/InstCombine/load-cmp.ll
@@ -22,7 +22,7 @@
 
 define i1 @test1(i32 %X) {
 ; CHECK-LABEL: @test1(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -33,7 +33,7 @@ define i1 @test1(i32 %X) {
 
 define i1 @test1_noinbounds(i32 %X) {
 ; CHECK-LABEL: @test1_noinbounds(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 9
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -44,7 +44,7 @@ define i1 @test1_noinbounds(i32 %X) {
 
 define i1 @test1_noinbounds_i64(i64 %X) {
 ; CHECK-LABEL: @test1_noinbounds_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %X to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -56,7 +56,7 @@ define i1 @test1_noinbounds_i64(i64 %X) {
 
 define i1 @test1_noinbounds_as1(i32 %x) {
 ; CHECK-LABEL: @test1_noinbounds_as1(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 %x to i16
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[X:%.*]] to i16
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i16 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -69,7 +69,7 @@ define i1 @test1_noinbounds_as1(i32 %x) {
 
 define i1 @test2(i32 %X) {
 ; CHECK-LABEL: @test2(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %X, 4
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 4
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [10 x i16], [10 x i16]* @G16, i32 0, i32 %X
@@ -80,7 +80,7 @@ define i1 @test2(i32 %X) {
 
 define i1 @test3(i32 %X) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 %X, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %P = getelementptr inbounds [6 x double], [6 x double]* @GD, i32 0, i32 %X
@@ -92,7 +92,7 @@ define i1 @test3(i32 %X) {
 
 define i1 @test4(i32 %X) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 933, %X
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 933, [[X:%.*]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP2]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -105,7 +105,7 @@ define i1 @test4(i32 %X) {
 
 define i1 @test4_i16(i16 %X) {
 ; CHECK-LABEL: @test4_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %X to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i32 933, [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = and i32 [[TMP2]], 1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP3]], 0
@@ -119,8 +119,8 @@ define i1 @test4_i16(i16 %X) {
 
 define i1 @test5(i32 %X) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 %X, 2
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 %X, 7
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[X]], 7
 ; CHECK-NEXT:    [[R:%.*]] = or i1 [[TMP1]], [[TMP2]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -132,7 +132,7 @@ define i1 @test5(i32 %X) {
 
 define i1 @test6(i32 %X) {
 ; CHECK-LABEL: @test6(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ult i32 [[TMP1]], 3
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -144,7 +144,7 @@ define i1 @test6(i32 %X) {
 
 define i1 @test7(i32 %X) {
 ; CHECK-LABEL: @test7(
-; CHECK-NEXT:    [[TMP1:%.*]] = add i32 %X, -1
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
 ; CHECK-NEXT:    [[R:%.*]] = icmp ugt i32 [[TMP1]], 2
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -156,7 +156,7 @@ define i1 @test7(i32 %X) {
 
 define i1 @test8(i32 %X) {
 ; CHECK-LABEL: @test8(
-; CHECK-NEXT:    [[TMP1:%.*]] = or i32 %X, 1
+; CHECK-NEXT:    [[TMP1:%.*]] = or i32 [[X:%.*]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq i32 [[TMP1]], 9
 ; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
@@ -176,9 +176,9 @@ define i1 @test8(i32 %X) {
 
 define i1 @test9(i32 %X) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[X_OFF:%.*]] = add i32 %X, -1
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i32 [[X_OFF]], 2
-; CHECK-NEXT:    ret i1 [[TMP1]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add i32 [[X:%.*]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i32 [[TMP1]], 2
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %P = getelementptr inbounds [4 x { i32, i32 } ], [4 x { i32, i32 } ]* @GA, i32 0, i32 %X, i32 1
   %Q = load i32, i32* %P
@@ -198,7 +198,7 @@ define i1 @test10_struct(i32 %x) {
 
 define i1 @test10_struct_noinbounds(i32 %x) {
 ; CHECK-LABEL: @test10_struct_noinbounds(
-; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 %x, i32 0
+; CHECK-NEXT:    [[P:%.*]] = getelementptr [[FOO:%.*]], %Foo* @GS, i32 [[X:%.*]], i32 0
 ; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 9
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -235,8 +235,8 @@ define i1 @test10_struct_i64(i64 %x){
 
 define i1 @test10_struct_noinbounds_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_noinbounds_i16(
-; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 %x to i32
-; CHECK-NEXT:    [[P:%.*]] = getelementptr %Foo, %Foo* @GS, i32 [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[X:%.*]] to i32
+; CHECK-NEXT:    [[P:%.*]] = getelementptr [[FOO:%.*]], %Foo* @GS, i32 [[TMP1]], i32 0
 ; CHECK-NEXT:    [[Q:%.*]] = load i32, i32* [[P]], align 8
 ; CHECK-NEXT:    [[R:%.*]] = icmp eq i32 [[Q]], 0
 ; CHECK-NEXT:    ret i1 [[R]]
@@ -249,7 +249,7 @@ define i1 @test10_struct_noinbounds_i16(i16 %x) {
 
 define i1 @test10_struct_arr(i32 %x) {
 ; CHECK-LABEL: @test10_struct_arr(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
@@ -260,7 +260,7 @@ define i1 @test10_struct_arr(i32 %x) {
 
 define i1 @test10_struct_arr_noinbounds(i32 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i32 %x, i32 2
@@ -271,7 +271,7 @@ define i1 @test10_struct_arr_noinbounds(i32 %x) {
 
 define i1 @test10_struct_arr_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_arr_i16(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr inbounds [4 x %Foo], [4 x %Foo]* @GStructArr, i16 0, i16 %x, i32 2
@@ -282,7 +282,7 @@ define i1 @test10_struct_arr_i16(i16 %x) {
 
 define i1 @test10_struct_arr_i64(i64 %x) {
 ; CHECK-LABEL: @test10_struct_arr_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
@@ -294,7 +294,7 @@ define i1 @test10_struct_arr_i64(i64 %x) {
 
 define i1 @test10_struct_arr_noinbounds_i16(i16 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds_i16(
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 %x, 1
+; CHECK-NEXT:    [[R:%.*]] = icmp ne i16 [[X:%.*]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %p = getelementptr [4 x %Foo], [4 x %Foo]* @GStructArr, i32 0, i16 %x, i32 2
@@ -305,7 +305,7 @@ define i1 @test10_struct_arr_noinbounds_i16(i16 %x) {
 
 define i1 @test10_struct_arr_noinbounds_i64(i64 %x) {
 ; CHECK-LABEL: @test10_struct_arr_noinbounds_i64(
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 %x to i32
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[X:%.*]] to i32
 ; CHECK-NEXT:    [[R:%.*]] = icmp ne i32 [[TMP1]], 1
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/log-pow.ll b/llvm/test/Transforms/InstCombine/log-pow.ll
index 3a6b4eaa4e7b1..b7085e955ef41 100644
--- a/llvm/test/Transforms/InstCombine/log-pow.ll
+++ b/llvm/test/Transforms/InstCombine/log-pow.ll
@@ -3,7 +3,7 @@
 
 define double @log_pow(double %x, double %y) {
 ; CHECK-LABEL: @log_pow(
-; CHECK-NEXT:    [[LOG1:%.*]] = call fast double @log(double [[X:%.*]]) #0
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast double @llvm.log.f64(double [[X:%.*]])
 ; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[LOG1]], [[Y:%.*]]
 ; CHECK-NEXT:    ret double [[MUL]]
 ;
@@ -12,24 +12,22 @@ define double @log_pow(double %x, double %y) {
   ret double %log
 }
 
-; FIXME: log10f() should also be simplified.
 define float @log10f_powf(float %x, float %y) {
 ; CHECK-LABEL: @log10f_powf(
-; CHECK-NEXT:    [[POW:%.*]] = call fast float @powf(float [[X:%.*]], float [[Y:%.*]])
-; CHECK-NEXT:    [[LOG:%.*]] = call fast float @log10f(float [[POW]])
-; CHECK-NEXT:    ret float [[LOG]]
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast float @llvm.log10.f32(float [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[LOG1]], [[Y:%.*]]
+; CHECK-NEXT:    ret float [[MUL]]
 ;
   %pow = call fast float @powf(float %x, float %y)
-  %log = call fast float @log10f(float %pow)
+  %log = call fast float @llvm.log10.f32(float %pow)
   ret float %log
 }
 
-; FIXME: Intrinsic log2() should also be simplified.
 define <2 x double> @log2v_powv(<2 x double> %x, <2 x double> %y) {
 ; CHECK-LABEL: @log2v_powv(
-; CHECK-NEXT:    [[POW:%.*]] = call fast <2 x double> @llvm.pow.v2f64(<2 x double> [[X:%.*]], <2 x double> [[Y:%.*]])
-; CHECK-NEXT:    [[LOG:%.*]] = call fast <2 x double> @llvm.log2.v2f64(<2 x double> [[POW]])
-; CHECK-NEXT:    ret <2 x double> [[LOG]]
+; CHECK-NEXT:    [[LOG1:%.*]] = call fast <2 x double> @llvm.log2.v2f64(<2 x double> [[X:%.*]])
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <2 x double> [[LOG1]], [[Y:%.*]]
+; CHECK-NEXT:    ret <2 x double> [[MUL]]
 ;
   %pow = call fast <2 x double> @llvm.pow.v2f64(<2 x double> %x, <2 x double> %y)
   %log = call fast <2 x double> @llvm.log2.v2f64(<2 x double> %pow)
@@ -58,39 +56,33 @@ define float @function_pointer(float ()* %fptr, float %p1) {
   ret float %log
 }
 
-; FIXME: The call to exp2() should be removed.
-define double @log_exp2(double %x) {
-; CHECK-LABEL: @log_exp2(
-; CHECK-NEXT:    [[EXP:%.*]] = call fast double @exp2(double [[X:%.*]])
-; CHECK-NEXT:    [[LOGMUL:%.*]] = fmul fast double [[X]], 0x3FE62E42FEFA39EF
-; CHECK-NEXT:    ret double [[LOGMUL]]
+define double @log10_exp(double %x) {
+; CHECK-LABEL: @log10_exp(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast double [[X:%.*]], 0x3FDBCB7B1526E50E
+; CHECK-NEXT:    ret double [[MUL]]
 ;
-  %exp = call fast double @exp2(double %x)
-  %log = call fast double @log(double %exp)
+  %exp = call fast double @exp(double %x)
+  %log = call fast double @log10(double %exp)
   ret double %log
 }
 
-; FIXME: Intrinsic logf() should also be simplified.
 define <2 x float> @logv_exp2v(<2 x float> %x) {
 ; CHECK-LABEL: @logv_exp2v(
-; CHECK-NEXT:    [[EXP:%.*]] = call fast <2 x float> @llvm.exp2.v2f32(<2 x float> [[X:%.*]])
-; CHECK-NEXT:    [[LOG:%.*]] = call fast <2 x float> @llvm.log.v2f32(<2 x float> [[EXP]])
-; CHECK-NEXT:    ret <2 x float> [[LOG]]
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast <2 x float> [[X:%.*]], <float 0x3FE62E4300000000, float 0x3FE62E4300000000>
+; CHECK-NEXT:    ret <2 x float> [[MUL]]
 ;
   %exp = call fast <2 x float> @llvm.exp2.v2f32(<2 x float> %x)
   %log = call fast <2 x float> @llvm.log.v2f32(<2 x float> %exp)
   ret <2 x float> %log
 }
 
-; FIXME: log10f() should also be simplified.
-define float @log10f_exp2f(float %x) {
-; CHECK-LABEL: @log10f_exp2f(
-; CHECK-NEXT:    [[EXP:%.*]] = call fast float @exp2f(float [[X:%.*]])
-; CHECK-NEXT:    [[LOG:%.*]] = call fast float @log10f(float [[EXP]])
-; CHECK-NEXT:    ret float [[LOG]]
+define float @log2f_exp10f(float %x) {
+; CHECK-LABEL: @log2f_exp10f(
+; CHECK-NEXT:    [[MUL:%.*]] = fmul fast float [[X:%.*]], 0x400A934F00000000
+; CHECK-NEXT:    ret float [[MUL]]
 ;
-  %exp = call fast float @exp2f(float %x)
-  %log = call fast float @log10f(float %exp)
+  %exp = call fast float @exp10f(float %x)
+  %log = call fast float @log2f(float %exp)
   ret float %log
 }
 
@@ -108,11 +100,13 @@ define double @log_exp2_not_fast(double %x) {
 declare double @log(double) #0
 declare float @logf(float) #0
 declare <2 x float> @llvm.log.v2f32(<2 x float>)
-declare double @log2(double) #0
+declare float @log2f(float) #0
 declare <2 x double> @llvm.log2.v2f64(<2 x double>)
-declare float @log10f(float) #0
-declare double @exp2(double)
-declare float @exp2f(float)
+declare double @log10(double) #0
+declare float @llvm.log10.f32(float)
+declare double @exp(double %x) #0
+declare double @exp2(double) #0
+declare float @exp10f(float) #0
 declare <2 x float> @llvm.exp2.v2f32(<2 x float>)
 declare double @pow(double, double) #0
 declare float @powf(float, float) #0
diff --git a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
index c288d1ce44fce..a0828e1b6b17f 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free-delete.ll
@@ -72,8 +72,8 @@ define void @test5(i8* %ptr, i8** %esc) {
 ; CHECK-NEXT:    [[E:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
 ; CHECK-NEXT:    [[F:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
 ; CHECK-NEXT:    [[G:%.*]] = call dereferenceable_or_null(700) i8* @malloc(i32 700)
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(32) [[PTR:%.*]], i8* align 1 dereferenceable(32) [[A]], i32 32, i1 false)
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* align 1 dereferenceable(32) [[PTR]], i8* align 1 dereferenceable(32) [[B]], i32 32, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(32) [[PTR:%.*]], i8* nonnull align 1 dereferenceable(32) [[A]], i32 32, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(32) [[PTR]], i8* nonnull align 1 dereferenceable(32) [[B]], i32 32, i1 false)
 ; CHECK-NEXT:    store i8* [[C]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* [[D]], i8* [[PTR]], i32 32, i1 true)
 ; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i32(i8* [[E]], i8* [[PTR]], i32 32, i1 true)
diff --git a/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll b/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
index b09269ccc6796..76eeeedfe4d30 100644
--- a/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
+++ b/llvm/test/Transforms/InstCombine/mem-deref-bytes-addrspaces.ll
@@ -3,11 +3,20 @@
 
 declare i32 @memcmp(i8 addrspace(1)* nocapture, i8* nocapture, i64)
 
-define i32 @memcmp_const_size_update_deref8(i8 addrspace(1)* nocapture readonly %d, i8* nocapture readonly %s) {
-; CHECK-LABEL: @memcmp_const_size_update_deref8(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8 addrspace(1)* dereferenceable(16) dereferenceable_or_null(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+define i32 @memcmp_const_size_update_deref(i8 addrspace(1)* nocapture readonly %d, i8* nocapture readonly %s) {
+; CHECK-LABEL: @memcmp_const_size_update_deref(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8 addrspace(1)* dereferenceable(16) dereferenceable_or_null(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8 addrspace(1)* dereferenceable_or_null(40) %d, i8* %s, i64 16)
   ret i32 %call
 }
+
+define i32 @memcmp_nonconst_size_nonnnull(i8 addrspace(1)* nocapture readonly %d, i8* nocapture readonly %s, i64 %n) {
+; CHECK-LABEL: @memcmp_nonconst_size_nonnnull(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8 addrspace(1)* nonnull dereferenceable_or_null(40) [[D:%.*]], i8* nonnull [[S:%.*]], i64 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8 addrspace(1)* nonnull dereferenceable_or_null(40) %d, i8* nonnull %s, i64 %n)
+  ret i32 %call
+}
diff --git a/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll b/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
index c27ebb2939e9f..c19597a4b4c68 100644
--- a/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
+++ b/llvm/test/Transforms/InstCombine/mem-deref-bytes.ll
@@ -12,7 +12,7 @@ declare void @llvm.memset.p0i8.i64(i8* nocapture, i8, i64, i1)
 
 define i32 @memcmp_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_set_deref(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(16) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* %d, i8* %s, i64 16)
@@ -21,7 +21,7 @@ define i32 @memcmp_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture
 
 define i32 @memcmp_const_size_update_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_update_deref(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(16) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* dereferenceable(4) %d, i8* dereferenceable(8) %s, i64 16)
@@ -30,7 +30,7 @@ define i32 @memcmp_const_size_update_deref(i8* nocapture readonly %d, i8* nocapt
 
 define i32 @memcmp_const_size_update_deref2(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_update_deref2(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(16) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* %d, i8* dereferenceable_or_null(8) %s, i64 16)
@@ -39,7 +39,7 @@ define i32 @memcmp_const_size_update_deref2(i8* nocapture readonly %d, i8* nocap
 
 define i32 @memcmp_const_size_update_deref3(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_update_deref3(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* dereferenceable(40) %d, i8* %s, i64 16)
@@ -48,7 +48,7 @@ define i32 @memcmp_const_size_update_deref3(i8* nocapture readonly %d, i8* nocap
 
 define i32 @memcmp_const_size_update_deref4(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_update_deref4(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(16) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(16) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* dereferenceable_or_null(16) %d, i8* %s, i64 16)
@@ -57,7 +57,7 @@ define i32 @memcmp_const_size_update_deref4(i8* nocapture readonly %d, i8* nocap
 
 define i32 @memcmp_const_size_update_deref5(i8* nocapture readonly %d, i8* nocapture readonly %s) "null-pointer-is-valid"="false" {
 ; CHECK-LABEL: @memcmp_const_size_update_deref5(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(16) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* dereferenceable_or_null(40) %d, i8* %s, i64 16)
@@ -73,9 +73,18 @@ define i32 @memcmp_const_size_update_deref6(i8* nocapture readonly %d, i8* nocap
   ret i32 %call
 }
 
+define i32 @memcmp_const_size_update_deref7(i8* nocapture readonly %d, i8* nocapture readonly %s) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @memcmp_const_size_update_deref7(
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* dereferenceable(16) [[S:%.*]], i64 16)
+; CHECK-NEXT:    ret i32 [[CALL]]
+;
+  %call = tail call i32 @memcmp(i8* nonnull dereferenceable_or_null(40) %d, i8* %s, i64 16)
+  ret i32 %call
+}
+
 define i32 @memcmp_const_size_no_update_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcmp_const_size_no_update_deref(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(40) [[D:%.*]], i8* dereferenceable(20) [[S:%.*]], i64 16)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(40) [[D:%.*]], i8* nonnull dereferenceable(20) [[S:%.*]], i64 16)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = tail call i32 @memcmp(i8* dereferenceable(40) %d, i8* dereferenceable(20) %s, i64 16)
@@ -93,7 +102,7 @@ define i32 @memcmp_nonconst_size(i8* nocapture readonly %d, i8* nocapture readon
 
 define i8* @memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcpy_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(64) [[D:%.*]], i8* align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[D:%.*]], i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
 ; CHECK-NEXT:    ret i8* [[D]]
 ;
   %call = tail call i8* @memcpy(i8* %d, i8* %s, i64 64)
@@ -102,7 +111,7 @@ define i8* @memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture
 
 define i8* @memmove_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memmove_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 dereferenceable(64) [[D:%.*]], i8* align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[D:%.*]], i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i64 64, i1 false)
 ; CHECK-NEXT:    ret i8* [[D]]
 ;
   %call = tail call i8* @memmove(i8* %d, i8* %s, i64 64)
@@ -111,7 +120,7 @@ define i8* @memmove_const_size_set_deref(i8* nocapture readonly %d, i8* nocaptur
 
 define i8* @memset_const_size_set_deref(i8* nocapture readonly %s, i8 %c) {
 ; CHECK-LABEL: @memset_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 dereferenceable(64) [[S:%.*]], i8 [[C:%.*]], i64 64, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 1 dereferenceable(64) [[S:%.*]], i8 [[C:%.*]], i64 64, i1 false)
 ; CHECK-NEXT:    ret i8* [[S]]
 ;
   %call = tail call i8* @memset(i8* %s, i8 %c, i64 64)
@@ -120,7 +129,7 @@ define i8* @memset_const_size_set_deref(i8* nocapture readonly %s, i8 %c) {
 
 define i8* @memchr_const_size_set_deref(i8* nocapture readonly %s, i32 %c) {
 ; CHECK-LABEL: @memchr_const_size_set_deref(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @memchr(i8* dereferenceable(64) [[S:%.*]], i32 [[C:%.*]], i64 64)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @memchr(i8* nonnull dereferenceable(64) [[S:%.*]], i32 [[C:%.*]], i64 64)
 ; CHECK-NEXT:    ret i8* [[CALL]]
 ;
   %call = tail call i8* @memchr(i8* %s, i32 %c, i64 64)
@@ -129,7 +138,7 @@ define i8* @memchr_const_size_set_deref(i8* nocapture readonly %s, i32 %c) {
 
 define i8* @llvm_memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @llvm_memcpy_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(16) [[D:%.*]], i8* align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    ret i8* [[D]]
 ;
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 %d, i8* align 1 %s, i64 16, i1 false)
@@ -138,7 +147,7 @@ define i8* @llvm_memcpy_const_size_set_deref(i8* nocapture readonly %d, i8* noca
 
 define i8* @llvm_memmove_const_size_set_deref(i8* nocapture readonly %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @llvm_memmove_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 dereferenceable(16) [[D:%.*]], i8* align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    ret i8* [[D]]
 ;
   call void @llvm.memmove.p0i8.p0i8.i64(i8* align 1 %d, i8* align 1 %s, i64 16, i1 false)
@@ -146,7 +155,7 @@ define i8* @llvm_memmove_const_size_set_deref(i8* nocapture readonly %d, i8* noc
 }
 define i8* @llvm_memset_const_size_set_deref(i8* nocapture readonly %s, i8 %c) {
 ; CHECK-LABEL: @llvm_memset_const_size_set_deref(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 1 dereferenceable(16) [[S:%.*]], i8 [[C:%.*]], i64 16, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i8 [[C:%.*]], i64 16, i1 false)
 ; CHECK-NEXT:    ret i8* [[S]]
 ;
   call void @llvm.memset.p0i8.i64(i8* align 1 %s, i8 %c, i64 16, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/memccpy.ll b/llvm/test/Transforms/InstCombine/memccpy.ll
new file mode 100644
index 0000000000000..d911da16278a8
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memccpy.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@hello = private constant [11 x i8] c"helloworld\00", align 1
+
+declare i8* @memccpy(i8*, i8*, i32, i64)
+
+define i8* @memccpy_to_memcpy(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 12) ; 114 is 'r'
+  ret i8* %call
+}
+
+define i8* @memccpy_to_memcpy2(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy2(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 5)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 5)
+  ret i8* %call
+}
+
+define void @memccpy_to_memcpy3(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_memcpy3(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 5)
+; CHECK-NEXT:    ret void
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 5)
+  ret void
+}
+
+define i8* @memccpy_to_null(i8* %dst, i8* %src, i32 %c) {
+; CHECK-LABEL: @memccpy_to_null(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* [[SRC:%.*]], i32 [[C:%.*]], i64 0)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %src, i32 %c, i64 0)
+  ret i8* %call
+}
+
+define i8* @memccpy_to_null2(i8* %dst) {
+; CHECK-LABEL: @memccpy_to_null2(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 5)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 115, i64 5) ; 115 is 's'
+  ret i8* %call
+}
+
+; Negative tests
+define i8* @unknown_src(i8* %dst, i8* %src) {
+; CHECK-LABEL: @unknown_src(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* [[SRC:%.*]], i32 114, i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* %src, i32 114, i64 12)
+  ret i8* %call
+}
+
+define i8* @unknown_stop_char(i8* %dst, i32 %c) {
+; CHECK-LABEL: @unknown_stop_char(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 [[C:%.*]], i64 12)
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 %c, i64 12)
+  ret i8* %call
+}
+
+define i8* @unknown_size_n(i8* %dst, i64 %n) {
+; CHECK-LABEL: @unknown_size_n(
+; CHECK-NEXT:    [[CALL:%.*]] = call i8* @memccpy(i8* [[DST:%.*]], i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[CALL]]
+;
+  %call = call i8* @memccpy(i8* %dst, i8* getelementptr inbounds ([11 x i8], [11 x i8]* @hello, i64 0, i64 0), i32 114, i64 %n)
+  ret i8* %call
+}
diff --git a/llvm/test/Transforms/InstCombine/memchr.ll b/llvm/test/Transforms/InstCombine/memchr.ll
index 808467955bc5c..2e2c29a07d57f 100644
--- a/llvm/test/Transforms/InstCombine/memchr.ll
+++ b/llvm/test/Transforms/InstCombine/memchr.ll
@@ -50,7 +50,7 @@ define void @test3() {
 
 define void @test4(i32 %chr) {
 ; CHECK-LABEL: @test4(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* dereferenceable(14) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(14) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
 ; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
 ; CHECK-NEXT:    ret void
 ;
@@ -148,7 +148,7 @@ define i1 @test11(i32 %C) {
 ; No 64 bits here
 define i1 @test12(i32 %C) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @spaces, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -185,7 +185,7 @@ define i1 @test14(i32 %C) {
 
 define i1 @test15(i32 %C) {
 ; CHECK-LABEL: @test15(
-; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @negative, i32 0, i32 0), i32 [[C:%.*]], i32 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i8* [[DST]], null
 ; CHECK-NEXT:    ret i1 [[CMP]]
 ;
@@ -202,3 +202,43 @@ define i8* @pr32124() {
   %res = tail call i8* @memchr(i8* getelementptr ([1 x i8], [1 x i8]* @s, i64 0, i64 0), i32 0, i32 1)
   ret i8* %res
 }
+
+define i8* @test16(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test16(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test17(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test17(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* nonnull [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* nonnull %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test18(i8* %str, i32 %c) {
+; CHECK-LABEL: @test18(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(5) [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test19(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test19(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memchr(i8* dereferenceable(5) [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
index 62a17ea3a0e2c..22b946375cd81 100644
--- a/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
+++ b/llvm/test/Transforms/InstCombine/memcmp-constant-fold.ll
@@ -55,7 +55,7 @@ define i1 @memcmp_4bytes_unaligned_constant_i16(i8* align 4 %x) {
 
 define i1 @memcmp_3bytes_aligned_constant_i32(i8* align 4 %x) {
 ; ALL-LABEL: @memcmp_3bytes_aligned_constant_i32(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(3) bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* dereferenceable(3) bitcast ([2 x i32]* @intbuf to i8*), i64 3)
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(3) bitcast (i32* getelementptr inbounds ([2 x i32], [2 x i32]* @intbuf, i64 0, i64 1) to i8*), i8* nonnull dereferenceable(3) bitcast ([2 x i32]* @intbuf to i8*), i64 3)
 ; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
 ; ALL-NEXT:    ret i1 [[CMPEQ0]]
 ;
@@ -68,7 +68,7 @@ define i1 @memcmp_3bytes_aligned_constant_i32(i8* align 4 %x) {
 
 define i1 @memcmp_4bytes_one_unaligned_i8(i8* align 4 %x, i8* align 1 %y) {
 ; ALL-LABEL: @memcmp_4bytes_one_unaligned_i8(
-; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* dereferenceable(4) [[X:%.*]], i8* dereferenceable(4) [[Y:%.*]], i64 4)
+; ALL-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* nonnull dereferenceable(4) [[X:%.*]], i8* nonnull dereferenceable(4) [[Y:%.*]], i64 4)
 ; ALL-NEXT:    [[CMPEQ0:%.*]] = icmp eq i32 [[CALL]], 0
 ; ALL-NEXT:    ret i1 [[CMPEQ0]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
index 2e5b83c91e165..96029fa59bfed 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-from-global.ll
@@ -78,7 +78,7 @@ define void @test2() {
 ; CHECK-LABEL: @test2(
 ; CHECK-NEXT:    [[B1:%.*]] = alloca [124 x i8], align 8
 ; CHECK-NEXT:    [[B1_SUB:%.*]] = getelementptr inbounds [124 x i8], [124 x i8]* [[B1]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(124) [[B1_SUB]], i8* align 16 dereferenceable(124) getelementptr inbounds (%T, %T* @G, i64 0, i32 0), i64 124, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 8 dereferenceable(124) [[B1_SUB]], i8* nonnull align 16 dereferenceable(124) getelementptr inbounds (%T, %T* @G, i64 0, i32 0), i64 124, i1 false)
 ; CHECK-NEXT:    call void @bar(i8* nonnull [[B1_SUB]])
 ; CHECK-NEXT:    ret void
 ;
@@ -234,7 +234,7 @@ define void @test8() {
 ; CHECK-LABEL: @test8(
 ; CHECK-NEXT:    [[AL:%.*]] = alloca [[U:%.*]], align 16
 ; CHECK-NEXT:    [[A:%.*]] = bitcast %U* [[AL]] to i8*
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(20) [[A]], i8* align 4 dereferenceable(20) bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*), i64 20, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(20) [[A]], i8* nonnull align 4 dereferenceable(20) bitcast (%U* getelementptr inbounds ([2 x %U], [2 x %U]* @H, i64 0, i64 1) to i8*), i64 20, i1 false)
 ; CHECK-NEXT:    call void @bar(i8* nonnull [[A]]) #2
 ; CHECK-NEXT:    ret void
 ;
@@ -294,8 +294,8 @@ define void @test9_small_global() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[CC:%.*]] = alloca [1000000 x i8], align 16
 ; CHECK-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [1000000 x i8], [1000000 x i8]* [[CC]], i64 0, i64 0
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(3) [[ARRAYDECAY]], i8* align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 dereferenceable(1000000) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* nonnull align 16 dereferenceable(1000000) [[ARRAYDECAY]], i64 1000000, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(3) [[ARRAYDECAY]], i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(1000000) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* nonnull align 16 dereferenceable(1000000) [[ARRAYDECAY]], i64 1000000, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -311,7 +311,7 @@ entry:
 define void @test10_same_global() {
 ; CHECK-LABEL: @test10_same_global(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 16 dereferenceable(3) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([1000000 x i8], [1000000 x i8]* @bbb, i64 0, i64 0), i8* nonnull align 16 dereferenceable(3) getelementptr inbounds ([3 x i8], [3 x i8]* @_ZL3KKK, i64 0, i64 0), i64 3, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
index 172c83d8a25ac..2f9d692687ae8 100644
--- a/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy-to-load.ll
@@ -34,7 +34,7 @@ define void @copy_2_bytes(i8* %d, i8* %s) {
 
 define void @copy_3_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_3_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(3) [[D:%.*]], i8* align 1 dereferenceable(3) [[S:%.*]], i32 3, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(3) [[D:%.*]], i8* nonnull align 1 dereferenceable(3) [[S:%.*]], i32 3, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 3, i1 false)
@@ -57,7 +57,7 @@ define void @copy_4_bytes(i8* %d, i8* %s) {
 
 define void @copy_5_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_5_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(5) [[D:%.*]], i8* align 1 dereferenceable(5) [[S:%.*]], i32 5, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(5) [[D:%.*]], i8* nonnull align 1 dereferenceable(5) [[S:%.*]], i32 5, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 5, i1 false)
@@ -78,7 +78,7 @@ define void @copy_8_bytes(i8* %d, i8* %s) {
 
 define void @copy_16_bytes(i8* %d, i8* %s) {
 ; ALL-LABEL: @copy_16_bytes(
-; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(16) [[D:%.*]], i8* align 1 dereferenceable(16) [[S:%.*]], i32 16, i1 false)
+; ALL-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(16) [[D:%.*]], i8* nonnull align 1 dereferenceable(16) [[S:%.*]], i32 16, i1 false)
 ; ALL-NEXT:    ret void
 ;
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* %d, i8* %s, i32 16, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/memcpy.ll b/llvm/test/Transforms/InstCombine/memcpy.ll
index 18e08535b5f14..4f52cb18eb429 100644
--- a/llvm/test/Transforms/InstCombine/memcpy.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy.ll
@@ -29,7 +29,7 @@ define void @test2(i8* %a) {
 
 define void @test3(i8* %d, i8* %s) {
 ; CHECK-LABEL: @test3(
-; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(17179869184) [[D:%.*]], i8* align 4 dereferenceable(17179869184) [[S:%.*]], i64 17179869184, i1 false)
+; CHECK-NEXT:    tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(17179869184) [[D:%.*]], i8* nonnull align 4 dereferenceable(17179869184) [[S:%.*]], i64 17179869184, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %d, i8* align 4 %s, i64 17179869184, i1 false)
diff --git a/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
index 155bcb19ba88d..c76e4f61f654b 100644
--- a/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memcpy_chk-1.ll
@@ -18,7 +18,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
 ;
   %dst = bitcast %struct.T1* @t1 to i8*
@@ -29,7 +29,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
 ;
   %dst = bitcast %struct.T1* @t1 to i8*
@@ -65,7 +65,7 @@ define i8* @test_no_simplify2() {
 define i8* @test_simplify_return_indcall(i8* ()* %alloc) {
 ; CHECK-LABEL: @test_simplify_return_indcall(
 ; CHECK-NEXT:    [[DST:%.*]] = call i8* [[ALLOC:%.*]]()
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(1824) [[DST]], i8* align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(1824) [[DST]], i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* [[DST]]
 ;
   %src = bitcast %struct.T2* @t2 to i8*
diff --git a/llvm/test/Transforms/InstCombine/memmove_chk-1.ll b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
index 79f10fd5f734e..16aa9db812ef9 100644
--- a/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memmove_chk-1.ll
@@ -18,7 +18,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T2* @t2 to i8*), i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
 ;
   %dst = bitcast %struct.T1* @t1 to i8*
@@ -30,7 +30,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memmove.p0i8.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T1* @t1 to i8*), i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T3* @t3 to i8*), i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T1* @t1 to i8*)
 ;
   %dst = bitcast %struct.T1* @t1 to i8*
diff --git a/llvm/test/Transforms/InstCombine/mempcpy.ll b/llvm/test/Transforms/InstCombine/mempcpy.ll
index 7aa6a31a61ac3..79158a3a0a6d5 100644
--- a/llvm/test/Transforms/InstCombine/mempcpy.ll
+++ b/llvm/test/Transforms/InstCombine/mempcpy.ll
@@ -45,7 +45,7 @@ define i8* @memcpy_small_const_n(i8* %d, i8* nocapture readonly %s) {
 
 define i8* @memcpy_big_const_n(i8* %d, i8* nocapture readonly %s) {
 ; CHECK-LABEL: @memcpy_big_const_n(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 1 dereferenceable(1024) [[D:%.*]], i8* align 1 dereferenceable(1024) [[S:%.*]], i64 1024, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 1 dereferenceable(1024) [[D:%.*]], i8* nonnull align 1 dereferenceable(1024) [[S:%.*]], i64 1024, i1 false)
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, i8* [[D]], i64 1024
 ; CHECK-NEXT:    ret i8* [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/memrchr.ll b/llvm/test/Transforms/InstCombine/memrchr.ll
new file mode 100644
index 0000000000000..feb384702f5ce
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/memrchr.ll
@@ -0,0 +1,55 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+
+declare i8* @memrchr(i8*, i32, i32)
+
+define i8* @test1(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* nonnull [[STR:%.*]], i32 [[C:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* nonnull %str, i32 %c, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test3(i8* %str, i32 %c) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test4(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test5(i8* %str, i32 %c) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @memrchr(i8* [[STR:%.*]], i32 [[C:%.*]], i32 0)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @memrchr(i8* %str, i32 %c, i32 0)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/memset-1.ll b/llvm/test/Transforms/InstCombine/memset-1.ll
index 7b6341d34b349..9ce8795c65623 100644
--- a/llvm/test/Transforms/InstCombine/memset-1.ll
+++ b/llvm/test/Transforms/InstCombine/memset-1.ll
@@ -49,7 +49,7 @@ define i8* @malloc_and_memset_intrinsic(i32 %n) #0 {
 define i8* @notmalloc_memset(i32 %size, i8*(i32)* %notmalloc) {
 ; CHECK-LABEL: @notmalloc_memset(
 ; CHECK-NEXT:    [[CALL1:%.*]] = call i8* [[NOTMALLOC:%.*]](i32 [[SIZE:%.*]]) #0
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL1]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[CALL1]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    ret i8* [[CALL1]]
 ;
   %call1 = call i8* %notmalloc(i32 %size) #1
@@ -68,7 +68,7 @@ define float* @pr25892(i32 %size) #0 {
 ; CHECK-NEXT:    br i1 [[CMP]], label [[CLEANUP:%.*]], label [[IF_END:%.*]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[BC:%.*]] = bitcast i8* [[CALL]] to float*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[CALL]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[CALL]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    br label [[CLEANUP]]
 ; CHECK:       cleanup:
 ; CHECK-NEXT:    [[RETVAL_0:%.*]] = phi float* [ [[BC]], [[IF_END]] ], [ null, [[ENTRY:%.*]] ]
@@ -93,7 +93,7 @@ define i8* @buffer_is_modified_then_memset(i32 %size) {
 ; CHECK-LABEL: @buffer_is_modified_then_memset(
 ; CHECK-NEXT:    [[PTR:%.*]] = tail call i8* @malloc(i32 [[SIZE:%.*]]) #0
 ; CHECK-NEXT:    store i8 1, i8* [[PTR]], align 1
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 [[PTR]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR]], i8 0, i32 [[SIZE]], i1 false) #0
 ; CHECK-NEXT:    ret i8* [[PTR]]
 ;
   %ptr = tail call i8* @malloc(i32 %size) #1
@@ -102,6 +102,103 @@ define i8* @buffer_is_modified_then_memset(i32 %size) {
   ret i8* %memset
 }
 
+define i8* @memset_size_select(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(10) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+
+define i8* @memset_size_select2(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select2(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable(80) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_size_select3(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select3(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false)
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* dereferenceable_or_null(40) %ptr, i32 0, i32 %size)
+  ret i8* %memset
+}
+
+define i8* @memset_size_select4(i1 %b, i8* %ptr) {
+; CHECK-LABEL: @memset_size_select4(
+; CHECK-NEXT:    [[SIZE:%.*]] = select i1 [[B:%.*]], i32 10, i32 50
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = select i1 %b, i32 10, i32 50
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_size_ashr(i1 %b, i8* %ptr, i32 %v) {
+; CHECK-LABEL: @memset_size_ashr(
+; CHECK-NEXT:    [[SIZE:%.*]] = ashr i32 -2, [[V:%.*]]
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR:%.*]], i8 0, i32 [[SIZE]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %size = ashr i32 -2, %v
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+define i8* @memset_attrs1(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs1(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* align 1 dereferenceable_or_null(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; be sure to drop nonnull since size is unknown and can be 0
+; do not change dereferenceable attribute
+define i8* @memset_attrs2(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs2(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; size is unknown, just copy attrs, no changes in attrs
+define i8* @memset_attrs3(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs3(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable_or_null(40) [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull dereferenceable_or_null(40) %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+; be sure to drop nonnull since size is unknown and can be 0
+define i8* @memset_attrs4(i1 %b, i8* %ptr, i32 %size) {
+; CHECK-LABEL: @memset_attrs4(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 [[PTR:%.*]], i8 0, i32 [[SIZE:%.*]], i1 false) #0
+; CHECK-NEXT:    ret i8* [[PTR]]
+;
+  %memset = tail call i8* @memset(i8* nonnull %ptr, i32 0, i32 %size) #1
+  ret i8* %memset
+}
+
+
 attributes #0 = { nounwind ssp uwtable }
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readnone }
diff --git a/llvm/test/Transforms/InstCombine/memset_chk-1.ll b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
index e6e9427bc1ca6..ad45fb86299d4 100644
--- a/llvm/test/Transforms/InstCombine/memset_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/memset_chk-1.ll
@@ -14,7 +14,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
 ;
   %dst = bitcast %struct.T* @t to i8*
@@ -25,7 +25,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
 ;
   %dst = bitcast %struct.T* @t to i8*
@@ -36,7 +36,7 @@ define i8* @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i64(i8* nonnull align 4 dereferenceable(1824) bitcast (%struct.T* @t to i8*), i8 0, i64 1824, i1 false)
 ; CHECK-NEXT:    ret i8* bitcast (%struct.T* @t to i8*)
 ;
   %dst = bitcast %struct.T* @t to i8*
@@ -73,11 +73,11 @@ define i8* @test_no_simplify2() {
 define i32 @test_rauw(i8* %a, i8* %b, i8** %c) {
 ; CHECK-LABEL: @test_rauw(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL49:%.*]] = call i64 @strlen(i8* [[A:%.*]])
+; CHECK-NEXT:    [[CALL49:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[A:%.*]])
 ; CHECK-NEXT:    [[ADD180:%.*]] = add i64 [[CALL49]], 1
 ; CHECK-NEXT:    [[YO107:%.*]] = call i64 @llvm.objectsize.i64.p0i8(i8* [[B:%.*]], i1 false, i1 false, i1 false)
 ; CHECK-NEXT:    [[CALL50:%.*]] = call i8* @__memmove_chk(i8* [[B]], i8* [[A]], i64 [[ADD180]], i64 [[YO107]])
-; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* [[B]])
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[B]])
 ; CHECK-NEXT:    [[STRCHR2:%.*]] = getelementptr i8, i8* [[B]], i64 [[STRLEN]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8** [[C:%.*]] to i64*
 ; CHECK-NEXT:    [[D1:%.*]] = load i64, i64* [[TMP0]], align 8
diff --git a/llvm/test/Transforms/InstCombine/minmax-fold.ll b/llvm/test/Transforms/InstCombine/minmax-fold.ll
index 264e579db1d6f..47588565c5ced 100644
--- a/llvm/test/Transforms/InstCombine/minmax-fold.ll
+++ b/llvm/test/Transforms/InstCombine/minmax-fold.ll
@@ -1080,8 +1080,8 @@ define i37 @add_umax_constant_limit(i37 %x) {
 
 define i37 @add_umax_simplify(i37 %x) {
 ; CHECK-LABEL: @add_umax_simplify(
-; CHECK-NEXT:    [[R:%.*]] = add nuw i37 [[X:%.*]], 42
-; CHECK-NEXT:    ret i37 [[R]]
+; CHECK-NEXT:    [[A:%.*]] = add i37 [[X:%.*]], 42
+; CHECK-NEXT:    ret i37 [[A]]
 ;
   %a = add nuw i37 %x, 42
   %c = icmp ugt i37 %a, 42
diff --git a/llvm/test/Transforms/InstCombine/mul.ll b/llvm/test/Transforms/InstCombine/mul.ll
index b1dc9d51332d7..f106026762bab 100644
--- a/llvm/test/Transforms/InstCombine/mul.ll
+++ b/llvm/test/Transforms/InstCombine/mul.ll
@@ -517,3 +517,81 @@ define i64 @test_mul_canonicalize_neg_is_not_undone(i64 %L1) {
   %B4 = mul i64 %B8, %L1
   ret i64 %B4
 }
+
+define i32 @negate_if_true(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND:%.*]], i32 [[TMP1]], i32 [[X]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sel = select i1 %cond, i32 -1, i32 1
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+define i32 @negate_if_false(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_false(
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND:%.*]], i32 [[X]], i32 [[TMP1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sel = select i1 %cond, i32 1, i32 -1
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+define <2 x i8> @negate_if_true_commute(<2 x i8> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_commute(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 42>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub nsw <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select i1 [[COND:%.*]], <2 x i8> [[TMP1]], <2 x i8> [[X]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 42>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x i8> <i8 -1, i8 -1>, <2 x i8> <i8 1, i8 1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
+
+define <2 x i8> @negate_if_false_commute(<2 x i8> %px, <2 x i1> %cond) {
+; CHECK-LABEL: @negate_if_false_commute(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 5>, [[PX:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub <2 x i8> zeroinitializer, [[X]]
+; CHECK-NEXT:    [[R:%.*]] = select <2 x i1> [[COND:%.*]], <2 x i8> [[X]], <2 x i8> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 5>, %px  ; thwart complexity-based canonicalization
+  %sel = select <2 x i1> %cond, <2 x i8> <i8 1, i8 undef>, <2 x i8> <i8 -1, i8 -1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
+
+; Negative test
+
+define i32 @negate_if_true_extra_use(i32 %x, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_extra_use(
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], i32 -1, i32 1
+; CHECK-NEXT:    call void @use32(i32 [[SEL]])
+; CHECK-NEXT:    [[R:%.*]] = mul i32 [[SEL]], [[X:%.*]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %sel = select i1 %cond, i32 -1, i32 1
+  call void @use32(i32 %sel)
+  %r = mul i32 %sel, %x
+  ret i32 %r
+}
+
+; Negative test
+
+define <2 x i8> @negate_if_true_wrong_constant(<2 x i8> %px, i1 %cond) {
+; CHECK-LABEL: @negate_if_true_wrong_constant(
+; CHECK-NEXT:    [[X:%.*]] = sdiv <2 x i8> <i8 42, i8 42>, [[PX:%.*]]
+; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[COND:%.*]], <2 x i8> <i8 -1, i8 0>, <2 x i8> <i8 1, i8 1>
+; CHECK-NEXT:    [[R:%.*]] = mul <2 x i8> [[X]], [[SEL]]
+; CHECK-NEXT:    ret <2 x i8> [[R]]
+;
+  %x = sdiv <2 x i8> <i8 42, i8 42>, %px  ; thwart complexity-based canonicalization
+  %sel = select i1 %cond, <2 x i8> <i8 -1, i8 0>, <2 x i8> <i8 1, i8 1>
+  %r = mul <2 x i8> %x, %sel
+  ret <2 x i8> %r
+}
diff --git a/llvm/test/Transforms/InstCombine/objsize.ll b/llvm/test/Transforms/InstCombine/objsize.ll
index ebf3c33b648a2..15f6b44a3a0ea 100644
--- a/llvm/test/Transforms/InstCombine/objsize.ll
+++ b/llvm/test/Transforms/InstCombine/objsize.ll
@@ -141,7 +141,7 @@ define i32 @test4(i8** %esc) nounwind ssp {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = alloca [[STRUCT_DATA:%.*]], align 8
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast %struct.data* [[TMP0]] to i8*
-; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 8 dereferenceable(1824) [[TMP1]], i8 0, i32 1824, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 8 dereferenceable(1824) [[TMP1]], i8 0, i32 1824, i1 false) #0
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i8** [[ESC:%.*]] to %struct.data**
 ; CHECK-NEXT:    store %struct.data* [[TMP0]], %struct.data** [[TMP2]], align 4
 ; CHECK-NEXT:    ret i32 0
@@ -163,7 +163,7 @@ define i8* @test5(i32 %n) nounwind ssp {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call noalias dereferenceable_or_null(20) i8* @malloc(i32 20) #0
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i8*, i8** @s, align 8
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(10) [[TMP0]], i8* align 1 dereferenceable(10) [[TMP1]], i32 10, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(10) [[TMP0]], i8* nonnull align 1 dereferenceable(10) [[TMP1]], i32 10, i1 false) #0
 ; CHECK-NEXT:    ret i8* [[TMP0]]
 ;
 entry:
@@ -227,7 +227,7 @@ declare noalias i8* @strndup(i8* nocapture, i32) nounwind
 
 define i32 @test9(i8** %esc) {
 ; CHECK-LABEL: @test9(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0)) #0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0)) #0
 ; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    ret i32 8
 ;
@@ -239,7 +239,7 @@ define i32 @test9(i8** %esc) {
 
 define i32 @test10(i8** %esc) {
 ; CHECK-LABEL: @test10(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 3) #0
+; CHECK-NEXT:    [[CALL:%.*]] = tail call dereferenceable_or_null(4) i8* @strndup(i8* dereferenceable(8) getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 3) #0
 ; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    ret i32 4
 ;
@@ -251,8 +251,8 @@ define i32 @test10(i8** %esc) {
 
 define i32 @test11(i8** %esc) {
 ; CHECK-LABEL: @test11(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 7) #0
-; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    ret i32 8
 ;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 7) nounwind
@@ -263,8 +263,8 @@ define i32 @test11(i8** %esc) {
 
 define i32 @test12(i8** %esc) {
 ; CHECK-LABEL: @test12(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 8) #0
-; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    ret i32 8
 ;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 8) nounwind
@@ -275,8 +275,8 @@ define i32 @test12(i8** %esc) {
 
 define i32 @test13(i8** %esc) {
 ; CHECK-LABEL: @test13(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0), i32 57) #0
-; CHECK-NEXT:    store i8* [[CALL]], i8** [[ESC:%.*]], align 8
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(8) i8* @strdup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i32 0, i32 0))
+; CHECK-NEXT:    store i8* [[STRDUP]], i8** [[ESC:%.*]], align 8
 ; CHECK-NEXT:    ret i32 8
 ;
   %call = tail call i8* @strndup(i8* getelementptr inbounds ([8 x i8], [8 x i8]* @.str, i64 0, i64 0), i32 57) nounwind
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll
new file mode 100644
index 0000000000000..bcaf6440efc7a
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-a.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   a)  (x & ((1 << maskNbits) - 1)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 1, %t0 ; shifting by nbits-1
+  %t2 = add i32 %t1, -1
+  %t3 = and i32 %t2, %x
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 undef, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 0, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 undef, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = shl <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %t0
+  %t2 = add <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 1, %t0 ; shifting by nbits-1
+  %t2 = add i32 %t1, -1
+  %t3 = and i32 %t2, %x ; this mask must be one-use.
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t3) ; BAD
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll
new file mode 100644
index 0000000000000..faf069cfec9d5
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-b.ll
@@ -0,0 +1,157 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   b)  (x & (~(-1 << maskNbits))) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (~(-1 << (maskNbits+shiftNbits)))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[T1]], -1
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 -1, %t0 ; shifting by nbits-1
+  %t2 = xor i32 %t1, -1
+  %t3 = and i32 %t2, %x
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 0, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 undef, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = add <8 x i32> [[NBITS:%.*]], <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+; CHECK-NEXT:    [[T1:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor <8 x i32> [[T1]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    [[T4:%.*]] = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, [[NBITS]]
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T4]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T4]]
+; CHECK-NEXT:    [[T5:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 0, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T5]]
+;
+  %t0 = add <8 x i32> %nbits, <i32 -33, i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32>
+  %t1 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %t0
+  %t2 = xor <8 x i32> %t1, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  %t3 = and <8 x i32> %t2, %x
+  %t4 = sub <8 x i32> <i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32, i32 32>, %nbits
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t2)
+  call void @use8xi32(<8 x i32> %t4)
+  %t5 = shl <8 x i32> %t3, %t4
+  ret <8 x i32> %t5
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = add i32 [[NBITS:%.*]], -1
+; CHECK-NEXT:    [[T1:%.*]] = shl i32 -1, [[T0]]
+; CHECK-NEXT:    [[T2:%.*]] = xor i32 [[T1]], -1
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[T2]], [[X:%.*]]
+; CHECK-NEXT:    [[T4:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    call void @use32(i32 [[T4]])
+; CHECK-NEXT:    [[T5:%.*]] = shl i32 [[T3]], [[T4]]
+; CHECK-NEXT:    ret i32 [[T5]]
+;
+  %t0 = add i32 %nbits, -1
+  %t1 = shl i32 -1, %t0 ; shifting by nbits-1
+  %t2 = xor i32 %t1, -1
+  %t3 = and i32 %t2, %x ; this mask must be one-use.
+  %t4 = sub i32 32, %nbits
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2)
+  call void @use32(i32 %t3) ; BAD
+  call void @use32(i32 %t4)
+  %t5 = shl i32 %t3, %t4
+  ret i32 %t5
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll
new file mode 100644
index 0000000000000..2c6bb31ebf772
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-c.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   c)  (x & (-1 >> maskNbits)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = lshr i32 -1, %nbits
+  %t1 = and i32 %t0, %x
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 -1, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = lshr <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = and <8 x i32> %t0, %x
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = lshr i32 -1, %nbits
+  %t1 = and i32 %t0, %x
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1) ; BAD
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll
new file mode 100644
index 0000000000000..fcbb7eb50d5d2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-d.ll
@@ -0,0 +1,137 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   d)  (x & ((-1 << maskNbits) >> maskNbits)) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %t0 = shl i32 -1, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = and i32 %t1, %x
+  %t3 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t3)
+  %t4 = shl i32 %t2, %t3 ; shift is smaller than mask
+  ret i32 %t4
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t2_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+define <8 x i32> @t2_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 -1, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+define <8 x i32> @t2_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t2_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr <8 x i32> [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T3:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T1]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T3]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X:%.*]], [[T3]]
+; CHECK-NEXT:    [[T4:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T4]]
+;
+  %t0 = shl <8 x i32> <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = and <8 x i32> %t1, %x
+  %t3 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t1)
+  call void @use8xi32(<8 x i32> %t3)
+  %t4 = shl <8 x i32> %t2, %t3 ; shift is smaller than mask
+  ret <8 x i32> %t4
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
+; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    call void @use32(i32 [[T3]])
+; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
+; CHECK-NEXT:    ret i32 [[T4]]
+;
+  %t0 = shl i32 -1, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = and i32 %t1, %x
+  %t3 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1)
+  call void @use32(i32 %t2) ; BAD
+  call void @use32(i32 %t3)
+  %t4 = shl i32 %t2, %t3 ; shift is smaller than mask
+  ret i32 %t4
+}
diff --git a/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll
new file mode 100644
index 0000000000000..2b3fb5098aacf
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/partally-redundant-left-shift-input-masking-variant-e.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have some pattern that leaves only some low bits set, and then performs
+; left-shift of those bits, we can combine those two shifts into a shift+mask.
+
+; There are many variants to this pattern:
+;   e)  ((x << maskNbits) l>> maskNbits) << shiftNbits
+; simplify to:
+;   (x << shiftNbits) & (-1 >> ((-(maskNbits+shiftNbits))+32))
+
+; Simple tests.
+
+declare void @use32(i32)
+
+define i32 @t0_basic(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @t0_basic(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and i32 [[TMP1]], 2147483647
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = shl i32 %x, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
+
+; Vectors
+
+declare void @use8xi32(<8 x i32>)
+
+define <8 x i32> @t1_vec_splat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_splat_undef(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_splat_undef(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647, i32 -1, i32 2147483647>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef, i32 -1>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+define <8 x i32> @t1_vec_nonsplat(<8 x i32> %x, <8 x i32> %nbits) {
+; CHECK-LABEL: @t1_vec_nonsplat(
+; CHECK-NEXT:    [[T0:%.*]] = shl <8 x i32> [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T2:%.*]] = add <8 x i32> [[NBITS]], <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T0]])
+; CHECK-NEXT:    call void @use8xi32(<8 x i32> [[T2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <8 x i32> [[X]], [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = and <8 x i32> [[TMP1]], <i32 undef, i32 1, i32 2147483647, i32 -1, i32 -1, i32 -1, i32 -1, i32 undef>
+; CHECK-NEXT:    ret <8 x i32> [[T3]]
+;
+  %t0 = shl <8 x i32> %x, %nbits
+  %t1 = lshr <8 x i32> %t0, %nbits
+  %t2 = add <8 x i32> %nbits, <i32 -32, i32 -31, i32 -1, i32 0, i32 1, i32 31, i32 32, i32 33>
+  call void @use8xi32(<8 x i32> %t0)
+  call void @use8xi32(<8 x i32> %t2)
+  %t3 = shl <8 x i32> %t1, %t2 ; shift is smaller than mask
+  ret <8 x i32> %t3
+}
+
+; Extra uses.
+
+define i32 @n3_extrause(i32 %x, i32 %nbits) {
+; CHECK-LABEL: @n3_extrause(
+; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
+; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
+; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
+; CHECK-NEXT:    call void @use32(i32 [[T0]])
+; CHECK-NEXT:    call void @use32(i32 [[T1]])
+; CHECK-NEXT:    call void @use32(i32 [[T2]])
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
+;
+  %t0 = shl i32 %x, %nbits
+  %t1 = lshr i32 %t0, %nbits
+  %t2 = add i32 %nbits, -1
+  call void @use32(i32 %t0)
+  call void @use32(i32 %t1) ; BAD
+  call void @use32(i32 %t2)
+  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
+  ret i32 %t3
+}
diff --git a/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll b/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll
new file mode 100644
index 0000000000000..1e5bf5190a591
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr43376-getFlippedStrictnessPredicateAndConstant-assert.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+; We used to hit an assertion in getFlippedStrictnessPredicateAndConstant due
+; to assuming that edge cases such as %cmp (ult x, 0) already has been
+; simplified. But that depends on the worklist order, so that is not always
+; guaranteed.
+
+define i16 @d(i16* %d.a, i16* %d.b) {
+; CHECK-LABEL: @d(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T0:%.*]] = load i16, i16* [[D_A:%.*]], align 1
+; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i16 [[T0]], 0
+; CHECK-NEXT:    br i1 [[TOBOOL]], label [[LAND_END:%.*]], label [[LAND_RHS:%.*]]
+; CHECK:       land.rhs:
+; CHECK-NEXT:    br label [[LAND_END]]
+; CHECK:       land.end:
+; CHECK-NEXT:    ret i16 -1
+;
+entry:
+  %t0 = load i16, i16* %d.a, align 1
+  %tobool = icmp ne i16 %t0, 0
+  br i1 %tobool, label %land.rhs, label %land.end
+
+land.rhs:
+  %t1 = load i16, i16* %d.b, align 1
+  %cmp = icmp ult i16 %t1, 0
+  br label %land.end
+
+land.end:
+  %t2 = phi i1 [ false, %entry ], [ %cmp, %land.rhs ]
+  %land.ext = zext i1 %t2 to i16
+  %mul = mul nsw i16 %land.ext, 3
+  %neg = xor i16 %mul, -1
+  ret i16 %neg
+}
diff --git a/llvm/test/Transforms/InstCombine/printf-1.ll b/llvm/test/Transforms/InstCombine/printf-1.ll
index 9d13b360f8138..6249ca5b1c027 100644
--- a/llvm/test/Transforms/InstCombine/printf-1.ll
+++ b/llvm/test/Transforms/InstCombine/printf-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the printf library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -22,110 +23,164 @@ declare i32 @printf(i8*, ...)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify1(
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("x") -> putchar('x'), even for '%'.
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify2(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Special case: printf("%%") -> putchar('%').
 
 define void @test_simplify2b() {
 ; CHECK-LABEL: @test_simplify2b(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify2b(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @h2, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 37)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify3(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 37)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @percent, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 37)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("foo\n") -> puts("foo").
 
 define void @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify4(
+; CHECK-IPRINTF-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* [[$STR]], i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("%c", chr) -> putchar(chr).
 
 define void @test_simplify5() {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify5(
+; CHECK-IPRINTF-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_c, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8 104)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf("%s\n", str) -> puts(str).
 
 define void @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_simplify6(
+; CHECK-IPRINTF-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Check printf(format, ...) -> iprintf(format, ...) if no floating point.
 
 define void @test_simplify7() {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_simplify7(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_d, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, i32 187)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify1() {
+; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-NEXT:    ret void
+;
 ; CHECK-IPRINTF-LABEL: @test_no_simplify1(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-IPRINTF-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
   call i32 (i8*, ...) @printf(i8* %fmt, double 1.87)
-; CHECK-IPRINTF-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
   ret void
-; CHECK-IPRINTF-NEXT: ret void
 }
 
 define void @test_no_simplify2(i8* %fmt, double %d) {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-NEXT:    ret void
+;
+; CHECK-IPRINTF-LABEL: @test_no_simplify2(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
+; CHECK-IPRINTF-NEXT:    ret void
+;
   call i32 (i8*, ...) @printf(i8* %fmt, double %d)
-; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* %fmt, double %d)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define i32 @test_no_simplify3() {
 ; CHECK-LABEL: @test_no_simplify3(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 (i8*, ...) @printf(i8* nonnull dereferenceable(1) getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+; CHECK-IPRINTF-LABEL: @test_no_simplify3(
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, ...) @iprintf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
+; CHECK-IPRINTF-NEXT:    ret i32 [[TMP1]]
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   %ret = call i32 (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([2 x i8], [2 x i8]* @h, i32 0, i32 0))
   ret i32 %ret
-; CHECK-NEXT: ret i32 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/printf-2.ll b/llvm/test/Transforms/InstCombine/printf-2.ll
index fbd5b1bb96c40..66678bc557cc7 100644
--- a/llvm/test/Transforms/InstCombine/printf-2.ll
+++ b/llvm/test/Transforms/InstCombine/printf-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the printf library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -16,38 +17,42 @@ declare void @printf(i8*, ...)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 104)
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [2 x i8], [2 x i8]* @h, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @putchar(i32 104)
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([12 x i8], [12 x i8]* @str, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[PUTS:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [4 x i8], [4 x i8]* @percent_s, i32 0, i32 0
   %str = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0))
   ret void
-; CHECK-NEXT: ret void
 }
 
 define void @test_simplify7() {
 ; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 97)
+; CHECK-NEXT:    ret void
+;
   %fmt = getelementptr [3 x i8], [3 x i8]* @format_str, i32 0, i32 0
   %str = getelementptr [2 x i8], [2 x i8]* @charstr, i32 0, i32 0
   call void (i8*, ...) @printf(i8* %fmt, i8* %str)
-; CHECK-NEXT: call i32 @putchar(i32 97)
   ret void
-; CHECK-NEXT: ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/puts-1.ll b/llvm/test/Transforms/InstCombine/puts-1.ll
index 21028684851d3..aa18a30f4f6df 100644
--- a/llvm/test/Transforms/InstCombine/puts-1.ll
+++ b/llvm/test/Transforms/InstCombine/puts-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the puts library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -12,20 +13,22 @@ declare i32 @puts(i8*)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[PUTCHAR:%.*]] = call i32 @putchar(i32 10)
+; CHECK-NEXT:    ret void
+;
   %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   call i32 @puts(i8* %str)
-; CHECK-NEXT: call i32 @putchar(i32 10)
   ret void
-; CHECK-NEXT: ret void
 }
 
 ; Don't simplify if the return value is used.
 
 define i32 @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @puts(i8* nonnull dereferenceable(1) getelementptr inbounds ([1 x i8], [1 x i8]* @empty, i32 0, i32 0))
+; CHECK-NEXT:    ret i32 [[RET]]
+;
   %str = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
   %ret = call i32 @puts(i8* %str)
-; CHECK-NEXT: call i32 @puts(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @empty, i32 0, i32 0))
   ret i32 %ret
-; CHECK-NEXT: ret i32 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
index 95527286a3822..e7145fcccd7d7 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-a.ll
@@ -407,28 +407,3 @@ define i32 @n13_not_minus_one(i32 %x, i32 %nbits) {
   %t4 = shl i32 %t2, %t3
   ret i32 %t4
 }
-
-define i32 @n14_insifficient_sum(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n14_insifficient_sum(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = add nsw i32 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub i32 31, [[NBITS]]
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 1, %nbits
-  %t1 = add nsw i32 %t0, -1
-  %t2 = and i32 %t1, %x
-  %t3 = sub i32 31, %nbits ; summary shift amount is less than 32
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  call void @use32(i32 %t3)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
index 347010dd106c3..94e786d38b088 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-b.ll
@@ -382,28 +382,3 @@ define i32 @n12_not_minus_one(i32 %x, i32 %nbits) {
   %t4 = shl i32 %t2, %t3
   ret i32 %t4
 }
-
-define i32 @n13_insufficient_sum(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n13_insufficient_sum(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = xor i32 [[T0]], -1
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = sub i32 31, [[NBITS]]
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    call void @use32(i32 [[T3]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 -1, %nbits
-  %t1 = xor i32 %t0, -1
-  %t2 = and i32 %t1, %x
-  %t3 = sub i32 31, %nbits ; summary shift amount is less than 32
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  call void @use32(i32 %t3)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
index 79a540d3452c3..fb480e3f3ecd7 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-c.ll
@@ -281,23 +281,3 @@ define i32 @n12_not_minus_one(i32 %x, i32 %nbits) {
   %t2 = shl i32 %t1, %nbits
   ret i32 %t2
 }
-
-define i32 @n13_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n13_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = lshr i32 -1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = and i32 [[T0]], [[X:%.*]]
-; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    ret i32 [[T2]]
-;
-  %t0 = lshr i32 -1, %nbits
-  %t1 = and i32 %t0, %x
-  %t2 = add i32 %nbits, -1
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
-  ret i32 %t2
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
index e01ac41055f64..0871e82dd28de 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-d.ll
@@ -361,26 +361,3 @@ define i32 @n13_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
   %t3 = shl i32 %t2, %nbits1
   ret i32 %t3
 }
-
-define i32 @n14_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n14_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 -1, [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
-; CHECK-NEXT:    [[T2:%.*]] = and i32 [[T1]], [[X:%.*]]
-; CHECK-NEXT:    [[T3:%.*]] = add i32 [[NBITS]], -1
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    [[T4:%.*]] = shl i32 [[T2]], [[T3]]
-; CHECK-NEXT:    ret i32 [[T4]]
-;
-  %t0 = shl i32 -1, %nbits
-  %t1 = lshr i32 %t0, %nbits
-  %t2 = and i32 %t1, %x
-  %t3 = add i32 %nbits, -1 ; shift is smaller than mask
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t4 = shl i32 %t2, %t3
-  ret i32 %t4
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
index 104340b08d5e9..f3682ef8cdde8 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-e.ll
@@ -234,23 +234,3 @@ define i32 @n10_different_shamts1(i32 %x, i32 %nbits0, i32 %nbits1) {
   %t2 = shl i32 %t1, %nbits1
   ret i32 %t2
 }
-
-define i32 @n11_shamt_is_smaller(i32 %x, i32 %nbits) {
-; CHECK-LABEL: @n11_shamt_is_smaller(
-; CHECK-NEXT:    [[T0:%.*]] = shl i32 [[X:%.*]], [[NBITS:%.*]]
-; CHECK-NEXT:    [[T1:%.*]] = lshr i32 [[T0]], [[NBITS]]
-; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
-; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
-; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    ret i32 [[T2]]
-;
-  %t0 = shl i32 %x, %nbits
-  %t1 = lshr i32 %t0, %nbits
-  %t2 = add i32 %nbits, -1
-  call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
-  call void @use32(i32 %t2)
-  %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
-  ret i32 %t2
-}
diff --git a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
index 331d85edb7316..3e2bb330ab6c2 100644
--- a/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
+++ b/llvm/test/Transforms/InstCombine/redundant-left-shift-input-masking-variant-f.ll
@@ -241,16 +241,15 @@ define i32 @n11_shamt_is_smaller(i32 %x, i32 %nbits) {
 ; CHECK-NEXT:    [[T1:%.*]] = ashr i32 [[T0]], [[NBITS]]
 ; CHECK-NEXT:    [[T2:%.*]] = add i32 [[NBITS]], -1
 ; CHECK-NEXT:    call void @use32(i32 [[T0]])
-; CHECK-NEXT:    call void @use32(i32 [[T1]])
 ; CHECK-NEXT:    call void @use32(i32 [[T2]])
-; CHECK-NEXT:    ret i32 [[T2]]
+; CHECK-NEXT:    [[T3:%.*]] = shl i32 [[T1]], [[T2]]
+; CHECK-NEXT:    ret i32 [[T3]]
 ;
   %t0 = shl i32 %x, %nbits
   %t1 = ashr i32 %t0, %nbits
   %t2 = add i32 %nbits, -1
   call void @use32(i32 %t0)
-  call void @use32(i32 %t1)
   call void @use32(i32 %t2)
   %t3 = shl i32 %t1, %t2 ; shift is smaller than mask
-  ret i32 %t2
+  ret i32 %t3
 }
diff --git a/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
index 1a376c42648e0..5b48b5e30ff45 100644
--- a/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
+++ b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
@@ -1,10 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt %s -instcombine -S | FileCheck %s
 
-; Here we add unsigned two values, check that addition did not underflow AND
-; that the result is non-zero. This can be simplified just to a comparison
-; between the base and negated offset.
-
 declare void @use8(i8)
 
 declare void @use1(i1)
@@ -35,10 +31,9 @@ define i1 @t1(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -58,10 +53,9 @@ define i1 @t2(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %offset, 0
   call void @llvm.assume(i1 %cmp)
@@ -83,9 +77,9 @@ define i1 @t3_oneuse0(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -104,11 +98,11 @@ define i1 @t4_oneuse1(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -153,10 +147,9 @@ define i1 @t6_commutativity0(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -174,10 +167,9 @@ define i1 @t7_commutativity1(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -185,7 +177,7 @@ define i1 @t7_commutativity1(i8 %base, i8 %offset) {
   %adjusted = add i8 %base, %offset
   call void @use8(i8 %adjusted)
   %not_null = icmp ne i8 %adjusted, 0
-  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %no_underflow = icmp ugt i8 %base, %adjusted ; swapped
   %r = and i1 %not_null, %no_underflow
   ret i1 %r
 }
@@ -195,10 +187,9 @@ define i1 @t7_commutativity3(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -206,7 +197,7 @@ define i1 @t7_commutativity3(i8 %base, i8 %offset) {
   %adjusted = add i8 %base, %offset
   call void @use8(i8 %adjusted)
   %not_null = icmp ne i8 %adjusted, 0
-  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %no_underflow = icmp ugt i8 %base, %adjusted ; swapped
   %r = and i1 %no_underflow, %not_null ; swapped
   ret i1 %r
 }
@@ -218,10 +209,9 @@ define i1 @t8(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
 ;
   %cmp = icmp slt i8 %base, 0
   call void @llvm.assume(i1 %cmp)
@@ -233,3 +223,25 @@ define i1 @t8(i8 %base, i8 %offset) {
   %r = or i1 %not_null, %no_underflow
   ret i1 %r
 }
+
+; The comparison can be with any of the values being added.
+define i1 @t9(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %cmp = icmp slt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %offset
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll
new file mode 100644
index 0000000000000..b8a29e4fab25e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/result-of-add-of-negative-or-zero-is-non-zero-and-no-underflow.ll
@@ -0,0 +1,161 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use8(i8)
+
+declare void @use1(i1)
+declare void @llvm.assume(i1)
+
+define i1 @t0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We need to produce extra instruction, so one of icmp's must go away.
+define i1 @t1_oneuse0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1_oneuse0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t2_oneuse1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t2_oneuse1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @n3_oneuse2_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n3_oneuse2_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %no_underflow = icmp ule i8 %adjusted, %base
+  call void @use1(i1 %no_underflow)
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t4_commutativity0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t4_commutativity0(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %base
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t5_commutativity1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5_commutativity1(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t6_commutativity3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t6_commutativity3(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[OFFSET]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %adjusted ; swapped
+  %r = and i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; We could have the opposite question, did we get null or overflow happened?
+define i1 @t7(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = add i8 [[ADJUSTED]], -1
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp uge i8 [[TMP1]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; The comparison can be with any of the values being added.
+define i1 @t8(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i8 0, [[BASE]]
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ult i8 [[TMP1]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP2]]
+;
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %adjusted, %offset
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll b/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
index 1cf669acdd814..d46da5ec863e4 100644
--- a/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
+++ b/llvm/test/Transforms/InstCombine/result-of-usub-is-non-zero-and-no-overflow.ll
@@ -24,8 +24,8 @@ define i1 @t0_noncanonical_ignoreme(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -45,8 +45,8 @@ define i1 @t1(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -65,8 +65,7 @@ define i1 @t1_strict(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -115,8 +114,8 @@ define i1 @t3_commutability0(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -135,8 +134,8 @@ define i1 @t4_commutability1(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -155,8 +154,8 @@ define i1 @t5_commutability2(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -206,8 +205,8 @@ define i1 @t7(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
 ; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NULL]], [[UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -226,8 +225,7 @@ define i1 @t7_nonstrict(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
 ; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NULL]], [[UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 [[UNDERFLOW]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -272,8 +270,8 @@ define i1 @t9_commutative(i8 %base, i8 %offset) {
 ; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
 ; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NULL]], [[UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -292,12 +290,12 @@ define i1 @t10(i64 %base, i64* nonnull %offsetptr) {
 ; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i64 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %offset = ptrtoint i64* %offsetptr to i64
 
@@ -315,12 +313,12 @@ define i1 @t11_commutative(i64 %base, i64* nonnull %offsetptr) {
 ; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i64 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i64 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %offset = ptrtoint i64* %offsetptr to i64
 
@@ -339,12 +337,12 @@ define i1 @t12(i64 %base, i64* nonnull %offsetptr) {
 ; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i64 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i64 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %offset = ptrtoint i64* %offsetptr to i64
 
@@ -362,12 +360,12 @@ define i1 @t13(i64 %base, i64* nonnull %offsetptr) {
 ; CHECK-NEXT:    [[OFFSET:%.*]] = ptrtoint i64* [[OFFSETPTR:%.*]] to i64
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i64 [[BASE:%.*]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use64(i64 [[ADJUSTED]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i64 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i64 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i64 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i64 [[OFFSET]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
 ;
   %offset = ptrtoint i64* %offsetptr to i64
 
@@ -401,3 +399,32 @@ define i1 @t14_bad(i64 %base, i64 %offset) {
   %r = and i1 %not_null, %no_underflow
   ret i1 %r
 }
+
+define i1 @base_ult_offset(i8 %base, i8 %offset) {
+; CHECK-LABEL: @base_ult_offset(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+define i1 @base_uge_offset(i8 %base, i8 %offset) {
+; CHECK-LABEL: @base_uge_offset(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[TMP1]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/shift-by-signext.ll b/llvm/test/Transforms/InstCombine/shift-by-signext.ll
new file mode 100644
index 0000000000000..b049dfac6f855
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/shift-by-signext.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+; If we have a shift by sign-extended value, we can replace sign-extension
+; with zero-extension.
+
+define i32 @t0_shl(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t0_shl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t1_lshr(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t1_lshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = lshr i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = lshr i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @t2_ashr(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t2_ashr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  %r = ashr i32 %x, %shamt_wide
+  ret i32 %r
+}
+
+define <2 x i32> @t3_vec_shl(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t3_vec_shl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = shl <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = shl <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+define <2 x i32> @t4_vec_lshr(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t4_vec_lshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = lshr <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = lshr <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+define <2 x i32> @t5_vec_ashr(<2 x i32> %x, <2 x i8> %shamt) {
+; CHECK-LABEL: @t5_vec_ashr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext <2 x i8> [[SHAMT:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[R:%.*]] = ashr <2 x i32> [[X:%.*]], [[SHAMT_WIDE1]]
+; CHECK-NEXT:    ret <2 x i32> [[R]]
+;
+  %shamt_wide = sext <2 x i8> %shamt to <2 x i32>
+  %r = ashr <2 x i32> %x, %shamt_wide
+  ret <2 x i32> %r
+}
+
+define i32 @t6_twoshifts(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @t6_twoshifts(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    br label [[WORK:%.*]]
+; CHECK:       work:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    [[N0:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[N0]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+bb:
+  %shamt_wide = sext i8 %shamt to i32
+  br label %work
+work:
+  %n0 = shl i32 %x, %shamt_wide
+  %r = ashr i32 %n0, %shamt_wide
+  br label %end
+end:
+  ret i32 %r
+}
+
+; This is not valid for funnel shifts in general
+declare i7 @llvm.fshl.i7(i7 %a, i7 %b, i7 %c)
+declare i7 @llvm.fshr.i7(i7 %a, i7 %b, i7 %c)
+define i7 @n7_fshl(i7 %x, i7 %y, i6 %shamt) {
+; CHECK-LABEL: @n7_fshl(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i6 [[SHAMT:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = call i7 @llvm.fshl.i7(i7 [[X:%.*]], i7 [[Y:%.*]], i7 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i7 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i7
+  %r = call i7 @llvm.fshl.i7(i7 %x, i7 %y, i7 %shamt_wide)
+  ret i7 %r
+}
+define i7 @n8_fshr(i7 %x, i7 %y, i6 %shamt) {
+; CHECK-LABEL: @n8_fshr(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i6 [[SHAMT:%.*]] to i7
+; CHECK-NEXT:    [[R:%.*]] = call i7 @llvm.fshr.i7(i7 [[X:%.*]], i7 [[Y:%.*]], i7 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i7 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i7
+  %r = call i7 @llvm.fshr.i7(i7 %x, i7 %y, i7 %shamt_wide)
+  ret i7 %r
+}
+; And the cases that are safe are handled by SimplifyDemandedBits().
+declare i8 @llvm.fshl.i8(i8 %a, i8 %b, i8 %c)
+declare i8 @llvm.fshr.i8(i8 %a, i8 %b, i8 %c)
+define i8 @t9_fshl(i8 %x, i8 %y, i6 %shamt) {
+; CHECK-LABEL: @t9_fshl(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i6 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshl.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SHAMT_WIDE1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i8
+  %r = call i8 @llvm.fshl.i8(i8 %x, i8 %y, i8 %shamt_wide)
+  ret i8 %r
+}
+define i8 @t10_fshr(i8 %x, i8 %y, i6 %shamt) {
+; CHECK-LABEL: @t10_fshr(
+; CHECK-NEXT:    [[SHAMT_WIDE1:%.*]] = zext i6 [[SHAMT:%.*]] to i8
+; CHECK-NEXT:    [[R:%.*]] = call i8 @llvm.fshr.i8(i8 [[X:%.*]], i8 [[Y:%.*]], i8 [[SHAMT_WIDE1]])
+; CHECK-NEXT:    ret i8 [[R]]
+;
+  %shamt_wide = sext i6 %shamt to i8
+  %r = call i8 @llvm.fshr.i8(i8 %x, i8 %y, i8 %shamt_wide)
+  ret i8 %r
+}
+
+declare void @use32(i32)
+define i32 @n11_extrause(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @n11_extrause(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[SHAMT_WIDE]])
+; CHECK-NEXT:    [[R:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  call void @use32(i32 %shamt_wide)
+  %r = shl i32 %x, %shamt_wide
+  ret i32 %r
+}
+define i32 @n12_twoshifts_and_extrause(i32 %x, i8 %shamt) {
+; CHECK-LABEL: @n12_twoshifts_and_extrause(
+; CHECK-NEXT:    [[SHAMT_WIDE:%.*]] = sext i8 [[SHAMT:%.*]] to i32
+; CHECK-NEXT:    br label [[WORK:%.*]]
+; CHECK:       work:
+; CHECK-NEXT:    br label [[END:%.*]]
+; CHECK:       end:
+; CHECK-NEXT:    [[N0:%.*]] = shl i32 [[X:%.*]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    [[R:%.*]] = ashr i32 [[N0]], [[SHAMT_WIDE]]
+; CHECK-NEXT:    call void @use32(i32 [[SHAMT_WIDE]])
+; CHECK-NEXT:    ret i32 [[R]]
+;
+  %shamt_wide = sext i8 %shamt to i32
+  br label %work
+work:
+  %n0 = shl i32 %x, %shamt_wide
+  %r = ashr i32 %n0, %shamt_wide
+  br label %end
+end:
+  call void @use32(i32 %shamt_wide)
+  ret i32 %r
+}
diff --git a/llvm/test/Transforms/InstCombine/snprintf.ll b/llvm/test/Transforms/InstCombine/snprintf.ll
index f323bf9bf5737..0b9d88048d740 100644
--- a/llvm/test/Transforms/InstCombine/snprintf.ll
+++ b/llvm/test/Transforms/InstCombine/snprintf.ll
@@ -10,13 +10,14 @@ declare i32 @snprintf(i8*, i64, i8*, ...) #1
 
 define void @test_not_const_fmt(i8* %buf, i8* %fmt) #0 {
 ; CHECK-LABEL: @test_not_const_fmt(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* [[FMT:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 32, i8* [[FMT:%.*]])
 ; CHECK-NEXT:    ret void
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* %fmt) #2
   ret void
 }
 
+; size is '0', do not add nonnull attribute
 define void @test_not_const_fmt_zero_size_return_value(i8* %buf, i8* %fmt) #0 {
 ; CHECK-LABEL: @test_not_const_fmt_zero_size_return_value(
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 0, i8* [[FMT:%.*]])
@@ -26,7 +27,6 @@ define void @test_not_const_fmt_zero_size_return_value(i8* %buf, i8* %fmt) #0 {
   ret void
 }
 
-
 define void @test_not_const_size(i8* %buf, i64 %size) #0 {
 ; CHECK-LABEL: @test_not_const_size(
 ; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 [[SIZE:%.*]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
@@ -47,7 +47,7 @@ define i32 @test_return_value(i8* %buf) #0 {
 
 define void @test_percentage(i8* %buf) #0 {
 ; CHECK-LABEL: @test_percentage(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0))
 ; CHECK-NEXT:    ret void
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 32, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.1, i64 0, i64 0)) #2
@@ -92,7 +92,7 @@ define i32 @test_char_zero_size(i8* %buf) #0 {
 
 define i32 @test_char_wrong_size(i8* %buf) #0 {
 ; CHECK-LABEL: @test_char_wrong_size(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65)
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.2, i64 0, i64 0), i32 65) #2
@@ -120,7 +120,7 @@ define i32 @test_str_zero_size(i8* %buf) #0 {
 
 define i32 @test_str_wrong_size(i8* %buf) #0 {
 ; CHECK-LABEL: @test_str_wrong_size(
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 (i8*, i64, i8*, ...) @snprintf(i8* nonnull dereferenceable(1) [[BUF:%.*]], i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0))
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %call = call i32 (i8*, i64, i8*, ...) @snprintf(i8* %buf, i64 1, i8* getelementptr inbounds ([3 x i8], [3 x i8]* @.str.3, i64 0, i64 0), i8* getelementptr inbounds ([4 x i8], [4 x i8]* @.str, i64 0, i64 0)) #2
diff --git a/llvm/test/Transforms/InstCombine/sprintf-1.ll b/llvm/test/Transforms/InstCombine/sprintf-1.ll
index 12957b3404e26..9dbfecaecb31b 100644
--- a/llvm/test/Transforms/InstCombine/sprintf-1.ll
+++ b/llvm/test/Transforms/InstCombine/sprintf-1.ll
@@ -21,11 +21,11 @@ declare i32 @sprintf(i8*, i8*, ...)
 
 define void @test_simplify1(i8* %dst) {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(13) [[DST:%.*]], i8* align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(13) [[DST:%.*]], i8* nonnull align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-IPRINTF-LABEL: @test_simplify1(
-; CHECK-IPRINTF-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(13) [[DST:%.*]], i8* align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
+; CHECK-IPRINTF-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(13) [[DST:%.*]], i8* nonnull align 1 dereferenceable(13) getelementptr inbounds ([13 x i8], [13 x i8]* @hello_world, i32 0, i32 0), i32 13, i1 false)
 ; CHECK-IPRINTF-NEXT:    ret void
 ;
   %fmt = getelementptr [13 x i8], [13 x i8]* @hello_world, i32 0, i32 0
@@ -85,13 +85,13 @@ define void @test_simplify4(i8* %dst) {
 
 define void @test_simplify5(i8* %dst, i8* %str) {
 ; CHECK-LABEL: @test_simplify5(
-; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* [[STR:%.*]])
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
 ; CHECK-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
 ; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[DST:%.*]], i8* align 1 [[STR]], i32 [[LENINC]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-IPRINTF-LABEL: @test_simplify5(
-; CHECK-IPRINTF-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* [[STR:%.*]])
+; CHECK-IPRINTF-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
 ; CHECK-IPRINTF-NEXT:    [[LENINC:%.*]] = add i32 [[STRLEN]], 1
 ; CHECK-IPRINTF-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 [[DST:%.*]], i8* align 1 [[STR]], i32 [[LENINC]], i1 false)
 ; CHECK-IPRINTF-NEXT:    ret void
@@ -105,7 +105,7 @@ define void @test_simplify5(i8* %dst, i8* %str) {
 
 define void @test_simplify6(i8* %dst) {
 ; CHECK-LABEL: @test_simplify6(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_d, i32 0, i32 0), i32 187)
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-IPRINTF-LABEL: @test_simplify6(
@@ -119,11 +119,11 @@ define void @test_simplify6(i8* %dst) {
 
 define void @test_no_simplify1(i8* %dst) {
 ; CHECK-LABEL: @test_no_simplify1(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-IPRINTF-LABEL: @test_no_simplify1(
-; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) getelementptr inbounds ([3 x i8], [3 x i8]* @percent_f, i32 0, i32 0), double 1.870000e+00)
 ; CHECK-IPRINTF-NEXT:    ret void
 ;
   %fmt = getelementptr [3 x i8], [3 x i8]* @percent_f, i32 0, i32 0
@@ -133,11 +133,11 @@ define void @test_no_simplify1(i8* %dst) {
 
 define void @test_no_simplify2(i8* %dst, i8* %fmt, double %d) {
 ; CHECK-LABEL: @test_no_simplify2(
-; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* [[FMT:%.*]], double [[D:%.*]])
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
 ; CHECK-NEXT:    ret void
 ;
 ; CHECK-IPRINTF-LABEL: @test_no_simplify2(
-; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* [[DST:%.*]], i8* [[FMT:%.*]], double [[D:%.*]])
+; CHECK-IPRINTF-NEXT:    [[TMP1:%.*]] = call i32 (i8*, i8*, ...) @sprintf(i8* nonnull dereferenceable(1) [[DST:%.*]], i8* nonnull dereferenceable(1) [[FMT:%.*]], double [[D:%.*]])
 ; CHECK-IPRINTF-NEXT:    ret void
 ;
   call i32 (i8*, i8*, ...) @sprintf(i8* %dst, i8* %fmt, double %d)
diff --git a/llvm/test/Transforms/InstCombine/sqrt.ll b/llvm/test/Transforms/InstCombine/sqrt.ll
index 57719eb8a08b4..55f952fe5cbc1 100644
--- a/llvm/test/Transforms/InstCombine/sqrt.ll
+++ b/llvm/test/Transforms/InstCombine/sqrt.ll
@@ -47,5 +47,17 @@ define float @test3(float* %v) nounwind uwtable ssp {
   ret float %conv38
 }
 
+; PR43347 - https://bugs.llvm.org/show_bug.cgi?id=43347
+
+define void @0(float %f) {
+; CHECK-LABEL: @0(
+; CHECK-NEXT:    [[SQRTF:%.*]] = call float @sqrtf(float [[F:%.*]]) #2
+; CHECK-NEXT:    ret void
+;
+  %d = fpext float %f to double
+  %r = call double @sqrt(double %d)
+  ret void
+}
+
 declare i32 @foo(double)
 declare double @sqrt(double) readnone
diff --git a/llvm/test/Transforms/InstCombine/stpcpy-1.ll b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
index c0aad563b9a69..260ccf6c0e783 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy-1.ll
@@ -14,7 +14,7 @@ declare i8* @stpcpy(i8*, i8*)
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 5)
 ;
 
@@ -27,7 +27,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [32 x i8], [32 x i8]* @a, i32 0, i32 [[STRLEN]]
 ; CHECK-NEXT:    ret i8* [[TMP1]]
 ;
@@ -38,6 +38,18 @@ define i8* @test_simplify2() {
   ret i8* %ret
 }
 
+define void @test_simplify3(i8* %dst) {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @stpcpy(i8* dereferenceable(80) %dst, i8* %src)
+  ret void
+}
+
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
 ; CHECK-NEXT:    [[RET:%.*]] = call i8* @stpcpy(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0))
diff --git a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
index b2f1f6a94f3e9..27244245adc8a 100644
--- a/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/stpcpy_chk-1.ll
@@ -14,7 +14,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -26,7 +26,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -38,7 +38,7 @@ define i8* @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 11)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -82,7 +82,7 @@ define i8* @test_simplify5() {
 
 define i8* @test_simplify6() {
 ; CHECK-LABEL: @test_simplify6(
-; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0))
 ; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 [[STRLEN]]
 ; CHECK-NEXT:    ret i8* [[TMP1]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/strchr-1.ll b/llvm/test/Transforms/InstCombine/strchr-1.ll
index 4fce378f59f43..081c3bf7a8ea2 100644
--- a/llvm/test/Transforms/InstCombine/strchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strchr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strchr library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -11,9 +12,10 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 declare i8* @strchr(i8*, i32)
 
 define void @test_simplify1() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %str, i32 119)
@@ -22,9 +24,10 @@ define void @test_simplify1() {
 }
 
 define void @test_simplify2() {
-; CHECK: store i8* null, i8** @chp, align 4
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %dst = call i8* @strchr(i8* %str, i32 119)
@@ -33,9 +36,10 @@ define void @test_simplify2() {
 }
 
 define void @test_simplify3() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 0)
@@ -44,9 +48,11 @@ define void @test_simplify3() {
 }
 
 define void @test_simplify4(i32 %chr) {
-; CHECK: call i8* @memchr
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[MEMCHR:%.*]] = call i8* @memchr(i8* nonnull dereferenceable(14) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]], i32 14)
+; CHECK-NEXT:    store i8* [[MEMCHR]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 %chr)
@@ -55,9 +61,10 @@ define void @test_simplify4(i32 %chr) {
 }
 
 define void @test_simplify5() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strchr(i8* %src, i32 65280)
@@ -67,11 +74,12 @@ define void @test_simplify5() {
 
 ; Check transformation strchr(p, 0) -> p + strlen(p)
 define void @test_simplify6(i8* %str) {
-; CHECK: %strlen = call i32 @strlen(i8* %str)
-; CHECK-NOT: call i8* @strchr
-; CHECK: %strchr = getelementptr i8, i8* %str, i32 %strlen
-; CHECK: store i8* %strchr, i8** @chp, align 4
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    [[STRCHR:%.*]] = getelementptr i8, i8* [[STR]], i32 [[STRLEN]]
+; CHECK-NEXT:    store i8* [[STRCHR]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %dst = call i8* @strchr(i8* %str, i32 0)
   store i8* %dst, i8** @chp
@@ -80,17 +88,38 @@ define void @test_simplify6(i8* %str) {
 
 ; Check transformation strchr("\r\n", C) != nullptr -> (C & 9217) != 0
 define i1 @test_simplify7(i32 %C) {
-; CHECK-LABEL: @test_simplify7
-; CHECK-NEXT: [[TRUNC:%.*]] = trunc i32 %C to i16
-; CHECK-NEXT: [[TRUNC_AND:%.*]] = and i16 [[TRUNC]], 255
-; CHECK-NEXT: %memchr.bounds = icmp ult i16 [[TRUNC_AND]], 16
-; CHECK-NEXT: [[SHL:%.*]] = shl i16 1, [[TRUNC_AND]]
-; CHECK-NEXT: [[AND:%.*]] = and i16 [[SHL]], 9217
-; CHECK-NEXT: %memchr.bits = icmp ne i16 [[AND]], 0
-; CHECK-NEXT: %memchr1 = and i1 %memchr.bounds, %memchr.bits
-; CHECK-NEXT: ret i1 %memchr1
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc i32 [[C:%.*]] to i16
+; CHECK-NEXT:    [[TMP2:%.*]] = and i16 [[TMP1]], 255
+; CHECK-NEXT:    [[MEMCHR_BOUNDS:%.*]] = icmp ult i16 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = shl i16 1, [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = and i16 [[TMP3]], 9217
+; CHECK-NEXT:    [[MEMCHR_BITS:%.*]] = icmp ne i16 [[TMP4]], 0
+; CHECK-NEXT:    [[MEMCHR1:%.*]] = and i1 [[MEMCHR_BOUNDS]], [[MEMCHR_BITS]]
+; CHECK-NEXT:    ret i1 [[MEMCHR1]]
+;
 
   %dst = call i8* @strchr(i8* getelementptr inbounds ([3 x i8], [3 x i8]* @newlines, i64 0, i64 0), i32 %C)
   %cmp = icmp ne i8* %dst, null
   ret i1 %cmp
 }
+
+define i8* @test1(i8* %str, i32 %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strchr(i8* [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strcmp-1.ll b/llvm/test/Transforms/InstCombine/strcmp-1.ll
index 7d86ec51f7fa9..9634999a346cb 100644
--- a/llvm/test/Transforms/InstCombine/strcmp-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcmp-1.ll
@@ -97,12 +97,12 @@ define i32 @test5(i1 %b) {
 ; CHECK: ret i32 %memcmp
 ; NOBCMP-LABEL: @test5(
 ; NOBCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; NOBCMP-NEXT:    ret i32 [[MEMCMP]]
 ;
 ; BCMP-LABEL: @test5(
 ; BCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; BCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; BCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; BCMP-NEXT:    ret i32 [[MEMCMP]]
 ;
   %str1 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -131,13 +131,13 @@ define i32 @test6(i8* %str) {
 define i1 @test7(i1 %b) {
 ; NOBCMP-LABEL: @test7(
 ; NOBCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; NOBCMP-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; NOBCMP-NEXT:    [[RES:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; NOBCMP-NEXT:    ret i1 [[RES]]
 ;
 ; BCMP-LABEL: @test7(
 ; BCMP-NEXT:    [[STR2:%.*]] = select i1 [[B:%.*]], i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([5 x i8], [5 x i8]* @bell, i32 0, i32 0)
-; BCMP-NEXT:    [[BCMP:%.*]] = call i32 @bcmp(i8* dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* dereferenceable(5) [[STR2]], i32 5)
+; BCMP-NEXT:    [[BCMP:%.*]] = call i32 @bcmp(i8* nonnull dereferenceable(5) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i8* nonnull dereferenceable(5) [[STR2]], i32 5)
 ; BCMP-NEXT:    [[RES:%.*]] = icmp eq i32 [[BCMP]], 0
 ; BCMP-NEXT:    ret i1 [[RES]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
index 14490339d1c93..4c89b81b8f28b 100644
--- a/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
+++ b/llvm/test/Transforms/InstCombine/strcmp-memcmp.ll
@@ -11,7 +11,7 @@ declare void @use(i32)
 define i32 @strcmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -28,7 +28,7 @@ declare i32 @strcmp(i8* nocapture, i8* nocapture)
 define i32 @strcmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -43,7 +43,7 @@ define i32 @strcmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -58,7 +58,7 @@ define i32 @strcmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -73,7 +73,7 @@ define i32 @strcmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp5([5 x i8]* dereferenceable (5) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [5 x i8], [5 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -88,7 +88,7 @@ define i32 @strcmp_memcmp5([5 x i8]* dereferenceable (5) %buf) {
 define i32 @strcmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -103,7 +103,7 @@ define i32 @strcmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp7(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
 ; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
 ;
@@ -117,7 +117,7 @@ define i32 @strcmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp8([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -132,7 +132,7 @@ define i32 @strcmp_memcmp8([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strcmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp9(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -148,7 +148,7 @@ define i32 @strcmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* dereferenceable(2) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* nonnull dereferenceable(2) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -165,7 +165,7 @@ declare i32 @strncmp(i8* nocapture, i8* nocapture, i64)
 define i32 @strncmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -180,7 +180,7 @@ define i32 @strncmp_memcmp2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -195,7 +195,7 @@ define i32 @strncmp_memcmp3([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -210,7 +210,7 @@ define i32 @strncmp_memcmp4([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp5([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -226,7 +226,7 @@ define i32 @strncmp_memcmp5([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ne i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -241,7 +241,7 @@ define i32 @strncmp_memcmp6([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp7(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -256,7 +256,7 @@ define i32 @strncmp_memcmp7([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp8([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(3) [[STRING]], i8* dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(3) [[STRING]], i8* nonnull dereferenceable(3) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 3)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -271,7 +271,7 @@ define i32 @strncmp_memcmp8([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp9(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -286,7 +286,7 @@ define i32 @strncmp_memcmp9([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp10([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp10(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[MEMCMP_LOBIT:%.*]] = lshr i32 [[MEMCMP]], 31
 ; CHECK-NEXT:    ret i32 [[MEMCMP_LOBIT]]
 ;
@@ -300,7 +300,7 @@ define i32 @strncmp_memcmp10([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp11([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp11(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -315,7 +315,7 @@ define i32 @strncmp_memcmp11([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp12([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp12(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(4) [[STRING]], i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -330,7 +330,7 @@ define i32 @strncmp_memcmp12([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp13([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp13(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* dereferenceable(2) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(2) [[STRING]], i8* nonnull dereferenceable(2) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -345,7 +345,7 @@ define i32 @strncmp_memcmp13([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp14([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp14(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
+; CHECK-NEXT:    [[MEMCMP:%.*]] = call i32 @memcmp(i8* nonnull dereferenceable(4) [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([8 x i8], [8 x i8]* @abc, i64 0, i64 0), i64 4)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[MEMCMP]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -361,7 +361,7 @@ define i32 @strncmp_memcmp14([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -376,7 +376,7 @@ define i32 @strcmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad2(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -391,7 +391,7 @@ define i32 @strcmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strcmp_memcmp_bad3([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad3(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    ret i32 [[CALL]]
 ;
   %string = getelementptr inbounds [12 x i8], [12 x i8]* %buf, i64 0, i64 0
@@ -402,7 +402,7 @@ define i32 @strcmp_memcmp_bad3([12 x i8]* dereferenceable (12) %buf) {
 
 define i32 @strcmp_memcmp_bad4(i8* nocapture readonly %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad4(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[BUF:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[BUF:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -417,7 +417,7 @@ define i32 @strcmp_memcmp_bad4(i8* nocapture readonly %buf) {
 define i32 @strcmp_memcmp_bad5([3 x i8]* dereferenceable (3) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad5(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [3 x i8], [3 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -432,7 +432,7 @@ define i32 @strcmp_memcmp_bad5([3 x i8]* dereferenceable (3) %buf) {
 define i32 @strcmp_memcmp_bad6([4 x i8]* dereferenceable (4) %buf, i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strcmp_memcmp_bad6(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* [[K:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(1) [[K:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -446,7 +446,7 @@ define i32 @strcmp_memcmp_bad6([4 x i8]* dereferenceable (4) %buf, i8* nocapture
 
 define i32 @strcmp_memcmp_bad7(i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strcmp_memcmp_bad7(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]])
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strcmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[K:%.*]])
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -460,7 +460,7 @@ define i32 @strcmp_memcmp_bad7(i8* nocapture readonly %k) {
 define i32 @strcmp_memcmp_bad8([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strcmp_memcmp_bad8(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -473,7 +473,7 @@ define i32 @strcmp_memcmp_bad8([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strncmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -489,7 +489,7 @@ define i32 @strncmp_memcmp_bad([12 x i8]* dereferenceable (12) %buf) {
 define i32 @strncmp_memcmp_bad1([12 x i8]* dereferenceable (12) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad1(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull [[STRING]], i64 5)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[CALL]], 3
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -518,7 +518,7 @@ define i32 @strncmp_memcmp_bad2([12 x i8]* dereferenceable (12) %buf, i64 %n) {
 
 define i32 @strncmp_memcmp_bad3(i8* nocapture readonly %k) {
 ; CHECK-LABEL: @strncmp_memcmp_bad3(
-; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strncmp(i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* [[K:%.*]], i64 2)
+; CHECK-NEXT:    [[CALL:%.*]] = tail call i32 @strncmp(i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i8* nonnull dereferenceable(1) [[K:%.*]], i64 2)
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
@@ -532,7 +532,7 @@ define i32 @strncmp_memcmp_bad3(i8* nocapture readonly %k) {
 define i32 @strncmp_memcmp_bad4([4 x i8]* dereferenceable (4) %buf) {
 ; CHECK-LABEL: @strncmp_memcmp_bad4(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [4 x i8], [4 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strncmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0), i64 2)
 ; CHECK-NEXT:    tail call void @use(i32 [[CALL]])
 ; CHECK-NEXT:    ret i32 0
 ;
@@ -545,7 +545,7 @@ define i32 @strncmp_memcmp_bad4([4 x i8]* dereferenceable (4) %buf) {
 define i32 @strcmp_memcmp_msan([12 x i8]* dereferenceable (12) %buf) sanitize_memory {
 ; CHECK-LABEL: @strcmp_memcmp_msan(
 ; CHECK-NEXT:    [[STRING:%.*]] = getelementptr inbounds [12 x i8], [12 x i8]* [[BUF:%.*]], i64 0, i64 0
-; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
+; CHECK-NEXT:    [[CALL:%.*]] = call i32 @strcmp(i8* nonnull [[STRING]], i8* nonnull dereferenceable(4) getelementptr inbounds ([4 x i8], [4 x i8]* @key, i64 0, i64 0))
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; CHECK-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; CHECK-NEXT:    ret i32 [[CONV]]
diff --git a/llvm/test/Transforms/InstCombine/strcpy-1.ll b/llvm/test/Transforms/InstCombine/strcpy-1.ll
index 24c70c18fc05d..0ccf983a73141 100644
--- a/llvm/test/Transforms/InstCombine/strcpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcpy library call simplifier works correctly.
 ; rdar://6839935
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,32 +15,50 @@ declare i8* @strcpy(i8*, i8*)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strcpy(i8* %dst, i8* %src)
-; CHECK: @llvm.memcpy.p0i8.p0i8.i32
   ret void
 }
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
 
   %ret = call i8* @strcpy(i8* %dst, i8* %dst)
-; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
   ret i8* %ret
 }
 
+
+define void @test_simplify3(i8* %dst) {
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(80) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
+
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+
+  call i8* @strcpy(i8* dereferenceable(80) %dst, i8* %src)
+  ret void
+}
+
 define i8* @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strcpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    ret i8* [[RET]]
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
 
   %ret = call i8* @strcpy(i8* %dst, i8* %src)
-; CHECK: call i8* @strcpy
   ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
index 292591f72819c..02a4b5cbdeac2 100644
--- a/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcpy_chk-1.ll
@@ -14,7 +14,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -26,7 +26,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -38,7 +38,7 @@ define i8* @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -52,7 +52,7 @@ define i8* @test_simplify3() {
 
 define i8* @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
-; CHECK-NEXT:    [[STRCPY:%.*]] = call i8* @strcpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
+; CHECK-NEXT:    [[STRCPY:%.*]] = call i8* @strcpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0))
 ; CHECK-NEXT:    ret i8* [[STRCPY]]
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strcspn-1.ll b/llvm/test/Transforms/InstCombine/strcspn-1.ll
index 8d441a9eecd06..c1f728584d83a 100644
--- a/llvm/test/Transforms/InstCombine/strcspn-1.ll
+++ b/llvm/test/Transforms/InstCombine/strcspn-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strcspn library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,44 +15,48 @@ declare i64 @strcspn(i8*, i8*)
 
 define i64 @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    ret i64 [[STRLEN]]
+;
   %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
-; CHECK-NEXT: [[VAR:%[a-z]+]] = call i64 @strlen(i8* %str)
   ret i64 %ret
-; CHECK-NEXT: ret i64 [[VAR]]
 }
 
 ; Check strcspn("", s) -> 0.
 
 define i64 @test_simplify2(i8* %pat) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i64 0
+;
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 0
 }
 
 ; Check strcspn(s1, s2), where s1 and s2 are constants.
 
 define i64 @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i64 0
+;
   %str = getelementptr [6 x i8], [6 x i8]* @abcba, i32 0, i32 0
   %pat = getelementptr [4 x i8], [4 x i8]* @abc, i32 0, i32 0
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 0
 }
 
 ; Check cases that shouldn't be simplified.
 
 define i64 @test_no_simplify1(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i64 @strcspn(i8* [[STR:%.*]], i8* [[PAT:%.*]])
+; CHECK-NEXT:    ret i64 [[RET]]
+;
 
   %ret = call i64 @strcspn(i8* %str, i8* %pat)
-; CHECK-NEXT: %ret = call i64 @strcspn(i8* %str, i8* %pat)
   ret i64 %ret
-; CHECK-NEXT: ret i64 %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll b/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll
new file mode 100644
index 0000000000000..9abb588a4ee0b
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strict-sub-underflow-check-to-comparison-of-sub-operands.ll
@@ -0,0 +1,111 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @llvm.assume(i1)
+declare i8 @gen8()
+declare void @use8(i8)
+
+define i1 @t0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
+define i1 @t1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp uge i8 %adjusted, %base
+  ret i1 %res
+}
+define i1 @t2(i8 %offset) {
+; CHECK-LABEL: @t2(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[BASE:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %base = call i8 @gen8()
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ugt i8 %base, %adjusted
+  ret i1 %res
+}
+define i1 @t3(i8 %offset) {
+; CHECK-LABEL: @t3(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[BASE:%.*]] = call i8 @gen8()
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %base = call i8 @gen8()
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ule i8 %base, %adjusted
+  ret i1 %res
+}
+
+; Here we don't know that offset is non-zero. Can't fold.
+define i1 @n4_maybezero(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n4_maybezero(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
+; We need to know that about %offset, %base won't do. Can't fold.
+define i1 @n5_wrongnonzero(i8 %base, i8 %offset) {
+; CHECK-LABEL: @n5_wrongnonzero(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i8 [[BASE:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[RES:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    ret i1 [[RES]]
+;
+  %cmp = icmp sgt i8 %base, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %res = icmp ult i8 %adjusted, %base
+  ret i1 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/strlen-1.ll b/llvm/test/Transforms/InstCombine/strlen-1.ll
index aaf1b89945a0f..0ecf233a26030 100644
--- a/llvm/test/Transforms/InstCombine/strlen-1.ll
+++ b/llvm/test/Transforms/InstCombine/strlen-1.ll
@@ -100,7 +100,7 @@ define i1 @test_simplify8(i8* %str_p) {
 
 define i32 @test_simplify9(i1 %x) {
 ; CHECK-LABEL: @test_simplify9(
-; CHECK-NEXT:    [[TMP1:%.*]] = select i1 %x, i32 5, i32 6
+; CHECK-NEXT:    [[TMP1:%.*]] = select i1 [[X:%.*]], i32 5, i32 6
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %hello = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -115,7 +115,7 @@ define i32 @test_simplify9(i1 %x) {
 
 define i32 @test_simplify10(i32 %x) {
 ; CHECK-LABEL: @test_simplify10(
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 5, %x
+; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 5, [[X:%.*]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
   %hello_p = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 %x
@@ -127,7 +127,7 @@ define i32 @test_simplify10(i32 %x) {
 
 define i32 @test_simplify11(i32 %x) {
 ; CHECK-LABEL: @test_simplify11(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 7
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 7
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub nuw nsw i32 9, [[AND]]
 ; CHECK-NEXT:    ret i32 [[TMP1]]
 ;
@@ -141,7 +141,7 @@ define i32 @test_simplify11(i32 %x) {
 
 define i32 @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
-; CHECK-NEXT:    [[A_L:%.*]] = call i32 @strlen(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[A_L:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
 ; CHECK-NEXT:    ret i32 [[A_L]]
 ;
   %a_p = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
@@ -153,7 +153,7 @@ define i32 @test_no_simplify1() {
 
 define i32 @test_no_simplify2(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify2(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 [[X:%.*]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -164,7 +164,7 @@ define i32 @test_no_simplify2(i32 %x) {
 
 define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify2_no_null_opt(
-; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 %x
+; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [7 x i8], [7 x i8]* @null_hello, i32 0, i32 [[X:%.*]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
 ;
@@ -177,7 +177,7 @@ define i32 @test_no_simplify2_no_null_opt(i32 %x) #0 {
 
 define i32 @test_no_simplify3(i32 %x) {
 ; CHECK-LABEL: @test_no_simplify3(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* nonnull [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
@@ -190,7 +190,7 @@ define i32 @test_no_simplify3(i32 %x) {
 
 define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
 ; CHECK-LABEL: @test_no_simplify3_on_null_opt(
-; CHECK-NEXT:    [[AND:%.*]] = and i32 %x, 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[X:%.*]], 15
 ; CHECK-NEXT:    [[HELLO_P:%.*]] = getelementptr inbounds [13 x i8], [13 x i8]* @null_hello_mid, i32 0, i32 [[AND]]
 ; CHECK-NEXT:    [[HELLO_L:%.*]] = call i32 @strlen(i8* [[HELLO_P]])
 ; CHECK-NEXT:    ret i32 [[HELLO_L]]
@@ -201,4 +201,22 @@ define i32 @test_no_simplify3_on_null_opt(i32 %x) #0 {
   ret i32 %hello_l
 }
 
+define i32 @test1(i8* %str) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]]) #1
+; CHECK-NEXT:    ret i32 [[LEN]]
+;
+  %len = tail call i32 @strlen(i8* %str) nounwind
+  ret i32 %len
+}
+
+define i32 @test2(i8* %str) #0 {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[LEN:%.*]] = tail call i32 @strlen(i8* [[STR:%.*]]) #1
+; CHECK-NEXT:    ret i32 [[LEN]]
+;
+  %len = tail call i32 @strlen(i8* %str) nounwind
+  ret i32 %len
+}
+
 attributes #0 = { "null-pointer-is-valid"="true" }
diff --git a/llvm/test/Transforms/InstCombine/strncat-2.ll b/llvm/test/Transforms/InstCombine/strncat-2.ll
index 3533e1eace708..4aed71829a6e5 100644
--- a/llvm/test/Transforms/InstCombine/strncat-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncat-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncat libcall simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -7,13 +8,15 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 @hello = constant [6 x i8] c"hello\00"
 @empty = constant [1 x i8] c"\00"
 @a = common global [32 x i8] zeroinitializer, align 1
-
 declare i8* @strncat(i8*, i8*, i32)
 
 define void @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NOT: call i8* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0))
+; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 [[STRLEN]]
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ENDPTR]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -23,7 +26,8 @@ define void @test_simplify1() {
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [1 x i8], [1 x i8]* @empty, i32 0, i32 0
@@ -33,7 +37,8 @@ define void @test_simplify2() {
 
 define void @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT: ret void
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -43,11 +48,66 @@ define void @test_simplify3() {
 
 define void @test_nosimplify1() {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i8* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 1)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
   call i8* @strncat(i8* %dst, i8* %src, i32 1)
   ret void
 }
+
+; strncat(nonnull x, nonnull y, n)  -> strncat(nonnull x, y, n)
+define i8* @test1(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* nonnull [[STR1:%.*]], i8* nonnull [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* nonnull %str1, i8* nonnull %str2, i32 %n)
+  ret i8* %temp1
+}
+
+; strncat(x, y, 0)  -> x
+define i8* @test2(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    ret i8* [[STR1:%.*]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 0)
+  ret i8* %temp1
+}
+
+; strncat(x, y, 5)  -> strncat(nonnull x, nonnull y, 5)
+define i8* @test3(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]], i32 5)
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 5)
+  ret i8* %temp1
+}
+
+define i8* @test4(i8* %str1, i8* %str2, i32 %n) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i8* @strncat(i8* [[STR1:%.*]], i8* [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[TEMP1]]
+;
+
+  %temp1 = call i8* @strncat(i8* %str1, i8* %str2, i32 %n)
+  ret i8* %temp1
+}
+
+define i8* @test5(i8* %str, i32 %n) {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i32 @strlen(i8* nonnull dereferenceable(1) [[STR:%.*]])
+; CHECK-NEXT:    [[ENDPTR:%.*]] = getelementptr i8, i8* [[STR]], i32 [[STRLEN]]
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ENDPTR]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret i8* [[STR]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %temp1 = call i8* @strncat(i8* %str, i8* %src, i32 10)
+  ret i8* %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncat-3.ll b/llvm/test/Transforms/InstCombine/strncat-3.ll
index 0f6964df9154f..d87688378b885 100644
--- a/llvm/test/Transforms/InstCombine/strncat-3.ll
+++ b/llvm/test/Transforms/InstCombine/strncat-3.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncat libcall simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -12,8 +13,9 @@ declare i16* @strncat(i8*, i8*, i32)
 
 define void @test_nosimplify1() {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i16* @strncat
-; CHECK: ret void
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16* @strncat(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 13)
+; CHECK-NEXT:    ret void
+;
 
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strncmp-1.ll b/llvm/test/Transforms/InstCombine/strncmp-1.ll
index a1121821fdf5d..4bb03cc7a9a81 100644
--- a/llvm/test/Transforms/InstCombine/strncmp-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncmp-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -13,10 +14,11 @@ declare i32 @strncmp(i8*, i8*, i32)
 ; strncmp("", x, n) -> -*x
 define i32 @test1(i8* %str2) {
 ; CHECK-LABEL: @test1(
-; CHECK: %strcmpload = load i8, i8* %str
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: %2 = sub nsw i32 0, %1
-; CHECK: ret i32 %2
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    [[TMP2:%.*]] = sub nsw i32 0, [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
+;
 
   %str1 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
@@ -26,9 +28,10 @@ define i32 @test1(i8* %str2) {
 ; strncmp(x, "", n) -> *x
 define i32 @test2(i8* %str1) {
 ; CHECK-LABEL: @test2(
-; CHECK: %strcmpload = load i8, i8* %str1
-; CHECK: %1 = zext i8 %strcmpload to i32
-; CHECK: ret i32 %1
+; CHECK-NEXT:    [[STRCMPLOAD:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = zext i8 [[STRCMPLOAD]] to i32
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
 
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 10)
@@ -38,7 +41,8 @@ define i32 @test2(i8* %str1) {
 ; strncmp(x, y, n)  -> cnst
 define i32 @test3() {
 ; CHECK-LABEL: @test3(
-; CHECK: ret i32 -1
+; CHECK-NEXT:    ret i32 -1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -48,7 +52,8 @@ define i32 @test3() {
 
 define i32 @test4() {
 ; CHECK-LABEL: @test4(
-; CHECK: ret i32 1
+; CHECK-NEXT:    ret i32 1
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [1 x i8], [1 x i8]* @null, i32 0, i32 0
@@ -58,7 +63,8 @@ define i32 @test4() {
 
 define i32 @test5() {
 ; CHECK-LABEL: @test5(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
@@ -69,12 +75,13 @@ define i32 @test5() {
 ; strncmp(x,y,1) -> memcmp(x,y,1)
 define i32 @test6(i8* %str1, i8* %str2) {
 ; CHECK-LABEL: @test6(
-; CHECK: [[LOAD1:%[a-z]+]] = load i8, i8* %str1, align 1
-; CHECK: [[ZEXT1:%[a-z]+]] = zext i8 [[LOAD1]] to i32
-; CHECK: [[LOAD2:%[a-z]+]] = load i8, i8* %str2, align 1
-; CHECK: [[ZEXT2:%[a-z]+]] = zext i8 [[LOAD2]] to i32
-; CHECK: [[RET:%[a-z]+]] = sub nsw i32 [[ZEXT1]], [[ZEXT2]]
-; CHECK: ret i32 [[RET]]
+; CHECK-NEXT:    [[LHSC:%.*]] = load i8, i8* [[STR1:%.*]], align 1
+; CHECK-NEXT:    [[LHSV:%.*]] = zext i8 [[LHSC]] to i32
+; CHECK-NEXT:    [[RHSC:%.*]] = load i8, i8* [[STR2:%.*]], align 1
+; CHECK-NEXT:    [[RHSV:%.*]] = zext i8 [[RHSC]] to i32
+; CHECK-NEXT:    [[CHARDIFF:%.*]] = sub nsw i32 [[LHSV]], [[RHSV]]
+; CHECK-NEXT:    ret i32 [[CHARDIFF]]
+;
 
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 1)
   ret i32 %temp1
@@ -83,7 +90,8 @@ define i32 @test6(i8* %str1, i8* %str2) {
 ; strncmp(x,y,0)   -> 0
 define i32 @test7(i8* %str1, i8* %str2) {
 ; CHECK-LABEL: @test7(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 0)
   ret i32 %temp1
@@ -92,8 +100,51 @@ define i32 @test7(i8* %str1, i8* %str2) {
 ; strncmp(x,x,n)  -> 0
 define i32 @test8(i8* %str, i32 %n) {
 ; CHECK-LABEL: @test8(
-; CHECK: ret i32 0
+; CHECK-NEXT:    ret i32 0
+;
 
   %temp1 = call i32 @strncmp(i8* %str, i8* %str, i32 %n)
   ret i32 %temp1
 }
+
+; strncmp(nonnull x, nonnull y, n)  -> strncmp(x, y, n)
+define i32 @test9(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test9(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* nonnull [[STR1:%.*]], i8* nonnull [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* nonnull %str1, i8* nonnull %str2, i32 %n)
+  ret i32 %temp1
+}
+
+; strncmp(nonnull x, nonnull y, 0)  -> 0
+define i32 @test10(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test10(
+; CHECK-NEXT:    ret i32 0
+;
+
+  %temp1 = call i32 @strncmp(i8* nonnull %str1, i8* nonnull %str2, i32 0)
+  ret i32 %temp1
+}
+
+; strncmp(x, y, 5)  -> strncmp(nonnull x, nonnull y, 5)
+define i32 @test11(i8* %str1, i8* %str2, i32 %n) {
+; CHECK-LABEL: @test11(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]], i32 5)
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 5)
+  ret i32 %temp1
+}
+
+define i32 @test12(i8* %str1, i8* %str2, i32 %n) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test12(
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i32 @strncmp(i8* [[STR1:%.*]], i8* [[STR2:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i32 [[TEMP1]]
+;
+
+  %temp1 = call i32 @strncmp(i8* %str1, i8* %str2, i32 %n)
+  ret i32 %temp1
+}
diff --git a/llvm/test/Transforms/InstCombine/strncmp-2.ll b/llvm/test/Transforms/InstCombine/strncmp-2.ll
index 9e19781e04c41..5afad9e196330 100644
--- a/llvm/test/Transforms/InstCombine/strncmp-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncmp-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncmp library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -10,8 +11,9 @@ declare i16 @strncmp(i8*, i8*, i32)
 
 define i16 @test_nosimplify() {
 ; CHECK-LABEL: @test_nosimplify(
-; CHECK: call i16 @strncmp
-; CHECK: ret i16 %temp1
+; CHECK-NEXT:    [[TEMP1:%.*]] = call i16 @strncmp(i8* getelementptr inbounds ([5 x i8], [5 x i8]* @hell, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 10)
+; CHECK-NEXT:    ret i16 [[TEMP1]]
+;
 
   %str1 = getelementptr inbounds [5 x i8], [5 x i8]* @hell, i32 0, i32 0
   %str2 = getelementptr inbounds [6 x i8], [6 x i8]* @hello, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strncpy-1.ll b/llvm/test/Transforms/InstCombine/strncpy-1.ll
index 28cb26b170471..969358eb4d7d3 100644
--- a/llvm/test/Transforms/InstCombine/strncpy-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncpy library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -17,8 +18,15 @@ declare i32 @puts(i8*)
 
 define i32 @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NOT: call i8* @strncpy
-; CHECK: call i32 @puts
+; CHECK-NEXT:    [[TARGET:%.*]] = alloca [1024 x i8], align 1
+; CHECK-NEXT:    [[ARG1:%.*]] = getelementptr inbounds [1024 x i8], [1024 x i8]* [[TARGET]], i32 0, i32 0
+; CHECK-NEXT:    store i8 0, i8* [[ARG1]], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) [[ARG1]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(42) [[ARG1]], i8 0, i32 42, i1 false)
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(42) [[ARG1]], i8 0, i32 42, i1 false)
+; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @puts(i8* nonnull [[ARG1]])
+; CHECK-NEXT:    ret i32 0
+;
   %target = alloca [1024 x i8]
   %arg1 = getelementptr [1024 x i8], [1024 x i8]* %target, i32 0, i32 0
   store i8 0, i8* %arg1
@@ -40,11 +48,13 @@ define i32 @test_simplify1() {
 
 define void @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* nonnull align 1 dereferenceable(32) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8 0, i32 32, i1 false)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 32)
-; CHECK: call void @llvm.memset.p0i8.i32
   ret void
 }
 
@@ -52,44 +62,110 @@ define void @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   %ret = call i8* @strncpy(i8* %dst, i8* %src, i32 0)
   ret i8* %ret
-; CHECK: ret i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0)
 }
 
 ; Check  strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant].
 
 define void @test_simplify4() {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6, i1 false)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 6)
-; CHECK: call void @llvm.memcpy.p0i8.p0i8.i32
   ret void
 }
 
+define void @test_simplify5(i8* %dst) {
+; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(8) [[DST:%.*]], i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(8) %dst, i8* %src, i32 32)
+  ret void
+}
+
+define void @test_simplify6(i8* %dst) {
+; CHECK-LABEL: @test_simplify6(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(80) [[DST:%.*]], i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(80) %dst, i8* %src, i32 32)
+  ret void
+}
+
+define void @test_simplify7(i8* %dst, i32 %n) {
+; CHECK-LABEL: @test_simplify7(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(80) [[DST:%.*]], i8* getelementptr inbounds ([1 x i8], [1 x i8]* @null, i32 0, i32 0), i32 [[N:%.*]])
+; CHECK-NEXT:    ret void
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  call i8* @strncpy(i8* dereferenceable(80) %dst, i8* %src, i32 %n)
+  ret void
+}
+
+define i8* @test1(i8* %dst, i8* %src, i32 %n) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strncpy(i8* nonnull [[DST:%.*]], i8* nonnull [[SRC:%.*]], i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strncpy(i8* nonnull %dst, i8* nonnull %src, i32 %n)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %dst) {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(5) [[DST:%.*]], i8* nonnull align 1 dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 5, i1 false)
+; CHECK-NEXT:    ret i8* [[DST]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strncpy(i8* nonnull %dst, i8* nonnull %src, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test3(i8* %dst, i32 %n) {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    call void @llvm.memset.p0i8.i32(i8* noalias nonnull align 1 dereferenceable(5) [[DST:%.*]], i8 0, i32 5, i1 false)
+; CHECK-NEXT:    ret i8* [[DST]]
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %ret = call i8* @strncpy(i8* noalias nonnull %dst, i8* nonnull %src, i32 5);
+  ret i8* %ret
+}
+
 ; Check cases that shouldn't be simplified.
 
 define void @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @b, i32 0, i32 0), i32 32)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [32 x i8], [32 x i8]* @b, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 32)
-; CHECK: call i8* @strncpy
   ret void
 }
 
 define void @test_no_simplify2() {
 ; CHECK-LABEL: @test_no_simplify2(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 8)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i8* @strncpy(i8* %dst, i8* %src, i32 8)
-; CHECK: call i8* @strncpy
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/strncpy-2.ll b/llvm/test/Transforms/InstCombine/strncpy-2.ll
index 5c45f9f24aee0..ed4de382ce5b7 100644
--- a/llvm/test/Transforms/InstCombine/strncpy-2.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy-2.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strncpy library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -13,10 +14,12 @@ declare i16* @strncpy(i8*, i8*, i32)
 
 define void @test_no_simplify1() {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[TMP1:%.*]] = call i16* @strncpy(i8* getelementptr inbounds ([32 x i8], [32 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i32 0, i32 0), i32 6)
+; CHECK-NEXT:    ret void
+;
   %dst = getelementptr [32 x i8], [32 x i8]* @a, i32 0, i32 0
   %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
 
   call i16* @strncpy(i8* %dst, i8* %src, i32 6)
-; CHECK: call i16* @strncpy
   ret void
 }
diff --git a/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
index 6f7b0415b1f6c..ed90303b28080 100644
--- a/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strncpy_chk-1.ll
@@ -14,7 +14,7 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 
 define i8* @test_simplify1() {
 ; CHECK-LABEL: @test_simplify1(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -26,7 +26,7 @@ define i8* @test_simplify1() {
 
 define i8* @test_simplify2() {
 ; CHECK-LABEL: @test_simplify2(
-; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0i8.p0i8.i32(i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull align 1 dereferenceable(12) getelementptr inbounds ([12 x i8], [12 x i8]* @.str, i32 0, i32 0), i32 12, i1 false)
 ; CHECK-NEXT:    ret i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0)
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
@@ -38,7 +38,7 @@ define i8* @test_simplify2() {
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
-; CHECK-NEXT:    [[STRNCPY:%.*]] = call i8* @strncpy(i8* getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 12)
+; CHECK-NEXT:    [[STRNCPY:%.*]] = call i8* @strncpy(i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @a, i32 0, i32 0), i8* nonnull dereferenceable(1) getelementptr inbounds ([60 x i8], [60 x i8]* @b, i32 0, i32 0), i32 12)
 ; CHECK-NEXT:    ret i8* [[STRNCPY]]
 ;
   %dst = getelementptr inbounds [60 x i8], [60 x i8]* @a, i32 0, i32 0
diff --git a/llvm/test/Transforms/InstCombine/strndup.ll b/llvm/test/Transforms/InstCombine/strndup.ll
new file mode 100644
index 0000000000000..b3f927934a752
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/strndup.ll
@@ -0,0 +1,67 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instcombine -S | FileCheck %s
+
+@hello = constant [6 x i8] c"hello\00"
+@null = constant [1 x i8] zeroinitializer
+
+declare i8* @strndup(i8*, i32)
+
+define i8* @test1() {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(1) i8* @strdup(i8* getelementptr inbounds ([1 x i8], [1 x i8]* @null, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 0)
+  ret i8* %ret
+}
+
+define i8* @test2() {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call dereferenceable_or_null(5) i8* @strndup(i8* dereferenceable(6) getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0), i32 4)
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 4)
+  ret i8* %ret
+}
+
+define i8* @test3() {
+; CHECK-LABEL: @test3(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 5)
+  ret i8* %ret
+}
+
+define i8* @test4() {
+; CHECK-LABEL: @test4(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 6)
+  ret i8* %ret
+}
+
+define i8* @test5() {
+; CHECK-LABEL: @test5(
+; CHECK-NEXT:    [[STRDUP:%.*]] = call dereferenceable_or_null(6) i8* @strdup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0))
+; CHECK-NEXT:    ret i8* [[STRDUP]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 7)
+  ret i8* %ret
+}
+
+define i8* @test6(i32 %n) {
+; CHECK-LABEL: @test6(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strndup(i8* getelementptr inbounds ([6 x i8], [6 x i8]* @hello, i64 0, i64 0), i32 [[N:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %src = getelementptr [6 x i8], [6 x i8]* @hello, i32 0, i32 0
+  %ret = call i8* @strndup(i8* %src, i32 %n)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strpbrk-1.ll b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
index a61100deb8757..c581409805cce 100644
--- a/llvm/test/Transforms/InstCombine/strpbrk-1.ll
+++ b/llvm/test/Transforms/InstCombine/strpbrk-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strpbrk library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -14,55 +15,60 @@ declare i8* @strpbrk(i8*, i8*)
 
 define i8* @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i8* null
+;
   %pat = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* null
 }
 
 ; Check strpbrk("", s) -> NULL.
 
 define i8* @test_simplify2(i8* %pat) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    ret i8* null
+;
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* null
 }
 
 ; Check strpbrk(s1, s2), where s1 and s2 are constants.
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([12 x i8], [12 x i8]* @hello, i32 0, i32 6)
+;
   %str = getelementptr [12 x i8], [12 x i8]* @hello, i32 0, i32 0
   %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* getelementptr inbounds ([12 x i8], [12 x i8]* @hello, i32 0, i32 6)
 }
 
 ; Check strpbrk(s, "a") -> strchr(s, 'a').
 
 define i8* @test_simplify4(i8* %str) {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    [[STRCHR:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 119)
+; CHECK-NEXT:    ret i8* [[STRCHR]]
+;
   %pat = getelementptr [2 x i8], [2 x i8]* @w, i32 0, i32 0
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
-; CHECK-NEXT: [[VAR:%[a-z]+]] = call i8* @strchr(i8* %str, i32 119)
   ret i8* %ret
-; CHECK-NEXT: ret i8* [[VAR]]
 }
 
 ; Check cases that shouldn't be simplified.
 
 define i8* @test_no_simplify1(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_no_simplify1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strpbrk(i8* [[STR:%.*]], i8* [[PAT:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
 
   %ret = call i8* @strpbrk(i8* %str, i8* %pat)
-; CHECK-NEXT: %ret = call i8* @strpbrk(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/strrchr-1.ll b/llvm/test/Transforms/InstCombine/strrchr-1.ll
index 3ae68fba96dfc..e60e9bcd36c77 100644
--- a/llvm/test/Transforms/InstCombine/strrchr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strrchr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strrchr library call simplifier works correctly.
 ; RUN: opt < %s -instcombine -S | FileCheck %s
 
@@ -10,9 +11,10 @@ target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f3
 declare i8* @strrchr(i8*, i32)
 
 define void @test_simplify1() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 6), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %str, i32 119)
@@ -21,9 +23,10 @@ define void @test_simplify1() {
 }
 
 define void @test_simplify2() {
-; CHECK: store i8* null, i8** @chp, align 4
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    store i8* null, i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %str = getelementptr [1 x i8], [1 x i8]* @null, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %str, i32 119)
@@ -32,9 +35,10 @@ define void @test_simplify2() {
 }
 
 define void @test_simplify3() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 0)
@@ -43,9 +47,10 @@ define void @test_simplify3() {
 }
 
 define void @test_simplify4() {
-; CHECK: store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13)
-; CHECK-NOT: call i8* @strrchr
-; CHECK: ret void
+; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    store i8* getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 13), i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 65280)
@@ -55,11 +60,33 @@ define void @test_simplify4() {
 
 define void @test_nosimplify1(i32 %chr) {
 ; CHECK-LABEL: @test_nosimplify1(
-; CHECK: call i8* @strrchr
-; CHECK: ret void
+; CHECK-NEXT:    [[DST:%.*]] = call i8* @strrchr(i8* nonnull dereferenceable(1) getelementptr inbounds ([14 x i8], [14 x i8]* @hello, i32 0, i32 0), i32 [[CHR:%.*]])
+; CHECK-NEXT:    store i8* [[DST]], i8** @chp, align 4
+; CHECK-NEXT:    ret void
+;
 
   %src = getelementptr [14 x i8], [14 x i8]* @hello, i32 0, i32 0
   %dst = call i8* @strrchr(i8* %src, i32 %chr)
   store i8* %dst, i8** @chp
   ret void
 }
+
+define i8* @test1(i8* %str, i32 %c) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strrchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strrchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str, i32 %c) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strrchr(i8* [[STR:%.*]], i32 [[C:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+
+  %ret = call i8* @strrchr(i8* %str, i32 %c)
+  ret i8* %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/strstr-1.ll b/llvm/test/Transforms/InstCombine/strstr-1.ll
index d57e56ccdaf37..4a9e62045bd45 100644
--- a/llvm/test/Transforms/InstCombine/strstr-1.ll
+++ b/llvm/test/Transforms/InstCombine/strstr-1.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; Test that the strstr library call simplifier works correctly.
 ;
 ; RUN: opt < %s -instcombine -S | FileCheck %s
@@ -15,51 +16,75 @@ declare i8* @strstr(i8*, i8*)
 
 define i8* @test_simplify1(i8* %str) {
 ; CHECK-LABEL: @test_simplify1(
+; CHECK-NEXT:    ret i8* [[STR:%.*]]
+;
   %pat = getelementptr inbounds [1 x i8], [1 x i8]* @.str, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %str
 }
 
 ; Check strstr(str, "a") -> strchr(str, 'a').
 
 define i8* @test_simplify2(i8* %str) {
 ; CHECK-LABEL: @test_simplify2(
+; CHECK-NEXT:    [[STRCHR:%.*]] = call i8* @strchr(i8* nonnull dereferenceable(1) [[STR:%.*]], i32 97)
+; CHECK-NEXT:    ret i8* [[STRCHR]]
+;
   %pat = getelementptr inbounds [2 x i8], [2 x i8]* @.str1, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: @strchr(i8* %str, i32 97)
 }
 
 ; Check strstr("abcde", "bcd") -> "abcde" + 1.
 
 define i8* @test_simplify3() {
 ; CHECK-LABEL: @test_simplify3(
+; CHECK-NEXT:    ret i8* getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 1)
+;
   %str = getelementptr inbounds [6 x i8], [6 x i8]* @.str2, i32 0, i32 0
   %pat = getelementptr inbounds [4 x i8], [4 x i8]* @.str3, i32 0, i32 0
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   ret i8* %ret
-; CHECK-NEXT: getelementptr inbounds ([6 x i8], [6 x i8]* @.str2, i64 0, i64 1)
 }
 
 ; Check strstr(str, str) -> str.
 
 define i8* @test_simplify4(i8* %str) {
 ; CHECK-LABEL: @test_simplify4(
+; CHECK-NEXT:    ret i8* [[STR:%.*]]
+;
   %ret = call i8* @strstr(i8* %str, i8* %str)
   ret i8* %ret
-; CHECK-NEXT: ret i8* %str
 }
 
 ; Check strstr(str, pat) == str -> strncmp(str, pat, strlen(str)) == 0.
 
 define i1 @test_simplify5(i8* %str, i8* %pat) {
 ; CHECK-LABEL: @test_simplify5(
+; CHECK-NEXT:    [[STRLEN:%.*]] = call i64 @strlen(i8* nonnull dereferenceable(1) [[PAT:%.*]])
+; CHECK-NEXT:    [[STRNCMP:%.*]] = call i32 @strncmp(i8* [[STR:%.*]], i8* [[PAT]], i64 [[STRLEN]])
+; CHECK-NEXT:    [[CMP1:%.*]] = icmp eq i32 [[STRNCMP]], 0
+; CHECK-NEXT:    ret i1 [[CMP1]]
+;
   %ret = call i8* @strstr(i8* %str, i8* %pat)
   %cmp = icmp eq i8* %ret, %str
   ret i1 %cmp
-; CHECK: [[LEN:%[a-z]+]] = call {{i[0-9]+}} @strlen(i8* %pat)
-; CHECK: [[NCMP:%[a-z]+]] = call {{i[0-9]+}} @strncmp(i8* %str, i8* %pat, {{i[0-9]+}} [[LEN]])
-; CHECK: icmp eq {{i[0-9]+}} [[NCMP]], 0
-; CHECK: ret i1
+}
+
+define i8* @test1(i8* %str1, i8* %str2) {
+; CHECK-LABEL: @test1(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strstr(i8* nonnull dereferenceable(1) [[STR1:%.*]], i8* nonnull dereferenceable(1) [[STR2:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strstr(i8* %str1, i8* %str2)
+  ret i8* %ret
+}
+
+define i8* @test2(i8* %str1, i8* %str2) "null-pointer-is-valid"="true" {
+; CHECK-LABEL: @test2(
+; CHECK-NEXT:    [[RET:%.*]] = call i8* @strstr(i8* [[STR1:%.*]], i8* [[STR2:%.*]])
+; CHECK-NEXT:    ret i8* [[RET]]
+;
+  %ret = call i8* @strstr(i8* %str1, i8* %str2)
+  ret i8* %ret
 }
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
new file mode 100644
index 0000000000000..66609f1418439
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-and-to-icmp-select.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+; Clamp negative to zero:
+; E.g., clamp0 implemented in a shifty way, could be optimized as v > 0 ? v : 0, where sub hasNoSignedWrap.
+; int32 clamp0(int32 v) {
+;   return ((-(v) >> 31) & (v));
+; }
+;
+
+; Scalar Types
+
+define i8 @sub_ashr_and_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_ashr_and_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i8 [[X]], i8 0
+; CHECK-NEXT:    ret i8 [[AND]]
+;
+  %sub = sub nsw i8 %y, %x
+  %shr = ashr i8 %sub, 7
+  %and = and i8 %shr, %x
+  ret i8 %and
+}
+
+define i16 @sub_ashr_and_i16(i16 %x, i16 %y) {
+; CHECK-LABEL: @sub_ashr_and_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i16 [[X]], i16 0
+; CHECK-NEXT:    ret i16 [[AND]]
+;
+
+  %sub = sub nsw i16 %y, %x
+  %shr = ashr i16 %sub, 15
+  %and = and i16 %shr, %x
+  ret i16 %and
+}
+
+define i32 @sub_ashr_and_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i64 @sub_ashr_and_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @sub_ashr_and_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i64 [[X]], i64 0
+; CHECK-NEXT:    ret i64 [[AND]]
+;
+  %sub = sub nsw i64 %y, %x
+  %shr = ashr i64 %sub, 63
+  %and = and i64 %shr, %x
+  ret i64 %and
+}
+
+; nuw nsw
+
+define i32 @sub_ashr_and_i32_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nuw nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+; Commute
+
+define i32 @sub_ashr_and_i32_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %x, %shr  ; commute %x and %shr
+  ret i32 %and
+}
+
+; Vector Types
+
+define <4 x i32> @sub_ashr_and_i32_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nuw nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[X]], <4 x i32> zeroinitializer
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %and = and <4 x i32> %x, %shr  ; commute %x and %shr
+  ret <4 x i32> %and
+}
+
+; Extra uses
+
+define i32 @sub_ashr_and_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  store i32 %sub, i32* %p
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i32 @sub_ashr_and_i32_extra_use_and(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_and(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 0
+; CHECK-NEXT:    store i32 [[AND]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %and = and i32 %shr, %x
+  store i32 %and, i32* %p
+  ret i32 %and
+}
+
+; Negative Tests
+
+define i32 @sub_ashr_and_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_and_i32_extra_use_ashr(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  store i32 %shr, i32* %p
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define i32 @sub_ashr_and_i32_no_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_no_nuw_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 7
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub i32 %y, %x
+  %shr = ashr i32 %sub, 7
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
+
+define <4 x i32> @sub_ashr_and_i32_vec_undef(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_vec_undef(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 31, i32 31, i32 31, i32 undef>
+; CHECK-NEXT:    [[AND:%.*]] = and <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[AND]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 undef>
+  %and = and <4 x i32> %shr, %x
+  ret <4 x i32> %and
+}
+
+define i32 @sub_ashr_and_i32_shift_wrong_bit(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_and_i32_shift_wrong_bit(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 15
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 15
+  %and = and i32 %shr, %x
+  ret i32 %and
+}
diff --git a/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
new file mode 100644
index 0000000000000..937bff62dc7a7
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-ashr-or-to-icmp-select.ll
@@ -0,0 +1,239 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -instcombine %s -S -o - | FileCheck %s
+
+; Clamp positive to allOnes:
+; E.g., clamp255 implemented in a shifty way, could be optimized as v > 255 ? 255 : v, where sub hasNoSignedWrap.
+; int32 clamp255(int32 v) {
+;   return (((255 - (v)) >> 31) | (v)) & 255;
+; }
+;
+
+; Scalar Types
+
+define i32 @clamp255_i32(i32 %x) {
+; CHECK-LABEL: @clamp255_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[X:%.*]], 255
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 [[X]], i32 255
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR]], 255
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %sub = sub nsw i32 255, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  %and = and i32 %or, 255
+  ret i32 %and
+}
+
+define i8 @sub_ashr_or_i8(i8 %x, i8 %y) {
+; CHECK-LABEL: @sub_ashr_or_i8(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i8 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i8 -1, i8 [[X]]
+; CHECK-NEXT:    ret i8 [[OR]]
+;
+  %sub = sub nsw i8 %y, %x
+  %shr = ashr i8 %sub, 7
+  %or = or i8 %shr, %x
+  ret i8 %or
+}
+
+define i16 @sub_ashr_or_i16(i16 %x, i16 %y) {
+; CHECK-LABEL: @sub_ashr_or_i16(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i16 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i16 -1, i16 [[X]]
+; CHECK-NEXT:    ret i16 [[OR]]
+;
+  %sub = sub nsw i16 %y, %x
+  %shr = ashr i16 %sub, 15
+  %or = or i16 %shr, %x
+  ret i16 %or
+}
+
+define i32 @sub_ashr_or_i32(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i64 @sub_ashr_or_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @sub_ashr_or_i64(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i64 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i64 -1, i64 [[X]]
+; CHECK-NEXT:    ret i64 [[OR]]
+;
+  %sub = sub nsw i64 %y, %x
+  %shr = ashr i64 %sub, 63
+  %or = or i64 %shr, %x
+  ret i64 %or
+}
+
+; nuw nsw
+
+define i32 @sub_ashr_or_i32_nuw_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nuw nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+; Commute
+
+define i32 @sub_ashr_or_i32_commute(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %x, %shr  ; commute %shr and %x
+  ret i32 %or
+}
+
+; Vector Types
+
+define <4 x i32> @sub_ashr_or_i32_vec(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_nuw_nsw(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_nuw_nsw(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nuw nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_commute(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_commute(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt <4 x i32> [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>, <4 x i32> [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> %y, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %x, %shr
+  ret <4 x i32> %or
+}
+
+; Extra uses
+
+define i32 @sub_ashr_or_i32_extra_use_sub(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    store i32 [[SUB]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[Y]], [[X]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  store i32 %sub, i32* %p
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i32 @sub_ashr_or_i32_extra_use_or(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_or(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[OR:%.*]] = select i1 [[TMP1]], i32 -1, i32 [[X]]
+; CHECK-NEXT:    store i32 [[OR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  store i32 %or, i32* %p
+  ret i32 %or
+}
+
+; Negative Tests
+
+define i32 @sub_ashr_or_i32_extra_use_ashr(i32 %x, i32 %y, i32* %p) {
+; CHECK-LABEL: @sub_ashr_or_i32_extra_use_ashr(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    store i32 [[SHR]], i32* [[P:%.*]], align 4
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  store i32 %shr, i32* %p
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define i32 @sub_ashr_or_i32_no_nsw_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_no_nsw_nuw(
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 31
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub i32 %y, %x
+  %shr = ashr i32 %sub, 31
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_undef1(<4 x i32> %x) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_undef1(
+; CHECK-NEXT:    [[SUB:%.*]] = sub <4 x i32> <i32 255, i32 255, i32 undef, i32 255>, [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 31, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub <4 x i32> <i32 255, i32 255, i32 undef, i32 255>, %x
+  %shr = ashr <4 x i32> %sub, <i32 31, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define <4 x i32> @sub_ashr_or_i32_vec_undef2(<4 x i32> %x) {
+; CHECK-LABEL: @sub_ashr_or_i32_vec_undef2(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw <4 x i32> <i32 255, i32 255, i32 255, i32 255>, [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr <4 x i32> [[SUB]], <i32 undef, i32 31, i32 31, i32 31>
+; CHECK-NEXT:    [[OR:%.*]] = or <4 x i32> [[SHR]], [[X]]
+; CHECK-NEXT:    ret <4 x i32> [[OR]]
+;
+  %sub = sub nsw <4 x i32> <i32 255, i32 255, i32 255, i32 255>, %x
+  %shr = ashr <4 x i32> %sub, <i32 undef, i32 31, i32 31, i32 31>
+  %or = or <4 x i32> %shr, %x
+  ret <4 x i32> %or
+}
+
+define i32 @sub_ashr_or_i32_shift_wrong_bit(i32 %x, i32 %y) {
+; CHECK-LABEL: @sub_ashr_or_i32_shift_wrong_bit(
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i32 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SUB]], 11
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[SHR]], [[X]]
+; CHECK-NEXT:    ret i32 [[OR]]
+;
+  %sub = sub nsw i32 %y, %x
+  %shr = ashr i32 %sub, 11
+  %or = or i32 %shr, %x
+  ret i32 %or
+}
diff --git a/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll
new file mode 100644
index 0000000000000..a5f38735a3738
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/variable-signext-of-variable-high-bit-extraction.ll
@@ -0,0 +1,575 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instcombine -S | FileCheck %s
+
+declare void @use16(i16)
+declare void @use32(i32)
+declare void @use64(i64)
+
+define i32 @t0(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t0(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t0_zext_of_nbits(i64 %data, i8 %nbits_narrow) {
+; CHECK-LABEL: @t0_zext_of_nbits(
+; CHECK-NEXT:    [[NBITS:%.*]] = zext i8 [[NBITS_NARROW:%.*]] to i16
+; CHECK-NEXT:    call void @use16(i16 [[NBITS]])
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub nsw i16 64, [[NBITS]]
+; CHECK-NEXT:    call void @use16(i16 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i16 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW:%.*]] = sub nsw i16 32, [[NBITS]]
+; CHECK-NEXT:    call void @use16(i16 [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = zext i16 [[NUM_HIGH_BITS_TO_SMEAR_NARROW_NARROW]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %nbits = zext i8 %nbits_narrow to i16
+  call void @use16(i16 %nbits)
+  %skip_high = sub i16 64, %nbits
+  call void @use16(i16 %skip_high)
+  %skip_high_wide = zext i16 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow_narrow = sub i16 32, %nbits
+  call void @use16(i16 %num_high_bits_to_smear_narrow_narrow)
+  %num_high_bits_to_smear_narrow = zext i16 %num_high_bits_to_smear_narrow_narrow to i32
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t0_exact(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t0_exact(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr exact i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr exact i64 %data, %skip_high_wide ; We can preserve `exact`-ness of the original shift.
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @t1_redundant_sext(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t1_redundant_sext(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED_WITH_SIGNEXTENSION:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED_WITH_SIGNEXTENSION]])
+; CHECK-NEXT:    [[EXTRACTED_WITH_SIGNEXTENSION_NARROW:%.*]] = trunc i64 [[EXTRACTED_WITH_SIGNEXTENSION]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    ret i32 [[EXTRACTED_WITH_SIGNEXTENSION_NARROW]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted_with_signextension = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted_with_signextension)
+  %extracted_with_signextension_narrow = trunc i64 %extracted_with_signextension to i32 ; this is already the answer.
+  call void @use32(i32 %extracted_with_signextension_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_with_signextension_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i64 @t2_notrunc(i64 %data, i64 %nbits) {
+; CHECK-LABEL: @t2_notrunc(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i64 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR:%.*]] = sub i64 64, [[NBITS]]
+; CHECK-NEXT:    call void @use64(i64 [[NUM_HIGH_BITS_TO_SMEAR]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i64 [[EXTRACTED]], [[NUM_HIGH_BITS_TO_SMEAR]]
+; CHECK-NEXT:    call void @use64(i64 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH]]
+; CHECK-NEXT:    ret i64 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i64 64, %nbits
+  call void @use64(i64 %skip_high)
+  %extracted = lshr i64 %data, %skip_high
+  call void @use64(i64 %extracted)
+  %num_high_bits_to_smear = sub i64 64, %nbits
+  call void @use64(i64 %num_high_bits_to_smear)
+  %signbit_positioned = shl i64 %extracted, %num_high_bits_to_smear ;
+  call void @use64(i64 %signbit_positioned)
+  %signextended = ashr i64 %signbit_positioned, %num_high_bits_to_smear ; can just shift %data itself.
+  ret i64 %signextended
+}
+
+define i64 @t3_notrunc_redundant_sext(i64 %data, i64 %nbits) {
+; CHECK-LABEL: @t3_notrunc_redundant_sext(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i64 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR:%.*]] = sub i64 64, [[NBITS]]
+; CHECK-NEXT:    call void @use64(i64 [[NUM_HIGH_BITS_TO_SMEAR]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i64 [[EXTRACTED]], [[NUM_HIGH_BITS_TO_SMEAR]]
+; CHECK-NEXT:    call void @use64(i64 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    ret i64 [[EXTRACTED]]
+;
+  %skip_high = sub i64 64, %nbits
+  call void @use64(i64 %skip_high)
+  %extracted = ashr i64 %data, %skip_high ; this is already the answer.
+  call void @use64(i64 %extracted)
+  %num_high_bits_to_smear = sub i64 64, %nbits
+  call void @use64(i64 %num_high_bits_to_smear)
+  %signbit_positioned = shl i64 %extracted, %num_high_bits_to_smear
+  call void @use64(i64 %signbit_positioned)
+  %signextended = ashr i64 %signbit_positioned, %num_high_bits_to_smear
+  ret i64 %signextended
+}
+
+define <2 x i32> @t4_vec(<2 x i64> %data, <2 x i32> %nbits) {
+; CHECK-LABEL: @t4_vec(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub <2 x i32> <i32 64, i32 64>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext <2 x i32> [[SKIP_HIGH]] to <2 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <2 x i64> [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc <2 x i64> [[TMP1]] to <2 x i32>
+; CHECK-NEXT:    ret <2 x i32> [[SIGNEXTENDED]]
+;
+  %skip_high = sub <2 x i32> <i32 64, i32 64>, %nbits
+  %skip_high_wide = zext <2 x i32> %skip_high to <2 x i64>
+  %extracted = lshr <2 x i64> %data, %skip_high_wide
+  %extracted_narrow = trunc <2 x i64> %extracted to <2 x i32>
+  %num_high_bits_to_smear_narrow = sub <2 x i32> <i32 32, i32 32>, %nbits
+  %signbit_positioned = shl <2 x i32> %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr <2 x i32> %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret <2 x i32> %signextended
+}
+
+define <3 x i32> @t5_vec_undef(<3 x i64> %data, <3 x i32> %nbits) {
+; CHECK-LABEL: @t5_vec_undef(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub <3 x i32> <i32 64, i32 64, i32 undef>, [[NBITS:%.*]]
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext <3 x i32> [[SKIP_HIGH]] to <3 x i64>
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <3 x i64> [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc <3 x i64> [[TMP1]] to <3 x i32>
+; CHECK-NEXT:    ret <3 x i32> [[SIGNEXTENDED]]
+;
+  %skip_high = sub <3 x i32> <i32 64, i32 64, i32 undef>, %nbits
+  %skip_high_wide = zext <3 x i32> %skip_high to <3 x i64>
+  %extracted = lshr <3 x i64> %data, %skip_high_wide
+  %extracted_narrow = trunc <3 x i64> %extracted to <3 x i32>
+  %num_high_bits_to_smear_narrow0 = sub <3 x i32> <i32 32, i32 32, i32 undef>, %nbits
+  %num_high_bits_to_smear_narrow1 = sub <3 x i32> <i32 undef, i32 32, i32 32>, %nbits
+  %signbit_positioned = shl <3 x i32> %extracted_narrow, %num_high_bits_to_smear_narrow0
+  %signextended = ashr <3 x i32> %signbit_positioned, %num_high_bits_to_smear_narrow1
+  ret <3 x i32> %signextended
+}
+
+; Extra-uses
+define i32 @t6_extrause_good0(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t6_extrause_good0(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow ; will go away
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+define i32 @t7_extrause_good1(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @t7_extrause_good1(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW0:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW0]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW0]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr i64 [[DATA]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = trunc i64 [[TMP1]] to i32
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow0 = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow0)
+  %num_high_bits_to_smear_narrow1 = sub i32 32, %nbits ; will go away.
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow0
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow1
+  ret i32 %signextended
+}
+define i32 @n8_extrause_bad(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n8_extrause_bad(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[SIGNBIT_POSITIONED]])
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %signbit_positioned)
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow ; neither of operands will go away.
+  ret i32 %signextended
+}
+
+; Negative tests
+define i32 @n9(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n9(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 63, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 63, %nbits ; not 64
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n10(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n10(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 31, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 31, %nbits ; not 32
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n11(i64 %data, i32 %nbits1, i32 %nbits2) {
+; CHECK-LABEL: @n11(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS1:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS2:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits1 ; not %nbits2
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits2 ; not %nbits1
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow
+  ret i32 %signextended
+}
+
+define i32 @n12(i64 %data, i32 %nbits1, i32 %nbits2) {
+; CHECK-LABEL: @n12(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS1:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW1:%.*]] = sub i32 32, [[NBITS1]]
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW2:%.*]] = sub i32 32, [[NBITS2:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW1]])
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW2]])
+; CHECK-NEXT:    [[SIGNBIT_POSITIONED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW1]]
+; CHECK-NEXT:    [[SIGNEXTENDED:%.*]] = ashr i32 [[SIGNBIT_POSITIONED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW2]]
+; CHECK-NEXT:    ret i32 [[SIGNEXTENDED]]
+;
+  %skip_high = sub i32 64, %nbits1
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow1 = sub i32 32, %nbits1 ; not %nbits2
+  %num_high_bits_to_smear_narrow2 = sub i32 32, %nbits2 ; not %nbits1
+  call void @use32(i32 %num_high_bits_to_smear_narrow1)
+  call void @use32(i32 %num_high_bits_to_smear_narrow2)
+  %signbit_positioned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow1
+  %signextended = ashr i32 %signbit_positioned, %num_high_bits_to_smear_narrow2
+  ret i32 %signextended
+}
+
+define i32 @n13(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n13(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 -1, [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[TMP1]], [[EXTRACTED_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n13_extrause(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n13_extrause(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = lshr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[HIGHBITS_CLEANED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[HIGHBITS_CLEANED]])
+; CHECK-NEXT:    [[RES:%.*]] = lshr i32 [[HIGHBITS_CLEANED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = lshr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %highbits_cleaned)
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n14(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n14(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 -1, [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    [[RES:%.*]] = and i32 [[TMP1]], [[EXTRACTED_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
+define i32 @n14_extrause(i64 %data, i32 %nbits) {
+; CHECK-LABEL: @n14_extrause(
+; CHECK-NEXT:    [[SKIP_HIGH:%.*]] = sub i32 64, [[NBITS:%.*]]
+; CHECK-NEXT:    call void @use32(i32 [[SKIP_HIGH]])
+; CHECK-NEXT:    [[SKIP_HIGH_WIDE:%.*]] = zext i32 [[SKIP_HIGH]] to i64
+; CHECK-NEXT:    call void @use64(i64 [[SKIP_HIGH_WIDE]])
+; CHECK-NEXT:    [[EXTRACTED:%.*]] = ashr i64 [[DATA:%.*]], [[SKIP_HIGH_WIDE]]
+; CHECK-NEXT:    call void @use64(i64 [[EXTRACTED]])
+; CHECK-NEXT:    [[EXTRACTED_NARROW:%.*]] = trunc i64 [[EXTRACTED]] to i32
+; CHECK-NEXT:    call void @use32(i32 [[EXTRACTED_NARROW]])
+; CHECK-NEXT:    [[NUM_HIGH_BITS_TO_SMEAR_NARROW:%.*]] = sub i32 32, [[NBITS]]
+; CHECK-NEXT:    call void @use32(i32 [[NUM_HIGH_BITS_TO_SMEAR_NARROW]])
+; CHECK-NEXT:    [[HIGHBITS_CLEANED:%.*]] = shl i32 [[EXTRACTED_NARROW]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    call void @use32(i32 [[HIGHBITS_CLEANED]])
+; CHECK-NEXT:    [[RES:%.*]] = lshr i32 [[HIGHBITS_CLEANED]], [[NUM_HIGH_BITS_TO_SMEAR_NARROW]]
+; CHECK-NEXT:    ret i32 [[RES]]
+;
+  %skip_high = sub i32 64, %nbits
+  call void @use32(i32 %skip_high)
+  %skip_high_wide = zext i32 %skip_high to i64
+  call void @use64(i64 %skip_high_wide)
+  %extracted = ashr i64 %data, %skip_high_wide
+  call void @use64(i64 %extracted)
+  %extracted_narrow = trunc i64 %extracted to i32
+  call void @use32(i32 %extracted_narrow)
+  %num_high_bits_to_smear_narrow = sub i32 32, %nbits
+  call void @use32(i32 %num_high_bits_to_smear_narrow)
+  %highbits_cleaned = shl i32 %extracted_narrow, %num_high_bits_to_smear_narrow
+  call void @use32(i32 %highbits_cleaned)
+  %res = lshr i32 %highbits_cleaned, %num_high_bits_to_smear_narrow ; not ashr
+  ret i32 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
index 3f172be3d8304..164061de5112c 100644
--- a/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
+++ b/llvm/test/Transforms/InstCombine/vec_demanded_elts.ll
@@ -651,3 +651,20 @@ define i32* @PR41624(<2 x { i32, i32 }*> %a) {
   %r = extractelement <2 x i32*> %w, i32 0
   ret i32* %r
 }
+
+@global = external global [0 x i32], align 4
+
+; Make sure we don't get stuck in a loop turning the zeroinitializer into
+; <0, undef, undef, undef> and then changing it back.
+define i32* @zero_sized_type_extract(<4 x i64> %arg, i64 %arg1) {
+; CHECK-LABEL: @zero_sized_type_extract(
+; CHECK-NEXT:  bb:
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* undef, [0 x i32]* undef, [0 x i32]* undef>, <4 x i64> <i64 0, i64 undef, i64 undef, i64 undef>, <4 x i64> [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32*> [[TMP]], i64 0
+; CHECK-NEXT:    ret i32* [[TMP2]]
+;
+bb:
+  %tmp = getelementptr inbounds [0 x i32], <4 x [0 x i32]*> <[0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global, [0 x i32]* @global>, <4 x i64> zeroinitializer, <4 x i64> %arg
+  %tmp2 = extractelement <4 x i32*> %tmp, i64 0
+  ret i32* %tmp2
+}
diff --git a/llvm/test/Transforms/InstSimplify/call.ll b/llvm/test/Transforms/InstSimplify/call.ll
index 27497e5e862bc..2fc0841a8e15b 100644
--- a/llvm/test/Transforms/InstSimplify/call.ll
+++ b/llvm/test/Transforms/InstSimplify/call.ll
@@ -742,3 +742,181 @@ define i9 @fshr_shift_undef(i9 %x, i9 %y) {
   ret i9 %r
 }
 
+declare double @llvm.fma.f64(double,double,double)
+declare double @llvm.fmuladd.f64(double,double,double)
+
+define double @fma_undef_op0(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double undef, double %x, double %y)
+  ret double %r
+}
+
+define double @fma_undef_op1(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double undef, double %y)
+  ret double %r
+}
+
+define double @fma_undef_op2(double %x, double %y) {
+; CHECK-LABEL: @fma_undef_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double %x, double %y, double undef)
+  ret double %r
+}
+
+define double @fmuladd_undef_op0(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double undef, double %x, double %y)
+  ret double %r
+}
+
+define double @fmuladd_undef_op1(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double undef, double %y)
+  ret double %r
+}
+
+define double @fmuladd_undef_op2(double %x, double %y) {
+; CHECK-LABEL: @fmuladd_undef_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double %y, double undef)
+  ret double %r
+}
+
+define double @fma_nan_op0(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op0(
+; CHECK-NEXT:    ret double 0x7FF8000000000000
+;
+  %r = call double @llvm.fma.f64(double 0x7ff8000000000000, double %x, double %y)
+  ret double %r
+}
+
+define double @fma_nan_op1(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000000001
+;
+  %r = call double @llvm.fma.f64(double %x, double 0x7ff8000000000001, double %y)
+  ret double %r
+}
+
+define double @fma_nan_op2(double %x, double %y) {
+; CHECK-LABEL: @fma_nan_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000000002
+;
+  %r = call double @llvm.fma.f64(double %x, double %y, double 0x7ff8000000000002)
+  ret double %r
+}
+
+define double @fmuladd_nan_op0_op1(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op0_op1(
+; CHECK-NEXT:    ret double 0x7FF8000000001234
+;
+  %r = call double @llvm.fmuladd.f64(double 0x7ff8000000001234, double 0x7ff800000000dead, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_op0_op2(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op0_op2(
+; CHECK-NEXT:    ret double 0x7FF8000000005678
+;
+  %r = call double @llvm.fmuladd.f64(double 0x7ff8000000005678, double %x, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @fmuladd_nan_op1_op2(double %x) {
+; CHECK-LABEL: @fmuladd_nan_op1_op2(
+; CHECK-NEXT:    ret double 0x7FF80000AAAAAAAA
+;
+  %r = call double @llvm.fmuladd.f64(double %x, double 0x7ff80000aaaaaaaa, double 0x7ff800000000dead)
+  ret double %r
+}
+
+define double @fma_nan_multiplicand_inf_zero(double %x) {
+; CHECK-LABEL: @fma_nan_multiplicand_inf_zero(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0x7FF0000000000000, double 0.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fma.f64(double 0x7ff0000000000000, double 0.0, double %x)
+  ret double %r
+}
+
+define double @fma_nan_multiplicand_zero_inf(double %x) {
+; CHECK-LABEL: @fma_nan_multiplicand_zero_inf(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0.000000e+00, double 0x7FF0000000000000, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fma.f64(double 0.0, double 0x7ff0000000000000, double %x)
+  ret double %r
+}
+
+define double @fma_nan_addend_inf_neginf(double %x, i32 %y) {
+; CHECK-LABEL: @fma_nan_addend_inf_neginf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = uitofp i32 [[Y:%.*]] to double
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double 0x7FF0000000000000, double [[NOTNAN]], double 0xFFF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = uitofp i32 %y to double
+  %r = call double @llvm.fma.f64(double 0x7ff0000000000000, double %notnan, double 0xfff0000000000000)
+  ret double %r
+}
+
+define double @fma_nan_addend_neginf_inf(double %x, i1 %y) {
+; CHECK-LABEL: @fma_nan_addend_neginf_inf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = select i1 [[Y:%.*]], double 4.200000e+01, double -1.000000e-01
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fma.f64(double [[NOTNAN]], double 0xFFF0000000000000, double 0x7FF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = select i1 %y, double 42.0, double -0.1
+  %r = call double @llvm.fma.f64(double %notnan, double 0xfff0000000000000, double 0x7ff0000000000000)
+  ret double %r
+}
+
+define double @fmuladd_nan_multiplicand_neginf_zero(double %x) {
+; CHECK-LABEL: @fmuladd_nan_multiplicand_neginf_zero(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double 0xFFF0000000000000, double 0.000000e+00, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fmuladd.f64(double 0xfff0000000000000, double 0.0, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_multiplicand_negzero_inf(double %x) {
+; CHECK-LABEL: @fmuladd_nan_multiplicand_negzero_inf(
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double -0.000000e+00, double 0x7FF0000000000000, double [[X:%.*]])
+; CHECK-NEXT:    ret double [[R]]
+;
+  %r = call double @llvm.fmuladd.f64(double -0.0, double 0x7ff0000000000000, double %x)
+  ret double %r
+}
+
+define double @fmuladd_nan_addend_inf_neginf(double %x, i32 %y) {
+; CHECK-LABEL: @fmuladd_nan_addend_inf_neginf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = sitofp i32 [[Y:%.*]] to double
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double 0x7FF0000000000000, double [[NOTNAN]], double 0xFFF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = sitofp i32 %y to double
+  %r = call double @llvm.fmuladd.f64(double 0x7ff0000000000000, double %notnan, double 0xfff0000000000000)
+  ret double %r
+}
+
+define double @fmuladd_nan_addend_neginf_inf(double %x, i1 %y) {
+; CHECK-LABEL: @fmuladd_nan_addend_neginf_inf(
+; CHECK-NEXT:    [[NOTNAN:%.*]] = select i1 [[Y:%.*]], double 4.200000e+01, double -1.000000e-01
+; CHECK-NEXT:    [[R:%.*]] = call double @llvm.fmuladd.f64(double [[NOTNAN]], double 0xFFF0000000000000, double 0x7FF0000000000000)
+; CHECK-NEXT:    ret double [[R]]
+;
+  %notnan = select i1 %y, double 42.0, double -0.1
+  %r = call double @llvm.fmuladd.f64(double %notnan, double 0xfff0000000000000, double 0x7ff0000000000000)
+  ret double %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll b/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
index 58a56c4d7babb..9521a71bd701f 100644
--- a/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
+++ b/llvm/test/Transforms/InstSimplify/result-of-add-of-negative-is-non-zero-and-no-underflow.ll
@@ -99,3 +99,27 @@ define i1 @t4_commutative(i8 %base, i8 %offset) {
   %r = and i1 %not_null, %no_underflow
   ret i1 %r
 }
+
+; We only need to know that any of the 'add' operands is non-zero,
+; not necessarily the one used in the comparison.
+define i1 @t5(i8 %base, i8 %offset) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[OFFSET:%.*]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = add i8 [[BASE:%.*]], [[OFFSET]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[ADJUSTED]], [[BASE]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %cmp = icmp slt i8 %offset, 0
+  call void @llvm.assume(i1 %cmp)
+
+  %adjusted = add i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %adjusted, %base
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll b/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
index 66ff797c252cb..fd473ac9d251f 100644
--- a/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
+++ b/llvm/test/Transforms/InstSimplify/result-of-usub-is-non-zero-and-no-overflow.ll
@@ -14,16 +14,15 @@ declare void @llvm.assume(i1)
 ; If we are checking that we either did not get null or got no overflow,
 ; this is tautological and is always true.
 
-define i1 @t1(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t1(
+define i1 @commutativity0(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity0(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
 ; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
@@ -34,107 +33,307 @@ define i1 @t1(i8 %base, i8 %offset) {
   %r = or i1 %not_null, %no_underflow
   ret i1 %r
 }
-define i1 @t1_strict_bad(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t1_strict_bad(
+define i1 @commutativity1(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity1(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
-  %no_underflow = icmp ugt i8 %base, %offset ; but not for non-strict predicate
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
   call void @use1(i1 %no_underflow)
   %not_null = icmp ne i8 %adjusted, 0
   call void @use1(i1 %not_null)
   %r = or i1 %not_null, %no_underflow
   ret i1 %r
 }
-define i1 @t1_commutativity(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t1_commutativity(
+define i1 @commutativity2(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity2(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %no_underflow = icmp uge i8 %base, %offset
+  call void @use1(i1 %no_underflow)
+  %not_null = icmp ne i8 %adjusted, 0
+  call void @use1(i1 %not_null)
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+define i1 @commutativity3(i8 %base, i8 %offset) {
+; CHECK-LABEL: @commutativity3(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
 ; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[OFFSET]], [[BASE]]
 ; CHECK-NEXT:    call void @use1(i1 [[NO_UNDERFLOW]])
 ; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
 ; CHECK-NEXT:    call void @use1(i1 [[NOT_NULL]])
-; CHECK-NEXT:    [[R:%.*]] = or i1 [[NOT_NULL]], [[NO_UNDERFLOW]]
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
-  %no_underflow = icmp ule i8 %offset, %base
+  %no_underflow = icmp ule i8 %offset, %base ; swapped
   call void @use1(i1 %no_underflow)
   %not_null = icmp ne i8 %adjusted, 0
   call void @use1(i1 %not_null)
-  %r = or i1 %not_null, %no_underflow
+  %r = or i1 %no_underflow, %not_null ; swapped
   ret i1 %r
 }
 
-; Likewise, if we are checking that we both got null and overflow happened,
-; it makes no sense and is always false.
+;-------------------------------------------------------------------------------
 
-define i1 @t2(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t2(
+define i1 @exaustive_t0_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t0_no_underflow(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
-; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
-; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NULL]], [[UNDERFLOW]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t1_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t1_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t2_false(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t2_false(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 false
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t3_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t3_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ult i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
-  %underflow = icmp ult i8 %base, %offset
-  call void @use1(i1 %underflow)
-  %null = icmp eq i8 %adjusted, 0
-  call void @use1(i1 %null)
-  %r = and i1 %null, %underflow
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ult i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
   ret i1 %r
 }
-define i1 @t2_nonstrict_bad(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t2_nonstrict_bad(
+
+define i1 @exaustive_t4_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t4_no_underflow(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp uge i8 [[ADJUSTED]], [[BASE]]
-; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
-; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NULL]], [[UNDERFLOW]]
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t5_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t5_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t6_false(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t6_false(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 false
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t7_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t7_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = or i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
-  %underflow = icmp uge i8 %adjusted, %base ; but not for non-strict predicate
-  call void @use1(i1 %underflow)
-  %null = icmp eq i8 %adjusted, 0
-  call void @use1(i1 %null)
-  %r = and i1 %null, %underflow
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ugt i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t8_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t8_bad(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
+; CHECK-NEXT:    ret i1 [[R]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t9_true(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t9_true(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t10_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t10_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t11_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t11_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp ule i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
   ret i1 %r
 }
-define i1 @t2_commutativity(i8 %base, i8 %offset) {
-; CHECK-LABEL: @t2_commutativity(
+
+define i1 @exaustive_t12_bad(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t12_bad(
 ; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
 ; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
-; CHECK-NEXT:    [[UNDERFLOW:%.*]] = icmp ugt i8 [[OFFSET]], [[BASE]]
-; CHECK-NEXT:    call void @use1(i1 [[UNDERFLOW]])
-; CHECK-NEXT:    [[NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
-; CHECK-NEXT:    call void @use1(i1 [[NULL]])
-; CHECK-NEXT:    [[R:%.*]] = and i1 [[NULL]], [[UNDERFLOW]]
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    [[R:%.*]] = and i1 [[NO_UNDERFLOW]], [[NOT_NULL]]
 ; CHECK-NEXT:    ret i1 [[R]]
 ;
   %adjusted = sub i8 %base, %offset
   call void @use8(i8 %adjusted)
-  %underflow = icmp ugt i8 %offset, %base
-  call void @use1(i1 %underflow)
-  %null = icmp eq i8 %adjusted, 0
-  call void @use1(i1 %null)
-  %r = and i1 %null, %underflow
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t13_true(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t13_true(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    ret i1 true
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp ne i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t14_not_null(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t14_not_null(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[ADJUSTED]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = and i1 %no_underflow, %not_null
+  ret i1 %r
+}
+
+define i1 @exaustive_t15_no_underflow(i8 %base, i8 %offset) {
+; CHECK-LABEL: @exaustive_t15_no_underflow(
+; CHECK-NEXT:    [[ADJUSTED:%.*]] = sub i8 [[BASE:%.*]], [[OFFSET:%.*]]
+; CHECK-NEXT:    call void @use8(i8 [[ADJUSTED]])
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp uge i8 [[BASE]], [[OFFSET]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %adjusted = sub i8 %base, %offset
+  call void @use8(i8 %adjusted)
+  %not_null = icmp eq i8 %adjusted, 0
+  %no_underflow = icmp uge i8 %base, %offset
+  %r = or i1 %no_underflow, %not_null
   ret i1 %r
 }
diff --git a/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll b/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll
new file mode 100644
index 0000000000000..e0c3a7c8d689e
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/unsigned-range-checks.ll
@@ -0,0 +1,130 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt %s -instsimplify -S | FileCheck %s
+
+; Here we add unsigned two values, check that addition did not underflow AND
+; that the result is non-zero. This can be simplified just to a comparison
+; between the base and negated offset.
+
+; If we are checking that the result is not null or no underflow happened,
+; it is tautological (always-true).
+define i1 @t1(i8 %x, i8 %y) {
+; CHECK-LABEL: @t1(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t2_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t2_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp uge i8 %x, %y ; swapped
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+define i1 @t3_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t3_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+define i1 @t4_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t4_commutative(
+; CHECK-NEXT:    ret i1 true
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp uge i8 %x, %y ; swapped
+  %r = or i1 %no_underflow, %not_null ; swapped
+  ret i1 %r
+}
+
+; If we are checking that the result is null and underflow happened,
+; it is tautological (always-false).
+define i1 @t5(i8 %x, i8 %y) {
+; CHECK-LABEL: @t5(
+; CHECK-NEXT:    ret i1 false
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+define i1 @t6_commutative(i8 %x, i8 %y) {
+; CHECK-LABEL: @t6_commutative(
+; CHECK-NEXT:    ret i1 false
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ult i8 %x, %y ; swapped
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; We only need to know that any of the 'add' operands is non-zero,
+; not necessarily the one used in the comparison.
+define i1 @t7(i8 %x, i8 %y) {
+; CHECK-LABEL: @t7(
+; CHECK-NEXT:    ret i1 true
+;
+  %cmp = icmp slt i8 %y, 0
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; If we check that no underflow happened and that the result is null,
+; we can just check for null.
+define i1 @t8(i8 %x, i8 %y) {
+; CHECK-LABEL: @t8(
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp eq i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
+; Likewise, if we check that result is non-null or underflow happened,
+; we can just check for null.
+define i1 @t9(i8 %x, i8 %y) {
+; CHECK-LABEL: @t9(
+; CHECK-NEXT:    [[NOT_NULL:%.*]] = icmp ne i8 [[Y:%.*]], 0
+; CHECK-NEXT:    ret i1 [[NOT_NULL]]
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+
+; If we check that no underflow happened or that the result is not null,
+; we can just check for lack of underflow.
+define i1 @t10(i8 %x, i8 %y) {
+; CHECK-LABEL: @t10(
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ule i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %not_null = icmp eq i8 %y, 0
+  %no_underflow = icmp ule i8 %y, %x
+  %r = or i1 %not_null, %no_underflow
+  ret i1 %r
+}
+; Likewise, if we check that underflow happened and that the result is not null,
+; we can just check for lack of underflow.
+define i1 @t11(i8 %x, i8 %y) {
+; CHECK-LABEL: @t11(
+; CHECK-NEXT:    [[NO_UNDERFLOW:%.*]] = icmp ugt i8 [[Y:%.*]], [[X:%.*]]
+; CHECK-NEXT:    ret i1 [[NO_UNDERFLOW]]
+;
+  %not_null = icmp ne i8 %y, 0
+  %no_underflow = icmp ugt i8 %y, %x
+  %r = and i1 %not_null, %no_underflow
+  ret i1 %r
+}
diff --git a/llvm/test/Transforms/LoopFusion/guarded.ll b/llvm/test/Transforms/LoopFusion/guarded.ll
new file mode 100644
index 0000000000000..1a131178d25e5
--- /dev/null
+++ b/llvm/test/Transforms/LoopFusion/guarded.ll
@@ -0,0 +1,67 @@
+; RUN: opt -S -loop-fusion < %s | FileCheck %s
+
+@B = common global [1024 x i32] zeroinitializer, align 16
+
+; CHECK: void @dep_free_parametric
+; CHECK-next: entry:
+; CHECK: br i1 %{{.*}}, label %[[LOOP1PREHEADER:bb[0-9]*]], label %[[LOOP1SUCC:bb[0-9]+]]
+; CHECK: [[LOOP1PREHEADER]]
+; CHECK-NEXT: br label %[[LOOP1BODY:bb[0-9]*]]
+; CHECK: [[LOOP1BODY]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP2BODY:bb[0-9]*]], label %[[LOOP2BODY]]
+; CHECK: [[LOOP2BODY]]
+; CHECK: br i1 %{{.*}}, label %[[LOOP1BODY]], label %[[LOOP2EXIT:bb[0-9]+]]
+; CHECK: [[LOOP2EXIT]]
+; CHECK: br label %[[LOOP1SUCC]]
+; CHECK: [[LOOP1SUCC]]
+; CHECK: ret void
+define void @dep_free_parametric(i32* noalias %A, i64 %N) {
+entry:
+  %cmp4 = icmp slt i64 0, %N
+  br i1 %cmp4, label %bb3, label %bb14
+
+bb3:                               ; preds = %entry
+  br label %bb5
+
+bb5:                                         ; preds = %bb3, %bb5
+  %i.05 = phi i64 [ %inc, %bb5 ], [ 0, %bb3 ]
+  %sub = sub nsw i64 %i.05, 3
+  %add = add nsw i64 %i.05, 3
+  %mul = mul nsw i64 %sub, %add
+  %rem = srem i64 %mul, %i.05
+  %conv = trunc i64 %rem to i32
+  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %i.05
+  store i32 %conv, i32* %arrayidx, align 4
+  %inc = add nsw i64 %i.05, 1
+  %cmp = icmp slt i64 %inc, %N
+  br i1 %cmp, label %bb5, label %bb10
+
+bb10:                                 ; preds = %bb5
+  br label %bb14
+
+bb14:                                          ; preds = %bb10, %entry
+  %cmp31 = icmp slt i64 0, %N
+  br i1 %cmp31, label %bb8, label %bb12
+
+bb8:                              ; preds = %bb14
+  br label %bb9
+
+bb9:                                        ; preds = %bb8, %bb9
+  %i1.02 = phi i64 [ %inc14, %bb9 ], [ 0, %bb8 ]
+  %sub7 = sub nsw i64 %i1.02, 3
+  %add8 = add nsw i64 %i1.02, 3
+  %mul9 = mul nsw i64 %sub7, %add8
+  %rem10 = srem i64 %mul9, %i1.02
+  %conv11 = trunc i64 %rem10 to i32
+  %arrayidx12 = getelementptr inbounds [1024 x i32], [1024 x i32]* @B, i64 0, i64 %i1.02
+  store i32 %conv11, i32* %arrayidx12, align 4
+  %inc14 = add nsw i64 %i1.02, 1
+  %cmp3 = icmp slt i64 %inc14, %N
+  br i1 %cmp3, label %bb9, label %bb15
+
+bb15:                               ; preds = %bb9
+  br label %bb12
+
+bb12:                                        ; preds = %bb15, %bb14
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
index c0bf00b938abf..b0b3a55787a1c 100644
--- a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
+++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-optsize.ll
@@ -136,7 +136,7 @@ define void @fully_unrolled_bigger() #0 {
 ; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds [4 x i32], [4 x i32]* [[ARR]], i64 0, i64 [[INDVARS_IV]]
 ; CHECK-NEXT:    store i32 [[OR]], i32* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
-; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 6
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV]], 7
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
 ; CHECK-NEXT:    [[PTR:%.*]] = bitcast [4 x i32]* [[ARR]] to i32*
@@ -156,7 +156,7 @@ for.body:                                         ; preds = %for.body, %entry
   %arrayidx = getelementptr inbounds [4 x i32], [4 x i32]* %arr, i64 0, i64 %indvars.iv
   store i32 %or, i32* %arrayidx, align 4
   %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
-  %exitcond = icmp eq i64 %indvars.iv, 6
+  %exitcond = icmp eq i64 %indvars.iv, 7
   br i1 %exitcond, label %for.cond.cleanup, label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.cond
diff --git a/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
new file mode 100644
index 0000000000000..c89948ad52cb7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/disable-full-unroll-by-opt.ll
@@ -0,0 +1,35 @@
+; Default behavior
+; RUN: opt < %s -passes='unroll' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; Pass option
+; RUN: opt < %s -passes='unroll<full-unroll-max=0>'  -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; cl::opt option
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=0 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+; Pass option has a priority over cl::opt
+; RUN: opt < %s -passes='unroll<full-unroll-max=30>' -unroll-full-max-count=36 -S | FileCheck %s -check-prefixes=DISABLE,COMMON
+; RUN: opt < %s -passes='unroll<full-unroll-max=36>' -unroll-full-max-count=30 -S | FileCheck %s -check-prefixes=ENABLE,COMMON
+
+define void @test() {
+; COMMON-LABEL: @test(
+ entry:
+  br label %loop
+
+ loop:
+  %idx = phi i32 [ 0, %entry ], [ %idx.inc, %loop ]
+  %idx.inc = add i32 %idx, 1
+  %be = icmp slt i32 %idx, 32
+  br i1 %be, label %loop, label %exit
+
+; COMMON: loop:
+; DISABLE:  %be = icmp slt i32 %idx, 32
+; ENABLE-NOT:  %be = icmp slt i32 %idx, 32
+
+ exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll b/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll
new file mode 100644
index 0000000000000..4ca397a1c770e
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/optsize-loop-size.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -loop-unroll -S < %s | FileCheck %s
+
+define i32 @test(i32 %a, i32 %b, i32 %c) optsize {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[REF_TMP:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    [[TMP0:%.*]] = bitcast [3 x i32]* [[REF_TMP]] to i8*
+; CHECK-NEXT:    [[ARRAYINIT_BEGIN:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 0
+; CHECK-NEXT:    store i32 [[A:%.*]], i32* [[ARRAYINIT_BEGIN]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 1
+; CHECK-NEXT:    store i32 [[B:%.*]], i32* [[ARRAYINIT_ELEMENT]], align 4
+; CHECK-NEXT:    [[ARRAYINIT_ELEMENT1:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 2
+; CHECK-NEXT:    store i32 [[C:%.*]], i32* [[ARRAYINIT_ELEMENT1]], align 4
+; CHECK-NEXT:    [[ADD_PTR_I_I:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 3
+; CHECK-NEXT:    [[CMP_I_I_I3:%.*]] = icmp slt i32 [[A]], [[B]]
+; CHECK-NEXT:    [[SPEC_SELECT_I_I4:%.*]] = select i1 [[CMP_I_I_I3]], i32* [[ARRAYINIT_ELEMENT]], i32* [[ARRAYINIT_BEGIN]]
+; CHECK-NEXT:    [[INCDEC_PTR_I_I5:%.*]] = getelementptr inbounds [3 x i32], [3 x i32]* [[REF_TMP]], i64 0, i64 2
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[DOTPRE:%.*]] = load i32, i32* [[SPEC_SELECT_I_I4]], align 4
+; CHECK-NEXT:    [[DOTPRE2:%.*]] = load i32, i32* [[INCDEC_PTR_I_I5]], align 4
+; CHECK-NEXT:    [[CMP_I_I_I:%.*]] = icmp slt i32 [[DOTPRE]], [[DOTPRE2]]
+; CHECK-NEXT:    [[SPEC_SELECT_I_I:%.*]] = select i1 [[CMP_I_I_I]], i32* [[INCDEC_PTR_I_I5]], i32* [[SPEC_SELECT_I_I4]]
+; CHECK-NEXT:    [[INCDEC_PTR_I_I:%.*]] = getelementptr inbounds i32, i32* [[INCDEC_PTR_I_I5]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* [[SPEC_SELECT_I_I]], align 4
+; CHECK-NEXT:    ret i32 [[TMP1]]
+;
+entry:
+  %ref.tmp = alloca [3 x i32], align 4
+  %0 = bitcast [3 x i32]* %ref.tmp to i8*
+  %arrayinit.begin = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 0
+  store i32 %a, i32* %arrayinit.begin, align 4
+  %arrayinit.element = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 1
+  store i32 %b, i32* %arrayinit.element, align 4
+  %arrayinit.element1 = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 2
+  store i32 %c, i32* %arrayinit.element1, align 4
+  %add.ptr.i.i = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 3
+  %cmp.i.i.i3 = icmp slt i32 %a, %b
+  %spec.select.i.i4 = select i1 %cmp.i.i.i3, i32* %arrayinit.element, i32* %arrayinit.begin
+  %incdec.ptr.i.i5 = getelementptr inbounds [3 x i32], [3 x i32]* %ref.tmp, i64 0, i64 2
+  br label %loop
+
+loop:          ; preds = %entry, %loop
+  %incdec.ptr.i.i7 = phi i32* [ %incdec.ptr.i.i5, %entry ], [ %incdec.ptr.i.i, %loop ]
+  %spec.select.i.i6 = phi i32* [ %spec.select.i.i4, %entry ], [ %spec.select.i.i, %loop ]
+  %.pre = load i32, i32* %spec.select.i.i6, align 4
+  %.pre2 = load i32, i32* %incdec.ptr.i.i7, align 4
+  %cmp.i.i.i = icmp slt i32 %.pre, %.pre2
+  %spec.select.i.i = select i1 %cmp.i.i.i, i32* %incdec.ptr.i.i7, i32* %spec.select.i.i6
+  %incdec.ptr.i.i = getelementptr inbounds i32, i32* %incdec.ptr.i.i7, i64 1
+  %cmp1.i.i = icmp eq i32* %incdec.ptr.i.i, %add.ptr.i.i
+  br i1 %cmp1.i.i, label %exit, label %loop
+
+exit:           ; preds = %loop
+  %1 = load i32, i32* %spec.select.i.i, align 4
+  ret i32 %1
+}
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll b/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll
new file mode 100644
index 0000000000000..95632a5a3bee7
--- /dev/null
+++ b/llvm/test/Transforms/LoopUnroll/runtime-small-upperbound.ll
@@ -0,0 +1,70 @@
+; RUN: opt -S -loop-unroll -unroll-runtime %s -o - | FileCheck %s
+; RUN: opt -S -loop-unroll -unroll-runtime -unroll-max-upperbound=6 %s -o - | FileCheck %s --check-prefix=UPPER
+
+target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
+
+@global = dso_local local_unnamed_addr global i32 0, align 4
+@global.1 = dso_local local_unnamed_addr global i8* null, align 4
+
+; Check that loop in hoge_3, with a runtime upperbound of 3, is not unrolled.
+; CHECK-LABEL: hoge_3
+; CHECK: loop:
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: br i1 %{{.*}}, label %loop
+; UPPER-LABEL: hoge_3
+; UPPER: loop:
+; UPPER: store
+; UPPER-NOT: store
+; UPPER: br i1 %{{.*}}, label %loop
+define dso_local void @hoge_3(i8 %arg) {
+entry:
+  %x = load i32, i32* @global, align 4
+  %y = load i8*, i8** @global.1, align 4
+  %0 = icmp ult i32 %x, 17
+  br i1 %0, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %x, %entry ], [ %iv.next, %loop ]
+  %ptr = phi i8* [ %y, %entry ], [ %ptr.next, %loop ]
+  %iv.next = add nuw i32 %iv, 8
+  %ptr.next = getelementptr inbounds i8, i8* %ptr, i32 1
+  store i8 %arg, i8* %ptr.next, align 1
+  %1 = icmp ult i32 %iv.next, 17
+  br i1 %1, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; Check that loop in hoge_5, with a runtime upperbound of 5, is unrolled when -unroll-max-upperbound=4
+; CHECK-LABEL: hoge_5
+; CHECK: loop:
+; CHECK: store
+; CHECK-NOT: store
+; CHECK: br i1 %{{.*}}, label %loop
+; UPPER-LABEL: hoge_5
+; UPPER: loop:
+; UPPER: store
+; UPPER: store
+; UPPER: store
+; UPPER: br i1 %{{.*}}, label %loop
+define dso_local void @hoge_5(i8 %arg) {
+entry:
+  %x = load i32, i32* @global, align 4
+  %y = load i8*, i8** @global.1, align 4
+  %0 = icmp ult i32 %x, 17
+  br i1 %0, label %loop, label %exit
+
+loop:
+  %iv = phi i32 [ %x, %entry ], [ %iv.next, %loop ]
+  %ptr = phi i8* [ %y, %entry ], [ %ptr.next, %loop ]
+  %iv.next = add nuw i32 %iv, 4
+  %ptr.next = getelementptr inbounds i8, i8* %ptr, i32 1
+  store i8 %arg, i8* %ptr.next, align 1
+  %1 = icmp ult i32 %iv.next, 17
+  br i1 %1, label %loop, label %exit
+
+exit:
+  ret void
+}
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll
new file mode 100644
index 0000000000000..38d838576bd81
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-maskedldst.ll
@@ -0,0 +1,40 @@
+; RUN: opt -loop-vectorize -enable-arm-maskedldst < %s -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1-m.main-none-eabi"
+
+; CHECK-LABEL: test
+; CHECK: llvm.masked.store.v4i32.p0v4i32
+define void @test(i32* nocapture %A, i32 %n) #0 {
+entry:
+  %cmp12 = icmp sgt i32 %n, 0
+  br i1 %cmp12, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:                               ; preds = %entry
+  br label %for.body
+
+for.body:                                         ; preds = %for.body.preheader, %for.inc
+  %i.013 = phi i32 [ %inc, %for.inc ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i32, i32* %A, i32 %i.013
+  %0 = load i32, i32* %arrayidx, align 4
+  %.off = add i32 %0, 9
+  %1 = icmp ult i32 %.off, 19
+  br i1 %1, label %if.then, label %for.inc
+
+if.then:                                          ; preds = %for.body
+  store i32 0, i32* %arrayidx, align 4
+  br label %for.inc
+
+for.inc:                                          ; preds = %for.body, %if.then
+  %inc = add nuw nsw i32 %i.013, 1
+  %exitcond = icmp eq i32 %inc, %n
+  br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
+
+for.cond.cleanup.loopexit:                        ; preds = %for.inc
+  br label %for.cond.cleanup
+
+for.cond.cleanup:                                 ; preds = %for.cond.cleanup.loopexit, %entry
+  ret void
+}
+
+attributes #0 = { "target-features"="+mve" }
diff --git a/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll b/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
new file mode 100644
index 0000000000000..fc9ef2b3b76f0
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
@@ -0,0 +1,89 @@
+; RUN: opt < %s -loop-vectorize -enable-arm-maskedldst -S | \
+; RUN:  FileCheck %s -check-prefixes=COMMON,CHECK
+
+; RUN: opt < %s -loop-vectorize -enable-arm-maskedldst -prefer-predicate-over-epilog -S | \
+; RUN:   FileCheck -check-prefixes=COMMON,PREDFLAG %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-unknown-eabihf"
+
+define dso_local void @tail_folding_enabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; COMMON-LABEL: tail_folding_enabled(
+; COMMON: vector.body:
+; COMMON:   %[[WML1:.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; COMMON:   %[[WML2:.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; COMMON:   %[[ADD:.*]] = add nsw <4 x i32> %[[WML2]], %[[WML1]]
+; COMMON:   call void @llvm.masked.store.v4i32.p0v4i32(<4 x i32> %[[ADD]]
+; COMMON:   br i1 %12, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !6
+}
+
+define dso_local void @tail_folding_disabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
+; CHECK-LABEL: tail_folding_disabled(
+; CHECK:      vector.body:
+; CHECK-NOT:  @llvm.masked.load.v8i32.p0v8i32(
+; CHECK-NOT:  @llvm.masked.store.v8i32.p0v8i32(
+; CHECK:      br i1 %{{.*}}, label {{.*}}, label %vector.body
+
+; PREDFLAG-LABEL: tail_folding_disabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; PREDFLAG:  %wide.masked.load1 = call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; PREDFLAG:  %{{.*}} = add nsw <4 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v4i32.p0v4i32(
+; PREDFLAG:  %index.next = add i64 %index, 4
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !4
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i64 %indvars.iv
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx2 = getelementptr inbounds i32, i32* %C, i64 %indvars.iv
+  %1 = load i32, i32* %arrayidx2, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx4 = getelementptr inbounds i32, i32* %A, i64 %indvars.iv
+  store i32 %add, i32* %arrayidx4, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 430
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+; CHECK:      !0 = distinct !{!0, !1}
+; CHECK-NEXT: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK-NEXT: !2 = distinct !{!2, !3, !1}
+; CHECK-NEXT: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK-NEXT: !4 = distinct !{!4, !1}
+; CHECK-NEXT: !5 = distinct !{!5, !3, !1}
+
+attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }
+
+!6 = distinct !{!6, !7, !8}
+!7 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!8 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!10 = distinct !{!10, !11, !12}
+!11 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
+!12 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
index 2a665e56ab067..3f462425658c0 100644
--- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll
+++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s  -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -dce -instcombine -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -disable-basicaa -S -pass-remarks-analysis='loop-vectorize' 2>&1 | FileCheck %s -check-prefix=FORCED_OPTSIZE
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 
@@ -159,6 +160,32 @@ loopexit:
 
 ; CHECK: !9 = !DILocation(line: 101, column: 1, scope: !{{.*}})
 
+define dso_local void @forced_optsize(i64* noalias nocapture readonly %x_p, i64* noalias nocapture readonly %y_p, i64* noalias nocapture %z_p) minsize optsize {
+;
+; FORCED_OPTSIZE: remark: <unknown>:0:0: Code-size may be reduced by not forcing vectorization, or by source-code modifications eliminating the need for runtime checks (e.g., adding 'restrict').
+; FORCED_OPTSIZE-LABEL: @forced_optsize(
+; FORCED_OPTSIZE:       vector.body:
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]
+  %arrayidx = getelementptr inbounds i64, i64* %x_p, i64 %indvars.iv
+  %0 = load i64, i64* %arrayidx, align 8
+  %arrayidx2 = getelementptr inbounds i64, i64* %y_p, i64 %indvars.iv
+  %1 = load i64, i64* %arrayidx2, align 8
+  %add = add nsw i64 %1, %0
+  %arrayidx4 = getelementptr inbounds i64, i64* %z_p, i64 %indvars.iv
+  store i64 %add, i64* %arrayidx4, align 8
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 128
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !12
+}
+
 !llvm.module.flags = !{!0, !1}
 !llvm.dbg.cu = !{!9}
 !0 = !{i32 2, !"Dwarf Version", i32 4}
@@ -177,3 +204,6 @@ loopexit:
                              splitDebugFilename: "abc.debug", emissionKind: 2)
 !10 = !DIFile(filename: "path/to/file", directory: "/path/to/dir")
 !11 = !{i32 2, !"Debug Info Version", i32 3}
+!12 = distinct !{!12, !13, !14}
+!13 = !{!"llvm.loop.vectorize.width", i32 2}
+!14 = !{!"llvm.loop.vectorize.enable", i1 true}
diff --git a/llvm/test/Transforms/ObjCARC/code-motion.ll b/llvm/test/Transforms/ObjCARC/code-motion.ll
new file mode 100644
index 0000000000000..7c3242ce857f1
--- /dev/null
+++ b/llvm/test/Transforms/ObjCARC/code-motion.ll
@@ -0,0 +1,39 @@
+; RUN: opt -objc-arc -S < %s | FileCheck %s
+
+declare void @alterRefCount()
+declare void @use(i8*)
+
+; Check that ARC optimizer doesn't reverse the order of the retain call and the
+; release call when there are debug instructions.
+
+; CHECK: call i8* @llvm.objc.retain(i8* %x)
+; CHECK: call void @llvm.objc.release(i8* %x)
+
+define i32 @test(i8* %x, i8* %y, i8 %z, i32 %i) {
+  %i.addr = alloca i32, align 4
+  store i32 %i, i32* %i.addr, align 4
+  %v1 = tail call i8* @llvm.objc.retain(i8* %x)
+  store i8 %z, i8* %x
+  call void @llvm.dbg.declare(metadata i32* %i.addr, metadata !9, metadata !DIExpression()), !dbg !10
+  call void @alterRefCount()
+  tail call void @llvm.objc.release(i8* %x)
+  ret i32 %i
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+declare i8* @llvm.objc.retain(i8*) local_unnamed_addr
+declare void @llvm.objc.release(i8*) local_unnamed_addr
+
+!llvm.module.flags = !{!0, !1}
+
+!0 = !{i32 2, !"Dwarf Version", i32 4}
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!2 = !DILocalVariable(name: "i", arg: 1, scope: !3, file: !4, line: 1, type: !7)
+!3 = distinct !DISubprogram(name: "test", scope: !4, file: !4, line: 1, type: !5, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !8, retainedNodes: !9)
+!4 = !DIFile(filename: "test.m", directory: "dir")
+!5 = !DISubroutineType(types: !6)
+!6 = !{!7, !7}
+!7 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!8 = distinct !DICompileUnit(language: DW_LANG_ObjC, file: !4, isOptimized: false, runtimeVersion: 2, emissionKind: FullDebug, enums: !9, nameTableKind: None)
+!9 = !{}
+!10 = !DILocation(line: 1, column: 14, scope: !3)
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
index 2dc2c2ec9f36e..230f44ba443d7 100644
--- a/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
+++ b/llvm/test/Transforms/PGOProfile/Inputs/func_entry.proftext
@@ -1,17 +1,25 @@
 # IR level Instrumentation Flag
 :ir
-foo
+hot
 # Func Hash:
 12884901887
 # Num Counters:
 1
 # Counter Values:
-9999
+9000
 
-bar
+cold
 # Func Hash:
 12884901887
 # Num Counters:
 1
 # Counter Values:
-0
+10
+
+med
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+50
diff --git a/llvm/test/Transforms/PGOProfile/chr.ll b/llvm/test/Transforms/PGOProfile/chr.ll
index 838b7f6af825a..0221a4fd000c4 100644
--- a/llvm/test/Transforms/PGOProfile/chr.ll
+++ b/llvm/test/Transforms/PGOProfile/chr.ll
@@ -468,15 +468,14 @@ define i32 @test_chr_5(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP11]], i32 [[SUM1_NONCHR]], i32 [[TMP12]], !prof !16
 ; CHECK-NEXT:    [[TMP13:%.*]] = and i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
-; CHECK-NEXT:    br i1 [[TMP14]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP15:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP16]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -569,15 +568,14 @@ define i32 @test_chr_5_1(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP14]], i32 [[SUM1_NONCHR]], i32 [[TMP15]], !prof !16
 ; CHECK-NEXT:    [[TMP16:%.*]] = and i32 [[SUM0]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[TMP16]], 0
-; CHECK-NEXT:    br i1 [[TMP17]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP18:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP19:%.*]] = icmp eq i32 [[TMP18]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP19]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP17]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP7]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -667,15 +665,14 @@ define i32 @test_chr_6(i32* %i, i32* %j, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[V4_NONCHR]], i32 [[SUM0]], i32 [[V8_NONCHR]], !prof !16
 ; CHECK-NEXT:    [[V9_NONCHR:%.*]] = and i32 [[J0]], 4
 ; CHECK-NEXT:    [[V10_NONCHR:%.*]] = icmp eq i32 [[V9_NONCHR]], 0
-; CHECK-NEXT:    br i1 [[V10_NONCHR]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[V11_NONCHR:%.*]] = and i32 [[I0]], 8
 ; CHECK-NEXT:    [[V12_NONCHR:%.*]] = icmp eq i32 [[V11_NONCHR]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[V12_NONCHR]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[V12_NONCHR]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[V10_NONCHR]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[V13]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[V13]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
@@ -1754,15 +1751,14 @@ define i32 @test_chr_19(i32* %i, i32 %sum0) !prof !14 {
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i32 [[TMP7]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = add i32 [[SUM0]], 85
 ; CHECK-NEXT:    [[SUM2_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM0]], i32 [[TMP9]], !prof !16
-; CHECK-NEXT:    br i1 [[TMP8]], label [[BB3]], label [[BB1_NONCHR:%.*]], !prof !16
-; CHECK:       bb1.nonchr:
 ; CHECK-NEXT:    [[TMP10:%.*]] = and i32 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp eq i32 [[TMP10]], 0
-; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88, !prof !16
+; CHECK-NEXT:    [[SUM4_NONCHR_V:%.*]] = select i1 [[TMP11]], i32 44, i32 88
 ; CHECK-NEXT:    [[SUM4_NONCHR:%.*]] = add i32 [[SUM2_NONCHR]], [[SUM4_NONCHR_V]]
+; CHECK-NEXT:    [[SUM5_NONCHR:%.*]] = select i1 [[TMP8]], i32 [[SUM2_NONCHR]], i32 [[SUM4_NONCHR]], !prof !16
 ; CHECK-NEXT:    br label [[BB3]]
 ; CHECK:       bb3:
-; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM2_NONCHR]], [[BB0_NONCHR]] ], [ [[SUM4_NONCHR]], [[BB1_NONCHR]] ]
+; CHECK-NEXT:    [[SUM6:%.*]] = phi i32 [ [[TMP4]], [[BB0]] ], [ [[SUM0]], [[ENTRY_SPLIT_NONCHR]] ], [ [[SUM5_NONCHR]], [[BB0_NONCHR]] ]
 ; CHECK-NEXT:    ret i32 [[SUM6]]
 ;
 entry:
diff --git a/llvm/test/Transforms/PGOProfile/comdat_internal.ll b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
index e88c61a27e59b..f0906bf2b3720 100644
--- a/llvm/test/Transforms/PGOProfile/comdat_internal.ll
+++ b/llvm/test/Transforms/PGOProfile/comdat_internal.ll
@@ -7,16 +7,15 @@ $foo = comdat any
 ; CHECK: $foo = comdat any
 
 ; CHECK: $__llvm_profile_raw_version = comdat any
-; CHECK: $__profv__stdin__foo.[[FOO_HASH:[0-9]+]] = comdat any
 
 @bar = global i32 ()* @foo, align 8
 
 ; CHECK: @__llvm_profile_raw_version = constant i64 {{[0-9]+}}, comdat
 ; CHECK-NOT: __profn__stdin__foo
-; CHECK: @__profc__stdin__foo.[[FOO_HASH]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK: @__profc__stdin__foo.[[FOO_HASH:[0-9]+]] = private global [1 x i64] zeroinitializer, section "__llvm_prf_cnts", comdat, align 8
 ; CHECK: @__profd__stdin__foo.[[FOO_HASH]] = private global { i64, i64, i64*, i8*, i8*, i32, [2 x i16] } { i64 -5640069336071256030, i64 [[FOO_HASH]], i64* getelementptr inbounds ([1 x i64], [1 x i64]* @__profc__stdin__foo.[[FOO_HASH]], i32 0, i32 0), i8* null
 ; CHECK-NOT: bitcast (i32 ()* @foo to i8*)
-; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat($__profv__stdin__foo.[[FOO_HASH]]), align 8
+; CHECK-SAME: , i8* null, i32 1, [2 x i16] zeroinitializer }, section "__llvm_prf_data", comdat, align 8
 ; CHECK: @__llvm_prf_nm
 ; CHECK: @llvm.used
 
diff --git a/llvm/test/Transforms/PGOProfile/func_entry.ll b/llvm/test/Transforms/PGOProfile/func_entry.ll
index 37fad27f2f2fe..fe1b44b0bd71d 100644
--- a/llvm/test/Transforms/PGOProfile/func_entry.ll
+++ b/llvm/test/Transforms/PGOProfile/func_entry.ll
@@ -6,8 +6,9 @@ target triple = "x86_64-unknown-linux-gnu"
 
 @s = common dso_local local_unnamed_addr global i32 0, align 4
 
-define void @bar() {
-; CHECK-LABEL: @bar
+define void @cold() {
+; CHECK-LABEL: @cold()
+; CHECK-SAME: #[[COLD_ATTR:[0-1]+]]
 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_ZERO:[0-9]+]]
 
 entry:
@@ -15,8 +16,9 @@ entry:
   ret void
 }
 
-define void @foo() {
-; CHECK-LABEL: @foo
+define void @hot() {
+; CHECK-LABEL: @hot()
+; CHECK-SAME: #[[HOT_ATTR:[0-1]+]]
 ; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_NON_ZERO:[0-9]+]]
 entry:
   %0 = load i32, i32* @s, align 4
@@ -25,5 +27,18 @@ entry:
   ret void
 }
 
-; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 0}
-; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9999}
+define void @med() {
+; CHECK-LABEL: @med
+; CHECK-NOT: #
+; CHECK-SAME: !prof ![[FUNC_ENTRY_COUNT_MED:[0-9]+]]
+
+entry:
+  store i32 1, i32* @s, align 4
+  ret void
+}
+
+; CHECK-DAG: attributes #[[COLD_ATTR]] = { cold }
+; CHECK-DAG: attributes #[[HOT_ATTR]] = { inlinehint }
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_ZERO]] = !{!"function_entry_count", i64 10}
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_NON_ZERO]] = !{!"function_entry_count", i64 9000}
+; CHECK-DAG: ![[FUNC_ENTRY_COUNT_MED]] = !{!"function_entry_count", i64 50}
diff --git a/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll b/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll
new file mode 100644
index 0000000000000..4d4a30e00eafb
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/two-shifts-by-sext.ll
@@ -0,0 +1,125 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -O1 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-01
+; RUN: opt -O2 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-02
+; RUN: opt -O3 -S < %s | FileCheck %s --check-prefixes=CHECK,CHECK-03
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; int two_shifts_by_sext(int val, char len) {
+;   return (val << len) >> len;
+; }
+;
+; int two_shifts_by_same_sext(int val, char len) {
+;   int wide_len = len;
+;   return (val << wide_len) >> wide_len;
+; }
+;
+; void use_int32(int);
+; int two_shifts_by_sext_with_extra_use(int val, char len) {
+;   use_int32(len);
+;   return (val << len) >> len;
+; }
+;
+; int two_shifts_by_same_sext_with_extra_use(int val, char len) {
+;   int wide_len = len;
+;   use_int32(wide_len);
+;   return (val << wide_len) >> wide_len;
+; }
+
+define i32 @two_shifts_by_sext(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_sext(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  %shl = shl i32 %val.reloaded, %conv
+  %len.reloaded.1 = load i8, i8* %len.addr, align 1
+  %conv1 = sext i8 %len.reloaded.1 to i32
+  %shr = ashr i32 %shl, %conv1
+  ret i32 %shr
+}
+
+define i32 @two_shifts_by_same_sext(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_same_sext(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  %wide_len = alloca i32, align 4
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  store i32 %conv, i32* %wide_len, align 4
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.1 = load i32, i32* %wide_len, align 4
+  %shl = shl i32 %val.reloaded, %len.reloaded.1
+  %len.reloaded.2 = load i32, i32* %wide_len, align 4
+  %shr = ashr i32 %shl, %len.reloaded.2
+  ret i32 %shr
+}
+
+define i32 @two_shifts_by_sext_with_extra_use(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_sext_with_extra_use(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    tail call void @use_int32(i32 [[CONV]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  call void @use_int32(i32 %conv)
+  %val.reloaded = load i32, i32* %val.addr, align 4
+  %len.reloaded.1 = load i8, i8* %len.addr, align 1
+  %conv1 = sext i8 %len.reloaded.1 to i32
+  %shl = shl i32 %val.reloaded, %conv1
+  %len.reloaded.2 = load i8, i8* %len.addr, align 1
+  %conv2 = sext i8 %len.reloaded.2 to i32
+  %shr = ashr i32 %shl, %conv2
+  ret i32 %shr
+}
+
+declare void @use_int32(i32)
+
+define i32 @two_shifts_by_same_sext_with_extra_use(i32 %val, i8 signext %len) {
+; CHECK-LABEL: @two_shifts_by_same_sext_with_extra_use(
+; CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[LEN:%.*]] to i32
+; CHECK-NEXT:    tail call void @use_int32(i32 [[CONV]])
+; CHECK-NEXT:    [[SHL:%.*]] = shl i32 [[VAL:%.*]], [[CONV]]
+; CHECK-NEXT:    [[SHR:%.*]] = ashr i32 [[SHL]], [[CONV]]
+; CHECK-NEXT:    ret i32 [[SHR]]
+;
+  %val.addr = alloca i32, align 4
+  %len.addr = alloca i8, align 1
+  %wide_len = alloca i32, align 4
+  store i32 %val, i32* %val.addr, align 4
+  store i8 %len, i8* %len.addr, align 1
+  %len.reloaded.0 = load i8, i8* %len.addr, align 1
+  %conv = sext i8 %len.reloaded.0 to i32
+  store i32 %conv, i32* %wide_len, align 4
+  %val.reloaded = load i32, i32* %wide_len, align 4
+  call void @use_int32(i32 %val.reloaded)
+  %len.reloaded.1 = load i32, i32* %val.addr, align 4
+  %len.reloaded.2 = load i32, i32* %wide_len, align 4
+  %shl = shl i32 %len.reloaded.1, %len.reloaded.2
+  %wide_len.reloaded = load i32, i32* %wide_len, align 4
+  %shr = ashr i32 %shl, %wide_len.reloaded
+  ret i32 %shr
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
index 5e6db8b483607..5f8cbd5c8d010 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/gather-root.ll
@@ -17,16 +17,8 @@ define void @PR28330(i32 %n) {
 ; DEFAULT:       for.body:
 ; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
-; DEFAULT-NEXT:    [[P20:%.*]] = add i32 [[P17]], undef
-; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
-; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
-; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
-; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
 ; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[P17]]
-; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
 ; DEFAULT-NEXT:    br label [[FOR_BODY]]
 ;
 ; GATHER-LABEL: @PR28330(
@@ -36,37 +28,30 @@ define void @PR28330(i32 %n) {
 ; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
 ; GATHER:       for.body:
 ; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
-; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
-; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
-; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
-; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
-; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
-; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
-; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP3:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = insertelement <8 x i1> undef, i1 [[TMP3]], i32 0
+; GATHER-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[TMP5]], i32 1
+; GATHER-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[TMP7]], i32 2
+; GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = insertelement <8 x i1> [[TMP8]], i1 [[TMP9]], i32 3
+; GATHER-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = insertelement <8 x i1> [[TMP10]], i1 [[TMP11]], i32 4
+; GATHER-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = insertelement <8 x i1> [[TMP12]], i1 [[TMP13]], i32 5
+; GATHER-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = insertelement <8 x i1> [[TMP14]], i1 [[TMP15]], i32 6
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP16]], i1 [[TMP2]], i32 7
 ; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
 ; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
-; GATHER-NEXT:    [[P20:%.*]] = add i32 [[P17]], [[TMP19]]
 ; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
-; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
 ; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
-; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
 ; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
-; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
 ; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
-; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
 ; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
-; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
 ; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
-; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
 ; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
 ; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
 ; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
@@ -78,7 +63,6 @@ define void @PR28330(i32 %n) {
 ; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
 ; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP34]])
 ; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], [[P17]]
-; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
 ; GATHER-NEXT:    br label [[FOR_BODY]]
 ;
 ; MAX-COST-LABEL: @PR28330(
@@ -169,16 +153,8 @@ define void @PR32038(i32 %n) {
 ; DEFAULT:       for.body:
 ; DEFAULT-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = select <8 x i1> [[TMP1]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
-; DEFAULT-NEXT:    [[P20:%.*]] = add i32 -5, undef
-; DEFAULT-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; DEFAULT-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; DEFAULT-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
-; DEFAULT-NEXT:    [[P28:%.*]] = add i32 [[P26]], undef
-; DEFAULT-NEXT:    [[P30:%.*]] = add i32 [[P28]], undef
-; DEFAULT-NEXT:    [[P32:%.*]] = add i32 [[P30]], undef
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP2]])
 ; DEFAULT-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], -5
-; DEFAULT-NEXT:    [[P34:%.*]] = add i32 [[P32]], undef
 ; DEFAULT-NEXT:    br label [[FOR_BODY]]
 ;
 ; GATHER-LABEL: @PR32038(
@@ -188,37 +164,30 @@ define void @PR32038(i32 %n) {
 ; GATHER-NEXT:    br label [[FOR_BODY:%.*]]
 ; GATHER:       for.body:
 ; GATHER-NEXT:    [[P17:%.*]] = phi i32 [ [[OP_EXTRA:%.*]], [[FOR_BODY]] ], [ 0, [[ENTRY:%.*]] ]
-; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
-; GATHER-NEXT:    [[TMP3:%.*]] = insertelement <8 x i1> undef, i1 [[TMP2]], i32 0
-; GATHER-NEXT:    [[TMP4:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
-; GATHER-NEXT:    [[TMP5:%.*]] = insertelement <8 x i1> [[TMP3]], i1 [[TMP4]], i32 1
-; GATHER-NEXT:    [[TMP6:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
-; GATHER-NEXT:    [[TMP7:%.*]] = insertelement <8 x i1> [[TMP5]], i1 [[TMP6]], i32 2
-; GATHER-NEXT:    [[TMP8:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
-; GATHER-NEXT:    [[TMP9:%.*]] = insertelement <8 x i1> [[TMP7]], i1 [[TMP8]], i32 3
-; GATHER-NEXT:    [[TMP10:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
-; GATHER-NEXT:    [[TMP11:%.*]] = insertelement <8 x i1> [[TMP9]], i1 [[TMP10]], i32 4
-; GATHER-NEXT:    [[TMP12:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
-; GATHER-NEXT:    [[TMP13:%.*]] = insertelement <8 x i1> [[TMP11]], i1 [[TMP12]], i32 5
-; GATHER-NEXT:    [[TMP14:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
-; GATHER-NEXT:    [[TMP15:%.*]] = insertelement <8 x i1> [[TMP13]], i1 [[TMP14]], i32 6
-; GATHER-NEXT:    [[TMP16:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
-; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP15]], i1 [[TMP16]], i32 7
+; GATHER-NEXT:    [[TMP2:%.*]] = extractelement <8 x i1> [[TMP1]], i32 7
+; GATHER-NEXT:    [[TMP3:%.*]] = extractelement <8 x i1> [[TMP1]], i32 0
+; GATHER-NEXT:    [[TMP4:%.*]] = insertelement <8 x i1> undef, i1 [[TMP3]], i32 0
+; GATHER-NEXT:    [[TMP5:%.*]] = extractelement <8 x i1> [[TMP1]], i32 1
+; GATHER-NEXT:    [[TMP6:%.*]] = insertelement <8 x i1> [[TMP4]], i1 [[TMP5]], i32 1
+; GATHER-NEXT:    [[TMP7:%.*]] = extractelement <8 x i1> [[TMP1]], i32 2
+; GATHER-NEXT:    [[TMP8:%.*]] = insertelement <8 x i1> [[TMP6]], i1 [[TMP7]], i32 2
+; GATHER-NEXT:    [[TMP9:%.*]] = extractelement <8 x i1> [[TMP1]], i32 3
+; GATHER-NEXT:    [[TMP10:%.*]] = insertelement <8 x i1> [[TMP8]], i1 [[TMP9]], i32 3
+; GATHER-NEXT:    [[TMP11:%.*]] = extractelement <8 x i1> [[TMP1]], i32 4
+; GATHER-NEXT:    [[TMP12:%.*]] = insertelement <8 x i1> [[TMP10]], i1 [[TMP11]], i32 4
+; GATHER-NEXT:    [[TMP13:%.*]] = extractelement <8 x i1> [[TMP1]], i32 5
+; GATHER-NEXT:    [[TMP14:%.*]] = insertelement <8 x i1> [[TMP12]], i1 [[TMP13]], i32 5
+; GATHER-NEXT:    [[TMP15:%.*]] = extractelement <8 x i1> [[TMP1]], i32 6
+; GATHER-NEXT:    [[TMP16:%.*]] = insertelement <8 x i1> [[TMP14]], i1 [[TMP15]], i32 6
+; GATHER-NEXT:    [[TMP17:%.*]] = insertelement <8 x i1> [[TMP16]], i1 [[TMP2]], i32 7
 ; GATHER-NEXT:    [[TMP18:%.*]] = select <8 x i1> [[TMP17]], <8 x i32> <i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720, i32 -720>, <8 x i32> <i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80, i32 -80>
 ; GATHER-NEXT:    [[TMP19:%.*]] = extractelement <8 x i32> [[TMP18]], i32 0
-; GATHER-NEXT:    [[P20:%.*]] = add i32 -5, [[TMP19]]
 ; GATHER-NEXT:    [[TMP20:%.*]] = extractelement <8 x i32> [[TMP18]], i32 1
-; GATHER-NEXT:    [[P22:%.*]] = add i32 [[P20]], [[TMP20]]
 ; GATHER-NEXT:    [[TMP21:%.*]] = extractelement <8 x i32> [[TMP18]], i32 2
-; GATHER-NEXT:    [[P24:%.*]] = add i32 [[P22]], [[TMP21]]
 ; GATHER-NEXT:    [[TMP22:%.*]] = extractelement <8 x i32> [[TMP18]], i32 3
-; GATHER-NEXT:    [[P26:%.*]] = add i32 [[P24]], [[TMP22]]
 ; GATHER-NEXT:    [[TMP23:%.*]] = extractelement <8 x i32> [[TMP18]], i32 4
-; GATHER-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[TMP23]]
 ; GATHER-NEXT:    [[TMP24:%.*]] = extractelement <8 x i32> [[TMP18]], i32 5
-; GATHER-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[TMP24]]
 ; GATHER-NEXT:    [[TMP25:%.*]] = extractelement <8 x i32> [[TMP18]], i32 6
-; GATHER-NEXT:    [[P32:%.*]] = add i32 [[P30]], [[TMP25]]
 ; GATHER-NEXT:    [[TMP26:%.*]] = insertelement <8 x i32> undef, i32 [[TMP19]], i32 0
 ; GATHER-NEXT:    [[TMP27:%.*]] = insertelement <8 x i32> [[TMP26]], i32 [[TMP20]], i32 1
 ; GATHER-NEXT:    [[TMP28:%.*]] = insertelement <8 x i32> [[TMP27]], i32 [[TMP21]], i32 2
@@ -230,7 +199,6 @@ define void @PR32038(i32 %n) {
 ; GATHER-NEXT:    [[TMP34:%.*]] = insertelement <8 x i32> [[TMP32]], i32 [[TMP33]], i32 7
 ; GATHER-NEXT:    [[TMP35:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP34]])
 ; GATHER-NEXT:    [[OP_EXTRA]] = add i32 [[TMP35]], -5
-; GATHER-NEXT:    [[P34:%.*]] = add i32 [[P32]], [[TMP33]]
 ; GATHER-NEXT:    br label [[FOR_BODY]]
 ;
 ; MAX-COST-LABEL: @PR32038(
@@ -259,18 +227,12 @@ define void @PR32038(i32 %n) {
 ; MAX-COST-NEXT:    [[TMP6:%.*]] = insertelement <4 x i1> [[TMP5]], i1 [[P5]], i32 2
 ; MAX-COST-NEXT:    [[TMP7:%.*]] = insertelement <4 x i1> [[TMP6]], i1 [[P7]], i32 3
 ; MAX-COST-NEXT:    [[TMP8:%.*]] = select <4 x i1> [[TMP7]], <4 x i32> <i32 -720, i32 -720, i32 -720, i32 -720>, <4 x i32> <i32 -80, i32 -80, i32 -80, i32 -80>
-; MAX-COST-NEXT:    [[P20:%.*]] = add i32 -5, undef
-; MAX-COST-NEXT:    [[P22:%.*]] = add i32 [[P20]], undef
-; MAX-COST-NEXT:    [[P24:%.*]] = add i32 [[P22]], undef
-; MAX-COST-NEXT:    [[P26:%.*]] = add i32 [[P24]], undef
 ; MAX-COST-NEXT:    [[P27:%.*]] = select i1 [[P9]], i32 -720, i32 -80
-; MAX-COST-NEXT:    [[P28:%.*]] = add i32 [[P26]], [[P27]]
 ; MAX-COST-NEXT:    [[P29:%.*]] = select i1 [[P11]], i32 -720, i32 -80
 ; MAX-COST-NEXT:    [[TMP9:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP8]])
 ; MAX-COST-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[P27]]
 ; MAX-COST-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], [[P29]]
 ; MAX-COST-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP11]], -5
-; MAX-COST-NEXT:    [[P30:%.*]] = add i32 [[P28]], [[P29]]
 ; MAX-COST-NEXT:    [[P31:%.*]] = select i1 [[P13]], i32 -720, i32 -80
 ; MAX-COST-NEXT:    [[P32:%.*]] = add i32 [[OP_EXTRA]], [[P31]]
 ; MAX-COST-NEXT:    [[P33:%.*]] = select i1 [[P15]], i32 -720, i32 -80
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
index 7e4b95d723499..b3ce90dabd843 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/horizontal.ll
@@ -46,12 +46,8 @@ define i32 @test_select(i32* noalias nocapture readonly %blk1, i32* noalias noca
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp slt <4 x i32> [[TMP4]], zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP4]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = select <4 x i1> [[TMP5]], <4 x i32> [[TMP6]], <4 x i32> [[TMP4]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_026]]
-; CHECK-NEXT:    [[ADD11:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD19:%.*]] = add nsw i32 [[ADD11]], undef
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP7]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP8]], [[S_026]]
-; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD19]], undef
 ; CHECK-NEXT:    [[ADD_PTR]] = getelementptr inbounds i32, i32* [[P1_023]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[ADD_PTR29]] = getelementptr inbounds i32, i32* [[P2_024]], i64 [[IDX_EXT]]
 ; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[J_025]], 1
@@ -173,12 +169,8 @@ define i32 @reduction_with_br(i32* noalias nocapture readonly %blk1, i32* noalia
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[P2_018]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul nsw <4 x i32> [[TMP3]], [[TMP1]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_020]]
-; CHECK-NEXT:    [[ADD5:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD9:%.*]] = add nsw i32 [[ADD5]], undef
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v4i32(<4 x i32> [[TMP4]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP5]], [[S_020]]
-; CHECK-NEXT:    [[ADD13:%.*]] = add nsw i32 [[ADD9]], undef
 ; CHECK-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP14]], label [[IF_END]], label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       if.end:
@@ -293,16 +285,8 @@ define i32 @test_unrolled_select(i8* noalias nocapture readonly %blk1, i8* noali
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp slt <8 x i32> [[TMP6]], zeroinitializer
 ; CHECK-NEXT:    [[TMP8:%.*]] = sub nsw <8 x i32> zeroinitializer, [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select <8 x i1> [[TMP7]], <8 x i32> [[TMP8]], <8 x i32> [[TMP6]]
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, [[S_047]]
-; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[ADD]], undef
-; CHECK-NEXT:    [[ADD27:%.*]] = add nsw i32 [[ADD16]], undef
-; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD27]], undef
-; CHECK-NEXT:    [[ADD49:%.*]] = add nsw i32 [[ADD38]], undef
-; CHECK-NEXT:    [[ADD60:%.*]] = add nsw i32 [[ADD49]], undef
-; CHECK-NEXT:    [[ADD71:%.*]] = add nsw i32 [[ADD60]], undef
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.experimental.vector.reduce.add.v8i32(<8 x i32> [[TMP9]])
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP10]], [[S_047]]
-; CHECK-NEXT:    [[ADD82:%.*]] = add nsw i32 [[ADD71]], undef
 ; CHECK-NEXT:    [[CMP83:%.*]] = icmp slt i32 [[OP_EXTRA]], [[LIM:%.*]]
 ; CHECK-NEXT:    br i1 [[CMP83]], label [[IF_END_86]], label [[FOR_END_LOOPEXIT:%.*]]
 ; CHECK:       if.end.86:
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
index 98a9fd482e872..90771e4eb3176 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/spillcost-di.ll
@@ -13,11 +13,11 @@ define void @patatino(i64 %n, i64 %i, %struct.S* %p) !dbg !7 {
 ; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 [[I:%.*]], metadata !19, metadata !DIExpression()), !dbg !24
 ; CHECK-NEXT:    call void @llvm.dbg.value(metadata %struct.S* [[P:%.*]], metadata !20, metadata !DIExpression()), !dbg !25
 ; CHECK-NEXT:    [[X1:%.*]] = getelementptr inbounds [[STRUCT_S:%.*]], %struct.S* [[P]], i64 [[N]], i32 0, !dbg !26
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !21, metadata !DIExpression()), !dbg !27
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata !2, metadata !21, metadata !DIExpression()), !dbg !27
 ; CHECK-NEXT:    [[Y3:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[N]], i32 1, !dbg !28
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i64* [[X1]] to <2 x i64>*, !dbg !26
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* [[TMP0]], align 8, !dbg !26, !tbaa !29
-; CHECK-NEXT:    call void @llvm.dbg.value(metadata i64 undef, metadata !22, metadata !DIExpression()), !dbg !33
+; CHECK-NEXT:    call void @llvm.dbg.value(metadata !2, metadata !22, metadata !DIExpression()), !dbg !33
 ; CHECK-NEXT:    [[X5:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 0, !dbg !34
 ; CHECK-NEXT:    [[Y7:%.*]] = getelementptr inbounds [[STRUCT_S]], %struct.S* [[P]], i64 [[I]], i32 1, !dbg !35
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i64* [[X5]] to <2 x i64>*, !dbg !36
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll
new file mode 100644
index 0000000000000..2fe342430de40
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR31847.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -o - -mtriple=i386 -mcpu=haswell < %s | FileCheck %s
+target datalayout = "e-m:e-p:32:32-f64:32:64-f80:32-n8:16:32-S128"
+
+@shift = common local_unnamed_addr global [10 x i32] zeroinitializer, align 4
+@data = common local_unnamed_addr global [10 x i8*] zeroinitializer, align 4
+
+define void @flat(i32 %intensity) {
+; CHECK-LABEL: @flat(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 0), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 1), align 4
+; CHECK-NEXT:    [[SHR:%.*]] = lshr i32 1, [[TMP0]]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i8, i8* [[TMP2]], i32 [[SHR]]
+; CHECK-NEXT:    [[SHR1:%.*]] = lshr i32 1, [[TMP1]]
+; CHECK-NEXT:    [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, i8* [[TMP3]], i32 [[SHR1]]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[D1_DATA_046:%.*]] = phi i8* [ [[TMP3]], [[ENTRY:%.*]] ], [ [[ADD_PTR23_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[Y_045:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC_1:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[TMP4]] to i32
+; CHECK-NEXT:    [[SUB:%.*]] = add nsw i32 [[CONV]], -128
+; CHECK-NEXT:    [[TMP5:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; CHECK-NEXT:    [[CONV3:%.*]] = zext i8 [[TMP5]] to i32
+; CHECK-NEXT:    [[SUB4:%.*]] = add nsw i32 [[CONV3]], -128
+; CHECK-NEXT:    [[CMP5:%.*]] = icmp sgt i32 [[SUB]], -1
+; CHECK-NEXT:    [[SUB7:%.*]] = sub nsw i32 128, [[CONV]]
+; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP5]], i32 [[SUB]], i32 [[SUB7]]
+; CHECK-NEXT:    [[CMP8:%.*]] = icmp sgt i32 [[SUB4]], -1
+; CHECK-NEXT:    [[SUB12:%.*]] = sub nsw i32 128, [[CONV3]]
+; CHECK-NEXT:    [[COND14:%.*]] = select i1 [[CMP8]], i32 [[SUB4]], i32 [[SUB12]]
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[COND14]], [[COND]]
+; CHECK-NEXT:    [[IDX_NEG:%.*]] = sub nsw i32 0, [[ADD]]
+; CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[IDX_NEG]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, i8* [[ADD_PTR]], align 1
+; CHECK-NEXT:    [[CONV15:%.*]] = zext i8 [[TMP6]] to i32
+; CHECK-NEXT:    [[ADD16:%.*]] = add nsw i32 [[CONV15]], [[INTENSITY:%.*]]
+; CHECK-NEXT:    [[CONV17:%.*]] = trunc i32 [[ADD16]] to i8
+; CHECK-NEXT:    store i8 [[CONV17]], i8* [[ADD_PTR]], align 1
+; CHECK-NEXT:    [[ADD_PTR18:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[ADD]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i8, i8* [[ADD_PTR18]], align 1
+; CHECK-NEXT:    [[NOT_TOBOOL:%.*]] = icmp eq i8 [[TMP7]], 0
+; CHECK-NEXT:    [[CONV21:%.*]] = zext i1 [[NOT_TOBOOL]] to i8
+; CHECK-NEXT:    store i8 [[CONV21]], i8* [[ADD_PTR18]], align 1
+; CHECK-NEXT:    [[ADD_PTR23:%.*]] = getelementptr inbounds i8, i8* [[D1_DATA_046]], i32 [[TMP1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i8, i8* [[ARRAYIDX]], align 1
+; CHECK-NEXT:    [[CONV_1:%.*]] = zext i8 [[TMP8]] to i32
+; CHECK-NEXT:    [[SUB_1:%.*]] = add nsw i32 [[CONV_1]], -128
+; CHECK-NEXT:    [[TMP9:%.*]] = load i8, i8* [[ARRAYIDX2]], align 1
+; CHECK-NEXT:    [[CONV3_1:%.*]] = zext i8 [[TMP9]] to i32
+; CHECK-NEXT:    [[SUB4_1:%.*]] = add nsw i32 [[CONV3_1]], -128
+; CHECK-NEXT:    [[CMP5_1:%.*]] = icmp sgt i32 [[SUB_1]], -1
+; CHECK-NEXT:    [[SUB7_1:%.*]] = sub nsw i32 128, [[CONV_1]]
+; CHECK-NEXT:    [[COND_1:%.*]] = select i1 [[CMP5_1]], i32 [[SUB_1]], i32 [[SUB7_1]]
+; CHECK-NEXT:    [[CMP8_1:%.*]] = icmp sgt i32 [[SUB4_1]], -1
+; CHECK-NEXT:    [[SUB12_1:%.*]] = sub nsw i32 128, [[CONV3_1]]
+; CHECK-NEXT:    [[COND14_1:%.*]] = select i1 [[CMP8_1]], i32 [[SUB4_1]], i32 [[SUB12_1]]
+; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 [[COND14_1]], [[COND_1]]
+; CHECK-NEXT:    [[IDX_NEG_1:%.*]] = sub nsw i32 0, [[ADD_1]]
+; CHECK-NEXT:    [[ADD_PTR_1:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[IDX_NEG_1]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i8, i8* [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[CONV15_1:%.*]] = zext i8 [[TMP10]] to i32
+; CHECK-NEXT:    [[ADD16_1:%.*]] = add nsw i32 [[CONV15_1]], [[INTENSITY]]
+; CHECK-NEXT:    [[CONV17_1:%.*]] = trunc i32 [[ADD16_1]] to i8
+; CHECK-NEXT:    store i8 [[CONV17_1]], i8* [[ADD_PTR_1]], align 1
+; CHECK-NEXT:    [[ADD_PTR18_1:%.*]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[ADD_1]]
+; CHECK-NEXT:    [[TMP11:%.*]] = load i8, i8* [[ADD_PTR18_1]], align 1
+; CHECK-NEXT:    [[NOT_TOBOOL_1:%.*]] = icmp eq i8 [[TMP11]], 0
+; CHECK-NEXT:    [[CONV21_1:%.*]] = zext i1 [[NOT_TOBOOL_1]] to i8
+; CHECK-NEXT:    store i8 [[CONV21_1]], i8* [[ADD_PTR18_1]], align 1
+; CHECK-NEXT:    [[ADD_PTR23_1]] = getelementptr inbounds i8, i8* [[ADD_PTR23]], i32 [[TMP1]]
+; CHECK-NEXT:    [[INC_1]] = add nsw i32 [[Y_045]], 2
+; CHECK-NEXT:    [[EXITCOND_1:%.*]] = icmp eq i32 [[INC_1]], 128
+; CHECK-NEXT:    br i1 [[EXITCOND_1]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 0), align 4
+  %1 = load i32, i32* getelementptr inbounds ([10 x i32], [10 x i32]* @shift, i32 0, i32 1), align 4
+  %2 = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 0), align 4
+  %3 = load i8*, i8** getelementptr inbounds ([10 x i8*], [10 x i8*]* @data, i32 0, i32 1), align 4
+  %shr = lshr i32 1, %0
+  %arrayidx = getelementptr inbounds i8, i8* %2, i32 %shr
+  %shr1 = lshr i32 1, %1
+  %arrayidx2 = getelementptr inbounds i8, i8* %3, i32 %shr1
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %for.body, %entry
+  %d1_data.046 = phi i8* [ %3, %entry ], [ %add.ptr23.1, %for.body ]
+  %y.045 = phi i32 [ 0, %entry ], [ %inc.1, %for.body ]
+  %4 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %4 to i32
+  %sub = add nsw i32 %conv, -128
+  %5 = load i8, i8* %arrayidx2, align 1
+  %conv3 = zext i8 %5 to i32
+  %sub4 = add nsw i32 %conv3, -128
+  %cmp5 = icmp sgt i32 %sub, -1
+  %sub7 = sub nsw i32 128, %conv
+  %cond = select i1 %cmp5, i32 %sub, i32 %sub7
+  %cmp8 = icmp sgt i32 %sub4, -1
+  %sub12 = sub nsw i32 128, %conv3
+  %cond14 = select i1 %cmp8, i32 %sub4, i32 %sub12
+  %add = add nsw i32 %cond14, %cond
+  %idx.neg = sub nsw i32 0, %add
+  %add.ptr = getelementptr inbounds i8, i8* %d1_data.046, i32 %idx.neg
+  %6 = load i8, i8* %add.ptr, align 1
+  %conv15 = zext i8 %6 to i32
+  %add16 = add nsw i32 %conv15, %intensity
+  %conv17 = trunc i32 %add16 to i8
+  store i8 %conv17, i8* %add.ptr, align 1
+  %add.ptr18 = getelementptr inbounds i8, i8* %d1_data.046, i32 %add
+  %7 = load i8, i8* %add.ptr18, align 1
+  %not.tobool = icmp eq i8 %7, 0
+  %conv21 = zext i1 %not.tobool to i8
+  store i8 %conv21, i8* %add.ptr18, align 1
+  %add.ptr23 = getelementptr inbounds i8, i8* %d1_data.046, i32 %1
+  %8 = load i8, i8* %arrayidx, align 1
+  %conv.1 = zext i8 %8 to i32
+  %sub.1 = add nsw i32 %conv.1, -128
+  %9 = load i8, i8* %arrayidx2, align 1
+  %conv3.1 = zext i8 %9 to i32
+  %sub4.1 = add nsw i32 %conv3.1, -128
+  %cmp5.1 = icmp sgt i32 %sub.1, -1
+  %sub7.1 = sub nsw i32 128, %conv.1
+  %cond.1 = select i1 %cmp5.1, i32 %sub.1, i32 %sub7.1
+  %cmp8.1 = icmp sgt i32 %sub4.1, -1
+  %sub12.1 = sub nsw i32 128, %conv3.1
+  %cond14.1 = select i1 %cmp8.1, i32 %sub4.1, i32 %sub12.1
+  %add.1 = add nsw i32 %cond14.1, %cond.1
+  %idx.neg.1 = sub nsw i32 0, %add.1
+  %add.ptr.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %idx.neg.1
+  %10 = load i8, i8* %add.ptr.1, align 1
+  %conv15.1 = zext i8 %10 to i32
+  %add16.1 = add nsw i32 %conv15.1, %intensity
+  %conv17.1 = trunc i32 %add16.1 to i8
+  store i8 %conv17.1, i8* %add.ptr.1, align 1
+  %add.ptr18.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %add.1
+  %11 = load i8, i8* %add.ptr18.1, align 1
+  %not.tobool.1 = icmp eq i8 %11, 0
+  %conv21.1 = zext i1 %not.tobool.1 to i8
+  store i8 %conv21.1, i8* %add.ptr18.1, align 1
+  %add.ptr23.1 = getelementptr inbounds i8, i8* %add.ptr23, i32 %1
+  %inc.1 = add nsw i32 %y.045, 2
+  %exitcond.1 = icmp eq i32 %inc.1, 128
+  br i1 %exitcond.1, label %for.cond.cleanup, label %for.body
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
index 625748f682093..e033ed768ca05 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_1.ll
@@ -18,23 +18,16 @@ define void @mainTest(i32* %ptr) #0  {
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[TMP4]], i32 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[TMP4]], i32 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <4 x i32> [[TMP4]], [[TMP4]]
-; CHECK-NEXT:    [[TMP9:%.*]] = add i32 1, undef
-; CHECK-NEXT:    [[TMP10:%.*]] = add i32 [[TMP9]], [[TMP7]]
-; CHECK-NEXT:    [[TMP11:%.*]] = add i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[TMP11]], [[TMP6]]
-; CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = sext i32 [[TMP6]] to i64
-; CHECK-NEXT:    [[TMP15:%.*]] = add i32 [[TMP13]], [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sext i32 [[TMP6]] to i64
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP8]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i32 [[TMP10]], 1
 ; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add i32 [[OP_EXTRA]], [[TMP7]]
 ; CHECK-NEXT:    [[OP_EXTRA4:%.*]] = add i32 [[OP_EXTRA3]], [[TMP6]]
 ; CHECK-NEXT:    [[OP_EXTRA5]] = add i32 [[OP_EXTRA4]], [[TMP5]]
-; CHECK-NEXT:    [[TMP17:%.*]] = add i32 [[TMP15]], undef
 ; CHECK-NEXT:    br label [[LOOP]]
 ; CHECK:       bail_out:
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
index 712ff040a9186..4916d283bf381 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR35628_2.ll
@@ -20,10 +20,6 @@ define void @test() #0 {
 ; CHECK-NEXT:    [[DUMMY_SHL:%.*]] = shl i64 [[TMP7]], 32
 ; CHECK-NEXT:    [[TMP8:%.*]] = add <4 x i64> <i64 1, i64 1, i64 1, i64 1>, [[TMP5]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = ashr exact <4 x i64> [[TMP8]], <i64 32, i64 32, i64 32, i64 32>
-; CHECK-NEXT:    [[SUM1:%.*]] = add i64 undef, undef
-; CHECK-NEXT:    [[SUM2:%.*]] = add i64 [[SUM1]], undef
-; CHECK-NEXT:    [[ZSUM:%.*]] = add i64 [[SUM2]], 0
-; CHECK-NEXT:    [[JOIN:%.*]] = add i64 [[TMP6]], [[ZSUM]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP9]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i64> [[TMP9]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -31,7 +27,6 @@ define void @test() #0 {
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[OP_EXTRA3]] = add i64 [[OP_EXTRA]], [[TMP6]]
-; CHECK-NEXT:    [[LAST:%.*]] = add i64 [[JOIN]], undef
 ; CHECK-NEXT:    br label [[LOOP]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
index 24f75b32c5d2d..db02d64ba9382 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR39774.ll
@@ -11,40 +11,6 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[SHUFFLE]], i32 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = add <8 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240, i32 1496, i32 8555, i32 12529, i32 13685>
-; CHECK-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
-; CHECK-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
-; CHECK-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
-; CHECK-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
-; CHECK-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
-; CHECK-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], undef
-; CHECK-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], undef
-; CHECK-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_40:%.*]] = and i32 [[VAL_38]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP3]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -52,7 +18,7 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP4]], [[TMP0:%.*]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = and i32 [[OP_EXTRA5]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = and i32 [[OP_EXTRA6]], [[TMP0]]
@@ -79,7 +45,6 @@ define void @Test(i32) {
 ; CHECK-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP0]]
 ; CHECK-NEXT:    [[OP_EXTRA30:%.*]] = and i32 [[OP_EXTRA29]], [[TMP0]]
-; CHECK-NEXT:    [[VAL_42:%.*]] = and i32 [[VAL_40]], undef
 ; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i32> undef, i32 [[OP_EXTRA30]], i32 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 14910, i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i32> undef, i32 [[TMP2]], i32 0
@@ -101,40 +66,8 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
 ; FORCE_REDUCTION-NEXT:    [[TMP2:%.*]] = extractelement <4 x i32> [[SHUFFLE]], i32 1
 ; FORCE_REDUCTION-NEXT:    [[TMP3:%.*]] = add <4 x i32> [[SHUFFLE]], <i32 0, i32 55, i32 285, i32 1240>
-; FORCE_REDUCTION-NEXT:    [[VAL_1:%.*]] = and i32 [[TMP2]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_2:%.*]] = and i32 [[VAL_1]], [[TMP0:%.*]]
-; FORCE_REDUCTION-NEXT:    [[VAL_3:%.*]] = and i32 [[VAL_2]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_4:%.*]] = and i32 [[VAL_3]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_5:%.*]] = and i32 [[VAL_4]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_7:%.*]] = and i32 [[VAL_5]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_8:%.*]] = and i32 [[VAL_7]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_9:%.*]] = and i32 [[VAL_8]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_10:%.*]] = and i32 [[VAL_9]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_12:%.*]] = and i32 [[VAL_10]], undef
-; FORCE_REDUCTION-NEXT:    [[VAL_13:%.*]] = and i32 [[VAL_12]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_14:%.*]] = and i32 [[VAL_13]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_15:%.*]] = and i32 [[VAL_14]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_16:%.*]] = and i32 [[VAL_15]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_17:%.*]] = and i32 [[VAL_16]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_19:%.*]] = and i32 [[VAL_17]], undef
 ; FORCE_REDUCTION-NEXT:    [[VAL_20:%.*]] = add i32 [[TMP2]], 1496
-; FORCE_REDUCTION-NEXT:    [[VAL_21:%.*]] = and i32 [[VAL_19]], [[VAL_20]]
-; FORCE_REDUCTION-NEXT:    [[VAL_22:%.*]] = and i32 [[VAL_21]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_23:%.*]] = and i32 [[VAL_22]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_24:%.*]] = and i32 [[VAL_23]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_25:%.*]] = and i32 [[VAL_24]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_26:%.*]] = and i32 [[VAL_25]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_27:%.*]] = and i32 [[VAL_26]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_28:%.*]] = and i32 [[VAL_27]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_29:%.*]] = and i32 [[VAL_28]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_30:%.*]] = and i32 [[VAL_29]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_31:%.*]] = and i32 [[VAL_30]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_32:%.*]] = and i32 [[VAL_31]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_33:%.*]] = and i32 [[VAL_32]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_34:%.*]] = add i32 [[TMP2]], 8555
-; FORCE_REDUCTION-NEXT:    [[VAL_35:%.*]] = and i32 [[VAL_33]], [[VAL_34]]
-; FORCE_REDUCTION-NEXT:    [[VAL_36:%.*]] = and i32 [[VAL_35]], [[TMP0]]
-; FORCE_REDUCTION-NEXT:    [[VAL_37:%.*]] = and i32 [[VAL_36]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP3]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; FORCE_REDUCTION-NEXT:    [[BIN_RDX:%.*]] = and <4 x i32> [[TMP3]], [[RDX_SHUF]]
 ; FORCE_REDUCTION-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -142,7 +75,7 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[TMP4:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; FORCE_REDUCTION-NEXT:    [[TMP5:%.*]] = and i32 [[TMP4]], [[VAL_20]]
 ; FORCE_REDUCTION-NEXT:    [[TMP6:%.*]] = and i32 [[TMP5]], [[VAL_34]]
-; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0]]
+; FORCE_REDUCTION-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP6]], [[TMP0:%.*]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA3:%.*]] = and i32 [[OP_EXTRA]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA4:%.*]] = and i32 [[OP_EXTRA3]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA5:%.*]] = and i32 [[OP_EXTRA4]], [[TMP0]]
@@ -170,7 +103,6 @@ define void @Test(i32) {
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA27:%.*]] = and i32 [[OP_EXTRA26]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA28:%.*]] = and i32 [[OP_EXTRA27]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[OP_EXTRA29:%.*]] = and i32 [[OP_EXTRA28]], [[TMP2]]
-; FORCE_REDUCTION-NEXT:    [[VAL_38:%.*]] = and i32 [[VAL_37]], [[TMP0]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_39:%.*]] = add i32 [[TMP2]], 12529
 ; FORCE_REDUCTION-NEXT:    [[VAL_40:%.*]] = and i32 [[OP_EXTRA29]], [[VAL_39]]
 ; FORCE_REDUCTION-NEXT:    [[VAL_41:%.*]] = add i32 [[TMP2]], 13685
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
index 2a0b66ee28174..4622c77e8c874 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/PR40310.ll
@@ -13,21 +13,6 @@ define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[SHUFFLE]], i32 15
 ; CHECK-NEXT:    store atomic i32 [[TMP3]], i32* [[VALS:%.*]] unordered, align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = add <16 x i32> [[SHUFFLE]], <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 -1>
-; CHECK-NEXT:    [[V14:%.*]] = and i32 [[TMP2]], undef
-; CHECK-NEXT:    [[V16:%.*]] = and i32 undef, [[V14]]
-; CHECK-NEXT:    [[V18:%.*]] = and i32 undef, [[V16]]
-; CHECK-NEXT:    [[V20:%.*]] = and i32 undef, [[V18]]
-; CHECK-NEXT:    [[V22:%.*]] = and i32 undef, [[V20]]
-; CHECK-NEXT:    [[V24:%.*]] = and i32 undef, [[V22]]
-; CHECK-NEXT:    [[V26:%.*]] = and i32 undef, [[V24]]
-; CHECK-NEXT:    [[V28:%.*]] = and i32 undef, [[V26]]
-; CHECK-NEXT:    [[V30:%.*]] = and i32 undef, [[V28]]
-; CHECK-NEXT:    [[V32:%.*]] = and i32 undef, [[V30]]
-; CHECK-NEXT:    [[V34:%.*]] = and i32 undef, [[V32]]
-; CHECK-NEXT:    [[V36:%.*]] = and i32 undef, [[V34]]
-; CHECK-NEXT:    [[V38:%.*]] = and i32 undef, [[V36]]
-; CHECK-NEXT:    [[V40:%.*]] = and i32 undef, [[V38]]
-; CHECK-NEXT:    [[V42:%.*]] = and i32 undef, [[V40]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP4]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <16 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -38,7 +23,6 @@ define void @mainTest(i32 %param, i32 * %vals, i32 %len) {
 ; CHECK-NEXT:    [[BIN_RDX6:%.*]] = and <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = and i32 [[TMP5]], [[TMP2]]
-; CHECK-NEXT:    [[V43:%.*]] = and i32 undef, [[V42]]
 ; CHECK-NEXT:    [[V44:%.*]] = add i32 [[TMP2]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i32> undef, i32 [[V44]], i32 0
 ; CHECK-NEXT:    [[TMP7]] = insertelement <2 x i32> [[TMP6]], i32 [[OP_EXTRA]], i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
index 4bd84d846e115..888985a1350ea 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-add-usat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -61,25 +61,6 @@ define void @add_v8i64() {
 ; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v8i64(
-; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.uadd.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
-; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v8i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
@@ -153,25 +134,6 @@ define void @add_v16i32() {
 ; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v16i32(
-; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.uadd.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
-; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v16i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
 ; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
@@ -277,25 +239,6 @@ define void @add_v32i16() {
 ; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @add_v32i16(
-; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.uadd.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
-; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @add_v32i16(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
 ; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
index a8897aac631fd..92fdbeb70cdb8 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fix.ll
@@ -149,22 +149,70 @@ define void @smul_v8i64() {
 
 define void @smul_v16i32() {
 ; SSE-LABEL: @smul_v16i32(
-; SSE-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SSE-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]], i32 3)
-; SSE-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]], i32 3)
-; SSE-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]], i32 3)
-; SSE-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.smul.fix.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]], i32 3)
-; SSE-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
+; SSE-NEXT:    [[A0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[A1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[A2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[A3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[A4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[A5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[A6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[A7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[A8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[A9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[A10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[A11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[A12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[A13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[A14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[A15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[B0:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 0), align 4
+; SSE-NEXT:    [[B1:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 1), align 4
+; SSE-NEXT:    [[B2:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 2), align 4
+; SSE-NEXT:    [[B3:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 3), align 4
+; SSE-NEXT:    [[B4:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4), align 4
+; SSE-NEXT:    [[B5:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 5), align 4
+; SSE-NEXT:    [[B6:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 6), align 4
+; SSE-NEXT:    [[B7:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 7), align 4
+; SSE-NEXT:    [[B8:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8), align 4
+; SSE-NEXT:    [[B9:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 9), align 4
+; SSE-NEXT:    [[B10:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 10), align 4
+; SSE-NEXT:    [[B11:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 11), align 4
+; SSE-NEXT:    [[B12:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12), align 4
+; SSE-NEXT:    [[B13:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 13), align 4
+; SSE-NEXT:    [[B14:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 14), align 4
+; SSE-NEXT:    [[B15:%.*]] = load i32, i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 15), align 4
+; SSE-NEXT:    [[R0:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A0]], i32 [[B0]], i32 3)
+; SSE-NEXT:    [[R1:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A1]], i32 [[B1]], i32 3)
+; SSE-NEXT:    [[R2:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A2]], i32 [[B2]], i32 3)
+; SSE-NEXT:    [[R3:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A3]], i32 [[B3]], i32 3)
+; SSE-NEXT:    [[R4:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A4]], i32 [[B4]], i32 3)
+; SSE-NEXT:    [[R5:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A5]], i32 [[B5]], i32 3)
+; SSE-NEXT:    [[R6:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A6]], i32 [[B6]], i32 3)
+; SSE-NEXT:    [[R7:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A7]], i32 [[B7]], i32 3)
+; SSE-NEXT:    [[R8:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A8]], i32 [[B8]], i32 3)
+; SSE-NEXT:    [[R9:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A9]], i32 [[B9]], i32 3)
+; SSE-NEXT:    [[R10:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A10]], i32 [[B10]], i32 3)
+; SSE-NEXT:    [[R11:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A11]], i32 [[B11]], i32 3)
+; SSE-NEXT:    [[R12:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A12]], i32 [[B12]], i32 3)
+; SSE-NEXT:    [[R13:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A13]], i32 [[B13]], i32 3)
+; SSE-NEXT:    [[R14:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A14]], i32 [[B14]], i32 3)
+; SSE-NEXT:    [[R15:%.*]] = call i32 @llvm.smul.fix.i32(i32 [[A15]], i32 [[B15]], i32 3)
+; SSE-NEXT:    store i32 [[R0]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 0), align 4
+; SSE-NEXT:    store i32 [[R1]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 1), align 4
+; SSE-NEXT:    store i32 [[R2]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 2), align 4
+; SSE-NEXT:    store i32 [[R3]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 3), align 4
+; SSE-NEXT:    store i32 [[R4]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4), align 4
+; SSE-NEXT:    store i32 [[R5]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 5), align 4
+; SSE-NEXT:    store i32 [[R6]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 6), align 4
+; SSE-NEXT:    store i32 [[R7]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 7), align 4
+; SSE-NEXT:    store i32 [[R8]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8), align 4
+; SSE-NEXT:    store i32 [[R9]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 9), align 4
+; SSE-NEXT:    store i32 [[R10]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 10), align 4
+; SSE-NEXT:    store i32 [[R11]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 11), align 4
+; SSE-NEXT:    store i32 [[R12]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12), align 4
+; SSE-NEXT:    store i32 [[R13]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 13), align 4
+; SSE-NEXT:    store i32 [[R14]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 14), align 4
+; SSE-NEXT:    store i32 [[R15]], i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 15), align 4
 ; SSE-NEXT:    ret void
 ;
 ; SLM-LABEL: @smul_v16i32(
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
index 1858e3a63cec7..9de83db6ab734 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-fp.ll
@@ -98,6 +98,14 @@ define <2 x double> @buildvector_div_2f64(<2 x double> %a, <2 x double> %b) {
 ; AVX-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
 ; AVX-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
 ; AVX-NEXT:    ret <2 x double> [[R1]]
+;
+; AVX512-LABEL: @buildvector_div_2f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <2 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <2 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <2 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    ret <2 x double> [[R1]]
 ;
   %a0 = extractelement <2 x double> %a, i32 0
   %a1 = extractelement <2 x double> %a, i32 1
@@ -381,6 +389,18 @@ define <4 x double> @buildvector_div_4f64(<4 x double> %a, <4 x double> %b) {
 ; AVX-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
 ; AVX-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
 ; AVX-NEXT:    ret <4 x double> [[R3]]
+;
+; AVX512-LABEL: @buildvector_div_4f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <4 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <4 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <4 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <4 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <4 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT:    ret <4 x double> [[R3]]
 ;
   %a0 = extractelement <4 x double> %a, i32 0
   %a1 = extractelement <4 x double> %a, i32 1
@@ -872,6 +892,26 @@ define <8 x double> @buildvector_div_8f64(<8 x double> %a, <8 x double> %b) {
 ; AVX-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
 ; AVX-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
 ; AVX-NEXT:    ret <8 x double> [[R7]]
+;
+; AVX512-LABEL: @buildvector_div_8f64(
+; AVX512-NEXT:    [[TMP1:%.*]] = fdiv <8 x double> [[A:%.*]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP2:%.*]] = extractelement <8 x double> [[TMP1]], i32 0
+; AVX512-NEXT:    [[R0:%.*]] = insertelement <8 x double> undef, double [[TMP2]], i32 0
+; AVX512-NEXT:    [[TMP3:%.*]] = extractelement <8 x double> [[TMP1]], i32 1
+; AVX512-NEXT:    [[R1:%.*]] = insertelement <8 x double> [[R0]], double [[TMP3]], i32 1
+; AVX512-NEXT:    [[TMP4:%.*]] = extractelement <8 x double> [[TMP1]], i32 2
+; AVX512-NEXT:    [[R2:%.*]] = insertelement <8 x double> [[R1]], double [[TMP4]], i32 2
+; AVX512-NEXT:    [[TMP5:%.*]] = extractelement <8 x double> [[TMP1]], i32 3
+; AVX512-NEXT:    [[R3:%.*]] = insertelement <8 x double> [[R2]], double [[TMP5]], i32 3
+; AVX512-NEXT:    [[TMP6:%.*]] = extractelement <8 x double> [[TMP1]], i32 4
+; AVX512-NEXT:    [[R4:%.*]] = insertelement <8 x double> [[R3]], double [[TMP6]], i32 4
+; AVX512-NEXT:    [[TMP7:%.*]] = extractelement <8 x double> [[TMP1]], i32 5
+; AVX512-NEXT:    [[R5:%.*]] = insertelement <8 x double> [[R4]], double [[TMP7]], i32 5
+; AVX512-NEXT:    [[TMP8:%.*]] = extractelement <8 x double> [[TMP1]], i32 6
+; AVX512-NEXT:    [[R6:%.*]] = insertelement <8 x double> [[R5]], double [[TMP8]], i32 6
+; AVX512-NEXT:    [[TMP9:%.*]] = extractelement <8 x double> [[TMP1]], i32 7
+; AVX512-NEXT:    [[R7:%.*]] = insertelement <8 x double> [[R6]], double [[TMP9]], i32 7
+; AVX512-NEXT:    ret <8 x double> [[R7]]
 ;
   %a0 = extractelement <8 x double> %a, i32 0
   %a1 = extractelement <8 x double> %a, i32 1
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
index ec554ff4fb514..698b2b941ea1e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/arith-sub-usat.ll
@@ -1,6 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SLM
+; RUN: opt < %s -mtriple=x86_64-unknown -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=slm -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,SSE,SLM
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=corei7-avx -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=core-avx2 -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
 ; RUN: opt < %s -mtriple=x86_64-unknown -mcpu=knl -basicaa -slp-vectorizer -S | FileCheck %s --check-prefixes=CHECK,AVX512,AVX512F
@@ -61,25 +61,6 @@ define void @sub_v8i64() {
 ; SSE-NEXT:    store i64 [[R7]], i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 7), align 8
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v8i64(
-; SLM-NEXT:    [[TMP1:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @a64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP2:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP3:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP4:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP5:%.*]] = load <2 x i64>, <2 x i64>* bitcast ([8 x i64]* @b64 to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP6:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP7:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP8:%.*]] = load <2 x i64>, <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @b64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    [[TMP9:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP1]], <2 x i64> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP2]], <2 x i64> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP3]], <2 x i64> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <2 x i64> @llvm.usub.sat.v2i64(<2 x i64> [[TMP4]], <2 x i64> [[TMP8]])
-; SLM-NEXT:    store <2 x i64> [[TMP9]], <2 x i64>* bitcast ([8 x i64]* @c64 to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP10]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 2) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP11]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 4) to <2 x i64>*), align 8
-; SLM-NEXT:    store <2 x i64> [[TMP12]], <2 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @c64, i32 0, i64 6) to <2 x i64>*), align 8
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v8i64(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <4 x i64>, <4 x i64>* bitcast ([8 x i64]* @a64 to <4 x i64>*), align 8
 ; AVX-NEXT:    [[TMP2:%.*]] = load <4 x i64>, <4 x i64>* bitcast (i64* getelementptr inbounds ([8 x i64], [8 x i64]* @a64, i32 0, i64 4) to <4 x i64>*), align 8
@@ -153,25 +134,6 @@ define void @sub_v16i32() {
 ; SSE-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v16i32(
-; SLM-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @a32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP2:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP3:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP4:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP5:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([16 x i32]* @b32 to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP7:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP8:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @b32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    [[TMP9:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP1]], <4 x i32> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP2]], <4 x i32> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP3]], <4 x i32> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <4 x i32> @llvm.usub.sat.v4i32(<4 x i32> [[TMP4]], <4 x i32> [[TMP8]])
-; SLM-NEXT:    store <4 x i32> [[TMP9]], <4 x i32>* bitcast ([16 x i32]* @c32 to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP10]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 4) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP11]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 8) to <4 x i32>*), align 4
-; SLM-NEXT:    store <4 x i32> [[TMP12]], <4 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @c32, i32 0, i64 12) to <4 x i32>*), align 4
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v16i32(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([16 x i32]* @a32 to <8 x i32>*), align 4
 ; AVX-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast (i32* getelementptr inbounds ([16 x i32], [16 x i32]* @a32, i32 0, i64 8) to <8 x i32>*), align 4
@@ -277,25 +239,6 @@ define void @sub_v32i16() {
 ; SSE-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
 ; SSE-NEXT:    ret void
 ;
-; SLM-LABEL: @sub_v32i16(
-; SLM-NEXT:    [[TMP1:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @a16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP2:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP3:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP4:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP5:%.*]] = load <8 x i16>, <8 x i16>* bitcast ([32 x i16]* @b16 to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP6:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP7:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP8:%.*]] = load <8 x i16>, <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @b16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    [[TMP9:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP1]], <8 x i16> [[TMP5]])
-; SLM-NEXT:    [[TMP10:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP2]], <8 x i16> [[TMP6]])
-; SLM-NEXT:    [[TMP11:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP3]], <8 x i16> [[TMP7]])
-; SLM-NEXT:    [[TMP12:%.*]] = call <8 x i16> @llvm.usub.sat.v8i16(<8 x i16> [[TMP4]], <8 x i16> [[TMP8]])
-; SLM-NEXT:    store <8 x i16> [[TMP9]], <8 x i16>* bitcast ([32 x i16]* @c16 to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP10]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 8) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP11]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 16) to <8 x i16>*), align 2
-; SLM-NEXT:    store <8 x i16> [[TMP12]], <8 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @c16, i32 0, i64 24) to <8 x i16>*), align 2
-; SLM-NEXT:    ret void
-;
 ; AVX-LABEL: @sub_v32i16(
 ; AVX-NEXT:    [[TMP1:%.*]] = load <16 x i16>, <16 x i16>* bitcast ([32 x i16]* @a16 to <16 x i16>*), align 2
 ; AVX-NEXT:    [[TMP2:%.*]] = load <16 x i16>, <16 x i16>* bitcast (i16* getelementptr inbounds ([32 x i16], [32 x i16]* @a16, i32 0, i64 16) to <16 x i16>*), align 2
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
new file mode 100644
index 0000000000000..e3452e194dbfb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/bad-reduction.ll
@@ -0,0 +1,306 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -slp-vectorizer -S -mtriple=x86_64-- -mattr=avx2 | FileCheck %s
+
+%v8i8 = type { i8, i8, i8, i8, i8, i8, i8, i8 }
+
+; https://bugs.llvm.org/show_bug.cgi?id=43146
+
+define i64 @load_bswap(%v8i8* %p) {
+; CHECK-LABEL: @load_bswap(
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds [[V8I8:%.*]], %v8i8* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[G0]] to <4 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[T4:%.*]] = load i8, i8* [[G4]]
+; CHECK-NEXT:    [[T5:%.*]] = load i8, i8* [[G5]]
+; CHECK-NEXT:    [[T6:%.*]] = load i8, i8* [[G6]]
+; CHECK-NEXT:    [[T7:%.*]] = load i8, i8* [[G7]]
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[Z4:%.*]] = zext i8 [[T4]] to i64
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[T5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[T6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[T7]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw <4 x i64> [[TMP3]], <i64 56, i64 48, i64 40, i64 32>
+; CHECK-NEXT:    [[SH4:%.*]] = shl nuw nsw i64 [[Z4]], 24
+; CHECK-NEXT:    [[SH5:%.*]] = shl nuw nsw i64 [[Z5]], 16
+; CHECK-NEXT:    [[SH6:%.*]] = shl nuw nsw i64 [[Z6]], 8
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <4 x i64> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = or <4 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP5]], [[SH4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], [[SH5]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP7]], [[SH6]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = or i64 [[TMP8]], [[Z7]]
+; CHECK-NEXT:    ret i64 [[OP_EXTRA]]
+;
+  %g0 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 0
+  %g1 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 1
+  %g2 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 2
+  %g3 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 3
+  %g4 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 4
+  %g5 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 5
+  %g6 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 6
+  %g7 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 7
+
+  %t0 = load i8, i8* %g0
+  %t1 = load i8, i8* %g1
+  %t2 = load i8, i8* %g2
+  %t3 = load i8, i8* %g3
+  %t4 = load i8, i8* %g4
+  %t5 = load i8, i8* %g5
+  %t6 = load i8, i8* %g6
+  %t7 = load i8, i8* %g7
+
+  %z0 = zext i8 %t0 to i64
+  %z1 = zext i8 %t1 to i64
+  %z2 = zext i8 %t2 to i64
+  %z3 = zext i8 %t3 to i64
+  %z4 = zext i8 %t4 to i64
+  %z5 = zext i8 %t5 to i64
+  %z6 = zext i8 %t6 to i64
+  %z7 = zext i8 %t7 to i64
+
+  %sh0 = shl nuw i64 %z0, 56
+  %sh1 = shl nuw nsw i64 %z1, 48
+  %sh2 = shl nuw nsw i64 %z2, 40
+  %sh3 = shl nuw nsw i64 %z3, 32
+  %sh4 = shl nuw nsw i64 %z4, 24
+  %sh5 = shl nuw nsw i64 %z5, 16
+  %sh6 = shl nuw nsw i64 %z6, 8
+;  %sh7 = shl nuw nsw i64 %z7, 0 <-- missing phantom shift
+
+  %or01 = or i64 %sh0, %sh1
+  %or012 = or i64 %or01, %sh2
+  %or0123 = or i64 %or012, %sh3
+  %or01234 = or i64 %or0123, %sh4
+  %or012345 = or i64 %or01234, %sh5
+  %or0123456 = or i64 %or012345, %sh6
+  %or01234567 = or i64 %or0123456, %z7
+  ret i64 %or01234567
+}
+
+define i64 @load_bswap_nop_shift(%v8i8* %p) {
+; CHECK-LABEL: @load_bswap_nop_shift(
+; CHECK-NEXT:    [[G0:%.*]] = getelementptr inbounds [[V8I8:%.*]], %v8i8* [[P:%.*]], i64 0, i32 0
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds [[V8I8]], %v8i8* [[P]], i64 0, i32 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[G0]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw <8 x i64> [[TMP3]], <i64 56, i64 48, i64 40, i64 32, i64 24, i64 16, i64 8, i64 0>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <8 x i64> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i64> [[BIN_RDX]], <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = or <8 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i64> [[BIN_RDX2]], <8 x i64> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = or <8 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %g0 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 0
+  %g1 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 1
+  %g2 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 2
+  %g3 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 3
+  %g4 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 4
+  %g5 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 5
+  %g6 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 6
+  %g7 = getelementptr inbounds %v8i8, %v8i8* %p, i64 0, i32 7
+
+  %t0 = load i8, i8* %g0
+  %t1 = load i8, i8* %g1
+  %t2 = load i8, i8* %g2
+  %t3 = load i8, i8* %g3
+  %t4 = load i8, i8* %g4
+  %t5 = load i8, i8* %g5
+  %t6 = load i8, i8* %g6
+  %t7 = load i8, i8* %g7
+
+  %z0 = zext i8 %t0 to i64
+  %z1 = zext i8 %t1 to i64
+  %z2 = zext i8 %t2 to i64
+  %z3 = zext i8 %t3 to i64
+  %z4 = zext i8 %t4 to i64
+  %z5 = zext i8 %t5 to i64
+  %z6 = zext i8 %t6 to i64
+  %z7 = zext i8 %t7 to i64
+
+  %sh0 = shl nuw i64 %z0, 56
+  %sh1 = shl nuw nsw i64 %z1, 48
+  %sh2 = shl nuw nsw i64 %z2, 40
+  %sh3 = shl nuw nsw i64 %z3, 32
+  %sh4 = shl nuw nsw i64 %z4, 24
+  %sh5 = shl nuw nsw i64 %z5, 16
+  %sh6 = shl nuw nsw i64 %z6, 8
+  %sh7 = shl nuw nsw i64 %z7, 0
+
+  %or01 = or i64 %sh0, %sh1
+  %or012 = or i64 %or01, %sh2
+  %or0123 = or i64 %or012, %sh3
+  %or01234 = or i64 %or0123, %sh4
+  %or012345 = or i64 %or01234, %sh5
+  %or0123456 = or i64 %or012345, %sh6
+  %or01234567 = or i64 %or0123456, %sh7
+  ret i64 %or01234567
+}
+
+; https://bugs.llvm.org/show_bug.cgi?id=42708
+
+define i64 @load64le(i8* %arg) {
+; CHECK-LABEL: @load64le(
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i8, i8* [[ARG:%.*]], i64 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 7
+; CHECK-NEXT:    [[LD0:%.*]] = load i8, i8* [[ARG]], align 1
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[G1]] to <4 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x i8>, <4 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[LD5:%.*]] = load i8, i8* [[G5]], align 1
+; CHECK-NEXT:    [[LD6:%.*]] = load i8, i8* [[G6]], align 1
+; CHECK-NEXT:    [[LD7:%.*]] = load i8, i8* [[G7]], align 1
+; CHECK-NEXT:    [[Z0:%.*]] = zext i8 [[LD0]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[Z5:%.*]] = zext i8 [[LD5]] to i64
+; CHECK-NEXT:    [[Z6:%.*]] = zext i8 [[LD6]] to i64
+; CHECK-NEXT:    [[Z7:%.*]] = zext i8 [[LD7]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw nsw <4 x i64> [[TMP3]], <i64 8, i64 16, i64 24, i64 32>
+; CHECK-NEXT:    [[S5:%.*]] = shl nuw nsw i64 [[Z5]], 40
+; CHECK-NEXT:    [[S6:%.*]] = shl nuw nsw i64 [[Z6]], 48
+; CHECK-NEXT:    [[S7:%.*]] = shl nuw i64 [[Z7]], 56
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <4 x i64> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i64> [[BIN_RDX]], <4 x i64> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = or <4 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = or i64 [[TMP5]], [[S5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = or i64 [[TMP6]], [[S6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = or i64 [[TMP7]], [[S7]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = or i64 [[TMP8]], [[Z0]]
+; CHECK-NEXT:    ret i64 [[OP_EXTRA]]
+;
+  %g1 = getelementptr inbounds i8, i8* %arg, i64 1
+  %g2 = getelementptr inbounds i8, i8* %arg, i64 2
+  %g3 = getelementptr inbounds i8, i8* %arg, i64 3
+  %g4 = getelementptr inbounds i8, i8* %arg, i64 4
+  %g5 = getelementptr inbounds i8, i8* %arg, i64 5
+  %g6 = getelementptr inbounds i8, i8* %arg, i64 6
+  %g7 = getelementptr inbounds i8, i8* %arg, i64 7
+
+  %ld0 = load i8, i8* %arg, align 1
+  %ld1 = load i8, i8* %g1, align 1
+  %ld2 = load i8, i8* %g2, align 1
+  %ld3 = load i8, i8* %g3, align 1
+  %ld4 = load i8, i8* %g4, align 1
+  %ld5 = load i8, i8* %g5, align 1
+  %ld6 = load i8, i8* %g6, align 1
+  %ld7 = load i8, i8* %g7, align 1
+
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z2 = zext i8 %ld2 to i64
+  %z3 = zext i8 %ld3 to i64
+  %z4 = zext i8 %ld4 to i64
+  %z5 = zext i8 %ld5 to i64
+  %z6 = zext i8 %ld6 to i64
+  %z7 = zext i8 %ld7 to i64
+
+;  %s0 = shl nuw nsw i64 %z0, 0 <-- missing phantom shift
+  %s1 = shl nuw nsw i64 %z1, 8
+  %s2 = shl nuw nsw i64 %z2, 16
+  %s3 = shl nuw nsw i64 %z3, 24
+  %s4 = shl nuw nsw i64 %z4, 32
+  %s5 = shl nuw nsw i64 %z5, 40
+  %s6 = shl nuw nsw i64 %z6, 48
+  %s7 = shl nuw i64 %z7, 56
+
+  %o1 = or i64 %s1, %z0
+  %o2 = or i64 %o1, %s2
+  %o3 = or i64 %o2, %s3
+  %o4 = or i64 %o3, %s4
+  %o5 = or i64 %o4, %s5
+  %o6 = or i64 %o5, %s6
+  %o7 = or i64 %o6, %s7
+  ret i64 %o7
+}
+
+define i64 @load64le_nop_shift(i8* %arg) {
+; CHECK-LABEL: @load64le_nop_shift(
+; CHECK-NEXT:    [[G1:%.*]] = getelementptr inbounds i8, i8* [[ARG:%.*]], i64 1
+; CHECK-NEXT:    [[G2:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 2
+; CHECK-NEXT:    [[G3:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 3
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 4
+; CHECK-NEXT:    [[G5:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 5
+; CHECK-NEXT:    [[G6:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 6
+; CHECK-NEXT:    [[G7:%.*]] = getelementptr inbounds i8, i8* [[ARG]], i64 7
+; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i8* [[ARG]] to <8 x i8>*
+; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i8>, <8 x i8>* [[TMP1]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <8 x i8> [[TMP2]] to <8 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw <8 x i64> [[TMP3]], <i64 0, i64 8, i64 16, i64 24, i64 32, i64 40, i64 48, i64 56>
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <8 x i64> [[TMP4]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i64> [[BIN_RDX]], <8 x i64> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX2:%.*]] = or <8 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i64> [[BIN_RDX2]], <8 x i64> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[BIN_RDX4:%.*]] = or <8 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> [[BIN_RDX4]], i32 0
+; CHECK-NEXT:    ret i64 [[TMP5]]
+;
+  %g1 = getelementptr inbounds i8, i8* %arg, i64 1
+  %g2 = getelementptr inbounds i8, i8* %arg, i64 2
+  %g3 = getelementptr inbounds i8, i8* %arg, i64 3
+  %g4 = getelementptr inbounds i8, i8* %arg, i64 4
+  %g5 = getelementptr inbounds i8, i8* %arg, i64 5
+  %g6 = getelementptr inbounds i8, i8* %arg, i64 6
+  %g7 = getelementptr inbounds i8, i8* %arg, i64 7
+
+  %ld0 = load i8, i8* %arg, align 1
+  %ld1 = load i8, i8* %g1, align 1
+  %ld2 = load i8, i8* %g2, align 1
+  %ld3 = load i8, i8* %g3, align 1
+  %ld4 = load i8, i8* %g4, align 1
+  %ld5 = load i8, i8* %g5, align 1
+  %ld6 = load i8, i8* %g6, align 1
+  %ld7 = load i8, i8* %g7, align 1
+
+  %z0 = zext i8 %ld0 to i64
+  %z1 = zext i8 %ld1 to i64
+  %z2 = zext i8 %ld2 to i64
+  %z3 = zext i8 %ld3 to i64
+  %z4 = zext i8 %ld4 to i64
+  %z5 = zext i8 %ld5 to i64
+  %z6 = zext i8 %ld6 to i64
+  %z7 = zext i8 %ld7 to i64
+
+  %s0 = shl nuw nsw i64 %z0, 0
+  %s1 = shl nuw nsw i64 %z1, 8
+  %s2 = shl nuw nsw i64 %z2, 16
+  %s3 = shl nuw nsw i64 %z3, 24
+  %s4 = shl nuw nsw i64 %z4, 32
+  %s5 = shl nuw nsw i64 %z5, 40
+  %s6 = shl nuw nsw i64 %z6, 48
+  %s7 = shl nuw i64 %z7, 56
+
+  %o1 = or i64 %s1, %s0
+  %o2 = or i64 %o1, %s2
+  %o3 = or i64 %o2, %s3
+  %o4 = or i64 %o3, %s4
+  %o5 = or i64 %o4, %s5
+  %o6 = or i64 %o5, %s6
+  %o7 = or i64 %o6, %s7
+  ret i64 %o7
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
index cb4d7b707e94b..56a70637013e5 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/consecutive-access.ll
@@ -549,14 +549,17 @@ for.body:                                         ; preds = %entry, %for.body
   br i1 %cmp, label %for.body, label %for.cond.cleanup
 }
 
+; Globals/constant expressions are not normal constants.
+; They should not be treated as the usual vectorization candidates.
+
 @g1 = external global i32, align 4
 @g2 = external global i32, align 4
 
 define void @PR33958(i32** nocapture %p) {
 ; CHECK-LABEL: @PR33958(
-; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32*, i32** [[P:%.*]], i64 1
-; CHECK-NEXT:    [[TMP1:%.*]] = bitcast i32** [[P]] to <2 x i32*>*
-; CHECK-NEXT:    store <2 x i32*> <i32* @g1, i32* @g2>, <2 x i32*>* [[TMP1]], align 8
+; CHECK-NEXT:    store i32* @g1, i32** [[P:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32*, i32** [[P]], i64 1
+; CHECK-NEXT:    store i32* @g2, i32** [[ARRAYIDX1]], align 8
 ; CHECK-NEXT:    ret void
 ;
   store i32* @g1, i32** %p, align 8
@@ -565,6 +568,19 @@ define void @PR33958(i32** nocapture %p) {
   ret void
 }
 
+define void @store_constant_expression(i64* %p) {
+; CHECK-LABEL: @store_constant_expression(
+; CHECK-NEXT:    store i64 ptrtoint (i32* @g1 to i64), i64* [[P:%.*]], align 8
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i64, i64* [[P]], i64 1
+; CHECK-NEXT:    store i64 ptrtoint (i32* @g2 to i64), i64* [[ARRAYIDX1]], align 8
+; CHECK-NEXT:    ret void
+;
+  store i64 ptrtoint (i32* @g1 to i64), i64* %p, align 8
+  %arrayidx1 = getelementptr inbounds i64, i64* %p, i64 1
+  store i64 ptrtoint (i32* @g2 to i64), i64* %arrayidx1, align 8
+  ret void
+}
+
 attributes #0 = { nounwind ssp uwtable "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" }
 
 !llvm.ident = !{!0}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll b/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll
new file mode 100644
index 0000000000000..87f82dfe4e78f
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/different-vec-widths.ll
@@ -0,0 +1,105 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -mattr=sse2 -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=SSE
+; RUN: opt < %s -mattr=avx2 -slp-vectorizer -S | FileCheck %s --check-prefix=CHECK --check-prefix=AVX
+
+; TODO:
+; With AVX, we are able to vectorize the 1st 4 elements as 256-bit vector ops,
+; but the final 2 elements remain scalar. They should get vectorized using
+; 128-bit ops identically to what happens with SSE.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+define void @PR28457(double* noalias nocapture align 32 %q, double* noalias nocapture readonly align 32 %p) {
+; SSE-LABEL: @PR28457(
+; SSE-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 0
+; SSE-NEXT:    [[P1:%.*]] = getelementptr inbounds double, double* [[P]], i64 1
+; SSE-NEXT:    [[P2:%.*]] = getelementptr inbounds double, double* [[P]], i64 2
+; SSE-NEXT:    [[P3:%.*]] = getelementptr inbounds double, double* [[P]], i64 3
+; SSE-NEXT:    [[P4:%.*]] = getelementptr inbounds double, double* [[P]], i64 4
+; SSE-NEXT:    [[P5:%.*]] = getelementptr inbounds double, double* [[P]], i64 5
+; SSE-NEXT:    [[Q0:%.*]] = getelementptr inbounds double, double* [[Q:%.*]], i64 0
+; SSE-NEXT:    [[Q1:%.*]] = getelementptr inbounds double, double* [[Q]], i64 1
+; SSE-NEXT:    [[Q2:%.*]] = getelementptr inbounds double, double* [[Q]], i64 2
+; SSE-NEXT:    [[Q3:%.*]] = getelementptr inbounds double, double* [[Q]], i64 3
+; SSE-NEXT:    [[Q4:%.*]] = getelementptr inbounds double, double* [[Q]], i64 4
+; SSE-NEXT:    [[Q5:%.*]] = getelementptr inbounds double, double* [[Q]], i64 5
+; SSE-NEXT:    [[TMP1:%.*]] = bitcast double* [[P0]] to <2 x double>*
+; SSE-NEXT:    [[TMP2:%.*]] = load <2 x double>, <2 x double>* [[TMP1]], align 8
+; SSE-NEXT:    [[TMP3:%.*]] = bitcast double* [[P2]] to <2 x double>*
+; SSE-NEXT:    [[TMP4:%.*]] = load <2 x double>, <2 x double>* [[TMP3]], align 8
+; SSE-NEXT:    [[TMP5:%.*]] = bitcast double* [[P4]] to <2 x double>*
+; SSE-NEXT:    [[TMP6:%.*]] = load <2 x double>, <2 x double>* [[TMP5]], align 8
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <2 x double> [[TMP2]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP8:%.*]] = fadd <2 x double> [[TMP4]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP6]], <double 1.000000e+00, double 1.000000e+00>
+; SSE-NEXT:    [[TMP10:%.*]] = bitcast double* [[Q0]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP7]], <2 x double>* [[TMP10]], align 8
+; SSE-NEXT:    [[TMP11:%.*]] = bitcast double* [[Q2]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP8]], <2 x double>* [[TMP11]], align 8
+; SSE-NEXT:    [[TMP12:%.*]] = bitcast double* [[Q4]] to <2 x double>*
+; SSE-NEXT:    store <2 x double> [[TMP9]], <2 x double>* [[TMP12]], align 8
+; SSE-NEXT:    ret void
+;
+; AVX-LABEL: @PR28457(
+; AVX-NEXT:    [[P0:%.*]] = getelementptr inbounds double, double* [[P:%.*]], i64 0
+; AVX-NEXT:    [[P1:%.*]] = getelementptr inbounds double, double* [[P]], i64 1
+; AVX-NEXT:    [[P2:%.*]] = getelementptr inbounds double, double* [[P]], i64 2
+; AVX-NEXT:    [[P3:%.*]] = getelementptr inbounds double, double* [[P]], i64 3
+; AVX-NEXT:    [[P4:%.*]] = getelementptr inbounds double, double* [[P]], i64 4
+; AVX-NEXT:    [[P5:%.*]] = getelementptr inbounds double, double* [[P]], i64 5
+; AVX-NEXT:    [[Q0:%.*]] = getelementptr inbounds double, double* [[Q:%.*]], i64 0
+; AVX-NEXT:    [[Q1:%.*]] = getelementptr inbounds double, double* [[Q]], i64 1
+; AVX-NEXT:    [[Q2:%.*]] = getelementptr inbounds double, double* [[Q]], i64 2
+; AVX-NEXT:    [[Q3:%.*]] = getelementptr inbounds double, double* [[Q]], i64 3
+; AVX-NEXT:    [[Q4:%.*]] = getelementptr inbounds double, double* [[Q]], i64 4
+; AVX-NEXT:    [[Q5:%.*]] = getelementptr inbounds double, double* [[Q]], i64 5
+; AVX-NEXT:    [[TMP1:%.*]] = bitcast double* [[P0]] to <4 x double>*
+; AVX-NEXT:    [[TMP2:%.*]] = load <4 x double>, <4 x double>* [[TMP1]], align 8
+; AVX-NEXT:    [[D4:%.*]] = load double, double* [[P4]]
+; AVX-NEXT:    [[D5:%.*]] = load double, double* [[P5]]
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP2]], <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double 1.000000e+00>
+; AVX-NEXT:    [[A4:%.*]] = fadd double [[D4]], 1.000000e+00
+; AVX-NEXT:    [[A5:%.*]] = fadd double [[D5]], 1.000000e+00
+; AVX-NEXT:    [[TMP4:%.*]] = bitcast double* [[Q0]] to <4 x double>*
+; AVX-NEXT:    store <4 x double> [[TMP3]], <4 x double>* [[TMP4]], align 8
+; AVX-NEXT:    store double [[A4]], double* [[Q4]]
+; AVX-NEXT:    store double [[A5]], double* [[Q5]]
+; AVX-NEXT:    ret void
+;
+  %p0 = getelementptr inbounds double, double* %p, i64 0
+  %p1 = getelementptr inbounds double, double* %p, i64 1
+  %p2 = getelementptr inbounds double, double* %p, i64 2
+  %p3 = getelementptr inbounds double, double* %p, i64 3
+  %p4 = getelementptr inbounds double, double* %p, i64 4
+  %p5 = getelementptr inbounds double, double* %p, i64 5
+
+  %q0 = getelementptr inbounds double, double* %q, i64 0
+  %q1 = getelementptr inbounds double, double* %q, i64 1
+  %q2 = getelementptr inbounds double, double* %q, i64 2
+  %q3 = getelementptr inbounds double, double* %q, i64 3
+  %q4 = getelementptr inbounds double, double* %q, i64 4
+  %q5 = getelementptr inbounds double, double* %q, i64 5
+
+  %d0 = load double, double* %p0
+  %d1 = load double, double* %p1
+  %d2 = load double, double* %p2
+  %d3 = load double, double* %p3
+  %d4 = load double, double* %p4
+  %d5 = load double, double* %p5
+
+  %a0 = fadd double %d0, 1.0
+  %a1 = fadd double %d1, 1.0
+  %a2 = fadd double %d2, 1.0
+  %a3 = fadd double %d3, 1.0
+  %a4 = fadd double %d4, 1.0
+  %a5 = fadd double %d5, 1.0
+
+  store double %a0, double* %q0
+  store double %a1, double* %q1
+  store double %a2, double* %q2
+  store double %a3, double* %q3
+  store double %a4, double* %q4
+  store double %a5, double* %q5
+  ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
index 6c3994b0a22b3..1a2c66c6879ab 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-list.ll
@@ -100,16 +100,8 @@ define float @bazz() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
 ; CHECK-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
-; CHECK-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
-; CHECK-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
-; CHECK-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
-; CHECK-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -119,7 +111,6 @@ define float @bazz() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
-; CHECK-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
 ; CHECK-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
@@ -131,16 +122,8 @@ define float @bazz() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr to <8 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([20 x float]* @arr1 to <8 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <8 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
 ; THRESHOLD-NEXT:    [[MUL5:%.*]] = shl nsw i32 [[TMP0]], 2
 ; THRESHOLD-NEXT:    [[CONV6:%.*]] = sitofp i32 [[MUL5]] to float
-; THRESHOLD-NEXT:    [[ADD7:%.*]] = fadd fast float [[ADD_3]], [[CONV6]]
-; THRESHOLD-NEXT:    [[ADD19:%.*]] = fadd fast float undef, [[ADD7]]
-; THRESHOLD-NEXT:    [[ADD19_1:%.*]] = fadd fast float undef, [[ADD19]]
-; THRESHOLD-NEXT:    [[ADD19_2:%.*]] = fadd fast float undef, [[ADD19_1]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -150,7 +133,6 @@ define float @bazz() {
 ; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP4]], [[CONV]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV6]]
-; THRESHOLD-NEXT:    [[ADD19_3:%.*]] = fadd fast float undef, [[ADD19_2]]
 ; THRESHOLD-NEXT:    store float [[OP_EXTRA5]], float* @res, align 4
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
@@ -205,17 +187,14 @@ define float @bazzz() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; CHECK-NEXT:    store float [[TMP8]], float* @res, align 4
-; CHECK-NEXT:    ret float [[TMP8]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; CHECK-NEXT:    store float [[TMP5]], float* @res, align 4
+; CHECK-NEXT:    ret float [[TMP5]]
 ;
 ; THRESHOLD-LABEL: @bazzz(
 ; THRESHOLD-NEXT:  entry:
@@ -224,17 +203,14 @@ define float @bazzz() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; THRESHOLD-NEXT:    store float [[TMP8]], float* @res, align 4
-; THRESHOLD-NEXT:    ret float [[TMP8]]
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; THRESHOLD-NEXT:    store float [[TMP5]], float* @res, align 4
+; THRESHOLD-NEXT:    ret float [[TMP5]]
 ;
 entry:
   %0 = load i32, i32* @n, align 4
@@ -267,16 +243,13 @@ define i32 @foo() {
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; CHECK-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; CHECK-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP5]] to i32
 ; CHECK-NEXT:    store i32 [[CONV4]], i32* @n, align 4
 ; CHECK-NEXT:    ret i32 [[CONV4]]
 ;
@@ -287,16 +260,13 @@ define i32 @foo() {
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP1]]
-; THRESHOLD-NEXT:    [[TMP4:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[TMP5:%.*]] = fadd fast float undef, [[TMP4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
-; THRESHOLD-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; THRESHOLD-NEXT:    [[TMP7:%.*]] = fadd fast float undef, [[TMP5]]
-; THRESHOLD-NEXT:    [[TMP8:%.*]] = fmul fast float [[CONV]], [[TMP6]]
-; THRESHOLD-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP8]] to i32
+; THRESHOLD-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
+; THRESHOLD-NEXT:    [[TMP5:%.*]] = fmul fast float [[CONV]], [[TMP4]]
+; THRESHOLD-NEXT:    [[CONV4:%.*]] = fptosi float [[TMP5]] to i32
 ; THRESHOLD-NEXT:    store i32 [[CONV4]], i32* @n, align 4
 ; THRESHOLD-NEXT:    ret i32 [[CONV4]]
 ;
@@ -330,11 +300,6 @@ define float @bar() {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
-; CHECK-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
-; CHECK-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
-; CHECK-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
-; CHECK-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
@@ -342,7 +307,6 @@ define float @bar() {
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
-; CHECK-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
 ; CHECK-NEXT:    store float [[TMP3]], float* @res, align 4
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -351,11 +315,6 @@ define float @bar() {
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <4 x float>, <4 x float>* bitcast ([20 x float]* @arr1 to <4 x float>*), align 16
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP0]]
-; THRESHOLD-NEXT:    [[CMP4:%.*]] = fcmp fast ogt float undef, undef
-; THRESHOLD-NEXT:    [[MAX_0_MUL3:%.*]] = select i1 [[CMP4]], float undef, float undef
-; THRESHOLD-NEXT:    [[CMP4_1:%.*]] = fcmp fast ogt float [[MAX_0_MUL3]], undef
-; THRESHOLD-NEXT:    [[MAX_0_MUL3_1:%.*]] = select i1 [[CMP4_1]], float [[MAX_0_MUL3]], float undef
-; THRESHOLD-NEXT:    [[CMP4_2:%.*]] = fcmp fast ogt float [[MAX_0_MUL3_1]], undef
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <4 x float> [[TMP2]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x float> [[TMP2]], <4 x float> [[RDX_SHUF]]
@@ -363,7 +322,6 @@ define float @bar() {
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = fcmp fast ogt <4 x float> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; THRESHOLD-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x float> [[RDX_MINMAX_SELECT]], <4 x float> [[RDX_SHUF1]]
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = extractelement <4 x float> [[RDX_MINMAX_SELECT3]], i32 0
-; THRESHOLD-NEXT:    [[MAX_0_MUL3_2:%.*]] = select i1 [[CMP4_2]], float [[MAX_0_MUL3_1]], float undef
 ; THRESHOLD-NEXT:    store float [[TMP3]], float* @res, align 4
 ; THRESHOLD-NEXT:    ret float [[TMP3]]
 ;
@@ -410,21 +368,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
 ; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; CHECK-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
 ; CHECK-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
@@ -459,37 +402,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
-; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
-; CHECK-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
-; CHECK-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
-; CHECK-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
-; CHECK-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
-; CHECK-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
-; CHECK-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
-; CHECK-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
-; CHECK-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
-; CHECK-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
-; CHECK-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
-; CHECK-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
-; CHECK-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
-; CHECK-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
-; CHECK-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
-; CHECK-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -511,7 +423,6 @@ define float @f(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
 ; CHECK-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
-; CHECK-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
 ; CHECK-NEXT:    ret float [[OP_RDX]]
 ;
 ; THRESHOLD-LABEL: @f(
@@ -533,21 +444,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <16 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <16 x float>, <16 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, undef
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; THRESHOLD-NEXT:    [[ARRAYIDX_17:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
 ; THRESHOLD-NEXT:    [[ARRAYIDX_18:%.*]] = getelementptr inbounds float, float* [[X]], i64 18
@@ -582,37 +478,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_47:%.*]] = getelementptr inbounds float, float* [[X]], i64 47
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_16]] to <32 x float>*
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <32 x float>, <32 x float>* [[TMP2]], align 4
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
-; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
-; THRESHOLD-NEXT:    [[ADD_32:%.*]] = fadd fast float undef, [[ADD_31]]
-; THRESHOLD-NEXT:    [[ADD_33:%.*]] = fadd fast float undef, [[ADD_32]]
-; THRESHOLD-NEXT:    [[ADD_34:%.*]] = fadd fast float undef, [[ADD_33]]
-; THRESHOLD-NEXT:    [[ADD_35:%.*]] = fadd fast float undef, [[ADD_34]]
-; THRESHOLD-NEXT:    [[ADD_36:%.*]] = fadd fast float undef, [[ADD_35]]
-; THRESHOLD-NEXT:    [[ADD_37:%.*]] = fadd fast float undef, [[ADD_36]]
-; THRESHOLD-NEXT:    [[ADD_38:%.*]] = fadd fast float undef, [[ADD_37]]
-; THRESHOLD-NEXT:    [[ADD_39:%.*]] = fadd fast float undef, [[ADD_38]]
-; THRESHOLD-NEXT:    [[ADD_40:%.*]] = fadd fast float undef, [[ADD_39]]
-; THRESHOLD-NEXT:    [[ADD_41:%.*]] = fadd fast float undef, [[ADD_40]]
-; THRESHOLD-NEXT:    [[ADD_42:%.*]] = fadd fast float undef, [[ADD_41]]
-; THRESHOLD-NEXT:    [[ADD_43:%.*]] = fadd fast float undef, [[ADD_42]]
-; THRESHOLD-NEXT:    [[ADD_44:%.*]] = fadd fast float undef, [[ADD_43]]
-; THRESHOLD-NEXT:    [[ADD_45:%.*]] = fadd fast float undef, [[ADD_44]]
-; THRESHOLD-NEXT:    [[ADD_46:%.*]] = fadd fast float undef, [[ADD_45]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP3]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP3]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -634,7 +499,6 @@ define float @f(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[BIN_RDX16:%.*]] = fadd fast <16 x float> [[BIN_RDX14]], [[RDX_SHUF15]]
 ; THRESHOLD-NEXT:    [[TMP5:%.*]] = extractelement <16 x float> [[BIN_RDX16]], i32 0
 ; THRESHOLD-NEXT:    [[OP_RDX:%.*]] = fadd fast float [[TMP4]], [[TMP5]]
-; THRESHOLD-NEXT:    [[ADD_47:%.*]] = fadd fast float undef, [[ADD_46]]
 ; THRESHOLD-NEXT:    ret float [[OP_RDX]]
 ;
   entry:
@@ -821,37 +685,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; CHECK-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -864,7 +697,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
-; CHECK-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA]]
 ;
 ; THRESHOLD-LABEL: @f1(
@@ -904,37 +736,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_31:%.*]] = getelementptr inbounds float, float* [[X]], i64 31
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <32 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <32 x float>, <32 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD:%.*]] = fadd fast float undef, [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
-; THRESHOLD-NEXT:    [[ADD_30:%.*]] = fadd fast float undef, [[ADD_29]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP1]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <32 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x float> [[BIN_RDX]], <32 x float> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -947,7 +748,6 @@ define float @f1(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[BIN_RDX8:%.*]] = fadd fast <32 x float> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <32 x float> [[BIN_RDX8]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD_31:%.*]] = fadd fast float undef, [[ADD_30]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA]]
 ;
   entry:
@@ -1058,17 +858,12 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
 ; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
-; CHECK-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
 ; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
 ; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
 ; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
 ; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
-; CHECK-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
 ; CHECK-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
@@ -1079,14 +874,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
-; CHECK-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; CHECK-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; CHECK-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; CHECK-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; CHECK-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; CHECK-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; CHECK-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; CHECK-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; CHECK-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; CHECK-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
@@ -1105,21 +892,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
 ; CHECK-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
 ; CHECK-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
-; CHECK-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; CHECK-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; CHECK-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; CHECK-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; CHECK-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; CHECK-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; CHECK-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; CHECK-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; CHECK-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; CHECK-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; CHECK-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; CHECK-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; CHECK-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; CHECK-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; CHECK-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1145,7 +917,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; CHECK-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
-; CHECK-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
 ; CHECK-NEXT:    ret float [[TMP12]]
 ;
 ; THRESHOLD-LABEL: @loadadd31(
@@ -1154,17 +925,12 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = load float, float* [[ARRAYIDX]], align 4
 ; THRESHOLD-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds float, float* [[X]], i64 2
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load float, float* [[ARRAYIDX_1]], align 4
-; THRESHOLD-NEXT:    [[ADD_1:%.*]] = fadd fast float [[TMP1]], [[TMP0]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds float, float* [[X]], i64 3
 ; THRESHOLD-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds float, float* [[X]], i64 4
 ; THRESHOLD-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds float, float* [[X]], i64 5
 ; THRESHOLD-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds float, float* [[X]], i64 6
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = bitcast float* [[ARRAYIDX_2]] to <4 x float>*
 ; THRESHOLD-NEXT:    [[TMP3:%.*]] = load <4 x float>, <4 x float>* [[TMP2]], align 4
-; THRESHOLD-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; THRESHOLD-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; THRESHOLD-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; THRESHOLD-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds float, float* [[X]], i64 8
 ; THRESHOLD-NEXT:    [[ARRAYIDX_8:%.*]] = getelementptr inbounds float, float* [[X]], i64 9
@@ -1175,14 +941,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_13:%.*]] = getelementptr inbounds float, float* [[X]], i64 14
 ; THRESHOLD-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX_6]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
-; THRESHOLD-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; THRESHOLD-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; THRESHOLD-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; THRESHOLD-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; THRESHOLD-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; THRESHOLD-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; THRESHOLD-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; THRESHOLD-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; THRESHOLD-NEXT:    [[ARRAYIDX_14:%.*]] = getelementptr inbounds float, float* [[X]], i64 15
 ; THRESHOLD-NEXT:    [[ARRAYIDX_15:%.*]] = getelementptr inbounds float, float* [[X]], i64 16
 ; THRESHOLD-NEXT:    [[ARRAYIDX_16:%.*]] = getelementptr inbounds float, float* [[X]], i64 17
@@ -1201,21 +959,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX_29:%.*]] = getelementptr inbounds float, float* [[X]], i64 30
 ; THRESHOLD-NEXT:    [[TMP6:%.*]] = bitcast float* [[ARRAYIDX_14]] to <16 x float>*
 ; THRESHOLD-NEXT:    [[TMP7:%.*]] = load <16 x float>, <16 x float>* [[TMP6]], align 4
-; THRESHOLD-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
-; THRESHOLD-NEXT:    [[ADD_15:%.*]] = fadd fast float undef, [[ADD_14]]
-; THRESHOLD-NEXT:    [[ADD_16:%.*]] = fadd fast float undef, [[ADD_15]]
-; THRESHOLD-NEXT:    [[ADD_17:%.*]] = fadd fast float undef, [[ADD_16]]
-; THRESHOLD-NEXT:    [[ADD_18:%.*]] = fadd fast float undef, [[ADD_17]]
-; THRESHOLD-NEXT:    [[ADD_19:%.*]] = fadd fast float undef, [[ADD_18]]
-; THRESHOLD-NEXT:    [[ADD_20:%.*]] = fadd fast float undef, [[ADD_19]]
-; THRESHOLD-NEXT:    [[ADD_21:%.*]] = fadd fast float undef, [[ADD_20]]
-; THRESHOLD-NEXT:    [[ADD_22:%.*]] = fadd fast float undef, [[ADD_21]]
-; THRESHOLD-NEXT:    [[ADD_23:%.*]] = fadd fast float undef, [[ADD_22]]
-; THRESHOLD-NEXT:    [[ADD_24:%.*]] = fadd fast float undef, [[ADD_23]]
-; THRESHOLD-NEXT:    [[ADD_25:%.*]] = fadd fast float undef, [[ADD_24]]
-; THRESHOLD-NEXT:    [[ADD_26:%.*]] = fadd fast float undef, [[ADD_25]]
-; THRESHOLD-NEXT:    [[ADD_27:%.*]] = fadd fast float undef, [[ADD_26]]
-; THRESHOLD-NEXT:    [[ADD_28:%.*]] = fadd fast float undef, [[ADD_27]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP7]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP7]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1241,7 +984,6 @@ define float @loadadd31(float* nocapture readonly %x) {
 ; THRESHOLD-NEXT:    [[OP_RDX17:%.*]] = fadd fast float [[OP_RDX]], [[TMP10]]
 ; THRESHOLD-NEXT:    [[TMP11:%.*]] = fadd fast float [[OP_RDX17]], [[TMP1]]
 ; THRESHOLD-NEXT:    [[TMP12:%.*]] = fadd fast float [[TMP11]], [[TMP0]]
-; THRESHOLD-NEXT:    [[ADD_29:%.*]] = fadd fast float undef, [[ADD_28]]
 ; THRESHOLD-NEXT:    ret float [[TMP12]]
 ;
   entry:
@@ -1352,14 +1094,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1369,7 +1103,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
 ; THRESHOLD-LABEL: @extra_args(
@@ -1386,14 +1119,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1403,7 +1128,6 @@ define float @extra_args(float* nocapture readonly %x, i32 %a, i32 %b) {
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
   entry:
@@ -1452,16 +1176,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1473,7 +1187,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA7]]
 ;
 ; THRESHOLD-LABEL: @extra_args_same_several_times(
@@ -1490,16 +1203,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD41:%.*]] = fadd fast float [[ADD4]], 5.000000e+00
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD41]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_11:%.*]] = fadd fast float [[ADD4_1]], 5.000000e+00
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_11]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD4_3]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1511,7 +1214,6 @@ define float @extra_args_same_several_times(float* nocapture readonly %x, i32 %a
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], 5.000000e+00
 ; THRESHOLD-NEXT:    [[OP_EXTRA6:%.*]] = fadd fast float [[OP_EXTRA5]], 5.000000e+00
 ; THRESHOLD-NEXT:    [[OP_EXTRA7:%.*]] = fadd fast float [[OP_EXTRA6]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA7]]
 ;
   entry:
@@ -1564,14 +1266,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; CHECK-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; CHECK-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; CHECK-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; CHECK-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
-; CHECK-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; CHECK-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; CHECK-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
-; CHECK-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1581,7 +1275,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; CHECK-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; CHECK-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; CHECK-NEXT:    ret float [[OP_EXTRA5]]
 ;
 ; THRESHOLD-LABEL: @extra_args_no_replace(
@@ -1600,14 +1293,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; THRESHOLD-NEXT:    [[ARRAYIDX3_6:%.*]] = getelementptr inbounds float, float* [[X]], i64 7
 ; THRESHOLD-NEXT:    [[TMP0:%.*]] = bitcast float* [[X]] to <8 x float>*
 ; THRESHOLD-NEXT:    [[TMP1:%.*]] = load <8 x float>, <8 x float>* [[TMP0]], align 4
-; THRESHOLD-NEXT:    [[ADD1:%.*]] = fadd fast float undef, [[ADD]]
-; THRESHOLD-NEXT:    [[ADD4:%.*]] = fadd fast float undef, [[ADD1]]
-; THRESHOLD-NEXT:    [[ADD4_1:%.*]] = fadd fast float undef, [[ADD4]]
-; THRESHOLD-NEXT:    [[ADD4_2:%.*]] = fadd fast float undef, [[ADD4_1]]
-; THRESHOLD-NEXT:    [[ADD4_3:%.*]] = fadd fast float undef, [[ADD4_2]]
-; THRESHOLD-NEXT:    [[ADD5:%.*]] = fadd fast float [[ADD4_3]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_4:%.*]] = fadd fast float undef, [[ADD5]]
-; THRESHOLD-NEXT:    [[ADD4_5:%.*]] = fadd fast float undef, [[ADD4_4]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP1]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1617,7 +1302,6 @@ define float @extra_args_no_replace(float* nocapture readonly %x, i32 %a, i32 %b
 ; THRESHOLD-NEXT:    [[TMP2:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = fadd fast float [[TMP2]], [[ADD]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA5:%.*]] = fadd fast float [[OP_EXTRA]], [[CONV]]
-; THRESHOLD-NEXT:    [[ADD4_6:%.*]] = fadd fast float undef, [[ADD4_5]]
 ; THRESHOLD-NEXT:    ret float [[OP_EXTRA5]]
 ;
   entry:
@@ -1668,10 +1352,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
 ; CHECK-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
-; CHECK-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
-; CHECK-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
-; CHECK-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
-; CHECK-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -1679,7 +1359,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
 ; CHECK-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
-; CHECK-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA3]]
 ;
 ; THRESHOLD-LABEL: @wobble(
@@ -1696,10 +1375,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; THRESHOLD-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[TMP8]], i32 3
 ; THRESHOLD-NEXT:    [[TMP10:%.*]] = icmp eq <4 x i32> [[TMP8]], zeroinitializer
 ; THRESHOLD-NEXT:    [[TMP11:%.*]] = sext <4 x i1> [[TMP10]] to <4 x i32>
-; THRESHOLD-NEXT:    [[R1:%.*]] = add nuw i32 [[ARG]], undef
-; THRESHOLD-NEXT:    [[R2:%.*]] = add nsw i32 [[R1]], undef
-; THRESHOLD-NEXT:    [[R3:%.*]] = add nsw i32 [[R2]], undef
-; THRESHOLD-NEXT:    [[R4:%.*]] = add nsw i32 [[R3]], undef
 ; THRESHOLD-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP11]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; THRESHOLD-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[TMP11]], [[RDX_SHUF]]
 ; THRESHOLD-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
@@ -1707,7 +1382,6 @@ define i32 @wobble(i32 %arg, i32 %bar) {
 ; THRESHOLD-NEXT:    [[TMP12:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; THRESHOLD-NEXT:    [[OP_EXTRA:%.*]] = add nuw i32 [[TMP12]], [[ARG]]
 ; THRESHOLD-NEXT:    [[OP_EXTRA3:%.*]] = add nsw i32 [[OP_EXTRA]], [[TMP9]]
-; THRESHOLD-NEXT:    [[R5:%.*]] = add nsw i32 [[R4]], [[TMP9]]
 ; THRESHOLD-NEXT:    ret i32 [[OP_EXTRA3]]
 ;
   bb:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
index 8432b910d9193..8b0ae669ae795 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-minmax.ll
@@ -12,19 +12,6 @@
 define i32 @maxi8(i32) {
 ; CHECK-LABEL: @maxi8(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr to <8 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP2]], <8 x i32> [[RDX_SHUF]]
@@ -34,9 +21,8 @@ define i32 @maxi8(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp sgt <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP16]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -66,35 +52,6 @@ define i32 @maxi8(i32) {
 define i32 @maxi16(i32) {
 ; CHECK-LABEL: @maxi16(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr to <16 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP2]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <16 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x i32> [[TMP2]], <16 x i32> [[RDX_SHUF]]
@@ -107,9 +64,8 @@ define i32 @maxi16(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = icmp sgt <16 x i32> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x i32> [[RDX_MINMAX_SELECT6]], <16 x i32> [[RDX_SHUF7]]
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i32> [[RDX_MINMAX_SELECT9]], i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x i32> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -163,67 +119,6 @@ define i32 @maxi16(i32) {
 define i32 @maxi32(i32) {
 ; CHECK-LABEL: @maxi32(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr to <32 x i32>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP5:%.*]] = icmp sgt i32 [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 undef
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP6]], i32 undef
-; CHECK-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; CHECK-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; CHECK-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; CHECK-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; CHECK-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; CHECK-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; CHECK-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 undef
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 undef
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], undef
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 undef
-; CHECK-NEXT:    [[TMP35:%.*]] = icmp sgt i32 [[TMP34]], undef
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], i32 [[TMP34]], i32 undef
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp sgt i32 [[TMP36]], undef
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], i32 [[TMP36]], i32 undef
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp sgt i32 [[TMP38]], undef
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], i32 [[TMP38]], i32 undef
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 [[TMP40]], undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 [[TMP40]], i32 undef
-; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP42]], i32 undef
-; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 [[TMP44]], undef
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[TMP44]], i32 undef
-; CHECK-NEXT:    [[TMP47:%.*]] = icmp sgt i32 [[TMP46]], undef
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i32 [[TMP46]], i32 undef
-; CHECK-NEXT:    [[TMP49:%.*]] = icmp sgt i32 [[TMP48]], undef
-; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], i32 [[TMP48]], i32 undef
-; CHECK-NEXT:    [[TMP51:%.*]] = icmp sgt i32 [[TMP50]], undef
-; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], i32 [[TMP50]], i32 undef
-; CHECK-NEXT:    [[TMP53:%.*]] = icmp sgt i32 [[TMP52]], undef
-; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], i32 [[TMP52]], i32 undef
-; CHECK-NEXT:    [[TMP55:%.*]] = icmp sgt i32 [[TMP54]], undef
-; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], i32 [[TMP54]], i32 undef
-; CHECK-NEXT:    [[TMP57:%.*]] = icmp sgt i32 [[TMP56]], undef
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], i32 [[TMP56]], i32 undef
-; CHECK-NEXT:    [[TMP59:%.*]] = icmp sgt i32 [[TMP58]], undef
-; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], i32 [[TMP58]], i32 undef
-; CHECK-NEXT:    [[TMP61:%.*]] = icmp sgt i32 [[TMP60]], undef
-; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], i32 [[TMP60]], i32 undef
-; CHECK-NEXT:    [[TMP63:%.*]] = icmp sgt i32 [[TMP62]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP2]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <32 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x i32> [[TMP2]], <32 x i32> [[RDX_SHUF]]
@@ -239,9 +134,8 @@ define i32 @maxi32(i32) {
 ; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = icmp sgt <32 x i32> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x i32> [[RDX_MINMAX_SELECT9]], <32 x i32> [[RDX_SHUF10]]
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x i32> [[RDX_MINMAX_SELECT12]], i32 0
-; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], i32 [[TMP62]], i32 undef
-; CHECK-NEXT:    ret i32 [[TMP64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <32 x i32> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -343,19 +237,6 @@ define i32 @maxi32(i32) {
 define float @maxf8(float) {
 ; CHECK-LABEL: @maxf8(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr1 to <8 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <8 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x float> [[TMP2]], <8 x float> [[RDX_SHUF]]
@@ -365,9 +246,8 @@ define float @maxf8(float) {
 ; CHECK-NEXT:    [[RDX_SHUF4:%.*]] = shufflevector <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = fcmp fast ogt <8 x float> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x float> [[RDX_MINMAX_SELECT3]], <8 x float> [[RDX_SHUF4]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[TMP17:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    ret float [[TMP16]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x float> [[RDX_MINMAX_SELECT6]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -397,35 +277,6 @@ define float @maxf8(float) {
 define float @maxf16(float) {
 ; CHECK-LABEL: @maxf16(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr1 to <16 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP2]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <16 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP]], <16 x float> [[TMP2]], <16 x float> [[RDX_SHUF]]
@@ -438,9 +289,8 @@ define float @maxf16(float) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP8:%.*]] = fcmp fast ogt <16 x float> [[RDX_MINMAX_SELECT6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT9:%.*]] = select <16 x i1> [[RDX_MINMAX_CMP8]], <16 x float> [[RDX_MINMAX_SELECT6]], <16 x float> [[RDX_SHUF7]]
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
-; CHECK-NEXT:    [[TMP33:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; CHECK-NEXT:    ret float [[TMP32]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <16 x float> [[RDX_MINMAX_SELECT9]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -494,67 +344,6 @@ define float @maxf16(float) {
 define float @maxf32(float) {
 ; CHECK-LABEL: @maxf32(
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <32 x float>, <32 x float>* bitcast ([32 x float]* @arr1 to <32 x float>*), align 16
-; CHECK-NEXT:    [[TMP3:%.*]] = fcmp fast ogt float undef, undef
-; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP3]], float undef, float undef
-; CHECK-NEXT:    [[TMP5:%.*]] = fcmp fast ogt float [[TMP4]], undef
-; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], float [[TMP4]], float undef
-; CHECK-NEXT:    [[TMP7:%.*]] = fcmp fast ogt float [[TMP6]], undef
-; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], float [[TMP6]], float undef
-; CHECK-NEXT:    [[TMP9:%.*]] = fcmp fast ogt float [[TMP8]], undef
-; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], float [[TMP8]], float undef
-; CHECK-NEXT:    [[TMP11:%.*]] = fcmp fast ogt float [[TMP10]], undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], float [[TMP10]], float undef
-; CHECK-NEXT:    [[TMP13:%.*]] = fcmp fast ogt float [[TMP12]], undef
-; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], float [[TMP12]], float undef
-; CHECK-NEXT:    [[TMP15:%.*]] = fcmp fast ogt float [[TMP14]], undef
-; CHECK-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], float [[TMP14]], float undef
-; CHECK-NEXT:    [[TMP17:%.*]] = fcmp fast ogt float [[TMP16]], undef
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], float [[TMP16]], float undef
-; CHECK-NEXT:    [[TMP19:%.*]] = fcmp fast ogt float [[TMP18]], undef
-; CHECK-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], float [[TMP18]], float undef
-; CHECK-NEXT:    [[TMP21:%.*]] = fcmp fast ogt float [[TMP20]], undef
-; CHECK-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], float [[TMP20]], float undef
-; CHECK-NEXT:    [[TMP23:%.*]] = fcmp fast ogt float [[TMP22]], undef
-; CHECK-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], float [[TMP22]], float undef
-; CHECK-NEXT:    [[TMP25:%.*]] = fcmp fast ogt float [[TMP24]], undef
-; CHECK-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], float [[TMP24]], float undef
-; CHECK-NEXT:    [[TMP27:%.*]] = fcmp fast ogt float [[TMP26]], undef
-; CHECK-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], float [[TMP26]], float undef
-; CHECK-NEXT:    [[TMP29:%.*]] = fcmp fast ogt float [[TMP28]], undef
-; CHECK-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], float [[TMP28]], float undef
-; CHECK-NEXT:    [[TMP31:%.*]] = fcmp fast ogt float [[TMP30]], undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], float [[TMP30]], float undef
-; CHECK-NEXT:    [[TMP33:%.*]] = fcmp fast ogt float [[TMP32]], undef
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], float [[TMP32]], float undef
-; CHECK-NEXT:    [[TMP35:%.*]] = fcmp fast ogt float [[TMP34]], undef
-; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP35]], float [[TMP34]], float undef
-; CHECK-NEXT:    [[TMP37:%.*]] = fcmp fast ogt float [[TMP36]], undef
-; CHECK-NEXT:    [[TMP38:%.*]] = select i1 [[TMP37]], float [[TMP36]], float undef
-; CHECK-NEXT:    [[TMP39:%.*]] = fcmp fast ogt float [[TMP38]], undef
-; CHECK-NEXT:    [[TMP40:%.*]] = select i1 [[TMP39]], float [[TMP38]], float undef
-; CHECK-NEXT:    [[TMP41:%.*]] = fcmp fast ogt float [[TMP40]], undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], float [[TMP40]], float undef
-; CHECK-NEXT:    [[TMP43:%.*]] = fcmp fast ogt float [[TMP42]], undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], float [[TMP42]], float undef
-; CHECK-NEXT:    [[TMP45:%.*]] = fcmp fast ogt float [[TMP44]], undef
-; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], float [[TMP44]], float undef
-; CHECK-NEXT:    [[TMP47:%.*]] = fcmp fast ogt float [[TMP46]], undef
-; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], float [[TMP46]], float undef
-; CHECK-NEXT:    [[TMP49:%.*]] = fcmp fast ogt float [[TMP48]], undef
-; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], float [[TMP48]], float undef
-; CHECK-NEXT:    [[TMP51:%.*]] = fcmp fast ogt float [[TMP50]], undef
-; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], float [[TMP50]], float undef
-; CHECK-NEXT:    [[TMP53:%.*]] = fcmp fast ogt float [[TMP52]], undef
-; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], float [[TMP52]], float undef
-; CHECK-NEXT:    [[TMP55:%.*]] = fcmp fast ogt float [[TMP54]], undef
-; CHECK-NEXT:    [[TMP56:%.*]] = select i1 [[TMP55]], float [[TMP54]], float undef
-; CHECK-NEXT:    [[TMP57:%.*]] = fcmp fast ogt float [[TMP56]], undef
-; CHECK-NEXT:    [[TMP58:%.*]] = select i1 [[TMP57]], float [[TMP56]], float undef
-; CHECK-NEXT:    [[TMP59:%.*]] = fcmp fast ogt float [[TMP58]], undef
-; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], float [[TMP58]], float undef
-; CHECK-NEXT:    [[TMP61:%.*]] = fcmp fast ogt float [[TMP60]], undef
-; CHECK-NEXT:    [[TMP62:%.*]] = select i1 [[TMP61]], float [[TMP60]], float undef
-; CHECK-NEXT:    [[TMP63:%.*]] = fcmp fast ogt float [[TMP62]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x float> [[TMP2]], <32 x float> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = fcmp fast ogt <32 x float> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP]], <32 x float> [[TMP2]], <32 x float> [[RDX_SHUF]]
@@ -570,9 +359,8 @@ define float @maxf32(float) {
 ; CHECK-NEXT:    [[RDX_SHUF10:%.*]] = shufflevector <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP11:%.*]] = fcmp fast ogt <32 x float> [[RDX_MINMAX_SELECT9]], [[RDX_SHUF10]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT12:%.*]] = select <32 x i1> [[RDX_MINMAX_CMP11]], <32 x float> [[RDX_MINMAX_SELECT9]], <32 x float> [[RDX_SHUF10]]
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <32 x float> [[RDX_MINMAX_SELECT12]], i32 0
-; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP63]], float [[TMP62]], float undef
-; CHECK-NEXT:    ret float [[TMP64]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <32 x float> [[RDX_MINMAX_SELECT12]], i32 0
+; CHECK-NEXT:    ret float [[TMP3]]
 ;
   %2 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 0), align 16
   %3 = load float, float* getelementptr inbounds ([32 x float], [32 x float]* @arr1, i64 0, i64 1), align 4
@@ -678,34 +466,24 @@ define i32 @maxi8_mutiple_uses(i32) {
 ; SSE-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
 ; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
 ; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; SSE-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; SSE-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; SSE-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; SSE-NEXT:    ret i32 [[TMP24]]
+; SSE-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], [[TMP7]]
+; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], [[TMP5]]
+; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP5]]
+; SSE-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP12]]
+; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[OP_EXTRA]], i32 [[TMP12]]
+; SSE-NEXT:    [[TMP15:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; SSE-NEXT:    store i32 [[TMP15]], i32* @var, align 8
+; SSE-NEXT:    ret i32 [[TMP14]]
 ;
 ; AVX-LABEL: @maxi8_mutiple_uses(
 ; AVX-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
@@ -713,34 +491,24 @@ define i32 @maxi8_mutiple_uses(i32) {
 ; AVX-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
 ; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
 ; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; AVX-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; AVX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; AVX-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; AVX-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; AVX-NEXT:    ret i32 [[TMP24]]
+; AVX-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], [[TMP7]]
+; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], [[TMP5]]
+; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP5]]
+; AVX-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP12]]
+; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[OP_EXTRA]], i32 [[TMP12]]
+; AVX-NEXT:    [[TMP15:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; AVX-NEXT:    store i32 [[TMP15]], i32* @var, align 8
+; AVX-NEXT:    ret i32 [[TMP14]]
 ;
 ; AVX2-LABEL: @maxi8_mutiple_uses(
 ; AVX2-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
@@ -748,34 +516,24 @@ define i32 @maxi8_mutiple_uses(i32) {
 ; AVX2-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[TMP2]], [[TMP3]]
 ; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
 ; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[TMP17:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX2-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[TMP17]], [[TMP15]]
-; AVX2-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[TMP17]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], [[TMP5]]
-; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP20]], i32 [[TMP19]], i32 [[TMP5]]
-; AVX2-NEXT:    [[TMP21:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP22]]
-; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[OP_EXTRA]], i32 [[TMP22]]
-; AVX2-NEXT:    [[TMP25:%.*]] = select i1 [[TMP4]], i32 3, i32 4
-; AVX2-NEXT:    store i32 [[TMP25]], i32* @var, align 8
-; AVX2-NEXT:    ret i32 [[TMP24]]
+; AVX2-NEXT:    [[TMP8:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], [[TMP7]]
+; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[TMP7]]
+; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], [[TMP5]]
+; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 [[TMP5]]
+; AVX2-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP12]]
+; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[OP_EXTRA]], i32 [[TMP12]]
+; AVX2-NEXT:    [[TMP15:%.*]] = select i1 [[TMP4]], i32 3, i32 4
+; AVX2-NEXT:    store i32 [[TMP15]], i32* @var, align 8
+; AVX2-NEXT:    ret i32 [[TMP14]]
 ;
 ; SKX-LABEL: @maxi8_mutiple_uses(
 ; SKX-NEXT:    [[TMP2:%.*]] = load <2 x i32>, <2 x i32>* bitcast ([32 x i32]* @arr to <2 x i32>*), align 16
@@ -797,26 +555,16 @@ define i32 @maxi8_mutiple_uses(i32) {
 ; SKX-NEXT:    [[TMP12:%.*]] = icmp sgt <2 x i32> [[TMP9]], [[TMP11]]
 ; SKX-NEXT:    [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x i32> [[TMP9]], <2 x i32> [[TMP11]]
 ; SKX-NEXT:    [[TMP14:%.*]] = extractelement <2 x i32> [[TMP13]], i32 1
-; SKX-NEXT:    [[TMP15:%.*]] = icmp sgt i32 [[TMP14]], undef
-; SKX-NEXT:    [[TMP16:%.*]] = select i1 [[TMP15]], i32 [[TMP14]], i32 undef
-; SKX-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP16]], undef
-; SKX-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP16]], i32 undef
-; SKX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP18]], undef
-; SKX-NEXT:    [[TMP20:%.*]] = select i1 [[TMP19]], i32 [[TMP18]], i32 undef
-; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP6]]
-; SKX-NEXT:    [[TMP24:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
-; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP14]]
-; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP14]]
-; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP6]]
-; SKX-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SKX-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP27]]
-; SKX-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 [[OP_EXTRA]], i32 [[TMP27]]
-; SKX-NEXT:    [[TMP30:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
-; SKX-NEXT:    [[TMP31:%.*]] = select i1 [[TMP30]], i32 3, i32 4
-; SKX-NEXT:    store i32 [[TMP31]], i32* @var, align 8
-; SKX-NEXT:    ret i32 [[TMP29]]
+; SKX-NEXT:    [[TMP15:%.*]] = extractelement <2 x i32> [[TMP13]], i32 0
+; SKX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP15]], [[TMP14]]
+; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP16]], i32 [[TMP15]], i32 [[TMP14]]
+; SKX-NEXT:    [[TMP17:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
+; SKX-NEXT:    [[TMP18:%.*]] = icmp sgt i32 [[OP_EXTRA]], [[TMP17]]
+; SKX-NEXT:    [[TMP19:%.*]] = select i1 [[TMP18]], i32 [[OP_EXTRA]], i32 [[TMP17]]
+; SKX-NEXT:    [[TMP20:%.*]] = extractelement <2 x i1> [[TMP12]], i32 1
+; SKX-NEXT:    [[TMP21:%.*]] = select i1 [[TMP20]], i32 3, i32 4
+; SKX-NEXT:    store i32 [[TMP21]], i32* @var, align 8
+; SKX-NEXT:    ret i32 [[TMP19]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
   %3 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 1), align 4
@@ -854,33 +602,21 @@ define i32 @maxi8_wrong_parent(i32) {
 ; SSE:       pp:
 ; SSE-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; SSE-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; SSE-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; SSE-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; SSE-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; SSE-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; SSE-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; SSE-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; SSE-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; SSE-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; SSE-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; SSE-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; SSE-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; SSE-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; SSE-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; SSE-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
 ; SSE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; SSE-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; SSE-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; SSE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; SSE-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; SSE-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; SSE-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; SSE-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; SSE-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; SSE-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; SSE-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; SSE-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; SSE-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; SSE-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; SSE-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; SSE-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; SSE-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; SSE-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
+; SSE-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP13]], [[TMP5]]
+; SSE-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP5]]
 ; SSE-NEXT:    ret i32 [[OP_EXTRA]]
 ;
 ; AVX-LABEL: @maxi8_wrong_parent(
@@ -891,33 +627,21 @@ define i32 @maxi8_wrong_parent(i32) {
 ; AVX:       pp:
 ; AVX-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; AVX-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; AVX-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
 ; AVX-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; AVX-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; AVX-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; AVX-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; AVX-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; AVX-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; AVX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; AVX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; AVX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; AVX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; AVX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; AVX-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; AVX-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; AVX-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; AVX-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
+; AVX-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP13]], [[TMP5]]
+; AVX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP5]]
 ; AVX-NEXT:    ret i32 [[OP_EXTRA]]
 ;
 ; AVX2-LABEL: @maxi8_wrong_parent(
@@ -928,33 +652,21 @@ define i32 @maxi8_wrong_parent(i32) {
 ; AVX2:       pp:
 ; AVX2-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i32 [[TMP2]], i32 [[TMP3]]
 ; AVX2-NEXT:    [[TMP6:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 2) to <4 x i32>*), align 8
-; AVX2-NEXT:    [[TMP7:%.*]] = icmp sgt i32 [[TMP5]], undef
-; AVX2-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP5]], i32 undef
-; AVX2-NEXT:    [[TMP9:%.*]] = icmp sgt i32 [[TMP8]], undef
-; AVX2-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 undef
-; AVX2-NEXT:    [[TMP11:%.*]] = icmp sgt i32 [[TMP10]], undef
-; AVX2-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 [[TMP10]], i32 undef
-; AVX2-NEXT:    [[TMP13:%.*]] = icmp sgt i32 [[TMP12]], undef
-; AVX2-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP12]], i32 undef
-; AVX2-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
-; AVX2-NEXT:    [[TMP16:%.*]] = icmp sgt i32 [[TMP14]], [[TMP15]]
-; AVX2-NEXT:    [[TMP17:%.*]] = select i1 [[TMP16]], i32 [[TMP14]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
-; AVX2-NEXT:    [[TMP19:%.*]] = icmp sgt i32 [[TMP17]], [[TMP18]]
+; AVX2-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 6), align 8
+; AVX2-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 7), align 4
 ; AVX2-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; AVX2-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; AVX2-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP6]], <4 x i32> [[RDX_SHUF]]
 ; AVX2-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; AVX2-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
 ; AVX2-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
-; AVX2-NEXT:    [[TMP20:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
-; AVX2-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], [[TMP15]]
-; AVX2-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 [[TMP15]]
-; AVX2-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], [[TMP18]]
-; AVX2-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 [[TMP18]]
-; AVX2-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], [[TMP5]]
-; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 [[TMP5]]
-; AVX2-NEXT:    [[TMP26:%.*]] = select i1 [[TMP19]], i32 [[TMP17]], i32 [[TMP18]]
+; AVX2-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; AVX2-NEXT:    [[TMP10:%.*]] = icmp sgt i32 [[TMP9]], [[TMP7]]
+; AVX2-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i32 [[TMP9]], i32 [[TMP7]]
+; AVX2-NEXT:    [[TMP12:%.*]] = icmp sgt i32 [[TMP11]], [[TMP8]]
+; AVX2-NEXT:    [[TMP13:%.*]] = select i1 [[TMP12]], i32 [[TMP11]], i32 [[TMP8]]
+; AVX2-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP13]], [[TMP5]]
+; AVX2-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP14]], i32 [[TMP13]], i32 [[TMP5]]
 ; AVX2-NEXT:    ret i32 [[OP_EXTRA]]
 ;
 ; SKX-LABEL: @maxi8_wrong_parent(
@@ -985,21 +697,9 @@ define i32 @maxi8_wrong_parent(i32) {
 ; SKX-NEXT:    [[TMP18:%.*]] = insertelement <2 x i32> [[TMP17]], i32 [[TMP4]], i32 1
 ; SKX-NEXT:    [[TMP19:%.*]] = select <2 x i1> [[TMP14]], <2 x i32> [[TMP16]], <2 x i32> [[TMP18]]
 ; SKX-NEXT:    [[TMP20:%.*]] = extractelement <2 x i32> [[TMP19]], i32 1
-; SKX-NEXT:    [[TMP21:%.*]] = icmp sgt i32 [[TMP20]], undef
-; SKX-NEXT:    [[TMP22:%.*]] = select i1 [[TMP21]], i32 [[TMP20]], i32 undef
-; SKX-NEXT:    [[TMP23:%.*]] = icmp sgt i32 [[TMP22]], undef
-; SKX-NEXT:    [[TMP24:%.*]] = select i1 [[TMP23]], i32 [[TMP22]], i32 undef
-; SKX-NEXT:    [[TMP25:%.*]] = icmp sgt i32 [[TMP24]], undef
-; SKX-NEXT:    [[TMP26:%.*]] = select i1 [[TMP25]], i32 [[TMP24]], i32 undef
-; SKX-NEXT:    [[TMP27:%.*]] = icmp sgt i32 [[TMP26]], undef
-; SKX-NEXT:    [[TMP28:%.*]] = select i1 [[TMP27]], i32 [[TMP26]], i32 undef
-; SKX-NEXT:    [[TMP29:%.*]] = icmp sgt i32 [[TMP28]], [[TMP7]]
-; SKX-NEXT:    [[TMP30:%.*]] = select i1 [[TMP29]], i32 [[TMP28]], i32 [[TMP7]]
-; SKX-NEXT:    [[TMP31:%.*]] = icmp sgt i32 [[TMP30]], [[TMP8]]
-; SKX-NEXT:    [[TMP32:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
-; SKX-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP20]]
-; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP33]], i32 [[TMP32]], i32 [[TMP20]]
-; SKX-NEXT:    [[TMP34:%.*]] = select i1 [[TMP31]], i32 [[TMP30]], i32 [[TMP8]]
+; SKX-NEXT:    [[TMP21:%.*]] = extractelement <2 x i32> [[TMP19]], i32 0
+; SKX-NEXT:    [[TMP22:%.*]] = icmp sgt i32 [[TMP21]], [[TMP20]]
+; SKX-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP22]], i32 [[TMP21]], i32 [[TMP20]]
 ; SKX-NEXT:    ret i32 [[OP_EXTRA]]
 ;
   %2 = load i32, i32* getelementptr inbounds ([32 x i32], [32 x i32]* @arr, i64 0, i64 0), align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
index 311d8a476c71d..ed0e5784e3026 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal.ll
@@ -37,14 +37,11 @@ define i32 @add_red(float* %A, i32 %n) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
-; CHECK-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
 ; CHECK-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
@@ -77,14 +74,11 @@ define i32 @add_red(float* %A, i32 %n) {
 ; STORE-NEXT:    [[TMP1:%.*]] = bitcast float* [[ARRAYIDX]] to <4 x float>*
 ; STORE-NEXT:    [[TMP2:%.*]] = load <4 x float>, <4 x float>* [[TMP1]], align 4
 ; STORE-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[TMP2]], <float 7.000000e+00, float 7.000000e+00, float 7.000000e+00, float 7.000000e+00>
-; STORE-NEXT:    [[ADD6:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD11:%.*]] = fadd fast float [[ADD6]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP3]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP3]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP4:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD16:%.*]] = fadd fast float [[ADD11]], undef
 ; STORE-NEXT:    [[ADD17]] = fadd fast float [[SUM_032]], [[TMP4]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_033]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP0]]
@@ -178,14 +172,11 @@ define i32 @mul_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; CHECK-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
@@ -223,14 +214,11 @@ define i32 @mul_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul <4 x float> [[TMP1]], [[TMP4]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; STORE-NEXT:    [[MUL21]] = fmul float [[SUM_039]], [[TMP6]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_040]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
@@ -350,13 +338,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
-; CHECK-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; CHECK-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
-; CHECK-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
-; CHECK-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
-; CHECK-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
-; CHECK-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
-; CHECK-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
 ; CHECK-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
 ; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
@@ -369,7 +350,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
-; CHECK-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
 ; CHECK-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
@@ -421,13 +401,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP4:%.*]] = bitcast float* [[ARRAYIDX2]] to <8 x float>*
 ; STORE-NEXT:    [[TMP5:%.*]] = load <8 x float>, <8 x float>* [[TMP4]], align 4
 ; STORE-NEXT:    [[TMP6:%.*]] = fmul fast <8 x float> [[TMP1]], [[TMP5]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
-; STORE-NEXT:    [[ADD26:%.*]] = fadd fast float [[ADD20]], undef
-; STORE-NEXT:    [[ADD32:%.*]] = fadd fast float [[ADD26]], undef
-; STORE-NEXT:    [[ADD38:%.*]] = fadd fast float [[ADD32]], undef
-; STORE-NEXT:    [[ADD44:%.*]] = fadd fast float [[ADD38]], undef
 ; STORE-NEXT:    [[ADD47:%.*]] = add nsw i64 [[MUL]], 8
 ; STORE-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds float, float* [[A]], i64 [[ADD47]]
 ; STORE-NEXT:    [[TMP7:%.*]] = load float, float* [[ARRAYIDX48]], align 4
@@ -440,7 +413,6 @@ define i32 @long_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP8:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
 ; STORE-NEXT:    [[TMP9:%.*]] = fadd fast float [[TMP8]], [[MUL49]]
-; STORE-NEXT:    [[ADD50:%.*]] = fadd fast float [[ADD44]], [[MUL49]]
 ; STORE-NEXT:    [[ADD51]] = fadd fast float [[SUM_082]], [[TMP9]]
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_083]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP3]]
@@ -576,16 +548,12 @@ define i32 @chain_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; CHECK-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
-; CHECK-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
-; CHECK-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
-; CHECK-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
-; CHECK-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
 ; CHECK-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
@@ -622,16 +590,12 @@ define i32 @chain_red(float* noalias %A, float* noalias %B, i32 %n) {
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP1]], [[TMP4]]
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float [[SUM_042]], undef
-; STORE-NEXT:    [[ADD9:%.*]] = fadd fast float [[ADD]], undef
-; STORE-NEXT:    [[ADD15:%.*]] = fadd fast float [[ADD9]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
 ; STORE-NEXT:    [[OP_EXTRA]] = fadd fast float [[TMP6]], [[SUM_042]]
-; STORE-NEXT:    [[ADD21:%.*]] = fadd fast float [[ADD15]], undef
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_043]], 1
 ; STORE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INC]], [[TMP2]]
 ; STORE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]], label [[FOR_BODY]]
@@ -1087,14 +1051,11 @@ define i32 @store_red(float* noalias %A, float* noalias %B, float* noalias %C, i
 ; STORE-NEXT:    [[TMP3:%.*]] = bitcast float* [[ARRAYIDX2]] to <4 x float>*
 ; STORE-NEXT:    [[TMP4:%.*]] = load <4 x float>, <4 x float>* [[TMP3]], align 4
 ; STORE-NEXT:    [[TMP5:%.*]] = fmul fast <4 x float> [[TMP2]], [[TMP4]]
-; STORE-NEXT:    [[ADD8:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD14:%.*]] = fadd fast float [[ADD8]], undef
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP5]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP5]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP6:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD20:%.*]] = fadd fast float [[ADD14]], undef
 ; STORE-NEXT:    store float [[TMP6]], float* [[C_ADDR_038]], align 4
 ; STORE-NEXT:    [[INCDEC_PTR]] = getelementptr inbounds float, float* [[C_ADDR_038]], i64 1
 ; STORE-NEXT:    [[INC]] = add nsw i64 [[I_039]], 1
@@ -1169,14 +1130,11 @@ define void @float_red_example4(float* %res) {
 ; STORE-LABEL: @float_red_example4(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <4 x float>, <4 x float>* bitcast ([32 x float]* @arr_float to <4 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x float> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1216,12 +1174,6 @@ define void @float_red_example8(float* %res) {
 ; STORE-LABEL: @float_red_example8(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x float>, <8 x float>* bitcast ([32 x float]* @arr_float to <8 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <8 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x float> [[BIN_RDX]], <8 x float> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1229,7 +1181,6 @@ define void @float_red_example8(float* %res) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x float> [[BIN_RDX2]], <8 x float> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = fadd fast <8 x float> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x float> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1293,20 +1244,6 @@ define void @float_red_example16(float* %res) {
 ; STORE-LABEL: @float_red_example16(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <16 x float>, <16 x float>* bitcast ([32 x float]* @arr_float to <16 x float>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = fadd fast float undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = fadd fast float undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = fadd fast float undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = fadd fast float undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = fadd fast float undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = fadd fast float undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = fadd fast float undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = fadd fast float undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = fadd fast float undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = fadd fast float undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = fadd fast float undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = fadd fast float undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = fadd fast float undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = fadd fast float undef, [[ADD_12]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x float> [[TMP0]], <16 x float> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = fadd fast <16 x float> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x float> [[BIN_RDX]], <16 x float> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1316,7 +1253,6 @@ define void @float_red_example16(float* %res) {
 ; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x float> [[BIN_RDX4]], <16 x float> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX6:%.*]] = fadd fast <16 x float> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x float> [[BIN_RDX6]], i32 0
-; STORE-NEXT:    [[ADD_14:%.*]] = fadd fast float undef, [[ADD_13]]
 ; STORE-NEXT:    store float [[TMP1]], float* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1372,14 +1308,11 @@ define void @i32_red_example4(i32* %res) {
 ; STORE-LABEL: @i32_red_example4(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <4 x i32>, <4 x i32>* bitcast ([32 x i32]* @arr_i32 to <4 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <4 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1419,12 +1352,6 @@ define void @i32_red_example8(i32* %res) {
 ; STORE-LABEL: @i32_red_example8(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1432,7 +1359,6 @@ define void @i32_red_example8(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1496,20 +1422,6 @@ define void @i32_red_example16(i32* %res) {
 ; STORE-LABEL: @i32_red_example16(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <16 x i32>, <16 x i32>* bitcast ([32 x i32]* @arr_i32 to <16 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[TMP0]], <16 x i32> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <16 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> undef, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1519,7 +1431,6 @@ define void @i32_red_example16(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX6:%.*]] = add nsw <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
-; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1631,36 +1542,6 @@ define void @i32_red_example32(i32* %res) {
 ; STORE-LABEL: @i32_red_example32(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <32 x i32>, <32 x i32>* bitcast ([32 x i32]* @arr_i32 to <32 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
-; STORE-NEXT:    [[ADD_7:%.*]] = add nsw i32 undef, [[ADD_6]]
-; STORE-NEXT:    [[ADD_8:%.*]] = add nsw i32 undef, [[ADD_7]]
-; STORE-NEXT:    [[ADD_9:%.*]] = add nsw i32 undef, [[ADD_8]]
-; STORE-NEXT:    [[ADD_10:%.*]] = add nsw i32 undef, [[ADD_9]]
-; STORE-NEXT:    [[ADD_11:%.*]] = add nsw i32 undef, [[ADD_10]]
-; STORE-NEXT:    [[ADD_12:%.*]] = add nsw i32 undef, [[ADD_11]]
-; STORE-NEXT:    [[ADD_13:%.*]] = add nsw i32 undef, [[ADD_12]]
-; STORE-NEXT:    [[ADD_14:%.*]] = add nsw i32 undef, [[ADD_13]]
-; STORE-NEXT:    [[ADD_15:%.*]] = add nsw i32 undef, [[ADD_14]]
-; STORE-NEXT:    [[ADD_16:%.*]] = add nsw i32 undef, [[ADD_15]]
-; STORE-NEXT:    [[ADD_17:%.*]] = add nsw i32 undef, [[ADD_16]]
-; STORE-NEXT:    [[ADD_18:%.*]] = add nsw i32 undef, [[ADD_17]]
-; STORE-NEXT:    [[ADD_19:%.*]] = add nsw i32 undef, [[ADD_18]]
-; STORE-NEXT:    [[ADD_20:%.*]] = add nsw i32 undef, [[ADD_19]]
-; STORE-NEXT:    [[ADD_21:%.*]] = add nsw i32 undef, [[ADD_20]]
-; STORE-NEXT:    [[ADD_22:%.*]] = add nsw i32 undef, [[ADD_21]]
-; STORE-NEXT:    [[ADD_23:%.*]] = add nsw i32 undef, [[ADD_22]]
-; STORE-NEXT:    [[ADD_24:%.*]] = add nsw i32 undef, [[ADD_23]]
-; STORE-NEXT:    [[ADD_25:%.*]] = add nsw i32 undef, [[ADD_24]]
-; STORE-NEXT:    [[ADD_26:%.*]] = add nsw i32 undef, [[ADD_25]]
-; STORE-NEXT:    [[ADD_27:%.*]] = add nsw i32 undef, [[ADD_26]]
-; STORE-NEXT:    [[ADD_28:%.*]] = add nsw i32 undef, [[ADD_27]]
-; STORE-NEXT:    [[ADD_29:%.*]] = add nsw i32 undef, [[ADD_28]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i32> [[TMP0]], <32 x i32> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <32 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i32> [[BIN_RDX]], <32 x i32> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1672,7 +1553,6 @@ define void @i32_red_example32(i32* %res) {
 ; STORE-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i32> [[BIN_RDX6]], <32 x i32> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX8:%.*]] = add nsw <32 x i32> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <32 x i32> [[BIN_RDX8]], i32 0
-; STORE-NEXT:    [[ADD_30:%.*]] = add nsw i32 undef, [[ADD_29]]
 ; STORE-NEXT:    store i32 [[TMP1]], i32* [[RES:%.*]], align 16
 ; STORE-NEXT:    ret void
 ;
@@ -1750,12 +1630,6 @@ define void @i32_red_call(i32 %val) {
 ; CHECK-LABEL: @i32_red_call(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1763,19 +1637,12 @@ define void @i32_red_call(i32 %val) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
 ; CHECK-NEXT:    ret void
 ;
 ; STORE-LABEL: @i32_red_call(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1783,7 +1650,6 @@ define void @i32_red_call(i32 %val) {
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    [[RES:%.*]] = call i32 @foobar(i32 [[TMP1]])
 ; STORE-NEXT:    ret void
 ;
@@ -1811,12 +1677,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; CHECK-LABEL: @i32_red_invoke(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; CHECK-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1824,7 +1684,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
 ; CHECK-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
 ; CHECK:       exception:
@@ -1837,12 +1696,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; STORE-LABEL: @i32_red_invoke(
 ; STORE-NEXT:  entry:
 ; STORE-NEXT:    [[TMP0:%.*]] = load <8 x i32>, <8 x i32>* bitcast ([32 x i32]* @arr_i32 to <8 x i32>*), align 16
-; STORE-NEXT:    [[ADD:%.*]] = add nsw i32 undef, undef
-; STORE-NEXT:    [[ADD_1:%.*]] = add nsw i32 undef, [[ADD]]
-; STORE-NEXT:    [[ADD_2:%.*]] = add nsw i32 undef, [[ADD_1]]
-; STORE-NEXT:    [[ADD_3:%.*]] = add nsw i32 undef, [[ADD_2]]
-; STORE-NEXT:    [[ADD_4:%.*]] = add nsw i32 undef, [[ADD_3]]
-; STORE-NEXT:    [[ADD_5:%.*]] = add nsw i32 undef, [[ADD_4]]
 ; STORE-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP0]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX:%.*]] = add nsw <8 x i32> [[TMP0]], [[RDX_SHUF]]
 ; STORE-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1850,7 +1703,6 @@ define void @i32_red_invoke(i32 %val) personality i32 (...)* @__gxx_personality_
 ; STORE-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; STORE-NEXT:    [[BIN_RDX4:%.*]] = add nsw <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; STORE-NEXT:    [[TMP1:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; STORE-NEXT:    [[ADD_6:%.*]] = add nsw i32 undef, [[ADD_5]]
 ; STORE-NEXT:    [[RES:%.*]] = invoke i32 @foobar(i32 [[TMP1]])
 ; STORE-NEXT:    to label [[NORMAL:%.*]] unwind label [[EXCEPTION:%.*]]
 ; STORE:       exception:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
index ffbdd9f1d148d..52c1c6bee1216 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/long_chains.ll
@@ -12,10 +12,10 @@ define i32 @test(double* nocapture %A, i8* nocapture %B) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i8* [[B:%.*]] to <2 x i8>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <2 x i8>, <2 x i8>* [[TMP0]], align 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i8> [[TMP1]], <i8 3, i8 3>
-; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP4:%.*]] = insertelement <2 x i8> undef, i8 [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
-; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP4]], i8 [[TMP5]], i32 1
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i8> [[TMP2]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i8> [[TMP2]], i32 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <2 x i8> undef, i8 [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP6:%.*]] = insertelement <2 x i8> [[TMP5]], i8 [[TMP3]], i32 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = sitofp <2 x i8> [[TMP6]] to <2 x double>
 ; CHECK-NEXT:    [[TMP8:%.*]] = fmul <2 x double> [[TMP7]], [[TMP7]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = fadd <2 x double> [[TMP8]], <double 1.000000e+00, double 1.000000e+00>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
index 13bbe5e579812..0b8c72b35a2a7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassociated-loads.ll
@@ -5,36 +5,6 @@ define signext i8 @Foo(<32 x i8>* %__v) {
 ; CHECK-LABEL: @Foo(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = load <32 x i8>, <32 x i8>* [[__V:%.*]], align 32
-; CHECK-NEXT:    [[ADD_I_1_I:%.*]] = add i8 undef, undef
-; CHECK-NEXT:    [[ADD_I_2_I:%.*]] = add i8 [[ADD_I_1_I]], undef
-; CHECK-NEXT:    [[ADD_I_3_I:%.*]] = add i8 [[ADD_I_2_I]], undef
-; CHECK-NEXT:    [[ADD_I_4_I:%.*]] = add i8 [[ADD_I_3_I]], undef
-; CHECK-NEXT:    [[ADD_I_5_I:%.*]] = add i8 [[ADD_I_4_I]], undef
-; CHECK-NEXT:    [[ADD_I_6_I:%.*]] = add i8 [[ADD_I_5_I]], undef
-; CHECK-NEXT:    [[ADD_I_7_I:%.*]] = add i8 [[ADD_I_6_I]], undef
-; CHECK-NEXT:    [[ADD_I_8_I:%.*]] = add i8 [[ADD_I_7_I]], undef
-; CHECK-NEXT:    [[ADD_I_9_I:%.*]] = add i8 [[ADD_I_8_I]], undef
-; CHECK-NEXT:    [[ADD_I_10_I:%.*]] = add i8 [[ADD_I_9_I]], undef
-; CHECK-NEXT:    [[ADD_I_11_I:%.*]] = add i8 [[ADD_I_10_I]], undef
-; CHECK-NEXT:    [[ADD_I_12_I:%.*]] = add i8 [[ADD_I_11_I]], undef
-; CHECK-NEXT:    [[ADD_I_13_I:%.*]] = add i8 [[ADD_I_12_I]], undef
-; CHECK-NEXT:    [[ADD_I_14_I:%.*]] = add i8 [[ADD_I_13_I]], undef
-; CHECK-NEXT:    [[ADD_I_15_I:%.*]] = add i8 [[ADD_I_14_I]], undef
-; CHECK-NEXT:    [[ADD_I_16_I:%.*]] = add i8 [[ADD_I_15_I]], undef
-; CHECK-NEXT:    [[ADD_I_17_I:%.*]] = add i8 [[ADD_I_16_I]], undef
-; CHECK-NEXT:    [[ADD_I_18_I:%.*]] = add i8 [[ADD_I_17_I]], undef
-; CHECK-NEXT:    [[ADD_I_19_I:%.*]] = add i8 [[ADD_I_18_I]], undef
-; CHECK-NEXT:    [[ADD_I_20_I:%.*]] = add i8 [[ADD_I_19_I]], undef
-; CHECK-NEXT:    [[ADD_I_21_I:%.*]] = add i8 [[ADD_I_20_I]], undef
-; CHECK-NEXT:    [[ADD_I_22_I:%.*]] = add i8 [[ADD_I_21_I]], undef
-; CHECK-NEXT:    [[ADD_I_23_I:%.*]] = add i8 [[ADD_I_22_I]], undef
-; CHECK-NEXT:    [[ADD_I_24_I:%.*]] = add i8 [[ADD_I_23_I]], undef
-; CHECK-NEXT:    [[ADD_I_25_I:%.*]] = add i8 [[ADD_I_24_I]], undef
-; CHECK-NEXT:    [[ADD_I_26_I:%.*]] = add i8 [[ADD_I_25_I]], undef
-; CHECK-NEXT:    [[ADD_I_27_I:%.*]] = add i8 [[ADD_I_26_I]], undef
-; CHECK-NEXT:    [[ADD_I_28_I:%.*]] = add i8 [[ADD_I_27_I]], undef
-; CHECK-NEXT:    [[ADD_I_29_I:%.*]] = add i8 [[ADD_I_28_I]], undef
-; CHECK-NEXT:    [[ADD_I_30_I:%.*]] = add i8 [[ADD_I_29_I]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <32 x i8> [[TMP0]], <32 x i8> undef, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <32 x i8> [[TMP0]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <32 x i8> [[BIN_RDX]], <32 x i8> undef, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -46,7 +16,6 @@ define signext i8 @Foo(<32 x i8>* %__v) {
 ; CHECK-NEXT:    [[RDX_SHUF7:%.*]] = shufflevector <32 x i8> [[BIN_RDX6]], <32 x i8> undef, <32 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX8:%.*]] = add <32 x i8> [[BIN_RDX6]], [[RDX_SHUF7]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <32 x i8> [[BIN_RDX8]], i32 0
-; CHECK-NEXT:    [[ADD_I_31_I:%.*]] = add i8 [[ADD_I_30_I]], undef
 ; CHECK-NEXT:    ret i8 [[TMP1]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
index 56539ab928eab..15ede8156fa5b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_loads.ll
@@ -35,13 +35,6 @@ define i32 @test(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = mul <8 x i32> [[TMP1]], <i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42, i32 42>
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP2]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP2]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -50,7 +43,6 @@ define i32 @test(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP3]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
@@ -138,13 +130,6 @@ define i32 @test2(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[Q]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -153,7 +138,6 @@ define i32 @test2(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP5]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
@@ -258,13 +242,6 @@ define i32 @test3(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = bitcast i32* [[Q]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP3:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul <8 x i32> [[REORDER_SHUFFLE]], [[TMP3]]
-; CHECK-NEXT:    [[ADD:%.*]] = add i32 undef, [[SUM]]
-; CHECK-NEXT:    [[ADD_1:%.*]] = add i32 undef, [[ADD]]
-; CHECK-NEXT:    [[ADD_2:%.*]] = add i32 undef, [[ADD_1]]
-; CHECK-NEXT:    [[ADD_3:%.*]] = add i32 undef, [[ADD_2]]
-; CHECK-NEXT:    [[ADD_4:%.*]] = add i32 undef, [[ADD_3]]
-; CHECK-NEXT:    [[ADD_5:%.*]] = add i32 undef, [[ADD_4]]
-; CHECK-NEXT:    [[ADD_6:%.*]] = add i32 undef, [[ADD_5]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP4]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -273,7 +250,6 @@ define i32 @test3(i32* nocapture readonly %p, i32* nocapture readonly %q) {
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add i32 [[TMP5]], [[SUM]]
-; CHECK-NEXT:    [[ADD_7:%.*]] = add i32 undef, [[ADD_6]]
 ; CHECK-NEXT:    br i1 true, label [[FOR_END:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.end:
 ; CHECK-NEXT:    ret i32 [[OP_EXTRA]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
index 4dd40876a70a8..7fdc953922357 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction_unrolled.ll
@@ -26,12 +26,6 @@ define i32 @test_add(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = add i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = add i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = add i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = add i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = add i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = add i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = add <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -39,7 +33,6 @@ define i32 @test_add(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = add i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -147,12 +140,6 @@ define i32 @test_and(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = and i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = and i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = and i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = and i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = and i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = and i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = and <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -160,7 +147,6 @@ define i32 @test_and(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = and <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = and i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -208,12 +194,6 @@ define i32 @test_or(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = or i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = or i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = or i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = or i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = or i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = or i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = or <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -221,7 +201,6 @@ define i32 @test_or(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = or <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = or i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -269,12 +248,6 @@ define i32 @test_xor(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, i32* [[P]], i64 7
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[P]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[MUL_18:%.*]] = xor i32 undef, undef
-; CHECK-NEXT:    [[MUL_29:%.*]] = xor i32 undef, [[MUL_18]]
-; CHECK-NEXT:    [[MUL_310:%.*]] = xor i32 undef, [[MUL_29]]
-; CHECK-NEXT:    [[MUL_411:%.*]] = xor i32 undef, [[MUL_310]]
-; CHECK-NEXT:    [[MUL_512:%.*]] = xor i32 undef, [[MUL_411]]
-; CHECK-NEXT:    [[MUL_613:%.*]] = xor i32 undef, [[MUL_512]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> undef, <8 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -282,7 +255,6 @@ define i32 @test_xor(i32* nocapture readonly %p) {
 ; CHECK-NEXT:    [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> undef, <8 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX4:%.*]] = xor <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
-; CHECK-NEXT:    [[MUL_714:%.*]] = xor i32 undef, [[MUL_613]]
 ; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
@@ -322,15 +294,12 @@ define i32 @PR37731(<4 x i32>* noalias nocapture dereferenceable(16) %self) unna
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl <4 x i32> [[TMP4]], <i32 18, i32 2, i32 7, i32 13>
 ; CHECK-NEXT:    [[TMP6:%.*]] = xor <4 x i32> [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    store <4 x i32> [[TMP6]], <4 x i32>* [[SELF]], align 16
-; CHECK-NEXT:    [[TMP7:%.*]] = xor i32 undef, undef
-; CHECK-NEXT:    [[TMP8:%.*]] = xor i32 [[TMP7]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP6]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX:%.*]] = xor <4 x i32> [[TMP6]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = xor <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
-; CHECK-NEXT:    [[TMP10:%.*]] = xor i32 [[TMP8]], undef
-; CHECK-NEXT:    ret i32 [[TMP9]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
+; CHECK-NEXT:    ret i32 [[TMP7]]
 ;
 entry:
   %0 = load <4 x i32>, <4 x i32>* %self, align 16
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
index 27997f6af3d73..4a69a35420b4b 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/remark_horcost.ll
@@ -33,11 +33,8 @@ define i32 @foo(i32* %diff) #0 {
 ; CHECK-NEXT:    [[TMP11:%.*]] = bitcast i32* [[ARRAYIDX2]] to <4 x i32>*
 ; CHECK-NEXT:    [[TMP12:%.*]] = load <4 x i32>, <4 x i32>* [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = add nsw <4 x i32> [[TMP12]], [[TMP9]]
-; CHECK-NEXT:    [[ADD10:%.*]] = add nsw i32 undef, [[A_088]]
 ; CHECK-NEXT:    [[ARRAYIDX20:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 1
-; CHECK-NEXT:    [[ADD24:%.*]] = add nsw i32 [[ADD10]], undef
 ; CHECK-NEXT:    [[ARRAYIDX34:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 2
-; CHECK-NEXT:    [[ADD38:%.*]] = add nsw i32 [[ADD24]], undef
 ; CHECK-NEXT:    [[ARRAYIDX48:%.*]] = getelementptr inbounds [8 x [8 x i32]], [8 x [8 x i32]]* [[M2]], i64 0, i64 [[INDVARS_IV]], i64 3
 ; CHECK-NEXT:    [[TMP14:%.*]] = bitcast i32* [[ARRAYIDX6]] to <4 x i32>*
 ; CHECK-NEXT:    store <4 x i32> [[TMP13]], <4 x i32>* [[TMP14]], align 16
@@ -47,7 +44,6 @@ define i32 @foo(i32* %diff) #0 {
 ; CHECK-NEXT:    [[BIN_RDX2:%.*]] = add nsw <4 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <4 x i32> [[BIN_RDX2]], i32 0
 ; CHECK-NEXT:    [[OP_EXTRA]] = add nsw i32 [[TMP15]], [[A_088]]
-; CHECK-NEXT:    [[ADD52:%.*]] = add nsw i32 [[ADD38]], undef
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 8
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]]
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
index 13884efd98dd8..768cba9c37bb6 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reorder_repeated_ops.ll
@@ -19,11 +19,6 @@ define void @hoge() {
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub <2 x i32> [[TMP3]], undef
 ; CHECK-NEXT:    [[SHUFFLE8:%.*]] = shufflevector <2 x i32> [[TMP4]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP5:%.*]] = add <4 x i32> [[SHUFFLE8]], <i32 undef, i32 15, i32 31, i32 47>
-; CHECK-NEXT:    [[TMP11:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP14:%.*]] = icmp sgt i32 [[TMP12]], undef
-; CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP12]], i32 undef
-; CHECK-NEXT:    [[TMP17:%.*]] = icmp sgt i32 [[TMP15]], undef
 ; CHECK-NEXT:    [[RDX_SHUF9:%.*]] = shufflevector <4 x i32> [[TMP5]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP10:%.*]] = icmp sgt <4 x i32> [[TMP5]], [[RDX_SHUF9]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT11:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP10]], <4 x i32> [[TMP5]], <4 x i32> [[RDX_SHUF9]]
@@ -31,28 +26,12 @@ define void @hoge() {
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP13:%.*]] = icmp sgt <4 x i32> [[RDX_MINMAX_SELECT11]], [[RDX_SHUF12]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT14:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP13]], <4 x i32> [[RDX_MINMAX_SELECT11]], <4 x i32> [[RDX_SHUF12]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT14]], i32 0
-; CHECK-NEXT:    [[TMP18:%.*]] = select i1 [[TMP17]], i32 [[TMP15]], i32 undef
 ; CHECK-NEXT:    [[TMP19:%.*]] = select i1 undef, i32 [[TMP6]], i32 undef
 ; CHECK-NEXT:    [[TMP20:%.*]] = icmp sgt i32 [[TMP19]], 63
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub nsw <2 x i32> undef, [[TMP2]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = sub <2 x i32> [[TMP7]], undef
 ; CHECK-NEXT:    [[SHUFFLE:%.*]] = shufflevector <2 x i32> [[TMP8]], <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
 ; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <4 x i32> [[SHUFFLE]], <i32 -49, i32 -33, i32 -33, i32 -17>
-; CHECK-NEXT:    [[TMP26:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP27:%.*]] = select i1 [[TMP26]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP28:%.*]] = icmp sgt i32 [[TMP27]], undef
-; CHECK-NEXT:    [[TMP29:%.*]] = select i1 [[TMP28]], i32 undef, i32 [[TMP27]]
-; CHECK-NEXT:    [[TMP31:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP33:%.*]] = icmp sgt i32 [[TMP32]], [[TMP29]]
-; CHECK-NEXT:    [[TMP34:%.*]] = select i1 [[TMP33]], i32 [[TMP29]], i32 [[TMP32]]
-; CHECK-NEXT:    [[TMP36:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP37:%.*]] = select i1 [[TMP36]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP38:%.*]] = icmp sgt i32 [[TMP37]], [[TMP34]]
-; CHECK-NEXT:    [[TMP39:%.*]] = select i1 [[TMP38]], i32 [[TMP34]], i32 [[TMP37]]
-; CHECK-NEXT:    [[TMP41:%.*]] = icmp sgt i32 undef, undef
-; CHECK-NEXT:    [[TMP42:%.*]] = select i1 [[TMP41]], i32 undef, i32 undef
-; CHECK-NEXT:    [[TMP43:%.*]] = icmp sgt i32 [[TMP42]], [[TMP39]]
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP9]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP9]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP9]], <4 x i32> [[RDX_SHUF]]
@@ -70,7 +49,6 @@ define void @hoge() {
 ; CHECK-NEXT:    [[OP_EXTRA6:%.*]] = select i1 [[TMP14]], i32 [[OP_EXTRA5]], i32 undef
 ; CHECK-NEXT:    [[TMP15:%.*]] = icmp slt i32 [[OP_EXTRA6]], undef
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP15]], i32 [[OP_EXTRA6]], i32 undef
-; CHECK-NEXT:    [[TMP44:%.*]] = select i1 [[TMP43]], i32 [[TMP39]], i32 [[TMP42]]
 ; CHECK-NEXT:    [[TMP45:%.*]] = icmp sgt i32 undef, [[OP_EXTRA7]]
 ; CHECK-NEXT:    unreachable
 ;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
index 7ecd5805f9313..822c362ec5294 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/undef_vect.ll
@@ -16,15 +16,6 @@ define void @_Z2azv() local_unnamed_addr {
 ; CHECK-NEXT:    [[DOTSROA_RAW_IDX_7:%.*]] = getelementptr inbounds %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76", %"struct.std::h.0.4.8.12.16.20.24.28.248.0.1.2.3.76"* undef, i64 7, i32 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = bitcast i32* [[DOTSROA_CAST_4]] to <8 x i32>*
 ; CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i32>, <8 x i32>* [[TMP0]], align 4
-; CHECK-NEXT:    [[CMP_I1_4:%.*]] = icmp slt i32 undef, undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_4:%.*]] = select i1 [[CMP_I1_4]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP_I1_5:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_4]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_5:%.*]] = select i1 [[CMP_I1_5]], i32 undef, i32 [[DOTSROA_SPECULATED_4]]
-; CHECK-NEXT:    [[CMP_I1_6:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_5]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_6:%.*]] = select i1 [[CMP_I1_6]], i32 undef, i32 [[DOTSROA_SPECULATED_5]]
-; CHECK-NEXT:    [[CMP_I1_7:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_6]], undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_7:%.*]] = select i1 [[CMP_I1_7]], i32 undef, i32 [[DOTSROA_SPECULATED_6]]
-; CHECK-NEXT:    [[CMP_I1_8:%.*]] = icmp slt i32 undef, undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp sgt <8 x i32> [[TMP1]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP1]], <8 x i32> [[RDX_SHUF]]
@@ -39,7 +30,6 @@ define void @_Z2azv() local_unnamed_addr {
 ; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP3]], i32 [[TMP2]], i32 undef
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp sgt i32 [[OP_EXTRA]], undef
 ; CHECK-NEXT:    [[OP_EXTRA7:%.*]] = select i1 [[TMP4]], i32 [[OP_EXTRA]], i32 undef
-; CHECK-NEXT:    [[DOTSROA_SPECULATED_8:%.*]] = select i1 [[CMP_I1_8]], i32 undef, i32 undef
 ; CHECK-NEXT:    [[DOTSROA_SPECULATED_9:%.*]] = select i1 undef, i32 undef, i32 [[OP_EXTRA7]]
 ; CHECK-NEXT:    [[CMP_I1_10:%.*]] = icmp slt i32 [[DOTSROA_SPECULATED_9]], undef
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
new file mode 100644
index 0000000000000..a8752bcff3e6d
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/used-reduced-op.ll
@@ -0,0 +1,529 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -slp-vectorizer -S -o - -mtriple=x86_64-unknown-linux-gnu -mcpu=haswell < %s | FileCheck %s
+
+@k = external dso_local constant [8 x [4 x i32]], align 16
+@l = external dso_local global [366 x i32], align 16
+
+; Function Attrs: nofree norecurse noreturn nounwind writeonly
+define void @n() local_unnamed_addr #0 {
+; CHECK-LABEL: @n(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 0), align 16
+; CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* bitcast (i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 1) to <4 x i32>*), align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 1), align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 2), align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 3), align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 0), align 16
+; CHECK-NEXT:    [[TMP6:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 1), align 4
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 2), align 8
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 3), align 4
+; CHECK-NEXT:    [[TMP9:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 0), align 16
+; CHECK-NEXT:    [[TMP10:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 1), align 4
+; CHECK-NEXT:    [[TMP11:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 2), align 8
+; CHECK-NEXT:    [[TMP12:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 3), align 4
+; CHECK-NEXT:    [[TMP13:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 0), align 16
+; CHECK-NEXT:    [[TMP14:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 1), align 4
+; CHECK-NEXT:    [[TMP15:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 2), align 8
+; CHECK-NEXT:    [[TMP16:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 3), align 4
+; CHECK-NEXT:    [[TMP17:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 0), align 16
+; CHECK-NEXT:    [[TMP18:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 1), align 4
+; CHECK-NEXT:    [[TMP19:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 2), align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 3), align 4
+; CHECK-NEXT:    [[TMP21:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 0), align 16
+; CHECK-NEXT:    [[TMP22:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 1), align 4
+; CHECK-NEXT:    [[TMP23:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 2), align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 3), align 4
+; CHECK-NEXT:    [[TMP25:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 0), align 16
+; CHECK-NEXT:    [[TMP26:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 1), align 4
+; CHECK-NEXT:    [[TMP27:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 2), align 8
+; CHECK-NEXT:    [[TMP28:%.*]] = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 3), align 4
+; CHECK-NEXT:    br label [[FOR_COND:%.*]]
+; CHECK:       for.cond:
+; CHECK-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], [[FOR_COND]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[B_0:%.*]] = phi i32 [ [[SPEC_SELECT8_3_7:%.*]], [[FOR_COND]] ], [ undef, [[ENTRY]] ]
+; CHECK-NEXT:    [[TMP29:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT:    [[TMP30:%.*]] = add i32 [[TMP29]], -183
+; CHECK-NEXT:    [[SUB:%.*]] = sub i32 [[TMP30]], [[TMP0]]
+; CHECK-NEXT:    [[TMP31:%.*]] = icmp slt i32 [[SUB]], 0
+; CHECK-NEXT:    [[NEG:%.*]] = sub nsw i32 0, [[SUB]]
+; CHECK-NEXT:    [[TMP32:%.*]] = select i1 [[TMP31]], i32 [[NEG]], i32 [[SUB]]
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <4 x i32> undef, i32 [[TMP30]], i32 0
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <4 x i32> [[TMP33]], i32 [[TMP30]], i32 1
+; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <4 x i32> [[TMP34]], i32 [[TMP30]], i32 2
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <4 x i32> [[TMP35]], i32 [[TMP30]], i32 3
+; CHECK-NEXT:    [[TMP37:%.*]] = sub <4 x i32> [[TMP36]], [[TMP1]]
+; CHECK-NEXT:    [[TMP38:%.*]] = icmp slt <4 x i32> [[TMP37]], zeroinitializer
+; CHECK-NEXT:    [[TMP39:%.*]] = sub nsw <4 x i32> zeroinitializer, [[TMP37]]
+; CHECK-NEXT:    [[TMP40:%.*]] = select <4 x i1> [[TMP38]], <4 x i32> [[TMP39]], <4 x i32> [[TMP37]]
+; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP40]], <4 x i32> undef, <4 x i32> <i32 2, i32 3, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp slt <4 x i32> [[TMP40]], [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP]], <4 x i32> [[TMP40]], <4 x i32> [[RDX_SHUF]]
+; CHECK-NEXT:    [[RDX_SHUF1:%.*]] = shufflevector <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> undef, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+; CHECK-NEXT:    [[RDX_MINMAX_CMP2:%.*]] = icmp slt <4 x i32> [[RDX_MINMAX_SELECT]], [[RDX_SHUF1]]
+; CHECK-NEXT:    [[RDX_MINMAX_SELECT3:%.*]] = select <4 x i1> [[RDX_MINMAX_CMP2]], <4 x i32> [[RDX_MINMAX_SELECT]], <4 x i32> [[RDX_SHUF1]]
+; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <4 x i32> [[RDX_MINMAX_SELECT3]], i32 0
+; CHECK-NEXT:    [[TMP42:%.*]] = icmp slt i32 [[TMP41]], [[TMP32]]
+; CHECK-NEXT:    [[TMP43:%.*]] = select i1 [[TMP42]], i32 [[TMP41]], i32 [[TMP32]]
+; CHECK-NEXT:    [[TMP44:%.*]] = icmp slt i32 [[TMP43]], [[B_0]]
+; CHECK-NEXT:    [[OP_EXTRA:%.*]] = select i1 [[TMP44]], i32 [[TMP43]], i32 [[B_0]]
+; CHECK-NEXT:    [[SUB_1_1:%.*]] = sub i32 [[TMP30]], [[TMP2]]
+; CHECK-NEXT:    [[TMP45:%.*]] = icmp slt i32 [[SUB_1_1]], 0
+; CHECK-NEXT:    [[NEG_1_1:%.*]] = sub nsw i32 0, [[SUB_1_1]]
+; CHECK-NEXT:    [[TMP46:%.*]] = select i1 [[TMP45]], i32 [[NEG_1_1]], i32 [[SUB_1_1]]
+; CHECK-NEXT:    [[CMP12_1_1:%.*]] = icmp slt i32 [[TMP46]], [[OP_EXTRA]]
+; CHECK-NEXT:    [[NARROW:%.*]] = or i1 [[CMP12_1_1]], undef
+; CHECK-NEXT:    [[SPEC_SELECT8_1_1:%.*]] = select i1 [[CMP12_1_1]], i32 [[TMP46]], i32 [[OP_EXTRA]]
+; CHECK-NEXT:    [[SUB_2_1:%.*]] = sub i32 [[TMP30]], [[TMP3]]
+; CHECK-NEXT:    [[TMP47:%.*]] = icmp slt i32 [[SUB_2_1]], 0
+; CHECK-NEXT:    [[NEG_2_1:%.*]] = sub nsw i32 0, [[SUB_2_1]]
+; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i32 [[NEG_2_1]], i32 [[SUB_2_1]]
+; CHECK-NEXT:    [[CMP12_2_1:%.*]] = icmp slt i32 [[TMP48]], [[SPEC_SELECT8_1_1]]
+; CHECK-NEXT:    [[NARROW34:%.*]] = or i1 [[CMP12_2_1]], [[NARROW]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_1:%.*]] = select i1 [[CMP12_2_1]], i32 [[TMP48]], i32 [[SPEC_SELECT8_1_1]]
+; CHECK-NEXT:    [[SUB_3_1:%.*]] = sub i32 [[TMP30]], [[TMP4]]
+; CHECK-NEXT:    [[TMP49:%.*]] = icmp slt i32 [[SUB_3_1]], 0
+; CHECK-NEXT:    [[NEG_3_1:%.*]] = sub nsw i32 0, [[SUB_3_1]]
+; CHECK-NEXT:    [[TMP50:%.*]] = select i1 [[TMP49]], i32 [[NEG_3_1]], i32 [[SUB_3_1]]
+; CHECK-NEXT:    [[CMP12_3_1:%.*]] = icmp slt i32 [[TMP50]], [[SPEC_SELECT8_2_1]]
+; CHECK-NEXT:    [[NARROW35:%.*]] = or i1 [[CMP12_3_1]], [[NARROW34]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_1:%.*]] = zext i1 [[NARROW35]] to i32
+; CHECK-NEXT:    [[SPEC_SELECT8_3_1:%.*]] = select i1 [[CMP12_3_1]], i32 [[TMP50]], i32 [[SPEC_SELECT8_2_1]]
+; CHECK-NEXT:    [[SUB_222:%.*]] = sub i32 [[TMP30]], [[TMP5]]
+; CHECK-NEXT:    [[TMP51:%.*]] = icmp slt i32 [[SUB_222]], 0
+; CHECK-NEXT:    [[NEG_223:%.*]] = sub nsw i32 0, [[SUB_222]]
+; CHECK-NEXT:    [[TMP52:%.*]] = select i1 [[TMP51]], i32 [[NEG_223]], i32 [[SUB_222]]
+; CHECK-NEXT:    [[CMP12_224:%.*]] = icmp slt i32 [[TMP52]], [[SPEC_SELECT8_3_1]]
+; CHECK-NEXT:    [[SPEC_SELECT8_226:%.*]] = select i1 [[CMP12_224]], i32 [[TMP52]], i32 [[SPEC_SELECT8_3_1]]
+; CHECK-NEXT:    [[SUB_1_2:%.*]] = sub i32 [[TMP30]], [[TMP6]]
+; CHECK-NEXT:    [[TMP53:%.*]] = icmp slt i32 [[SUB_1_2]], 0
+; CHECK-NEXT:    [[NEG_1_2:%.*]] = sub nsw i32 0, [[SUB_1_2]]
+; CHECK-NEXT:    [[TMP54:%.*]] = select i1 [[TMP53]], i32 [[NEG_1_2]], i32 [[SUB_1_2]]
+; CHECK-NEXT:    [[CMP12_1_2:%.*]] = icmp slt i32 [[TMP54]], [[SPEC_SELECT8_226]]
+; CHECK-NEXT:    [[TMP55:%.*]] = or i1 [[CMP12_1_2]], [[CMP12_224]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_2:%.*]] = select i1 [[CMP12_1_2]], i32 [[TMP54]], i32 [[SPEC_SELECT8_226]]
+; CHECK-NEXT:    [[SUB_2_2:%.*]] = sub i32 [[TMP30]], [[TMP7]]
+; CHECK-NEXT:    [[TMP56:%.*]] = icmp slt i32 [[SUB_2_2]], 0
+; CHECK-NEXT:    [[NEG_2_2:%.*]] = sub nsw i32 0, [[SUB_2_2]]
+; CHECK-NEXT:    [[TMP57:%.*]] = select i1 [[TMP56]], i32 [[NEG_2_2]], i32 [[SUB_2_2]]
+; CHECK-NEXT:    [[CMP12_2_2:%.*]] = icmp slt i32 [[TMP57]], [[SPEC_SELECT8_1_2]]
+; CHECK-NEXT:    [[TMP58:%.*]] = or i1 [[CMP12_2_2]], [[TMP55]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_2:%.*]] = select i1 [[CMP12_2_2]], i32 [[TMP57]], i32 [[SPEC_SELECT8_1_2]]
+; CHECK-NEXT:    [[SUB_3_2:%.*]] = sub i32 [[TMP30]], [[TMP8]]
+; CHECK-NEXT:    [[TMP59:%.*]] = icmp slt i32 [[SUB_3_2]], 0
+; CHECK-NEXT:    [[NEG_3_2:%.*]] = sub nsw i32 0, [[SUB_3_2]]
+; CHECK-NEXT:    [[TMP60:%.*]] = select i1 [[TMP59]], i32 [[NEG_3_2]], i32 [[SUB_3_2]]
+; CHECK-NEXT:    [[CMP12_3_2:%.*]] = icmp slt i32 [[TMP60]], [[SPEC_SELECT8_2_2]]
+; CHECK-NEXT:    [[TMP61:%.*]] = or i1 [[CMP12_3_2]], [[TMP58]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_2:%.*]] = select i1 [[TMP61]], i32 2, i32 [[SPEC_SELECT_3_1]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_2:%.*]] = select i1 [[CMP12_3_2]], i32 [[TMP60]], i32 [[SPEC_SELECT8_2_2]]
+; CHECK-NEXT:    [[SUB_328:%.*]] = sub i32 [[TMP30]], [[TMP9]]
+; CHECK-NEXT:    [[TMP62:%.*]] = icmp slt i32 [[SUB_328]], 0
+; CHECK-NEXT:    [[NEG_329:%.*]] = sub nsw i32 0, [[SUB_328]]
+; CHECK-NEXT:    [[TMP63:%.*]] = select i1 [[TMP62]], i32 [[NEG_329]], i32 [[SUB_328]]
+; CHECK-NEXT:    [[CMP12_330:%.*]] = icmp slt i32 [[TMP63]], [[SPEC_SELECT8_3_2]]
+; CHECK-NEXT:    [[SPEC_SELECT8_332:%.*]] = select i1 [[CMP12_330]], i32 [[TMP63]], i32 [[SPEC_SELECT8_3_2]]
+; CHECK-NEXT:    [[SUB_1_3:%.*]] = sub i32 [[TMP30]], [[TMP10]]
+; CHECK-NEXT:    [[TMP64:%.*]] = icmp slt i32 [[SUB_1_3]], 0
+; CHECK-NEXT:    [[NEG_1_3:%.*]] = sub nsw i32 0, [[SUB_1_3]]
+; CHECK-NEXT:    [[TMP65:%.*]] = select i1 [[TMP64]], i32 [[NEG_1_3]], i32 [[SUB_1_3]]
+; CHECK-NEXT:    [[CMP12_1_3:%.*]] = icmp slt i32 [[TMP65]], [[SPEC_SELECT8_332]]
+; CHECK-NEXT:    [[TMP66:%.*]] = or i1 [[CMP12_1_3]], [[CMP12_330]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_3:%.*]] = select i1 [[CMP12_1_3]], i32 [[TMP65]], i32 [[SPEC_SELECT8_332]]
+; CHECK-NEXT:    [[SUB_2_3:%.*]] = sub i32 [[TMP30]], [[TMP11]]
+; CHECK-NEXT:    [[TMP67:%.*]] = icmp slt i32 [[SUB_2_3]], 0
+; CHECK-NEXT:    [[NEG_2_3:%.*]] = sub nsw i32 0, [[SUB_2_3]]
+; CHECK-NEXT:    [[TMP68:%.*]] = select i1 [[TMP67]], i32 [[NEG_2_3]], i32 [[SUB_2_3]]
+; CHECK-NEXT:    [[CMP12_2_3:%.*]] = icmp slt i32 [[TMP68]], [[SPEC_SELECT8_1_3]]
+; CHECK-NEXT:    [[TMP69:%.*]] = or i1 [[CMP12_2_3]], [[TMP66]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_3:%.*]] = select i1 [[CMP12_2_3]], i32 [[TMP68]], i32 [[SPEC_SELECT8_1_3]]
+; CHECK-NEXT:    [[SUB_3_3:%.*]] = sub i32 [[TMP30]], [[TMP12]]
+; CHECK-NEXT:    [[TMP70:%.*]] = icmp slt i32 [[SUB_3_3]], 0
+; CHECK-NEXT:    [[NEG_3_3:%.*]] = sub nsw i32 0, [[SUB_3_3]]
+; CHECK-NEXT:    [[TMP71:%.*]] = select i1 [[TMP70]], i32 [[NEG_3_3]], i32 [[SUB_3_3]]
+; CHECK-NEXT:    [[CMP12_3_3:%.*]] = icmp slt i32 [[TMP71]], [[SPEC_SELECT8_2_3]]
+; CHECK-NEXT:    [[TMP72:%.*]] = or i1 [[CMP12_3_3]], [[TMP69]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_3:%.*]] = select i1 [[TMP72]], i32 3, i32 [[SPEC_SELECT_3_2]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_3:%.*]] = select i1 [[CMP12_3_3]], i32 [[TMP71]], i32 [[SPEC_SELECT8_2_3]]
+; CHECK-NEXT:    [[SUB_4:%.*]] = sub i32 [[TMP30]], [[TMP13]]
+; CHECK-NEXT:    [[TMP73:%.*]] = icmp slt i32 [[SUB_4]], 0
+; CHECK-NEXT:    [[NEG_4:%.*]] = sub nsw i32 0, [[SUB_4]]
+; CHECK-NEXT:    [[TMP74:%.*]] = select i1 [[TMP73]], i32 [[NEG_4]], i32 [[SUB_4]]
+; CHECK-NEXT:    [[CMP12_4:%.*]] = icmp slt i32 [[TMP74]], [[SPEC_SELECT8_3_3]]
+; CHECK-NEXT:    [[SPEC_SELECT8_4:%.*]] = select i1 [[CMP12_4]], i32 [[TMP74]], i32 [[SPEC_SELECT8_3_3]]
+; CHECK-NEXT:    [[SUB_1_4:%.*]] = sub i32 [[TMP30]], [[TMP14]]
+; CHECK-NEXT:    [[TMP75:%.*]] = icmp slt i32 [[SUB_1_4]], 0
+; CHECK-NEXT:    [[NEG_1_4:%.*]] = sub nsw i32 0, [[SUB_1_4]]
+; CHECK-NEXT:    [[TMP76:%.*]] = select i1 [[TMP75]], i32 [[NEG_1_4]], i32 [[SUB_1_4]]
+; CHECK-NEXT:    [[CMP12_1_4:%.*]] = icmp slt i32 [[TMP76]], [[SPEC_SELECT8_4]]
+; CHECK-NEXT:    [[TMP77:%.*]] = or i1 [[CMP12_1_4]], [[CMP12_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_4:%.*]] = select i1 [[CMP12_1_4]], i32 [[TMP76]], i32 [[SPEC_SELECT8_4]]
+; CHECK-NEXT:    [[SUB_2_4:%.*]] = sub i32 [[TMP30]], [[TMP15]]
+; CHECK-NEXT:    [[TMP78:%.*]] = icmp slt i32 [[SUB_2_4]], 0
+; CHECK-NEXT:    [[NEG_2_4:%.*]] = sub nsw i32 0, [[SUB_2_4]]
+; CHECK-NEXT:    [[TMP79:%.*]] = select i1 [[TMP78]], i32 [[NEG_2_4]], i32 [[SUB_2_4]]
+; CHECK-NEXT:    [[CMP12_2_4:%.*]] = icmp slt i32 [[TMP79]], [[SPEC_SELECT8_1_4]]
+; CHECK-NEXT:    [[TMP80:%.*]] = or i1 [[CMP12_2_4]], [[TMP77]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_4:%.*]] = select i1 [[CMP12_2_4]], i32 [[TMP79]], i32 [[SPEC_SELECT8_1_4]]
+; CHECK-NEXT:    [[SUB_3_4:%.*]] = sub i32 [[TMP30]], [[TMP16]]
+; CHECK-NEXT:    [[TMP81:%.*]] = icmp slt i32 [[SUB_3_4]], 0
+; CHECK-NEXT:    [[NEG_3_4:%.*]] = sub nsw i32 0, [[SUB_3_4]]
+; CHECK-NEXT:    [[TMP82:%.*]] = select i1 [[TMP81]], i32 [[NEG_3_4]], i32 [[SUB_3_4]]
+; CHECK-NEXT:    [[CMP12_3_4:%.*]] = icmp slt i32 [[TMP82]], [[SPEC_SELECT8_2_4]]
+; CHECK-NEXT:    [[TMP83:%.*]] = or i1 [[CMP12_3_4]], [[TMP80]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_4:%.*]] = select i1 [[TMP83]], i32 4, i32 [[SPEC_SELECT_3_3]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_4:%.*]] = select i1 [[CMP12_3_4]], i32 [[TMP82]], i32 [[SPEC_SELECT8_2_4]]
+; CHECK-NEXT:    [[SUB_5:%.*]] = sub i32 [[TMP30]], [[TMP17]]
+; CHECK-NEXT:    [[TMP84:%.*]] = icmp slt i32 [[SUB_5]], 0
+; CHECK-NEXT:    [[NEG_5:%.*]] = sub nsw i32 0, [[SUB_5]]
+; CHECK-NEXT:    [[TMP85:%.*]] = select i1 [[TMP84]], i32 [[NEG_5]], i32 [[SUB_5]]
+; CHECK-NEXT:    [[CMP12_5:%.*]] = icmp slt i32 [[TMP85]], [[SPEC_SELECT8_3_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_5:%.*]] = select i1 [[CMP12_5]], i32 [[TMP85]], i32 [[SPEC_SELECT8_3_4]]
+; CHECK-NEXT:    [[SUB_1_5:%.*]] = sub i32 [[TMP30]], [[TMP18]]
+; CHECK-NEXT:    [[TMP86:%.*]] = icmp slt i32 [[SUB_1_5]], 0
+; CHECK-NEXT:    [[NEG_1_5:%.*]] = sub nsw i32 0, [[SUB_1_5]]
+; CHECK-NEXT:    [[TMP87:%.*]] = select i1 [[TMP86]], i32 [[NEG_1_5]], i32 [[SUB_1_5]]
+; CHECK-NEXT:    [[CMP12_1_5:%.*]] = icmp slt i32 [[TMP87]], [[SPEC_SELECT8_5]]
+; CHECK-NEXT:    [[TMP88:%.*]] = or i1 [[CMP12_1_5]], [[CMP12_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_5:%.*]] = select i1 [[CMP12_1_5]], i32 [[TMP87]], i32 [[SPEC_SELECT8_5]]
+; CHECK-NEXT:    [[SUB_2_5:%.*]] = sub i32 [[TMP30]], [[TMP19]]
+; CHECK-NEXT:    [[TMP89:%.*]] = icmp slt i32 [[SUB_2_5]], 0
+; CHECK-NEXT:    [[NEG_2_5:%.*]] = sub nsw i32 0, [[SUB_2_5]]
+; CHECK-NEXT:    [[TMP90:%.*]] = select i1 [[TMP89]], i32 [[NEG_2_5]], i32 [[SUB_2_5]]
+; CHECK-NEXT:    [[CMP12_2_5:%.*]] = icmp slt i32 [[TMP90]], [[SPEC_SELECT8_1_5]]
+; CHECK-NEXT:    [[TMP91:%.*]] = or i1 [[CMP12_2_5]], [[TMP88]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_5:%.*]] = select i1 [[CMP12_2_5]], i32 [[TMP90]], i32 [[SPEC_SELECT8_1_5]]
+; CHECK-NEXT:    [[SUB_3_5:%.*]] = sub i32 [[TMP30]], [[TMP20]]
+; CHECK-NEXT:    [[TMP92:%.*]] = icmp slt i32 [[SUB_3_5]], 0
+; CHECK-NEXT:    [[NEG_3_5:%.*]] = sub nsw i32 0, [[SUB_3_5]]
+; CHECK-NEXT:    [[TMP93:%.*]] = select i1 [[TMP92]], i32 [[NEG_3_5]], i32 [[SUB_3_5]]
+; CHECK-NEXT:    [[CMP12_3_5:%.*]] = icmp slt i32 [[TMP93]], [[SPEC_SELECT8_2_5]]
+; CHECK-NEXT:    [[TMP94:%.*]] = or i1 [[CMP12_3_5]], [[TMP91]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_5:%.*]] = select i1 [[TMP94]], i32 5, i32 [[SPEC_SELECT_3_4]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_5:%.*]] = select i1 [[CMP12_3_5]], i32 [[TMP93]], i32 [[SPEC_SELECT8_2_5]]
+; CHECK-NEXT:    [[SUB_6:%.*]] = sub i32 [[TMP30]], [[TMP21]]
+; CHECK-NEXT:    [[TMP95:%.*]] = icmp slt i32 [[SUB_6]], 0
+; CHECK-NEXT:    [[NEG_6:%.*]] = sub nsw i32 0, [[SUB_6]]
+; CHECK-NEXT:    [[TMP96:%.*]] = select i1 [[TMP95]], i32 [[NEG_6]], i32 [[SUB_6]]
+; CHECK-NEXT:    [[CMP12_6:%.*]] = icmp slt i32 [[TMP96]], [[SPEC_SELECT8_3_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_6:%.*]] = select i1 [[CMP12_6]], i32 [[TMP96]], i32 [[SPEC_SELECT8_3_5]]
+; CHECK-NEXT:    [[SUB_1_6:%.*]] = sub i32 [[TMP30]], [[TMP22]]
+; CHECK-NEXT:    [[TMP97:%.*]] = icmp slt i32 [[SUB_1_6]], 0
+; CHECK-NEXT:    [[NEG_1_6:%.*]] = sub nsw i32 0, [[SUB_1_6]]
+; CHECK-NEXT:    [[TMP98:%.*]] = select i1 [[TMP97]], i32 [[NEG_1_6]], i32 [[SUB_1_6]]
+; CHECK-NEXT:    [[CMP12_1_6:%.*]] = icmp slt i32 [[TMP98]], [[SPEC_SELECT8_6]]
+; CHECK-NEXT:    [[TMP99:%.*]] = or i1 [[CMP12_1_6]], [[CMP12_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_6:%.*]] = select i1 [[CMP12_1_6]], i32 [[TMP98]], i32 [[SPEC_SELECT8_6]]
+; CHECK-NEXT:    [[SUB_2_6:%.*]] = sub i32 [[TMP30]], [[TMP23]]
+; CHECK-NEXT:    [[TMP100:%.*]] = icmp slt i32 [[SUB_2_6]], 0
+; CHECK-NEXT:    [[NEG_2_6:%.*]] = sub nsw i32 0, [[SUB_2_6]]
+; CHECK-NEXT:    [[TMP101:%.*]] = select i1 [[TMP100]], i32 [[NEG_2_6]], i32 [[SUB_2_6]]
+; CHECK-NEXT:    [[CMP12_2_6:%.*]] = icmp slt i32 [[TMP101]], [[SPEC_SELECT8_1_6]]
+; CHECK-NEXT:    [[TMP102:%.*]] = or i1 [[CMP12_2_6]], [[TMP99]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_6:%.*]] = select i1 [[CMP12_2_6]], i32 [[TMP101]], i32 [[SPEC_SELECT8_1_6]]
+; CHECK-NEXT:    [[SUB_3_6:%.*]] = sub i32 [[TMP30]], [[TMP24]]
+; CHECK-NEXT:    [[TMP103:%.*]] = icmp slt i32 [[SUB_3_6]], 0
+; CHECK-NEXT:    [[NEG_3_6:%.*]] = sub nsw i32 0, [[SUB_3_6]]
+; CHECK-NEXT:    [[TMP104:%.*]] = select i1 [[TMP103]], i32 [[NEG_3_6]], i32 [[SUB_3_6]]
+; CHECK-NEXT:    [[CMP12_3_6:%.*]] = icmp slt i32 [[TMP104]], [[SPEC_SELECT8_2_6]]
+; CHECK-NEXT:    [[TMP105:%.*]] = or i1 [[CMP12_3_6]], [[TMP102]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_6:%.*]] = select i1 [[TMP105]], i32 6, i32 [[SPEC_SELECT_3_5]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_6:%.*]] = select i1 [[CMP12_3_6]], i32 [[TMP104]], i32 [[SPEC_SELECT8_2_6]]
+; CHECK-NEXT:    [[SUB_7:%.*]] = sub i32 [[TMP30]], [[TMP25]]
+; CHECK-NEXT:    [[TMP106:%.*]] = icmp slt i32 [[SUB_7]], 0
+; CHECK-NEXT:    [[NEG_7:%.*]] = sub nsw i32 0, [[SUB_7]]
+; CHECK-NEXT:    [[TMP107:%.*]] = select i1 [[TMP106]], i32 [[NEG_7]], i32 [[SUB_7]]
+; CHECK-NEXT:    [[CMP12_7:%.*]] = icmp slt i32 [[TMP107]], [[SPEC_SELECT8_3_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_7:%.*]] = select i1 [[CMP12_7]], i32 [[TMP107]], i32 [[SPEC_SELECT8_3_6]]
+; CHECK-NEXT:    [[SUB_1_7:%.*]] = sub i32 [[TMP30]], [[TMP26]]
+; CHECK-NEXT:    [[TMP108:%.*]] = icmp slt i32 [[SUB_1_7]], 0
+; CHECK-NEXT:    [[NEG_1_7:%.*]] = sub nsw i32 0, [[SUB_1_7]]
+; CHECK-NEXT:    [[TMP109:%.*]] = select i1 [[TMP108]], i32 [[NEG_1_7]], i32 [[SUB_1_7]]
+; CHECK-NEXT:    [[CMP12_1_7:%.*]] = icmp slt i32 [[TMP109]], [[SPEC_SELECT8_7]]
+; CHECK-NEXT:    [[TMP110:%.*]] = or i1 [[CMP12_1_7]], [[CMP12_7]]
+; CHECK-NEXT:    [[SPEC_SELECT8_1_7:%.*]] = select i1 [[CMP12_1_7]], i32 [[TMP109]], i32 [[SPEC_SELECT8_7]]
+; CHECK-NEXT:    [[SUB_2_7:%.*]] = sub i32 [[TMP30]], [[TMP27]]
+; CHECK-NEXT:    [[TMP111:%.*]] = icmp slt i32 [[SUB_2_7]], 0
+; CHECK-NEXT:    [[NEG_2_7:%.*]] = sub nsw i32 0, [[SUB_2_7]]
+; CHECK-NEXT:    [[TMP112:%.*]] = select i1 [[TMP111]], i32 [[NEG_2_7]], i32 [[SUB_2_7]]
+; CHECK-NEXT:    [[CMP12_2_7:%.*]] = icmp slt i32 [[TMP112]], [[SPEC_SELECT8_1_7]]
+; CHECK-NEXT:    [[TMP113:%.*]] = or i1 [[CMP12_2_7]], [[TMP110]]
+; CHECK-NEXT:    [[SPEC_SELECT8_2_7:%.*]] = select i1 [[CMP12_2_7]], i32 [[TMP112]], i32 [[SPEC_SELECT8_1_7]]
+; CHECK-NEXT:    [[SUB_3_7:%.*]] = sub i32 [[TMP30]], [[TMP28]]
+; CHECK-NEXT:    [[TMP114:%.*]] = icmp slt i32 [[SUB_3_7]], 0
+; CHECK-NEXT:    [[NEG_3_7:%.*]] = sub nsw i32 0, [[SUB_3_7]]
+; CHECK-NEXT:    [[TMP115:%.*]] = select i1 [[TMP114]], i32 [[NEG_3_7]], i32 [[SUB_3_7]]
+; CHECK-NEXT:    [[CMP12_3_7:%.*]] = icmp slt i32 [[TMP115]], [[SPEC_SELECT8_2_7]]
+; CHECK-NEXT:    [[TMP116:%.*]] = or i1 [[CMP12_3_7]], [[TMP113]]
+; CHECK-NEXT:    [[SPEC_SELECT_3_7:%.*]] = select i1 [[TMP116]], i32 7, i32 [[SPEC_SELECT_3_6]]
+; CHECK-NEXT:    [[SPEC_SELECT8_3_7]] = select i1 [[CMP12_3_7]], i32 [[TMP115]], i32 [[SPEC_SELECT8_2_7]]
+; CHECK-NEXT:    [[K:%.*]] = getelementptr inbounds [366 x i32], [366 x i32]* @l, i64 0, i64 [[INDVARS_IV]]
+; CHECK-NEXT:    store i32 [[SPEC_SELECT_3_7]], i32* [[K]], align 4
+; CHECK-NEXT:    [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT:    br label [[FOR_COND]]
+;
+entry:
+  %0 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 0), align 16
+  %1 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 1), align 4
+  %2 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 2), align 8
+  %3 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 0, i64 3), align 4
+  %4 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 0), align 16
+  %5 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 1), align 4
+  %6 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 2), align 8
+  %7 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 1, i64 3), align 4
+  %8 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 0), align 16
+  %9 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 1), align 4
+  %10 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 2), align 8
+  %11 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 2, i64 3), align 4
+  %12 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 0), align 16
+  %13 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 1), align 4
+  %14 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 2), align 8
+  %15 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 3, i64 3), align 4
+  %16 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 0), align 16
+  %17 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 1), align 4
+  %18 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 2), align 8
+  %19 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 4, i64 3), align 4
+  %20 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 0), align 16
+  %21 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 1), align 4
+  %22 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 2), align 8
+  %23 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 5, i64 3), align 4
+  %24 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 0), align 16
+  %25 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 1), align 4
+  %26 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 2), align 8
+  %27 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 6, i64 3), align 4
+  %28 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 0), align 16
+  %29 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 1), align 4
+  %30 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 2), align 8
+  %31 = load i32, i32* getelementptr inbounds ([8 x [4 x i32]], [8 x [4 x i32]]* @k, i64 0, i64 7, i64 3), align 4
+  br label %for.cond
+
+for.cond:                                         ; preds = %for.cond, %entry
+  %indvars.iv = phi i64 [ %indvars.iv.next, %for.cond ], [ 0, %entry ]
+  %b.0 = phi i32 [ %spec.select8.3.7, %for.cond ], [ undef, %entry ]
+  %32 = trunc i64 %indvars.iv to i32
+  %33 = add i32 %32, -183
+  %sub = sub i32 %33, %0
+  %34 = icmp slt i32 %sub, 0
+  %neg = sub nsw i32 0, %sub
+  %35 = select i1 %34, i32 %neg, i32 %sub
+  %cmp12 = icmp slt i32 %35, %b.0
+  %spec.select8 = select i1 %cmp12, i32 %35, i32 %b.0
+  %sub.1 = sub i32 %33, %1
+  %36 = icmp slt i32 %sub.1, 0
+  %neg.1 = sub nsw i32 0, %sub.1
+  %37 = select i1 %36, i32 %neg.1, i32 %sub.1
+  %cmp12.1 = icmp slt i32 %37, %spec.select8
+  %spec.select8.1 = select i1 %cmp12.1, i32 %37, i32 %spec.select8
+  %sub.2 = sub i32 %33, %2
+  %38 = icmp slt i32 %sub.2, 0
+  %neg.2 = sub nsw i32 0, %sub.2
+  %39 = select i1 %38, i32 %neg.2, i32 %sub.2
+  %cmp12.2 = icmp slt i32 %39, %spec.select8.1
+  %spec.select8.2 = select i1 %cmp12.2, i32 %39, i32 %spec.select8.1
+  %sub.3 = sub i32 %33, %3
+  %40 = icmp slt i32 %sub.3, 0
+  %neg.3 = sub nsw i32 0, %sub.3
+  %41 = select i1 %40, i32 %neg.3, i32 %sub.3
+  %cmp12.3 = icmp slt i32 %41, %spec.select8.2
+  %spec.select8.3 = select i1 %cmp12.3, i32 %41, i32 %spec.select8.2
+  %sub.116 = sub i32 %33, %4
+  %42 = icmp slt i32 %sub.116, 0
+  %neg.117 = sub nsw i32 0, %sub.116
+  %43 = select i1 %42, i32 %neg.117, i32 %sub.116
+  %cmp12.118 = icmp slt i32 %43, %spec.select8.3
+  %spec.select8.120 = select i1 %cmp12.118, i32 %43, i32 %spec.select8.3
+  %sub.1.1 = sub i32 %33, %5
+  %44 = icmp slt i32 %sub.1.1, 0
+  %neg.1.1 = sub nsw i32 0, %sub.1.1
+  %45 = select i1 %44, i32 %neg.1.1, i32 %sub.1.1
+  %cmp12.1.1 = icmp slt i32 %45, %spec.select8.120
+  %narrow = or i1 %cmp12.1.1, %cmp12.118
+  %spec.select8.1.1 = select i1 %cmp12.1.1, i32 %45, i32 %spec.select8.120
+  %sub.2.1 = sub i32 %33, %6
+  %46 = icmp slt i32 %sub.2.1, 0
+  %neg.2.1 = sub nsw i32 0, %sub.2.1
+  %47 = select i1 %46, i32 %neg.2.1, i32 %sub.2.1
+  %cmp12.2.1 = icmp slt i32 %47, %spec.select8.1.1
+  %narrow34 = or i1 %cmp12.2.1, %narrow
+  %spec.select8.2.1 = select i1 %cmp12.2.1, i32 %47, i32 %spec.select8.1.1
+  %sub.3.1 = sub i32 %33, %7
+  %48 = icmp slt i32 %sub.3.1, 0
+  %neg.3.1 = sub nsw i32 0, %sub.3.1
+  %49 = select i1 %48, i32 %neg.3.1, i32 %sub.3.1
+  %cmp12.3.1 = icmp slt i32 %49, %spec.select8.2.1
+  %narrow35 = or i1 %cmp12.3.1, %narrow34
+  %spec.select.3.1 = zext i1 %narrow35 to i32
+  %spec.select8.3.1 = select i1 %cmp12.3.1, i32 %49, i32 %spec.select8.2.1
+  %sub.222 = sub i32 %33, %8
+  %50 = icmp slt i32 %sub.222, 0
+  %neg.223 = sub nsw i32 0, %sub.222
+  %51 = select i1 %50, i32 %neg.223, i32 %sub.222
+  %cmp12.224 = icmp slt i32 %51, %spec.select8.3.1
+  %spec.select8.226 = select i1 %cmp12.224, i32 %51, i32 %spec.select8.3.1
+  %sub.1.2 = sub i32 %33, %9
+  %52 = icmp slt i32 %sub.1.2, 0
+  %neg.1.2 = sub nsw i32 0, %sub.1.2
+  %53 = select i1 %52, i32 %neg.1.2, i32 %sub.1.2
+  %cmp12.1.2 = icmp slt i32 %53, %spec.select8.226
+  %54 = or i1 %cmp12.1.2, %cmp12.224
+  %spec.select8.1.2 = select i1 %cmp12.1.2, i32 %53, i32 %spec.select8.226
+  %sub.2.2 = sub i32 %33, %10
+  %55 = icmp slt i32 %sub.2.2, 0
+  %neg.2.2 = sub nsw i32 0, %sub.2.2
+  %56 = select i1 %55, i32 %neg.2.2, i32 %sub.2.2
+  %cmp12.2.2 = icmp slt i32 %56, %spec.select8.1.2
+  %57 = or i1 %cmp12.2.2, %54
+  %spec.select8.2.2 = select i1 %cmp12.2.2, i32 %56, i32 %spec.select8.1.2
+  %sub.3.2 = sub i32 %33, %11
+  %58 = icmp slt i32 %sub.3.2, 0
+  %neg.3.2 = sub nsw i32 0, %sub.3.2
+  %59 = select i1 %58, i32 %neg.3.2, i32 %sub.3.2
+  %cmp12.3.2 = icmp slt i32 %59, %spec.select8.2.2
+  %60 = or i1 %cmp12.3.2, %57
+  %spec.select.3.2 = select i1 %60, i32 2, i32 %spec.select.3.1
+  %spec.select8.3.2 = select i1 %cmp12.3.2, i32 %59, i32 %spec.select8.2.2
+  %sub.328 = sub i32 %33, %12
+  %61 = icmp slt i32 %sub.328, 0
+  %neg.329 = sub nsw i32 0, %sub.328
+  %62 = select i1 %61, i32 %neg.329, i32 %sub.328
+  %cmp12.330 = icmp slt i32 %62, %spec.select8.3.2
+  %spec.select8.332 = select i1 %cmp12.330, i32 %62, i32 %spec.select8.3.2
+  %sub.1.3 = sub i32 %33, %13
+  %63 = icmp slt i32 %sub.1.3, 0
+  %neg.1.3 = sub nsw i32 0, %sub.1.3
+  %64 = select i1 %63, i32 %neg.1.3, i32 %sub.1.3
+  %cmp12.1.3 = icmp slt i32 %64, %spec.select8.332
+  %65 = or i1 %cmp12.1.3, %cmp12.330
+  %spec.select8.1.3 = select i1 %cmp12.1.3, i32 %64, i32 %spec.select8.332
+  %sub.2.3 = sub i32 %33, %14
+  %66 = icmp slt i32 %sub.2.3, 0
+  %neg.2.3 = sub nsw i32 0, %sub.2.3
+  %67 = select i1 %66, i32 %neg.2.3, i32 %sub.2.3
+  %cmp12.2.3 = icmp slt i32 %67, %spec.select8.1.3
+  %68 = or i1 %cmp12.2.3, %65
+  %spec.select8.2.3 = select i1 %cmp12.2.3, i32 %67, i32 %spec.select8.1.3
+  %sub.3.3 = sub i32 %33, %15
+  %69 = icmp slt i32 %sub.3.3, 0
+  %neg.3.3 = sub nsw i32 0, %sub.3.3
+  %70 = select i1 %69, i32 %neg.3.3, i32 %sub.3.3
+  %cmp12.3.3 = icmp slt i32 %70, %spec.select8.2.3
+  %71 = or i1 %cmp12.3.3, %68
+  %spec.select.3.3 = select i1 %71, i32 3, i32 %spec.select.3.2
+  %spec.select8.3.3 = select i1 %cmp12.3.3, i32 %70, i32 %spec.select8.2.3
+  %sub.4 = sub i32 %33, %16
+  %72 = icmp slt i32 %sub.4, 0
+  %neg.4 = sub nsw i32 0, %sub.4
+  %73 = select i1 %72, i32 %neg.4, i32 %sub.4
+  %cmp12.4 = icmp slt i32 %73, %spec.select8.3.3
+  %spec.select8.4 = select i1 %cmp12.4, i32 %73, i32 %spec.select8.3.3
+  %sub.1.4 = sub i32 %33, %17
+  %74 = icmp slt i32 %sub.1.4, 0
+  %neg.1.4 = sub nsw i32 0, %sub.1.4
+  %75 = select i1 %74, i32 %neg.1.4, i32 %sub.1.4
+  %cmp12.1.4 = icmp slt i32 %75, %spec.select8.4
+  %76 = or i1 %cmp12.1.4, %cmp12.4
+  %spec.select8.1.4 = select i1 %cmp12.1.4, i32 %75, i32 %spec.select8.4
+  %sub.2.4 = sub i32 %33, %18
+  %77 = icmp slt i32 %sub.2.4, 0
+  %neg.2.4 = sub nsw i32 0, %sub.2.4
+  %78 = select i1 %77, i32 %neg.2.4, i32 %sub.2.4
+  %cmp12.2.4 = icmp slt i32 %78, %spec.select8.1.4
+  %79 = or i1 %cmp12.2.4, %76
+  %spec.select8.2.4 = select i1 %cmp12.2.4, i32 %78, i32 %spec.select8.1.4
+  %sub.3.4 = sub i32 %33, %19
+  %80 = icmp slt i32 %sub.3.4, 0
+  %neg.3.4 = sub nsw i32 0, %sub.3.4
+  %81 = select i1 %80, i32 %neg.3.4, i32 %sub.3.4
+  %cmp12.3.4 = icmp slt i32 %81, %spec.select8.2.4
+  %82 = or i1 %cmp12.3.4, %79
+  %spec.select.3.4 = select i1 %82, i32 4, i32 %spec.select.3.3
+  %spec.select8.3.4 = select i1 %cmp12.3.4, i32 %81, i32 %spec.select8.2.4
+  %sub.5 = sub i32 %33, %20
+  %83 = icmp slt i32 %sub.5, 0
+  %neg.5 = sub nsw i32 0, %sub.5
+  %84 = select i1 %83, i32 %neg.5, i32 %sub.5
+  %cmp12.5 = icmp slt i32 %84, %spec.select8.3.4
+  %spec.select8.5 = select i1 %cmp12.5, i32 %84, i32 %spec.select8.3.4
+  %sub.1.5 = sub i32 %33, %21
+  %85 = icmp slt i32 %sub.1.5, 0
+  %neg.1.5 = sub nsw i32 0, %sub.1.5
+  %86 = select i1 %85, i32 %neg.1.5, i32 %sub.1.5
+  %cmp12.1.5 = icmp slt i32 %86, %spec.select8.5
+  %87 = or i1 %cmp12.1.5, %cmp12.5
+  %spec.select8.1.5 = select i1 %cmp12.1.5, i32 %86, i32 %spec.select8.5
+  %sub.2.5 = sub i32 %33, %22
+  %88 = icmp slt i32 %sub.2.5, 0
+  %neg.2.5 = sub nsw i32 0, %sub.2.5
+  %89 = select i1 %88, i32 %neg.2.5, i32 %sub.2.5
+  %cmp12.2.5 = icmp slt i32 %89, %spec.select8.1.5
+  %90 = or i1 %cmp12.2.5, %87
+  %spec.select8.2.5 = select i1 %cmp12.2.5, i32 %89, i32 %spec.select8.1.5
+  %sub.3.5 = sub i32 %33, %23
+  %91 = icmp slt i32 %sub.3.5, 0
+  %neg.3.5 = sub nsw i32 0, %sub.3.5
+  %92 = select i1 %91, i32 %neg.3.5, i32 %sub.3.5
+  %cmp12.3.5 = icmp slt i32 %92, %spec.select8.2.5
+  %93 = or i1 %cmp12.3.5, %90
+  %spec.select.3.5 = select i1 %93, i32 5, i32 %spec.select.3.4
+  %spec.select8.3.5 = select i1 %cmp12.3.5, i32 %92, i32 %spec.select8.2.5
+  %sub.6 = sub i32 %33, %24
+  %94 = icmp slt i32 %sub.6, 0
+  %neg.6 = sub nsw i32 0, %sub.6
+  %95 = select i1 %94, i32 %neg.6, i32 %sub.6
+  %cmp12.6 = icmp slt i32 %95, %spec.select8.3.5
+  %spec.select8.6 = select i1 %cmp12.6, i32 %95, i32 %spec.select8.3.5
+  %sub.1.6 = sub i32 %33, %25
+  %96 = icmp slt i32 %sub.1.6, 0
+  %neg.1.6 = sub nsw i32 0, %sub.1.6
+  %97 = select i1 %96, i32 %neg.1.6, i32 %sub.1.6
+  %cmp12.1.6 = icmp slt i32 %97, %spec.select8.6
+  %98 = or i1 %cmp12.1.6, %cmp12.6
+  %spec.select8.1.6 = select i1 %cmp12.1.6, i32 %97, i32 %spec.select8.6
+  %sub.2.6 = sub i32 %33, %26
+  %99 = icmp slt i32 %sub.2.6, 0
+  %neg.2.6 = sub nsw i32 0, %sub.2.6
+  %100 = select i1 %99, i32 %neg.2.6, i32 %sub.2.6
+  %cmp12.2.6 = icmp slt i32 %100, %spec.select8.1.6
+  %101 = or i1 %cmp12.2.6, %98
+  %spec.select8.2.6 = select i1 %cmp12.2.6, i32 %100, i32 %spec.select8.1.6
+  %sub.3.6 = sub i32 %33, %27
+  %102 = icmp slt i32 %sub.3.6, 0
+  %neg.3.6 = sub nsw i32 0, %sub.3.6
+  %103 = select i1 %102, i32 %neg.3.6, i32 %sub.3.6
+  %cmp12.3.6 = icmp slt i32 %103, %spec.select8.2.6
+  %104 = or i1 %cmp12.3.6, %101
+  %spec.select.3.6 = select i1 %104, i32 6, i32 %spec.select.3.5
+  %spec.select8.3.6 = select i1 %cmp12.3.6, i32 %103, i32 %spec.select8.2.6
+  %sub.7 = sub i32 %33, %28
+  %105 = icmp slt i32 %sub.7, 0
+  %neg.7 = sub nsw i32 0, %sub.7
+  %106 = select i1 %105, i32 %neg.7, i32 %sub.7
+  %cmp12.7 = icmp slt i32 %106, %spec.select8.3.6
+  %spec.select8.7 = select i1 %cmp12.7, i32 %106, i32 %spec.select8.3.6
+  %sub.1.7 = sub i32 %33, %29
+  %107 = icmp slt i32 %sub.1.7, 0
+  %neg.1.7 = sub nsw i32 0, %sub.1.7
+  %108 = select i1 %107, i32 %neg.1.7, i32 %sub.1.7
+  %cmp12.1.7 = icmp slt i32 %108, %spec.select8.7
+  %109 = or i1 %cmp12.1.7, %cmp12.7
+  %spec.select8.1.7 = select i1 %cmp12.1.7, i32 %108, i32 %spec.select8.7
+  %sub.2.7 = sub i32 %33, %30
+  %110 = icmp slt i32 %sub.2.7, 0
+  %neg.2.7 = sub nsw i32 0, %sub.2.7
+  %111 = select i1 %110, i32 %neg.2.7, i32 %sub.2.7
+  %cmp12.2.7 = icmp slt i32 %111, %spec.select8.1.7
+  %112 = or i1 %cmp12.2.7, %109
+  %spec.select8.2.7 = select i1 %cmp12.2.7, i32 %111, i32 %spec.select8.1.7
+  %sub.3.7 = sub i32 %33, %31
+  %113 = icmp slt i32 %sub.3.7, 0
+  %neg.3.7 = sub nsw i32 0, %sub.3.7
+  %114 = select i1 %113, i32 %neg.3.7, i32 %sub.3.7
+  %cmp12.3.7 = icmp slt i32 %114, %spec.select8.2.7
+  %115 = or i1 %cmp12.3.7, %112
+  %spec.select.3.7 = select i1 %115, i32 7, i32 %spec.select.3.6
+  %spec.select8.3.7 = select i1 %cmp12.3.7, i32 %114, i32 %spec.select8.2.7
+  %k = getelementptr inbounds [366 x i32], [366 x i32]* @l, i64 0, i64 %indvars.iv
+  store i32 %spec.select.3.7, i32* %k, align 4
+  %indvars.iv.next = add i64 %indvars.iv, 1
+  br label %for.cond
+}
+
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
index 889bba80b7a49..779d5260506af 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/vectorize-reorder-reuse.ll
@@ -18,19 +18,6 @@ define i32 @foo(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -41,7 +28,6 @@ define i32 @foo(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a4
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
@@ -92,19 +78,6 @@ define i32 @foo1(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -115,7 +88,6 @@ define i32 @foo1(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
@@ -170,19 +142,6 @@ define i32 @foo2(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[A7:%.*]], i32 6
 ; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[A8:%.*]], i32 7
 ; CHECK-NEXT:    [[TMP10:%.*]] = add <8 x i32> [[SHUFFLE]], [[TMP9]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i32 undef, undef
-; CHECK-NEXT:    [[COND:%.*]] = select i1 [[CMP]], i32 undef, i32 undef
-; CHECK-NEXT:    [[CMP15:%.*]] = icmp ult i32 [[COND]], undef
-; CHECK-NEXT:    [[COND19:%.*]] = select i1 [[CMP15]], i32 [[COND]], i32 undef
-; CHECK-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[COND19]], undef
-; CHECK-NEXT:    [[COND24:%.*]] = select i1 [[CMP20]], i32 [[COND19]], i32 undef
-; CHECK-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[COND24]], undef
-; CHECK-NEXT:    [[COND29:%.*]] = select i1 [[CMP25]], i32 [[COND24]], i32 undef
-; CHECK-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[COND29]], undef
-; CHECK-NEXT:    [[COND34:%.*]] = select i1 [[CMP30]], i32 [[COND29]], i32 undef
-; CHECK-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[COND34]], undef
-; CHECK-NEXT:    [[COND39:%.*]] = select i1 [[CMP35]], i32 [[COND34]], i32 undef
-; CHECK-NEXT:    [[CMP40:%.*]] = icmp ult i32 [[COND39]], undef
 ; CHECK-NEXT:    [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[TMP10]], <8 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef>
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP:%.*]] = icmp ult <8 x i32> [[TMP10]], [[RDX_SHUF]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP]], <8 x i32> [[TMP10]], <8 x i32> [[RDX_SHUF]]
@@ -193,7 +152,6 @@ define i32 @foo2(i32* nocapture readonly %arr, i32 %a1, i32 %a2, i32 %a3, i32 %a
 ; CHECK-NEXT:    [[RDX_MINMAX_CMP5:%.*]] = icmp ult <8 x i32> [[RDX_MINMAX_SELECT3]], [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[RDX_MINMAX_SELECT6:%.*]] = select <8 x i1> [[RDX_MINMAX_CMP5]], <8 x i32> [[RDX_MINMAX_SELECT3]], <8 x i32> [[RDX_SHUF4]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i32> [[RDX_MINMAX_SELECT6]], i32 0
-; CHECK-NEXT:    [[COND44:%.*]] = select i1 [[CMP40]], i32 [[COND39]], i32 undef
 ; CHECK-NEXT:    ret i32 [[TMP11]]
 ;
 entry:
diff --git a/llvm/test/Transforms/SROA/vector-promotion-different-size.ll b/llvm/test/Transforms/SROA/vector-promotion-different-size.ll
new file mode 100644
index 0000000000000..56e1f1f2160a7
--- /dev/null
+++ b/llvm/test/Transforms/SROA/vector-promotion-different-size.ll
@@ -0,0 +1,24 @@
+; RUN: opt < %s -sroa -S | FileCheck %s
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-n8:16:32:64"
+
+define <4 x i1> @vector_bitcast() {
+  ; CHECK-LABEL: @vector_bitcast
+  ; CHECK: alloca i1
+
+    %a = alloca <3 x i1>
+    store <3 x i1> <i1 1,i1 0,i1 1>, <3 x i1>* %a
+    %cast = bitcast <3 x i1>* %a to <4 x i1>*
+    %vec = load <4 x i1>, <4 x i1>* %cast
+    ret <4 x i1> %vec
+}
+
+define void @vector_bitcast_2() {
+  ; CHECK-LABEL: @vector_bitcast_2
+  ; CHECK: alloca <32 x i16>
+
+    %"sum$1.host2" = alloca <32 x i16>
+    store <32 x i16> undef, <32 x i16>* %"sum$1.host2"
+    %bc = bitcast <32 x i16>* %"sum$1.host2" to <64 x i16>*
+    %bcl = load <64 x i16>, <64 x i16>* %bc
+    ret void
+}
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo b/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo
new file mode 100644
index 0000000000000..c05af6fb4a6dc
Binary files /dev/null and b/llvm/test/Transforms/SampleProfile/Inputs/profsampleacc.extbinary.afdo differ
diff --git a/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll b/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
index dfd66219f7626..6626567722993 100644
--- a/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
+++ b/llvm/test/Transforms/SampleProfile/compressed-profile-symbol-list.ll
@@ -1,5 +1,5 @@
 ; REQUIRES: zlib
 ; Append inline.prof with profile symbol list and save it after compression.
 ; RUN: llvm-profdata merge --sample --prof-sym-list=%S/Inputs/profile-symbol-list.text --compress-prof-sym-list=true --extbinary %S/Inputs/inline.prof --output=%t.profdata
-; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-sample-accurate -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
-; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-sample-accurate -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
diff --git a/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll b/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll
new file mode 100644
index 0000000000000..bfade9d90106e
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/profile-sample-accurate.ll
@@ -0,0 +1,157 @@
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+
+; RUN: llvm-profdata merge -sample -extbinary -prof-sym-list=%S/Inputs/profile-symbol-list.text %S/Inputs/profsampleacc.extbinary.afdo -o %t.symlist.afdo
+; RUN: opt < %s -sample-profile -sample-profile-file=%t.symlist.afdo -profile-summary-cutoff-hot=600000 -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=PROFSYMLIST
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t.symlist.afdo -profile-summary-cutoff-hot=600000 -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=PROFSYMLIST
+;
+; If -profile-accurate-for-symsinlist and -profile-sample-accurate both present,
+; -profile-sample-accurate will override -profile-accurate-for-symsinlist.
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=600000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_WARM
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profsampleacc.extbinary.afdo -profile-summary-cutoff-hot=900000 -profile-sample-accurate -profile-accurate-for-symsinlist -S | FileCheck %s --check-prefix=CALL_SUM_IS_HOT
+;
+; Original C++ test case
+;
+; #include <stdio.h>
+;
+; int sum(int x, int y) {
+;   return x + y;
+; }
+;
+; int main() {
+;   int s, i = 0;
+;   while (i++ < 20000 * 20000)
+;     if (i != 100) s = sum(i, s); else s = 30;
+;   printf("sum is %d\n", s);
+;   return 0;
+; }
+;
+@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1
+
+; Check _Z3sumii's function entry count will be 0 when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_HOT: define i32 @_Z3sumii{{.*}}!prof ![[ZERO_ID:[0-9]+]]
+;
+; Check _Z3sumii's function entry count will be nonzero when
+; profile-sample-accurate is enabled because the callsite is warm and not
+; inlined so its function entry count is adjusted to nonzero.
+; CALL_SUM_IS_WARM: define i32 @_Z3sumii{{.*}}!prof ![[NONZERO_ID:[0-9]+]]
+;
+; Check _Z3sumii's function entry count will be initialized to -1 when
+; profile-accurate-for-profsymlist is enabled and _Z3sumii exists in the
+; profile symbol list because it also shows up in the profile as inline
+; instance.
+; PROFSYMLIST: define i32 @_Z3sumii{{.*}}!prof ![[UNKNOWN_ID:[0-9]+]]
+;
+; Function Attrs: nounwind uwtable
+define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 {
+entry:
+  %x.addr = alloca i32, align 4
+  %y.addr = alloca i32, align 4
+  store i32 %x, i32* %x.addr, align 4
+  store i32 %y, i32* %y.addr, align 4
+  %0 = load i32, i32* %x.addr, align 4, !dbg !11
+  %1 = load i32, i32* %y.addr, align 4, !dbg !11
+  %add = add nsw i32 %0, %1, !dbg !11
+  ret i32 %add, !dbg !11
+}
+
+; Function Attrs: uwtable
+define i32 @main() !dbg !7 {
+entry:
+  %retval = alloca i32, align 4
+  %s = alloca i32, align 4
+  %i = alloca i32, align 4
+  store i32 0, i32* %retval
+  store i32 0, i32* %i, align 4, !dbg !12
+  br label %while.cond, !dbg !13
+
+while.cond:                                       ; preds = %if.end, %entry
+  %0 = load i32, i32* %i, align 4, !dbg !14
+  %inc = add nsw i32 %0, 1, !dbg !14
+  store i32 %inc, i32* %i, align 4, !dbg !14
+  %cmp = icmp slt i32 %0, 400000000, !dbg !14
+  br i1 %cmp, label %while.body, label %while.end, !dbg !14
+
+while.body:                                       ; preds = %while.cond
+  %1 = load i32, i32* %i, align 4, !dbg !16
+  %cmp1 = icmp ne i32 %1, 100, !dbg !16
+  br i1 %cmp1, label %if.then, label %if.else, !dbg !16
+
+; With the hot cutoff being set to 600000, the inline instance of _Z3sumii
+; in main is neither hot nor cold. Check it won't be inlined when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_WARM: if.then:
+; CALL_SUM_IS_WARM: call i32 @_Z3sumii
+; CALL_SUM_IS_WARM: if.else:
+;
+; With the hot cutoff being set to 900000, the inline instance of _Z3sumii
+; in main is hot. Check the callsite of _Z3sumii will be inlined when
+; profile-sample-accurate is enabled.
+; CALL_SUM_IS_HOT: if.then:
+; CALL_SUM_IS_HOT-NOT: call i32 @_Z3sumii
+; CALL_SUM_IS_HOT: if.else:
+;
+; Check _Z3sumii will be inlined when profile-accurate-for-profsymlist is
+; enabled
+; PROFSYMLIST: if.then:
+; PROFSYMLIST-NOT: call i32 @_Z3sumii
+; PROFSYMLIST: if.else:
+if.then:                                          ; preds = %while.body
+  %2 = load i32, i32* %i, align 4, !dbg !18
+  %3 = load i32, i32* %s, align 4, !dbg !18
+  %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18
+  store i32 %call, i32* %s, align 4, !dbg !18
+  br label %if.end, !dbg !18
+
+if.else:                                          ; preds = %while.body
+  store i32 30, i32* %s, align 4, !dbg !20
+  br label %if.end
+
+if.end:                                           ; preds = %if.else, %if.then
+  br label %while.cond, !dbg !22
+
+while.end:                                        ; preds = %while.cond
+  %4 = load i32, i32* %s, align 4, !dbg !24
+  %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24
+  ret i32 0, !dbg !25
+}
+
+declare i32 @printf(i8*, ...) #2
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!8, !9}
+!llvm.ident = !{!10}
+
+; CALL_SUM_IS_HOT: ![[ZERO_ID]] = !{!"function_entry_count", i64 0}
+; CALL_SUM_IS_WARM: ![[NONZERO_ID]] = !{!"function_entry_count", i64 5179}
+; PROFSYMLIST: ![[UNKNOWN_ID]] = !{!"function_entry_count", i64 -1}
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2)
+!1 = !DIFile(filename: "calls.cc", directory: ".")
+!2 = !{}
+!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!5 = !DIFile(filename: "calls.cc", directory: ".")
+!6 = !DISubroutineType(types: !2)
+!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2)
+!8 = !{i32 2, !"Dwarf Version", i32 4}
+!9 = !{i32 1, !"Debug Info Version", i32 3}
+!10 = !{!"clang version 3.5 "}
+!11 = !DILocation(line: 4, scope: !4)
+!12 = !DILocation(line: 8, scope: !7)
+!13 = !DILocation(line: 9, scope: !7)
+!14 = !DILocation(line: 9, scope: !15)
+!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7)
+!16 = !DILocation(line: 10, scope: !17)
+!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7)
+!18 = !DILocation(line: 10, scope: !19)
+!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17)
+!20 = !DILocation(line: 10, scope: !21)
+!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17)
+!22 = !DILocation(line: 10, scope: !23)
+!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17)
+!24 = !DILocation(line: 11, scope: !7)
+!25 = !DILocation(line: 12, scope: !7)
diff --git a/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll b/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
index ed737651927a0..abe562d7ebbe3 100644
--- a/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
+++ b/llvm/test/Transforms/SampleProfile/uncompressed-profile-symbol-list.ll
@@ -1,4 +1,4 @@
 ; Append inline.prof with profile symbol list and save it without compression.
 ; RUN: llvm-profdata merge --sample --prof-sym-list=%S/Inputs/profile-symbol-list.text --compress-prof-sym-list=false --extbinary %S/Inputs/inline.prof --output=%t.profdata
-; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-sample-accurate -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
-; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-sample-accurate -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
+; RUN: opt < %S/Inputs/profile-symbol-list.ll -passes=sample-profile -profile-accurate-for-symsinlist -sample-profile-file=%t.profdata -S | FileCheck %S/Inputs/profile-symbol-list.ll
diff --git a/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
index 5f498d62d7eb6..4d7435ce3c254 100644
--- a/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
+++ b/llvm/test/Transforms/SimplifyCFG/PhiEliminate3.ll
@@ -7,55 +7,21 @@
 ; with various folding thresholds
 
 define i32 @test(i1 %a, i1 %b, i32 %i, i32 %j, i32 %k) {
-; CHECK-ONE-LABEL: @test(
-; CHECK-ONE-NEXT:  entry:
-; CHECK-ONE-NEXT:    br i1 [[A:%.*]], label [[M:%.*]], label [[O:%.*]]
-; CHECK-ONE:       O:
-; CHECK-ONE-NEXT:    br i1 [[B:%.*]], label [[P:%.*]], label [[Q:%.*]]
-; CHECK-ONE:       P:
-; CHECK-ONE-NEXT:    [[IAJ:%.*]] = add i32 [[I:%.*]], [[J:%.*]]
-; CHECK-ONE-NEXT:    [[IAJAK:%.*]] = add i32 [[IAJ]], [[K:%.*]]
-; CHECK-ONE-NEXT:    br label [[N:%.*]]
-; CHECK-ONE:       Q:
-; CHECK-ONE-NEXT:    [[IXJ:%.*]] = xor i32 [[I]], [[J]]
-; CHECK-ONE-NEXT:    [[IXJXK:%.*]] = xor i32 [[IXJ]], [[K]]
-; CHECK-ONE-NEXT:    br label [[N]]
-; CHECK-ONE:       N:
-; CHECK-ONE-NEXT:    [[WP:%.*]] = phi i32 [ [[IAJAK]], [[P]] ], [ [[IXJXK]], [[Q]] ]
-; CHECK-ONE-NEXT:    [[WP2:%.*]] = add i32 [[WP]], [[WP]]
-; CHECK-ONE-NEXT:    br label [[M]]
-; CHECK-ONE:       M:
-; CHECK-ONE-NEXT:    [[W:%.*]] = phi i32 [ [[WP2]], [[N]] ], [ 2, [[ENTRY:%.*]] ]
-; CHECK-ONE-NEXT:    [[R:%.*]] = add i32 [[W]], 1
-; CHECK-ONE-NEXT:    ret i32 [[R]]
-;
-; CHECK-TWO-LABEL: @test(
-; CHECK-TWO-NEXT:  entry:
-; CHECK-TWO-NEXT:    br i1 [[A:%.*]], label [[M:%.*]], label [[O:%.*]]
-; CHECK-TWO:       O:
-; CHECK-TWO-NEXT:    [[IAJ:%.*]] = add i32 [[I:%.*]], [[J:%.*]]
-; CHECK-TWO-NEXT:    [[IAJAK:%.*]] = add i32 [[IAJ]], [[K:%.*]]
-; CHECK-TWO-NEXT:    [[IXJ:%.*]] = xor i32 [[I]], [[J]]
-; CHECK-TWO-NEXT:    [[IXJXK:%.*]] = xor i32 [[IXJ]], [[K]]
-; CHECK-TWO-NEXT:    [[WP:%.*]] = select i1 [[B:%.*]], i32 [[IAJAK]], i32 [[IXJXK]]
-; CHECK-TWO-NEXT:    [[WP2:%.*]] = add i32 [[WP]], [[WP]]
-; CHECK-TWO-NEXT:    br label [[M]]
-; CHECK-TWO:       M:
-; CHECK-TWO-NEXT:    [[W:%.*]] = phi i32 [ [[WP2]], [[O]] ], [ 2, [[ENTRY:%.*]] ]
-; CHECK-TWO-NEXT:    [[R:%.*]] = add i32 [[W]], 1
-; CHECK-TWO-NEXT:    ret i32 [[R]]
-;
-; CHECK-SEVEN-LABEL: @test(
-; CHECK-SEVEN-NEXT:  entry:
-; CHECK-SEVEN-NEXT:    [[IAJ:%.*]] = add i32 [[I:%.*]], [[J:%.*]]
-; CHECK-SEVEN-NEXT:    [[IAJAK:%.*]] = add i32 [[IAJ]], [[K:%.*]]
-; CHECK-SEVEN-NEXT:    [[IXJ:%.*]] = xor i32 [[I]], [[J]]
-; CHECK-SEVEN-NEXT:    [[IXJXK:%.*]] = xor i32 [[IXJ]], [[K]]
-; CHECK-SEVEN-NEXT:    [[WP:%.*]] = select i1 [[B:%.*]], i32 [[IAJAK]], i32 [[IXJXK]]
-; CHECK-SEVEN-NEXT:    [[WP2:%.*]] = add i32 [[WP]], [[WP]]
-; CHECK-SEVEN-NEXT:    [[W:%.*]] = select i1 [[A:%.*]], i32 2, i32 [[WP2]]
-; CHECK-SEVEN-NEXT:    [[R:%.*]] = add i32 [[W]], 1
-; CHECK-SEVEN-NEXT:    ret i32 [[R]]
+; ALL-LABEL: @test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    br i1 [[A:%.*]], label [[M:%.*]], label [[O:%.*]]
+; ALL:       O:
+; ALL-NEXT:    [[IAJ:%.*]] = add i32 [[I:%.*]], [[J:%.*]]
+; ALL-NEXT:    [[IAJAK:%.*]] = add i32 [[IAJ]], [[K:%.*]]
+; ALL-NEXT:    [[IXJ:%.*]] = xor i32 [[I]], [[J]]
+; ALL-NEXT:    [[IXJXK:%.*]] = xor i32 [[IXJ]], [[K]]
+; ALL-NEXT:    [[WP:%.*]] = select i1 [[B:%.*]], i32 [[IAJAK]], i32 [[IXJXK]]
+; ALL-NEXT:    [[WP2:%.*]] = add i32 [[WP]], [[WP]]
+; ALL-NEXT:    br label [[M]]
+; ALL:       M:
+; ALL-NEXT:    [[W:%.*]] = phi i32 [ [[WP2]], [[O]] ], [ 2, [[ENTRY:%.*]] ]
+; ALL-NEXT:    [[R:%.*]] = add i32 [[W]], 1
+; ALL-NEXT:    ret i32 [[R]]
 ;
 entry:
   br i1 %a, label %M, label %O
diff --git a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
index e9e1d73a7b6ec..c21edd0c2ad8f 100644
--- a/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
+++ b/llvm/test/Transforms/SimplifyCFG/SpeculativeExec.ll
@@ -8,14 +8,10 @@ define i32 @test1(i32 %a, i32 %b, i32 %c) nounwind  {
 ; CHECK-LABEL: @test1(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[B:%.*]], 0
-; CHECK-NEXT:    br i1 [[TMP1]], label [[BB1:%.*]], label [[BB3:%.*]]
-; CHECK:       bb1:
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp sgt i32 [[C:%.*]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i32 [[A:%.*]], 1
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP2]], i32 [[TMP3]], i32 [[A]]
-; CHECK-NEXT:    br label [[BB3]]
-; CHECK:       bb3:
-; CHECK-NEXT:    [[TMP4:%.*]] = phi i32 [ [[B]], [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[BB1]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = select i1 [[TMP1]], i32 [[SPEC_SELECT]], i32 [[B]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i32 [[TMP4]], 1
 ; CHECK-NEXT:    ret i32 [[TMP5]]
 ;
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
index bc5567bc52975..4a7f7d7918483 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/merge-cond-stores-cost.ll
@@ -8,24 +8,16 @@ define void @test_costly(i32* %p, i32 %a, i32 %b, i32 %c, i32 %d) {
 ; CHECK-NEXT:    br i1 [[X1]], label [[FALLTHROUGH:%.*]], label [[YES1:%.*]]
 ; CHECK:       yes1:
 ; CHECK-NEXT:    [[VAL0:%.*]] = sdiv i32 [[D:%.*]], [[C:%.*]]
+; CHECK-NEXT:    store i32 [[VAL0]], i32* [[P:%.*]]
 ; CHECK-NEXT:    br label [[FALLTHROUGH]]
 ; CHECK:       fallthrough:
-; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE:%.*]] = phi i32 [ [[VAL0]], [[YES1]] ], [ undef, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[X2:%.*]] = icmp eq i32 [[B:%.*]], 0
 ; CHECK-NEXT:    br i1 [[X2]], label [[END:%.*]], label [[YES2:%.*]]
 ; CHECK:       yes2:
 ; CHECK-NEXT:    [[VAL1:%.*]] = sdiv i32 [[C]], [[D]]
+; CHECK-NEXT:    store i32 [[VAL1]], i32* [[P]]
 ; CHECK-NEXT:    br label [[END]]
 ; CHECK:       end:
-; CHECK-NEXT:    [[SIMPLIFYCFG_MERGE1:%.*]] = phi i32 [ [[VAL1]], [[YES2]] ], [ [[SIMPLIFYCFG_MERGE]], [[FALLTHROUGH]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = xor i1 [[X1]], true
-; CHECK-NEXT:    [[TMP1:%.*]] = xor i1 [[X2]], true
-; CHECK-NEXT:    [[TMP2:%.*]] = or i1 [[TMP0]], [[TMP1]]
-; CHECK-NEXT:    br i1 [[TMP2]], label [[TMP3:%.*]], label [[TMP4:%.*]]
-; CHECK:       3:
-; CHECK-NEXT:    store i32 [[SIMPLIFYCFG_MERGE1]], i32* [[P:%.*]], align 4
-; CHECK-NEXT:    br label [[TMP4]]
-; CHECK:       4:
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
index 11ba3984f35d4..3a2f067a2ee32 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/speculate-cttz-ctlz.ll
@@ -223,37 +223,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 ; for the target.
 
 define i64 @test1e(i32 %x) {
-; BMI-LABEL: @test1e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
-; BMI-NEXT:    ret i64 [[COND]]
-;
-; LZCNT-LABEL: @test1e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i64 [[COND]]
-;
-; GENERIC-LABEL: @test1e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i64 [[COND]]
+; ALL-LABEL: @test1e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
+; ALL-NEXT:    ret i64 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -270,37 +246,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i32 @test2e(i64 %x) {
-; BMI-LABEL: @test2e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
-; BMI-NEXT:    ret i32 [[COND]]
-;
-; LZCNT-LABEL: @test2e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i32 [[COND]]
-;
-; GENERIC-LABEL: @test2e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i32 [[COND]]
+; ALL-LABEL: @test2e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
+; ALL-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -317,37 +269,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i64 @test3e(i32 %x) {
-; BMI-LABEL: @test3e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i64 [[COND]]
-;
-; LZCNT-LABEL: @test3e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
-; LZCNT-NEXT:    ret i64 [[COND]]
-;
-; GENERIC-LABEL: @test3e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i64 [ [[PHITMP2]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i64 [[COND]]
+; ALL-LABEL: @test3e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[PHITMP2:%.*]] = zext i32 [[TMP0]] to i64
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i64 32, i64 [[PHITMP2]]
+; ALL-NEXT:    ret i64 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -364,37 +292,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i32 @test4e(i64 %x) {
-; BMI-LABEL: @test4e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i32 [[COND]]
-;
-; LZCNT-LABEL: @test4e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
-; LZCNT-NEXT:    ret i32 [[COND]]
-;
-; GENERIC-LABEL: @test4e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i32 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i32 [[COND]]
+; ALL-LABEL: @test4e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i32
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i32 64, i32 [[CAST]]
+; ALL-NEXT:    ret i32 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -411,37 +315,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test5e(i64 %x) {
-; BMI-LABEL: @test5e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test5e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test5e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test5e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.ctlz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -458,37 +338,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test6e(i32 %x) {
-; BMI-LABEL: @test6e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; BMI:       cond.true:
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; BMI-NEXT:    br label [[COND_END]]
-; BMI:       cond.end:
-; BMI-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test6e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; LZCNT-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test6e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test6e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.ctlz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
@@ -505,37 +361,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test7e(i64 %x) {
-; BMI-LABEL: @test7e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test7e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test7e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 64, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test7e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i64 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.cttz.i64(i64 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i64 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 64, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i64 %x, 0
@@ -552,37 +384,13 @@ cond.end:                                         ; preds = %entry, %cond.true
 }
 
 define i16 @test8e(i32 %x) {
-; BMI-LABEL: @test8e(
-; BMI-NEXT:  entry:
-; BMI-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; BMI-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; BMI-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; BMI-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
-; BMI-NEXT:    ret i16 [[COND]]
-;
-; LZCNT-LABEL: @test8e(
-; LZCNT-NEXT:  entry:
-; LZCNT-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; LZCNT-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; LZCNT:       cond.true:
-; LZCNT-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; LZCNT-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; LZCNT-NEXT:    br label [[COND_END]]
-; LZCNT:       cond.end:
-; LZCNT-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; LZCNT-NEXT:    ret i16 [[COND]]
-;
-; GENERIC-LABEL: @test8e(
-; GENERIC-NEXT:  entry:
-; GENERIC-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
-; GENERIC-NEXT:    br i1 [[TOBOOL]], label [[COND_END:%.*]], label [[COND_TRUE:%.*]]
-; GENERIC:       cond.true:
-; GENERIC-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
-; GENERIC-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
-; GENERIC-NEXT:    br label [[COND_END]]
-; GENERIC:       cond.end:
-; GENERIC-NEXT:    [[COND:%.*]] = phi i16 [ [[CAST]], [[COND_TRUE]] ], [ 32, [[ENTRY:%.*]] ]
-; GENERIC-NEXT:    ret i16 [[COND]]
+; ALL-LABEL: @test8e(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
+; ALL-NEXT:    [[TMP0:%.*]] = tail call i32 @llvm.cttz.i32(i32 [[X]], i1 true)
+; ALL-NEXT:    [[CAST:%.*]] = trunc i32 [[TMP0]] to i16
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[TOBOOL]], i16 32, i16 [[CAST]]
+; ALL-NEXT:    ret i16 [[COND]]
 ;
 entry:
   %tobool = icmp eq i32 %x, 0
diff --git a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
index c77438974b2d2..597b5b969a739 100644
--- a/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
+++ b/llvm/test/Transforms/SimplifyCFG/X86/switch_to_lookup_table.ll
@@ -1437,14 +1437,10 @@ define i32 @no_reuse_cmp2(i32 %x, i32 %y) {
 ; CHECK-LABEL: @no_reuse_cmp2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[EC:%.*]] = icmp ne i32 [[Y:%.*]], 0
-; CHECK-NEXT:    br i1 [[EC]], label [[SWITCH_ENTRY:%.*]], label [[SW_EPILOG:%.*]]
-; CHECK:       switch.entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp ult i32 [[X:%.*]], 4
 ; CHECK-NEXT:    [[SWITCH_OFFSET:%.*]] = add i32 [[X]], 10
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[TMP0]], i32 [[SWITCH_OFFSET]], i32 0
-; CHECK-NEXT:    br label [[SW_EPILOG]]
-; CHECK:       sw.epilog:
-; CHECK-NEXT:    [[R_0:%.*]] = phi i32 [ 100, [[ENTRY:%.*]] ], [ [[SPEC_SELECT]], [[SWITCH_ENTRY]] ]
+; CHECK-NEXT:    [[R_0:%.*]] = select i1 [[EC]], i32 [[SPEC_SELECT]], i32 100
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[R_0]], 0
 ; CHECK-NEXT:    [[DOTR_0:%.*]] = select i1 [[CMP]], i32 100, i32 [[R_0]]
 ; CHECK-NEXT:    ret i32 [[DOTR_0]]
diff --git a/llvm/test/Transforms/SimplifyCFG/safe-abs.ll b/llvm/test/Transforms/SimplifyCFG/safe-abs.ll
index 550fcefdde36f..6d8028f8d9433 100644
--- a/llvm/test/Transforms/SimplifyCFG/safe-abs.ll
+++ b/llvm/test/Transforms/SimplifyCFG/safe-abs.ll
@@ -8,14 +8,10 @@ define i32 @abs_with_clamp(i32 %arg) {
 ; CHECK-LABEL: @abs_with_clamp(
 ; CHECK-NEXT:  begin:
 ; CHECK-NEXT:    [[IS_POSITIVE:%.*]] = icmp sgt i32 [[ARG:%.*]], 0
-; CHECK-NEXT:    br i1 [[IS_POSITIVE]], label [[END:%.*]], label [[NEGATIVE:%.*]]
-; CHECK:       negative:
 ; CHECK-NEXT:    [[IS_INT_MIN:%.*]] = icmp eq i32 [[ARG]], -2147483648
 ; CHECK-NEXT:    [[NEGATED:%.*]] = sub nsw i32 0, [[ARG]]
 ; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[IS_INT_MIN]], i32 2147483647, i32 [[NEGATED]]
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[TMP6:%.*]] = phi i32 [ [[ARG]], [[BEGIN:%.*]] ], [ [[ABS]], [[NEGATIVE]] ]
+; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[IS_POSITIVE]], i32 [[ARG]], i32 [[ABS]]
 ; CHECK-NEXT:    ret i32 [[TMP6]]
 ;
 begin:
diff --git a/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll b/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
index 354cfff7ba72a..19a6313aa1d0d 100644
--- a/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
+++ b/llvm/test/Transforms/SimplifyCFG/safe-low-bit-extract.ll
@@ -9,14 +9,10 @@ define i32 @extract_low_bits(i32 %input, i32 %nbits) {
 ; CHECK-LABEL: @extract_low_bits(
 ; CHECK-NEXT:  begin:
 ; CHECK-NEXT:    [[SHOULD_MASK:%.*]] = icmp ult i32 [[NBITS:%.*]], 32
-; CHECK-NEXT:    br i1 [[SHOULD_MASK]], label [[PERFORM_MASKING:%.*]], label [[END:%.*]]
-; CHECK:       perform_masking:
 ; CHECK-NEXT:    [[MASK_NOT:%.*]] = shl nsw i32 -1, [[NBITS]]
 ; CHECK-NEXT:    [[MASK:%.*]] = xor i32 [[MASK_NOT]], -1
 ; CHECK-NEXT:    [[MASKED:%.*]] = and i32 [[MASK]], [[INPUT:%.*]]
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[MASKED]], [[PERFORM_MASKING]] ], [ [[INPUT]], [[BEGIN:%.*]] ]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[SHOULD_MASK]], i32 [[MASKED]], i32 [[INPUT]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 begin:
diff --git a/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll b/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
index ea013ef9e6c6b..e955e0679dc12 100644
--- a/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
+++ b/llvm/test/Transforms/SimplifyCFG/signbit-like-value-extension.ll
@@ -11,14 +11,10 @@ define i32 @extend_value(i32 %storage, i32 %nbits) {
 ; CHECK-NEXT:    [[SKIPNBITS:%.*]] = sub i32 32, [[NBITS:%.*]]
 ; CHECK-NEXT:    [[VALUE:%.*]] = lshr i32 [[STORAGE:%.*]], [[SKIPNBITS]]
 ; CHECK-NEXT:    [[SHOULDEXTEND:%.*]] = icmp sgt i32 [[STORAGE]], -1
-; CHECK-NEXT:    br i1 [[SHOULDEXTEND]], label [[EXTEND:%.*]], label [[END:%.*]]
-; CHECK:       extend:
 ; CHECK-NEXT:    [[HIGHBITMASK:%.*]] = shl nsw i32 -1, [[NBITS]]
 ; CHECK-NEXT:    [[HIGHBITMASKPLUSONE:%.*]] = add nsw i32 [[HIGHBITMASK]], 1
 ; CHECK-NEXT:    [[EXTENDED:%.*]] = add i32 [[HIGHBITMASKPLUSONE]], [[VALUE]]
-; CHECK-NEXT:    br label [[END]]
-; CHECK:       end:
-; CHECK-NEXT:    [[RES:%.*]] = phi i32 [ [[EXTENDED]], [[EXTEND]] ], [ [[VALUE]], [[BB:%.*]] ]
+; CHECK-NEXT:    [[RES:%.*]] = select i1 [[SHOULDEXTEND]], i32 [[EXTENDED]], i32 [[VALUE]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
 bb:
diff --git a/llvm/test/Transforms/SimplifyCFG/speculate-math.ll b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
index 9edffafa23626..e3fe5ed2fda78 100644
--- a/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
+++ b/llvm/test/Transforms/SimplifyCFG/speculate-math.ll
@@ -12,23 +12,12 @@ declare float @llvm.minimum.f32(float, float) nounwind readonly
 declare float @llvm.maximum.f32(float, float) nounwind readonly
 
 define double @fdiv_test(double %a, double %b) {
-; EXPENSIVE-LABEL: @fdiv_test(
-; EXPENSIVE-NEXT:  entry:
-; EXPENSIVE-NEXT:    [[CMP:%.*]] = fcmp ogt double [[A:%.*]], 0.000000e+00
-; EXPENSIVE-NEXT:    [[DIV:%.*]] = fdiv double [[B:%.*]], [[A]]
-; EXPENSIVE-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[DIV]], double 0.000000e+00
-; EXPENSIVE-NEXT:    ret double [[COND]]
-;
-; CHEAP-LABEL: @fdiv_test(
-; CHEAP-NEXT:  entry:
-; CHEAP-NEXT:    [[CMP:%.*]] = fcmp ogt double [[A:%.*]], 0.000000e+00
-; CHEAP-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_END:%.*]]
-; CHEAP:       cond.true:
-; CHEAP-NEXT:    [[DIV:%.*]] = fdiv double [[B:%.*]], [[A]]
-; CHEAP-NEXT:    br label [[COND_END]]
-; CHEAP:       cond.end:
-; CHEAP-NEXT:    [[COND:%.*]] = phi double [ [[DIV]], [[COND_TRUE]] ], [ 0.000000e+00, [[ENTRY:%.*]] ]
-; CHEAP-NEXT:    ret double [[COND]]
+; ALL-LABEL: @fdiv_test(
+; ALL-NEXT:  entry:
+; ALL-NEXT:    [[CMP:%.*]] = fcmp ogt double [[A:%.*]], 0.000000e+00
+; ALL-NEXT:    [[DIV:%.*]] = fdiv double [[B:%.*]], [[A]]
+; ALL-NEXT:    [[COND:%.*]] = select i1 [[CMP]], double [[DIV]], double 0.000000e+00
+; ALL-NEXT:    ret double [[COND]]
 ;
 entry:
   %cmp = fcmp ogt double %a, 0.0
diff --git a/llvm/test/Transforms/StackProtector/X86/lit.local.cfg b/llvm/test/Transforms/StackProtector/X86/lit.local.cfg
deleted file mode 100644
index c8625f4d9d248..0000000000000
--- a/llvm/test/Transforms/StackProtector/X86/lit.local.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-if not 'X86' in config.root.targets:
-    config.unsupported = True
diff --git a/llvm/test/Transforms/Util/flattencfg.ll b/llvm/test/Transforms/Util/flattencfg.ll
index 4fcb77ab02371..3b31476df0b0b 100644
--- a/llvm/test/Transforms/Util/flattencfg.ll
+++ b/llvm/test/Transforms/Util/flattencfg.ll
@@ -24,3 +24,63 @@ b1:                                       ; preds = %entry, %b0
 exit:                               ; preds = %entry, %b0, %b1
   ret void
 }
+
+; CHECK-LABEL: @test_not_crash2
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %0 = fcmp ult float %a
+; CHECK-NEXT:    %1 = fcmp ult float %b
+; CHECK-NEXT:    [[COND:%[a-z0-9]+]] = or i1 %0, %1
+; CHECK-NEXT:    br i1 [[COND]], label %bb4, label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    br label %bb4
+; CHECK:       bb4:
+; CHECK-NEXT:    ret void
+define void @test_not_crash2(float %a, float %b) #0 {
+entry:
+  %0 = fcmp ult float %a, 1.000000e+00
+  br i1 %0, label %bb0, label %bb1
+
+bb3:                                               ; preds = %bb0
+  br label %bb4
+
+bb4:                                               ; preds = %bb0, %bb3
+  ret void
+
+bb1:                                               ; preds = %entry
+  br label %bb0
+
+bb0:                                               ; preds = %bb1, %entry
+  %1 = fcmp ult float %b, 1.000000e+00
+  br i1 %1, label %bb4, label %bb3
+}
+
+; CHECK-LABEL: @test_not_crash3
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    %a_eq_0 = icmp eq i32 %a, 0
+; CHECK-NEXT:    %a_eq_1 = icmp eq i32 %a, 1
+; CHECK-NEXT:    [[COND:%[a-z0-9]+]] = or i1 %a_eq_0, %a_eq_1
+; CHECK-NEXT:    br i1 [[COND]], label %bb2, label %bb3
+; CHECK:       bb2:
+; CHECK-NEXT:    br label %bb3
+; CHECK:       bb3:
+; CHECK-NEXT:    %check_badref = phi i32 [ 17, %entry ], [ 11, %bb2 ]
+; CHECK-NEXT:    ret void
+define void @test_not_crash3(i32 %a) #0 {
+entry:
+  %a_eq_0 = icmp eq i32 %a, 0
+  br i1 %a_eq_0, label %bb0, label %bb1
+
+bb0:                                              ; preds = %entry
+  br label %bb1
+
+bb1:                                              ; preds = %bb0, %entry
+  %a_eq_1 = icmp eq i32 %a, 1
+  br i1 %a_eq_1, label %bb2, label %bb3
+
+bb2:                                              ; preds = %bb1
+  br label %bb3
+
+bb3:                                              ; preds = %bb2, %bb1
+  %check_badref = phi i32 [ 17, %bb1 ], [ 11, %bb2 ]
+  ret void
+}
diff --git a/llvm/test/Verifier/Mips/intrinsic-immarg.ll b/llvm/test/Verifier/Mips/intrinsic-immarg.ll
deleted file mode 100644
index dfe5aa94ab97f..0000000000000
--- a/llvm/test/Verifier/Mips/intrinsic-immarg.ll
+++ /dev/null
@@ -1,82 +0,0 @@
-; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
-
-define void @ld_b(<16 x i8> * %ptr, i8 * %ldptr, i32 %offset) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: %a = call <16 x i8> @llvm.mips.ld.b(i8* %ldptr, i32 %offset)
-  %a = call <16 x i8> @llvm.mips.ld.b(i8* %ldptr, i32 %offset)
-  store <16 x i8> %a, <16 x i8> * %ptr, align 16
-  ret void
-}
-
-define void @st_b(<16 x i8> * %ptr, i8 * %ldptr, i32 %offset, i8 * %stptr) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: call void @llvm.mips.st.b(<16 x i8> %a, i8* %stptr, i32 %offset)
-  %a = call <16 x i8> @llvm.mips.ld.b(i8* %ldptr, i32 0)
-  call void @llvm.mips.st.b(<16 x i8> %a, i8* %stptr, i32 %offset)
-  ret void
-}
-
-define void @ld_w(<4 x i32> * %ptr, i8 * %ldptr, i32 %offset) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: %a = call <4 x i32> @llvm.mips.ld.w(i8* %ldptr, i32 %offset)
-  %a = call <4 x i32> @llvm.mips.ld.w(i8* %ldptr, i32 %offset)
-  store <4 x i32> %a, <4 x i32> * %ptr, align 16
-  ret void
-}
-
-define void @st_w(<8 x i16> * %ptr, i8 * %ldptr, i32 %offset, i8 * %stptr) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: call void @llvm.mips.st.w(<4 x i32> %a, i8* %stptr, i32 %offset)
-  %a = call <4 x i32> @llvm.mips.ld.w(i8* %ldptr, i32 0)
-  call void @llvm.mips.st.w(<4 x i32> %a, i8* %stptr, i32 %offset)
-  ret void
-}
-
-define void @ld_h(<8 x i16> * %ptr, i8 * %ldptr, i32 %offset) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: %a = call <8 x i16> @llvm.mips.ld.h(i8* %ldptr, i32 %offset)
-  %a = call <8 x i16> @llvm.mips.ld.h(i8* %ldptr, i32 %offset)
-  store <8 x i16> %a, <8 x i16> * %ptr, align 16
-  ret void
-}
-
-define void @st_h(<8 x i16> * %ptr, i8 * %ldptr, i32 %offset, i8 * %stptr) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: call void @llvm.mips.st.h(<8 x i16> %a, i8* %stptr, i32 %offset)
-  %a = call <8 x i16> @llvm.mips.ld.h(i8* %ldptr, i32 0)
-  call void @llvm.mips.st.h(<8 x i16> %a, i8* %stptr, i32 %offset)
-  ret void
-}
-
-define void @ld_d(<2 x i64> * %ptr, i8 * %ldptr, i32 %offset) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: %a = call <2 x i64> @llvm.mips.ld.d(i8* %ldptr, i32 %offset)
-  %a = call <2 x i64> @llvm.mips.ld.d(i8* %ldptr, i32 %offset)
-  store <2 x i64> %a, <2 x i64> * %ptr, align 16
-  ret void
-}
-
-define void @st_d(<2 x i64> * %ptr, i8 * %ldptr, i32 %offset, i8 * %stptr) {
-  ; CHECK: immarg operand has non-immediate parameter
-  ; CHECK-NEXT: i32 %offset
-  ; CHECK-NEXT: call void @llvm.mips.st.d(<2 x i64> %a, i8* %stptr, i32 %offset)
-  %a = call <2 x i64> @llvm.mips.ld.d(i8* %ldptr, i32 0)
-  call void @llvm.mips.st.d(<2 x i64> %a, i8* %stptr, i32 %offset)
-  ret void
-}
-
-declare <16 x i8> @llvm.mips.ld.b(i8*, i32)
-declare <8 x i16> @llvm.mips.ld.h(i8*, i32)
-declare <4 x i32> @llvm.mips.ld.w(i8*, i32)
-declare <2 x i64> @llvm.mips.ld.d(i8*, i32)
-declare void @llvm.mips.st.b(<16 x i8>, i8*, i32)
-declare void @llvm.mips.st.h(<8 x i16>, i8*, i32)
-declare void @llvm.mips.st.w(<4 x i32>, i8*, i32)
-declare void @llvm.mips.st.d(<2 x i64>, i8*, i32)
diff --git a/llvm/test/Verifier/blockbyref.ll b/llvm/test/Verifier/blockbyref.ll
index 037da7f095210..14d705647b74a 100644
--- a/llvm/test/Verifier/blockbyref.ll
+++ b/llvm/test/Verifier/blockbyref.ll
@@ -1,6 +1,6 @@
 ; RUN: llvm-as -disable-output <%s 2>&1| FileCheck %s
 
-; CHECK: BlockByRef variable without complex expression
+; CHECK: DIBlockByRefStruct on DICompositeType is no longer supported
 ; CHECK: warning: ignoring invalid debug info
 
 define void @foo() {
@@ -16,4 +16,4 @@ declare void @llvm.dbg.declare(metadata, metadata, metadata)
 !0 = !{i32 2, !"Debug Info Version", i32 3}
 !1 = distinct !DISubprogram()
 !2 = !DILocalVariable(scope: !1, type: !3)
-!3 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagBlockByrefStruct)
+!3 = !DICompositeType(tag: DW_TAG_structure_type, flags: DIFlagReservedBit4)
diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll
new file mode 100644
index 0000000000000..403cc573d0dd0
--- /dev/null
+++ b/llvm/test/Verifier/callbr.ll
@@ -0,0 +1,50 @@
+; RUN: not opt -S %s -verify 2>&1 | FileCheck %s
+
+; CHECK: Indirect label missing from arglist.
+define void @foo() {
+  ; The %4 in the indirect label list is not found in the blockaddresses in the
+  ; arg list (bad).
+  callbr void asm sideeffect "${0:l} {1:l}", "X,X"(i8* blockaddress(@foo, %3), i8* blockaddress(@foo, %2))
+  to label %1 [label %4, label %2]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
+
+; CHECK-NOT: Indirect label missing from arglist.
+define void @bar() {
+  ; %4 and %2 are both in the indirect label list and the arg list (good).
+  callbr void asm sideeffect "${0:l} ${1:l}", "X,X"(i8* blockaddress(@bar, %4), i8* blockaddress(@bar, %2))
+  to label %1 [label %4, label %2]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
+
+; CHECK-NOT: Indirect label missing from arglist.
+define void @baz() {
+  ; note %2 blockaddress. Such a case is possible when passing the address of
+  ; a label as an input to the inline asm (both address of label and asm goto
+  ; use blockaddress constants; we're testing that the indirect label list from
+  ; the asm goto is in the arg list to the asm).
+  callbr void asm sideeffect "${0:l} ${1:l} ${2:l}", "X,X,X"(i8* blockaddress(@baz, %4), i8* blockaddress(@baz, %2), i8* blockaddress(@baz, %3))
+  to label %1 [label %3, label %4]
+1:
+  ret void
+2:
+  ret void
+3:
+  ret void
+4:
+  ret void
+}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 6771b9e7fee8c..4a2181397ced1 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -124,6 +124,11 @@ def get_asan_rtlib():
 
 opt_viewer_cmd = '%s %s/tools/opt-viewer/opt-viewer.py' % (sys.executable, config.llvm_src_root)
 
+llvm_locstats_tool = os.path.join(config.llvm_tools_dir, 'llvm-locstats')
+config.substitutions.append(
+    ('%llvm-locstats', "'%s' %s" % (config.python_executable, llvm_locstats_tool)))
+config.llvm_locstats_used = os.path.exists(llvm_locstats_tool)
+
 tools = [
     ToolSubst('%lli', FindTool('lli'), post='.', extra_args=lli_args),
     ToolSubst('%llc_dwarf', FindTool('llc'), extra_args=llc_args),
diff --git a/llvm/test/lit.site.cfg.py.in b/llvm/test/lit.site.cfg.py.in
index 39096a80e7e03..c7e5748e867fe 100644
--- a/llvm/test/lit.site.cfg.py.in
+++ b/llvm/test/lit.site.cfg.py.in
@@ -29,7 +29,8 @@ config.llvm_bindings = "@LLVM_BINDINGS@".split(' ')
 config.host_os = "@HOST_OS@"
 config.host_cc = "@HOST_CC@"
 config.host_cxx = "@HOST_CXX@"
-config.host_ldflags = "@HOST_LDFLAGS@"
+# Note: ldflags can contain double-quoted paths, so must use single quotes here.
+config.host_ldflags = '@HOST_LDFLAGS@'
 config.llvm_use_intel_jitevents = @LLVM_USE_INTEL_JITEVENTS@
 config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
 config.have_zlib = @HAVE_LIBZ@
diff --git a/llvm/test/Object/Inputs/mri-crlf.mri b/llvm/test/tools/llvm-ar/Inputs/mri-crlf.mri
similarity index 100%
rename from llvm/test/Object/Inputs/mri-crlf.mri
rename to llvm/test/tools/llvm-ar/Inputs/mri-crlf.mri
diff --git a/llvm/test/tools/llvm-ar/count.test b/llvm/test/tools/llvm-ar/count.test
index 81b30bc1d6c10..fa6141ff13f39 100644
--- a/llvm/test/tools/llvm-ar/count.test
+++ b/llvm/test/tools/llvm-ar/count.test
@@ -31,9 +31,9 @@ RUN: not llvm-ar xN 4 %t/archive.a foo.txt 2>&1 | FileCheck %s --check-prefix=ER
 # N only applies to x/d.
 RUN: not llvm-ar rN 1 %t/archive.a foo.txt 2>&1 | FileCheck %s --check-prefix=ERR-BAD-OP
 
-ERR-NOT-NUM:   error: Value for [count] must be numeric
-ERR-NOT-POS:   error: Value for [count] must be positive
-ERR-BAD-OP:    error: The 'N' modifier can only be specified with the 'x' or 'd' operations
+ERR-NOT-NUM:   error: value for [count] must be numeric
+ERR-NOT-POS:   error: value for [count] must be positive
+ERR-BAD-OP:    error: the 'N' modifier can only be specified with the 'x' or 'd' operations
 ERR-NOT-FOUND: error: 'foo.txt' was not found
 
 # Extract individual items.
diff --git a/llvm/test/tools/llvm-ar/help-message.test b/llvm/test/tools/llvm-ar/help-message.test
new file mode 100644
index 0000000000000..ed17667291dce
--- /dev/null
+++ b/llvm/test/tools/llvm-ar/help-message.test
@@ -0,0 +1,6 @@
+# RUN: llvm-ar h | FileCheck %s
+# RUN: llvm-ar -h | FileCheck %s
+# RUN: llvm-ar -help | FileCheck %s
+# RUN: llvm-ar --help | FileCheck %s
+
+# CHECK: USAGE: llvm-ar
diff --git a/llvm/test/tools/llvm-ar/insert-after.test b/llvm/test/tools/llvm-ar/insert-after.test
index 7c01e736149aa..a3f51a6d4d792 100644
--- a/llvm/test/tools/llvm-ar/insert-after.test
+++ b/llvm/test/tools/llvm-ar/insert-after.test
@@ -43,7 +43,7 @@ RUN: not llvm-ar ra invalid.txt %t-invalid.a %t2.txt 2>&1 \
 RUN:   | FileCheck %s --check-prefix=ERROR
 RUN: llvm-ar t %t-invalid.a | FileCheck %s --check-prefix=INVALID
 
-ERROR: error: Insertion point not found.
+ERROR: error: insertion point not found
 INVALID:      1.txt
 INVALID-NEXT: 2.txt
 INVALID-NEXT: 3.txt
diff --git a/llvm/test/tools/llvm-ar/insert-before.test b/llvm/test/tools/llvm-ar/insert-before.test
index 4abf3a000287c..f2a6bdd4e170f 100644
--- a/llvm/test/tools/llvm-ar/insert-before.test
+++ b/llvm/test/tools/llvm-ar/insert-before.test
@@ -43,7 +43,7 @@ RUN: not llvm-ar rb invalid.txt %t-invalid.a %t2.txt 2>&1 \
 RUN:   | FileCheck %s --check-prefix=ERROR
 RUN: llvm-ar t %t-invalid.a | FileCheck %s --check-prefix=INVALID
 
-ERROR: error: Insertion point not found.
+ERROR: error: insertion point not found
 INVALID:      1.txt
 INVALID-NEXT: 2.txt
 INVALID-NEXT: 3.txt
diff --git a/llvm/test/tools/llvm-ar/invalid-command-line.test b/llvm/test/tools/llvm-ar/invalid-command-line.test
index 2bfcc1a457dbd..0dcd36a9c4e2d 100644
--- a/llvm/test/tools/llvm-ar/invalid-command-line.test
+++ b/llvm/test/tools/llvm-ar/invalid-command-line.test
@@ -1,5 +1,5 @@
 Test that llvm-ar exits with 1 when there is an error.
 
 RUN: not llvm-ar e 2>&1 | FileCheck %s
-CHECK: unknown option e.
+CHECK: error: unknown option e
 CHECK: OVERVIEW: LLVM Archiver
diff --git a/llvm/test/tools/llvm-ar/invalid-object-file.test b/llvm/test/tools/llvm-ar/invalid-object-file.test
new file mode 100644
index 0000000000000..d967d8763a921
--- /dev/null
+++ b/llvm/test/tools/llvm-ar/invalid-object-file.test
@@ -0,0 +1,8 @@
+## Test that we get a proper error message with a file that is not a recognized object file.
+
+# RUN: rm -f %t.txt
+# RUN: touch %t.txt
+# RUN: not llvm-ar sc %t.txt 2>&1 > /dev/null | FileCheck %s -DFILE=%t.txt
+
+# CHECK:     error: unable to load '[[FILE]]': file too small to be an archive
+# CHECK-NOT: {{.}}
diff --git a/llvm/test/tools/llvm-ar/move-after.test b/llvm/test/tools/llvm-ar/move-after.test
index 871d120c73b78..8bd83216c6895 100644
--- a/llvm/test/tools/llvm-ar/move-after.test
+++ b/llvm/test/tools/llvm-ar/move-after.test
@@ -43,7 +43,7 @@ RUN: not llvm-ar ma invalid.txt %t-invalid.a %t2.txt 2>&1 \
 RUN:   | FileCheck %s --check-prefix=ERROR
 RUN: llvm-ar t %t-invalid.a | FileCheck %s --check-prefix=INVALID
 
-ERROR: error: Insertion point not found.
+ERROR: error: insertion point not found
 INVALID:      1.txt
 INVALID-NEXT: 2.txt
 INVALID-NEXT: 3.txt
diff --git a/llvm/test/tools/llvm-ar/move-before.test b/llvm/test/tools/llvm-ar/move-before.test
index 15e27359d0e40..bab8592f8f978 100644
--- a/llvm/test/tools/llvm-ar/move-before.test
+++ b/llvm/test/tools/llvm-ar/move-before.test
@@ -43,7 +43,7 @@ RUN: not llvm-ar mb invalid.txt %t-invalid.a %t2.txt 2>&1 \
 RUN:   | FileCheck %s --check-prefix=ERROR
 RUN: llvm-ar t %t-invalid.a | FileCheck %s --check-prefix=INVALID
 
-ERROR: error: Insertion point not found.
+ERROR: error: insertion point not found
 INVALID:      1.txt
 INVALID-NEXT: 2.txt
 INVALID-NEXT: 3.txt
diff --git a/llvm/test/tools/llvm-ar/move.test b/llvm/test/tools/llvm-ar/move.test
index 268fbf94cf19f..e62b8ccd609d2 100644
--- a/llvm/test/tools/llvm-ar/move.test
+++ b/llvm/test/tools/llvm-ar/move.test
@@ -77,14 +77,14 @@
 # RUN: not llvm-ar m 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=NO-ARCHIVE
 
-# NO-ARCHIVE: error: An archive name must be specified.
+# NO-ARCHIVE: error: an archive name must be specified
 
 ## Member does not exist:
 # RUN: llvm-ar rc %t/missing.a %t/1.o %t/2.o %t/3.o
 # RUN: not llvm-ar m %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory.
+# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-ar/mri-addlib.test b/llvm/test/tools/llvm-ar/mri-addlib.test
index cf62acbf0b063..ee692ea9fbddd 100644
--- a/llvm/test/tools/llvm-ar/mri-addlib.test
+++ b/llvm/test/tools/llvm-ar/mri-addlib.test
@@ -22,10 +22,10 @@
 # RUN: not llvm-ar -M < %t/badlib.mri 2>&1 | FileCheck --check-prefix=PARSE %s
 # RUN: not ls %t/badlib.a
 
-# PARSE: Could not parse library
+# PARSE: error: script line 2: could not parse library
 
 ## No create command.
-# RUN: echo "ADDLIB %t/f.a"  > %t/nocreate.mri
+# RUN: echo "ADDLIB %t/f.a" > %t/nocreate.mri
 # RUN: echo "SAVE" >> %t/nocreate.mri
 # RUN: not llvm-ar -M < %t/nocreate.mri
 
diff --git a/llvm/test/Object/mri-crlf.test b/llvm/test/tools/llvm-ar/mri-crlf.test
similarity index 100%
rename from llvm/test/Object/mri-crlf.test
rename to llvm/test/tools/llvm-ar/mri-crlf.test
diff --git a/llvm/test/tools/llvm-ar/mri-errors.test b/llvm/test/tools/llvm-ar/mri-errors.test
new file mode 100644
index 0000000000000..a6980f27caff3
--- /dev/null
+++ b/llvm/test/tools/llvm-ar/mri-errors.test
@@ -0,0 +1,41 @@
+## Test different MRI comment formats and whitespace.
+
+# RUN: rm -rf %t && mkdir -p %t
+
+# RUN: echo "create %t/mri.ar" > %t/script1.mri
+# RUN: echo "addlib %t/missing.a" >> %t/script1.mri
+
+# RUN: not llvm-ar -M < %t/script1.mri 2>&1 | FileCheck --check-prefix=SCRIPT1 %s
+# SCRIPT1: error: script line 2: could not open library
+
+# RUN: echo "create %t/mri.ar" > %t/script2.mri
+# RUN: echo "addlib %t/bad.a" >> %t/script2.mri
+
+# RUN: echo "bad archive" > %t/bad.a
+
+# RUN: not llvm-ar -M < %t/script2.mri 2>&1 | FileCheck --check-prefix=SCRIPT2 %s
+# SCRIPT2: error: script line 2: could not parse library
+
+# RUN: echo "create %t/mri.ar" > %t/script3.mri
+# RUN: echo "create %t/second.ar" >> %t/script3.mri
+
+# RUN: not llvm-ar -M < %t/script3.mri 2>&1 | FileCheck --check-prefix=SCRIPT3 %s
+# SCRIPT3: error: script line 2: editing multiple archives not supported
+
+# RUN: echo "save" > %t/script4.mri
+# RUN: echo "create %t/mri.ar" >> %t/script4.mri
+
+# RUN: not llvm-ar -M < %t/script4.mri 2>&1 | FileCheck --check-prefix=SCRIPT4 %s
+# SCRIPT4: error: script line 2: file already saved
+
+# RUN: echo "create %t/mri.ar" > %t/script5.mri
+# RUN: echo "bad command" >> %t/script5.mri
+
+# RUN: not llvm-ar -M < %t/script5.mri 2>&1 | FileCheck --check-prefix=SCRIPT5 %s
+# SCRIPT5: error: script line 2: unknown command: bad
+
+# RUN: echo "bad command" | not llvm-ar -M 2>&1 | FileCheck --check-prefix=SCRIPT6 %s
+# SCRIPT6: error: script line 1: unknown command: bad
+
+# RUN: not llvm-ar -M rc %t/mri.ar 2>&1 | FileCheck --check-prefix=SCRIPT7 %s
+# SCRIPT7: error: cannot mix -M and other options
diff --git a/llvm/test/Object/mri1.test b/llvm/test/tools/llvm-ar/mri1.test
similarity index 100%
rename from llvm/test/Object/mri1.test
rename to llvm/test/tools/llvm-ar/mri1.test
diff --git a/llvm/test/Object/mri2.test b/llvm/test/tools/llvm-ar/mri2.test
similarity index 70%
rename from llvm/test/Object/mri2.test
rename to llvm/test/tools/llvm-ar/mri2.test
index 0c24179341779..2e90171c6acb8 100644
--- a/llvm/test/Object/mri2.test
+++ b/llvm/test/tools/llvm-ar/mri2.test
@@ -4,4 +4,4 @@
 ; RUN: echo end >> %t.mri
 
 ; RUN: not llvm-ar -M  < %t.mri 2>&1 | FileCheck %s
-; CHECK: Editing multiple archives not supported
+; CHECK: error: script line 2: editing multiple archives not supported
diff --git a/llvm/test/Object/mri3.test b/llvm/test/tools/llvm-ar/mri3.test
similarity index 73%
rename from llvm/test/Object/mri3.test
rename to llvm/test/tools/llvm-ar/mri3.test
index bdc53991510e8..697878bfff4b7 100644
--- a/llvm/test/Object/mri3.test
+++ b/llvm/test/tools/llvm-ar/mri3.test
@@ -3,4 +3,4 @@
 ; RUN: echo end >> %t.mri
 
 ; RUN: not llvm-ar -M  < %t.mri 2>&1 | FileCheck %s
-; CHECK: File already saved.
+; CHECK: error: script line 2: file already saved
diff --git a/llvm/test/Object/mri4.test b/llvm/test/tools/llvm-ar/mri4.test
similarity index 60%
rename from llvm/test/Object/mri4.test
rename to llvm/test/tools/llvm-ar/mri4.test
index a24c14d7d2896..bdca3c9d57e13 100644
--- a/llvm/test/Object/mri4.test
+++ b/llvm/test/tools/llvm-ar/mri4.test
@@ -1,4 +1,4 @@
 ; RUN: echo abc > %t.mri
 
 ; RUN: not llvm-ar -M  < %t.mri 2>&1 | FileCheck %s
-; CHECK: Unknown command: abc.
+; CHECK: error: script line 1: unknown command: abc
diff --git a/llvm/test/Object/mri5.test b/llvm/test/tools/llvm-ar/mri5.test
similarity index 50%
rename from llvm/test/Object/mri5.test
rename to llvm/test/tools/llvm-ar/mri5.test
index 98114248b69c3..4c39349903ee1 100644
--- a/llvm/test/Object/mri5.test
+++ b/llvm/test/tools/llvm-ar/mri5.test
@@ -1,2 +1,2 @@
 ; RUN: not llvm-ar -M t < %s 2>&1 | FileCheck %s
-; CHECK: Cannot mix -M and other options.
+; CHECK: error: cannot mix -M and other options
diff --git a/llvm/test/tools/llvm-ar/print.test b/llvm/test/tools/llvm-ar/print.test
index 4081a36ac4eb3..ec7fdc4137d29 100644
--- a/llvm/test/tools/llvm-ar/print.test
+++ b/llvm/test/tools/llvm-ar/print.test
@@ -69,13 +69,13 @@ RUN:   | FileCheck %s --check-prefix=SAME -DFILE=%t/2.txt
 RUN: not llvm-ar p 2>&1 \
 RUN:   | FileCheck %s --check-prefix=NO-ARCHIVE
 
-NO-ARCHIVE: error: An archive name must be specified.
+NO-ARCHIVE: error: an archive name must be specified
 
 # Archive does not exist:
 RUN: not llvm-ar p %t/missing.a 2>&1 \
 RUN:   | FileCheck %s --check-prefix=MISSING-ARCHIVE -DARCHIVE=%t/missing.a
 
-MISSING-ARCHIVE: error: error loading '[[ARCHIVE]]': {{[nN]}}o such file or directory.
+MISSING-ARCHIVE: error: error loading '[[ARCHIVE]]': {{[nN]}}o such file or directory
 
 # Member does not exist:
 RUN: not llvm-ar p %t/archive.a %t-missing.txt 2>&1 \
diff --git a/llvm/test/tools/llvm-ar/quick-append.test b/llvm/test/tools/llvm-ar/quick-append.test
index 607ba413ae09d..857b01a7aa174 100644
--- a/llvm/test/tools/llvm-ar/quick-append.test
+++ b/llvm/test/tools/llvm-ar/quick-append.test
@@ -54,13 +54,13 @@
 # RUN: not llvm-ar qc 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=NO-ARCHIVE
 
-# NO-ARCHIVE: error: An archive name must be specified.
+# NO-ARCHIVE: error: an archive name must be specified
 
 ## Member does not exist:
 # RUN: not llvm-ar qc %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory.
+# MISSING-FILE: error: [[FILE]]: {{[nN]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-ar/read-only-archive.test b/llvm/test/tools/llvm-ar/read-only-archive.test
index 1d638f45cac36..0365f6ad054da 100644
--- a/llvm/test/tools/llvm-ar/read-only-archive.test
+++ b/llvm/test/tools/llvm-ar/read-only-archive.test
@@ -13,7 +13,7 @@
 # RUN: not llvm-ar r %t/archive.a %t/3.txt \
 # RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a
 
-# ERROR: error: [[FILE]]: no such file or directory.
+# ERROR: error: [[FILE]]: no such file or directory
 
 # RUN: not llvm-ar q %t/archive.a %t/3.txt \
 # RUN:   FileCheck %s --check-prefix=ERROR -DFILE=%t/archive.a
@@ -21,7 +21,7 @@
 # RUN: not llvm-ar m %t/archive.a t/1.txt \
 # RUN:   FileCheck %s --check-prefix=ERROR-2 -DFILE=%t/archive.a
 
-# ERROR-2: error: [[FILE]]: permission denied.
+# ERROR-2: error: [[FILE]]: permission denied
 
 # RUN: llvm-ar t %t/archive.a \
 # RUN:   | FileCheck %s --check-prefix=ARCHIVE --match-full-lines --implicit-check-not {{.}}
diff --git a/llvm/test/tools/llvm-ar/replace.test b/llvm/test/tools/llvm-ar/replace.test
index 0c1722a23ef5a..9e38d5a7502cf 100644
--- a/llvm/test/tools/llvm-ar/replace.test
+++ b/llvm/test/tools/llvm-ar/replace.test
@@ -84,14 +84,14 @@
 # RUN: not llvm-ar r 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=NO-ARCHIVE
 
-# NO-ARCHIVE: error: An archive name must be specified.
+# NO-ARCHIVE: error: an archive name must be specified
 
 ## Member does not exist:
 # RUN: llvm-ar rc %t/missing.a %t/1.o %t/2.o %t/3.o
 # RUN: not llvm-ar r %t/missing.a %t/missing.txt 2>&1 \
 # RUN:   | FileCheck %s --check-prefix=MISSING-FILE -DFILE=%t/missing.txt
 
-# MISSING-FILE: error: [[FILE]]: {{[Nn]}}o such file or directory.
+# MISSING-FILE: error: [[FILE]]: {{[Nn]}}o such file or directory
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/debug_loc_base_address.s b/llvm/test/tools/llvm-dwarfdump/X86/debug_loc_base_address.s
new file mode 100644
index 0000000000000..fde942b2ebf8d
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/debug_loc_base_address.s
@@ -0,0 +1,34 @@
+# RUN: llvm-mc %s -filetype obj -triple x86_64-pc-linux -o %t.o
+# RUN: llvm-dwarfdump --debug-loc %t.o | FileCheck %s
+
+# CHECK:         .debug_loc contents:
+# CHECK-NEXT:    0x00000000:
+# CHECK-NEXT:    [0xffffffffffffffff, 0x000000000000002a):
+# CHECK-NEXT:    [0x0000000000000003, 0x0000000000000007): DW_OP_consts +3, DW_OP_stack_value
+
+	.section	.debug_loc,"",@progbits
+	.quad	0xffffffffffffffff
+	.quad	42
+	.quad	3
+	.quad	7
+	.short	3                       # Loc expr size
+	.byte	17                      # DW_OP_consts
+	.byte	3                       # 3
+	.byte	159                     # DW_OP_stack_value
+	.quad	0
+	.quad	0
+	.section	.debug_abbrev,"",@progbits
+	.byte	1                       # Abbreviation Code
+	.byte	17                      # DW_TAG_compile_unit
+	.byte	0                       # DW_CHILDREN_no
+	.byte	0                       # EOM(1)
+	.byte	0                       # EOM(2)
+	.byte	0                       # EOM(3)
+	.section	.debug_info,"",@progbits
+	.long	.Ldebug_info_end0-.Ldebug_info_start0 # Length of Unit
+.Ldebug_info_start0:
+	.short	4                       # DWARF version number
+	.long	.debug_abbrev           # Offset Into Abbrev. Section
+	.byte	8                       # Address Size (in bytes)
+	.byte	1                       # Abbrev [1] DW_TAG_compile_unit
+.Ldebug_info_end0:
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
index e88667390ea7b..a295ad8e402e7 100644
--- a/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
+++ b/llvm/test/tools/llvm-dwarfdump/X86/locstats.ll
@@ -71,7 +71,7 @@
 ; CHECK: "vars with 80-89% of its scope covered":1
 ; CHECK: "vars with 90-99% of its scope covered":0
 ; CHECK: "vars with 100% of its scope covered":1
-; CHECK: "vars (excluding the debug entry values) with 0% of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with 0% of its scope covered":1
 ; CHECK: "vars (excluding the debug entry values) with 1-9% of its scope covered":0
 ; CHECK: "vars (excluding the debug entry values) with 10-19% of its scope covered":0
 ; CHECK: "vars (excluding the debug entry values) with 20-29% of its scope covered":0
@@ -80,9 +80,9 @@
 ; CHECK: "vars (excluding the debug entry values) with 50-59% of its scope covered":1
 ; CHECK: "vars (excluding the debug entry values) with 60-69% of its scope covered":0
 ; CHECK: "vars (excluding the debug entry values) with 70-79% of its scope covered":0
-; CHECK: "vars (excluding the debug entry values) with 80-89% of its scope covered":0
+; CHECK: "vars (excluding the debug entry values) with 80-89% of its scope covered":1
 ; CHECK: "vars (excluding the debug entry values) with 90-99% of its scope covered":0
-; CHECK: "vars (excluding the debug entry values) with 100% of its scope covered":1}
+; CHECK: "vars (excluding the debug entry values) with 100% of its scope covered":1
 ;
 ; The source code of the test case:
 ; extern void fn3(int *);
diff --git a/llvm/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s b/llvm/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s
index 49cb84792034d..61c4280db749c 100644
--- a/llvm/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s
+++ b/llvm/test/tools/llvm-exegesis/X86/inverse_throughput-by-opcode-name.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=inverse_throughput -opcode-name=ADD32rr | FileCheck %s
+# RUN: llvm-exegesis -mode=inverse_throughput -opcode-name=ADD32rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=inverse_throughput -opcode-name=ADD32rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      ---
 CHECK-NEXT: mode: inverse_throughput
diff --git a/llvm/test/tools/llvm-exegesis/X86/latency-CMOV32rr.s b/llvm/test/tools/llvm-exegesis/X86/latency-CMOV32rr.s
index 8eec5c0c9ed16..5def574faaa8d 100644
--- a/llvm/test/tools/llvm-exegesis/X86/latency-CMOV32rr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/latency-CMOV32rr.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=latency -opcode-name=CMOV32rr | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=CMOV32rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=CMOV32rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
diff --git a/llvm/test/tools/llvm-exegesis/X86/latency-SBB8rr.s b/llvm/test/tools/llvm-exegesis/X86/latency-SBB8rr.s
index d98d547cbd3bb..4ac8330f84fe0 100644
--- a/llvm/test/tools/llvm-exegesis/X86/latency-SBB8rr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/latency-SBB8rr.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=latency -opcode-name=SBB8rr | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=SBB8rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=SBB8rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
diff --git a/llvm/test/tools/llvm-exegesis/X86/latency-by-opcode-name.s b/llvm/test/tools/llvm-exegesis/X86/latency-by-opcode-name.s
index da808b73ab5d3..f69f263935005 100644
--- a/llvm/test/tools/llvm-exegesis/X86/latency-by-opcode-name.s
+++ b/llvm/test/tools/llvm-exegesis/X86/latency-by-opcode-name.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=latency -opcode-name=ADD32rr | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=ADD32rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=latency -opcode-name=ADD32rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      ---
 CHECK-NEXT: mode: latency
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mi8.s b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mi8.s
index e3b3b80efc258..6aef789b3c602 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mi8.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mi8.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mi8 | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mi8 -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mi8 -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mr.s b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mr.s
index 80ecb3033b0fc..0c3628f03a0c4 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32mr.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mr | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32mr -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32rm.s b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32rm.s
index 0e6bdb587b9a1..b63c6e7c04a04 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-ADD32rm.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-ADD32rm.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rm | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rm -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rm -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-BEXTR32rm.s b/llvm/test/tools/llvm-exegesis/X86/uops-BEXTR32rm.s
index c4d2c7d840f12..a33023cb36d6a 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-BEXTR32rm.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-BEXTR32rm.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=BEXTR32rm | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BEXTR32rm -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BEXTR32rm -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-BSF16rm.s b/llvm/test/tools/llvm-exegesis/X86/uops-BSF16rm.s
index 9cb278dc0524a..fdf99ff0ff041 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-BSF16rm.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-BSF16rm.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=BSF16rm | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BSF16rm -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BSF16rm -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-BTR64mr.s b/llvm/test/tools/llvm-exegesis/X86/uops-BTR64mr.s
index 6d4544b5c521f..da622e445a545 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-BTR64mr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-BTR64mr.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=BTR64mr | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BTR64mr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=BTR64mr -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-VFMADDSS4rm.s b/llvm/test/tools/llvm-exegesis/X86/uops-VFMADDSS4rm.s
index c323395ef5b0a..84a928cb23d8a 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-VFMADDSS4rm.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-VFMADDSS4rm.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=VFMADDSS4rm | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=VFMADDSS4rm -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=VFMADDSS4rm -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-XCHG64rr.s b/llvm/test/tools/llvm-exegesis/X86/uops-XCHG64rr.s
index 66bf57e9094a9..c73b2bc7b5529 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-XCHG64rr.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-XCHG64rr.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=XCHG64rr | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=XCHG64rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=XCHG64rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-exegesis/X86/uops-by-opcode-name.s b/llvm/test/tools/llvm-exegesis/X86/uops-by-opcode-name.s
index 982bd4fdf40a8..5d13f20630391 100644
--- a/llvm/test/tools/llvm-exegesis/X86/uops-by-opcode-name.s
+++ b/llvm/test/tools/llvm-exegesis/X86/uops-by-opcode-name.s
@@ -1,4 +1,5 @@
-# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rr | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rr -repetition-mode=duplicate | FileCheck %s
+# RUN: llvm-exegesis -mode=uops -opcode-name=ADD32rr -repetition-mode=loop | FileCheck %s
 
 CHECK:      mode:            uops
 CHECK-NEXT: key:
diff --git a/llvm/test/tools/llvm-lib/duplicate.test b/llvm/test/tools/llvm-lib/duplicate.test
new file mode 100644
index 0000000000000..3c503ca7f58f2
--- /dev/null
+++ b/llvm/test/tools/llvm-lib/duplicate.test
@@ -0,0 +1,14 @@
+If the same file is specified more than once as an input file,
+llvm-lib should ignore all but the first occurrence of the file.
+
+RUN: rm -rf %t
+RUN: mkdir -p %t
+
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/foo.o %S/Inputs/a.s
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/bar.o %S/Inputs/b.s
+RUN: llvm-lib -out:%t/foo.lib %t/foo.o %t/foo.o %t/bar.o
+
+RUN: llvm-ar t %t/foo.lib | FileCheck %s
+CHECK: foo.o
+CHECK-NOT: foo.o
+CHECK: bar.o
diff --git a/llvm/test/tools/llvm-lib/invalid.test b/llvm/test/tools/llvm-lib/invalid.test
index 2978177a431e2..57266400cdc87 100644
--- a/llvm/test/tools/llvm-lib/invalid.test
+++ b/llvm/test/tools/llvm-lib/invalid.test
@@ -1,2 +1,2 @@
 RUN: not llvm-lib %S/Inputs/cl-gl.obj 2>&1 | FileCheck %s
-CHECK: not a COFF object, bitcode or resource file
+CHECK: not a COFF object, bitcode, archive or resource file
diff --git a/llvm/test/tools/llvm-lib/nest.test b/llvm/test/tools/llvm-lib/nest.test
new file mode 100644
index 0000000000000..627c847b13339
--- /dev/null
+++ b/llvm/test/tools/llvm-lib/nest.test
@@ -0,0 +1,15 @@
+If an archive file is specified as an input file, its members
+are added to an output file. This test verifies that beahvior.
+
+RUN: rm -rf %t
+RUN: mkdir -p %t
+
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/foo.o %S/Inputs/a.s
+RUN: llvm-lib -out:%t/foo.lib %t/foo.o
+
+RUN: llvm-mc -triple=x86_64-pc-windows-msvc -filetype=obj -o %t/bar.o %S/Inputs/b.s
+RUN: llvm-lib -out:%t/bar.lib %t/foo.lib %t/bar.o
+
+RUN: llvm-ar t %t/bar.lib | FileCheck %s
+CHECK: foo.o
+CHECK: bar.o
diff --git a/llvm/test/tools/llvm-lipo/create-arch.test b/llvm/test/tools/llvm-lipo/create-arch.test
new file mode 100644
index 0000000000000..bae216566de65
--- /dev/null
+++ b/llvm/test/tools/llvm-lipo/create-arch.test
@@ -0,0 +1,17 @@
+# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o
+# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal.o
+# RUN: llvm-lipo %t-i386.o -arch x86_64 %t-x86_64.o -create -output %t-universal-1.o
+# RUN: cmp %t-universal.o %t-universal-1.o
+# RUN: llvm-lipo -arch i386 %t-i386.o -arch x86_64 %t-x86_64.o -create -output %t-universal-2.o
+# RUN: cmp %t-universal.o %t-universal-2.o
+#
+# RUN: not llvm-lipo -arch armv7 %t-i386.o -arch x86_64 %t-x86_64.o -create -output /dev/null 2>&1 | FileCheck --check-prefix=ARCH_NOT_MATCHED %s
+# ARCH_NOT_MATCHED: error: specified architecture: armv7 for file: {{.*}} does not match the file's architecture (i386)
+#
+# # RUN: not llvm-lipo -arch i3866 %t-32.o -create -o /dev/null 2>&1 | FileCheck --check-prefix=INVALID_ARCH %s
+# INVALID_ARCH: error: Invalid architecture: i3866
+#
+# RUN: not llvm-lipo -arch i386 %t-33.o -create -o /dev/null 2>&1 | FileCheck --check-prefix=INVALID_FILE %s
+# INVALID_FILE: {{[nN]}}o such file or directory
diff --git a/llvm/test/tools/llvm-lipo/create-archive-input.test b/llvm/test/tools/llvm-lipo/create-archive-input.test
new file mode 100644
index 0000000000000..8f4307e9c8303
--- /dev/null
+++ b/llvm/test/tools/llvm-lipo/create-archive-input.test
@@ -0,0 +1,26 @@
+# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o
+# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o
+# RUN: yaml2obj %p/Inputs/i386-x86_64-universal.yaml > %t-universal.o
+
+# RUN: llvm-ar cr %t.empty.a
+# RUN: not llvm-lipo %t.empty.a -create -output /dev/null 2>&1 | FileCheck --check-prefix=EMPTY-ARCHIVE %s
+
+# RUN: llvm-ar cr %t.different_architectures.a %t-i386.o %t-x86_64.o
+# RUN: not llvm-lipo %t.different_architectures.a -create -output /dev/null 2>&1 | FileCheck --check-prefix=ARCHIVE-WITH-DIFFERENT-ARCHS %s
+
+# RUN: llvm-ar cr %t.contains_fat_binary.a %t-universal.o 
+# RUN: not llvm-lipo %t.contains_fat_binary.a -create -output /dev/null 2>&1 | FileCheck --check-prefix=ARCHIVE-WITH-FAT-BINARY %s
+
+# RUN: llvm-ar cr %t-i386-lib.a %t-i386.o
+# RUN: llvm-lipo %t-i386-lib.a %t-x86_64.o -create -output %t-i386-x86_64-universal.o
+# RUN: llvm-lipo %t-i386-x86_64-universal.o -info | FileCheck --check-prefix=INFO-i386-x86_64 %s
+# RUN: llvm-lipo %t-i386-x86_64-universal.o -thin i386 -output %t-extracted-i386-lib.a
+# RUN: cmp %t-extracted-i386-lib.a %t-i386-lib.a
+# RUN: llvm-lipo %t-i386-x86_64-universal.o -thin x86_64 -output %t-extracted-x86_64.o
+# RUN: cmp %t-extracted-x86_64.o %t-x86_64.o
+#
+# EMPTY-ARCHIVE: empty archive
+# ARCHIVE-WITH-DIFFERENT-ARCHS: all members must match
+# ARCHIVE-WITH-FAT-BINARY: fat file (not allowed in an archive)
+#
+# INFO-i386-x86_64: i386 x86_64
diff --git a/llvm/test/tools/llvm-lipo/replace-invalid-input.test b/llvm/test/tools/llvm-lipo/replace-invalid-input.test
index b1c5fa2cd7789..38e9805bcba7d 100644
--- a/llvm/test/tools/llvm-lipo/replace-invalid-input.test
+++ b/llvm/test/tools/llvm-lipo/replace-invalid-input.test
@@ -5,20 +5,20 @@
 # RUN: not llvm-lipo %t-universal.o -replace %t-32.o 2>&1 | FileCheck --check-prefix=MISSING_ARG %s
 # MISSING_ARG: error: replace is missing an argument: expects -replace arch_type file_name
 
-# RUN: not llvm-lipo %t-universal.o -replace %t-32.o i386 2>&1 | FileCheck --check-prefix=OUTPUT_FILE %s
+# RUN: not llvm-lipo %t-universal.o -replace i386 %t-32.o 2>&1 | FileCheck --check-prefix=OUTPUT_FILE %s
 # OUTPUT_FILE: error: replace expects a single output file to be specified
 
-# RUN: not llvm-lipo %t-universal.o %t-universal.o -replace %t-32.o i386 -o %t.o 2>&1 | FileCheck --check-prefix=INPUT_ARGS %s
+# RUN: not llvm-lipo %t-universal.o %t-universal.o -replace i386 %t-32.o -o %t.o 2>&1 | FileCheck --check-prefix=INPUT_ARGS %s
 # INPUT_ARGS: error: replace expects a single input file
 
-# RUN: not llvm-lipo %t-universal.o -replace %t-32.o i386 -o %t.o 2>&1 | FileCheck --check-prefix=INVALID_FILE %s
-# INVALID_FILE: error: 'i386': {{[nN]}}o such file or directory
+# RUN: not llvm-lipo %t-universal.o -replace i386 %t-33.o  -o %t.o 2>&1 | FileCheck --check-prefix=INVALID_FILE %s
+# INVALID_FILE: {{[nN]}}o such file or directory
 
 # RUN: not llvm-lipo %t-universal.o -replace i3866 %t-32.o -o %t.o 2>&1 | FileCheck --check-prefix=INVALID_ARCH %s
 # INVALID_ARCH: error: Invalid architecture: i3866
 
 # RUN: not llvm-lipo %t-universal.o -replace arm64 %t-32.o -o %t.o 2>&1 | FileCheck --check-prefix=ARCH_NOT_MATCHED %s
-# ARCH_NOT_MATCHED: error: specified architecture: arm64 for replacement file: {{.*}} does not match the file's architecture
+# ARCH_NOT_MATCHED: error: specified architecture: arm64 for file: {{.*}} does not match the file's architecture (i386)
 
 # RUN: not llvm-lipo %t-universal.o -replace arm64 %t-arm64.o -o %t.o 2>&1 | FileCheck --check-prefix=ARCH_NOT_IN_INPUT %s
 # ARCH_NOT_IN_INPUT: error: -replace arm64 <file_name> specified but fat file: {{.*}} does not contain that architecture
diff --git a/llvm/test/tools/llvm-locstats/lit.local.cfg b/llvm/test/tools/llvm-locstats/lit.local.cfg
new file mode 100644
index 0000000000000..0f2d477e161bb
--- /dev/null
+++ b/llvm/test/tools/llvm-locstats/lit.local.cfg
@@ -0,0 +1,2 @@
+if not config.llvm_locstats_used:
+    config.unsupported = True
diff --git a/llvm/test/tools/llvm-locstats/locstats.ll b/llvm/test/tools/llvm-locstats/locstats.ll
new file mode 100644
index 0000000000000..394d5129df7dd
--- /dev/null
+++ b/llvm/test/tools/llvm-locstats/locstats.ll
@@ -0,0 +1,175 @@
+; UNSUPPORTED: system-windows
+; REQUIRES: x86-registered-target
+; RUN: llc %s -o %t0.o -filetype=obj
+; RUN: %llvm-locstats %t0.o | FileCheck %s --check-prefix=LOCSTATS
+;
+; Test the llvm-locstats output.
+; LOCSTATS: 0% 0 0%
+; LOCSTATS: 1-9% 0 0%
+; LOCSTATS: 10-19% 0 0%
+; LOCSTATS: 20-29% 1 11%
+; LOCSTATS: 30-39% 0 0%
+; LOCSTATS: 40-49% 1 11%
+; LOCSTATS: 50-59% 1 11%
+; LOCSTATS: 60-69% 1 11%
+; LOCSTATS: 70-79% 0 0%
+; LOCSTATS: 80-89% 2 22%
+; LOCSTATS: 90-99% 1 11%
+; LOCSTATS: 100% 2 22%
+;
+; The source code of the test case:
+;extern int fn2 (int);
+;
+;__attribute__((noinline))
+;int
+;fn1 (int *x, int *y)
+;{
+;  int a = *x;
+;  int b = *y;
+;  int local = a + b;
+;  if (a > 1) {
+;    local += 2;
+;    ++local;
+;    if (local > 200)
+;      local -= fn2(a);
+;  } else {
+;    local += 3;
+;    ++local;
+;    local += fn2(a);
+; }
+;  if (b > 4)
+;   local += a;
+;  int local2 = 7;
+;  local -= fn2 (local2);
+;  return local;
+;}
+;
+;__attribute__((noinline))
+;int f()
+;{
+;  int l, k;
+;  int res = 0;
+;  res += fn1 (&l, &k);
+;  return res;
+;}
+;
+; ModuleID = 'locstats.c'
+source_filename = "locstats.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @fn1(i32* nocapture readonly %0, i32* nocapture readonly %1) local_unnamed_addr !dbg !7 {
+  call void @llvm.dbg.value(metadata i32* %0, metadata !13, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32* %1, metadata !14, metadata !DIExpression()), !dbg !19
+  %3 = load i32, i32* %0, align 4, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %3, metadata !15, metadata !DIExpression()), !dbg !19
+  %4 = load i32, i32* %1, align 4, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %4, metadata !16, metadata !DIExpression()), !dbg !19
+  %5 = add nsw i32 %4, %3, !dbg !20
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression()), !dbg !19
+  %6 = icmp sgt i32 %3, 1, !dbg !20
+  br i1 %6, label %7, label %13, !dbg !22
+
+7:                                                ; preds = %2
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression(DW_OP_plus_uconst, 2, DW_OP_stack_value)), !dbg !19
+  %8 = add nsw i32 %5, 3, !dbg !23
+  call void @llvm.dbg.value(metadata i32 %8, metadata !17, metadata !DIExpression()), !dbg !19
+  %9 = icmp sgt i32 %8, 200, !dbg !25
+  br i1 %9, label %10, label %17, !dbg !27
+
+10:                                               ; preds = %7
+  %11 = tail call i32 @fn2(i32 %3), !dbg !27
+  %12 = sub nsw i32 %8, %11, !dbg !27
+  call void @llvm.dbg.value(metadata i32 %12, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %17, !dbg !27
+
+13:                                               ; preds = %2
+  call void @llvm.dbg.value(metadata i32 %5, metadata !17, metadata !DIExpression(DW_OP_plus_uconst, 3, DW_OP_stack_value)), !dbg !19
+  %14 = add nsw i32 %5, 4, !dbg !28
+  call void @llvm.dbg.value(metadata i32 %14, metadata !17, metadata !DIExpression()), !dbg !19
+  %15 = tail call i32 @fn2(i32 %3), !dbg !30
+  %16 = add nsw i32 %14, %15, !dbg !30
+  call void @llvm.dbg.value(metadata i32 %16, metadata !17, metadata !DIExpression()), !dbg !19
+  br label %17
+
+17:                                               ; preds = %7, %10, %13
+  %18 = phi i32 [ %12, %10 ], [ %8, %7 ], [ %16, %13 ], !dbg !31
+  call void @llvm.dbg.value(metadata i32 %18, metadata !17, metadata !DIExpression()), !dbg !19
+  %19 = icmp sgt i32 %4, 4, !dbg !32
+  %20 = select i1 %19, i32 %3, i32 0, !dbg !34
+  %21 = add nsw i32 %18, %20, !dbg !34
+  call void @llvm.dbg.value(metadata i32 %21, metadata !17, metadata !DIExpression()), !dbg !19
+  call void @llvm.dbg.value(metadata i32 7, metadata !18, metadata !DIExpression()), !dbg !19
+  %22 = tail call i32 @fn2(i32 7), !dbg !34
+  %23 = sub i32 %21, %22, !dbg !34
+  call void @llvm.dbg.value(metadata i32 %23, metadata !17, metadata !DIExpression()), !dbg !19
+  ret i32 %23, !dbg !34
+}
+
+declare dso_local i32 @fn2(i32) local_unnamed_addr
+
+; Function Attrs: noinline nounwind uwtable
+define dso_local i32 @f() local_unnamed_addr !dbg !35 {
+  %1 = alloca i32, align 4
+  %2 = alloca i32, align 4
+  %3 = bitcast i32* %1 to i8*, !dbg !42
+  %4 = bitcast i32* %2 to i8*, !dbg !42
+  call void @llvm.dbg.value(metadata i32 0, metadata !41, metadata !DIExpression()), !dbg !42
+  call void @llvm.dbg.value(metadata i32* %1, metadata !39, metadata !DIExpression(DW_OP_deref)), !dbg !42
+  call void @llvm.dbg.value(metadata i32* %2, metadata !40, metadata !DIExpression(DW_OP_deref)), !dbg !42
+  %5 = call i32 @fn1(i32* nonnull %1, i32* nonnull %2), !dbg !42
+  call void @llvm.dbg.value(metadata i32 %5, metadata !41, metadata !DIExpression()), !dbg !42
+  ret i32 %5, !dbg !42
+}
+
+; Function Attrs: nounwind readnone speculatable willreturn
+declare void @llvm.dbg.value(metadata, metadata, metadata)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+!llvm.ident = !{!6}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 10.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, nameTableKind: None)
+!1 = !DIFile(filename: "locstats.c", directory: "/dir")
+!2 = !{}
+!3 = !{i32 2, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = !{!"clang version 10.0.0"}
+!7 = distinct !DISubprogram(name: "fn1", scope: !1, file: !1, line: 5, type: !8, scopeLine: 6, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !12)
+!8 = !DISubroutineType(types: !9)
+!9 = !{!10, !11, !11}
+!10 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !10, size: 64)
+!12 = !{!13, !14, !15, !16, !17, !18}
+!13 = !DILocalVariable(name: "x", arg: 1, scope: !7, file: !1, line: 5, type: !11)
+!14 = !DILocalVariable(name: "y", arg: 2, scope: !7, file: !1, line: 5, type: !11)
+!15 = !DILocalVariable(name: "a", scope: !7, file: !1, line: 7, type: !10)
+!16 = !DILocalVariable(name: "b", scope: !7, file: !1, line: 8, type: !10)
+!17 = !DILocalVariable(name: "local", scope: !7, file: !1, line: 9, type: !10)
+!18 = !DILocalVariable(name: "local2", scope: !7, file: !1, line: 22, type: !10)
+!19 = !DILocation(line: 0, scope: !7)
+!20 = !DILocation(line: 7, column: 11, scope: !7)
+!21 = distinct !DILexicalBlock(scope: !7, file: !1, line: 10, column: 7)
+!22 = !DILocation(line: 10, column: 7, scope: !7)
+!23 = !DILocation(line: 12, column: 5, scope: !24)
+!24 = distinct !DILexicalBlock(scope: !21, file: !1, line: 10, column: 14)
+!25 = !DILocation(line: 13, column: 15, scope: !26)
+!26 = distinct !DILexicalBlock(scope: !24, file: !1, line: 13, column: 9)
+!27 = !DILocation(line: 13, column: 9, scope: !24)
+!28 = !DILocation(line: 17, column: 5, scope: !26)
+!29 = distinct !DILexicalBlock(scope: !21, file: !1, line: 15, column: 10)
+!30 = !DILocation(line: 18, column: 14, scope: !29)
+!31 = !DILocation(line: 0, scope: !21)
+!32 = !DILocation(line: 20, column: 9, scope: !33)
+!33 = distinct !DILexicalBlock(scope: !7, file: !1, line: 20, column: 7)
+!34 = !DILocation(line: 20, column: 7, scope: !7)
+!35 = distinct !DISubprogram(name: "f", scope: !1, file: !1, line: 28, type: !36, scopeLine: 29, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !38)
+!36 = !DISubroutineType(types: !37)
+!37 = !{!10}
+!38 = !{!39, !40, !41}
+!39 = !DILocalVariable(name: "l", scope: !35, file: !1, line: 30, type: !10)
+!40 = !DILocalVariable(name: "k", scope: !35, file: !1, line: 30, type: !10)
+!41 = !DILocalVariable(name: "res", scope: !35, file: !1, line: 31, type: !10)
+!42 = !DILocation(line: 30, column: 3, scope: !35)
diff --git a/llvm/test/tools/llvm-mca/ARM/m4-targetfeatures.s b/llvm/test/tools/llvm-mca/ARM/m4-targetfeatures.s
new file mode 100644
index 0000000000000..033d7046d66d0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/ARM/m4-targetfeatures.s
@@ -0,0 +1,29 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=arm-none-none-eabi -mcpu=cortex-m4 -mattr=+fp64 -instruction-tables < %s | FileCheck %s
+
+vadd.f32 s0, s2, s2
+vadd.f64 d0, d2, d2
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        vadd.f32	s0, s2, s2
+# CHECK-NEXT:  1      1     1.00                        vadd.f64	d0, d2, d2
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - M4Unit
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]
+# CHECK-NEXT: 2.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    Instructions:
+# CHECK-NEXT: 1.00   vadd.f32	s0, s2, s2
+# CHECK-NEXT: 1.00   vadd.f64	d0, d2, d2
diff --git a/llvm/test/tools/llvm-mca/X86/SkylakeClient/bottleneck-analysis.s b/llvm/test/tools/llvm-mca/X86/SkylakeClient/bottleneck-analysis.s
index 1f5e2dbf46903..98c5376ff824f 100644
--- a/llvm/test/tools/llvm-mca/X86/SkylakeClient/bottleneck-analysis.s
+++ b/llvm/test/tools/llvm-mca/X86/SkylakeClient/bottleneck-analysis.s
@@ -67,18 +67,14 @@
 # CHECK-NEXT:  |      11.   vmovaps	%xmm7, %xmm4
 # CHECK-NEXT:  +----> 12.   vfmadd213ps	%xmm1, %xmm14, %xmm2      ## REGISTER dependency:  %xmm1
 # CHECK-NEXT:  |      13.   vmovaps	%xmm6, %xmm1
-# CHECK-NEXT:  |      14.   vfmadd213ps	%xmm2, %xmm15, %xmm3
-# CHECK-NEXT:  |      15.   vpermilps	$170, %xmm3, %xmm0
+# CHECK-NEXT:  +----> 14.   vfmadd213ps	%xmm2, %xmm15, %xmm3      ## REGISTER dependency:  %xmm2
+# CHECK-NEXT:  +----> 15.   vpermilps	$170, %xmm3, %xmm0                ## REGISTER dependency:  %xmm3
 # CHECK-NEXT:  |      16.   vmovups	%xmm3, (%rdx,%rax)
 # CHECK-NEXT:  |      17.   vpermilps	$255, %xmm3, %xmm2
 # CHECK-NEXT:  |      18.   addq	$16, %rax
 # CHECK-NEXT:  |      19.   decl	%ecx
-# CHECK-NEXT:  |      20.   vmovaps	%xmm0, %xmm3
-# CHECK-NEXT:  |      21.   jne	.LBB0_4
-# CHECK-NEXT:  |
-# CHECK-NEXT:  |    < loop carried >
-# CHECK-NEXT:  |
-# CHECK-NEXT:  +----> 2.    vmulps	-24(%rsp), %xmm7, %xmm8           ## RESOURCE interference:  SKLPort1 [ probability: 45% ]
+# CHECK-NEXT:  +----> 20.   vmovaps	%xmm0, %xmm3                      ## REGISTER dependency:  %xmm0
+# CHECK-NEXT:         21.   jne	.LBB0_4
 
 # CHECK:      Instruction Info:
 # CHECK-NEXT: [1]: #uOps
diff --git a/llvm/test/tools/llvm-objcopy/ELF/binary-input-and-output.test b/llvm/test/tools/llvm-objcopy/ELF/binary-input-and-output.test
index 0fc2a4bc3c17c..68331f23986b3 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/binary-input-and-output.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/binary-input-and-output.test
@@ -3,13 +3,21 @@
 # Preserve input to verify it is not modified
 # RUN: cp %t.txt %t-copy.txt
 
-# -I binary -O binary preserves payload through in-memory representation
-# RUN: llvm-objcopy -I binary -B i386:x86-64 -O binary %t.txt %t.2.txt
+## -I binary -O binary preserves payload through in-memory representation.
+# RUN: llvm-objcopy -I binary -O binary %t.txt %t.2.txt
 # RUN: cmp %t.txt %t.2.txt
 # RUN: cmp %t.txt %t-copy.txt
 
-# -I binary -O binary preserves payload through an intermediate object file
-# RUN: llvm-objcopy -I binary -B i386:x86-64 %t.txt %t.o
-# RUN: llvm-objcopy -O binary %t.o %t.3.txt
+## If -O is not specified, it defaults to -I, i.e. "binary".
+# RUN: llvm-objcopy -I binary %t.txt %t.3.txt
 # RUN: cmp %t.txt %t.3.txt
-# RUN: cmp %t.txt %t-copy.txt
+
+## -I binary -O elf* creates an ELF. -O binary transforms it back to the
+## original file.
+# RUN: llvm-objcopy -I binary -O elf64-x86-64 %t.txt %t.o
+# RUN: llvm-objcopy -O binary %t.o %t.4.txt
+# RUN: cmp %t.txt %t.4.txt
+
+## -B is ignored.
+# RUN: llvm-objcopy -I binary -B i386:x86-64 -O binary %t.txt %t.5.txt
+# RUN: cmp %t.txt %t.5.txt
diff --git a/llvm/test/tools/llvm-objcopy/ELF/binary-input-error.test b/llvm/test/tools/llvm-objcopy/ELF/binary-input-error.test
index 08cc22c750e9e..40ce3edce0152 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/binary-input-error.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/binary-input-error.test
@@ -1,10 +1,4 @@
 # RUN: echo abcd > %t.txt
 
-# RUN: not llvm-objcopy -I binary %t.txt %t.o 2>&1 \
-# RUN:   | FileCheck %s --check-prefix=MISSING-BINARY-ARCH
-
-# RUN: not llvm-objcopy -I binary -B xyz %t.txt %t.o 2>&1 \
-# RUN:   | FileCheck %s --check-prefix=BAD-BINARY-ARCH
-
-# MISSING-BINARY-ARCH: specified binary input without specifiying an architecture
-# BAD-BINARY-ARCH: invalid architecture: 'xyz'
+## -B is ignored.
+# RUN: llvm-objcopy -I binary -B xyz %t.txt %t.o
diff --git a/llvm/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test b/llvm/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test
deleted file mode 100644
index ff2e82c6917bc..0000000000000
--- a/llvm/test/tools/llvm-objcopy/ELF/binary-input-with-arch.test
+++ /dev/null
@@ -1,26 +0,0 @@
-# RUN: echo -n abcd > %t.x-txt
-# Preserve input to verify it is not modified.
-# RUN: cp %t.x-txt %t-copy.txt
-# RUN: llvm-objcopy -I binary -B i386 -O elf64-x86-64 %t.x-txt %t.o
-# RUN: llvm-readobj --file-headers %t.o | FileCheck %s --check-prefixes=CHECK,SYSV
-# RUN: cmp %t.x-txt %t-copy.txt
-
-# Many uses of objcopy use no spaces in the flags, make sure that also works.
-# RUN: llvm-objcopy -Ibinary -Bi386 -Oelf64-x86-64 %t.x-txt %t-no-spaces.o
-# RUN: cmp %t.o %t-no-spaces.o
-
-# elf64-x86-64-freebsd sets the OS/ABI field.
-# RUN: llvm-objcopy -I binary -B i386 -O elf64-x86-64-freebsd %t.x-txt %t.freebsd.o
-# RUN: llvm-readobj --file-headers %t.freebsd.o | FileCheck %s --check-prefixes=CHECK,FREEBSD
-
-# CHECK:      Format: ELF64-x86-64
-# CHECK-NEXT: Arch: x86_64
-# CHECK-NEXT: AddressSize: 64bit
-
-# CHECK: Class: 64-bit
-# CHECK: DataEncoding: LittleEndian
-# SYSV:    OS/ABI: SystemV
-# FREEBSD: OS/ABI: FreeBSD
-# CHECK: Machine: EM_X86_64
-# CHECK: HeaderSize: 64
-# CHECK: SectionHeaderEntrySize: 64
diff --git a/llvm/test/tools/llvm-objcopy/ELF/binary-input.test b/llvm/test/tools/llvm-objcopy/ELF/binary-input.test
index d7a708dcea537..f232296ded825 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/binary-input.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/binary-input.test
@@ -1,7 +1,7 @@
 # RUN: echo -n abcd > %t.x-txt
 # Preserve input to verify it is not modified
 # RUN: cp %t.x-txt %t-copy.txt
-# RUN: llvm-objcopy -I binary -B i386:x86-64 %t.x-txt %t.o
+# RUN: llvm-objcopy -I binary -O elf64-x86-64 %t.x-txt %t.o
 # RUN: llvm-readobj --sections --symbols %t.o | FileCheck %s
 # RUN: cmp %t.x-txt %t-copy.txt
 
@@ -110,3 +110,11 @@
 # CHECK-NEXT:     Section: Absolute
 # CHECK-NEXT:   }
 # CHECK-NEXT: ]
+
+## The alignment can be changed by --set-section-alignment.
+# RUN: llvm-objcopy -I binary -O elf64-x86-64 --set-section-alignment .data=8 %t.x-txt %t2.o
+# RUN: llvm-readobj --sections %t2.o | FileCheck --check-prefix=ALIGN %s
+
+# ALIGN:      Name: .data
+# ALIGN:      AddressAlignment:
+# ALIGN-SAME:                   8{{$}}
diff --git a/llvm/test/tools/llvm-objcopy/ELF/binary-input-arch.test b/llvm/test/tools/llvm-objcopy/ELF/binary-output-target.test
similarity index 65%
rename from llvm/test/tools/llvm-objcopy/ELF/binary-input-arch.test
rename to llvm/test/tools/llvm-objcopy/ELF/binary-output-target.test
index 2b63e48866cf8..4af2d0514bf49 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/binary-input-arch.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/binary-output-target.test
@@ -1,38 +1,35 @@
 # RUN: echo abcd > %t.txt
 
-# RUN: llvm-objcopy -I binary -B aarch64 %t.txt %t.aarch64.o
+# RUN: llvm-objcopy -I binary -O elf64-littleaarch64 %t.txt %t.aarch64.o
 # RUN: llvm-readobj --file-headers %t.aarch64.o | FileCheck %s --check-prefixes=CHECK,LE,AARCH64,64
 
-# RUN: llvm-objcopy -I binary -B arm %t.txt %t.arm.o
+# RUN: llvm-objcopy -I binary -O elf32-littlearm %t.txt %t.arm.o
 # RUN: llvm-readobj --file-headers %t.arm.o | FileCheck %s --check-prefixes=CHECK,LE,ARM,32
 
-# RUN: llvm-objcopy -I binary -B i386 %t.txt %t.i386.o
+# RUN: llvm-objcopy -I binary -O elf32-i386 %t.txt %t.i386.o
 # RUN: llvm-readobj --file-headers %t.i386.o | FileCheck %s --check-prefixes=CHECK,LE,I386,32
 
-# RUN: llvm-objcopy -I binary -B i386:x86-64 %t.txt %t.i386_x86-64.o
-# RUN: llvm-readobj --file-headers %t.i386_x86-64.o | FileCheck %s --check-prefixes=CHECK,LE,X86-64,64
+# RUN: llvm-objcopy -I binary -O elf64-x86-64 %t.txt %t.x86-64.o
+# RUN: llvm-readobj --file-headers %t.x86-64.o | FileCheck %s --check-prefixes=CHECK,LE,X86-64,64
 
-# RUN: llvm-objcopy -I binary -B mips %t.txt %t.mips.o
+# RUN: llvm-objcopy -I binary -O elf32-bigmips %t.txt %t.mips.o
 # RUN: llvm-readobj --file-headers %t.mips.o | FileCheck %s --check-prefixes=CHECK,BE,MIPS,32
 
-# RUN: llvm-objcopy -I binary -B powerpc:common64 %t.txt %t.powerpc_common64.o
-# RUN: llvm-readobj --file-headers %t.powerpc_common64.o | FileCheck %s --check-prefixes=CHECK,LE,PPC,64
+# RUN: llvm-objcopy -I binary -O elf64-powerpcle %t.txt %t.ppc64le.o
+# RUN: llvm-readobj --file-headers %t.ppc64le.o | FileCheck %s --check-prefixes=CHECK,LE,PPC64,64
 
-# RUN: llvm-objcopy -I binary -B riscv:rv32 %t.txt %t.rv32.o
+# RUN: llvm-objcopy -I binary -O elf32-littleriscv %t.txt %t.rv32.o
 # RUN: llvm-readobj --file-headers %t.rv32.o | FileCheck %s --check-prefixes=CHECK,LE,RISCV32,32
 
-# RUN: llvm-objcopy -I binary -B riscv:rv64 %t.txt %t.rv64.o
+# RUN: llvm-objcopy -I binary -O elf64-littleriscv %t.txt %t.rv64.o
 # RUN: llvm-readobj --file-headers %t.rv64.o | FileCheck %s --check-prefixes=CHECK,LE,RISCV64,64
 
-# RUN: llvm-objcopy -I binary -B sparc %t.txt %t.sparc.o
+# RUN: llvm-objcopy -I binary -O elf32-sparc %t.txt %t.sparc.o
 # RUN: llvm-readobj --file-headers %t.sparc.o | FileCheck %s --check-prefixes=CHECK,BE,SPARC,32
 
-# RUN: llvm-objcopy -I binary -B sparcel %t.txt %t.sparcel.o
+# RUN: llvm-objcopy -I binary -O elf32-sparcel %t.txt %t.sparcel.o
 # RUN: llvm-readobj --file-headers %t.sparcel.o | FileCheck %s --check-prefixes=CHECK,LE,SPARCEL,32
 
-# RUN: llvm-objcopy -I binary -B x86-64 %t.txt %t.x86-64.o
-# RUN: llvm-readobj --file-headers %t.x86-64.o | FileCheck %s --check-prefixes=CHECK,LE,X86-64,64
-
 # CHECK: Format:
 # 32-SAME:      ELF32-
 # 64-SAME:      ELF64-
@@ -42,7 +39,8 @@
 # MIPS-SAME:    mips{{$}}
 # RISCV32-SAME: riscv{{$}}
 # RISCV64-SAME: riscv{{$}}
-# PPC-SAME:     ppc64
+# PPC-SAME:     ppc{{$}}
+# PPC64le-SAME: ppc64
 # SPARC-SAME:   sparc
 # SPARCEL-SAME: sparc
 # X86-64-SAME:  x86-64
@@ -50,8 +48,9 @@
 # AARCH64-NEXT: Arch: aarch64
 # ARM-NEXT:     Arch: arm
 # I386-NEXT:    Arch: i386
-# MIPS-NEXT:    Arch: mips{{$$}}
-# PPC-NEXT:     Arch: powerpc64le
+# MIPS-NEXT:    Arch: mips{{$}}
+# PPC-NEXT:     Arch: powerpc{{$}}
+# PPC64-NEXT:   Arch: powerpc64le
 # RISCV32-NEXT: Arch: riscv32
 # RISCV64-NEXT: Arch: riscv64
 # SPARC-NEXT:   Arch: sparc{{$}}
@@ -74,16 +73,22 @@
 # CHECK-NEXT:       Unused: (00 00 00 00 00 00 00)
 # CHECK-NEXT:     }
 # CHECK-NEXT:     Type: Relocatable (0x1)
+
+## In GNU objcopy, e_machine is EM_NONE unless -B is specifid.
+## It is not very clear why it behaves this way.
+## We fill in the field regardless of -B.
 # AARCH64-NEXT:   Machine: EM_AARCH64 (0xB7)
 # ARM-NEXT:       Machine: EM_ARM (0x28)
 # I386-NEXT:      Machine: EM_386 (0x3)
 # MIPS-NEXT:      Machine: EM_MIPS (0x8)
-# PPC-NEXT:       Machine: EM_PPC64 (0x15)
+# PPC-NEXT:       Machine: EM_PPC (0x14)
+# PPC64-NEXT:     Machine: EM_PPC64 (0x15)
 # RISCV32-NEXT:   Machine: EM_RISCV (0xF3)
 # RISCV64-NEXT:   Machine: EM_RISCV (0xF3)
 # SPARC-NEXT:     Machine: EM_SPARC (0x2)
 # SPARCEL-NEXT:   Machine: EM_SPARC (0x2)
 # X86-64-NEXT:    Machine: EM_X86_64 (0x3E)
+
 # CHECK-NEXT:     Version: 1
 # CHECK-NEXT:     Entry: 0x0
 # CHECK-NEXT:     ProgramHeaderOffset:
@@ -100,3 +105,12 @@
 # CHECK-NEXT:     SectionHeaderCount: 4
 # CHECK-NEXT:     StringTableSectionIndex:
 # CHECK-NEXT:   }
+
+## ELFOSABI_NONE = ELFOSABI_SYSV = 0 is the default.
+## "-freebsd" suffixed target name can change the field to ELFOSABI_FREEBSD.
+
+## Many uses of objcopy use no spaces in the flags, make sure that also works.
+# RUN: llvm-objcopy -Ibinary -O elf64-x86-64-freebsd %t.txt %t.freebsd.o
+# RUN: llvm-readobj --file-headers %t.freebsd.o | FileCheck %s --check-prefixes=FREEBSD
+
+# FREEBSD: OS/ABI: FreeBSD
diff --git a/llvm/test/tools/llvm-objcopy/ELF/help-message.test b/llvm/test/tools/llvm-objcopy/ELF/help-message.test
index 2f199a331af7d..0a093fe45c83e 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/help-message.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/help-message.test
@@ -14,6 +14,10 @@
 
 
 # OBJCOPY-USAGE:  USAGE: llvm-objcopy
+# OBJCOPY-USAGE:  @FILE
+
 # STRIP-USAGE:    USAGE: llvm-strip
+# STRIP-USAGE:    @FILE
+
 # UNKNOWN-ARG:    unknown argument '{{-+}}abcabc'
 # NO-INPUT-FILES: no input file specified
diff --git a/llvm/test/tools/llvm-objcopy/ELF/input-output-target.test b/llvm/test/tools/llvm-objcopy/ELF/input-output-target.test
index 0661aca7bc923..fba52d029c521 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/input-output-target.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/input-output-target.test
@@ -4,18 +4,18 @@
 # RUN: cp %t.txt %t-copy.txt
 
 # -F <target> is equivalent to -I <target> -O <target>
-# RUN: llvm-objcopy -F binary -B i386:x86-64 %t.txt %t.2.txt
+# RUN: llvm-objcopy -F binary %t.txt %t.2.txt
 # RUN: cmp %t-copy.txt %t.2.txt
 
 # --target <target> is equivalent to --input-target <target> --output-target <target>
-# RUN: llvm-objcopy --target binary -B i386:x86-64 %t.txt %t.3.txt
+# RUN: llvm-objcopy --target binary %t.txt %t.3.txt
 # RUN: cmp %t-copy.txt %t.3.txt
 
 # --target is incompatibile with --input-target/--output-target
-# RUN: not llvm-objcopy --target binary --input-target binary -B i386:x86-64 \
+# RUN: not llvm-objcopy --target binary --input-target binary \
 # RUN:     %t.txt %t.4.txt 2>&1 \
 # RUN:     | FileCheck %s --check-prefix=BAD-FLAG
-# RUN: not llvm-objcopy --target binary --output-target binary -B i386:x86-64 \
+# RUN: not llvm-objcopy --target binary --output-target binary \
 # RUN:     %t.txt %t.4.txt 2>&1 \
 # RUN:     | FileCheck %s --check-prefix=BAD-FLAG
 
diff --git a/llvm/test/tools/llvm-objcopy/ELF/new-symbol-visibility.test b/llvm/test/tools/llvm-objcopy/ELF/new-symbol-visibility.test
index 143a18c0a7e57..c0112050be4e1 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/new-symbol-visibility.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/new-symbol-visibility.test
@@ -1,22 +1,22 @@
 ## Ensure that the visibility of the start, end and size symbols created when
 ## using binary input can be specified with the --new-symbol-visibility switch.
 
-# RUN: llvm-objcopy -I binary -B i386:x86-64 %s %t.unspecified
+# RUN: llvm-objcopy -I binary -O elf64-x86-64 %s %t.unspecified
 # RUN: llvm-readelf -s %t.unspecified | FileCheck %s --check-prefix=BINARY -DVISIBILITY=DEFAULT
-# RUN: llvm-objcopy --new-symbol-visibility default -I binary -B i386:x86-64 %s %t.default
+# RUN: llvm-objcopy --new-symbol-visibility default -I binary -O elf64-x86-64 %s %t.default
 # RUN: llvm-readelf -s %t.default | FileCheck %s --check-prefix=BINARY -DVISIBILITY=DEFAULT
-# RUN: llvm-objcopy --new-symbol-visibility hidden -I binary -B i386:x86-64 %s %t.hidden
+# RUN: llvm-objcopy --new-symbol-visibility hidden -I binary -O elf64-x86-64 %s %t.hidden
 # RUN: llvm-readelf -s %t.hidden | FileCheck %s --check-prefix=BINARY -DVISIBILITY=HIDDEN
-# RUN: llvm-objcopy --new-symbol-visibility protected -I binary -B i386:x86-64 %s %t.protected
+# RUN: llvm-objcopy --new-symbol-visibility protected -I binary -O elf64-x86-64 %s %t.protected
 # RUN: llvm-readelf -s %t.protected | FileCheck %s --check-prefix=BINARY -DVISIBILITY=PROTECTED
-# RUN: llvm-objcopy --new-symbol-visibility internal -I binary -B i386:x86-64 %s %t.internal
+# RUN: llvm-objcopy --new-symbol-visibility internal -I binary -O elf64-x86-64 %s %t.internal
 # RUN: llvm-readelf -s %t.internal | FileCheck %s --check-prefix=BINARY -DVISIBILITY=INTERNAL
 
 # BINARY:      [[VISIBILITY]] {{.*}} _binary{{.*}}_start
 # BINARY-NEXT: [[VISIBILITY]] {{.*}} _binary{{.*}}_end
 # BINARY-NEXT: [[VISIBILITY]] {{.*}} _binary{{.*}}_size
 
-# RUN: not llvm-objcopy --new-symbol-visibility fluff -I binary -B i386:x86-64 %s %t.error 2>&1 \
-# RUN:   | FileCheck %s --check-prefix=ERR
+# RUN: not llvm-objcopy --new-symbol-visibility fluff -I binary -O elf64-x86-64 \
+# RUN:   %s %t.error 2>&1 | FileCheck %s --check-prefix=ERR
 
 # ERR: error: 'fluff' is not a valid symbol visibility
diff --git a/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections-in-segments.test b/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections-in-segments.test
index cd92a7eb643e2..1c7cb2814fe2f 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections-in-segments.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections-in-segments.test
@@ -4,17 +4,6 @@
 ## inputs somehow.
 # RUN: yaml2obj %s -o %t.o
 
-## First, check the address of the section header table.
-# RUN: llvm-readobj --file-headers %t.o | FileCheck %s --check-prefix=SHDRS-OFFSET
-# SHDRS-OFFSET: SectionHeaderOffset: 0x1050{{$}}
-
-## Binary edit the section header sh_offset field of the second section to
-## (e_shoff+64*2+24 = 0x10e8) overlap the first one.
-# RUN: %python -c "with open('%/t.o', 'r+b') as input: import struct; bytes = struct.pack('<Q', 0x1001); input.seek(0x10e8); input.write(bytes)"
-
-## Sanity check that the binary editing modified the correct field.
-# RUN: llvm-readobj --section-headers %t.o | FileCheck %s
-
 ## Check that the contents are as expected before the copy.
 # RUN: llvm-readobj -x .first -x .second %t.o | FileCheck %s --check-prefix=CONTENTS
 
@@ -49,6 +38,7 @@ Sections:
   - Name: .second
     Type: SHT_PROGBITS
     Content: '89abcdef'
+    ShOffset: 0x1001
 ProgramHeaders:
   - Type:     PT_LOAD
     FileSize: 5
diff --git a/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections.test b/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections.test
index 2c470e6adc16a..49f59804e9dba 100644
--- a/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections.test
+++ b/llvm/test/tools/llvm-objcopy/ELF/overlapping-sections.test
@@ -3,22 +3,6 @@
 ## inputs somehow.
 # RUN: yaml2obj %s -o %t.o
 
-## First, check the address of the section header table.
-# RUN: llvm-readobj --file-headers %t.o | FileCheck %s --check-prefix=SHDRS-OFFSET
-# SHDRS-OFFSET: SectionHeaderOffset: 0x1050{{$}}
-
-## Binary edit the section header sh_offset field of the second section to
-## (e_shoff+64*2+24 = 0x10e8) overlap the first one.
-# RUN: %python -c "with open('%/t.o', 'r+b') as input: import struct; bytes = struct.pack('<Q', 0x1001); input.seek(0x10e8); input.write(bytes)"
-
-## Sanity check that the binary editing modified the correct field.
-# RUN: llvm-readobj --section-headers %t.o | FileCheck %s --check-prefix=VALIDATE
-
-# VALIDATE: Name: .first
-# VALIDATE: Offset: 0x1000
-# VALIDATE: Name: .second
-# VALIDATE: Offset: 0x1001
-
 ## Check that the contents are as expected before the copy.
 # RUN: llvm-readobj -x .first -x .second %t.o | FileCheck %s --check-prefix=CONTENTS
 
@@ -54,3 +38,4 @@ Sections:
   - Name: .second
     Type: SHT_PROGBITS
     Content: '89abcdef'
+    ShOffset: 0x1001
diff --git a/llvm/test/tools/llvm-objcopy/ELF/response-file.test b/llvm/test/tools/llvm-objcopy/ELF/response-file.test
new file mode 100644
index 0000000000000..45784cc06e467
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/response-file.test
@@ -0,0 +1,20 @@
+## Check that we support response files.
+# RUN: yaml2obj %s -o %t.o
+# RUN: echo "--strip-debug %t.o" > %t-response
+# RUN: llvm-objcopy @%t-response %t2.o
+# RUN: llvm-strip @%t-response
+
+# RUN: llvm-readobj -S %t.o | FileCheck %s
+# RUN: cmp %t.o %t2.o
+
+# CHECK-NOT: .debug_foo
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .debug_foo
+    Type: SHT_PROGBITS
diff --git a/llvm/test/tools/llvm-objcopy/ELF/set-section-alignment.test b/llvm/test/tools/llvm-objcopy/ELF/set-section-alignment.test
new file mode 100644
index 0000000000000..79c7eddffddc3
--- /dev/null
+++ b/llvm/test/tools/llvm-objcopy/ELF/set-section-alignment.test
@@ -0,0 +1,54 @@
+# RUN: yaml2obj %s -o %t
+
+# RUN: llvm-objcopy --set-section-alignment .foo=4 --set-section-alignment .bar=0x5 \
+# RUN:   --set-section-alignment .baz=0 %t %t.2
+# RUN: llvm-readobj --sections %t.2 | FileCheck --check-prefix=CHECK %s
+
+# CHECK:      Name: .foo
+# CHECK:      AddressAlignment:
+# CHECK-SAME:                   4{{$}}
+# CHECK:      Name: .bar
+# CHECK:      AddressAlignment:
+# CHECK-SAME:                   5{{$}}
+# CHECK:      Name: .baz
+# CHECK:      AddressAlignment:
+# CHECK-SAME:                   0{{$}}
+
+## If a section is specified multiple times, the last wins.
+# RUN: llvm-objcopy --set-section-alignment .foo=4 --set-section-alignment=.foo=7 %t %t.3
+# RUN: llvm-readobj --sections %t.3 | FileCheck --check-prefix=MULTI %s
+
+# MULTI:      Name: .foo
+# MULTI:      AddressAlignment:
+# MULTI-SAME:                   7{{$}}
+
+## Ignore the option if the section does not exist.
+# RUN: llvm-objcopy --set-section-alignment .not_exist=4 %t.3 %t.4
+# RUN: cmp %t.3 %t.4
+
+# RUN: not llvm-objcopy --set-section-alignment=.foo %t /dev/null 2>&1 | \
+# RUN:   FileCheck --check-prefix=MISSING-EQUAL %s
+# MISSING-EQUAL:   error: bad format for --set-section-alignment: missing '='
+
+# RUN: not llvm-objcopy --set-section-alignment==4 %t /dev/null 2>&1 | \
+# RUN:   FileCheck --check-prefix=MISSING-SECTION %s
+# MISSING-SECTION: error: bad format for --set-section-alignment: missing section name
+
+# RUN: not llvm-objcopy --set-section-alignment=.foo=bar %t /dev/null 2>&1 | \
+# RUN:   FileCheck --check-prefix=INVALID-ALIGN %s
+# INVALID-ALIGN:   error: invalid alignment for --set-section-alignment: 'bar'
+
+!ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name:         .foo
+    Type:         SHT_PROGBITS
+  - Name:         .bar
+    Type:         SHT_NOBITS
+  - Name:         .baz
+    Type:         SHT_NOTE
+    AddressAlign: 4
diff --git a/llvm/test/tools/llvm-profdata/show-prof-size.test b/llvm/test/tools/llvm-profdata/show-prof-size.test
new file mode 100644
index 0000000000000..a15efdcb0de8d
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/show-prof-size.test
@@ -0,0 +1,8 @@
+; RUN: llvm-profdata merge -sample -extbinary -prof-sym-list=%S/Inputs/profile-symbol-list-1.text %S/Inputs/sample-profile.proftext -o %t.1.output
+; RUN: ls -l %t.1.output |tr -s ' ' |cut -f5 -d ' ' > %t.txt
+; RUN: llvm-profdata show -sample -show-sec-info-only %t.1.output >> %t.txt
+; RUN: FileCheck %s --input-file=%t.txt
+; Check llvm-profdata shows the correct file size.
+; CHECK: [[FILESIZE:.*]]
+; CHECK: [[FILESIZE]]
+; UNSUPPORTED: system-windows
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-0.s b/llvm/test/tools/llvm-readobj/ARM/attribute-0.s
index 24049941492e5..dfee541125651 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-0.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-0.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 0
 @CHECK:   .eabi_attribute 6, 0
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-1.s b/llvm/test/tools/llvm-readobj/ARM/attribute-1.s
index e493a426576a2..d7cc1aaf12077 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-1.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-1.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 1
 @CHECK:   .eabi_attribute 6, 1
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-10.s b/llvm/test/tools/llvm-readobj/ARM/attribute-10.s
index 570e27b08f513..1c62fee75ade0 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-10.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-10.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 10
 @CHECK:   .eabi_attribute 6, 10
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-11.s b/llvm/test/tools/llvm-readobj/ARM/attribute-11.s
index b70e5a0c5e4a9..04a2e41065db5 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-11.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-11.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 11
 @CHECK:   .eabi_attribute 6, 11
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-12.s b/llvm/test/tools/llvm-readobj/ARM/attribute-12.s
index 1bedf6dc4c4e6..e36ed9b652109 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-12.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-12.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 12
 @CHECK:   .eabi_attribute 6, 12
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-13.s b/llvm/test/tools/llvm-readobj/ARM/attribute-13.s
index d3b9fb70953c6..407995c01a578 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-13.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-13.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 13
 @CHECK:   .eabi_attribute 6, 13
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-136.s b/llvm/test/tools/llvm-readobj/ARM/attribute-136.s
index 5952b34ccc4c8..674970a654ea0 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-136.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-136.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_compatibility, 136, "Foo Corp"
 @CHECK:   .eabi_attribute 32, 136
 @CHECK-OBJ: Tag: 32
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-14.s b/llvm/test/tools/llvm-readobj/ARM/attribute-14.s
index aad44332af76f..c15a5cd535f6d 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-14.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-14.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 14
 @CHECK:   .eabi_attribute 6, 14
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-15.s b/llvm/test/tools/llvm-readobj/ARM/attribute-15.s
index efe4c7a3e581c..1fd1869c9c713 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-15.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-15.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_compatibility, 15, "Longer Corporation NaMe"
 @CHECK:   .eabi_attribute 32, 15
 @CHECK-OBJ: Tag: 32
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-2.s b/llvm/test/tools/llvm-readobj/ARM/attribute-2.s
index dd7fc9e4ca79a..33aeca8a1a783 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-2.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-2.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 2
 @CHECK:   .eabi_attribute 6, 2
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-3.s b/llvm/test/tools/llvm-readobj/ARM/attribute-3.s
index 3ef7dde5cdc01..284e6ed829590 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-3.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-3.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 3
 @CHECK:   .eabi_attribute 6, 3
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-4.s b/llvm/test/tools/llvm-readobj/ARM/attribute-4.s
index 01cd6d412068d..5f5f2b9e9d65c 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-4.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-4.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 4
 @CHECK:   .eabi_attribute 6, 4
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-5.s b/llvm/test/tools/llvm-readobj/ARM/attribute-5.s
index cadf1fc1d695d..d1d386ef97819 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-5.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-5.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 5
 @CHECK:   .eabi_attribute 6, 5
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-6.s b/llvm/test/tools/llvm-readobj/ARM/attribute-6.s
index cf603971cbf42..63c1019a3acc8 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-6.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-6.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 6
 @CHECK:   .eabi_attribute 6, 6
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-7.s b/llvm/test/tools/llvm-readobj/ARM/attribute-7.s
index 36242c0aa882a..7f511674aef19 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-7.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-7.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 7
 @CHECK:   .eabi_attribute 6, 7
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-8.s b/llvm/test/tools/llvm-readobj/ARM/attribute-8.s
index 42a6116e6af0b..42e2c878408bb 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-8.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-8.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 8
 @CHECK:   .eabi_attribute 6, 8
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-9.s b/llvm/test/tools/llvm-readobj/ARM/attribute-9.s
index 1c75784932fca..1d15ddeb79f5a 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-9.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-9.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch, 9
 @CHECK:   .eabi_attribute 6, 9
 @CHECK-OBJ: Tag: 6
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-A.s b/llvm/test/tools/llvm-readobj/ARM/attribute-A.s
index 8c08c7eb2d13f..105e70532aab3 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-A.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-A.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch_profile, 'A'
 @CHECK:   .eabi_attribute 7, 65
 @CHECK-OBJ: Tag: 7
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-M.s b/llvm/test/tools/llvm-readobj/ARM/attribute-M.s
index 56c0c953ae261..72ea70f30c702 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-M.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-M.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch_profile, 'M'
 @CHECK:   .eabi_attribute 7, 77
 @CHECK-OBJ: Tag: 7
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-R.s b/llvm/test/tools/llvm-readobj/ARM/attribute-R.s
index add4192ff6994..6a4c2620bab86 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-R.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-R.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch_profile, 'R'
 @CHECK:   .eabi_attribute 7, 82
 @CHECK-OBJ: Tag: 7
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-S.s b/llvm/test/tools/llvm-readobj/ARM/attribute-S.s
index 0bd4924adceca..a8cd27c156c07 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-S.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-S.s
@@ -1,6 +1,6 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_CPU_arch_profile, 'S'
 @CHECK:   .eabi_attribute 7, 83
 @CHECK-OBJ: Tag: 7
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-1.s b/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-1.s
index cfa426d155f66..2ceaca2ccede0 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-1.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-1.s
@@ -1,6 +1,8 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
+@ RUN:   | llvm-readobj -A - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_conformance, "0"
 @CHECK:   .eabi_attribute 67, "0"
 @CHECK-OBJ: Tag: 67
diff --git a/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-2.s b/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-2.s
index 00d08fbcc6483..2ac459aa23799 100644
--- a/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-2.s
+++ b/llvm/test/tools/llvm-readobj/ARM/attribute-conformance-2.s
@@ -1,6 +1,8 @@
 @ RUN: llvm-mc -triple armv7-elf -filetype asm -o - %s | FileCheck %s
 @ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
-@ RUN:   | llvm-readobj --arm-attributes - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN:   | llvm-readobj --arch-specific - | FileCheck %s --check-prefix=CHECK-OBJ
+@ RUN: llvm-mc -triple armv7-eabi -filetype obj -o - %s \
+@ RUN:   | llvm-readobj -A - | FileCheck %s --check-prefix=CHECK-OBJ
 .eabi_attribute  Tag_conformance, "A.long--non numeric oddity...!!"
 @CHECK:   .eabi_attribute 67, "A.long--non numeric oddity...!!"
 @CHECK-OBJ: Tag: 67
diff --git a/llvm/test/tools/llvm-readobj/Inputs/relocs.obj.elf-ppc64 b/llvm/test/tools/llvm-readobj/Inputs/relocs.obj.elf-ppc64
deleted file mode 100644
index c46e4c04285be..0000000000000
Binary files a/llvm/test/tools/llvm-readobj/Inputs/relocs.obj.elf-ppc64 and /dev/null differ
diff --git a/llvm/test/tools/llvm-readobj/all.test b/llvm/test/tools/llvm-readobj/all.test
index b8ec174d5aa3f..17c5a007adfa5 100644
--- a/llvm/test/tools/llvm-readobj/all.test
+++ b/llvm/test/tools/llvm-readobj/all.test
@@ -13,3 +13,4 @@ ALL: Relocations [
 ALL: Symbols [
 ALL: ProgramHeaders [
 ALL: Notes [
+ALL: StackSizes [
diff --git a/llvm/test/tools/llvm-readobj/archive.test b/llvm/test/tools/llvm-readobj/archive.test
index 1bb70d36270f3..2cf0200b51e8d 100644
--- a/llvm/test/tools/llvm-readobj/archive.test
+++ b/llvm/test/tools/llvm-readobj/archive.test
@@ -15,6 +15,8 @@
 # LLVM: Symbols [
 # LLVM: Notes [
 # LLVM: ]
+# LLVM: StackSizes [
+# LLVM: ]
 
 # GNU: {{^$}}
 # GNU-NEXT: File: [[ARFILE]](trivial.obj.elf-x86-64)
@@ -35,6 +37,8 @@
 # LLVM: Symbols [
 # LLVM: Notes [
 # LLVM: ]
+# LLVM: StackSizes [
+# LLVM: ]
 
 # GNU-EMPTY:
 # GNU-NEXT: File: [[ARFILE]](trivial.obj.elf-i386)
diff --git a/llvm/test/tools/llvm-readobj/elf-dynamic-malformed.test b/llvm/test/tools/llvm-readobj/elf-dynamic-malformed.test
index b377eaee21341..afdcbe6b6b5ec 100644
--- a/llvm/test/tools/llvm-readobj/elf-dynamic-malformed.test
+++ b/llvm/test/tools/llvm-readobj/elf-dynamic-malformed.test
@@ -7,26 +7,21 @@
 # RUN: llvm-readelf --all %t.bad-size 2>&1 \
 # RUN:   | FileCheck %s -DFILE=%t.bad-size --implicit-check-not=warning --check-prefix WARN-GNU
 
-# WARN-NOT: warning
 # WARN: warning: '[[FILE]]': invalid section size (4) or entity size (16)
+# WARN: warning: '[[FILE]]': invalid section size (4) or entity size (16)
+# WARN: warning: '[[FILE]]': no valid dynamic table was found
 # WARN-EMPTY:
-# WARN-NEXT: File:
-# WARN:      Symbols [
-# WARN:      ]
-# WARN-EMPTY:
-## A warning is printed at the place where a normal dynamic table should be. 
-# WARN-NEXT: warning: '[[FILE]]': invalid section size (4) or entity size (16)
-# WARN-NEXT: ProgramHeaders [
-
-# WARN-GNU-NOT:  warning
-# WARN-GNU:      warning: '[[FILE]]': invalid section size (4) or entity size (16)
+# WARN: File:
+# WARN: Symbols [
+# WARN: ]
+# WARN: ProgramHeaders [
+
+# WARN-GNU: warning: '[[FILE]]': invalid section size (4) or entity size (16)
+# WARN-GNU: warning: '[[FILE]]': invalid section size (4) or entity size (16)
+# WARN-GNU: warning: '[[FILE]]': no valid dynamic table was found
 # WARN-GNU-NEXT: ELF Header:
 # WARN-GNU:      Symbol table '.symtab' contains 1 entries:
 # WARN-GNU:        0:
-# WARN-GNU-EMPTY:
-## A warning is printed at the place where a normal dynamic table should be. 
-# WARN-GNU:      warning: '[[FILE]]': invalid section size (4) or entity size (16)
-# WARN-GNU-EMPTY:
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/elf-dynamic-not-in-pt-dynamic.test b/llvm/test/tools/llvm-readobj/elf-dynamic-not-in-pt-dynamic.test
index 163e61033cd71..cb8da637e006a 100644
--- a/llvm/test/tools/llvm-readobj/elf-dynamic-not-in-pt-dynamic.test
+++ b/llvm/test/tools/llvm-readobj/elf-dynamic-not-in-pt-dynamic.test
@@ -1,24 +1,86 @@
-## Show that llvm-readobj/llvm-readelf tools can dump the .dynamic
-## section when it is not in a PT_DYNAMIC segment.
+## Show that llvm-readobj/llvm-readelf tools sometimes can dump the
+## dynamic table when it is not in a PT_DYNAMIC segment.
 
-# RUN: yaml2obj %s -o %t.o
-# RUN: llvm-readobj --dynamic-table %t.o 2>&1 \
-# RUN:   | FileCheck -DFILE=%t.o --check-prefixes=WARNING,LLVM %s
-# RUN: llvm-readelf --dynamic-table %t.o 2>&1 \
-# RUN:   | FileCheck -DFILE=%t.o --check-prefixes=WARNING,GNU %s
+## Case 1: The dynamic table found using the dynamic program header is corrupted
+##         (<size of data> % <size of dynamic entry> != 0). So the table is taken
+##         from the section header.
 
-# WARNING:   warning: '[[FILE]]': The SHT_DYNAMIC section '.dynamic' is not contained within the PT_DYNAMIC segment
+# RUN: yaml2obj --docnum=1 %s -o %t1.o
+# RUN: llvm-readobj --dynamic-table %t1.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t1.o --check-prefixes=WARNING1,LLVM1 %s
+# RUN: llvm-readelf --dynamic-table %t1.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t1.o --check-prefixes=WARNING1,GNU1 %s
 
-# LLVM:      DynamicSection [ (2 entries)
-# LLVM-NEXT:   Tag                Type     Name/Value
-# LLVM-NEXT:   0x0000000000000018 BIND_NOW 0x1
-# LLVM-NEXT:   0x0000000000000000 NULL     0x0
-# LLVM-NEXT: ]
+# WARNING1: warning: '[[FILE]]': The SHT_DYNAMIC section '.dynamic' is not contained within the PT_DYNAMIC segment
+# WARNING1: warning: '[[FILE]]': invalid section size (1) or entity size (16)
+# WARNING1: warning: '[[FILE]]': SHT_DYNAMIC section header and PT_DYNAMIC program header disagree about the location of the dynamic table
+# WARNING1: warning: '[[FILE]]': PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used
 
-# GNU:      Dynamic section at offset 0x{{.*}} contains 2 entries:
-# GNU-NEXT:   Tag                Type       Name/Value
-# GNU-NEXT:   0x0000000000000018 (BIND_NOW) 0x1
-# GNU-NEXT:   0x0000000000000000 (NULL)     0x0
+# LLVM1:      DynamicSection [ (2 entries)
+# LLVM1-NEXT:   Tag                Type     Name/Value
+# LLVM1-NEXT:   0x0000000000000018 BIND_NOW 0x1
+# LLVM1-NEXT:   0x0000000000000000 NULL     0x0
+# LLVM1-NEXT: ]
+
+# GNU1:      Dynamic section at offset 0x{{.*}} contains 2 entries:
+# GNU1-NEXT:   Tag                Type       Name/Value
+# GNU1-NEXT:   0x0000000000000018 (BIND_NOW) 0x1
+# GNU1-NEXT:   0x0000000000000000 (NULL)     0x0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Flags: [SHF_ALLOC]
+    Address: 0x1000
+    AddressAlign: 0x1000
+    Entries:
+      - Tag:   DT_BIND_NOW
+        Value: 0x1
+      - Tag:   DT_NULL
+        Value: 0x0
+  - Name: .text
+    Type: SHT_PROGBITS
+    Flags: [SHF_ALLOC]
+    Address: 0x1100
+    AddressAlign: 0x100
+    Content: "00"
+ProgramHeaders:
+  - Type: PT_LOAD
+    VAddr: 0x1000
+    Sections:
+      - Section: .dynamic
+      - Section: .text
+  - Type: PT_DYNAMIC
+    VAddr: 0x1000
+    Sections:
+      - Section: .text
+
+## Case 2: The dynamic table found using the dynamic program header is different from the
+##         table found using the section header table.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --dynamic-table %t2.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t2.o --check-prefixes=WARNING2,LLVM2 %s
+# RUN: llvm-readelf --dynamic-table %t2.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t2.o --check-prefixes=WARNING2,GNU2 %s
+
+# WARNING2: warning: '[[FILE]]': The SHT_DYNAMIC section '.dynamic' is not contained within the PT_DYNAMIC segment
+# WARNING2: warning: '[[FILE]]': SHT_DYNAMIC section header and PT_DYNAMIC program header disagree about the location of the dynamic table
+
+# LLVM2:      DynamicSection [ (1 entries)
+# LLVM2-NEXT:   Tag                Type     Name/Value
+# LLVM2-NEXT:   0x0000000000000000 NULL     0x0
+# LLVM2-NEXT: ]
+
+# GNU2:      Dynamic section at offset 0x{{.*}} contains 1 entries:
+# GNU2-NEXT:   Tag                Type       Name/Value
+# GNU2-NEXT:   0x0000000000000000 (NULL)     0x0
 
 --- !ELF
 FileHeader:
@@ -37,6 +99,48 @@ Sections:
         Value: 0x1
       - Tag:   DT_NULL
         Value: 0x0
+  - Name: .text
+    Type: SHT_PROGBITS
+    Flags: [SHF_ALLOC]
+    Address: 0x1100
+    AddressAlign: 0x100
+    Content: "00000000000000000000000000000000"
+ProgramHeaders:
+  - Type: PT_LOAD
+    VAddr: 0x1000
+    Sections:
+      - Section: .dynamic
+      - Section: .text
+  - Type: PT_DYNAMIC
+    VAddr: 0x1000
+    Sections:
+      - Section: .text
+
+## Case 3: Both dynamic tables found using SHT_DYNAMIC/PT_DYNAMIC are corrupted.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --dynamic-table %t3.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t3.o --check-prefix=WARNING3 --implicit-check-not="Dynamic" %s
+# RUN: llvm-readelf --dynamic-table %t3.o 2>&1 \
+# RUN:   | FileCheck -DFILE=%t3.o --check-prefix=WARNING3 --implicit-check-not="Dynamic" %s
+
+# WARNING3: warning: '[[FILE]]': invalid section size (1) or entity size (16)
+# WARNING3: warning: '[[FILE]]': SHT_DYNAMIC section header and PT_DYNAMIC program header disagree about the location of the dynamic table
+# WARNING3: warning: '[[FILE]]': no valid dynamic table was found
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Flags: [SHF_ALLOC]
+    Address: 0x1000
+    AddressAlign: 0x1000
+    Content: "00"
   - Name: .text
     Type: SHT_PROGBITS
     Flags: [SHF_ALLOC]
diff --git a/llvm/test/tools/llvm-readobj/elf-hash-symbols.test b/llvm/test/tools/llvm-readobj/elf-hash-symbols.test
index b0140a2e9c1ce..4ffecf9fcc6d4 100644
--- a/llvm/test/tools/llvm-readobj/elf-hash-symbols.test
+++ b/llvm/test/tools/llvm-readobj/elf-hash-symbols.test
@@ -361,3 +361,43 @@ ProgramHeaders:
     PAddr: 0x1000
     Sections:
       - Section: .dynamic
+
+## Show that we report a warning for a hash table which contains an entry of
+## the bucket array pointing to a cycle.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6.so
+# RUN: llvm-readelf --hash-symbols %t6.so 2>&1 | FileCheck %s -DFILE=%t6.so --check-prefix=BROKEN
+
+# BROKEN:      Symbol table of .hash for image:
+# BROKEN-NEXT:  Num Buc:    Value  Size   Type   Bind Vis      Ndx Name
+# BROKEN-NEXT:    1   0: 00000000     0 NOTYPE  LOCAL  DEFAULT UND aaa
+# BROKEN: warning: '[[FILE]]': .hash section is invalid: bucket 1: a cycle was detected in the linked chain
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:    .hash
+    Type:    SHT_HASH
+    Link:    .dynsym
+    Bucket:  [ 1 ]
+    Chain:   [ 1, 1 ]
+  - Name:  .dynamic
+    Type:  SHT_DYNAMIC
+    Entries:
+## llvm-readelf will read the hash table from the file offset
+## p_offset + (p_vaddr - DT_HASH) = p_offset + (0 - 0) = p_offset,
+## which is the start of PT_LOAD, i.e. the file offset of .hash.
+      - Tag:   DT_HASH
+        Value: 0x0
+DynamicSymbols:
+  - Name:    aaa
+  - Name:    bbb
+ProgramHeaders:
+  - Type:  PT_LOAD
+    Sections:
+      - Section: .hash
+      - Section: .dynamic
diff --git a/llvm/test/tools/llvm-readobj/elf-non-dynamic-in-pt-dynamic.test b/llvm/test/tools/llvm-readobj/elf-non-dynamic-in-pt-dynamic.test
index 08bae9467af78..77dea95903e3e 100644
--- a/llvm/test/tools/llvm-readobj/elf-non-dynamic-in-pt-dynamic.test
+++ b/llvm/test/tools/llvm-readobj/elf-non-dynamic-in-pt-dynamic.test
@@ -4,13 +4,16 @@
 ## In the first case .text is placed before .dynamic.
 ## We check that we warn about this case.
 
-# RUN: yaml2obj --docnum=1 %s -o %t.o
-# RUN: llvm-readobj --dynamic-table %t.o 2>&1 \
-# RUN:   | FileCheck %s --DFILE=%t.o --check-prefixes=WARNING,LLVM
-# RUN: llvm-readelf --dynamic-table %t.o 2>&1 \
-# RUN:   | FileCheck %s --DFILE=%t.o --check-prefixes=WARNING,GNU
+# RUN: yaml2obj --docnum=1 %s -o %t1.o
+# RUN: llvm-readobj --dynamic-table %t1.o 2>&1 \
+# RUN:   | FileCheck %s --DFILE=%t1.o --check-prefixes=WARNING,LLVM
+# RUN: llvm-readelf --dynamic-table %t1.o 2>&1 \
+# RUN:   | FileCheck %s --DFILE=%t1.o --check-prefixes=WARNING,GNU
 
-# WARNING:    warning: '[[FILE]]': The SHT_DYNAMIC section '.dynamic' is not at the start of PT_DYNAMIC segment
+# WARNING: warning: '[[FILE]]': The SHT_DYNAMIC section '.dynamic' is not at the start of PT_DYNAMIC segment
+# WARNING: warning: '[[FILE]]': invalid section size (33) or entity size (16)
+# WARNING: warning: '[[FILE]]': SHT_DYNAMIC section header and PT_DYNAMIC program header disagree about the location of the dynamic table
+# WARNING: warning: '[[FILE]]': PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used
 
 # LLVM:      DynamicSection [ (2 entries)
 # LLVM-NEXT:   Tag                Type     Name/Value
@@ -39,8 +42,7 @@ Sections:
   - Name: .dynamic
     Type: SHT_DYNAMIC
     Flags: [SHF_ALLOC]
-    Address: 0x1100
-    AddressAlign: 0x1000
+    Address: 0x1001
     Entries:
       - Tag:   DT_BIND_NOW
         Value: 0x1
@@ -58,11 +60,81 @@ ProgramHeaders:
       - Section: .text
       - Section: .dynamic
 
-## In the second case .text goes after .dynamic and we don't display any warnings.
+## In this case .text goes after .dynamic and we don't display any warnings,
+## though the content of the .text is used for dumping the dynamic table.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2.o
+# RUN: llvm-readobj --dynamic-table %t2.o 2>&1 | FileCheck %s --check-prefix=LLVM2
+# RUN: llvm-readelf --dynamic-table %t2.o 2>&1 | FileCheck %s --check-prefix=GNU2
+
+# LLVM2:      DynamicSection [ (3 entries)
+# LLVM2-NEXT:   Tag                Type     Name/Value
+# LLVM2-NEXT:   0x0000000000000018 BIND_NOW 0x1
+# LLVM2-NEXT:   0x0000000000000018 BIND_NOW 0x2
+# LLVM2-NEXT:   0x0000000000000000 NULL     0x0
+# LLVM2-NEXT: ]
+
+# GNU2:      Dynamic section at offset 0x{{.*}} contains 3 entries:
+# GNU2-NEXT:   Tag                Type       Name/Value
+# GNU2-NEXT:   0x0000000000000018 (BIND_NOW) 0x1
+# GNU2-NEXT:   0x0000000000000018 (BIND_NOW) 0x2
+# GNU2-NEXT:   0x0000000000000000 (NULL)     0x0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .dynamic
+    Type: SHT_DYNAMIC
+    Flags: [SHF_ALLOC]
+    Address: 0x1000
+    AddressAlign: 0x1000
+    Entries:
+      - Tag:   DT_BIND_NOW
+        Value: 0x1
+      - Tag:   DT_BIND_NOW
+        Value: 0x2
+  - Name: .text
+    Type: SHT_PROGBITS
+    Flags: [SHF_ALLOC]
+    Address: 0x1100
+    AddressAlign: 0x100
+    Content: "00000000000000000000000000000000"
+ProgramHeaders:
+  - Type: PT_LOAD
+    VAddr: 0x1000
+    Sections:
+      - Section: .dynamic
+      - Section: .text
+  - Type: PT_DYNAMIC
+    VAddr: 0x1000
+    Sections:
+      - Section: .dynamic
+      - Section: .text
+
+## In this case .text goes after .dynamic, but (PT_DYNAMIC segment size % dynamic entry size != 0)
+## and we have to use the information from the section header instead.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3.o
+# RUN: llvm-readobj --dynamic-table %t3.o 2>&1 | FileCheck %s --DFILE=%t3.o --check-prefixes=WARNING2,LLVM3
+# RUN: llvm-readelf --dynamic-table %t3.o 2>&1 | FileCheck %s --DFILE=%t3.o --check-prefixes=WARNING2,GNU3
+
+# WARNING2: warning: '[[FILE]]': invalid section size (257) or entity size (16)
+# WARNING2: warning: '[[FILE]]': PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used
+
+# LLVM3:      DynamicSection [ (2 entries)
+# LLVM3-NEXT:   Tag                Type     Name/Value
+# LLVM3-NEXT:   0x0000000000000018 BIND_NOW 0x1
+# LLVM3-NEXT:   0x0000000000000000 NULL     0x0
+# LLVM3-NEXT: ]
 
-# RUN: yaml2obj --docnum=2 %s -o %t.o
-# RUN: llvm-readobj --dynamic-table %t.o | FileCheck %s --implicit-check-not="warning"
-# RUN: llvm-readelf --dynamic-table %t.o | FileCheck %s --implicit-check-not="warning"
+# GNU3:      Dynamic section at offset 0x{{.*}} contains 2 entries:
+# GNU3-NEXT:   Tag                Type       Name/Value
+# GNU3-NEXT:   0x0000000000000018 (BIND_NOW) 0x1
+# GNU3-NEXT:   0x0000000000000000 (NULL)     0x0
 
 --- !ELF
 FileHeader:
diff --git a/llvm/test/tools/llvm-readobj/elf-reloc-zero-name-or-value.test b/llvm/test/tools/llvm-readobj/elf-reloc-zero-name-or-value.test
index 6b1a41637b432..c6481badc14c1 100644
--- a/llvm/test/tools/llvm-readobj/elf-reloc-zero-name-or-value.test
+++ b/llvm/test/tools/llvm-readobj/elf-reloc-zero-name-or-value.test
@@ -5,23 +5,23 @@
 # RUN: yaml2obj %s -o %t
 # RUN: llvm-readelf --relocations --dyn-relocations %t | FileCheck %s
 
-# CHECK:      Relocation section '.rela.text' at offset {{.*}} contains 2 entries:
+# CHECK:      Relocation section '.rela.text' at offset {{.*}} contains 3 entries:
 # CHECK-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
 # CHECK-NEXT: 0000000000000000  0000000000000000 R_X86_64_NONE                           1
 # CHECK-NEXT: 0000000000000000  0000000100000000 R_X86_64_NONE          0000000000000000 sym + 1
+# CHECK-NEXT: 0000000000000000  0000000200000000 R_X86_64_NONE          0000000000000123 456
 
-# CHECK:      Relocation section '.rela.dyn' at offset {{.*}} contains 2 entries:
+# CHECK:      Relocation section '.rela.dyn' at offset {{.*}} contains 3 entries:
 # CHECK-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
 # CHECK-NEXT: 0000000000000000  0000000000000000 R_X86_64_NONE                           1
 # CHECK-NEXT: 0000000000000000  0000000100000000 R_X86_64_NONE          0000000000000000 sym + 1
+# CHECK-NEXT: 0000000000000000  0000000200000000 R_X86_64_NONE          0000000000000123 456
 
-# CHECK:      'RELA' relocation section at offset {{.*}} contains 48 bytes:
+# CHECK:      'RELA' relocation section at offset {{.*}} contains 72 bytes:
 # CHECK-NEXT:     Offset             Info             Type               Symbol's Value  Symbol's Name + Addend
 # CHECK-NEXT: 0000000000000000  0000000000000000 R_X86_64_NONE                           1
 # CHECK-NEXT: 0000000000000000  0000000100000000 R_X86_64_NONE          0000000000000000 sym + 1
-
-# TODO: Add test for symbol with no name but with a value once yaml2obj allows
-# referencing symbols with no name from relocations.
+# CHECK-NEXT: 0000000000000000  0000000200000000 R_X86_64_NONE          0000000000000123 456
 
 --- !ELF
 FileHeader:
@@ -45,6 +45,10 @@ Sections:
         Type: R_X86_64_NONE
         Addend: 1
         Symbol: sym
+      - Offset: 0
+        Type: R_X86_64_NONE
+        Addend: 0x456
+        Symbol: 2
   - Name: .dynamic
     Type: SHT_DYNAMIC
     Flags: [SHF_ALLOC]
@@ -54,7 +58,7 @@ Sections:
       - Tag: DT_RELA
         Value: 0x1100
       - Tag: DT_RELASZ
-        Value: 48
+        Value: 72
       - Tag: DT_RELAENT
         Value: 24
       - Tag: DT_NULL
@@ -73,16 +77,26 @@ Sections:
         Type: R_X86_64_NONE
         Addend: 1
         Symbol: sym
+      - Offset: 0
+        Type: R_X86_64_NONE
+        Addend: 0x456
+        Symbol: 2
 Symbols:
   - Name:    sym
     Value:   0
     Section: .text
     Binding: STB_GLOBAL
+  - Value:   0x123
+    Section: .text
+    Binding: STB_GLOBAL
 DynamicSymbols:
   - Name:    sym
     Value:   0
     Section: .text
     Binding: STB_GLOBAL
+  - Value:   0x123
+    Section: .text
+    Binding: STB_GLOBAL
 ProgramHeaders:
   - Type: PT_LOAD
     VAddr: 0x1000
diff --git a/llvm/test/tools/llvm-readobj/elf-section-types.test b/llvm/test/tools/llvm-readobj/elf-section-types.test
index 2c4e921456ad8..aad9f43c8a301 100644
--- a/llvm/test/tools/llvm-readobj/elf-section-types.test
+++ b/llvm/test/tools/llvm-readobj/elf-section-types.test
@@ -150,6 +150,7 @@ Sections:
     Info: progbits
   - Name: hash
     Type: SHT_HASH
+    Content: ''
   - Name: dynamic
     Type: SHT_DYNAMIC
   - Name: note
diff --git a/llvm/test/tools/llvm-readobj/reloc-types-elf-ppc64.test b/llvm/test/tools/llvm-readobj/reloc-types-elf-ppc64.test
index f1102c3e218c7..7e081b7e4e229 100644
--- a/llvm/test/tools/llvm-readobj/reloc-types-elf-ppc64.test
+++ b/llvm/test/tools/llvm-readobj/reloc-types-elf-ppc64.test
@@ -1,38 +1,293 @@
-## Test that llvm-readobj/llvm-readelf shows proper relocation type
+## Test that llvm-readobj shows proper relocation type
 ## names and values for ppc64 target.
 
-## TODO: Use yaml2obj. obj2yaml now hangs on dumping relocs.obj.elf-ppc64
-# RUN: llvm-readobj -r --expand-relocs %p/Inputs/relocs.obj.elf-ppc64 | FileCheck %s
+# RUN: yaml2obj %s -o %t-ppc64.o
+# RUN: llvm-readobj -r --expand-relocs %t-ppc64.o | FileCheck %s
 
 # CHECK: Type: R_PPC64_NONE (0)
 # CHECK: Type: R_PPC64_ADDR32 (1)
+# CHECK: Type: R_PPC64_ADDR24 (2)
+# CHECK: Type: R_PPC64_ADDR16 (3)
 # CHECK: Type: R_PPC64_ADDR16_LO (4)
 # CHECK: Type: R_PPC64_ADDR16_HI (5)
+# CHECK: Type: R_PPC64_ADDR16_HA (6)
 # CHECK: Type: R_PPC64_ADDR14 (7)
+# CHECK: Type: R_PPC64_ADDR14_BRTAKEN (8)
+# CHECK: Type: R_PPC64_ADDR14_BRNTAKEN (9)
 # CHECK: Type: R_PPC64_REL24 (10)
+# CHECK: Type: R_PPC64_REL14 (11)
+# CHECK: Type: R_PPC64_REL14_BRTAKEN (12)
+# CHECK: Type: R_PPC64_REL14_BRNTAKEN (13)
+# CHECK: Type: R_PPC64_GOT16 (14)
+# CHECK: Type: R_PPC64_GOT16_LO (15)
+# CHECK: Type: R_PPC64_GOT16_HI (16)
+# CHECK: Type: R_PPC64_GOT16_HA (17)
+# CHECK: Type: R_PPC64_GLOB_DAT (20)
+# CHECK: Type: R_PPC64_JMP_SLOT (21)
+# CHECK: Type: R_PPC64_RELATIVE (22)
 # CHECK: Type: R_PPC64_REL32 (26)
 # CHECK: Type: R_PPC64_ADDR64 (38)
 # CHECK: Type: R_PPC64_ADDR16_HIGHER (39)
+# CHECK: Type: R_PPC64_ADDR16_HIGHERA (40)
 # CHECK: Type: R_PPC64_ADDR16_HIGHEST (41)
+# CHECK: Type: R_PPC64_ADDR16_HIGHESTA (42)
 # CHECK: Type: R_PPC64_REL64 (44)
 # CHECK: Type: R_PPC64_TOC16 (47)
 # CHECK: Type: R_PPC64_TOC16_LO (48)
+# CHECK: Type: R_PPC64_TOC16_HI (49)
 # CHECK: Type: R_PPC64_TOC16_HA (50)
 # CHECK: Type: R_PPC64_TOC (51)
 # CHECK: Type: R_PPC64_ADDR16_DS (56)
 # CHECK: Type: R_PPC64_ADDR16_LO_DS (57)
+# CHECK: Type: R_PPC64_GOT16_DS (58)
+# CHECK: Type: R_PPC64_GOT16_LO_DS (59)
 # CHECK: Type: R_PPC64_TOC16_DS (63)
 # CHECK: Type: R_PPC64_TOC16_LO_DS (64)
 # CHECK: Type: R_PPC64_TLS (67)
+# CHECK: Type: R_PPC64_DTPMOD64 (68)
+# CHECK: Type: R_PPC64_TPREL16 (69)
 # CHECK: Type: R_PPC64_TPREL16_LO (70)
+# CHECK: Type: R_PPC64_TPREL16_HI (71)
 # CHECK: Type: R_PPC64_TPREL16_HA (72)
+# CHECK: Type: R_PPC64_TPREL64 (73)
+# CHECK: Type: R_PPC64_DTPREL16 (74)
 # CHECK: Type: R_PPC64_DTPREL16_LO (75)
+# CHECK: Type: R_PPC64_DTPREL16_HI (76)
 # CHECK: Type: R_PPC64_DTPREL16_HA (77)
+# CHECK: Type: R_PPC64_DTPREL64 (78)
+# CHECK: Type: R_PPC64_GOT_TLSGD16 (79)
 # CHECK: Type: R_PPC64_GOT_TLSGD16_LO (80)
+# CHECK: Type: R_PPC64_GOT_TLSGD16_HI (81)
 # CHECK: Type: R_PPC64_GOT_TLSGD16_HA (82)
+# CHECK: Type: R_PPC64_GOT_TLSLD16 (83)
 # CHECK: Type: R_PPC64_GOT_TLSLD16_LO (84)
+# CHECK: Type: R_PPC64_GOT_TLSLD16_HI (85)
 # CHECK: Type: R_PPC64_GOT_TLSLD16_HA (86)
+# CHECK: Type: R_PPC64_GOT_TPREL16_DS (87)
 # CHECK: Type: R_PPC64_GOT_TPREL16_LO_DS (88)
+# CHECK: Type: R_PPC64_GOT_TPREL16_HI (89)
 # CHECK: Type: R_PPC64_GOT_TPREL16_HA (90)
+# CHECK: Type: R_PPC64_GOT_DTPREL16_DS (91)
+# CHECK: Type: R_PPC64_GOT_DTPREL16_LO_DS (92)
+# CHECK: Type: R_PPC64_GOT_DTPREL16_HI (93)
+# CHECK: Type: R_PPC64_GOT_DTPREL16_HA (94)
+# CHECK: Type: R_PPC64_TPREL16_DS (95)
+# CHECK: Type: R_PPC64_TPREL16_LO_DS (96)
+# CHECK: Type: R_PPC64_TPREL16_HIGHER (97)
+# CHECK: Type: R_PPC64_TPREL16_HIGHERA (98)
+# CHECK: Type: R_PPC64_TPREL16_HIGHEST (99)
+# CHECK: Type: R_PPC64_TPREL16_HIGHESTA (100)
+# CHECK: Type: R_PPC64_DTPREL16_DS (101)
+# CHECK: Type: R_PPC64_DTPREL16_LO_DS (102)
+# CHECK: Type: R_PPC64_DTPREL16_HIGHER (103)
+# CHECK: Type: R_PPC64_DTPREL16_HIGHERA (104)
+# CHECK: Type: R_PPC64_DTPREL16_HIGHEST (105)
+# CHECK: Type: R_PPC64_DTPREL16_HIGHESTA (106)
 # CHECK: Type: R_PPC64_TLSGD (107)
 # CHECK: Type: R_PPC64_TLSLD (108)
+# CHECK: Type: R_PPC64_ADDR16_HIGH (110)
+# CHECK: Type: R_PPC64_ADDR16_HIGHA (111)
+# CHECK: Type: R_PPC64_TPREL16_HIGH (112)
+# CHECK: Type: R_PPC64_TPREL16_HIGHA (113)
+# CHECK: Type: R_PPC64_DTPREL16_HIGH (114)
+# CHECK: Type: R_PPC64_DTPREL16_HIGHA (115)
+# CHECK: Type: R_PPC64_IRELATIVE (248)
+# CHECK: Type: R_PPC64_REL16 (249)
+# CHECK: Type: R_PPC64_REL16_LO (250)
+# CHECK: Type: R_PPC64_REL16_HI (251)
+# CHECK: Type: R_PPC64_REL16_HA (252)
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_PPC64
+Sections:
+  - Name: .rela.text
+    Type: SHT_RELA
+    Relocations:
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_NONE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR32
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR24
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14_BRTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14_BRNTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL24
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14_BRTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14_BRNTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GLOB_DAT
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_JMP_SLOT
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_RELATIVE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL32
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPMOD64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLSGD
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLSLD
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_IRELATIVE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_HA
diff --git a/llvm/test/tools/llvm-readobj/stack-sizes.test b/llvm/test/tools/llvm-readobj/stack-sizes.test
index 5a4c4c2b12fa4..8786ec665352f 100644
--- a/llvm/test/tools/llvm-readobj/stack-sizes.test
+++ b/llvm/test/tools/llvm-readobj/stack-sizes.test
@@ -3,13 +3,29 @@
 
 # RUN: yaml2obj --docnum=1 %s > %t01
 # RUN: llvm-readelf --stack-sizes %t01 \
-# RUN:   | FileCheck %s --check-prefix=RELOC --strict-whitespace --match-full-lines
-
-#      RELOC:         Size     Function
-# RELOC-NEXT:           16     referenced_by_symbol_foo
-# RELOC-NEXT:           32     referenced_via_section_bar
-# RELOC-NEXT:            8     separate_text_section_baz
-#  RELOC-NOT:{{.}}
+# RUN:   | FileCheck %s --check-prefix=RELOC-GNU --strict-whitespace --match-full-lines
+# RUN: llvm-readobj --stack-sizes %t01 | FileCheck %s --check-prefix=RELOC-LLVM
+
+#      RELOC-GNU:         Size     Function
+# RELOC-GNU-NEXT:           16     referenced_by_symbol_foo
+# RELOC-GNU-NEXT:           32     referenced_via_section_bar
+# RELOC-GNU-NEXT:            8     separate_text_section_baz
+#  RELOC-GNU-NOT:{{.}}
+
+# RELOC-LLVM:      StackSizes [
+# RELOC-LLVM-NEXT:   Entry {
+# RELOC-LLVM-NEXT:     Function: referenced_by_symbol_foo
+# RELOC-LLVM-NEXT:     Size: 0x10
+# RELOC-LLVM-NEXT:   }
+# RELOC-LLVM-NEXT:   Entry {
+# RELOC-LLVM-NEXT:     Function: referenced_via_section_bar
+# RELOC-LLVM-NEXT:     Size: 0x20
+# RELOC-LLVM-NEXT:   }
+# RELOC-LLVM-NEXT:   Entry {
+# RELOC-LLVM-NEXT:     Function: separate_text_section_baz
+# RELOC-LLVM-NEXT:     Size: 0x8
+# RELOC-LLVM-NEXT:   }
+# RELOC-LLVM-NEXT: ]
 
 --- !ELF
 FileHeader:
@@ -28,14 +44,15 @@ Sections:
     Size:    16
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
-## 2 stack size entries. Each consists of an address (subject to relocation)
-## followed by a ULEB for the size.
-    Content: "000000000000000010000000000000000020"
+    Entries:
+      - Size: 0x10
+      - Size: 0x20
     Link:    .text
-  - Name:    .stack_sizes.baz
+  - Name:    '.stack_sizes [1]'
     Type:    SHT_PROGBITS
-## One stack size entry.
-    Content: "200000000000000008"
+    Entries:
+      - Address: 0x20
+        Size:    0x8
     Link:    .text.baz
   - Name:    .rela.stack_sizes
     Type:    SHT_RELA
@@ -50,9 +67,9 @@ Sections:
         Addend: 16
         Symbol: .text
         Type:   R_X86_64_64
-  - Name:   .rela.stack_sizes.baz
+  - Name:   '.rela.stack_sizes [1]'
     Type:   SHT_RELA
-    Info:   .stack_sizes.baz
+    Info:   '.stack_sizes [1]'
     Relocations:
       - Offset: 0
         Symbol: separate_text_section_baz
@@ -79,12 +96,24 @@ Symbols:
 
 # RUN: yaml2obj --docnum=2 %s > %t02
 # RUN: llvm-readelf --stack-sizes %t02 \
-# RUN:   | FileCheck %s --check-prefix=EXEC --strict-whitespace --match-full-lines
-
-#      EXEC:         Size     Function
-# EXEC-NEXT:           16     foo
-# EXEC-NEXT:           32     bar
-#  EXEC-NOT:{{.}}
+# RUN:   | FileCheck %s --check-prefix=EXEC-GNU --strict-whitespace --match-full-lines
+# RUN: llvm-readobj --stack-sizes %t02 | FileCheck %s --check-prefix=EXEC-LLVM
+
+#      EXEC-GNU:         Size     Function
+# EXEC-GNU-NEXT:           16     foo
+# EXEC-GNU-NEXT:           32     bar
+#  EXEC-GNU-NOT:{{.}}
+
+# EXEC-LLVM:      StackSizes [
+# EXEC-LLVM-NEXT:   Entry {
+# EXEC-LLVM-NEXT:     Function: foo
+# EXEC-LLVM-NEXT:     Size: 0x10
+# EXEC-LLVM-NEXT:   }
+# EXEC-LLVM-NEXT:   Entry {
+# EXEC-LLVM-NEXT:     Function: bar
+# EXEC-LLVM-NEXT:     Size: 0x20
+# EXEC-LLVM-NEXT:   }
+# EXEC-LLVM-NEXT: ]
 
 --- !ELF
 FileHeader:
@@ -99,7 +128,11 @@ Sections:
     Size:    16
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
-    Content: "100000000000000010200000000000000020"
+    Entries:
+      - Address: 0x10
+        Size:    0x10
+      - Address: 0x20
+        Size:    0x20
     Link:    .text
 Symbols:
   - Name:    foo
@@ -118,6 +151,7 @@ Symbols:
 
 # RUN: yaml2obj --docnum=3 %s > %t03
 # RUN: not llvm-readelf --stack-sizes %t03 2>&1 | FileCheck %s --check-prefix=SHORT -DFILE=%t03
+# RUN: not llvm-readobj --stack-sizes %t03 2>&1 | FileCheck %s --check-prefix=SHORT -DFILE=%t03
 
 # SHORT: error: '[[FILE]]': found invalid relocation offset into section .stack_sizes while trying to extract a stack size entry
 
@@ -134,7 +168,7 @@ Sections:
     Size:    16
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
-    Size:    1
+    Content: "00"
     Link:    .text
   - Name:    .rela.stack_sizes
     Type:    SHT_RELA
@@ -153,12 +187,26 @@ Symbols:
 ## that is referenced by the stack sizes section's sh_link.
 
 # RUN: yaml2obj --docnum=4 %s > %t04
-# RUN: llvm-readelf --stack-sizes %t04 2> %t04.err | FileCheck %s --check-prefix=WRONGSECTION
-# RUN: FileCheck %s < %t04.err --check-prefix=WRONGSECTION-ERR -DFILE=%t04
+# RUN: llvm-readelf --stack-sizes %t04 2> %t04-gnu.err | FileCheck %s --check-prefix=WRONGSECTION-GNU
+# RUN: FileCheck %s < %t04-gnu.err --check-prefix=WRONGSECTION-ERR -DFILE=%t04
+# RUN: llvm-readobj --stack-sizes %t04 2> %t04-llvm.err | FileCheck %s --check-prefix=WRONGSECTION-LLVM
+# RUN: FileCheck %s < %t04-llvm.err --check-prefix=WRONGSECTION-ERR -DFILE=%t04
+
+# RUN: llvm-readelf --stack-sizes --demangle %t04 2>&1 | FileCheck %s --check-prefix=WRONGSECTION-DEMANGLE-ERR -DFILE=%t04
+# RUN: llvm-readobj --stack-sizes --demangle %t04 2>&1 | FileCheck %s --check-prefix=WRONGSECTION-DEMANGLE-ERR -DFILE=%t04
+
+# WRONGSECTION-GNU:      Size Function
+# WRONGSECTION-GNU-NEXT: 8 _Z3foof
+
+# WRONGSECTION-LLVM:      StackSizes [
+# WRONGSECTION-LLVM-NEXT:   Entry {
+# WRONGSECTION-LLVM-NEXT:     Function: _Z3foof
+# WRONGSECTION-LLVM-NEXT:     Size: 0x8
+# WRONGSECTION-LLVM-NEXT:   }
+# WRONGSECTION-LLVM-NEXT: ]
 
-# WRONGSECTION:      Size Function
-# WRONGSECTION-NEXT: 8 foo
-# WRONGSECTION-ERR:  warning: '[[FILE]]': relocation symbol foo is not in the expected section
+# WRONGSECTION-ERR:           warning: '[[FILE]]': relocation symbol '_Z3foof' is not in the expected section
+# WRONGSECTION-DEMANGLE-ERR:  warning: '[[FILE]]': relocation symbol 'foo(float)' is not in the expected section
 
 --- !ELF
 FileHeader:
@@ -176,17 +224,18 @@ Sections:
     Flags:   [SHF_ALLOC]
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
-    Content: "000000000000000008"
+    Entries:
+      - Size: 0x8
     Link:    .text2
   - Name:    .rela.stack_sizes
     Type:    SHT_RELA
     Info:    .stack_sizes
     Relocations:
       - Offset: 0
-        Symbol: foo 
+        Symbol: _Z3foof
         Type:   R_X86_64_64
 Symbols:
-  - Name:    foo
+  - Name:    _Z3foof
     Section: .text
     Type:    STT_FUNC
     Binding: STB_GLOBAL
@@ -194,7 +243,9 @@ Symbols:
 ## Check that we report an error when a stack sizes section ends with an incomplete stack size entry.
 
 # RUN: yaml2obj --docnum=5 %s > %t05
-# RUN: not llvm-readelf --stack-sizes %t05 2>&1 %t05.err | \
+# RUN: not llvm-readelf --stack-sizes %t05 2>&1 | \
+# RUN:    FileCheck %s --check-prefix=SUDDENEND -DFILE=%t05
+# RUN: not llvm-readobj --stack-sizes %t05 2>&1 | \
 # RUN:    FileCheck %s --check-prefix=SUDDENEND -DFILE=%t05
 
 # SUDDENEND: error: '[[FILE]]': section .stack_sizes ended while trying to extract a stack size entry
@@ -226,6 +277,7 @@ Symbols:
 
 # RUN: yaml2obj --docnum=6 %s > %t06
 # RUN: not llvm-readelf --stack-sizes %t06 2>&1 | FileCheck %s --check-prefix=BADSIZE -DFILE=%t06
+# RUN: not llvm-readobj --stack-sizes %t06 2>&1 | FileCheck %s --check-prefix=BADSIZE -DFILE=%t06
 
 # BADSIZE: error: '[[FILE]]': could not extract a valid stack size in section .stack_sizes
 
@@ -256,12 +308,26 @@ Symbols:
 ## output.
 
 # RUN: yaml2obj --docnum=7 %s > %t07
-# RUN: llvm-readelf --stack-sizes %t07 2> %t07.err | FileCheck %s --check-prefix=BADSECTION-OUT
-# RUN: FileCheck %s < %t07.err --check-prefix=BADSECTION-ERR -DFILE=%t07
+# RUN: llvm-readelf --stack-sizes %t07 2> %t07-gnu.err | FileCheck %s --check-prefix=BADSECTION-OUT-GNU
+# RUN: FileCheck %s < %t07-gnu.err --check-prefix=BADSECTION-ERR -DFILE=%t07
+# RUN: llvm-readobj --stack-sizes %t07 2> %t07-llvm.err | FileCheck %s --check-prefix=BADSECTION-OUT-LLVM
+# RUN: FileCheck %s < %t07-llvm.err --check-prefix=BADSECTION-ERR -DFILE=%t07
+
+# RUN: llvm-readelf --stack-sizes --demangle %t07 2>&1 | FileCheck %s --check-prefix=BADSECTION-DEMANGLE-ERR -DFILE=%t07
+# RUN: llvm-readobj --stack-sizes --demangle %t07 2>&1 | FileCheck %s --check-prefix=BADSECTION-DEMANGLE-ERR -DFILE=%t07
 
-# BADSECTION-OUT: Size Function
-# BADSECTION-OUT:    8 ?
-# BADSECTION-ERR: warning: '[[FILE]]': cannot identify the section for relocation symbol foo
+# BADSECTION-OUT-GNU: Size Function
+# BADSECTION-OUT-GNU:    8 ?
+
+# BADSECTION-OUT-LLVM:      StackSizes [
+# BADSECTION-OUT-LLVM-NEXT:   Entry {
+# BADSECTION-OUT-LLVM-NEXT:     Function: ?
+# BADSECTION-OUT-LLVM-NEXT:     Size: 0x8
+# BADSECTION-OUT-LLVM-NEXT:   }
+# BADSECTION-OUT-LLVM-NEXT: ]
+
+# BADSECTION-ERR: warning: '[[FILE]]': cannot identify the section for relocation symbol '_Z3foof'
+# BADSECTION-DEMANGLE-ERR: warning: '[[FILE]]': cannot identify the section for relocation symbol 'foo(float)'
 
 --- !ELF
 FileHeader:
@@ -276,16 +342,17 @@ Sections:
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
     Link:    .text
-    Content: "000000000000000008"
+    Entries:
+      - Size: 0x8
   - Name:    .rela.stack_sizes
     Type:    SHT_RELA
     Info:    .stack_sizes
     Relocations:
     - Offset: 0
-      Symbol: foo 
+      Symbol: _Z3foof
       Type:   R_X86_64_64
 Symbols:
-  - Name:    foo
+  - Name:    _Z3foof
 ## An invalid section index.
     Index:   10
     Type:    STT_FUNC
@@ -295,12 +362,18 @@ Symbols:
 ## a corresponding relocation section.
 
 # RUN: yaml2obj --docnum=8 %s > %t08
-# RUN: llvm-readelf --stack-sizes %t08 2> %t08.err | FileCheck %s --check-prefix=NORELOCSECTION-OUT
-# RUN: FileCheck %s < %t08.err --check-prefix=NORELOCSECTION-ERR -DFILE=%t08
+# RUN: llvm-readelf --stack-sizes %t08 2> %t08-gnu.err | FileCheck %s --check-prefix=NORELOCSECTION-OUT-GNU
+# RUN: FileCheck %s < %t08-gnu.err --check-prefix=NORELOCSECTION-ERR -DFILE=%t08
+# RUN: llvm-readobj --stack-sizes %t08 2> %t08-llvm.err | FileCheck %s --check-prefix=NORELOCSECTION-OUT-LLVM
+# RUN: FileCheck %s < %t08-llvm.err --check-prefix=NORELOCSECTION-ERR -DFILE=%t08
+
+# NORELOCSECTION-OUT-GNU:     Size Function
+# NORELOCSECTION-OUT-GNU-NOT: {{.}}
 
-# NORELOCSECTION-OUT:     Size Function
-# NORELOCSECTION-OUT-NOT: {{.}}
-# NORELOCSECTION-ERR:     warning: '[[FILE]]': section .stack_sizes does not have a corresponding relocation section
+# NORELOCSECTION-OUT-LLVM:      StackSizes [
+# NORELOCSECTION-OUT-LLVM-NEXT: ]
+
+# NORELOCSECTION-ERR: warning: '[[FILE]]': section .stack_sizes does not have a corresponding relocation section
 
 --- !ELF
 FileHeader:
@@ -315,7 +388,8 @@ Sections:
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
     Link:    .text
-    Content: "000000000000000008"
+    Entries:
+      - Size: 0x1
 
 ## Check that we handle multiple object files, separately and when they
 ## are in an archive. This also checks whether we have blank lines between the
@@ -323,36 +397,79 @@ Sections:
 
 # RUN: llvm-ar rc %t1.a %t01 %t02
 # RUN: llvm-readelf --stack-sizes %t01 %t02 \
-# RUN:   | FileCheck %s --check-prefixes=MULTIPLE,OBJECT -DFILE1=%t01 -DFILE2=%t02
+# RUN:   | FileCheck %s --check-prefixes=MULTIPLE-GNU,OBJECT -DFILE1=%t01 -DFILE2=%t02
 # RUN: llvm-readelf --stack-sizes %t1.a \
-# RUN:   | FileCheck %s --check-prefixes=MULTIPLE,ARCHIVE --strict-whitespace\
+# RUN:   | FileCheck %s --check-prefixes=MULTIPLE-GNU,ARCHIVE --strict-whitespace\
 # RUN:   --match-full-lines -DFILE=%t1.a
+# RUN: llvm-readobj --stack-sizes %t01 %t02 \
+# RUN:   | FileCheck %s --check-prefixes=MULTIPLE-LLVM,OBJECT -DFILE1=%t01 -DFILE2=%t02
+# RUN: llvm-readobj --stack-sizes %t1.a \
+# RUN:   | FileCheck %s --check-prefixes=MULTIPLE-LLVM,ARCHIVE -DFILE=%t1.a
 
 #        OBJECT:File: [[FILE1]]
 #       ARCHIVE:File: [[FILE]]({{.*01}})
-#      MULTIPLE:Stack Sizes:
-# MULTIPLE-NEXT:         Size     Function
-# MULTIPLE-NEXT:           16     referenced_by_symbol_foo
-# MULTIPLE-NEXT:           32     referenced_via_section_bar
-# MULTIPLE-NEXT:            8     separate_text_section_baz
-# MULTIPLE-EMPTY:
+
+#      MULTIPLE-GNU:Stack Sizes:
+# MULTIPLE-GNU-NEXT:         Size     Function
+# MULTIPLE-GNU-NEXT:           16     referenced_by_symbol_foo
+# MULTIPLE-GNU-NEXT:           32     referenced_via_section_bar
+# MULTIPLE-GNU-NEXT:            8     separate_text_section_baz
+# MULTIPLE-GNU-EMPTY:
+
+# MULTIPLE-LLVM:      StackSizes [
+# MULTIPLE-LLVM-NEXT:   Entry {
+# MULTIPLE-LLVM-NEXT:     Function: referenced_by_symbol_foo
+# MULTIPLE-LLVM-NEXT:     Size: 0x10
+# MULTIPLE-LLVM-NEXT:   }
+# MULTIPLE-LLVM-NEXT:   Entry {
+# MULTIPLE-LLVM-NEXT:     Function: referenced_via_section_bar
+# MULTIPLE-LLVM-NEXT:     Size: 0x20
+# MULTIPLE-LLVM-NEXT:   }
+# MULTIPLE-LLVM-NEXT:   Entry {
+# MULTIPLE-LLVM-NEXT:     Function: separate_text_section_baz
+# MULTIPLE-LLVM-NEXT:     Size: 0x8
+# MULTIPLE-LLVM-NEXT:   }
+# MULTIPLE-LLVM-NEXT: ]
+
 #        OBJECT:File: [[FILE2]]
 #       ARCHIVE:File: [[FILE]]({{.*02}})
-# MULTIPLE-EMPTY:
-# MULTIPLE-NEXT:Stack Sizes:
-# MULTIPLE-NEXT:         Size     Function
-# MULTIPLE-NEXT:           16     foo
-# MULTIPLE-NEXT:           32     bar
+
+# MULTIPLE-GNU-EMPTY:
+# MULTIPLE-GNU-NEXT:Stack Sizes:
+# MULTIPLE-GNU-NEXT:         Size     Function
+# MULTIPLE-GNU-NEXT:           16     foo
+# MULTIPLE-GNU-NEXT:           32     bar
+
+# MULTIPLE-LLVM:      StackSizes [
+# MULTIPLE-LLVM-NEXT:   Entry {
+# MULTIPLE-LLVM-NEXT:     Function: foo
+# MULTIPLE-LLVM-NEXT:     Size: 0x10
+# MULTIPLE-LLVM-NEXT:   }
+# MULTIPLE-LLVM-NEXT:   Entry {
+# MULTIPLE-LLVM-NEXT:     Function: bar
+# MULTIPLE-LLVM-NEXT:     Size: 0x20
+# MULTIPLE-LLVM-NEXT:   }
+# MULTIPLE-LLVM-NEXT: ]
 
 ## Check that we do not consider symbols that are not function symbols, even though
 ## a relocation references them. 
 
 # RUN: yaml2obj --docnum=9 %s > %t14
-# RUN: llvm-readelf --stack-sizes %t14 2> %t14.err | FileCheck %s --check-prefix=NONFUNCTIONSYM
-# RUN: FileCheck %s < %t14.err --check-prefix=NONFUNCTIONSYM-ERR -DFILE=%t14
+# RUN: llvm-readelf --stack-sizes %t14 2> %t14-gnu.err | FileCheck %s --check-prefix=NONFUNCTIONSYM-GNU
+# RUN: FileCheck %s < %t14-gnu.err --check-prefix=NONFUNCTIONSYM-ERR -DFILE=%t14
+# RUN: llvm-readobj --stack-sizes %t14 2> %t14-llvm.err | FileCheck %s --check-prefix=NONFUNCTIONSYM-LLVM
+# RUN: FileCheck %s < %t14-llvm.err --check-prefix=NONFUNCTIONSYM-ERR -DFILE=%t14
+
+# NONFUNCTIONSYM-GNU:     Stack Sizes:
+# NONFUNCTIONSYM-GNU:     0 ?
+
+# NONFUNCTIONSYM-LLVM:      StackSizes [
+# NONFUNCTIONSYM-LLVM-NEXT:   Entry {
+# NONFUNCTIONSYM-LLVM-NEXT:     Function: ?
+# NONFUNCTIONSYM-LLVM-NEXT:     Size: 0x0
+# NONFUNCTIONSYM-LLVM-NEXT:   }
+# NONFUNCTIONSYM-LLVM-NEXT: ]
 
-# NONFUNCTIONSYM:     Stack Sizes:
-# NONFUNCTIONSYM:     0 ?
 # NONFUNCTIONSYM-ERR: warning: '[[FILE]]': could not identify function symbol for stack size entry
 
 --- !ELF
@@ -367,7 +484,8 @@ Sections:
     Size: 16
   - Name: .stack_sizes
     Type: SHT_PROGBITS
-    Size: 9
+    Entries:
+      - Size: 0
     Link: .text
   - Name: .rela.stack_sizes
     Type: SHT_RELA
@@ -387,6 +505,7 @@ Symbols:
 
 # RUN: yaml2obj --docnum=10 %s > %t15
 # RUN: not llvm-readelf --stack-sizes %t15 2>&1 | FileCheck %s --check-prefix=UNSUPPRELOC -DFILE=%t15
+# RUN: not llvm-readobj --stack-sizes %t15 2>&1 | FileCheck %s --check-prefix=UNSUPPRELOC -DFILE=%t15
 
 # UNSUPPRELOC: error: '[[FILE]]': unsupported relocation type in section .rela.stack_sizes: R_X86_64_RELATIVE
 
@@ -402,8 +521,8 @@ Sections:
     Size: 8
   - Name: .stack_sizes
     Type: SHT_PROGBITS
-    Size: 16
     Link: .text
+    Entries: []
   - Name: .rela.stack_sizes
     Type: SHT_RELA
     Info: .stack_sizes
@@ -422,28 +541,56 @@ Symbols:
 ## is still dumped.
 
 # RUN: llvm-ar rc %t2.a %t04 %t01
-# RUN: llvm-readelf --stack-sizes %t2.a 2>&1 | FileCheck %s --check-prefix=ARCHIVEWARN \
+# RUN: llvm-readelf --stack-sizes %t2.a 2>&1 | FileCheck %s --check-prefix=ARCHIVEWARN-GNU \
 # RUN:   -DFILE=%t2.a --strict-whitespace --match-full-lines
-
-#      ARCHIVEWARN:File: [[FILE]]({{.*04}})
-#      ARCHIVEWARN:Stack Sizes:
-# ARCHIVEWARN-NEXT:         Size     Function
-#      ARCHIVEWARN:            8     foo
-#      ARCHIVEWARN:File: [[FILE]]({{.*01}})
-#      ARCHIVEWARN:Stack Sizes:
-# ARCHIVEWARN-NEXT:         Size     Function
-# ARCHIVEWARN-NEXT:           16     referenced_by_symbol_foo
-# ARCHIVEWARN-NEXT:           32     referenced_via_section_bar
-# ARCHIVEWARN-NEXT:            8     separate_text_section_baz
-#  ARCHIVEWARN-NOT:{{.}}
-#  ARCHIVEWARN-ERR:warning: '[[FILE]]': relocation symbol foo is not in the expected section
+# RUN: llvm-readobj --stack-sizes %t2.a 2>&1 | FileCheck %s --check-prefix=ARCHIVEWARN-LLVM -DFILE=%t2.a
+
+#      ARCHIVEWARN-GNU:File: [[FILE]]({{.*04}})
+#      ARCHIVEWARN-GNU:Stack Sizes:
+# ARCHIVEWARN-GNU-NEXT:         Size     Function
+#      ARCHIVEWARN-GNU:{{.*}}: warning: '{{.*04}}': relocation symbol '_Z3foof' is not in the expected section
+#      ARCHIVEWARN-GNU:            8     _Z3foof
+#      ARCHIVEWARN-GNU:File: [[FILE]]({{.*01}})
+#      ARCHIVEWARN-GNU:Stack Sizes:
+# ARCHIVEWARN-GNU-NEXT:         Size     Function
+# ARCHIVEWARN-GNU-NEXT:           16     referenced_by_symbol_foo
+# ARCHIVEWARN-GNU-NEXT:           32     referenced_via_section_bar
+# ARCHIVEWARN-GNU-NEXT:            8     separate_text_section_baz
+#  ARCHIVEWARN-GNU-NOT:{{.}}
+
+
+# ARCHIVEWARN-LLVM:      File: [[FILE]]({{.*04}})
+# ARCHIVEWARN-LLVM:      StackSizes [
+# ARCHIVEWARN-LLVM:      warning: '{{.*04}}': relocation symbol '_Z3foof' is not in the expected section
+# ARCHIVEWARN-LLVM-NEXT:   Entry {
+# ARCHIVEWARN-LLVM-NEXT:     Function: _Z3foof
+# ARCHIVEWARN-LLVM-NEXT:     Size: 0x8
+# ARCHIVEWARN-LLVM-NEXT:   }
+# ARCHIVEWARN-LLVM-NEXT: ]
+# ARCHIVEWARN-LLVM:      File: [[FILE]]({{.*01}})
+# ARCHIVEWARN-LLVM:      StackSizes [
+# ARCHIVEWARN-LLVM-NEXT:   Entry {
+# ARCHIVEWARN-LLVM-NEXT:     Function: referenced_by_symbol_foo
+# ARCHIVEWARN-LLVM-NEXT:     Size: 0x10
+# ARCHIVEWARN-LLVM-NEXT:   }
+# ARCHIVEWARN-LLVM-NEXT:   Entry {
+# ARCHIVEWARN-LLVM-NEXT:     Function: referenced_via_section_bar
+# ARCHIVEWARN-LLVM-NEXT:     Size: 0x20
+# ARCHIVEWARN-LLVM-NEXT:   }
+# ARCHIVEWARN-LLVM-NEXT:   Entry {
+# ARCHIVEWARN-LLVM-NEXT:     Function: separate_text_section_baz
+# ARCHIVEWARN-LLVM-NEXT:     Size: 0x8
+# ARCHIVEWARN-LLVM-NEXT:   }
+# ARCHIVEWARN-LLVM-NEXT: ]
 
 ## Check that we demangle function names when requested.
 
 # RUN: yaml2obj --docnum=11 %s > %t16
-# RUN: llvm-readelf --stack-sizes --demangle %t16 | FileCheck %s --check-prefix=DEMANGLE
+# RUN: llvm-readelf --stack-sizes --demangle %t16 | FileCheck %s --check-prefix=DEMANGLE-GNU
+# RUN: llvm-readobj --stack-sizes --demangle %t16 | FileCheck %s --check-prefix=DEMANGLE-LLVM
 
-# DEMANGLE: 16 foo(float)
+# DEMANGLE-GNU:  16 foo(float)
+# DEMANGLE-LLVM: Function: foo(float)
 
 --- !ELF
 FileHeader:
@@ -458,7 +605,9 @@ Sections:
     Size:    16
   - Name:    .stack_sizes
     Type:    SHT_PROGBITS
-    Content: "100000000000000010"
+    Entries:
+      - Address: 0x10
+        Size:    0x10
     Link:    .text
 Symbols:
   - Name:    _Z3foof
@@ -467,10 +616,28 @@ Symbols:
     Type:    STT_FUNC
     Binding: STB_GLOBAL
 
-## Check that we emit a 'not implemented' message for an attempt to dump stack-sizes
-## sections LLVM-style, i.e. when invoking llvm-readobj. 
-## FIXME: Replace this test with something functional when the feature is implemented.
+## Check that we report an error when we are unable to resolve a relocation for a given ELF architecture.
+## Here we have a 64-bit relocation used in a 32-bit object.
+
+# RUN: yaml2obj --docnum=12 %s > %t17
+# RUN: not llvm-readelf --stack-sizes %t17 2>&1 | FileCheck %s -DFILE=%t17 --check-prefix=UNSUPPRELOC2
+# RUN: not llvm-readobj --stack-sizes %t17 2>&1 | FileCheck %s -DFILE=%t17 --check-prefix=UNSUPPRELOC2
 
-# RUN: llvm-readobj --stack-sizes %t01 | FileCheck %s --check-prefix=NOTIMPLEMENTED 
+# UNSUPPRELOC2: error: '[[FILE]]': unsupported relocation type in section .rela.stack_sizes: R_X86_64_64
 
-# NOTIMPLEMENTED: Dumping of stack sizes in LLVM style is not implemented yet
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+    Content: "00"
+  - Name: .rela.stack_sizes
+    Type: SHT_RELA
+    Info: .stack_sizes
+    Relocations:
+      - Offset: 0
+        Type:   R_X86_64_64
diff --git a/llvm/test/tools/llvm-size/elf-sysv.test b/llvm/test/tools/llvm-size/elf-sysv.test
index 2ce909a301367..54f978069424a 100644
--- a/llvm/test/tools/llvm-size/elf-sysv.test
+++ b/llvm/test/tools/llvm-size/elf-sysv.test
@@ -63,6 +63,7 @@ Sections:
     Address: 0x20000
   - Name:    .hash
     Type:    SHT_HASH
+    Content: ''
     ShSize:  0x40
     Address: 0x10000
   - Name:    .dynamic
diff --git a/llvm/test/tools/obj2yaml/elf-emachine.yaml b/llvm/test/tools/obj2yaml/elf-emachine.yaml
new file mode 100644
index 0000000000000..d351505aa2845
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/elf-emachine.yaml
@@ -0,0 +1,39 @@
+## Check how obj2yaml dumps e_machine field.
+
+## Check it dumps an unknown e_machine as a number.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: obj2yaml %t1 | FileCheck %s --check-prefix=UNKNOWN
+
+# UNKNOWN:      --- !ELF
+# UNKNOWN-NEXT: FileHeader:
+# UNKNOWN-NEXT:   Class:   ELFCLASS64
+# UNKNOWN-NEXT:   Data:    ELFDATA2MSB
+# UNKNOWN-NEXT:   Type:    ET_REL
+# UNKNOWN-NEXT:   Machine: 0x1234
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0x1234
+
+## Check it dumps a known e_machine value as an enum string.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: obj2yaml %t2 | FileCheck %s --check-prefix=KNOWN
+
+# KNOWN:      --- !ELF
+# KNOWN-NEXT: FileHeader:
+# KNOWN-NEXT:   Class:   ELFCLASS64
+# KNOWN-NEXT:   Data:    ELFDATA2MSB
+# KNOWN-NEXT:   Type:    ET_REL
+# KNOWN-NEXT:   Machine: EM_NONE
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0
diff --git a/llvm/test/tools/obj2yaml/elf-hash-section.yaml b/llvm/test/tools/obj2yaml/elf-hash-section.yaml
new file mode 100644
index 0000000000000..4f49254771a04
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/elf-hash-section.yaml
@@ -0,0 +1,78 @@
+## Check how obj2yaml produces SHT_HASH section descriptions.
+
+## Check that obj2yaml uses "Bucket" and "Chain" tags to describe
+## a SHT_HASH section when it has content of a correct size.
+## I.e. data size == 4 * (2 + nbucket + nchain).
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: obj2yaml %t1 | FileCheck %s --check-prefix=CHAIN-BUCKET
+
+# CHAIN-BUCKET:      - Name:  .hash1
+# CHAIN-BUCKET-NEXT:  Type:   SHT_HASH
+# CHAIN-BUCKET-NEXT:  Bucket: [ 3 ]
+# CHAIN-BUCKET-NEXT:  Chain:  [ 4, 5 ]
+# CHAIN-BUCKET:      - Name:  .hash2
+# CHAIN-BUCKET-NEXT:  Type:   SHT_HASH
+# CHAIN-BUCKET-NEXT:  Bucket: [  ]
+# CHAIN-BUCKET-NEXT:  Chain:  [  ]
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+## Case 1: A non-empty hash table: nbucket == 1, nchain == 2. 
+  - Name:    .hash1
+    Type:    SHT_HASH
+    Content: '0100000002000000030000000400000005000000'
+## Case 2: An empty hash table: nbucket == 0, nchain == 0.
+  - Name:    .hash2
+    Type:    SHT_HASH
+    Content: '0000000000000000'
+
+## Check that obj2yaml falls back to using the "Content" tag when
+## hash sections are broken.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: obj2yaml %t2 | FileCheck %s --check-prefix=CONTENT
+
+# CONTENT:      - Name:    .empty_hash
+# CONTENT-NEXT:   Type:    SHT_HASH
+# CONTENT-NEXT:   Content: ''
+# CONTENT-NEXT: - Name:    .invalid_header
+# CONTENT-NEXT:   Type:    SHT_HASH
+# CONTENT-NEXT:   Content: '00'
+# CONTENT-NEXT: - Name:    .truncated
+# CONTENT-NEXT:   Type:    SHT_HASH
+# CONTENT-NEXT:   Content: '01000000020000000300000004000000'
+# CONTENT-NEXT: - Name:    .oversized
+# CONTENT-NEXT:   Type:    SHT_HASH
+# CONTENT-NEXT:   Content: '0100000002000000030000000400000000'
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+## Case 1: section has no data.
+  - Name:    .empty_hash
+    Type:    SHT_HASH
+    Content: ''
+## Case 2: section size is less than 2 * 4.
+  - Name:    .invalid_header
+    Type:    SHT_HASH
+    Content: '00'
+## Case 3: nbucket == 1, nchain == 2.
+##         Section size is less than (2 * nbucket + nchain) * 4.
+  - Name:    .truncated
+    Type:    SHT_HASH
+    Content: '01000000020000000300000004000000'
+## Case 4: nbucket == 1, nchain == 2.
+##         Section size is greater than (2 * nbucket + nchain) * 4.
+  - Name:    .oversized
+    Type:    SHT_HASH
+    Content: '0100000002000000030000000400000000'
diff --git a/llvm/test/tools/obj2yaml/elf-ppc64-relocations.yaml b/llvm/test/tools/obj2yaml/elf-ppc64-relocations.yaml
new file mode 100644
index 0000000000000..cee6a8b667220
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/elf-ppc64-relocations.yaml
@@ -0,0 +1,488 @@
+## Check obj2yaml is able to decode all PPC64 relocations.
+
+# RUN: yaml2obj %s -o %t
+# RUN: obj2yaml %t | FileCheck %s
+
+# CHECK:      --- !ELF
+# CHECK-NEXT: FileHeader:
+# CHECK-NEXT:   Class:   ELFCLASS64
+# CHECK-NEXT:   Data:    ELFDATA2MSB
+# CHECK-NEXT:   Type:    ET_REL
+# CHECK-NEXT:   Machine: EM_PPC64
+# CHECK-NEXT: Sections:
+# CHECK-NEXT:   - Name:    .rela.text
+# CHECK-NEXT:     Type:    SHT_RELA
+# CHECK-NEXT:     Link:    .symtab
+# CHECK-NEXT:     EntSize: 0x0000000000000018
+# CHECK-NEXT:     Relocations:
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_NONE
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR32
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR24
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR14
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR14_BRTAKEN
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR14_BRNTAKEN
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL24
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL14
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL14_BRTAKEN
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL14_BRNTAKEN
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GLOB_DAT
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_JMP_SLOT
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_RELATIVE
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL32
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR64
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGHER
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGHERA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGHEST
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGHESTA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL64
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TOC16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TLS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPMOD64
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL64
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL64
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSGD16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSGD16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSGD16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSGD16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSLD16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSLD16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSLD16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TLSLD16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TPREL16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TPREL16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TPREL16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_TPREL16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_DTPREL16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_DTPREL16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_DTPREL16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_GOT_DTPREL16_HA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGHER
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGHERA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGHEST
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGHESTA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_LO_DS
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGHER
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGHERA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGHEST
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGHESTA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TLSGD
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TLSLD
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGH
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_ADDR16_HIGHA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGH
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_TPREL16_HIGHA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGH
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_DTPREL16_HIGHA
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_IRELATIVE
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL16
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL16_LO
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL16_HI
+# CHECK-NEXT:       - Offset: 0x0000000000000000
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   R_PPC64_REL16_HA
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_PPC64
+Sections:
+  - Name: .rela.text
+    Type: SHT_RELA
+    Relocations:
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_NONE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR32
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR24
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14_BRTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR14_BRNTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL24
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14_BRTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL14_BRNTAKEN
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GLOB_DAT
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_JMP_SLOT
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_RELATIVE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL32
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TOC16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPMOD64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL64
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSGD16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TLSLD16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_TPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_GOT_DTPREL16_HA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_LO_DS
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHER
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHERA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHEST
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHESTA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLSGD
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TLSLD
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_ADDR16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_TPREL16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGH
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_DTPREL16_HIGHA
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_IRELATIVE
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_LO
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_HI
+      - Offset: 0x0000000000000000
+        Type:   R_PPC64_REL16_HA
diff --git a/llvm/test/tools/obj2yaml/elf-stack-sizes.yaml b/llvm/test/tools/obj2yaml/elf-stack-sizes.yaml
new file mode 100644
index 0000000000000..56a05739d10aa
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/elf-stack-sizes.yaml
@@ -0,0 +1,88 @@
+## Check how obj2yaml produces YAML .stack_sizes descriptions.
+
+## Check that obj2yaml uses the "Entries" tag to describe a .stack_sizes section
+## when it can extract <address, size> pairs.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: obj2yaml %t1 | FileCheck %s --check-prefix=VALID
+
+# VALID:      --- !ELF
+# VALID-NEXT: FileHeader:
+# VALID-NEXT:   Class:   ELFCLASS64
+# VALID-NEXT:   Data:    ELFDATA2LSB
+# VALID-NEXT:   Type:    ET_EXEC
+# VALID-NEXT:   Machine: EM_X86_64
+# VALID-NEXT: Sections:
+# VALID-NEXT:   - Name: .stack_sizes
+# VALID-NEXT:     Type: SHT_PROGBITS
+# VALID-NEXT:     Entries:
+# VALID-NEXT:       - Address: 0x0000000000000010
+# VALID-NEXT:         Size:    0x0000000000000020
+# VALID-NEXT:       - Address: 0x0000000000000030
+# VALID-NEXT:         Size:    0x0000000000000040
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Content: "100000000000000020300000000000000040"
+
+## Check that obj2yaml uses the "Content" tag to describe a .stack_sizes section
+## when it can't extract the entries, for example, when section data is truncated.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: obj2yaml %t2 | FileCheck %s --check-prefix=INVALID
+
+# INVALID:      --- !ELF
+# INVALID-NEXT: FileHeader:
+# INVALID-NEXT:   Class:   ELFCLASS64
+# INVALID-NEXT:   Data:    ELFDATA2LSB
+# INVALID-NEXT:   Type:    ET_EXEC
+# INVALID-NEXT:   Machine: EM_X86_64
+# INVALID-NEXT: Sections:
+# INVALID-NEXT:   - Name:    .stack_sizes
+# INVALID-NEXT:     Type:    SHT_PROGBITS
+# INVALID-NEXT:     Content: '1000000000000000203000000000000000'
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Content: "1000000000000000203000000000000000"
+
+## Check obj2yaml can dump empty .stack_sizes.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: obj2yaml %t3 | FileCheck %s --check-prefix=EMPTY
+
+# EMPTY:      --- !ELF
+# EMPTY-NEXT: FileHeader:
+# EMPTY-NEXT:   Class:   ELFCLASS64
+# EMPTY-NEXT:   Data:    ELFDATA2LSB
+# EMPTY-NEXT:   Type:    ET_EXEC
+# EMPTY-NEXT:   Machine: EM_X86_64
+# EMPTY-NEXT: Sections:
+# EMPTY-NEXT:   - Name:    .stack_sizes
+# EMPTY-NEXT:     Type:    SHT_PROGBITS
+# EMPTY-NEXT:     Content: ''
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Content: ""
diff --git a/llvm/test/tools/obj2yaml/relocation-unsupported-machine.yaml b/llvm/test/tools/obj2yaml/relocation-unsupported-machine.yaml
new file mode 100644
index 0000000000000..e55420f07b393
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/relocation-unsupported-machine.yaml
@@ -0,0 +1,38 @@
+## Show that obj2yaml is able to dump relocations when `e_machine` is unknown.
+
+# RUN: yaml2obj %s -o %t1
+# RUN: obj2yaml %t1 | FileCheck %s
+
+# CHECK:      --- !ELF
+# CHECK-NEXT: FileHeader:
+# CHECK-NEXT:   Class:   ELFCLASS64
+# CHECK-NEXT:   Data:    ELFDATA2MSB
+# CHECK-NEXT:   Type:    ET_REL
+# CHECK-NEXT:   Machine: 0x1234
+# CHECK-NEXT: Sections:
+# CHECK-NEXT:   - Name:    .rela.text
+# CHECK-NEXT:     Type:    SHT_RELA
+# CHECK-NEXT:     Link:    .symtab
+# CHECK-NEXT:     EntSize: 0x0000000000000018
+# CHECK-NEXT:     Relocations:
+# CHECK-NEXT:       - Offset: 0x0000000000000001
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   0x00000002
+# CHECK-NEXT:       - Offset: 0x0000000000000003
+# CHECK-NEXT:         Symbol: ''
+# CHECK-NEXT:         Type:   0x00000004
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0x1234
+Sections:
+  - Name: .rela.text
+    Type: SHT_RELA
+    Relocations:
+      - Offset: 0x1
+        Type:   0x2
+      - Offset: 0x3
+        Type:   0x4
diff --git a/llvm/test/tools/yaml2obj/elf-emachine.yaml b/llvm/test/tools/yaml2obj/elf-emachine.yaml
new file mode 100644
index 0000000000000..206be42c7dc5c
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/elf-emachine.yaml
@@ -0,0 +1,55 @@
+## Test how the Machine YAML field can be used to set the e_machine ELF header field.
+
+## Test we can use an arbitrary value.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: llvm-readelf %t1 --file-headers | FileCheck %s --check-prefix=UNKNOWN
+
+# UNKNOWN: Machine: 1234
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0x1234
+
+## Test we can't use a number that doesn't fit into 2 bytes.
+
+# RUN: not yaml2obj --docnum=2 %s 2>&1 | FileCheck %s --check-prefix=ERR
+
+# ERR: error: out of range hex16 number
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0x12345
+
+## Test we can use a known named constant.
+
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: llvm-readelf %t3 --file-headers | FileCheck %s --check-prefix=NONE
+
+# NONE: Machine: None
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_NONE
+
+## Test we can't use an unknown string constant.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=ERR2
+
+# ERR2: error: invalid hex16 number
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: EM_UNKNOWN_FOO
diff --git a/llvm/test/tools/yaml2obj/elf-hash-section.yaml b/llvm/test/tools/yaml2obj/elf-hash-section.yaml
new file mode 100644
index 0000000000000..4aad9c11fd592
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/elf-hash-section.yaml
@@ -0,0 +1,278 @@
+## Check how yaml2obj produces SHT_HASH sections.
+
+## Check we can describe a SHT_HASH section using the "Content" tag.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: llvm-readobj --sections --section-data %t1 | FileCheck %s --check-prefix=CONTENT
+
+# CONTENT:      Name: .hash
+# CONTENT-NEXT: Type: SHT_HASH
+# CONTENT-NEXT: Flags [
+# CONTENT-NEXT: ]
+# CONTENT-NEXT: Address: 0x0
+# CONTENT-NEXT: Offset: 0x44
+# CONTENT-NEXT: Size: 20
+# CONTENT-NEXT: Link: 1
+# CONTENT-NEXT: Info: 0
+# CONTENT-NEXT: AddressAlignment: 0
+# CONTENT-NEXT: EntrySize: 0
+# CONTENT-NEXT: SectionData (
+# CONTENT-NEXT:   0000: 01000000 02000000 03000000 04000000
+# CONTENT-NEXT:   0010: 05000000
+# CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+## SHT_HASH is linked to dynamic symbol table by default.
+  - Name: .dynsym
+    Type: SHT_DYNSYM
+  - Name:    .hash
+    Type:    SHT_HASH
+    Content: '0100000002000000030000000400000005000000'
+
+## Check we can describe a SHT_HASH section using "Bucket" and "Chain" tags.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s --check-prefix=BUCKET-CHAIN
+
+# BUCKET-CHAIN:      Name: .hash
+# BUCKET-CHAIN:      Size:
+# BUCKET-CHAIN-SAME: 28
+# BUCKET-CHAIN:      Link:
+# BUCKET-CHAIN-SAME: 0
+# BUCKET-CHAIN:      SectionData (
+# BUCKET-CHAIN-NEXT:   0000: 02000000 03000000 01000000 02000000  |
+# BUCKET-CHAIN-NEXT:   0010: 03000000 04000000 05000000           |
+# BUCKET-CHAIN-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Bucket: [ 1, 2 ]
+    Chain:  [ 3, 4, 5 ]
+
+## Check we can't use "Content" and "Bucket" tags together.
+
+# RUN: not yaml2obj --docnum=3 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-BUCKET
+
+# CONTENT-BUCKET: "Bucket" cannot be used with "Content" or "Size"
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Bucket: [ 1 ]
+    Content: '00'
+
+## Check we can't use "Content" and "Chain" tags together.
+
+# RUN: not yaml2obj --docnum=4 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-CHAIN
+
+# CONTENT-CHAIN: "Chain" cannot be used with "Content" or "Size"
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Chain:  [ 1 ]
+    Content: '00'
+
+## Check we can't use "Bucket" without "Chain".
+
+# RUN: not yaml2obj --docnum=5 %s 2>&1 | FileCheck %s --check-prefix=NO-BUCKET-OR-CHAIN
+
+# NO-BUCKET-OR-CHAIN: error: "Bucket" and "Chain" must be used together
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:   .hash
+    Type:   SHT_HASH
+    Bucket: [ 1 ]
+
+## Check we can't use "Chain" without "Bucket".
+
+# RUN: not yaml2obj --docnum=6 %s 2>&1 | FileCheck %s --check-prefix=NO-BUCKET-OR-CHAIN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:  .hash
+    Type:  SHT_HASH
+    Chain: [ 1 ]
+
+## Check we report an error if neither "Bucket", "Chain" nor "Content" were set.
+
+# RUN: not yaml2obj --docnum=7 %s 2>&1 | FileCheck %s --check-prefix=NO-TAGS
+
+# NO-TAGS: error: one of "Content", "Size", "Bucket" or "Chain" must be specified
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name:  .hash
+    Type:  SHT_HASH
+
+## Check we can set any sh_link value with use of the "Link" tag.
+
+# RUN: yaml2obj --docnum=8 %s -o %t8
+# RUN: llvm-readobj --sections %t8 | FileCheck %s --check-prefix=LINK
+
+# LINK:      Name: .hash1
+# LINK:      Link:
+# LINK-SAME: 123
+
+# LINK:      Name: .hash2
+# LINK:      Link:
+# LINK-SAME: 1
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_DYN
+  Machine: EM_386
+Sections:
+  - Name: .hash1
+    Type: SHT_HASH
+    Content: ""
+    Link: 123
+  - Name: .hash2
+    Type: SHT_HASH
+    Content: ""
+    Link: .hash1
+## SHT_HASH is linked to dynamic symbol table by default if it exists.
+  - Name: .dynsym
+    Type: SHT_DYNSYM
+
+## Check we can use only "Size" to create a SHT_HASH section.
+
+# RUN: yaml2obj --docnum=9 %s -o %t9
+# RUN: llvm-readobj --sections --section-data %t9 | FileCheck %s --check-prefix=SIZE
+
+# SIZE:      Name: .hash
+# SIZE:      Size:
+# SIZE-SAME: 17
+# SIZE:      SectionData (
+# SIZE-NEXT:  0000: 00000000 00000000 00000000 00000000  |
+# SIZE-NEXT:  0010: 00                                   |
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .hash
+    Type: SHT_HASH
+    Size: 0x11
+
+## Check we can use "Size" and "Content" together to create a SHT_HASH section.
+
+# RUN: yaml2obj --docnum=10 %s -o %t10
+# RUN: llvm-readobj --sections --section-data %t10 | FileCheck %s --check-prefix=SIZE-CONTENT
+
+# SIZE-CONTENT:      Name: .hash
+# SIZE-CONTENT:      Size:
+# SIZE-CONTENT-SAME: 5
+# SIZE-CONTENT:      SectionData (
+# SIZE-CONTENT-NEXT:  0000: 11223300 00 |
+# SIZE-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .hash
+    Type: SHT_HASH
+    Size: 0x5
+    Content: "112233"
+
+## Check that when "Size" and "Content" are used together, the size
+## must be greater than or equal to the content size.
+
+# RUN: not yaml2obj --docnum=11 %s 2>&1 | FileCheck %s --check-prefix=SIZE-CONTENT-ERR
+
+# SIZE-CONTENT-ERR: error: "Size" must be greater than or equal to the content size
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .hash
+    Type: SHT_HASH
+    Size: 0x1
+    Content: "1122"
+
+## Check we can't use "Size" and "Bucket" tags together.
+
+# RUN: not yaml2obj --docnum=12 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-BUCKET
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .hash
+    Type: SHT_HASH
+    Size: 0x1
+    Bucket: [ 1 ]
+
+## Check we can't use "Size" and "Chain" tags together.
+
+# RUN: not yaml2obj --docnum=13 %s 2>&1 | FileCheck %s --check-prefix=CONTENT-CHAIN
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .hash
+    Type: SHT_HASH
+    Size: 0x1
+    Chain: [ 1 ]
diff --git a/llvm/test/tools/yaml2obj/elf-stack-sizes.yaml b/llvm/test/tools/yaml2obj/elf-stack-sizes.yaml
new file mode 100644
index 0000000000000..e05a3def6a43c
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/elf-stack-sizes.yaml
@@ -0,0 +1,344 @@
+## Check how yaml2obj produces .stack_sizes sections.
+
+## Test the following cases when the .stack_sizes Content field is specified:
+## 1) We can produce a .stack_sizes section from a description with
+##    a valid section content.
+## 2) We can produce an incorrect .stack_sizes section from a description with
+##    a broken (truncated) section content.
+## 3) We can produce an empty .stack_sizes section from a description with
+##    empty section content.
+
+# RUN: yaml2obj --docnum=1 %s -o %t1
+# RUN: llvm-readobj --sections --section-data %t1 | FileCheck %s
+
+## Case 1: valid content.
+# CHECK:      Section {
+# CHECK:        Index: 1
+# CHECK-NEXT:   Name: .stack_sizes (1)
+# CHECK-NEXT:   Type: SHT_PROGBITS (0x1)
+# CHECK-NEXT:   Flags [ (0x0)
+# CHECK-NEXT:   ]
+# CHECK-NEXT:   Address: 0x0
+# CHECK-NEXT:   Offset: 0x40
+# CHECK-NEXT:   Size: 9
+# CHECK-NEXT:   Link: 0
+# CHECK-NEXT:   Info: 0
+# CHECK-NEXT:   AddressAlignment: 0
+# CHECK-NEXT:   EntrySize: 0
+# CHECK-NEXT:   SectionData (
+# CHECK-NEXT:     0000: 10000000 00000000 20
+# CHECK-NEXT:   )
+# CHECK-NEXT: }
+
+## Case 2: truncated content.
+# CHECK:      Name: .stack_sizes
+# CHECK:      Size:
+# CHECK-SAME: 8
+# CHECK:      SectionData (
+# CHECK-NEXT:  0000: 10000000 00000000
+
+## Case 3: empty content.
+# CHECK:      Name: .stack_sizes
+# CHECK:      Size:
+# CHECK-SAME: 0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+## Valid.
+  - Name:    '.stack_sizes [1]'
+    Type:    SHT_PROGBITS
+    Content: "100000000000000020"
+## Truncated.
+  - Name:    '.stack_sizes [2]'
+    Type:    SHT_PROGBITS
+    Content: "1000000000000000"
+## Empty.
+  - Name:    '.stack_sizes [3]'
+    Type:    SHT_PROGBITS
+    Content: ""
+
+## Check we can describe .stack_sizes section using <address, size> pairs.
+
+# RUN: yaml2obj --docnum=2 %s -o %t2
+# RUN: llvm-readobj --sections --section-data %t2 | FileCheck %s --check-prefix=ENTRIES-LE64-BOTH
+# RUN: yaml2obj --docnum=3 %s -o %t3
+# RUN: llvm-readobj --sections --section-data %t3 | FileCheck %s --check-prefix=ENTRIES-BE64-BOTH
+# RUN: yaml2obj --docnum=4 %s -o %t4
+# RUN: llvm-readobj --sections --section-data %t4 | FileCheck %s --check-prefix=ENTRIES-LE32-BOTH
+# RUN: yaml2obj --docnum=5 %s -o %t5
+# RUN: llvm-readobj --sections --section-data %t5 | FileCheck %s --check-prefix=ENTRIES-BE32-BOTH
+
+# ENTRIES-LE64-BOTH:      Name: .stack_sizes
+# ENTRIES-LE64-BOTH:      SectionData (
+# ENTRIES-LE64-BOTH-NEXT:  0000: 10000000 00000000 20300000 00000000 |
+# ENTRIES-LE64-BOTH-NEXT:  0010: 0040                                |
+
+# ENTRIES-BE64-BOTH:      Name: .stack_sizes
+# ENTRIES-BE64-BOTH:      SectionData (
+# ENTRIES-BE64-BOTH-NEXT:  0000: 00000000 00000010 20000000 00000000 |
+# ENTRIES-BE64-BOTH-NEXT:  0010: 3040
+
+# ENTRIES-LE32-BOTH:      Name: .stack_sizes
+# ENTRIES-LE32-BOTH:      SectionData (
+# ENTRIES-LE32-BOTH-NEXT:  0000: 10000000 20300000 0040 |
+
+# ENTRIES-BE32-BOTH:      Name: .stack_sizes
+# ENTRIES-BE32-BOTH:      SectionData (
+# ENTRIES-BE32-BOTH-NEXT:  0000: 00000010 20000000 3040 |
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+      - Address: 0x30
+        Size:    0x40
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+      - Address: 0x30
+        Size:    0x40
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_386
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+      - Address: 0x30
+        Size:    0x40
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS32
+  Data:    ELFDATA2MSB
+  Type:    ET_EXEC
+  Machine: EM_386
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+      - Address: 0x30
+        Size:    0x40
+
+## Check we can omit the "Address" tag. In this case the address will be zero.
+
+# RUN: yaml2obj --docnum=6 %s -o %t6
+# RUN: llvm-readobj --sections --section-data %t6 | FileCheck %s --check-prefix=ENTRIES-NOADDR
+
+# ENTRIES-NOADDR:      Name: .stack_sizes
+# ENTRIES-NOADDR:      SectionData (
+# ENTRIES-NOADDR-NEXT:  0000: 00000000 00000000 10000000 00000000 |
+# ENTRIES-NOADDR-NEXT:  0010: 0020                                |
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+       - Size: 0x10
+       - Size: 0x20
+
+## Check that "Size" tag is mandatory when we describe .stack_sizes using "Entries".
+
+# RUN: not yaml2obj --docnum=7 %s 2>&1 | FileCheck %s --check-prefix=ENTRIES-NOSIZE
+
+# ENTRIES-NOSIZE: error: missing required key 'Size'
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Entries:
+       - Address: 0x10
+
+## Check we can't use both "Content" and "Entries" tags at the same time.
+
+# RUN: not yaml2obj --docnum=8 %s 2>&1 | FileCheck %s --check-prefix=ENTRIES-AND-CONTENT
+
+# ENTRIES-AND-CONTENT: error: .stack_sizes: Content and Entries cannot be used together
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    .stack_sizes
+    Type:    SHT_PROGBITS
+    Content: "00"
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+
+## Check we must specify either "Content", "Entries" or "Size" tag when describing .stack_sizes.
+
+# RUN: not yaml2obj --docnum=9 %s 2>&1 | FileCheck %s --check-prefix=NO-TAGS
+
+# NO-TAGS: .stack_sizes: one of Content, Entries and Size must be specified
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+
+## Check we can't use both "Size" and "Entries" tags at the same time.
+
+# RUN: not yaml2obj --docnum=10 %s 2>&1 | FileCheck %s --check-prefix=ENTRIES-AND-SIZE
+
+# ENTRIES-AND-SIZE: .stack_sizes: Size and Entries cannot be used together
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+    Size: 0x1
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+
+## Check we can use only "Size" to create .stack_sizes section.
+
+# RUN: yaml2obj --docnum=11 %s -o %t11
+# RUN: llvm-readobj --sections --section-data %t11 | FileCheck %s --check-prefix=SIZE
+
+# SIZE:      Name: .stack_sizes
+# SIZE:      Size:
+# SIZE-SAME: 17
+# SIZE:      SectionData (
+# SIZE-NEXT:  0000: 00000000 00000000 00000000 00000000  |
+# SIZE-NEXT:  0010: 00                                   |
+# SIZE-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+    Size: 0x11
+
+## Check we can use "Size" and "Content" together to create .stack_sizes section.
+
+# RUN: yaml2obj --docnum=12 %s -o %t12
+# RUN: llvm-readobj --sections --section-data %t12 | FileCheck %s --check-prefix=SIZE-CONTENT
+
+# SIZE-CONTENT:      Name: .stack_sizes
+# SIZE-CONTENT:      Size:
+# SIZE-CONTENT-SAME: 5
+# SIZE-CONTENT:      SectionData (
+# SIZE-CONTENT-NEXT:  0000: 11223300 00 |
+# SIZE-CONTENT-NEXT: )
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+    Size: 0x5
+    Content: "112233"
+
+# RUN: not yaml2obj --docnum=13 %s 2>&1 | FileCheck %s --check-prefix=SIZE-CONTENT-ERR
+
+# SIZE-CONTENT-ERR: error: .stack_sizes: Size must be greater than or equal to the content size
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name: .stack_sizes
+    Type: SHT_PROGBITS
+    Size: 0x1
+    Content: "1122"
+
+## Check we can describe multiple .stack_sizes sections using unique suffixes.
+
+# RUN: yaml2obj --docnum=14 %s -o %t11
+# RUN: llvm-readobj --sections --section-data %t11 | FileCheck %s --check-prefix=UNIQUE
+
+# UNIQUE:      Name: .stack_sizes
+# UNIQUE:      SectionData (
+# UNIQUE-NEXT:   0000: 10000000 00000000 20 |
+
+# UNIQUE:      Name: .stack_sizes
+# UNIQUE:      SectionData (
+# UNIQUE-NEXT:   0000: 30000000 00000000 40 |
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2LSB
+  Type:    ET_EXEC
+  Machine: EM_X86_64
+Sections:
+  - Name:    '.stack_sizes [1]'
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x10
+        Size:    0x20
+  - Name:    '.stack_sizes [2]'
+    Type:    SHT_PROGBITS
+    Entries:
+      - Address: 0x30
+        Size:    0x40
diff --git a/llvm/test/tools/yaml2obj/relocation-unsupported-machine.yaml b/llvm/test/tools/yaml2obj/relocation-unsupported-machine.yaml
new file mode 100644
index 0000000000000..3943e64d14ef8
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/relocation-unsupported-machine.yaml
@@ -0,0 +1,24 @@
+## Show that yaml2obj is able to write relocations for an unknown e_machine properly.
+
+# RUN: yaml2obj %s -o %t1
+# RUN: llvm-readelf %t1 --relocations | FileCheck %s
+
+# CHECK: Relocation section '.rela.text' at offset 0x40 contains 2 entries:
+# CHECK:      Offset            Info             Type    Symbol's Value  Symbol's Name + Addend
+# CHECK-NEXT: 0000000000000001  0000000000000002 Unknown                                 0
+# CHECK-NEXT: 0000000000000003  0000000000000004 Unknown                                 0
+
+--- !ELF
+FileHeader:
+  Class:   ELFCLASS64
+  Data:    ELFDATA2MSB
+  Type:    ET_REL
+  Machine: 0x1234
+Sections:
+  - Name: .rela.text
+    Type: SHT_RELA
+    Relocations:
+      - Offset: 0x1
+        Type:   0x2
+      - Offset: 0x3
+        Type:   0x4
diff --git a/llvm/tools/bugpoint/BugDriver.h b/llvm/tools/bugpoint/BugDriver.h
index 75f166b21b2c0..fe5201eb2e6c3 100644
--- a/llvm/tools/bugpoint/BugDriver.h
+++ b/llvm/tools/bugpoint/BugDriver.h
@@ -217,8 +217,7 @@ class BugDriver {
   /// returning the transformed module on success, or a null pointer on failure.
   std::unique_ptr<Module> runPassesOn(Module *M,
                                       const std::vector<std::string> &Passes,
-                                      unsigned NumExtraArgs = 0,
-                                      const char *const *ExtraArgs = nullptr);
+                                      ArrayRef<std::string> ExtraArgs = {});
 
   /// runPasses - Run the specified passes on Program, outputting a bitcode
   /// file and writting the filename into OutputFile if successful.  If the
@@ -231,8 +230,8 @@ class BugDriver {
   ///
   bool runPasses(Module &Program, const std::vector<std::string> &PassesToRun,
                  std::string &OutputFilename, bool DeleteOutput = false,
-                 bool Quiet = false, unsigned NumExtraArgs = 0,
-                 const char *const *ExtraArgs = nullptr) const;
+                 bool Quiet = false,
+                 ArrayRef<std::string> ExtraArgs = {}) const;
 
   /// runPasses - Just like the method above, but this just returns true or
   /// false indicating whether or not the optimizer crashed on the specified
diff --git a/llvm/tools/bugpoint/ExtractFunction.cpp b/llvm/tools/bugpoint/ExtractFunction.cpp
index 105702de3f1d8..d9047acd30e11 100644
--- a/llvm/tools/bugpoint/ExtractFunction.cpp
+++ b/llvm/tools/bugpoint/ExtractFunction.cpp
@@ -407,11 +407,10 @@ BugDriver::extractMappedBlocksFromModule(const std::vector<BasicBlock *> &BBs,
 
   std::string uniqueFN = "--extract-blocks-file=";
   uniqueFN += Temp->TmpName;
-  const char *ExtraArg = uniqueFN.c_str();
 
   std::vector<std::string> PI;
   PI.push_back("extract-blocks");
-  std::unique_ptr<Module> Ret = runPassesOn(M, PI, 1, &ExtraArg);
+  std::unique_ptr<Module> Ret = runPassesOn(M, PI, {uniqueFN});
 
   if (!Ret) {
     outs() << "*** Basic Block extraction failed, please report a bug!\n";
diff --git a/llvm/tools/bugpoint/OptimizerDriver.cpp b/llvm/tools/bugpoint/OptimizerDriver.cpp
index e093389bdfefa..64af81fcc8a1a 100644
--- a/llvm/tools/bugpoint/OptimizerDriver.cpp
+++ b/llvm/tools/bugpoint/OptimizerDriver.cpp
@@ -130,8 +130,7 @@ static cl::list<std::string> OptArgs("opt-args", cl::Positional,
 bool BugDriver::runPasses(Module &Program,
                           const std::vector<std::string> &Passes,
                           std::string &OutputFilename, bool DeleteOutput,
-                          bool Quiet, unsigned NumExtraArgs,
-                          const char *const *ExtraArgs) const {
+                          bool Quiet, ArrayRef<std::string> ExtraArgs) const {
   // setup the output file name
   outs().flush();
   SmallString<128> UniqueFilename;
@@ -223,8 +222,7 @@ bool BugDriver::runPasses(Module &Program,
        I != E; ++I)
     Args.push_back(I->c_str());
   Args.push_back(Temp->TmpName.c_str());
-  for (unsigned i = 0; i < NumExtraArgs; ++i)
-    Args.push_back(*ExtraArgs);
+  Args.append(ExtraArgs.begin(), ExtraArgs.end());
 
   LLVM_DEBUG(errs() << "\nAbout to run:\t";
              for (unsigned i = 0, e = Args.size() - 1; i != e; ++i) errs()
@@ -268,10 +266,10 @@ bool BugDriver::runPasses(Module &Program,
 
 std::unique_ptr<Module>
 BugDriver::runPassesOn(Module *M, const std::vector<std::string> &Passes,
-                       unsigned NumExtraArgs, const char *const *ExtraArgs) {
+                       ArrayRef<std::string> ExtraArgs) {
   std::string BitcodeResult;
   if (runPasses(*M, Passes, BitcodeResult, false /*delete*/, true /*quiet*/,
-                NumExtraArgs, ExtraArgs)) {
+                ExtraArgs)) {
     return nullptr;
   }
 
diff --git a/llvm/tools/bugpoint/ToolRunner.cpp b/llvm/tools/bugpoint/ToolRunner.cpp
index da4244345e3bc..19b2ea2c0181e 100644
--- a/llvm/tools/bugpoint/ToolRunner.cpp
+++ b/llvm/tools/bugpoint/ToolRunner.cpp
@@ -170,7 +170,7 @@ Expected<int> LLI::ExecuteProgram(const std::string &Bitcode,
                                   const std::vector<std::string> &SharedLibs,
                                   unsigned Timeout, unsigned MemoryLimit) {
   std::vector<StringRef> LLIArgs;
-  LLIArgs.push_back(LLIPath.c_str());
+  LLIArgs.push_back(LLIPath);
   LLIArgs.push_back("-force-interpreter=true");
 
   for (std::vector<std::string>::const_iterator i = SharedLibs.begin(),
@@ -266,15 +266,15 @@ Error CustomCompiler::compileProgram(const std::string &Bitcode,
                                      unsigned Timeout, unsigned MemoryLimit) {
 
   std::vector<StringRef> ProgramArgs;
-  ProgramArgs.push_back(CompilerCommand.c_str());
+  ProgramArgs.push_back(CompilerCommand);
 
-  for (std::size_t i = 0; i < CompilerArgs.size(); ++i)
-    ProgramArgs.push_back(CompilerArgs.at(i).c_str());
+  for (const auto &Arg : CompilerArgs)
+    ProgramArgs.push_back(Arg);
   ProgramArgs.push_back(Bitcode);
 
   // Add optional parameters to the running program from Argv
-  for (unsigned i = 0, e = CompilerArgs.size(); i != e; ++i)
-    ProgramArgs.push_back(CompilerArgs[i].c_str());
+  for (const auto &Arg : CompilerArgs)
+    ProgramArgs.push_back(Arg);
 
   if (RunProgramWithTimeout(CompilerCommand, ProgramArgs, "", "", "", Timeout,
                             MemoryLimit))
@@ -559,7 +559,7 @@ Expected<int> JIT::ExecuteProgram(const std::string &Bitcode,
                                   unsigned Timeout, unsigned MemoryLimit) {
   // Construct a vector of parameters, incorporating those from the command-line
   std::vector<StringRef> JITArgs;
-  JITArgs.push_back(LLIPath.c_str());
+  JITArgs.push_back(LLIPath);
   JITArgs.push_back("-force-interpreter=false");
 
   // Add any extra LLI args.
@@ -570,7 +570,7 @@ Expected<int> JIT::ExecuteProgram(const std::string &Bitcode,
     JITArgs.push_back("-load");
     JITArgs.push_back(SharedLibs[i]);
   }
-  JITArgs.push_back(Bitcode.c_str());
+  JITArgs.push_back(Bitcode);
   // Add optional parameters to the running program from Argv
   for (unsigned i = 0, e = Args.size(); i != e; ++i)
     JITArgs.push_back(Args[i]);
diff --git a/llvm/tools/bugpoint/bugpoint.cpp b/llvm/tools/bugpoint/bugpoint.cpp
index 2d5322a351ad1..c7644e75ae4b0 100644
--- a/llvm/tools/bugpoint/bugpoint.cpp
+++ b/llvm/tools/bugpoint/bugpoint.cpp
@@ -80,6 +80,10 @@ static cl::opt<bool> OptLevelOs(
     cl::desc(
         "Like -O2 with extra optimizations for size. Similar to clang -Os"));
 
+static cl::opt<bool>
+OptLevelOz("Oz",
+           cl::desc("Like -Os but reduces code size further. Similar to clang -Oz"));
+
 static cl::opt<bool>
     OptLevelO3("O3", cl::desc("Optimization level 3. Identical to 'opt -O3'"));
 
@@ -109,6 +113,26 @@ class AddToDriver : public legacy::FunctionPassManager {
 };
 }
 
+// This routine adds optimization passes based on selected optimization level,
+// OptLevel.
+//
+// OptLevel - Optimization Level
+static void AddOptimizationPasses(legacy::FunctionPassManager &FPM,
+                                  unsigned OptLevel,
+                                  unsigned SizeLevel) {
+  PassManagerBuilder Builder;
+  Builder.OptLevel = OptLevel;
+  Builder.SizeLevel = SizeLevel;
+
+  if (OptLevel > 1)
+    Builder.Inliner = createFunctionInliningPass(OptLevel, SizeLevel, false);
+  else
+    Builder.Inliner = createAlwaysInlinerLegacyPass();
+
+  Builder.populateFunctionPassManager(FPM);
+  Builder.populateModulePassManager(FPM);
+}
+
 #ifdef LINK_POLLY_INTO_TOOLS
 namespace polly {
 void initializePollyPasses(llvm::PassRegistry &Registry);
@@ -189,18 +213,16 @@ int main(int argc, char **argv) {
     Builder.populateLTOPassManager(PM);
   }
 
-  if (OptLevelO1 || OptLevelO2 || OptLevelO3) {
-    PassManagerBuilder Builder;
-    if (OptLevelO1)
-      Builder.Inliner = createAlwaysInlinerLegacyPass();
-    else if (OptLevelOs || OptLevelO2)
-      Builder.Inliner = createFunctionInliningPass(
-          2, OptLevelOs ? 1 : 0, false);
-    else
-      Builder.Inliner = createFunctionInliningPass(275);
-    Builder.populateFunctionPassManager(PM);
-    Builder.populateModulePassManager(PM);
-  }
+  if (OptLevelO1)
+    AddOptimizationPasses(PM, 1, 0);
+  else if (OptLevelO2)
+    AddOptimizationPasses(PM, 2, 0);
+  else if (OptLevelO3)
+    AddOptimizationPasses(PM, 3, 0);
+  else if (OptLevelOs)
+    AddOptimizationPasses(PM, 2, 1);
+  else if (OptLevelOz)
+    AddOptimizationPasses(PM, 2, 2);
 
   for (const PassInfo *PI : PassList)
     D.addPass(PI->getPassArgument());
diff --git a/llvm/tools/dsymutil/DwarfStreamer.cpp b/llvm/tools/dsymutil/DwarfStreamer.cpp
index 5d0f7c1067cd8..732260f0346f8 100644
--- a/llvm/tools/dsymutil/DwarfStreamer.cpp
+++ b/llvm/tools/dsymutil/DwarfStreamer.cpp
@@ -260,7 +260,7 @@ void DwarfStreamer::emitAppleTypes(
 /// Emit the swift_ast section stored in \p Buffers.
 void DwarfStreamer::emitSwiftAST(StringRef Buffer) {
   MCSection *SwiftASTSection = MOFI->getDwarfSwiftASTSection();
-  SwiftASTSection->setAlignment(llvm::Align(32));
+  SwiftASTSection->setAlignment(Align(32));
   MS->SwitchSection(SwiftASTSection);
   MS->EmitBytes(Buffer);
 }
@@ -339,7 +339,7 @@ void DwarfStreamer::emitUnitRangesEntries(CompileUnit &Unit,
         sizeof(int8_t);   // Segment Size (in bytes)
 
     unsigned TupleSize = AddressSize * 2;
-    unsigned Padding = offsetToAlignment(HeaderSize, llvm::Align(TupleSize));
+    unsigned Padding = offsetToAlignment(HeaderSize, Align(TupleSize));
 
     Asm->EmitLabelDifference(EndLabel, BeginLabel, 4); // Arange length
     Asm->OutStreamer->EmitLabel(BeginLabel);
diff --git a/llvm/tools/llc/llc.cpp b/llvm/tools/llc/llc.cpp
index 74ad643496b8d..574b15b399c37 100644
--- a/llvm/tools/llc/llc.cpp
+++ b/llvm/tools/llc/llc.cpp
@@ -538,8 +538,9 @@ static int compileModule(char **argv, LLVMContext &Context) {
     }
 
     const char *argv0 = argv[0];
-    LLVMTargetMachine &LLVMTM = static_cast<LLVMTargetMachine&>(*Target);
-    MachineModuleInfo *MMI = new MachineModuleInfo(&LLVMTM);
+    LLVMTargetMachine &LLVMTM = static_cast<LLVMTargetMachine &>(*Target);
+    MachineModuleInfoWrapperPass *MMIWP =
+        new MachineModuleInfoWrapperPass(&LLVMTM);
 
     // Construct a custom pass pipeline that starts after instruction
     // selection.
@@ -559,7 +560,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
 
       TPC.setDisableVerify(NoVerify);
       PM.add(&TPC);
-      PM.add(MMI);
+      PM.add(MMIWP);
       TPC.printAndVerify("");
       for (const std::string &RunPassName : *RunPassNames) {
         if (addPass(PM, argv0, RunPassName, TPC))
@@ -570,7 +571,7 @@ static int compileModule(char **argv, LLVMContext &Context) {
       PM.add(createFreeMachineFunctionPass());
     } else if (Target->addPassesToEmitFile(PM, *OS,
                                            DwoOut ? &DwoOut->os() : nullptr,
-                                           FileType, NoVerify, MMI)) {
+                                           FileType, NoVerify, MMIWP)) {
       WithColor::warning(errs(), argv[0])
           << "target does not support generation of this"
           << " file type!\n";
@@ -578,8 +579,8 @@ static int compileModule(char **argv, LLVMContext &Context) {
     }
 
     if (MIR) {
-      assert(MMI && "Forgot to create MMI?");
-      if (MIR->parseMachineFunctions(*M, *MMI))
+      assert(MMIWP && "Forgot to create MMIWP?");
+      if (MIR->parseMachineFunctions(*M, MMIWP->getMMI()))
         return 1;
     }
 
diff --git a/llvm/tools/llvm-ar/llvm-ar.cpp b/llvm/tools/llvm-ar/llvm-ar.cpp
index c83a6798bcc1c..4eb50a124907c 100644
--- a/llvm/tools/llvm-ar/llvm-ar.cpp
+++ b/llvm/tools/llvm-ar/llvm-ar.cpp
@@ -70,14 +70,14 @@ USAGE: llvm-ar [options] [-]<operation>[modifiers] [relpos] [count] <archive> [f
        llvm-ar -M [<mri-script]
 
 OPTIONS:
-  --format              - Archive format to create
+  --format              - archive format to create
     =default            -   default
     =gnu                -   gnu
     =darwin             -   darwin
     =bsd                -   bsd
-  --plugin=<string>     - Ignored for compatibility
-  --help                - Display available options
-  --version             - Display the version of this program
+  --plugin=<string>     - ignored for compatibility
+  -h --help             - display this help and exit
+  --version             - print the version and exit
   @<file>               - read options from <file>
 
 OPERATIONS:
@@ -116,10 +116,19 @@ void printHelpMessage() {
     outs() << ArHelp;
 }
 
+static unsigned MRILineNumber;
+static bool ParsingMRIScript;
+
 // Show the error message and exit.
 LLVM_ATTRIBUTE_NORETURN static void fail(Twine Error) {
-  WithColor::error(errs(), ToolName) << Error << ".\n";
-  printHelpMessage();
+  if (ParsingMRIScript) {
+    WithColor::error(errs(), ToolName)
+        << "script line " << MRILineNumber << ": " << Error << "\n";
+  } else {
+    WithColor::error(errs(), ToolName) << Error << "\n";
+    printHelpMessage();
+  }
+
   exit(1);
 }
 
@@ -209,7 +218,7 @@ static BumpPtrAllocator Alloc;
 // associated with a, b, and i modifiers
 static void getRelPos() {
   if (PositionalArgs.empty())
-    fail("Expected [relpos] for a, b, or i modifier");
+    fail("expected [relpos] for 'a', 'b', or 'i' modifier");
   RelPos = PositionalArgs[0];
   PositionalArgs.erase(PositionalArgs.begin());
 }
@@ -218,19 +227,19 @@ static void getRelPos() {
 // associated with the N modifier
 static void getCountParam() {
   if (PositionalArgs.empty())
-    fail("Expected [count] for N modifier");
+    fail("expected [count] for 'N' modifier");
   auto CountParamArg = StringRef(PositionalArgs[0]);
   if (CountParamArg.getAsInteger(10, CountParam))
-    fail("Value for [count] must be numeric, got: " + CountParamArg);
+    fail("value for [count] must be numeric, got: " + CountParamArg);
   if (CountParam < 1)
-    fail("Value for [count] must be positive, got: " + CountParamArg);
+    fail("value for [count] must be positive, got: " + CountParamArg);
   PositionalArgs.erase(PositionalArgs.begin());
 }
 
 // Get the archive file name from the command line
 static void getArchive() {
   if (PositionalArgs.empty())
-    fail("An archive name must be specified");
+    fail("an archive name must be specified");
   ArchiveName = PositionalArgs[0];
   PositionalArgs.erase(PositionalArgs.begin());
 }
@@ -246,12 +255,12 @@ std::vector<std::unique_ptr<object::Archive>> Archives;
 
 static object::Archive &readLibrary(const Twine &Library) {
   auto BufOrErr = MemoryBuffer::getFile(Library, -1, false);
-  failIfError(BufOrErr.getError(), "Could not open library " + Library);
+  failIfError(BufOrErr.getError(), "could not open library " + Library);
   ArchiveBuffers.push_back(std::move(*BufOrErr));
   auto LibOrErr =
       object::Archive::create(ArchiveBuffers.back()->getMemBufferRef());
   failIfError(errorToErrorCode(LibOrErr.takeError()),
-              "Could not parse library");
+              "could not parse library");
   Archives.push_back(std::move(*LibOrErr));
   return *Archives.back();
 }
@@ -264,7 +273,7 @@ static void runMRIScript();
 static ArchiveOperation parseCommandLine() {
   if (MRI) {
     if (!PositionalArgs.empty() || !Options.empty())
-      fail("Cannot mix -M and other options");
+      fail("cannot mix -M and other options");
     runMRIScript();
   }
 
@@ -383,31 +392,31 @@ static ArchiveOperation parseCommandLine() {
     NumOperations = 1;
     Operation = CreateSymTab;
     if (!Members.empty())
-      fail("The s operation takes only an archive as argument");
+      fail("the 's' operation takes only an archive as argument");
   }
 
   // Perform various checks on the operation/modifier specification
   // to make sure we are dealing with a legal request.
   if (NumOperations == 0)
-    fail("You must specify at least one of the operations");
+    fail("you must specify at least one of the operations");
   if (NumOperations > 1)
-    fail("Only one operation may be specified");
+    fail("only one operation may be specified");
   if (NumPositional > 1)
-    fail("You may only specify one of a, b, and i modifiers");
+    fail("you may only specify one of 'a', 'b', and 'i' modifiers");
   if (AddAfter || AddBefore)
     if (Operation != Move && Operation != ReplaceOrInsert)
-      fail("The 'a', 'b' and 'i' modifiers can only be specified with "
+      fail("the 'a', 'b' and 'i' modifiers can only be specified with "
            "the 'm' or 'r' operations");
   if (CountParam)
     if (Operation != Extract && Operation != Delete)
-      fail("The 'N' modifier can only be specified with the 'x' or 'd' "
+      fail("the 'N' modifier can only be specified with the 'x' or 'd' "
            "operations");
   if (OriginalDates && Operation != Extract)
-    fail("The 'o' modifier is only applicable to the 'x' operation");
+    fail("the 'o' modifier is only applicable to the 'x' operation");
   if (OnlyUpdate && Operation != ReplaceOrInsert)
-    fail("The 'u' modifier is only applicable to the 'r' operation");
+    fail("the 'u' modifier is only applicable to the 'r' operation");
   if (AddLibrary && Operation != QuickAppend)
-    fail("The 'L' modifier is only applicable to the 'q' operation");
+    fail("the 'L' modifier is only applicable to the 'q' operation");
 
   // Return the parsed operation to the caller
   return Operation;
@@ -589,7 +598,7 @@ static void addChildMember(std::vector<NewArchiveMember> &Members,
                            const object::Archive::Child &M,
                            bool FlattenArchive = false) {
   if (Thin && !M.getParent()->isThin())
-    fail("Cannot convert a regular archive to a thin one");
+    fail("cannot convert a regular archive to a thin one");
   Expected<NewArchiveMember> NMOrErr =
       NewArchiveMember::getOldMember(M, Deterministic);
   failIfError(NMOrErr.takeError());
@@ -783,7 +792,7 @@ computeNewArchiveMembers(ArchiveOperation Operation,
     return Ret;
 
   if (!RelPos.empty() && InsertPos == -1)
-    fail("Insertion point not found");
+    fail("insertion point not found");
 
   if (RelPos.empty())
     InsertPos = Ret.size();
@@ -859,12 +868,12 @@ static void performWriteOperation(ArchiveOperation Operation,
     break;
   case BSD:
     if (Thin)
-      fail("Only the gnu format has a thin mode");
+      fail("only the gnu format has a thin mode");
     Kind = object::Archive::K_BSD;
     break;
   case DARWIN:
     if (Thin)
-      fail("Only the gnu format has a thin mode");
+      fail("only the gnu format has a thin mode");
     Kind = object::Archive::K_DARWIN;
     break;
   case Unknown:
@@ -922,14 +931,12 @@ static int performOperation(ArchiveOperation Operation,
       MemoryBuffer::getFile(ArchiveName, -1, false);
   std::error_code EC = Buf.getError();
   if (EC && EC != errc::no_such_file_or_directory)
-    fail("error opening '" + ArchiveName + "': " + EC.message() + "!");
+    fail("error opening '" + ArchiveName + "': " + EC.message());
 
   if (!EC) {
     Error Err = Error::success();
     object::Archive Archive(Buf.get()->getMemBufferRef(), Err);
-    EC = errorToErrorCode(std::move(Err));
-    failIfError(EC,
-                "error loading '" + ArchiveName + "': " + EC.message() + "!");
+    failIfError(std::move(Err), "unable to load '" + ArchiveName + "'");
     if (Archive.isThin())
       CompareFullPath = true;
     performOperation(Operation, &Archive, std::move(Buf.get()), NewMembers);
@@ -960,8 +967,10 @@ static void runMRIScript() {
   const MemoryBuffer &Ref = *Buf.get();
   bool Saved = false;
   std::vector<NewArchiveMember> NewMembers;
+  ParsingMRIScript = true;
 
   for (line_iterator I(Ref, /*SkipBlanks*/ false), E; I != E; ++I) {
+    ++MRILineNumber;
     StringRef Line = *I;
     Line = Line.split(';').first;
     Line = Line.split('*').first;
@@ -1003,9 +1012,9 @@ static void runMRIScript() {
     case MRICommand::Create:
       Create = true;
       if (!ArchiveName.empty())
-        fail("Editing multiple archives not supported");
+        fail("editing multiple archives not supported");
       if (Saved)
-        fail("File already saved");
+        fail("file already saved");
       ArchiveName = Rest;
       break;
     case MRICommand::Delete: {
@@ -1020,10 +1029,12 @@ static void runMRIScript() {
     case MRICommand::End:
       break;
     case MRICommand::Invalid:
-      fail("Unknown command: " + CommandStr);
+      fail("unknown command: " + CommandStr);
     }
   }
-
+  
+  ParsingMRIScript = false;
+  
   // Nothing to do if not saved.
   if (Saved)
     performOperation(ReplaceOrInsert, &NewMembers);
@@ -1031,7 +1042,7 @@ static void runMRIScript() {
 }
 
 static bool handleGenericOption(StringRef arg) {
-  if (arg == "-help" || arg == "--help") {
+  if (arg == "h" || arg.startswith("-h") || arg == "--help") {
     printHelpMessage();
     return true;
   }
@@ -1108,7 +1119,7 @@ static int ranlib_main(int argc, char **argv) {
       return 0;
     } else {
       if (ArchiveSpecified)
-        fail("Exactly one archive should be specified");
+        fail("exactly one archive should be specified");
       ArchiveSpecified = true;
       ArchiveName = argv[i];
     }
@@ -1136,5 +1147,5 @@ int main(int argc, char **argv) {
 
   if (Stem.contains_lower("ar"))
     return ar_main(argc, argv);
-  fail("Not ranlib, ar, lib or dlltool!");
+  fail("not ranlib, ar, lib or dlltool");
 }
diff --git a/llvm/tools/llvm-c-test/echo.cpp b/llvm/tools/llvm-c-test/echo.cpp
index 8341b11aa45fc..4d10a4bbfc058 100644
--- a/llvm/tools/llvm-c-test/echo.cpp
+++ b/llvm/tools/llvm-c-test/echo.cpp
@@ -576,6 +576,8 @@ struct FunCloner {
         LLVMValueRef Ptr = CloneValue(LLVMGetOperand(Src, 0));
         Dst = LLVMBuildLoad(Builder, Ptr, Name);
         LLVMSetAlignment(Dst, LLVMGetAlignment(Src));
+        LLVMSetOrdering(Dst, LLVMGetOrdering(Src));
+        LLVMSetVolatile(Dst, LLVMGetVolatile(Src));
         break;
       }
       case LLVMStore: {
@@ -583,6 +585,8 @@ struct FunCloner {
         LLVMValueRef Ptr = CloneValue(LLVMGetOperand(Src, 1));
         Dst = LLVMBuildStore(Builder, Val, Ptr);
         LLVMSetAlignment(Dst, LLVMGetAlignment(Src));
+        LLVMSetOrdering(Dst, LLVMGetOrdering(Src));
+        LLVMSetVolatile(Dst, LLVMGetVolatile(Src));
         break;
       }
       case LLVMGetElementPtr: {
@@ -597,6 +601,17 @@ struct FunCloner {
           Dst = LLVMBuildGEP(Builder, Ptr, Idx.data(), NumIdx, Name);
         break;
       }
+      case LLVMAtomicRMW: {
+        LLVMValueRef Ptr = CloneValue(LLVMGetOperand(Src, 0));
+        LLVMValueRef Val = CloneValue(LLVMGetOperand(Src, 1));
+        LLVMAtomicRMWBinOp BinOp = LLVMGetAtomicRMWBinOp(Src);
+        LLVMAtomicOrdering Ord = LLVMGetOrdering(Src);
+        LLVMBool SingleThread = LLVMIsAtomicSingleThread(Src);
+        Dst = LLVMBuildAtomicRMW(Builder, BinOp, Ptr, Val, Ord, SingleThread);
+        LLVMSetVolatile(Dst, LLVMGetVolatile(Src));
+        LLVMSetValueName2(Dst, Name, NameLen);
+        break;
+      }
       case LLVMAtomicCmpXchg: {
         LLVMValueRef Ptr = CloneValue(LLVMGetOperand(Src, 0));
         LLVMValueRef Cmp = CloneValue(LLVMGetOperand(Src, 1));
@@ -607,7 +622,11 @@ struct FunCloner {
 
         Dst = LLVMBuildAtomicCmpXchg(Builder, Ptr, Cmp, New, Succ, Fail,
                                      SingleThread);
-      } break;
+        LLVMSetVolatile(Dst, LLVMGetVolatile(Src));
+        LLVMSetWeak(Dst, LLVMGetWeak(Src));
+        LLVMSetValueName2(Dst, Name, NameLen);
+        break;
+      }
       case LLVMBitCast: {
         LLVMValueRef V = CloneValue(LLVMGetOperand(Src, 0));
         Dst = LLVMBuildBitCast(Builder, V, CloneType(Src), Name);
diff --git a/llvm/tools/llvm-cov/TestingSupport.cpp b/llvm/tools/llvm-cov/TestingSupport.cpp
index 39f809a8f0d67..b99bd83157d03 100644
--- a/llvm/tools/llvm-cov/TestingSupport.cpp
+++ b/llvm/tools/llvm-cov/TestingSupport.cpp
@@ -100,7 +100,7 @@ int convertForTestingMain(int argc, const char *argv[]) {
   encodeULEB128(ProfileNamesAddress, OS);
   OS << ProfileNamesData;
   // Coverage mapping data is expected to have an alignment of 8.
-  for (unsigned Pad = offsetToAlignment(OS.tell(), llvm::Align(8)); Pad; --Pad)
+  for (unsigned Pad = offsetToAlignment(OS.tell(), Align(8)); Pad; --Pad)
     OS.write(uint8_t(0));
   OS << CoverageMappingData;
 
diff --git a/llvm/tools/llvm-dwarfdump/Statistics.cpp b/llvm/tools/llvm-dwarfdump/Statistics.cpp
index 268ab5f79b679..c29ad783a9e61 100644
--- a/llvm/tools/llvm-dwarfdump/Statistics.cpp
+++ b/llvm/tools/llvm-dwarfdump/Statistics.cpp
@@ -5,6 +5,7 @@
 #include "llvm/DebugInfo/DWARF/DWARFContext.h"
 #include "llvm/DebugInfo/DWARF/DWARFDebugLoc.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/JSON.h"
 
 #define DEBUG_TYPE "dwarfdump"
 using namespace llvm;
@@ -92,11 +93,11 @@ struct GlobalStats {
   unsigned CallSiteParamDIEs = 0;
   /// Total byte size of concrete functions. This byte size includes
   /// inline functions contained in the concrete functions.
-  uint64_t FunctionSize = 0;
+  unsigned FunctionSize = 0;
   /// Total byte size of inlined functions. This is the total number of bytes
   /// for the top inline functions within concrete functions. This can help
   /// tune the inline settings when compiling to match user expectations.
-  uint64_t InlineFunctionSize = 0;
+  unsigned InlineFunctionSize = 0;
 };
 
 /// Holds accumulated debug location statistics about local variables and
@@ -449,11 +450,7 @@ static void collectStatsRecursive(DWARFDie Die, uint64_t UnitLowPC, std::string
 /// Print machine-readable output.
 /// The machine-readable format is single-line JSON output.
 /// \{
-static void printDatum(raw_ostream &OS, const char *Key, StringRef Value) {
-  OS << ",\"" << Key << "\":\"" << Value << '"';
-  LLVM_DEBUG(llvm::dbgs() << Key << ": " << Value << '\n');
-}
-static void printDatum(raw_ostream &OS, const char *Key, uint64_t Value) {
+static void printDatum(raw_ostream &OS, const char *Key, json::Value Value) {
   OS << ",\"" << Key << "\":" << Value;
   LLVM_DEBUG(llvm::dbgs() << Key << ": " << Value << '\n');
 }
@@ -603,7 +600,7 @@ bool collectStatsForObjectFile(ObjectFile &Obj, DWARFContext &DICtx,
   printDatum(OS, "total vars procesed by location statistics", LocStats.NumVar);
   printLocationStats(OS, "vars", LocStats.VarLocStats);
   printLocationStats(OS, "vars (excluding the debug entry values)",
-                     LocStats.ParamNonEntryValLocStats);
+                     LocStats.VarNonEntryValLocStats);
   OS << "}\n";
   LLVM_DEBUG(
       llvm::dbgs() << "Total Availability: "
diff --git a/llvm/tools/llvm-exegesis/lib/Assembler.cpp b/llvm/tools/llvm-exegesis/lib/Assembler.cpp
index 9695c79b57ae2..229bf36da6bf9 100644
--- a/llvm/tools/llvm-exegesis/lib/Assembler.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Assembler.cpp
@@ -8,6 +8,7 @@
 
 #include "Assembler.h"
 
+#include "SnippetRepetitor.h"
 #include "Target.h"
 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
@@ -28,21 +29,22 @@ namespace exegesis {
 static constexpr const char ModuleID[] = "ExegesisInfoTest";
 static constexpr const char FunctionID[] = "foo";
 
-static std::vector<llvm::MCInst>
+// Fills the given basic block with register setup code, and returns true if
+// all registers could be setup correctly.
+static bool
 generateSnippetSetupCode(const ExegesisTarget &ET,
                          const llvm::MCSubtargetInfo *const MSI,
                          llvm::ArrayRef<RegisterValue> RegisterInitialValues,
-                         bool &IsSnippetSetupComplete) {
-  IsSnippetSetupComplete = true;
-  std::vector<llvm::MCInst> Result;
+                         BasicBlockFiller &BBF) {
+  bool IsSnippetSetupComplete = true;
   for (const RegisterValue &RV : RegisterInitialValues) {
     // Load a constant in the register.
     const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value);
     if (SetRegisterCode.empty())
       IsSnippetSetupComplete = false;
-    Result.insert(Result.end(), SetRegisterCode.begin(), SetRegisterCode.end());
+    BBF.addInstructions(SetRegisterCode);
   }
-  return Result;
+  return IsSnippetSetupComplete;
 }
 
 // Small utility function to add named passes.
@@ -67,8 +69,7 @@ static bool addPass(llvm::PassManagerBase &PM, llvm::StringRef PassName,
   return false;
 }
 
-// Creates a void(int8*) MachineFunction.
-static llvm::MachineFunction &
+llvm::MachineFunction &
 createVoidVoidPtrMachineFunction(llvm::StringRef FunctionID,
                                  llvm::Module *Module,
                                  llvm::MachineModuleInfo *MMI) {
@@ -85,38 +86,43 @@ createVoidVoidPtrMachineFunction(llvm::StringRef FunctionID,
   return MMI->getOrCreateMachineFunction(*F);
 }
 
-static void fillMachineFunction(llvm::MachineFunction &MF,
-                                llvm::ArrayRef<unsigned> LiveIns,
-                                llvm::ArrayRef<llvm::MCInst> Instructions) {
-  llvm::MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
-  MF.push_back(MBB);
-  for (const unsigned Reg : LiveIns)
-    MBB->addLiveIn(Reg);
-  const llvm::MCInstrInfo *MCII = MF.getTarget().getMCInstrInfo();
-  llvm::DebugLoc DL;
-  for (const llvm::MCInst &Inst : Instructions) {
-    const unsigned Opcode = Inst.getOpcode();
-    const llvm::MCInstrDesc &MCID = MCII->get(Opcode);
-    llvm::MachineInstrBuilder Builder = llvm::BuildMI(MBB, DL, MCID);
-    for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
-         ++OpIndex) {
-      const llvm::MCOperand &Op = Inst.getOperand(OpIndex);
-      if (Op.isReg()) {
-        const bool IsDef = OpIndex < MCID.getNumDefs();
-        unsigned Flags = 0;
-        const llvm::MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
-        if (IsDef && !OpInfo.isOptionalDef())
-          Flags |= llvm::RegState::Define;
-        Builder.addReg(Op.getReg(), Flags);
-      } else if (Op.isImm()) {
-        Builder.addImm(Op.getImm());
-      } else if (!Op.isValid()) {
-        llvm_unreachable("Operand is not set");
-      } else {
-        llvm_unreachable("Not yet implemented");
-      }
+BasicBlockFiller::BasicBlockFiller(llvm::MachineFunction &MF,
+                                   llvm::MachineBasicBlock *MBB,
+                                   const llvm::MCInstrInfo *MCII)
+    : MF(MF), MBB(MBB), MCII(MCII) {}
+
+void BasicBlockFiller::addInstruction(const llvm::MCInst &Inst,
+                                      const llvm::DebugLoc &DL) {
+  const unsigned Opcode = Inst.getOpcode();
+  const llvm::MCInstrDesc &MCID = MCII->get(Opcode);
+  llvm::MachineInstrBuilder Builder = llvm::BuildMI(MBB, DL, MCID);
+  for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
+       ++OpIndex) {
+    const llvm::MCOperand &Op = Inst.getOperand(OpIndex);
+    if (Op.isReg()) {
+      const bool IsDef = OpIndex < MCID.getNumDefs();
+      unsigned Flags = 0;
+      const llvm::MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
+      if (IsDef && !OpInfo.isOptionalDef())
+        Flags |= llvm::RegState::Define;
+      Builder.addReg(Op.getReg(), Flags);
+    } else if (Op.isImm()) {
+      Builder.addImm(Op.getImm());
+    } else if (!Op.isValid()) {
+      llvm_unreachable("Operand is not set");
+    } else {
+      llvm_unreachable("Not yet implemented");
     }
   }
+}
+
+void BasicBlockFiller::addInstructions(ArrayRef<llvm::MCInst> Insts,
+                                       const llvm::DebugLoc &DL) {
+  for (const MCInst &Inst : Insts)
+    addInstruction(Inst, DL);
+}
+
+void BasicBlockFiller::addReturn(const llvm::DebugLoc &DL) {
   // Insert the return code.
   const llvm::TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
   if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
@@ -128,6 +134,21 @@ static void fillMachineFunction(llvm::MachineFunction &MF,
   }
 }
 
+FunctionFiller::FunctionFiller(llvm::MachineFunction &MF,
+                               std::vector<unsigned> RegistersSetUp)
+    : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
+      RegistersSetUp(std::move(RegistersSetUp)) {}
+
+BasicBlockFiller FunctionFiller::addBasicBlock() {
+  llvm::MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
+  MF.push_back(MBB);
+  return BasicBlockFiller(MF, MBB, MCII);
+}
+
+ArrayRef<unsigned> FunctionFiller::getRegistersSetUp() const {
+  return RegistersSetUp;
+}
+
 static std::unique_ptr<llvm::Module>
 createModule(const std::unique_ptr<llvm::LLVMContext> &Context,
              const llvm::DataLayout DL) {
@@ -142,11 +163,11 @@ llvm::BitVector getFunctionReservedRegs(const llvm::TargetMachine &TM) {
   std::unique_ptr<llvm::Module> Module =
       createModule(Context, TM.createDataLayout());
   // TODO: This only works for targets implementing LLVMTargetMachine.
-  const LLVMTargetMachine &LLVMTM = static_cast<const LLVMTargetMachine&>(TM);
-  std::unique_ptr<llvm::MachineModuleInfo> MMI =
-      std::make_unique<llvm::MachineModuleInfo>(&LLVMTM);
-  llvm::MachineFunction &MF =
-      createVoidVoidPtrMachineFunction(FunctionID, Module.get(), MMI.get());
+  const LLVMTargetMachine &LLVMTM = static_cast<const LLVMTargetMachine &>(TM);
+  std::unique_ptr<llvm::MachineModuleInfoWrapperPass> MMIWP =
+      std::make_unique<llvm::MachineModuleInfoWrapperPass>(&LLVMTM);
+  llvm::MachineFunction &MF = createVoidVoidPtrMachineFunction(
+      FunctionID, Module.get(), &MMIWP.get()->getMMI());
   // Saving reserved registers for client.
   return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
 }
@@ -155,47 +176,52 @@ void assembleToStream(const ExegesisTarget &ET,
                       std::unique_ptr<llvm::LLVMTargetMachine> TM,
                       llvm::ArrayRef<unsigned> LiveIns,
                       llvm::ArrayRef<RegisterValue> RegisterInitialValues,
-                      llvm::ArrayRef<llvm::MCInst> Instructions,
+                      const FillFunction &Fill,
                       llvm::raw_pwrite_stream &AsmStream) {
   std::unique_ptr<llvm::LLVMContext> Context =
       std::make_unique<llvm::LLVMContext>();
   std::unique_ptr<llvm::Module> Module =
       createModule(Context, TM->createDataLayout());
-  std::unique_ptr<llvm::MachineModuleInfo> MMI =
-      std::make_unique<llvm::MachineModuleInfo>(TM.get());
-  llvm::MachineFunction &MF =
-      createVoidVoidPtrMachineFunction(FunctionID, Module.get(), MMI.get());
+  std::unique_ptr<llvm::MachineModuleInfoWrapperPass> MMIWP =
+      std::make_unique<llvm::MachineModuleInfoWrapperPass>(TM.get());
+  llvm::MachineFunction &MF = createVoidVoidPtrMachineFunction(
+      FunctionID, Module.get(), &MMIWP.get()->getMMI());
 
   // We need to instruct the passes that we're done with SSA and virtual
   // registers.
   auto &Properties = MF.getProperties();
   Properties.set(llvm::MachineFunctionProperties::Property::NoVRegs);
   Properties.reset(llvm::MachineFunctionProperties::Property::IsSSA);
+  Properties.set(llvm::MachineFunctionProperties::Property::NoPHIs);
 
   for (const unsigned Reg : LiveIns)
     MF.getRegInfo().addLiveIn(Reg);
 
-  bool IsSnippetSetupComplete;
-  std::vector<llvm::MCInst> Code =
-      generateSnippetSetupCode(ET, TM->getMCSubtargetInfo(),
-                               RegisterInitialValues, IsSnippetSetupComplete);
+  std::vector<unsigned> RegistersSetUp;
+  for (const auto &InitValue : RegisterInitialValues) {
+    RegistersSetUp.push_back(InitValue.Register);
+  }
+  FunctionFiller Sink(MF, std::move(RegistersSetUp));
+  auto Entry = Sink.getEntry();
+  for (const unsigned Reg : LiveIns)
+    Entry.MBB->addLiveIn(Reg);
 
-  Code.insert(Code.end(), Instructions.begin(), Instructions.end());
+  const bool IsSnippetSetupComplete = generateSnippetSetupCode(
+      ET, TM->getMCSubtargetInfo(), RegisterInitialValues, Entry);
 
   // If the snippet setup is not complete, we disable liveliness tracking. This
   // means that we won't know what values are in the registers.
   if (!IsSnippetSetupComplete)
     Properties.reset(llvm::MachineFunctionProperties::Property::TracksLiveness);
 
+  Fill(Sink);
+
   // prologue/epilogue pass needs the reserved registers to be frozen, this
   // is usually done by the SelectionDAGISel pass.
   MF.getRegInfo().freezeReservedRegs(MF);
 
-  // Fill the MachineFunction from the instructions.
-  fillMachineFunction(MF, LiveIns, Code);
-
   // We create the pass manager, run the passes to populate AsmBuffer.
-  llvm::MCContext &MCContext = MMI->getContext();
+  llvm::MCContext &MCContext = MMIWP->getMMI().getContext();
   llvm::legacy::PassManager PM;
 
   llvm::TargetLibraryInfoImpl TLII(llvm::Triple(Module->getTargetTriple()));
@@ -203,7 +229,7 @@ void assembleToStream(const ExegesisTarget &ET,
 
   llvm::TargetPassConfig *TPC = TM->createPassConfig(PM);
   PM.add(TPC);
-  PM.add(MMI.release());
+  PM.add(MMIWP.release());
   TPC->printAndVerify("MachineFunctionGenerator::assemble");
   // Add target-specific passes.
   ET.addTargetSpecificPasses(PM);
diff --git a/llvm/tools/llvm-exegesis/lib/Assembler.h b/llvm/tools/llvm-exegesis/lib/Assembler.h
index ff1e8db1a0c35..5937de9f053e1 100644
--- a/llvm/tools/llvm-exegesis/lib/Assembler.h
+++ b/llvm/tools/llvm-exegesis/lib/Assembler.h
@@ -40,6 +40,49 @@ class ExegesisTarget;
 // convention and target machine).
 llvm::BitVector getFunctionReservedRegs(const llvm::TargetMachine &TM);
 
+// Helper to fill in a basic block.
+class BasicBlockFiller {
+public:
+  BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
+                   const MCInstrInfo *MCII);
+
+  void addInstruction(const MCInst &Inst, const DebugLoc &DL = DebugLoc());
+  void addInstructions(ArrayRef<llvm::MCInst> Insts,
+                       const DebugLoc &DL = DebugLoc());
+
+  void addReturn(const DebugLoc &DL = DebugLoc());
+
+  MachineFunction &MF;
+  MachineBasicBlock *const MBB;
+  const MCInstrInfo *const MCII;
+};
+
+// Helper to fill in a function.
+class FunctionFiller {
+public:
+  FunctionFiller(MachineFunction &MF, std::vector<unsigned> RegistersSetUp);
+
+  // Adds a basic block to the function.
+  BasicBlockFiller addBasicBlock();
+
+  // Returns the function entry point.
+  BasicBlockFiller getEntry() { return Entry; }
+
+  MachineFunction &MF;
+  const MCInstrInfo *const MCII;
+
+  // Returns the set of registers in the snippet setup code.
+  ArrayRef<unsigned> getRegistersSetUp() const;
+
+private:
+  BasicBlockFiller Entry;
+  // The set of registers that are set up in the basic block.
+  std::vector<unsigned> RegistersSetUp;
+};
+
+// A callback that fills a function.
+using FillFunction = std::function<void(FunctionFiller &)>;
+
 // Creates a temporary `void foo(char*)` function containing the provided
 // Instructions. Runs a set of llvm Passes to provide correct prologue and
 // epilogue. Once the MachineFunction is ready, it is assembled for TM to
@@ -48,7 +91,7 @@ void assembleToStream(const ExegesisTarget &ET,
                       std::unique_ptr<llvm::LLVMTargetMachine> TM,
                       llvm::ArrayRef<unsigned> LiveIns,
                       llvm::ArrayRef<RegisterValue> RegisterInitialValues,
-                      llvm::ArrayRef<llvm::MCInst> Instructions,
+                      const FillFunction &Fill,
                       llvm::raw_pwrite_stream &AsmStream);
 
 // Creates an ObjectFile in the format understood by the host.
@@ -81,6 +124,12 @@ struct ExecutableFunction {
   llvm::StringRef FunctionBytes;
 };
 
+// Creates a void(int8*) MachineFunction.
+llvm::MachineFunction &
+createVoidVoidPtrMachineFunction(llvm::StringRef FunctionID,
+                                 llvm::Module *Module,
+                                 llvm::MachineModuleInfo *MMI);
+
 } // namespace exegesis
 } // namespace llvm
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
index 685f31a2b7d10..790e253709cc6 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.cpp
@@ -8,6 +8,7 @@
 
 #include "BenchmarkResult.h"
 #include "BenchmarkRunner.h"
+#include "Error.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringMap.h"
@@ -343,7 +344,7 @@ InstructionBenchmark::readYaml(const LLVMState &State,
     if (Yin.setCurrentDocument())
       llvm::yaml::yamlize(Yin, Benchmark, /*unused*/ true, Context);
     if (!Context.getLastError().empty())
-      return llvm::make_error<BenchmarkFailure>(Context.getLastError());
+      return make_error<Failure>(Context.getLastError());
     return Benchmark;
   } else {
     return ExpectedMemoryBuffer.takeError();
@@ -364,7 +365,7 @@ InstructionBenchmark::readYamls(const LLVMState &State,
       if (Yin.error())
         return llvm::errorCodeToError(Yin.error());
       if (!Context.getLastError().empty())
-        return llvm::make_error<BenchmarkFailure>(Context.getLastError());
+        return make_error<Failure>(Context.getLastError());
       Yin.nextDocument();
     }
     return Benchmarks;
@@ -381,7 +382,7 @@ llvm::Error InstructionBenchmark::writeYamlTo(const LLVMState &State,
   Yout.beginDocuments();
   llvm::yaml::yamlize(Yout, *this, /*unused*/ true, Context);
   if (!Context.getLastError().empty())
-    return llvm::make_error<BenchmarkFailure>(Context.getLastError());
+    return make_error<Failure>(Context.getLastError());
   Yout.endDocuments();
   return Error::success();
 }
@@ -393,7 +394,7 @@ llvm::Error InstructionBenchmark::readYamlFrom(const LLVMState &State,
   if (Yin.setCurrentDocument())
     llvm::yaml::yamlize(Yin, *this, /*unused*/ true, Context);
   if (!Context.getLastError().empty())
-    return llvm::make_error<BenchmarkFailure>(Context.getLastError());
+    return make_error<Failure>(Context.getLastError());
   return Error::success();
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
index e54e8c2a8dac9..17ffd0a8c8703 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h
@@ -66,6 +66,8 @@ struct InstructionBenchmark {
   // The number of instructions inside the repeated snippet. For example, if a
   // snippet of 3 instructions is repeated 4 times, this is 12.
   int NumRepetitions = 0;
+  enum RepetitionModeE { Duplicate, Loop };
+  RepetitionModeE RepetitionMode;
   // Note that measurements are per instruction.
   std::vector<BenchmarkMeasure> Measurements;
   std::string Error;
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index 38abf3eefa3e1..4b541f4d829de 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -11,6 +11,7 @@
 
 #include "Assembler.h"
 #include "BenchmarkRunner.h"
+#include "Error.h"
 #include "MCInstrDescView.h"
 #include "PerfHelper.h"
 #include "llvm/ADT/StringExtras.h"
@@ -24,26 +25,12 @@
 namespace llvm {
 namespace exegesis {
 
-BenchmarkFailure::BenchmarkFailure(const llvm::Twine &S)
-    : llvm::StringError(S, llvm::inconvertibleErrorCode()) {}
-
 BenchmarkRunner::BenchmarkRunner(const LLVMState &State,
                                  InstructionBenchmark::ModeE Mode)
     : State(State), Mode(Mode), Scratch(std::make_unique<ScratchSpace>()) {}
 
 BenchmarkRunner::~BenchmarkRunner() = default;
 
-// Repeat the snippet until there are at least MinInstructions in the resulting
-// code.
-static std::vector<llvm::MCInst>
-GenerateInstructions(const BenchmarkCode &BC, const size_t MinInstructions) {
-  if (BC.Instructions.empty())
-    return {};
-  std::vector<llvm::MCInst> Code = BC.Instructions;
-  for (int I = 0; Code.size() < MinInstructions; ++I)
-    Code.push_back(BC.Instructions[I % BC.Instructions.size()]);
-  return Code;
-}
 
 namespace {
 class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
@@ -82,8 +69,7 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
         llvm::CrashRecoveryContext::Disable();
         // FIXME: Better diagnosis.
         if (Crashed)
-          return llvm::make_error<BenchmarkFailure>(
-              "snippet crashed while running");
+          return make_error<Failure>("snippet crashed while running");
       }
       CounterValue += Counter.read();
     }
@@ -95,10 +81,9 @@ class FunctionExecutorImpl : public BenchmarkRunner::FunctionExecutor {
 };
 } // namespace
 
-InstructionBenchmark
-BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
-                                  unsigned NumRepetitions,
-                                  bool DumpObjectToDisk) const {
+InstructionBenchmark BenchmarkRunner::runConfiguration(
+    const BenchmarkCode &BC, unsigned NumRepetitions,
+    const SnippetRepetitor &Repetitor, bool DumpObjectToDisk) const {
   InstructionBenchmark InstrBenchmark;
   InstrBenchmark.Mode = Mode;
   InstrBenchmark.CpuName = State.getTargetMachine().getTargetCPU();
@@ -119,9 +104,10 @@ BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
   {
     llvm::SmallString<0> Buffer;
     llvm::raw_svector_ostream OS(Buffer);
-    assembleToStream(State.getExegesisTarget(), State.createTargetMachine(),
-                     BC.LiveIns, BC.RegisterInitialValues,
-                     GenerateInstructions(BC, kMinInstructionsForSnippet), OS);
+    assembleToStream(
+        State.getExegesisTarget(), State.createTargetMachine(), BC.LiveIns,
+        BC.RegisterInitialValues,
+        Repetitor.Repeat(BC.Instructions, kMinInstructionsForSnippet), OS);
     const ExecutableFunction EF(State.createTargetMachine(),
                                 getObjectFromBuffer(OS.str()));
     const auto FnBytes = EF.getFunctionBytes();
@@ -130,11 +116,12 @@ BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
 
   // Assemble NumRepetitions instructions repetitions of the snippet for
   // measurements.
-  const auto Code = GenerateInstructions(BC, InstrBenchmark.NumRepetitions);
+  const auto Filler =
+      Repetitor.Repeat(BC.Instructions, InstrBenchmark.NumRepetitions);
 
   llvm::object::OwningBinary<llvm::object::ObjectFile> ObjectFile;
   if (DumpObjectToDisk) {
-    auto ObjectFilePath = writeObjectFile(BC, Code);
+    auto ObjectFilePath = writeObjectFile(BC, Filler);
     if (llvm::Error E = ObjectFilePath.takeError()) {
       InstrBenchmark.Error = llvm::toString(std::move(E));
       return InstrBenchmark;
@@ -146,7 +133,7 @@ BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
     llvm::SmallString<0> Buffer;
     llvm::raw_svector_ostream OS(Buffer);
     assembleToStream(State.getExegesisTarget(), State.createTargetMachine(),
-                     BC.LiveIns, BC.RegisterInitialValues, Code, OS);
+                     BC.LiveIns, BC.RegisterInitialValues, Filler, OS);
     ObjectFile = getObjectFromBuffer(OS.str());
   }
 
@@ -172,7 +159,7 @@ BenchmarkRunner::runConfiguration(const BenchmarkCode &BC,
 
 llvm::Expected<std::string>
 BenchmarkRunner::writeObjectFile(const BenchmarkCode &BC,
-                                 llvm::ArrayRef<llvm::MCInst> Code) const {
+                                 const FillFunction &FillFunction) const {
   int ResultFD = 0;
   llvm::SmallString<256> ResultPath;
   if (llvm::Error E = llvm::errorCodeToError(llvm::sys::fs::createTemporaryFile(
@@ -180,7 +167,7 @@ BenchmarkRunner::writeObjectFile(const BenchmarkCode &BC,
     return std::move(E);
   llvm::raw_fd_ostream OFS(ResultFD, true /*ShouldClose*/);
   assembleToStream(State.getExegesisTarget(), State.createTargetMachine(),
-                   BC.LiveIns, BC.RegisterInitialValues, Code, OFS);
+                   BC.LiveIns, BC.RegisterInitialValues, FillFunction, OFS);
   return ResultPath.str();
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
index 29e98619301ae..2f6ea0f5b5a18 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h
@@ -20,6 +20,7 @@
 #include "BenchmarkResult.h"
 #include "LlvmState.h"
 #include "MCInstrDescView.h"
+#include "SnippetRepetitor.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/Support/Error.h"
 #include <cstdlib>
@@ -29,13 +30,6 @@
 namespace llvm {
 namespace exegesis {
 
-// A class representing failures that happened during Benchmark, they are used
-// to report informations to the user.
-class BenchmarkFailure : public llvm::StringError {
-public:
-  BenchmarkFailure(const llvm::Twine &S);
-};
-
 // Common code for all benchmark modes.
 class BenchmarkRunner {
 public:
@@ -46,6 +40,7 @@ class BenchmarkRunner {
 
   InstructionBenchmark runConfiguration(const BenchmarkCode &Configuration,
                                         unsigned NumRepetitions,
+                                        const SnippetRepetitor &Repetitor,
                                         bool DumpObjectToDisk) const;
 
   // Scratch space to run instructions that touch memory.
@@ -84,7 +79,7 @@ class BenchmarkRunner {
 
   llvm::Expected<std::string>
   writeObjectFile(const BenchmarkCode &Configuration,
-                  llvm::ArrayRef<llvm::MCInst> Code) const;
+                  const FillFunction &Fill) const;
 
   const std::unique_ptr<ScratchSpace> Scratch;
 };
diff --git a/llvm/tools/llvm-exegesis/lib/CMakeLists.txt b/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
index ef1a411d9ef26..3b58df7db7ec2 100644
--- a/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
+++ b/llvm/tools/llvm-exegesis/lib/CMakeLists.txt
@@ -30,7 +30,9 @@ add_library(LLVMExegesis
   RegisterAliasing.cpp
   RegisterValue.cpp
   SchedClassResolution.cpp
+  SnippetFile.cpp
   SnippetGenerator.cpp
+  SnippetRepetitor.cpp
   Target.cpp
   Uops.cpp
   )
@@ -45,6 +47,7 @@ llvm_map_components_to_libnames(libs
   MC
   MCDisassembler
   MCJIT
+  MCParser
   Object
   ObjectYAML
   Support
diff --git a/llvm/tools/llvm-exegesis/lib/Error.h b/llvm/tools/llvm-exegesis/lib/Error.h
new file mode 100644
index 0000000000000..720cc6829f3e3
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/Error.h
@@ -0,0 +1,28 @@
+//===-- Error.h -------------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_EXEGESIS_ERROR_H
+#define LLVM_TOOLS_LLVM_EXEGESIS_ERROR_H
+
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace exegesis {
+
+// A class representing failures that happened within llvm-exegesis, they are
+// used to report informations to the user.
+class Failure : public StringError {
+public:
+  Failure(const Twine &S) : StringError(S, inconvertibleErrorCode()) {}
+};
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-exegesis/lib/Latency.cpp b/llvm/tools/llvm-exegesis/lib/Latency.cpp
index 2f3fbaef4f4e5..87c895e4ab698 100644
--- a/llvm/tools/llvm-exegesis/lib/Latency.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Latency.cpp
@@ -39,7 +39,8 @@ static constexpr size_t kMaxAliasingInstructions = 10;
 
 static std::vector<Instruction>
 computeAliasingInstructions(const LLVMState &State, const Instruction &Instr,
-                            size_t MaxAliasingInstructions) {
+                            size_t MaxAliasingInstructions,
+                            const BitVector &ForbiddenRegisters) {
   // Randomly iterate the set of instructions.
   std::vector<unsigned> Opcodes;
   Opcodes.resize(State.getInstrInfo().getNumOpcodes());
@@ -53,15 +54,16 @@ computeAliasingInstructions(const LLVMState &State, const Instruction &Instr,
     const Instruction &OtherInstr = State.getIC().getInstr(OtherOpcode);
     if (OtherInstr.hasMemoryOperands())
       continue;
-    if (Instr.hasAliasingRegistersThrough(OtherInstr))
-      AliasingInstructions.push_back(std::move(OtherInstr));
+    if (Instr.hasAliasingRegistersThrough(OtherInstr, ForbiddenRegisters))
+      AliasingInstructions.push_back(OtherInstr);
     if (AliasingInstructions.size() >= MaxAliasingInstructions)
       break;
   }
   return AliasingInstructions;
 }
 
-static ExecutionMode getExecutionModes(const Instruction &Instr) {
+static ExecutionMode getExecutionModes(const Instruction &Instr,
+                                       const BitVector &ForbiddenRegisters) {
   ExecutionMode EM = ExecutionMode::UNKNOWN;
   if (Instr.hasAliasingImplicitRegisters())
     EM |= ExecutionMode::ALWAYS_SERIAL_IMPLICIT_REGS_ALIAS;
@@ -70,7 +72,7 @@ static ExecutionMode getExecutionModes(const Instruction &Instr) {
   if (Instr.hasMemoryOperands())
     EM |= ExecutionMode::SERIAL_VIA_MEMORY_INSTR;
   else {
-    if (Instr.hasAliasingRegisters())
+    if (Instr.hasAliasingRegisters(ForbiddenRegisters))
       EM |= ExecutionMode::SERIAL_VIA_EXPLICIT_REGS;
     if (Instr.hasOneUseOrOneDef())
       EM |= ExecutionMode::SERIAL_VIA_NON_MEMORY_INSTR;
@@ -80,6 +82,7 @@ static ExecutionMode getExecutionModes(const Instruction &Instr) {
 
 static void appendCodeTemplates(const LLVMState &State,
                                 const Instruction &Instr,
+                                const BitVector &ForbiddenRegisters,
                                 ExecutionMode ExecutionModeBit,
                                 llvm::StringRef ExecutionClassDescription,
                                 std::vector<CodeTemplate> &CodeTemplates) {
@@ -122,8 +125,8 @@ static void appendCodeTemplates(const LLVMState &State,
   }
   case ExecutionMode::SERIAL_VIA_NON_MEMORY_INSTR: {
     // Select back-to-back non-memory instruction.
-    for (const auto OtherInstr :
-         computeAliasingInstructions(State, Instr, kMaxAliasingInstructions)) {
+    for (const auto OtherInstr : computeAliasingInstructions(
+             State, Instr, kMaxAliasingInstructions, ForbiddenRegisters)) {
       const AliasingConfigurations Forward(Instr, OtherInstr);
       const AliasingConfigurations Back(OtherInstr, Instr);
       InstructionTemplate ThisIT(Instr);
@@ -149,18 +152,19 @@ static void appendCodeTemplates(const LLVMState &State,
 LatencySnippetGenerator::~LatencySnippetGenerator() = default;
 
 llvm::Expected<std::vector<CodeTemplate>>
-LatencySnippetGenerator::generateCodeTemplates(const Instruction &Instr) const {
+LatencySnippetGenerator::generateCodeTemplates(
+    const Instruction &Instr, const BitVector &ForbiddenRegisters) const {
   std::vector<CodeTemplate> Results;
-  const ExecutionMode EM = getExecutionModes(Instr);
+  const ExecutionMode EM = getExecutionModes(Instr, ForbiddenRegisters);
   for (const auto EC : kExecutionClasses) {
     for (const auto ExecutionModeBit : getExecutionModeBits(EM & EC.Mask))
-      appendCodeTemplates(State, Instr, ExecutionModeBit, EC.Description,
-                          Results);
+      appendCodeTemplates(State, Instr, ForbiddenRegisters, ExecutionModeBit,
+                          EC.Description, Results);
     if (!Results.empty())
       break;
   }
   if (Results.empty())
-    return llvm::make_error<BenchmarkFailure>(
+    return make_error<Failure>(
         "No strategy found to make the execution serial");
   return std::move(Results);
 }
diff --git a/llvm/tools/llvm-exegesis/lib/Latency.h b/llvm/tools/llvm-exegesis/lib/Latency.h
index 7d6d96a195efb..503f91daf8958 100644
--- a/llvm/tools/llvm-exegesis/lib/Latency.h
+++ b/llvm/tools/llvm-exegesis/lib/Latency.h
@@ -15,6 +15,7 @@
 #define LLVM_TOOLS_LLVM_EXEGESIS_LATENCY_H
 
 #include "BenchmarkRunner.h"
+#include "Error.h"
 #include "MCInstrDescView.h"
 #include "SnippetGenerator.h"
 
@@ -27,7 +28,8 @@ class LatencySnippetGenerator : public SnippetGenerator {
   ~LatencySnippetGenerator() override;
 
   llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const override;
+  generateCodeTemplates(const Instruction &Instr,
+                        const BitVector &ForbiddenRegisters) const override;
 };
 
 class LatencyBenchmarkRunner : public BenchmarkRunner {
diff --git a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
index de36d5cda0c0d..f44dcbdf717ee 100644
--- a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
+++ b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.cpp
@@ -183,16 +183,29 @@ bool Instruction::hasAliasingImplicitRegisters() const {
   return ImplDefRegs.anyCommon(ImplUseRegs);
 }
 
-bool Instruction::hasAliasingImplicitRegistersThrough(
-    const Instruction &OtherInstr) const {
-  return ImplDefRegs.anyCommon(OtherInstr.ImplUseRegs) &&
-         OtherInstr.ImplDefRegs.anyCommon(ImplUseRegs);
+// Returns true if there are registers that are both in `A` and `B` but not in
+// `Forbidden`.
+static bool anyCommonExcludingForbidden(const BitVector &A, const BitVector &B,
+                                        const BitVector &Forbidden) {
+  assert(A.size() == B.size() && B.size() == Forbidden.size());
+  const auto Size = A.size();
+  for (int AIndex = A.find_first(); AIndex != -1;) {
+    const int BIndex = B.find_first_in(AIndex, Size);
+    if (BIndex == -1)
+      return false;
+    if (AIndex == BIndex && !Forbidden.test(AIndex))
+      return true;
+    AIndex = A.find_first_in(BIndex + 1, Size);
+  }
+  return false;
 }
 
 bool Instruction::hasAliasingRegistersThrough(
-    const Instruction &OtherInstr) const {
-  return AllDefRegs.anyCommon(OtherInstr.AllUseRegs) &&
-         OtherInstr.AllDefRegs.anyCommon(AllUseRegs);
+    const Instruction &OtherInstr, const BitVector &ForbiddenRegisters) const {
+  return anyCommonExcludingForbidden(AllDefRegs, OtherInstr.AllUseRegs,
+                                     ForbiddenRegisters) &&
+         anyCommonExcludingForbidden(OtherInstr.AllDefRegs, AllUseRegs,
+                                     ForbiddenRegisters);
 }
 
 bool Instruction::hasTiedRegisters() const {
@@ -200,8 +213,10 @@ bool Instruction::hasTiedRegisters() const {
       Variables, [](const Variable &Var) { return Var.hasTiedOperands(); });
 }
 
-bool Instruction::hasAliasingRegisters() const {
-  return AllDefRegs.anyCommon(AllUseRegs);
+bool Instruction::hasAliasingRegisters(
+    const BitVector &ForbiddenRegisters) const {
+  return anyCommonExcludingForbidden(AllDefRegs, AllUseRegs,
+                                     ForbiddenRegisters);
 }
 
 bool Instruction::hasOneUseOrOneDef() const {
@@ -209,6 +224,7 @@ bool Instruction::hasOneUseOrOneDef() const {
 }
 
 void Instruction::dump(const llvm::MCRegisterInfo &RegInfo,
+                       const RegisterAliasingTrackerCache &RATC,
                        llvm::raw_ostream &Stream) const {
   Stream << "- " << Name << "\n";
   for (const auto &Op : Operands) {
@@ -257,7 +273,7 @@ void Instruction::dump(const llvm::MCRegisterInfo &RegInfo,
     Stream << "- hasAliasingImplicitRegisters (execution is always serial)\n";
   if (hasTiedRegisters())
     Stream << "- hasTiedRegisters (execution is always serial)\n";
-  if (hasAliasingRegisters())
+  if (hasAliasingRegisters(RATC.emptyRegisters()))
     Stream << "- hasAliasingRegisters\n";
 }
 
diff --git a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.h b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.h
index b333b92bb0e62..1c67ef8ec29cc 100644
--- a/llvm/tools/llvm-exegesis/lib/MCInstrDescView.h
+++ b/llvm/tools/llvm-exegesis/lib/MCInstrDescView.h
@@ -112,14 +112,11 @@ struct Instruction {
   // Repeating this instruction may execute sequentially by picking aliasing
   // Use and Def registers. It may also execute in parallel by picking non
   // aliasing Use and Def registers.
-  bool hasAliasingRegisters() const;
-
-  // Whether this instruction's implicit registers alias with OtherInstr's
-  // implicit registers.
-  bool hasAliasingImplicitRegistersThrough(const Instruction &OtherInstr) const;
+  bool hasAliasingRegisters(const BitVector &ForbiddenRegisters) const;
 
   // Whether this instruction's registers alias with OtherInstr's registers.
-  bool hasAliasingRegistersThrough(const Instruction &OtherInstr) const;
+  bool hasAliasingRegistersThrough(const Instruction &OtherInstr,
+                                   const BitVector &ForbiddenRegisters) const;
 
   // Returns whether this instruction has Memory Operands.
   // Repeating this instruction executes sequentially with an instruction that
@@ -133,6 +130,7 @@ struct Instruction {
 
   // Convenient function to help with debugging.
   void dump(const llvm::MCRegisterInfo &RegInfo,
+            const RegisterAliasingTrackerCache &RATC,
             llvm::raw_ostream &Stream) const;
 
   const llvm::MCInstrDesc *Description; // Never nullptr.
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
new file mode 100644
index 0000000000000..f5666ecab9548
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp
@@ -0,0 +1,163 @@
+//===-- SnippetFile.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SnippetFile.h"
+#include "Error.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/MC/MCParser/MCTargetAsmParser.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
+#include <string>
+
+namespace llvm {
+namespace exegesis {
+namespace {
+
+// An MCStreamer that reads a BenchmarkCode definition from a file.
+class BenchmarkCodeStreamer : public MCStreamer, public AsmCommentConsumer {
+public:
+  explicit BenchmarkCodeStreamer(MCContext *Context,
+                                 const MCRegisterInfo *TheRegInfo,
+                                 BenchmarkCode *Result)
+      : MCStreamer(*Context), RegInfo(TheRegInfo), Result(Result) {}
+
+  // Implementation of the MCStreamer interface. We only care about
+  // instructions.
+  void EmitInstruction(const MCInst &Instruction,
+                       const MCSubtargetInfo &STI) override {
+    Result->Instructions.push_back(Instruction);
+  }
+
+  // Implementation of the AsmCommentConsumer.
+  void HandleComment(SMLoc Loc, StringRef CommentText) override {
+    CommentText = CommentText.trim();
+    if (!CommentText.consume_front("LLVM-EXEGESIS-"))
+      return;
+    if (CommentText.consume_front("DEFREG")) {
+      // LLVM-EXEGESIS-DEFREF <reg> <hex_value>
+      RegisterValue RegVal;
+      SmallVector<StringRef, 2> Parts;
+      CommentText.split(Parts, ' ', /*unlimited splits*/ -1,
+                        /*do not keep empty strings*/ false);
+      if (Parts.size() != 2) {
+        errs() << "invalid comment 'LLVM-EXEGESIS-DEFREG " << CommentText
+               << "', expected two parameters <REG> <HEX_VALUE>\n";
+        ++InvalidComments;
+        return;
+      }
+      if (!(RegVal.Register = findRegisterByName(Parts[0].trim()))) {
+        errs() << "unknown register '" << Parts[0]
+               << "' in 'LLVM-EXEGESIS-DEFREG " << CommentText << "'\n";
+        ++InvalidComments;
+        return;
+      }
+      const StringRef HexValue = Parts[1].trim();
+      RegVal.Value = APInt(
+          /* each hex digit is 4 bits */ HexValue.size() * 4, HexValue, 16);
+      Result->RegisterInitialValues.push_back(std::move(RegVal));
+      return;
+    }
+    if (CommentText.consume_front("LIVEIN")) {
+      // LLVM-EXEGESIS-LIVEIN <reg>
+      const auto RegName = CommentText.ltrim();
+      if (unsigned Reg = findRegisterByName(RegName))
+        Result->LiveIns.push_back(Reg);
+      else {
+        errs() << "unknown register '" << RegName
+               << "' in 'LLVM-EXEGESIS-LIVEIN " << CommentText << "'\n";
+        ++InvalidComments;
+      }
+      return;
+    }
+  }
+
+  unsigned numInvalidComments() const { return InvalidComments; }
+
+private:
+  // We only care about instructions, we don't implement this part of the API.
+  void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                        unsigned ByteAlignment) override {}
+  bool EmitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override {
+    return false;
+  }
+  void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value,
+                            unsigned ValueSize,
+                            unsigned MaxBytesToEmit) override {}
+  void EmitZerofill(MCSection *Section, MCSymbol *Symbol, uint64_t Size,
+                    unsigned ByteAlignment, SMLoc Loc) override {}
+
+  unsigned findRegisterByName(const StringRef RegName) const {
+    // FIXME: Can we do better than this ?
+    for (unsigned I = 0, E = RegInfo->getNumRegs(); I < E; ++I) {
+      if (RegName == RegInfo->getName(I))
+        return I;
+    }
+    errs() << "'" << RegName
+           << "' is not a valid register name for the target\n";
+    return 0;
+  }
+
+  const MCRegisterInfo *const RegInfo;
+  BenchmarkCode *const Result;
+  unsigned InvalidComments = 0;
+};
+
+} // namespace
+
+// Reads code snippets from file `Filename`.
+Expected<std::vector<BenchmarkCode>> readSnippets(const LLVMState &State,
+                                                  StringRef Filename) {
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufferPtr =
+      MemoryBuffer::getFileOrSTDIN(Filename);
+  if (std::error_code EC = BufferPtr.getError()) {
+    return make_error<Failure>("cannot read snippet: " + Filename + ": " +
+                               EC.message());
+  }
+  SourceMgr SM;
+  SM.AddNewSourceBuffer(std::move(BufferPtr.get()), SMLoc());
+
+  BenchmarkCode Result;
+
+  MCObjectFileInfo ObjectFileInfo;
+  const TargetMachine &TM = State.getTargetMachine();
+  MCContext Context(TM.getMCAsmInfo(), TM.getMCRegisterInfo(), &ObjectFileInfo);
+  ObjectFileInfo.InitMCObjectFileInfo(TM.getTargetTriple(), /*PIC*/ false,
+                                      Context);
+  BenchmarkCodeStreamer Streamer(&Context, TM.getMCRegisterInfo(), &Result);
+  const std::unique_ptr<MCAsmParser> AsmParser(
+      createMCAsmParser(SM, Context, Streamer, *TM.getMCAsmInfo()));
+  if (!AsmParser)
+    return make_error<Failure>("cannot create asm parser");
+  AsmParser->getLexer().setCommentConsumer(&Streamer);
+
+  const std::unique_ptr<MCTargetAsmParser> TargetAsmParser(
+      TM.getTarget().createMCAsmParser(*TM.getMCSubtargetInfo(), *AsmParser,
+                                       *TM.getMCInstrInfo(),
+                                       MCTargetOptions()));
+
+  if (!TargetAsmParser)
+    return make_error<Failure>("cannot create target asm parser");
+  AsmParser->setTargetParser(*TargetAsmParser);
+
+  if (AsmParser->Run(false))
+    return make_error<Failure>("cannot parse asm file");
+  if (Streamer.numInvalidComments())
+    return make_error<Failure>(Twine("found ")
+                                   .concat(Twine(Streamer.numInvalidComments()))
+                                   .concat(" invalid LLVM-EXEGESIS comments"));
+  return std::vector<BenchmarkCode>{std::move(Result)};
+}
+
+} // namespace exegesis
+} // namespace llvm
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetFile.h b/llvm/tools/llvm-exegesis/lib/SnippetFile.h
new file mode 100644
index 0000000000000..e5ece91917fb7
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/SnippetFile.h
@@ -0,0 +1,35 @@
+//===-- SnippetFile.cpp -----------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Utilities to read a snippet file.
+/// Snippet files are just asm files with additional comments to specify which
+/// registers should be defined or are live on entry.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_EXEGESIS_SNIPPETFILE_H
+#define LLVM_TOOLS_LLVM_EXEGESIS_SNIPPETFILE_H
+
+#include "BenchmarkCode.h"
+#include "LlvmState.h"
+#include "llvm/Support/Error.h"
+
+#include <vector>
+
+namespace llvm {
+namespace exegesis {
+
+// Reads code snippets from file `Filename`.
+Expected<std::vector<BenchmarkCode>> readSnippets(const LLVMState &State,
+                                                  StringRef Filename);
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif
\ No newline at end of file
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
index d5c790dbefced..267ab13163345 100644
--- a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
+++ b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.cpp
@@ -10,6 +10,7 @@
 #include <string>
 
 #include "Assembler.h"
+#include "Error.h"
 #include "MCInstrDescView.h"
 #include "SnippetGenerator.h"
 #include "Target.h"
@@ -37,15 +38,35 @@ SnippetGenerator::SnippetGenerator(const LLVMState &State) : State(State) {}
 SnippetGenerator::~SnippetGenerator() = default;
 
 llvm::Expected<std::vector<BenchmarkCode>>
-SnippetGenerator::generateConfigurations(const Instruction &Instr) const {
-  if (auto E = generateCodeTemplates(Instr)) {
-    const auto &RATC = State.getRATC();
+SnippetGenerator::generateConfigurations(
+    const Instruction &Instr, const llvm::BitVector &ExtraForbiddenRegs) const {
+  llvm::BitVector ForbiddenRegs = State.getRATC().reservedRegisters();
+  ForbiddenRegs |= ExtraForbiddenRegs;
+  // If the instruction has memory registers, prevent the generator from
+  // using the scratch register and its aliasing registers.
+  if (Instr.hasMemoryOperands()) {
+    const auto &ET = State.getExegesisTarget();
+    unsigned ScratchSpacePointerInReg =
+        ET.getScratchMemoryRegister(State.getTargetMachine().getTargetTriple());
+    if (ScratchSpacePointerInReg == 0)
+      return make_error<Failure>(
+          "Infeasible : target does not support memory instructions");
+    const auto &ScratchRegAliases =
+        State.getRATC().getRegister(ScratchSpacePointerInReg).aliasedBits();
+    // If the instruction implicitly writes to ScratchSpacePointerInReg , abort.
+    // FIXME: We could make a copy of the scratch register.
+    for (const auto &Op : Instr.Operands) {
+      if (Op.isDef() && Op.isImplicitReg() &&
+          ScratchRegAliases.test(Op.getImplicitReg()))
+        return make_error<Failure>(
+            "Infeasible : memory instruction uses scratch memory register");
+    }
+    ForbiddenRegs |= ScratchRegAliases;
+  }
+
+  if (auto E = generateCodeTemplates(Instr, ForbiddenRegs)) {
     std::vector<BenchmarkCode> Output;
     for (CodeTemplate &CT : E.get()) {
-      const llvm::BitVector &ForbiddenRegs =
-          CT.ScratchSpacePointerInReg
-              ? RATC.getRegister(CT.ScratchSpacePointerInReg).aliasedBits()
-              : RATC.emptyRegisters();
       // TODO: Generate as many BenchmarkCode as needed.
       {
         BenchmarkCode BC;
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.h b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.h
index ef44bd9946484..c2ea1c124738e 100644
--- a/llvm/tools/llvm-exegesis/lib/SnippetGenerator.h
+++ b/llvm/tools/llvm-exegesis/lib/SnippetGenerator.h
@@ -57,7 +57,8 @@ class SnippetGenerator {
 
   // Calls generateCodeTemplate and expands it into one or more BenchmarkCode.
   llvm::Expected<std::vector<BenchmarkCode>>
-  generateConfigurations(const Instruction &Instr) const;
+  generateConfigurations(const Instruction &Instr,
+                         const llvm::BitVector &ExtraForbiddenRegs) const;
 
   // Given a snippet, computes which registers the setup code needs to define.
   std::vector<RegisterValue> computeRegisterInitialValues(
@@ -69,7 +70,8 @@ class SnippetGenerator {
 private:
   // API to be implemented by subclasses.
   virtual llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const = 0;
+  generateCodeTemplates(const Instruction &Instr,
+                        const BitVector &ForbiddenRegisters) const = 0;
 };
 
 // A global Random Number Generator to randomize configurations.
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
new file mode 100644
index 0000000000000..954471659e593
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.cpp
@@ -0,0 +1,117 @@
+//===-- SnippetRepetitor.cpp ------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <array>
+#include <string>
+
+#include "SnippetRepetitor.h"
+#include "Target.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+
+namespace llvm {
+namespace exegesis {
+namespace {
+
+class DuplicateSnippetRepetitor : public SnippetRepetitor {
+public:
+  using SnippetRepetitor::SnippetRepetitor;
+
+  // Repeats the snippet until there are at least MinInstructions in the
+  // resulting code.
+  FillFunction Repeat(ArrayRef<MCInst> Instructions,
+                      unsigned MinInstructions) const override {
+    return [Instructions, MinInstructions](FunctionFiller &Filler) {
+      auto Entry = Filler.getEntry();
+      if (!Instructions.empty()) {
+        // Add the whole snippet at least once.
+        Entry.addInstructions(Instructions);
+        for (unsigned I = Instructions.size(); I < MinInstructions; ++I) {
+          Entry.addInstruction(Instructions[I % Instructions.size()]);
+        }
+      }
+      Entry.addReturn();
+    };
+  }
+
+  BitVector getReservedRegs() const override {
+    // We're using no additional registers.
+    return State.getRATC().emptyRegisters();
+  }
+};
+
+class LoopSnippetRepetitor : public SnippetRepetitor {
+public:
+  explicit LoopSnippetRepetitor(const LLVMState &State)
+      : SnippetRepetitor(State),
+        LoopCounter(State.getExegesisTarget().getLoopCounterRegister(
+            State.getTargetMachine().getTargetTriple())) {}
+
+  // Loop over the snippet ceil(MinInstructions / Instructions.Size()) times.
+  FillFunction Repeat(ArrayRef<MCInst> Instructions,
+                      unsigned MinInstructions) const override {
+    return [this, Instructions, MinInstructions](FunctionFiller &Filler) {
+      const auto &ET = State.getExegesisTarget();
+      auto Entry = Filler.getEntry();
+      auto Loop = Filler.addBasicBlock();
+      auto Exit = Filler.addBasicBlock();
+
+      // Set loop counter to the right value:
+      const APInt LoopCount(32, (MinInstructions + Instructions.size() - 1) /
+                                    Instructions.size());
+      for (const MCInst &Inst :
+           ET.setRegTo(State.getSubtargetInfo(), LoopCounter, LoopCount))
+        Entry.addInstruction(Inst);
+
+      // Set up the loop basic block.
+      Entry.MBB->addSuccessor(Loop.MBB, llvm::BranchProbability::getOne());
+      Loop.MBB->addSuccessor(Loop.MBB, llvm::BranchProbability::getOne());
+      // The live ins are: the loop counter, the registers that were setup by
+      // the entry block, and entry block live ins.
+      Loop.MBB->addLiveIn(LoopCounter);
+      for (unsigned Reg : Filler.getRegistersSetUp())
+        Loop.MBB->addLiveIn(Reg);
+      for (const auto &LiveIn : Entry.MBB->liveins())
+        Loop.MBB->addLiveIn(LiveIn);
+      Loop.addInstructions(Instructions);
+      ET.decrementLoopCounterAndLoop(*Loop.MBB, State.getInstrInfo());
+
+      // Set up the exit basic block.
+      Loop.MBB->addSuccessor(Exit.MBB, llvm::BranchProbability::getZero());
+      Exit.addReturn();
+    };
+  }
+
+  BitVector getReservedRegs() const override {
+    // We're using a single loop counter, but we have to reserve all aliasing
+    // registers.
+    return State.getRATC().getRegister(LoopCounter).aliasedBits();
+  }
+
+private:
+  const unsigned LoopCounter;
+};
+
+} // namespace
+
+SnippetRepetitor::~SnippetRepetitor() {}
+
+std::unique_ptr<const SnippetRepetitor>
+SnippetRepetitor::Create(InstructionBenchmark::RepetitionModeE Mode,
+                         const LLVMState &State) {
+  switch (Mode) {
+  case InstructionBenchmark::Duplicate:
+    return std::make_unique<DuplicateSnippetRepetitor>(State);
+  case InstructionBenchmark::Loop:
+    return std::make_unique<LoopSnippetRepetitor>(State);
+  }
+  llvm_unreachable("Unknown RepetitionModeE enum");
+}
+
+} // namespace exegesis
+} // namespace llvm
diff --git a/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.h b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.h
new file mode 100644
index 0000000000000..038dcac1e16db
--- /dev/null
+++ b/llvm/tools/llvm-exegesis/lib/SnippetRepetitor.h
@@ -0,0 +1,53 @@
+//===-- SnippetRepetitor.h --------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// Defines helpers to fill functions with repetitions of a snippet.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_EXEGESIS_FUNCTIONFILLER_H
+#define LLVM_TOOLS_LLVM_EXEGESIS_FUNCTIONFILLER_H
+
+#include "Assembler.h"
+#include "BenchmarkResult.h"
+#include "LlvmState.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
+#include "llvm/Object/Binary.h"
+
+namespace llvm {
+namespace exegesis {
+
+class SnippetRepetitor {
+public:
+  static std::unique_ptr<const SnippetRepetitor>
+  Create(InstructionBenchmark::RepetitionModeE Mode, const LLVMState &State);
+
+  virtual ~SnippetRepetitor();
+
+  // Returns the set of registers that are reserved by the repetitor.
+  virtual BitVector getReservedRegs() const = 0;
+
+  // Returns a functor that repeats `Instructions` so that the function executes
+  // at least `MinInstructions` instructions.
+  virtual FillFunction Repeat(ArrayRef<MCInst> Instructions,
+                              unsigned MinInstructions) const = 0;
+
+  explicit SnippetRepetitor(const LLVMState &State) : State(State) {}
+
+protected:
+  const LLVMState &State;
+};
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h
index 0c5747808650a..4b0c9d17dd7fe 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.h
+++ b/llvm/tools/llvm-exegesis/lib/Target.h
@@ -85,11 +85,22 @@ class ExegesisTarget {
   // Fills memory operands with references to the address at [Reg] + Offset.
   virtual void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
                                   unsigned Offset) const {
-
     llvm_unreachable(
         "fillMemoryOperands() requires getScratchMemoryRegister() > 0");
   }
 
+  // Returns a counter usable as a loop counter.
+  virtual unsigned getLoopCounterRegister(const llvm::Triple &) const {
+    return 0;
+  }
+
+  // Adds the code to decrement the loop counter and
+  virtual void decrementLoopCounterAndLoop(MachineBasicBlock &MBB,
+                                           const llvm::MCInstrInfo &MII) const {
+    llvm_unreachable("decrementLoopCounterAndBranch() requires "
+                     "getLoopCounterRegister() > 0");
+  }
+
   // Returns a list of unavailable registers.
   // Targets can use this to prevent some registers to be automatically selected
   // for use in snippets.
diff --git a/llvm/tools/llvm-exegesis/lib/Uops.cpp b/llvm/tools/llvm-exegesis/lib/Uops.cpp
index 8fc8fd278d41d..6823ec8fd175c 100644
--- a/llvm/tools/llvm-exegesis/lib/Uops.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Uops.cpp
@@ -126,7 +126,7 @@ void UopsSnippetGenerator::instantiateMemoryOperands(
 static std::vector<InstructionTemplate> generateSnippetUsingStaticRenaming(
     const LLVMState &State, const InstructionTemplate &IT,
     const ArrayRef<const Variable *> TiedVariables,
-    const BitVector *ScratchSpaceAliasedRegs) {
+    const BitVector &ForbiddenRegisters) {
   std::vector<InstructionTemplate> Instructions;
   // Assign registers to variables in a round-robin manner. This is simple but
   // ensures that the most register-constrained variable does not get starved.
@@ -135,12 +135,8 @@ static std::vector<InstructionTemplate> generateSnippetUsingStaticRenaming(
     assert(Var);
     const Operand &Op = IT.Instr.getPrimaryOperand(*Var);
     assert(Op.isReg());
-    BitVector PossibleRegs = State.getRATC().emptyRegisters();
-    if (ScratchSpaceAliasedRegs) {
-      PossibleRegs |= *ScratchSpaceAliasedRegs;
-    }
-    PossibleRegs.flip();
-    PossibleRegs &= Op.getRegisterAliasing().sourceBits();
+    BitVector PossibleRegs = Op.getRegisterAliasing().sourceBits();
+    remove(PossibleRegs, ForbiddenRegisters);
     PossibleRegsForVar.push_back(std::move(PossibleRegs));
   }
   SmallVector<int, 2> Iterators(TiedVariables.size(), 0);
@@ -168,28 +164,14 @@ static std::vector<InstructionTemplate> generateSnippetUsingStaticRenaming(
 }
 
 llvm::Expected<std::vector<CodeTemplate>>
-UopsSnippetGenerator::generateCodeTemplates(const Instruction &Instr) const {
+UopsSnippetGenerator::generateCodeTemplates(
+    const Instruction &Instr, const BitVector &ForbiddenRegisters) const {
   CodeTemplate CT;
-  const llvm::BitVector *ScratchSpaceAliasedRegs = nullptr;
-  if (Instr.hasMemoryOperands()) {
-    const auto &ET = State.getExegesisTarget();
-    CT.ScratchSpacePointerInReg =
-        ET.getScratchMemoryRegister(State.getTargetMachine().getTargetTriple());
-    if (CT.ScratchSpacePointerInReg == 0)
-      return llvm::make_error<BenchmarkFailure>(
-          "Infeasible : target does not support memory instructions");
-    ScratchSpaceAliasedRegs =
-        &State.getRATC().getRegister(CT.ScratchSpacePointerInReg).aliasedBits();
-    // If the instruction implicitly writes to ScratchSpacePointerInReg , abort.
-    // FIXME: We could make a copy of the scratch register.
-    for (const auto &Op : Instr.Operands) {
-      if (Op.isDef() && Op.isImplicitReg() &&
-          ScratchSpaceAliasedRegs->test(Op.getImplicitReg()))
-        return llvm::make_error<BenchmarkFailure>(
-            "Infeasible : memory instruction uses scratch memory register");
-    }
-  }
-
+  CT.ScratchSpacePointerInReg =
+      Instr.hasMemoryOperands()
+          ? State.getExegesisTarget().getScratchMemoryRegister(
+                State.getTargetMachine().getTargetTriple())
+          : 0;
   const AliasingConfigurations SelfAliasing(Instr, Instr);
   InstructionTemplate IT(Instr);
   if (SelfAliasing.empty()) {
@@ -208,20 +190,17 @@ UopsSnippetGenerator::generateCodeTemplates(const Instruction &Instr) const {
   if (!TiedVariables.empty()) {
     CT.Info = "instruction has tied variables, using static renaming.";
     CT.Instructions = generateSnippetUsingStaticRenaming(
-        State, IT, TiedVariables, ScratchSpaceAliasedRegs);
+        State, IT, TiedVariables, ForbiddenRegisters);
     instantiateMemoryOperands(CT.ScratchSpacePointerInReg, CT.Instructions);
     return getSingleton(std::move(CT));
   }
-  const auto &ReservedRegisters = State.getRATC().reservedRegisters();
   // No tied variables, we pick random values for defs.
   llvm::BitVector Defs(State.getRegInfo().getNumRegs());
   for (const auto &Op : Instr.Operands) {
     if (Op.isReg() && Op.isExplicit() && Op.isDef() && !Op.isMemory()) {
       auto PossibleRegisters = Op.getRegisterAliasing().sourceBits();
-      remove(PossibleRegisters, ReservedRegisters);
-      // Do not use the scratch memory address register.
-      if (ScratchSpaceAliasedRegs)
-        remove(PossibleRegisters, *ScratchSpaceAliasedRegs);
+      // Do not use forbidden registers.
+      remove(PossibleRegisters, ForbiddenRegisters);
       assert(PossibleRegisters.any() && "No register left to choose from");
       const auto RandomReg = randomBit(PossibleRegisters);
       Defs.set(RandomReg);
@@ -233,10 +212,7 @@ UopsSnippetGenerator::generateCodeTemplates(const Instruction &Instr) const {
   for (const auto &Op : Instr.Operands) {
     if (Op.isReg() && Op.isExplicit() && Op.isUse() && !Op.isMemory()) {
       auto PossibleRegisters = Op.getRegisterAliasing().sourceBits();
-      remove(PossibleRegisters, ReservedRegisters);
-      // Do not use the scratch memory address register.
-      if (ScratchSpaceAliasedRegs)
-        remove(PossibleRegisters, *ScratchSpaceAliasedRegs);
+      remove(PossibleRegisters, ForbiddenRegisters);
       remove(PossibleRegisters, DefAliases);
       assert(PossibleRegisters.any() && "No register left to choose from");
       const auto RandomReg = randomBit(PossibleRegisters);
diff --git a/llvm/tools/llvm-exegesis/lib/Uops.h b/llvm/tools/llvm-exegesis/lib/Uops.h
index 077c9a85bd51a..23caff2665813 100644
--- a/llvm/tools/llvm-exegesis/lib/Uops.h
+++ b/llvm/tools/llvm-exegesis/lib/Uops.h
@@ -26,7 +26,8 @@ class UopsSnippetGenerator : public SnippetGenerator {
   ~UopsSnippetGenerator() override;
 
   llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const override;
+  generateCodeTemplates(const Instruction &Instr,
+                        const BitVector &ForbiddenRegisters) const override;
 
   static constexpr const size_t kMinNumDifferentAddresses = 6;
 
diff --git a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
index cebcb1c1aea41..bf008e8bbc7a5 100644
--- a/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/X86/Target.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 #include "../Target.h"
 
+#include "../Error.h"
 #include "../Latency.h"
 #include "../SnippetGenerator.h"
 #include "../Uops.h"
@@ -112,9 +113,11 @@ static Error isInvalidMemoryInstr(const Instruction &Instr) {
   case X86II::RawFrmImm8:
     return Error::success();
   case X86II::AddRegFrm:
-    return (Instr.Description->Opcode == X86::POP16r || Instr.Description->Opcode == X86::POP32r ||
-            Instr.Description->Opcode == X86::PUSH16r || Instr.Description->Opcode == X86::PUSH32r)
-               ? make_error<BenchmarkFailure>(
+    return (Instr.Description->Opcode == X86::POP16r ||
+            Instr.Description->Opcode == X86::POP32r ||
+            Instr.Description->Opcode == X86::PUSH16r ||
+            Instr.Description->Opcode == X86::PUSH32r)
+               ? make_error<Failure>(
                      "unsupported opcode: unsupported memory access")
                : Error::success();
   // These access memory and are handled.
@@ -140,19 +143,17 @@ static Error isInvalidMemoryInstr(const Instruction &Instr) {
   case X86II::RawFrmSrc:
   case X86II::RawFrmDst:
   case X86II::RawFrmDstSrc:
-    return make_error<BenchmarkFailure>(
-        "unsupported opcode: non uniform memory access");
+    return make_error<Failure>("unsupported opcode: non uniform memory access");
   }
 }
 
 static llvm::Error IsInvalidOpcode(const Instruction &Instr) {
   const auto OpcodeName = Instr.Name;
   if ((Instr.Description->TSFlags & X86II::FormMask) == X86II::Pseudo)
-    return llvm::make_error<BenchmarkFailure>(
-        "unsupported opcode: pseudo instruction");
+    return llvm::make_error<Failure>("unsupported opcode: pseudo instruction");
   if (OpcodeName.startswith("POPF") || OpcodeName.startswith("PUSHF") ||
       OpcodeName.startswith("ADJCALLSTACK"))
-    return llvm::make_error<BenchmarkFailure>(
+    return llvm::make_error<Failure>(
         "unsupported opcode: Push/Pop/AdjCallStack");
   if (llvm::Error Error = isInvalidMemoryInstr(Instr))
     return Error;
@@ -160,14 +161,14 @@ static llvm::Error IsInvalidOpcode(const Instruction &Instr) {
   for (const Operand &Op : Instr.Operands)
     if (Op.isExplicit() &&
         Op.getExplicitOperandInfo().OperandType == llvm::MCOI::OPERAND_PCREL)
-      return llvm::make_error<BenchmarkFailure>(
+      return llvm::make_error<Failure>(
           "unsupported opcode: PC relative operand");
   // We do not handle second-form X87 instructions. We only handle first-form
   // ones (_Fp), see comment in X86InstrFPStack.td.
   for (const Operand &Op : Instr.Operands)
     if (Op.isReg() && Op.isExplicit() &&
         Op.getExplicitOperandInfo().RegClass == llvm::X86::RSTRegClassID)
-      return llvm::make_error<BenchmarkFailure>(
+      return llvm::make_error<Failure>(
           "unsupported second-form X87 instruction");
   return llvm::Error::success();
 }
@@ -182,25 +183,27 @@ class X86LatencySnippetGenerator : public LatencySnippetGenerator {
   using LatencySnippetGenerator::LatencySnippetGenerator;
 
   llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const override;
+  generateCodeTemplates(const Instruction &Instr,
+                        const BitVector &ForbiddenRegisters) const override;
 };
 } // namespace
 
 llvm::Expected<std::vector<CodeTemplate>>
 X86LatencySnippetGenerator::generateCodeTemplates(
-    const Instruction &Instr) const {
+    const Instruction &Instr, const BitVector &ForbiddenRegisters) const {
   if (auto E = IsInvalidOpcode(Instr))
     return std::move(E);
 
   switch (getX86FPFlags(Instr)) {
   case llvm::X86II::NotFP:
-    return LatencySnippetGenerator::generateCodeTemplates(Instr);
+    return LatencySnippetGenerator::generateCodeTemplates(Instr,
+                                                          ForbiddenRegisters);
   case llvm::X86II::ZeroArgFP:
   case llvm::X86II::OneArgFP:
   case llvm::X86II::SpecialFP:
   case llvm::X86II::CompareFP:
   case llvm::X86II::CondMovFP:
-    return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
+    return llvm::make_error<Failure>("Unsupported x87 Instruction");
   case llvm::X86II::OneArgFPRW:
   case llvm::X86II::TwoArgFP:
     // These are instructions like
@@ -219,23 +222,25 @@ class X86UopsSnippetGenerator : public UopsSnippetGenerator {
   using UopsSnippetGenerator::UopsSnippetGenerator;
 
   llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const override;
+  generateCodeTemplates(const Instruction &Instr,
+                        const BitVector &ForbiddenRegisters) const override;
 };
 } // namespace
 
 llvm::Expected<std::vector<CodeTemplate>>
 X86UopsSnippetGenerator::generateCodeTemplates(
-    const Instruction &Instr) const {
+    const Instruction &Instr, const BitVector &ForbiddenRegisters) const {
   if (auto E = IsInvalidOpcode(Instr))
     return std::move(E);
 
   switch (getX86FPFlags(Instr)) {
   case llvm::X86II::NotFP:
-    return UopsSnippetGenerator::generateCodeTemplates(Instr);
+    return UopsSnippetGenerator::generateCodeTemplates(Instr,
+                                                       ForbiddenRegisters);
   case llvm::X86II::ZeroArgFP:
   case llvm::X86II::OneArgFP:
   case llvm::X86II::SpecialFP:
-    return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
+    return llvm::make_error<Failure>("Unsupported x87 Instruction");
   case llvm::X86II::OneArgFPRW:
   case llvm::X86II::TwoArgFP:
     // These are instructions like
@@ -432,6 +437,8 @@ class ExegesisX86Target : public ExegesisTarget {
 
   unsigned getScratchMemoryRegister(const llvm::Triple &TT) const override;
 
+  unsigned getLoopCounterRegister(const llvm::Triple &) const override;
+
   unsigned getMaxMemoryAccessSize() const override { return 64; }
 
   void randomizeMCOperand(const Instruction &Instr, const Variable &Var,
@@ -441,6 +448,9 @@ class ExegesisX86Target : public ExegesisTarget {
   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
                           unsigned Offset) const override;
 
+  void decrementLoopCounterAndLoop(MachineBasicBlock &MBB,
+                                   const llvm::MCInstrInfo &MII) const override;
+
   std::vector<llvm::MCInst> setRegTo(const llvm::MCSubtargetInfo &STI,
                                      unsigned Reg,
                                      const llvm::APInt &Value) const override;
@@ -472,6 +482,12 @@ class ExegesisX86Target : public ExegesisTarget {
 // prefix.
 const unsigned ExegesisX86Target::kUnavailableRegisters[4] = {X86::AH, X86::BH,
                                                               X86::CH, X86::DH};
+
+// We're using one of R8-R15 because these registers are never hardcoded in
+// instructions (e.g. MOVS writes to EDI, ESI, EDX), so they have less
+// conflicts.
+constexpr const unsigned kLoopCounterReg = X86::R8;
+
 } // namespace
 
 void ExegesisX86Target::addTargetSpecificPasses(
@@ -490,6 +506,14 @@ ExegesisX86Target::getScratchMemoryRegister(const llvm::Triple &TT) const {
   return TT.isOSWindows() ? llvm::X86::RCX : llvm::X86::RDI;
 }
 
+unsigned
+ExegesisX86Target::getLoopCounterRegister(const llvm::Triple &TT) const {
+  if (!TT.isArch64Bit()) {
+    return 0;
+  }
+  return kLoopCounterReg;
+}
+
 void ExegesisX86Target::randomizeMCOperand(
     const Instruction &Instr, const Variable &Var,
     llvm::MCOperand &AssignedValue,
@@ -534,6 +558,17 @@ void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
   SetOp(MemOpIdx + 4, MCOperand::createReg(0));      // Segment
 }
 
+void ExegesisX86Target::decrementLoopCounterAndLoop(
+    MachineBasicBlock &MBB, const llvm::MCInstrInfo &MII) const {
+  BuildMI(&MBB, DebugLoc(), MII.get(X86::ADD64ri8))
+      .addDef(kLoopCounterReg)
+      .addUse(kLoopCounterReg)
+      .addImm(-1);
+  BuildMI(&MBB, DebugLoc(), MII.get(X86::JCC_1))
+      .addMBB(&MBB)
+      .addImm(X86::COND_NE);
+}
+
 std::vector<llvm::MCInst>
 ExegesisX86Target::setRegTo(const llvm::MCSubtargetInfo &STI, unsigned Reg,
                             const llvm::APInt &Value) const {
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 91bb30703a9ae..616b7c96232a7 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -15,8 +15,11 @@
 #include "lib/BenchmarkResult.h"
 #include "lib/BenchmarkRunner.h"
 #include "lib/Clustering.h"
+#include "lib/Error.h"
 #include "lib/LlvmState.h"
 #include "lib/PerfHelper.h"
+#include "lib/SnippetFile.h"
+#include "lib/SnippetRepetitor.h"
 #include "lib/Target.h"
 #include "lib/TargetSelect.h"
 #include "llvm/ADT/StringExtras.h"
@@ -26,7 +29,6 @@
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCTargetAsmParser.h"
 #include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/CommandLine.h"
@@ -80,6 +82,14 @@ static cl::opt<exegesis::InstructionBenchmark::ModeE> BenchmarkMode(
                clEnumValN(exegesis::InstructionBenchmark::Unknown, "analysis",
                           "Analysis")));
 
+static cl::opt<exegesis::InstructionBenchmark::RepetitionModeE> RepetitionMode(
+    "repetition-mode", cl::desc("how to repeat the instruction snippet"),
+    cl::cat(BenchmarkOptions),
+    cl::values(clEnumValN(exegesis::InstructionBenchmark::Duplicate,
+                          "duplicate", "Duplicate the snippet"),
+               clEnumValN(exegesis::InstructionBenchmark::Loop, "loop",
+                          "Loop over the snippet")));
+
 static cl::opt<unsigned>
     NumRepetitions("num-repetitions",
                    cl::desc("number of time to repeat the asm snippet"),
@@ -192,167 +202,23 @@ getOpcodesOrDie(const llvm::MCInstrInfo &MCInstrInfo) {
 
 // Generates code snippets for opcode `Opcode`.
 static llvm::Expected<std::vector<BenchmarkCode>>
-generateSnippets(const LLVMState &State, unsigned Opcode) {
+generateSnippets(const LLVMState &State, unsigned Opcode,
+                 const llvm::BitVector &ForbiddenRegs) {
   const Instruction &Instr = State.getIC().getInstr(Opcode);
   const llvm::MCInstrDesc &InstrDesc = *Instr.Description;
   // Ignore instructions that we cannot run.
   if (InstrDesc.isPseudo())
-    return llvm::make_error<BenchmarkFailure>("Unsupported opcode: isPseudo");
+    return make_error<Failure>("Unsupported opcode: isPseudo");
   if (InstrDesc.isBranch() || InstrDesc.isIndirectBranch())
-    return llvm::make_error<BenchmarkFailure>(
-        "Unsupported opcode: isBranch/isIndirectBranch");
+    return make_error<Failure>("Unsupported opcode: isBranch/isIndirectBranch");
   if (InstrDesc.isCall() || InstrDesc.isReturn())
-    return llvm::make_error<BenchmarkFailure>(
-        "Unsupported opcode: isCall/isReturn");
+    return make_error<Failure>("Unsupported opcode: isCall/isReturn");
 
   const std::unique_ptr<SnippetGenerator> Generator =
       State.getExegesisTarget().createSnippetGenerator(BenchmarkMode, State);
   if (!Generator)
     llvm::report_fatal_error("cannot create snippet generator");
-  return Generator->generateConfigurations(Instr);
-}
-
-namespace {
-
-// An MCStreamer that reads a BenchmarkCode definition from a file.
-// The BenchmarkCode definition is just an asm file, with additional comments to
-// specify which registers should be defined or are live on entry.
-class BenchmarkCodeStreamer : public llvm::MCStreamer,
-                              public llvm::AsmCommentConsumer {
-public:
-  explicit BenchmarkCodeStreamer(llvm::MCContext *Context,
-                                 const llvm::MCRegisterInfo *TheRegInfo,
-                                 BenchmarkCode *Result)
-      : llvm::MCStreamer(*Context), RegInfo(TheRegInfo), Result(Result) {}
-
-  // Implementation of the llvm::MCStreamer interface. We only care about
-  // instructions.
-  void EmitInstruction(const llvm::MCInst &Instruction,
-                       const llvm::MCSubtargetInfo &STI) override {
-    Result->Instructions.push_back(Instruction);
-  }
-
-  // Implementation of the llvm::AsmCommentConsumer.
-  void HandleComment(llvm::SMLoc Loc, llvm::StringRef CommentText) override {
-    CommentText = CommentText.trim();
-    if (!CommentText.consume_front("LLVM-EXEGESIS-"))
-      return;
-    if (CommentText.consume_front("DEFREG")) {
-      // LLVM-EXEGESIS-DEFREF <reg> <hex_value>
-      RegisterValue RegVal;
-      llvm::SmallVector<llvm::StringRef, 2> Parts;
-      CommentText.split(Parts, ' ', /*unlimited splits*/ -1,
-                        /*do not keep empty strings*/ false);
-      if (Parts.size() != 2) {
-        llvm::errs() << "invalid comment 'LLVM-EXEGESIS-DEFREG " << CommentText
-                     << "\n";
-        ++InvalidComments;
-      }
-      if (!(RegVal.Register = findRegisterByName(Parts[0].trim()))) {
-        llvm::errs() << "unknown register in 'LLVM-EXEGESIS-DEFREG "
-                     << CommentText << "\n";
-        ++InvalidComments;
-        return;
-      }
-      const llvm::StringRef HexValue = Parts[1].trim();
-      RegVal.Value = llvm::APInt(
-          /* each hex digit is 4 bits */ HexValue.size() * 4, HexValue, 16);
-      Result->RegisterInitialValues.push_back(std::move(RegVal));
-      return;
-    }
-    if (CommentText.consume_front("LIVEIN")) {
-      // LLVM-EXEGESIS-LIVEIN <reg>
-      if (unsigned Reg = findRegisterByName(CommentText.ltrim()))
-        Result->LiveIns.push_back(Reg);
-      else {
-        llvm::errs() << "unknown register in 'LLVM-EXEGESIS-LIVEIN "
-                     << CommentText << "\n";
-        ++InvalidComments;
-      }
-      return;
-    }
-  }
-
-  unsigned numInvalidComments() const { return InvalidComments; }
-
-private:
-  // We only care about instructions, we don't implement this part of the API.
-  void EmitCommonSymbol(llvm::MCSymbol *Symbol, uint64_t Size,
-                        unsigned ByteAlignment) override {}
-  bool EmitSymbolAttribute(llvm::MCSymbol *Symbol,
-                           llvm::MCSymbolAttr Attribute) override {
-    return false;
-  }
-  void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value,
-                            unsigned ValueSize,
-                            unsigned MaxBytesToEmit) override {}
-  void EmitZerofill(llvm::MCSection *Section, llvm::MCSymbol *Symbol,
-                    uint64_t Size, unsigned ByteAlignment,
-                    llvm::SMLoc Loc) override {}
-
-  unsigned findRegisterByName(const llvm::StringRef RegName) const {
-    // FIXME: Can we do better than this ?
-    for (unsigned I = 0, E = RegInfo->getNumRegs(); I < E; ++I) {
-      if (RegName == RegInfo->getName(I))
-        return I;
-    }
-    llvm::errs() << "'" << RegName
-                 << "' is not a valid register name for the target\n";
-    return 0;
-  }
-
-  const llvm::MCRegisterInfo *const RegInfo;
-  BenchmarkCode *const Result;
-  unsigned InvalidComments = 0;
-};
-
-} // namespace
-
-// Reads code snippets from file `Filename`.
-static llvm::Expected<std::vector<BenchmarkCode>>
-readSnippets(const LLVMState &State, llvm::StringRef Filename) {
-  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> BufferPtr =
-      llvm::MemoryBuffer::getFileOrSTDIN(Filename);
-  if (std::error_code EC = BufferPtr.getError()) {
-    return llvm::make_error<BenchmarkFailure>(
-        "cannot read snippet: " + Filename + ": " + EC.message());
-  }
-  llvm::SourceMgr SM;
-  SM.AddNewSourceBuffer(std::move(BufferPtr.get()), llvm::SMLoc());
-
-  BenchmarkCode Result;
-
-  llvm::MCObjectFileInfo ObjectFileInfo;
-  const llvm::TargetMachine &TM = State.getTargetMachine();
-  llvm::MCContext Context(TM.getMCAsmInfo(), TM.getMCRegisterInfo(),
-                          &ObjectFileInfo);
-  ObjectFileInfo.InitMCObjectFileInfo(TM.getTargetTriple(), /*PIC*/ false,
-                                      Context);
-  BenchmarkCodeStreamer Streamer(&Context, TM.getMCRegisterInfo(), &Result);
-  const std::unique_ptr<llvm::MCAsmParser> AsmParser(
-      llvm::createMCAsmParser(SM, Context, Streamer, *TM.getMCAsmInfo()));
-  if (!AsmParser)
-    return llvm::make_error<BenchmarkFailure>("cannot create asm parser");
-  AsmParser->getLexer().setCommentConsumer(&Streamer);
-
-  const std::unique_ptr<llvm::MCTargetAsmParser> TargetAsmParser(
-      TM.getTarget().createMCAsmParser(*TM.getMCSubtargetInfo(), *AsmParser,
-                                       *TM.getMCInstrInfo(),
-                                       llvm::MCTargetOptions()));
-
-  if (!TargetAsmParser)
-    return llvm::make_error<BenchmarkFailure>(
-        "cannot create target asm parser");
-  AsmParser->setTargetParser(*TargetAsmParser);
-
-  if (AsmParser->Run(false))
-    return llvm::make_error<BenchmarkFailure>("cannot parse asm file");
-  if (Streamer.numInvalidComments())
-    return llvm::make_error<BenchmarkFailure>(
-        llvm::Twine("found ")
-            .concat(llvm::Twine(Streamer.numInvalidComments()))
-            .concat(" invalid LLVM-EXEGESIS comments"));
-  return std::vector<BenchmarkCode>{std::move(Result)};
+  return Generator->generateConfigurations(Instr, ForbiddenRegs);
 }
 
 void benchmarkMain() {
@@ -372,6 +238,8 @@ void benchmarkMain() {
   const LLVMState State(CpuName);
   const auto Opcodes = getOpcodesOrDie(State.getInstrInfo());
 
+  const auto Repetitor = SnippetRepetitor::Create(RepetitionMode, State);
+
   std::vector<BenchmarkCode> Configurations;
   if (!Opcodes.empty()) {
     for (const unsigned Opcode : Opcodes) {
@@ -383,7 +251,8 @@ void benchmarkMain() {
                      << ": ignoring instruction without sched class\n";
         continue;
       }
-      auto ConfigsForInstr = generateSnippets(State, Opcode);
+      auto ConfigsForInstr =
+          generateSnippets(State, Opcode, Repetitor->getReservedRegs());
       if (!ConfigsForInstr) {
         llvm::logAllUnhandledErrors(
             ConfigsForInstr.takeError(), llvm::errs(),
@@ -411,8 +280,8 @@ void benchmarkMain() {
     BenchmarkFile = "-";
 
   for (const BenchmarkCode &Conf : Configurations) {
-    InstructionBenchmark Result =
-        Runner->runConfiguration(Conf, NumRepetitions, DumpObjectToDisk);
+    InstructionBenchmark Result = Runner->runConfiguration(
+        Conf, NumRepetitions, *Repetitor, DumpObjectToDisk);
     ExitOnErr(Result.writeYaml(State, BenchmarkFile));
   }
   exegesis::pfm::pfmTerminate();
diff --git a/llvm/tools/llvm-ifs/llvm-ifs.cpp b/llvm/tools/llvm-ifs/llvm-ifs.cpp
index 9d6d8d9ec4c04..f329b46336321 100644
--- a/llvm/tools/llvm-ifs/llvm-ifs.cpp
+++ b/llvm/tools/llvm-ifs/llvm-ifs.cpp
@@ -220,21 +220,6 @@ static Expected<std::unique_ptr<IFSStub>> readInputFile(StringRef FilePath) {
 
 int writeTbdStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
                  const StringRef Format, raw_ostream &Out) {
-  auto ArchOrError =
-      [](const llvm::Triple &T) -> llvm::Expected<llvm::MachO::Architecture> {
-    switch (T.getArch()) {
-    default:
-      return createStringError(errc::not_supported, "Invalid Architecture.\n");
-    case llvm::Triple::ArchType::x86:
-      return AK_i386;
-    case llvm::Triple::ArchType::x86_64:
-      return AK_x86_64;
-    case llvm::Triple::ArchType::arm:
-      return AK_armv7;
-    case llvm::Triple::ArchType::aarch64:
-      return AK_arm64;
-    }
-  }(T);
 
   auto PlatformKindOrError =
       [](const llvm::Triple &T) -> llvm::Expected<llvm::MachO::PlatformKind> {
@@ -256,19 +241,15 @@ int writeTbdStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
     return createStringError(errc::not_supported, "Invalid Platform.\n");
   }(T);
 
-  if (!ArchOrError)
-    return -1;
-
   if (!PlatformKindOrError)
     return -1;
 
-  Architecture Arch = ArchOrError.get();
   PlatformKind Plat = PlatformKindOrError.get();
+  TargetList Targets({Target(llvm::MachO::mapToArchitecture(T), Plat)});
 
   InterfaceFile File;
   File.setFileType(FileType::TBD_V3); // Only supporting v3 for now.
-  File.setArchitectures(Arch);
-  File.setPlatform(Plat);
+  File.addTargets(Targets);
 
   for (const auto &Symbol : Symbols) {
     auto Name = Symbol.Name;
@@ -286,9 +267,9 @@ int writeTbdStub(const llvm::Triple &T, const std::set<IFSSymbol> &Symbols,
       break;
     }
     if (Symbol.Weak)
-      File.addSymbol(Kind, Name, Arch, SymbolFlags::WeakDefined);
+      File.addSymbol(Kind, Name, Targets, SymbolFlags::WeakDefined);
     else
-      File.addSymbol(Kind, Name, Arch);
+      File.addSymbol(Kind, Name, Targets);
   }
 
   SmallString<4096> Buffer;
diff --git a/llvm/tools/llvm-lipo/LipoOpts.td b/llvm/tools/llvm-lipo/LipoOpts.td
index e2f6f05106c87..344e407ca2396 100644
--- a/llvm/tools/llvm-lipo/LipoOpts.td
+++ b/llvm/tools/llvm-lipo/LipoOpts.td
@@ -12,6 +12,10 @@ def segalign
                "architecture when creating a universal binary file. The "
                "alignment is a hexadecimal number that is a power of 2.">;
 
+def arch
+    : MultiArg<["-", "--"], "arch", 2>,
+      HelpText<"Specifies the architecture and the corresponding input file">;
+
 def action_group : OptionGroup<"action group">;
 
 def verify_arch
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
index da5889f37d53e..5eb3332c02e05 100644
--- a/llvm/tools/llvm-lipo/llvm-lipo.cpp
+++ b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -86,27 +86,171 @@ enum class LipoAction {
   ReplaceArch,
 };
 
-struct Replacement {
-  StringRef ArchType;
+struct InputFile {
+  Optional<StringRef> ArchType;
   StringRef FileName;
 };
 
 struct Config {
-  SmallVector<std::string, 1> InputFiles;
+  SmallVector<InputFile, 1> InputFiles;
   SmallVector<std::string, 1> VerifyArchList;
-  SmallVector<Replacement, 1> Replacements;
+  SmallVector<InputFile, 1> ReplacementFiles;
   StringMap<const uint32_t> SegmentAlignments;
   std::string ThinArchType;
   std::string OutputFile;
   LipoAction ActionToPerform;
 };
 
-struct Slice {
-  const MachOObjectFile *ObjectFile;
-  // Requires Alignment field to store slice alignment values from universal
-  // binaries. Also needed to order the slices using compareSlices, so the total
+// For compatibility with cctools lipo, a file's alignment is calculated as the
+// minimum aligment of all segments. For object files, the file's alignment is
+// the maximum alignment of its sections.
+static uint32_t calculateFileAlignment(const MachOObjectFile &O) {
+  uint32_t P2CurrentAlignment;
+  uint32_t P2MinAlignment = MachOUniversalBinary::MaxSectionAlignment;
+  const bool Is64Bit = O.is64Bit();
+
+  for (const auto &LC : O.load_commands()) {
+    if (LC.C.cmd != (Is64Bit ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT))
+      continue;
+    if (O.getHeader().filetype == MachO::MH_OBJECT) {
+      unsigned NumberOfSections =
+          (Is64Bit ? O.getSegment64LoadCommand(LC).nsects
+                   : O.getSegmentLoadCommand(LC).nsects);
+      P2CurrentAlignment = NumberOfSections ? 2 : P2MinAlignment;
+      for (unsigned SI = 0; SI < NumberOfSections; ++SI) {
+        P2CurrentAlignment = std::max(P2CurrentAlignment,
+                                      (Is64Bit ? O.getSection64(LC, SI).align
+                                               : O.getSection(LC, SI).align));
+      }
+    } else {
+      P2CurrentAlignment =
+          countTrailingZeros(Is64Bit ? O.getSegment64LoadCommand(LC).vmaddr
+                                     : O.getSegmentLoadCommand(LC).vmaddr);
+    }
+    P2MinAlignment = std::min(P2MinAlignment, P2CurrentAlignment);
+  }
+  // return a value >= 4 byte aligned, and less than MachO MaxSectionAlignment
+  return std::max(
+      static_cast<uint32_t>(2),
+      std::min(P2MinAlignment, static_cast<uint32_t>(
+                                   MachOUniversalBinary::MaxSectionAlignment)));
+}
+
+static uint32_t calculateAlignment(const MachOObjectFile *ObjectFile) {
+  switch (ObjectFile->getHeader().cputype) {
+  case MachO::CPU_TYPE_I386:
+  case MachO::CPU_TYPE_X86_64:
+  case MachO::CPU_TYPE_POWERPC:
+  case MachO::CPU_TYPE_POWERPC64:
+    return 12; // log2 value of page size(4k) for x86 and PPC
+  case MachO::CPU_TYPE_ARM:
+  case MachO::CPU_TYPE_ARM64:
+  case MachO::CPU_TYPE_ARM64_32:
+    return 14; // log2 value of page size(16k) for Darwin ARM
+  default:
+    return calculateFileAlignment(*ObjectFile);
+  }
+}
+
+class Slice {
+  const Binary *B;
+  uint32_t CPUType;
+  uint32_t CPUSubType;
+  std::string ArchName;
+
+  // P2Alignment field stores slice alignment values from universal
+  // binaries. This is also needed to order the slices so the total
   // file size can be calculated before creating the output buffer.
-  uint32_t Alignment;
+  uint32_t P2Alignment;
+
+public:
+  Slice(const MachOObjectFile *O, uint32_t Align)
+      : B(O), CPUType(O->getHeader().cputype),
+        CPUSubType(O->getHeader().cpusubtype),
+        ArchName(O->getArchTriple().getArchName()), P2Alignment(Align) {}
+
+  explicit Slice(const MachOObjectFile *O) : Slice(O, calculateAlignment(O)){};
+
+  explicit Slice(const Archive *A) : B(A) {
+    Error Err = Error::success();
+    std::unique_ptr<MachOObjectFile> FO = nullptr;
+    for (const Archive::Child &Child : A->children(Err)) {
+      Expected<std::unique_ptr<Binary>> ChildOrErr = Child.getAsBinary();
+      if (!ChildOrErr)
+        reportError(A->getFileName(), ChildOrErr.takeError());
+      Binary *Bin = ChildOrErr.get().get();
+      if (Bin->isMachOUniversalBinary())
+        reportError(("archive member " + Bin->getFileName() +
+                     " is a fat file (not allowed in an archive)")
+                        .str());
+      if (!Bin->isMachO())
+        reportError(("archive member " + Bin->getFileName() +
+                     " is not a MachO file (not allowed in an archive)"));
+      MachOObjectFile *O = cast<MachOObjectFile>(Bin);
+      if (FO &&
+          std::tie(FO->getHeader().cputype, FO->getHeader().cpusubtype) !=
+              std::tie(O->getHeader().cputype, O->getHeader().cpusubtype)) {
+        reportError(("archive member " + O->getFileName() + " cputype (" +
+                     Twine(O->getHeader().cputype) + ") and cpusubtype(" +
+                     Twine(O->getHeader().cpusubtype) +
+                     ") does not match previous archive members cputype (" +
+                     Twine(FO->getHeader().cputype) + ") and cpusubtype(" +
+                     Twine(FO->getHeader().cpusubtype) +
+                     ") (all members must match) " + FO->getFileName())
+                        .str());
+      }
+      if (!FO) {
+        ChildOrErr.get().release();
+        FO.reset(O);
+      }
+    }
+    if (Err)
+      reportError(A->getFileName(), std::move(Err));
+    if (!FO)
+      reportError(("empty archive with no architecture specification: " +
+                   A->getFileName() + " (can't determine architecture for it)")
+                      .str());
+    CPUType = FO->getHeader().cputype;
+    CPUSubType = FO->getHeader().cpusubtype;
+    ArchName = FO->getArchTriple().getArchName();
+    // Replicate the behavior of cctools lipo.
+    P2Alignment = FO->is64Bit() ? 3 : 2;
+  }
+
+  void setP2Alignment(uint32_t Align) { P2Alignment = Align; }
+
+  const Binary *getBinary() const { return B; }
+
+  uint32_t getCPUType() const { return CPUType; }
+
+  uint32_t getCPUSubType() const { return CPUSubType; }
+
+  uint32_t getP2Alignment() const { return P2Alignment; }
+
+  uint64_t getCPUID() const {
+    return static_cast<uint64_t>(CPUType) << 32 | CPUSubType;
+  }
+
+  std::string getArchString() const {
+    if (!ArchName.empty())
+      return ArchName;
+    return ("unknown(" + Twine(CPUType) + "," +
+            Twine(CPUSubType & ~MachO::CPU_SUBTYPE_MASK) + ")")
+        .str();
+  }
+
+  friend bool operator<(const Slice &Lhs, const Slice &Rhs) {
+    if (Lhs.CPUType == Rhs.CPUType)
+      return Lhs.CPUSubType < Rhs.CPUSubType;
+    // force arm64-family to follow after all other slices for
+    // compatibility with cctools lipo
+    if (Lhs.CPUType == MachO::CPU_TYPE_ARM64)
+      return false;
+    if (Rhs.CPUType == MachO::CPU_TYPE_ARM64)
+      return true;
+    // Sort by alignment to minimize file size
+    return Lhs.P2Alignment < Rhs.P2Alignment;
+  }
 };
 
 } // end namespace
@@ -157,7 +301,15 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
     reportError("unknown argument '" + Arg->getAsString(InputArgs) + "'");
 
   for (auto Arg : InputArgs.filtered(LIPO_INPUT))
-    C.InputFiles.push_back(Arg->getValue());
+    C.InputFiles.push_back({None, Arg->getValue()});
+  for (auto Arg : InputArgs.filtered(LIPO_arch)) {
+    validateArchitectureName(Arg->getValue(0));
+    if (!Arg->getValue(1))
+      reportError(
+          "arch is missing an argument: expects -arch arch_type file_name");
+    C.InputFiles.push_back({StringRef(Arg->getValue(0)), Arg->getValue(1)});
+  }
+
   if (C.InputFiles.empty())
     reportError("at least one input file should be specified");
 
@@ -258,8 +410,9 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
         reportError(
             "replace is missing an argument: expects -replace arch_type "
             "file_name");
-      C.Replacements.push_back(
-          Replacement{Action->getValue(0), Action->getValue(1)});
+      validateArchitectureName(Action->getValue(0));
+      C.ReplacementFiles.push_back(
+          {StringRef(Action->getValue(0)), Action->getValue(1)});
     }
 
     if (C.OutputFile.empty())
@@ -275,19 +428,25 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
 }
 
 static SmallVector<OwningBinary<Binary>, 1>
-readInputBinaries(ArrayRef<std::string> InputFiles) {
+readInputBinaries(ArrayRef<InputFile> InputFiles) {
   SmallVector<OwningBinary<Binary>, 1> InputBinaries;
-  for (StringRef InputFile : InputFiles) {
-    Expected<OwningBinary<Binary>> BinaryOrErr = createBinary(InputFile);
+  for (const InputFile &IF : InputFiles) {
+    Expected<OwningBinary<Binary>> BinaryOrErr = createBinary(IF.FileName);
     if (!BinaryOrErr)
-      reportError(InputFile, BinaryOrErr.takeError());
-    // TODO: Add compatibility for archive files
-    if (BinaryOrErr->getBinary()->isArchive())
-      reportError("File " + InputFile +
-                  " is an archive file and is not yet supported.");
-    if (!BinaryOrErr->getBinary()->isMachO() &&
-        !BinaryOrErr->getBinary()->isMachOUniversalBinary())
-      reportError("File " + InputFile + " has unsupported binary format");
+      reportError(IF.FileName, BinaryOrErr.takeError());
+    const Binary *B = BinaryOrErr->getBinary();
+    if (!B->isArchive() && !B->isMachO() && !B->isMachOUniversalBinary())
+      reportError("File " + IF.FileName + " has unsupported binary format");
+    if (IF.ArchType && (B->isMachO() || B->isArchive())) {
+      const auto ArchType =
+          B->isMachO() ? Slice(cast<MachOObjectFile>(B)).getArchString()
+                       : Slice(cast<Archive>(B)).getArchString();
+      if (Triple(*IF.ArchType).getArch() != Triple(ArchType).getArch())
+        reportError("specified architecture: " + *IF.ArchType +
+                    " for file: " + B->getFileName() +
+                    " does not match the file's architecture (" + ArchType +
+                    ")");
+    }
     InputBinaries.push_back(std::move(*BinaryOrErr));
   }
   return InputBinaries;
@@ -306,7 +465,7 @@ static void verifyArch(ArrayRef<OwningBinary<Binary>> InputBinaries,
   if (auto UO =
           dyn_cast<MachOUniversalBinary>(InputBinaries.front().getBinary())) {
     for (StringRef Arch : VerifyArchList) {
-      Expected<std::unique_ptr<MachOObjectFile>> Obj =
+      Expected<MachOUniversalBinary::ObjectForArch> Obj =
           UO->getObjectForArch(Arch);
       if (!Obj)
         exit(EXIT_FAILURE);
@@ -323,34 +482,29 @@ static void verifyArch(ArrayRef<OwningBinary<Binary>> InputBinaries,
   exit(EXIT_SUCCESS);
 }
 
-// Returns a string of the given Object file's architecture type
-// Unknown architectures formatted unknown(CPUType,CPUSubType) for compatibility
-// with cctools lipo
-static std::string getArchString(const MachOObjectFile &ObjectFile) {
-  const Triple T = ObjectFile.getArchTriple();
-  const StringRef ObjectArch = T.getArchName();
-  if (!ObjectArch.empty())
-    return ObjectArch;
-  return ("unknown(" + Twine(ObjectFile.getHeader().cputype) + "," +
-          Twine(ObjectFile.getHeader().cpusubtype & ~MachO::CPU_SUBTYPE_MASK) +
-          ")")
-      .str();
-}
-
 static void printBinaryArchs(const Binary *Binary, raw_ostream &OS) {
   // Prints trailing space for compatibility with cctools lipo.
   if (auto UO = dyn_cast<MachOUniversalBinary>(Binary)) {
     for (const auto &O : UO->objects()) {
-      Expected<std::unique_ptr<MachOObjectFile>> BinaryOrError =
+      Expected<std::unique_ptr<MachOObjectFile>> MachOObjOrError =
           O.getAsObjectFile();
-      if (!BinaryOrError)
-        reportError(Binary->getFileName(), BinaryOrError.takeError());
-      OS << getArchString(*BinaryOrError.get().get()) << " ";
+      if (MachOObjOrError) {
+        OS << Slice(MachOObjOrError->get()).getArchString() << " ";
+        continue;
+      }
+      Expected<std::unique_ptr<Archive>> ArchiveOrError = O.getAsArchive();
+      if (ArchiveOrError) {
+        consumeError(MachOObjOrError.takeError());
+        OS << Slice(ArchiveOrError->get()).getArchString() << " ";
+        continue;
+      }
+      consumeError(ArchiveOrError.takeError());
+      reportError(Binary->getFileName(), MachOObjOrError.takeError());
     }
     OS << "\n";
     return;
   }
-  OS << getArchString(*cast<MachOObjectFile>(Binary)) << " \n";
+  OS << Slice(cast<MachOObjectFile>(Binary)).getArchString() << " \n";
 }
 
 LLVM_ATTRIBUTE_NORETURN
@@ -399,22 +553,24 @@ static void extractSlice(ArrayRef<OwningBinary<Binary>> InputBinaries,
 
   auto *UO = cast<MachOUniversalBinary>(InputBinaries.front().getBinary());
   Expected<std::unique_ptr<MachOObjectFile>> Obj =
-      UO->getObjectForArch(ThinArchType);
-  if (!Obj)
+      UO->getMachOObjectForArch(ThinArchType);
+  Expected<std::unique_ptr<Archive>> Ar = UO->getArchiveForArch(ThinArchType);
+  if (!Obj && !Ar)
     reportError("fat input file " + UO->getFileName() +
                 " does not contain the specified architecture " + ThinArchType +
                 " to thin it to");
-
+  Binary *B = Obj ? static_cast<Binary *>(Obj->get())
+                  : static_cast<Binary *>(Ar->get());
   Expected<std::unique_ptr<FileOutputBuffer>> OutFileOrError =
       FileOutputBuffer::create(OutputFileName,
-                               Obj.get()->getMemoryBufferRef().getBufferSize(),
+                               B->getMemoryBufferRef().getBufferSize(),
                                sys::fs::can_execute(UO->getFileName())
                                    ? FileOutputBuffer::F_executable
                                    : 0);
   if (!OutFileOrError)
     reportError(OutputFileName, OutFileOrError.takeError());
-  std::copy(Obj.get()->getMemoryBufferRef().getBufferStart(),
-            Obj.get()->getMemoryBufferRef().getBufferEnd(),
+  std::copy(B->getMemoryBufferRef().getBufferStart(),
+            B->getMemoryBufferRef().getBufferEnd(),
             OutFileOrError.get()->getBufferStart());
   if (Error E = OutFileOrError.get()->commit())
     reportError(OutputFileName, std::move(E));
@@ -422,98 +578,24 @@ static void extractSlice(ArrayRef<OwningBinary<Binary>> InputBinaries,
 }
 
 static void checkArchDuplicates(const ArrayRef<Slice> &Slices) {
-  DenseMap<uint64_t, const MachOObjectFile *> CPUIds;
-  auto CPUIDForSlice = [](const Slice &S) {
-    return static_cast<uint64_t>(S.ObjectFile->getHeader().cputype) << 32 |
-           S.ObjectFile->getHeader().cpusubtype;
-  };
+  DenseMap<uint64_t, const Binary *> CPUIds;
   for (const auto &S : Slices) {
-    auto Entry = CPUIds.try_emplace(CPUIDForSlice(S), S.ObjectFile);
+    auto Entry = CPUIds.try_emplace(S.getCPUID(), S.getBinary());
     if (!Entry.second)
       reportError(Entry.first->second->getFileName() + " and " +
-                  S.ObjectFile->getFileName() + " have the same architecture " +
-                  getArchString(*S.ObjectFile) +
+                  S.getBinary()->getFileName() +
+                  " have the same architecture " + S.getArchString() +
                   " and therefore cannot be in the same universal binary");
   }
 }
 
-// For compatibility with cctools lipo, alignment is calculated as the minimum
-// aligment of all segments. Each segments's alignment is the maximum alignment
-// from its sections
-static uint32_t calculateSegmentAlignment(const MachOObjectFile &O) {
-  uint32_t P2CurrentAlignment;
-  uint32_t P2MinAlignment = MachOUniversalBinary::MaxSectionAlignment;
-  const bool Is64Bit = O.is64Bit();
-
-  for (const auto &LC : O.load_commands()) {
-    if (LC.C.cmd != (Is64Bit ? MachO::LC_SEGMENT_64 : MachO::LC_SEGMENT))
-      continue;
-    if (O.getHeader().filetype == MachO::MH_OBJECT) {
-      unsigned NumberOfSections =
-          (Is64Bit ? O.getSegment64LoadCommand(LC).nsects
-                   : O.getSegmentLoadCommand(LC).nsects);
-      P2CurrentAlignment = NumberOfSections ? 2 : P2MinAlignment;
-      for (unsigned SI = 0; SI < NumberOfSections; ++SI) {
-        P2CurrentAlignment = std::max(P2CurrentAlignment,
-                                      (Is64Bit ? O.getSection64(LC, SI).align
-                                               : O.getSection(LC, SI).align));
-      }
-    } else {
-      P2CurrentAlignment =
-          countTrailingZeros(Is64Bit ? O.getSegment64LoadCommand(LC).vmaddr
-                                     : O.getSegmentLoadCommand(LC).vmaddr);
-    }
-    P2MinAlignment = std::min(P2MinAlignment, P2CurrentAlignment);
-  }
-  // return a value >= 4 byte aligned, and less than MachO MaxSectionAlignment
-  return std::max(
-      static_cast<uint32_t>(2),
-      std::min(P2MinAlignment, static_cast<uint32_t>(
-                                   MachOUniversalBinary::MaxSectionAlignment)));
-}
-
-static uint32_t calculateAlignment(const MachOObjectFile *ObjectFile) {
-  switch (ObjectFile->getHeader().cputype) {
-  case MachO::CPU_TYPE_I386:
-  case MachO::CPU_TYPE_X86_64:
-  case MachO::CPU_TYPE_POWERPC:
-  case MachO::CPU_TYPE_POWERPC64:
-    return 12; // log2 value of page size(4k) for x86 and PPC
-  case MachO::CPU_TYPE_ARM:
-  case MachO::CPU_TYPE_ARM64:
-  case MachO::CPU_TYPE_ARM64_32:
-    return 14; // log2 value of page size(16k) for Darwin ARM
-  default:
-    return calculateSegmentAlignment(*ObjectFile);
-  }
-}
-
-// This function replicates ordering from cctools lipo for consistency
-static bool compareSlices(const Slice &Lhs, const Slice &Rhs) {
-  if (Lhs.ObjectFile->getHeader().cputype ==
-      Rhs.ObjectFile->getHeader().cputype)
-    return Lhs.ObjectFile->getHeader().cpusubtype <
-           Rhs.ObjectFile->getHeader().cpusubtype;
-
-  // force arm64-family to follow after all other slices for compatibility
-  // with cctools lipo
-  if (Lhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64)
-    return false;
-  if (Rhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64)
-    return true;
-
-  // Sort by alignment to minimize file size
-  return Lhs.Alignment < Rhs.Alignment;
-}
-
 template <typename Range>
-static void
-updateSegmentAlignments(Range &Slices,
-                        const StringMap<const uint32_t> &Alignments) {
+static void updateAlignments(Range &Slices,
+                             const StringMap<const uint32_t> &Alignments) {
   for (auto &Slice : Slices) {
-    auto Alignment = Alignments.find(getArchString(*Slice.ObjectFile));
+    auto Alignment = Alignments.find(Slice.getArchString());
     if (Alignment != Alignments.end())
-      Slice.Alignment = Alignment->second;
+      Slice.setP2Alignment(Alignment->second);
   }
 }
 
@@ -521,7 +603,7 @@ static void checkUnusedAlignments(ArrayRef<Slice> Slices,
                                   const StringMap<const uint32_t> &Alignments) {
   auto HasArch = [&](StringRef Arch) {
     return llvm::find_if(Slices, [Arch](Slice S) {
-             return getArchString(*S.ObjectFile) == Arch;
+             return S.getArchString() == Arch;
            }) != Slices.end();
   };
   for (StringRef Arch : Alignments.keys())
@@ -547,15 +629,17 @@ static SmallVector<Slice, 2> buildSlices(
         if (!BinaryOrError)
           reportError(InputBinary->getFileName(), BinaryOrError.takeError());
         ExtractedObjects.push_back(std::move(BinaryOrError.get()));
-        Slices.push_back(Slice{ExtractedObjects.back().get(), O.getAlign()});
+        Slices.emplace_back(ExtractedObjects.back().get(), O.getAlign());
       }
     } else if (auto O = dyn_cast<MachOObjectFile>(InputBinary)) {
-      Slices.push_back(Slice{O, calculateAlignment(O)});
+      Slices.emplace_back(O);
+    } else if (auto A = dyn_cast<Archive>(InputBinary)) {
+      Slices.emplace_back(A);
     } else {
       llvm_unreachable("Unexpected binary format");
     }
   }
-  updateSegmentAlignments(Slices, Alignments);
+  updateAlignments(Slices, Alignments);
   return Slices;
 }
 
@@ -565,22 +649,20 @@ buildFatArchList(ArrayRef<Slice> Slices) {
   uint64_t Offset =
       sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
 
-  for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
-    Offset = alignTo(Offset, 1ull << Slices[Index].Alignment);
-    const MachOObjectFile *ObjectFile = Slices[Index].ObjectFile;
+  for (const auto &S : Slices) {
+    Offset = alignTo(Offset, 1ull << S.getP2Alignment());
     if (Offset > UINT32_MAX)
       reportError("fat file too large to be created because the offset "
                   "field in struct fat_arch is only 32-bits and the offset " +
-                  Twine(Offset) + " for " + ObjectFile->getFileName() +
-                  " for architecture " + getArchString(*ObjectFile) +
-                  "exceeds that.");
+                  Twine(Offset) + " for " + S.getBinary()->getFileName() +
+                  " for architecture " + S.getArchString() + "exceeds that.");
 
     MachO::fat_arch FatArch;
-    FatArch.cputype = ObjectFile->getHeader().cputype;
-    FatArch.cpusubtype = ObjectFile->getHeader().cpusubtype;
+    FatArch.cputype = S.getCPUType();
+    FatArch.cpusubtype = S.getCPUSubType();
     FatArch.offset = Offset;
-    FatArch.size = ObjectFile->getMemoryBufferRef().getBufferSize();
-    FatArch.align = Slices[Index].Alignment;
+    FatArch.size = S.getBinary()->getMemoryBufferRef().getBufferSize();
+    FatArch.align = S.getP2Alignment();
     Offset += FatArch.size;
     FatArchList.push_back(FatArch);
   }
@@ -593,11 +675,11 @@ static void createUniversalBinary(SmallVectorImpl<Slice> &Slices,
   FatHeader.magic = MachO::FAT_MAGIC;
   FatHeader.nfat_arch = Slices.size();
 
-  stable_sort(Slices, compareSlices);
+  stable_sort(Slices);
   SmallVector<MachO::fat_arch, 2> FatArchList = buildFatArchList(Slices);
 
   const bool IsExecutable = any_of(Slices, [](Slice S) {
-    return sys::fs::can_execute(S.ObjectFile->getFileName());
+    return sys::fs::can_execute(S.getBinary()->getFileName());
   });
   const uint64_t OutputFileSize =
       FatArchList.back().offset + FatArchList.back().size;
@@ -615,7 +697,7 @@ static void createUniversalBinary(SmallVectorImpl<Slice> &Slices,
   std::memcpy(OutFile->getBufferStart(), &FatHeader, sizeof(MachO::fat_header));
 
   for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
-    MemoryBufferRef BufferRef = Slices[Index].ObjectFile->getMemoryBufferRef();
+    MemoryBufferRef BufferRef = Slices[Index].getBinary()->getMemoryBufferRef();
     std::copy(BufferRef.getBufferStart(), BufferRef.getBufferEnd(),
               OutFile->getBufferStart() + FatArchList[Index].offset);
   }
@@ -652,40 +734,27 @@ static void createUniversalBinary(ArrayRef<OwningBinary<Binary>> InputBinaries,
 
 static StringMap<Slice>
 buildReplacementSlices(ArrayRef<OwningBinary<Binary>> ReplacementBinaries,
-                       const StringMap<const uint32_t> &Alignments,
-                       ArrayRef<Replacement> Replacements) {
-  assert(ReplacementBinaries.size() == Replacements.size() &&
-         "Number of replacment binaries does not match the number of "
-         "replacements");
+                       const StringMap<const uint32_t> &Alignments) {
   StringMap<Slice> Slices;
   // populates StringMap of slices to replace with; error checks for mismatched
   // replace flag args, fat files, and duplicate arch_types
-  for (size_t Index = 0, Size = Replacements.size(); Index < Size; ++Index) {
-    StringRef ReplacementArch = Replacements[Index].ArchType;
-    const Binary *ReplacementBinary = ReplacementBinaries[Index].getBinary();
-    validateArchitectureName(ReplacementArch);
-
+  for (const auto &OB : ReplacementBinaries) {
+    const Binary *ReplacementBinary = OB.getBinary();
     auto O = dyn_cast<MachOObjectFile>(ReplacementBinary);
     if (!O)
       reportError("replacement file: " + ReplacementBinary->getFileName() +
                   " is a fat file (must be a thin file)");
-
-    if (O->getArch() != Triple(ReplacementArch).getArch())
-      reportError("specified architecture: " + ReplacementArch +
-                  " for replacement file: " + ReplacementBinary->getFileName() +
-                  " does not match the file's architecture");
-
-    auto Entry =
-        Slices.try_emplace(ReplacementArch, Slice{O, calculateAlignment(O)});
+    Slice S(O);
+    auto Entry = Slices.try_emplace(S.getArchString(), S);
     if (!Entry.second)
-      reportError("-replace " + ReplacementArch +
+      reportError("-replace " + S.getArchString() +
                   " <file_name> specified multiple times: " +
-                  Entry.first->second.ObjectFile->getFileName() + ", " +
+                  Entry.first->second.getBinary()->getFileName() + ", " +
                   O->getFileName());
   }
   auto SlicesMapRange = map_range(
       Slices, [](StringMapEntry<Slice> &E) -> Slice & { return E.getValue(); });
-  updateSegmentAlignments(SlicesMapRange, Alignments);
+  updateAlignments(SlicesMapRange, Alignments);
   return Slices;
 }
 
@@ -693,7 +762,7 @@ LLVM_ATTRIBUTE_NORETURN
 static void replaceSlices(ArrayRef<OwningBinary<Binary>> InputBinaries,
                           const StringMap<const uint32_t> &Alignments,
                           StringRef OutputFileName,
-                          ArrayRef<Replacement> Replacements) {
+                          ArrayRef<InputFile> ReplacementFiles) {
   assert(InputBinaries.size() == 1 && "Incorrect number of input binaries");
   assert(!OutputFileName.empty() && "Replace expects a single output file");
 
@@ -702,20 +771,17 @@ static void replaceSlices(ArrayRef<OwningBinary<Binary>> InputBinaries,
                 InputBinaries.front().getBinary()->getFileName() +
                 " must be a fat file when the -replace option is specified");
 
-  SmallVector<std::string, 1> ReplacementFiles;
-  for (const auto &R : Replacements)
-    ReplacementFiles.push_back(R.FileName);
   SmallVector<OwningBinary<Binary>, 1> ReplacementBinaries =
       readInputBinaries(ReplacementFiles);
 
   StringMap<Slice> ReplacementSlices =
-      buildReplacementSlices(ReplacementBinaries, Alignments, Replacements);
+      buildReplacementSlices(ReplacementBinaries, Alignments);
   SmallVector<std::unique_ptr<MachOObjectFile>, 2> ExtractedObjects;
   SmallVector<Slice, 2> Slices =
       buildSlices(InputBinaries, Alignments, ExtractedObjects);
 
   for (auto &Slice : Slices) {
-    auto It = ReplacementSlices.find(getArchString(*Slice.ObjectFile));
+    auto It = ReplacementSlices.find(Slice.getArchString());
     if (It != ReplacementSlices.end()) {
       Slice = It->second;
       ReplacementSlices.erase(It); // only keep remaining replacing arch_types
@@ -757,7 +823,7 @@ int main(int argc, char **argv) {
     break;
   case LipoAction::ReplaceArch:
     replaceSlices(InputBinaries, C.SegmentAlignments, C.OutputFile,
-                  C.Replacements);
+                  C.ReplacementFiles);
     break;
   }
   return EXIT_SUCCESS;
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
index 560c6c6e8a334..feff0cd6d5240 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.cpp
@@ -165,10 +165,33 @@ void DependencyGraph::dumpDependencyEdge(raw_ostream &OS,
            "Unsupported dependency type!");
     OS << " - RESOURCE MASK: " << DE.ResourceOrRegID;
   }
-  OS << " - CYCLES: " << DE.Cost << '\n';
+  OS << " - COST: " << DE.Cost << '\n';
 }
 #endif // NDEBUG
 
+void DependencyGraph::pruneEdges(unsigned Iterations) {
+  for (DGNode &N : Nodes) {
+    unsigned NumPruned = 0;
+    const unsigned Size = N.OutgoingEdges.size();
+    // Use a cut-off threshold to prune edges with a low frequency.
+    for (unsigned I = 0, E = Size; I < E; ++I) {
+      DependencyEdge &Edge = N.OutgoingEdges[I];
+      if (Edge.Frequency == Iterations)
+        continue;
+      double Factor = (double)Edge.Frequency / Iterations;
+      if (0.10 < Factor)
+        continue;
+      Nodes[Edge.ToIID].NumPredecessors--;
+      std::swap(Edge, N.OutgoingEdges[E - 1]);
+      --E;
+      ++NumPruned;
+    }
+
+    if (NumPruned)
+      N.OutgoingEdges.resize(Size - NumPruned);
+  }
+}
+
 void DependencyGraph::initializeRootSet(
     SmallVectorImpl<unsigned> &RootSet) const {
   for (unsigned I = 0, E = Nodes.size(); I < E; ++I) {
@@ -179,7 +202,7 @@ void DependencyGraph::initializeRootSet(
 }
 
 void DependencyGraph::propagateThroughEdges(
-    SmallVectorImpl<unsigned> &RootSet) {
+    SmallVectorImpl<unsigned> &RootSet, unsigned Iterations) {
   SmallVector<unsigned, 8> ToVisit;
 
   // A critical sequence is computed as the longest path from a node of the
@@ -189,6 +212,10 @@ void DependencyGraph::propagateThroughEdges(
   // Each node of the graph starts with an initial default cost of zero.  The
   // cost of a node is a measure of criticality: the higher the cost, the bigger
   // is the performance impact.
+  // For register and memory dependencies, the cost is a function of the write
+  // latency as well as the actual delay (in cycles) caused to users.
+  // For processor resource dependencies, the cost is a function of the resource
+  // pressure. Resource interferences with low frequency values are ignored.
   //
   // This algorithm is very similar to a (reverse) Dijkstra.  Every iteration of
   // the inner loop selects (i.e. visits) a node N from a set of `unvisited
@@ -277,6 +304,10 @@ static void printInstruction(formatted_raw_ostream &FOS,
 }
 
 void BottleneckAnalysis::printCriticalSequence(raw_ostream &OS) const {
+  // Early exit if no bottlenecks were found during the simulation.
+  if (!SeenStallCycles || !BPI.PressureIncreaseCycles)
+    return;
+
   SmallVector<const DependencyEdge *, 16> Seq;
   DG.getCriticalSequence(Seq);
   if (Seq.empty())
@@ -432,7 +463,6 @@ void BottleneckAnalysis::addRegisterDep(unsigned From, unsigned To,
   bool IsLoopCarried = From >= To;
   unsigned SourceSize = Source.size();
   if (IsLoopCarried) {
-    Cost *= Iterations / 2;
     DG.addRegisterDep(From, To + SourceSize, RegID, Cost);
     DG.addRegisterDep(From + SourceSize, To + (SourceSize * 2), RegID, Cost);
     return;
@@ -445,7 +475,6 @@ void BottleneckAnalysis::addMemoryDep(unsigned From, unsigned To,
   bool IsLoopCarried = From >= To;
   unsigned SourceSize = Source.size();
   if (IsLoopCarried) {
-    Cost *= Iterations / 2;
     DG.addMemoryDep(From, To + SourceSize, Cost);
     DG.addMemoryDep(From + SourceSize, To + (SourceSize * 2), Cost);
     return;
@@ -458,7 +487,6 @@ void BottleneckAnalysis::addResourceDep(unsigned From, unsigned To,
   bool IsLoopCarried = From >= To;
   unsigned SourceSize = Source.size();
   if (IsLoopCarried) {
-    Cost *= Iterations / 2;
     DG.addResourceDep(From, To + SourceSize, Mask, Cost);
     DG.addResourceDep(From + SourceSize, To + (SourceSize * 2), Mask, Cost);
     return;
@@ -514,7 +542,7 @@ void BottleneckAnalysis::onEvent(const HWInstructionEvent &Event) {
 
   // Check if this is the last simulated instruction.
   if (IID == ((Iterations * Source.size()) - 1))
-    DG.finalizeGraph();
+    DG.finalizeGraph(Iterations);
 }
 
 void BottleneckAnalysis::onEvent(const HWPressureEvent &Event) {
diff --git a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
index 7564b1a482068..9e3bd5978f093 100644
--- a/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
+++ b/llvm/tools/llvm-mca/Views/BottleneckAnalysis.h
@@ -236,8 +236,9 @@ class DependencyGraph {
   void addDependency(unsigned From, unsigned To,
                      DependencyEdge::Dependency &&DE);
 
+  void pruneEdges(unsigned Iterations);
   void initializeRootSet(SmallVectorImpl<unsigned> &RootSet) const;
-  void propagateThroughEdges(SmallVectorImpl<unsigned> &RootSet);
+  void propagateThroughEdges(SmallVectorImpl<unsigned> &RootSet, unsigned Iterations);
 
 #ifndef NDEBUG
   void dumpDependencyEdge(raw_ostream &OS, const DependencyEdge &DE,
@@ -263,10 +264,11 @@ class DependencyGraph {
 
   // Called by the bottleneck analysis at the end of simulation to propagate
   // costs through the edges of the graph, and compute a critical path.
-  void finalizeGraph() {
+  void finalizeGraph(unsigned Iterations) {
     SmallVector<unsigned, 16> RootSet;
+    pruneEdges(Iterations);
     initializeRootSet(RootSet);
-    propagateThroughEdges(RootSet);
+    propagateThroughEdges(RootSet, Iterations);
   }
 
   // Returns a sequence of edges representing the critical sequence based on the
diff --git a/llvm/tools/llvm-mca/llvm-mca.cpp b/llvm/tools/llvm-mca/llvm-mca.cpp
index dde42c117300b..291ed540a8de8 100644
--- a/llvm/tools/llvm-mca/llvm-mca.cpp
+++ b/llvm/tools/llvm-mca/llvm-mca.cpp
@@ -89,6 +89,11 @@ static cl::opt<std::string>
          cl::desc("Target a specific cpu type (-mcpu=help for details)"),
          cl::value_desc("cpu-name"), cl::cat(ToolOptions), cl::init("native"));
 
+static cl::opt<std::string>
+    MATTR("mattr",
+          cl::desc("Additional target features."),
+          cl::cat(ToolOptions));
+
 static cl::opt<int>
     OutputAsmVariant("output-asm-variant",
                      cl::desc("Syntax variant to use for output printing"),
@@ -322,7 +327,7 @@ int main(int argc, char **argv) {
     MCPU = llvm::sys::getHostCPUName();
 
   std::unique_ptr<MCSubtargetInfo> STI(
-      TheTarget->createMCSubtargetInfo(TripleName, MCPU, /* FeaturesStr */ ""));
+      TheTarget->createMCSubtargetInfo(TripleName, MCPU, MATTR));
   if (!STI->isCPUStringValid(MCPU))
     return 1;
 
diff --git a/llvm/tools/llvm-objcopy/CMakeLists.txt b/llvm/tools/llvm-objcopy/CMakeLists.txt
index d281bf887f692..50c890b59769e 100644
--- a/llvm/tools/llvm-objcopy/CMakeLists.txt
+++ b/llvm/tools/llvm-objcopy/CMakeLists.txt
@@ -21,6 +21,7 @@ add_llvm_tool(llvm-objcopy
   COFF/Object.cpp
   COFF/Reader.cpp
   COFF/Writer.cpp
+  ELF/ELFConfig.cpp
   ELF/ELFObjcopy.cpp
   ELF/Object.cpp
   MachO/MachOObjcopy.cpp
diff --git a/llvm/tools/llvm-objcopy/COFF/COFFObjcopy.cpp b/llvm/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
index 28de0eb6efd32..60f70d71349aa 100644
--- a/llvm/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/COFF/COFFObjcopy.cpp
@@ -208,10 +208,11 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj) {
       !Config.SymbolsToGlobalize.empty() || !Config.SymbolsToKeep.empty() ||
       !Config.SymbolsToLocalize.empty() || !Config.SymbolsToWeaken.empty() ||
       !Config.SymbolsToKeepGlobal.empty() || !Config.SectionsToRename.empty() ||
-      !Config.SetSectionFlags.empty() || !Config.SymbolsToRename.empty() ||
-      Config.ExtractDWO || Config.KeepFileSymbols || Config.LocalizeHidden ||
-      Config.PreserveDates || Config.StripDWO || Config.StripNonAlloc ||
-      Config.StripSections || Config.Weaken || Config.DecompressDebugSections ||
+      !Config.SetSectionAlignment.empty() || !Config.SetSectionFlags.empty() ||
+      !Config.SymbolsToRename.empty() || Config.ExtractDWO ||
+      Config.KeepFileSymbols || Config.LocalizeHidden || Config.PreserveDates ||
+      Config.StripDWO || Config.StripNonAlloc || Config.StripSections ||
+      Config.Weaken || Config.DecompressDebugSections ||
       Config.DiscardMode == DiscardType::Locals ||
       !Config.SymbolsToAdd.empty() || Config.EntryExpr) {
     return createStringError(llvm::errc::invalid_argument,
diff --git a/llvm/tools/llvm-objcopy/CopyConfig.cpp b/llvm/tools/llvm-objcopy/CopyConfig.cpp
index b3a62a3a6d5e7..9b51b745d50e1 100644
--- a/llvm/tools/llvm-objcopy/CopyConfig.cpp
+++ b/llvm/tools/llvm-objcopy/CopyConfig.cpp
@@ -155,6 +155,25 @@ static Expected<SectionRename> parseRenameSectionValue(StringRef FlagValue) {
   return SR;
 }
 
+static Expected<std::pair<StringRef, uint64_t>>
+parseSetSectionAlignment(StringRef FlagValue) {
+  if (!FlagValue.contains('='))
+    return createStringError(
+        errc::invalid_argument,
+        "bad format for --set-section-alignment: missing '='");
+  auto Split = StringRef(FlagValue).split('=');
+  if (Split.first.empty())
+    return createStringError(
+        errc::invalid_argument,
+        "bad format for --set-section-alignment: missing section name");
+  uint64_t NewAlign;
+  if (Split.second.getAsInteger(0, NewAlign))
+    return createStringError(errc::invalid_argument,
+                             "invalid alignment for --set-section-alignment: '%s'",
+                             Split.second.str().c_str());
+  return std::make_pair(Split.first, NewAlign);
+}
+
 static Expected<SectionFlagsUpdate>
 parseSetSectionFlagValue(StringRef FlagValue) {
   if (!StringRef(FlagValue).contains('='))
@@ -177,110 +196,6 @@ parseSetSectionFlagValue(StringRef FlagValue) {
   return SFU;
 }
 
-static Expected<NewSymbolInfo> parseNewSymbolInfo(StringRef FlagValue,
-                                                  uint8_t DefaultVisibility) {
-  // Parse value given with --add-symbol option and create the
-  // new symbol if possible. The value format for --add-symbol is:
-  //
-  // <name>=[<section>:]<value>[,<flags>]
-  //
-  // where:
-  // <name> - symbol name, can be empty string
-  // <section> - optional section name. If not given ABS symbol is created
-  // <value> - symbol value, can be decimal or hexadecimal number prefixed
-  //           with 0x.
-  // <flags> - optional flags affecting symbol type, binding or visibility:
-  //           The following are currently supported:
-  //
-  //           global, local, weak, default, hidden, file, section, object,
-  //           indirect-function.
-  //
-  //           The following flags are ignored and provided for GNU
-  //           compatibility only:
-  //
-  //           warning, debug, constructor, indirect, synthetic,
-  //           unique-object, before=<symbol>.
-  NewSymbolInfo SI;
-  StringRef Value;
-  std::tie(SI.SymbolName, Value) = FlagValue.split('=');
-  if (Value.empty())
-    return createStringError(
-        errc::invalid_argument,
-        "bad format for --add-symbol, missing '=' after '%s'",
-        SI.SymbolName.str().c_str());
-
-  if (Value.contains(':')) {
-    std::tie(SI.SectionName, Value) = Value.split(':');
-    if (SI.SectionName.empty() || Value.empty())
-      return createStringError(
-          errc::invalid_argument,
-          "bad format for --add-symbol, missing section name or symbol value");
-  }
-
-  SmallVector<StringRef, 6> Flags;
-  Value.split(Flags, ',');
-  if (Flags[0].getAsInteger(0, SI.Value))
-    return createStringError(errc::invalid_argument, "bad symbol value: '%s'",
-                             Flags[0].str().c_str());
-
-  SI.Visibility = DefaultVisibility;
-
-  using Functor = std::function<void(void)>;
-  SmallVector<StringRef, 6> UnsupportedFlags;
-  for (size_t I = 1, NumFlags = Flags.size(); I < NumFlags; ++I)
-    static_cast<Functor>(
-        StringSwitch<Functor>(Flags[I])
-            .CaseLower("global", [&SI] { SI.Bind = ELF::STB_GLOBAL; })
-            .CaseLower("local", [&SI] { SI.Bind = ELF::STB_LOCAL; })
-            .CaseLower("weak", [&SI] { SI.Bind = ELF::STB_WEAK; })
-            .CaseLower("default", [&SI] { SI.Visibility = ELF::STV_DEFAULT; })
-            .CaseLower("hidden", [&SI] { SI.Visibility = ELF::STV_HIDDEN; })
-            .CaseLower("protected", [&SI] { SI.Visibility = ELF::STV_PROTECTED; })
-            .CaseLower("file", [&SI] { SI.Type = ELF::STT_FILE; })
-            .CaseLower("section", [&SI] { SI.Type = ELF::STT_SECTION; })
-            .CaseLower("object", [&SI] { SI.Type = ELF::STT_OBJECT; })
-            .CaseLower("function", [&SI] { SI.Type = ELF::STT_FUNC; })
-            .CaseLower("indirect-function",
-                       [&SI] { SI.Type = ELF::STT_GNU_IFUNC; })
-            .CaseLower("debug", [] {})
-            .CaseLower("constructor", [] {})
-            .CaseLower("warning", [] {})
-            .CaseLower("indirect", [] {})
-            .CaseLower("synthetic", [] {})
-            .CaseLower("unique-object", [] {})
-            .StartsWithLower("before", [] {})
-            .Default([&] { UnsupportedFlags.push_back(Flags[I]); }))();
-  if (!UnsupportedFlags.empty())
-    return createStringError(errc::invalid_argument,
-                             "unsupported flag%s for --add-symbol: '%s'",
-                             UnsupportedFlags.size() > 1 ? "s" : "",
-                             join(UnsupportedFlags, "', '").c_str());
-  return SI;
-}
-
-static const StringMap<MachineInfo> ArchMap{
-    // Name, {EMachine, 64bit, LittleEndian}
-    {"aarch64", {ELF::EM_AARCH64, true, true}},
-    {"arm", {ELF::EM_ARM, false, true}},
-    {"i386", {ELF::EM_386, false, true}},
-    {"i386:x86-64", {ELF::EM_X86_64, true, true}},
-    {"mips", {ELF::EM_MIPS, false, false}},
-    {"powerpc:common64", {ELF::EM_PPC64, true, true}},
-    {"riscv:rv32", {ELF::EM_RISCV, false, true}},
-    {"riscv:rv64", {ELF::EM_RISCV, true, true}},
-    {"sparc", {ELF::EM_SPARC, false, false}},
-    {"sparcel", {ELF::EM_SPARC, false, true}},
-    {"x86-64", {ELF::EM_X86_64, true, true}},
-};
-
-static Expected<const MachineInfo &> getMachineInfo(StringRef Arch) {
-  auto Iter = ArchMap.find(Arch);
-  if (Iter == std::end(ArchMap))
-    return createStringError(errc::invalid_argument,
-                             "invalid architecture: '%s'", Arch.str().c_str());
-  return Iter->getValue();
-}
-
 struct TargetInfo {
   FileFormat Format;
   MachineInfo Machine;
@@ -410,6 +325,16 @@ template <class T> static ErrorOr<T> getAsInteger(StringRef Val) {
   return Result;
 }
 
+static void printHelp(const opt::OptTable &OptTable, raw_ostream &OS,
+                      StringRef ToolName) {
+  OptTable.PrintHelp(OS, (ToolName + " input [output]").str().c_str(),
+                     (ToolName + " tool").str().c_str());
+  // TODO: Replace this with libOption call once it adds extrahelp support.
+  // The CommandLine library has a cl::extrahelp class to support this,
+  // but libOption does not have that yet.
+  OS << "\nPass @FILE as argument to read options from FILE.\n";
+}
+
 // ParseObjcopyOptions returns the config and sets the input arguments. If a
 // help flag is set then ParseObjcopyOptions will print the help messege and
 // exit.
@@ -421,12 +346,12 @@ Expected<DriverConfig> parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
       T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
 
   if (InputArgs.size() == 0) {
-    T.PrintHelp(errs(), "llvm-objcopy input [output]", "objcopy tool");
+    printHelp(T, errs(), "llvm-objcopy");
     exit(1);
   }
 
   if (InputArgs.hasArg(OBJCOPY_help)) {
-    T.PrintHelp(outs(), "llvm-objcopy input [output]", "objcopy tool");
+    printHelp(T, outs(), "llvm-objcopy");
     exit(0);
   }
 
@@ -479,43 +404,26 @@ Expected<DriverConfig> parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
                            .Case("binary", FileFormat::Binary)
                            .Case("ihex", FileFormat::IHex)
                            .Default(FileFormat::Unspecified);
-  if (Config.InputFormat == FileFormat::Binary) {
-    auto BinaryArch = InputArgs.getLastArgValue(OBJCOPY_binary_architecture);
-    if (BinaryArch.empty())
-      return createStringError(
-          errc::invalid_argument,
-          "specified binary input without specifiying an architecture");
-    Expected<const MachineInfo &> MI = getMachineInfo(BinaryArch);
-    if (!MI)
-      return MI.takeError();
-    Config.BinaryArch = *MI;
-  }
 
-  if (opt::Arg *A = InputArgs.getLastArg(OBJCOPY_new_symbol_visibility)) {
-    const uint8_t Invalid = 0xff;
-    Config.NewSymbolVisibility = StringSwitch<uint8_t>(A->getValue())
-                                     .Case("default", ELF::STV_DEFAULT)
-                                     .Case("hidden", ELF::STV_HIDDEN)
-                                     .Case("internal", ELF::STV_INTERNAL)
-                                     .Case("protected", ELF::STV_PROTECTED)
-                                     .Default(Invalid);
-
-    if (Config.NewSymbolVisibility == Invalid)
-      return createStringError(
-          errc::invalid_argument, "'%s' is not a valid symbol visibility",
-          InputArgs.getLastArgValue(OBJCOPY_new_symbol_visibility).str().c_str());
-  }
+  if (InputArgs.hasArg(OBJCOPY_new_symbol_visibility))
+    Config.NewSymbolVisibility =
+        InputArgs.getLastArgValue(OBJCOPY_new_symbol_visibility);
 
   Config.OutputFormat = StringSwitch<FileFormat>(OutputFormat)
                             .Case("binary", FileFormat::Binary)
                             .Case("ihex", FileFormat::IHex)
                             .Default(FileFormat::Unspecified);
-  if (Config.OutputFormat == FileFormat::Unspecified && !OutputFormat.empty()) {
-    Expected<TargetInfo> Target = getOutputTargetInfoByTargetName(OutputFormat);
-    if (!Target)
-      return Target.takeError();
-    Config.OutputFormat = Target->Format;
-    Config.OutputArch = Target->Machine;
+  if (Config.OutputFormat == FileFormat::Unspecified) {
+    if (OutputFormat.empty()) {
+      Config.OutputFormat = Config.InputFormat;
+    } else {
+      Expected<TargetInfo> Target =
+          getOutputTargetInfoByTargetName(OutputFormat);
+      if (!Target)
+        return Target.takeError();
+      Config.OutputFormat = Target->Format;
+      Config.OutputArch = Target->Machine;
+    }
   }
 
   if (auto Arg = InputArgs.getLastArg(OBJCOPY_compress_debug_sections,
@@ -600,6 +508,13 @@ Expected<DriverConfig> parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
                                "multiple renames of section '%s'",
                                SR->OriginalName.str().c_str());
   }
+  for (auto Arg : InputArgs.filtered(OBJCOPY_set_section_alignment)) {
+    Expected<std::pair<StringRef, uint64_t>> NameAndAlign =
+        parseSetSectionAlignment(Arg->getValue());
+    if (!NameAndAlign)
+      return NameAndAlign.takeError();
+    Config.SetSectionAlignment[NameAndAlign->first] = NameAndAlign->second;
+  }
   for (auto Arg : InputArgs.filtered(OBJCOPY_set_section_flags)) {
     Expected<SectionFlagsUpdate> SFU =
         parseSetSectionFlagValue(Arg->getValue());
@@ -713,14 +628,8 @@ Expected<DriverConfig> parseObjcopyOptions(ArrayRef<const char *> ArgsArr) {
     if (Error E = addSymbolsFromFile(Config.SymbolsToKeep, DC.Alloc,
                                      Arg->getValue(), UseRegex))
       return std::move(E);
-  for (auto Arg : InputArgs.filtered(OBJCOPY_add_symbol)) {
-    Expected<NewSymbolInfo> NSI = parseNewSymbolInfo(
-        Arg->getValue(),
-        Config.NewSymbolVisibility.getValueOr((uint8_t)ELF::STV_DEFAULT));
-    if (!NSI)
-      return NSI.takeError();
-    Config.SymbolsToAdd.push_back(*NSI);
-  }
+  for (auto Arg : InputArgs.filtered(OBJCOPY_add_symbol))
+    Config.SymbolsToAdd.push_back(Arg->getValue());
 
   Config.AllowBrokenLinks = InputArgs.hasArg(OBJCOPY_allow_broken_links);
 
@@ -790,12 +699,12 @@ parseStripOptions(ArrayRef<const char *> ArgsArr,
       T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount);
 
   if (InputArgs.size() == 0) {
-    T.PrintHelp(errs(), "llvm-strip [options] file...", "strip tool");
+    printHelp(T, errs(), "llvm-strip");
     exit(1);
   }
 
   if (InputArgs.hasArg(STRIP_help)) {
-    T.PrintHelp(outs(), "llvm-strip [options] file...", "strip tool");
+    printHelp(T, outs(), "llvm-strip");
     exit(0);
   }
 
diff --git a/llvm/tools/llvm-objcopy/CopyConfig.h b/llvm/tools/llvm-objcopy/CopyConfig.h
index e7f293d7fc20e..745af0ce480f0 100644
--- a/llvm/tools/llvm-objcopy/CopyConfig.h
+++ b/llvm/tools/llvm-objcopy/CopyConfig.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_TOOLS_LLVM_OBJCOPY_COPY_CONFIG_H
 #define LLVM_TOOLS_LLVM_OBJCOPY_COPY_CONFIG_H
 
+#include "ELF/ELFConfig.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/Optional.h"
@@ -111,25 +112,17 @@ class NameMatcher {
   bool empty() const { return Matchers.empty(); }
 };
 
-struct NewSymbolInfo {
-  StringRef SymbolName;
-  StringRef SectionName;
-  uint64_t Value = 0;
-  uint8_t Type = ELF::STT_NOTYPE;
-  uint8_t Bind = ELF::STB_GLOBAL;
-  uint8_t Visibility = ELF::STV_DEFAULT;
-};
-
 // Configuration for copying/stripping a single file.
 struct CopyConfig {
+  // Format-specific options to be initialized lazily when needed.
+  Optional<elf::ELFCopyConfig> ELF;
+
   // Main input/output options
   StringRef InputFilename;
   FileFormat InputFormat;
   StringRef OutputFilename;
   FileFormat OutputFormat;
 
-  // Only applicable for --input-format=binary
-  MachineInfo BinaryArch;
   // Only applicable when --output-format!=binary (e.g. elf64-x86-64).
   Optional<MachineInfo> OutputArch;
 
@@ -145,12 +138,12 @@ struct CopyConfig {
   StringRef SymbolsPrefix;
   StringRef AllocSectionsPrefix;
   DiscardType DiscardMode = DiscardType::None;
-  Optional<uint8_t> NewSymbolVisibility;
+  Optional<StringRef> NewSymbolVisibility;
 
   // Repeated options
   std::vector<StringRef> AddSection;
   std::vector<StringRef> DumpSection;
-  std::vector<NewSymbolInfo> SymbolsToAdd;
+  std::vector<StringRef> SymbolsToAdd;
 
   // Section matchers
   NameMatcher KeepSection;
@@ -168,6 +161,7 @@ struct CopyConfig {
 
   // Map options
   StringMap<SectionRename> SectionsToRename;
+  StringMap<uint64_t> SetSectionAlignment;
   StringMap<SectionFlagsUpdate> SetSectionFlags;
   StringMap<StringRef> SymbolsToRename;
 
@@ -196,6 +190,18 @@ struct CopyConfig {
   bool Weaken = false;
   bool DecompressDebugSections = false;
   DebugCompressionType CompressionType = DebugCompressionType::None;
+
+  // parseELFConfig performs ELF-specific command-line parsing. Fills `ELF` on
+  // success or returns an Error otherwise.
+  Error parseELFConfig() {
+    if (!ELF) {
+      Expected<elf::ELFCopyConfig> ELFConfig = elf::parseConfig(*this);
+      if (!ELFConfig)
+        return ELFConfig.takeError();
+      ELF = *ELFConfig;
+    }
+    return Error::success();
+  }
 };
 
 // Configuration for the overall invocation of this tool. When invoked as
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFConfig.cpp b/llvm/tools/llvm-objcopy/ELF/ELFConfig.cpp
new file mode 100644
index 0000000000000..40993760add79
--- /dev/null
+++ b/llvm/tools/llvm-objcopy/ELF/ELFConfig.cpp
@@ -0,0 +1,133 @@
+//===- ELFConfig.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CopyConfig.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Errc.h"
+#include "llvm/Support/Error.h"
+
+namespace llvm {
+namespace objcopy {
+namespace elf {
+
+static Expected<NewSymbolInfo> parseNewSymbolInfo(StringRef FlagValue,
+                                                  uint8_t DefaultVisibility) {
+  // Parse value given with --add-symbol option and create the
+  // new symbol if possible. The value format for --add-symbol is:
+  //
+  // <name>=[<section>:]<value>[,<flags>]
+  //
+  // where:
+  // <name> - symbol name, can be empty string
+  // <section> - optional section name. If not given ABS symbol is created
+  // <value> - symbol value, can be decimal or hexadecimal number prefixed
+  //           with 0x.
+  // <flags> - optional flags affecting symbol type, binding or visibility:
+  //           The following are currently supported:
+  //
+  //           global, local, weak, default, hidden, file, section, object,
+  //           indirect-function.
+  //
+  //           The following flags are ignored and provided for GNU
+  //           compatibility only:
+  //
+  //           warning, debug, constructor, indirect, synthetic,
+  //           unique-object, before=<symbol>.
+  NewSymbolInfo SI;
+  StringRef Value;
+  std::tie(SI.SymbolName, Value) = FlagValue.split('=');
+  if (Value.empty())
+    return createStringError(
+        errc::invalid_argument,
+        "bad format for --add-symbol, missing '=' after '%s'",
+        SI.SymbolName.str().c_str());
+
+  if (Value.contains(':')) {
+    std::tie(SI.SectionName, Value) = Value.split(':');
+    if (SI.SectionName.empty() || Value.empty())
+      return createStringError(
+          errc::invalid_argument,
+          "bad format for --add-symbol, missing section name or symbol value");
+  }
+
+  SmallVector<StringRef, 6> Flags;
+  Value.split(Flags, ',');
+  if (Flags[0].getAsInteger(0, SI.Value))
+    return createStringError(errc::invalid_argument, "bad symbol value: '%s'",
+                             Flags[0].str().c_str());
+
+  SI.Visibility = DefaultVisibility;
+
+  using Functor = std::function<void(void)>;
+  SmallVector<StringRef, 6> UnsupportedFlags;
+  for (size_t I = 1, NumFlags = Flags.size(); I < NumFlags; ++I)
+    static_cast<Functor>(
+        StringSwitch<Functor>(Flags[I])
+            .CaseLower("global", [&SI] { SI.Bind = ELF::STB_GLOBAL; })
+            .CaseLower("local", [&SI] { SI.Bind = ELF::STB_LOCAL; })
+            .CaseLower("weak", [&SI] { SI.Bind = ELF::STB_WEAK; })
+            .CaseLower("default", [&SI] { SI.Visibility = ELF::STV_DEFAULT; })
+            .CaseLower("hidden", [&SI] { SI.Visibility = ELF::STV_HIDDEN; })
+            .CaseLower("protected",
+                       [&SI] { SI.Visibility = ELF::STV_PROTECTED; })
+            .CaseLower("file", [&SI] { SI.Type = ELF::STT_FILE; })
+            .CaseLower("section", [&SI] { SI.Type = ELF::STT_SECTION; })
+            .CaseLower("object", [&SI] { SI.Type = ELF::STT_OBJECT; })
+            .CaseLower("function", [&SI] { SI.Type = ELF::STT_FUNC; })
+            .CaseLower("indirect-function",
+                       [&SI] { SI.Type = ELF::STT_GNU_IFUNC; })
+            .CaseLower("debug", [] {})
+            .CaseLower("constructor", [] {})
+            .CaseLower("warning", [] {})
+            .CaseLower("indirect", [] {})
+            .CaseLower("synthetic", [] {})
+            .CaseLower("unique-object", [] {})
+            .StartsWithLower("before", [] {})
+            .Default([&] { UnsupportedFlags.push_back(Flags[I]); }))();
+  if (!UnsupportedFlags.empty())
+    return createStringError(errc::invalid_argument,
+                             "unsupported flag%s for --add-symbol: '%s'",
+                             UnsupportedFlags.size() > 1 ? "s" : "",
+                             join(UnsupportedFlags, "', '").c_str());
+  return SI;
+}
+
+Expected<ELFCopyConfig> parseConfig(const CopyConfig &Config) {
+  ELFCopyConfig ELFConfig;
+  if (Config.NewSymbolVisibility) {
+    const uint8_t Invalid = 0xff;
+    ELFConfig.NewSymbolVisibility =
+        StringSwitch<uint8_t>(*Config.NewSymbolVisibility)
+            .Case("default", ELF::STV_DEFAULT)
+            .Case("hidden", ELF::STV_HIDDEN)
+            .Case("internal", ELF::STV_INTERNAL)
+            .Case("protected", ELF::STV_PROTECTED)
+            .Default(Invalid);
+
+    if (ELFConfig.NewSymbolVisibility == Invalid)
+      return createStringError(errc::invalid_argument,
+                               "'%s' is not a valid symbol visibility",
+                               Config.NewSymbolVisibility->str().c_str());
+  }
+
+  for (StringRef Arg : Config.SymbolsToAdd) {
+    Expected<elf::NewSymbolInfo> NSI = parseNewSymbolInfo(
+        Arg,
+        ELFConfig.NewSymbolVisibility.getValueOr((uint8_t)ELF::STV_DEFAULT));
+    if (!NSI)
+      return NSI.takeError();
+    ELFConfig.SymbolsToAdd.push_back(*NSI);
+  }
+
+  return ELFConfig;
+}
+
+} // end namespace elf
+} // end namespace objcopy
+} // end namespace llvm
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFConfig.h b/llvm/tools/llvm-objcopy/ELF/ELFConfig.h
new file mode 100644
index 0000000000000..977efbc4166fa
--- /dev/null
+++ b/llvm/tools/llvm-objcopy/ELF/ELFConfig.h
@@ -0,0 +1,44 @@
+//===- ELFConfig.h ----------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_OBJCOPY_ELFCONFIG_H
+#define LLVM_TOOLS_OBJCOPY_ELFCONFIG_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Object/ELFTypes.h"
+#include "llvm/Support/Error.h"
+#include <vector>
+
+namespace llvm {
+namespace objcopy {
+struct CopyConfig;
+
+namespace elf {
+
+struct NewSymbolInfo {
+  StringRef SymbolName;
+  StringRef SectionName;
+  uint64_t Value = 0;
+  uint8_t Type = ELF::STT_NOTYPE;
+  uint8_t Bind = ELF::STB_GLOBAL;
+  uint8_t Visibility = ELF::STV_DEFAULT;
+};
+
+struct ELFCopyConfig {
+  Optional<uint8_t> NewSymbolVisibility;
+  std::vector<NewSymbolInfo> SymbolsToAdd;
+};
+
+Expected<ELFCopyConfig> parseConfig(const CopyConfig &Config);
+
+} // namespace elf
+} // namespace objcopy
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
index 1b1bc8e12a066..dd6a7d7e14b86 100644
--- a/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/ELFObjcopy.cpp
@@ -670,6 +670,14 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj,
     }
   }
 
+  if (!Config.SetSectionAlignment.empty()) {
+    for (SectionBase &Sec : Obj.sections()) {
+      auto I = Config.SetSectionAlignment.find(Sec.Name);
+      if (I != Config.SetSectionAlignment.end())
+        Sec.Align = I->second;
+    }
+  }
+
   if (!Config.SetSectionFlags.empty()) {
     for (auto &Sec : Obj.sections()) {
       const auto Iter = Config.SetSectionFlags.find(Sec.Name);
@@ -710,7 +718,7 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj,
     Obj.addSection<GnuDebugLinkSection>(Config.AddGnuDebugLink,
                                         Config.GnuDebugLinkCRC32);
 
-  for (const NewSymbolInfo &SI : Config.SymbolsToAdd) {
+  for (const NewSymbolInfo &SI : Config.ELF->SymbolsToAdd) {
     SectionBase *Sec = Obj.findSection(SI.SectionName);
     uint64_t Value = Sec ? Sec->Addr + SI.Value : SI.Value;
     Obj.SymbolTable->addSymbol(
@@ -737,7 +745,7 @@ Error executeObjcopyOnIHex(const CopyConfig &Config, MemoryBuffer &In,
   IHexReader Reader(&In);
   std::unique_ptr<Object> Obj = Reader.create();
   const ElfType OutputElfType =
-      getOutputElfType(Config.OutputArch.getValueOr(Config.BinaryArch));
+    getOutputElfType(Config.OutputArch.getValueOr(MachineInfo()));
   if (Error E = handleArgs(Config, *Obj, Reader, OutputElfType))
     return E;
   return writeOutput(Config, *Obj, Out, OutputElfType);
@@ -746,14 +754,14 @@ Error executeObjcopyOnIHex(const CopyConfig &Config, MemoryBuffer &In,
 Error executeObjcopyOnRawBinary(const CopyConfig &Config, MemoryBuffer &In,
                                 Buffer &Out) {
   uint8_t NewSymbolVisibility =
-      Config.NewSymbolVisibility.getValueOr((uint8_t)ELF::STV_DEFAULT);
-  BinaryReader Reader(Config.BinaryArch, &In, NewSymbolVisibility);
+      Config.ELF->NewSymbolVisibility.getValueOr((uint8_t)ELF::STV_DEFAULT);
+  BinaryReader Reader(&In, NewSymbolVisibility);
   std::unique_ptr<Object> Obj = Reader.create();
 
   // Prefer OutputArch (-O<format>) if set, otherwise fallback to BinaryArch
   // (-B<arch>).
   const ElfType OutputElfType =
-      getOutputElfType(Config.OutputArch.getValueOr(Config.BinaryArch));
+      getOutputElfType(Config.OutputArch.getValueOr(MachineInfo()));
   if (Error E = handleArgs(Config, *Obj, Reader, OutputElfType))
     return E;
   return writeOutput(Config, *Obj, Out, OutputElfType);
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.cpp b/llvm/tools/llvm-objcopy/ELF/Object.cpp
index 3d1527cc2aeef..0220b2b5a9229 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.cpp
+++ b/llvm/tools/llvm-objcopy/ELF/Object.cpp
@@ -397,7 +397,7 @@ void SectionWriter::visit(const OwnedDataSection &Sec) {
   llvm::copy(Sec.Data, Out.getBufferStart() + Sec.Offset);
 }
 
-static constexpr std::array<uint8_t, 4> ZlibGnuMagic = {'Z', 'L', 'I', 'B'};
+static constexpr std::array<uint8_t, 4> ZlibGnuMagic = {{'Z', 'L', 'I', 'B'}};
 
 static bool isDataGnuCompressed(ArrayRef<uint8_t> Data) {
   return Data.size() > ZlibGnuMagic.size() &&
@@ -1112,7 +1112,7 @@ void BasicELFBuilder::initFileHeader() {
   Obj->OSABI = ELFOSABI_NONE;
   Obj->ABIVersion = 0;
   Obj->Entry = 0x0;
-  Obj->Machine = EMachine;
+  Obj->Machine = EM_NONE;
   Obj->Version = 1;
 }
 
@@ -1606,7 +1606,7 @@ Writer::~Writer() {}
 Reader::~Reader() {}
 
 std::unique_ptr<Object> BinaryReader::create() const {
-  return BinaryELFBuilder(MInfo.EMachine, MemBuf, NewSymbolVisibility).build();
+  return BinaryELFBuilder(MemBuf, NewSymbolVisibility).build();
 }
 
 Expected<std::vector<IHexRecord>> IHexReader::parse() const {
diff --git a/llvm/tools/llvm-objcopy/ELF/Object.h b/llvm/tools/llvm-objcopy/ELF/Object.h
index 0ff455d540fae..d74b5f410afb0 100644
--- a/llvm/tools/llvm-objcopy/ELF/Object.h
+++ b/llvm/tools/llvm-objcopy/ELF/Object.h
@@ -873,7 +873,6 @@ using object::OwningBinary;
 
 class BasicELFBuilder {
 protected:
-  uint16_t EMachine;
   std::unique_ptr<Object> Obj;
 
   void initFileHeader();
@@ -883,8 +882,7 @@ class BasicELFBuilder {
   void initSections();
 
 public:
-  BasicELFBuilder(uint16_t EM)
-      : EMachine(EM), Obj(std::make_unique<Object>()) {}
+  BasicELFBuilder() : Obj(std::make_unique<Object>()) {}
 };
 
 class BinaryELFBuilder : public BasicELFBuilder {
@@ -893,8 +891,8 @@ class BinaryELFBuilder : public BasicELFBuilder {
   void addData(SymbolTableSection *SymTab);
 
 public:
-  BinaryELFBuilder(uint16_t EM, MemoryBuffer *MB, uint8_t NewSymbolVisibility)
-      : BasicELFBuilder(EM), MemBuf(MB),
+  BinaryELFBuilder(MemoryBuffer *MB, uint8_t NewSymbolVisibility)
+      : BasicELFBuilder(), MemBuf(MB),
         NewSymbolVisibility(NewSymbolVisibility) {}
 
   std::unique_ptr<Object> build();
@@ -907,7 +905,7 @@ class IHexELFBuilder : public BasicELFBuilder {
 
 public:
   IHexELFBuilder(const std::vector<IHexRecord> &Records)
-      : BasicELFBuilder(ELF::EM_386), Records(Records) {}
+      : BasicELFBuilder(), Records(Records) {}
 
   std::unique_ptr<Object> build();
 };
@@ -942,14 +940,12 @@ template <class ELFT> class ELFBuilder {
 };
 
 class BinaryReader : public Reader {
-  const MachineInfo &MInfo;
   MemoryBuffer *MemBuf;
   uint8_t NewSymbolVisibility;
 
 public:
-  BinaryReader(const MachineInfo &MI, MemoryBuffer *MB,
-               const uint8_t NewSymbolVisibility)
-      : MInfo(MI), MemBuf(MB), NewSymbolVisibility(NewSymbolVisibility) {}
+  BinaryReader(MemoryBuffer *MB, const uint8_t NewSymbolVisibility)
+      : MemBuf(MB), NewSymbolVisibility(NewSymbolVisibility) {}
   std::unique_ptr<Object> create() const override;
 };
 
diff --git a/llvm/tools/llvm-objcopy/MachO/MachOLayoutBuilder.cpp b/llvm/tools/llvm-objcopy/MachO/MachOLayoutBuilder.cpp
index 0b8b3f1c2dd1d..f621f3aa09cfa 100644
--- a/llvm/tools/llvm-objcopy/MachO/MachOLayoutBuilder.cpp
+++ b/llvm/tools/llvm-objcopy/MachO/MachOLayoutBuilder.cpp
@@ -146,7 +146,7 @@ uint64_t MachOLayoutBuilder::layoutSegments() {
           Sec.Offset = 0;
         } else {
           uint64_t PaddingSize =
-              offsetToAlignment(SegFileSize, llvm::Align(1ull << Sec.Align));
+              offsetToAlignment(SegFileSize, Align(1ull << Sec.Align));
           Sec.Offset = SegOffset + SegFileSize + PaddingSize;
           Sec.Size = Sec.Content.size();
           SegFileSize += PaddingSize + Sec.Size;
diff --git a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
index a52931e469fd3..6d586e7d73f10 100644
--- a/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
+++ b/llvm/tools/llvm-objcopy/MachO/MachOObjcopy.cpp
@@ -31,13 +31,14 @@ static Error handleArgs(const CopyConfig &Config, Object &Obj) {
       !Config.SymbolsToKeepGlobal.empty() || !Config.SectionsToRename.empty() ||
       !Config.SymbolsToRename.empty() ||
       !Config.UnneededSymbolsToRemove.empty() ||
-      !Config.SetSectionFlags.empty() || !Config.ToRemove.empty() ||
-      Config.ExtractDWO || Config.KeepFileSymbols || Config.LocalizeHidden ||
-      Config.PreserveDates || Config.StripDWO || Config.StripNonAlloc ||
-      Config.StripSections || Config.Weaken || Config.DecompressDebugSections ||
-      Config.StripDebug || Config.StripNonAlloc || Config.StripSections ||
-      Config.StripUnneeded || Config.DiscardMode != DiscardType::None ||
-      !Config.SymbolsToAdd.empty() || Config.EntryExpr) {
+      !Config.SetSectionAlignment.empty() || !Config.SetSectionFlags.empty() ||
+      !Config.ToRemove.empty() || Config.ExtractDWO || Config.KeepFileSymbols ||
+      Config.LocalizeHidden || Config.PreserveDates || Config.StripDWO ||
+      Config.StripNonAlloc || Config.StripSections || Config.Weaken ||
+      Config.DecompressDebugSections || Config.StripDebug ||
+      Config.StripNonAlloc || Config.StripSections || Config.StripUnneeded ||
+      Config.DiscardMode != DiscardType::None || !Config.SymbolsToAdd.empty() ||
+      Config.EntryExpr) {
     return createStringError(llvm::errc::invalid_argument,
                              "option not supported by llvm-objcopy for MachO");
   }
diff --git a/llvm/tools/llvm-objcopy/ObjcopyOpts.td b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
index 55e26fc881e93..9e6b6f0005cd1 100644
--- a/llvm/tools/llvm-objcopy/ObjcopyOpts.td
+++ b/llvm/tools/llvm-objcopy/ObjcopyOpts.td
@@ -1,8 +1,7 @@
 include "CommonOpts.td"
 
 defm binary_architecture
-    : Eq<"binary-architecture", "Used when transforming an architecture-less "
-                                "format (such as binary) to another format">;
+    : Eq<"binary-architecture", "Ignored for compatibility">;
 def B : JoinedOrSeparate<["-"], "B">,
         Alias<binary_architecture>,
         HelpText<"Alias for --binary-architecture">;
@@ -76,6 +75,10 @@ defm add_section
          "Make a section named <section> with the contents of <file>.">,
       MetaVarName<"section=file">;
 
+defm set_section_alignment
+    : Eq<"set-section-alignment", "Set alignment for a given section.">,
+      MetaVarName<"section=align">;
+
 defm set_section_flags
     : Eq<"set-section-flags",
          "Set section flags for a given section. Flags supported for GNU "
diff --git a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
index 9c0145ffee503..db6d751024e32 100644
--- a/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
+++ b/llvm/tools/llvm-objcopy/llvm-objcopy.cpp
@@ -29,6 +29,7 @@
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
@@ -36,6 +37,7 @@
 #include "llvm/Support/Memory.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
+#include "llvm/Support/StringSaver.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
@@ -130,16 +132,18 @@ static Error deepWriteArchive(StringRef ArcName,
 
 /// The function executeObjcopyOnIHex does the dispatch based on the format
 /// of the output specified by the command line options.
-static Error executeObjcopyOnIHex(const CopyConfig &Config, MemoryBuffer &In,
+static Error executeObjcopyOnIHex(CopyConfig &Config, MemoryBuffer &In,
                                   Buffer &Out) {
   // TODO: support output formats other than ELF.
+  if (Error E = Config.parseELFConfig())
+    return E;
   return elf::executeObjcopyOnIHex(Config, In, Out);
 }
 
 /// The function executeObjcopyOnRawBinary does the dispatch based on the format
 /// of the output specified by the command line options.
-static Error executeObjcopyOnRawBinary(const CopyConfig &Config,
-                                       MemoryBuffer &In, Buffer &Out) {
+static Error executeObjcopyOnRawBinary(CopyConfig &Config, MemoryBuffer &In,
+                                       Buffer &Out) {
   switch (Config.OutputFormat) {
   case FileFormat::ELF:
   // FIXME: Currently, we call elf::executeObjcopyOnRawBinary even if the
@@ -148,6 +152,8 @@ static Error executeObjcopyOnRawBinary(const CopyConfig &Config,
   case FileFormat::Binary:
   case FileFormat::IHex:
   case FileFormat::Unspecified:
+    if (Error E = Config.parseELFConfig())
+      return E;
     return elf::executeObjcopyOnRawBinary(Config, In, Out);
   }
 
@@ -156,11 +162,13 @@ static Error executeObjcopyOnRawBinary(const CopyConfig &Config,
 
 /// The function executeObjcopyOnBinary does the dispatch based on the format
 /// of the input binary (ELF, MachO or COFF).
-static Error executeObjcopyOnBinary(const CopyConfig &Config,
-                                    object::Binary &In, Buffer &Out) {
-  if (auto *ELFBinary = dyn_cast<object::ELFObjectFileBase>(&In))
+static Error executeObjcopyOnBinary(CopyConfig &Config, object::Binary &In,
+                                    Buffer &Out) {
+  if (auto *ELFBinary = dyn_cast<object::ELFObjectFileBase>(&In)) {
+    if (Error E = Config.parseELFConfig())
+      return E;
     return elf::executeObjcopyOnBinary(Config, *ELFBinary, Out);
-  else if (auto *COFFBinary = dyn_cast<object::COFFObjectFile>(&In))
+  } else if (auto *COFFBinary = dyn_cast<object::COFFObjectFile>(&In))
     return coff::executeObjcopyOnBinary(Config, *COFFBinary, Out);
   else if (auto *MachOBinary = dyn_cast<object::MachOObjectFile>(&In))
     return macho::executeObjcopyOnBinary(Config, *MachOBinary, Out);
@@ -169,8 +177,7 @@ static Error executeObjcopyOnBinary(const CopyConfig &Config,
                              "unsupported object file format");
 }
 
-static Error executeObjcopyOnArchive(const CopyConfig &Config,
-                                     const Archive &Ar) {
+static Error executeObjcopyOnArchive(CopyConfig &Config, const Archive &Ar) {
   std::vector<NewArchiveMember> NewArchiveMembers;
   Error Err = Error::success();
   for (const Archive::Child &Child : Ar.children(Err)) {
@@ -246,7 +253,7 @@ static Error restoreStatOnFile(StringRef Filename,
 /// The function executeObjcopy does the higher level dispatch based on the type
 /// of input (raw binary, archive or single object file) and takes care of the
 /// format-agnostic modifications, i.e. preserving dates.
-static Error executeObjcopy(const CopyConfig &Config) {
+static Error executeObjcopy(CopyConfig &Config) {
   sys::fs::file_status Stat;
   if (Config.InputFilename != "-") {
     if (auto EC = sys::fs::status(Config.InputFilename, Stat))
@@ -255,7 +262,7 @@ static Error executeObjcopy(const CopyConfig &Config) {
     Stat.permissions(static_cast<sys::fs::perms>(0777));
   }
 
-  typedef Error (*ProcessRawFn)(const CopyConfig &, MemoryBuffer &, Buffer &);
+  using ProcessRawFn = Error (*)(CopyConfig &, MemoryBuffer &, Buffer &);
   ProcessRawFn ProcessRaw;
   switch (Config.InputFormat) {
   case FileFormat::Binary:
@@ -310,15 +317,31 @@ int main(int argc, char **argv) {
   InitLLVM X(argc, argv);
   ToolName = argv[0];
   bool IsStrip = sys::path::stem(ToolName).contains("strip");
+
+  // Expand response files.
+  // TODO: Move these lines, which are copied from lib/Support/CommandLine.cpp,
+  // into a separate function in the CommandLine library and call that function
+  // here. This is duplicated code.
+  SmallVector<const char *, 20> NewArgv(argv, argv + argc);
+  BumpPtrAllocator A;
+  StringSaver Saver(A);
+  cl::ExpandResponseFiles(Saver,
+                          Triple(sys::getProcessTriple()).isOSWindows()
+                              ? cl::TokenizeWindowsCommandLine
+                              : cl::TokenizeGNUCommandLine,
+                          NewArgv);
+
+  auto Args = makeArrayRef(NewArgv).drop_front();
+
   Expected<DriverConfig> DriverConfig =
-      IsStrip ? parseStripOptions(makeArrayRef(argv + 1, argc), reportWarning)
-              : parseObjcopyOptions(makeArrayRef(argv + 1, argc));
+      IsStrip ? parseStripOptions(Args, reportWarning)
+              : parseObjcopyOptions(Args);
   if (!DriverConfig) {
     logAllUnhandledErrors(DriverConfig.takeError(),
                           WithColor::error(errs(), ToolName));
     return 1;
   }
-  for (const CopyConfig &CopyConfig : DriverConfig->CopyConfigs) {
+  for (CopyConfig &CopyConfig : DriverConfig->CopyConfigs) {
     if (Error E = executeObjcopy(CopyConfig)) {
       logAllUnhandledErrors(std::move(E), WithColor::error(errs(), ToolName));
       return 1;
diff --git a/llvm/tools/llvm-objdump/MachODump.cpp b/llvm/tools/llvm-objdump/MachODump.cpp
index 5f4ff0c636c5c..284f9b2a7af73 100644
--- a/llvm/tools/llvm-objdump/MachODump.cpp
+++ b/llvm/tools/llvm-objdump/MachODump.cpp
@@ -7340,10 +7340,24 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
     // A separate DSym file path was specified, parse it as a macho file,
     // get the sections and supply it to the section name parsing machinery.
     if (!DSYMFile.empty()) {
+      std::string DSYMPath(DSYMFile);
+
+      // If DSYMPath is a .dSYM directory, append the Mach-O file.
+      if (llvm::sys::fs::is_directory(DSYMPath) &&
+          llvm::sys::path::extension(DSYMPath) == ".dSYM") {
+        SmallString<128> ShortName(llvm::sys::path::filename(DSYMPath));
+        llvm::sys::path::replace_extension(ShortName, "");
+        SmallString<1024> FullPath(DSYMPath);
+        llvm::sys::path::append(FullPath, "Contents", "Resources", "DWARF",
+                                ShortName);
+        DSYMPath = FullPath.str();
+      }
+
+      // Load the file.
       ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
-          MemoryBuffer::getFileOrSTDIN(DSYMFile);
+          MemoryBuffer::getFileOrSTDIN(DSYMPath);
       if (std::error_code EC = BufOrErr.getError()) {
-        reportError(errorCodeToError(EC), DSYMFile);
+        reportError(errorCodeToError(EC), DSYMPath);
         return;
       }
 
@@ -7353,13 +7367,12 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
       Expected<std::unique_ptr<Binary>> BinaryOrErr =
       createBinary(DSYMBuf.get()->getMemBufferRef());
       if (!BinaryOrErr) {
-        reportError(BinaryOrErr.takeError(), DSYMFile);
+        reportError(BinaryOrErr.takeError(), DSYMPath);
         return;
       }
 
-      // We need to keep the Binary elive with the buffer
+      // We need to keep the Binary alive with the buffer
       DSYMBinary = std::move(BinaryOrErr.get());
-    
       if (ObjectFile *O = dyn_cast<ObjectFile>(DSYMBinary.get())) {
         // this is a Mach-O object file, use it
         if (MachOObjectFile *MachDSYM = dyn_cast<MachOObjectFile>(&*O)) {
@@ -7367,7 +7380,7 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
         }
         else {
           WithColor::error(errs(), "llvm-objdump")
-            << DSYMFile << " is not a Mach-O file type.\n";
+            << DSYMPath << " is not a Mach-O file type.\n";
           return;
         }
       }
@@ -7387,19 +7400,19 @@ static void DisassembleMachO(StringRef Filename, MachOObjectFile *MachOOF,
         Triple T = MachOObjectFile::getArchTriple(CPUType, CPUSubType, nullptr,
                                                   &ArchFlag);
         Expected<std::unique_ptr<MachOObjectFile>> MachDSYM =
-            UB->getObjectForArch(ArchFlag);
+            UB->getMachOObjectForArch(ArchFlag);
         if (!MachDSYM) {
-          reportError(MachDSYM.takeError(), DSYMFile);
+          reportError(MachDSYM.takeError(), DSYMPath);
           return;
         }
     
-        // We need to keep the Binary elive with the buffer
+        // We need to keep the Binary alive with the buffer
         DbgObj = &*MachDSYM.get();
         DSYMBinary = std::move(*MachDSYM);
       }
       else {
         WithColor::error(errs(), "llvm-objdump")
-          << DSYMFile << " is not a Mach-O or Universal file type.\n";
+          << DSYMPath << " is not a Mach-O or Universal file type.\n";
         return;
       }
     }
diff --git a/llvm/tools/llvm-opt-report/OptReport.cpp b/llvm/tools/llvm-opt-report/OptReport.cpp
index a0bc5dccbcfa6..4aecec2749e45 100644
--- a/llvm/tools/llvm-opt-report/OptReport.cpp
+++ b/llvm/tools/llvm-opt-report/OptReport.cpp
@@ -158,16 +158,17 @@ static bool readLocationInfo(LocationInfoTy &LocationInfo) {
   if (!Format) {
     handleAllErrors(Format.takeError(), [&](const ErrorInfoBase &PE) {
       PE.log(WithColor::error());
-      WithColor::error() << '\n';
+      errs() << '\n';
     });
     return false;
   }
 
   Expected<std::unique_ptr<remarks::RemarkParser>> MaybeParser =
-      remarks::createRemarkParser(*Format, (*Buf)->getBuffer());
+      remarks::createRemarkParserFromMeta(*Format, (*Buf)->getBuffer());
   if (!MaybeParser) {
     handleAllErrors(MaybeParser.takeError(), [&](const ErrorInfoBase &PE) {
       PE.log(WithColor::error());
+      errs() << '\n';
     });
     return false;
   }
@@ -182,8 +183,9 @@ static bool readLocationInfo(LocationInfoTy &LocationInfo) {
         consumeError(std::move(E));
         break;
       }
-      handleAllErrors(MaybeRemark.takeError(), [&](const ErrorInfoBase &PE) {
+      handleAllErrors(std::move(E), [&](const ErrorInfoBase &PE) {
         PE.log(WithColor::error());
+        errs() << '\n';
       });
       return false;
     }
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 4f825dfb5b87f..e311c1069f722 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -682,7 +682,7 @@ static void overlapInstrProfile(const std::string &BaseFilename,
   WriterContext Context(false, ErrorLock, WriterErrorCodes);
   WeightedFile WeightedInput{BaseFilename, 1};
   OverlapStats Overlap;
-  Error E = Overlap.accumuateCounts(BaseFilename, TestFilename, IsCS);
+  Error E = Overlap.accumulateCounts(BaseFilename, TestFilename, IsCS);
   if (E)
     exitWithError(std::move(E), "Error in getting profile count sums");
   if (Overlap.Base.CountSum < 1.0f) {
@@ -982,10 +982,21 @@ static int showInstrProfile(const std::string &Filename, bool ShowCounts,
   return 0;
 }
 
+static void showSectionInfo(sampleprof::SampleProfileReader *Reader,
+                            raw_fd_ostream &OS) {
+  if (!Reader->dumpSectionInfo(OS)) {
+    WithColor::warning() << "-show-sec-info-only is only supported for "
+                         << "sample profile in extbinary format and is "
+                         << "ignored for other formats.\n";
+    return;
+  }
+}
+
 static int showSampleProfile(const std::string &Filename, bool ShowCounts,
                              bool ShowAllFunctions,
                              const std::string &ShowFunction,
-                             bool ShowProfileSymbolList, raw_fd_ostream &OS) {
+                             bool ShowProfileSymbolList,
+                             bool ShowSectionInfoOnly, raw_fd_ostream &OS) {
   using namespace sampleprof;
   LLVMContext Context;
   auto ReaderOrErr = SampleProfileReader::create(Filename, Context);
@@ -993,6 +1004,12 @@ static int showSampleProfile(const std::string &Filename, bool ShowCounts,
     exitWithErrorCode(EC, Filename);
 
   auto Reader = std::move(ReaderOrErr.get());
+
+  if (ShowSectionInfoOnly) {
+    showSectionInfo(Reader.get(), OS);
+    return 0;
+  }
+
   if (std::error_code EC = Reader->read())
     exitWithErrorCode(EC, Filename);
 
@@ -1062,6 +1079,11 @@ static int show_main(int argc, const char *argv[]) {
   cl::opt<bool> ShowProfileSymbolList(
       "show-prof-sym-list", cl::init(false),
       cl::desc("Show profile symbol list if it exists in the profile. "));
+  cl::opt<bool> ShowSectionInfoOnly(
+      "show-sec-info-only", cl::init(false),
+      cl::desc("Show the information of each section in the sample profile. "
+               "The flag is only usable when the sample profile is in "
+               "extbinary format"));
 
   cl::ParseCommandLineOptions(argc, argv, "LLVM profile data summary\n");
 
@@ -1090,7 +1112,8 @@ static int show_main(int argc, const char *argv[]) {
                             OnlyListBelow, ShowFunction, TextFormat, OS);
   else
     return showSampleProfile(Filename, ShowCounts, ShowAllFunctions,
-                             ShowFunction, ShowProfileSymbolList, OS);
+                             ShowFunction, ShowProfileSymbolList,
+                             ShowSectionInfoOnly, OS);
 }
 
 int main(int argc, const char *argv[]) {
diff --git a/llvm/tools/llvm-readobj/ARMWinEHPrinter.cpp b/llvm/tools/llvm-readobj/ARMWinEHPrinter.cpp
index 140a3510a3de3..3e026f58871b8 100644
--- a/llvm/tools/llvm-readobj/ARMWinEHPrinter.cpp
+++ b/llvm/tools/llvm-readobj/ARMWinEHPrinter.cpp
@@ -842,8 +842,10 @@ bool Decoder::dumpXDataRecord(const COFFObjectFile &COFF,
 
   if ((int64_t)(Contents.size() - Offset - 4 * HeaderWords(XData) -
                 (XData.E() ? 0 : XData.EpilogueCount() * 4) -
-                (XData.X() ? 8 : 0)) < (int64_t)ByteCodeLength)
+                (XData.X() ? 8 : 0)) < (int64_t)ByteCodeLength) {
+    SW.flush();
     report_fatal_error("Malformed unwind data");
+  }
 
   if (XData.E()) {
     ArrayRef<uint8_t> UC = XData.UnwindByteCode();
diff --git a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
index fdd01e122830c..683243f8cd4c6 100644
--- a/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
+++ b/llvm/tools/llvm-readobj/DwarfCFIEHPrinter.h
@@ -114,11 +114,9 @@ void PrinterContext<ELFT>::printEHFrameHdr(uint64_t EHFrameHdrOffset,
     W.printString("Corresponding Section", *SectionName);
   }
 
-  DataExtractor DE(
-      StringRef(reinterpret_cast<const char *>(Obj->base()) + EHFrameHdrOffset,
-                EHFrameHdrSize),
-      ELFT::TargetEndianness == support::endianness::little,
-      ELFT::Is64Bits ? 8 : 4);
+  DataExtractor DE(makeArrayRef(Obj->base() + EHFrameHdrOffset, EHFrameHdrSize),
+                   ELFT::TargetEndianness == support::endianness::little,
+                   ELFT::Is64Bits ? 8 : 4);
 
   DictScope D(W, "Header");
   uint64_t Offset = 0;
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 3631efa38fd66..1b3e8f4851df0 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -218,6 +218,8 @@ template <typename ELFT> class ELFDumper : public ObjDumper {
                      S->sh_entsize, ObjF->getFileName()});
   }
 
+  std::pair<const Elf_Phdr *, const Elf_Shdr *>
+  findDynamic(const ELFFile<ELFT> *Obj);
   void loadDynamicTable(const ELFFile<ELFT> *Obj);
   void parseDynamicTable();
 
@@ -1417,7 +1419,8 @@ static const char *getElfMipsOptionsOdkType(unsigned Odk) {
 }
 
 template <typename ELFT>
-void ELFDumper<ELFT>::loadDynamicTable(const ELFFile<ELFT> *Obj) {
+std::pair<const typename ELFT::Phdr *, const typename ELFT::Shdr *>
+ELFDumper<ELFT>::findDynamic(const ELFFile<ELFT> *Obj) {
   // Try to locate the PT_DYNAMIC header.
   const Elf_Phdr *DynamicPhdr = nullptr;
   for (const Elf_Phdr &Phdr :
@@ -1438,53 +1441,112 @@ void ELFDumper<ELFT>::loadDynamicTable(const ELFFile<ELFT> *Obj) {
     break;
   }
 
-  // Information in the section header has priority over the information
-  // in a PT_DYNAMIC header.
+  if (DynamicPhdr && DynamicPhdr->p_offset + DynamicPhdr->p_filesz >
+                         ObjF->getMemoryBufferRef().getBufferSize()) {
+    reportWarning(
+        createError(
+            "PT_DYNAMIC segment offset + size exceeds the size of the file"),
+        ObjF->getFileName());
+    // Don't use the broken dynamic header.
+    DynamicPhdr = nullptr;
+  }
+
+  if (DynamicPhdr && DynamicSec) {
+    StringRef Name =
+        unwrapOrError(ObjF->getFileName(), Obj->getSectionName(DynamicSec));
+    if (DynamicSec->sh_addr + DynamicSec->sh_size >
+            DynamicPhdr->p_vaddr + DynamicPhdr->p_memsz ||
+        DynamicSec->sh_addr < DynamicPhdr->p_vaddr)
+      reportWarning(createError("The SHT_DYNAMIC section '" + Name +
+                                "' is not contained within the "
+                                "PT_DYNAMIC segment"),
+                    ObjF->getFileName());
+
+    if (DynamicSec->sh_addr != DynamicPhdr->p_vaddr)
+      reportWarning(createError("The SHT_DYNAMIC section '" + Name +
+                                "' is not at the start of "
+                                "PT_DYNAMIC segment"),
+                    ObjF->getFileName());
+  }
+
+  return std::make_pair(DynamicPhdr, DynamicSec);
+}
+
+template <typename ELFT>
+void ELFDumper<ELFT>::loadDynamicTable(const ELFFile<ELFT> *Obj) {
+  const Elf_Phdr *DynamicPhdr;
+  const Elf_Shdr *DynamicSec;
+  std::tie(DynamicPhdr, DynamicSec) = findDynamic(Obj);
+  if (!DynamicPhdr && !DynamicSec)
+    return;
+
+  DynRegionInfo FromPhdr(ObjF->getFileName());
+  bool IsPhdrTableValid = false;
+  if (DynamicPhdr) {
+    FromPhdr = createDRIFrom(DynamicPhdr, sizeof(Elf_Dyn));
+    IsPhdrTableValid = !FromPhdr.getAsArrayRef<Elf_Dyn>().empty();
+  }
+
+  // Locate the dynamic table described in a section header.
   // Ignore sh_entsize and use the expected value for entry size explicitly.
-  // This allows us to dump the dynamic sections with a broken sh_entsize
+  // This allows us to dump dynamic sections with a broken sh_entsize
   // field.
+  DynRegionInfo FromSec(ObjF->getFileName());
+  bool IsSecTableValid = false;
   if (DynamicSec) {
-    DynamicTable =
+    FromSec =
         checkDRI({ObjF->getELFFile()->base() + DynamicSec->sh_offset,
                   DynamicSec->sh_size, sizeof(Elf_Dyn), ObjF->getFileName()});
-    parseDynamicTable();
+    IsSecTableValid = !FromSec.getAsArrayRef<Elf_Dyn>().empty();
   }
 
-  // If we have a PT_DYNAMIC header, we will either check the found dynamic
-  // section or take the dynamic table data directly from the header.
-  if (!DynamicPhdr)
-    return;
-
-  if (DynamicPhdr->p_offset + DynamicPhdr->p_filesz >
-      ObjF->getMemoryBufferRef().getBufferSize()) {
-    reportWarning(
-        createError(
-            "PT_DYNAMIC segment offset + size exceeds the size of the file"),
-        ObjF->getFileName());
+  // When we only have information from one of the SHT_DYNAMIC section header or
+  // PT_DYNAMIC program header, just use that.
+  if (!DynamicPhdr || !DynamicSec) {
+    if ((DynamicPhdr && IsPhdrTableValid) || (DynamicSec && IsSecTableValid)) {
+      DynamicTable = DynamicPhdr ? FromPhdr : FromSec;
+      parseDynamicTable();
+    } else {
+      reportWarning(createError("no valid dynamic table was found"),
+                    ObjF->getFileName());
+    }
     return;
   }
 
-  if (!DynamicSec) {
-    DynamicTable = createDRIFrom(DynamicPhdr, sizeof(Elf_Dyn));
-    parseDynamicTable();
-    return;
-  }
+  // At this point we have tables found from the section header and from the
+  // dynamic segment. Usually they match, but we have to do sanity checks to
+  // verify that.
 
-  StringRef Name =
-      unwrapOrError(ObjF->getFileName(), Obj->getSectionName(DynamicSec));
-  if (DynamicSec->sh_addr + DynamicSec->sh_size >
-          DynamicPhdr->p_vaddr + DynamicPhdr->p_memsz ||
-      DynamicSec->sh_addr < DynamicPhdr->p_vaddr)
-    reportWarning(createError("The SHT_DYNAMIC section '" + Name +
-                              "' is not contained within the "
-                              "PT_DYNAMIC segment"),
+  if (FromPhdr.Addr != FromSec.Addr)
+    reportWarning(createError("SHT_DYNAMIC section header and PT_DYNAMIC "
+                              "program header disagree about "
+                              "the location of the dynamic table"),
                   ObjF->getFileName());
 
-  if (DynamicSec->sh_addr != DynamicPhdr->p_vaddr)
-    reportWarning(createError("The SHT_DYNAMIC section '" + Name +
-                              "' is not at the start of "
-                              "PT_DYNAMIC segment"),
+  if (!IsPhdrTableValid && !IsSecTableValid) {
+    reportWarning(createError("no valid dynamic table was found"),
                   ObjF->getFileName());
+    return;
+  }
+
+  // Information in the PT_DYNAMIC program header has priority over the information
+  // in a section header.
+  if (IsPhdrTableValid) {
+    if (!IsSecTableValid)
+      reportWarning(
+          createError(
+              "SHT_DYNAMIC dynamic table is invalid: PT_DYNAMIC will be used"),
+          ObjF->getFileName());
+    DynamicTable = FromPhdr;
+  } else {
+    reportWarning(
+        createError(
+            "PT_DYNAMIC dynamic table is invalid: SHT_DYNAMIC will be used"),
+        ObjF->getFileName());
+    DynamicTable = FromSec;
+  }
+
+  parseDynamicTable();
 }
 
 template <typename ELFT>
@@ -3375,10 +3437,21 @@ template <class ELFT> void GNUStyle<ELFT>::printHashSymbols(const ELFO *Obj) {
     for (uint32_t Buc = 0; Buc < SysVHash->nbucket; Buc++) {
       if (Buckets[Buc] == ELF::STN_UNDEF)
         continue;
+      std::vector<bool> Visited(SysVHash->nchain);
       for (uint32_t Ch = Buckets[Buc]; Ch < SysVHash->nchain; Ch = Chains[Ch]) {
         if (Ch == ELF::STN_UNDEF)
           break;
+
+        if (Visited[Ch]) {
+          reportWarning(
+              createError(".hash section is invalid: bucket " + Twine(Ch) +
+                          ": a cycle was detected in the linked chain"),
+              this->FileName);
+          break;
+        }
+
         printHashedSymbol(Obj, &DynSyms[0], Ch, StringTable, Buc);
+        Visited[Ch] = true;
       }
     }
   }
@@ -4577,10 +4650,9 @@ void GNUStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
         OS << "    " << N.Type << ":\n        " << N.Value << '\n';
     } else if (Name == "CORE") {
       if (Type == ELF::NT_FILE) {
-        DataExtractor DescExtractor(
-            StringRef(reinterpret_cast<const char *>(Descriptor.data()),
-                      Descriptor.size()),
-            ELFT::TargetEndianness == support::little, sizeof(Elf_Addr));
+        DataExtractor DescExtractor(Descriptor,
+                                    ELFT::TargetEndianness == support::little,
+                                    sizeof(Elf_Addr));
         Expected<CoreNote> Note = readCoreNote(DescExtractor);
         if (Note)
           printCoreNote<ELFT>(OS, *Note);
@@ -4627,6 +4699,26 @@ void GNUStyle<ELFT>::printELFLinkerOptions(const ELFFile<ELFT> *Obj) {
   OS << "printELFLinkerOptions not implemented!\n";
 }
 
+// Used for printing section names in places where possible errors can be
+// ignored.
+static StringRef getSectionName(const SectionRef &Sec) {
+  Expected<StringRef> NameOrErr = Sec.getName();
+  if (NameOrErr)
+    return *NameOrErr;
+  consumeError(NameOrErr.takeError());
+  return "<?>";
+}
+
+// Used for printing symbol names in places where possible errors can be
+// ignored.
+static std::string getSymbolName(const ELFSymbolRef &Sym) {
+  Expected<StringRef> NameOrErr = Sym.getName();
+  if (NameOrErr)
+    return maybeDemangle(*NameOrErr);
+  consumeError(NameOrErr.takeError());
+  return "<?>";
+}
+
 template <class ELFT>
 void DumpStyle<ELFT>::printFunctionStackSize(
     const ELFObjectFile<ELFT> *Obj, uint64_t SymValue, SectionRef FunctionSec,
@@ -4651,18 +4743,12 @@ void DumpStyle<ELFT>::printFunctionStackSize(
 
   std::string FuncName = "?";
   // A valid SymbolRef has a non-null object file pointer.
-  if (FuncSym.BasicSymbolRef::getObject()) {
-    // Extract the symbol name.
-    Expected<StringRef> FuncNameOrErr = FuncSym.getName();
-    if (FuncNameOrErr)
-      FuncName = maybeDemangle(*FuncNameOrErr);
-    else
-      consumeError(FuncNameOrErr.takeError());
-  } else {
+  if (FuncSym.BasicSymbolRef::getObject())
+    FuncName = getSymbolName(FuncSym);
+  else
     reportWarning(
         createError("could not identify function symbol for stack size entry"),
         Obj->getFileName());
-  }
 
   // Extract the size. The expectation is that Offset is pointing to the right
   // place, i.e. past the function address.
@@ -4704,23 +4790,17 @@ void DumpStyle<ELFT>::printStackSize(const ELFObjectFile<ELFT> *Obj,
     // Ensure that the relocation symbol is in the function section, i.e. the
     // section where the functions whose stack sizes we are reporting are
     // located.
-    StringRef SymName = "?";
-    Expected<StringRef> NameOrErr = RelocSym->getName();
-    if (NameOrErr)
-      SymName = *NameOrErr;
-    else
-      consumeError(NameOrErr.takeError());
-
     auto SectionOrErr = RelocSym->getSection();
     if (!SectionOrErr) {
       reportWarning(
-          createError("cannot identify the section for relocation symbol " +
-                      SymName),
+          createError("cannot identify the section for relocation symbol '" +
+                      getSymbolName(*RelocSym) + "'"),
           FileStr);
       consumeError(SectionOrErr.takeError());
     } else if (*SectionOrErr != FunctionSec) {
-      reportWarning(createError("relocation symbol " + SymName +
-                                " is not in the expected section"),
+      reportWarning(createError("relocation symbol '" +
+                                getSymbolName(*RelocSym) +
+                                "' is not in the expected section"),
                     FileStr);
       // Pretend that the symbol is in the correct section and report its
       // stack size anyway.
@@ -4749,13 +4829,6 @@ void DumpStyle<ELFT>::printStackSize(const ELFObjectFile<ELFT> *Obj,
                                Data, &Offset);
 }
 
-template <class ELFT>
-SectionRef toSectionRef(const ObjectFile *Obj, const typename ELFT::Shdr *Sec) {
-  DataRefImpl DRI;
-  DRI.p = reinterpret_cast<uintptr_t>(Sec);
-  return SectionRef(DRI, Obj);
-}
-
 template <class ELFT>
 void DumpStyle<ELFT>::printNonRelocatableStackSizes(
     const ELFObjectFile<ELFT> *Obj, std::function<void()> PrintHeader) {
@@ -4764,22 +4837,14 @@ void DumpStyle<ELFT>::printNonRelocatableStackSizes(
   const ELFFile<ELFT> *EF = Obj->getELFFile();
   StringRef FileStr = Obj->getFileName();
   for (const SectionRef &Sec : Obj->sections()) {
-    StringRef SectionName;
-    if (Expected<StringRef> NameOrErr = Sec.getName())
-      SectionName = *NameOrErr;
-    else
-      consumeError(NameOrErr.takeError());
-
-    const Elf_Shdr *ElfSec = Obj->getSection(Sec.getRawDataRefImpl());
-    if (!SectionName.startswith(".stack_sizes"))
+    StringRef SectionName = getSectionName(Sec);
+    if (SectionName != ".stack_sizes")
       continue;
     PrintHeader();
+    const Elf_Shdr *ElfSec = Obj->getSection(Sec.getRawDataRefImpl());
     ArrayRef<uint8_t> Contents =
         unwrapOrError(this->FileName, EF->getSectionContents(ElfSec));
-    DataExtractor Data(
-        StringRef(reinterpret_cast<const char *>(Contents.data()),
-                  Contents.size()),
-        Obj->isLittleEndian(), sizeof(Elf_Addr));
+    DataExtractor Data(Contents, Obj->isLittleEndian(), sizeof(Elf_Addr));
     // A .stack_sizes section header's sh_link field is supposed to point
     // to the section that contains the functions whose stack sizes are
     // described in it.
@@ -4798,8 +4863,7 @@ void DumpStyle<ELFT>::printNonRelocatableStackSizes(
             FileStr);
       }
       uint64_t SymValue = Data.getAddress(&Offset);
-      printFunctionStackSize(Obj, SymValue,
-                             toSectionRef<ELFT>(Obj, FunctionELFSec),
+      printFunctionStackSize(Obj, SymValue, Obj->toSectionRef(FunctionELFSec),
                              SectionName, Data, &Offset);
     }
   }
@@ -4824,7 +4888,7 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
 
     // A stack size section that we haven't encountered yet is mapped to the
     // null section until we find its corresponding relocation section.
-    if (SectionName.startswith(".stack_sizes"))
+    if (SectionName == ".stack_sizes")
       if (StackSizeRelocMap.count(Sec) == 0) {
         StackSizeRelocMap[Sec] = NullSection;
         continue;
@@ -4845,10 +4909,10 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
       consumeError(ContentsSectionNameOrErr.takeError());
       continue;
     }
-    if (!ContentsSectionNameOrErr->startswith(".stack_sizes"))
+    if (*ContentsSectionNameOrErr != ".stack_sizes")
       continue;
     // Insert a mapping from the stack sizes section to its relocation section.
-    StackSizeRelocMap[toSectionRef<ELFT>(Obj, ContentsSec)] = Sec;
+    StackSizeRelocMap[Obj->toSectionRef(ContentsSec)] = Sec;
   }
 
   for (const auto &StackSizeMapEntry : StackSizeRelocMap) {
@@ -4857,12 +4921,7 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
     const SectionRef &RelocSec = StackSizeMapEntry.second;
 
     // Warn about stack size sections without a relocation section.
-    StringRef StackSizeSectionName;
-    if (Expected<StringRef> NameOrErr = StackSizesSec.getName())
-      StackSizeSectionName = *NameOrErr;
-    else
-      consumeError(NameOrErr.takeError());
-
+    StringRef StackSizeSectionName = getSectionName(StackSizesSec);
     if (RelocSec == NullSection) {
       reportWarning(createError("section " + StackSizeSectionName +
                                 " does not have a corresponding "
@@ -4876,34 +4935,22 @@ void DumpStyle<ELFT>::printRelocatableStackSizes(
     // described in it.
     const Elf_Shdr *StackSizesELFSec =
         Obj->getSection(StackSizesSec.getRawDataRefImpl());
-    const SectionRef FunctionSec = toSectionRef<ELFT>(
-        Obj, unwrapOrError(this->FileName,
-                           EF->getSection(StackSizesELFSec->sh_link)));
+    const SectionRef FunctionSec = Obj->toSectionRef(unwrapOrError(
+        this->FileName, EF->getSection(StackSizesELFSec->sh_link)));
 
     bool (*IsSupportedFn)(uint64_t);
     RelocationResolver Resolver;
     std::tie(IsSupportedFn, Resolver) = getRelocationResolver(*Obj);
     auto Contents = unwrapOrError(this->FileName, StackSizesSec.getContents());
-    DataExtractor Data(
-        StringRef(reinterpret_cast<const char *>(Contents.data()),
-                  Contents.size()),
-        Obj->isLittleEndian(), sizeof(Elf_Addr));
+    DataExtractor Data(Contents, Obj->isLittleEndian(), sizeof(Elf_Addr));
     for (const RelocationRef &Reloc : RelocSec.relocations()) {
-      if (!IsSupportedFn(Reloc.getType())) {
-        StringRef RelocSectionName;
-        Expected<StringRef> NameOrErr = RelocSec.getName();
-        if (NameOrErr)
-          RelocSectionName = *NameOrErr;
-        else
-          consumeError(NameOrErr.takeError());
-
-        StringRef RelocName = EF->getRelocationTypeName(Reloc.getType());
-        reportError(
-            createStringError(object_error::parse_failed,
-                              "unsupported relocation type in section %s: %s",
-                              RelocSectionName.data(), RelocName.data()),
-            Obj->getFileName());
-      }
+      if (!IsSupportedFn || !IsSupportedFn(Reloc.getType()))
+        reportError(createStringError(
+                        object_error::parse_failed,
+                        "unsupported relocation type in section %s: %s",
+                        getSectionName(RelocSec).data(),
+                        EF->getRelocationTypeName(Reloc.getType()).data()),
+                    Obj->getFileName());
       this->printStackSize(Obj, Reloc, FunctionSec, StackSizeSectionName,
                            Resolver, Data);
     }
@@ -5786,10 +5833,9 @@ void LLVMStyle<ELFT>::printNotes(const ELFFile<ELFT> *Obj) {
         W.printString(N.Type, N.Value);
     } else if (Name == "CORE") {
       if (Type == ELF::NT_FILE) {
-        DataExtractor DescExtractor(
-            StringRef(reinterpret_cast<const char *>(Descriptor.data()),
-                      Descriptor.size()),
-            ELFT::TargetEndianness == support::little, sizeof(Elf_Addr));
+        DataExtractor DescExtractor(Descriptor,
+                                    ELFT::TargetEndianness == support::little,
+                                    sizeof(Elf_Addr));
         Expected<CoreNote> Note = readCoreNote(DescExtractor);
         if (Note)
           printCoreNoteLLVMStyle(*Note, W);
@@ -5854,13 +5900,18 @@ void LLVMStyle<ELFT>::printELFLinkerOptions(const ELFFile<ELFT> *Obj) {
 
 template <class ELFT>
 void LLVMStyle<ELFT>::printStackSizes(const ELFObjectFile<ELFT> *Obj) {
-  W.printString(
-      "Dumping of stack sizes in LLVM style is not implemented yet\n");
+  ListScope L(W, "StackSizes");
+  if (Obj->isRelocatableObject())
+    this->printRelocatableStackSizes(Obj, []() {});
+  else
+    this->printNonRelocatableStackSizes(Obj, []() {});
 }
 
 template <class ELFT>
 void LLVMStyle<ELFT>::printStackSizeEntry(uint64_t Size, StringRef FuncName) {
-  // FIXME: Implement this function for LLVM-style dumping.
+  DictScope D(W, "Entry");
+  W.printString("Function", FuncName);
+  W.printHex("Size", Size);
 }
 
 template <class ELFT>
diff --git a/llvm/tools/llvm-readobj/llvm-readobj.cpp b/llvm/tools/llvm-readobj/llvm-readobj.cpp
index 32e8cbb6c1dc7..5919a7eed3e3a 100644
--- a/llvm/tools/llvm-readobj/llvm-readobj.cpp
+++ b/llvm/tools/llvm-readobj/llvm-readobj.cpp
@@ -231,9 +231,11 @@ namespace opts {
       "codeview-subsection-bytes",
       cl::desc("Dump raw contents of codeview debug sections and records"));
 
-  // --arm-attributes
-  cl::opt<bool> ARMAttributes("arm-attributes",
-                              cl::desc("Display the ARM attributes section"));
+  // --arch-specific
+  cl::opt<bool> ArchSpecificInfo("arch-specific",
+                              cl::desc("Displays architecture-specific information, if there is any."));
+  cl::alias ArchSpecifcInfoShort("A", cl::desc("Alias for --arch-specific"),
+                                 cl::aliasopt(ArchSpecificInfo), cl::NotHidden);
 
   // --mips-plt-got
   cl::opt<bool>
@@ -517,8 +519,8 @@ static void dumpObject(const ObjectFile *Obj, ScopedPrinter &Writer,
   if (Obj->isELF()) {
     if (opts::ELFLinkerOptions)
       Dumper->printELFLinkerOptions();
-    if (Obj->getArch() == llvm::Triple::arm)
-      if (opts::ARMAttributes)
+    if (opts::ArchSpecificInfo)
+      if (Obj->getArch() == llvm::Triple::arm)
         Dumper->printAttributes();
     if (isMipsArch(Obj->getArch())) {
       if (opts::MipsPLTGOT)
@@ -728,10 +730,8 @@ int main(int argc, const char *argv[]) {
     opts::UnwindInfo = true;
     opts::SectionGroups = true;
     opts::HashHistogram = true;
-    // FIXME: As soon as we implement LLVM-style printing of the .stack_size
-    // section, we will enable it with --all (only for LLVM-style).
     if (opts::Output == opts::LLVM)
-      opts::PrintStackSizes = false;
+      opts::PrintStackSizes = true;
   }
 
   if (opts::Headers) {
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index c64b642f8250a..48de0ffa78a10 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -18,6 +18,8 @@ add_llvm_tool(llvm-reduce
   deltas/ReduceGlobalVars.cpp
   deltas/ReduceMetadata.cpp
   deltas/ReduceArguments.cpp
+  deltas/ReduceBasicBlocks.cpp
+  deltas/ReduceInstructions.cpp
 
   DEPENDS
   intrinsics_gen
diff --git a/llvm/tools/llvm-reduce/DeltaManager.h b/llvm/tools/llvm-reduce/DeltaManager.h
index bee15777edc2f..2309c3adf4e6c 100644
--- a/llvm/tools/llvm-reduce/DeltaManager.h
+++ b/llvm/tools/llvm-reduce/DeltaManager.h
@@ -14,18 +14,22 @@
 #include "TestRunner.h"
 #include "deltas/Delta.h"
 #include "deltas/ReduceArguments.h"
+#include "deltas/ReduceBasicBlocks.h"
 #include "deltas/ReduceFunctions.h"
 #include "deltas/ReduceGlobalVars.h"
 #include "deltas/ReduceMetadata.h"
+#include "deltas/ReduceInstructions.h"
 
 namespace llvm {
 
 // TODO: Add CLI option to run only specified Passes (for unit tests)
 inline void runDeltaPasses(TestRunner &Tester) {
   reduceFunctionsDeltaPass(Tester);
+  reduceBasicBlocksDeltaPass(Tester);
   reduceGlobalsDeltaPass(Tester);
   reduceMetadataDeltaPass(Tester);
   reduceArgumentsDeltaPass(Tester);
+  reduceInstructionsDeltaPass(Tester);
   // TODO: Implement the remaining Delta Passes
 }
 
diff --git a/llvm/tools/llvm-reduce/TestRunner.cpp b/llvm/tools/llvm-reduce/TestRunner.cpp
index 50b277b6f0eab..d0e195d5697c9 100644
--- a/llvm/tools/llvm-reduce/TestRunner.cpp
+++ b/llvm/tools/llvm-reduce/TestRunner.cpp
@@ -10,25 +10,8 @@
 
 using namespace llvm;
 
-/// Gets Current Working Directory and tries to create a Tmp Directory
-static SmallString<128> initializeTmpDirectory() {
-  SmallString<128> CWD;
-  if (std::error_code EC = sys::fs::current_path(CWD)) {
-    errs() << "Error getting current directory: " << EC.message() << "!\n";
-    exit(1);
-  }
-
-  SmallString<128> TmpDirectory;
-  sys::path::append(TmpDirectory, CWD, "tmp");
-  if (std::error_code EC = sys::fs::create_directory(TmpDirectory))
-    errs() << "Error creating tmp directory: " << EC.message() << "!\n";
-
-  return TmpDirectory;
-}
-
-TestRunner::TestRunner(StringRef TestName, std::vector<std::string> TestArgs)
-    : TestName(TestName), TestArgs(std::move(TestArgs)) {
-  TmpDirectory = initializeTmpDirectory();
+TestRunner::TestRunner(StringRef TestName, const std::vector<std::string> &TestArgs)
+    : TestName(TestName), TestArgs(TestArgs) {
 }
 
 /// Runs the interestingness test, passes file to be tested as first argument
@@ -36,16 +19,16 @@ TestRunner::TestRunner(StringRef TestName, std::vector<std::string> TestArgs)
 int TestRunner::run(StringRef Filename) {
   std::vector<StringRef> ProgramArgs;
   ProgramArgs.push_back(TestName);
-  ProgramArgs.push_back(Filename);
 
-  for (auto Arg : TestArgs)
-    ProgramArgs.push_back(Arg.c_str());
+  for (const auto &Arg : TestArgs)
+    ProgramArgs.push_back(Arg);
+
+  ProgramArgs.push_back(Filename);
 
-  Optional<StringRef> Redirects[3]; // STDIN, STDOUT, STDERR
   std::string ErrMsg;
-  int Result =
-      sys::ExecuteAndWait(TestName, ProgramArgs, None, Redirects,
-                          /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
+  int Result = sys::ExecuteAndWait(
+      TestName, ProgramArgs, /*Env=*/None, /*Redirects=*/None,
+      /*SecondsToWait=*/0, /*MemoryLimit=*/0, &ErrMsg);
 
   if (Result < 0) {
     Error E = make_error<StringError>("Error running interesting-ness test: " +
diff --git a/llvm/tools/llvm-reduce/TestRunner.h b/llvm/tools/llvm-reduce/TestRunner.h
index 35235e2adebd5..2270d6bd90b27 100644
--- a/llvm/tools/llvm-reduce/TestRunner.h
+++ b/llvm/tools/llvm-reduce/TestRunner.h
@@ -24,23 +24,20 @@ namespace llvm {
 // respective filename.
 class TestRunner {
 public:
-  TestRunner(StringRef TestName, std::vector<std::string> TestArgs);
+  TestRunner(StringRef TestName, const std::vector<std::string> &TestArgs);
 
   /// Runs the interesting-ness test for the specified file
   /// @returns 0 if test was successful, 1 if otherwise
   int run(StringRef Filename);
 
-  /// Directory where tmp files are created
-  StringRef getTmpDir() const { return TmpDirectory; }
   /// Returns the most reduced version of the original testcase
   Module *getProgram() const { return Program.get(); }
 
   void setProgram(std::unique_ptr<Module> P) { Program = std::move(P); }
 
 private:
-  SmallString<128> TestName;
-  std::vector<std::string> TestArgs;
-  SmallString<128> TmpDirectory;
+  StringRef TestName;
+  const std::vector<std::string> &TestArgs;
   std::unique_ptr<Module> Program;
 };
 
diff --git a/llvm/tools/llvm-reduce/deltas/Delta.cpp b/llvm/tools/llvm-reduce/deltas/Delta.cpp
index 0b5571f74de7f..0642241ddebd6 100644
--- a/llvm/tools/llvm-reduce/deltas/Delta.cpp
+++ b/llvm/tools/llvm-reduce/deltas/Delta.cpp
@@ -44,8 +44,8 @@ bool IsReduced(Module &M, TestRunner &Test, SmallString<128> &CurrentFilepath) {
 }
 
 /// Counts the amount of lines for a given file
-static unsigned getLines(StringRef Filepath) {
-  unsigned Lines = 0;
+static int getLines(StringRef Filepath) {
+  int Lines = 0;
   std::string CurrLine;
   std::ifstream FileStream(Filepath);
 
@@ -66,7 +66,7 @@ static bool increaseGranularity(std::vector<Chunk> &Chunks) {
     if (C.end - C.begin == 0)
       NewChunks.push_back(C);
     else {
-      unsigned Half = (C.begin + C.end) / 2;
+      int Half = (C.begin + C.end) / 2;
       NewChunks.push_back({C.begin, Half});
       NewChunks.push_back({Half + 1, C.end});
       SplitOne = true;
@@ -88,9 +88,10 @@ static bool increaseGranularity(std::vector<Chunk> &Chunks) {
 /// reduces the amount of chunks that are considered interesting by the
 /// given test.
 void llvm::runDeltaPass(
-    TestRunner &Test, unsigned Targets,
+    TestRunner &Test, int Targets,
     std::function<void(const std::vector<Chunk> &, Module *)>
         ExtractChunksFromModule) {
+  assert(Targets >= 0);
   if (!Targets) {
     errs() << "\nNothing to reduce\n";
     return;
diff --git a/llvm/tools/llvm-reduce/deltas/Delta.h b/llvm/tools/llvm-reduce/deltas/Delta.h
index d6cf52f8d6343..dbb18e4bd07fe 100644
--- a/llvm/tools/llvm-reduce/deltas/Delta.h
+++ b/llvm/tools/llvm-reduce/deltas/Delta.h
@@ -23,11 +23,11 @@
 namespace llvm {
 
 struct Chunk {
-  unsigned begin;
-  unsigned end;
+  int begin;
+  int end;
 
   /// Helper function to verify if a given Target-index is inside the Chunk
-  bool contains(unsigned Index) const { return Index >= begin && Index <= end; }
+  bool contains(int Index) const { return Index >= begin && Index <= end; }
 
   void print() const {
     errs() << "[" << begin;
@@ -68,7 +68,7 @@ struct Chunk {
 ///
 /// Other implementations of the Delta Debugging algorithm can also be found in
 /// the CReduce, Delta, and Lithium projects.
-void runDeltaPass(TestRunner &Test, unsigned Targets,
+void runDeltaPass(TestRunner &Test, int Targets,
                   std::function<void(const std::vector<Chunk> &, Module *)>
                       ExtractChunksFromModule);
 } // namespace llvm
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
index 40bffa5d8de43..f5f14b83f42cd 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceArguments.cpp
@@ -40,10 +40,9 @@ static void replaceFunctionCalls(Function &OldF, Function &NewF,
 
 /// Removes out-of-chunk arguments from functions, and modifies their calls
 /// accordingly. It also removes allocations of out-of-chunk arguments.
-/// @returns the Module stripped of out-of-chunk functions
 static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
                                        Module *Program) {
-  unsigned I = 0, ArgCount = 0;
+  int I = 0, ArgCount = 0;
   std::set<Argument *> ArgsToKeep;
   std::vector<Function *> Funcs;
   // Get inside-chunk arguments, as well as their parent function
@@ -51,7 +50,7 @@ static void extractArgumentsFromModule(std::vector<Chunk> ChunksToKeep,
     if (!F.isDeclaration()) {
       Funcs.push_back(&F);
       for (auto &A : F.args())
-        if (I < ChunksToKeep.size()) {
+        if (I < (int)ChunksToKeep.size()) {
           if (ChunksToKeep[I].contains(++ArgCount))
             ArgsToKeep.insert(&A);
           if (ChunksToKeep[I].end == ArgCount)
@@ -121,6 +120,6 @@ static int countArguments(Module *Program) {
 
 void llvm::reduceArgumentsDeltaPass(TestRunner &Test) {
   outs() << "*** Reducing Arguments...\n";
-  unsigned ArgCount = countArguments(Test.getProgram());
+  int ArgCount = countArguments(Test.getProgram());
   runDeltaPass(Test, ArgCount, extractArgumentsFromModule);
 }
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
new file mode 100644
index 0000000000000..03c3962d2fd9d
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.cpp
@@ -0,0 +1,146 @@
+//===- ReduceArguments.cpp - Specialized Delta Pass -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting Arguments from defined functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ReduceBasicBlocks.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/raw_ostream.h"
+#include <vector>
+
+using namespace llvm;
+
+/// Replaces BB Terminator with one that only contains Chunk BBs
+static void replaceBranchTerminator(BasicBlock &BB,
+                                    std::set<BasicBlock *> BBsToKeep) {
+  auto Term = BB.getTerminator();
+  std::vector<BasicBlock *> ChunkSucessors;
+  for (auto Succ : successors(&BB))
+    if (BBsToKeep.count(Succ))
+      ChunkSucessors.push_back(Succ);
+
+  // BB only references Chunk BBs
+  if (ChunkSucessors.size() == Term->getNumSuccessors())
+    return;
+
+  bool IsBranch = isa<BranchInst>(Term);
+  Value *Address = nullptr;
+  if (auto IndBI = dyn_cast<IndirectBrInst>(Term))
+    Address = IndBI->getAddress();
+
+  Term->eraseFromParent();
+
+  if (ChunkSucessors.empty()) {
+    ReturnInst::Create(BB.getContext(), nullptr, &BB);
+    return;
+  }
+
+  if (IsBranch)
+    BranchInst::Create(ChunkSucessors[0], &BB);
+
+  if (Address) {
+    auto NewIndBI =
+        IndirectBrInst::Create(Address, ChunkSucessors.size(), &BB);
+    for (auto Dest : ChunkSucessors)
+      NewIndBI->addDestination(Dest);
+  }
+}
+
+/// Removes uninteresting BBs from switch, if the default case ends up being
+/// uninteresting, the switch is replaced with a void return (since it has to be
+/// replace with something)
+static void removeUninterestingBBsFromSwitch(SwitchInst &SwInst,
+                                             std::set<BasicBlock *> BBsToKeep) {
+  if (!BBsToKeep.count(SwInst.getDefaultDest())) {
+    ReturnInst::Create(SwInst.getContext(), nullptr, SwInst.getParent());
+    SwInst.eraseFromParent();
+  } else
+    for (int I = 0, E = SwInst.getNumCases(); I != E; ++I) {
+      auto Case = SwInst.case_begin() + I;
+      if (!BBsToKeep.count(Case->getCaseSuccessor())) {
+        SwInst.removeCase(Case);
+        --I;
+        --E;
+      }
+    }
+}
+
+/// Removes out-of-chunk arguments from functions, and modifies their calls
+/// accordingly. It also removes allocations of out-of-chunk arguments.
+static void extractBasicBlocksFromModule(std::vector<Chunk> ChunksToKeep,
+                                         Module *Program) {
+  int I = 0, BBCount = 0;
+  std::set<BasicBlock *> BBsToKeep;
+
+  for (auto &F : *Program)
+    for (auto &BB : F)
+      if (I < (int)ChunksToKeep.size()) {
+        if (ChunksToKeep[I].contains(++BBCount))
+          BBsToKeep.insert(&BB);
+        if (ChunksToKeep[I].end == BBCount)
+          ++I;
+      }
+
+  std::vector<BasicBlock *> BBsToDelete;
+  for (auto &F : *Program)
+    for (auto &BB : F) {
+      if (!BBsToKeep.count(&BB)) {
+        BBsToDelete.push_back(&BB);
+        // Remove out-of-chunk BB from successor phi nodes
+        for (auto *Succ : successors(&BB))
+          Succ->removePredecessor(&BB);
+      }
+    }
+
+  // Replace terminators that reference out-of-chunk BBs
+  for (auto &F : *Program)
+    for (auto &BB : F) {
+      if (auto *SwInst = dyn_cast<SwitchInst>(BB.getTerminator()))
+        removeUninterestingBBsFromSwitch(*SwInst, BBsToKeep);
+      else
+        replaceBranchTerminator(BB, BBsToKeep);
+    }
+
+  // Replace out-of-chunk switch uses
+  for (auto &BB : BBsToDelete) {
+    // Instructions might be referenced in other BBs
+    for (auto &I : *BB)
+      I.replaceAllUsesWith(UndefValue::get(I.getType()));
+    BB->eraseFromParent();
+  }
+}
+
+/// Counts the amount of basic blocks and prints their name & respective index
+static int countBasicBlocks(Module *Program) {
+  // TODO: Silence index with --quiet flag
+  outs() << "----------------------------\n";
+  int BBCount = 0;
+  for (auto &F : *Program)
+    for (auto &BB : F) {
+      if (BB.hasName())
+        outs() << "\t" << ++BBCount << ": " << BB.getName() << "\n";
+      else
+        outs() << "\t" << ++BBCount << ": Unnamed\n";
+    }
+
+  return BBCount;
+}
+
+void llvm::reduceBasicBlocksDeltaPass(TestRunner &Test) {
+  outs() << "*** Reducing Basic Blocks...\n";
+  int BBCount = countBasicBlocks(Test.getProgram());
+  runDeltaPass(Test, BBCount, extractBasicBlocksFromModule);
+}
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.h b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.h
new file mode 100644
index 0000000000000..cf76a0abbcd7a
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceBasicBlocks.h
@@ -0,0 +1,20 @@
+//===- ReduceArguments.h - Specialized Delta Pass -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting Arguments from defined functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Delta.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+namespace llvm {
+void reduceBasicBlocksDeltaPass(TestRunner &Test);
+} // namespace llvm
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
index 84dc842d9724a..3382f35a945a4 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceFunctions.cpp
@@ -25,9 +25,9 @@ static void extractFunctionsFromModule(const std::vector<Chunk> &ChunksToKeep,
                                        Module *Program) {
   // Get functions inside desired chunks
   std::set<Function *> FuncsToKeep;
-  unsigned I = 0, FunctionCount = 0;
+  int I = 0, FunctionCount = 0;
   for (auto &F : *Program)
-    if (I < ChunksToKeep.size()) {
+    if (I < (int)ChunksToKeep.size()) {
       if (ChunksToKeep[I].contains(++FunctionCount))
         FuncsToKeep.insert(&F);
       if (FunctionCount == ChunksToKeep[I].end)
@@ -57,11 +57,11 @@ static void extractFunctionsFromModule(const std::vector<Chunk> &ChunksToKeep,
 
 /// Counts the amount of non-declaration functions and prints their
 /// respective name & index
-static unsigned countFunctions(Module *Program) {
+static int countFunctions(Module *Program) {
   // TODO: Silence index with --quiet flag
   errs() << "----------------------------\n";
   errs() << "Function Index Reference:\n";
-  unsigned FunctionCount = 0;
+  int FunctionCount = 0;
   for (auto &F : *Program)
     errs() << "\t" << ++FunctionCount << ": " << F.getName() << "\n";
 
@@ -71,7 +71,7 @@ static unsigned countFunctions(Module *Program) {
 
 void llvm::reduceFunctionsDeltaPass(TestRunner &Test) {
   errs() << "*** Reducing Functions...\n";
-  unsigned Functions = countFunctions(Test.getProgram());
+  int Functions = countFunctions(Test.getProgram());
   runDeltaPass(Test, Functions, extractFunctionsFromModule);
   errs() << "----------------------------\n";
 }
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp b/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
index b2c78d170786c..5732208ee0a96 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceGlobalVars.cpp
@@ -21,9 +21,9 @@ static void extractGVsFromModule(std::vector<Chunk> ChunksToKeep,
                                  Module *Program) {
   // Get GVs inside desired chunks
   std::set<GlobalVariable *> GVsToKeep;
-  unsigned I = 0, GVCount = 0;
+  int I = 0, GVCount = 0;
   for (auto &GV : Program->globals())
-    if (GV.hasInitializer() && I < ChunksToKeep.size()) {
+    if (GV.hasInitializer() && I < (int)ChunksToKeep.size()) {
       if (ChunksToKeep[I].contains(++GVCount))
         GVsToKeep.insert(&GV);
       if (GVCount == ChunksToKeep[I].end)
@@ -69,6 +69,6 @@ static int countGVs(Module *Program) {
 
 void llvm::reduceGlobalsDeltaPass(TestRunner &Test) {
   outs() << "*** Reducing GVs...\n";
-  unsigned GVCount = countGVs(Test.getProgram());
+  int GVCount = countGVs(Test.getProgram());
   runDeltaPass(Test, GVCount, extractGVsFromModule);
 }
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
new file mode 100644
index 0000000000000..b3497ad2dc02d
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.cpp
@@ -0,0 +1,65 @@
+//===- ReduceArguments.cpp - Specialized Delta Pass -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting Arguments from defined functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ReduceInstructions.h"
+
+using namespace llvm;
+
+/// Removes out-of-chunk arguments from functions, and modifies their calls
+/// accordingly. It also removes allocations of out-of-chunk arguments.
+static void extractInstrFromModule(std::vector<Chunk> ChunksToKeep,
+                                   Module *Program) {
+  int I = 0, InstCount = 0;
+  std::set<Instruction *> InstToKeep;
+
+  for (auto &F : *Program)
+    for (auto &BB : F)
+      for (auto &Inst : BB)
+        if (I < (int)ChunksToKeep.size()) {
+          if (ChunksToKeep[I].contains(++InstCount))
+            InstToKeep.insert(&Inst);
+          if (ChunksToKeep[I].end == InstCount)
+            ++I;
+        }
+
+  std::vector<Instruction *> InstToDelete;
+  for (auto &F : *Program)
+    for (auto &BB : F)
+      for (auto &Inst : BB)
+        if (!InstToKeep.count(&Inst)) {
+          Inst.replaceAllUsesWith(UndefValue::get(Inst.getType()));
+          InstToDelete.push_back(&Inst);
+        }
+
+  for (auto &I : InstToDelete)
+    I->eraseFromParent();
+}
+
+/// Counts the amount of basic blocks and prints their name & respective index
+static unsigned countInstructions(Module *Program) {
+  // TODO: Silence index with --quiet flag
+  outs() << "----------------------------\n";
+  int InstCount = 0;
+  for (auto &F : *Program)
+    for (auto &BB : F)
+        InstCount += BB.getInstList().size();
+  outs() << "Number of instructions: " << InstCount << "\n";
+
+  return InstCount;
+}
+
+void llvm::reduceInstructionsDeltaPass(TestRunner &Test) {
+  outs() << "*** Reducing Insructions...\n";
+  unsigned InstCount = countInstructions(Test.getProgram());
+  runDeltaPass(Test, InstCount, extractInstrFromModule);
+}
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructions.h b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.h
new file mode 100644
index 0000000000000..a9266acd051a3
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructions.h
@@ -0,0 +1,20 @@
+//===- ReduceArguments.h - Specialized Delta Pass -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function which calls the Generic Delta pass in order
+// to reduce uninteresting Arguments from defined functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Delta.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+namespace llvm {
+void reduceInstructionsDeltaPass(TestRunner &Test);
+} // namespace llvm
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
index 56e69ce873daf..4ea223546efa3 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
@@ -21,7 +21,7 @@ using namespace llvm;
 
 /// Adds all Unnamed Metadata Nodes that are inside desired Chunks to set
 template <class T>
-static void getChunkMetadataNodes(T &MDUser, unsigned &I,
+static void getChunkMetadataNodes(T &MDUser, int &I,
                                   const std::vector<Chunk> &ChunksToKeep,
                                   std::set<MDNode *> &SeenNodes,
                                   std::set<MDNode *> &NodesToKeep) {
@@ -29,10 +29,10 @@ static void getChunkMetadataNodes(T &MDUser, unsigned &I,
   MDUser.getAllMetadata(MDs);
   for (auto &MD : MDs) {
     SeenNodes.insert(MD.second);
-    if (I < ChunksToKeep.size()) {
+    if (I < (int)ChunksToKeep.size()) {
       if (ChunksToKeep[I].contains(SeenNodes.size()))
         NodesToKeep.insert(MD.second);
-      if (ChunksToKeep[I].end == SeenNodes.size())
+      if (ChunksToKeep[I].end == (int)SeenNodes.size())
         ++I;
     }
   }
@@ -55,7 +55,7 @@ static void extractMetadataFromModule(const std::vector<Chunk> &ChunksToKeep,
                                       Module *Program) {
   std::set<MDNode *> SeenNodes;
   std::set<MDNode *> NodesToKeep;
-  unsigned I = 0;
+  int I = 0;
 
   // Add chunk MDNodes used by GVs, Functions, and Instructions to set
   for (auto &GV : Program->globals())
@@ -84,10 +84,10 @@ static void extractMetadataFromModule(const std::vector<Chunk> &ChunksToKeep,
   unsigned MetadataCount = SeenNodes.size();
   std::vector<NamedMDNode *> NamedNodesToDelete;
   for (auto &MD : Program->named_metadata()) {
-    if (I < ChunksToKeep.size()) {
+    if (I < (int)ChunksToKeep.size()) {
       if (!ChunksToKeep[I].contains(++MetadataCount))
         NamedNodesToDelete.push_back(&MD);
-      if (ChunksToKeep[I].end == SeenNodes.size())
+      if (ChunksToKeep[I].end == (int)SeenNodes.size())
         ++I;
     } else
       NamedNodesToDelete.push_back(&MD);
@@ -111,7 +111,7 @@ static void addMetadataToSet(T &MDUser, std::set<MDNode *> &UnnamedNodes) {
 }
 
 /// Returns the amount of Named and Unnamed Metadata Nodes
-static unsigned countMetadataTargets(Module *Program) {
+static int countMetadataTargets(Module *Program) {
   std::set<MDNode *> UnnamedNodes;
   int NamedMetadataNodes = Program->named_metadata_size();
 
@@ -132,7 +132,7 @@ static unsigned countMetadataTargets(Module *Program) {
 
 void llvm::reduceMetadataDeltaPass(TestRunner &Test) {
   outs() << "*** Reducing Metadata...\n";
-  unsigned MDCount = countMetadataTargets(Test.getProgram());
+  int MDCount = countMetadataTargets(Test.getProgram());
   runDeltaPass(Test, MDCount, extractMetadataFromModule);
   outs() << "----------------------------\n";
 }
diff --git a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
index 09539c457a9a9..3a36e77094839 100644
--- a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
+++ b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp
@@ -984,4 +984,5 @@ int main(int argc, char **argv) {
     Result = linkAndVerify();
     break;
   }
+  return Result;
 }
diff --git a/llvm/tools/lto/lto.cpp b/llvm/tools/lto/lto.cpp
index ce670c22c7fb8..9ff16e85db4c7 100644
--- a/llvm/tools/lto/lto.cpp
+++ b/llvm/tools/lto/lto.cpp
@@ -652,3 +652,9 @@ extern const char *lto_input_get_dependent_library(lto_input_t input,
                                                    size_t *size) {
   return LTOModule::getDependentLibrary(unwrap(input), index, size);
 }
+
+extern const char *const *lto_runtime_lib_symbols_list(size_t *size) {
+  auto symbols = lto::LTO::getRuntimeLibcallSymbols();
+  *size = symbols.size();
+  return symbols.data();
+}
diff --git a/llvm/tools/lto/lto.exports b/llvm/tools/lto/lto.exports
index 6913e0e70dc30..d24809fe762f5 100644
--- a/llvm/tools/lto/lto.exports
+++ b/llvm/tools/lto/lto.exports
@@ -76,3 +76,4 @@ lto_input_create
 lto_input_dispose
 lto_input_get_num_dependent_libraries
 lto_input_get_dependent_library
+lto_runtime_lib_symbols_list
diff --git a/llvm/tools/obj2yaml/elf2yaml.cpp b/llvm/tools/obj2yaml/elf2yaml.cpp
index aa3891dcf3115..c4b6eb79d18c0 100644
--- a/llvm/tools/obj2yaml/elf2yaml.cpp
+++ b/llvm/tools/obj2yaml/elf2yaml.cpp
@@ -11,6 +11,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/ObjectYAML/ELFYAML.h"
+#include "llvm/Support/DataExtractor.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/YAMLTraits.h"
 
@@ -62,11 +63,16 @@ class ELFDumper {
   Expected<ELFYAML::SymtabShndxSection *>
   dumpSymtabShndxSection(const Elf_Shdr *Shdr);
   Expected<ELFYAML::NoBitsSection *> dumpNoBitsSection(const Elf_Shdr *Shdr);
+  Expected<ELFYAML::HashSection *> dumpHashSection(const Elf_Shdr *Shdr);
   Expected<ELFYAML::VerdefSection *> dumpVerdefSection(const Elf_Shdr *Shdr);
   Expected<ELFYAML::SymverSection *> dumpSymverSection(const Elf_Shdr *Shdr);
   Expected<ELFYAML::VerneedSection *> dumpVerneedSection(const Elf_Shdr *Shdr);
   Expected<ELFYAML::Group *> dumpGroup(const Elf_Shdr *Shdr);
   Expected<ELFYAML::MipsABIFlags *> dumpMipsABIFlags(const Elf_Shdr *Shdr);
+  Expected<ELFYAML::StackSizesSection *>
+  dumpStackSizesSection(const Elf_Shdr *Shdr);
+
+  Expected<ELFYAML::Section *> dumpSpecialSection(const Elf_Shdr *Shdr);
 
 public:
   ELFDumper(const object::ELFFile<ELFT> &O);
@@ -250,6 +256,13 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
       Y->Sections.emplace_back(*SecOrErr);
       break;
     }
+    case ELF::SHT_HASH: {
+      Expected<ELFYAML::HashSection *> SecOrErr = dumpHashSection(&Sec);
+      if (!SecOrErr)
+        return SecOrErr.takeError();
+      Y->Sections.emplace_back(*SecOrErr);
+      break;
+    }
     case ELF::SHT_GNU_verdef: {
       Expected<ELFYAML::VerdefSection *> SecOrErr = dumpVerdefSection(&Sec);
       if (!SecOrErr)
@@ -284,6 +297,17 @@ template <class ELFT> Expected<ELFYAML::Object *> ELFDumper<ELFT>::dump() {
       LLVM_FALLTHROUGH;
     }
     default: {
+      // Recognize some special SHT_PROGBITS sections by name.
+      if (Sec.sh_type == ELF::SHT_PROGBITS) {
+        Expected<ELFYAML::Section *> SpecialSecOrErr = dumpSpecialSection(&Sec);
+        if (!SpecialSecOrErr)
+          return SpecialSecOrErr.takeError();
+        if (*SpecialSecOrErr) {
+          Y->Sections.emplace_back(*SpecialSecOrErr);
+          break;
+        }
+      }
+
       Expected<ELFYAML::RawContentSection *> SecOrErr =
           dumpContentSection(&Sec);
       if (!SecOrErr)
@@ -432,6 +456,18 @@ Error ELFDumper<ELFT>::dumpCommonSection(const Elf_Shdr *Shdr,
   return Error::success();
 }
 
+template <class ELFT>
+Expected<ELFYAML::Section *>
+ELFDumper<ELFT>::dumpSpecialSection(const Elf_Shdr *Shdr) {
+  auto NameOrErr = getUniquedSectionName(Shdr);
+  if (!NameOrErr)
+    return NameOrErr.takeError();
+
+  if (ELFYAML::StackSizesSection::nameMatches(*NameOrErr))
+    return dumpStackSizesSection(Shdr);
+  return nullptr;
+}
+
 template <class ELFT>
 Error ELFDumper<ELFT>::dumpCommonRelocationSection(
     const Elf_Shdr *Shdr, ELFYAML::RelocationSection &S) {
@@ -450,6 +486,39 @@ Error ELFDumper<ELFT>::dumpCommonRelocationSection(
   return Error::success();
 }
 
+template <class ELFT>
+Expected<ELFYAML::StackSizesSection *>
+ELFDumper<ELFT>::dumpStackSizesSection(const Elf_Shdr *Shdr) {
+  auto S = std::make_unique<ELFYAML::StackSizesSection>();
+  if (Error E = dumpCommonSection(Shdr, *S))
+    return std::move(E);
+
+  auto ContentOrErr = Obj.getSectionContents(Shdr);
+  if (!ContentOrErr)
+    return ContentOrErr.takeError();
+
+  ArrayRef<uint8_t> Content = *ContentOrErr;
+  DataExtractor Data(Content, Obj.isLE(), ELFT::Is64Bits ? 8 : 4);
+
+  std::vector<ELFYAML::StackSizeEntry> Entries;
+  DataExtractor::Cursor Cur(0);
+  while (Cur && Cur.tell() < Content.size()) {
+    uint64_t Address = Data.getAddress(Cur);
+    uint64_t Size = Data.getULEB128(Cur);
+    Entries.push_back({Address, Size});
+  }
+
+  if (Content.empty() || !Cur) {
+    // If .stack_sizes cannot be decoded, we dump it as an array of bytes.
+    consumeError(Cur.takeError());
+    S->Content = yaml::BinaryRef(Content);
+  } else {
+    S->Entries = std::move(Entries);
+  }
+
+  return S.release();
+}
+
 template <class ELFT>
 Expected<ELFYAML::DynamicSection *>
 ELFDumper<ELFT>::dumpDynamicSection(const Elf_Shdr *Shdr) {
@@ -555,6 +624,47 @@ ELFDumper<ELFT>::dumpNoBitsSection(const Elf_Shdr *Shdr) {
   return S.release();
 }
 
+template <class ELFT>
+Expected<ELFYAML::HashSection *>
+ELFDumper<ELFT>::dumpHashSection(const Elf_Shdr *Shdr) {
+  auto S = std::make_unique<ELFYAML::HashSection>();
+  if (Error E = dumpCommonSection(Shdr, *S))
+    return std::move(E);
+
+  auto ContentOrErr = Obj.getSectionContents(Shdr);
+  if (!ContentOrErr)
+    return ContentOrErr.takeError();
+
+  ArrayRef<uint8_t> Content = *ContentOrErr;
+  if (Content.size() % 4 != 0 || Content.size() < 8) {
+    S->Content = yaml::BinaryRef(Content);
+    return S.release();
+  }
+
+  DataExtractor::Cursor Cur(0);
+  DataExtractor Data(Content, Obj.isLE(), /*AddressSize=*/0);
+  uint32_t NBucket = Data.getU32(Cur);
+  uint32_t NChain = Data.getU32(Cur);
+  if (Content.size() != (2 + NBucket + NChain) * 4) {
+    S->Content = yaml::BinaryRef(Content);
+    if (Cur)
+      return S.release();
+    llvm_unreachable("entries were not read correctly");
+  }
+
+  S->Bucket.emplace(NBucket);
+  for (uint32_t &V : *S->Bucket)
+    V = Data.getU32(Cur);
+
+  S->Chain.emplace(NChain);
+  for (uint32_t &V : *S->Chain)
+    V = Data.getU32(Cur);
+
+  if (Cur)
+    return S.release();
+  llvm_unreachable("entries were not read correctly");
+}
+
 template <class ELFT>
 Expected<ELFYAML::VerdefSection *>
 ELFDumper<ELFT>::dumpVerdefSection(const Elf_Shdr *Shdr) {
diff --git a/llvm/tools/opt-viewer/CMakeLists.txt b/llvm/tools/opt-viewer/CMakeLists.txt
index 19b6069330820..ead73ec13a8f5 100644
--- a/llvm/tools/opt-viewer/CMakeLists.txt
+++ b/llvm/tools/opt-viewer/CMakeLists.txt
@@ -11,3 +11,10 @@ foreach (file ${files})
     DESTINATION share/opt-viewer
     COMPONENT opt-viewer)
 endforeach (file)
+
+add_custom_target(opt-viewer DEPENDS ${files})
+if(NOT LLVM_ENABLE_IDE)
+  add_llvm_install_targets("install-opt-viewer"
+                           DEPENDS opt-viewer
+                           COMPONENT opt-viewer)
+endif()
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index ac4859b95c49b..45f2d149bae62 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -523,7 +523,6 @@ int main(int argc, char **argv) {
   initializeDwarfEHPreparePass(Registry);
   initializeSafeStackLegacyPassPass(Registry);
   initializeSjLjEHPreparePass(Registry);
-  initializeStackProtectorPass(Registry);
   initializePreISelIntrinsicLoweringLegacyPassPass(Registry);
   initializeGlobalMergePass(Registry);
   initializeIndirectBrExpandPassPass(Registry);
diff --git a/llvm/tools/sancov/sancov.cpp b/llvm/tools/sancov/sancov.cpp
index 9f5de69ee55af..9645183c2bc36 100644
--- a/llvm/tools/sancov/sancov.cpp
+++ b/llvm/tools/sancov/sancov.cpp
@@ -123,8 +123,8 @@ static const uint32_t BinCoverageMagic = 0xC0BFFFFF;
 static const uint32_t Bitness32 = 0xFFFFFF32;
 static const uint32_t Bitness64 = 0xFFFFFF64;
 
-static Regex SancovFileRegex("(.*)\\.[0-9]+\\.sancov");
-static Regex SymcovFileRegex(".*\\.symcov");
+static const Regex SancovFileRegex("(.*)\\.[0-9]+\\.sancov");
+static const Regex SymcovFileRegex(".*\\.symcov");
 
 // --------- MAIN DATASTRUCTURES ----------
 
diff --git a/llvm/tools/vfabi-demangle-fuzzer/CMakeLists.txt b/llvm/tools/vfabi-demangle-fuzzer/CMakeLists.txt
new file mode 100644
index 0000000000000..908364690f5e0
--- /dev/null
+++ b/llvm/tools/vfabi-demangle-fuzzer/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Analysis
+  Support
+)
+add_llvm_fuzzer(vfabi-demangler-fuzzer
+  vfabi-demangler-fuzzer.cpp
+)
diff --git a/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
new file mode 100644
index 0000000000000..13657effbbebf
--- /dev/null
+++ b/llvm/tools/vfabi-demangle-fuzzer/vfabi-demangler-fuzzer.cpp
@@ -0,0 +1,26 @@
+//===-- vfabi-demangler-fuzzer.cpp - Fuzzer VFABI using lib/Fuzzer   ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Build tool to fuzz the demangler for the vector function ABI names.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/VectorUtils.h"
+
+using namespace llvm;
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
+  const StringRef MangledName((const char *)Data, Size);
+  const auto Info = VFABI::tryDemangleForVFABI(MangledName);
+
+  // Do not optimize away the return value. Inspired by
+  // https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307-L345
+  asm volatile("" : : "r,m"(Info) : "memory");
+
+  return 0;
+}
diff --git a/llvm/unittests/ADT/APIntTest.cpp b/llvm/unittests/ADT/APIntTest.cpp
index ce511ce06f370..a58d31439e7c1 100644
--- a/llvm/unittests/ADT/APIntTest.cpp
+++ b/llvm/unittests/ADT/APIntTest.cpp
@@ -1858,6 +1858,64 @@ TEST(APIntTest, insertBits) {
   EXPECT_EQ(i260.extractBits(4, 256).getZExtValue(), 0x000000000000000Full);
 }
 
+TEST(APIntTest, insertBitsUInt64) {
+  // Tests cloned from insertBits but adapted to the numBits <= 64 constraint
+  uint64_t iSrc = 0x00123456;
+
+  // Direct copy.
+  APInt i31(31, 0x76543210ull);
+  i31.insertBits(iSrc, 0, 31);
+  EXPECT_EQ(static_cast<int64_t>(0x00123456ull), i31.getSExtValue());
+
+  // Single word src/dst insertion.
+  APInt i63(63, 0x01234567FFFFFFFFull);
+  i63.insertBits(iSrc, 4, 31);
+  EXPECT_EQ(static_cast<int64_t>(0x012345600123456Full), i63.getSExtValue());
+
+  // Insert single word src into one word of dst.
+  APInt i120(120, UINT64_MAX, true);
+  i120.insertBits(iSrc, 8, 31);
+  EXPECT_EQ(static_cast<int64_t>(0xFFFFFF80123456FFull), i120.getSExtValue());
+
+  // Insert single word src into two words of dst.
+  APInt i127(127, UINT64_MAX, true);
+  i127.insertBits(iSrc, 48, 31);
+  EXPECT_EQ(i127.extractBits(64, 0).getZExtValue(), 0x3456FFFFFFFFFFFFull);
+  EXPECT_EQ(i127.extractBits(63, 64).getZExtValue(), 0x7FFFFFFFFFFF8012ull);
+
+  // Insert on word boundaries.
+  APInt i128(128, 0);
+  i128.insertBits(UINT64_MAX, 0, 64);
+  i128.insertBits(UINT64_MAX, 64, 64);
+  EXPECT_EQ(-1, i128.getSExtValue());
+
+  APInt i256(256, UINT64_MAX, true);
+  i256.insertBits(0, 0, 64);
+  i256.insertBits(0, 64, 1);
+  i256.insertBits(0, 64, 64);
+  i256.insertBits(0, 128, 5);
+  i256.insertBits(0, 128, 64);
+  i256.insertBits(0, 192, 64);
+  EXPECT_EQ(0u, i256.getSExtValue());
+
+  APInt i257(257, 0);
+  i257.insertBits(APInt(96, UINT64_MAX, true), 64);
+  EXPECT_EQ(i257.extractBitsAsZExtValue(64, 0), 0x0000000000000000ull);
+  EXPECT_EQ(i257.extractBitsAsZExtValue(64, 64), 0xFFFFFFFFFFFFFFFFull);
+  EXPECT_EQ(i257.extractBitsAsZExtValue(64, 128), 0x00000000FFFFFFFFull);
+  EXPECT_EQ(i257.extractBitsAsZExtValue(64, 192), 0x0000000000000000ull);
+  EXPECT_EQ(i257.extractBitsAsZExtValue(1, 256), 0x0000000000000000ull);
+
+  // General insertion.
+  APInt i260(260, UINT64_MAX, true);
+  i260.insertBits(APInt(129, 1ull << 48), 15);
+  EXPECT_EQ(i260.extractBitsAsZExtValue(64, 0), 0x8000000000007FFFull);
+  EXPECT_EQ(i260.extractBitsAsZExtValue(64, 64), 0x0000000000000000ull);
+  EXPECT_EQ(i260.extractBitsAsZExtValue(64, 128), 0xFFFFFFFFFFFF0000ull);
+  EXPECT_EQ(i260.extractBitsAsZExtValue(64, 192), 0xFFFFFFFFFFFFFFFFull);
+  EXPECT_EQ(i260.extractBitsAsZExtValue(4, 256), 0x000000000000000Full);
+}
+
 TEST(APIntTest, extractBits) {
   APInt i32(32, 0x1234567);
   EXPECT_EQ(0x3456, i32.extractBits(16, 4));
@@ -1881,6 +1939,33 @@ TEST(APIntTest, extractBits) {
             APInt(144, "281474976710655", 10).extractBits(48, 1));
 }
 
+TEST(APIntTest, extractBitsAsZExtValue) {
+  // Tests based on extractBits
+  APInt i32(32, 0x1234567);
+  EXPECT_EQ(0x3456u, i32.extractBitsAsZExtValue(16, 4));
+
+  APInt i257(257, 0xFFFFFFFFFF0000FFull, true);
+  EXPECT_EQ(0xFFu, i257.extractBitsAsZExtValue(16, 0));
+  EXPECT_EQ((0xFFu >> 1), i257.extractBitsAsZExtValue(16, 1));
+  EXPECT_EQ(0xFFFFFFFFull, i257.extractBitsAsZExtValue(32, 64));
+  EXPECT_EQ(0xFFFFFFFFFFFFFFFFull, i257.extractBitsAsZExtValue(64, 128));
+  EXPECT_EQ(0xFFFFFFFFFFFFFFFFull, i257.extractBitsAsZExtValue(64, 192));
+  EXPECT_EQ(0xFFFFFFFFFFFFFFFFull, i257.extractBitsAsZExtValue(64, 191));
+  EXPECT_EQ(0x3u, i257.extractBitsAsZExtValue(2, 255));
+  EXPECT_EQ(0xFFFFFFFFFF80007Full, i257.extractBitsAsZExtValue(64, 1));
+  EXPECT_EQ(0xFFFFFFFFFFFFFFFFull, i257.extractBitsAsZExtValue(64, 65));
+  EXPECT_EQ(0xFFFFFFFFFF80007Full, i257.extractBitsAsZExtValue(64, 1));
+  EXPECT_EQ(0xFFFFFFFFFFFFFFFFull, i257.extractBitsAsZExtValue(64, 65));
+  EXPECT_EQ(0x1ull, i257.extractBitsAsZExtValue(1, 129));
+
+  EXPECT_EQ(APInt(48, 0),
+            APInt(144, "281474976710655", 10).extractBitsAsZExtValue(48, 48));
+  EXPECT_EQ(APInt(48, 0x0000ffffffffffffull),
+            APInt(144, "281474976710655", 10).extractBitsAsZExtValue(48, 0));
+  EXPECT_EQ(APInt(48, 0x00007fffffffffffull),
+            APInt(144, "281474976710655", 10).extractBitsAsZExtValue(48, 1));
+}
+
 TEST(APIntTest, getLowBitsSet) {
   APInt i128lo64 = APInt::getLowBitsSet(128, 64);
   EXPECT_EQ(0u, i128lo64.countLeadingOnes());
diff --git a/llvm/unittests/ADT/ArrayRefTest.cpp b/llvm/unittests/ADT/ArrayRefTest.cpp
index 5424742b6b65a..6a3a120449b37 100644
--- a/llvm/unittests/ADT/ArrayRefTest.cpp
+++ b/llvm/unittests/ADT/ArrayRefTest.cpp
@@ -251,7 +251,7 @@ TEST(ArrayRefTest, OwningArrayRef) {
 }
 
 TEST(ArrayRefTest, makeArrayRefFromStdArray) {
-  std::array<int, 5> A1{42, -5, 0, 1000000, -1000000};
+  std::array<int, 5> A1{{42, -5, 0, 1000000, -1000000}};
   ArrayRef<int> A2 = makeArrayRef(A1);
 
   EXPECT_EQ(A1.size(), A2.size());
diff --git a/llvm/unittests/ADT/StringMapTest.cpp b/llvm/unittests/ADT/StringMapTest.cpp
index 4038d4d3d2f7c..5f0c83d0de6ea 100644
--- a/llvm/unittests/ADT/StringMapTest.cpp
+++ b/llvm/unittests/ADT/StringMapTest.cpp
@@ -75,6 +75,16 @@ const char* StringMapTest::testKeyFirst = testKey;
 size_t StringMapTest::testKeyLength = sizeof(testKey) - 1;
 const std::string StringMapTest::testKeyStr(testKey);
 
+struct CountCopyAndMove {
+  CountCopyAndMove() = default;
+  CountCopyAndMove(const CountCopyAndMove &) { copy = 1; }
+  CountCopyAndMove(CountCopyAndMove &&) { move = 1; }
+  void operator=(const CountCopyAndMove &) { ++copy; }
+  void operator=(CountCopyAndMove &&) { ++move; }
+  int copy = 0;
+  int move = 0;
+};
+
 // Empty map tests.
 TEST_F(StringMapTest, EmptyMapTest) {
   assertEmptyMap();
@@ -269,6 +279,27 @@ TEST_F(StringMapTest, InsertRehashingPairTest) {
   EXPECT_EQ(42u, It->second);
 }
 
+TEST_F(StringMapTest, InsertOrAssignTest) {
+  struct A : CountCopyAndMove {
+    A(int v) : v(v) {}
+    int v;
+  };
+  StringMap<A> t(0);
+
+  auto try1 = t.insert_or_assign("A", A(1));
+  EXPECT_TRUE(try1.second);
+  EXPECT_EQ(1, try1.first->second.v);
+  EXPECT_EQ(1, try1.first->second.move);
+
+  auto try2 = t.insert_or_assign("A", A(2));
+  EXPECT_FALSE(try2.second);
+  EXPECT_EQ(2, try2.first->second.v);
+  EXPECT_EQ(2, try1.first->second.move);
+
+  EXPECT_EQ(try1.first, try2.first);
+  EXPECT_EQ(0, try1.first->second.copy);
+}
+
 TEST_F(StringMapTest, IterMapKeys) {
   StringMap<int> Map;
   Map["A"] = 1;
diff --git a/llvm/unittests/Analysis/CMakeLists.txt b/llvm/unittests/Analysis/CMakeLists.txt
index a1858d25b3682..4b4ea416ed8dd 100644
--- a/llvm/unittests/Analysis/CMakeLists.txt
+++ b/llvm/unittests/Analysis/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_unittest(AnalysisTests
   PhiValuesTest.cpp
   ProfileSummaryInfoTest.cpp
   ScalarEvolutionTest.cpp
+  VectorFunctionABITest.cpp
   SparsePropagation.cpp
   TargetLibraryInfoTest.cpp
   TBAATest.cpp
diff --git a/llvm/unittests/Analysis/LoopInfoTest.cpp b/llvm/unittests/Analysis/LoopInfoTest.cpp
index 75e74295033c0..2528078ac67b7 100644
--- a/llvm/unittests/Analysis/LoopInfoTest.cpp
+++ b/llvm/unittests/Analysis/LoopInfoTest.cpp
@@ -868,6 +868,126 @@ TEST(LoopInfoTest, ZextIndVar) {
       });
 }
 
+TEST(LoopInfoTest, MultiExitingLoop) {
+  const char *ModuleStr =
+      "define void @foo(i32* %A, i32 %ub, i1 %cond) {\n"
+      "entry:\n"
+      "  %guardcmp = icmp slt i32 0, %ub\n"
+      "  br i1 %guardcmp, label %for.preheader, label %for.end\n"
+      "for.preheader:\n"
+      "  br label %for.body\n"
+      "for.body:\n"
+      "  %i = phi i32 [ 0, %for.preheader ], [ %inc, %for.body.1 ]\n"
+      "  br i1 %cond, label %for.body.1, label %for.exit\n"
+      "for.body.1:\n"
+      "  %idxprom = sext i32 %i to i64\n"
+      "  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %idxprom\n"
+      "  store i32 %i, i32* %arrayidx, align 4\n"
+      "  %inc = add nsw i32 %i, 1\n"
+      "  %cmp = icmp slt i32 %inc, %ub\n"
+      "  br i1 %cmp, label %for.body, label %for.exit\n"
+      "for.exit:\n"
+      "  br label %for.end\n"
+      "for.end:\n"
+      "  ret void\n"
+      "}\n";
+
+  // Parse the module.
+  LLVMContext Context;
+  std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleStr);
+
+  runWithLoopInfoPlus(
+      *M, "foo",
+      [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+        Function::iterator FI = F.begin();
+        BasicBlock *Entry = &*(FI);
+        BranchInst *Guard = dyn_cast<BranchInst>(Entry->getTerminator());
+        // First two basic block are entry and for.preheader - skip them.
+        ++FI;
+        BasicBlock *Header = &*(++FI);
+        assert(Header->getName() == "for.body");
+        Loop *L = LI.getLoopFor(Header);
+        EXPECT_NE(L, nullptr);
+
+        Optional<Loop::LoopBounds> Bounds = L->getBounds(SE);
+        EXPECT_NE(Bounds, None);
+        ConstantInt *InitialIVValue =
+            dyn_cast<ConstantInt>(&Bounds->getInitialIVValue());
+        EXPECT_TRUE(InitialIVValue && InitialIVValue->isZero());
+        EXPECT_EQ(Bounds->getStepInst().getName(), "inc");
+        ConstantInt *StepValue =
+            dyn_cast_or_null<ConstantInt>(Bounds->getStepValue());
+        EXPECT_TRUE(StepValue && StepValue->isOne());
+        EXPECT_EQ(Bounds->getFinalIVValue().getName(), "ub");
+        EXPECT_EQ(Bounds->getCanonicalPredicate(), ICmpInst::ICMP_SLT);
+        EXPECT_EQ(Bounds->getDirection(),
+                  Loop::LoopBounds::Direction::Increasing);
+        EXPECT_EQ(L->getInductionVariable(SE)->getName(), "i");
+        EXPECT_EQ(L->getLoopGuardBranch(), Guard);
+        EXPECT_TRUE(L->isGuarded());
+      });
+}
+
+TEST(LoopInfoTest, MultiExitLoop) {
+  const char *ModuleStr =
+      "define void @foo(i32* %A, i32 %ub, i1 %cond) {\n"
+      "entry:\n"
+      "  %guardcmp = icmp slt i32 0, %ub\n"
+      "  br i1 %guardcmp, label %for.preheader, label %for.end\n"
+      "for.preheader:\n"
+      "  br label %for.body\n"
+      "for.body:\n"
+      "  %i = phi i32 [ 0, %for.preheader ], [ %inc, %for.body.1 ]\n"
+      "  br i1 %cond, label %for.body.1, label %for.exit\n"
+      "for.body.1:\n"
+      "  %idxprom = sext i32 %i to i64\n"
+      "  %arrayidx = getelementptr inbounds i32, i32* %A, i64 %idxprom\n"
+      "  store i32 %i, i32* %arrayidx, align 4\n"
+      "  %inc = add nsw i32 %i, 1\n"
+      "  %cmp = icmp slt i32 %inc, %ub\n"
+      "  br i1 %cmp, label %for.body, label %for.exit.1\n"
+      "for.exit:\n"
+      "  br label %for.end\n"
+      "for.exit.1:\n"
+      "  br label %for.end\n"
+      "for.end:\n"
+      "  ret void\n"
+      "}\n";
+
+  // Parse the module.
+  LLVMContext Context;
+  std::unique_ptr<Module> M = makeLLVMModule(Context, ModuleStr);
+
+  runWithLoopInfoPlus(
+      *M, "foo",
+      [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+        Function::iterator FI = F.begin();
+        // First two basic block are entry and for.preheader - skip them.
+        ++FI;
+        BasicBlock *Header = &*(++FI);
+        assert(Header->getName() == "for.body");
+        Loop *L = LI.getLoopFor(Header);
+        EXPECT_NE(L, nullptr);
+
+        Optional<Loop::LoopBounds> Bounds = L->getBounds(SE);
+        EXPECT_NE(Bounds, None);
+        ConstantInt *InitialIVValue =
+            dyn_cast<ConstantInt>(&Bounds->getInitialIVValue());
+        EXPECT_TRUE(InitialIVValue && InitialIVValue->isZero());
+        EXPECT_EQ(Bounds->getStepInst().getName(), "inc");
+        ConstantInt *StepValue =
+            dyn_cast_or_null<ConstantInt>(Bounds->getStepValue());
+        EXPECT_TRUE(StepValue && StepValue->isOne());
+        EXPECT_EQ(Bounds->getFinalIVValue().getName(), "ub");
+        EXPECT_EQ(Bounds->getCanonicalPredicate(), ICmpInst::ICMP_SLT);
+        EXPECT_EQ(Bounds->getDirection(),
+                  Loop::LoopBounds::Direction::Increasing);
+        EXPECT_EQ(L->getInductionVariable(SE)->getName(), "i");
+        EXPECT_EQ(L->getLoopGuardBranch(), nullptr);
+        EXPECT_FALSE(L->isGuarded());
+      });
+}
+
 TEST(LoopInfoTest, UnguardedLoop) {
   const char *ModuleStr =
       "define void @foo(i32* %A, i32 %ub) {\n"
diff --git a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
index 002901d943b37..8072e05c7cea0 100644
--- a/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
+++ b/llvm/unittests/Analysis/ProfileSummaryInfoTest.cpp
@@ -137,6 +137,18 @@ TEST_F(ProfileSummaryInfoTest, TestCommon) {
   EXPECT_FALSE(PSI.isColdCount(100));
   EXPECT_FALSE(PSI.isHotCount(100));
 
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(990000, 400));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(990000, 2));
+
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 400));
+  EXPECT_TRUE(PSI.isHotCountNthPercentile(999999, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(999999, 2));
+
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 400));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 100));
+  EXPECT_FALSE(PSI.isHotCountNthPercentile(10000, 2));
+
   EXPECT_TRUE(PSI.isFunctionEntryHot(F));
   EXPECT_FALSE(PSI.isFunctionEntryHot(G));
   EXPECT_FALSE(PSI.isFunctionEntryHot(H));
@@ -160,6 +172,21 @@ TEST_F(ProfileSummaryInfoTest, InstrProf) {
   EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI));
   EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI));
 
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI));
+
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI));
+
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI));
+
   CallSite CS1(BB1->getFirstNonPHI());
   auto *CI2 = BB2->getFirstNonPHI();
   CallSite CS2(CI2);
@@ -192,6 +219,21 @@ TEST_F(ProfileSummaryInfoTest, SampleProf) {
   EXPECT_FALSE(PSI.isHotBlock(BB2, &BFI));
   EXPECT_TRUE(PSI.isHotBlock(BB3, &BFI));
 
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(990000, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(990000, BB3, &BFI));
+
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, &BB0, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB1, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB2, &BFI));
+  EXPECT_TRUE(PSI.isHotBlockNthPercentile(999900, BB3, &BFI));
+
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, &BB0, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB1, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB2, &BFI));
+  EXPECT_FALSE(PSI.isHotBlockNthPercentile(10000, BB3, &BFI));
+
   CallSite CS1(BB1->getFirstNonPHI());
   auto *CI2 = BB2->getFirstNonPHI();
   // Manually attach branch weights metadata to the call instruction.
diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
index 42c708da6d4fe..c42ebf656dd41 100644
--- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
+++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp
@@ -1745,4 +1745,186 @@ TEST_F(ScalarEvolutionsTest, SCEVComputeConstantDifference) {
   });
 }
 
+// Test expansion of nested addrecs in CanonicalMode.
+// Expanding nested addrecs in canonical mode requiers a canonical IV of a
+// type wider than the type of the addrec itself. Currently, SCEVExpander
+// just falls back to literal mode for nested addrecs.
+TEST_F(ScalarEvolutionsTest, SCEVExpandNonAffineAddRec) {
+  LLVMContext C;
+  SMDiagnostic Err;
+
+  // Expand the addrec produced by GetAddRec into a loop without a canonical IV.
+  auto TestNoCanonicalIV = [&](std::function<const SCEVAddRecExpr *(
+                                   ScalarEvolution & SE, Loop * L)> GetAddRec) {
+    std::unique_ptr<Module> M =
+        parseAssemblyString("define i32 @test(i32 %limit) { "
+                            "entry: "
+                            "  br label %loop "
+                            "loop: "
+                            "  %i = phi i32 [ 1, %entry ], [ %i.inc, %loop ] "
+                            "  %i.inc = add nsw i32 %i, 1 "
+                            "  %cont = icmp slt i32 %i.inc, %limit "
+                            "  br i1 %cont, label %loop, label %exit "
+                            "exit: "
+                            "  ret i32 %i.inc "
+                            "}",
+                            Err, C);
+
+    assert(M && "Could not parse module?");
+    assert(!verifyModule(*M) && "Must have been well formed!");
+
+    runWithSE(*M, "test", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+      auto &I = GetInstByName(F, "i");
+      auto *Loop = LI.getLoopFor(I.getParent());
+      EXPECT_FALSE(Loop->getCanonicalInductionVariable());
+
+      auto *AR = GetAddRec(SE, Loop);
+      EXPECT_FALSE(AR->isAffine());
+
+      SCEVExpander Exp(SE, M->getDataLayout(), "expander");
+      auto *InsertAt = I.getNextNode();
+      Value *V = Exp.expandCodeFor(AR, nullptr, InsertAt);
+      auto *ExpandedAR = SE.getSCEV(V);
+      // Check that the expansion happened literally.
+      EXPECT_EQ(AR, ExpandedAR);
+    });
+  };
+
+  // Expand the addrec produced by GetAddRec into a loop with a canonical IV
+  // which is narrower than addrec type.
+  auto TestNarrowCanonicalIV = [&](
+      std::function<const SCEVAddRecExpr *(ScalarEvolution & SE, Loop * L)>
+          GetAddRec) {
+    std::unique_ptr<Module> M = parseAssemblyString(
+        "define i32 @test(i32 %limit) { "
+        "entry: "
+        "  br label %loop "
+        "loop: "
+        "  %i = phi i32 [ 1, %entry ], [ %i.inc, %loop ] "
+        "  %canonical.iv = phi i8 [ 0, %entry ], [ %canonical.iv.inc, %loop ] "
+        "  %i.inc = add nsw i32 %i, 1 "
+        "  %canonical.iv.inc = add i8 %canonical.iv, 1 "
+        "  %cont = icmp slt i32 %i.inc, %limit "
+        "  br i1 %cont, label %loop, label %exit "
+        "exit: "
+        "  ret i32 %i.inc "
+        "}",
+        Err, C);
+
+    assert(M && "Could not parse module?");
+    assert(!verifyModule(*M) && "Must have been well formed!");
+
+    runWithSE(*M, "test", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+      auto &I = GetInstByName(F, "i");
+
+      auto *LoopHeaderBB = I.getParent();
+      auto *Loop = LI.getLoopFor(LoopHeaderBB);
+      PHINode *CanonicalIV = Loop->getCanonicalInductionVariable();
+      EXPECT_EQ(CanonicalIV, &GetInstByName(F, "canonical.iv"));
+
+      auto *AR = GetAddRec(SE, Loop);
+      EXPECT_FALSE(AR->isAffine());
+
+      unsigned ExpectedCanonicalIVWidth = SE.getTypeSizeInBits(AR->getType());
+      unsigned CanonicalIVBitWidth =
+          cast<IntegerType>(CanonicalIV->getType())->getBitWidth();
+      EXPECT_LT(CanonicalIVBitWidth, ExpectedCanonicalIVWidth);
+
+      SCEVExpander Exp(SE, M->getDataLayout(), "expander");
+      auto *InsertAt = I.getNextNode();
+      Value *V = Exp.expandCodeFor(AR, nullptr, InsertAt);
+      auto *ExpandedAR = SE.getSCEV(V);
+      // Check that the expansion happened literally.
+      EXPECT_EQ(AR, ExpandedAR);
+    });
+  };
+
+  // Expand the addrec produced by GetAddRec into a loop with a canonical IV
+  // of addrec width.
+  auto TestMatchingCanonicalIV = [&](
+      std::function<const SCEVAddRecExpr *(ScalarEvolution & SE, Loop * L)>
+          GetAddRec,
+      unsigned ARBitWidth) {
+    auto ARBitWidthTypeStr = "i" + std::to_string(ARBitWidth);
+    std::unique_ptr<Module> M = parseAssemblyString(
+        "define i32 @test(i32 %limit) { "
+        "entry: "
+        "  br label %loop "
+        "loop: "
+        "  %i = phi i32 [ 1, %entry ], [ %i.inc, %loop ] "
+        "  %canonical.iv = phi " + ARBitWidthTypeStr +
+            " [ 0, %entry ], [ %canonical.iv.inc, %loop ] "
+        "  %i.inc = add nsw i32 %i, 1 "
+        "  %canonical.iv.inc = add " + ARBitWidthTypeStr +
+            " %canonical.iv, 1 "
+        "  %cont = icmp slt i32 %i.inc, %limit "
+        "  br i1 %cont, label %loop, label %exit "
+        "exit: "
+        "  ret i32 %i.inc "
+        "}",
+        Err, C);
+
+    assert(M && "Could not parse module?");
+    assert(!verifyModule(*M) && "Must have been well formed!");
+
+    runWithSE(*M, "test", [&](Function &F, LoopInfo &LI, ScalarEvolution &SE) {
+      auto &I = GetInstByName(F, "i");
+      auto &CanonicalIV = GetInstByName(F, "canonical.iv");
+
+      auto *LoopHeaderBB = I.getParent();
+      auto *Loop = LI.getLoopFor(LoopHeaderBB);
+      EXPECT_EQ(&CanonicalIV, Loop->getCanonicalInductionVariable());
+      unsigned CanonicalIVBitWidth =
+          cast<IntegerType>(CanonicalIV.getType())->getBitWidth();
+
+      auto *AR = GetAddRec(SE, Loop);
+      EXPECT_FALSE(AR->isAffine());
+      EXPECT_EQ(ARBitWidth, SE.getTypeSizeInBits(AR->getType()));
+      EXPECT_EQ(CanonicalIVBitWidth, ARBitWidth);
+
+      SCEVExpander Exp(SE, M->getDataLayout(), "expander");
+      auto *InsertAt = I.getNextNode();
+      Value *V = Exp.expandCodeFor(AR, nullptr, InsertAt);
+      auto *ExpandedAR = SE.getSCEV(V);
+      // Check that the expansion happened literally.
+      EXPECT_EQ(AR, ExpandedAR);
+    });
+  };
+
+  unsigned ARBitWidth = 16;
+  Type *ARType = IntegerType::get(C, ARBitWidth);
+
+  // Expand {5,+,1,+,1}
+  auto GetAR3 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
+    SmallVector<const SCEV *, 3> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
+                                        SE.getOne(ARType), SE.getOne(ARType)};
+    return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
+  };
+  TestNoCanonicalIV(GetAR3);
+  TestNarrowCanonicalIV(GetAR3);
+  TestMatchingCanonicalIV(GetAR3, ARBitWidth);
+
+  // Expand {5,+,1,+,1,+,1}
+  auto GetAR4 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
+    SmallVector<const SCEV *, 4> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
+                                        SE.getOne(ARType), SE.getOne(ARType),
+                                        SE.getOne(ARType)};
+    return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
+  };
+  TestNoCanonicalIV(GetAR4);
+  TestNarrowCanonicalIV(GetAR4);
+  TestMatchingCanonicalIV(GetAR4, ARBitWidth);
+
+  // Expand {5,+,1,+,1,+,1,+,1}
+  auto GetAR5 = [&](ScalarEvolution &SE, Loop *L) -> const SCEVAddRecExpr * {
+    SmallVector<const SCEV *, 5> Ops = {SE.getConstant(APInt(ARBitWidth, 5)),
+                                        SE.getOne(ARType), SE.getOne(ARType),
+                                        SE.getOne(ARType), SE.getOne(ARType)};
+    return cast<SCEVAddRecExpr>(SE.getAddRecExpr(Ops, L, SCEV::FlagAnyWrap));
+  };
+  TestNoCanonicalIV(GetAR5);
+  TestNarrowCanonicalIV(GetAR5);
+  TestMatchingCanonicalIV(GetAR5, ARBitWidth);
+}
+
 }  // end namespace llvm
diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp
new file mode 100644
index 0000000000000..4dbc13076618e
--- /dev/null
+++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp
@@ -0,0 +1,439 @@
+//===------- VectorFunctionABITest.cpp - VFABI Unittests  ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/VectorUtils.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+// This test makes sure that the getFromVFABI method succeeds only on
+// valid values of the string.
+TEST(VectorFunctionABITests, OnlyValidNames) {
+  // Incomplete string.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGV").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVn").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2v").hasValue());
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2v_").hasValue());
+  // Missing parameters.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2_foo").hasValue());
+  // Missing _ZGV prefix.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZVnN2v_foo").hasValue());
+  // Missing <isa>.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVN2v_foo").hasValue());
+  // Missing <mask>.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVn2v_foo").hasValue());
+  // Missing <vlen>.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnNv_foo").hasValue());
+  // Missing <scalarname>.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2v_").hasValue());
+  // Missing _ separator.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2vfoo").hasValue());
+  // Missing <vectorname>.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2v_foo()").hasValue());
+  // Unterminated name.
+  EXPECT_FALSE(VFABI::tryDemangleForVFABI("_ZGVnN2v_foo(bar").hasValue());
+}
+
+TEST(VectorFunctionABITests, ParamListParsing) {
+  // Testing "vl16Ls32R3l"
+  const auto OptVFS = VFABI::tryDemangleForVFABI("_ZGVnN2vl16Ls32R3l_foo");
+  EXPECT_TRUE(OptVFS.hasValue());
+  const VFInfo VFS = OptVFS.getValue();
+  EXPECT_EQ(VFS.Shape.Parameters.size(), (unsigned)5);
+  EXPECT_EQ(VFS.Shape.Parameters[0], VFParameter({0, VFParamKind::Vector, 0}));
+  EXPECT_EQ(VFS.Shape.Parameters[1],
+            VFParameter({1, VFParamKind::OMP_Linear, 16}));
+  EXPECT_EQ(VFS.Shape.Parameters[2],
+            VFParameter({2, VFParamKind::OMP_LinearValPos, 32}));
+  EXPECT_EQ(VFS.Shape.Parameters[3],
+            VFParameter({3, VFParamKind::OMP_LinearRef, 3}));
+  EXPECT_EQ(VFS.Shape.Parameters[4],
+            VFParameter({4, VFParamKind::OMP_Linear, 1}));
+}
+
+TEST(VectorFunctionABITests, ScalarNameAndVectorName) {
+  // Parse Scalar Name
+  const auto A = VFABI::tryDemangleForVFABI("_ZGVnM2v_sin");
+  const auto B = VFABI::tryDemangleForVFABI("_ZGVnM2v_sin(UserFunc)");
+  const auto C = VFABI::tryDemangleForVFABI("_ZGVnM2v___sin_sin_sin");
+  EXPECT_TRUE(A.hasValue());
+  EXPECT_TRUE(B.hasValue());
+  EXPECT_TRUE(C.hasValue());
+  EXPECT_EQ(A.getValue().ScalarName, "sin");
+  EXPECT_EQ(B.getValue().ScalarName, "sin");
+  EXPECT_EQ(C.getValue().ScalarName, "__sin_sin_sin");
+  EXPECT_EQ(A.getValue().VectorName, "_ZGVnM2v_sin");
+  EXPECT_EQ(B.getValue().VectorName, "UserFunc");
+  EXPECT_EQ(C.getValue().VectorName, "_ZGVnM2v___sin_sin_sin");
+}
+
+namespace {
+// Test fixture needed that holds the veariables needed by the parser.
+class VFABIParserTest : public ::testing::Test {
+private:
+  // Parser output.
+  VFInfo Info;
+  // Reset the parser output references.
+  void reset() { Info = VFInfo(); }
+
+protected:
+  // Referencies to the parser output field.
+  unsigned &VF = Info.Shape.VF;
+  VFISAKind &ISA = Info.Shape.ISA;
+  SmallVector<VFParameter, 8> &Parameters = Info.Shape.Parameters;
+  StringRef &ScalarName = Info.ScalarName;
+  StringRef &VectorName = Info.VectorName;
+  bool &IsScalable = Info.Shape.IsScalable;
+  // Invoke the parser.
+  bool invokeParser(const StringRef MangledName) {
+    reset();
+    const auto OptInfo = VFABI::tryDemangleForVFABI(MangledName);
+    if (OptInfo.hasValue()) {
+      Info = OptInfo.getValue();
+      return true;
+    }
+
+    return false;
+  }
+  // Checks that 1. the last Parameter in the Shape is of type
+  // VFParamKind::GlobalPredicate and 2. it is the only one of such
+  // type.
+  bool IsMasked() const {
+    const auto NGlobalPreds =
+        std::count_if(Info.Shape.Parameters.begin(),
+                      Info.Shape.Parameters.end(), [](const VFParameter PK) {
+                        return PK.ParamKind == VFParamKind::GlobalPredicate;
+                      });
+    return NGlobalPreds == 1 && Info.Shape.Parameters.back().ParamKind ==
+                                    VFParamKind::GlobalPredicate;
+  }
+};
+} // unnamed namespace
+
+TEST_F(VFABIParserTest, Parse) {
+  EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_FALSE(IsMasked());
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(Parameters.size(), (unsigned)9);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearPos, 2}));
+  EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearValPos, 27}));
+  EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearUValPos, 4}));
+  EXPECT_EQ(Parameters[4], VFParameter({4, VFParamKind::OMP_LinearRefPos, 5}));
+  EXPECT_EQ(Parameters[5], VFParameter({5, VFParamKind::OMP_Linear, 1}));
+  EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearVal, 10}));
+  EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, 100}));
+  EXPECT_EQ(Parameters[8], VFParameter({8, VFParamKind::OMP_LinearRef, 1000}));
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin");
+}
+
+TEST_F(VFABIParserTest, ParseVectorName) {
+  EXPECT_TRUE(invokeParser("_ZGVnN2v_sin(my_v_sin)"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_FALSE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  EXPECT_EQ(Parameters.size(), (unsigned)1);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0}));
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "my_v_sin");
+}
+
+TEST_F(VFABIParserTest, LinearWithCompileTimeNegativeStep) {
+  EXPECT_TRUE(invokeParser("_ZGVnN2ln1Ln10Un100Rn1000_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_FALSE(IsMasked());
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(Parameters.size(), (unsigned)4);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, -1}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, -10}));
+  EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearUVal, -100}));
+  EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearRef, -1000}));
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "_ZGVnN2ln1Ln10Un100Rn1000_sin");
+}
+
+TEST_F(VFABIParserTest, ParseScalableSVE) {
+  EXPECT_TRUE(invokeParser("_ZGVsMxv_sin"));
+  EXPECT_EQ(VF, (unsigned)0);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_TRUE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::SVE);
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "_ZGVsMxv_sin");
+}
+
+TEST_F(VFABIParserTest, ParseFixedWidthSVE) {
+  EXPECT_TRUE(invokeParser("_ZGVsM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::SVE);
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "_ZGVsM2v_sin");
+}
+
+TEST_F(VFABIParserTest, NotAVectorFunctionABIName) {
+  // Vector names should start with `_ZGV`.
+  EXPECT_FALSE(invokeParser("ZGVnN2v_sin"));
+}
+
+TEST_F(VFABIParserTest, LinearWithRuntimeStep) {
+  EXPECT_FALSE(invokeParser("_ZGVnN2ls_sin"))
+      << "A number should be present after \"ls\".";
+  EXPECT_TRUE(invokeParser("_ZGVnN2ls2_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVnN2Rs_sin"))
+      << "A number should be present after \"Rs\".";
+  EXPECT_TRUE(invokeParser("_ZGVnN2Rs4_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVnN2Ls_sin"))
+      << "A number should be present after \"Ls\".";
+  EXPECT_TRUE(invokeParser("_ZGVnN2Ls6_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVnN2Us_sin"))
+      << "A number should be present after \"Us\".";
+  EXPECT_TRUE(invokeParser("_ZGVnN2Us8_sin"));
+}
+
+TEST_F(VFABIParserTest, LinearWithoutCompileTime) {
+  EXPECT_TRUE(invokeParser("_ZGVnN3lLRUlnLnRnUn_sin"));
+  EXPECT_EQ(Parameters.size(), (unsigned)8);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, 1}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, 1}));
+  EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearRef, 1}));
+  EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearUVal, 1}));
+  EXPECT_EQ(Parameters[4], VFParameter({4, VFParamKind::OMP_Linear, -1}));
+  EXPECT_EQ(Parameters[5], VFParameter({5, VFParamKind::OMP_LinearVal, -1}));
+  EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearRef, -1}));
+  EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, -1}));
+}
+
+TEST_F(VFABIParserTest, ISA) {
+  EXPECT_TRUE(invokeParser("_ZGVqN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::Unknown);
+
+  EXPECT_TRUE(invokeParser("_ZGVnN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+
+  EXPECT_TRUE(invokeParser("_ZGVsN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::SVE);
+
+  EXPECT_TRUE(invokeParser("_ZGVbN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::SSE);
+
+  EXPECT_TRUE(invokeParser("_ZGVcN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX);
+
+  EXPECT_TRUE(invokeParser("_ZGVdN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX2);
+
+  EXPECT_TRUE(invokeParser("_ZGVeN2v_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX512);
+}
+
+TEST_F(VFABIParserTest, InvalidMask) {
+  EXPECT_FALSE(invokeParser("_ZGVsK2v_sin"));
+}
+
+TEST_F(VFABIParserTest, InvalidParameter) {
+  EXPECT_FALSE(invokeParser("_ZGVsM2vX_sin"));
+}
+
+TEST_F(VFABIParserTest, Align) {
+  EXPECT_TRUE(invokeParser("_ZGVsN2l2a2_sin"));
+  EXPECT_EQ(Parameters.size(), (unsigned)1);
+  EXPECT_EQ(Parameters[0].Alignment, Align(2));
+
+  // Missing alignement value.
+  EXPECT_FALSE(invokeParser("_ZGVsM2l2a_sin"));
+  // Invalid alignment token "x".
+  EXPECT_FALSE(invokeParser("_ZGVsM2l2ax_sin"));
+  // Alignment MUST be associated to a paramater.
+  EXPECT_FALSE(invokeParser("_ZGVsM2a2_sin"));
+  // Alignment must be a power of 2.
+  EXPECT_FALSE(invokeParser("_ZGVsN2l2a0_sin"));
+  EXPECT_TRUE(invokeParser("_ZGVsN2l2a1_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVsN2l2a3_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVsN2l2a6_sin"));
+}
+
+TEST_F(VFABIParserTest, ParseUniform) {
+  EXPECT_TRUE(invokeParser("_ZGVnN2u0_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_FALSE(IsMasked());
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(Parameters.size(), (unsigned)1);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Uniform, 0}));
+  EXPECT_EQ(ScalarName, "sin");
+  EXPECT_EQ(VectorName, "_ZGVnN2u0_sin");
+
+  EXPECT_FALSE(invokeParser("_ZGVnN2u_sin"));
+  EXPECT_FALSE(invokeParser("_ZGVnN2ul_sin"));
+}
+
+TEST_F(VFABIParserTest, ISAIndependentMangling) {
+  // This test makes sure that the mangling of the parameters in
+  // independent on the <isa> token.
+  const SmallVector<VFParameter, 8> ExpectedParams = {
+      VFParameter({0, VFParamKind::Vector, 0}),
+      VFParameter({1, VFParamKind::OMP_LinearPos, 2}),
+      VFParameter({2, VFParamKind::OMP_LinearValPos, 27}),
+      VFParameter({3, VFParamKind::OMP_LinearUValPos, 4}),
+      VFParameter({4, VFParamKind::OMP_LinearRefPos, 5}),
+      VFParameter({5, VFParamKind::OMP_Linear, 1}),
+      VFParameter({6, VFParamKind::OMP_LinearVal, 10}),
+      VFParameter({7, VFParamKind::OMP_LinearUVal, 100}),
+      VFParameter({8, VFParamKind::OMP_LinearRef, 1000}),
+      VFParameter({9, VFParamKind::OMP_Uniform, 2}),
+  };
+
+#define __COMMON_CHECKS                                                        \
+  do {                                                                         \
+    EXPECT_EQ(VF, (unsigned)2);                                                \
+    EXPECT_FALSE(IsMasked());                                                  \
+    EXPECT_FALSE(IsScalable);                                                  \
+    EXPECT_EQ(Parameters.size(), (unsigned)10);                                \
+    EXPECT_EQ(Parameters, ExpectedParams);                                     \
+    EXPECT_EQ(ScalarName, "sin");                                              \
+  } while (0)
+
+  // Advanced SIMD: <isa> = "n"
+  EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // SVE: <isa> = "s"
+  EXPECT_TRUE(invokeParser("_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::SVE);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // SSE: <isa> = "b"
+  EXPECT_TRUE(invokeParser("_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::SSE);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // AVX: <isa> = "c"
+  EXPECT_TRUE(invokeParser("_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // AVX2: <isa> = "d"
+  EXPECT_TRUE(invokeParser("_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX2);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // AVX512: <isa> = "e"
+  EXPECT_TRUE(invokeParser("_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::AVX512);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+  // Unknown ISA (randomly using "q"). This test will need update if
+  // some targets decide to use "q" as their ISA token.
+  EXPECT_TRUE(invokeParser("_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin"));
+  EXPECT_EQ(ISA, VFISAKind::Unknown);
+  __COMMON_CHECKS;
+  EXPECT_EQ(VectorName, "_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u2_sin");
+
+#undef __COMMON_CHECKS
+}
+
+TEST_F(VFABIParserTest, MissingScalarName) {
+  EXPECT_FALSE(invokeParser("_ZGVnN2v_"));
+}
+
+TEST_F(VFABIParserTest, MissingVectorName) {
+  EXPECT_FALSE(invokeParser("_ZGVnN2v_foo()"));
+}
+
+TEST_F(VFABIParserTest, MissingVectorNameTermination) {
+  EXPECT_FALSE(invokeParser("_ZGVnN2v_foo(bar"));
+}
+
+TEST_F(VFABIParserTest, ParseMaskingNEON) {
+  EXPECT_TRUE(invokeParser("_ZGVnM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
+
+TEST_F(VFABIParserTest, ParseMaskingSVE) {
+  EXPECT_TRUE(invokeParser("_ZGVsM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::SVE);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
+
+TEST_F(VFABIParserTest, ParseMaskingSSE) {
+  EXPECT_TRUE(invokeParser("_ZGVbM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::SSE);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
+
+TEST_F(VFABIParserTest, ParseMaskingAVX) {
+  EXPECT_TRUE(invokeParser("_ZGVcM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::AVX);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
+
+TEST_F(VFABIParserTest, ParseMaskingAVX2) {
+  EXPECT_TRUE(invokeParser("_ZGVdM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::AVX2);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
+
+TEST_F(VFABIParserTest, ParseMaskingAVX512) {
+  EXPECT_TRUE(invokeParser("_ZGVeM2v_sin"));
+  EXPECT_EQ(VF, (unsigned)2);
+  EXPECT_TRUE(IsMasked());
+  EXPECT_FALSE(IsScalable);
+  EXPECT_EQ(ISA, VFISAKind::AVX512);
+  EXPECT_EQ(Parameters.size(), (unsigned)2);
+  EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector}));
+  EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate}));
+  EXPECT_EQ(ScalarName, "sin");
+}
diff --git a/llvm/unittests/Bitcode/CMakeLists.txt b/llvm/unittests/Bitcode/CMakeLists.txt
index 7e9d1bc43fefe..e329e98226d96 100644
--- a/llvm/unittests/Bitcode/CMakeLists.txt
+++ b/llvm/unittests/Bitcode/CMakeLists.txt
@@ -8,4 +8,5 @@ set(LLVM_LINK_COMPONENTS
 
 add_llvm_unittest(BitcodeTests
   BitReaderTest.cpp
+  DataLayoutUpgradeTest.cpp
   )
diff --git a/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
new file mode 100644
index 0000000000000..c92418136e571
--- /dev/null
+++ b/llvm/unittests/Bitcode/DataLayoutUpgradeTest.cpp
@@ -0,0 +1,59 @@
+//===- DataLayoutUpgradeTest.cpp - Tests for DataLayout upgrades ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/IR/AutoUpgrade.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+namespace {
+
+TEST(DataLayoutUpgradeTest, ValidDataLayoutUpgrade) {
+  std::string DL1 =
+      UpgradeDataLayoutString("e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128",
+                              "x86_64-unknown-linux-gnu");
+  std::string DL2 = UpgradeDataLayoutString(
+      "e-m:w-p:32:32-i64:64-f80:32-n8:16:32-S32", "i686-pc-windows-msvc");
+  std::string DL3 = UpgradeDataLayoutString("e-m:o-i64:64-i128:128-n32:64-S128",
+                                            "x86_64-apple-macosx");
+  EXPECT_EQ(DL1, "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64"
+                 "-f80:128-n8:16:32:64-S128");
+  EXPECT_EQ(DL2, "e-m:w-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64"
+                 "-f80:32-n8:16:32-S32");
+  EXPECT_EQ(DL3, "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128"
+                 "-n32:64-S128");
+}
+
+TEST(DataLayoutUpgradeTest, NoDataLayoutUpgrade) {
+  std::string DL1 = UpgradeDataLayoutString(
+      "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32"
+      "-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128"
+      "-n8:16:32:64-S128",
+      "x86_64-unknown-linux-gnu");
+  std::string DL2 = UpgradeDataLayoutString("e-p:32:32", "i686-apple-darwin9");
+  std::string DL3 = UpgradeDataLayoutString("e-m:e-i64:64-n32:64",
+                                            "powerpc64le-unknown-linux-gnu");
+  std::string DL4 =
+      UpgradeDataLayoutString("e-m:o-i64:64-i128:128-n32:64-S128", "aarch64--");
+  EXPECT_EQ(DL1, "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64"
+                 "-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64"
+                 "-f80:128:128-n8:16:32:64-S128");
+  EXPECT_EQ(DL2, "e-p:32:32");
+  EXPECT_EQ(DL3, "e-m:e-i64:64-n32:64");
+  EXPECT_EQ(DL4, "e-m:o-i64:64-i128:128-n32:64-S128");
+}
+
+TEST(DataLayoutUpgradeTest, EmptyDataLayout) {
+  std::string DL1 = UpgradeDataLayoutString("", "x86_64-unknown-linux-gnu");
+  std::string DL2 = UpgradeDataLayoutString(
+      "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128", "");
+  EXPECT_EQ(DL1, "");
+  EXPECT_EQ(DL2, "e-m:e-p:32:32-i64:64-f80:128-n8:16:32:64-S128");
+}
+
+} // end namespace
diff --git a/llvm/unittests/CMakeLists.txt b/llvm/unittests/CMakeLists.txt
index 6bb2fb8eb9239..9384bdad0434a 100644
--- a/llvm/unittests/CMakeLists.txt
+++ b/llvm/unittests/CMakeLists.txt
@@ -31,8 +31,9 @@ add_subdirectory(Remarks)
 add_subdirectory(Passes)
 add_subdirectory(ProfileData)
 add_subdirectory(Support)
-add_subdirectory(TextAPI)
+add_subdirectory(TableGen)
 add_subdirectory(Target)
+add_subdirectory(TextAPI)
 add_subdirectory(Transforms)
 add_subdirectory(XRay)
 add_subdirectory(tools)
diff --git a/llvm/unittests/CodeGen/GlobalISel/GISelMITest.h b/llvm/unittests/CodeGen/GlobalISel/GISelMITest.h
index 0af63b70eec67..408fea7f1cbed 100644
--- a/llvm/unittests/CodeGen/GlobalISel/GISelMITest.h
+++ b/llvm/unittests/CodeGen/GlobalISel/GISelMITest.h
@@ -195,12 +195,11 @@ static inline bool CheckMachineFunction(const MachineFunction &MF,
   SM.AddNewSourceBuffer(MemoryBuffer::getMemBuffer(CheckFileText, "CheckFile"),
                         SMLoc());
   Regex PrefixRE = FC.buildCheckPrefixRegex();
-  std::vector<FileCheckString> CheckStrings;
-  if (FC.ReadCheckFile(SM, CheckFileText, PrefixRE, CheckStrings))
+  if (FC.readCheckFile(SM, CheckFileText, PrefixRE))
     return false;
 
   auto OutBuffer = OutputBuf->getBuffer();
   SM.AddNewSourceBuffer(std::move(OutputBuf), SMLoc());
-  return FC.CheckInput(SM, OutBuffer, CheckStrings);
+  return FC.checkInput(SM, OutBuffer);
 }
 #endif
diff --git a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
index 203c12afa6e6c..421544ee1d40d 100644
--- a/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
+++ b/llvm/unittests/DebugInfo/GSYM/GSYMTest.cpp
@@ -9,6 +9,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/DebugInfo/GSYM/Header.h"
 #include "llvm/DebugInfo/GSYM/FileEntry.h"
 #include "llvm/DebugInfo/GSYM/FileWriter.h"
 #include "llvm/DebugInfo/GSYM/FunctionInfo.h"
@@ -24,6 +25,23 @@
 using namespace llvm;
 using namespace gsym;
 
+void checkError(ArrayRef<std::string> ExpectedMsgs, Error Err) {
+  ASSERT_TRUE(bool(Err));
+  size_t WhichMsg = 0;
+  Error Remaining =
+      handleErrors(std::move(Err), [&](const ErrorInfoBase &Actual) {
+        ASSERT_LT(WhichMsg, ExpectedMsgs.size());
+        // Use .str(), because googletest doesn't visualise a StringRef
+        // properly.
+        EXPECT_EQ(Actual.message(), ExpectedMsgs[WhichMsg++]);
+      });
+  EXPECT_EQ(WhichMsg, ExpectedMsgs.size());
+  EXPECT_FALSE(Remaining);
+}
+
+void checkError(std::string ExpectedMsg, Error Err) {
+  checkError(ArrayRef<std::string>{ExpectedMsg}, std::move(Err));
+}
 TEST(GSYMTest, TestFileEntry) {
   // Make sure default constructed GSYM FileEntry has zeroes in the
   // directory and basename string table indexes.
@@ -137,22 +155,160 @@ TEST(GSYMTest, TestFunctionInfo) {
   EXPECT_LT(FIWithLines, FIWithLinesWithHigherAddress);
 }
 
-void checkError(ArrayRef<std::string> ExpectedMsgs, Error Err) {
-  ASSERT_TRUE(Err.operator bool());
-  size_t WhichMsg = 0;
-  Error Remaining =
-      handleErrors(std::move(Err), [&](const ErrorInfoBase &Actual) {
-        ASSERT_LT(WhichMsg, ExpectedMsgs.size());
-        // Use .str(), because googletest doesn't visualise a StringRef
-        // properly.
-        EXPECT_EQ(Actual.message(), ExpectedMsgs[WhichMsg++]);
-      });
-  EXPECT_EQ(WhichMsg, ExpectedMsgs.size());
-  EXPECT_FALSE(Remaining);
+static void TestFunctionInfoDecodeError(llvm::support::endianness ByteOrder,
+                                        std::string Bytes,
+                                        const uint64_t BaseAddr,
+                                        std::string ExpectedErrorMsg) {
+  uint8_t AddressSize = 4;
+  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  llvm::Expected<FunctionInfo> Decoded = FunctionInfo::decode(Data, BaseAddr);
+  // Make sure decoding fails.
+  ASSERT_FALSE((bool)Decoded);
+  // Make sure decoded object is the same as the one we encoded.
+  checkError(ExpectedErrorMsg, Decoded.takeError());
 }
 
-void checkError(std::string ExpectedMsg, Error Err) {
-  checkError(ArrayRef<std::string>{ExpectedMsg}, std::move(Err));
+TEST(GSYMTest, TestFunctionInfoDecodeErrors) {
+  // Test decoding FunctionInfo objects that ensure we report an appropriate
+  // error message.
+  const llvm::support::endianness ByteOrder = llvm::support::little;
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  const uint64_t BaseAddr = 0x100;
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x00000000: missing FunctionInfo Size");
+  FW.writeU32(0x100); // Function size.
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x00000004: missing FunctionInfo Name");
+  // Write out an invalid Name string table offset of zero.
+  FW.writeU32(0);
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x00000004: invalid FunctionInfo Name value 0x00000000");
+  // Modify the Name to be 0x00000001, which is a valid value.
+  FW.fixup32(0x00000001, 4);
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x00000008: missing FunctionInfo InfoType value");
+  auto FixupOffset = FW.tell();
+  FW.writeU32(1); // InfoType::LineTableInfo.
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x0000000c: missing FunctionInfo InfoType length");
+  FW.fixup32(4, FixupOffset); // Write an invalid InfoType enumeration value
+  FW.writeU32(0); // LineTableInfo InfoType data length.
+  TestFunctionInfoDecodeError(ByteOrder, OutStrm.str(), BaseAddr,
+      "0x00000008: unsupported InfoType 4");
+}
+
+static void TestFunctionInfoEncodeError(llvm::support::endianness ByteOrder,
+                                      const FunctionInfo &FI,
+                                      std::string ExpectedErrorMsg) {
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  Expected<uint64_t> ExpectedOffset = FI.encode(FW);
+  ASSERT_FALSE(ExpectedOffset);
+  checkError(ExpectedErrorMsg, ExpectedOffset.takeError());
+}
+
+TEST(GSYMTest, TestFunctionInfoEncodeErrors) {
+  const uint64_t FuncAddr = 0x1000;
+  const uint64_t FuncSize = 0x100;
+  const uint32_t InvalidName = 0;
+  const uint32_t ValidName = 1;
+  FunctionInfo InvalidNameFI(FuncAddr, FuncSize, InvalidName);
+  TestFunctionInfoEncodeError(llvm::support::little, InvalidNameFI,
+      "attempted to encode invalid FunctionInfo object");
+
+  FunctionInfo InvalidLineTableFI(FuncAddr, FuncSize, ValidName);
+  // Empty line tables are not valid. Verify if the encoding of anything
+  // in our line table fails, that we see get the error propagated.
+  InvalidLineTableFI.OptLineTable = LineTable();
+  TestFunctionInfoEncodeError(llvm::support::little, InvalidLineTableFI,
+      "attempted to encode invalid LineTable object");
+
+  FunctionInfo InvalidInlineInfoFI(FuncAddr, FuncSize, ValidName);
+  // Empty line tables are not valid. Verify if the encoding of anything
+  // in our line table fails, that we see get the error propagated.
+  InvalidInlineInfoFI.Inline = InlineInfo();
+  TestFunctionInfoEncodeError(llvm::support::little, InvalidInlineInfoFI,
+      "attempted to encode invalid InlineInfo object");
+}
+
+static void TestFunctionInfoEncodeDecode(llvm::support::endianness ByteOrder,
+                                         const FunctionInfo &FI) {
+  // Test encoding and decoding FunctionInfo objects.
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  llvm::Expected<uint64_t> ExpectedOffset = FI.encode(FW);
+  ASSERT_TRUE(bool(ExpectedOffset));
+  // Verify we got the encoded offset back from the encode function.
+  ASSERT_EQ(ExpectedOffset.get(), 0ULL);
+  std::string Bytes(OutStrm.str());
+  uint8_t AddressSize = 4;
+  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  llvm::Expected<FunctionInfo> Decoded = FunctionInfo::decode(Data,
+                                                              FI.Range.Start);
+  // Make sure decoding succeeded.
+  ASSERT_TRUE((bool)Decoded);
+  // Make sure decoded object is the same as the one we encoded.
+  EXPECT_EQ(FI, Decoded.get());
+}
+
+static void AddLines(uint64_t FuncAddr, uint32_t FileIdx, FunctionInfo &FI) {
+    FI.OptLineTable = LineTable();
+    LineEntry Line0(FuncAddr + 0x000, FileIdx, 10);
+    LineEntry Line1(FuncAddr + 0x010, FileIdx, 11);
+    LineEntry Line2(FuncAddr + 0x100, FileIdx, 1000);
+    FI.OptLineTable->push(Line0);
+    FI.OptLineTable->push(Line1);
+    FI.OptLineTable->push(Line2);
+}
+
+
+static void AddInline(uint64_t FuncAddr, uint64_t FuncSize, FunctionInfo &FI) {
+    FI.Inline = InlineInfo();
+    FI.Inline->Ranges.insert(AddressRange(FuncAddr, FuncAddr + FuncSize));
+    InlineInfo Inline1;
+    Inline1.Ranges.insert(AddressRange(FuncAddr + 0x10, FuncAddr + 0x30));
+    Inline1.Name = 1;
+    Inline1.CallFile = 1;
+    Inline1.CallLine = 11;
+    FI.Inline->Children.push_back(Inline1);
+}
+
+TEST(GSYMTest, TestFunctionInfoEncoding) {
+  constexpr uint64_t FuncAddr = 0x1000;
+  constexpr uint64_t FuncSize = 0x100;
+  constexpr uint32_t FuncName = 1;
+  constexpr uint32_t FileIdx = 1;
+  // Make sure that we can encode and decode a FunctionInfo with no line table
+  // or inline info.
+  FunctionInfo FI(FuncAddr, FuncSize, FuncName);
+  TestFunctionInfoEncodeDecode(llvm::support::little, FI);
+  TestFunctionInfoEncodeDecode(llvm::support::big, FI);
+
+  // Make sure that we can encode and decode a FunctionInfo with a line table
+  // and no inline info.
+  FunctionInfo FILines(FuncAddr, FuncSize, FuncName);
+  AddLines(FuncAddr, FileIdx, FILines);
+  TestFunctionInfoEncodeDecode(llvm::support::little, FILines);
+  TestFunctionInfoEncodeDecode(llvm::support::big, FILines);
+
+  // Make sure that we can encode and decode a FunctionInfo with no line table
+  // and with inline info.
+  FunctionInfo FIInline(FuncAddr, FuncSize, FuncName);
+  AddInline(FuncAddr, FuncSize, FIInline);
+  TestFunctionInfoEncodeDecode(llvm::support::little, FIInline);
+  TestFunctionInfoEncodeDecode(llvm::support::big, FIInline);
+
+  // Make sure that we can encode and decode a FunctionInfo with no line table
+  // and with inline info.
+  FunctionInfo FIBoth(FuncAddr, FuncSize, FuncName);
+  AddLines(FuncAddr, FileIdx, FIBoth);
+  AddInline(FuncAddr, FuncSize, FIBoth);
+  TestFunctionInfoEncodeDecode(llvm::support::little, FIBoth);
+  TestFunctionInfoEncodeDecode(llvm::support::big, FIBoth);
 }
 
 static void TestInlineInfoEncodeDecode(llvm::support::endianness ByteOrder,
@@ -789,3 +945,104 @@ TEST(GSYMTest, TestLineTableEncodeErrors) {
              LT.encode(FW, BaseAddr));
   LT.clear();
 }
+
+static void TestHeaderEncodeError(const Header &H,
+                                  std::string ExpectedErrorMsg) {
+  const support::endianness ByteOrder = llvm::support::little;
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  llvm::Error Err = H.encode(FW);
+  checkError(ExpectedErrorMsg, std::move(Err));
+}
+
+static void TestHeaderDecodeError(std::string Bytes,
+                                  std::string ExpectedErrorMsg) {
+  const support::endianness ByteOrder = llvm::support::little;
+  uint8_t AddressSize = 4;
+  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  llvm::Expected<Header> Decoded = Header::decode(Data);
+  // Make sure decoding fails.
+  ASSERT_FALSE((bool)Decoded);
+  // Make sure decoded object is the same as the one we encoded.
+  checkError(ExpectedErrorMsg, Decoded.takeError());
+}
+
+// Populate a GSYM header with valid values.
+static void InitHeader(Header &H) {
+  H.Magic = GSYM_MAGIC;
+  H.Version = GSYM_VERSION;
+  H.AddrOffSize = 4;
+  H.UUIDSize = 16;
+  H.BaseAddress = 0x1000;
+  H.NumAddresses = 1;
+  H.StrtabOffset= 0x2000;
+  H.StrtabSize = 0x1000;
+  for (size_t i=0; i<GSYM_MAX_UUID_SIZE; ++i) {
+    if (i < H.UUIDSize)
+      H.UUID[i] = i;
+    else
+      H.UUID[i] = 0;
+  }
+}
+
+TEST(GSYMTest, TestHeaderEncodeErrors) {
+  Header H;
+  InitHeader(H);
+  H.Magic = 12;
+  TestHeaderEncodeError(H, "invalid GSYM magic 0x0000000c");
+  InitHeader(H);
+  H.Version = 12;
+  TestHeaderEncodeError(H, "unsupported GSYM version 12");
+  InitHeader(H);
+  H.AddrOffSize = 12;
+  TestHeaderEncodeError(H, "invalid address offset size 12");
+  InitHeader(H);
+  H.UUIDSize = 128;
+  TestHeaderEncodeError(H, "invalid UUID size 128");
+}
+
+TEST(GSYMTest, TestHeaderDecodeErrors) {
+  const llvm::support::endianness ByteOrder = llvm::support::little;
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  Header H;
+  InitHeader(H);
+  llvm::Error Err = H.encode(FW);
+  ASSERT_FALSE(Err);
+  FW.fixup32(12, offsetof(Header, Magic));
+  TestHeaderDecodeError(OutStrm.str(), "invalid GSYM magic 0x0000000c");
+  FW.fixup32(GSYM_MAGIC, offsetof(Header, Magic));
+  FW.fixup32(12, offsetof(Header, Version));
+  TestHeaderDecodeError(OutStrm.str(), "unsupported GSYM version 12");
+  FW.fixup32(GSYM_VERSION, offsetof(Header, Version));
+  FW.fixup32(12, offsetof(Header, AddrOffSize));
+  TestHeaderDecodeError(OutStrm.str(), "invalid address offset size 12");
+  FW.fixup32(4, offsetof(Header, AddrOffSize));
+  FW.fixup32(128, offsetof(Header, UUIDSize));
+  TestHeaderDecodeError(OutStrm.str(), "invalid UUID size 128");
+}
+
+static void TestHeaderEncodeDecode(const Header &H,
+                                   support::endianness ByteOrder) {
+  uint8_t AddressSize = 4;
+  SmallString<512> Str;
+  raw_svector_ostream OutStrm(Str);
+  FileWriter FW(OutStrm, ByteOrder);
+  llvm::Error Err = H.encode(FW);
+  ASSERT_FALSE(Err);
+  std::string Bytes(OutStrm.str());
+  DataExtractor Data(Bytes, ByteOrder == llvm::support::little, AddressSize);
+  llvm::Expected<Header> Decoded = Header::decode(Data);
+  // Make sure decoding succeeded.
+  ASSERT_TRUE((bool)Decoded);
+  EXPECT_EQ(H, Decoded.get());
+
+}
+TEST(GSYMTest, TestHeaderEncodeDecode) {
+  Header H;
+  InitHeader(H);
+  TestHeaderEncodeDecode(H, llvm::support::little);
+  TestHeaderEncodeDecode(H, llvm::support::big);
+}
diff --git a/llvm/unittests/IR/BasicBlockTest.cpp b/llvm/unittests/IR/BasicBlockTest.cpp
index 95bc242ec30a6..ad446ee6ebab2 100644
--- a/llvm/unittests/IR/BasicBlockTest.cpp
+++ b/llvm/unittests/IR/BasicBlockTest.cpp
@@ -122,6 +122,9 @@ TEST(BasicBlockTest, TestInstructionsWithoutDebug) {
   CHECK_ITERATORS(BB1->instructionsWithoutDebug(), Exp);
   CHECK_ITERATORS(BBConst->instructionsWithoutDebug(), Exp);
 
+  EXPECT_EQ(static_cast<size_t>(BB1->sizeWithoutDebug()), Exp.size());
+  EXPECT_EQ(static_cast<size_t>(BBConst->sizeWithoutDebug()), Exp.size());
+
   delete M;
   delete V;
 }
diff --git a/llvm/unittests/IR/ConstantRangeTest.cpp b/llvm/unittests/IR/ConstantRangeTest.cpp
index c0166b21039c8..7c43d2fb09cc9 100644
--- a/llvm/unittests/IR/ConstantRangeTest.cpp
+++ b/llvm/unittests/IR/ConstantRangeTest.cpp
@@ -669,6 +669,262 @@ TEST_F(ConstantRangeTest, AddWithNoSignedWrap) {
             ConstantRange(APInt(8, 110), APInt(8, INT8_MIN-10)));
 }
 
+template <typename Fn1, typename Fn2>
+static void TestAddWithNoSignedWrapExhaustive(Fn1 RangeFn, Fn2 IntFn) {
+  unsigned Bits = 4;
+  EnumerateTwoConstantRanges(Bits, [&](const ConstantRange &CR1,
+                                       const ConstantRange &CR2) {
+    ConstantRange CR = RangeFn(CR1, CR2);
+    APInt Min = APInt::getSignedMaxValue(Bits);
+    APInt Max = APInt::getSignedMinValue(Bits);
+    ForeachNumInConstantRange(CR1, [&](const APInt &N1) {
+      ForeachNumInConstantRange(CR2, [&](const APInt &N2) {
+        bool IsOverflow = false;
+        APInt N = IntFn(IsOverflow, N1, N2);
+        if (!IsOverflow) {
+          if (N.slt(Min))
+            Min = N;
+          if (N.sgt(Max))
+            Max = N;
+          EXPECT_TRUE(CR.contains(N));
+        }
+      });
+    });
+    if (!CR1.isSignWrappedSet() && !CR2.isSignWrappedSet()) {
+      if (Min.sgt(Max)) {
+        EXPECT_TRUE(CR.isEmptySet());
+        return;
+      }
+
+      ConstantRange Exact = ConstantRange::getNonEmpty(Min, Max + 1);
+      EXPECT_EQ(Exact, CR);
+    }
+  });
+}
+
+template <typename Fn1, typename Fn2>
+static void TestAddWithNoUnsignedWrapExhaustive(Fn1 RangeFn, Fn2 IntFn) {
+  unsigned Bits = 4;
+  EnumerateTwoConstantRanges(Bits, [&](const ConstantRange &CR1,
+                                       const ConstantRange &CR2) {
+    ConstantRange CR = RangeFn(CR1, CR2);
+    APInt Min = APInt::getMaxValue(Bits);
+    APInt Max = APInt::getMinValue(Bits);
+    ForeachNumInConstantRange(CR1, [&](const APInt &N1) {
+      ForeachNumInConstantRange(CR2, [&](const APInt &N2) {
+        bool IsOverflow = false;
+        APInt N = IntFn(IsOverflow, N1, N2);
+        if (!IsOverflow) {
+          if (N.ult(Min))
+            Min = N;
+          if (N.ugt(Max))
+            Max = N;
+          EXPECT_TRUE(CR.contains(N));
+        }
+      });
+    });
+
+    if (!CR1.isWrappedSet() && !CR2.isWrappedSet()) {
+      if (Min.ugt(Max)) {
+        EXPECT_TRUE(CR.isEmptySet());
+        return;
+      }
+
+      ConstantRange Exact = ConstantRange::getNonEmpty(Min, Max + 1);
+      EXPECT_EQ(Exact, CR);
+    }
+  });
+}
+
+template <typename Fn1, typename Fn2, typename Fn3>
+static void TestAddWithNoSignedUnsignedWrapExhaustive(Fn1 RangeFn,
+                                                      Fn2 IntFnSigned,
+                                                      Fn3 IntFnUnsigned) {
+  unsigned Bits = 4;
+  EnumerateTwoConstantRanges(
+      Bits, [&](const ConstantRange &CR1, const ConstantRange &CR2) {
+        ConstantRange CR = RangeFn(CR1, CR2);
+        APInt UMin = APInt::getMaxValue(Bits);
+        APInt UMax = APInt::getMinValue(Bits);
+        APInt SMin = APInt::getSignedMaxValue(Bits);
+        APInt SMax = APInt::getSignedMinValue(Bits);
+        ForeachNumInConstantRange(CR1, [&](const APInt &N1) {
+          ForeachNumInConstantRange(CR2, [&](const APInt &N2) {
+            bool IsOverflow = false, IsSignedOverflow = false;
+            APInt N = IntFnSigned(IsSignedOverflow, N1, N2);
+            (void) IntFnUnsigned(IsOverflow, N1, N2);
+            if (!IsSignedOverflow && !IsOverflow) {
+              if (N.slt(SMin))
+                SMin = N;
+              if (N.sgt(SMax))
+                SMax = N;
+              if (N.ult(UMin))
+                UMin = N;
+              if (N.ugt(UMax))
+                UMax = N;
+              EXPECT_TRUE(CR.contains(N));
+            }
+          });
+        });
+
+        if (!CR1.isWrappedSet() && !CR2.isWrappedSet() &&
+            !CR1.isSignWrappedSet() && !CR2.isSignWrappedSet()) {
+          if (UMin.ugt(UMax) || SMin.sgt(SMax)) {
+            EXPECT_TRUE(CR.isEmptySet());
+            return;
+          }
+
+          ConstantRange Exact =
+              ConstantRange::getNonEmpty(SMin, SMax + 1)
+                  .intersectWith(ConstantRange::getNonEmpty(UMin, UMax + 1));
+          EXPECT_EQ(Exact, CR);
+        }
+      });
+}
+
+TEST_F(ConstantRangeTest, AddWithNoWrap) {
+  typedef OverflowingBinaryOperator OBO;
+  EXPECT_EQ(Empty.addWithNoWrap(Some, OBO::NoSignedWrap), Empty);
+  EXPECT_EQ(Some.addWithNoWrap(Empty, OBO::NoSignedWrap), Empty);
+  EXPECT_EQ(Full.addWithNoWrap(Full, OBO::NoSignedWrap), Full);
+  EXPECT_NE(Full.addWithNoWrap(Some, OBO::NoSignedWrap), Full);
+  EXPECT_NE(Some.addWithNoWrap(Full, OBO::NoSignedWrap), Full);
+  EXPECT_EQ(Full.addWithNoWrap(ConstantRange(APInt(16, 1), APInt(16, 2)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(16, INT16_MIN + 1), APInt(16, INT16_MIN)));
+  EXPECT_EQ(ConstantRange(APInt(16, 1), APInt(16, 2))
+                .addWithNoWrap(Full, OBO::NoSignedWrap),
+            ConstantRange(APInt(16, INT16_MIN + 1), APInt(16, INT16_MIN)));
+  EXPECT_EQ(Full.addWithNoWrap(ConstantRange(APInt(16, -1), APInt(16, 0)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(16, INT16_MIN), APInt(16, INT16_MAX)));
+  EXPECT_EQ(ConstantRange(APInt(8, 100), APInt(8, 120))
+                .addWithNoWrap(ConstantRange(APInt(8, 120), APInt(8, 123)),
+                               OBO::NoSignedWrap),
+            ConstantRange(8, false));
+  EXPECT_EQ(ConstantRange(APInt(8, -120), APInt(8, -100))
+                .addWithNoWrap(ConstantRange(APInt(8, -110), APInt(8, -100)),
+                               OBO::NoSignedWrap),
+            ConstantRange(8, false));
+  EXPECT_EQ(ConstantRange(APInt(8, 0), APInt(8, 101))
+                .addWithNoWrap(ConstantRange(APInt(8, -128), APInt(8, 28)),
+                               OBO::NoSignedWrap),
+            ConstantRange(8, true));
+  EXPECT_EQ(ConstantRange(APInt(8, 0), APInt(8, 101))
+                .addWithNoWrap(ConstantRange(APInt(8, -120), APInt(8, 29)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, -120), APInt(8, -128)));
+  EXPECT_EQ(ConstantRange(APInt(8, -50), APInt(8, 50))
+                .addWithNoWrap(ConstantRange(APInt(8, 10), APInt(8, 20)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, -40), APInt(8, 69)));
+  EXPECT_EQ(ConstantRange(APInt(8, 10), APInt(8, 20))
+                .addWithNoWrap(ConstantRange(APInt(8, -50), APInt(8, 50)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, -40), APInt(8, 69)));
+  EXPECT_EQ(ConstantRange(APInt(8, 120), APInt(8, -10))
+                .addWithNoWrap(ConstantRange(APInt(8, 5), APInt(8, 20)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, 125), APInt(8, 9)));
+  EXPECT_EQ(ConstantRange(APInt(8, 5), APInt(8, 20))
+                .addWithNoWrap(ConstantRange(APInt(8, 120), APInt(8, -10)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, 125), APInt(8, 9)));
+
+  TestAddWithNoSignedWrapExhaustive(
+      [](const ConstantRange &CR1, const ConstantRange &CR2) {
+        return CR1.addWithNoWrap(CR2, OBO::NoSignedWrap);
+      },
+      [](bool &IsOverflow, const APInt &N1, const APInt &N2) {
+        return N1.sadd_ov(N2, IsOverflow);
+      });
+
+  EXPECT_EQ(Empty.addWithNoWrap(Some, OBO::NoUnsignedWrap), Empty);
+  EXPECT_EQ(Some.addWithNoWrap(Empty, OBO::NoUnsignedWrap), Empty);
+  EXPECT_EQ(Full.addWithNoWrap(Full, OBO::NoUnsignedWrap), Full);
+  EXPECT_NE(Full.addWithNoWrap(Some, OBO::NoUnsignedWrap), Full);
+  EXPECT_NE(Some.addWithNoWrap(Full, OBO::NoUnsignedWrap), Full);
+  EXPECT_EQ(Full.addWithNoWrap(ConstantRange(APInt(16, 1), APInt(16, 2)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(16, 1), APInt(16, 0)));
+  EXPECT_EQ(ConstantRange(APInt(16, 1), APInt(16, 2))
+                .addWithNoWrap(Full, OBO::NoUnsignedWrap),
+            ConstantRange(APInt(16, 1), APInt(16, 0)));
+  EXPECT_EQ(ConstantRange(APInt(8, 200), APInt(8, 220))
+                .addWithNoWrap(ConstantRange(APInt(8, 100), APInt(8, 123)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(8, false));
+  EXPECT_EQ(ConstantRange(APInt(8, 0), APInt(8, 101))
+                .addWithNoWrap(ConstantRange(APInt(8, 0), APInt(8, 156)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(8, true));
+  EXPECT_EQ(ConstantRange(APInt(8, 0), APInt(8, 101))
+                .addWithNoWrap(ConstantRange(APInt(8, 10), APInt(8, 29)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 10), APInt(8, 129)));
+  EXPECT_EQ(ConstantRange(APInt(8, 20), APInt(8, 10))
+                .addWithNoWrap(ConstantRange(APInt(8, 50), APInt(8, 200)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 50), APInt(8, 0)));
+  EXPECT_EQ(ConstantRange(APInt(8, 10), APInt(8, 20))
+                .addWithNoWrap(ConstantRange(APInt(8, 50), APInt(8, 200)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 60), APInt(8, -37)));
+  EXPECT_EQ(ConstantRange(APInt(8, 20), APInt(8, -30))
+                .addWithNoWrap(ConstantRange(APInt(8, 5), APInt(8, 20)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 25), APInt(8, -11)));
+  EXPECT_EQ(ConstantRange(APInt(8, 5), APInt(8, 20))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, -30)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 25), APInt(8, -11)));
+
+  TestAddWithNoUnsignedWrapExhaustive(
+      [](const ConstantRange &CR1, const ConstantRange &CR2) {
+        return CR1.addWithNoWrap(CR2, OBO::NoUnsignedWrap);
+      },
+      [](bool &IsOverflow, const APInt &N1, const APInt &N2) {
+        return N1.uadd_ov(N2, IsOverflow);
+      });
+
+  EXPECT_EQ(ConstantRange(APInt(8, 50), APInt(8, 100))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 70)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, 70), APInt(8, -128)));
+  EXPECT_EQ(ConstantRange(APInt(8, 50), APInt(8, 100))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 70)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 70), APInt(8, 169)));
+  EXPECT_EQ(ConstantRange(APInt(8, 50), APInt(8, 100))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 70)),
+                               OBO::NoUnsignedWrap | OBO::NoSignedWrap),
+            ConstantRange(APInt(8, 70), APInt(8, -128)));
+
+  EXPECT_EQ(ConstantRange(APInt(8, -100), APInt(8, -50))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 30)),
+                               OBO::NoSignedWrap),
+            ConstantRange(APInt(8, -80), APInt(8, -21)));
+  EXPECT_EQ(ConstantRange(APInt(8, -100), APInt(8, -50))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 30)),
+                               OBO::NoUnsignedWrap),
+            ConstantRange(APInt(8, 176), APInt(8, 235)));
+  EXPECT_EQ(ConstantRange(APInt(8, -100), APInt(8, -50))
+                .addWithNoWrap(ConstantRange(APInt(8, 20), APInt(8, 30)),
+                               OBO::NoUnsignedWrap | OBO::NoSignedWrap),
+            ConstantRange(APInt(8, 176), APInt(8, 235)));
+
+  TestAddWithNoSignedUnsignedWrapExhaustive(
+      [](const ConstantRange &CR1, const ConstantRange &CR2) {
+        return CR1.addWithNoWrap(CR2, OBO::NoUnsignedWrap | OBO::NoSignedWrap);
+      },
+      [](bool &IsOverflow, const APInt &N1, const APInt &N2) {
+        return N1.sadd_ov(N2, IsOverflow);
+      },
+      [](bool &IsOverflow, const APInt &N1, const APInt &N2) {
+        return N1.uadd_ov(N2, IsOverflow);
+      });
+}
+
 TEST_F(ConstantRangeTest, Sub) {
   EXPECT_EQ(Full.sub(APInt(16, 4)), Full);
   EXPECT_EQ(Full.sub(Full), Full);
diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp
index b2ea3844d335b..afd579400ff33 100644
--- a/llvm/unittests/IR/InstructionsTest.cpp
+++ b/llvm/unittests/IR/InstructionsTest.cpp
@@ -1034,13 +1034,16 @@ TEST(InstructionsTest, SkipDebug) {
   EXPECT_EQ(nullptr, Term->getNextNonDebugInstruction());
 }
 
-TEST(InstructionsTest, PhiIsNotFPMathOperator) {
+TEST(InstructionsTest, PhiMightNotBeFPMathOperator) {
   LLVMContext Context;
   IRBuilder<> Builder(Context);
   MDBuilder MDHelper(Context);
-  Instruction *I = Builder.CreatePHI(Builder.getDoubleTy(), 0);
+  Instruction *I = Builder.CreatePHI(Builder.getInt32Ty(), 0);
   EXPECT_FALSE(isa<FPMathOperator>(I));
   I->deleteValue();
+  Instruction *FP = Builder.CreatePHI(Builder.getDoubleTy(), 0);
+  EXPECT_TRUE(isa<FPMathOperator>(FP));
+  FP->deleteValue();
 }
 
 TEST(InstructionsTest, FNegInstruction) {
diff --git a/llvm/unittests/IR/PatternMatch.cpp b/llvm/unittests/IR/PatternMatch.cpp
index fe8c518fa82ef..c2c38b73a2789 100644
--- a/llvm/unittests/IR/PatternMatch.cpp
+++ b/llvm/unittests/IR/PatternMatch.cpp
@@ -471,6 +471,42 @@ TEST_F(PatternMatchTest, Unless) {
   EXPECT_FALSE(m_Unless(m_c_Add(m_Zero(), m_One())).match(X));
 }
 
+TEST_F(PatternMatchTest, ZExtSExtSelf) {
+  LLVMContext &Ctx = IRB.getContext();
+
+  Value *One32 = IRB.getInt32(1);
+  Value *One64Z = IRB.CreateZExt(One32, IntegerType::getInt64Ty(Ctx));
+  Value *One64S = IRB.CreateSExt(One32, IntegerType::getInt64Ty(Ctx));
+
+  EXPECT_TRUE(m_One().match(One32));
+  EXPECT_FALSE(m_One().match(One64Z));
+  EXPECT_FALSE(m_One().match(One64S));
+
+  EXPECT_FALSE(m_ZExt(m_One()).match(One32));
+  EXPECT_TRUE(m_ZExt(m_One()).match(One64Z));
+  EXPECT_FALSE(m_ZExt(m_One()).match(One64S));
+
+  EXPECT_FALSE(m_SExt(m_One()).match(One32));
+  EXPECT_FALSE(m_SExt(m_One()).match(One64Z));
+  EXPECT_TRUE(m_SExt(m_One()).match(One64S));
+
+  EXPECT_TRUE(m_ZExtOrSelf(m_One()).match(One32));
+  EXPECT_TRUE(m_ZExtOrSelf(m_One()).match(One64Z));
+  EXPECT_FALSE(m_ZExtOrSelf(m_One()).match(One64S));
+
+  EXPECT_TRUE(m_SExtOrSelf(m_One()).match(One32));
+  EXPECT_FALSE(m_SExtOrSelf(m_One()).match(One64Z));
+  EXPECT_TRUE(m_SExtOrSelf(m_One()).match(One64S));
+
+  EXPECT_FALSE(m_ZExtOrSExt(m_One()).match(One32));
+  EXPECT_TRUE(m_ZExtOrSExt(m_One()).match(One64Z));
+  EXPECT_TRUE(m_ZExtOrSExt(m_One()).match(One64S));
+
+  EXPECT_TRUE(m_ZExtOrSExtOrSelf(m_One()).match(One32));
+  EXPECT_TRUE(m_ZExtOrSExtOrSelf(m_One()).match(One64Z));
+  EXPECT_TRUE(m_ZExtOrSExtOrSelf(m_One()).match(One64S));
+}
+
 TEST_F(PatternMatchTest, Power2) {
   Value *C128 = IRB.getInt32(128);
   Value *CNeg128 = ConstantExpr::getNeg(cast<Constant>(C128));
@@ -1045,6 +1081,34 @@ TEST_F(PatternMatchTest, FloatingPointFNeg) {
   EXPECT_FALSE(match(V3, m_FNeg(m_Value(Match))));
 }
 
+TEST_F(PatternMatchTest, CondBranchTest) {
+  BasicBlock *TrueBB = BasicBlock::Create(Ctx, "TrueBB", F);
+  BasicBlock *FalseBB = BasicBlock::Create(Ctx, "FalseBB", F);
+  Value *Br1 = IRB.CreateCondBr(IRB.getTrue(), TrueBB, FalseBB);
+
+  EXPECT_TRUE(match(Br1, m_Br(m_Value(), m_BasicBlock(), m_BasicBlock())));
+
+  BasicBlock *A, *B;
+  EXPECT_TRUE(match(Br1, m_Br(m_Value(), m_BasicBlock(A), m_BasicBlock(B))));
+  EXPECT_EQ(TrueBB, A);
+  EXPECT_EQ(FalseBB, B);
+
+  EXPECT_FALSE(
+      match(Br1, m_Br(m_Value(), m_SpecificBB(FalseBB), m_BasicBlock())));
+  EXPECT_FALSE(
+      match(Br1, m_Br(m_Value(), m_BasicBlock(), m_SpecificBB(TrueBB))));
+  EXPECT_FALSE(
+      match(Br1, m_Br(m_Value(), m_SpecificBB(FalseBB), m_BasicBlock(TrueBB))));
+  EXPECT_TRUE(
+      match(Br1, m_Br(m_Value(), m_SpecificBB(TrueBB), m_BasicBlock(FalseBB))));
+
+  // Check we can use m_Deferred with branches.
+  EXPECT_FALSE(match(Br1, m_Br(m_Value(), m_BasicBlock(A), m_Deferred(A))));
+  Value *Br2 = IRB.CreateCondBr(IRB.getTrue(), TrueBB, TrueBB);
+  A = nullptr;
+  EXPECT_TRUE(match(Br2, m_Br(m_Value(), m_BasicBlock(A), m_Deferred(A))));
+}
+
 template <typename T> struct MutableConstTest : PatternMatchTest { };
 
 typedef ::testing::Types<std::tuple<Value*, Instruction*>,
diff --git a/llvm/unittests/MI/LiveIntervalTest.cpp b/llvm/unittests/MI/LiveIntervalTest.cpp
index 5ee9d13dbd923..c81f13f6e67f1 100644
--- a/llvm/unittests/MI/LiveIntervalTest.cpp
+++ b/llvm/unittests/MI/LiveIntervalTest.cpp
@@ -63,10 +63,10 @@ std::unique_ptr<Module> parseMIR(LLVMContext &Context,
 
   M->setDataLayout(TM.createDataLayout());
 
-  MachineModuleInfo *MMI = new MachineModuleInfo(&TM);
-  if (MIR->parseMachineFunctions(*M, *MMI))
+  MachineModuleInfoWrapperPass *MMIWP = new MachineModuleInfoWrapperPass(&TM);
+  if (MIR->parseMachineFunctions(*M, MMIWP->getMMI()))
     return nullptr;
-  PM.add(MMI);
+  PM.add(MMIWP);
 
   return M;
 }
@@ -309,8 +309,8 @@ TEST(LiveIntervalTest, MoveUndefUse) {
 
 TEST(LiveIntervalTest, MoveUpValNos) {
   // handleMoveUp() had a bug where it would reuse the value number of the
-  // destination segment, even though we have no guarntee that this valno wasn't
-  // used in other segments.
+  // destination segment, even though we have no guarantee that this valno
+  // wasn't used in other segments.
   liveIntervalTest(R"MIR(
     successors: %bb.1, %bb.2
     %0 = IMPLICIT_DEF
diff --git a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
index be923e865954d..0ebb47809f562 100644
--- a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
+++ b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
@@ -22,21 +22,18 @@
 
 using namespace llvm;
 
-static void check(remarks::SerializerMode Mode, ArrayRef<remarks::Remark> Rs,
+static void check(remarks::Format SerializerFormat,
+                  remarks::SerializerMode Mode, ArrayRef<remarks::Remark> Rs,
                   StringRef ExpectedR, Optional<StringRef> ExpectedMeta,
-                  bool UseStrTab = false,
                   Optional<remarks::StringTable> StrTab = None) {
   std::string Buf;
   raw_string_ostream OS(Buf);
   Expected<std::unique_ptr<remarks::RemarkSerializer>> MaybeS = [&] {
-    if (UseStrTab) {
-      if (StrTab)
-        return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS,
-                                      std::move(*StrTab));
-      else
-        return createRemarkSerializer(remarks::Format::YAMLStrTab, Mode, OS);
-    } else
-      return createRemarkSerializer(remarks::Format::YAML, Mode, OS);
+    if (StrTab)
+      return createRemarkSerializer(SerializerFormat, Mode, OS,
+                                    std::move(*StrTab));
+    else
+      return createRemarkSerializer(SerializerFormat, Mode, OS);
   }();
   EXPECT_FALSE(errorToBool(MaybeS.takeError()));
   std::unique_ptr<remarks::RemarkSerializer> S = std::move(*MaybeS);
@@ -54,18 +51,20 @@ static void check(remarks::SerializerMode Mode, ArrayRef<remarks::Remark> Rs,
   }
 }
 
-static void check(const remarks::Remark &R, StringRef ExpectedR,
-                  StringRef ExpectedMeta, bool UseStrTab = false,
+static void check(remarks::Format SerializerFormat, const remarks::Remark &R,
+                  StringRef ExpectedR, StringRef ExpectedMeta,
                   Optional<remarks::StringTable> StrTab = None) {
-  return check(remarks::SerializerMode::Separate, makeArrayRef(&R, &R + 1), ExpectedR, ExpectedMeta,
-               UseStrTab, std::move(StrTab));
+  return check(SerializerFormat, remarks::SerializerMode::Separate,
+               makeArrayRef(&R, &R + 1), ExpectedR, ExpectedMeta,
+               std::move(StrTab));
 }
 
-static void checkStandalone(const remarks::Remark &R, StringRef ExpectedR,
+static void checkStandalone(remarks::Format SerializerFormat,
+                            const remarks::Remark &R, StringRef ExpectedR,
                             Optional<remarks::StringTable> StrTab = None) {
-  bool UseStrTab = StrTab.hasValue();
-  return check(remarks::SerializerMode::Standalone, makeArrayRef(&R, &R +1), ExpectedR,
-               /*ExpectedMeta=*/None, UseStrTab, std::move(StrTab));
+  return check(SerializerFormat, remarks::SerializerMode::Standalone,
+               makeArrayRef(&R, &R + 1), ExpectedR,
+               /*ExpectedMeta=*/None, std::move(StrTab));
 }
 
 TEST(YAMLRemarks, SerializerRemark) {
@@ -83,7 +82,7 @@ TEST(YAMLRemarks, SerializerRemark) {
   R.Args.back().Key = "keydebug";
   R.Args.back().Val = "valuedebug";
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  check(R,
+  check(remarks::Format::YAML, R,
         "--- !Missed\n"
         "Pass:            pass\n"
         "Name:            name\n"
@@ -97,8 +96,7 @@ TEST(YAMLRemarks, SerializerRemark) {
         "...\n",
         StringRef("REMARKS\0"
                   "\0\0\0\0\0\0\0\0"
-                  "\0\0\0\0\0\0\0\0"
-                  EXTERNALFILETESTPATH"\0",
+                  "\0\0\0\0\0\0\0\0" EXTERNALFILETESTPATH "\0",
                   38));
 }
 
@@ -118,7 +116,7 @@ TEST(YAMLRemarks, SerializerRemarkStandalone) {
   R.Args.back().Val = "valuedebug";
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
   checkStandalone(
-      R,
+      remarks::Format::YAML, R,
       StringRef("--- !Missed\n"
                 "Pass:            pass\n"
                 "Name:            name\n"
@@ -147,7 +145,7 @@ TEST(YAMLRemarks, SerializerRemarkStrTab) {
   R.Args.back().Key = "keydebug";
   R.Args.back().Val = "valuedebug";
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  check(R,
+  check(remarks::Format::YAMLStrTab, R,
         "--- !Missed\n"
         "Pass:            0\n"
         "Name:            1\n"
@@ -162,10 +160,9 @@ TEST(YAMLRemarks, SerializerRemarkStrTab) {
         StringRef("REMARKS\0"
                   "\0\0\0\0\0\0\0\0"
                   "\x2d\0\0\0\0\0\0\0"
-                  "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
-                  EXTERNALFILETESTPATH"\0",
-                  83),
-        /*UseStrTab=*/true);
+                  "pass\0name\0func\0path\0value\0valuedebug\0argpath"
+                  "\0" EXTERNALFILETESTPATH "\0",
+                  83));
 }
 
 TEST(YAMLRemarks, SerializerRemarkParsedStrTab) {
@@ -184,7 +181,7 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTab) {
   R.Args.back().Key = "keydebug";
   R.Args.back().Val = "valuedebug";
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  check(R,
+  check(remarks::Format::YAMLStrTab, R,
         "--- !Missed\n"
         "Pass:            0\n"
         "Name:            1\n"
@@ -199,13 +196,47 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTab) {
         StringRef("REMARKS\0"
                   "\0\0\0\0\0\0\0\0"
                   "\x2d\0\0\0\0\0\0\0"
-                  "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
-                  EXTERNALFILETESTPATH"\0",
+                  "pass\0name\0func\0path\0value\0valuedebug\0argpath"
+                  "\0" EXTERNALFILETESTPATH "\0",
                   83),
-        /*UseStrTab=*/true,
         remarks::StringTable(remarks::ParsedStringTable(StrTab)));
 }
 
+TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneNoStrTab) {
+  // Check that we don't use the string table even if it was provided.
+  StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
+  remarks::ParsedStringTable ParsedStrTab(StrTab);
+  remarks::StringTable PreFilledStrTab(ParsedStrTab);
+  remarks::Remark R;
+  R.RemarkType = remarks::Type::Missed;
+  R.PassName = "pass";
+  R.RemarkName = "name";
+  R.FunctionName = "func";
+  R.Loc = remarks::RemarkLocation{"path", 3, 4};
+  R.Hotness = 5;
+  R.Args.emplace_back();
+  R.Args.back().Key = "key";
+  R.Args.back().Val = "value";
+  R.Args.emplace_back();
+  R.Args.back().Key = "keydebug";
+  R.Args.back().Val = "valuedebug";
+  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
+  checkStandalone(
+      remarks::Format::YAML, R,
+      StringRef("--- !Missed\n"
+                "Pass:            pass\n"
+                "Name:            name\n"
+                "DebugLoc:        { File: path, Line: 3, Column: 4 }\n"
+                "Function:        func\n"
+                "Hotness:         5\n"
+                "Args:\n"
+                "  - key:             value\n"
+                "  - keydebug:        valuedebug\n"
+                "    DebugLoc:        { File: argpath, Line: 6, Column: 7 }\n"
+                "...\n"),
+      std::move(PreFilledStrTab));
+}
+
 TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) {
   StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
   remarks::ParsedStringTable ParsedStrTab(StrTab);
@@ -225,7 +256,7 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) {
   R.Args.back().Val = "valuedebug";
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
   checkStandalone(
-      R,
+      remarks::Format::YAMLStrTab, R,
       StringRef("REMARKS\0"
                 "\0\0\0\0\0\0\0\0"
                 "\x2d\0\0\0\0\0\0\0"
@@ -266,7 +297,7 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneMultipleRemarks) {
   R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
   Rs.emplace_back(R.clone());
   Rs.emplace_back(std::move(R));
-  check(remarks::SerializerMode::Standalone, Rs,
+  check(remarks::Format::YAMLStrTab, remarks::SerializerMode::Standalone, Rs,
         StringRef("REMARKS\0"
                   "\0\0\0\0\0\0\0\0"
                   "\x2d\0\0\0\0\0\0\0"
@@ -294,6 +325,5 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneMultipleRemarks) {
                   "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
                   "...\n",
                   561),
-        /*ExpectedMeta=*/None,
-        /*UseStrTab=*/true, std::move(PreFilledStrTab));
+        /*ExpectedMeta=*/None, std::move(PreFilledStrTab));
 }
diff --git a/llvm/unittests/Support/AlignmentTest.cpp b/llvm/unittests/Support/AlignmentTest.cpp
index 420772541d4a5..0b1435912b930 100644
--- a/llvm/unittests/Support/AlignmentTest.cpp
+++ b/llvm/unittests/Support/AlignmentTest.cpp
@@ -28,7 +28,10 @@ std::vector<uint64_t> getValidAlignments() {
   return Out;
 }
 
-TEST(AlignmentTest, AlignDefaultCTor) { EXPECT_EQ(Align().value(), 1ULL); }
+TEST(AlignmentTest, AlignDefaultCTor) {
+  EXPECT_EQ(Align().value(), 1ULL);
+  EXPECT_EQ(Align::None().value(), 1ULL);
+}
 
 TEST(AlignmentTest, MaybeAlignDefaultCTor) {
   EXPECT_FALSE(MaybeAlign().hasValue());
@@ -224,6 +227,29 @@ TEST(AlignmentTest, AlignComparisons) {
   }
 }
 
+TEST(AlignmentTest, Max) {
+  // We introduce std::max here to test ADL.
+  using std::max;
+
+  // Uses llvm::max.
+  EXPECT_EQ(max(MaybeAlign(), Align(2)), Align(2));
+  EXPECT_EQ(max(Align(2), MaybeAlign()), Align(2));
+
+  EXPECT_EQ(max(MaybeAlign(1), Align(2)), Align(2));
+  EXPECT_EQ(max(Align(2), MaybeAlign(1)), Align(2));
+
+  EXPECT_EQ(max(MaybeAlign(2), Align(2)), Align(2));
+  EXPECT_EQ(max(Align(2), MaybeAlign(2)), Align(2));
+
+  EXPECT_EQ(max(MaybeAlign(4), Align(2)), Align(4));
+  EXPECT_EQ(max(Align(2), MaybeAlign(4)), Align(4));
+
+  // Uses std::max.
+  EXPECT_EQ(max(Align(2), Align(4)), Align(4));
+  EXPECT_EQ(max(MaybeAlign(2), MaybeAlign(4)), MaybeAlign(4));
+  EXPECT_EQ(max(MaybeAlign(), MaybeAlign()), MaybeAlign());
+}
+
 TEST(AlignmentTest, AssumeAligned) {
   EXPECT_EQ(assumeAligned(0), Align(1));
   EXPECT_EQ(assumeAligned(0), Align());
diff --git a/llvm/unittests/Support/FileCheckTest.cpp b/llvm/unittests/Support/FileCheckTest.cpp
index fd1df57ca73c1..3fbc06b467be8 100644
--- a/llvm/unittests/Support/FileCheckTest.cpp
+++ b/llvm/unittests/Support/FileCheckTest.cpp
@@ -7,10 +7,12 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/FileCheck.h"
+#include "../lib/Support/FileCheckImpl.h"
 #include "gtest/gtest.h"
 #include <unordered_set>
 
 using namespace llvm;
+
 namespace {
 
 class FileCheckTest : public ::testing::Test {};
diff --git a/llvm/unittests/TableGen/CMakeLists.txt b/llvm/unittests/TableGen/CMakeLists.txt
new file mode 100644
index 0000000000000..47bde04cad0db
--- /dev/null
+++ b/llvm/unittests/TableGen/CMakeLists.txt
@@ -0,0 +1,10 @@
+set(LLVM_LINK_COMPONENTS
+  TableGen
+  Support
+  )
+
+add_llvm_unittest(TableGenTests
+  CodeExpanderTest.cpp
+  )
+include_directories(${CMAKE_SOURCE_DIR}/utils/TableGen)
+target_link_libraries(TableGenTests PRIVATE LLVMTableGenGlobalISel)
diff --git a/llvm/unittests/TableGen/CodeExpanderTest.cpp b/llvm/unittests/TableGen/CodeExpanderTest.cpp
new file mode 100644
index 0000000000000..75b9b73737073
--- /dev/null
+++ b/llvm/unittests/TableGen/CodeExpanderTest.cpp
@@ -0,0 +1,203 @@
+//===- llvm/unittest/TableGen/CodeExpanderTest.cpp - Tests ----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "GlobalISel/CodeExpander.h"
+#include "GlobalISel/CodeExpansions.h"
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+static StringRef bufferize(StringRef Str) {
+  std::unique_ptr<MemoryBuffer> Buffer =
+      MemoryBuffer::getMemBufferCopy(Str, "TestBuffer");
+  StringRef StrBufferRef = Buffer->getBuffer();
+  SrcMgr.AddNewSourceBuffer(std::move(Buffer), SMLoc());
+  return StrBufferRef;
+}
+
+class RAIIDiagnosticChecker {
+  std::string EmittedDiags;
+  raw_string_ostream OS;
+  std::vector<SMDiagnostic> Expected;
+  std::vector<SMDiagnostic> Received;
+
+public:
+  RAIIDiagnosticChecker() : OS(EmittedDiags) {
+    SrcMgr.setDiagHandler(handler, this);
+  }
+  ~RAIIDiagnosticChecker() {
+    SrcMgr.setDiagHandler(nullptr);
+    EXPECT_EQ(Received.size(), Expected.size());
+    for (unsigned i = 0; i < Received.size() && i < Expected.size(); ++i) {
+      EXPECT_EQ(Received[i].getLoc(), Expected[i].getLoc());
+      EXPECT_EQ(Received[i].getFilename(), Expected[i].getFilename());
+      EXPECT_EQ(Received[i].getKind(), Expected[i].getKind());
+      EXPECT_EQ(Received[i].getLineNo(), Expected[i].getLineNo());
+      EXPECT_EQ(Received[i].getColumnNo(), Expected[i].getColumnNo());
+      EXPECT_EQ(Received[i].getMessage(), Expected[i].getMessage());
+      EXPECT_EQ(Received[i].getLineContents(), Expected[i].getLineContents());
+      EXPECT_EQ(Received[i].getRanges(), Expected[i].getRanges());
+    }
+
+    if (testing::Test::HasFailure())
+      errs() << "Emitted diagnostic:\n" << OS.str();
+  }
+
+  void expect(SMDiagnostic D) { Expected.push_back(D); }
+
+  void diag(const SMDiagnostic &D) {
+    Received.push_back(D);
+  }
+
+  static void handler(const SMDiagnostic &D, void *Context) {
+    RAIIDiagnosticChecker *Self = static_cast<RAIIDiagnosticChecker *>(Context);
+    Self->diag(D);
+    SrcMgr.setDiagHandler(nullptr);
+    SrcMgr.PrintMessage(Self->OS, D);
+    SrcMgr.setDiagHandler(handler, Context);
+  };
+};
+
+TEST(CodeExpander, NoExpansions) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions");
+}
+
+// Indentation is applied to all lines except the first
+TEST(CodeExpander, Indentation) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("No expansions\nsecond line\nthird line", Expansions, SMLoc(),
+               false, "  ")
+      .emit(OS);
+  EXPECT_EQ(OS.str(), "No expansions\n  second line\n  third line");
+}
+
+// \ is an escape character that removes special meanings from the next
+// character.
+TEST(CodeExpander, Escape) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\\\\\a\\$", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "\\a$");
+}
+
+// $foo is not an expansion. It should warn though.
+TEST(CodeExpander, NotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" $foo");
+  CodeExpander(" $foo", Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), " $foo");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 1), "TestBuffer", 1, 1,
+      SourceMgr::DK_Warning, "Assuming missing escape character", " $foo", {}));
+}
+
+// \$foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedNotAnExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\$foo", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "$foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedUnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo");
+}
+
+// \${foo is not an expansion but shouldn't warn as it's using the escape.
+TEST(CodeExpander, EscapedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("\\${foo}", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "${foo}");
+}
+
+// ${foo} is an undefined expansion and should error.
+TEST(CodeExpander, UndefinedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  CodeExpander("${foo}${bar}", Expansions, SMLoc(), false).emit(OS);
+  EXPECT_EQ(OS.str(), "expansion");
+  DiagChecker.expect(
+      SMDiagnostic(SrcMgr, SMLoc(), "<unknown>", 0, -1, SourceMgr::DK_Error,
+                   "Attempting to expand an undeclared variable foo", "", {}));
+}
+
+// ${foo} is an undefined expansion and should error. When given a valid
+// location for the start of the buffer it should correctly point at the
+// expansion being performed.
+TEST(CodeExpander, UndefinedExpansionWithLoc) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize("Padding ${foo}${bar}");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), "Padding expansion");
+  DiagChecker.expect(SMDiagnostic(
+      SrcMgr, SMLoc::getFromPointer(In.data() + 8), "TestBuffer", 1, 8,
+      SourceMgr::DK_Error, "Attempting to expand an undeclared variable foo",
+      "Padding ${foo}${bar}", {}));
+}
+
+// ${bar is an unterminated expansion. Warn and implicitly terminate it.
+TEST(CodeExpander, UnterminatedExpansion) {
+  std::string Result;
+  raw_string_ostream OS(Result);
+  CodeExpansions Expansions;
+  Expansions.declare("bar", "expansion");
+
+  RAIIDiagnosticChecker DiagChecker;
+  StringRef In = bufferize(" ${bar");
+  CodeExpander(In, Expansions, SMLoc::getFromPointer(In.data()), false)
+      .emit(OS);
+  EXPECT_EQ(OS.str(), " expansion");
+  DiagChecker.expect(SMDiagnostic(SrcMgr, SMLoc::getFromPointer(In.data() + 1),
+                                  "TestBuffer", 1, 1, SourceMgr::DK_Warning,
+                                  "Unterminated expansion", " ${bar", {}));
+}
diff --git a/llvm/unittests/Target/ARM/CMakeLists.txt b/llvm/unittests/Target/ARM/CMakeLists.txt
new file mode 100644
index 0000000000000..0fdd955273067
--- /dev/null
+++ b/llvm/unittests/Target/ARM/CMakeLists.txt
@@ -0,0 +1,17 @@
+include_directories(
+  ${CMAKE_SOURCE_DIR}/lib/Target/ARM
+  ${CMAKE_BINARY_DIR}/lib/Target/ARM
+  )
+
+set(LLVM_LINK_COMPONENTS
+  ARMCodeGen
+  ARMDesc
+  ARMInfo
+  MC
+  Support
+  Target
+)
+
+add_llvm_unittest(ARMTests
+  MachineInstrTest.cpp
+  )
diff --git a/llvm/unittests/Target/ARM/MachineInstrTest.cpp b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
new file mode 100644
index 0000000000000..77bba08dcbc8c
--- /dev/null
+++ b/llvm/unittests/Target/ARM/MachineInstrTest.cpp
@@ -0,0 +1,166 @@
+#include "ARMBaseInstrInfo.h"
+#include "ARMSubtarget.h"
+#include "ARMTargetMachine.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
+
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+// Test for instructions that aren't immediately obviously valid within a
+// tail-predicated loop. This should be marked up in their tablegen
+// descriptions. Currently the horizontal vector operations are tagged.
+// TODO Add instructions that perform:
+// - truncation,
+// - extensions,
+// - byte swapping,
+// - others?
+TEST(MachineInstrInvalidTailPredication, IsCorrect) {
+  LLVMInitializeARMTargetInfo();
+  LLVMInitializeARMTarget();
+  LLVMInitializeARMTargetMC();
+
+  auto TT(Triple::normalize("thumbv8.1m.main-arm-none-eabi"));
+  std::string Error;
+  const Target *T = TargetRegistry::lookupTarget(TT, Error);
+  if (!T) {
+    dbgs() << Error;
+    return;
+  }
+
+  TargetOptions Options;
+  auto TM = std::unique_ptr<LLVMTargetMachine>(
+    static_cast<LLVMTargetMachine*>(
+      T->createTargetMachine(TT, "generic", "", Options, None, None,
+                             CodeGenOpt::Default)));
+  auto MII = TM->getMCInstrInfo();
+
+  using namespace ARM;
+
+  auto IsInvalidTPOpcode = [](unsigned Opcode) {
+    switch (Opcode) {
+    case MVE_VABAVs8:
+    case MVE_VABAVs16:
+    case MVE_VABAVs32:
+    case MVE_VABAVu8:
+    case MVE_VABAVu16:
+    case MVE_VABAVu32:
+    case MVE_VADDVs8acc:
+    case MVE_VADDVs16acc:
+    case MVE_VADDVs32acc:
+    case MVE_VADDVu8acc:
+    case MVE_VADDVu16acc:
+    case MVE_VADDVu32acc:
+    case MVE_VADDVs8no_acc:
+    case MVE_VADDVs16no_acc:
+    case MVE_VADDVs32no_acc:
+    case MVE_VADDVu8no_acc:
+    case MVE_VADDVu16no_acc:
+    case MVE_VADDVu32no_acc:
+    case MVE_VADDLVs32no_acc:
+    case MVE_VADDLVu32no_acc:
+    case MVE_VADDLVs32acc:
+    case MVE_VADDLVu32acc:
+    case MVE_VMLADAVas16:
+    case MVE_VMLADAVas32:
+    case MVE_VMLADAVas8:
+    case MVE_VMLADAVau16:
+    case MVE_VMLADAVau32:
+    case MVE_VMLADAVau8:
+    case MVE_VMLADAVaxs16:
+    case MVE_VMLADAVaxs32:
+    case MVE_VMLADAVaxs8:
+    case MVE_VMLADAVs16:
+    case MVE_VMLADAVs32:
+    case MVE_VMLADAVs8:
+    case MVE_VMLADAVu16:
+    case MVE_VMLADAVu32:
+    case MVE_VMLADAVu8:
+    case MVE_VMLADAVxs16:
+    case MVE_VMLADAVxs32:
+    case MVE_VMLADAVxs8:
+    case MVE_VMLALDAVas16:
+    case MVE_VMLALDAVas32:
+    case MVE_VMLALDAVau16:
+    case MVE_VMLALDAVau32:
+    case MVE_VMLALDAVaxs16:
+    case MVE_VMLALDAVaxs32:
+    case MVE_VMLALDAVs16:
+    case MVE_VMLALDAVs32:
+    case MVE_VMLALDAVu16:
+    case MVE_VMLALDAVu32:
+    case MVE_VMLALDAVxs16:
+    case MVE_VMLALDAVxs32:
+    case MVE_VMLSDAVas16:
+    case MVE_VMLSDAVas32:
+    case MVE_VMLSDAVas8:
+    case MVE_VMLSDAVaxs16:
+    case MVE_VMLSDAVaxs32:
+    case MVE_VMLSDAVaxs8:
+    case MVE_VMLSDAVs16:
+    case MVE_VMLSDAVs32:
+    case MVE_VMLSDAVs8:
+    case MVE_VMLSDAVxs16:
+    case MVE_VMLSDAVxs32:
+    case MVE_VMLSDAVxs8:
+    case MVE_VMLSLDAVas16:
+    case MVE_VMLSLDAVas32:
+    case MVE_VMLSLDAVaxs16:
+    case MVE_VMLSLDAVaxs32:
+    case MVE_VMLSLDAVs16:
+    case MVE_VMLSLDAVs32:
+    case MVE_VMLSLDAVxs16:
+    case MVE_VMLSLDAVxs32:
+    case MVE_VRMLALDAVHas32:
+    case MVE_VRMLALDAVHau32:
+    case MVE_VRMLALDAVHaxs32:
+    case MVE_VRMLALDAVHs32:
+    case MVE_VRMLALDAVHu32:
+    case MVE_VRMLALDAVHxs32:
+    case MVE_VRMLSLDAVHas32:
+    case MVE_VRMLSLDAVHaxs32:
+    case MVE_VRMLSLDAVHs32:
+    case MVE_VRMLSLDAVHxs32:
+    case MVE_VMAXNMVf16:
+    case MVE_VMINNMVf16:
+    case MVE_VMAXNMVf32:
+    case MVE_VMINNMVf32:
+    case MVE_VMAXNMAVf16:
+    case MVE_VMINNMAVf16:
+    case MVE_VMAXNMAVf32:
+    case MVE_VMINNMAVf32:
+    case MVE_VMAXVs8:
+    case MVE_VMAXVs16:
+    case MVE_VMAXVs32:
+    case MVE_VMAXVu8:
+    case MVE_VMAXVu16:
+    case MVE_VMAXVu32:
+    case MVE_VMINVs8:
+    case MVE_VMINVs16:
+    case MVE_VMINVs32:
+    case MVE_VMINVu8:
+    case MVE_VMINVu16:
+    case MVE_VMINVu32:
+    case MVE_VMAXAVs8:
+    case MVE_VMAXAVs16:
+    case MVE_VMAXAVs32:
+    case MVE_VMINAVs8:
+    case MVE_VMINAVs16:
+    case MVE_VMINAVs32:
+      return true;
+    default:
+      return false;
+    }
+  };
+
+  for (unsigned i = 0; i < ARM::INSTRUCTION_LIST_END; ++i) {
+    uint64_t Flags = MII->get(i).TSFlags;
+    bool Invalid = (Flags & ARMII::InvalidForTailPredication) != 0;
+    ASSERT_EQ(IsInvalidTPOpcode(i), Invalid)
+        << MII->getName(i)
+        << ": mismatched expectation for tail-predicated safety\n";
+  }
+}
diff --git a/llvm/unittests/TextAPI/TextStubV1Tests.cpp b/llvm/unittests/TextAPI/TextStubV1Tests.cpp
index 383bc41d4f104..4472b1c599350 100644
--- a/llvm/unittests/TextAPI/TextStubV1Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV1Tests.cpp
@@ -87,8 +87,13 @@ TEST(TBDv1, ReadFile) {
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
   auto Archs = AK_armv7 | AK_armv7s | AK_armv7k | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
   EXPECT_EQ(Archs, File->getArchitectures());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
   EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
   EXPECT_EQ(PackedVersion(2, 3, 4), File->getCurrentVersion());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
@@ -97,8 +102,8 @@ TEST(TBDv1, ReadFile) {
   EXPECT_TRUE(File->isTwoLevelNamespace());
   EXPECT_TRUE(File->isApplicationExtensionSafe());
   EXPECT_FALSE(File->isInstallAPI());
-  InterfaceFileRef client("clientA", Archs);
-  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Archs);
+  InterfaceFileRef client("clientA", Targets);
+  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Targets);
   EXPECT_EQ(1U, File->allowableClients().size());
   EXPECT_EQ(client, File->allowableClients().front());
   EXPECT_EQ(1U, File->reexportedLibraries().size());
@@ -117,6 +122,10 @@ TEST(TBDv1, ReadFile) {
   EXPECT_EQ(sizeof(TBDv1Symbols) / sizeof(ExportedSymbol), Exports.size());
   EXPECT_TRUE(
       std::equal(Exports.begin(), Exports.end(), std::begin(TBDv1Symbols)));
+
+  File->addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", {Targets[1]});
+  File->addSymbol(SymbolKind::ObjectiveCInstanceVariable, "Class1._ivar1",
+                  {Targets[1]});
 }
 
 TEST(TBDv1, ReadFile2) {
@@ -131,8 +140,13 @@ TEST(TBDv1, ReadFile2) {
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
   auto Archs = AK_armv7 | AK_armv7s | AK_armv7k | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
   EXPECT_EQ(Archs, File->getArchitectures());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
   EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCurrentVersion());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
@@ -169,25 +183,27 @@ TEST(TBDv1, WriteFile) {
       "...\n";
 
   InterfaceFile File;
+  TargetList Targets;
+  for (auto &&arch : AK_i386 | AK_x86_64)
+    Targets.emplace_back(Target(arch, PlatformKind::macOS));
   File.setPath("libfoo.dylib");
   File.setInstallName("/usr/lib/libfoo.dylib");
   File.setFileType(FileType::TBD_V1);
-  File.setArchitectures(AK_i386 | AK_x86_64);
-  File.setPlatform(PlatformKind::macOS);
+  File.addTargets(Targets);
   File.setCurrentVersion(PackedVersion(1, 2, 3));
   File.setSwiftABIVersion(5);
   File.setObjCConstraint(ObjCConstraintType::Retain_Release);
-  File.addAllowableClient("clientA", AK_x86_64);
-  File.addReexportedLibrary("/usr/lib/libfoo.dylib", AK_x86_64);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", AK_i386);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", AK_i386,
+  File.addAllowableClient("clientA", Targets[1]);
+  File.addReexportedLibrary("/usr/lib/libfoo.dylib", Targets[1]);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", {Targets[0]});
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", {Targets[0]},
                  SymbolFlags::WeakDefined);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", AK_i386,
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", {Targets[0]},
                  SymbolFlags::ThreadLocalValue);
-  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", AK_x86_64);
-  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", AK_x86_64);
+  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", {Targets[1]});
+  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", {Targets[1]});
   File.addSymbol(SymbolKind::ObjectiveCInstanceVariable, "Class1._ivar1",
-                 AK_x86_64);
+                 {Targets[1]});
 
   SmallString<4096> Buffer;
   raw_svector_ostream OS(Buffer);
@@ -206,9 +222,11 @@ TEST(TBDv1, Platform_macOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_macos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::macOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
-  EXPECT_EQ(PlatformKind::macOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv1, Platform_iOS) {
@@ -221,9 +239,11 @@ TEST(TBDv1, Platform_iOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_ios, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::iOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv1, Platform_watchOS) {
@@ -236,9 +256,11 @@ TEST(TBDv1, Platform_watchOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_watchos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::watchOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
-  EXPECT_EQ(PlatformKind::watchOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv1, Platform_tvOS) {
@@ -251,9 +273,11 @@ TEST(TBDv1, Platform_tvOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_tvos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::tvOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
-  EXPECT_EQ(PlatformKind::tvOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv1, Platform_bridgeOS) {
@@ -266,9 +290,11 @@ TEST(TBDv1, Platform_bridgeOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_bridgeos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::bridgeOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V1, File->getFileType());
-  EXPECT_EQ(PlatformKind::bridgeOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv1, Swift_1_0) {
diff --git a/llvm/unittests/TextAPI/TextStubV2Tests.cpp b/llvm/unittests/TextAPI/TextStubV2Tests.cpp
index 6350b61db5d75..1cc622bc4503b 100644
--- a/llvm/unittests/TextAPI/TextStubV2Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV2Tests.cpp
@@ -89,8 +89,13 @@ TEST(TBDv2, ReadFile) {
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
   auto Archs = AK_armv7 | AK_armv7s | AK_armv7k | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
   EXPECT_EQ(Archs, File->getArchitectures());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
   EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
   EXPECT_EQ(PackedVersion(2, 3, 4), File->getCurrentVersion());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
@@ -99,8 +104,8 @@ TEST(TBDv2, ReadFile) {
   EXPECT_TRUE(File->isTwoLevelNamespace());
   EXPECT_TRUE(File->isApplicationExtensionSafe());
   EXPECT_TRUE(File->isInstallAPI());
-  InterfaceFileRef client("clientA", Archs);
-  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Archs);
+  InterfaceFileRef client("clientA", Targets);
+  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Targets);
   EXPECT_EQ(1U, File->allowableClients().size());
   EXPECT_EQ(client, File->allowableClients().front());
   EXPECT_EQ(1U, File->reexportedLibraries().size());
@@ -155,8 +160,13 @@ TEST(TBDv2, ReadFile2) {
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
   auto Archs = AK_armv7 | AK_armv7s | AK_armv7k | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
   EXPECT_EQ(Archs, File->getArchitectures());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
   EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCurrentVersion());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
@@ -192,27 +202,29 @@ TEST(TBDv2, WriteFile) {
       "...\n";
 
   InterfaceFile File;
+  TargetList Targets;
+  for (auto &&arch : AK_i386 | AK_x86_64)
+    Targets.emplace_back(Target(arch, PlatformKind::macOS));
   File.setPath("libfoo.dylib");
   File.setInstallName("/usr/lib/libfoo.dylib");
   File.setFileType(FileType::TBD_V2);
-  File.setArchitectures(AK_i386 | AK_x86_64);
-  File.setPlatform(PlatformKind::macOS);
+  File.addTargets(Targets);
   File.setCurrentVersion(PackedVersion(1, 2, 3));
   File.setTwoLevelNamespace();
   File.setApplicationExtensionSafe();
   File.setSwiftABIVersion(5);
   File.setObjCConstraint(ObjCConstraintType::Retain_Release);
-  File.addAllowableClient("clientA", AK_x86_64);
-  File.addReexportedLibrary("/usr/lib/libfoo.dylib", AK_x86_64);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", AK_i386);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", AK_i386,
+  File.addAllowableClient("clientA", Targets[1]);
+  File.addReexportedLibrary("/usr/lib/libfoo.dylib", Targets[1]);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", {Targets[0]});
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", {Targets[0]},
                  SymbolFlags::WeakDefined);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", AK_i386,
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", {Targets[0]},
                  SymbolFlags::ThreadLocalValue);
-  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", AK_x86_64);
-  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", AK_x86_64);
+  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", {Targets[1]});
+  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", {Targets[1]});
   File.addSymbol(SymbolKind::ObjectiveCInstanceVariable, "Class1._ivar1",
-                 AK_x86_64);
+                 {Targets[1]});
 
   SmallString<4096> Buffer;
   raw_svector_ostream OS(Buffer);
@@ -232,8 +244,10 @@ TEST(TBDv2, Platform_macOS) {
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_macos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
+  auto Platform = PlatformKind::macOS;
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
-  EXPECT_EQ(PlatformKind::macOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv2, Platform_iOS) {
@@ -246,9 +260,11 @@ TEST(TBDv2, Platform_iOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_ios, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::iOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv2, Platform_watchOS) {
@@ -261,9 +277,11 @@ TEST(TBDv2, Platform_watchOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_watchos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::watchOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
-  EXPECT_EQ(PlatformKind::watchOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv2, Platform_tvOS) {
@@ -276,9 +294,11 @@ TEST(TBDv2, Platform_tvOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_tvos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::tvOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
-  EXPECT_EQ(PlatformKind::tvOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv2, Platform_bridgeOS) {
@@ -291,9 +311,11 @@ TEST(TBDv2, Platform_bridgeOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_bridgeos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::bridgeOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V2, File->getFileType());
-  EXPECT_EQ(PlatformKind::bridgeOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv2, Swift_1_0) {
@@ -415,7 +437,6 @@ TEST(TBDv2, UnknownArchitecture) {
       "platform: macosx\n"
       "install-name: Test.dylib\n"
       "...\n";
-
   auto Result = TextAPIReader::get(
       MemoryBufferRef(tbd_v2_file_unknown_architecture, "Test.tbd"));
   EXPECT_TRUE(!!Result);
@@ -423,9 +444,9 @@ TEST(TBDv2, UnknownArchitecture) {
 
 TEST(TBDv2, UnknownPlatform) {
   static const char tbd_v2_file_unknown_platform[] = "--- !tapi-tbd-v2\n"
-                                                     "archs: [ i386 ]\n"
-                                                     "platform: newOS\n"
-                                                     "...\n";
+	                                                     "archs: [ i386 ]\n"
+	                                                     "platform: newOS\n"
+	                                                     "...\n";
 
   auto Result = TextAPIReader::get(
       MemoryBufferRef(tbd_v2_file_unknown_platform, "Test.tbd"));
@@ -436,6 +457,23 @@ TEST(TBDv2, UnknownPlatform) {
             errorMessage);
 }
 
+TEST(TBDv2, InvalidPlatform) {
+  static const char tbd_v2_file_invalid_platform[] =
+      "--- !tapi-tbd-v2\n"
+      "archs: [ i386 ]\n"
+      "platform: iosmac\n"
+      "install-name: Test.dylib\n"
+      "...\n";
+
+  auto Result = TextAPIReader::get(
+      MemoryBufferRef(tbd_v2_file_invalid_platform, "Test.tbd"));
+  EXPECT_FALSE(!!Result);
+  auto errorMessage = toString(Result.takeError());
+  EXPECT_EQ("malformed file\nTest.tbd:3:11: error: invalid platform\nplatform: "
+            "iosmac\n          ^~~~~~\n",
+            errorMessage);
+}
+
 TEST(TBDv2, MalformedFile1) {
   static const char malformed_file1[] = "--- !tapi-tbd-v2\n"
                                         "archs: [ arm64 ]\n"
diff --git a/llvm/unittests/TextAPI/TextStubV3Tests.cpp b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
index 9904e7bc7358d..c8b16dbb7d7b8 100644
--- a/llvm/unittests/TextAPI/TextStubV3Tests.cpp
+++ b/llvm/unittests/TextAPI/TextStubV3Tests.cpp
@@ -23,7 +23,7 @@ struct ExportedSymbol {
   bool ThreadLocalValue;
 };
 using ExportedSymbolSeq = std::vector<ExportedSymbol>;
-using UUIDs = std::vector<std::pair<Architecture, std::string>>;
+using UUIDs = std::vector<std::pair<Target, std::string>>;
 
 inline bool operator<(const ExportedSymbol &lhs, const ExportedSymbol &rhs) {
   return std::tie(lhs.Kind, lhs.Name) < std::tie(rhs.Kind, rhs.Name);
@@ -93,11 +93,18 @@ TEST(TBDv3, ReadFile) {
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
   auto Archs = AK_armv7 | AK_arm64;
+  auto Platform = PlatformKind::iOS;
+  TargetList Targets;
+  for (auto &&arch : Archs)
+    Targets.emplace_back(Target(arch, Platform));
   EXPECT_EQ(Archs, File->getArchitectures());
-  UUIDs uuids = {{AK_armv7, "00000000-0000-0000-0000-000000000000"},
-                 {AK_arm64, "11111111-1111-1111-1111-111111111111"}};
-  EXPECT_EQ(uuids, File->uuids());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  UUIDs Uuids = {{Target(AK_armv7, PlatformKind::unknown),
+                  "00000000-0000-0000-0000-000000000000"},
+                 {Target(AK_arm64, PlatformKind::unknown),
+                  "11111111-1111-1111-1111-111111111111"}};
+  EXPECT_EQ(Uuids, File->uuids());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
   EXPECT_EQ(std::string("Test.dylib"), File->getInstallName());
   EXPECT_EQ(PackedVersion(2, 3, 4), File->getCurrentVersion());
   EXPECT_EQ(PackedVersion(1, 0, 0), File->getCompatibilityVersion());
@@ -106,8 +113,8 @@ TEST(TBDv3, ReadFile) {
   EXPECT_TRUE(File->isTwoLevelNamespace());
   EXPECT_TRUE(File->isApplicationExtensionSafe());
   EXPECT_TRUE(File->isInstallAPI());
-  InterfaceFileRef client("clientA", Archs);
-  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Archs);
+  InterfaceFileRef client("clientA", Targets);
+  InterfaceFileRef reexport("/usr/lib/libfoo.dylib", Targets);
   EXPECT_EQ(1U, File->allowableClients().size());
   EXPECT_EQ(client, File->allowableClients().front());
   EXPECT_EQ(1U, File->reexportedLibraries().size());
@@ -151,27 +158,29 @@ TEST(TBDv3, WriteFile) {
       "...\n";
 
   InterfaceFile File;
+  TargetList Targets;
+  for (auto &&arch : AK_i386 | AK_x86_64)
+    Targets.emplace_back(Target(arch, PlatformKind::macOS));
   File.setPath("libfoo.dylib");
   File.setInstallName("/usr/lib/libfoo.dylib");
   File.setFileType(FileType::TBD_V3);
-  File.setArchitectures(AK_i386 | AK_x86_64);
-  File.setPlatform(PlatformKind::macOS);
+  File.addTargets(Targets);
   File.setCurrentVersion(PackedVersion(1, 2, 3));
   File.setTwoLevelNamespace();
   File.setApplicationExtensionSafe();
   File.setSwiftABIVersion(5);
   File.setObjCConstraint(ObjCConstraintType::Retain_Release);
-  File.addAllowableClient("clientA", AK_x86_64);
-  File.addReexportedLibrary("/usr/lib/libfoo.dylib", AK_x86_64);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", AK_i386);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", AK_i386,
+  File.addAllowableClient("clientA", Targets[1]);
+  File.addReexportedLibrary("/usr/lib/libfoo.dylib", Targets[1]);
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym1", {Targets[0]});
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym2", {Targets[0]},
                  SymbolFlags::WeakDefined);
-  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", AK_i386,
+  File.addSymbol(SymbolKind::GlobalSymbol, "_sym3", {Targets[0]},
                  SymbolFlags::ThreadLocalValue);
-  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", AK_x86_64);
-  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", AK_x86_64);
+  File.addSymbol(SymbolKind::ObjectiveCClass, "Class1", {Targets[1]});
+  File.addSymbol(SymbolKind::ObjectiveCClassEHType, "Class1", {Targets[1]});
   File.addSymbol(SymbolKind::ObjectiveCInstanceVariable, "Class1._ivar1",
-                 AK_x86_64);
+                 {Targets[1]});
 
   SmallString<4096> Buffer;
   raw_svector_ostream OS(Buffer);
@@ -190,9 +199,11 @@ TEST(TBDv3, Platform_macOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_macos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::macOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
-  EXPECT_EQ(PlatformKind::macOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv3, Platform_iOS) {
@@ -205,9 +216,11 @@ TEST(TBDv3, Platform_iOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_ios, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::iOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
-  EXPECT_EQ(PlatformKind::iOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv3, Platform_watchOS) {
@@ -220,9 +233,11 @@ TEST(TBDv3, Platform_watchOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_watchos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::watchOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
-  EXPECT_EQ(PlatformKind::watchOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv3, Platform_tvOS) {
@@ -236,8 +251,10 @@ TEST(TBDv3, Platform_tvOS) {
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_tvos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
   auto File = std::move(Result.get());
+  auto Platform = PlatformKind::tvOS;
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
-  EXPECT_EQ(PlatformKind::tvOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
 }
 
 TEST(TBDv3, Platform_bridgeOS) {
@@ -250,9 +267,48 @@ TEST(TBDv3, Platform_bridgeOS) {
   auto Result =
       TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_bridgeos, "Test.tbd"));
   EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::bridgeOS;
   auto File = std::move(Result.get());
   EXPECT_EQ(FileType::TBD_V3, File->getFileType());
-  EXPECT_EQ(PlatformKind::bridgeOS, File->getPlatform());
+  EXPECT_EQ(File->getPlatforms().size(), 1U);
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+}
+
+TEST(TBDv3, Platform_macCatalyst) {
+  static const char tbd_v1_platform_iosmac[] = "--- !tapi-tbd-v3\n"
+                                                 "archs: [ armv7k ]\n"
+                                                 "platform: iosmac\n"
+                                                 "install-name: Test.dylib\n"
+                                                 "...\n";
+
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_iosmac, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto Platform = PlatformKind::macCatalyst;
+  auto File = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+  EXPECT_EQ(Platform, *File->getPlatforms().begin());
+}
+
+TEST(TBDv3, Platform_zippered) {
+  static const char tbd_v1_platform_zip[] = "--- !tapi-tbd-v3\n"
+                                                 "archs: [ armv7k ]\n"
+                                                 "platform: zippered\n"
+                                                 "install-name: Test.dylib\n"
+                                                 "...\n";
+
+  auto Result =
+      TextAPIReader::get(MemoryBufferRef(tbd_v1_platform_zip, "Test.tbd"));
+  EXPECT_TRUE(!!Result);
+  auto File = std::move(Result.get());
+  EXPECT_EQ(FileType::TBD_V3, File->getFileType());
+
+  PlatformSet Platforms;
+  Platforms.insert(PlatformKind::macOS);
+  Platforms.insert(PlatformKind::macCatalyst);
+  EXPECT_EQ(Platforms.size(), File->getPlatforms().size());
+  for (auto Platform : File->getPlatforms())
+	    EXPECT_EQ(Platforms.count(Platform), 1U);
 }
 
 TEST(TBDv3, Swift_1_0) {
diff --git a/llvm/unittests/Transforms/Utils/LocalTest.cpp b/llvm/unittests/Transforms/Utils/LocalTest.cpp
index 55f04e98d77dc..1f67a1ec84c76 100644
--- a/llvm/unittests/Transforms/Utils/LocalTest.cpp
+++ b/llvm/unittests/Transforms/Utils/LocalTest.cpp
@@ -867,12 +867,42 @@ TEST(Local, RemoveUnreachableBlocks) {
       bb2:
         br label %bb1
       }
+
+      declare i32 @__gxx_personality_v0(...)
+
+      define void @invoke_terminator() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) {
+      entry:
+        br i1 undef, label %invoke.block, label %exit
+
+      invoke.block:
+        %cond = invoke zeroext i1 @invokable()
+                to label %continue.block unwind label %lpad.block
+
+      continue.block:
+        br i1 %cond, label %if.then, label %if.end
+
+      if.then:
+        unreachable
+
+      if.end:
+        unreachable
+
+      lpad.block:
+        %lp = landingpad { i8*, i32 }
+                catch i8* null
+        br label %exit
+
+      exit:
+        ret void
+      }
+
+      declare i1 @invokable()
       )");
 
   auto runEager = [&](Function &F, DominatorTree *DT) {
     PostDominatorTree PDT = PostDominatorTree(F);
     DomTreeUpdater DTU(*DT, PDT, DomTreeUpdater::UpdateStrategy::Eager);
-    removeUnreachableBlocks(F, nullptr, &DTU);
+    removeUnreachableBlocks(F, &DTU);
     EXPECT_TRUE(DTU.getDomTree().verify());
     EXPECT_TRUE(DTU.getPostDomTree().verify());
   };
@@ -880,7 +910,7 @@ TEST(Local, RemoveUnreachableBlocks) {
   auto runLazy = [&](Function &F, DominatorTree *DT) {
     PostDominatorTree PDT = PostDominatorTree(F);
     DomTreeUpdater DTU(*DT, PDT, DomTreeUpdater::UpdateStrategy::Lazy);
-    removeUnreachableBlocks(F, nullptr, &DTU);
+    removeUnreachableBlocks(F, &DTU);
     EXPECT_TRUE(DTU.getDomTree().verify());
     EXPECT_TRUE(DTU.getPostDomTree().verify());
   };
@@ -890,12 +920,14 @@ TEST(Local, RemoveUnreachableBlocks) {
   runWithDomTree(*M, "br_self_loop", runEager);
   runWithDomTree(*M, "br_constant", runEager);
   runWithDomTree(*M, "br_loop", runEager);
+  runWithDomTree(*M, "invoke_terminator", runEager);
 
   // Test removeUnreachableBlocks under Lazy UpdateStrategy.
   runWithDomTree(*M, "br_simple", runLazy);
   runWithDomTree(*M, "br_self_loop", runLazy);
   runWithDomTree(*M, "br_constant", runLazy);
   runWithDomTree(*M, "br_loop", runLazy);
+  runWithDomTree(*M, "invoke_terminator", runLazy);
 
   M = parseIR(C,
               R"(
@@ -909,8 +941,8 @@ TEST(Local, RemoveUnreachableBlocks) {
 
   auto checkRUBlocksRetVal = [&](Function &F, DominatorTree *DT) {
     DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
-    EXPECT_TRUE(removeUnreachableBlocks(F, nullptr, &DTU));
-    EXPECT_FALSE(removeUnreachableBlocks(F, nullptr, &DTU));
+    EXPECT_TRUE(removeUnreachableBlocks(F, &DTU));
+    EXPECT_FALSE(removeUnreachableBlocks(F, &DTU));
     EXPECT_TRUE(DTU.getDomTree().verify());
   };
 
diff --git a/llvm/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h b/llvm/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
index fcbff94052838..9a348e4e9de84 100644
--- a/llvm/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
+++ b/llvm/unittests/tools/llvm-exegesis/Common/AssemblerUtils.h
@@ -43,12 +43,15 @@ class MachineFunctionGeneratorBaseTest : public ::testing::Test {
   }
 
   template <class... Bs>
-  inline void Check(llvm::ArrayRef<RegisterValue> RegisterInitialValues,
-                    llvm::MCInst MCInst, Bs... Bytes) {
+  inline void Check(ArrayRef<RegisterValue> RegisterInitialValues, MCInst Inst,
+                    Bs... Bytes) {
     ExecutableFunction Function =
-        (MCInst.getOpcode() == 0)
-            ? assembleToFunction(RegisterInitialValues, {})
-            : assembleToFunction(RegisterInitialValues, {MCInst});
+        (Inst.getOpcode() == 0)
+            ? assembleToFunction(RegisterInitialValues, [](FunctionFiller &) {})
+            : assembleToFunction(RegisterInitialValues,
+                                 [Inst](FunctionFiller &Filler) {
+                                   Filler.getEntry().addInstruction(Inst);
+                                 });
     ASSERT_THAT(Function.getFunctionBytes().str(),
                 testing::ElementsAre(Bytes...));
     if (CanExecute) {
@@ -73,11 +76,11 @@ class MachineFunctionGeneratorBaseTest : public ::testing::Test {
 
   ExecutableFunction
   assembleToFunction(llvm::ArrayRef<RegisterValue> RegisterInitialValues,
-                     llvm::ArrayRef<llvm::MCInst> Instructions) {
+                     FillFunction Fill) {
     llvm::SmallString<256> Buffer;
     llvm::raw_svector_ostream AsmStream(Buffer);
     assembleToStream(*ET, createTargetMachine(), /*LiveIns=*/{},
-                     RegisterInitialValues, Instructions, AsmStream);
+                     RegisterInitialValues, Fill, AsmStream);
     return ExecutableFunction(createTargetMachine(),
                               getObjectFromBuffer(AsmStream.str()));
   }
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/CMakeLists.txt b/llvm/unittests/tools/llvm-exegesis/X86/CMakeLists.txt
index 88c38d0d7e8ad..c9de60d6c64b1 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/CMakeLists.txt
+++ b/llvm/unittests/tools/llvm-exegesis/X86/CMakeLists.txt
@@ -18,7 +18,9 @@ add_llvm_unittest(LLVMExegesisX86Tests
   BenchmarkResultTest.cpp
   RegisterAliasingTest.cpp
   SchedClassResolutionTest.cpp
+  SnippetFileTest.cpp
   SnippetGeneratorTest.cpp
+  SnippetRepetitorTest.cpp
   TargetTest.cpp
   )
 target_link_libraries(LLVMExegesisX86Tests PRIVATE
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
index b65b2a9315462..4c9dc4512c007 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/X86/RegisterAliasingTest.cpp
@@ -12,6 +12,7 @@
 #include <cassert>
 #include <memory>
 
+#include "TestBase.h"
 #include "X86InstrInfo.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
@@ -22,37 +23,10 @@ namespace llvm {
 namespace exegesis {
 namespace {
 
-class RegisterAliasingTest : public ::testing::Test {
-protected:
-  RegisterAliasingTest() {
-    const std::string TT = "x86_64-unknown-linux";
-    std::string error;
-    const llvm::Target *const TheTarget =
-        llvm::TargetRegistry::lookupTarget(TT, error);
-    if (!TheTarget) {
-      llvm::errs() << error << "\n";
-      return;
-    }
-    MCRegInfo.reset(TheTarget->createMCRegInfo(TT));
-  }
-
-  static void SetUpTestCase() {
-    LLVMInitializeX86TargetInfo();
-    LLVMInitializeX86Target();
-    LLVMInitializeX86TargetMC();
-  }
-
-  const llvm::MCRegisterInfo &getMCRegInfo() {
-    assert(MCRegInfo);
-    return *MCRegInfo;
-  }
-
-private:
-  std::unique_ptr<const llvm::MCRegisterInfo> MCRegInfo;
-};
+class RegisterAliasingTest : public X86TestBase {};
 
 TEST_F(RegisterAliasingTest, TrackSimpleRegister) {
-  const auto &RegInfo = getMCRegInfo();
+  const auto &RegInfo = State.getRegInfo();
   const RegisterAliasingTracker tracker(RegInfo, llvm::X86::EAX);
   std::set<llvm::MCPhysReg> ActualAliasedRegisters;
   for (unsigned I : tracker.aliasedBits().set_bits())
@@ -69,7 +43,7 @@ TEST_F(RegisterAliasingTest, TrackSimpleRegister) {
 TEST_F(RegisterAliasingTest, TrackRegisterClass) {
   // The alias bits for GR8_ABCD_LRegClassID are the union of the alias bits for
   // AL, BL, CL and DL.
-  const auto &RegInfo = getMCRegInfo();
+  const auto &RegInfo = State.getRegInfo();
   const llvm::BitVector NoReservedReg(RegInfo.getNumRegs());
 
   const RegisterAliasingTracker RegClassTracker(
@@ -87,7 +61,7 @@ TEST_F(RegisterAliasingTest, TrackRegisterClass) {
 
 TEST_F(RegisterAliasingTest, TrackRegisterClassCache) {
   // Fetching twice the same tracker yields the same pointers.
-  const auto &RegInfo = getMCRegInfo();
+  const auto &RegInfo = State.getRegInfo();
   const llvm::BitVector NoReservedReg(RegInfo.getNumRegs());
   RegisterAliasingTrackerCache Cache(RegInfo, NoReservedReg);
   ASSERT_THAT(&Cache.getRegister(llvm::X86::AX),
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SchedClassResolutionTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SchedClassResolutionTest.cpp
index 9e745461142d4..7e158daf7349e 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/SchedClassResolutionTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/X86/SchedClassResolutionTest.cpp
@@ -11,6 +11,7 @@
 #include <cassert>
 #include <memory>
 
+#include "TestBase.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "gmock/gmock.h"
@@ -23,21 +24,11 @@ namespace {
 using testing::Pair;
 using testing::UnorderedElementsAre;
 
-class SchedClassResolutionTest : public ::testing::Test {
+class SchedClassResolutionTest : public X86TestBase {
 protected:
-  SchedClassResolutionTest() {
-    const std::string TT = "x86_64-unknown-linux";
-    std::string error;
-    const llvm::Target *const TheTarget =
-        llvm::TargetRegistry::lookupTarget(TT, error);
-    if (!TheTarget) {
-      llvm::errs() << error << "\n";
-      return;
-    }
-    STI.reset(TheTarget->createMCSubtargetInfo(TT, "haswell", ""));
-
+  SchedClassResolutionTest() : STI(State.getSubtargetInfo()) {
     // Compute the ProxResIdx of ports uses in tests.
-    const auto &SM = STI->getSchedModel();
+    const auto &SM = STI.getSchedModel();
     for (unsigned I = 0, E = SM.getNumProcResourceKinds(); I < E; ++I) {
       const std::string Name = SM.getProcResource(I)->Name;
       if (Name == "HWPort0") {
@@ -62,14 +53,8 @@ class SchedClassResolutionTest : public ::testing::Test {
     EXPECT_NE(P0156Idx, 0);
   }
 
-  static void SetUpTestCase() {
-    LLVMInitializeX86TargetInfo();
-    LLVMInitializeX86Target();
-    LLVMInitializeX86TargetMC();
-  }
-
 protected:
-  std::unique_ptr<const llvm::MCSubtargetInfo> STI;
+  const llvm::MCSubtargetInfo &STI;
   uint16_t P0Idx = 0;
   uint16_t P1Idx = 0;
   uint16_t P5Idx = 0;
@@ -80,20 +65,20 @@ class SchedClassResolutionTest : public ::testing::Test {
 
 TEST_F(SchedClassResolutionTest, ComputeIdealizedProcResPressure_2P0) {
   const auto Pressure =
-      computeIdealizedProcResPressure(STI->getSchedModel(), {{P0Idx, 2}});
+      computeIdealizedProcResPressure(STI.getSchedModel(), {{P0Idx, 2}});
   EXPECT_THAT(Pressure, UnorderedElementsAre(Pair(P0Idx, 2.0)));
 }
 
 TEST_F(SchedClassResolutionTest, ComputeIdealizedProcResPressure_2P05) {
   const auto Pressure =
-      computeIdealizedProcResPressure(STI->getSchedModel(), {{P05Idx, 2}});
+      computeIdealizedProcResPressure(STI.getSchedModel(), {{P05Idx, 2}});
   EXPECT_THAT(Pressure,
               UnorderedElementsAre(Pair(P0Idx, 1.0), Pair(P5Idx, 1.0)));
 }
 
 TEST_F(SchedClassResolutionTest, ComputeIdealizedProcResPressure_2P05_2P0156) {
   const auto Pressure = computeIdealizedProcResPressure(
-      STI->getSchedModel(), {{P05Idx, 2}, {P0156Idx, 2}});
+      STI.getSchedModel(), {{P05Idx, 2}, {P0156Idx, 2}});
   EXPECT_THAT(Pressure,
               UnorderedElementsAre(Pair(P0Idx, 1.0), Pair(P1Idx, 1.0),
                                    Pair(P5Idx, 1.0), Pair(P6Idx, 1.0)));
@@ -102,7 +87,7 @@ TEST_F(SchedClassResolutionTest, ComputeIdealizedProcResPressure_2P05_2P0156) {
 TEST_F(SchedClassResolutionTest,
        ComputeIdealizedProcResPressure_1P1_1P05_2P0156) {
   const auto Pressure = computeIdealizedProcResPressure(
-      STI->getSchedModel(), {{P1Idx, 1}, {P05Idx, 1}, {P0156Idx, 2}});
+      STI.getSchedModel(), {{P1Idx, 1}, {P05Idx, 1}, {P0156Idx, 2}});
   EXPECT_THAT(Pressure,
               UnorderedElementsAre(Pair(P0Idx, 1.0), Pair(P1Idx, 1.0),
                                    Pair(P5Idx, 1.0), Pair(P6Idx, 1.0)));
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp
new file mode 100644
index 0000000000000..69dd689fc492d
--- /dev/null
+++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp
@@ -0,0 +1,120 @@
+//===-- SnippetFileTest.cpp -------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "SnippetFile.h"
+
+#include "LlvmState.h"
+#include "TestBase.h"
+#include "X86InstrInfo.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/raw_ostream.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+namespace exegesis {
+
+void InitializeX86ExegesisTarget();
+
+namespace {
+
+using testing::AllOf;
+using testing::ElementsAre;
+using testing::Eq;
+using testing::Field;
+using testing::Property;
+using testing::SizeIs;
+
+class X86SnippetFileTest : public X86TestBase {
+protected:
+  Expected<std::vector<BenchmarkCode>> TestCommon(StringRef Contents) {
+    SmallString<64> Filename;
+    std::error_code EC;
+    EC = sys::fs::createUniqueDirectory("SnippetFileTestDir", Filename);
+    EXPECT_FALSE(EC);
+    sys::path::append(Filename, "snippet.s");
+    errs() << Filename << "-------\n";
+    {
+      raw_fd_ostream FOS(Filename, EC);
+      FOS << Contents;
+      EXPECT_FALSE(EC);
+    }
+    return readSnippets(State, Filename);
+  }
+};
+
+// FIXME: Refactor these to ../Common/Matchers.h
+static auto HasOpcode = [](unsigned Opcode) {
+  return Property(&MCInst::getOpcode, Eq(Opcode));
+};
+
+MATCHER_P2(RegisterInitialValueIs, Reg, Val, "") {
+  if (arg.Register == Reg &&
+      arg.Value.getLimitedValue() == static_cast<uint64_t>(Val))
+    return true;
+  *result_listener << "expected: {" << Reg << ", " << Val << "} ";
+  *result_listener << "actual: {" << arg.Register << ", "
+                   << arg.Value.getLimitedValue() << "}";
+  return false;
+}
+
+TEST_F(X86SnippetFileTest, Works) {
+  auto Snippets = TestCommon(R"(
+    # LLVM-EXEGESIS-DEFREG RAX 0f
+    # LLVM-EXEGESIS-DEFREG SIL 0
+    # LLVM-EXEGESIS-LIVEIN RDI
+    # LLVM-EXEGESIS-LIVEIN DL
+    incq %rax
+  )");
+  EXPECT_FALSE((bool)Snippets.takeError());
+  ASSERT_THAT(*Snippets, SizeIs(1));
+  const auto &Snippet = (*Snippets)[0];
+  ASSERT_THAT(Snippet.Instructions, ElementsAre(HasOpcode(X86::INC64r)));
+  ASSERT_THAT(Snippet.RegisterInitialValues,
+              ElementsAre(RegisterInitialValueIs(X86::RAX, 15),
+                          RegisterInitialValueIs(X86::SIL, 0)));
+  ASSERT_THAT(Snippet.LiveIns, ElementsAre(X86::RDI, X86::DL));
+}
+
+TEST_F(X86SnippetFileTest, BadDefregParam) {
+  auto Error = TestCommon(R"(
+    # LLVM-EXEGESIS-DEFREG DOESNOEXIST 0
+    incq %rax
+  )")
+                   .takeError();
+  EXPECT_TRUE((bool)Error);
+  consumeError(std::move(Error));
+}
+
+TEST_F(X86SnippetFileTest, NoDefregValue) {
+  auto Error = TestCommon(R"(
+    # LLVM-EXEGESIS-DEFREG RAX
+    incq %rax
+  )")
+                   .takeError();
+  EXPECT_TRUE((bool)Error);
+  consumeError(std::move(Error));
+}
+
+TEST_F(X86SnippetFileTest, MissingParam) {
+  auto Error = TestCommon(R"(
+    # LLVM-EXEGESIS-LIVEIN
+    incq %rax
+  )")
+                   .takeError();
+  EXPECT_TRUE((bool)Error);
+  consumeError(std::move(Error));
+}
+
+} // namespace
+} // namespace exegesis
+} // namespace llvm
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
index 8b110f8c8a566..c5a69250999e0 100644
--- a/llvm/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
+++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetGeneratorTest.cpp
@@ -11,6 +11,7 @@
 #include "LlvmState.h"
 #include "MCInstrDescView.h"
 #include "RegisterAliasing.h"
+#include "TestBase.h"
 #include "Uops.h"
 #include "X86InstrInfo.h"
 
@@ -34,23 +35,11 @@ using testing::UnorderedElementsAre;
 MATCHER(IsInvalid, "") { return !arg.isValid(); }
 MATCHER(IsReg, "") { return arg.isReg(); }
 
-class X86SnippetGeneratorTest : public ::testing::Test {
+class X86SnippetGeneratorTest : public X86TestBase {
 protected:
-  X86SnippetGeneratorTest()
-      : State("x86_64-unknown-linux", "haswell"),
-        MCInstrInfo(State.getInstrInfo()), MCRegisterInfo(State.getRegInfo()) {}
-
-  static void SetUpTestCase() {
-    LLVMInitializeX86TargetInfo();
-    LLVMInitializeX86TargetMC();
-    LLVMInitializeX86Target();
-    LLVMInitializeX86AsmPrinter();
-    InitializeX86ExegesisTarget();
-  }
+  X86SnippetGeneratorTest() : InstrInfo(State.getInstrInfo()) {}
 
-  const LLVMState State;
-  const llvm::MCInstrInfo &MCInstrInfo;
-  const llvm::MCRegisterInfo &MCRegisterInfo;
+  const llvm::MCInstrInfo &InstrInfo;
 };
 
 template <typename SnippetGeneratorT>
@@ -61,7 +50,8 @@ class SnippetGeneratorTest : public X86SnippetGeneratorTest {
   std::vector<CodeTemplate> checkAndGetCodeTemplates(unsigned Opcode) {
     randomGenerator().seed(0); // Initialize seed.
     const Instruction &Instr = State.getIC().getInstr(Opcode);
-    auto CodeTemplateOrError = Generator.generateCodeTemplates(Instr);
+    auto CodeTemplateOrError = Generator.generateCodeTemplates(
+        Instr, State.getRATC().emptyRegisters());
     EXPECT_FALSE(CodeTemplateOrError.takeError()); // Valid configuration.
     return std::move(CodeTemplateOrError.get());
   }
@@ -85,10 +75,10 @@ TEST_F(LatencySnippetGeneratorTest, ImplicitSelfDependencyThroughImplicitReg) {
   // - hasAliasingImplicitRegisters (execution is always serial)
   // - hasAliasingRegisters
   const unsigned Opcode = llvm::X86::ADC16i16;
-  EXPECT_THAT(MCInstrInfo.get(Opcode).getImplicitDefs()[0], llvm::X86::AX);
-  EXPECT_THAT(MCInstrInfo.get(Opcode).getImplicitDefs()[1], llvm::X86::EFLAGS);
-  EXPECT_THAT(MCInstrInfo.get(Opcode).getImplicitUses()[0], llvm::X86::AX);
-  EXPECT_THAT(MCInstrInfo.get(Opcode).getImplicitUses()[1], llvm::X86::EFLAGS);
+  EXPECT_THAT(InstrInfo.get(Opcode).getImplicitDefs()[0], llvm::X86::AX);
+  EXPECT_THAT(InstrInfo.get(Opcode).getImplicitDefs()[1], llvm::X86::EFLAGS);
+  EXPECT_THAT(InstrInfo.get(Opcode).getImplicitUses()[0], llvm::X86::AX);
+  EXPECT_THAT(InstrInfo.get(Opcode).getImplicitUses()[1], llvm::X86::EFLAGS);
   const auto CodeTemplates = checkAndGetCodeTemplates(Opcode);
   ASSERT_THAT(CodeTemplates, SizeIs(1));
   const auto &CT = CodeTemplates[0];
@@ -111,7 +101,7 @@ TEST_F(LatencySnippetGeneratorTest, ImplicitSelfDependencyThroughTiedRegs) {
   // - hasTiedRegisters (execution is always serial)
   // - hasAliasingRegisters
   const unsigned Opcode = llvm::X86::ADD16ri;
-  EXPECT_THAT(MCInstrInfo.get(Opcode).getImplicitDefs()[0], llvm::X86::EFLAGS);
+  EXPECT_THAT(InstrInfo.get(Opcode).getImplicitDefs()[0], llvm::X86::EFLAGS);
   const auto CodeTemplates = checkAndGetCodeTemplates(Opcode);
   ASSERT_THAT(CodeTemplates, SizeIs(1));
   const auto &CT = CodeTemplates[0];
@@ -148,6 +138,26 @@ TEST_F(LatencySnippetGeneratorTest, ImplicitSelfDependencyThroughExplicitRegs) {
       << "Op0 is either set to Op1 or to Op2";
 }
 
+TEST_F(LatencySnippetGeneratorTest,
+       ImplicitSelfDependencyThroughExplicitRegsForbidAll) {
+  // - VXORPSrr
+  // - Op0 Explicit Def RegClass(VR128)
+  // - Op1 Explicit Use RegClass(VR128)
+  // - Op2 Explicit Use RegClass(VR128)
+  // - Var0 [Op0]
+  // - Var1 [Op1]
+  // - Var2 [Op2]
+  // - hasAliasingRegisters
+  const unsigned Opcode = llvm::X86::VXORPSrr;
+  randomGenerator().seed(0); // Initialize seed.
+  const Instruction &Instr = State.getIC().getInstr(Opcode);
+  auto AllRegisters = State.getRATC().emptyRegisters();
+  AllRegisters.flip();
+  auto Error = Generator.generateCodeTemplates(Instr, AllRegisters).takeError();
+  EXPECT_TRUE((bool)Error);
+  llvm::consumeError(std::move(Error));
+}
+
 TEST_F(LatencySnippetGeneratorTest, DependencyThroughOtherOpcode) {
   // - CMP64rr
   // - Op0 Explicit Use RegClass(GR64)
@@ -323,30 +333,6 @@ TEST_F(UopsSnippetGeneratorTest, MemoryUse) {
   EXPECT_EQ(IT.VariableValues[5].getReg(), 0u);
 }
 
-TEST_F(UopsSnippetGeneratorTest, MemoryUse_Movsb) {
-  // MOVSB writes to scratch memory register.
-  // - MOVSB
-  // - Op0 Explicit Use Memory RegClass(GR8)
-  // - Op1 Explicit Use Memory RegClass(GR8)
-  // - Op2 Explicit Use Memory RegClass(SEGMENT_REG)
-  // - Op3 Implicit Def Reg(EDI)
-  // - Op4 Implicit Def Reg(ESI)
-  // - Op5 Implicit Use Reg(EDI)
-  // - Op6 Implicit Use Reg(ESI)
-  // - Op7 Implicit Use Reg(DF)
-  // - Var0 [Op0]
-  // - Var1 [Op1]
-  // - Var2 [Op2]
-  // - hasMemoryOperands
-  // - hasAliasingImplicitRegisters (execution is always serial)
-  // - hasAliasingRegisters
-  const unsigned Opcode = llvm::X86::MOVSB;
-  const Instruction &Instr = State.getIC().getInstr(Opcode);
-  auto Error = Generator.generateCodeTemplates(Instr).takeError();
-  EXPECT_TRUE((bool)Error);
-  llvm::consumeError(std::move(Error));
-}
-
 class FakeSnippetGenerator : public SnippetGenerator {
 public:
   FakeSnippetGenerator(const LLVMState &State) : SnippetGenerator(State) {}
@@ -357,7 +343,7 @@ class FakeSnippetGenerator : public SnippetGenerator {
 
 private:
   llvm::Expected<std::vector<CodeTemplate>>
-  generateCodeTemplates(const Instruction &Instr) const override {
+  generateCodeTemplates(const Instruction &, const BitVector &) const override {
     return llvm::make_error<llvm::StringError>("not implemented",
                                                llvm::inconvertibleErrorCode());
   }
@@ -371,6 +357,32 @@ testing::Matcher<const RegisterValue &> IsRegisterValue(unsigned Reg,
                         testing::Field(&RegisterValue::Value, Value));
 }
 
+TEST_F(FakeSnippetGeneratorTest, MemoryUse_Movsb) {
+  // MOVSB writes to scratch memory register.
+  // - MOVSB
+  // - Op0 Explicit Use Memory RegClass(GR8)
+  // - Op1 Explicit Use Memory RegClass(GR8)
+  // - Op2 Explicit Use Memory RegClass(SEGMENT_REG)
+  // - Op3 Implicit Def Reg(EDI)
+  // - Op4 Implicit Def Reg(ESI)
+  // - Op5 Implicit Use Reg(EDI)
+  // - Op6 Implicit Use Reg(ESI)
+  // - Op7 Implicit Use Reg(DF)
+  // - Var0 [Op0]
+  // - Var1 [Op1]
+  // - Var2 [Op2]
+  // - hasMemoryOperands
+  // - hasAliasingImplicitRegisters (execution is always serial)
+  // - hasAliasingRegisters
+  const unsigned Opcode = llvm::X86::MOVSB;
+  const Instruction &Instr = State.getIC().getInstr(Opcode);
+  auto Error =
+      Generator.generateConfigurations(Instr, State.getRATC().emptyRegisters())
+          .takeError();
+  EXPECT_TRUE((bool)Error);
+  llvm::consumeError(std::move(Error));
+}
+
 TEST_F(FakeSnippetGeneratorTest, ComputeRegisterInitialValuesAdd16ri) {
   // ADD16ri:
   // explicit def 0       : reg RegClass=GR16
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetRepetitorTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetRepetitorTest.cpp
new file mode 100644
index 0000000000000..562fd058b61d8
--- /dev/null
+++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetRepetitorTest.cpp
@@ -0,0 +1,97 @@
+//===-- SnippetRepetitorTest.cpp --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "../Common/AssemblerUtils.h"
+#include "Latency.h"
+#include "LlvmState.h"
+#include "MCInstrDescView.h"
+#include "RegisterAliasing.h"
+#include "TestBase.h"
+#include "Uops.h"
+#include "X86InstrInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+
+namespace llvm {
+namespace exegesis {
+
+void InitializeX86ExegesisTarget();
+
+namespace {
+
+using testing::ElementsAre;
+using testing::Eq;
+using testing::Field;
+using testing::Property;
+using testing::UnorderedElementsAre;
+
+class X86SnippetRepetitorTest : public X86TestBase {
+protected:
+  void SetUp() {
+    TM = State.createTargetMachine();
+    Context = std::make_unique<LLVMContext>();
+    Mod =
+        std::make_unique<Module>("X86SnippetRepetitorTest", *Context);
+    Mod->setDataLayout(TM->createDataLayout());
+    MMI = std::make_unique<MachineModuleInfo>(TM.get());
+    MF = &createVoidVoidPtrMachineFunction("TestFn", Mod.get(), MMI.get());
+  }
+
+  void TestCommon(InstructionBenchmark::RepetitionModeE RepetitionMode) {
+    const auto Repetitor = SnippetRepetitor::Create(RepetitionMode, State);
+    const std::vector<MCInst> Instructions = {MCInstBuilder(X86::NOOP)};
+    FunctionFiller Sink(*MF, {X86::EAX});
+    const auto Fill = Repetitor->Repeat(Instructions, kMinInstructions);
+    Fill(Sink);
+  }
+
+  static constexpr const unsigned kMinInstructions = 3;
+
+  std::unique_ptr<LLVMTargetMachine> TM;
+  std::unique_ptr<LLVMContext> Context;
+  std::unique_ptr<Module> Mod;
+  std::unique_ptr<MachineModuleInfo> MMI;
+  MachineFunction *MF = nullptr;
+};
+
+static auto HasOpcode = [](unsigned Opcode) {
+  return Property(&MachineInstr::getOpcode, Eq(Opcode));
+};
+
+static auto LiveReg = [](unsigned Reg) {
+  return Field(&MachineBasicBlock::RegisterMaskPair::PhysReg, Eq(Reg));
+};
+
+TEST_F(X86SnippetRepetitorTest, Duplicate) {
+  TestCommon(InstructionBenchmark::Duplicate);
+  // Duplicating creates a single basic block that repeats the instructions.
+  ASSERT_EQ(MF->getNumBlockIDs(), 1u);
+  EXPECT_THAT(MF->getBlockNumbered(0)->instrs(),
+              ElementsAre(HasOpcode(X86::NOOP), HasOpcode(X86::NOOP),
+                          HasOpcode(X86::NOOP), HasOpcode(X86::RETQ)));
+}
+
+TEST_F(X86SnippetRepetitorTest, Loop) {
+  TestCommon(InstructionBenchmark::Loop);
+  // Duplicating creates an entry block, a loop body and a ret block.
+  ASSERT_EQ(MF->getNumBlockIDs(), 3u);
+  const auto &LoopBlock = *MF->getBlockNumbered(1);
+  EXPECT_THAT(LoopBlock.instrs(),
+              ElementsAre(HasOpcode(X86::NOOP), HasOpcode(X86::ADD64ri8),
+                          HasOpcode(X86::JCC_1)));
+  EXPECT_THAT(LoopBlock.liveins(),
+              UnorderedElementsAre(
+                  LiveReg(X86::EAX),
+                  LiveReg(State.getExegesisTarget().getLoopCounterRegister(
+                      State.getTargetMachine().getTargetTriple()))));
+  EXPECT_THAT(MF->getBlockNumbered(2)->instrs(),
+              ElementsAre(HasOpcode(X86::RETQ)));
+}
+
+} // namespace
+} // namespace exegesis
+} // namespace llvm
diff --git a/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h b/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
new file mode 100644
index 0000000000000..c60cd76e53c34
--- /dev/null
+++ b/llvm/unittests/tools/llvm-exegesis/X86/TestBase.h
@@ -0,0 +1,44 @@
+//===-- TestBase.cpp --------------------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// Test fixture common to all X86 tests.
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UNITTESTS_TOOLS_LLVMEXEGESIS_X86_TESTBASE_H
+#define LLVM_UNITTESTS_TOOLS_LLVMEXEGESIS_X86_TESTBASE_H
+
+#include "LlvmState.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace llvm {
+namespace exegesis {
+
+void InitializeX86ExegesisTarget();
+
+class X86TestBase : public ::testing::Test {
+protected:
+  X86TestBase() : State("x86_64-unknown-linux", "haswell") {}
+
+  static void SetUpTestCase() {
+    LLVMInitializeX86TargetInfo();
+    LLVMInitializeX86TargetMC();
+    LLVMInitializeX86Target();
+    LLVMInitializeX86AsmPrinter();
+    LLVMInitializeX86AsmParser();
+    InitializeX86ExegesisTarget();
+  }
+
+  const LLVMState State;
+};
+
+} // namespace exegesis
+} // namespace llvm
+
+#endif
diff --git a/llvm/utils/FileCheck/FileCheck.cpp b/llvm/utils/FileCheck/FileCheck.cpp
index 2710ccb406c7a..8718be28ac997 100644
--- a/llvm/utils/FileCheck/FileCheck.cpp
+++ b/llvm/utils/FileCheck/FileCheck.cpp
@@ -597,8 +597,7 @@ int main(int argc, char **argv) {
                             CheckFileText, CheckFile.getBufferIdentifier()),
                         SMLoc());
 
-  std::vector<FileCheckString> CheckStrings;
-  if (FC.ReadCheckFile(SM, CheckFileText, PrefixRE, CheckStrings))
+  if (FC.readCheckFile(SM, CheckFileText, PrefixRE))
     return 2;
 
   // Open the file to check and add it to SourceMgr.
@@ -628,7 +627,7 @@ int main(int argc, char **argv) {
     DumpInput = DumpInputOnFailure ? DumpInputFail : DumpInputNever;
 
   std::vector<FileCheckDiag> Diags;
-  int ExitCode = FC.CheckInput(SM, InputFileText, CheckStrings,
+  int ExitCode = FC.checkInput(SM, InputFileText,
                                DumpInput == DumpInputNever ? nullptr : &Diags)
                      ? EXIT_SUCCESS
                      : 1;
diff --git a/llvm/utils/TableGen/AsmMatcherEmitter.cpp b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
index d32366e2986d2..1d39b300091fc 100644
--- a/llvm/utils/TableGen/AsmMatcherEmitter.cpp
+++ b/llvm/utils/TableGen/AsmMatcherEmitter.cpp
@@ -1111,6 +1111,7 @@ static std::string getEnumNameForToken(StringRef Str) {
     case '<': Res += "_LT_"; break;
     case '>': Res += "_GT_"; break;
     case '-': Res += "_MINUS_"; break;
+    case '#': Res += "_HASH_"; break;
     default:
       if ((*it >= 'A' && *it <= 'Z') ||
           (*it >= 'a' && *it <= 'z') ||
diff --git a/llvm/utils/TableGen/CMakeLists.txt b/llvm/utils/TableGen/CMakeLists.txt
index c88365a2b8cec..77ef764d2f22a 100644
--- a/llvm/utils/TableGen/CMakeLists.txt
+++ b/llvm/utils/TableGen/CMakeLists.txt
@@ -1,3 +1,5 @@
+add_subdirectory(GlobalISel)
+
 set(LLVM_LINK_COMPONENTS Support)
 
 add_tablegen(llvm-tblgen LLVM
@@ -24,6 +26,7 @@ add_tablegen(llvm-tblgen LLVM
   ExegesisEmitter.cpp
   FastISelEmitter.cpp
   FixedLenDecoderEmitter.cpp
+  GICombinerEmitter.cpp
   GlobalISelEmitter.cpp
   InfoByHwMode.cpp
   InstrInfoEmitter.cpp
@@ -49,4 +52,5 @@ add_tablegen(llvm-tblgen LLVM
   WebAssemblyDisassemblerEmitter.cpp
   CTagsEmitter.cpp
   )
+target_link_libraries(llvm-tblgen PRIVATE LLVMTableGenGlobalISel)
 set_target_properties(llvm-tblgen PROPERTIES FOLDER "Tablegenning")
diff --git a/llvm/utils/TableGen/CodeEmitterGen.cpp b/llvm/utils/TableGen/CodeEmitterGen.cpp
index a653238c70417..b021d08881fb4 100644
--- a/llvm/utils/TableGen/CodeEmitterGen.cpp
+++ b/llvm/utils/TableGen/CodeEmitterGen.cpp
@@ -16,6 +16,7 @@
 #include "CodeGenTarget.h"
 #include "SubtargetFeatureInfo.h"
 #include "Types.h"
+#include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/Casting.h"
@@ -45,12 +46,19 @@ class CodeEmitterGen {
 private:
   int getVariableBit(const std::string &VarName, BitsInit *BI, int bit);
   std::string getInstructionCase(Record *R, CodeGenTarget &Target);
+  std::string getInstructionCaseForEncoding(Record *R, Record *EncodingDef,
+                                            CodeGenTarget &Target);
   void AddCodeToMergeInOperand(Record *R, BitsInit *BI,
                                const std::string &VarName,
                                unsigned &NumberedOp,
                                std::set<unsigned> &NamedOpIndices,
                                std::string &Case, CodeGenTarget &Target);
 
+  void emitInstructionBaseValues(
+      raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
+      CodeGenTarget &Target, int HwMode = -1);
+  unsigned BitWidth;
+  bool UseAPInt;
 };
 
 // If the VarBitInit at position 'bit' matches the specified variable then
@@ -126,7 +134,10 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName,
   
   std::pair<unsigned, unsigned> SO = CGI.Operands.getSubOperandNumber(OpIdx);
   std::string &EncoderMethodName = CGI.Operands[SO.first].EncoderMethodName;
-  
+
+  if (UseAPInt)
+    Case += "      op.clearAllBits();\n";
+
   // If the source operand has a custom encoder, use it. This will
   // get the encoding for all of the suboperands.
   if (!EncoderMethodName.empty()) {
@@ -134,18 +145,54 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName,
     // sub-operands, if there are more than one, so only
     // query the encoder once per source operand.
     if (SO.second == 0) {
-      Case += "      // op: " + VarName + "\n" +
-              "      op = " + EncoderMethodName + "(MI, " + utostr(OpIdx);
-      Case += ", Fixups, STI";
-      Case += ");\n";
+      Case += "      // op: " + VarName + "\n";
+      if (UseAPInt) {
+        Case += "      " + EncoderMethodName + "(MI, " + utostr(OpIdx);
+        Case += ", op";
+      } else {
+        Case += "      op = " + EncoderMethodName + "(MI, " + utostr(OpIdx);
+      }
+      Case += ", Fixups, STI);\n";
     }
   } else {
-    Case += "      // op: " + VarName + "\n" +
-      "      op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")";
-    Case += ", Fixups, STI";
+    Case += "      // op: " + VarName + "\n";
+    if (UseAPInt) {
+      Case += "      getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")";
+      Case += ", op, Fixups, STI";
+    } else {
+      Case += "      op = getMachineOpValue(MI, MI.getOperand(" + utostr(OpIdx) + ")";
+      Case += ", Fixups, STI";
+    }
     Case += ");\n";
   }
-  
+
+  // Precalculate the number of lits this variable contributes to in the
+  // operand. If there is a single lit (consecutive range of bits) we can use a
+  // destructive sequence on APInt that reduces memory allocations.
+  int numOperandLits = 0;
+  for (int tmpBit = bit; tmpBit >= 0;) {
+    int varBit = getVariableBit(VarName, BI, tmpBit);
+
+    // If this bit isn't from a variable, skip it.
+    if (varBit == -1) {
+      --tmpBit;
+      continue;
+    }
+
+    // Figure out the consecutive range of bits covered by this operand, in
+    // order to generate better encoding code.
+    int beginVarBit = varBit;
+    int N = 1;
+    for (--tmpBit; tmpBit >= 0;) {
+      varBit = getVariableBit(VarName, BI, tmpBit);
+      if (varBit == -1 || varBit != (beginVarBit - N))
+        break;
+      ++N;
+      --tmpBit;
+    }
+    ++numOperandLits;
+  }
+
   for (; bit >= 0; ) {
     int varBit = getVariableBit(VarName, BI, bit);
     
@@ -166,20 +213,52 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName,
       ++N;
       --bit;
     }
-     
-    uint64_t opMask = ~(uint64_t)0 >> (64-N);
-    int opShift = beginVarBit - N + 1;
-    opMask <<= opShift;
-    opShift = beginInstBit - beginVarBit;
-    
-    if (opShift > 0) {
-      Case += "      Value |= (op & UINT64_C(" + utostr(opMask) + ")) << " +
-              itostr(opShift) + ";\n";
-    } else if (opShift < 0) {
-      Case += "      Value |= (op & UINT64_C(" + utostr(opMask) + ")) >> " + 
-              itostr(-opShift) + ";\n";
+
+    std::string maskStr;
+    int opShift;
+
+    unsigned loBit = beginVarBit - N + 1;
+    unsigned hiBit = loBit + N;
+    unsigned loInstBit = beginInstBit - N + 1;
+    if (UseAPInt) {
+      std::string extractStr;
+      if (N >= 64) {
+        extractStr = "op.extractBits(" + itostr(hiBit - loBit) + ", " +
+                     itostr(loBit) + ")";
+        Case += "      Value.insertBits(" + extractStr + ", " +
+                itostr(loInstBit) + ");\n";
+      } else {
+        extractStr = "op.extractBitsAsZExtValue(" + itostr(hiBit - loBit) +
+                     ", " + itostr(loBit) + ")";
+        Case += "      Value.insertBits(" + extractStr + ", " +
+                itostr(loInstBit) + ", " + itostr(hiBit - loBit) + ");\n";
+      }
     } else {
-      Case += "      Value |= op & UINT64_C(" + utostr(opMask) + ");\n";
+      uint64_t opMask = ~(uint64_t)0 >> (64 - N);
+      opShift = beginVarBit - N + 1;
+      opMask <<= opShift;
+      maskStr = "UINT64_C(" + utostr(opMask) + ")";
+      opShift = beginInstBit - beginVarBit;
+
+      if (numOperandLits == 1) {
+        Case += "      op &= " + maskStr + ";\n";
+        if (opShift > 0) {
+          Case += "      op <<= " + itostr(opShift) + ";\n";
+        } else if (opShift < 0) {
+          Case += "      op >>= " + itostr(-opShift) + ";\n";
+        }
+        Case += "      Value |= op;\n";
+      } else {
+        if (opShift > 0) {
+          Case += "      Value |= (op & " + maskStr + ") << " +
+                  itostr(opShift) + ";\n";
+        } else if (opShift < 0) {
+          Case += "      Value |= (op & " + maskStr + ") >> " +
+                  itostr(-opShift) + ";\n";
+        } else {
+          Case += "      Value |= (op & " + maskStr + ");\n";
+        }
+      }
     }
   }
 }
@@ -187,7 +266,29 @@ AddCodeToMergeInOperand(Record *R, BitsInit *BI, const std::string &VarName,
 std::string CodeEmitterGen::getInstructionCase(Record *R,
                                                CodeGenTarget &Target) {
   std::string Case;
-  BitsInit *BI = R->getValueAsBitsInit("Inst");
+  if (const RecordVal *RV = R->getValue("EncodingInfos")) {
+    if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+      const CodeGenHwModes &HWM = Target.getHwModes();
+      EncodingInfoByHwMode EBM(DI->getDef(), HWM);
+      Case += "      switch (HwMode) {\n";
+      Case += "      default: llvm_unreachable(\"Unhandled HwMode\");\n";
+      for (auto &KV : EBM.Map) {
+        Case += "      case " + itostr(KV.first) + ": {\n";
+        Case += getInstructionCaseForEncoding(R, KV.second, Target);
+        Case += "      break;\n";
+        Case += "      }\n";
+      }
+      Case += "      }\n";
+      return Case;
+    }
+  }
+  return getInstructionCaseForEncoding(R, R, Target);
+}
+
+std::string CodeEmitterGen::getInstructionCaseForEncoding(Record *R, Record *EncodingDef,
+                                                          CodeGenTarget &Target) {
+  std::string Case;
+  BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst");
   unsigned NumberedOp = 0;
   std::set<unsigned> NamedOpIndices;
 
@@ -207,7 +308,7 @@ std::string CodeEmitterGen::getInstructionCase(Record *R,
 
   // Loop over all of the fields in the instruction, determining which are the
   // operands to the instruction.
-  for (const RecordVal &RV : R->getValues()) { 
+  for (const RecordVal &RV : EncodingDef->getValues()) {
     // Ignore fixed fields in the record, we're looking for values like:
     //    bits<5> RST = { ?, ?, ?, ?, ? };
     if (RV.getPrefix() || RV.getValue()->isComplete())
@@ -237,6 +338,53 @@ getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset) {
   return Name;
 }
 
+static void emitInstBits(raw_ostream &OS, const APInt &Bits) {
+  for (unsigned I = 0; I < Bits.getNumWords(); ++I)
+    OS << ((I > 0) ? ", " : "") << "UINT64_C(" << utostr(Bits.getRawData()[I])
+       << ")";
+}
+
+void CodeEmitterGen::emitInstructionBaseValues(
+    raw_ostream &o, ArrayRef<const CodeGenInstruction *> NumberedInstructions,
+    CodeGenTarget &Target, int HwMode) {
+  const CodeGenHwModes &HWM = Target.getHwModes();
+  if (HwMode == -1)
+    o << "  static const uint64_t InstBits[] = {\n";
+  else
+    o << "  static const uint64_t InstBits_" << HWM.getMode(HwMode).Name
+      << "[] = {\n";
+
+  for (const CodeGenInstruction *CGI : NumberedInstructions) {
+    Record *R = CGI->TheDef;
+
+    if (R->getValueAsString("Namespace") == "TargetOpcode" ||
+        R->getValueAsBit("isPseudo")) {
+      o << "    "; emitInstBits(o, APInt(BitWidth, 0)); o << ",\n";
+      continue;
+    }
+
+    Record *EncodingDef = R;
+    if (const RecordVal *RV = R->getValue("EncodingInfos")) {
+      if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
+        EncodingDef = EBM.get(HwMode);
+      }
+    }
+    BitsInit *BI = EncodingDef->getValueAsBitsInit("Inst");
+
+    // Start by filling in fixed values.
+    APInt Value(BitWidth, 0);
+    for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) {
+      if (BitInit *B = dyn_cast<BitInit>(BI->getBit(e - i - 1)))
+        Value |= APInt(BitWidth, (uint64_t)B->getValue()) << (e - i - 1);
+    }
+    o << "    ";
+    emitInstBits(o, Value);
+    o << "," << '\t' << "// " << R->getName() << "\n";
+  }
+  o << "    UINT64_C(0)\n  };\n";
+}
+
 void CodeEmitterGen::run(raw_ostream &o) {
   CodeGenTarget Target(Records);
   std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction");
@@ -247,34 +395,66 @@ void CodeEmitterGen::run(raw_ostream &o) {
   ArrayRef<const CodeGenInstruction*> NumberedInstructions =
     Target.getInstructionsByEnumValue();
 
-  // Emit function declaration
-  o << "uint64_t " << Target.getName();
-  o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
-    << "    SmallVectorImpl<MCFixup> &Fixups,\n"
-    << "    const MCSubtargetInfo &STI) const {\n";
-
-  // Emit instruction base values
-  o << "  static const uint64_t InstBits[] = {\n";
+  const CodeGenHwModes &HWM = Target.getHwModes();
+  // The set of HwModes used by instruction encodings.
+  std::set<unsigned> HwModes;
+  BitWidth = 0;
   for (const CodeGenInstruction *CGI : NumberedInstructions) {
     Record *R = CGI->TheDef;
-
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
-        R->getValueAsBit("isPseudo")) {
-      o << "    UINT64_C(0),\n";
+        R->getValueAsBit("isPseudo"))
       continue;
-    }
 
+    if (const RecordVal *RV = R->getValue("EncodingInfos")) {
+      if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
+        for (auto &KV : EBM.Map) {
+          BitsInit *BI = KV.second->getValueAsBitsInit("Inst");
+          BitWidth = std::max(BitWidth, BI->getNumBits());
+          HwModes.insert(KV.first);
+        }
+        continue;
+      }
+    }
     BitsInit *BI = R->getValueAsBitsInit("Inst");
+    BitWidth = std::max(BitWidth, BI->getNumBits());
+  }
+  UseAPInt = BitWidth > 64;
+  
+  // Emit function declaration
+  if (UseAPInt) {
+    o << "void " << Target.getName()
+      << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
+      << "    SmallVectorImpl<MCFixup> &Fixups,\n"
+      << "    APInt &Inst,\n"
+      << "    APInt &Scratch,\n"
+      << "    const MCSubtargetInfo &STI) const {\n";
+  } else {
+    o << "uint64_t " << Target.getName();
+    o << "MCCodeEmitter::getBinaryCodeForInstr(const MCInst &MI,\n"
+      << "    SmallVectorImpl<MCFixup> &Fixups,\n"
+      << "    const MCSubtargetInfo &STI) const {\n";
+  }
+  
+  // Emit instruction base values
+  if (HwModes.empty()) {
+    emitInstructionBaseValues(o, NumberedInstructions, Target, -1);
+  } else {
+    for (unsigned HwMode : HwModes)
+      emitInstructionBaseValues(o, NumberedInstructions, Target, (int)HwMode);
+  }
 
-    // Start by filling in fixed values.
-    uint64_t Value = 0;
-    for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i) {
-      if (BitInit *B = dyn_cast<BitInit>(BI->getBit(e-i-1)))
-        Value |= (uint64_t)B->getValue() << (e-i-1);
+  if (!HwModes.empty()) {
+    o << "  const uint64_t *InstBits;\n";
+    o << "  unsigned HwMode = STI.getHwMode();\n";
+    o << "  switch (HwMode) {\n";
+    o << "  default: llvm_unreachable(\"Unknown hardware mode!\"); break;\n";
+    for (unsigned I : HwModes) {
+      o << "  case " << I << ": InstBits = InstBits_" << HWM.getMode(I).Name
+        << "; break;\n";
     }
-    o << "    UINT64_C(" << Value << ")," << '\t' << "// " << R->getName() << "\n";
+    o << "  };\n";
   }
-  o << "    UINT64_C(0)\n  };\n";
 
   // Map to accumulate all the cases.
   std::map<std::string, std::vector<std::string>> CaseMap;
@@ -294,11 +474,26 @@ void CodeEmitterGen::run(raw_ostream &o) {
   }
 
   // Emit initial function code
-  o << "  const unsigned opcode = MI.getOpcode();\n"
-    << "  uint64_t Value = InstBits[opcode];\n"
-    << "  uint64_t op = 0;\n"
-    << "  (void)op;  // suppress warning\n"
-    << "  switch (opcode) {\n";
+  if (UseAPInt) {
+    int NumWords = APInt::getNumWords(BitWidth);
+    int NumBytes = (BitWidth + 7) / 8;
+    o << "  const unsigned opcode = MI.getOpcode();\n"
+      << "  if (Inst.getBitWidth() != " << BitWidth << ")\n"
+      << "    Inst = Inst.zext(" << BitWidth << ");\n"
+      << "  if (Scratch.getBitWidth() != " << BitWidth << ")\n"
+      << "    Scratch = Scratch.zext(" << BitWidth << ");\n"
+      << "  LoadIntFromMemory(Inst, (uint8_t*)&InstBits[opcode * " << NumWords
+      << "], " << NumBytes << ");\n"
+      << "  APInt &Value = Inst;\n"
+      << "  APInt &op = Scratch;\n"
+      << "  switch (opcode) {\n";
+  } else {
+    o << "  const unsigned opcode = MI.getOpcode();\n"
+      << "  uint64_t Value = InstBits[opcode];\n"
+      << "  uint64_t op = 0;\n"
+      << "  (void)op;  // suppress warning\n"
+      << "  switch (opcode) {\n";
+  }
 
   // Emit each case statement
   std::map<std::string, std::vector<std::string>>::iterator IE, EE;
@@ -322,9 +517,12 @@ void CodeEmitterGen::run(raw_ostream &o) {
     << "    raw_string_ostream Msg(msg);\n"
     << "    Msg << \"Not supported instr: \" << MI;\n"
     << "    report_fatal_error(Msg.str());\n"
-    << "  }\n"
-    << "  return Value;\n"
-    << "}\n\n";
+    << "  }\n";
+  if (UseAPInt)
+    o << "  Inst = Value;\n";
+  else
+    o << "  return Value;\n";
+  o << "}\n\n";
 
   const auto &All = SubtargetFeatureInfo::getAll(Records);
   std::map<Record *, SubtargetFeatureInfo, LessRecordByID> SubtargetFeatures;
diff --git a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
index 07fffaec73d11..46f986ca0176b 100644
--- a/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/CodeGenDAGPatterns.cpp
@@ -769,7 +769,10 @@ void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out,
         for (MVT T : MVT::integer_valuetypes())
           if (Legal.count(T))
             Out.insert(T);
-        for (MVT T : MVT::integer_vector_valuetypes())
+        for (MVT T : MVT::integer_fixedlen_vector_valuetypes())
+          if (Legal.count(T))
+            Out.insert(T);
+        for (MVT T : MVT::integer_scalable_vector_valuetypes())
           if (Legal.count(T))
             Out.insert(T);
         return;
@@ -777,7 +780,10 @@ void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out,
         for (MVT T : MVT::fp_valuetypes())
           if (Legal.count(T))
             Out.insert(T);
-        for (MVT T : MVT::fp_vector_valuetypes())
+        for (MVT T : MVT::fp_fixedlen_vector_valuetypes())
+          if (Legal.count(T))
+            Out.insert(T);
+        for (MVT T : MVT::fp_scalable_vector_valuetypes())
           if (Legal.count(T))
             Out.insert(T);
         return;
@@ -2791,6 +2797,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(Init *TheInit,
 
     if (Operator->isSubClassOf("SDNode") &&
         Operator->getName() != "imm" &&
+        Operator->getName() != "timm" &&
         Operator->getName() != "fpimm" &&
         Operator->getName() != "tglobaltlsaddr" &&
         Operator->getName() != "tconstpool" &&
diff --git a/llvm/utils/TableGen/CodeGenMapTable.cpp b/llvm/utils/TableGen/CodeGenMapTable.cpp
index 7a5175ac35462..793bb61481e7a 100644
--- a/llvm/utils/TableGen/CodeGenMapTable.cpp
+++ b/llvm/utils/TableGen/CodeGenMapTable.cpp
@@ -132,7 +132,7 @@ class InstrMap {
         MapRec->getName() + "' has empty " + "`ValueCols' field!");
 
     for (Init *I : ColValList->getValues()) {
-      ListInit *ColI = dyn_cast<ListInit>(I);
+      auto *ColI = cast<ListInit>(I);
 
       // Make sure that all the sub-lists in 'ValueCols' have same number of
       // elements as the fields in 'ColFields'.
@@ -521,7 +521,7 @@ static void emitEnums(raw_ostream &OS, RecordKeeper &Records) {
     unsigned ListSize = List->size();
 
     for (unsigned j = 0; j < ListSize; j++) {
-      ListInit *ListJ = dyn_cast<ListInit>(List->getElement(j));
+      auto *ListJ = cast<ListInit>(List->getElement(j));
 
       if (ListJ->size() != ColFields->size())
         PrintFatalError("Record `" + CurMap->getName() + "', field "
diff --git a/llvm/utils/TableGen/CodeGenSchedule.cpp b/llvm/utils/TableGen/CodeGenSchedule.cpp
index cb05f78fba411..f12d7d484a8ea 100644
--- a/llvm/utils/TableGen/CodeGenSchedule.cpp
+++ b/llvm/utils/TableGen/CodeGenSchedule.cpp
@@ -1083,9 +1083,13 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
             if (RWD->getValueAsDef("SchedModel") == RWModelDef &&
                 RWModelDef->getValueAsBit("FullInstRWOverlapCheck")) {
               for (Record *Inst : InstDefs) {
-                PrintFatalError(InstRWDef->getLoc(), "Overlapping InstRW def " +
-                            Inst->getName() + " also matches " +
-                            RWD->getValue("Instrs")->getValue()->getAsString());
+                PrintFatalError
+                    (InstRWDef->getLoc(),
+                     "Overlapping InstRW definition for \"" +
+                     Inst->getName() +
+                     "\" also matches previous \"" +
+                     RWD->getValue("Instrs")->getValue()->getAsString() +
+                     "\".");
               }
             }
           }
@@ -1115,9 +1119,13 @@ void CodeGenSchedModels::createInstRWClass(Record *InstRWDef) {
       for (Record *OldRWDef : SchedClasses[OldSCIdx].InstRWs) {
         if (OldRWDef->getValueAsDef("SchedModel") == RWModelDef) {
           for (Record *InstDef : InstDefs) {
-            PrintFatalError(OldRWDef->getLoc(), "Overlapping InstRW def " +
-                       InstDef->getName() + " also matches " +
-                       OldRWDef->getValue("Instrs")->getValue()->getAsString());
+            PrintFatalError
+                (InstRWDef->getLoc(),
+                 "Overlapping InstRW definition for \"" +
+                 InstDef->getName() +
+                 "\" also matches previous \"" +
+                 OldRWDef->getValue("Instrs")->getValue()->getAsString() +
+                 "\".");
           }
         }
         assert(OldRWDef != InstRWDef &&
diff --git a/llvm/utils/TableGen/CodeGenTarget.cpp b/llvm/utils/TableGen/CodeGenTarget.cpp
index 478bbb7e4809f..fa8b842c97f99 100644
--- a/llvm/utils/TableGen/CodeGenTarget.cpp
+++ b/llvm/utils/TableGen/CodeGenTarget.cpp
@@ -478,7 +478,8 @@ void CodeGenTarget::reverseBitsForLittleEndianEncoding() {
   if (!isLittleEndianEncoding())
     return;
 
-  std::vector<Record*> Insts = Records.getAllDerivedDefinitions("Instruction");
+  std::vector<Record *> Insts =
+      Records.getAllDerivedDefinitions("InstructionEncoding");
   for (Record *R : Insts) {
     if (R->getValueAsString("Namespace") == "TargetOpcode" ||
         R->getValueAsBit("isPseudo"))
diff --git a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
index 7d51b07693720..6746fdd676a7f 100644
--- a/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
+++ b/llvm/utils/TableGen/DAGISelMatcherOpt.cpp
@@ -409,13 +409,14 @@ static void FactorNodes(std::unique_ptr<Matcher> &InputMatcherPtr) {
     DenseMap<unsigned, unsigned> TypeEntry;
     SmallVector<std::pair<MVT::SimpleValueType, Matcher*>, 8> Cases;
     for (unsigned i = 0, e = NewOptionsToMatch.size(); i != e; ++i) {
-      CheckTypeMatcher *CTM =
-        cast_or_null<CheckTypeMatcher>(FindNodeWithKind(NewOptionsToMatch[i],
-                                                        Matcher::CheckType));
+      Matcher* M = FindNodeWithKind(NewOptionsToMatch[i], Matcher::CheckType);
+      assert(M && isa<CheckTypeMatcher>(M) && "Unknown Matcher type");
+
+      auto *CTM = cast<CheckTypeMatcher>(M);
       Matcher *MatcherWithoutCTM = NewOptionsToMatch[i]->unlinkNode(CTM);
       MVT::SimpleValueType CTMTy = CTM->getType();
       delete CTM;
-      
+
       unsigned &Entry = TypeEntry[CTMTy];
       if (Entry != 0) {
         // If we have unfactored duplicate types, then we should factor them.
diff --git a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
index 01f4115aa1dd6..ac69b431607df 100644
--- a/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
+++ b/llvm/utils/TableGen/FixedLenDecoderEmitter.cpp
@@ -13,6 +13,7 @@
 
 #include "CodeGenInstruction.h"
 #include "CodeGenTarget.h"
+#include "InfoByHwMode.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/CachedHashString.h"
@@ -97,9 +98,11 @@ struct DecoderTableInfo {
 struct EncodingAndInst {
   const Record *EncodingDef;
   const CodeGenInstruction *Inst;
+  StringRef HwModeName;
 
-  EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst)
-      : EncodingDef(EncodingDef), Inst(Inst) {}
+  EncodingAndInst(const Record *EncodingDef, const CodeGenInstruction *Inst,
+                  StringRef HwModeName = "")
+      : EncodingDef(EncodingDef), Inst(Inst), HwModeName(HwModeName) {}
 };
 
 struct EncodingIDAndOpcode {
@@ -2382,12 +2385,50 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
   Target.reverseBitsForLittleEndianEncoding();
 
   // Parameterize the decoders based on namespace and instruction width.
+  std::set<StringRef> HwModeNames;
   const auto &NumberedInstructions = Target.getInstructionsByEnumValue();
   NumberedEncodings.reserve(NumberedInstructions.size());
   DenseMap<Record *, unsigned> IndexOfInstruction;
+  // First, collect all HwModes referenced by the target.
   for (const auto &NumberedInstruction : NumberedInstructions) {
     IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size();
-    NumberedEncodings.emplace_back(NumberedInstruction->TheDef, NumberedInstruction);
+
+    if (const RecordVal *RV =
+            NumberedInstruction->TheDef->getValue("EncodingInfos")) {
+      if (auto *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+        const CodeGenHwModes &HWM = Target.getHwModes();
+        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
+        for (auto &KV : EBM.Map)
+          HwModeNames.insert(HWM.getMode(KV.first).Name);
+      }
+    }
+  }
+
+  // If HwModeNames is empty, add the empty string so we always have one HwMode.
+  if (HwModeNames.empty())
+    HwModeNames.insert("");
+
+  for (const auto &NumberedInstruction : NumberedInstructions) {
+    IndexOfInstruction[NumberedInstruction->TheDef] = NumberedEncodings.size();
+
+    if (const RecordVal *RV =
+            NumberedInstruction->TheDef->getValue("EncodingInfos")) {
+      if (DefInit *DI = dyn_cast_or_null<DefInit>(RV->getValue())) {
+        const CodeGenHwModes &HWM = Target.getHwModes();
+        EncodingInfoByHwMode EBM(DI->getDef(), HWM);
+        for (auto &KV : EBM.Map) {
+          NumberedEncodings.emplace_back(KV.second, NumberedInstruction,
+                                         HWM.getMode(KV.first).Name);
+          HwModeNames.insert(HWM.getMode(KV.first).Name);
+        }
+        continue;
+      }
+    }
+    // This instruction is encoded the same on all HwModes. Emit it for all
+    // HwModes.
+    for (StringRef HwModeName : HwModeNames)
+      NumberedEncodings.emplace_back(NumberedInstruction->TheDef,
+                                     NumberedInstruction, HwModeName);
   }
   for (const auto &NumberedAlias : RK.getAllDerivedDefinitions("AdditionalEncoding"))
     NumberedEncodings.emplace_back(
@@ -2415,13 +2456,19 @@ void FixedLenDecoderEmitter::run(raw_ostream &o) {
       NumInstructions++;
     NumEncodings++;
 
-    StringRef DecoderNamespace = EncodingDef->getValueAsString("DecoderNamespace");
+    if (!Size)
+      continue;
 
-    if (Size) {
-      if (populateInstruction(Target, *EncodingDef, *Inst, i, Operands)) {
-        OpcMap[std::make_pair(DecoderNamespace, Size)].emplace_back(i, IndexOfInstruction.find(Def)->second);
-      } else
-        NumEncodingsOmitted++;
+    if (populateInstruction(Target, *EncodingDef, *Inst, i, Operands)) {
+      std::string DecoderNamespace =
+          EncodingDef->getValueAsString("DecoderNamespace");
+      if (!NumberedEncodings[i].HwModeName.empty())
+        DecoderNamespace +=
+            std::string("_") + NumberedEncodings[i].HwModeName.str();
+      OpcMap[std::make_pair(DecoderNamespace, Size)].emplace_back(
+          i, IndexOfInstruction.find(Def)->second);
+    } else {
+      NumEncodingsOmitted++;
     }
   }
 
diff --git a/llvm/utils/TableGen/GICombinerEmitter.cpp b/llvm/utils/TableGen/GICombinerEmitter.cpp
new file mode 100644
index 0000000000000..c2b64bcfb7c5e
--- /dev/null
+++ b/llvm/utils/TableGen/GICombinerEmitter.cpp
@@ -0,0 +1,105 @@
+//===- GlobalCombinerEmitter.cpp - Generate a combiner --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Generate a combiner implementation for GlobalISel from a declarative
+/// syntax
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Timer.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include "CodeGenTarget.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "gicombiner-emitter"
+
+cl::OptionCategory
+    GICombinerEmitterCat("Options for -gen-global-isel-combiner");
+static cl::list<std::string>
+    SelectedCombiners("combiners", cl::desc("Emit the specified combiners"),
+                      cl::cat(GICombinerEmitterCat), cl::CommaSeparated);
+static cl::opt<bool> ShowExpansions(
+    "gicombiner-show-expansions",
+    cl::desc("Use C++ comments to indicate occurence of code expansion"),
+    cl::cat(GICombinerEmitterCat));
+
+namespace {
+class GICombinerEmitter {
+  StringRef Name;
+  Record *Combiner;
+public:
+  explicit GICombinerEmitter(RecordKeeper &RK, StringRef Name,
+                             Record *Combiner);
+  ~GICombinerEmitter() {}
+
+  StringRef getClassName() const {
+    return Combiner->getValueAsString("Classname");
+  }
+  void run(raw_ostream &OS);
+
+};
+
+GICombinerEmitter::GICombinerEmitter(RecordKeeper &RK, StringRef Name,
+                                     Record *Combiner)
+    : Name(Name), Combiner(Combiner) {}
+
+void GICombinerEmitter::run(raw_ostream &OS) {
+  NamedRegionTimer T("Emit", "Time spent emitting the combiner",
+                     "Code Generation", "Time spent generating code",
+                     TimeRegions);
+  OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n"
+     << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_DEPS\n\n";
+
+  OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n"
+     << "class " << getClassName() << " {\n"
+     << "public:\n"
+     << "  bool tryCombineAll(\n"
+     << "    GISelChangeObserver &Observer,\n"
+     << "    MachineInstr &MI,\n"
+     << "    MachineIRBuilder &B) const;\n"
+     << "};\n";
+  OS << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_H\n\n";
+
+  OS << "#ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n"
+     << "\n"
+     << "bool " << getClassName() << "::tryCombineAll(\n"
+     << "    GISelChangeObserver &Observer,\n"
+     << "    MachineInstr &MI,\n"
+     << "    MachineIRBuilder &B) const {\n"
+     << "  MachineBasicBlock *MBB = MI.getParent();\n"
+     << "  MachineFunction *MF = MBB->getParent();\n"
+     << "  MachineRegisterInfo &MRI = MF->getRegInfo();\n"
+     << "  (void)MBB; (void)MF; (void)MRI;\n\n";
+  OS << "\n  return false;\n"
+     << "}\n"
+     << "#endif // ifdef " << Name.upper() << "_GENCOMBINERHELPER_CPP\n";
+}
+
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+
+namespace llvm {
+void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS) {
+  CodeGenTarget Target(RK);
+  emitSourceFileHeader("Global Combiner", OS);
+
+  if (SelectedCombiners.empty())
+    PrintFatalError("No combiners selected with -combiners");
+  for (const auto &Combiner : SelectedCombiners) {
+    Record *CombinerDef = RK.getDef(Combiner);
+    if (!CombinerDef)
+      PrintFatalError("Could not find " + Combiner);
+    GICombinerEmitter(RK, Combiner, CombinerDef).run(OS);
+  }
+}
+
+} // namespace llvm
diff --git a/llvm/utils/TableGen/GlobalISel/CMakeLists.txt b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
new file mode 100644
index 0000000000000..ea4713d8b29ab
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISel/CMakeLists.txt
@@ -0,0 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  )
+
+llvm_add_library(LLVMTableGenGlobalISel STATIC
+                 CodeExpander.cpp
+                )
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp b/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp
new file mode 100644
index 0000000000000..d59a9b8e3b65e
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISel/CodeExpander.cpp
@@ -0,0 +1,93 @@
+//===- CodeExpander.cpp - Expand variables in a string --------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#include "CodeExpander.h"
+#include "CodeExpansions.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/TableGen/Error.h"
+
+using namespace llvm;
+
+void CodeExpander::emit(raw_ostream &OS) const {
+  StringRef Current = Code;
+
+  while (!Current.empty()) {
+    size_t Pos = Current.find_first_of("$\n\\");
+    if (Pos == StringRef::npos) {
+      OS << Current;
+      Current = "";
+      continue;
+    }
+
+    OS << Current.substr(0, Pos);
+    Current = Current.substr(Pos);
+
+    if (Current.startswith("\n")) {
+      OS << "\n" << Indent;
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("\\$") || Current.startswith("\\\\")) {
+      OS << Current[1];
+      Current = Current.drop_front(2);
+      continue;
+    }
+
+    if (Current.startswith("\\")) {
+      Current = Current.drop_front(1);
+      continue;
+    }
+
+    if (Current.startswith("${")) {
+      StringRef StartVar = Current;
+      Current = Current.drop_front(2);
+      StringRef Var;
+      std::tie(Var, Current) = Current.split("}");
+
+      // Warn if we split because no terminator was found.
+      StringRef EndVar = StartVar.drop_front(2 /* ${ */ + Var.size());
+      if (EndVar.empty()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintWarning(
+            Loc.size() > 0 && Loc[0].isValid()
+                ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                : SMLoc(),
+            "Unterminated expansion");
+      }
+
+      auto ValueI = Expansions.find(Var);
+      if (ValueI == Expansions.end()) {
+        size_t LocOffset = StartVar.data() - Code.data();
+        PrintError(Loc.size() > 0 && Loc[0].isValid()
+                       ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                       : SMLoc(),
+                   "Attempting to expand an undeclared variable " + Var);
+      }
+      if (ShowExpansions)
+        OS << "/*$" << Var << "{*/";
+      OS << Expansions.lookup(Var);
+      if (ShowExpansions)
+        OS << "/*}*/";
+      continue;
+    }
+
+    size_t LocOffset = Current.data() - Code.data();
+    PrintWarning(Loc.size() > 0 && Loc[0].isValid()
+                     ? SMLoc::getFromPointer(Loc[0].getPointer() + LocOffset)
+                     : SMLoc(),
+                 "Assuming missing escape character");
+    OS << "$";
+    Current = Current.drop_front(1);
+  }
+}
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpander.h b/llvm/utils/TableGen/GlobalISel/CodeExpander.h
new file mode 100644
index 0000000000000..bd6946de59258
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISel/CodeExpander.h
@@ -0,0 +1,55 @@
+//===- CodeExpander.h - Expand variables in a string ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Expand the variables in a string.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/SMLoc.h"
+
+namespace llvm {
+class CodeExpansions;
+class raw_ostream;
+
+/// Emit the given code with all '${foo}' placeholders expanded to their
+/// replacements.
+///
+/// It's an error to use an undefined expansion and expansion-like output that
+/// needs to be emitted verbatim can be escaped as '\${foo}'
+///
+/// The emitted code can be given a custom indent to enable both indentation by
+/// an arbitrary amount of whitespace and emission of the code as a comment.
+class CodeExpander {
+  StringRef Code;
+  const CodeExpansions &Expansions;
+  const ArrayRef<SMLoc> &Loc;
+  bool ShowExpansions;
+  StringRef Indent;
+
+public:
+  CodeExpander(StringRef Code, const CodeExpansions &Expansions,
+               const ArrayRef<SMLoc> &Loc, bool ShowExpansions,
+               StringRef Indent = "    ")
+      : Code(Code), Expansions(Expansions), Loc(Loc),
+        ShowExpansions(ShowExpansions), Indent(Indent) {}
+
+  void emit(raw_ostream &OS) const;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const CodeExpander &Expander) {
+  Expander.emit(OS);
+  return OS;
+}
+} // end namespace llvm
+
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANDER_H
diff --git a/llvm/utils/TableGen/GlobalISel/CodeExpansions.h b/llvm/utils/TableGen/GlobalISel/CodeExpansions.h
new file mode 100644
index 0000000000000..bb890ec8f57ef
--- /dev/null
+++ b/llvm/utils/TableGen/GlobalISel/CodeExpansions.h
@@ -0,0 +1,43 @@
+//===- CodeExpansions.h - Record expansions for CodeExpander --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file Record the expansions to use in a CodeExpander.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringMap.h"
+
+#ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+#define LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
+namespace llvm {
+class CodeExpansions {
+public:
+  using const_iterator = StringMap<std::string>::const_iterator;
+
+protected:
+  StringMap<std::string> Expansions;
+
+public:
+  void declare(StringRef Name, StringRef Expansion) {
+    bool Inserted = Expansions.try_emplace(Name, Expansion).second;
+    assert(Inserted && "Declared variable twice");
+    (void)Inserted;
+  }
+
+  std::string lookup(StringRef Variable) const {
+    return Expansions.lookup(Variable);
+  }
+
+  const_iterator begin() const { return Expansions.begin(); }
+  const_iterator end() const { return Expansions.end(); }
+  const_iterator find(StringRef Variable) const {
+    return Expansions.find(Variable);
+  }
+};
+} // end namespace llvm
+#endif // ifndef LLVM_UTILS_TABLEGEN_CODEEXPANSIONS_H
diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp
index fbf0b4bd9ddd4..d8d4c9f4f55cf 100644
--- a/llvm/utils/TableGen/GlobalISelEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp
@@ -1062,6 +1062,7 @@ class PredicateMatcher {
     IPM_Opcode,
     IPM_NumOperands,
     IPM_ImmPredicate,
+    IPM_Imm,
     IPM_AtomicOrderingMMO,
     IPM_MemoryLLTSize,
     IPM_MemoryVsLLTSize,
@@ -1340,6 +1341,23 @@ class MBBOperandMatcher : public OperandPredicateMatcher {
   }
 };
 
+class ImmOperandMatcher : public OperandPredicateMatcher {
+public:
+  ImmOperandMatcher(unsigned InsnVarID, unsigned OpIdx)
+      : OperandPredicateMatcher(IPM_Imm, InsnVarID, OpIdx) {}
+
+  static bool classof(const PredicateMatcher *P) {
+    return P->getKind() == IPM_Imm;
+  }
+
+  void emitPredicateOpcodes(MatchTable &Table,
+                            RuleMatcher &Rule) const override {
+    Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI")
+          << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
+          << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+  }
+};
+
 /// Generates code to check that an operand is a G_CONSTANT with a particular
 /// int.
 class ConstantIntOperandMatcher : public OperandPredicateMatcher {
@@ -3794,6 +3812,10 @@ Error GlobalISelEmitter::importChildMatcher(RuleMatcher &Rule,
         OM.addPredicate<MBBOperandMatcher>();
         return Error::success();
       }
+      if (SrcChild->getOperator()->getName() == "timm") {
+        OM.addPredicate<ImmOperandMatcher>();
+        return Error::success();
+      }
     }
   }
 
@@ -3943,7 +3965,10 @@ Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
     // rendered as operands.
     // FIXME: The target should be able to choose sign-extended when appropriate
     //        (e.g. on Mips).
-    if (DstChild->getOperator()->getName() == "imm") {
+    if (DstChild->getOperator()->getName() == "timm") {
+      DstMIBuilder.addRenderer<CopyRenderer>(DstChild->getName());
+      return InsertPt;
+    } else if (DstChild->getOperator()->getName() == "imm") {
       DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(DstChild->getName());
       return InsertPt;
     } else if (DstChild->getOperator()->getName() == "fpimm") {
diff --git a/llvm/utils/TableGen/InfoByHwMode.cpp b/llvm/utils/TableGen/InfoByHwMode.cpp
index d9662889a5dba..7cd1b0f08132d 100644
--- a/llvm/utils/TableGen/InfoByHwMode.cpp
+++ b/llvm/utils/TableGen/InfoByHwMode.cpp
@@ -192,6 +192,17 @@ void RegSizeInfoByHwMode::writeToStream(raw_ostream &OS) const {
   OS << '}';
 }
 
+EncodingInfoByHwMode::EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH) {
+  const HwModeSelect &MS = CGH.getHwModeSelect(R);
+  for (const HwModeSelect::PairType &P : MS.Items) {
+    assert(P.second && P.second->isSubClassOf("InstructionEncoding") &&
+           "Encoding must subclass InstructionEncoding");
+    auto I = Map.insert({P.first, P.second});
+    assert(I.second && "Duplicate entry?");
+    (void)I;
+  }
+}
+
 namespace llvm {
   raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T) {
     T.writeToStream(OS);
diff --git a/llvm/utils/TableGen/InfoByHwMode.h b/llvm/utils/TableGen/InfoByHwMode.h
index 9e5cc3d5f2a4b..d92e5901a7f3b 100644
--- a/llvm/utils/TableGen/InfoByHwMode.h
+++ b/llvm/utils/TableGen/InfoByHwMode.h
@@ -184,6 +184,11 @@ raw_ostream &operator<<(raw_ostream &OS, const ValueTypeByHwMode &T);
 raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfo &T);
 raw_ostream &operator<<(raw_ostream &OS, const RegSizeInfoByHwMode &T);
 
+struct EncodingInfoByHwMode : public InfoByHwMode<Record*> {
+  EncodingInfoByHwMode(Record *R, const CodeGenHwModes &CGH);
+  EncodingInfoByHwMode() = default;
+};
+
 } // namespace llvm
 
 #endif // LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H
diff --git a/llvm/utils/TableGen/InstrInfoEmitter.cpp b/llvm/utils/TableGen/InstrInfoEmitter.cpp
index ae48d383cdf72..b7961efbf9636 100644
--- a/llvm/utils/TableGen/InstrInfoEmitter.cpp
+++ b/llvm/utils/TableGen/InstrInfoEmitter.cpp
@@ -332,6 +332,10 @@ void InstrInfoEmitter::emitOperandTypeMappings(
 
   StringRef Namespace = Target.getInstNamespace();
   std::vector<Record *> Operands = Records.getAllDerivedDefinitions("Operand");
+  std::vector<Record *> RegisterOperands =
+      Records.getAllDerivedDefinitions("RegisterOperand");
+  std::vector<Record *> RegisterClasses =
+      Records.getAllDerivedDefinitions("RegisterClass");
 
   OS << "#ifdef GET_INSTRINFO_OPERAND_TYPES_ENUM\n";
   OS << "#undef GET_INSTRINFO_OPERAND_TYPES_ENUM\n";
@@ -341,10 +345,13 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "enum OperandType {\n";
 
   unsigned EnumVal = 0;
-  for (const Record *Op : Operands) {
-    if (!Op->isAnonymous())
-      OS << "  " << Op->getName() << " = " << EnumVal << ",\n";
-    ++EnumVal;
+  for (const std::vector<Record *> *RecordsToAdd :
+       {&Operands, &RegisterOperands, &RegisterClasses}) {
+    for (const Record *Op : *RecordsToAdd) {
+      if (!Op->isAnonymous())
+        OS << "  " << Op->getName() << " = " << EnumVal << ",\n";
+      ++EnumVal;
+    }
   }
 
   OS << "  OPERAND_TYPE_LIST_END" << "\n};\n";
@@ -358,7 +365,8 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "namespace llvm {\n";
   OS << "namespace " << Namespace << " {\n";
   OS << "LLVM_READONLY\n";
-  OS << "int getOperandType(uint16_t Opcode, uint16_t OpIdx) {\n";
+  OS << "static int getOperandType(uint16_t Opcode, uint16_t OpIdx) {\n";
+  // TODO: Factor out instructions with same operands to compress the tables.
   if (!NumberedInstructions.empty()) {
     std::vector<int> OperandOffsets;
     std::vector<Record *> OperandRecords;
@@ -399,7 +407,10 @@ void InstrInfoEmitter::emitOperandTypeMappings(
           OS << "/**/\n    ";
       }
       Record *OpR = OperandRecords[I];
-      if (OpR->isSubClassOf("Operand") && !OpR->isAnonymous())
+      if ((OpR->isSubClassOf("Operand") ||
+           OpR->isSubClassOf("RegisterOperand") ||
+           OpR->isSubClassOf("RegisterClass")) &&
+          !OpR->isAnonymous())
         OS << "OpTypes::" << OpR->getName();
       else
         OS << -1;
@@ -414,7 +425,7 @@ void InstrInfoEmitter::emitOperandTypeMappings(
   OS << "}\n";
   OS << "} // end namespace " << Namespace << "\n";
   OS << "} // end namespace llvm\n";
-  OS << "#endif //GET_INSTRINFO_OPERAND_TYPE\n\n";
+  OS << "#endif // GET_INSTRINFO_OPERAND_TYPE\n\n";
 }
 
 void InstrInfoEmitter::emitMCIIHelperMethods(raw_ostream &OS,
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 3779dc7178468..e01f91c20456c 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -221,7 +221,10 @@ enum IIT_Info {
   IIT_STRUCT8 = 40,
   IIT_F128 = 41,
   IIT_VEC_ELEMENT = 42,
-  IIT_SCALABLE_VEC = 43
+  IIT_SCALABLE_VEC = 43,
+  IIT_SUBDIVIDE2_ARG = 44,
+  IIT_SUBDIVIDE4_ARG = 45,
+  IIT_VEC_OF_BITCASTS_TO_INT = 46
 };
 
 static void EncodeFixedValueType(MVT::SimpleValueType VT,
@@ -293,6 +296,12 @@ static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
       Sig.push_back(IIT_PTR_TO_ELT);
     else if (R->isSubClassOf("LLVMVectorElementType"))
       Sig.push_back(IIT_VEC_ELEMENT);
+    else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
+      Sig.push_back(IIT_SUBDIVIDE2_ARG);
+    else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
+      Sig.push_back(IIT_SUBDIVIDE4_ARG);
+    else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt"))
+      Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT);
     else
       Sig.push_back(IIT_ARG);
     return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/);
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index 41589b6b10de0..f970572365022 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -1746,7 +1746,10 @@ void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
      << "    return " << Target << "_MC"
      << "::resolveVariantSchedClassImpl(SchedClass, MI, CPUID); \n";
   OS << "  }\n";
+  if (TGT.getHwModes().getNumModeIds() > 1)
+    OS << "  unsigned getHwMode() const override;\n";
   OS << "};\n";
+  EmitHwModeCheck(Target + "GenMCSubtargetInfo", OS);
 }
 
 void SubtargetEmitter::EmitMCInstrAnalysisPredicateFunctions(raw_ostream &OS) {
diff --git a/llvm/utils/TableGen/TableGen.cpp b/llvm/utils/TableGen/TableGen.cpp
index c485ed2feb7af..817663b06e29b 100644
--- a/llvm/utils/TableGen/TableGen.cpp
+++ b/llvm/utils/TableGen/TableGen.cpp
@@ -49,6 +49,7 @@ enum ActionType {
   GenAttributes,
   GenSearchableTables,
   GenGlobalISel,
+  GenGICombiner,
   GenX86EVEX2VEXTables,
   GenX86FoldTables,
   GenRegisterBank,
@@ -62,75 +63,73 @@ bool TimeRegions = false;
 } // end namespace llvm
 
 namespace {
-  cl::opt<ActionType>
-  Action(cl::desc("Action to perform:"),
-         cl::values(clEnumValN(PrintRecords, "print-records",
-                               "Print all records to stdout (default)"),
-                    clEnumValN(DumpJSON, "dump-json",
-                               "Dump all records as machine-readable JSON"),
-                    clEnumValN(GenEmitter, "gen-emitter",
-                               "Generate machine code emitter"),
-                    clEnumValN(GenRegisterInfo, "gen-register-info",
-                               "Generate registers and register classes info"),
-                    clEnumValN(GenInstrInfo, "gen-instr-info",
-                               "Generate instruction descriptions"),
-                    clEnumValN(GenInstrDocs, "gen-instr-docs",
-                               "Generate instruction documentation"),
-                    clEnumValN(GenCallingConv, "gen-callingconv",
-                               "Generate calling convention descriptions"),
-                    clEnumValN(GenAsmWriter, "gen-asm-writer",
-                               "Generate assembly writer"),
-                    clEnumValN(GenDisassembler, "gen-disassembler",
-                               "Generate disassembler"),
-                    clEnumValN(GenPseudoLowering, "gen-pseudo-lowering",
-                               "Generate pseudo instruction lowering"),
-                    clEnumValN(GenCompressInst, "gen-compress-inst-emitter",
-                               "Generate RISCV compressed instructions."),
-                    clEnumValN(GenAsmMatcher, "gen-asm-matcher",
-                               "Generate assembly instruction matcher"),
-                    clEnumValN(GenDAGISel, "gen-dag-isel",
-                               "Generate a DAG instruction selector"),
-                    clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer",
-                               "Generate DFA Packetizer for VLIW targets"),
-                    clEnumValN(GenFastISel, "gen-fast-isel",
-                               "Generate a \"fast\" instruction selector"),
-                    clEnumValN(GenSubtarget, "gen-subtarget",
-                               "Generate subtarget enumerations"),
-                    clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums",
-                               "Generate intrinsic enums"),
-                    clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl",
-                               "Generate intrinsic information"),
-                    clEnumValN(GenTgtIntrinsicEnums, "gen-tgt-intrinsic-enums",
-                               "Generate target intrinsic enums"),
-                    clEnumValN(GenTgtIntrinsicImpl, "gen-tgt-intrinsic-impl",
-                               "Generate target intrinsic information"),
-                    clEnumValN(PrintEnums, "print-enums",
-                               "Print enum values for a class"),
-                    clEnumValN(PrintSets, "print-sets",
-                               "Print expanded sets for testing DAG exprs"),
-                    clEnumValN(GenOptParserDefs, "gen-opt-parser-defs",
-                               "Generate option definitions"),
-                    clEnumValN(GenCTags, "gen-ctags",
-                               "Generate ctags-compatible index"),
-                    clEnumValN(GenAttributes, "gen-attrs",
-                               "Generate attributes"),
-                    clEnumValN(GenSearchableTables, "gen-searchable-tables",
-                               "Generate generic binary-searchable table"),
-                    clEnumValN(GenGlobalISel, "gen-global-isel",
-                               "Generate GlobalISel selector"),
-                    clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables",
-                               "Generate X86 EVEX to VEX compress tables"),
-                    clEnumValN(GenX86FoldTables, "gen-x86-fold-tables",
-                               "Generate X86 fold tables"),
-                    clEnumValN(GenRegisterBank, "gen-register-bank",
-                               "Generate registers bank descriptions"),
-                    clEnumValN(GenExegesis, "gen-exegesis",
-                               "Generate llvm-exegesis tables")));
+cl::opt<ActionType> Action(
+    cl::desc("Action to perform:"),
+    cl::values(
+        clEnumValN(PrintRecords, "print-records",
+                   "Print all records to stdout (default)"),
+        clEnumValN(DumpJSON, "dump-json",
+                   "Dump all records as machine-readable JSON"),
+        clEnumValN(GenEmitter, "gen-emitter", "Generate machine code emitter"),
+        clEnumValN(GenRegisterInfo, "gen-register-info",
+                   "Generate registers and register classes info"),
+        clEnumValN(GenInstrInfo, "gen-instr-info",
+                   "Generate instruction descriptions"),
+        clEnumValN(GenInstrDocs, "gen-instr-docs",
+                   "Generate instruction documentation"),
+        clEnumValN(GenCallingConv, "gen-callingconv",
+                   "Generate calling convention descriptions"),
+        clEnumValN(GenAsmWriter, "gen-asm-writer", "Generate assembly writer"),
+        clEnumValN(GenDisassembler, "gen-disassembler",
+                   "Generate disassembler"),
+        clEnumValN(GenPseudoLowering, "gen-pseudo-lowering",
+                   "Generate pseudo instruction lowering"),
+        clEnumValN(GenCompressInst, "gen-compress-inst-emitter",
+                   "Generate RISCV compressed instructions."),
+        clEnumValN(GenAsmMatcher, "gen-asm-matcher",
+                   "Generate assembly instruction matcher"),
+        clEnumValN(GenDAGISel, "gen-dag-isel",
+                   "Generate a DAG instruction selector"),
+        clEnumValN(GenDFAPacketizer, "gen-dfa-packetizer",
+                   "Generate DFA Packetizer for VLIW targets"),
+        clEnumValN(GenFastISel, "gen-fast-isel",
+                   "Generate a \"fast\" instruction selector"),
+        clEnumValN(GenSubtarget, "gen-subtarget",
+                   "Generate subtarget enumerations"),
+        clEnumValN(GenIntrinsicEnums, "gen-intrinsic-enums",
+                   "Generate intrinsic enums"),
+        clEnumValN(GenIntrinsicImpl, "gen-intrinsic-impl",
+                   "Generate intrinsic information"),
+        clEnumValN(GenTgtIntrinsicEnums, "gen-tgt-intrinsic-enums",
+                   "Generate target intrinsic enums"),
+        clEnumValN(GenTgtIntrinsicImpl, "gen-tgt-intrinsic-impl",
+                   "Generate target intrinsic information"),
+        clEnumValN(PrintEnums, "print-enums", "Print enum values for a class"),
+        clEnumValN(PrintSets, "print-sets",
+                   "Print expanded sets for testing DAG exprs"),
+        clEnumValN(GenOptParserDefs, "gen-opt-parser-defs",
+                   "Generate option definitions"),
+        clEnumValN(GenCTags, "gen-ctags", "Generate ctags-compatible index"),
+        clEnumValN(GenAttributes, "gen-attrs", "Generate attributes"),
+        clEnumValN(GenSearchableTables, "gen-searchable-tables",
+                   "Generate generic binary-searchable table"),
+        clEnumValN(GenGlobalISel, "gen-global-isel",
+                   "Generate GlobalISel selector"),
+        clEnumValN(GenGICombiner, "gen-global-isel-combiner",
+                   "Generate GlobalISel combiner"),
+        clEnumValN(GenX86EVEX2VEXTables, "gen-x86-EVEX2VEX-tables",
+                   "Generate X86 EVEX to VEX compress tables"),
+        clEnumValN(GenX86FoldTables, "gen-x86-fold-tables",
+                   "Generate X86 fold tables"),
+        clEnumValN(GenRegisterBank, "gen-register-bank",
+                   "Generate registers bank descriptions"),
+        clEnumValN(GenExegesis, "gen-exegesis",
+                   "Generate llvm-exegesis tables")));
 
-  cl::OptionCategory PrintEnumsCat("Options for -print-enums");
-  cl::opt<std::string>
-  Class("class", cl::desc("Print Enum list for this class"),
-        cl::value_desc("class name"), cl::cat(PrintEnumsCat));
+cl::OptionCategory PrintEnumsCat("Options for -print-enums");
+cl::opt<std::string> Class("class", cl::desc("Print Enum list for this class"),
+                           cl::value_desc("class name"),
+                           cl::cat(PrintEnumsCat));
 
 cl::opt<bool, true>
     TimeRegionsOpt("time-regions",
@@ -235,6 +234,9 @@ bool LLVMTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenGlobalISel:
     EmitGlobalISel(Records, OS);
     break;
+  case GenGICombiner:
+    EmitGICombiner(Records, OS);
+    break;
   case GenRegisterBank:
     EmitRegisterBank(Records, OS);
     break;
@@ -263,11 +265,16 @@ int main(int argc, char **argv) {
   return TableGenMain(argv[0], &LLVMTableGenMain);
 }
 
-#ifdef __has_feature
-#if __has_feature(address_sanitizer)
+#ifndef __has_feature
+#define __has_feature(x) 0
+#endif
+
+#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) ||       \
+    __has_feature(leak_sanitizer)
+
 #include <sanitizer/lsan_interface.h>
 // Disable LeakSanitizer for this binary as it has too many leaks that are not
 // very interesting to fix. See compiler-rt/include/sanitizer/lsan_interface.h .
 LLVM_ATTRIBUTE_USED int __lsan_is_turned_off() { return 1; }
-#endif  // __has_feature(address_sanitizer)
-#endif  // defined(__has_feature)
+
+#endif
diff --git a/llvm/utils/TableGen/TableGenBackends.h b/llvm/utils/TableGen/TableGenBackends.h
index 135ec65c0f954..9c21ef3544072 100644
--- a/llvm/utils/TableGen/TableGenBackends.h
+++ b/llvm/utils/TableGen/TableGenBackends.h
@@ -85,6 +85,7 @@ void EmitCTags(RecordKeeper &RK, raw_ostream &OS);
 void EmitAttributes(RecordKeeper &RK, raw_ostream &OS);
 void EmitSearchableTables(RecordKeeper &RK, raw_ostream &OS);
 void EmitGlobalISel(RecordKeeper &RK, raw_ostream &OS);
+void EmitGICombiner(RecordKeeper &RK, raw_ostream &OS);
 void EmitX86EVEX2VEXTables(RecordKeeper &RK, raw_ostream &OS);
 void EmitX86FoldTables(RecordKeeper &RK, raw_ostream &OS);
 void EmitRegisterBank(RecordKeeper &RK, raw_ostream &OS);
diff --git a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
index 3df14f40e4a9b..6dc7e31e0dab6 100644
--- a/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86EVEX2VEXTablesEmitter.cpp
@@ -98,6 +98,7 @@ class IsMatch {
     bool EVEX_W1_VEX_W0 = RecE->getValueAsBit("EVEX_W1_VEX_W0");
 
     if (RecV->getValueAsDef("OpEnc")->getName().str() != "EncVEX" ||
+        RecV->getValueAsBit("isCodeGenOnly") != RecE->getValueAsBit("isCodeGenOnly") ||
         // VEX/EVEX fields
         RecV->getValueAsDef("OpPrefix") != RecE->getValueAsDef("OpPrefix") ||
         RecV->getValueAsDef("OpMap") != RecE->getValueAsDef("OpMap") ||
diff --git a/llvm/utils/emacs/tablegen-mode.el b/llvm/utils/emacs/tablegen-mode.el
index 035455d1b9005..a5da0fdac5a5a 100644
--- a/llvm/utils/emacs/tablegen-mode.el
+++ b/llvm/utils/emacs/tablegen-mode.el
@@ -39,17 +39,14 @@
 
      '("^[ \t]*\\(@.+\\)" 1 'td-decorators-face)
      ;; Keywords
-     (cons (concat kw "[ \n\t(]") 1)
-
+     kw
      ;; Type keywords
-     (cons (concat type-kw "[ \n\t(]") 1)
+     type-kw
      ))
   "Additional expressions to highlight in TableGen mode.")
 (put 'tablegen-mode 'font-lock-defaults '(tablegen-font-lock-keywords))
 
 ;; ---------------------- Syntax table ---------------------------
-;; Shamelessly ripped from jasmin.el
-;; URL: http://www.neilvandyke.org/jasmin-emacs/jasmin.el
 
 (defvar tablegen-mode-syntax-table nil
   "Syntax table used in `tablegen-mode' buffers.")
diff --git a/llvm/utils/git-svn/git-llvm b/llvm/utils/git-svn/git-llvm
index 549a45558dd49..debe6a5ea2007 100755
--- a/llvm/utils/git-svn/git-llvm
+++ b/llvm/utils/git-svn/git-llvm
@@ -53,6 +53,7 @@ LLVM_MONOREPO_SVN_MAPPING = {
         'debuginfo-tests',
         'dragonegg',
         'klee',
+        'libc',
         'libclc',
         'libcxx',
         'libcxxabi',
@@ -190,16 +191,16 @@ def program_exists(cmd):
 
 
 def get_default_rev_range():
-    # Get the branch tracked by the current branch, as set by
-    # git branch --set-upstream-to  See http://serverfault.com/a/352236/38694.
-    cur_branch = git('rev-parse', '--symbolic-full-name', 'HEAD')
-    upstream_branch = git('for-each-ref', '--format=%(upstream:short)',
-                          cur_branch)
-    if not upstream_branch:
-        upstream_branch = 'origin/master'
-
     # Get the newest common ancestor between HEAD and our upstream branch.
-    upstream_rev = git('merge-base', 'HEAD', upstream_branch)
+    upstream_rev = git('merge-base', 'HEAD', '@{upstream}', ignore_errors=True)
+    if not upstream_rev:
+        eprint("Warning: git-llvm assumes that origin/master is the upstream "
+               "branch but git does not.")
+        eprint("To make this warning go away: git branch -u origin/master")
+        eprint("To avoid this warning when creating branches: "
+               "git checkout -b MyBranchName origin/master")
+        upstream_rev = git('merge-base', 'HEAD', 'origin/master')
+
     return '%s..' % upstream_rev
 
 
diff --git a/llvm/utils/gn/TODO.txt b/llvm/utils/gn/TODO.txt
index ebf601e8d2518..2b35b4b809ce8 100644
--- a/llvm/utils/gn/TODO.txt
+++ b/llvm/utils/gn/TODO.txt
@@ -4,7 +4,7 @@ Ideas for things to do:
   - once there are more projects, have an llvm_enable_projects arg, modeled
     after llvm_targets_to_build in the GN build
   - a check-all build target that runs test of all projects
-- more targets (ARC AVR MSP430 XCore)
+- more targets (ARC MSP430 XCore)
   - example: https://reviews.llvm.org/D56416
 - investigate feasibility of working `gn check`
 
diff --git a/llvm/utils/gn/build/write_cmake_config.py b/llvm/utils/gn/build/write_cmake_config.py
index d57435285d1a0..d76d14b651078 100755
--- a/llvm/utils/gn/build/write_cmake_config.py
+++ b/llvm/utils/gn/build/write_cmake_config.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-"""Emulates the bits of CMake's configure_file() function needed in LLVM.
+r"""Emulates the bits of CMake's configure_file() function needed in LLVM.
 
 The CMake build uses configure_file() for several things.  This emulates that
 function for the GN build.  In the GN build, this runs at build time instead
@@ -62,7 +62,8 @@ def main():
     # Matches e.g. '${FOO}' or '@FOO@' and captures FOO in group 1 or 2.
     var_re = re.compile(r'\$\{([^}]*)\}|@([^@]*)@')
 
-    in_lines = open(args.input).readlines()
+    with open(args.input) as f:
+        in_lines = f.readlines()
     out_lines = []
     for in_line in in_lines:
         def repl(m):
@@ -102,8 +103,13 @@ def repl(m):
             file=sys.stderr)
         return 1
 
-    if not os.path.exists(args.output) or open(args.output).read() != output:
-        open(args.output, 'w').write(output)
+    def read(filename):
+        with open(args.output) as f:
+            return f.read()
+
+    if not os.path.exists(args.output) or read(args.output) != output:
+        with open(args.output, 'w') as f:
+            f.write(output)
         os.chmod(args.output, os.stat(args.input).st_mode & 0o777)
 
 
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
index 615d5d29c2b46..e37930fc44a66 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/BUILD.gn
@@ -37,3 +37,32 @@ static_library("clang-tidy") {
     "GlobList.cpp",
   ]
 }
+
+group("all-checks") {
+  # If you add a check, also add it to ClangTidyForceLinker.h.
+  deps = [
+    "//clang-tools-extra/clang-tidy/abseil",
+    "//clang-tools-extra/clang-tidy/android",
+    "//clang-tools-extra/clang-tidy/boost",
+    "//clang-tools-extra/clang-tidy/bugprone",
+    "//clang-tools-extra/clang-tidy/cert",
+    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
+    "//clang-tools-extra/clang-tidy/darwin",
+    "//clang-tools-extra/clang-tidy/fuchsia",
+    "//clang-tools-extra/clang-tidy/google",
+    "//clang-tools-extra/clang-tidy/hicpp",
+    "//clang-tools-extra/clang-tidy/linuxkernel",
+    "//clang-tools-extra/clang-tidy/llvm",
+    "//clang-tools-extra/clang-tidy/misc",
+    "//clang-tools-extra/clang-tidy/modernize",
+    "//clang-tools-extra/clang-tidy/objc",
+    "//clang-tools-extra/clang-tidy/openmp",
+    "//clang-tools-extra/clang-tidy/performance",
+    "//clang-tools-extra/clang-tidy/portability",
+    "//clang-tools-extra/clang-tidy/readability",
+    "//clang-tools-extra/clang-tidy/zircon",
+  ]
+  if (clang_enable_static_analyzer) {
+    deps += [ "//clang-tools-extra/clang-tidy/mpi" ]
+  }
+}
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
index 2bb5e6822d46c..c4cb8dbba0e88 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/bugprone/BUILD.gn
@@ -28,6 +28,7 @@ static_library("bugprone") {
     "ForwardingReferenceOverloadCheck.cpp",
     "InaccurateEraseCheck.cpp",
     "IncorrectRoundingsCheck.cpp",
+    "InfiniteLoopCheck.cpp",
     "IntegerDivisionCheck.cpp",
     "LambdaFunctionNameCheck.cpp",
     "MacroParenthesesCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
index a467d4947bffb..5d36602f97efb 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
@@ -18,6 +18,7 @@ static_library("cppcoreguidelines") {
   sources = [
     "AvoidGotoCheck.cpp",
     "CppCoreGuidelinesTidyModule.cpp",
+    "InitVariablesCheck.cpp",
     "InterfacesGlobalInitCheck.cpp",
     "MacroUsageCheck.cpp",
     "NarrowingConversionsCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/darwin/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/darwin/BUILD.gn
new file mode 100644
index 0000000000000..e9739190f65a1
--- /dev/null
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/darwin/BUILD.gn
@@ -0,0 +1,19 @@
+static_library("darwin") {
+  output_name = "clangTidyDarwinModule"
+  configs += [ "//llvm/utils/gn/build:clang_code" ]
+  deps = [
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy/utils",
+    "//clang/lib/AST",
+    "//clang/lib/ASTMatchers",
+    "//clang/lib/Analysis",
+    "//clang/lib/Basic",
+    "//clang/lib/Lex",
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "AvoidSpinlockCheck.cpp",
+    "DarwinTidyModule.cpp",
+    "DispatchOnceNonstaticCheck.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn
index 714a4a1fab021..bd6b2ad1c8548 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/objc/BUILD.gn
@@ -12,8 +12,8 @@ static_library("objc") {
   ]
   sources = [
     "AvoidNSErrorInitCheck.cpp",
-    "AvoidSpinlockCheck.cpp",
     "ForbiddenSubclassingCheck.cpp",
+    "MissingHashCheck.cpp",
     "ObjCTidyModule.cpp",
     "PropertyDeclarationCheck.cpp",
     "SuperSelfCheck.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn
index 476ef117811f6..4f635adb96837 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/plugin/BUILD.gn
@@ -1,29 +1,9 @@
-import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
-
 static_library("plugin") {
   output_name = "clangTidyPlugin"
   configs += [ "//llvm/utils/gn/build:clang_code" ]
   deps = [
     "//clang-tools-extra/clang-tidy",
-    "//clang-tools-extra/clang-tidy/abseil",
-    "//clang-tools-extra/clang-tidy/android",
-    "//clang-tools-extra/clang-tidy/boost",
-    "//clang-tools-extra/clang-tidy/bugprone",
-    "//clang-tools-extra/clang-tidy/cert",
-    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
-    "//clang-tools-extra/clang-tidy/fuchsia",
-    "//clang-tools-extra/clang-tidy/google",
-    "//clang-tools-extra/clang-tidy/hicpp",
-    "//clang-tools-extra/clang-tidy/linuxkernel",
-    "//clang-tools-extra/clang-tidy/llvm",
-    "//clang-tools-extra/clang-tidy/misc",
-    "//clang-tools-extra/clang-tidy/modernize",
-    "//clang-tools-extra/clang-tidy/objc",
-    "//clang-tools-extra/clang-tidy/openmp",
-    "//clang-tools-extra/clang-tidy/performance",
-    "//clang-tools-extra/clang-tidy/portability",
-    "//clang-tools-extra/clang-tidy/readability",
-    "//clang-tools-extra/clang-tidy/zircon",
+    "//clang-tools-extra/clang-tidy:all-checks",
     "//clang/lib/AST",
     "//clang/lib/ASTMatchers",
     "//clang/lib/Basic",
@@ -32,10 +12,6 @@ static_library("plugin") {
     "//clang/lib/Tooling",
   ]
 
-  if (clang_enable_static_analyzer) {
-    deps += [ "//clang-tools-extra/clang-tidy/mpi" ]
-  }
-
   sources = [
     "ClangTidyPlugin.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn
index 46efc8ac99feb..43c7fbbfd0647 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/tool/BUILD.gn
@@ -1,28 +1,8 @@
-import("//clang/lib/StaticAnalyzer/Frontend/enable.gni")
-
 executable("clang-tidy") {
   configs += [ "//llvm/utils/gn/build:clang_code" ]
   deps = [
     "//clang-tools-extra/clang-tidy",
-    "//clang-tools-extra/clang-tidy/abseil",
-    "//clang-tools-extra/clang-tidy/android",
-    "//clang-tools-extra/clang-tidy/boost",
-    "//clang-tools-extra/clang-tidy/bugprone",
-    "//clang-tools-extra/clang-tidy/cert",
-    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
-    "//clang-tools-extra/clang-tidy/fuchsia",
-    "//clang-tools-extra/clang-tidy/google",
-    "//clang-tools-extra/clang-tidy/hicpp",
-    "//clang-tools-extra/clang-tidy/linuxkernel",
-    "//clang-tools-extra/clang-tidy/llvm",
-    "//clang-tools-extra/clang-tidy/misc",
-    "//clang-tools-extra/clang-tidy/modernize",
-    "//clang-tools-extra/clang-tidy/objc",
-    "//clang-tools-extra/clang-tidy/openmp",
-    "//clang-tools-extra/clang-tidy/performance",
-    "//clang-tools-extra/clang-tidy/portability",
-    "//clang-tools-extra/clang-tidy/readability",
-    "//clang-tools-extra/clang-tidy/zircon",
+    "//clang-tools-extra/clang-tidy:all-checks",
     "//clang/lib/AST",
     "//clang/lib/ASTMatchers",
     "//clang/lib/Basic",
@@ -33,9 +13,6 @@ executable("clang-tidy") {
     "//llvm/lib/Target:AllTargetsDescs",
     "//llvm/lib/Target:AllTargetsInfos",
   ]
-  if (clang_enable_static_analyzer) {
-    deps += [ "//clang-tools-extra/clang-tidy/mpi" ]
-  }
   sources = [
     "ClangTidyMain.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
index f75dae06b84f2..5d4392cb3e7e3 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/BUILD.gn
@@ -24,25 +24,8 @@ static_library("clangd") {
   configs += [ "//llvm/utils/gn/build:clang_code" ]
   deps = [
     ":features",
-    "//clang-tools-extra/clang-tidy/abseil",
-    "//clang-tools-extra/clang-tidy/android",
-    "//clang-tools-extra/clang-tidy/boost",
-    "//clang-tools-extra/clang-tidy/bugprone",
-    "//clang-tools-extra/clang-tidy/cert",
-    "//clang-tools-extra/clang-tidy/cppcoreguidelines",
-    "//clang-tools-extra/clang-tidy/fuchsia",
-    "//clang-tools-extra/clang-tidy/google",
-    "//clang-tools-extra/clang-tidy/hicpp",
-    "//clang-tools-extra/clang-tidy/linuxkernel",
-    "//clang-tools-extra/clang-tidy/llvm",
-    "//clang-tools-extra/clang-tidy/misc",
-    "//clang-tools-extra/clang-tidy/modernize",
-    "//clang-tools-extra/clang-tidy/objc",
-    "//clang-tools-extra/clang-tidy/openmp",
-    "//clang-tools-extra/clang-tidy/performance",
-    "//clang-tools-extra/clang-tidy/portability",
-    "//clang-tools-extra/clang-tidy/readability",
-    "//clang-tools-extra/clang-tidy/zircon",
+    "//clang-tools-extra/clang-tidy",
+    "//clang-tools-extra/clang-tidy:all-checks",
     "//clang/lib/AST",
     "//clang/lib/ASTMatchers",
     "//clang/lib/Basic",
@@ -90,6 +73,7 @@ static_library("clangd") {
     "FormattedString.cpp",
     "FuzzyMatch.cpp",
     "GlobalCompilationDatabase.cpp",
+    "HeaderSourceSwitch.cpp",
     "Headers.cpp",
     "IncludeFixer.cpp",
     "JSONTransport.cpp",
@@ -102,6 +86,7 @@ static_library("clangd") {
     "RIFF.cpp",
     "Selection.cpp",
     "SemanticHighlighting.cpp",
+    "SemanticSelection.cpp",
     "SourceCode.cpp",
     "TUScheduler.cpp",
     "Threading.cpp",
diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
index eea48f3c8e867..b1598518a89e0 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clangd/unittests/BUILD.gn
@@ -52,6 +52,7 @@ unittest("ClangdTests") {
     "FunctionTests.cpp",
     "FuzzyMatchTests.cpp",
     "GlobalCompilationDatabaseTests.cpp",
+    "HeaderSourceSwitchTests.cpp",
     "HeadersTests.cpp",
     "IndexActionTests.cpp",
     "IndexTests.cpp",
@@ -63,6 +64,7 @@ unittest("ClangdTests") {
     "RenameTests.cpp",
     "SelectionTests.cpp",
     "SemanticHighlightingTests.cpp",
+    "SemanticSelectionTests.cpp",
     "SerializationTests.cpp",
     "SourceCodeTests.cpp",
     "SymbolCollectorTests.cpp",
diff --git a/llvm/utils/gn/secondary/clang/include/clang/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/include/clang/AST/BUILD.gn
index 6ac4a72890604..3bb292886db81 100644
--- a/llvm/utils/gn/secondary/clang/include/clang/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/include/clang/AST/BUILD.gn
@@ -55,6 +55,11 @@ clang_tablegen("DeclNodes") {
   td_file = "../Basic/DeclNodes.td"
 }
 
+clang_tablegen("TypeNodes") {
+  args = [ "-gen-clang-type-nodes" ]
+  td_file = "../Basic/TypeNodes.td"
+}
+
 clang_tablegen("CommentNodes") {
   args = [ "-gen-clang-comment-nodes" ]
   td_file = "../Basic/CommentNodes.td"
diff --git a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
index 84e80aa8048da..36f10eb3b34de 100644
--- a/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/AST/BUILD.gn
@@ -33,6 +33,7 @@ static_library("AST") {
     "//clang/include/clang/AST:CommentCommandList",
     "//clang/include/clang/AST:CommentNodes",
     "//clang/include/clang/AST:StmtNodes",
+    "//clang/include/clang/AST:TypeNodes",
   ]
   sources = [
     "APValue.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
index 66d209717f683..a70c69fc1b8cd 100644
--- a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -67,6 +67,7 @@ static_library("Driver") {
     "ToolChains/Haiku.cpp",
     "ToolChains/Hexagon.cpp",
     "ToolChains/Hurd.cpp",
+    "ToolChains/InterfaceStubs.cpp",
     "ToolChains/Linux.cpp",
     "ToolChains/MSP430.cpp",
     "ToolChains/MSVC.cpp",
diff --git a/llvm/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn
index ef25bec41a0fd..f93debc1fae5a 100644
--- a/llvm/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Tooling/Refactoring/BUILD.gn
@@ -27,6 +27,7 @@ static_library("Refactoring") {
     "Rename/USRFindingAction.cpp",
     "Rename/USRLocFinder.cpp",
     "SourceCode.cpp",
+    "SourceCodeBuilders.cpp",
     "Stencil.cpp",
     "Transformer.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
index ef23de0ac2707..06b28de392735 100644
--- a/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/AST/BUILD.gn
@@ -15,6 +15,7 @@ unittest("ASTTests") {
     "ASTContextParentMapTest.cpp",
     "ASTImporterFixtures.cpp",
     "ASTImporterGenericRedeclTest.cpp",
+    "ASTImporterODRStrategiesTest.cpp",
     "ASTImporterTest.cpp",
     "ASTImporterVisibilityTest.cpp",
     "ASTTraverserTest.cpp",
diff --git a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
index b0c3b03de09c4..cbd629c3a5571 100644
--- a/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/unittests/Tooling/BUILD.gn
@@ -62,6 +62,7 @@ unittest("ToolingTests") {
     "RefactoringTest.cpp",
     "ReplacementsYamlTest.cpp",
     "RewriterTest.cpp",
+    "SourceCodeBuildersTest.cpp",
     "SourceCodeTest.cpp",
     "StencilTest.cpp",
     "ToolingTest.cpp",
diff --git a/llvm/utils/gn/secondary/clang/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/clang/utils/TableGen/BUILD.gn
index 7281cd546ff87..1bb9a2ec7d4ba 100644
--- a/llvm/utils/gn/secondary/clang/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/utils/TableGen/BUILD.gn
@@ -15,6 +15,7 @@ executable("clang-tblgen") {
     "ClangOpenCLBuiltinEmitter.cpp",
     "ClangOptionDocEmitter.cpp",
     "ClangSACheckersEmitter.cpp",
+    "ClangTypeNodesEmitter.cpp",
     "NeonEmitter.cpp",
     "TableGen.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/lld/COFF/BUILD.gn b/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
index c565b0bca48c5..4e0a2074a3a41 100644
--- a/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/COFF/BUILD.gn
@@ -15,6 +15,7 @@ static_library("COFF") {
     "//llvm/lib/DebugInfo/CodeView",
     "//llvm/lib/DebugInfo/MSF",
     "//llvm/lib/DebugInfo/PDB",
+    "//llvm/lib/DebugInfo/Symbolize",
     "//llvm/lib/IR",
     "//llvm/lib/LTO",
     "//llvm/lib/MC",
diff --git a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
index 364559540b64b..99870a7a6b803 100644
--- a/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/ELF/BUILD.gn
@@ -24,6 +24,7 @@ static_library("ELF") {
   include_dirs = [ "." ]
   sources = [
     "AArch64ErrataFix.cpp",
+    "ARMErrataFix.cpp",
     "Arch/AArch64.cpp",
     "Arch/AMDGPU.cpp",
     "Arch/ARM.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
index 06ac3d21df2ed..cbbe1499a560b 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Analysis/BUILD.gn
@@ -36,9 +36,11 @@ static_library("Analysis") {
     "CodeMetrics.cpp",
     "ConstantFolding.cpp",
     "CostModel.cpp",
+    "DDG.cpp",
     "Delinearization.cpp",
     "DemandedBits.cpp",
     "DependenceAnalysis.cpp",
+    "DependenceGraphBuilder.cpp",
     "DivergenceAnalysis.cpp",
     "DomPrinter.cpp",
     "DomTreeUpdater.cpp",
@@ -105,6 +107,7 @@ static_library("Analysis") {
     "Trace.cpp",
     "TypeBasedAliasAnalysis.cpp",
     "TypeMetadataUtils.cpp",
+    "VFABIDemangling.cpp",
     "ValueLattice.cpp",
     "ValueLatticeUtils.cpp",
     "ValueTracking.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
index f70195eb05554..8f7c27ebf0b64 100644
--- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn
@@ -103,6 +103,7 @@ static_library("CodeGen") {
     "MachineInstrBundle.cpp",
     "MachineLICM.cpp",
     "MachineLoopInfo.cpp",
+    "MachineLoopUtils.cpp",
     "MachineModuleInfo.cpp",
     "MachineModuleInfoImpls.cpp",
     "MachineOperand.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/GSYM/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/GSYM/BUILD.gn
index f2717b85c9794..ceaf5e509d56c 100644
--- a/llvm/utils/gn/secondary/llvm/lib/DebugInfo/GSYM/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/DebugInfo/GSYM/BUILD.gn
@@ -6,6 +6,7 @@ static_library("GSYM") {
   sources = [
     "FileWriter.cpp",
     "FunctionInfo.cpp",
+    "Header.cpp",
     "InlineInfo.cpp",
     "LineTable.cpp",
     "Range.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
index b05e0891d124e..0c27c11e2e0ba 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AArch64/BUILD.gn
@@ -30,6 +30,15 @@ tablegen("AArch64GenGlobalISel") {
   td_file = "AArch64.td"
 }
 
+tablegen("AArch64GenGICombiner") {
+  visibility = [ ":LLVMAArch64CodeGen" ]
+  args = [
+    "-gen-global-isel-combiner",
+    "-combiners=AArch64PreLegalizerCombinerHelper",
+  ]
+  td_file = "AArch64.td"
+}
+
 tablegen("AArch64GenMCPseudoLowering") {
   visibility = [ ":LLVMAArch64CodeGen" ]
   args = [ "-gen-pseudo-lowering" ]
@@ -48,6 +57,7 @@ static_library("LLVMAArch64CodeGen") {
     ":AArch64GenCallingConv",
     ":AArch64GenDAGISel",
     ":AArch64GenFastISel",
+    ":AArch64GenGICombiner",
     ":AArch64GenGlobalISel",
     ":AArch64GenMCPseudoLowering",
     ":AArch64GenRegisterBank",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn
new file mode 100644
index 0000000000000..6a99c37c01aa5
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/AsmParser/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AVRGenAsmMatcher") {
+  visibility = [ ":AsmParser" ]
+  args = [ "-gen-asm-matcher" ]
+  td_file = "../AVR.td"
+}
+
+static_library("AsmParser") {
+  output_name = "LLVMAVRAsmParser"
+  deps = [
+    ":AVRGenAsmMatcher",
+    "//llvm/lib/MC",
+    "//llvm/lib/MC/MCParser",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AVR/MCTargetDesc",
+    "//llvm/lib/Target/AVR/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AVRAsmParser.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn
new file mode 100644
index 0000000000000..40ab4378196bb
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/BUILD.gn
@@ -0,0 +1,61 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AVRGenCallingConv") {
+  visibility = [ ":LLVMAVRCodeGen" ]
+  args = [ "-gen-callingconv" ]
+  td_file = "AVR.td"
+}
+
+tablegen("AVRGenDAGISel") {
+  visibility = [ ":LLVMAVRCodeGen" ]
+  args = [ "-gen-dag-isel" ]
+  td_file = "AVR.td"
+}
+
+static_library("LLVMAVRCodeGen") {
+  deps = [
+    ":AVRGenCallingConv",
+    ":AVRGenDAGISel",
+    "MCTargetDesc",
+    "TargetInfo",
+    "//llvm/include/llvm/Config:llvm-config",
+    "//llvm/lib/CodeGen",
+    "//llvm/lib/CodeGen/AsmPrinter",
+    "//llvm/lib/CodeGen/SelectionDAG",
+    "//llvm/lib/IR",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+  ]
+  include_dirs = [ "." ]
+  sources = [
+    "AVRAsmPrinter.cpp",
+    "AVRExpandPseudoInsts.cpp",
+    "AVRFrameLowering.cpp",
+    "AVRISelDAGToDAG.cpp",
+    "AVRISelLowering.cpp",
+    "AVRInstrInfo.cpp",
+    "AVRMCInstLower.cpp",
+    "AVRRegisterInfo.cpp",
+    "AVRRelaxMemOperations.cpp",
+    "AVRSubtarget.cpp",
+    "AVRTargetMachine.cpp",
+    "AVRTargetObjectFile.cpp",
+  ]
+}
+
+# This is a bit different from most build files: Due to this group
+# having the directory's name, "//llvm/lib/Target/AVR" will refer to this
+# target, which pulls in the code in this directory *and all subdirectories*.
+# For most other directories, "//llvm/lib/Foo" only pulls in the code directly
+# in "llvm/lib/Foo". The forwarding targets in //llvm/lib/Target expect this
+# different behavior.
+group("AVR") {
+  deps = [
+    ":LLVMAVRCodeGen",
+    "AsmParser",
+    "Disassembler",
+    "MCTargetDesc",
+    "TargetInfo",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn
new file mode 100644
index 0000000000000..ce678f9a7e883
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/Disassembler/BUILD.gn
@@ -0,0 +1,23 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AVRGenDisassemblerTables") {
+  visibility = [ ":Disassembler" ]
+  args = [ "-gen-disassembler" ]
+  td_file = "../AVR.td"
+}
+
+static_library("Disassembler") {
+  output_name = "LLVMAVRDisassembler"
+  deps = [
+    ":AVRGenDisassemblerTables",
+    "//llvm/lib/MC/MCDisassembler",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AVR:LLVMAVRCodeGen",
+    "//llvm/lib/Target/AVR/MCTargetDesc",
+    "//llvm/lib/Target/AVR/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AVRDisassembler.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/MCTargetDesc/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/MCTargetDesc/BUILD.gn
new file mode 100644
index 0000000000000..d2d5399f4c851
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/MCTargetDesc/BUILD.gn
@@ -0,0 +1,64 @@
+import("//llvm/utils/TableGen/tablegen.gni")
+
+tablegen("AVRGenAsmWriter") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [ "-gen-asm-writer" ]
+  td_file = "../AVR.td"
+}
+
+tablegen("AVRGenInstrInfo") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [ "-gen-instr-info" ]
+  td_file = "../AVR.td"
+}
+
+tablegen("AVRGenMCCodeEmitter") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [ "-gen-emitter" ]
+  td_file = "../AVR.td"
+}
+
+tablegen("AVRGenRegisterInfo") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [ "-gen-register-info" ]
+  td_file = "../AVR.td"
+}
+
+tablegen("AVRGenSubtargetInfo") {
+  visibility = [ ":MCTargetDesc" ]
+  args = [ "-gen-subtarget" ]
+  td_file = "../AVR.td"
+}
+
+static_library("MCTargetDesc") {
+  output_name = "LLVMAVRDesc"
+
+  # This should contain tablegen targets generating .inc files included
+  # by other targets. .inc files only used by .cpp files in this directory
+  # should be in deps instead.
+  public_deps = [
+    ":AVRGenInstrInfo",
+    ":AVRGenRegisterInfo",
+    ":AVRGenSubtargetInfo",
+  ]
+  deps = [
+    ":AVRGenAsmWriter",
+    ":AVRGenMCCodeEmitter",
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target/AVR/TargetInfo",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AVRAsmBackend.cpp",
+    "AVRELFObjectWriter.cpp",
+    "AVRELFStreamer.cpp",
+    "AVRInstPrinter.cpp",
+    "AVRMCAsmInfo.cpp",
+    "AVRMCCodeEmitter.cpp",
+    "AVRMCELFStreamer.cpp",
+    "AVRMCExpr.cpp",
+    "AVRMCTargetDesc.cpp",
+    "AVRTargetStreamer.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AVR/TargetInfo/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/TargetInfo/BUILD.gn
new file mode 100644
index 0000000000000..e069cac242f31
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AVR/TargetInfo/BUILD.gn
@@ -0,0 +1,10 @@
+static_library("TargetInfo") {
+  output_name = "LLVMAVRInfo"
+  deps = [
+    "//llvm/lib/Support",
+  ]
+  include_dirs = [ ".." ]
+  sources = [
+    "AVRTargetInfo.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn
index 034645ffba26e..51f65f954c908 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/SystemZ/BUILD.gn
@@ -35,7 +35,6 @@ static_library("LLVMSystemZCodeGen") {
     "SystemZCallingConv.cpp",
     "SystemZConstantPoolValue.cpp",
     "SystemZElimCompare.cpp",
-    "SystemZExpandPseudo.cpp",
     "SystemZFrameLowering.cpp",
     "SystemZHazardRecognizer.cpp",
     "SystemZISelDAGToDAG.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
index 94111f7f489e2..e82dd775e07af 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/targets.gni
@@ -5,6 +5,25 @@ declare_args() {
   llvm_targets_to_build = "host"
 }
 
+# FIXME: Port the remaining targets.
+llvm_all_targets = [
+  "AArch64",
+  "AMDGPU",
+  "ARM",
+  "AVR",
+  "BPF",
+  "Hexagon",
+  "Lanai",
+  "Mips",
+  "NVPTX",
+  "PowerPC",
+  "RISCV",
+  "Sparc",
+  "SystemZ",
+  "WebAssembly",
+  "X86",
+]
+
 if (llvm_targets_to_build == "host") {
   if (host_cpu == "arm64") {
     llvm_targets_to_build = [ "AArch64" ]
@@ -18,26 +37,12 @@ if (llvm_targets_to_build == "host") {
     assert(false, "add your host_cpu above")
   }
 } else if (llvm_targets_to_build == "all") {
-  # FIXME: Port the remaining targets.
-  llvm_targets_to_build = [
-    "AArch64",
-    "AMDGPU",
-    "ARM",
-    "BPF",
-    "Hexagon",
-    "Lanai",
-    "Mips",
-    "NVPTX",
-    "PowerPC",
-    "Sparc",
-    "SystemZ",
-    "WebAssembly",
-    "X86",
-  ]
+  llvm_targets_to_build = llvm_all_targets
 }
 
 # Validate that llvm_targets_to_build is set to a list of valid targets,
-# and remember which targets are built.
+# and remember which targets are built where needed (for conditionally-built
+# unittest targets).
 llvm_build_AArch64 = false
 llvm_build_ARM = false
 llvm_build_BPF = false
@@ -47,35 +52,28 @@ llvm_build_X86 = false
 foreach(target, llvm_targets_to_build) {
   if (target == "AArch64") {
     llvm_build_AArch64 = true
-  } else if (target == "AMDGPU") {
-    # Nothing to do.
   } else if (target == "ARM") {
     llvm_build_ARM = true
   } else if (target == "BPF") {
     llvm_build_BPF = true
-  } else if (target == "Hexagon") {
-    # Nothing to do.
-  } else if (target == "Lanai") {
-    # Nothing to do.
-  } else if (target == "Mips") {
-    # Nothing to do.
-  } else if (target == "NVPTX") {
-    # Nothing to do.
   } else if (target == "PowerPC") {
     llvm_build_PowerPC = true
-  } else if (target == "RISCV") {
-    # Nothing to do.
-  } else if (target == "Sparc") {
-    # Nothing to do.
-  } else if (target == "SystemZ") {
-    # Nothing to do.
   } else if (target == "WebAssembly") {
     llvm_build_WebAssembly = true
   } else if (target == "X86") {
     llvm_build_X86 = true
+  } else if (target == "AMDGPU" || target == "AVR" || target == "Hexagon" ||
+             target == "Lanai" || target == "Mips" || target == "NVPTX" ||
+             target == "RISCV" || target == "Sparc" || target == "SystemZ") {
+    # Nothing to do.
   } else {
-    # FIXME: Port the remaining targets.
-    assert(false, "Unknown target '$target'.")
+    all_targets_string = ""
+    foreach(target, llvm_all_targets) {
+      all_targets_string += "$0x0a    " + target
+    }
+    assert(false,
+           "Unknown target '$target' in llvm_targets_to_build. " +
+               "Known targets:" + all_targets_string)
   }
 }
 
diff --git a/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn
index 509d589086698..aa695da4bf30d 100644
--- a/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/TextAPI/BUILD.gn
@@ -12,7 +12,9 @@ static_library("TextAPI") {
     "MachO/ArchitectureSet.cpp",
     "MachO/InterfaceFile.cpp",
     "MachO/PackedVersion.cpp",
+    "MachO/Platform.cpp",
     "MachO/Symbol.cpp",
+    "MachO/Target.cpp",
     "MachO/TextStub.cpp",
     "MachO/TextStubCommon.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn
index 5cea22c8476e8..199c0331ba4e1 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-exegesis/lib/BUILD.gn
@@ -28,7 +28,9 @@ static_library("lib") {
     "RegisterAliasing.cpp",
     "RegisterValue.cpp",
     "SchedClassResolution.cpp",
+    "SnippetFile.cpp",
     "SnippetGenerator.cpp",
+    "SnippetRepetitor.cpp",
     "Target.cpp",
     "Uops.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
index 1e1a4901846a8..25aa19e313b7f 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-objcopy/BUILD.gn
@@ -54,6 +54,7 @@ executable("llvm-objcopy") {
     "COFF/Reader.cpp",
     "COFF/Writer.cpp",
     "CopyConfig.cpp",
+    "ELF/ELFConfig.cpp",
     "ELF/ELFObjcopy.cpp",
     "ELF/Object.cpp",
     "MachO/MachOLayoutBuilder.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
index a689ccc21e7ab..efa80c1b86d80 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-reduce/BUILD.gn
@@ -12,8 +12,10 @@ executable("llvm-reduce") {
     "TestRunner.cpp",
     "deltas/Delta.cpp",
     "deltas/ReduceArguments.cpp",
+    "deltas/ReduceBasicBlocks.cpp",
     "deltas/ReduceFunctions.cpp",
     "deltas/ReduceGlobalVars.cpp",
+    "deltas/ReduceInstructions.cpp",
     "deltas/ReduceMetadata.cpp",
     "llvm-reduce.cpp",
   ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
index 9be8d428ef10e..1e4b1581722fa 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Analysis/BUILD.gn
@@ -36,6 +36,7 @@ unittest("AnalysisTests") {
     "UnrollAnalyzerTest.cpp",
     "ValueLatticeTest.cpp",
     "ValueTrackingTest.cpp",
+    "VectorFunctionABITest.cpp",
     "VectorUtilsTest.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
index e2f813dcb4253..3d607dd774711 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/BUILD.gn
@@ -34,6 +34,7 @@ group("unittests") {
     "Remarks:RemarksTests",
     "Support:SupportTests",
     "Support/DynamicLibrary:DynamicLibraryTests",
+    "TableGen:TableGenTests",
     "TextAPI:TextAPITests",
     "Transforms/IPO:IPOTests",
     "Transforms/Scalar:ScalarTests",
@@ -55,7 +56,10 @@ group("unittests") {
     ]
   }
   if (llvm_build_ARM) {
-    deps += [ "tools/llvm-exegesis/ARM:LLVMExegesisARMTests" ]
+    deps += [
+      "Target/ARM:ARMTests",
+      "tools/llvm-exegesis/ARM:LLVMExegesisARMTests",
+    ]
   }
   if (llvm_build_WebAssembly) {
     deps += [ "Target/WebAssembly:WebAssemblyTests" ]
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn
index 9a3817ebd35ae..6200a0f07cd62 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Bitcode/BUILD.gn
@@ -10,5 +10,6 @@ unittest("BitcodeTests") {
   ]
   sources = [
     "BitReaderTest.cpp",
+    "DataLayoutUpgradeTest.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn
new file mode 100644
index 0000000000000..3f128f8ce2aaf
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/unittests/TableGen/BUILD.gn
@@ -0,0 +1,13 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("TableGenTests") {
+  deps = [
+    "//llvm/lib/Support",
+    "//llvm/lib/TableGen",
+    "//llvm/utils/TableGen/GlobalISel",
+  ]
+  include_dirs = [ "//llvm/utils/TableGen" ]
+  sources = [
+    "CodeExpanderTest.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Target/ARM/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Target/ARM/BUILD.gn
new file mode 100644
index 0000000000000..fd9db37196edb
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/unittests/Target/ARM/BUILD.gn
@@ -0,0 +1,17 @@
+import("//llvm/utils/unittest/unittest.gni")
+
+unittest("ARMTests") {
+  deps = [
+    "//llvm/lib/MC",
+    "//llvm/lib/Support",
+    "//llvm/lib/Target",
+    "//llvm/lib/Target/ARM:LLVMARMCodeGen",
+    "//llvm/lib/Target/ARM/MCTargetDesc",
+    "//llvm/lib/Target/ARM/TargetInfo",
+    "//llvm/lib/Target/ARM/Utils",
+  ]
+  include_dirs = [ "//llvm/lib/Target/ARM" ]
+  sources = [
+    "MachineInstrTest.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn
index 53c5e4c6bc455..9807b58977feb 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/tools/llvm-exegesis/X86/BUILD.gn
@@ -25,7 +25,9 @@ unittest("LLVMExegesisX86Tests") {
     "BenchmarkResultTest.cpp",
     "RegisterAliasingTest.cpp",
     "SchedClassResolutionTest.cpp",
+    "SnippetFileTest.cpp",
     "SnippetGeneratorTest.cpp",
+    "SnippetRepetitorTest.cpp",
     "TargetTest.cpp",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
index 01219543d2db7..952b2f9160627 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/BUILD.gn
@@ -4,6 +4,7 @@ executable("llvm-tblgen") {
     "//llvm/lib/MC",
     "//llvm/lib/Support",
     "//llvm/lib/TableGen",
+    "//llvm/utils/TableGen/GlobalISel",
   ]
   sources = [
     "AsmMatcherEmitter.cpp",
@@ -30,6 +31,7 @@ executable("llvm-tblgen") {
     "ExegesisEmitter.cpp",
     "FastISelEmitter.cpp",
     "FixedLenDecoderEmitter.cpp",
+    "GICombinerEmitter.cpp",
     "GlobalISelEmitter.cpp",
     "InfoByHwMode.cpp",
     "InstrDocsEmitter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
new file mode 100644
index 0000000000000..fe703cf9cad86
--- /dev/null
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/GlobalISel/BUILD.gn
@@ -0,0 +1,8 @@
+source_set("GlobalISel") {
+  deps = [
+    "//llvm/lib/Support",
+  ]
+  sources = [
+    "CodeExpander.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni b/llvm/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
index cb588abbaa6a5..a0e2e7967d6b5 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/tablegen.gni
@@ -64,6 +64,11 @@ template("tablegen") {
     depfile = "$gen_output.d"
     td_file = rebase_path(td_file, root_build_dir)
 
+    # FIXME: The cmake build lets tablegen write to a temp file and then copies
+    # it over the final output only if it has changed, for ninja's restat
+    # optimization. Instead of doing that in cmake, llvm-tblgen should do this
+    # itself. r330742 tried this, but it caused problems. Fix those and reland,
+    # so that the gn build has the optimization too.
     args = [
              rebase_path(tblgen_executable, root_build_dir),
              "-I",
diff --git a/llvm/utils/llvm-locstats/CMakeLists.txt b/llvm/utils/llvm-locstats/CMakeLists.txt
new file mode 100644
index 0000000000000..aa5aeca149f01
--- /dev/null
+++ b/llvm/utils/llvm-locstats/CMakeLists.txt
@@ -0,0 +1,7 @@
+if (LLVM_BUILD_UTILS AND LLVM_BUILD_TOOLS)
+  add_custom_target(llvm-locstats ALL
+    COMMAND  ${CMAKE_COMMAND} -E copy ${LLVM_MAIN_SRC_DIR}/utils/llvm-locstats/llvm-locstats.py ${LLVM_TOOLS_BINARY_DIR}/llvm-locstats
+    COMMENT "Copying llvm-locstats into ${LLVM_TOOLS_BINARY_DIR}"
+    )
+  set_target_properties(llvm-locstats PROPERTIES FOLDER "Tools")
+endif()
diff --git a/llvm/utils/llvm-locstats/llvm-locstats.py b/llvm/utils/llvm-locstats/llvm-locstats.py
new file mode 100755
index 0000000000000..22e5ae6b223bb
--- /dev/null
+++ b/llvm/utils/llvm-locstats/llvm-locstats.py
@@ -0,0 +1,208 @@
+#!/usr/bin/env python
+#
+# This is a tool that works like debug location coverage calculator.
+# It parses the llvm-dwarfdump --statistics output by reporting it
+# in a more human readable way.
+#
+
+from __future__ import print_function
+import argparse
+import os
+import sys
+from json import loads
+from math import ceil
+from subprocess import Popen, PIPE
+
+def coverage_buckets():
+  yield '0%'
+  yield '1-9%'
+  for start in range(10, 91, 10):
+    yield '{0}-{1}%'.format(start, start + 9)
+  yield '100%'
+
+def locstats_output(
+  variables_total,
+  variables_total_locstats,
+  variables_with_loc,
+  scope_bytes_covered,
+  scope_bytes_from_first_def,
+  variables_coverage_map
+  ):
+
+  pc_ranges_covered = int(ceil(scope_bytes_covered * 100.0)
+              / scope_bytes_from_first_def)
+  variables_coverage_per_map = {}
+  for cov_bucket in coverage_buckets():
+    variables_coverage_per_map[cov_bucket] = \
+      int(ceil(variables_coverage_map[cov_bucket] * 100.0) \
+               / variables_total_locstats)
+
+  print (' =================================================')
+  print ('            Debug Location Statistics       ')
+  print (' =================================================')
+  print ('     cov%          samples          percentage(~)  ')
+  print (' -------------------------------------------------')
+  for cov_bucket in coverage_buckets():
+    print ('   {0:6}        {1:8d}             {2:3d}%'. \
+      format(cov_bucket, variables_coverage_map[cov_bucket], \
+             variables_coverage_per_map[cov_bucket]))
+  print (' =================================================')
+  print (' -the number of debug variables processed: ' \
+    + str(variables_total_locstats))
+  print (' -PC ranges covered: ' + str(pc_ranges_covered) + '%')
+
+  # Only if we are processing all the variables output the total
+  # availability.
+  if variables_total and variables_with_loc:
+    total_availability = int(ceil(variables_with_loc * 100.0) \
+                                  / variables_total)
+    print (' -------------------------------------------------')
+    print (' -total availability: ' + str(total_availability) + '%')
+  print (' =================================================')
+
+def parse_program_args(parser):
+  parser.add_argument('-only-variables', action='store_true',
+            default=False,
+            help='calculate the location statistics only for '
+               'local variables'
+            )
+  parser.add_argument('-only-formal-parameters', action='store_true',
+            default=False,
+            help='calculate the location statistics only for '
+               'formal parameters'
+            )
+  parser.add_argument('-ignore-debug-entry-values', action='store_true',
+            default=False,
+            help='ignore the location statistics on locations with '
+               'entry values'
+            )
+  parser.add_argument('file_name', type=str, help='file to process')
+  return parser.parse_args()
+
+
+def Main():
+  parser = argparse.ArgumentParser()
+  results = parse_program_args(parser)
+
+  if len(sys.argv) < 2:
+    print ('error: Too few arguments.')
+    parser.print_help()
+    sys.exit(1)
+
+  if results.only_variables and results.only_formal_parameters:
+    print ('error: Please use just one only* option.')
+    parser.print_help()
+    sys.exit(1)
+
+  # These will be different due to different options enabled.
+  variables_total = None
+  variables_total_locstats = None
+  variables_with_loc = None
+  variables_scope_bytes_covered = None
+  variables_scope_bytes_from_first_def = None
+  variables_scope_bytes_entry_values = None
+  variables_coverage_map = {}
+  binary = results.file_name
+
+  # Get the directory of the LLVM tools.
+  llvm_dwarfdump_cmd = os.path.join(os.path.dirname(__file__), \
+                                    "llvm-dwarfdump")
+  # The statistics llvm-dwarfdump option.
+  llvm_dwarfdump_stats_opt = "--statistics"
+
+  subproc = Popen([llvm_dwarfdump_cmd, llvm_dwarfdump_stats_opt, binary], \
+                  stdin=PIPE, stdout=PIPE, stderr=PIPE, \
+                  universal_newlines = True)
+  cmd_stdout, cmd_stderr = subproc.communicate()
+
+  # Get the JSON and parse it.
+  json_parsed = None
+
+  try:
+    json_parsed = loads(cmd_stdout)
+  except:
+    print ('error: No valid llvm-dwarfdump statistics found.')
+    sys.exit(1)
+
+  if results.only_variables:
+    # Read the JSON only for local variables.
+    variables_total_locstats = \
+      json_parsed['total vars procesed by location statistics']
+    variables_scope_bytes_covered = \
+      json_parsed['vars scope bytes covered']
+    variables_scope_bytes_from_first_def = \
+      json_parsed['vars scope bytes total']
+    if not results.ignore_debug_entry_values:
+      for cov_bucket in coverage_buckets():
+        cov_category = "vars with {} of its scope covered".format(cov_bucket)
+        variables_coverage_map[cov_bucket] = json_parsed[cov_category]
+    else:
+      variables_scope_bytes_entry_values = \
+        json_parsed['vars entry value scope bytes covered']
+      variables_scope_bytes_covered = variables_scope_bytes_covered \
+         - variables_scope_bytes_entry_values
+      for cov_bucket in coverage_buckets():
+        cov_category = \
+          "vars (excluding the debug entry values) " \
+          "with {} of its scope covered".format(cov_bucket)
+        variables_coverage_map[cov_bucket] = json_parsed[cov_category]
+  elif results.only_formal_parameters:
+    # Read the JSON only for formal parameters.
+    variables_total_locstats = \
+      json_parsed['total params procesed by location statistics']
+    variables_scope_bytes_covered = \
+      json_parsed['formal params scope bytes covered']
+    variables_scope_bytes_from_first_def = \
+      json_parsed['formal params scope bytes total']
+    if not results.ignore_debug_entry_values:
+      for cov_bucket in coverage_buckets():
+        cov_category = "params with {} of its scope covered".format(cov_bucket)
+        variables_coverage_map[cov_bucket] = json_parsed[cov_category]
+    else:
+      variables_scope_bytes_entry_values = \
+        json_parsed['formal params entry value scope bytes covered']
+      variables_scope_bytes_covered = variables_scope_bytes_covered \
+        - variables_scope_bytes_entry_values
+      for cov_bucket in coverage_buckets():
+        cov_category = \
+          "params (excluding the debug entry values) " \
+          "with {} of its scope covered".format(cov_bucket)
+  else:
+    # Read the JSON for both local variables and formal parameters.
+    variables_total = \
+      json_parsed['source variables']
+    variables_with_loc = json_parsed['variables with location']
+    variables_total_locstats = \
+      json_parsed['total variables procesed by location statistics']
+    variables_scope_bytes_covered = \
+      json_parsed['scope bytes covered']
+    variables_scope_bytes_from_first_def = \
+      json_parsed['scope bytes total']
+    if not results.ignore_debug_entry_values:
+      for cov_bucket in coverage_buckets():
+        cov_category = "variables with {} of its scope covered". \
+                       format(cov_bucket)
+        variables_coverage_map[cov_bucket] = json_parsed[cov_category]
+    else:
+      variables_scope_bytes_entry_values = \
+        json_parsed['entry value scope bytes covered']
+      variables_scope_bytes_covered = variables_scope_bytes_covered \
+        - variables_scope_bytes_entry_values
+      for cov_bucket in coverage_buckets():
+        cov_category = "variables (excluding the debug entry values) " \
+                       "with {} of its scope covered". format(cov_bucket)
+        variables_coverage_map[cov_bucket] = json_parsed[cov_category]
+
+  # Pretty print collected info.
+  locstats_output(
+    variables_total,
+    variables_total_locstats,
+    variables_with_loc,
+    variables_scope_bytes_covered,
+    variables_scope_bytes_from_first_def,
+    variables_coverage_map
+    )
+
+if __name__ == '__main__':
+  Main()
+  sys.exit(0)
diff --git a/llvm/utils/release/build_llvm_package.bat b/llvm/utils/release/build_llvm_package.bat
index af613cdfe8827..6c48d20723c22 100755
--- a/llvm/utils/release/build_llvm_package.bat
+++ b/llvm/utils/release/build_llvm_package.bat
@@ -8,9 +8,9 @@ REM Usage: build_llvm_package.bat <revision>
 
 REM Prerequisites:
 REM
-REM   Visual Studio 2017, CMake, Ninja, SVN, GNUWin32, SWIG, Python 3,
+REM   Visual Studio 2019, CMake, Ninja, SVN, GNUWin32, SWIG, Python 3,
 REM   NSIS with the strlen_8192 patch,
-REM   Visual Studio 2017 SDK and Nuget (for the clang-format plugin),
+REM   Visual Studio 2019 SDK and Nuget (for the clang-format plugin),
 REM   Perl (for the OpenMP run-time).
 REM
 REM
@@ -19,7 +19,7 @@ REM   https://github.com/swig/swig/issues/769
 
 
 REM You need to modify the paths below:
-set vsdevcmd=C:\Program Files (x86)\Microsoft Visual Studio\2017\Professional\Common7\Tools\VsDevCmd.bat
+set vsdevcmd=C:\Program Files (x86)\Microsoft Visual Studio\2019\Professional\Common7\Tools\VsDevCmd.bat
 
 set python32_dir=C:\Users\%USERNAME%\AppData\Local\Programs\Python\Python36-32
 set python64_dir=C:\Users\%USERNAME%\AppData\Local\Programs\Python\Python36
@@ -72,8 +72,7 @@ set CC=
 set CXX=
 mkdir build32_stage0
 cd build32_stage0
-REM Work around VS2017 bug by using MinSizeRel.
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPYTHON_EXECUTABLE=%python32_dir%\python.exe -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python32_dir% -DPYTHON_EXECUTABLE=%python32_dir%\python.exe ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
@@ -97,6 +96,8 @@ cd ..
 REM The plug-in is built separately as it uses a statically linked clang-format.exe.
 mkdir build_vsix
 cd build_vsix
+REM Having VSSDKINSTALL set makes devenv *not* find the SDK for some reason.
+set VSSDKINSTALL=
 set CC=..\build32_stage0\bin\clang-cl
 set CXX=..\build32_stage0\bin\clang-cl
 cmake -GNinja %cmake_flags% -DLLVM_USE_CRT_RELEASE=MT -DBUILD_CLANG_FORMAT_VS_PLUGIN=ON -DPYTHON_HOME=%python32_dir% -DPYTHON_EXECUTABLE=%python32_dir%\python.exe ..\llvm || exit /b
@@ -111,8 +112,7 @@ set CC=
 set CXX=
 mkdir build64_stage0
 cd build64_stage0
-REM Work around VS2017 bug by using MinSizeRel.
-cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPYTHON_EXECUTABLE=%python64_dir%\python.exe -DCMAKE_BUILD_TYPE=MinSizeRel ..\llvm || exit /b
+cmake -GNinja %cmake_flags% -DPYTHON_HOME=%python64_dir% -DPYTHON_EXECUTABLE=%python64_dir%\python.exe ..\llvm || exit /b
 ninja all || ninja all || ninja all || exit /b
 ninja check || ninja check || ninja check || exit /b
 ninja check-clang || ninja check-clang || ninja check-clang || exit /b
diff --git a/llvm/utils/update_llc_test_checks.py b/llvm/utils/update_llc_test_checks.py
index 1f7d939155e2a..6fbab0b6a898b 100755
--- a/llvm/utils/update_llc_test_checks.py
+++ b/llvm/utils/update_llc_test_checks.py
@@ -19,7 +19,7 @@
 
 from UpdateTestChecks import asm, common
 
-ADVERT = '; NOTE: Assertions have been autogenerated by '
+ADVERT = ' NOTE: Assertions have been autogenerated by '
 
 
 def main():
@@ -44,7 +44,6 @@ def main():
   args = parser.parse_args()
 
   script_name = os.path.basename(__file__)
-  autogenerated_note = (ADVERT + 'utils/' + script_name)
 
   test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
   for test in test_paths:
@@ -92,6 +91,7 @@ def main():
 
       commands = [cmd.strip() for cmd in l.split('|', 1)]
       llc_cmd = commands[0]
+      llc_tool = llc_cmd.split(' ')[0]
 
       triple_in_cmd = None
       m = common.TRIPLE_ARG_RE.search(llc_cmd)
@@ -107,7 +107,7 @@ def main():
       if len(commands) > 1:
         filecheck_cmd = commands[1]
       common.verify_filecheck_prefixes(filecheck_cmd)
-      if not llc_cmd.startswith('llc '):
+      if llc_tool != 'llc':
         common.warn('Skipping non-llc RUN line: ' + l)
         continue
 
@@ -115,8 +115,15 @@ def main():
         common.warn('Skipping non-FileChecked RUN line: ' + l)
         continue
 
-      llc_cmd_args = llc_cmd[len('llc'):].strip()
+      llc_cmd_args = llc_cmd[len(llc_tool):].strip()
       llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
+      if test.endswith('.mir'):
+        llc_cmd_args += ' -x mir'
+        comment_sym = '#'
+        check_indent = '  '
+      else:
+        comment_sym = ';'
+        check_indent = ''
 
       check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
                                for item in m.group(1).split(',')]
@@ -127,6 +134,8 @@ def main():
       # now, we just ignore all but the last.
       run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
 
+    autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)
+
     func_dict = {}
     for p in run_list:
       prefixes = p[0]
@@ -134,7 +143,7 @@ def main():
         func_dict.update({prefix: dict()})
     for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
       if args.verbose:
-        print('Extracted LLC cmd: llc ' + llc_args, file=sys.stderr)
+        print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr)
         print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
 
       raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)
@@ -165,7 +174,7 @@ def main():
             continue
 
         # Print out the various check lines here.
-        asm.add_asm_checks(output_lines, ';', run_list, func_dict, func_name)
+        asm.add_asm_checks(output_lines, check_indent + ';', run_list, func_dict, func_name)
         is_in_function_start = False
 
       if is_in_function:
@@ -179,7 +188,7 @@ def main():
         continue
 
       # Discard any previous script advertising.
-      if input_line.startswith(ADVERT):
+      if input_line.startswith(comment_sym + ADVERT):
         continue
 
       # If it's outside a function, it just gets copied to the output.
diff --git a/llvm/utils/update_test_checks.py b/llvm/utils/update_test_checks.py
index fa8e1d45f1bde..5e0d4bac22f07 100755
--- a/llvm/utils/update_test_checks.py
+++ b/llvm/utils/update_test_checks.py
@@ -76,13 +76,13 @@ def main():
     sys.exit(1)
   opt_basename = 'opt'
 
-  test_paths = []
   for test in args.tests:
     if not glob.glob(test):
-      common.warn("Test file '%s' was not found. Ignoring it." % (test,))
+      common.warn("Test file pattern '%s' was not found. Ignoring it." % (test,))
       continue
-    test_paths.append(test)
 
+  # On Windows we must expand the patterns ourselves.
+  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
   for test in test_paths:
     if args.verbose:
       print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
diff --git a/openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp b/openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp
index 4648e51e95cd2..00e58d870f4c5 100644
--- a/openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp
+++ b/openmp/libomptarget/plugins/generic-elf-64bit/src/rtl.cpp
@@ -50,7 +50,7 @@ static int DebugLevel = 0;
 #include "../../common/elf_common.c"
 
 #define NUMBER_OF_DEVICES 4
-#define OFFLOADSECTIONNAME ".omp_offloading.entries"
+#define OFFLOADSECTIONNAME "omp_offloading_entries"
 
 /// Array of Dynamic libraries loaded for this target.
 struct DynLibTy {
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index fdb9dbb519810..a0b4202011dd8 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2181,10 +2181,9 @@ struct kmp_dephash_entry {
 typedef struct kmp_dephash {
   kmp_dephash_entry_t **buckets;
   size_t size;
-#ifdef KMP_DEBUG
+  size_t generation;
   kmp_uint32 nelements;
   kmp_uint32 nconflicts;
-#endif
 } kmp_dephash_t;
 
 typedef struct kmp_task_affinity_info {
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index db79deac3127f..f8aa51dd904a2 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -54,12 +54,64 @@ static inline kmp_depnode_t *__kmp_node_ref(kmp_depnode_t *node) {
 
 enum { KMP_DEPHASH_OTHER_SIZE = 97, KMP_DEPHASH_MASTER_SIZE = 997 };
 
+size_t sizes[] = { 997, 2003, 4001, 8191, 16001, 32003, 64007, 131071, 270029 };
+const size_t MAX_GEN = 8;
+
 static inline kmp_int32 __kmp_dephash_hash(kmp_intptr_t addr, size_t hsize) {
   // TODO alternate to try: set = (((Addr64)(addrUsefulBits * 9.618)) %
   // m_num_sets );
   return ((addr >> 6) ^ (addr >> 2)) % hsize;
 }
 
+static kmp_dephash_t *__kmp_dephash_extend(kmp_info_t *thread,
+                                           kmp_dephash_t *current_dephash) {
+  kmp_dephash_t *h;
+
+  size_t gen = current_dephash->generation + 1;
+  if (gen >= MAX_GEN)
+    return current_dephash;
+  size_t new_size = sizes[gen];
+
+  kmp_int32 size_to_allocate =
+      new_size * sizeof(kmp_dephash_entry_t *) + sizeof(kmp_dephash_t);
+
+#if USE_FAST_MEMORY
+  h = (kmp_dephash_t *)__kmp_fast_allocate(thread, size_to_allocate);
+#else
+  h = (kmp_dephash_t *)__kmp_thread_malloc(thread, size_to_allocate);
+#endif
+
+  h->size = new_size;
+  h->nelements = current_dephash->nelements;
+  h->buckets = (kmp_dephash_entry **)(h + 1);
+  h->generation = gen;
+
+  // insert existing elements in the new table
+  for (size_t i = 0; i < current_dephash->size; i++) {
+    kmp_dephash_entry_t *next;
+    for (kmp_dephash_entry_t *entry = current_dephash->buckets[i]; entry; entry = next) {
+      next = entry->next_in_bucket;
+      // Compute the new hash using the new size, and insert the entry in
+      // the new bucket.
+      kmp_int32 new_bucket = __kmp_dephash_hash(entry->addr, h->size);
+      if (entry->next_in_bucket) {
+        h->nconflicts++;
+      }
+      entry->next_in_bucket = h->buckets[new_bucket];
+      h->buckets[new_bucket] = entry;
+    }
+  }
+
+  // Free old hash table
+#if USE_FAST_MEMORY
+  __kmp_fast_free(thread, current_dephash);
+#else
+  __kmp_thread_free(thread, current_dephash);
+#endif
+
+  return h;
+}
+
 static kmp_dephash_t *__kmp_dephash_create(kmp_info_t *thread,
                                            kmp_taskdata_t *current_task) {
   kmp_dephash_t *h;
@@ -81,10 +133,9 @@ static kmp_dephash_t *__kmp_dephash_create(kmp_info_t *thread,
 #endif
   h->size = h_size;
 
-#ifdef KMP_DEBUG
+  h->generation = 0;
   h->nelements = 0;
   h->nconflicts = 0;
-#endif
   h->buckets = (kmp_dephash_entry **)(h + 1);
 
   for (size_t i = 0; i < h_size; i++)
@@ -97,7 +148,13 @@ static kmp_dephash_t *__kmp_dephash_create(kmp_info_t *thread,
 #define ENTRY_LAST_MTXS 1
 
 static kmp_dephash_entry *
-__kmp_dephash_find(kmp_info_t *thread, kmp_dephash_t *h, kmp_intptr_t addr) {
+__kmp_dephash_find(kmp_info_t *thread, kmp_dephash_t **hash, kmp_intptr_t addr) {
+  kmp_dephash_t *h = *hash;
+  if (h->nelements != 0
+      && h->nconflicts/h->size >= 1) {
+    *hash = __kmp_dephash_extend(thread, h);
+    h = *hash;
+  }
   kmp_int32 bucket = __kmp_dephash_hash(addr, h->size);
 
   kmp_dephash_entry_t *entry;
@@ -122,11 +179,9 @@ __kmp_dephash_find(kmp_info_t *thread, kmp_dephash_t *h, kmp_intptr_t addr) {
     entry->mtx_lock = NULL;
     entry->next_in_bucket = h->buckets[bucket];
     h->buckets[bucket] = entry;
-#ifdef KMP_DEBUG
     h->nelements++;
     if (entry->next_in_bucket)
       h->nconflicts++;
-#endif
   }
   return entry;
 }
@@ -232,7 +287,7 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
 
 template <bool filter>
 static inline kmp_int32
-__kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
+__kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
                    bool dep_barrier, kmp_int32 ndeps,
                    kmp_depend_info_t *dep_list, kmp_task_t *task) {
   KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d processing %d dependencies : "
@@ -352,7 +407,7 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *hash,
 
 // returns true if the task has any outstanding dependence
 static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
-                             kmp_task_t *task, kmp_dephash_t *hash,
+                             kmp_task_t *task, kmp_dephash_t **hash,
                              bool dep_barrier, kmp_int32 ndeps,
                              kmp_depend_info_t *dep_list,
                              kmp_int32 ndeps_noalias,
@@ -552,7 +607,7 @@ kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
     __kmp_init_node(node);
     new_taskdata->td_depnode = node;
 
-    if (__kmp_check_deps(gtid, node, new_task, current_task->td_dephash,
+    if (__kmp_check_deps(gtid, node, new_task, &current_task->td_dephash,
                          NO_DEP_BARRIER, ndeps, dep_list, ndeps_noalias,
                          noalias_dep_list)) {
       KA_TRACE(10, ("__kmpc_omp_task_with_deps(exit): T#%d task had blocking "
@@ -633,7 +688,7 @@ void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 gtid, kmp_int32 ndeps,
   kmp_depnode_t node = {0};
   __kmp_init_node(&node);
 
-  if (!__kmp_check_deps(gtid, &node, NULL, current_task->td_dephash,
+  if (!__kmp_check_deps(gtid, &node, NULL, &current_task->td_dephash,
                         DEP_BARRIER, ndeps, dep_list, ndeps_noalias,
                         noalias_dep_list)) {
     KA_TRACE(10, ("__kmpc_omp_wait_deps(exit): T#%d has no blocking "
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index 5f572ab0b0ed4..871be6283c294 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -50,6 +50,9 @@
 #include <mach/mach.h>
 #include <sys/sysctl.h>
 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/user.h>
 #include <pthread_np.h>
 #elif KMP_OS_NETBSD
 #include <sys/types.h>
@@ -1979,7 +1982,7 @@ int __kmp_is_address_mapped(void *addr) {
   int found = 0;
   int rc;
 
-#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_HURD
+#if KMP_OS_LINUX || KMP_OS_HURD
 
   /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the address
      ranges mapped into the address space. */
@@ -2017,6 +2020,44 @@ int __kmp_is_address_mapped(void *addr) {
   // Free resources.
   fclose(file);
   KMP_INTERNAL_FREE(name);
+#elif KMP_OS_FREEBSD
+  char *buf;
+  size_t lstsz;
+  int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
+  rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
+  if (rc < 0)
+     return 0;
+  // We pass from number of vm entry's semantic
+  // to size of whole entry map list.
+  lstsz = lstsz * 4 / 3;
+  buf = reinterpret_cast<char *>(kmpc_malloc(lstsz));
+  rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
+  if (rc < 0) {
+     kmpc_free(buf);
+     return 0;
+  }
+
+  char *lw = buf;
+  char *up = buf + lstsz;
+
+  while (lw < up) {
+      struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
+      size_t cursz = cur->kve_structsize;
+      if (cursz == 0)
+          break;
+      void *start = reinterpret_cast<void *>(cur->kve_start);
+      void *end = reinterpret_cast<void *>(cur->kve_end);
+      // Readable/Writable addresses within current map entry
+      if ((addr >= start) && (addr < end)) {
+          if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
+              (cur->kve_protection & KVME_PROT_WRITE) != 0) {
+              found = 1;
+              break;
+          }
+      }
+      lw += cursz;
+  }
+  kmpc_free(buf);
 
 #elif KMP_OS_DARWIN
 
diff --git a/openmp/runtime/test/tasking/omp_fill_taskqueue.c b/openmp/runtime/test/tasking/omp_fill_taskqueue.c
index e95f97a448076..17e1de380bcbc 100644
--- a/openmp/runtime/test/tasking/omp_fill_taskqueue.c
+++ b/openmp/runtime/test/tasking/omp_fill_taskqueue.c
@@ -29,7 +29,6 @@ int main()
 {
   int i;
   int block = 1;
-  int tid;
   int throttling = strcmp(getenv("KMP_ENABLE_TASK_THROTTLING"), "1") == 0;
   int enqueued = 0;
   int failed = -1;
@@ -41,6 +40,7 @@ int main()
       enqueued++;
       #pragma omp task
       {
+        int tid;
         tid = omp_get_thread_num();
         if (tid == 0) {
           // As soon as the master thread starts executing task we should unlock
diff --git a/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c b/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c
new file mode 100644
index 0000000000000..03f12b06dfe19
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_task_depend_resize_hashmap.c
@@ -0,0 +1,38 @@
+// RUN: %libomp-compile && env KMP_ENABLE_TASK_THROTTLING=0 %libomp-run
+
+#include<omp.h>
+#include<stdlib.h>
+#include<string.h>
+
+// The first hashtable static size is 997
+#define NUM_DEPS 4000
+
+
+int main()
+{
+  int *deps = calloc(NUM_DEPS, sizeof(int));
+  int i;
+  int failed = 0;
+
+  #pragma omp parallel
+  #pragma omp master
+  {
+    for (i = 0; i < NUM_DEPS; i++) {
+      #pragma omp task firstprivate(i) depend(inout: deps[i])
+      {
+        deps[i] = 1;
+      }
+      #pragma omp task firstprivate(i) depend(inout: deps[i])
+      {
+        deps[i] = 2;
+      }
+    }
+  }
+
+  for (i = 0; i < NUM_DEPS; i++) {
+    if (deps[i] != 2)
+      failed++;
+  }
+
+  return failed;
+}
diff --git a/polly/lib/CodeGen/BlockGenerators.cpp b/polly/lib/CodeGen/BlockGenerators.cpp
index e15209915d24a..da9ed989efa0d 100644
--- a/polly/lib/CodeGen/BlockGenerators.cpp
+++ b/polly/lib/CodeGen/BlockGenerators.cpp
@@ -1061,7 +1061,7 @@ Value *VectorBlockGenerator::generateStrideOneLoad(
   LoadInst *VecLoad =
       Builder.CreateLoad(VectorPtr, Load->getName() + "_p_vec_full");
   if (!Aligned)
-    VecLoad->setAlignment(8);
+    VecLoad->setAlignment(Align(8));
 
   if (NegativeStride) {
     SmallVector<Constant *, 16> Indices;
@@ -1089,7 +1089,7 @@ Value *VectorBlockGenerator::generateStrideZeroLoad(
       Builder.CreateLoad(VectorPtr, Load->getName() + "_p_splat_one");
 
   if (!Aligned)
-    ScalarLoad->setAlignment(8);
+    ScalarLoad->setAlignment(Align(8));
 
   Constant *SplatVector = Constant::getNullValue(
       VectorType::get(Builder.getInt32Ty(), getVectorWidth()));
diff --git a/polly/lib/CodeGen/IslNodeBuilder.cpp b/polly/lib/CodeGen/IslNodeBuilder.cpp
index 492944cf7c325..23b2891eade1a 100644
--- a/polly/lib/CodeGen/IslNodeBuilder.cpp
+++ b/polly/lib/CodeGen/IslNodeBuilder.cpp
@@ -231,11 +231,12 @@ void addReferencesFromStmt(const ScopStmt *Stmt, void *UserPtr,
 
   if (Stmt->isBlockStmt())
     findReferencesInBlock(References, Stmt, Stmt->getBasicBlock());
-  else {
-    assert(Stmt->isRegionStmt() &&
-           "Stmt was neither block nor region statement");
+  else if (Stmt->isRegionStmt()) {
     for (BasicBlock *BB : Stmt->getRegion()->blocks())
       findReferencesInBlock(References, Stmt, BB);
+  } else {
+    assert(Stmt->isCopyStmt());
+    // Copy Stmts have no instructions that we need to consider.
   }
 
   for (auto &Access : *Stmt) {
@@ -1212,7 +1213,8 @@ Value *IslNodeBuilder::preloadUnconditionally(isl_set *AccessRange,
   Ptr = Builder.CreatePointerCast(Ptr, Ty->getPointerTo(AS), Name + ".cast");
   PreloadVal = Builder.CreateLoad(Ptr, Name + ".load");
   if (LoadInst *PreloadInst = dyn_cast<LoadInst>(PreloadVal))
-    PreloadInst->setAlignment(dyn_cast<LoadInst>(AccInst)->getAlignment());
+    PreloadInst->setAlignment(
+        MaybeAlign(dyn_cast<LoadInst>(AccInst)->getAlignment()));
 
   // TODO: This is only a hot fix for SCoP sequences that use the same load
   //       instruction contained and hoisted by one of the SCoPs.
@@ -1495,7 +1497,8 @@ void IslNodeBuilder::allocateNewArrays(BBPair StartExitBlocks) {
 
       auto *CreatedArray = new AllocaInst(NewArrayType, DL.getAllocaAddrSpace(),
                                           SAI->getName(), &*InstIt);
-      CreatedArray->setAlignment(PollyTargetFirstLevelCacheLineSize);
+      CreatedArray->setAlignment(
+          MaybeAlign(PollyTargetFirstLevelCacheLineSize));
       SAI->setBasePtr(CreatedArray);
     }
   }
diff --git a/polly/test/DeLICM/reduction_looprotate_hoisted.ll b/polly/test/DeLICM/reduction_looprotate_hoisted.ll
index 3a96ff6138b99..47f449564508e 100644
--- a/polly/test/DeLICM/reduction_looprotate_hoisted.ll
+++ b/polly/test/DeLICM/reduction_looprotate_hoisted.ll
@@ -70,7 +70,7 @@ return:
 ; CHECK-NEXT:     Stmt_reduction_preheader
 ; CHECK-NEXT:             MustWriteAccess :=  [Reduction Type: NONE] [Scalar: 1]
 ; CHECK-NEXT:                 [Start] -> { Stmt_reduction_preheader[i0] -> MemRef_phi__phi[] };
-; CHECK-NEXT:            new: [Start] -> { Stmt_reduction_preheader[i0] -> MemRef_A[i0] };
+; CHECK-NEXT:            new: [Start] -> { Stmt_reduction_preheader[i0] -> MemRef_A[i0] : Start >= 2147483648 or Start <= 2147483646 };
 ; CHECK-NEXT:     Stmt_reduction_for
 ; CHECK-NEXT:             ReadAccess :=    [Reduction Type: NONE] [Scalar: 1]
 ; CHECK-NEXT:                 [Start] -> { Stmt_reduction_for[i0, i1] -> MemRef_phi__phi[] };
diff --git a/polly/test/Isl/CodeGen/OpenMP/matmul-parallel.ll b/polly/test/Isl/CodeGen/OpenMP/matmul-parallel.ll
new file mode 100644
index 0000000000000..4a9d9c8faa641
--- /dev/null
+++ b/polly/test/Isl/CodeGen/OpenMP/matmul-parallel.ll
@@ -0,0 +1,72 @@
+; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-ast -disable-output -debug-only=polly-ast < %s 2>&1 | FileCheck --check-prefix=AST %s
+; RUN: opt %loadPolly -polly-parallel -polly-opt-isl -polly-codegen -S < %s | FileCheck --check-prefix=CODEGEN %s
+; REQUIRES: asserts
+
+; Parellization of detected matrix-multiplication. The allocations
+; Packed_A and Packed_B must be passed to the outlined function.
+; llvm.org/PR43164
+;
+; #define N 1536
+; int foo(float A[N][N],float B[N][N],float C[N][N]) {
+;     for (int i = 0; i < N; i++) {
+;         for (int j = 0; j < N; j++) {
+;             for (int k = 0; k < N; k++)
+;                 C[i][j] = C[i][j] + A[i][k] * B[k][j];
+;         }
+;     }
+;     return 0;
+; }
+
+target datalayout = "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-pc-windows-msvc19.16.27034"
+
+define i32 @foo([1536 x float]* nocapture readonly %A, [1536 x float]* nocapture readonly %B, [1536 x float]* nocapture %C) {
+entry:
+  br label %entry.split
+
+entry.split:
+  br label %for.cond1.preheader
+
+for.cond1.preheader:
+  %indvars.iv50 = phi i64 [ 0, %entry.split ], [ %indvars.iv.next51, %for.cond.cleanup3 ]
+  br label %for.cond5.preheader
+
+for.cond.cleanup:
+  ret i32 0
+
+for.cond5.preheader:
+  %indvars.iv47 = phi i64 [ 0, %for.cond1.preheader ], [ %indvars.iv.next48, %for.cond.cleanup7 ]
+  %arrayidx10 = getelementptr inbounds [1536 x float], [1536 x float]* %C, i64 %indvars.iv50, i64 %indvars.iv47
+  br label %for.body8
+
+for.cond.cleanup3:
+  %indvars.iv.next51 = add nuw nsw i64 %indvars.iv50, 1
+  %exitcond52 = icmp eq i64 %indvars.iv.next51, 1536
+  br i1 %exitcond52, label %for.cond.cleanup, label %for.cond1.preheader
+
+for.cond.cleanup7:
+  %indvars.iv.next48 = add nuw nsw i64 %indvars.iv47, 1
+  %exitcond49 = icmp eq i64 %indvars.iv.next48, 1536
+  br i1 %exitcond49, label %for.cond.cleanup3, label %for.cond5.preheader
+
+for.body8:
+  %indvars.iv = phi i64 [ 0, %for.cond5.preheader ], [ %indvars.iv.next, %for.body8 ]
+  %0 = load float, float* %arrayidx10, align 4
+  %arrayidx14 = getelementptr inbounds [1536 x float], [1536 x float]* %A, i64 %indvars.iv50, i64 %indvars.iv
+  %1 = load float, float* %arrayidx14, align 4
+  %arrayidx18 = getelementptr inbounds [1536 x float], [1536 x float]* %B, i64 %indvars.iv, i64 %indvars.iv47
+  %2 = load float, float* %arrayidx18, align 4
+  %mul = fmul float %1, %2
+  %add = fadd float %0, %mul
+  store float %add, float* %arrayidx10, align 4
+  %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+  %exitcond = icmp eq i64 %indvars.iv.next, 1536
+  br i1 %exitcond, label %for.cond.cleanup7, label %for.body8
+}
+
+
+; AST: #pragma omp parallel for
+
+; CODGEN-LABEL: define internal void @init_array_polly_subfn(i8* %polly.par.userContext)
+; CODEGEN: %polly.subfunc.arg.Packed_A = load
+; CODEGEN: %polly.subfunc.arg.Packed_B = load